From 62e388f307336d823cf37f27c8d15a6cfafe734a Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Sun, 30 Aug 2026 04:25:37 -0700 Subject: [PATCH 01/61] dasLLAMA: IQ4_XS native tier, CPU slice - KqFmt.iq4xs (schema id 44), loader transcode with a decoded 20B scale row, plane pair + every dispatch ladder, portable/reference kernels, repack, tests Co-Authored-By: Claude Fable 5 --- modules/dasLLAMA/dasllama/dasllama_blocks.das | 2 +- modules/dasLLAMA/dasllama/dasllama_common.das | 33 ++++- modules/dasLLAMA/dasllama/dasllama_config.das | 3 +- .../dasLLAMA/dasllama/dasllama_convert.das | 40 ++++++ .../dasLLAMA/dasllama/dasllama_gemm_gen.das | 9 ++ .../dasllama/dasllama_gemm_schema.das | 9 +- modules/dasLLAMA/dasllama/dasllama_gguf.das | 44 ++++++ .../dasllama/dasllama_gpu_resident.das | 5 + modules/dasLLAMA/dasllama/dasllama_image.das | 4 +- .../dasLLAMA/dasllama/dasllama_kqformat.das | 26 ++-- modules/dasLLAMA/dasllama/dasllama_layout.das | 16 ++- modules/dasLLAMA/dasllama/dasllama_load.das | 37 ++++- modules/dasLLAMA/dasllama/dasllama_math.das | 9 +- .../dasllama/dasllama_math_default.das | 110 +++++++++++--- .../dasLLAMA/dasllama/dasllama_math_gen.das | 97 ++++++++++++- modules/dasLLAMA/dasllama/dasllama_ple.das | 4 + modules/dasLLAMA/dasllama/dasllama_repack.das | 44 ++++++ modules/dasLLAMA/tests/test_kqformat.das | 23 ++- modules/dasLLAMA/tests/test_kquant.das | 134 +++++++++++++++--- 19 files changed, 582 insertions(+), 67 deletions(-) diff --git a/modules/dasLLAMA/dasllama/dasllama_blocks.das b/modules/dasLLAMA/dasllama/dasllama_blocks.das index cb8a1d4df8..63f369b3d4 100644 --- a/modules/dasLLAMA/dasllama/dasllama_blocks.das +++ b/modules/dasLLAMA/dasllama/dasllama_blocks.das @@ -1394,7 +1394,7 @@ def private kq_bytes_per_weight(f : KqFmt) : float { if (f == KqFmt.q40) { return 144.0 / 256.0 // 4-bit quants plus an f16 scale per 32 } - if (f == KqFmt.k4) { + if (f == KqFmt.k4 || f == KqFmt.iq4xs) { return 148.0 / 256.0 } if (f == KqFmt.k5) { diff --git a/modules/dasLLAMA/dasllama/dasllama_common.das b/modules/dasLLAMA/dasllama/dasllama_common.das index accbe371e0..9aab895280 100644 --- a/modules/dasLLAMA/dasllama/dasllama_common.das +++ b/modules/dasLLAMA/dasllama/dasllama_common.das @@ -835,6 +835,9 @@ struct Model { // the q51 tier's plane pair, per 32-BLOCK: 20B nibbles+qh / 4B f16 d+m (never repacked) q51q : array q51s : array + // the iq4xs tier's plane pair: 128B nibbles in the q40 tiling + a 20B decoded scale row per superblock + iq4xsq : array + iq4xss : array kquant_native : bool = false // kq stage 4: the kq planes were repacked at load into the active backend's grp layout. // From then on every kq matmul MUST run the backend slots — disk-order portable kernels would @@ -845,6 +848,7 @@ struct Model { kq_repack_mr5 : int64 = 4l kq_repack_mr6 : int64 = 4l kq_repack_mr40 : int64 = 4l + kq_repack_mr44 : int64 = 4l // per-layer weight format tags (KqFmt; empty = all q8). Filled by detect_kq_formats before // layout_offsets so the layout walks each tensor into its format's cursor. wq_fmt : array @@ -1455,6 +1459,7 @@ def private dlim_cpu_source_impl(var c : DlimCpuConfig) { c.kq_mr5 = active_kq_layout_mr(5) c.kq_mr6 = active_kq_layout_mr(6) c.kq_mr40 = active_kq_layout_mr(40) + c.kq_mr44 = active_kq_layout_mr(44) c.q51_mr = active_q51_layout_mr() // OUTCOME, not request: a backend without s16 twins keeps f32 scale planes (wscale_convert_f16) c.wscale_f16 = g_wscale_f16 && kernel_backend_has_wscale16() @@ -1547,7 +1552,7 @@ def kq_active_mr(t : Model; f : KqFmt) : int64 { if (!t.kq_repacked) { return 1l } - return f == KqFmt.k4 ? t.kq_repack_mr4 : (f == KqFmt.k5 ? t.kq_repack_mr5 : (f == KqFmt.k6 ? t.kq_repack_mr6 : t.kq_repack_mr40)) + return f == KqFmt.k4 ? t.kq_repack_mr4 : (f == KqFmt.k5 ? t.kq_repack_mr5 : (f == KqFmt.k6 ? t.kq_repack_mr6 : (f == KqFmt.q40 ? t.kq_repack_mr40 : t.kq_repack_mr44))) } //! Load a llama-architecture GGUF into the split layout at the requested precision — straight into @@ -3285,7 +3290,7 @@ def mm_b_q51_groupn(var y : array; t : Model; offs : array; nregio // KqFmt -> the kernel-layer format id (matmul_kq*/kq_rows_fn/kq_qsb take the int form). q8 maps // to 0, a poison id no kq kernel accepts — kq_rows_for calls this for EVERY tag, so it must stay // total even though q8 branches never read the result. -def private kq_fi(fmt : KqFmt) : int => fmt == KqFmt.k4 ? 4 : (fmt == KqFmt.k5 ? 5 : (fmt == KqFmt.k6 ? 6 : (fmt == KqFmt.q40 ? 40 : 0))) +def private kq_fi(fmt : KqFmt) : int => fmt == KqFmt.k4 ? 4 : (fmt == KqFmt.k5 ? 5 : (fmt == KqFmt.k6 ? 6 : (fmt == KqFmt.q40 ? 40 : (fmt == KqFmt.iq4xs ? 44 : 0)))) // K-quant GEMV against a PRE-quantized activation (xq/xs/xbs already filled by the bs quantizer). // kq_repacked loads run the backend's stamped cores; disk-order loads run the portable kernels. @@ -3299,6 +3304,8 @@ def mm_at_kq_pre(var y : array; t : Model; fmt : KqFmt; woff : int64; xq matmul_kq_active(6, y, t.k6q, t.k6s, woff, xq, xs, xbs, n, d, yoff) } elif (fmt == KqFmt.q40) { matmul_kq_active(40, y, t.q40q, t.q40s, woff, xq, xs, xbs, n, d, yoff) + } elif (fmt == KqFmt.iq4xs) { + matmul_kq_active(44, y, t.iq4xsq, t.iq4xss, woff, xq, xs, xbs, n, d, yoff) } else { panic("mm_at_kq_pre: '{fmt}' has no kq plane pair") // a new KqFmt must claim its arm, never ride q40's } @@ -3310,6 +3317,8 @@ def mm_at_kq_pre(var y : array; t : Model; fmt : KqFmt; woff : int64; xq matmul_kq(6, y, t.k6q, t.k6s, woff, xq, xs, xbs, n, d, yoff) } elif (fmt == KqFmt.q40) { matmul_kq(40, y, t.q40q, t.q40s, woff, xq, xs, xbs, n, d, yoff) + } elif (fmt == KqFmt.iq4xs) { + matmul_kq(44, y, t.iq4xsq, t.iq4xss, woff, xq, xs, xbs, n, d, yoff) } else { panic("mm_at_kq_pre: '{fmt}' has no kq plane pair") } @@ -3341,6 +3350,8 @@ def mm_b_kq(var s : Session; var y : array; t : Model; fmt : KqFmt; woff matmul_kq_batch(6, y, t.k6q, t.k6s, woff, s.xqb, s.xsb, s.xbsb, n, d, npos) } elif (fmt == KqFmt.q40) { matmul_kq_batch(40, y, t.q40q, t.q40s, woff, s.xqb, s.xsb, s.xbsb, n, d, npos) + } elif (fmt == KqFmt.iq4xs) { + matmul_kq_batch(44, y, t.iq4xsq, t.iq4xss, woff, s.xqb, s.xsb, s.xbsb, n, d, npos) } else { panic("mm_b_kq: '{fmt}' has no kq plane pair") } @@ -3368,6 +3379,8 @@ def mm_at_kq_groupn(var y : array; t : Model; fmt : KqFmt; offs : array; t : Model; fmt : KqFmt; offs : array; t : Model; fmt : KqFmt; offs : array; t : Model; fmt : KqFmt; woff : int64; xq : matmul_kq_batch(6, y, t.k6q, t.k6s, woff, xq, xs, xbs, n, d, ntok) } elif (fmt == KqFmt.q40) { matmul_kq_batch(40, y, t.q40q, t.q40s, woff, xq, xs, xbs, n, d, ntok) + } elif (fmt == KqFmt.iq4xs) { + matmul_kq_batch(44, y, t.iq4xsq, t.iq4xss, woff, xq, xs, xbs, n, d, ntok) } else { panic("mm_b_kq_pre: '{fmt}' has no kq plane pair") } @@ -3425,6 +3444,7 @@ def kq_plane_q(t : Model; fmt : KqFmt; woff : int64) : uint8 const? { if (fmt == KqFmt.k5) return unsafe(addr(t.k5q[sb * K5_QSB])) if (fmt == KqFmt.k6) return unsafe(addr(t.k6q[sb * K6_QSB])) if (fmt == KqFmt.q40) return unsafe(addr(t.q40q[sb * Q40_QSB])) + if (fmt == KqFmt.iq4xs) return unsafe(addr(t.iq4xsq[sb * IQ4XS_QSB])) return null } @@ -3434,6 +3454,7 @@ def kq_plane_s(t : Model; fmt : KqFmt; woff : int64) : uint8 const? { if (fmt == KqFmt.k5) return unsafe(addr(t.k5s[sb * K5_SSB])) if (fmt == KqFmt.k6) return unsafe(addr(t.k6s[sb * K6_SSB])) if (fmt == KqFmt.q40) return unsafe(addr(t.q40s[sb * Q40_SSB])) + if (fmt == KqFmt.iq4xs) return unsafe(addr(t.iq4xss[sb * IQ4XS_SSB])) return null } @@ -3794,6 +3815,8 @@ def embed_row(t : Model; token : int64; var dst : float?) { // nolint:STYLE037 dequant_kq_row_grp(fmt, addr(t.embq[sbg * K6_QSB]), addr(t.embs[sbg * K6_SSB]), token % mr, mr, dim, dst) } elif (t.emb_fmt == KqFmt.q40) { dequant_kq_row_grp(fmt, addr(t.embq[sbg * Q40_QSB]), addr(t.embs[sbg * Q40_SSB]), token % mr, mr, dim, dst) + } elif (t.emb_fmt == KqFmt.iq4xs) { + dequant_kq_row_grp(fmt, addr(t.embq[sbg * IQ4XS_QSB]), addr(t.embs[sbg * IQ4XS_SSB]), token % mr, mr, dim, dst) } else { panic("embed_row: '{t.emb_fmt}' has no kq plane pair") } @@ -3805,6 +3828,8 @@ def embed_row(t : Model; token : int64; var dst : float?) { // nolint:STYLE037 dequant_kq_row_grp(fmt, addr(t.k6q[sbg * K6_QSB]), addr(t.k6s[sbg * K6_SSB]), token % mr, mr, dim, dst) } elif (t.emb_fmt == KqFmt.q40) { dequant_kq_row_grp(fmt, addr(t.q40q[sbg * Q40_QSB]), addr(t.q40s[sbg * Q40_SSB]), token % mr, mr, dim, dst) + } elif (t.emb_fmt == KqFmt.iq4xs) { + dequant_kq_row_grp(fmt, addr(t.iq4xsq[sbg * IQ4XS_QSB]), addr(t.iq4xss[sbg * IQ4XS_SSB]), token % mr, mr, dim, dst) } else { panic("embed_row: '{t.emb_fmt}' has no kq plane pair") } @@ -3827,6 +3852,8 @@ def embed_row(t : Model; token : int64; var dst : float?) { // nolint:STYLE037 dequant_k6_plane_superblock(t.embq, (sb0 + s) * K6_QSB, t.embs, (sb0 + s) * K6_SSB, row, s * 256l) } elif (t.emb_fmt == KqFmt.q40) { dequant_q40_plane_superblock(t.embq, (sb0 + s) * Q40_QSB, t.embs, (sb0 + s) * Q40_SSB, row, s * 256l) + } elif (t.emb_fmt == KqFmt.iq4xs) { + dequant_iq4xs_plane_superblock(t.embq, (sb0 + s) * IQ4XS_QSB, t.embs, (sb0 + s) * IQ4XS_SSB, row, s * 256l) } else { panic("embed_row: '{t.emb_fmt}' has no kq plane pair") } @@ -3842,6 +3869,8 @@ def embed_row(t : Model; token : int64; var dst : float?) { // nolint:STYLE037 } } elif (t.emb_fmt == KqFmt.q40) { dequant_q40_plane_superblock(t.q40q, (sb0 + s) * Q40_QSB, t.q40s, (sb0 + s) * Q40_SSB, row, s * 256l) + } elif (t.emb_fmt == KqFmt.iq4xs) { + dequant_iq4xs_plane_superblock(t.iq4xsq, (sb0 + s) * IQ4XS_QSB, t.iq4xss, (sb0 + s) * IQ4XS_SSB, row, s * 256l) } else { panic("embed_row: '{t.emb_fmt}' has no kq plane pair") } diff --git a/modules/dasLLAMA/dasllama/dasllama_config.das b/modules/dasLLAMA/dasllama/dasllama_config.das index 88b181b4fe..9104fdd776 100644 --- a/modules/dasLLAMA/dasllama/dasllama_config.das +++ b/modules/dasLLAMA/dasllama/dasllama_config.das @@ -27,6 +27,7 @@ struct public DlimCpuConfig { kq_mr5 : int64 kq_mr6 : int64 kq_mr40 : int64 + kq_mr44 : int64 q51_mr : int64 wscale_f16 : bool kquant_native : bool @@ -139,7 +140,7 @@ def public dlim_config_current(quant : string = "q8") : DlimConfiguration { def public dlim_identity(c : DlimConfiguration; image_version : int; tag : string = "") : string { return ("v{image_version}|{c.quant}|{c.cpu.backend}|{c.cpu.wscale_f16 ? "s16" : "s32"}" + "|q8 mr{c.cpu.q8_mr} b{c.cpu.q8_wbias} g{c.cpu.q8_kgroup}" - + "|kq {c.cpu.kq_mr4}/{c.cpu.kq_mr5}/{c.cpu.kq_mr6}/{c.cpu.kq_mr40}" + + "|kq {c.cpu.kq_mr4}/{c.cpu.kq_mr5}/{c.cpu.kq_mr6}/{c.cpu.kq_mr40}/{c.cpu.kq_mr44}" + "|q51 mr{c.cpu.q51_mr}" + "|nat {c.cpu.kquant_native ? 1 : 0}{c.cpu.kq_q40_native ? 1 : 0}{c.cpu.kq_q50_native ? 1 : 0}{c.cpu.kq_q51_native ? 1 : 0}" + (tag != "" ? "|{tag}" : "")) diff --git a/modules/dasLLAMA/dasllama/dasllama_convert.das b/modules/dasLLAMA/dasllama/dasllama_convert.das index 0fd918bf2f..915902db45 100644 --- a/modules/dasLLAMA/dasllama/dasllama_convert.das +++ b/modules/dasLLAMA/dasllama/dasllama_convert.das @@ -615,6 +615,46 @@ def dequant_q40_plane_superblock(kq : array | #; kqo : int64; ks : array< } } +//! Transcode one IQ4_XS superblock (136 bytes at `bo`: f16 d, u16 scales_h, 4 scales_l bytes, +//! 128 nibble bytes) into the iq4xs planes: nibbles verbatim (the disk k/k+16 pairing IS the +//! q40 tiling), the scale row DECODED — f16 d, 2 pad, 8 x int8 (ls - 32), 8 pad (exact). +def transcode_iq4xs_superblock(bytes : array | #; bo : int64; var kq : array; kqo : int64; var ks : array; kso : int64) { + ks[kso] = bytes[bo] + ks[kso + 1l] = bytes[bo + 1l] + ks[kso + 2l] = uint8(0) + ks[kso + 3l] = uint8(0) + let sh = uint(bytes[bo + 2l]) | (uint(bytes[bo + 3l]) << 8u) + for (blk in range64(8l)) { + let lo = (uint(bytes[bo + 4l + blk / 2l]) >> uint(4l * (blk % 2l))) & 15u + let ls = int(lo | (((sh >> uint(2l * blk)) & 3u) << 4u)) + ks[kso + 4l + blk] = uint8(ls - 32) + } + for (i in range64(8l)) { + ks[kso + 12l + i] = uint8(0) + } + for (i in range64(128l)) { + kq[kqo + i] = bytes[bo + 8l + i] + } +} + +//! The signed sub-scale of an iq4xs plane row: byte 4 + blk of the 20B row, read as int8. +def iq4xs_sc(ks : array | #; kso, blk : int64) : int => rd_i8(ks, kso + 4l + blk) + +//! Reference dequant of one iq4xs-plane superblock: w = (d * sc) * IQ4NL_LUT[q] per 32-block +//! (ggml's own float order: dl = d * (ls - 32), then dl * kvalue), q = nibble (low = k, high = +//! k+16 — the disk pairing, kept verbatim by the transcode). +def dequant_iq4xs_plane_superblock(kq : array | #; kqo : int64; ks : array | #; kso : int64; var dst : array | #; doff : int64) { + let d = f16_to_f32(rd_u16(ks, kso)) + for (blk in range64(8l)) { + let dl = d * float(iq4xs_sc(ks, kso, blk)) + for (l in range64(16l)) { + let b = int(kq[kqo + blk * 16l + l]) + dst[doff + blk * 32l + l] = dl * float(IQ4NL_LUT[b & 15]) + dst[doff + blk * 32l + 16l + l] = dl * float(IQ4NL_LUT[b >> 4]) + } + } +} + //! Transcode one Q5_1 disk block (24 bytes at `bo`: d f16, m f16, qh u32, 16 nibble bytes) into //! the q51 planes — verbatim splits, exact. The per-block array form the tests drive; the bulk //! loader (gguf_transcode_q51) runs the same split pointerized and threaded. diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das index 2cd27152fc..f958331941 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das @@ -2462,6 +2462,13 @@ def private kq_tile_gen_impl(var gc : LlvmCodeCtx; fmt : int) : bool { return true } +// iq4xs: the LUT-nibble emitter lands with the kernel arc; until then both stamps decline +// (reference bodies serve) — a declined generator is the framework's own fallback path. +[unused_argument(gc)] +def private iq4xs_gemv_gen(var gc : LlvmCodeCtx) : bool => false +[unused_argument(gc)] +def private iq4xs_tile_gen(var gc : LlvmCodeCtx) : bool => false + def private k4_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 4) def private k5_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 5) def private k6_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 6) @@ -2539,5 +2546,7 @@ def public register_dasllama_gemm_generators { register_llvm_code_generator("dasllama_gemm_gen::k6_tile", @@k6_tile_gen) register_llvm_code_generator("dasllama_gemm_gen::q40_gemv", @@q40_gemv_gen) register_llvm_code_generator("dasllama_gemm_gen::q40_tile", @@q40_tile_gen) + register_llvm_code_generator("dasllama_gemm_gen::iq4xs_gemv", @@iq4xs_gemv_gen) + register_llvm_code_generator("dasllama_gemm_gen::iq4xs_tile", @@iq4xs_tile_gen) register_llvm_code_generator("dasllama_gemm_gen::q8q8_witness", @@witness_gen) } diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_schema.das b/modules/dasLLAMA/dasllama/dasllama_gemm_schema.das index 9a2f769760..b76cfa8b60 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_schema.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_schema.das @@ -58,24 +58,27 @@ let Q51_QPB = 20l let Q51_SPB = 4l //! Quant-plane bytes per 256-weight superblock per row for a kq format id (4/5/6 = Q4_K/Q5_K/ -//! Q6_K, 40 = Q4_0). The ONE stride source for every fmt-branched kq walker/kernel/repack — -//! an unknown id panics instead of silently walking another format's stride. +//! Q6_K, 40 = Q4_0, 44 = IQ4_XS). The ONE stride source for every fmt-branched kq walker/ +//! kernel/repack — an unknown id panics instead of silently walking another format's stride. def kq_qsb(fmt : int) : int64 { if (fmt == 4) return kq_qsb(KqFmt.k4) if (fmt == 5) return kq_qsb(KqFmt.k5) if (fmt == 6) return kq_qsb(KqFmt.k6) if (fmt == 40) return kq_qsb(KqFmt.q40) + if (fmt == 44) return kq_qsb(KqFmt.iq4xs) panic("kq_qsb: unknown kq format id {fmt}") return 0l } //! Scale-plane bytes per 256-weight superblock per row (see kq_qsb): k4/k5 20 (16B disk block -//! + 4B pad, decoded in place at repack), k6 18 (native), q40 16 (8 x f16 d, verbatim). +//! + 4B pad, decoded in place at repack), k6 18 (native), q40 16 (8 x f16 d, verbatim), +//! iq4xs 20 (f16 d + 8 int8 sub-scales, decoded at transcode, in the k4 row shape). def kq_ssb(fmt : int) : int64 { if (fmt == 4) return kq_ssb(KqFmt.k4) if (fmt == 5) return kq_ssb(KqFmt.k5) if (fmt == 6) return kq_ssb(KqFmt.k6) if (fmt == 40) return kq_ssb(KqFmt.q40) + if (fmt == 44) return kq_ssb(KqFmt.iq4xs) panic("kq_ssb: unknown kq format id {fmt}") return 0l } diff --git a/modules/dasLLAMA/dasllama/dasllama_gguf.das b/modules/dasLLAMA/dasllama/dasllama_gguf.das index 9ce5b4b1ca..af90b089f2 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gguf.das +++ b/modules/dasLLAMA/dasllama/dasllama_gguf.das @@ -50,6 +50,7 @@ let GGML_TYPE_Q8_0 = 8 let GGML_TYPE_Q4_K = 12 let GGML_TYPE_Q5_K = 13 let GGML_TYPE_Q6_K = 14 +let GGML_TYPE_IQ4_XS = 23 let GGML_TYPE_BF16 = 30 let GGML_TYPE_MXFP4 = 39 @@ -701,6 +702,49 @@ def gguf_transcode_q40(m : GGUFMeta; srcbytes : array | #; name : string; } } +//! Transcode an IQ4_XS tensor into the iq4xs planes (see gguf_transcode_q4k; strides 128/20, +//! exact): the 136B disk superblock splits into the verbatim 128 nibble bytes and a DECODED +//! 20B scale row (f16 d, 2 pad, 8 x int8 (ls - 32), 8 pad — transcode_iq4xs_superblock). +def gguf_transcode_iq4xs(m : GGUFMeta; srcbytes : array | #; name : string; var kq : array; var ks : array; eloff, expect_n : int64; src_off : int64 = 0l) { + let ti = kq_transcode_check(m, name, GGML_TYPE_IQ4_XS, "IQ4_XS", src_off, expect_n) + let nb = expect_n / 256l + if (nb <= 0l) { + return + } + guard_dst(name, "iq4xs quant plane", (eloff / 256l) * IQ4XS_QSB, nb * IQ4XS_QSB, long_length(kq)) + guard_dst(name, "iq4xs scale plane", (eloff / 256l) * IQ4XS_SSB, nb * IQ4XS_SSB, long_length(ks)) + with_tensor_view(m, srcbytes, ti) $(bytes, tbase) { + let bo = tbase + (src_off / 256l) * 136l + unsafe { + let srcp = addr(bytes[bo]) + var kqp = addr(kq[(eloff / 256l) * IQ4XS_QSB]) + var ksp = addr(ks[(eloff / 256l) * IQ4XS_SSB]) + maybe_parallel_for(0, int(nb), transcode_jobs(nb, 136l)) $(rb, re) { + unsafe { + for (sb in range64(int64(rb), int64(re))) { + let src = srcp + sb * 136l + var row = ksp + sb * IQ4XS_SSB + row[0] = src[0] + row[1] = src[1] + row[2] = uint8(0) + row[3] = uint8(0) + let sh = uint(src[2]) | (uint(src[3]) << 8u) + for (blk in range64(8l)) { + let lo = (uint(src[4l + blk / 2l]) >> uint(4l * (blk % 2l))) & 15u + let ls = int(lo | (((sh >> uint(2l * blk)) & 3u) << 4u)) + row[4l + blk] = uint8(ls - 32) + } + for (i in range64(8l)) { + row[12l + i] = uint8(0) + } + bcopy(kqp + sb * IQ4XS_QSB, src + 8l, 128l) + } + } + } + } + } +} + //! Transcode a Q5_1 tensor into the q51 planes (per 32-BLOCK strides 20/4 — see Q51_QB): each //! 24B disk block splits into 2B d + 2B m (scale plane) and 16B nibbles + 4B qh (quant plane), //! all verbatim. Exact; `eloff`/`src_off`/`expect_n` are element offsets, % 32. diff --git a/modules/dasLLAMA/dasllama/dasllama_gpu_resident.das b/modules/dasLLAMA/dasllama/dasllama_gpu_resident.das index 48b7a198b7..652c4db8a0 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gpu_resident.das +++ b/modules/dasLLAMA/dasllama/dasllama_gpu_resident.das @@ -278,6 +278,11 @@ def trim_model_planes(var t : Model) : bool { t.embs |> reserve_resize(nsb * Q40_SSB) memcpy(addr(t.embq[0]), addr(t.q40q[sb0 * Q40_QSB]), nsb * Q40_QSB) memcpy(addr(t.embs[0]), addr(t.q40s[sb0 * Q40_SSB]), nsb * Q40_SSB) + } elif (t.emb_fmt == KqFmt.iq4xs) { + t.embq |> reserve_resize(nsb * IQ4XS_QSB) + t.embs |> reserve_resize(nsb * IQ4XS_SSB) + memcpy(addr(t.embq[0]), addr(t.iq4xsq[sb0 * IQ4XS_QSB]), nsb * IQ4XS_QSB) + memcpy(addr(t.embs[0]), addr(t.iq4xss[sb0 * IQ4XS_SSB]), nsb * IQ4XS_SSB) } else { to_log(LOG_WARNING, "dasLLAMA trim: emb fmt '{t.emb_fmt}' has no kq plane pair - declining\n") return false diff --git a/modules/dasLLAMA/dasllama/dasllama_image.das b/modules/dasLLAMA/dasllama/dasllama_image.das index 4329231b91..a7bacc72d3 100644 --- a/modules/dasLLAMA/dasllama/dasllama_image.das +++ b/modules/dasLLAMA/dasllama/dasllama_image.das @@ -38,7 +38,7 @@ require dasllama/dasllama_load // WhisperModel.enc) contribute their planes under dotted names ("enc.fblob"); string-array // fields ride the meta blob via serialize_strings — raw string pointers can't be planes. -let IMAGE_VERSION = 17 // 17: merged lineages - master's 14 and this branch's 14-16 (qwen3v f16 twin plane, devwf16 bake) numbered independently +let IMAGE_VERSION = 18 // 18: the iq4xs plane pair (IQ4_XS native tier) //! The metal (blob-only) flavor's identity tag: q8 planes ride the 34B block_q8_0 blob and the //! kq scale planes their GPU forms (convert_model_to_metal_blob) — flavors are per-config and @@ -957,7 +957,7 @@ def save_model_image(var t : Model; path : string; tag : string = ""; quant : st // streamed planes size as empty, so their bytes plus a page each are added here; over-reserve is free (dwrite_close truncates) def private stream_extra_bytes(t : Model; jobs : array) : uint64 { var extra = 0ul - for (fname in ["qblob", "mblob", "k4q", "k5q", "k6q", "q40q", "q51q"]) { + for (fname in ["qblob", "mblob", "k4q", "k5q", "k6q", "q40q", "q51q", "iq4xsq"]) { let sb = stream_plane_bytes(t, jobs, fname) if (sb > 0l) { extra += uint64(sb) + uint64(IMAGE_PAGE) diff --git a/modules/dasLLAMA/dasllama/dasllama_kqformat.das b/modules/dasLLAMA/dasllama/dasllama_kqformat.das index 2cfacb9033..38e7eb8bd4 100644 --- a/modules/dasLLAMA/dasllama/dasllama_kqformat.das +++ b/modules/dasLLAMA/dasllama/dasllama_kqformat.das @@ -13,7 +13,8 @@ module dasllama_kqformat shared public require dasllama/dasllama_lint public //! Per-weight storage format under a q8-mode load with native planes: q8 = qblob/qscales; -//! k4/k5/k6/q40 = superblock plane pairs (mixed PER TENSOR); q51 = native Q5_1 per-32 planes. +//! k4/k5/k6/q40/iq4xs = superblock plane pairs (mixed PER TENSOR); q51 = native Q5_1 per-32 +//! planes. New members append — the int value is the device stack tag and the image plane id. enum KqFmt : uint8 { q8 k4 @@ -21,19 +22,20 @@ enum KqFmt : uint8 { k6 q40 q51 + iq4xs } //! The superblock-lattice formats (Q8_K-form activations, % 256 rows, repack + stamped kq //! kernels). q51 deliberately fails this: it rides per-32 planes with Q8_0-form activations — //! a `fmt != KqFmt.q8` test does not imply the kq lattice; branch on kq_sb where it does. -def kq_sb(f : KqFmt) : bool => f == KqFmt.k4 || f == KqFmt.k5 || f == KqFmt.k6 || f == KqFmt.q40 +def kq_sb(f : KqFmt) : bool => f == KqFmt.k4 || f == KqFmt.k5 || f == KqFmt.k6 || f == KqFmt.q40 || f == KqFmt.iq4xs //! The same predicate over the int id space the GPU drivers carry per stack/plane. -def kq_sb(fmt : int) : bool => fmt == int(KqFmt.k4) || fmt == int(KqFmt.k5) || fmt == int(KqFmt.k6) || fmt == int(KqFmt.q40) +def kq_sb(fmt : int) : bool => fmt == int(KqFmt.k4) || fmt == int(KqFmt.k5) || fmt == int(KqFmt.k6) || fmt == int(KqFmt.q40) || fmt == int(KqFmt.iq4xs) -//! KqFmt -> the kernel/IR format id space (4/5/6/40) — the ONE bridge between the enum and the -//! int ids the generated kernels take as runtime params. q8/q51 are not kq superblock -//! formats and panic. +//! KqFmt -> the kernel/IR format id space — the ONE bridge between the enum and the int ids the +//! generated kernels take as runtime params. Mnemonic ids: a K-quant is its bit width (4/5/6), +//! Q4_0 is 40, an i-quant is bit width x 10 + a variant digit (IQ4_XS = 44). q8/q51 panic. def kq_schema_id(f : KqFmt) : int { if (f == KqFmt.k4) { return 4 @@ -47,6 +49,9 @@ def kq_schema_id(f : KqFmt) : int { if (f == KqFmt.q40) { return 40 } + if (f == KqFmt.iq4xs) { + return 44 + } panic("kq_schema_id: not a kq superblock format") return 0 } @@ -63,12 +68,17 @@ let Q40_QSB = 128l // Q4_0 quant plane: 128 nibble bytes (k4 tiling) let Q40_SSB = 16l // Q4_0 scale plane: 8 x f16 d, verbatim let Q51_QB = 20l // q5_1 quant plane PER 32-BLOCK: 16 nibble bytes (k/k+16 pairing) + 4 qh let Q51_SB = 4l // q5_1 scale plane PER 32-BLOCK: f16 d + f16 m +let IQ4XS_QSB = 128l // IQ4_XS quant plane: 128 nibble bytes, the disk k/k+16 pairing (q40 tiling) +let IQ4XS_SSB = 20l // IQ4_XS scale plane: f16 d, 2 pad, 8 x int8 (ls - 32), 8 pad — the k4 decoded-row shape + +//! The IQ4_NL / IQ4_XS nibble codebook: weight = scale x IQ4NL_LUT[nibble] (ggml's kvalues_iq4nl). +let IQ4NL_LUT = fixed_array(-127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113) //! Quant-plane bytes per stride unit — the typed stride surface. The stride unit is one //! 256-weight superblock for the kq lattice and one 32-weight block for q51 (see kq_elems). //! q8 panics (its planes are byte-per-elem, not stride-walked here). def kq_qsb(f : KqFmt) : int64 { - if (f == KqFmt.k4 || f == KqFmt.q40) { + if (f == KqFmt.k4 || f == KqFmt.q40 || f == KqFmt.iq4xs) { return K4_QSB } if (f == KqFmt.k5) { @@ -86,7 +96,7 @@ def kq_qsb(f : KqFmt) : int64 { //! Scale-plane bytes per stride unit (see kq_qsb for the unit). def kq_ssb(f : KqFmt) : int64 { - if (f == KqFmt.k4 || f == KqFmt.k5) { + if (f == KqFmt.k4 || f == KqFmt.k5 || f == KqFmt.iq4xs) { return K4_SSB } if (f == KqFmt.k6) { diff --git a/modules/dasLLAMA/dasllama/dasllama_layout.das b/modules/dasLLAMA/dasllama/dasllama_layout.das index aa0e5e6766..fea50859a5 100644 --- a/modules/dasLLAMA/dasllama/dasllama_layout.das +++ b/modules/dasLLAMA/dasllama/dasllama_layout.das @@ -292,6 +292,10 @@ def private repack_regions(var t : Model; regs : array) { var q40sp : uint8? = null var q51qp : uint8? = null var q51sp : uint8? = null + var iq4xsqp : uint8? = null + var iq4xssp : uint8? = null + if (!empty(t.iq4xsq)) { iq4xsqp = addr(t.iq4xsq[0]) } + if (!empty(t.iq4xss)) { iq4xssp = addr(t.iq4xss[0]) } if (!empty(t.qblob)) { qbp = addr(t.qblob[0]) } if (!empty(t.qscales)) { qsp = addr(t.qscales[0]) } if (!empty(t.mxq)) { mxqp = addr(t.mxq[0]) } @@ -323,8 +327,8 @@ def private repack_regions(var t : Model; regs : array) { let sb = rp[i].off / 256l let qsb = kq_qsb(f) let ssb = kq_ssb(f) - var kqp = f == 4 ? k4qp : (f == 5 ? k5qp : (f == 6 ? k6qp : q40qp)) - var ksp = f == 4 ? k4sp : (f == 5 ? k5sp : (f == 6 ? k6sp : q40sp)) + var kqp = f == 4 ? k4qp : (f == 5 ? k5qp : (f == 6 ? k6qp : (f == 40 ? q40qp : iq4xsqp))) + var ksp = f == 4 ? k4sp : (f == 5 ? k5sp : (f == 6 ? k6sp : (f == 40 ? q40sp : iq4xssp))) invoke(rkq, f, kqp + sb * qsb, ksp + sb * ssb, rp[i].n, rp[i].d) } } @@ -450,6 +454,8 @@ def private push_repack_kq(var regs : array; fmt : KqFmt; woff, n, d push_repack(regs, 6, woff, n, d) } elif (fmt == KqFmt.q40) { push_repack(regs, 40, woff, n, d) + } elif (fmt == KqFmt.iq4xs) { + push_repack(regs, 44, woff, n, d) } } @@ -722,10 +728,12 @@ def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice let sb0 = woff / KQ_SUPERBLOCK_ELEMS let qp = (fmt == KqFmt.k4 ? addr(t.k4q[sb0 * kq_qsb(4)]) : (fmt == KqFmt.k5 ? addr(t.k5q[sb0 * kq_qsb(5)]) - : (fmt == KqFmt.k6 ? addr(t.k6q[sb0 * kq_qsb(6)]) : addr(t.q40q[sb0 * kq_qsb(40)])))) + : (fmt == KqFmt.k6 ? addr(t.k6q[sb0 * kq_qsb(6)]) + : (fmt == KqFmt.q40 ? addr(t.q40q[sb0 * kq_qsb(40)]) : addr(t.iq4xsq[sb0 * kq_qsb(44)]))))) let sp = (fmt == KqFmt.k4 ? addr(t.k4s[sb0 * kq_ssb(4)]) : (fmt == KqFmt.k5 ? addr(t.k5s[sb0 * kq_ssb(5)]) - : (fmt == KqFmt.k6 ? addr(t.k6s[sb0 * kq_ssb(6)]) : addr(t.q40s[sb0 * kq_ssb(40)])))) + : (fmt == KqFmt.k6 ? addr(t.k6s[sb0 * kq_ssb(6)]) + : (fmt == KqFmt.q40 ? addr(t.q40s[sb0 * kq_ssb(40)]) : addr(t.iq4xss[sb0 * kq_ssb(44)]))))) var wqp = addr(wq[0]) var wsp = addr(ws[0]) let njobs = is_job_que_available() ? min(nslices, 4 * (get_total_hw_jobs() + 1)) : 1 diff --git a/modules/dasLLAMA/dasllama/dasllama_load.das b/modules/dasLLAMA/dasllama/dasllama_load.das index 8b07acb97d..5c4e0429aa 100644 --- a/modules/dasLLAMA/dasllama/dasllama_load.das +++ b/modules/dasLLAMA/dasllama/dasllama_load.das @@ -41,6 +41,7 @@ struct private LayoutSizes { k6_n : int64 q40_n : int64 q51_n : int64 + iq4xs_n : int64 pleq8_n : int64 // the dedicated q8 plane holding ONLY the PLE table (fp32/q4 serving) } @@ -53,6 +54,7 @@ struct private KqCursors { k6 : int64 q40 : int64 q51 : int64 + iq4xs : int64 } def private kq_take(var cur : KqCursors; f : KqFmt; n : int64) : int64 { @@ -81,6 +83,11 @@ def private kq_take(var cur : KqCursors; f : KqFmt; n : int64) : int64 { cur.q51 += n return o } + if (f == KqFmt.iq4xs) { + let o = cur.iq4xs + cur.iq4xs += n + return o + } let o = cur.wo cur.wo += n return o @@ -380,7 +387,7 @@ def private layout_offsets(var t : Model) : LayoutSizes { // nolint:STYLE037,S fo += dim } return LayoutSizes(fblob_n = fo, wblob_n = cur.wo, mx_n = mx, bf16_n = bf16, - k4_n = cur.k4, k5_n = cur.k5, k6_n = cur.k6, q40_n = cur.q40, q51_n = cur.q51, pleq8_n = pleq8) + k4_n = cur.k4, k5_n = cur.k5, k6_n = cur.k6, q40_n = cur.q40, q51_n = cur.q51, iq4xs_n = cur.iq4xs, pleq8_n = pleq8) } @@ -639,6 +646,8 @@ def stream_field_of(t : Model; fmt : KqFmt) : string { return "q40q" } elif (fmt == KqFmt.q51) { return "q51q" + } elif (fmt == KqFmt.iq4xs) { + return "iq4xsq" } elif (t.quant == QuantMode.q8) { // the blob flavor carries q8 as gguf-native 34B blocks in ONE plane, never the split pair return t.metal_blob ? "mblob" : "qblob" @@ -755,6 +764,8 @@ def fill_stream_plane(m : GGUFMeta; bytes : array | #; var t : Model; fie memcpy(addr(t.k6s[(j.woff / 256l) * K6_SSB]), addr(temp_s[0]), sh.sb) } elif (j.fmt == KqFmt.q40) { memcpy(addr(t.q40s[(j.woff / 256l) * Q40_SSB]), addr(temp_s[0]), sh.sb) + } elif (j.fmt == KqFmt.iq4xs) { + memcpy(addr(t.iq4xss[(j.woff / 256l) * IQ4XS_SSB]), addr(temp_s[0]), sh.sb) } elif (j.fmt == KqFmt.q51) { memcpy(addr(t.q51s[(j.woff / 32l) * Q51_SB]), addr(temp_s[0]), sh.sb) } elif (sh.sb == 0l && j.n > 0l && !t.metal_blob) { @@ -786,6 +797,9 @@ def fill_stream_plane(m : GGUFMeta; bytes : array | #; var t : Model; fie } elif (r.fmt == 40) { invoke(g_stream_repack, 40, addr(temp_q[((r.off - j.woff) / 256l) * Q40_QSB]), addr(t.q40s[(r.off / 256l) * Q40_SSB]), r.n, r.d) + } elif (r.fmt == 44) { + invoke(g_stream_repack, 44, addr(temp_q[((r.off - j.woff) / 256l) * IQ4XS_QSB]), + addr(t.iq4xss[(r.off / 256l) * IQ4XS_SSB]), r.n, r.d) } } } @@ -998,6 +1012,9 @@ def private transcode_kq_tensor(m : GGUFMeta; bytes : array | #; name : s } elif (fmt == KqFmt.q40) { gguf_transcode_q40(m, bytes, name, kq, ks, eloff, n, src_off) return "q40 transcode (Q4_0)" + } elif (fmt == KqFmt.iq4xs) { + gguf_transcode_iq4xs(m, bytes, name, kq, ks, eloff, n, src_off) + return "iq4xs transcode (IQ4_XS)" } gguf_transcode_q51(m, bytes, name, kq, ks, eloff, n, src_off) return "q51 transcode (Q5_1)" @@ -1036,6 +1053,8 @@ def private load_big(m : GGUFMeta; bytes : array | #; name : string; var kind = transcode_kq_tensor(m, bytes, name, fmt, t.k6q, t.k6s, woff, n, src_off, scratch) } elif (fmt == KqFmt.q40) { kind = transcode_kq_tensor(m, bytes, name, fmt, t.q40q, t.q40s, woff, n, src_off, scratch) + } elif (fmt == KqFmt.iq4xs) { + kind = transcode_kq_tensor(m, bytes, name, fmt, t.iq4xsq, t.iq4xss, woff, n, src_off, scratch) } elif (fmt == KqFmt.q51) { kind = transcode_kq_tensor(m, bytes, name, fmt, t.q51q, t.q51s, woff, n, src_off, scratch) } elif (t.quant == QuantMode.q8) { @@ -1135,8 +1154,8 @@ def private log_load_report(t : Model) { to_log(LOG_INFO, "dasLLAMA noisy: backend '{active_kernel_backend()}' | kquant_native {get_kquant_native()} q40_native {get_kq_q40_native()} q50_native {get_kq_q50_native()}\n") // the tier arms at [init], before a --noisy CLI flag can land — restate it here so the flag alone suffices to_log(LOG_INFO, "dasLLAMA noisy: GPU tier installed {moe_gpu_tier_installed()} | want auto {gpu_want_auto()}, moe layers {gpu_want_moe_layers()}, stream {gpu_want_moe_stream()}\n") - to_log(LOG_INFO, "dasLLAMA noisy: format tags - q8 {tally_fmt(t, KqFmt.q8)}, k4 {tally_fmt(t, KqFmt.k4)}, k5 {tally_fmt(t, KqFmt.k5)}, k6 {tally_fmt(t, KqFmt.k6)}, q40 {tally_fmt(t, KqFmt.q40)}, q51 {tally_fmt(t, KqFmt.q51)}\n") - to_log(LOG_INFO, "dasLLAMA noisy: plane MB - q8 {float(long_length(t.qblob)) / mb} (+scales {float(long_length(t.qscales) * 4l) / mb}), k4 {float(long_length(t.k4q) + long_length(t.k4s)) / mb}, k5 {float(long_length(t.k5q) + long_length(t.k5s)) / mb}, k6 {float(long_length(t.k6q) + long_length(t.k6s)) / mb}, q40 {float(long_length(t.q40q) + long_length(t.q40s)) / mb}, q51 {float(long_length(t.q51q) + long_length(t.q51s)) / mb}, f32 {float(long_length(t.wblob) * 4l) / mb}\n") + to_log(LOG_INFO, "dasLLAMA noisy: format tags - q8 {tally_fmt(t, KqFmt.q8)}, k4 {tally_fmt(t, KqFmt.k4)}, k5 {tally_fmt(t, KqFmt.k5)}, k6 {tally_fmt(t, KqFmt.k6)}, q40 {tally_fmt(t, KqFmt.q40)}, q51 {tally_fmt(t, KqFmt.q51)}, iq4xs {tally_fmt(t, KqFmt.iq4xs)}\n") + to_log(LOG_INFO, "dasLLAMA noisy: plane MB - q8 {float(long_length(t.qblob)) / mb} (+scales {float(long_length(t.qscales) * 4l) / mb}), k4 {float(long_length(t.k4q) + long_length(t.k4s)) / mb}, k5 {float(long_length(t.k5q) + long_length(t.k5s)) / mb}, k6 {float(long_length(t.k6q) + long_length(t.k6s)) / mb}, q40 {float(long_length(t.q40q) + long_length(t.q40s)) / mb}, q51 {float(long_length(t.q51q) + long_length(t.q51s)) / mb}, iq4xs {float(long_length(t.iq4xsq) + long_length(t.iq4xss)) / mb}, f32 {float(long_length(t.wblob) * 4l) / mb}\n") } // GGML disk type -> plane format tag (non-K-quant types ride the classic q8 path) @@ -1150,6 +1169,9 @@ def private kq_fmt_of(gt : int) : KqFmt { if (gt == GGML_TYPE_Q6_K) { return KqFmt.k6 } + if (gt == GGML_TYPE_IQ4_XS) { + return KqFmt.iq4xs + } if (gt == GGML_TYPE_Q4_0 && get_kq_q40_native()) { return KqFmt.q40 } @@ -1983,6 +2005,7 @@ def private load_gguf_parsed(var t : Model; m : GGUFMeta; bytes : array | g_stream_plane_total["k6q"] = (sz.k6_n / 256l) * K6_QSB g_stream_plane_total["q40q"] = (sz.q40_n / 256l) * Q40_QSB g_stream_plane_total["q51q"] = (sz.q51_n / 32l) * Q51_QB + g_stream_plane_total["iq4xsq"] = (sz.iq4xs_n / 256l) * IQ4XS_QSB let wb = mode == QuantMode.q8 ? sz.wblob_n : 0l g_stream_plane_total[t.metal_blob ? "mblob" : "qblob"] = (t.metal_blob ? (wb / Q8_BLOCK_ELEMS) * (Q8_QPB + Q8_SPB) : wb) @@ -2027,6 +2050,14 @@ def private load_gguf_parsed(var t : Model; m : GGUFMeta; bytes : array | t.q51s |> reserve((sz.q51_n / 32l) * Q51_SB) t.q51s |> resize((sz.q51_n / 32l) * Q51_SB) } + if (sz.iq4xs_n > 0l) { + if (!stream_q) { + t.iq4xsq |> reserve((sz.iq4xs_n / 256l) * IQ4XS_QSB) + t.iq4xsq |> resize((sz.iq4xs_n / 256l) * IQ4XS_QSB) + } + t.iq4xss |> reserve((sz.iq4xs_n / 256l) * IQ4XS_SSB) + t.iq4xss |> resize((sz.iq4xs_n / 256l) * IQ4XS_SSB) + } if (mode == QuantMode.q8) { if (!stream_q) { t.qblob |> reserve(sz.wblob_n) diff --git a/modules/dasLLAMA/dasllama/dasllama_math.das b/modules/dasLLAMA/dasllama/dasllama_math.das index 3dbe5e8d37..77384d6c18 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math.das +++ b/modules/dasLLAMA/dasllama/dasllama_math.das @@ -1084,6 +1084,7 @@ struct KernelBackend { kq_rows_k5 : MatmulKqRowsFn = @@kq_unset_rows kq_rows_k6 : MatmulKqRowsFn = @@kq_unset_rows kq_rows_q40 : MatmulKqRowsFn = @@kq_unset_rows + kq_rows_iq4xs : MatmulKqRowsFn = @@kq_unset_rows kq_batch : MatmulKqBatchFn = @@kq_unset_batch kq_groupn : MatmulKqGroupNFn = @@kq_unset_groupn kq_batch_groupn : MatmulKqBatchGroupNFn = @@kq_unset_batch_groupn @@ -1146,6 +1147,7 @@ var g_kq_rows_k4 = @@kq_unset_rows var g_kq_rows_k5 = @@kq_unset_rows var g_kq_rows_k6 = @@kq_unset_rows var g_kq_rows_q40 = @@kq_unset_rows +var g_kq_rows_iq4xs = @@kq_unset_rows var g_mm_kq_batch = @@kq_unset_batch var g_mm_kq_groupn = @@kq_unset_groupn var g_mm_kq_batch_groupn = @@kq_unset_batch_groupn @@ -1473,7 +1475,7 @@ def active_q8_layout_mr() : int64 => g_bake_cpu_override ? g_bake_cpu.q8_mr : g_ //! family split; the loader freezes these onto the Model at repack time. def active_kq_layout_mr(fmt : int) : int64 { if (g_bake_cpu_override) { - return fmt == 4 ? g_bake_cpu.kq_mr4 : (fmt == 5 ? g_bake_cpu.kq_mr5 : (fmt == 40 ? g_bake_cpu.kq_mr40 : g_bake_cpu.kq_mr6)) + return fmt == 4 ? g_bake_cpu.kq_mr4 : (fmt == 5 ? g_bake_cpu.kq_mr5 : (fmt == 40 ? g_bake_cpu.kq_mr40 : (fmt == 44 ? g_bake_cpu.kq_mr44 : g_bake_cpu.kq_mr6))) } return int64(invoke(g_active_kq_layout, fmt)) } @@ -1649,6 +1651,7 @@ def private activate(be : KernelBackend) { g_kq_rows_k5 = be.kq_rows_k5 g_kq_rows_k6 = be.kq_rows_k6 g_kq_rows_q40 = be.kq_rows_q40 + g_kq_rows_iq4xs = be.kq_rows_iq4xs g_mm_kq_batch = be.kq_batch g_mm_kq_groupn = be.kq_groupn g_mm_kq_batch_groupn = be.kq_batch_groupn @@ -1851,6 +1854,7 @@ def kq_rows_fn(fmt : int) : MatmulKqRowsFn { if (fmt == 5) return g_kq_rows_k5 if (fmt == 6) return g_kq_rows_k6 if (fmt == 40) return g_kq_rows_q40 + if (fmt == 44) return g_kq_rows_iq4xs return @@kq_unset_rows } @@ -3211,6 +3215,9 @@ def private restore_math_fn_defaults { // nolint:STYLE037,STYLE038 — flat seam if (g_kq_rows_q40 == null) { g_kq_rows_q40 = @@kq_unset_rows } + if (g_kq_rows_iq4xs == null) { + g_kq_rows_iq4xs = @@kq_unset_rows + } if (g_mm_kq_batch == null) { g_mm_kq_batch = @@kq_unset_batch } diff --git a/modules/dasLLAMA/dasllama/dasllama_math_default.das b/modules/dasLLAMA/dasllama/dasllama_math_default.das index e90019a36f..27a93de290 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_default.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_default.das @@ -546,6 +546,35 @@ def dot_q40q8(kqrow : uint8 const?; ksrow : uint8 const?; xqp : int8 const?; xsp return acc } +//! IQ4_XS row dot off the DISK-ORDER planes: per 32-block idot = sum IQ4NL_LUT[q] * x (signed, +//! exact int), folded as sum sc_blk * idot_blk over the superblock, then * (d * xs). No bsum +//! term — the codebook carries the sign, so xbsp is unused. +[unused_argument(xbsp)] +def dot_iq4xs_q8(kqrow : uint8 const?; ksrow : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n : int64) : float { + var acc = 0.0 + unsafe { + for (sb in range64(n / 256l)) { + let kqo = sb * 128l + let kso = sb * 20l + let dd = f16_to_f32(uint(ksrow[kso]) | (uint(ksrow[kso + 1l]) << 8u)) + var isum = 0 + for (blk in range64(8l)) { + let ab = (sb * 8l + blk) * 32l + var idot = 0 + for (l in range64(16l)) { + let q = int(kqrow[kqo + blk * 16l + l]) + idot += int(IQ4NL_LUT[q & 15]) * int(xqp[ab + l]) + idot += int(IQ4NL_LUT[q >> 4]) * int(xqp[ab + 16l + l]) + } + let u = int(ksrow[kso + 4l + blk]) + isum += (u < 128 ? u : u - 256) * idot + } + acc += float(isum) * (dd * xsp[sb]) + } + } + return acc +} + // Row-range cores + full GEMVs per format. Not `private`: invoked through hoisted function // pointers from lifted worker lambdas (the fused chains) and the dispatch wrappers in common. def k4_rows_kernel(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, rb, re : int64) { @@ -584,6 +613,15 @@ def q40_rows_kernel(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp } } +def iq4xs_rows_kernel(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, rb, re : int64) { + let nsb = n / 256l + unsafe { + for (i in range64(rb, re)) { + yp[i] = dot_iq4xs_q8(kqp + i * nsb * 128l, ksp + i * nsb * 20l, xqp, xsp, xbsp, n) + } + } +} + def private kq_gemv_kernel(fmt : int; var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d : int64) { var myp = yp maybe_parallel_for(0, int(d), matmul_chunks_gemv(int(d), 1, n * d)) $(rb, re) { @@ -593,12 +631,59 @@ def private kq_gemv_kernel(fmt : int; var yp : float?; kqp : uint8 const?; ksp : k5_rows_kernel(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb), int64(re)) } elif (fmt == 40) { q40_rows_kernel(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb), int64(re)) + } elif (fmt == 44) { + iq4xs_rows_kernel(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb), int64(re)) } else { k6_rows_kernel(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb), int64(re)) } } } +//! q40 grp row dequant: the k4 nibble tiling with a per-block f16 d plane — w = d·(q − 8). +def private dequant_q40_row_grp(kqg : uint8 const?; ksg : uint8 const?; r, mr, nsb : int64; var dst : float?) { + unsafe { + for (sbi in range64(nsb)) { + let qb = sbi * 128l * mr + let sb = sbi * 16l * mr + for (blk in range64(8l)) { + let kb = sbi * 256l + blk * 32l + let d = f16_to_f32(uint(ksg[sb + blk * 2l * mr + 2l * r]) | (uint(ksg[sb + blk * 2l * mr + 2l * r + 1l]) << 8u)) + for (j in range64(4l)) { + for (t in range64(4l)) { + let nib = uint(kqg[qb + ((blk * 4l + j) * mr + r) * 4l + t]) + dst[kb + j * 4l + t] = d * float(int(nib & 15u) - 8) + dst[kb + 16l + j * 4l + t] = d * float(int(nib >> 4u) - 8) + } + } + } + } + } +} + +//! iq4xs grp row dequant: the k4 tiling + the k4 scale-row interleave (d, signed int8 sc, no +//! min) — w = (d·sc)·IQ4NL_LUT[q]. +def private dequant_iq4xs_row_grp(kqg : uint8 const?; ksg : uint8 const?; r, mr, nsb : int64; var dst : float?) { + unsafe { + for (sbi in range64(nsb)) { + let qb = sbi * 128l * mr + let sb = sbi * 20l * mr + let d = f16_to_f32(uint(ksg[sb + 2l * r]) | (uint(ksg[sb + 2l * r + 1l]) << 8u)) + for (blk in range64(8l)) { + let kb = sbi * 256l + blk * 32l + let u = int(ksg[sb + 4l * mr + blk * mr + r]) + let dl = d * float(u < 128 ? u : u - 256) + for (j in range64(4l)) { + for (t in range64(4l)) { + let nib = int(kqg[qb + ((blk * 4l + j) * mr + r) * 4l + t]) + dst[kb + j * 4l + t] = dl * float(IQ4NL_LUT[nib & 15]) + dst[kb + 16l + j * 4l + t] = dl * float(IQ4NL_LUT[nib >> 4]) + } + } + } + } + } +} + //! Dequant one row off the grp-REPACKED K-quant planes (repack_k4/k5/k6_grp layout). //! kqg/ksg = row's GROUP plane base (group g = row/mr); r = row % mr. //! embed_row's kq_repacked path — per-token, so scalar is fine. @@ -607,24 +692,11 @@ def dequant_kq_row_grp(fmt : int64; kqg : uint8 const?; ksg : uint8 const?; r, m let qsb = kq_qsb(int(fmt)) let ssb = kq_ssb(int(fmt)) if (fmt == 40l) { - // q40 grp: the k4 nibble tiling with a per-block f16 d plane — w = d·(q − 8) - unsafe { - for (sbi in range64(nsb)) { - let qb = sbi * qsb * mr - let sb = sbi * ssb * mr - for (blk in range64(8l)) { - let kb = sbi * 256l + blk * 32l - let d = f16_to_f32(uint(ksg[sb + blk * 2l * mr + 2l * r]) | (uint(ksg[sb + blk * 2l * mr + 2l * r + 1l]) << 8u)) - for (j in range64(4l)) { - for (t in range64(4l)) { - let nib = uint(kqg[qb + ((blk * 4l + j) * mr + r) * 4l + t]) - dst[kb + j * 4l + t] = d * float(int(nib & 15u) - 8) - dst[kb + 16l + j * 4l + t] = d * float(int(nib >> 4u) - 8) - } - } - } - } - } + dequant_q40_row_grp(kqg, ksg, r, mr, nsb, dst) + return + } + if (fmt == 44l) { + dequant_iq4xs_row_grp(kqg, ksg, r, mr, nsb, dst) return } unsafe { @@ -721,6 +793,8 @@ def matmul_kq_groupn(fmt : int; var y : array | #; kq : array | #; myp[ii] = dot_k5q8(kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) } elif (fmt == 40) { myp[ii] = dot_q40q8(kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) + } elif (fmt == 44) { + myp[ii] = dot_iq4xs_q8(kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) } else { myp[ii] = dot_k6q8(kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) } diff --git a/modules/dasLLAMA/dasllama/dasllama_math_gen.das b/modules/dasLLAMA/dasllama/dasllama_math_gen.das index c7e75be3ed..d1e9b307df 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_gen.das @@ -243,6 +243,10 @@ def q40q8_layout_gen() : int { return q8q8_repack_type(GEMM_REFERENCE_MR).interleave } +def iq4xsq8_layout_gen() : int { + return q8q8_repack_type(GEMM_REFERENCE_MR).interleave +} + // the KernelBackend.kq_layout slot form of kq_layout_of (registered on both gen backends) def private kq_layout_fmt_gen(fmt : int) : int => int(kq_layout_of(fmt)) @@ -258,6 +262,9 @@ def kq_layout_of(fmt : int) : int64 { if (fmt == 40) { return int64(q40q8_layout_gen()) } + if (fmt == 44) { + return int64(iq4xsq8_layout_gen()) + } return int64(k6q8_layout_gen()) } @@ -293,6 +300,38 @@ def q40_grp_row_dot(kqg : uint8 const?; ksg : uint8 const?; r, mr : int64; xqp : return acc } +//! One row's dot off the grp iq4xs planes, scalar — the iq4xs stubs' reference body and the +//! repack oracle: per 32-block idot = sum IQ4NL_LUT[q] * x, folded sum sc·idot over the +//! superblock (sc = the row's signed int8 sub-scale), then * (d * xs). No bsum term. +[unused_argument(xbsp)] +def iq4xs_grp_row_dot(kqg : uint8 const?; ksg : uint8 const?; r, mr : int64; xqp : int8 const?; xsp : float const?; xbsp : int const?; n : int64) : float { + var acc = 0.0 + let nsb = n / 256l + unsafe { + for (sbi in range64(nsb)) { + let qb = sbi * 128l * mr + let sb = sbi * 20l * mr + let d = f16_to_f32(uint(ksg[sb + 2l * r]) | (uint(ksg[sb + 2l * r + 1l]) << 8u)) + var isum = 0 + for (blk in range64(8l)) { + let b = sbi * 8l + blk + var idot = 0 + for (j in range64(4l)) { + for (t in range64(4l)) { + let nib = int(kqg[qb + ((blk * 4l + j) * mr + r) * 4l + t]) + idot += int(IQ4NL_LUT[nib & 15]) * int(xqp[b * 32l + j * 4l + t]) + idot += int(IQ4NL_LUT[nib >> 4]) * int(xqp[b * 32l + 16l + j * 4l + t]) + } + } + let u = int(ksg[sb + 4l * mr + blk * mr + r]) + isum += (u < 128 ? u : u - 256) * idot + } + acc += float(isum) * (d * xsp[sbi]) + } + } + return acc +} + //! The K-quant GEMV kernels (kq stage 4): rows [rb, re) of one plane-region pair off the grp //! kq planes (each format's OWN layout companion). Reference bodies = the scalar grp walk above, //! declining in lockstep. Not `private`: they ARE the kq_rows_* slots. @@ -431,6 +470,38 @@ def q40q8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp } } +[hint(unsafe_range_check, noalias = kqp, noalias = ksp, noalias = xqp, noalias = xsp, noalias = xbsp)] +def iq4xsq8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, rb, re : int64) : void { + let mr = int64(iq4xsq8_layout_gen()) + let nsb = n / 256l + unsafe { + for (i in range64(rb, re)) { + let g = i / mr + yp[i] = iq4xs_grp_row_dot(kqp + g * mr * nsb * 128l, ksp + g * mr * nsb * 20l, i % mr, mr, xqp, xsp, xbsp, n) + } + } +} + +[tune_perm(mr = 4), tune_perm(mr = 4, nrsplit = 2), tune_perm(mr = 8), tune_perm(mr = 8, nrsplit = 2), + tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "maddubs", width = 256, mr = 8, nrsplit = 2, requires = "avx2"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, nrsplit = 2, requires = "avxvnni|avx512vnni"), + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, nrsplit = 2, requires = "avx512vnni,avx512bw"), + tune_companion(fn = "iq4xsq8_gemv_gen", gen = "dasllama_gemm_gen::iq4xs_gemv"), + tune_companion(fn = "iq4xsq8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), + tune(gen = "dasllama_gemm_gen::iq4xs_tile", + fallback = "dot_vpdpbusd_width512_mr16;dot_vpdpbusd_width256_mr8;dot_maddubs_width256_mr8;mr4"), + hint(unsafe_range_check, noalias = kqg, noalias = ksg, noalias = xqp, noalias = xsp, noalias = xbsp)] +def iq4xsq8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d, g, t0 : int64) : void { + let mr = int64(iq4xsq8_layout_gen()) + unsafe { + for (t in range64(4l)) { + for (r in range64(mr)) { + yp[(t0 + t) * d + g * mr + r] = iq4xs_grp_row_dot(kqg, ksg, r, mr, xqp + (t0 + t) * n, xsp + (t0 + t) * (n / 256l), xbsp + (t0 + t) * (n / 16l), n) + } + } + } +} + //! The f16-scale GEMV companion (wscale_f16 rail): q8q8_gemv_gen's twin over the raw binary16 //! group-scale plane. Same rows-range contract; the generated body differs only in the scale //! fold's widen (fcvtl/vcvtph2ps). Not `private`: it IS the mm_rows_s16 slot. @@ -748,6 +819,8 @@ def private repack_kq_gen(fmt : int; var kq : uint8?; var ks : uint8?; n, d : in repack_k5_grp(kq, ks, n, d, kq_layout_of(5)) } elif (fmt == 40) { repack_q40_grp(kq, ks, n, d, kq_layout_of(40)) + } elif (fmt == 44) { + repack_iq4xs_grp(kq, ks, n, d, kq_layout_of(44)) } else { repack_k6_grp(kq, ks, n, d, kq_layout_of(6)) } @@ -760,6 +833,8 @@ def private repack_kq_bake(fmt : int; var kq : uint8?; var ks : uint8?; n, d : i repack_k5_grp(kq, ks, n, d, active_kq_layout_mr(5)) } elif (fmt == 40) { repack_q40_grp(kq, ks, n, d, active_kq_layout_mr(40)) + } elif (fmt == 44) { + repack_iq4xs_grp(kq, ks, n, d, active_kq_layout_mr(44)) } else { repack_k6_grp(kq, ks, n, d, active_kq_layout_mr(6)) } @@ -1270,6 +1345,8 @@ def private kq_kernel_gen(fmt : int; var yp : float?; kqp : uint8 const?; ksp : k5q8_gemv_gen(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb) * mr, int64(re) * mr) } elif (fmt == 40) { q40q8_gemv_gen(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb) * mr, int64(re) * mr) + } elif (fmt == 44) { + iq4xsq8_gemv_gen(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb) * mr, int64(re) * mr) } else { k6q8_gemv_gen(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb) * mr, int64(re) * mr) } @@ -1282,6 +1359,8 @@ def private kq_kernel_gen(fmt : int; var yp : float?; kqp : uint8 const?; ksp : myp[i] = dot_k5q8(kqp + i * nsb * 160l, ksp + i * nsb * 20l, xqp, xsp, xbsp, n) } elif (fmt == 40) { myp[i] = dot_q40q8(kqp + i * nsb * 128l, ksp + i * nsb * 16l, xqp, xsp, xbsp, n) + } elif (fmt == 44) { + myp[i] = dot_iq4xs_q8(kqp + i * nsb * 128l, ksp + i * nsb * 20l, xqp, xsp, xbsp, n) } else { myp[i] = dot_k6q8(kqp + i * nsb * 192l, ksp + i * nsb * 18l, xqp, xsp, xbsp, n) } @@ -1297,7 +1376,7 @@ def private kq_batch_cell_gen(fmt : int; var myp : float?; kqp : uint8 const?; k let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) let nb16 = n / 16l - let packed = fmt == 4 || fmt == 40 // pure-nibble planes: the tile reads them directly + let packed = fmt == 4 || fmt == 40 || fmt == 44 // pure-nibble planes: the tile reads them directly var scratch : array if (!packed) { scratch |> resize(mr * nsb * 256l) @@ -1323,6 +1402,8 @@ def private kq_batch_cell_gen(fmt : int; var myp : float?; kqp : uint8 const?; k k5q8_tile_gen(myp, scp, ksg, xqp, xsp, xbsp, n, d, g, tk) } elif (fmt == 40) { q40q8_tile_gen(myp, kqg, ksg, xqp, xsp, xbsp, n, d, g, tk) + } elif (fmt == 44) { + iq4xsq8_tile_gen(myp, kqg, ksg, xqp, xsp, xbsp, n, d, g, tk) } else { k6q8_tile_gen(myp, scp, ksg, xqp, xsp, xbsp, n, d, g, tk) } @@ -1335,6 +1416,8 @@ def private kq_batch_cell_gen(fmt : int; var myp : float?; kqp : uint8 const?; k k5q8_gemv_gen(myp + tk * d, kqp, ksp, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n, g * mr, (g + 1l) * mr) } elif (fmt == 40) { q40q8_gemv_gen(myp + tk * d, kqp, ksp, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n, g * mr, (g + 1l) * mr) + } elif (fmt == 44) { + iq4xsq8_gemv_gen(myp + tk * d, kqp, ksp, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n, g * mr, (g + 1l) * mr) } else { k6q8_gemv_gen(myp + tk * d, kqp, ksp, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n, g * mr, (g + 1l) * mr) } @@ -1368,6 +1451,8 @@ def private kq_batch_kernel_gen(fmt : int; var yp : float?; kqp : uint8 const?; myp[tk * d + i] = dot_k5q8(kqp + i * nsb * 160l, ksp + i * nsb * 20l, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * (n / 16l), n) } elif (fmt == 40) { myp[tk * d + i] = dot_q40q8(kqp + i * nsb * 128l, ksp + i * nsb * 16l, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * (n / 16l), n) + } elif (fmt == 44) { + myp[tk * d + i] = dot_iq4xs_q8(kqp + i * nsb * 128l, ksp + i * nsb * 20l, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * (n / 16l), n) } else { myp[tk * d + i] = dot_k6q8(kqp + i * nsb * 192l, ksp + i * nsb * 18l, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * (n / 16l), n) } @@ -1416,6 +1501,8 @@ def private kq_batch_groupn_gen(fmt : int; var yp : float?; kqp : uint8 const?; myp[tk * d + i] = dot_k5q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n) } elif (fmt == 40) { myp[tk * d + i] = dot_q40q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n) + } elif (fmt == 44) { + myp[tk * d + i] = dot_iq4xs_q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n) } else { myp[tk * d + i] = dot_k6q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n) } @@ -1573,6 +1660,8 @@ def kq_groupn_gen(fmt : int; var yp : float?; kqp : uint8 const?; ksp : uint8 co k5q8_gemv_gen(myp + r * d, kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n, gLocal * mr, gEnd * mr) } elif (fmt == 40) { q40q8_gemv_gen(myp + r * d, kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n, gLocal * mr, gEnd * mr) + } elif (fmt == 44) { + iq4xsq8_gemv_gen(myp + r * d, kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n, gLocal * mr, gEnd * mr) } else { k6q8_gemv_gen(myp + r * d, kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n, gLocal * mr, gEnd * mr) } @@ -1591,6 +1680,8 @@ def kq_groupn_gen(fmt : int; var yp : float?; kqp : uint8 const?; ksp : uint8 co myp[r * d + i] = dot_k5q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) } elif (fmt == 40) { myp[r * d + i] = dot_q40q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) + } elif (fmt == 44) { + myp[r * d + i] = dot_iq4xs_q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) } else { myp[r * d + i] = dot_k6q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) } @@ -1884,7 +1975,7 @@ def dasllama_math_gen_register() { // the no-witness stance above covers kq too mm_kq = @@kq_kernel_gen, repack_kq = @@repack_kq_gen, kq_rows_k4 = @@k4q8_gemv_gen, kq_rows_k5 = @@k5q8_gemv_gen, - kq_rows_k6 = @@k6q8_gemv_gen, kq_rows_q40 = @@q40q8_gemv_gen, + kq_rows_k6 = @@k6q8_gemv_gen, kq_rows_q40 = @@q40q8_gemv_gen, kq_rows_iq4xs = @@iq4xsq8_gemv_gen, kq_batch = @@kq_batch_kernel_gen, kq_groupn = @@kq_groupn_gen, kq_batch_groupn = @@kq_batch_groupn_gen, // q51 family: tile + GEMV off the grp planes (the tile family's companions stamp @@ -1913,7 +2004,7 @@ def dasllama_math_gen_register() { mm_rows_s16 = @@q8q8_gemv_s16_gen, mm_kq = @@kq_kernel_gen, repack_kq = @@repack_kq_gen, kq_rows_k4 = @@k4q8_gemv_gen, kq_rows_k5 = @@k5q8_gemv_gen, - kq_rows_k6 = @@k6q8_gemv_gen, kq_rows_q40 = @@q40q8_gemv_gen, + kq_rows_k6 = @@k6q8_gemv_gen, kq_rows_q40 = @@q40q8_gemv_gen, kq_rows_iq4xs = @@iq4xsq8_gemv_gen, kq_batch = @@kq_batch_kernel_gen, kq_groupn = @@kq_groupn_gen, kq_batch_groupn = @@kq_batch_groupn_gen, groupn_q51 = @@q51q8_groupn_gen, repack_q51 = @@repack_q51_gen, diff --git a/modules/dasLLAMA/dasllama/dasllama_ple.das b/modules/dasLLAMA/dasllama/dasllama_ple.das index ddc47b4f1d..5d24984178 100644 --- a/modules/dasLLAMA/dasllama/dasllama_ple.das +++ b/modules/dasLLAMA/dasllama/dasllama_ple.das @@ -62,6 +62,8 @@ def ple_check_table(t : Model; origin : string) { have = (long_length(t.k6q) / K6_QSB) * 256l } elif (t.ple_emb_fmt == KqFmt.q40) { have = (long_length(t.q40q) / Q40_QSB) * 256l + } elif (t.ple_emb_fmt == KqFmt.iq4xs) { + have = (long_length(t.iq4xsq) / IQ4XS_QSB) * 256l } if (have < need) { panic("dasLLAMA: the PLE token table is not resident ({origin}): fmt {t.ple_emb_fmt} plane holds {have} of {need} elements - regenerate this carrier (it predates the pinned-table rail)") @@ -95,6 +97,8 @@ def private ple_gather_row(t : Model; row : int64; var dst : array; doff } } elif (t.ple_emb_fmt == KqFmt.q40) { dequant_q40_plane_superblock(t.q40q, sb * Q40_QSB, t.q40s, sb * Q40_SSB, dst, doff + blk * 256l) + } elif (t.ple_emb_fmt == KqFmt.iq4xs) { + dequant_iq4xs_plane_superblock(t.iq4xsq, sb * IQ4XS_QSB, t.iq4xss, sb * IQ4XS_SSB, dst, doff + blk * 256l) } else { panic("ple_gather_row: '{t.ple_emb_fmt}' has no kq plane pair") } diff --git a/modules/dasLLAMA/dasllama/dasllama_repack.das b/modules/dasLLAMA/dasllama/dasllama_repack.das index 03cf97166b..98d3ed70dd 100644 --- a/modules/dasLLAMA/dasllama/dasllama_repack.das +++ b/modules/dasLLAMA/dasllama/dasllama_repack.das @@ -373,6 +373,50 @@ def repack_q40_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { delete tq delete ts } + +//! iq4xs grp layout: the q40 quant gather (the disk k/k+16 pairing IS the k4 tiling) and the k4 +//! scale-row interleave over the DECODED 20B row — [mr x f16 d][mr x f16 0][8 x mr int8 sc][8 x mr 0]. +def repack_iq4xs_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { + let nsb = n / 256l + let qrow = nsb * 128l + let srow = nsb * 20l + let ng = d / mr + var tq : array + var ts : array + tq |> resize(d * qrow) + ts |> resize(d * srow) + unsafe { + var tqp = addr(tq[0]) + var tsp = addr(ts[0]) + memcpy(tqp, kq, d * qrow) + memcpy(tsp, ks, d * srow) + for (g in range64(ng)) { + for (sbi in range64(nsb)) { + let dq = g * mr * qrow + sbi * 128l * mr + let ds = g * mr * srow + sbi * 20l * mr + for (r in range64(mr)) { + let sq = (g * mr + r) * qrow + sbi * 128l + let ss = (g * mr + r) * srow + sbi * 20l + for (blk in range64(8l)) { + for (j in range64(4l)) { + for (t in range64(4l)) { + kq[dq + ((blk * 4l + j) * mr + r) * 4l + t] = tqp[sq + blk * 16l + j * 4l + t] + } + } + ks[ds + 4l * mr + blk * mr + r] = tsp[ss + 4l + blk] + ks[ds + 12l * mr + blk * mr + r] = uint8(0) + } + ks[ds + 2l * r] = tsp[ss] + ks[ds + 2l * r + 1l] = tsp[ss + 1l] + ks[ds + 2l * mr + 2l * r] = uint8(0) + ks[ds + 2l * mr + 2l * r + 1l] = uint8(0) + } + } + } + } + delete tq + delete ts +} // ===== grp panel unpack (the byte-expanded tile form the kq stamps load) ===== //! Unpack ONE group's packed grp k5/k6 quant panel into the BYTE-EXPANDED tile form (wlo/whi diff --git a/modules/dasLLAMA/tests/test_kqformat.das b/modules/dasLLAMA/tests/test_kqformat.das index 22c6207011..1ab1eed111 100644 --- a/modules/dasLLAMA/tests/test_kqformat.das +++ b/modules/dasLLAMA/tests/test_kqformat.das @@ -20,6 +20,7 @@ def test_kqfmt_enum_values(t : T?) { t |> equal(int(KqFmt.k6), 3) t |> equal(int(KqFmt.q40), 4) t |> equal(int(KqFmt.q51), 5) + t |> equal(int(KqFmt.iq4xs), 6) } } @@ -32,6 +33,8 @@ def test_kqfmt_predicates(t : T?) { t |> success(kq_sb(KqFmt.k6), "k6 is superblock") t |> success(kq_sb(KqFmt.q40), "q40 is superblock") t |> success(!kq_sb(KqFmt.q51), "q51 rides per-32 planes") + t |> success(kq_sb(KqFmt.iq4xs), "iq4xs is superblock") + t |> success(kq_sb(int(KqFmt.iq4xs)), "the int-id predicate agrees for iq4xs") } } @@ -48,6 +51,8 @@ def test_kqfmt_strides(t : T?) { t |> equal(kq_ssb(KqFmt.q40), 16l) // 8 x f16 d t |> equal(kq_qsb(KqFmt.q51), 20l) // per 32-BLOCK: 16 nibbles + 4 qh t |> equal(kq_ssb(KqFmt.q51), 4l) // per 32-BLOCK: f16 d + f16 m + t |> equal(kq_qsb(KqFmt.iq4xs), 128l) // the q40 tiling (disk k/k+16 pairing) + t |> equal(kq_ssb(KqFmt.iq4xs), 20l) // f16 d + 8 int8 sub-scales in the k4 row shape } t |> run("named constants agree with the literals") @(t : T?) { t |> equal(K4_QSB, 128l) @@ -60,19 +65,31 @@ def test_kqfmt_strides(t : T?) { t |> equal(Q40_SSB, 16l) t |> equal(Q51_QB, 20l) t |> equal(Q51_SB, 4l) + t |> equal(IQ4XS_QSB, 128l) + t |> equal(IQ4XS_SSB, 20l) + } + t |> run("the IQ4_NL codebook is ggml's kvalues_iq4nl") @(t : T?) { + t |> equal(int(IQ4NL_LUT[0]), -127) + t |> equal(int(IQ4NL_LUT[7]), -10) + t |> equal(int(IQ4NL_LUT[8]), 1) + t |> equal(int(IQ4NL_LUT[15]), 113) + for (i in range(15)) { + t |> success(int(IQ4NL_LUT[i]) < int(IQ4NL_LUT[i + 1]), "the codebook is monotonic") + } } } [test] def test_kqfmt_id_bridge(t : T?) { - t |> run("kq_schema_id is the one enum->kernel-id bridge (4/5/6/40)") @(t : T?) { + t |> run("kq_schema_id is the one enum->kernel-id bridge (4/5/6/40/44)") @(t : T?) { t |> equal(kq_schema_id(KqFmt.k4), 4) t |> equal(kq_schema_id(KqFmt.k5), 5) t |> equal(kq_schema_id(KqFmt.k6), 6) t |> equal(kq_schema_id(KqFmt.q40), 40) + t |> equal(kq_schema_id(KqFmt.iq4xs), 44) } t |> run("gemm_schema's int-id forms delegate to the same truth") @(t : T?) { - for (f in [KqFmt.k4, KqFmt.k5, KqFmt.k6, KqFmt.q40]) { + for (f in [KqFmt.k4, KqFmt.k5, KqFmt.k6, KqFmt.q40, KqFmt.iq4xs]) { t |> equal(kq_qsb(kq_schema_id(f)), kq_qsb(f)) t |> equal(kq_ssb(kq_schema_id(f)), kq_ssb(f)) } @@ -86,6 +103,7 @@ def test_kqfmt_elems_and_stream(t : T?) { t |> equal(kq_elems(KqFmt.k5), 256l) t |> equal(kq_elems(KqFmt.k6), 256l) t |> equal(kq_elems(KqFmt.q40), 256l) + t |> equal(kq_elems(KqFmt.iq4xs), 256l) t |> equal(kq_elems(KqFmt.q8), 32l) t |> equal(kq_elems(KqFmt.q51), 32l) } @@ -96,5 +114,6 @@ def test_kqfmt_elems_and_stream(t : T?) { t |> equal(kq_stream_code(KqFmt.k5), 5) t |> equal(kq_stream_code(KqFmt.k6), 6) t |> equal(kq_stream_code(KqFmt.q40), 40) + t |> equal(kq_stream_code(KqFmt.iq4xs), 44) } } diff --git a/modules/dasLLAMA/tests/test_kquant.das b/modules/dasLLAMA/tests/test_kquant.das index 63a665431d..35896af6f9 100644 --- a/modules/dasLLAMA/tests/test_kquant.das +++ b/modules/dasLLAMA/tests/test_kquant.das @@ -1,8 +1,8 @@ options gen2 options stack = 524288 // every dasLLAMA program root takes this budget (options stack does not unify up from libs) options persistent_heap // the layout arm loads a multi-GB model; explicit delete below -options _cyclomatic_complexity = 35 // every gate here is the same flat fmt == 4/5/6/40 ladder repeated per stage -options _function_length = 170 // and each gate carries its whole fixture build + oracle inline +options _cyclomatic_complexity = 40 // every gate here is the same flat fmt == 4/5/6/40/44 ladder repeated per stage +options _function_length = 180 // and each gate carries its whole fixture build + oracle inline options _dasllama_internal = true require dastest/testing_boost public @@ -102,9 +102,35 @@ def private build_q6k_block() : array { return <- blkb } -// loop-generated test labels: "k4"/"k5"/"k6" for the K-quants, "q40" for fmt 40 — matches the -// kernel/repack function names (dot_q40q8, repack_q40_grp) so failures grep 1:1 -def private kq_tag(fmt : int) : string => fmt == 40 ? "q40" : "k{fmt}" +// loop-generated test labels: "k4"/"k5"/"k6" for the K-quants, "q40" for fmt 40, "iq4xs" for +// fmt 44 — matches the kernel/repack function names (dot_q40q8, repack_q40_grp) so failures grep 1:1 +def private kq_tag(fmt : int) : string => fmt == 40 ? "q40" : (fmt == 44 ? "iq4xs" : "k{fmt}") + +// iq4xs synthetic superblock: 136B disk block — f16 d = SYNTH_D, the 8 six-bit sub-scales are +// SYNTH_SC (packed low nibbles into scales_l, top two bits into scales_h), the shared nibble +// pattern in the k/k+16 pairing — weight k = (d * (SYNTH_SC[k/32] - 32)) * IQ4NL_LUT[q4_pat(k)] +def private build_iq4xs_block() : array { + var blkb : array + blkb |> resize(136) + let dbits = f32_to_f16(SYNTH_D) + blkb[0] = uint8(dbits & 0xFF) + blkb[1] = uint8(dbits >> 8u) + var sh = 0 + for (blk in range(8)) { + sh |= ((SYNTH_SC[blk] >> 4) & 3) << (2 * blk) + } + blkb[2] = uint8(sh & 0xFF) + blkb[3] = uint8(sh >> 8) + for (j in range(4)) { // scales_l[j]: block 2j's low nibble, block 2j+1's in the high nibble + blkb[4 + j] = uint8((SYNTH_SC[2 * j] & 15) | ((SYNTH_SC[2 * j + 1] & 15) << 4)) + } + for (blk in range(8)) { + for (j in range(16)) { // byte j: weight blk*32+j in the low nibble, +16 in the high + blkb[8 + blk * 16 + j] = uint8(q4_pat(blk * 32 + j) | (q4_pat(blk * 32 + 16 + j) << 4)) + } + } + return <- blkb +} // q40 synthetic superblock: 8 x 18B q4_0 disk blocks, per-block f16-exact d = (blk+1)/16 and // the shared nibble pattern — weight k = d(k/32) * (q4_pat(k) - 8) @@ -259,6 +285,24 @@ def test_kq_transcode_planes(t : T?) { t |> success(dst[k] == expected, "q40 plane element must match hand-packed value exactly") } } + t |> run("iq4xs planes decode the 6-bit sub-scales and the codebook exactly") @(t : T?) { + let blkb <- build_iq4xs_block() + var kq : array + var ks : array + kq |> resize(128) + ks |> resize(20) + transcode_iq4xs_superblock(blkb, 0l, kq, 0l, ks, 0l) + for (blk in range(8)) { + t |> equal(iq4xs_sc(ks, 0l, int64(blk)), SYNTH_SC[blk] - 32, "decoded sub-scale is ls - 32, signed") + } + var dst : array + dst |> resize(256) + dequant_iq4xs_plane_superblock(kq, 0l, ks, 0l, dst, 0l) + for (k in range(256)) { + let expected = (SYNTH_D * float(SYNTH_SC[k / 32] - 32)) * float(IQ4NL_LUT[q4_pat(k)]) + t |> success(dst[k] == expected, "iq4xs plane element must match the ggml dequant exactly") + } + } t |> run("q51 planes unpack the hand-packed Q5_1 blocks exactly") @(t : T?) { for (blk in range(4)) { let blkb <- build_q51_block(blk) @@ -505,7 +549,7 @@ def test_q51_kernels(t : T?) { // elementwise dot over the plane dequant (the kernels' declared oracle). def private kq_dot_gate(t : T?; fmt : int; n : int64 = 512l) { let nsb = n / 256l - let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : build_q6k_block())) + let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : build_q6k_block()))) let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) var kq : array @@ -519,6 +563,8 @@ def private kq_dot_gate(t : T?; fmt : int; n : int64 = 512l) { transcode_q5k_superblock(blk1, 0l, kq, s * qsb, ks, s * ssb) } elif (fmt == 40) { transcode_q40_superblock(blk1, 0l, kq, s * qsb, ks, s * ssb) + } elif (fmt == 44) { + transcode_iq4xs_superblock(blk1, 0l, kq, s * qsb, ks, s * ssb) } else { transcode_q6k_superblock(blk1, 0l, kq, s * qsb, ks, s * ssb) } @@ -538,6 +584,8 @@ def private kq_dot_gate(t : T?; fmt : int; n : int64 = 512l) { dequant_k5_plane_superblock(kq, s * qsb, ks, s * ssb, w, s * 256l) } elif (fmt == 40) { dequant_q40_plane_superblock(kq, s * qsb, ks, s * ssb, w, s * 256l) + } elif (fmt == 44) { + dequant_iq4xs_plane_superblock(kq, s * qsb, ks, s * ssb, w, s * 256l) } else { dequant_k6_plane_superblock(kq, s * qsb, ks, s * ssb, w, s * 256l) } @@ -569,6 +617,8 @@ def private kq_dot_gate(t : T?; fmt : int; n : int64 = 512l) { got = dot_k5q8(addr(kq[0]), addr(ks[0]), addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } elif (fmt == 40) { got = dot_q40q8(addr(kq[0]), addr(ks[0]), addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + } elif (fmt == 44) { + got = dot_iq4xs_q8(addr(kq[0]), addr(ks[0]), addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } else { got = dot_k6q8(addr(kq[0]), addr(ks[0]), addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } @@ -591,9 +641,12 @@ def test_kq_dots(t : T?) { t |> run("dot_q40q8 matches the fp64 plane-dequant reference") @(t : T?) { kq_dot_gate(t, 40) } + t |> run("dot_iq4xs_q8 matches the fp64 plane-dequant reference") @(t : T?) { + kq_dot_gate(t, 44) + } // 122B-class row widths: n=3072 (12 superblocks — qwen35moe-122B gate/up/cls) and n=1024 // (its k5 down_exps). Every earlier kq model kept n at 512..2560. - for (fmt in [4, 5, 6, 40]) { + for (fmt in [4, 5, 6, 40, 44]) { for (nn in [1024l, 3072l]) { t |> run("dot_{kq_tag(fmt)}q8 matches the fp64 reference at n={int(nn)}") @(t : T?) { kq_dot_gate(t, fmt, nn) @@ -612,7 +665,7 @@ def private kq_gemv_rows_gate(t : T?; fmt : int; n : int64 = 512l) { let nsb = n / 256l let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) - let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : build_q6k_block())) + let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : build_q6k_block()))) var kq : array var ks : array kq |> resize(d * nsb * qsb) @@ -625,6 +678,8 @@ def private kq_gemv_rows_gate(t : T?; fmt : int; n : int64 = 512l) { transcode_q5k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } elif (fmt == 40) { transcode_q40_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 44) { + transcode_iq4xs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } else { transcode_q6k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } @@ -663,6 +718,8 @@ def private kq_gemv_rows_gate(t : T?; fmt : int; n : int64 = 512l) { want = dot_k5q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } elif (fmt == 40) { want = dot_q40q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + } elif (fmt == 44) { + want = dot_iq4xs_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } else { want = dot_k6q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } @@ -695,7 +752,10 @@ def test_kq_gemv_rows(t : T?) { t |> run("portable q40 GEMV rows bit-match per-row disk dots") @(t : T?) { kq_gemv_rows_gate(t, 40) } - for (fmt in [4, 5, 6, 40]) { + t |> run("portable iq4xs GEMV rows bit-match per-row disk dots") @(t : T?) { + kq_gemv_rows_gate(t, 44) + } + for (fmt in [4, 5, 6, 40, 44]) { for (nn in [1024l, 3072l]) { t |> run("portable {kq_tag(fmt)} GEMV rows bit-match per-row disk dots at n={int(nn)}") @(t : T?) { kq_gemv_rows_gate(t, fmt, nn) @@ -715,7 +775,7 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { let nsb = n / 256l let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) - let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : build_q6k_block())) + let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : build_q6k_block()))) var kq : array var ks : array kq |> resize(d * nsb * qsb) @@ -728,6 +788,8 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { transcode_q5k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } elif (fmt == 40) { transcode_q40_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 44) { + transcode_iq4xs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } else { transcode_q6k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } @@ -765,6 +827,8 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { want[r] = dot_k5q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } elif (fmt == 40) { want[r] = dot_q40q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + } elif (fmt == 44) { + want[r] = dot_iq4xs_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } else { want[r] = dot_k6q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } @@ -775,6 +839,8 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { dequant_k5_plane_superblock(kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb, wrow, r * n + s * 256l) } elif (fmt == 40) { dequant_q40_plane_superblock(kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb, wrow, r * n + s * 256l) + } elif (fmt == 44) { + dequant_iq4xs_plane_superblock(kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb, wrow, r * n + s * 256l) } else { dequant_k6_plane_superblock(kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb, wrow, r * n + s * 256l) } @@ -786,6 +852,8 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { repack_k5_grp(addr(kq[0]), addr(ks[0]), n, d, mr) } elif (fmt == 40) { repack_q40_grp(addr(kq[0]), addr(ks[0]), n, d, mr) + } elif (fmt == 44) { + repack_iq4xs_grp(addr(kq[0]), addr(ks[0]), n, d, mr) } else { repack_k6_grp(addr(kq[0]), addr(ks[0]), n, d, mr) } @@ -799,7 +867,9 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { let ksg = addr(ks[g * mr * nsb * ssb]) let got = (fmt == 40 ? q40_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) - : kq_grp_row_dot(int64(fmt), kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n)) + : (fmt == 44 + ? iq4xs_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + : kq_grp_row_dot(int64(fmt), kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n))) if (got != want[r]) { dotbad++ } @@ -826,7 +896,7 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { [test] def test_kq_repack_grp(t : T?) { - for (fmt in [4, 5, 6, 40]) { + for (fmt in [4, 5, 6, 40, 44]) { for (mr in [4l, 8l, 16l]) { t |> run("repack_{kq_tag(fmt)}_grp mr={mr} preserves dots and row dequants") @(t : T?) { kq_repack_gate(t, fmt, mr) @@ -851,7 +921,7 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { let mr = kq_layout_of(fmt) // the format's OWN layout companion (per-format tune families) let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) - let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : build_q6k_block())) + let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : build_q6k_block()))) var kq : array var ks : array kq |> resize(d * nsb * qsb) @@ -864,6 +934,8 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { transcode_q5k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } elif (fmt == 40) { transcode_q40_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 44) { + transcode_iq4xs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } else { transcode_q6k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } @@ -881,6 +953,8 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { repack_k5_grp(addr(kq[0]), addr(ks[0]), n, d, mr) } elif (fmt == 40) { repack_q40_grp(addr(kq[0]), addr(ks[0]), n, d, mr) + } elif (fmt == 44) { + repack_iq4xs_grp(addr(kq[0]), addr(ks[0]), n, d, mr) } else { repack_k6_grp(addr(kq[0]), addr(ks[0]), n, d, mr) } @@ -909,7 +983,7 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { // k5/k6 tiles read the byte-expanded panel the batch cell unpacks per (group, token-block) // — the gate covers unpack_kq_panel_grp too (tile-over-panel vs gemv-over-packed planes); // k4/q40 tiles read the packed planes directly - let packed = fmt == 4 || fmt == 40 + let packed = fmt == 4 || fmt == 40 || fmt == 44 var panel : array if (!packed) { panel |> resize(mr * nsb * 256l) @@ -930,6 +1004,8 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { k5q8_tile_gen(addr(ytile[0]), addr(panel[0]), ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) } elif (fmt == 40) { q40q8_tile_gen(addr(ytile[0]), kqg, ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) + } elif (fmt == 44) { + iq4xsq8_tile_gen(addr(ytile[0]), kqg, ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) } else { k6q8_tile_gen(addr(ytile[0]), addr(panel[0]), ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) } @@ -942,6 +1018,8 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { k5q8_gemv_gen(addr(ytile[tk * d]), addr(kq[0]), addr(ks[0]), addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, g * mr, (g + 1l) * mr) } elif (fmt == 40) { q40q8_gemv_gen(addr(ytile[tk * d]), addr(kq[0]), addr(ks[0]), addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, g * mr, (g + 1l) * mr) + } elif (fmt == 44) { + iq4xsq8_gemv_gen(addr(ytile[tk * d]), addr(kq[0]), addr(ks[0]), addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, g * mr, (g + 1l) * mr) } else { k6q8_gemv_gen(addr(ytile[tk * d]), addr(kq[0]), addr(ks[0]), addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, g * mr, (g + 1l) * mr) } @@ -955,6 +1033,8 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { k5q8_gemv_gen(addr(ygemv[p * d]), addr(kq[0]), addr(ks[0]), addr(xq[p * n]), addr(xs[p * (n / 256l)]), addr(xbs[p * (n / 16l)]), n, 0l, d) } elif (fmt == 40) { q40q8_gemv_gen(addr(ygemv[p * d]), addr(kq[0]), addr(ks[0]), addr(xq[p * n]), addr(xs[p * (n / 256l)]), addr(xbs[p * (n / 16l)]), n, 0l, d) + } elif (fmt == 44) { + iq4xsq8_gemv_gen(addr(ygemv[p * d]), addr(kq[0]), addr(ks[0]), addr(xq[p * n]), addr(xs[p * (n / 256l)]), addr(xbs[p * (n / 16l)]), n, 0l, d) } else { k6q8_gemv_gen(addr(ygemv[p * d]), addr(kq[0]), addr(ks[0]), addr(xq[p * n]), addr(xs[p * (n / 256l)]), addr(xbs[p * (n / 16l)]), n, 0l, d) } @@ -980,7 +1060,7 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { [test] def test_kq_tile(t : T?) { - for (fmt in [4, 5, 6, 40]) { + for (fmt in [4, 5, 6, 40, 44]) { t |> run("{kq_tag(fmt)} 4-token tile bit-matches per-token GEMVs") @(t : T?) { kq_tile_gate(t, fmt) } @@ -1001,7 +1081,7 @@ def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; let nsb = n / 256l let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) - let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : build_q6k_block())) + let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : build_q6k_block()))) var kq : array var ks : array kq |> resize(nreg * d * nsb * qsb) @@ -1014,6 +1094,8 @@ def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; transcode_q5k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } elif (fmt == 40) { transcode_q40_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 44) { + transcode_iq4xs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } else { transcode_q6k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } @@ -1071,6 +1153,9 @@ def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; } elif (fmt == 40) { want_sh[r * d + row] = dot_q40q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) want_pr[r * d + row] = dot_q40q8(kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n) + } elif (fmt == 44) { + want_sh[r * d + row] = dot_iq4xs_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + want_pr[r * d + row] = dot_iq4xs_q8(kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n) } else { want_sh[r * d + row] = dot_k6q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) want_pr[r * d + row] = dot_k6q8(kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n) @@ -1113,6 +1198,8 @@ def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; repack_k5_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) } elif (fmt == 40) { repack_q40_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) + } elif (fmt == 44) { + repack_iq4xs_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) } else { repack_k6_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) } @@ -1129,6 +1216,9 @@ def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; } elif (fmt == 40) { q40q8_gemv_gen(addr(want_sh[r * d]), kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, 0l, d) q40q8_gemv_gen(addr(want_pr[r * d]), kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n, 0l, d) + } elif (fmt == 44) { + iq4xsq8_gemv_gen(addr(want_sh[r * d]), kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, 0l, d) + iq4xsq8_gemv_gen(addr(want_pr[r * d]), kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n, 0l, d) } else { k6q8_gemv_gen(addr(want_sh[r * d]), kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, 0l, d) k6q8_gemv_gen(addr(want_pr[r * d]), kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n, 0l, d) @@ -1164,7 +1254,7 @@ def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; [test] def test_kq_groupn(t : T?) { - for (fmt in [4, 5, 6, 40]) { + for (fmt in [4, 5, 6, 40, 44]) { t |> run("{kq_tag(fmt)} region-list GEMV bit-matches per-row dots (disk + grp slices)") @(t : T?) { kq_groupn_gate(t, fmt) } @@ -1208,7 +1298,7 @@ def private kq_batch_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 for (c in cnts) { nk += c } - let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : build_q6k_block())) + let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : build_q6k_block()))) var kq : array var ks : array kq |> resize(nreg * d * nsb * qsb) @@ -1221,6 +1311,8 @@ def private kq_batch_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 transcode_q5k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } elif (fmt == 40) { transcode_q40_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 44) { + transcode_iq4xs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } else { transcode_q6k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } @@ -1239,6 +1331,8 @@ def private kq_batch_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 repack_k5_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) } elif (fmt == 40) { repack_q40_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) + } elif (fmt == 44) { + repack_iq4xs_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) } else { repack_k6_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) } @@ -1283,6 +1377,8 @@ def private kq_batch_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 k5q8_gemv_gen(addr(want[tk * d]), kqr, ksr, addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, 0l, d) } elif (fmt == 40) { q40q8_gemv_gen(addr(want[tk * d]), kqr, ksr, addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, 0l, d) + } elif (fmt == 44) { + iq4xsq8_gemv_gen(addr(want[tk * d]), kqr, ksr, addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, 0l, d) } else { k6q8_gemv_gen(addr(want[tk * d]), kqr, ksr, addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, 0l, d) } @@ -1314,7 +1410,7 @@ def private kq_batch_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 [test] def test_kq_batch_groupn(t : T?) { - for (fmt in [4, 5, 6, 40]) { + for (fmt in [4, 5, 6, 40, 44]) { t |> run("{kq_tag(fmt)} batch groupn bit-matches per-token rows-core calls") @(t : T?) { kq_batch_groupn_gate(t, fmt) } From 5a48b4a1b179db695d1fc691a1c2a33be743f5a8 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Sun, 30 Aug 2026 05:34:34 -0700 Subject: [PATCH 02/61] dasLLAMA: IQ4_XS end to end on CPU - the repack-mr freeze and worker-context codebook fixes, tuner family, image meta; HOW_TO_ADD_A_FORMAT pilot + module CLAUDE.md Co-Authored-By: Claude Fable 5 --- CLAUDE.md | 1 + modules/dasLLAMA/CLAUDE.md | 36 +++ modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md | 254 ++++++++++++++++++ modules/dasLLAMA/LAWS.md | 9 + .../dasLLAMA/dasllama/dasllama_convert.das | 5 +- modules/dasLLAMA/dasllama/dasllama_image.das | 5 +- .../dasLLAMA/dasllama/dasllama_kqformat.das | 6 + modules/dasLLAMA/dasllama/dasllama_load.das | 4 +- .../dasllama/dasllama_math_default.das | 10 +- .../dasLLAMA/dasllama/dasllama_math_gen.das | 5 +- modules/dasLLAMA/harness/gen_tune_probe.das | 60 ++++- 11 files changed, 373 insertions(+), 22 deletions(-) create mode 100644 modules/dasLLAMA/CLAUDE.md create mode 100644 modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md diff --git a/CLAUDE.md b/CLAUDE.md index 30e9cdb205..d099e6c7ec 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -162,6 +162,7 @@ Task-specific instructions are split into skill files under `skills/`. You MUST | `skills/daslang/references/queries.md` | Filter/map/sort/group/aggregate transforms - comprehension -> linq_boost -> plain `for`; avoid `daslib/functional` for new code | | `skills/decs.md` | Programming with `daslib/decs` / `decs_boost` - entities, components, queries, `[decs_template]`, stages | | `skills/internal/aot_hash_desync_debugging.md` | `error[50101]: AOT link failed` - semantic-hash desync diagnostics | +| `modules/dasLLAMA/CLAUDE.md` | Any work under `modules/dasLLAMA/` - the module's HOW_TO series (`HOW_TO_ADD_A_FORMAT.md` for a new weight format) and its architecture/review set | Multiple skill files may apply to one task: creating a new daslib module needs `skills/das_formatting.md`, `skills/daslib_modules.md`, and possibly `skills/internal/documentation_rst.md`. diff --git a/modules/dasLLAMA/CLAUDE.md b/modules/dasLLAMA/CLAUDE.md new file mode 100644 index 0000000000..bd5bc7fc44 --- /dev/null +++ b/modules/dasLLAMA/CLAUDE.md @@ -0,0 +1,36 @@ +# dasLLAMA module instructions + +dasLLAMA is the daslang LLM / ASR / vision engine, in-tree at `modules/dasLLAMA/`. **How it is +built and why is the `ARCHITECTURE*.md` set beside this file** (`ARCHITECTURE.md` routes to the +engine, image, GPU, Vulkan, Metal, measurement and media companions) - read the section you +are about to work in before writing code here. The rules binding a diff are `REVIEW*.md`; +`ENVIRONMENT.md` lists every knob; `followup_general.md` / `followup_vulkan.md` are the ledgers; +`PERF_LEDGER.md` is the measured record; `tests/CLAUDE.md` is the test discipline (run suites +ONLY through `tests/run.das`). + +Follow the daslang **gen2** conventions - the root `CLAUDE.md` rules apply to every `.das` file +here. + +## HOW_TO documents (REQUIRED for the task they name) + +A HOW_TO is a procedure: imperative, ordered, each step citing the architecture section that +owns it, validated by execution, with a QUIRKS ledger of every place the pattern broke so a +follow-up arc can unquirk it. Read the one that matches your task before the first edit, and +fix it in the same session when a step turns out wrong. + +| Document | Read BEFORE... | +|---|---| +| `HOW_TO_ADD_A_FORMAT.md` | Adding a weight format (a new `KqFmt`): GGUF type -> planes -> CPU kernels -> tune family -> Vulkan -> Metal -> tests | +| `BRINGUP.md` | Bringing a profiling box up from zero (the records rig; `METHODOLOGY.md` is the published method) | + +Planned entries in the series: adding a model family, a vision tower, an audio tower, TTS. + +## Skill files (REQUIRED) + +| Skill file | Read BEFORE... | +|---|---| +| `skills/tune.md` | Touching any `[tune]` / `[tune_perm]` kernel family or the sidecar | +| `skills/internal/llvm_tune_internals.md` | Editing the tune framework itself | +| `skills/perf_lint.md` / `skills/style_lint.md` | Suppressing any lint finding here | +| `skills/internal/tests_in_repo.md` | Adding a test (the deep-engine rules: `options stack`, `T?`-free helpers) | +| `skills/writing_benchmarks.md` / `skills/internal/benchmarks_in_repo.md` | Anything under `benchmarks/` or `performance/` | diff --git a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md new file mode 100644 index 0000000000..82310aeed2 --- /dev/null +++ b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md @@ -0,0 +1,254 @@ +# How to add a weight format to dasLLAMA + +A HOW_TO is the fourth document mood: `REVIEW.md` binds a diff, `ARCHITECTURE.md` says how the +system is, `/history` says what happened - this says **do this, then this**. It walks the code in +the order a new format lands, cites the architecture section that owns each step instead of +restating it, and keeps a QUIRKS ledger of every step where the pattern broke. It is validated by +execution: every step below was run in the session that wrote it, on the format named in the +per-format notes, or is marked untested. A step that turns out wrong during the next format's +bring-up is fixed here in that session. + +The worked example is IQ4_XS (the pilot). The per-format notes at the end say what each later +format changed in the walk. + +## 0. Decide the format's shape before touching code + +Three questions decide which existing family the format rides; answer them from +`ggml-common.h` (the block struct) and `ggml-quants.c` (`dequantize_row_`): + +1. **Block geometry.** 256-weight superblock (every K-quant, every i-quant) or 32-weight block + (`Q4_0`, `Q5_1`, `IQ4_NL`)? A superblock format joins the `kq_sb` lattice (Q8_K activations, + `% 256` rows, the grp repack, the stamped kq kernels); a 32-block format rides per-32 + planes like `q51`. `ARCHITECTURE_ENGINE.md` sec.1.2 owns the lattice split. +2. **Weight reconstruction.** Shift/mask nibbles (`q4_0`, K-quants), a 16-entry codebook on the + nibble (`IQ4_XS`, `IQ4_NL`), or a grid gather (`IQ2_*`, `IQ3_*`)? Shift/mask and codebook + formats keep the k4 nibble tiling and add an unpack step; grid formats decode to bytes first + (the byte-expanded panel k5/k6 already use). +3. **Scale structure and its fold.** Per-block f16 (`q40`), superblock d/dmin + 6-bit sub-scale + and min (`k4`/`k5`), superblock d + signed int8 sub-scale (`k6`, `IQ4_XS`). The fold decides + whether the dot needs the activation block sums (`xbsp`): a min or an unsigned-nibble offset + needs them, a signed reconstruction (codebook, k6's -32) does not. + +Write the answers down; they are the first lines of the per-format note. + +## 1. Identity - `dasllama/dasllama_kqformat.das` + +The taxonomy every other file keys off. One edit here, then the compiler finds the ladders. + +- Append the member to `KqFmt` - **append, never reorder**: the int value is the device stack + tag (`vk_kq_schema_id`) and the image plane id. +- `kq_sb` (both overloads) if it is a superblock format. +- `kq_schema_id`: the kernel/IR id. The ids are mnemonics - K-quants by bit width (4/5/6), + Q4_0 = 40, i-quants = bit width x 10 + a variant digit (IQ4_XS = 44). Three id spaces exist + (QUIRK 5); this is the only bridge between two of them. +- The stride constants `_QSB` / `_SSB` (bytes per superblock row of the quant and + scale planes) and the `kq_qsb` / `kq_ssb` arms. A codebook goes here too (`IQ4NL_LUT`). +- `dasllama_gemm_schema.das`: the int-id twins `kq_qsb(int)` / `kq_ssb(int)`. +- `tests/test_kqformat.das`: pin the enum value, the predicate, the strides, the id, the stream + code, and the codebook's edge values. + +## 2. Codec - `dasllama/dasllama_convert.das`, `dasllama/dasllama_gguf.das` + +- `GGML_TYPE_` constant (`dasllama_gguf.das`). +- `transcode__superblock(bytes, bo, kq, kqo, ks, kso)` - the per-superblock disk -> plane + split, array form (what the tests drive). +- `dequant__plane_superblock` - the reference dequant off the planes, in ggml's own float + order (`dequantize_row_` decides the order; match it operation for operation so the + plane dequant is bit-exact against the file dequant). +- `gguf_transcode_` (`dasllama_gguf.das`) - the bulk, threaded, pointerized twin of the + superblock transcode (`kq_transcode_check` + `guard_dst` + `with_tensor_view` + + `maybe_parallel_for` over superblocks). +- `tests/test_kquant.das`: a hand-packed synthetic superblock (`build__block`, written in + the PACK direction so a misread cannot cancel) and an arm asserting the plane dequant equals + the hand-computed weights exactly. + +**Plane layout rule.** Quant plane = the disk nibbles verbatim when the disk pairing is already +what the kernels want (Q4_0 and IQ4_XS pair k/k+16 inside a 32-block - the k4 tiling; Q4_K +pairs k/k+32 and re-pairs at repack). Scale plane = a fixed-stride row per superblock. The kq +v2 rule keeps the DISK scale block verbatim and decodes at repack (k4/k5: 16B block + 4B pad); +a format whose decoded form FITS the row may decode at transcode instead (IQ4_XS: f16 d, 2 pad, +8 x int8 sub-scale, 8 pad = the k4 decoded-row shape, so k4's repack addressing serves). + +## 3. Planes and the loader - `dasllama_common.das`, `dasllama_load.das`, `dasllama_layout.das` + +This is the ladder walk (QUIRK 1). Every site is a flat `if (fmt == KqFmt.k4) ... elif` chain; +add the arm next to `q40`'s. The compiler does not find these - a missing arm falls to the +`else` panic (good) or silently to k6 (bad, the `kq_kernel_gen` shape); grep `KqFmt.q40` and +`== 40` and visit every hit. + +- `Model`: the plane pair `q` / `s` and `kq_repack_mr`. +- `dasllama_load.das`: `LayoutSizes._n`, `KqCursors.`, `kq_take`, the `LayoutSizes` + constructor, `stream_field_of` (the streamed image plane name), the scale-half landing + `memcpy`, the stream repack `invoke(g_stream_repack, , ...)`, `transcode_kq_tensor`, + `load_big`, `kq_fmt_of` (GGML type -> tag), the two `noisy` log lines, + `g_stream_plane_total["q"]`, the plane `reserve`/`resize` block. +- `dasllama_common.das`: `kq_active_mr`, `kq_fi`, `mm_at_kq_pre` (two arms), `mm_b_kq`, + `mm_at_kq_groupn` (two arms), `mm_b_kq_groupn`, `mm_b_kq_pre`, `kq_plane_q`, `kq_plane_s`, + `embed_row` (four arms: grp and plane form, trimmed and untrimmed), the bake config fill + (`c.kq_mr = active_kq_layout_mr()`). +- `dasllama_layout.das`: the plane base pointers + the `rkq` ternary, `push_repack_kq`, the + `moe_gpu_gather_stack_kq` plane ternaries. +- `dasllama_gpu_resident.das` (embed trim), `dasllama_ple.das` (two arms), + `dasllama_blocks.das` (`kq_bytes_per_weight`). +- `dasllama_config.das`: `DlimCpuConfig.kq_mr` AND the identity string in + `dlim_identity` (QUIRK 3); `dasllama_image.das`: `IMAGE_VERSION` bump + the streamed plane + name list in `stream_extra_bytes`. + +## 4. CPU kernels - `dasllama_math_default.das`, `dasllama_math_gen.das`, `dasllama_math.das`, `dasllama_repack.das` + +- `dot__q8` - the portable disk-order row dot (exact integer inner sums, one float fold per + superblock), `_rows_kernel`, the `kq_gemv_kernel` arm, the `matmul_kq_groupn` arm. +- `dequant__row_grp` - the grp row dequant (own helper; `dequant_kq_row_grp` dispatches). +- `repack__grp` (`dasllama_repack.das`) - disk-order -> grp planes; tail rows (d % mr) + stay disk-order untouched. +- `dasllama_math_gen.das`: `q8_layout_gen` (the layout companion), `_grp_row_dot` + (the scalar grp reference = the stubs' body and the repack oracle), `q8_gemv_gen` + + `q8_tile_gen` with the `[tune_perm]` grid copied from q40's and + `tune(gen = "dasllama_gemm_gen::_tile", ...)`, `kq_layout_of`, `repack_kq_gen` / + `repack_kq_bake`, `kq_kernel_gen` (two ladders), `kq_batch_cell_gen` (`packed` + tile + + tail ladders), `kq_batch_kernel_gen` tail, `kq_batch_groupn_gen` tail, `kq_groupn_gen` (two + ladders), both `register_kernel_backend` rows (`kq_rows_ = @@q8_gemv_gen`). +- `dasllama_math.das`: the `KernelBackend.kq_rows_` slot, its `g_kq_rows_` global, + the activation copy, `kq_rows_fn`, the null-guard, the bake arm in `active_kq_layout_mr`. +- `dasllama_gemm_gen.das`: `register_llvm_code_generator` for `_gemv` and `_tile`. + Until the emitter arm exists, register generators that return `false` - a declined + generator IS the framework's fallback (QUIRK 4); the reference bodies serve. +- `harness/gen_tune_probe.das`: `pack_kq_disk_block`, `repack_kq_grp_fmt`, `build_kq_fixture` + (block size, transcode, the yref oracle), `kq_tile_variants`, `kq_gemv_variants_by_suffix`, + `kq_layout_mrs`, `run_kq_tile`'s `packed`, `kq_tile_entry`, the family list. Without this + the scope's completeness check demands a sidecar entry the tuner never writes, and every + start re-tunes. +- `tests/test_kquant.das`: add the format to every gate's ladder and to the `[4, 5, 6, 40]` + lists (QUIRK 2). The gates: dot vs the fp64 plane-dequant oracle, portable GEMV rows, + repack at mr 4/8/16 (dots and row dequants bit-exact), 4-token tile vs per-token GEMVs, + groupn (disk + grp slices), batch groupn. + +Gate: `test_kqformat` + `test_kquant` on the interpreter binary (the stubs' reference bodies). + +## 5. The JIT emitter - `dasllama_gemm_gen.das` + +Untested for IQ4_XS at the time of writing (the decline stubs stand). What the emitter offers: +`kq_tile_gen_impl(gc, fmt)` / `kq_gemv_gen_impl(gc, fmt)` are one emitter specialized on `fmt` +at generation time; `emit_block_kqv2` carries the nibble unpack + integer fold for 4/5/6/40; +`emit_block_mx4` carries the codebook path (`lut_lookup` = `tbl1` / `pshufb`, `dot_lane` with +`abs_w` + `psign` for signed weights on the u8 x s8 ISAs). A codebook format is mx4's block +with the kq scale fold. + +## 6. Vulkan - `dasllama_vulkan_classes.das`, `dasllama_vulkan_common.das`, `dasllama_vulkan_prefill.das` + +Untested at the time of writing. The class hierarchy: `KqGemvBase` + `def override blk_contrib` +per format; `KqBatchBase` + `stage_w` / `stage_ws` / `blk_fma`; the cm2 tiles get a +`[spirv_decode] def decode_` (`followup_vulkan.md` item 24 rules the template). Plus +`vk_kq_schema_id`, the `kq_batch_cls_*` / `gemv_cls_*` / `cm2_cls_*` ladders, `pf_f16_feed`, +the gather's format arms, and the oracle cells in `tests/test_vulkan_kernels.das` + +`tests/_vk_kq_fixtures.das`. + +## 7. Metal + +Untested at the time of writing (no Mac in the session). `MetalKqGemvK4` / `MetalKqMulMmK45T` +/ `MetalKqMvK4T` are the twins to subclass; `dasllama_metal_shapes.das` carries the servable +predicates, `dasllama_metal_common.das` the plane buffers, `dasllama_metal_prefill.das` the +PSO ladders. + +## 8. End to end + +A real file whose every tensor type is now loadable (the header census script in the session +scratchpad, or `harness/gguf_dump.das`), through `examples/dasLLAMA/run.das` against +`simple_ids.exe` from the llama.cpp reference build for the same prompt; then `test_model_image` +for the image round trip, then the records rig. + +## QUIRKS + +Every step where adding a format was not "the template plus one condition". Each entry: what, +where, why it is so today, what unquirked looks like. An empty ledger is a legitimate outcome. + +1. **Per-format plane fields and ~40 hand ladders.** `Model` holds `k4q/k4s, k5q/k5s, k6q/k6s, + q40q/q40s, q51q/q51s, iq4xsq/iq4xss`, and every consumer selects the pair with an + `if (fmt == KqFmt.k4) ... elif` chain: 12 ladders in `dasllama_load.das`, 13 in + `dasllama_common.das`, 3 in `dasllama_layout.das`, 8 in `dasllama_math_gen.das`, 3 in + `dasllama_math_default.das`, 4 in `dasllama_math.das`, one each in `ple`, `gpu_resident`, + `blocks`, `config`, `image`. Why: the planes grew one format at a time and each arm carries + a different literal stride. Unquirked: one `KqPlanes` (quant, scale, mr) indexed by `KqFmt` + on `Model`, and `kq_plane_q/s` as the only accessors - every ladder above collapses to one + table lookup, and a new format is the enum member plus its strides. +2. **The test file ladders per format too.** `tests/test_kquant.das` builds fixtures, transcodes, + dequants, dots, repacks and calls the stubs through the same `fmt == 4/5/6/40` chains in + five gates (28 arms for one format), and raises `_cyclomatic_complexity` / + `_function_length` per format added. `harness/gen_tune_probe.das` repeats the shape (9 + arms). Unquirked: per-format dispatch helpers in one `_kq_fixtures.das` shared by the test + and the harness (`kq_transcode_sb`, `kq_dequant_sb`, `kq_dot`, `kq_repack`, `kq_gemv_gen`, + `kq_tile_gen`), each a single ladder. +3. **The bake identity is hand-formatted.** `DlimCpuConfig` gains `kq_mr`, and + `dlim_identity` must ALSO append it to the identity string by hand - a field added without + the string leaves two images with different repack interleaves keyed identically. Why: the + identity is a formatter, not a reflection. Unquirked: format the `kq_mr*` fields from a table + the `KqFmt` enum drives. +4. **The `[tune]` family must exist before the emitter does.** `iq4xsq8_tile_gen` carries the + full `[tune_perm]` grid and a `tune(gen = ...)` key from day one because the loader's kq + dispatch, the layout companion and the tuner's completeness check all resolve the family by + name; the generator behind the key is a two-line `=> false` until the emitter arm lands. + Why: a declined generator is the framework's own fallback path, so the stubs are the + cheapest way to have a real family. Not a defect - record it so nobody "fixes" the stubs. +5. **Three int id spaces.** `int(KqFmt)` (device stack tags, image plane ids, `vk_kq_schema_id`'s + input), the kernel/IR id (`kq_schema_id`: 4/5/6/40/44 - runtime params of generated + kernels, `kq_qsb(int)`, `kq_rows_fn`), and the stream/repack region code (`kq_stream_code`: + 0/2 for q8/q51, else the kernel id). A new format touches all three; `test_kqformat` pins + them. Unquirked: one id, or at least one table that derives the other two. +6. **`|=` is not defined on `uint8`** in daslang (`blkb[i] |= uint8(x)` is a compile error). + Build the byte in an `int` and store it once. +7. **The image meta list is hand-maintained behind a count tripwire.** `kq_repack_mr` is a + non-array `Model` field, so `serialize_image_meta` (`dasllama_image.das`) must + `serialize_raw` it AND `IMAGE_META_FIELDS` must grow by one, or the first model load + dies on `verify(count_meta_fields(t) == IMAGE_META_FIELDS)` - after the tokenizer build, + not at compile time. The tripwire is deliberate (it catches a forgotten field); the hand + list is the quirk. Unquirked: the per-format `mr` moves into the `KqPlanes` table of + QUIRK 1 and the image serializes the table. +8. **The repack interleave is frozen onto the Model in a ladder of its own.** `layout_repack_kq` + repacks every format with the layout companion's `mr`, and a SEPARATE ladder in + `dasllama_load.das` (two copies: the streamed-save arm and the eager arm) records + `t.kq_repack_mr = active_kq_layout_mr()`. A format missing from that ladder keeps + the field's default (4) while its planes sit at the companion's `mr` (8 on this box): the + GEMV rail still reads the companion directly and is right, every `kq_active_mr` consumer - + the fused decode chains, `embed_row`, the grp row dequant - reads the wrong interleave, and + the model emits "time time time" with no diagnostic. Caught only by an end-to-end run; the + unit gates build their planes in-process and never see the freeze. Unquirked: the repack + records its own `mr` (one field per format in the `KqPlanes` table of QUIRK 1), and the + freeze ladder disappears. +9. **A kernel that runs on a forked worker reads a module-level table as zeros.** The fused + chains invoke the rows cores inside `team_parallel_stages` / `maybe_parallel_for` job + contexts; a `let` global (`IQ4NL_LUT`) read there returns 0, so a codebook format's rows + came out zero on every worker-run row and right on the main-context rows - the GEMV probes + (no job queue) all passed, the forward was garbage. The k4/k5/k6/q40 cores never had a + table; the mx4 path already hands its workers a `lutp` pointer for exactly this reason. + Rule: a kernel that can run on a worker takes its table as a per-call local + (`iq4nl_lut()`) or a pointer argument, never a module global; keep the global for tests, + oracles and the emitter's constant bake. Probe that catches it: any kernel gate run under + `with_job_que() { setup_dasllama_jobque() ... }` with enough rows to fork. +10. **The worktree session sees a stale binder.** LSP/MCP diagnostics in a fresh worktree come + from the main tree's binary and report the new enum member as "not inferred" and every + dasllama file as broken (`get_total_perf_cores` missing). Trust only the worktree binary: + `bin/Release/daslang.exe dastest/dastest.das -- --test `. Run the session inside the + worktree once it is bootstrapped. + +## Per-format notes + +### IQ4_XS (the pilot, 2026-08-30) + +Shape: 256-superblock, codebook nibble (`kvalues_iq4nl`, signed, no offset), scale = f16 d x +6-bit sub-scale - 32. Disk block 136 B: f16 d, u16 scales_h, 4 scales_l bytes, 128 nibbles +paired k/k+16 per 32-block. Plane pair: quants verbatim (the q40 tiling), scale row 20 B decoded +at transcode ([f16 d][2 pad][8 x int8 (ls-32)][8 pad] - the k4 decoded-row shape, so +`repack_k4_grp`'s scale addressing is reused verbatim in `repack_iq4xs_grp`). Dot fold: +`sum_blk sc * sum_k LUT[q] * x`, then `* (d * xs)` per superblock; no `xbsp` term. Ids: +`KqFmt.iq4xs` = 6, kernel id 44, stream code 44. Gates green on the interpreter binary +AND under `-jit`: `test_kqformat` 14/14, `test_kquant` 114/114 (iq4xs arms: transcode+dequant +exact, dot vs fp64 at n=512/1024/3072, GEMV rows, repack mr=4/8/16, tile, groupn). End to end: +`Llama-3.2-1B-Instruct-IQ4_XS.gguf` (bartowski; IQ4_XS x96 + Q5_K + Q6_K) through +`examples/dasLLAMA/run.das` reproduces llama.cpp's greedy continuation of "Once upon a time" +for ~45 tokens before the fast-math divergence (gen 42 t/s, reference bodies). Two bugs the +unit gates could not see, both caught only by the run: QUIRK 8 (the repack-mr freeze) and +QUIRK 9 (the codebook global on workers). Method that found them: a Python oracle that +dequantizes rows straight from the GGUF bytes, a daslang probe calling `mm_at_kq_pre` on the +same rows (run it UNDER the job queue), a layer bisect (`config.n_layers = L`, logits cosine +against a sibling quant of the same model), and a dump of the image's group-0 plane bytes +against the grp layout computed in Python. JIT emitter, Vulkan, Metal: pending. diff --git a/modules/dasLLAMA/LAWS.md b/modules/dasLLAMA/LAWS.md index 024423a9eb..62a7e34bf1 100644 --- a/modules/dasLLAMA/LAWS.md +++ b/modules/dasLLAMA/LAWS.md @@ -66,3 +66,12 @@ clause - a bump's diff also carries the LAWS.md entry recording the declared release; the declaration itself stays his call, on feel ("the intent was to bump due to major changes ... i go on feel on that one" - "4. agreee"). + +- **2026-08-30** (`HOW_TO_ADD_A_FORMAT.md`, `CLAUDE.md`): Boris opened the HOW_TO document + series with this pilot - "document entire process on how to add new format, as well as all + the quirks which popped up. so that the followup arc can unquirk them (if need be)"; a + separate kernel per card "is fine, as long as its a similar template setup ... just another + cond"; the series continues with adding a model family, vision, audio, TTS. Linked from the + module's new `CLAUDE.md` and the root table. Arc ruling the same day: the community i-quants + are read as-is ("B for sure. we are not big enough for people to bother with our own fork"), + Qwen3.8-27B on a 16 GB card is the Vulkan arc's endgame. diff --git a/modules/dasLLAMA/dasllama/dasllama_convert.das b/modules/dasLLAMA/dasllama/dasllama_convert.das index 915902db45..38c64d4a46 100644 --- a/modules/dasLLAMA/dasllama/dasllama_convert.das +++ b/modules/dasLLAMA/dasllama/dasllama_convert.das @@ -645,12 +645,13 @@ def iq4xs_sc(ks : array | #; kso, blk : int64) : int => rd_i8(ks, kso + 4 //! k+16 — the disk pairing, kept verbatim by the transcode). def dequant_iq4xs_plane_superblock(kq : array | #; kqo : int64; ks : array | #; kso : int64; var dst : array | #; doff : int64) { let d = f16_to_f32(rd_u16(ks, kso)) + let lut = iq4nl_lut() for (blk in range64(8l)) { let dl = d * float(iq4xs_sc(ks, kso, blk)) for (l in range64(16l)) { let b = int(kq[kqo + blk * 16l + l]) - dst[doff + blk * 32l + l] = dl * float(IQ4NL_LUT[b & 15]) - dst[doff + blk * 32l + 16l + l] = dl * float(IQ4NL_LUT[b >> 4]) + dst[doff + blk * 32l + l] = dl * float(lut[b & 15]) + dst[doff + blk * 32l + 16l + l] = dl * float(lut[b >> 4]) } } } diff --git a/modules/dasLLAMA/dasllama/dasllama_image.das b/modules/dasLLAMA/dasllama/dasllama_image.das index a7bacc72d3..932201fd80 100644 --- a/modules/dasLLAMA/dasllama/dasllama_image.das +++ b/modules/dasLLAMA/dasllama/dasllama_image.das @@ -634,6 +634,7 @@ def serialize_image_meta(var arch : Archive; var t : Model) { arch |> serialize_raw(t.kq_repack_mr5) arch |> serialize_raw(t.kq_repack_mr6) arch |> serialize_raw(t.kq_repack_mr40) + arch |> serialize_raw(t.kq_repack_mr44) arch |> serialize_raw(t.wcls_fmt) arch |> serialize_raw(t.emb_fmt) arch |> serialize_raw(t.ple_emb_fmt) @@ -672,8 +673,8 @@ def serialize_image_meta(var arch : Archive; var t : Model) { arch |> serialize_raw(t.mtp_headnorm_off) } -// 66 serialized fields + 3 deliberate skips (blocks, image_map, image_bytes) -let IMAGE_META_FIELDS = 66 + 3 +// 67 serialized fields + 3 deliberate skips (blocks, image_map, image_bytes) +let IMAGE_META_FIELDS = 67 + 3 //! Count of meta-carried fields of any struct: non-arrays plus string arrays (those cannot //! be raw planes — see serialize_strings). Pair with a per-type field-count constant to diff --git a/modules/dasLLAMA/dasllama/dasllama_kqformat.das b/modules/dasLLAMA/dasllama/dasllama_kqformat.das index 38e7eb8bd4..813bfacaa0 100644 --- a/modules/dasLLAMA/dasllama/dasllama_kqformat.das +++ b/modules/dasLLAMA/dasllama/dasllama_kqformat.das @@ -72,8 +72,14 @@ let IQ4XS_QSB = 128l // IQ4_XS quant plane: 128 nibble bytes, the disk k/k+16 let IQ4XS_SSB = 20l // IQ4_XS scale plane: f16 d, 2 pad, 8 x int8 (ls - 32), 8 pad — the k4 decoded-row shape //! The IQ4_NL / IQ4_XS nibble codebook: weight = scale x IQ4NL_LUT[nibble] (ggml's kvalues_iq4nl). +//! Main-context use only (tests, oracles, the emitter's constant bake): a kernel that can run on a +//! forked worker reads a module global as zeros - kernels take `iq4nl_lut()` as a local instead. let IQ4NL_LUT = fixed_array(-127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113) +//! The same codebook as a per-call local (16 bytes on the caller's frame) - the form every +//! worker-run kernel reads. +def iq4nl_lut() : int8[16] => fixed_array(-127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113) + //! Quant-plane bytes per stride unit — the typed stride surface. The stride unit is one //! 256-weight superblock for the kq lattice and one 32-weight block for q51 (see kq_elems). //! q8 panics (its planes are byte-per-elem, not stride-walked here). diff --git a/modules/dasLLAMA/dasllama/dasllama_load.das b/modules/dasLLAMA/dasllama/dasllama_load.das index 5c4e0429aa..834afc8d9e 100644 --- a/modules/dasLLAMA/dasllama/dasllama_load.das +++ b/modules/dasLLAMA/dasllama/dasllama_load.das @@ -2364,6 +2364,7 @@ def private load_gguf_parsed(var t : Model; m : GGUFMeta; bytes : array | t.kq_repack_mr5 = active_kq_layout_mr(5) t.kq_repack_mr6 = active_kq_layout_mr(6) t.kq_repack_mr40 = active_kq_layout_mr(40) + t.kq_repack_mr44 = active_kq_layout_mr(44) } } else { layout_repack_q8(t) @@ -2380,7 +2381,8 @@ def private load_gguf_parsed(var t : Model; m : GGUFMeta; bytes : array | t.kq_repack_mr5 = active_kq_layout_mr(5) t.kq_repack_mr6 = active_kq_layout_mr(6) t.kq_repack_mr40 = active_kq_layout_mr(40) - to_log(LOG_INFO, "dasLLAMA: K-quant planes repacked to the '{active_kernel_backend()}' layouts (k4 grp{t.kq_repack_mr4}, k5 grp{t.kq_repack_mr5}, k6 grp{t.kq_repack_mr6}, q40 grp{t.kq_repack_mr40})\n") + t.kq_repack_mr44 = active_kq_layout_mr(44) + to_log(LOG_INFO, "dasLLAMA: K-quant planes repacked to the '{active_kernel_backend()}' layouts (k4 grp{t.kq_repack_mr4}, k5 grp{t.kq_repack_mr5}, k6 grp{t.kq_repack_mr6}, q40 grp{t.kq_repack_mr40}, iq4xs grp{t.kq_repack_mr44})\n") } } } diff --git a/modules/dasLLAMA/dasllama/dasllama_math_default.das b/modules/dasLLAMA/dasllama/dasllama_math_default.das index 27a93de290..ede5add94c 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_default.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_default.das @@ -552,6 +552,7 @@ def dot_q40q8(kqrow : uint8 const?; ksrow : uint8 const?; xqp : int8 const?; xsp [unused_argument(xbsp)] def dot_iq4xs_q8(kqrow : uint8 const?; ksrow : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n : int64) : float { var acc = 0.0 + let lut = iq4nl_lut() unsafe { for (sb in range64(n / 256l)) { let kqo = sb * 128l @@ -563,8 +564,8 @@ def dot_iq4xs_q8(kqrow : uint8 const?; ksrow : uint8 const?; xqp : int8 const?; var idot = 0 for (l in range64(16l)) { let q = int(kqrow[kqo + blk * 16l + l]) - idot += int(IQ4NL_LUT[q & 15]) * int(xqp[ab + l]) - idot += int(IQ4NL_LUT[q >> 4]) * int(xqp[ab + 16l + l]) + idot += int(lut[q & 15]) * int(xqp[ab + l]) + idot += int(lut[q >> 4]) * int(xqp[ab + 16l + l]) } let u = int(ksrow[kso + 4l + blk]) isum += (u < 128 ? u : u - 256) * idot @@ -663,6 +664,7 @@ def private dequant_q40_row_grp(kqg : uint8 const?; ksg : uint8 const?; r, mr, n //! iq4xs grp row dequant: the k4 tiling + the k4 scale-row interleave (d, signed int8 sc, no //! min) — w = (d·sc)·IQ4NL_LUT[q]. def private dequant_iq4xs_row_grp(kqg : uint8 const?; ksg : uint8 const?; r, mr, nsb : int64; var dst : float?) { + let lut = iq4nl_lut() unsafe { for (sbi in range64(nsb)) { let qb = sbi * 128l * mr @@ -675,8 +677,8 @@ def private dequant_iq4xs_row_grp(kqg : uint8 const?; ksg : uint8 const?; r, mr, for (j in range64(4l)) { for (t in range64(4l)) { let nib = int(kqg[qb + ((blk * 4l + j) * mr + r) * 4l + t]) - dst[kb + j * 4l + t] = dl * float(IQ4NL_LUT[nib & 15]) - dst[kb + 16l + j * 4l + t] = dl * float(IQ4NL_LUT[nib >> 4]) + dst[kb + j * 4l + t] = dl * float(lut[nib & 15]) + dst[kb + 16l + j * 4l + t] = dl * float(lut[nib >> 4]) } } } diff --git a/modules/dasLLAMA/dasllama/dasllama_math_gen.das b/modules/dasLLAMA/dasllama/dasllama_math_gen.das index d1e9b307df..755b70252f 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_gen.das @@ -307,6 +307,7 @@ def q40_grp_row_dot(kqg : uint8 const?; ksg : uint8 const?; r, mr : int64; xqp : def iq4xs_grp_row_dot(kqg : uint8 const?; ksg : uint8 const?; r, mr : int64; xqp : int8 const?; xsp : float const?; xbsp : int const?; n : int64) : float { var acc = 0.0 let nsb = n / 256l + let lut = iq4nl_lut() unsafe { for (sbi in range64(nsb)) { let qb = sbi * 128l * mr @@ -319,8 +320,8 @@ def iq4xs_grp_row_dot(kqg : uint8 const?; ksg : uint8 const?; r, mr : int64; xqp for (j in range64(4l)) { for (t in range64(4l)) { let nib = int(kqg[qb + ((blk * 4l + j) * mr + r) * 4l + t]) - idot += int(IQ4NL_LUT[nib & 15]) * int(xqp[b * 32l + j * 4l + t]) - idot += int(IQ4NL_LUT[nib >> 4]) * int(xqp[b * 32l + 16l + j * 4l + t]) + idot += int(lut[nib & 15]) * int(xqp[b * 32l + j * 4l + t]) + idot += int(lut[nib >> 4]) * int(xqp[b * 32l + 16l + j * 4l + t]) } } let u = int(ksg[sb + 4l * mr + blk * mr + r]) diff --git a/modules/dasLLAMA/harness/gen_tune_probe.das b/modules/dasLLAMA/harness/gen_tune_probe.das index 329b457e35..c6a9d1f020 100644 --- a/modules/dasLLAMA/harness/gen_tune_probe.das +++ b/modules/dasLLAMA/harness/gen_tune_probe.das @@ -440,6 +440,9 @@ struct KqFixture { def kq_qsb(fmt : int64) : int64 => dasllama_gemm_schema::kq_qsb(int(fmt)) def kq_ssb(fmt : int64) : int64 => dasllama_gemm_schema::kq_ssb(int(fmt)) +// the family's sidecar entry = its [tune] function name (the K-quants by bit width, q40/iq4xs by tag) +def kq_tile_entry(fmt : int64) : string => fmt == 40l ? "q40q8_tile_gen" : (fmt == 44l ? "iq4xsq8_tile_gen" : "k{fmt}q8_tile_gen") + // synthetic disk superblocks, varied per (row, superblock) so no two blocks repeat: any byte // is a legal quant, and the small d/dmin keep the f16 (s, o) pairs finite and row sums tame def pack_kq_scale_header(var blkb : array; base : int) { @@ -470,6 +473,19 @@ def pack_kq_disk_block(fmt : int64; var blkb : array; base : int) { blkb[blk * 18 + 2 + i] = uint8((base * 37 + blk * 59 + i * 101 + 29) % 256) } } + } elif (fmt == 44l) { + // 136B IQ4_XS disk block: f16 d, u16 scales_h, 4 scales_l bytes, 128 nibble bytes — every byte pattern is legal + let dbits = f32_to_f16(0.002 + 0.0001 * float(base % 13)) + blkb[0] = uint8(dbits & 0xFF) + blkb[1] = uint8(dbits >> 8u) + blkb[2] = uint8((base * 7 + 3) % 256) + blkb[3] = uint8((base * 11 + 5) % 256) + for (i in range(4)) { + blkb[4 + i] = uint8((base * 13 + i * 17 + 9) % 256) + } + for (i in range(128)) { + blkb[8 + i] = uint8((base * 37 + i * 101 + 29) % 256) + } } elif (fmt == 4l) { pack_kq_scale_header(blkb, base) for (i in range(128)) { @@ -506,6 +522,8 @@ def repack_kq_grp_fmt(fmt : int64; var kq : uint8?; var ks : uint8?; n, d, mr : repack_k5_grp(kq, ks, n, d, mr) } elif (fmt == 40l) { repack_q40_grp(kq, ks, n, d, mr) + } elif (fmt == 44l) { + repack_iq4xs_grp(kq, ks, n, d, mr) } else { repack_k6_grp(kq, ks, n, d, mr) } @@ -518,7 +536,7 @@ def build_kq_fixture(fmt : int64; n, d, ntok : int64) : KqFixture { fx.kq |> resize(d * fx.nsb * qsb) fx.ks |> resize(d * fx.nsb * ssb) var blkb : array - blkb |> resize(fmt == 4l || fmt == 40l ? 144l : (fmt == 5l ? 176l : 210l)) + blkb |> resize(fmt == 4l || fmt == 40l ? 144l : (fmt == 5l ? 176l : (fmt == 44l ? 136l : 210l))) for (r in range64(d)) { for (s in range64(fx.nsb)) { let base = int(r * 31l + s * 7l) @@ -530,6 +548,8 @@ def build_kq_fixture(fmt : int64; n, d, ntok : int64) : KqFixture { transcode_q5k_superblock(blkb, 0l, fx.kq, sb * qsb, fx.ks, sb * ssb) } elif (fmt == 40l) { transcode_q40_superblock(blkb, 0l, fx.kq, sb * qsb, fx.ks, sb * ssb) + } elif (fmt == 44l) { + transcode_iq4xs_superblock(blkb, 0l, fx.kq, sb * qsb, fx.ks, sb * ssb) } else { transcode_q6k_superblock(blkb, 0l, fx.kq, sb * qsb, fx.ks, sb * ssb) } @@ -564,8 +584,11 @@ def build_kq_fixture(fmt : int64; n, d, ntok : int64) : KqFixture { fx.yref[tk * d + g * 4l + r] = (fmt == 40l ? q40_grp_row_dot(kqg, ksg, r, 4l, addr(fx.xq[0]) + tk * n, addr(fx.xs[0]) + tk * (n / 256l), addr(fx.xbs[0]) + tk * (n / 16l), n) - : kq_grp_row_dot(fmt, kqg, ksg, r, 4l, - addr(fx.xq[0]) + tk * n, addr(fx.xs[0]) + tk * (n / 256l), addr(fx.xbs[0]) + tk * (n / 16l), n)) + : (fmt == 44l + ? iq4xs_grp_row_dot(kqg, ksg, r, 4l, + addr(fx.xq[0]) + tk * n, addr(fx.xs[0]) + tk * (n / 256l), addr(fx.xbs[0]) + tk * (n / 16l), n) + : kq_grp_row_dot(fmt, kqg, ksg, r, 4l, + addr(fx.xq[0]) + tk * n, addr(fx.xs[0]) + tk * (n / 256l), addr(fx.xbs[0]) + tk * (n / 16l), n))) } } } @@ -587,6 +610,9 @@ def kq_tile_variants(fmt : int64) : array> { if (fmt == 40l) { return <- q40q8_tile_gen_variants() } + if (fmt == 44l) { + return <- iq4xsq8_tile_gen_variants() + } return <- k6q8_tile_gen_variants() } @@ -610,6 +636,12 @@ def kq_gemv_variants_by_suffix(fmt : int64) : table { tab[v._0] = v._1 } delete gvs + } elif (fmt == 44l) { + var gvs <- iq4xsq8_gemv_gen_variants() + for (v in gvs) { + tab[v._0] = v._1 + } + delete gvs } else { var gvs <- k6q8_gemv_gen_variants() for (v in gvs) { @@ -642,6 +674,12 @@ def kq_layout_mrs(fmt : int64) : table { mrs[v._0] = invoke(v._1) } delete lvs + } elif (fmt == 44l) { + var lvs <- iq4xsq8_layout_gen_variants() + for (v in lvs) { + mrs[v._0] = invoke(v._1) + } + delete lvs } else { var lvs <- k6q8_layout_gen_variants() for (v in lvs) { @@ -659,7 +697,7 @@ def kq_layout_mrs(fmt : int64) : table { def run_kq_tile(tile; fx : KqFixture; kq : array; ks : array; mr : int64; var y : array) { let ng = fx.d / mr verify(fx.ntok % 4l == 0l) - let packed = fx.fmt == 4l || fx.fmt == 40l // pure-nibble planes: the tile reads them directly + let packed = fx.fmt == 4l || fx.fmt == 40l || fx.fmt == 44l // pure-nibble planes: the tile reads them directly var panel : array if (!packed) { panel |> resize(int(mr * fx.nsb * 256l)) @@ -961,7 +999,7 @@ def kq_test_family(fmt : int64; kfxs : array) : bool { // and the gemv (streamed + hot decode shapes), print the table, and return the tile-best // suffix ("" = a gate failed). The kq gemv is nrsplit-independent — same-mr rows share the // plane and the gemv shape — so the tile bench decides the family entry. -def kq_tune_family(fmt : int64; var rows_out : table) : string { // nolint:STYLE037,STYLE038 — one family race: shared repacked buffers couple gate, bench and pick +def kq_tune_family(fmt : int64; var rows_out : table) : string { // nolint:STYLE038 — one family race: shared repacked buffers couple gate, bench and pick // batch shape: fat STREAMING ffn (weights > the L2 budget, multi-token-block walk) — the shape // class that carries ~90% of prefill time. The old 2048x512x64 kv-projection probe was L2-hot // and crowned hot-shape winners that lose double-digit % at model scale (mr4 vs mr8, M1). @@ -1086,7 +1124,7 @@ def kq_tune_family(fmt : int64; var rows_out : table) : stri tune_detail("k{fmt} {tvs[vi]._0} (mr={vmr[vi]}): tile {macs / best[vi] / 1000.0lf} GMAC/s, gemv stream {gmacs / gbest[vi] / 1000.0lf} GMAC/s ({gwbytes / gbest[vi] / 1000.0lf} GB/s w), hot {hmacs / hbest[vi] / 1000.0lf} GMAC/s (best of {ROUNDS})\n") } var tnames <- [for (t in tvs); t._0] - winner = gen_sticky(fmt == 40l ? "q40q8_tile_gen" : "k{fmt}q8_tile_gen", tnames, best, winner) + winner = gen_sticky(kq_tile_entry(fmt), tnames, best, winner) delete tnames let wname = tvs[winner]._0 delete keep @@ -1720,15 +1758,15 @@ def tune_mode_run : bool { // nolint:STYLE037,STYLE038 — flat family-by-family print("aborting mid-run - no sidecar written\n") return false } - for (fmt in fixed_array(4l, 5l, 6l, 40l)) { + for (fmt in fixed_array(4l, 5l, 6l, 40l, 44l)) { interrupt_gate("k{fmt}q8_tile_gen") let kq_t0 = ref_time_ticks() - tune_progress_kernel_begin("k{fmt}q8_tile_gen", ROUNDS) + let entry = kq_tile_entry(fmt) + tune_progress_kernel_begin(entry, ROUNDS) var krows : table let w = kq_tune_family(fmt, krows) - tune_detail("TUNE_GEN_TIME k{fmt}_family {get_time_usec(kq_t0) / 1000} ms\n") - tune_progress_kernel_end("k{fmt}q8_tile_gen", w, empty(w) ? "rejected" : "beats") - let entry = fmt == 40l ? "q40q8_tile_gen" : "k{fmt}q8_tile_gen" + tune_detail("TUNE_GEN_TIME {entry} family {get_time_usec(kq_t0) / 1000} ms\n") + tune_progress_kernel_end(entry, w, empty(w) ? "rejected" : "beats") // gate failure (the broken perm is named above): queue the always-correct reference // body so the scope still COMPLETES — a missing key re-tunes every start pending[entry] = empty(w) ? "reference" : w From 2de34d9890b07f03c549ee6ea2db1d95893a61cb Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Sun, 30 Aug 2026 08:11:57 -0700 Subject: [PATCH 03/61] dasLLAMA: IQ4_XS JIT emitter - emit_block_iq4xs (mx4 LUT decode + signed dot_lane + the k4 fold), tuner test-mode family, HOW_TO section 5 + QUIRK 11 The kq gemv/tile walks take a third block body: the q40-tiled nibbles decode through the kvalues_iq4nl LUT baked as a constant vector (tbl1/pshufb, the mx4 primitive) into signed int8, dotted on dot_lane's sign-trick lattice, folded as sum_blk sc*idot * d * d8 with no min term. setup_tile_emit wires the mx4 primitive for fmt 44; the decline stubs become the real stamps. gen_tune_probe's test mode gains the 44 family (it was only in the tune-mode list). Gates: tuner family gate 10/10 perms (live stamp dot_maddubs_width256_mr8, maxdiff 1.5e-5), test_kquant -jit 121/125 (4 absent-model skips), test_kqformat 14/14, 1B IQ4_XS end to end text unchanged; after the sidecar re-mint decode 59-60 t/s vs 39 t/s reference bodies. HOW_TO: section 5 rewritten from the executed steps; QUIRK 11 (a sidecar minted while the stubs declined pins "reference" and a .das-only emitter change never invalidates it) and the QUIRK 2 addendum (the probe's two family lists). Co-Authored-By: Claude Fable 5 --- modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md | 58 ++++++++-- .../dasLLAMA/dasllama/dasllama_gemm_gen.das | 101 +++++++++++++++--- modules/dasLLAMA/harness/gen_tune_probe.das | 3 + 3 files changed, 139 insertions(+), 23 deletions(-) diff --git a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md index 82310aeed2..b2656e4ce2 100644 --- a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md +++ b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md @@ -127,12 +127,32 @@ Gate: `test_kqformat` + `test_kquant` on the interpreter binary (the stubs' refe ## 5. The JIT emitter - `dasllama_gemm_gen.das` -Untested for IQ4_XS at the time of writing (the decline stubs stand). What the emitter offers: `kq_tile_gen_impl(gc, fmt)` / `kq_gemv_gen_impl(gc, fmt)` are one emitter specialized on `fmt` -at generation time; `emit_block_kqv2` carries the nibble unpack + integer fold for 4/5/6/40; -`emit_block_mx4` carries the codebook path (`lut_lookup` = `tbl1` / `pshufb`, `dot_lane` with -`abs_w` + `psign` for signed weights on the u8 x s8 ISAs). A codebook format is mx4's block -with the kq scale fold. +at generation time: the group walk, the slice/loop machinery (`emit_slice`) and the store +epilogue are format-agnostic; only the block body and the primitives `setup_tile_emit` wires +differ. IQ4_XS took: + +1. A block body, `emit_block_iq4xs`: `emit_block_mx4`'s nibble unpack + `lut_lookup` (`tbl1` + / `pshufb`, the codebook baked as a constant vector) + `dot_lane` with `abs_w` - the SIGNED + sign-trick lattice, because LUT weights are int8; `emit_block_kqv2`'s `kq_dot_lane` / + `kq_dot_mem` run the unsigned-nibble lattice and do not apply - then k4's fold with the min + term dropped: per block `iacc += sext(sc) * (idot_lo + idot_hi)`, per superblock + `f += iacc * (d * d8)`. Plane addressing is the repack's: nibbles at + `wb + ((blk*16 + j*4)*mr + qd*w8)`, `sc` at `sb + 4*mr + blk*mr + r`, `d` at `sb + 2*r`. +2. `emit_one_block`: `te.kq == 44` routes to the new body ahead of the `te.kq != 0` arm. +3. `setup_tile_emit(te, gc, p, needMx4 = fmt == 44)`: the mx4 primitive wiring is reused as + is; the LUT bake picks `iq4nl_lut()` over the e2m1 table on `te.kq == 44`. +4. The two stubs become `=> kq_gemv_gen_impl(gc, 44)` / `kq_tile_gen_impl(gc, 44)`. +5. No `perm_declines` change: the tbl1 rail already sits on every sdot perm (it is the mx4 + companion's), and pshufb is implied by the x64 tiers. + +Gates, in order: `DAS_TUNE_MODE=test bin/Release/daslang.exe -jit harness/gen_tune_probe.das` +(the family must be in BOTH of the probe's lists - QUIRK 2) gates every perm of the family's +grid against the scalar oracle - a stamped perm reports its layout companion's `mr` (8 on this +box) and a fast-math-sized maxdiff (1.5e-5), a declined one `mr=4` and maxdiff 0 (that is the +reference body answering, not the emitter); then `test_kquant` under `-jit` (the tile-vs-GEMV +gate is bit-exact by construction); then the end-to-end run - and read QUIRK 11 before +trusting its numbers. ## 6. Vulkan - `dasllama_vulkan_classes.das`, `dasllama_vulkan_common.das`, `dasllama_vulkan_prefill.das` @@ -175,9 +195,13 @@ where, why it is so today, what unquirked looks like. An empty ledger is a legit dequants, dots, repacks and calls the stubs through the same `fmt == 4/5/6/40` chains in five gates (28 arms for one format), and raises `_cyclomatic_complexity` / `_function_length` per format added. `harness/gen_tune_probe.das` repeats the shape (9 - arms). Unquirked: per-format dispatch helpers in one `_kq_fixtures.das` shared by the test - and the harness (`kq_transcode_sb`, `kq_dequant_sb`, `kq_dot`, `kq_repack`, `kq_gemv_gen`, - `kq_tile_gen`), each a single ladder. + arms), and its test mode gates a hand list of families (the `kq_test_family(4l/5l/6l/40l)` + calls plus their fixture arrays) SEPARATE from the tune-mode family array - IQ4_XS sat in + the tune list and not the test list, so `GEN TUNE TEST OK` said nothing about it until the + call was added. Unquirked: per-format dispatch helpers in one `_kq_fixtures.das` shared by + the test and the harness (`kq_transcode_sb`, `kq_dequant_sb`, `kq_dot`, `kq_repack`, + `kq_gemv_gen`, `kq_tile_gen`), each a single ladder, and ONE family array both probe modes + walk. 3. **The bake identity is hand-formatted.** `DlimCpuConfig` gains `kq_mr`, and `dlim_identity` must ALSO append it to the identity string by hand - a field added without the string leaves two images with different repack interleaves keyed identically. Why: the @@ -229,6 +253,17 @@ where, why it is so today, what unquirked looks like. An empty ledger is a legit dasllama file as broken (`get_total_perf_cores` missing). Trust only the worktree binary: `bin/Release/daslang.exe dastest/dastest.das -- --test `. Run the session inside the worktree once it is bootstrapped. +11. **A sidecar minted while the stubs declined pins the family to `"reference"`.** The app's + auto-policy tune ran during the first end-to-end (QUIRK 4's stubs in place), every perm of + `iq4xsq8_tile_gen` declined, and `examples/dasLLAMA/run.tune.json` recorded + `"iq4xsq8_tile_gen" : "reference"` - the framework's explicit-reference form, which forces + the original body. Sidecar staleness keys on the binary's mtime, and the emitter is `.das` + (JIT-compiled), so landing it invalidates nothing: the next run logs the same + `27 tune-stamped`, serves the reference body, and its text and t/s match the pre-emitter run + exactly - a "the emitter changed nothing" reading that is false. Re-mint with `-- --tune` + on the app (a whole-scope re-tune) or delete the sidecar. Unquirked: fold the family's + generator hash (the JIT DLL cache key already carries it) into the sidecar identity, so a + generator change reads as stale. ## Per-format notes @@ -251,4 +286,9 @@ QUIRK 9 (the codebook global on workers). Method that found them: a Python oracl dequantizes rows straight from the GGUF bytes, a daslang probe calling `mm_at_kq_pre` on the same rows (run it UNDER the job queue), a layer bisect (`config.n_layers = L`, logits cosine against a sibling quant of the same model), and a dump of the image's group-0 plane bytes -against the grp layout computed in Python. JIT emitter, Vulkan, Metal: pending. +against the grp layout computed in Python. JIT emitter: `emit_block_iq4xs` (section 5); +family gate 10/10 perms, live stamp on this box `dot_maddubs_width256_mr8` (mr 8, maxdiff +1.5e-5), `test_kquant` 121/125 under `-jit`; after the sidecar re-mint (QUIRK 11) the 1B +decodes at 59-60 t/s against 39 t/s on the reference body, same text. The body rides mx4's +chunk-load + lane-splat dot path; `emit_block_kqv2`'s x64 `vpbroadcastd` / `madd16` chains are +the untried next lever. Vulkan, Metal: pending. diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das index f958331941..1664657274 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das @@ -269,7 +269,7 @@ struct private TileEmit { mx4 : bool // block emitter: mx4 LUT-dequant instead of Q8 loads q51 : bool // block emitter: q51 nibble+qh unpack (32-weight blocks, // per-block f16 d/m scale pairs + xbs min-term) - kq : int // 4/5/6 = K-quant superblock emitter (0 = not kq): the + kq : int // 4/5/6/40 = K-quant superblock emitter, 44 = IQ4_XS (0 = not kq): the // block unit is a 256-weight superblock over the grp // kq planes; dots run unsigned-q (kq_dot_lane) kqBytes : bool // kq weights are a BYTE-EXPANDED panel (the tile's @@ -1195,11 +1195,84 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f } } +// One 256-weight SUPERBLOCK, IQ4_XS grp form (te.kq = 44): mx4's LUT decode (tbl1/pshufb) +// yields SIGNED int8 weights, so the dots ride dot_lane's sign-trick lattice, not kq_dot_lane; +// the fold is k4's integer sub-scale sum with no min term: f += (sum_blk sc * idot) * d * d8. +def private emit_block_iq4xs(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f : LLVMOpaqueValue? [8]; tokBase, tokCount : int) { + let b = te.builder + let rq = te.rq + let mr = te.interleave + let w8 = te.width / 8 + var wb = LLVMBuildMul(b, sbi, te.types->ConstI64(uint64(mr * 128)), "wb") + var sb = LLVMBuildMul(b, sbi, te.types->ConstI64(uint64(mr * 20)), "sb") + var xb = LLVMBuildMul(b, sbi, te.types->ConstI64(0x100ul), "xb") + var vri8 = LLVMVectorType(te.types.t_int8, uint(te.rv)) + var dv : LLVMOpaqueValue? [2] + for (qd in range(rq)) { + dv[qd] = load_f16_vec_at(te, te.sg, LLVMBuildAdd(b, sb, te.types->ConstI64(uint64(2 * (qd * te.rv))), ""), "d{qd}") + } + var iacc : LLVMOpaqueValue? [8] + for (i in range(tokCount * rq)) { + iacc[i] = LLVMConstNull(te.vni32) + } + for (blk in range(8)) { + var xoff0 = LLVMBuildAdd(b, xb, te.types->ConstI64(uint64(blk * 32)), "") + var xoff1 = LLVMBuildAdd(b, xb, te.types->ConstI64(uint64(blk * 32 + 16)), "") + var xv0 : LLVMOpaqueValue? [4] + var xv1 : LLVMOpaqueValue? [4] + for (i in range(tokCount)) { + xv0[i] = load_v16i8(te, te.x[tokBase + i], xoff0, "x{tokBase + i}_{blk}lo") + xv1[i] = load_v16i8(te, te.x[tokBase + i], xoff1, "x{tokBase + i}_{blk}hi") + } + var a : LLVMOpaqueValue? [8] // [token * rq + qd]; tokCount*rq <= 8 (nrsplit*vq rail) + for (i in range(tokCount * rq)) { + a[i] = LLVMConstNull(te.vni32) + } + for (j in range(4)) { + for (qd in range(rq)) { + var noff = LLVMBuildAdd(b, wb, te.types->ConstI64(uint64((blk * 16 + j * 4) * mr + qd * w8)), "") + var nv = load_vec(te, te.vwi8, te.wg, noff, "nv{blk}_{j * rq + qd}") + var loIdx = LLVMBuildAnd(b, nv, splat_i8w(te, 15), "nib.lo") + var hiIdx = LLVMBuildLShr(b, nv, splat_i8w(te, 4), "nib.hi") + var wlo = lut_lookup(te, loIdx, "wlo{blk}_{j * rq + qd}") + var whi = lut_lookup(te, hiIdx, "whi{blk}_{j * rq + qd}") + var ulo = abs_w(te, wlo, "ulo{blk}_{j * rq + qd}") + var uhi = abs_w(te, whi, "uhi{blk}_{j * rq + qd}") + for (i in range(tokCount)) { + a[i * rq + qd] = dot_lane(te, dot_lane(te, a[i * rq + qd], wlo, ulo, xv0[i], j), whi, uhi, xv1[i], j) + } + } + } + var scv : LLVMOpaqueValue? [2] + for (qd in range(rq)) { + var scp = LLVMBuildGEP2(b, te.types.t_int8, te.sg, LLVMBuildAdd(b, sb, te.types->ConstI64(uint64(4 * mr + blk * mr + qd * te.rv)), ""), "scp{blk}_{qd}") + scv[qd] = LLVMBuildSExt(b, LLVMBuildLoad2Aligned(b, vri8, scp, 1u, ""), te.vni32, "sc{blk}_{qd}") + } + for (i in range(tokCount)) { + for (qd in range(rq)) { + iacc[i * rq + qd] = LLVMBuildAdd(b, iacc[i * rq + qd], LLVMBuildMul(b, scv[qd], a[i * rq + qd], ""), "ia{tokBase + i}_{blk}_{qd}") + } + } + } + for (i in range(tokCount)) { + let tk = tokBase + i + var qp = LLVMBuildGEP2(b, te.types.t_float, te.xs[tk], sbi, "qp{tk}") + var d8 = LLVMBuildLoad2Aligned(b, te.types.t_float, qp, 4u, "d8{tk}") + var d8v = splat_f32(te, d8, "d8v{tk}") + for (qd in range(rq)) { + var ds = LLVMBuildFMul(b, dv[qd], d8v, "ds{tk}_{qd}") + f[i * rq + qd] = fold_fma(te, LLVMBuildSIToFP(b, iacc[i * rq + qd], te.vnf32, ""), ds, f[i * rq + qd], "f{tk}_{qd}s") + } + } +} + // Block-emitter dispatch: the slice/loop machinery is quant-agnostic. The MMA form needs a // token PAIR — single-token slices ride the sdot lattice instead. kq blocks are SUPERBLOCKS // (the kq gemv/tile drive emit_slice with nb = n/256). def private emit_one_block(var te : TileEmit; var bi : LLVMOpaqueValue?; var f : LLVMOpaqueValue? [8]; tokBase, tokCount : int) { - if (te.kq != 0) { + if (te.kq == 44) { + emit_block_iq4xs(te, bi, f, tokBase, tokCount) + } elif (te.kq != 0) { emit_block_kqv2(te, bi, f, tokBase, tokCount) } elif (te.q51) { emit_block_q51(te, bi, f, tokBase, tokCount) @@ -1459,13 +1532,15 @@ def private setup_tile_emit(var te : TileEmit; var gc : LlvmCodeCtx; p : TilePer } } if (needMx4) { - // the doubled-e2m1 LUT as a constant vector — no runtime table pointer; broadcast per 16-byte lane on x64 - let kv = fixed_array(0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12) + // the nibble codebook as a constant vector (mx4: doubled e2m1; kq 44: kvalues_iq4nl) - no runtime table pointer; broadcast per 16-byte lane on x64 + let e2m1 = fixed_array(0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12) + let iq4nl = iq4nl_lut() let lanes = te.dotKind == DOT_SDOT ? 1 : te.width / 128 var lutElems : array lutElems |> reserve(16 * lanes) for (_lane in range(lanes)) { - for (v in kv) { + for (i in range(16)) { + let v = te.kq == 44 ? int(iq4nl[i]) : e2m1[i] lutElems |> push(LLVMConstInt(te.types.t_int8, uint64(v & 255), 0)) } } @@ -2290,7 +2365,7 @@ def private kq_gemv_gen_impl(var gc : LlvmCodeCtx; fmt : int) : bool { let p = companion_perm(p0) var te = TileEmit(kq = fmt) - if (!setup_tile_emit(te, gc, p, false)) return false + if (!setup_tile_emit(te, gc, p, fmt == 44)) return false let b = gc.jit.builder var entry = LLVMAppendBasicBlockInContext(gc.jit.ctx, gc.impl, "entry") @@ -2421,8 +2496,8 @@ def private kq_tile_gen_impl(var gc : LlvmCodeCtx; fmt : int) : bool { if (perm_declines(gc, p0)) return false let p = companion_perm(p0) - var te = TileEmit(kq = fmt, kqBytes = fmt == 5 || fmt == 6) // k4/q40 tiles read the packed planes - if (!setup_tile_emit(te, gc, p, false)) return false + var te = TileEmit(kq = fmt, kqBytes = fmt == 5 || fmt == 6) // k4/q40/iq4xs tiles read the packed planes + if (!setup_tile_emit(te, gc, p, fmt == 44)) return false let b = gc.jit.builder var entry = LLVMAppendBasicBlockInContext(gc.jit.ctx, gc.impl, "entry") @@ -2462,12 +2537,10 @@ def private kq_tile_gen_impl(var gc : LlvmCodeCtx; fmt : int) : bool { return true } -// iq4xs: the LUT-nibble emitter lands with the kernel arc; until then both stamps decline -// (reference bodies serve) — a declined generator is the framework's own fallback path. -[unused_argument(gc)] -def private iq4xs_gemv_gen(var gc : LlvmCodeCtx) : bool => false -[unused_argument(gc)] -def private iq4xs_tile_gen(var gc : LlvmCodeCtx) : bool => false +// iq4xs rides the kq gemv/tile walks with emit_block_iq4xs as the block body (LUT decode + +// sign-trick dots); the tbl1/pshufb rail is the mx4 companion's, already in perm_declines. +def private iq4xs_gemv_gen(var gc : LlvmCodeCtx) : bool => kq_gemv_gen_impl(gc, 44) +def private iq4xs_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 44) def private k4_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 4) def private k5_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 5) diff --git a/modules/dasLLAMA/harness/gen_tune_probe.das b/modules/dasLLAMA/harness/gen_tune_probe.das index c6a9d1f020..633b47321d 100644 --- a/modules/dasLLAMA/harness/gen_tune_probe.das +++ b/modules/dasLLAMA/harness/gen_tune_probe.das @@ -1177,6 +1177,7 @@ def test_mode : bool { // nolint:STYLE037,STYLE038 — flat per-variant correctn var kfxs5 <- [ <- build_kq_fixture(5l, 256l, 32l, 8l), <- build_kq_fixture(5l, 768l, 32l, 8l), <- build_kq_fixture(5l, 2048l, 32l, 8l)] var kfxs6 <- [ <- build_kq_fixture(6l, 256l, 32l, 8l), <- build_kq_fixture(6l, 768l, 32l, 8l), <- build_kq_fixture(6l, 2048l, 32l, 8l)] var kfxs40 <- [ <- build_kq_fixture(40l, 256l, 32l, 8l), <- build_kq_fixture(40l, 768l, 32l, 8l), <- build_kq_fixture(40l, 2048l, 32l, 8l)] + var kfxs44 <- [ <- build_kq_fixture(44l, 256l, 32l, 8l), <- build_kq_fixture(44l, 768l, 32l, 8l), <- build_kq_fixture(44l, 2048l, 32l, 8l)] var vs <- q8q8_tile_gen_variants() var mrs <- variant_mrs() var wbs <- variant_wbias() @@ -1301,6 +1302,7 @@ def test_mode : bool { // nolint:STYLE037,STYLE038 — flat per-variant correctn allok = kq_test_family(5l, kfxs5) && allok allok = kq_test_family(6l, kfxs6) && allok allok = kq_test_family(40l, kfxs40) && allok + allok = kq_test_family(44l, kfxs44) && allok delete wtab delete mtab delete gtab @@ -1316,6 +1318,7 @@ def test_mode : bool { // nolint:STYLE037,STYLE038 — flat per-variant correctn delete kfxs5 delete kfxs6 delete kfxs40 + delete kfxs44 return allok } From 7d277111969d7b24e407dc624ecce00a1e1c4fba Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Sun, 30 Aug 2026 08:25:23 -0700 Subject: [PATCH 04/61] dasLLAMA: IQ4_XS on the Vulkan tier - KqGemvIq4xs + KqBatchIq4xs (codebook decode into signed sdot4 lanes), the id bridge, five-format kernel cells + a float witness; HOW_TO section 6 vk_kq_schema_id gains the 6 -> 44 arm (the arena plan panicked on the first iq4xs stack without it). iq4_word decodes one word of nibble lanes through the kvalues_iq4nl LUT (a fixed_array local the SPIR-V emitter lowers to a Function-storage variable) into signed int8 lanes, so the dots are plain OpSDot with no block-sum term; iq4_sc reads the signed (ls - 32) sub-scale off the k4-shaped 5-word row. The GEMV class overrides blk_contrib, the batch tile stage_w / stage_ws / blk_fma; the five class ladders gain an arm and gemv_cls_set keys on kq_sb. No cm2 decode tile, by followup_vulkan item 24 (new formats land on the class template, not as more hand-stamped bodies) - iq4xs prefill rides the kq batch tile as q40 does. Gates: test_vulkan_kernels 64/64 with the family cells at five formats plus iq4xs_gemv_float_oracle (a dequant straight off the plane bytes - the witness a wrong codebook pack cannot pass while staying class-consistent); the 1B IQ4_XS model on the resident driver reproduces the CPU text at gen 102 t/s. Co-Authored-By: Claude Fable 5 --- modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md | 41 +++++-- .../dasllama/dasllama_vulkan_classes.das | 104 +++++++++++++++++- .../dasllama/dasllama_vulkan_common.das | 7 +- modules/dasLLAMA/tests/_vkd_oracles.das | 38 +++++++ .../dasLLAMA/tests/test_vulkan_kernels.das | 30 +++-- 5 files changed, 193 insertions(+), 27 deletions(-) diff --git a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md index b2656e4ce2..b3ad8d4810 100644 --- a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md +++ b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md @@ -154,14 +154,36 @@ reference body answering, not the emitter); then `test_kquant` under `-jit` (the gate is bit-exact by construction); then the end-to-end run - and read QUIRK 11 before trusting its numbers. -## 6. Vulkan - `dasllama_vulkan_classes.das`, `dasllama_vulkan_common.das`, `dasllama_vulkan_prefill.das` - -Untested at the time of writing. The class hierarchy: `KqGemvBase` + `def override blk_contrib` -per format; `KqBatchBase` + `stage_w` / `stage_ws` / `blk_fma`; the cm2 tiles get a -`[spirv_decode] def decode_` (`followup_vulkan.md` item 24 rules the template). Plus -`vk_kq_schema_id`, the `kq_batch_cls_*` / `gemv_cls_*` / `cm2_cls_*` ladders, `pf_f16_feed`, -the gather's format arms, and the oracle cells in `tests/test_vulkan_kernels.das` + -`tests/_vk_kq_fixtures.das`. +## 6. Vulkan - `dasllama_vulkan_classes.das`, `dasllama_vulkan_common.das` + +The tier reads the CPU planes verbatim (`stack_plane_bytes` -> `arena_block_bytes` -> +`kq_qsb(vk_kq_schema_id(fmt))` x `KQ_DEV_SSB`), so a format whose plane pair already has the +20 B decoded scale row needs no upload work - only the id bridge and the kernels. IQ4_XS took: + +1. `vk_kq_schema_id` (`dasllama_vulkan_common.das`): the `int(KqFmt)` -> kernel-id arm + (`6 -> 44`). This is the third id space of QUIRK 5 at its Vulkan seam; without the arm the + arena plan panics on the first iq4xs stack. +2. `KqGemvIq4xs : KqGemvBase` - `def override blk_contrib`: the q40 nibble tiling + (`wq4[wsb * 8 + blk]`), each nibble word decoded through `iq4_word` (a `fixed_array` LUT + local - the SPIR-V emitter lowers a `let` fixed array to a Function-storage variable and + indexes it) into SIGNED lanes for `sdot4` (OpSDot, signed x signed - the block-sum trick of + q40/k4 does not apply and is not needed), scale `d * sc` with `sc` the signed byte off word + 1..2 of the 5-word row (`unpack8` sign-extends, the k6 spelling). +3. `KqBatchIq4xs : KqBatchBase` - `stage_w` decodes the staged words through `iq4_word` + (k4's staging otherwise), `stage_ws` fills ONE plane with `d * sc`, `blk_fma` is + `xscl * ws * idot` (q40's without the `- 8 * bsum`). +4. Ladders: `kq_batch_cls_ensure` / `kq_batch_cls_enc_for` / `gemv_cls_ensure` / + `gemv_cls_enc` gain an arm; `gemv_cls_set`'s four-way `||` became `kq_sb(fmt)`. +5. Tests: `tests/_vkd_oracles.das` `kq_cls_ref` arm (the class-on-CPU oracle), the two family + cells in `tests/test_vulkan_kernels.das` go to five formats, and - because the codebook pack + is new bit-math that a class-vs-device compare cannot see (both sides run the same + `iq4_word`) - `iq4xs_gemv_float_oracle`, a float dequant straight off the plane bytes that + the class oracle must match. + +Not done, by ruling: a cm2 decode-in-load tile (`[spirv_decode] def decode_iq4xs`). The f16 feed +admits q8/k4/k6 only (`pf_f16_feed`), q40 and k5 have no cm2 tile either, and +`followup_vulkan.md` item 24 rules that new formats land on the one class template, not as +three more hand-stamped bodies. IQ4_XS prefill rides the kq batch tile like q40 does. ## 7. Metal @@ -291,4 +313,5 @@ family gate 10/10 perms, live stamp on this box `dot_maddubs_width256_mr8` (mr 8 1.5e-5), `test_kquant` 121/125 under `-jit`; after the sidecar re-mint (QUIRK 11) the 1B decodes at 59-60 t/s against 39 t/s on the reference body, same text. The body rides mx4's chunk-load + lane-splat dot path; `emit_block_kqv2`'s x64 `vpbroadcastd` / `madd16` chains are -the untried next lever. Vulkan, Metal: pending. +the untried next lever. Vulkan (section 6): `KqGemvIq4xs` + `KqBatchIq4xs`, the kernel suite +64/64 with the five-format family cells and the float witness, and the 1B IQ4_XS model on the resident driver reproduces the CPU text at gen 102 t/s (prefill 38 t/s on the 5-token prompt). Metal: pending. diff --git a/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das b/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das index 4289abce11..c541080579 100644 --- a/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das +++ b/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das @@ -2464,6 +2464,23 @@ def private k5_dep(hb : uint) : uint { return ((hb & 1u) << 4u) | ((hb & 2u) << 11u) | ((hb & 4u) << 18u) | ((hb & 8u) << 25u) } +// IQ4_XS codebook decode of one word of 4 nibble lanes (bits 7..4 of each byte clear) into 4 +// SIGNED int8 lanes (ggml's kvalues_iq4nl) — the sdot4 operand form; batch and gemv share it +def private iq4_word(n : uint) : uint { + let lut = fixed_array(-127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113) + let b0 = uint(lut[int(n & 15u)]) & 0xFFu + let b1 = uint(lut[int((n >> 8u) & 15u)]) & 0xFFu + let b2 = uint(lut[int((n >> 16u) & 15u)]) & 0xFFu + let b3 = uint(lut[int((n >> 24u) & 15u)]) & 0xFFu + return b0 | (b1 << 8u) | (b2 << 16u) | (b3 << 24u) +} + +// the iq4xs scale row's sub-scale for block blk: word 1..2 of the 5-word row, SIGNED (ls - 32) +def private iq4_sc(w : uint; blk : uint) : int { + let s4 = int4(unpack8(int(w))) + return blk % 4u == 0u ? s4.x : (blk % 4u == 1u ? s4.y : (blk % 4u == 2u ? s4.z : s4.w)) +} + class KqBatchBase { @ssbo @binding = 0 wq : array // weight quant planes (per-format block layout) @ssbo @binding = 1 wsu : array // scale planes (5-word superblock layout) @@ -2682,6 +2699,51 @@ class KqBatchQ40 : KqBatchK4 { } } +// IQ4_XS tile: the k4 staging with the codebook decode in stage_w (signed weight lanes, so no +// block-sum term) and one signed-scale plane — per block acc += xs * d*sc * idot +[vk_dispatch(name = "kq_batch_iq4xs_cls", kernel = "run", family = "kq_batch_cls", grid = "wgs", params = "wgs : int64")] +class KqBatchIq4xs : KqBatchBase { + def override stage_w(wsb0, wcol, nsb, s, word, grp : uint) { + for (ki in range(2)) { + let k = word + uint(ki) * 8u + let bu = k / 2u + let sh = (k % 2u) * 4u + var wv = uint4(0u) + if (wcol < pa.d) { + let wsb = wsb0 + wcol * nsb + s + let rb4 = wsb * 32u + bu * 4u + wv = uint4(iq4_word((wq[rb4] >> sh) & 0x0F0F0F0F), iq4_word((wq[rb4 + 1u] >> sh) & 0x0F0F0F0F), + iq4_word((wq[rb4 + 2u] >> sh) & 0x0F0F0F0F), iq4_word((wq[rb4 + 3u] >> sh) & 0x0F0F0F0F)) + } + ww4[grp * 17u + k] = wv + } + } + + def override stage_ws(wsb0, wt, nsb, s, tid : uint) { + let slane = tid % 32u + let sblk = tid / 32u + var sa = 0.0 + if (wt * 32u + slane < pa.d) { + let wsb = wsb0 + (wt * 32u + slane) * nsb + s + sa = unpackHalf2x16(wsu[wsb * 5u]).x * float(iq4_sc(wsu[wsb * 5u + 1u + sblk / 4u], sblk)) + } + wsw[sblk * 32u + slane] = sa + } + + def override blk_fma(bu, grp, word : uint; xscl : float) : float4 { + var i = int4(0) + dots(bu, grp, word, i) + let ws = float4(wsw[bu * 32u + word], wsw[bu * 32u + word + 8u], + wsw[bu * 32u + word + 16u], wsw[bu * 32u + word + 24u]) + return xscl * ws * float4(i) + } + + [spirv_kernel(local_size_x = 256, name = "kq_batch_iq4xs_cls_spv")] + def run { + tile_shell() + } +} + // Q5_K tile: the k4 compose with the 5th bit OR'd in from the 40B block's qh words [vk_dispatch(name = "kq_batch_k5_cls", kernel = "run", family = "kq_batch_cls", grid = "wgs", params = "wgs : int64")] class KqBatchK5 : KqBatchBase { @@ -5576,7 +5638,7 @@ let private KQC_ONES = 0x01010101u // sdot4 against this sums 4 signed bytes class KqGemvBase { @ssbo @binding = 0 @role = "alias" wq : array // weight quant words (k5's qh rides this view) @ssbo @binding = 0 wq4 : array // vec4 view of the same words - @ssbo @binding = 1 wsu : array // scale planes ([f16 d][f16 dmin][8 sc][8 mn] / q40 half pairs / k6 int8+d) + @ssbo @binding = 1 wsu : array // scale planes ([f16 d][f16 dmin][8 sc][8 mn] / q40 half pairs / k6 int8+d / iq4xs d + 8 signed sc) @ssbo @binding = 2 regions : array // (rel_wblock, xblock) pairs per region — bulk data, stays SSBO @ssbo @binding = 3 xq4 : array // activation quant words, vec4 view @ssbo @binding = 4 axs : array // activation block scales @@ -5725,6 +5787,8 @@ def kq_batch_cls_ensure(fmt : int; q40cm : bool) : bool { return ensure_kq_batch_k6_cls() } elif (fmt == int(KqFmt.q40)) { return ensure_kq_batch_q40_cls() + } elif (fmt == int(KqFmt.iq4xs)) { + return ensure_kq_batch_iq4xs_cls() } panic("dasLLAMA vulkan tier: no kq batch class for format {fmt}") return false @@ -5751,6 +5815,8 @@ def kq_batch_cls_enc_for(fmt : int; q40cm : bool; raw : VkCommandBuffer; var h : enc_kq_batch_k6_cls(raw, h, set_, pc, wgs) } elif (fmt == int(KqFmt.q40)) { enc_kq_batch_q40_cls(raw, h, set_, pc, wgs) + } elif (fmt == int(KqFmt.iq4xs)) { + enc_kq_batch_iq4xs_cls(raw, h, set_, pc, wgs) } else { panic("dasLLAMA vulkan tier: no kq batch class for format {fmt}") } @@ -5771,6 +5837,8 @@ def gemv_cls_ensure(fmt : int) : bool { return ensure_kq_gemv_k6_cls() } elif (fmt == int(KqFmt.q40)) { return ensure_kq_gemv_q40_cls() + } elif (fmt == int(KqFmt.iq4xs)) { + return ensure_kq_gemv_iq4xs_cls() } panic("dasLLAMA vulkan tier: no GEMV class for format {fmt}") return false @@ -5779,8 +5847,8 @@ def gemv_cls_ensure(fmt : int) : bool { def gemv_cls_set(fmt : int; bufs : uint64 const[6]; sizes : int64 const[6]; gbits : uint const[6]) : VkDescriptorSet { if (fmt == int(KqFmt.q8)) { return set_q8_gemv_cls(bufs, sizes, gbits) - } elif (fmt == int(KqFmt.k4) || fmt == int(KqFmt.k5) || fmt == int(KqFmt.k6) || fmt == int(KqFmt.q40)) { - return set_kq_gemv_cls(bufs, sizes, gbits) // the family shares one layout — one set serves all four + } elif (kq_sb(fmt)) { + return set_kq_gemv_cls(bufs, sizes, gbits) // the family shares one layout — one set serves every kq format } panic("dasLLAMA vulkan tier: no GEMV class for format {fmt}") return VkDescriptorSet() @@ -5797,14 +5865,16 @@ def gemv_cls_enc(fmt : int; raw : VkCommandBuffer; var h : VkHaz; var set_ : VkD enc_kq_gemv_k6_cls(raw, h, set_, pc, wgs) } elif (fmt == int(KqFmt.q40)) { enc_kq_gemv_q40_cls(raw, h, set_, pc, wgs) + } elif (fmt == int(KqFmt.iq4xs)) { + enc_kq_gemv_iq4xs_cls(raw, h, set_, pc, wgs) } else { panic("dasLLAMA vulkan tier: no GEMV class for format {fmt}") } } // Q4_K: w = d*sc*q - dmin*mn*1, q in [0,15] — per 32-block axs*(d*sc*idot - dmin*mn*bsum). -// The four formats share family=kq_gemv_cls: ONE set layout + ONE set_kq_gemv_cls serves all -// four pipelines (identical inherited members), each leaf keeping its own ensure_/enc_ names. +// The five formats share family=kq_gemv_cls: ONE set layout + ONE set_kq_gemv_cls serves all +// five pipelines (identical inherited members), each leaf keeping its own ensure_/enc_ names. [vk_dispatch(name = "kq_gemv_k4_cls", kernel = "run", family = "kq_gemv_cls", grid = "wgs", params = "wgs : int64")] class KqGemvK4 : KqGemvBase { def override blk_contrib(wsb, blk, xsb : uint) : float { @@ -5843,6 +5913,30 @@ class KqGemvQ40 : KqGemvBase { } } +// IQ4_XS: w = d*sc*LUT[q] — the q40 nibble tiling decoded through the codebook into signed +// lanes (no block-sum term), the k4 scale-row shape with SIGNED (ls - 32) sub-scales and no min +[vk_dispatch(name = "kq_gemv_iq4xs_cls", kernel = "run", family = "kq_gemv_cls", grid = "wgs", params = "wgs : int64")] +class KqGemvIq4xs : KqGemvBase { + def iq4_dot(wv, xl, xh : uint4) : int { + var idot = sdot4(iq4_word(wv.x & KQC_LOW), xl.x) + sdot4(iq4_word(wv.y & KQC_LOW), xl.y) + idot += sdot4(iq4_word(wv.z & KQC_LOW), xl.z) + sdot4(iq4_word(wv.w & KQC_LOW), xl.w) + idot += sdot4(iq4_word((wv.x >> 4u) & KQC_LOW), xh.x) + sdot4(iq4_word((wv.y >> 4u) & KQC_LOW), xh.y) + return idot + sdot4(iq4_word((wv.z >> 4u) & KQC_LOW), xh.z) + sdot4(iq4_word((wv.w >> 4u) & KQC_LOW), xh.w) + } + + def override blk_contrib(wsb, blk, xsb : uint) : float { + let xb4 = xsb * 16u + blk * 2u + let idot = iq4_dot(wq4[wsb * 8u + blk], xq4[xb4], xq4[xb4 + 1u]) + let dd = unpackHalf2x16(wsu[wsb * 5u]).x + return axs[xsb] * dd * float(iq4_sc(wsu[wsb * 5u + 1u + blk / 4u], blk) * idot) + } + + [spirv_kernel(local_size_x = 64, name = "kq_gemv_iq4xs_cls_spv")] + def run { + gemv_shell() + } +} + // Q5_K: the k4 fold with the 5th bit deposited from the block's qh word [vk_dispatch(name = "kq_gemv_k5_cls", kernel = "run", family = "kq_gemv_cls", grid = "wgs", params = "wgs : int64")] class KqGemvK5 : KqGemvBase { diff --git a/modules/dasLLAMA/dasllama/dasllama_vulkan_common.das b/modules/dasLLAMA/dasllama/dasllama_vulkan_common.das index 95018ef0ef..59c9b847b0 100644 --- a/modules/dasLLAMA/dasllama/dasllama_vulkan_common.das +++ b/modules/dasLLAMA/dasllama/dasllama_vulkan_common.das @@ -1709,7 +1709,7 @@ struct ArenaSlab { } struct ArenaFmt { - fmt : int // int(KqFmt): 0 = q8, 1/2/3 = k4/k5/k6, 4 = q40 + fmt : int // int(KqFmt): 0 = q8, 1/2/3 = k4/k5/k6, 4 = q40, 6 = iq4xs slabs : array cur : int // slab the bump cursor is in slab_cap : int64 // max blocks one slab may carry (msr over both plane strides) @@ -1740,17 +1740,18 @@ def arena_local_blk(blk : int64) : int64 => blk & ((1l << 32l) - 1l) // per-block plane strides — the block is 32 weights for q8 and a 256-weight superblock for kq. // Mirrors stack_plane_bytes: it computes the same totals as count * these strides. -// the tier's stack fmt space is int(KqFmt) — bridge to the gemm_schema kq id space (4/5/6/40) +// the tier's stack fmt space is int(KqFmt) — bridge to the gemm_schema kq id space (4/5/6/40/44) def vk_kq_schema_id(fmt : int) : int { if (fmt == 1) return 4 if (fmt == 2) return 5 if (fmt == 3) return 6 if (fmt == 4) return 40 + if (fmt == 6) return 44 panic("vk_kq_schema_id: not a kq stack fmt") return 0 } -// fmt space here = int(KqFmt): 0 q8 (32-elem blocks), 1-4 the kq lattice + q40 +// fmt space here = int(KqFmt): 0 q8 (32-elem blocks), 1-4 and 6 the kq lattice (+ q40, iq4xs) def vk_fmt_b32(fmt : int) : bool => fmt == 0 // meta/activation block unit of a stack format — q8 counts 32-weight blocks, the kq diff --git a/modules/dasLLAMA/tests/_vkd_oracles.das b/modules/dasLLAMA/tests/_vkd_oracles.das index 7207ac95cb..7d1f87985e 100644 --- a/modules/dasLLAMA/tests/_vkd_oracles.das +++ b/modules/dasLLAMA/tests/_vkd_oracles.das @@ -169,6 +169,8 @@ def kq_cls_ref(fmt : int; wq, ws, xq : array; xs : array) : KqGemvB k = new KqGemvQ40() } elif (fmt == int(KqFmt.k5)) { k = new KqGemvK5() + } elif (fmt == int(KqFmt.iq4xs)) { + k = new KqGemvIq4xs() } else { k = new KqGemvK6() } @@ -205,6 +207,42 @@ def kq_gemv_oracle(fmt : int; wq, ws, xq : array; xs : array; return <- y } +// IQ4_XS GEMV float oracle, INDEPENDENT of the class bit-math: w = d*sc*LUT[q] straight off the +// plane bytes (q40 nibble tiling, the k4 row shape with signed sc), x off the Q8_K bytes — the +// gate that a wrong codebook pack in iq4_word cannot pass while staying self-consistent +def iq4xs_gemv_float_oracle(wq, ws, xq : array; xs : array; + regions : array; n, d, nreg : int) : array { + let lut = iq4nl_lut() + var y : array + y |> resize(nreg * d) + let nsb = n / 256 + for (r in range(nreg)) { + for (row in range(d)) { + let w0 = int(regions[r * 2]) + row * nsb + let x0 = int(regions[r * 2 + 1]) + var acc = 0.0 + for (s in range(nsb)) { + let sb = w0 + s + let xsb = x0 + s + let dd = unpackHalf2x16(ws[sb * 5]).x + for (blk in range(8)) { + let sc = byte_s8(ws, sb * 20 + 4 + blk) + var bacc = 0.0 + for (m in range(16)) { + let b = byte_u8(wq, sb * 128 + blk * 16 + m) + let xlo = byte_s8(xq, xsb * 256 + blk * 32 + m) + let xhi = byte_s8(xq, xsb * 256 + blk * 32 + 16 + m) + bacc += float(lut[b & 15]) * float(xlo) + float(lut[b >> 4]) * float(xhi) + } + acc += xs[xsb] * dd * float(sc) * bacc + } + } + y[r * d + row] = acc + } + } + return <- y +} + // kq batch GEMM oracle over region records — same blk_contrib per (row, col, block) def kq_gemm_oracle(fmt : int; wq, ws, xq : array; xs : array; recs : array; nrec, n, d : int; var y : array) { diff --git a/modules/dasLLAMA/tests/test_vulkan_kernels.das b/modules/dasLLAMA/tests/test_vulkan_kernels.das index 17258b0bff..fba6cb47c8 100644 --- a/modules/dasLLAMA/tests/test_vulkan_kernels.das +++ b/modules/dasLLAMA/tests/test_vulkan_kernels.das @@ -379,9 +379,9 @@ def private attn_row_oracle(q : array; qbase : int; kk : array; kb [test] def test_vkd_kq_gemv_family(t0 : T?) { - t0 |> run("kq GEMV class family == the CPU oracle (all 4 formats)") <| @(t : T?) { + t0 |> run("kq GEMV class family == the CPU oracle (all 5 formats)") <| @(t : T?) { static_if (typeinfo builtin_module_exists(vulkan)) { - if (!(ensure_kq_gemv_k4_cls() && ensure_kq_gemv_q40_cls() && ensure_kq_gemv_k5_cls() && ensure_kq_gemv_k6_cls())) { + if (!(ensure_kq_gemv_k4_cls() && ensure_kq_gemv_q40_cls() && ensure_kq_gemv_k5_cls() && ensure_kq_gemv_k6_cls() && ensure_kq_gemv_iq4xs_cls())) { feint("no Vulkan device - skipping\n") return } @@ -389,9 +389,9 @@ def test_vkd_kq_gemv_family(t0 : T?) { let d = 3 // rows per region let nreg = 2 let rows = nreg * d - let nfmt = 4 - let fmts = fixed_array(int(KqFmt.k4), int(KqFmt.q40), int(KqFmt.k5), int(KqFmt.k6)) - let stride4s = fixed_array(8, 8, 10, 12) // uint4s per superblock + let nfmt = 5 + let fmts = fixed_array(int(KqFmt.k4), int(KqFmt.q40), int(KqFmt.k5), int(KqFmt.k6), int(KqFmt.iq4xs)) + let stride4s = fixed_array(8, 8, 10, 12, 8) // uint4s per superblock for (fi in range(nfmt)) { let fmt = fmts[fi] let stride4 = stride4s[fi] @@ -461,6 +461,8 @@ def test_vkd_kq_gemv_family(t0 : T?) { enc_kq_gemv_q40_cls(raw2, h2, s_cls, pc, wgs) } elif (fmt == int(KqFmt.k5)) { enc_kq_gemv_k5_cls(raw2, h2, s_cls, pc, wgs) + } elif (fmt == int(KqFmt.iq4xs)) { + enc_kq_gemv_iq4xs_cls(raw2, h2, s_cls, pc, wgs) } else { enc_kq_gemv_k6_cls(raw2, h2, s_cls, pc, wgs) } @@ -473,6 +475,12 @@ def test_vkd_kq_gemv_family(t0 : T?) { var y_ref <- kq_gemv_oracle(fmt, wqh, wsh, xqh, xsh, regs, 512, d, nreg) let bad = mismatch_approx(y_cls, y_ref) t |> success(bad == 0, "fmt {fmt}: class kernel matches the CPU oracle ({bad} of {rows} rows off)") + if (fmt == int(KqFmt.iq4xs)) { // the codebook pack has an independent float witness + var y_flt <- iq4xs_gemv_float_oracle(wqh, wsh, xqh, xsh, regs, 512, d, nreg) + let badf = mismatch_approx(y_ref, y_flt) + t |> success(badf == 0, "iq4xs: the class bit-math matches the float dequant oracle ({badf} of {rows} rows off)") + delete y_flt + } delete regs delete y_ref delete wqh @@ -991,9 +999,9 @@ def test_vkd_q8_batch_tile(t0 : T?) { [test] def test_vkd_kq_batch_family(t0 : T?) { - t0 |> run("kq batch tile class family == the CPU oracle (all 4 formats)") <| @(t : T?) { + t0 |> run("kq batch tile class family == the CPU oracle (all 5 formats)") <| @(t : T?) { static_if (typeinfo builtin_module_exists(vulkan)) { - if (!(ensure_kq_batch_k4_cls() && ensure_kq_batch_q40_cls() && ensure_kq_batch_k5_cls() && ensure_kq_batch_k6_cls())) { + if (!(ensure_kq_batch_k4_cls() && ensure_kq_batch_q40_cls() && ensure_kq_batch_k5_cls() && ensure_kq_batch_k6_cls() && ensure_kq_batch_iq4xs_cls())) { feint("no Vulkan device - skipping\n") return } @@ -1008,9 +1016,9 @@ def test_vkd_kq_batch_family(t0 : T?) { let wgs1 = ((cnt1 + 31) / 32) * wtiles let wgs = wgs0 + wgs1 let totsb = 2 * d * nsb // superblocks across both regions - let nfmt = 4 - let wstrides = fixed_array(32, 32, 40, 48) // uint words per superblock: k4 q40 k5 k6 - let pipes = fixed_array(int(KqFmt.k4), int(KqFmt.q40), int(KqFmt.k5), int(KqFmt.k6)) + let nfmt = 5 + let wstrides = fixed_array(32, 32, 40, 48, 32) // uint words per superblock: k4 q40 k5 k6 iq4xs + let pipes = fixed_array(int(KqFmt.k4), int(KqFmt.q40), int(KqFmt.k5), int(KqFmt.k6), int(KqFmt.iq4xs)) for (fi in range(nfmt)) { let fmt = pipes[fi] let wstride = wstrides[fi] @@ -1083,6 +1091,8 @@ def test_vkd_kq_batch_family(t0 : T?) { enc_kq_batch_q40_cls(raw2, h2, sc, pc, int64(wgs)) } elif (fmt == int(KqFmt.k5)) { enc_kq_batch_k5_cls(raw2, h2, sc, pc, int64(wgs)) + } elif (fmt == int(KqFmt.iq4xs)) { + enc_kq_batch_iq4xs_cls(raw2, h2, sc, pc, int64(wgs)) } else { enc_kq_batch_k6_cls(raw2, h2, sc, pc, int64(wgs)) } From f80fe16c3a8e3d252d28f614e88b2aa0518ca3f5 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Sun, 30 Aug 2026 08:26:44 -0700 Subject: [PATCH 05/61] dasLLAMA HOW_TO: section 7 (Metal) rewritten from the census - the tier fails closed on IQ4_XS, no q40 precedent, the compact scale-block layout, the ladders a Mac session walks Co-Authored-By: Claude Fable 5 --- modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md | 31 ++++++++++++++++++++----- 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md index b3ad8d4810..33d8a0e031 100644 --- a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md +++ b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md @@ -185,12 +185,31 @@ admits q8/k4/k6 only (`pf_f16_feed`), q40 and k5 have no cm2 tile either, and `followup_vulkan.md` item 24 rules that new formats land on the one class template, not as three more hand-stamped bodies. IQ4_XS prefill rides the kq batch tile like q40 does. -## 7. Metal - -Untested at the time of writing (no Mac in the session). `MetalKqGemvK4` / `MetalKqMulMmK45T` -/ `MetalKqMvK4T` are the twins to subclass; `dasllama_metal_shapes.das` carries the servable -predicates, `dasllama_metal_common.das` the plane buffers, `dasllama_metal_prefill.das` the -PSO ladders. +## 7. Metal - a Mac session's step, not this one's + +Not done for IQ4_XS (no Mac in the session; nothing here compiles MSL or runs it). What IS +true today, and what the format needs on that tier: + +- **The tier fails closed.** `kq_fmt_gpu_supported` (`dasllama_metal_shapes.das`) admits + q8/k4/k5/k6 only and every resident-driver plan runs through it, so an IQ4_XS model DECLINES + `kquant_native` on Metal and serves from the CPU - never a wrong-layout kernel branch (the + dispatchers `enc_kq_gemv` / `enc_kq_mvb` treat "not k4/k6" as k5, which is why the gate + exists). Keep it closed until the kernels land; do not add the enum member to the gate first. +- **No q40 precedent.** Metal serves no per-32-scale format; IQ4_XS would be the first + non-K-quant there, so "the q40 class plus a LUT" - the CPU, JIT and Vulkan recipe - has no + twin to copy. The nearest shapes are k4's: `MetalKqGemvK4`, the `MetalKqMvK4T` small-batch + twins, `MetalKqMulMmK45T` (prefill mul_mm), each a class the k5/k6 versions subclass. +- **The scale plane is a DIFFERENT layout.** Metal binds `kq_scales_of` - "16B compact blocks + at soff" pre-baked by the blob transform (`t.metal_blob`), not the 20 B decoded device row + Vulkan reads verbatim. An IQ4_XS arm means a transform arm (the row's f16 d + 8 signed sc into + that 16 B form) plus `kq_quants_of` (128 B nibbles, the k4 arm's stride) - both ladders in + `dasllama_metal_common.das`. +- **Ladders:** `kq_quants_of` / `kq_scales_of`, `enc_kq_gemv`, `enc_kq_mvb` (B2/B4/B8 twins), + the mul_mm PSO ladders in `dasllama_metal_prefill.das` (`g_pf_pso_moe_mm__th` and the + dense twins, the `kfmt` picks near 4350/4425), `moe_site_ok` + the `sb1/sb2/sb3` superblock + predicates in `dasllama_metal_shapes.das`, and finally the gate. +- **Gates:** `tests/test_metal_gemv_kernels.das` (kq GEMV cells), the prefill/decode parity + suites, the support-matrix test, then the 1B IQ4_XS end to end on the M-box. ## 8. End to end From 3f771dbd8c2d87800a0d3726076cd9fb4208bfa9 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Sun, 30 Aug 2026 09:26:43 -0700 Subject: [PATCH 06/61] dasLLAMA: IQ4_XS on the Metal tier - the k6 split scale form, iq4_lut, GEMV + B2/B4/B8 mv twins + the mul_mm IQ4XS arm, the ladders and the gate; tests at fmt 44; HOW_TO section 7 + QUIRKS 12-15 The Metal blob rebakes the iq4xs scale plane into the k6 split form ([nsb x 16B strips = the row's bytes 4..19][nsb x f16 d]), so kq_scales_of, metal_blob_off_ok and every kernel bind are the k6 arms over t.iq4xss; quants bind verbatim. iq4_lut packs the codebook into four words (no table memory, the k6 sign trick widens the byte). Kernels: MetalKqGemvIq4xs, the MetalKqMvIq4xsT B2/B4 pair, MetalKqMvB8Iq4xs, and an IQ4XS arm in MetalKqMulMmK45T (MetalKqMulMmIq4xs). Ladders: enc_kq_gemv / enc_kq_mvb / enc_kq_gemm_mm_b, pf_enc_kq_site_mm (the base mul_mm only), pf_devw_panel_kq (declines), pf_enc_kq_gemv, moe_site_ok + the sb predicates, kq_fmt_gpu_supported. dequant_iq4xs_plane_superblock_at is the split-layout dequant twin the tests and the CPU row both use. Gates on the M1 Max: test_metal_gemv_kernels 2/2 and test_metal_gemm_kernels 2/2 with the iq4xs GEMV / mvb2,4,8 / mul_mm cells; a one-byte codebook mutation turns them red; lint clean. The first run on the box minted run.tune.json (iq4xsq8_tile_gen: mr8, verdict=beats - the emitter's NEON tbl1 path); run.das on the CPU/NEON tier reproduces the text at gen 131 t/s, and the Metal tier (set_metal_mode(required) before load_model_ + convert_model_to_metal_blob) decodes the same text at gen 153 t/s - metal decode/prefill resident paths live on the M1 Max. Not done, ledgered for the M5 kernel pass: the tensor / tall / dev-W mul_mm twins and the MoE GEMV / mul_mm trio for the format. Co-Authored-By: Claude Fable 5 --- modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md | 88 +++-- .../dasLLAMA/dasllama/dasllama_convert.das | 11 +- modules/dasLLAMA/dasllama/dasllama_layout.das | 23 +- .../dasllama/dasllama_metal_common.das | 16 +- .../dasllama/dasllama_metal_kernels.das | 329 ++++++++++++++++-- .../dasllama/dasllama_metal_prefill.das | 11 +- .../dasllama/dasllama_metal_shapes.das | 10 +- modules/dasLLAMA/followup_general.md | 17 + .../dasLLAMA/tests/_metal_kernel_common.das | 12 +- .../tests/test_metal_gemm_kernels.das | 24 +- .../tests/test_metal_gemv_kernels.das | 38 +- 11 files changed, 480 insertions(+), 99 deletions(-) diff --git a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md index 33d8a0e031..2c07d10406 100644 --- a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md +++ b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md @@ -185,31 +185,51 @@ admits q8/k4/k6 only (`pf_f16_feed`), q40 and k5 have no cm2 tile either, and `followup_vulkan.md` item 24 rules that new formats land on the one class template, not as three more hand-stamped bodies. IQ4_XS prefill rides the kq batch tile like q40 does. -## 7. Metal - a Mac session's step, not this one's - -Not done for IQ4_XS (no Mac in the session; nothing here compiles MSL or runs it). What IS -true today, and what the format needs on that tier: - -- **The tier fails closed.** `kq_fmt_gpu_supported` (`dasllama_metal_shapes.das`) admits - q8/k4/k5/k6 only and every resident-driver plan runs through it, so an IQ4_XS model DECLINES - `kquant_native` on Metal and serves from the CPU - never a wrong-layout kernel branch (the - dispatchers `enc_kq_gemv` / `enc_kq_mvb` treat "not k4/k6" as k5, which is why the gate - exists). Keep it closed until the kernels land; do not add the enum member to the gate first. -- **No q40 precedent.** Metal serves no per-32-scale format; IQ4_XS would be the first - non-K-quant there, so "the q40 class plus a LUT" - the CPU, JIT and Vulkan recipe - has no - twin to copy. The nearest shapes are k4's: `MetalKqGemvK4`, the `MetalKqMvK4T` small-batch - twins, `MetalKqMulMmK45T` (prefill mul_mm), each a class the k5/k6 versions subclass. -- **The scale plane is a DIFFERENT layout.** Metal binds `kq_scales_of` - "16B compact blocks - at soff" pre-baked by the blob transform (`t.metal_blob`), not the 20 B decoded device row - Vulkan reads verbatim. An IQ4_XS arm means a transform arm (the row's f16 d + 8 signed sc into - that 16 B form) plus `kq_quants_of` (128 B nibbles, the k4 arm's stride) - both ladders in - `dasllama_metal_common.das`. -- **Ladders:** `kq_quants_of` / `kq_scales_of`, `enc_kq_gemv`, `enc_kq_mvb` (B2/B4/B8 twins), - the mul_mm PSO ladders in `dasllama_metal_prefill.das` (`g_pf_pso_moe_mm__th` and the - dense twins, the `kfmt` picks near 4350/4425), `moe_site_ok` + the `sb1/sb2/sb3` superblock - predicates in `dasllama_metal_shapes.das`, and finally the gate. -- **Gates:** `tests/test_metal_gemv_kernels.das` (kq GEMV cells), the prefill/decode parity - suites, the support-matrix test, then the 1B IQ4_XS end to end on the M-box. +## 7. Metal - `dasllama_metal_kernels.das`, `_common`, `_prefill`, `_shapes`, `dasllama_layout.das` + +Done for IQ4_XS over ssh on the M1 (the tier compiles MSL and runs only there). The tier is +NOT a plane-verbatim consumer like Vulkan: a Metal-served model is the `metal_blob` flavor, +whose kq scale planes are rebaked into device forms (k4/k5: the 16 B disk block, k6: the split +form) and whose CPU never reads a plane again. So a format on Metal starts with a device-form +decision, and every ladder below keys on `KqFmt` with an `else` that means "k5" or "k6" - +without its own arm a new format runs a wrong-layout kernel silently, which is why the tier's +gate (`kq_fmt_gpu_supported`) is closed by default. IQ4_XS took: + +1. **The device scale form: the k6 split form.** `metal_blob_scale_plane("iq4xss")` + (`dasllama_layout.das`) turns the 20 B CPU row into `[nsb x 16 B strips][nsb x f16 d]` - + the strip is the row's bytes 4..19 (`[8 signed sc][8 pad]`), d is bytes 0..1; 18 B/sb, the + k6 numbers exactly. That choice makes `kq_scales_of` the k6 arm over `t.iq4xss` + (`soff = sb0*16`, `doff = plane_sb*16 + sb0*2`), `metal_blob_off_ok` the k6 rule + (`off % 512`), and every kernel bind the k6 pair (d plane at `s0off = doff`, strips at + `soff`). `metal_blob_commit` lists the plane by name. Quants: `kq_quants_of` binds + `t.iq4xsq` verbatim (128 B/sb). +2. **The codebook in MSL, `iq4_lut`:** four packed words selected by nibble range, byte + extracted, widened with the k6 sign trick - no table memory, no fixed-array literal (the MSL + emitter takes fixed arrays as `var` locals with per-index stores, not as initializers). +3. **Kernels** (`dasllama_metal_kernels.das`), each a copy of the k4/k6 shape with the per-32 + pairing (lane = one 32-block of every 4th superblock: 4 uints, 16 lo + 16 hi weights) and + `d * sc * LUT[q]` with no min term: `MetalKqGemvIq4xs` (decode GEMV), `MetalKqMvIq4xsT` + B2/B4 + `MetalKqMvB8Iq4xs` (small batch), and an `IQ4XS` arm in `MetalKqMulMmK45T` + (prefill mul_mm; `MetalKqMulMmIq4xs` stamps it). PSO globals + `compile_pso` + + `release_pso` lines in the three ladders. +4. **Ladders:** `enc_kq_gemv`, `enc_kq_mvb`, `enc_kq_gemm_mm_b` (kernels), `pf_enc_kq_site_mm` + (the base mul_mm only - no tensor / tall / dev-W twins: those are the M5 kernel pass), + `pf_devw_panel_kq` (returns false - its `dq` pick would otherwise be k5's), `pf_enc_kq_gemv` + (prefill), `moe_site_ok` + the `sb1/2/3` predicates (shapes), and last the gate. +5. **Tests:** `dequant_iq4xs_plane_superblock_at` (`dasllama_convert.das`, the split-layout + twin the CPU row now calls), fixtures at fmt 44 in `tests/_metal_kernel_common.das`, the + ladders + calls in `test_metal_gemv_kernels.das` (GEMV, B2/B4/B8) and + `test_metal_gemm_kernels.das` (mul_mm, base form). Gate proof: a one-byte codebook mutation + turns the iq4xs cells red. + +Result: `test_metal_gemv_kernels` 2/2 and `test_metal_gemm_kernels` 2/2 on the M1 Max; the +tuner's first mint on that box crowned `iq4xsq8_tile_gen: mr8` (the NEON `tbl1` path of the +JIT emitter, `verdict=beats`), and `run.das` on the CPU/NEON tier reproduces the text at gen 131 t/s; +on the Metal tier (`set_metal_mode(MetalMode.required)` BEFORE `load_model_`, then +`convert_model_to_metal_blob` - `run.das` itself declares CPU intent, and `DASLLAMA_GPU=1` on a +Mac build with dasVulkan arms Vulkan-on-MoltenVK, 96 t/s, not Metal) the 1B IQ4_XS decodes the +same text at gen 153 t/s with `metal decode/prefill: resident path live on Apple M1 Max`. Not done: the tensor / tall / dev-W mul_mm twins and +the MoE GEMV/mul_mm trio for the format - ledgered for the M5 kernel pass. ## 8. End to end @@ -305,6 +325,24 @@ where, why it is so today, what unquirked looks like. An empty ledger is a legit on the app (a whole-scope re-tune) or delete the sidecar. Unquirked: fold the family's generator hash (the JIT DLL cache key already carries it) into the sidecar identity, so a generator change reads as stale. +12. **A fresh worktree has no JIT until `lib/LLVM.dll` is staged - on every platform.** The + M1 worktree's first `-jit` run died on `can't load library LLVM.dll`; `utils/mcp/setup.das` + stages the codegen backend on Windows only. Copy `
/lib/LLVM.dll` (+ `.version`) into + the worktree's `lib/` by hand on macOS/Linux. Unquirked: the posix arm of `stage_jit_backend`. +13. **The Metal test ladders are nested ternaries with an `else` = k6.** `kq_gemv_gate`, + `kq_mvb_gate`, `kq_mulmm_gate` and the fixtures pick MSL sources / entries / fastmath / + tgmem names per format in four parallel ternary chains each; a format missing from any one + of them silently tests k6's kernel under the new format's tag. Same shape as the tier's + own dispatch ladders (QUIRK 1's cousin on Metal). Unquirked: one per-format record + (`src, entry, fastmath, tgmem`) per kernel family, indexed by format. +14. **The mul_mm template has no `static_if ... elif`.** Adding a format arm to + `MetalKqMulMmK45T` beside `SIXBIT` meant re-nesting the k4/k5 arm one level deeper + (`static_if (IQ4XS) {...} else { }`); the diff is mostly indentation. + Unquirked: an `elif` on `static_if` in the MSL emitter. +15. **First run on a fresh box tunes before it serves.** `run.das` on the M1 spent its first + minutes minting `run.tune.json` (every family, the confirm pass included) before loading + the model - the auto policy, working as designed, but a Mac session's first end-to-end + "hang" is that mint. Watch `@tune begin/end` lines in the log. ## Per-format notes diff --git a/modules/dasLLAMA/dasllama/dasllama_convert.das b/modules/dasLLAMA/dasllama/dasllama_convert.das index 38c64d4a46..f2e3c0368b 100644 --- a/modules/dasLLAMA/dasllama/dasllama_convert.das +++ b/modules/dasLLAMA/dasllama/dasllama_convert.das @@ -644,10 +644,17 @@ def iq4xs_sc(ks : array | #; kso, blk : int64) : int => rd_i8(ks, kso + 4 //! (ggml's own float order: dl = d * (ls - 32), then dl * kvalue), q = nibble (low = k, high = //! k+16 — the disk pairing, kept verbatim by the transcode). def dequant_iq4xs_plane_superblock(kq : array | #; kqo : int64; ks : array | #; kso : int64; var dst : array | #; doff : int64) { - let d = f16_to_f32(rd_u16(ks, kso)) + dequant_iq4xs_plane_superblock_at(kq, kqo, ks, kso + 4l, kso, dst, doff) +} + +//! The split-layout twin: `dso` names the f16 d halfword's byte offset independently of the +//! 8-byte signed sub-scale strip at kso — metal-blob models keep iq4xss in the k6 split form +//! ([nsb x 16B strips][nsb x f16 d]); the CPU row is the same math at kso + 4 / kso. +def dequant_iq4xs_plane_superblock_at(kq : array | #; kqo : int64; ks : array | #; kso, dso : int64; var dst : array | #; doff : int64) { + let d = f16_to_f32(rd_u16(ks, dso)) let lut = iq4nl_lut() for (blk in range64(8l)) { - let dl = d * float(iq4xs_sc(ks, kso, blk)) + let dl = d * float(rd_i8(ks, kso + blk)) for (l in range64(16l)) { let b = int(kq[kqo + blk * 16l + l]) dst[doff + blk * 32l + l] = dl * float(lut[b & 15]) diff --git a/modules/dasLLAMA/dasllama/dasllama_layout.das b/modules/dasLLAMA/dasllama/dasllama_layout.das index fea50859a5..5741059610 100644 --- a/modules/dasLLAMA/dasllama/dasllama_layout.das +++ b/modules/dasLLAMA/dasllama/dasllama_layout.das @@ -29,7 +29,7 @@ require math def private metal_blob_off_ok(off : int64; fmt : KqFmt) : bool { if (off < 0l) return true if (fmt == KqFmt.q8) return off % 256l == 0l - if (fmt == KqFmt.k6) return off % 512l == 0l + if (fmt == KqFmt.k6 || fmt == KqFmt.iq4xs) return off % 512l == 0l // the split scale form: the (off/256)*2 d-plane bind stays 4B-aligned if (fmt == KqFmt.q51) return off % 128l == 0l // (off/32)*20 and (off/32)*4 both 16B-aligned return true } @@ -169,6 +169,25 @@ def metal_blob_scale_plane(var t : Model; name : string) : bool { } return true } + if (name == "iq4xss") { + // iq4xs: the k6 split form — strips = bytes 4..19 of the 20B row ([8 signed sc][8 pad]), d = bytes 0..1 + if (!empty(t.iq4xss)) { + var @exact_size xc : array + let nsb = long_length(t.iq4xss) / IQ4XS_SSB + xc |> reserve_resize(nsb * 18l) + unsafe { + var dp = addr(xc[0]) + let sp = addr < uint8 const? >(t.iq4xss[0]) + for (sb in range64(nsb)) { + memcpy(reinterpret(dp + sb * 16l), reinterpret(sp + sb * IQ4XS_SSB + 4l), 16) + dp[nsb * 16l + sb * 2l] = sp[sb * IQ4XS_SSB] + dp[nsb * 16l + sb * 2l + 1l] = sp[sb * IQ4XS_SSB + 1l] + } + } + t.iq4xss <- xc + } + return true + } if (name != "k6s") { return false } @@ -198,7 +217,7 @@ def metal_blob_commit(var t : Model) : bool { if (!metal_blob_eligible(t)) { return false } - for (nm in ["k4s", "k5s", "k6s"]) { + for (nm in ["k4s", "k5s", "k6s", "iq4xss"]) { metal_blob_scale_plane(t, nm) } t.metal_blob = true diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_common.das b/modules/dasLLAMA/dasllama/dasllama_metal_common.das index 9e564dabb0..feb45e2750 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_common.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_common.das @@ -244,6 +244,7 @@ var g_pso_kq_k4 : MetalComputePipeline? var g_pso_kq_k5 : MetalComputePipeline? var g_pso_kq_k5c : MetalComputePipeline? // the select-form nr0=1 twin — n < 3072 / small-d sites var g_pso_kq_k6 : MetalComputePipeline? +var g_pso_kq_iq4xs : MetalComputePipeline? var g_pso_kq_mvb2_k4 : MetalComputePipeline? // batch B=2..8: the kq ext small-batch mv twins var g_pso_kq_mvb4_k4 : MetalComputePipeline? var g_pso_kq_mvb2_k5 : MetalComputePipeline? @@ -253,9 +254,13 @@ var g_pso_kq_mvb4_k6 : MetalComputePipeline? var g_pso_kq_mvb8_k4 : MetalComputePipeline? // B=5..8 single-pass twins (in-graph A/B vs 2 x B4 groups) var g_pso_kq_mvb8_k5 : MetalComputePipeline? var g_pso_kq_mvb8_k6 : MetalComputePipeline? +var g_pso_kq_mvb2_iq4xs : MetalComputePipeline? +var g_pso_kq_mvb4_iq4xs : MetalComputePipeline? +var g_pso_kq_mvb8_iq4xs : MetalComputePipeline? var g_pso_kq_mm_b4 : MetalComputePipeline? // batch B>=9: the prefill-owned kq mul_mm twins var g_pso_kq_mm_b5 : MetalComputePipeline? -var g_pso_kq_mm_b6 : MetalComputePipeline? // fused W1|W3+swiglu (s16 path) +var g_pso_kq_mm_b6 : MetalComputePipeline? +var g_pso_kq_mm_b44 : MetalComputePipeline? // fused W1|W3+swiglu (s16 path) // the batched-step set (P4): fixed-B GEMVs + the M-pad-32 GEMM twin (f32 X) + row-table kernels var g_pso_gemv_b2 : MetalComputePipeline? var g_pso_gemv_b4 : MetalComputePipeline? @@ -834,6 +839,10 @@ def kq_quants_of(dev : MetalDevice?; t : Model; fmt : KqFmt; off : int64) : tupl return (buf = plane_buffer(dev, addr < void? >(t.k5q[0]), uint64(long_length(t.k5q)), t.image_map != null), qoff = uint64(sb0 * 160l)) } + if (fmt == KqFmt.iq4xs) { + return (buf = plane_buffer(dev, addr < void? >(t.iq4xsq[0]), uint64(long_length(t.iq4xsq)), t.image_map != null), + qoff = uint64(sb0 * 128l)) + } return (buf = plane_buffer(dev, addr < void? >(t.k6q[0]), uint64(long_length(t.k6q)), t.image_map != null), qoff = uint64(sb0 * 192l)) } @@ -855,6 +864,11 @@ def kq_scales_of(dev : MetalDevice?; t : Model; fmt : KqFmt; off : int64) : tupl return (buf = plane_buffer(dev, addr < void? >(t.k4s[0]), uint64(long_length(t.k4s)), t.image_map != null), soff = uint64(sb0 * 16l), doff = 0ul) } + if (fmt == KqFmt.iq4xs) { // the k6 split form over the 18B/sb device plane + let plane_sb = long_length(t.iq4xss) / 18l + return (buf = plane_buffer(dev, addr < void? >(t.iq4xss[0]), uint64(long_length(t.iq4xss)), t.image_map != null), + soff = uint64(sb0 * 16l), doff = uint64(plane_sb * 16l + sb0 * 2l)) + } return (buf = plane_buffer(dev, addr < void? >(t.k5s[0]), uint64(long_length(t.k5s)), t.image_map != null), soff = uint64(sb0 * 16l), doff = 0ul) } diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das index be4c0c32d6..208445edbd 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das @@ -3636,6 +3636,11 @@ def metal_decode_init : bool { // nolint:STYLE038 — a flat compile_pso per k g_pso_kq_mm_b4 = compile_pso(MetalKqMulMmK4_metal_kq_mulmm_msl, MetalKqMulMmK4_metal_kq_mulmm_msl_entry, MetalKqMulMmK4_metal_kq_mulmm_msl_fastmath, ok) g_pso_kq_mm_b5 = compile_pso(MetalKqMulMmK5_metal_kq_mulmm_msl, MetalKqMulMmK5_metal_kq_mulmm_msl_entry, MetalKqMulMmK5_metal_kq_mulmm_msl_fastmath, ok) g_pso_kq_mm_b6 = compile_pso(MetalKqMulMmK6_metal_kq_mulmm_msl, MetalKqMulMmK6_metal_kq_mulmm_msl_entry, MetalKqMulMmK6_metal_kq_mulmm_msl_fastmath, ok) + g_pso_kq_iq4xs = compile_pso(metal_kq_gemv_iq4xs_msl, metal_kq_gemv_iq4xs_msl_entry, metal_kq_gemv_iq4xs_msl_fastmath, ok) + g_pso_kq_mvb2_iq4xs = compile_pso(MetalKqMvB2Iq4xs_metal_kq_mv_iq4xs_msl, MetalKqMvB2Iq4xs_metal_kq_mv_iq4xs_msl_entry, MetalKqMvB2Iq4xs_metal_kq_mv_iq4xs_msl_fastmath, ok) + g_pso_kq_mvb4_iq4xs = compile_pso(MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl, MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl_entry, MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl_fastmath, ok) + g_pso_kq_mvb8_iq4xs = compile_pso(metal_kq_mvb8_iq4xs_msl, metal_kq_mvb8_iq4xs_msl_entry, metal_kq_mvb8_iq4xs_msl_fastmath, ok) + g_pso_kq_mm_b44 = compile_pso(MetalKqMulMmIq4xs_metal_kq_mulmm_msl, MetalKqMulMmIq4xs_metal_kq_mulmm_msl_entry, MetalKqMulMmIq4xs_metal_kq_mulmm_msl_fastmath, ok) // the batched-step set (P4) g_pso_gemv_b2 = compile_pso(MetalGemvB2_metal_gemv_msl, MetalGemvB2_metal_gemv_msl_entry, MetalGemvB2_metal_gemv_msl_fastmath, ok) g_pso_gemv_b4 = compile_pso(MetalGemvB4_metal_gemv_msl, MetalGemvB4_metal_gemv_msl_entry, MetalGemvB4_metal_gemv_msl_fastmath, ok) @@ -3702,7 +3707,9 @@ def enc_gemv(enc : MetalComputeEncoder?; bw : MetalBuffer?; boff : uint64; bx, b def enc_kq_gemv(enc : MetalComputeEncoder?; fmt : KqFmt; bq : MetalBuffer?; qoff : uint64; bs : MetalBuffer?; soff, doff : uint64; bx, by, bn, bd : MetalBuffer?; rows, n : int64; yoff : uint64 = 0ul; xoff : uint64 = 0ul) { - if (fmt == KqFmt.k6) { + if (fmt == KqFmt.iq4xs) { + enc_kq_iq4xs_c(enc, bs, doff, bs, soff, bq, qoff, bx, xoff, by, yoff, bn, bd, rows, n) + } elif (fmt == KqFmt.k6) { enc_kq_k6_c(enc, bs, doff, bs, soff, bq, qoff, bx, xoff, by, yoff, bn, bd, rows, n) } elif (fmt == KqFmt.k5 && (n < 3072l || rows <= 1024l)) { // the select-form nr0=1 twin: wins every d at n < 3072 (gemv lab, 4B round) and the @@ -3738,7 +3745,15 @@ def enc_kq_mvb(enc : MetalComputeEncoder?; fmt : KqFmt; bq : MetalBuffer?; qoff let two = nlive == 2l let b8 = nlive >= 5l && kq_b8_on() let gcols = (nlive + 3l) / 4l - if (fmt == KqFmt.k6) { + if (fmt == KqFmt.iq4xs) { + if (b8) { + enc_kq_mvb8_iq4xs_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows) + } elif (two) { + enc_kq_mvb2_iq4xs_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows) + } else { + enc_kq_mvb4_iq4xs_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows, gcols) + } + } elif (fmt == KqFmt.k6) { if (b8) { enc_kq_mvb8_k6_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows) } elif (two) { @@ -3770,7 +3785,9 @@ def enc_kq_mvb(enc : MetalComputeEncoder?; fmt : KqFmt; bq : MetalBuffer?; qoff def enc_kq_gemm_mm_b(enc : MetalComputeEncoder?; fmt : KqFmt; bq : MetalBuffer?; qoff : uint64; bs : MetalBuffer?; soff, doff : uint64; bx, by : MetalBuffer?; yoff : uint64; bk, bn : MetalBuffer?; mp, rows : int64) { - if (fmt == KqFmt.k6) { + if (fmt == KqFmt.iq4xs) { + enc_kq_mm_iq4xs_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, bk, bn, mp, rows) + } elif (fmt == KqFmt.k6) { enc_kq_mm_k6_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, bk, bn, mp, rows) } elif (fmt == KqFmt.k4) { enc_kq_mm_k4_c(enc, bs, soff, bs, soff, bq, qoff, bx, by, yoff, bk, bn, mp, rows) @@ -4196,6 +4213,11 @@ def metal_kernels_release { // nolint:STYLE037,STYLE038 — a flat release_pso release_pso(g_pso_kq_mm_b4) release_pso(g_pso_kq_mm_b5) release_pso(g_pso_kq_mm_b6) + release_pso(g_pso_kq_iq4xs) + release_pso(g_pso_kq_mvb2_iq4xs) + release_pso(g_pso_kq_mvb4_iq4xs) + release_pso(g_pso_kq_mvb8_iq4xs) + release_pso(g_pso_kq_mm_b44) if (g_pso_rms != null) { metal_release(g_pso_rms) g_pso_rms = null @@ -5421,6 +5443,7 @@ class template MetalKqMulMmK45T : MetalMmTileBase { @template_constant BLK : uint = 32u // superblock quant-plane stride in uints (unused in the SIXBIT arm — k6 strides are hardcoded 48/16/8) @template_constant QH : bool = false // q5_K: the 5th-bit overlay plane rides after the nibbles @template_constant SIXBIT : bool = false // q6_K: int8 sub-scales x super-d, ql/qh compose + @template_constant IQ4XS : bool = false // iq4xs: the k6 split scale form, the q40 pairing, codebook decode [metal_kernel] def metal_kq_mulmm { // nolint:STYLE038 — MSL kernel body; the mul_mm tiles are simdgroup-barrier coupled @@ -5489,39 +5512,55 @@ class template MetalKqMulMmK45T : MetalMmTileBase { va[k * 4 + 3] = float16(dsc3 * float(qv & 4278190080u) - dmn) } } else { - let sv = ks4[blk] - var sc = 0u - var mn = 0u - if (js < 4u) { - sc = (sv.y >> (8u * js)) & 63u - mn = (sv.z >> (8u * js)) & 63u - } else { - let j2 = js - 4u - let hi = sv.w >> (8u * j2) - sc = (hi & 15u) | (((sv.y >> (8u * j2 + 6u)) & 3u) << 4u) - mn = ((hi >> 4u) & 15u) | (((sv.z >> (8u * j2 + 6u)) & 3u) << 4u) - } - let dsc = float(ksh[blk * 8u]) * float(sc) - let dmn = float(ksh[blk * 8u + 1u]) * float(mn) - let qb = blk * BLK + (js / 2u) * 8u + il0 * 4u - static_if (QH) { - let hb = blk * BLK + 32u + il0 * 4u - let nsh = (js & 1u) * 4u + static_if (IQ4XS) { + // iq4xs: strip byte js is the SIGNED sub-scale, super-d off the d plane (the k6 split + // form); block js = 4 uints of the q40 pairing, il0 picks the nibble half; codebook per element + let sv = ks4[blk] + let scw = js < 4u ? sv.x : sv.y + let dsc = float(ksh[blk]) * float(((int(scw >> ((js & 3u) * 8u)) & 255) ^ 128) - 128) + let qb = blk * 32u + js * 4u + let nsh = il0 * 4u for [unroll_full] (k in range(4)) { let u = kqu[qb + uint(k)] - let hu = kqu[hb + uint(k)] for [unroll_full] (c in range(4)) { - let q = ((u >> (8u * uint(c) + nsh)) & 15u) | (((hu >> (8u * uint(c) + js)) & 1u) << 4u) - va[k * 4 + c] = float16(dsc * float(q) - dmn) + va[k * 4 + c] = float16(dsc * iq4_lut((u >> (8u * uint(c) + nsh)) & 15u)) } } } else { - let nsh = (js & 1u) * 4u - for [unroll_full] (k in range(4)) { - let u = kqu[qb + uint(k)] - for [unroll_full] (c in range(4)) { - let q = (u >> (8u * uint(c) + nsh)) & 15u - va[k * 4 + c] = float16(dsc * float(q) - dmn) + let sv = ks4[blk] + var sc = 0u + var mn = 0u + if (js < 4u) { + sc = (sv.y >> (8u * js)) & 63u + mn = (sv.z >> (8u * js)) & 63u + } else { + let j2 = js - 4u + let hi = sv.w >> (8u * j2) + sc = (hi & 15u) | (((sv.y >> (8u * j2 + 6u)) & 3u) << 4u) + mn = ((hi >> 4u) & 15u) | (((sv.z >> (8u * j2 + 6u)) & 3u) << 4u) + } + let dsc = float(ksh[blk * 8u]) * float(sc) + let dmn = float(ksh[blk * 8u + 1u]) * float(mn) + let qb = blk * BLK + (js / 2u) * 8u + il0 * 4u + static_if (QH) { + let hb = blk * BLK + 32u + il0 * 4u + let nsh = (js & 1u) * 4u + for [unroll_full] (k in range(4)) { + let u = kqu[qb + uint(k)] + let hu = kqu[hb + uint(k)] + for [unroll_full] (c in range(4)) { + let q = ((u >> (8u * uint(c) + nsh)) & 15u) | (((hu >> (8u * uint(c) + js)) & 1u) << 4u) + va[k * 4 + c] = float16(dsc * float(q) - dmn) + } + } + } else { + let nsh = (js & 1u) * 4u + for [unroll_full] (k in range(4)) { + let u = kqu[qb + uint(k)] + for [unroll_full] (c in range(4)) { + let q = (u >> (8u * uint(c) + nsh)) & 15u + va[k * 4 + c] = float16(dsc * float(q) - dmn) + } } } } @@ -5564,6 +5603,12 @@ class MetalKqMulMmK6 : MetalKqMulMmK45T { override SIXBIT = true } +// iq4xs rides the IQ4XS arm; the split scale plane binds twice like k6 (d plane at s0off = doff) +[metal_dispatch(name = "enc_kq_mm_iq4xs_c", pso = "g_pso_kq_mm_b44", tgmem = "MetalKqMulMmIq4xs_metal_kq_mulmm_msl_tgmem", tg = 128, grid = "mp/32, rows/64", params = "mp : int64, rows : int64")] +class MetalKqMulMmIq4xs : MetalKqMulMmK45T { + override IQ4XS = true +} + // Router GEMV over the uploaded fp32 router slab: y[ne] = W[ne x dim] . x per stream. ne <= 256 // rows. Each sg serves NR positions per weight load (slab SLC traffic /NR); BATCHED stamps // clamp tails via ns; singles fold the clamp away and the gate erases ns outright. @@ -5841,6 +5886,13 @@ def private kmask_scales(sv : uint4; iq : uint) : uint4 { ((hw4 >> 4u) & 0x0F0F) | ((hw2 & 0xC0C0) >> 2u)) } +// IQ4_XS codebook (ggml's kvalues_iq4nl) as four packed words, selected + byte-extracted — +// no table memory; the k6 sign trick widens the byte +def private iq4_lut(q : uint) : float { + let w = q < 8u ? (q < 4u ? 3215825025u : 4142587343u) : (q < 12u ? 639175937u : 1901675829u) + return float((int((w >> ((q & 3u) * 8u)) & 255u) ^ 128) - 128) +} + // Expert-indexed Q4_K GEMV — MetalKqGemvK4's dot; the superblock index shifts by sel[slot] * esb. // x rides a float4 view: the 32-scalar-x-load form was load-issue-bound at the routed expert // shapes (nfe-row planes) — the vector view is 2.25x there (lab: bench_metal_moe_lab, 142→321 wGB/s). @@ -6835,6 +6887,223 @@ class MetalKqGemvK6 { } } +// IQ4_XS: w = d*sc*LUT[q] with sc the SIGNED (ls - 32) strip byte and no min plane; each lane owns +// one 32-block of every 4th superblock (the q40 pairing: byte l -> elems l and l + 16), the scale +// plane is the k6 split form (d plane at s0off = doff, 16B sub-scale strips at soff). +[metal_dispatch(name = "enc_kq_iq4xs_c", pso = "g_pso_kq_iq4xs", tg = 64, grid = "rows/4", params = "rows : int64, n : int64")] +class MetalKqGemvIq4xs { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d plane — the scale buffer bound at byte nsb*16 (s0off = the caller's doff) + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // 16B strips: 8 signed sub-scales + 8 pad, byte view (same buffer at 0) + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array // iq4xs quant plane, uint view (block bu of blk at 32*blk + 4*bu) + @ssbo @binding = 3 @role = "read" @off = "xoff" @span = "n*4" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" @span = "rows*4" y : array + @uniform @binding = 5 ndim : uint + @uniform @binding = 6 ddim : uint + + [metal_kernel(name="metal_kq_gemv_iq4xs_msl")] + def metal_kq_gemv_iq4xs { + let lane = gl_SubgroupInvocationID + let ix = lane / 8u + let bu = lane % 8u + let nb = ndim / 256u + let first_row = (gl_WorkGroupID.x * 2u + gl_SubgroupID) * 2u + var yl : float[16] + var yh : float[16] + var sumf : float[2] + var ib = ix + while (ib < nb) { + let yb = ib * 256u + bu * 32u + for [unroll_full] (i in range(16)) { + yl[i] = x[yb + uint(i)] + yh[i] = x[yb + 16u + uint(i)] + } + for [unroll_full] (r in range(2)) { + let blk = (first_row + uint(r)) * nb + ib + let qb = blk * 32u + bu * 4u + var acc = 0.0 + for [unroll_full] (u in range(4)) { + let qv = kqu[qb + uint(u)] + for [unroll_full] (e in range(4)) { + let byv = (qv >> (8u * uint(e))) & 255u + acc += yl[4 * u + e] * iq4_lut(byv & 15u) + yh[4 * u + e] * iq4_lut(byv >> 4u) + } + } + let sc = float((int(kscb[blk * 16u + bu]) ^ 128) - 128) + sumf[r] += float(kdh[blk]) * sc * acc + } + ib += 4u + } + for [unroll_full] (r in range(2)) { + let s = simd_sum(sumf[r]) + if (lane == 0u && first_row + uint(r) < ddim) { + y[first_row + uint(r)] = s + } + } + } +} + +// The iq4xs B2/B4 pair: the K4T shell (row/colbase/TILED twins) over the per-32-block lane map of +// the GEMV above — lane tx owns block tx of each superblock, 16 lo + 16 hi weights as float4s. +[ |> template_struct_instance] +class template MetalKqMvIq4xsT { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d plane — the scale buffer bound at byte nsb*16 + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // 16B strips: 8 signed sub-scales + 8 pad (same buffer at 0) + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array // iq4xs quant plane, uint view + @ssbo @binding = 3 @role = "read" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" y : array + @uniform @binding = 5 ka : KqMvArgs + @template_constant NR : int = 2 + @template_constant NRU : uint = 2u // NR's uint spelling (colbase math) — override BOTH together + @template_constant TILED : bool = false + + [metal_kernel] + def metal_kq_mv_iq4xs { + let lane = gl_SubgroupInvocationID + let tx = lane % 8u + let ty = lane / 8u + let row = gl_WorkGroupID.x * 8u + gl_SubgroupID * 4u + ty + let colbase = gl_WorkGroupID.y * NRU // nolint:LINT002 dead in the TILED=false stamp — measured free + let nb = ka.ndim / 256u + let nb4 = ka.ndim / 4u + var sumf : float[NR] + var sb = 0u + while (sb < nb) { + let blk = row * nb + sb + let dl = float(kdh[blk]) * float((int(kscb[blk * 16u + tx]) ^ 128) - 128) + let qb = blk * 32u + tx * 4u + var wl : float4[4] + var wh : float4[4] + for [unroll_full] (u in range(4)) { + let qv = kqu[qb + uint(u)] + wl[u] = float4(iq4_lut(qv & 15u), iq4_lut((qv >> 8u) & 15u), iq4_lut((qv >> 16u) & 15u), iq4_lut((qv >> 24u) & 15u)) * dl + wh[u] = float4(iq4_lut((qv >> 4u) & 15u), iq4_lut((qv >> 12u) & 15u), iq4_lut((qv >> 20u) & 15u), iq4_lut(qv >> 28u)) * dl + } + let cx4 = sb * 64u + tx * 8u + static_if (TILED) { + for [unroll_full] (b in range(NR)) { + let xb4 = (colbase + uint(b)) * nb4 + cx4 + var acc = 0.0 + for [unroll_full] (u in range(4)) { + let v = x[xb4 + uint(u)] + let vh = x[xb4 + 4u + uint(u)] + acc += v.x * wl[u].x + v.y * wl[u].y + v.z * wl[u].z + v.w * wl[u].w + acc += vh.x * wh[u].x + vh.y * wh[u].y + vh.z * wh[u].z + vh.w * wh[u].w + } + sumf[b] += acc + } + } else { + for [unroll_full] (b in range(NR)) { + let xb4 = uint(b) * nb4 + cx4 + var acc = 0.0 + for [unroll_full] (u in range(4)) { + let v = x[xb4 + uint(u)] + let vh = x[xb4 + 4u + uint(u)] + acc += v.x * wl[u].x + v.y * wl[u].y + v.z * wl[u].z + v.w * wl[u].w + acc += vh.x * wh[u].x + vh.y * wh[u].y + vh.z * wh[u].z + vh.w * wh[u].w + } + sumf[b] += acc + } + } + sb++ + } + static_if (TILED) { + for [unroll_full] (b in range(NR)) { + var s = sumf[b] + s += simd_shuffle_down(s, 4u) + s += simd_shuffle_down(s, 2u) + s += simd_shuffle_down(s, 1u) + if (tx == 0u && row < ka.ddim && colbase + uint(b) < ka.nr) { + y[(colbase + uint(b)) * ka.ys + row] = s + } + } + } else { + for [unroll_full] (b in range(NR)) { + var s = sumf[b] + s += simd_shuffle_down(s, 4u) + s += simd_shuffle_down(s, 2u) + s += simd_shuffle_down(s, 1u) + if (tx == 0u && row < ka.ddim && uint(b) < ka.nr) { + y[uint(b) * ka.ys + row] = s + } + } + } + } +} + +[metal_dispatch(name = "enc_kq_mvb2_iq4xs_c", pso = "g_pso_kq_mvb2_iq4xs", tg = 64, grid = "rows/8", params = "rows : int64")] +class MetalKqMvB2Iq4xs : MetalKqMvIq4xsT { + override NR = 2 + override NRU = 2u + override TILED = false +} + +[metal_dispatch(name = "enc_kq_mvb4_iq4xs_c", pso = "g_pso_kq_mvb4_iq4xs", tg = 64, grid = "rows/8, gcols", params = "rows : int64, gcols : int64")] +class MetalKqMvB4Iq4xs : MetalKqMvIq4xsT { + override NR = 4 + override NRU = 4u + override TILED = true +} + +// the iq4xs B8 twin: the K45 B8 shell (one superblock's X panel staged per threadgroup) over the +// per-32-block lane map +[metal_dispatch(name = "enc_kq_mvb8_iq4xs_c", pso = "g_pso_kq_mvb8_iq4xs", tgmem = "metal_kq_mvb8_iq4xs_msl_tgmem", tg = 64, grid = "rows/8", params = "rows : int64")] +class MetalKqMvB8Iq4xs { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d plane — the scale buffer bound at byte nsb*16 + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // 16B strips: 8 signed sub-scales + 8 pad (same buffer at 0) + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array // iq4xs quant plane, uint view + @ssbo @binding = 3 @role = "read" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" y : array + @uniform @binding = 5 ka : KqMvArgs + @workgroup txp : float4[512] // [8 streams x 64 float4] one superblock's X panel + + [metal_kernel(name="metal_kq_mvb8_iq4xs_msl")] + def metal_kq_mvb8_iq4xs { + let lid = gl_LocalInvocationID.x + let lane = gl_SubgroupInvocationID + let tx = lane % 8u + let ty = lane / 8u + let row = gl_WorkGroupID.x * 8u + gl_SubgroupID * 4u + ty + let nb = ka.ndim / 256u + let nb4 = ka.ndim / 4u + var acc : float4[8] + var sb = 0u + while (sb < nb) { + barrier() + for [unroll_full] (j in range(8)) { + let slot = uint(j) * 64u + lid + txp[slot] = x[(slot / 64u) * nb4 + sb * 64u + (slot % 64u)] + } + barrier() + let blk = row * nb + sb + let dl = float(kdh[blk]) * float((int(kscb[blk * 16u + tx]) ^ 128) - 128) + let qb = blk * 32u + tx * 4u + var wl : float4[4] + var wh : float4[4] + for [unroll_full] (u in range(4)) { + let qv = kqu[qb + uint(u)] + wl[u] = float4(iq4_lut(qv & 15u), iq4_lut((qv >> 8u) & 15u), iq4_lut((qv >> 16u) & 15u), iq4_lut((qv >> 24u) & 15u)) * dl + wh[u] = float4(iq4_lut((qv >> 4u) & 15u), iq4_lut((qv >> 12u) & 15u), iq4_lut((qv >> 20u) & 15u), iq4_lut(qv >> 28u)) * dl + } + let cxl = tx * 8u + for [unroll_full] (b in range(8)) { + for [unroll_full] (u in range(4)) { + acc[b] += txp[uint(b) * 64u + cxl + uint(u)] * wl[u] + txp[uint(b) * 64u + cxl + 4u + uint(u)] * wh[u] + } + } + sb++ + } + for [unroll_full] (b in range(8)) { + var s = acc[b].x + acc[b].y + acc[b].z + acc[b].w + s += simd_shuffle_down(s, 4u) + s += simd_shuffle_down(s, 2u) + s += simd_shuffle_down(s, 1u) + if (tx == 0u && row < ka.ddim && uint(b) < ka.nr) { + y[uint(b) * ka.ys + row] = s + } + } + } +} + // The B2/B4 width pair as one template per format (the B8 trio below is a DIFFERENT algorithm and // stays hand-written). TILED branch-duplicates the b-loop/writeback because a LIVE colbase on B2 // costs +2% (k4) / +0.5% (k6) at cls (bench_metal_gemv_kernels KqMv round); B4 stamps byte-identical. diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das b/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das index 9fcbf636cb..a400bbf51d 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das @@ -4411,6 +4411,9 @@ def private pf_devw_panel(enc : MetalComputeEncoder?; bwblob : MetalBuffer?; wbo //! quant superblocks, plus the k6 2B d tail def private pf_devw_panel_kq(enc : MetalComputeEncoder?; t : Model; fmt : KqFmt; woff : int64; bxh, by, bk, bn : MetalBuffer?; rows, d, kdim : int64; yoff : uint64) : bool { + if (fmt == KqFmt.iq4xs) { // no dev-W dequant kernel for the codebook format yet + return false + } let dq = fmt == KqFmt.k6 ? g_pf_pso_dq_k6 : (fmt == KqFmt.k4 ? g_pf_pso_dq_k4 : g_pf_pso_dq_k5) if (dq == null) { return false @@ -4573,6 +4576,10 @@ def private pf_enc_kq_site_mm(enc : MetalComputeEncoder?; t : Model; fmt : KqFmt let fmt_tensor = fmt == KqFmt.k6 ? g_pf_kq_mm6_tensor : (fmt == KqFmt.k4 ? g_pf_kq_mm4_tensor : g_pf_kq_mm5_tensor) let bq = kq_quants_of(g_dev, t, fmt, woff) let bs = kq_scales_of(g_dev, t, fmt, woff) + if (fmt == KqFmt.iq4xs) { // the base mul_mm only - no tensor / tall / dev-W twins yet + enc_kq_mm_iq4xs_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, by, yoff, bk, bn, mp, rows) + return + } //! deep-class arm: a panel dev-W would split past 8 tiles is DRAM-resident - the tall //! in-kernel-dequant stamp reads the quant plane once per 128-row tile instead of //! materializing and re-streaming a 2B/element f16 panel @@ -4676,7 +4683,9 @@ def private pf_enc_kq_gemv(enc : MetalComputeEncoder?; t : Model; fmt : KqFmt; w bx : MetalBuffer?; xoff : uint64; by, bn, bd : MetalBuffer?) { let bq = kq_quants_of(g_dev, t, fmt, woff) let bs = kq_scales_of(g_dev, t, fmt, woff) - if (fmt == KqFmt.k6) { + if (fmt == KqFmt.iq4xs) { + enc_kq_iq4xs_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, xoff, by, 0ul, bn, bd, rows, n) + } elif (fmt == KqFmt.k6) { enc_kq_k6_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, xoff, by, 0ul, bn, bd, rows, n) } elif (fmt == KqFmt.k4) { enc_kq_k4_c(enc, bs.buf, bs.soff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, xoff, by, 0ul, bn, bd, rows, n) diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_shapes.das b/modules/dasLLAMA/dasllama/dasllama_metal_shapes.das index eaf6031e26..45b5eda15b 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_shapes.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_shapes.das @@ -180,7 +180,7 @@ def record_needs(var tab : table; missing : MetalNeed) { // formats) must DECLINE kquant_native instead of falling into a wrong-layout kernel branch // (the dispatchers below treat "not k4/k6" as k5) def kq_fmt_gpu_supported(f : KqFmt) : bool { - return f == KqFmt.q8 || f == KqFmt.k4 || f == KqFmt.k5 || f == KqFmt.k6 + return f == KqFmt.q8 || f == KqFmt.k4 || f == KqFmt.k5 || f == KqFmt.k6 || f == KqFmt.iq4xs } def private kq_fmts_gpu_supported(a : array) : bool { @@ -243,7 +243,7 @@ def moe_site_ok(fmt : KqFmt; off, ege : int64) : bool { if (fmt == KqFmt.k4 || fmt == KqFmt.k5) { return off >= 0l && (off % 256l) == 0l && (ege % 256l) == 0l } - if (fmt == KqFmt.k6) { + if (fmt == KqFmt.k6 || fmt == KqFmt.iq4xs) { // the split scale form: off % 512 keeps the d-plane bind 4B-aligned return off >= 0l && (off % 512l) == 0l && (ege % 256l) == 0l } if (fmt == KqFmt.q51) { // per-32 planes: off % 128 keeps the 20B/4B binds 16B-aligned @@ -324,9 +324,9 @@ def moe_metal_ok(t : Model) : bool { // nolint:STYLE038 — flat per-shape ref let f3 = fmt_at(t.we3_fmt, l) // per-site alignment + reduction dims: superblock kq kernels iterate whole 256-rows, // q51 whole 32-blocks (+ the mul_mm's 64-col output tiles) - let sb1 = f1 == KqFmt.k4 || f1 == KqFmt.k5 || f1 == KqFmt.k6 - let sb3 = f3 == KqFmt.k4 || f3 == KqFmt.k5 || f3 == KqFmt.k6 - let sb2 = f2 == KqFmt.k4 || f2 == KqFmt.k5 || f2 == KqFmt.k6 + let sb1 = f1 == KqFmt.k4 || f1 == KqFmt.k5 || f1 == KqFmt.k6 || f1 == KqFmt.iq4xs + let sb3 = f3 == KqFmt.k4 || f3 == KqFmt.k5 || f3 == KqFmt.k6 || f3 == KqFmt.iq4xs + let sb2 = f2 == KqFmt.k4 || f2 == KqFmt.k5 || f2 == KqFmt.k6 || f2 == KqFmt.iq4xs if (!moe_site_ok(f1, t.we1_offs[l], ege) || !moe_site_ok(f2, t.we2_offs[l], ege) || !moe_site_ok(f3, t.we3_offs[l], ege) || diff --git a/modules/dasLLAMA/followup_general.md b/modules/dasLLAMA/followup_general.md index d262d58a86..2033d89fdd 100644 --- a/modules/dasLLAMA/followup_general.md +++ b/modules/dasLLAMA/followup_general.md @@ -714,3 +714,20 @@ once already. The tests checklist ledgers the residue; the fix is an in-process equivalent of `DASLLAMA_IMAGE=0` (`g_env_engine.image` is a `let` read at load), so such cells can run image-free instead of risking the purge. + +57. **Plane types have no `long_length`.** `length(PlaneF)` / `length(PlaneU16)` return + `int`, so every `uint64(length(t.blob) * 4l)` spelling caps a plane at 2^31 elements + before the widening - headroom-only today (whisper large-v3's twin is ~632M elements). + Done = `long_length` overloads in `dasllama_plane` and the buffer-sizing call sites + moved onto them. + +58. **The M5 pass on the IQ4_XS Metal kernels (Boris, 2026-08-30: "we'll ledger M5 pass on + new kernels for later").** The format's Metal set is correctness-first: `MetalKqGemvIq4xs`, + the `MetalKqMvIq4xsT` B2/B4 pair, `MetalKqMvB8Iq4xs` and the `IQ4XS` arm of + `MetalKqMulMmK45T` copy the k4/k6 lane maps with a per-element `iq4_lut` (four packed + words, select + shift + sign trick) and no measurement behind them; the prefill site takes + the base mul_mm only - no tensor (`_t`), tall (`_th128`), double-buffered (`_thdb`) or dev-W + dequant twins, and no MoE GEMV / mul_mm trio for the format. Done = the twins stamped on + the existing templates, the LUT cost measured against a `constant` table and against a + byte-pair decode on the M5, and `bench_metal_gemv_kernels` / `bench_metal_kq_mm_lab` rows + for the format beside k4's. diff --git a/modules/dasLLAMA/tests/_metal_kernel_common.das b/modules/dasLLAMA/tests/_metal_kernel_common.das index aea4f78ca0..2d4579ba18 100644 --- a/modules/dasLLAMA/tests/_metal_kernel_common.das +++ b/modules/dasLLAMA/tests/_metal_kernel_common.das @@ -171,17 +171,18 @@ def kq_hash_byte(i : int) : uint8 { let KQ_SAFE_SCALES = fixed_array(0.25, 0.5, 1.0, 0.125, 2.0, 0.75, 1.5, 0.375) // quant plane = hash bytes (any bit pattern is valid); scale plane K4/K5 = [d f16][dmin f16] -// [12 packed 6-bit bytes] per superblock, K6 = [16B sub-scales] x nsb + f16 d tail +// [12 packed 6-bit bytes] per superblock, K6 = [16B sub-scales] x nsb + f16 d tail, IQ4_XS (44) = +// the same split form with [8 signed sub-scales in -31..31][8 zero] strips def kq_fill_planes(fmt, nsb : int; var kq : array; var ks : array) { - let qsb = fmt == 4 ? 128 : (fmt == 5 ? 160 : 192) + let qsb = fmt == 4 || fmt == 44 ? 128 : (fmt == 5 ? 160 : 192) kq |> resize(nsb * qsb) for (i in range(nsb * qsb)) { kq[i] = kq_hash_byte(i * 3 + fmt) } - if (fmt == 6) { + if (fmt == 6 || fmt == 44) { ks |> resize(nsb * 18) for (i in range(nsb * 16)) { - ks[i] = kq_hash_byte(i * 7 + 13) + ks[i] = fmt == 6 ? kq_hash_byte(i * 7 + 13) : uint8(i % 16 < 8 ? int(kq_hash_byte(i * 7 + 13)) % 63 - 31 : 0) } for (b in range(nsb)) { let dbits = f32_to_f16(KQ_SAFE_SCALES[b % 8]) @@ -244,6 +245,9 @@ def kq_row_ref(fmt, sb_base, nb, nsb_total : int; kq, ks : array; var wro dequant_k4_plane_superblock(kq, int64(sb * 128), ks, int64(sb * 16), wrow, int64(ib * 256)) } elif (fmt == 5) { dequant_k5_plane_superblock(kq, int64(sb * 160), ks, int64(sb * 16), wrow, int64(ib * 256)) + } elif (fmt == 44) { + dequant_iq4xs_plane_superblock_at(kq, int64(sb * 128), ks, int64(sb * 16), + int64(nsb_total * 16 + sb * 2), wrow, int64(ib * 256)) } else { dequant_k6_plane_superblock_at(kq, int64(sb * 192), ks, int64(sb * 16), int64(nsb_total * 16 + sb * 2), wrow, int64(ib * 256)) diff --git a/modules/dasLLAMA/tests/test_metal_gemm_kernels.das b/modules/dasLLAMA/tests/test_metal_gemm_kernels.das index 106ae00e62..4eb7e07ae3 100644 --- a/modules/dasLLAMA/tests/test_metal_gemm_kernels.das +++ b/modules/dasLLAMA/tests/test_metal_gemm_kernels.das @@ -36,28 +36,28 @@ def private kq_mulmm_gate(t : T?; dev, queue; fmt : int; tensor : bool; m, kdim, : halfx ? (fmt == 4 ? MetalKqMulMmK4TH_metal_kq_mulmm_t_msl : (fmt == 5 ? MetalKqMulMmK5TH_metal_kq_mulmm_t_msl : MetalKqMulMmK6TH_metal_kq_mulmm_k6_t_msl)) : (fmt == 4 ? MetalKqMulMmK4T_metal_kq_mulmm_t_msl : (fmt == 5 ? MetalKqMulMmK5T_metal_kq_mulmm_t_msl : MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl))) - : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl : MetalKqMulMmK6_metal_kq_mulmm_msl))) + : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl : MetalKqMulMmK6_metal_kq_mulmm_msl)))) let entry = (tensor ? (tall ? (fmt == 4 ? MetalKqMulMmK4TH128_metal_kq_mulmm_t_msl_entry : (fmt == 5 ? MetalKqMulMmK5TH128_metal_kq_mulmm_t_msl_entry : MetalKqMulMmK6TH128_metal_kq_mulmm_k6_t_msl_entry)) : halfx ? (fmt == 4 ? MetalKqMulMmK4TH_metal_kq_mulmm_t_msl_entry : (fmt == 5 ? MetalKqMulMmK5TH_metal_kq_mulmm_t_msl_entry : MetalKqMulMmK6TH_metal_kq_mulmm_k6_t_msl_entry)) : (fmt == 4 ? MetalKqMulMmK4T_metal_kq_mulmm_t_msl_entry : (fmt == 5 ? MetalKqMulMmK5T_metal_kq_mulmm_t_msl_entry : MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl_entry))) - : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_entry : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_entry : MetalKqMulMmK6_metal_kq_mulmm_msl_entry))) + : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl_entry : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_entry : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_entry : MetalKqMulMmK6_metal_kq_mulmm_msl_entry)))) let fm = (tensor ? (tall ? (fmt == 4 ? MetalKqMulMmK4TH128_metal_kq_mulmm_t_msl_fastmath : (fmt == 5 ? MetalKqMulMmK5TH128_metal_kq_mulmm_t_msl_fastmath : MetalKqMulMmK6TH128_metal_kq_mulmm_k6_t_msl_fastmath)) : halfx ? (fmt == 4 ? MetalKqMulMmK4TH_metal_kq_mulmm_t_msl_fastmath : (fmt == 5 ? MetalKqMulMmK5TH_metal_kq_mulmm_t_msl_fastmath : MetalKqMulMmK6TH_metal_kq_mulmm_k6_t_msl_fastmath)) : (fmt == 4 ? MetalKqMulMmK4T_metal_kq_mulmm_t_msl_fastmath : (fmt == 5 ? MetalKqMulMmK5T_metal_kq_mulmm_t_msl_fastmath : MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl_fastmath))) - : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_fastmath : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_fastmath : MetalKqMulMmK6_metal_kq_mulmm_msl_fastmath))) + : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl_fastmath : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_fastmath : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_fastmath : MetalKqMulMmK6_metal_kq_mulmm_msl_fastmath)))) let tgm = (tensor ? (tall ? (fmt == 4 ? MetalKqMulMmK4TH128_metal_kq_mulmm_t_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5TH128_metal_kq_mulmm_t_msl_tgmem : MetalKqMulMmK6TH128_metal_kq_mulmm_k6_t_msl_tgmem)) : halfx ? (fmt == 4 ? MetalKqMulMmK4TH_metal_kq_mulmm_t_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5TH_metal_kq_mulmm_t_msl_tgmem : MetalKqMulMmK6TH_metal_kq_mulmm_k6_t_msl_tgmem)) : (fmt == 4 ? MetalKqMulMmK4T_metal_kq_mulmm_t_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5T_metal_kq_mulmm_t_msl_tgmem : MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl_tgmem))) - : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_tgmem : MetalKqMulMmK6_metal_kq_mulmm_msl_tgmem))) + : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl_tgmem : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_tgmem : MetalKqMulMmK6_metal_kq_mulmm_msl_tgmem)))) var pso = pipeline_from_source(dev, src, entry, fm, err) if (tensor && pso == null && !metal4_tensor_available(dev)) { to_log(LOG_INFO, "{tag}: no Metal-4 tensor toolchain on this box - arm skipped\n") @@ -123,7 +123,7 @@ def private kq_mulmm_gate(t : T?; dev, queue; fmt : int; tensor : bool; m, kdim, let ran = with_compute_encoder(queue, err) $(enc : MetalComputeEncoder?) { metal_set_pipeline(enc, pso) metal_set_threadgroup_memory_length(enc, tgm, 0) - metal_set_buffer(enc, bks, fmt == 6 ? uint64(nsb * 16) : 0ul, 0) // k6: the f16 d tail + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs: the f16 d tail metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bx, 0ul, 3) @@ -134,7 +134,7 @@ def private kq_mulmm_gate(t : T?; dev, queue; fmt : int; tensor : bool; m, kdim, if (rem_pso != null) { metal_set_pipeline(enc, rem_pso) metal_set_threadgroup_memory_length(enc, fmt == 4 ? MetalKqMulMmK4TH_metal_kq_mulmm_t_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5TH_metal_kq_mulmm_t_msl_tgmem : MetalKqMulMmK6TH_metal_kq_mulmm_k6_t_msl_tgmem), 0) - metal_set_buffer(enc, bks, fmt == 6 ? uint64(nsb * 16) : 0ul, 0) + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 ? uint64(nsb * 16) : 0ul, 0) metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bx, uint64(r0 * kdim * 2), 3) @@ -484,7 +484,7 @@ def private kq_devw_gate(t : T?; dev, queue; fmt : int; m, kdim, ndim : int) { var dka = CvtArgs(total = uint(nblk)) let ran = with_compute_encoder(queue, err) $(enc : MetalComputeEncoder?) { metal_set_pipeline(enc, dq_pso) - metal_set_buffer(enc, bks, fmt == 6 ? uint64(nsb * 16) : 0ul, 0) // k6: the f16 d tail + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs: the f16 d tail metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bwh, 0ul, 3) @@ -1461,7 +1461,7 @@ def private moe_mulmm_kq_gate(t : T?; dev, queue; fmt : int; kdim, ndim : int; c let ran = with_compute_encoder(queue, err) $(enc : MetalComputeEncoder?) { metal_set_pipeline(enc, pso) metal_set_threadgroup_memory_length(enc, tgm, 0) - metal_set_buffer(enc, bks, fmt == 6 ? uint64(nsb * 16) : 0ul, 0) // k6: the f16 d tail + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs: the f16 d tail metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bx, 0ul, 3) @@ -1498,7 +1498,7 @@ def private moe_mulmm_kq_gate(t : T?; dev, queue; fmt : int; kdim, ndim : int; c metal_set_threadgroup_memory_length(enc, fmt == 4 ? MetalMoeMulMmK4TH_metal_moe_mulmm_k45_t_msl_tgmem : (fmt == 5 ? MetalMoeMulMmK5TH_metal_moe_mulmm_k45_t_msl_tgmem : MetalMoeMulMmK6TH_metal_moe_mulmm_k6_t_msl_tgmem), 0) - metal_set_buffer(enc, bks, fmt == 6 ? uint64(nsb * 16) : 0ul, 0) + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 ? uint64(nsb * 16) : 0ul, 0) metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bxh, 0ul, 3) @@ -1538,7 +1538,7 @@ def private moe_mulmm_kq_gate(t : T?; dev, queue; fmt : int; kdim, ndim : int; c metal_set_threadgroup_memory_length(enc, fmt == 4 ? MetalMoeMulMmK4TH128_metal_moe_mulmm_k45_t_msl_tgmem : (fmt == 5 ? MetalMoeMulMmK5TH128_metal_moe_mulmm_k45_t_msl_tgmem : MetalMoeMulMmK6TH128_metal_moe_mulmm_k6_t_msl_tgmem), 0) - metal_set_buffer(enc, bks, fmt == 6 ? uint64(nsb * 16) : 0ul, 0) + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 ? uint64(nsb * 16) : 0ul, 0) metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bxh, 0ul, 3) @@ -1551,7 +1551,7 @@ def private moe_mulmm_kq_gate(t : T?; dev, queue; fmt : int; kdim, ndim : int; c metal_set_threadgroup_memory_length(enc, fmt == 4 ? MetalMoeMulMmK4THR_metal_moe_mulmm_k45_t_msl_tgmem : (fmt == 5 ? MetalMoeMulMmK5THR_metal_moe_mulmm_k45_t_msl_tgmem : MetalMoeMulMmK6THR_metal_moe_mulmm_k6_t_msl_tgmem), 0) - metal_set_buffer(enc, bks, fmt == 6 ? uint64(nsb * 16) : 0ul, 0) + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 ? uint64(nsb * 16) : 0ul, 0) metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bxh, 0ul, 3) @@ -2047,6 +2047,8 @@ def test_metal_gemm_kernels(t : T?) { kq_mulmm_gate(t, dev, queue, fmt, true, 320, 512, 64, halfx = true, tall = true) // off the 128 floor: the remainder pair at nonzero x/y offsets } kq_mulmm_gate(t, dev, queue, 4, false, 32, 256, 128) // second shape: 2 col tiles + kq_mulmm_gate(t, dev, queue, 44, false, 64, 512, 64) // iq4xs: the base form only (no tensor / tall twins) + kq_mulmm_gate(t, dev, queue, 44, false, 32, 256, 128) q8_mulmm_t_gate(t, dev, queue, 64, 256, 64) q8_mulmm_t_gate(t, dev, queue, 64, 256, 64, halfx = true) q8_mulmm_t_gate(t, dev, queue, 64, 160, 64) // kdim % 64 == 32: the bk=64 helper's 32-tail chunk diff --git a/modules/dasLLAMA/tests/test_metal_gemv_kernels.das b/modules/dasLLAMA/tests/test_metal_gemv_kernels.das index 6a9958a8f2..6b58d9dd2d 100644 --- a/modules/dasLLAMA/tests/test_metal_gemv_kernels.das +++ b/modules/dasLLAMA/tests/test_metal_gemv_kernels.das @@ -23,12 +23,12 @@ require math // ===== single-stream kq GEMV (MetalKqGemvK4 / K5 / K5C / K6) ===== def private kq_gemv_gate(t : T?; dev, queue; vform : string; n, d : int) { - let fmt = vform == "k4" ? 4 : (vform == "k6" ? 6 : 5) + let fmt = vform == "k4" ? 4 : (vform == "k6" ? 6 : (vform == "iq4xs" ? 44 : 5)) let tag = "kq_gemv_{vform} n={n} d={d}" var err : string - let src = vform == "k4" ? metal_kq_gemv_k4_msl : (vform == "k5" ? metal_kq_gemv_k5_msl : (vform == "k5c" ? metal_kq_gemv_k5c_msl : metal_kq_gemv_k6_msl)) - let entry = vform == "k4" ? metal_kq_gemv_k4_msl_entry : (vform == "k5" ? metal_kq_gemv_k5_msl_entry : (vform == "k5c" ? metal_kq_gemv_k5c_msl_entry : metal_kq_gemv_k6_msl_entry)) - let fm = vform == "k4" ? metal_kq_gemv_k4_msl_fastmath : (vform == "k5" ? metal_kq_gemv_k5_msl_fastmath : (vform == "k5c" ? metal_kq_gemv_k5c_msl_fastmath : metal_kq_gemv_k6_msl_fastmath)) + let src = vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl : (vform == "k4" ? metal_kq_gemv_k4_msl : (vform == "k5" ? metal_kq_gemv_k5_msl : (vform == "k5c" ? metal_kq_gemv_k5c_msl : metal_kq_gemv_k6_msl))) + let entry = vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl_entry : (vform == "k4" ? metal_kq_gemv_k4_msl_entry : (vform == "k5" ? metal_kq_gemv_k5_msl_entry : (vform == "k5c" ? metal_kq_gemv_k5c_msl_entry : metal_kq_gemv_k6_msl_entry))) + let fm = vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl_fastmath : (vform == "k4" ? metal_kq_gemv_k4_msl_fastmath : (vform == "k5" ? metal_kq_gemv_k5_msl_fastmath : (vform == "k5c" ? metal_kq_gemv_k5c_msl_fastmath : metal_kq_gemv_k6_msl_fastmath))) var pso = pipeline_from_source(dev, src, entry, fm, err) t |> success(pso != null, "{tag}: pipeline: {err}") return if (pso == null) @@ -69,7 +69,7 @@ def private kq_gemv_gate(t : T?; dev, queue; vform : string; n, d : int) { let groups = vform == "k5c" ? (d + 1) / 2 : (d + 3) / 4 let ran = with_compute_encoder(queue, err) $(enc : MetalComputeEncoder?) { metal_set_pipeline(enc, pso) - metal_set_buffer(enc, bks, fmt == 6 ? uint64(nsb * 16) : 0ul, 0) // k6: the f16 d tail + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs: the f16 d tail metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bx, 0ul, 3) @@ -102,15 +102,15 @@ def private kq_gemv_gate(t : T?; dev, queue; vform : string; n, d : int) { def private kq_mvb_gate(t : T?; dev, queue; fmt, bwidth, n, d, nr, ys : int) { let tag = "kq_mvb{bwidth}_k{fmt} n={n} d={d} nr={nr} ys={ys}" var err : string - let src = (bwidth == 2 ? (fmt == 4 ? MetalKqMvB2K4_metal_kq_mv_k4_msl : (fmt == 5 ? MetalKqMvB2K5_metal_kq_mv_k5_msl : MetalKqMvB2K6_metal_kq_mv_k6_msl)) - : (bwidth == 4 ? (fmt == 4 ? MetalKqMvB4K4_metal_kq_mv_k4_msl : (fmt == 5 ? MetalKqMvB4K5_metal_kq_mv_k5_msl : MetalKqMvB4K6_metal_kq_mv_k6_msl)) - : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl : metal_kq_mvb8_k6_msl)))) - let entry = (bwidth == 2 ? (fmt == 4 ? MetalKqMvB2K4_metal_kq_mv_k4_msl_entry : (fmt == 5 ? MetalKqMvB2K5_metal_kq_mv_k5_msl_entry : MetalKqMvB2K6_metal_kq_mv_k6_msl_entry)) - : (bwidth == 4 ? (fmt == 4 ? MetalKqMvB4K4_metal_kq_mv_k4_msl_entry : (fmt == 5 ? MetalKqMvB4K5_metal_kq_mv_k5_msl_entry : MetalKqMvB4K6_metal_kq_mv_k6_msl_entry)) - : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_entry : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_entry : metal_kq_mvb8_k6_msl_entry)))) - let fm = (bwidth == 2 ? (fmt == 4 ? MetalKqMvB2K4_metal_kq_mv_k4_msl_fastmath : (fmt == 5 ? MetalKqMvB2K5_metal_kq_mv_k5_msl_fastmath : MetalKqMvB2K6_metal_kq_mv_k6_msl_fastmath)) - : (bwidth == 4 ? (fmt == 4 ? MetalKqMvB4K4_metal_kq_mv_k4_msl_fastmath : (fmt == 5 ? MetalKqMvB4K5_metal_kq_mv_k5_msl_fastmath : MetalKqMvB4K6_metal_kq_mv_k6_msl_fastmath)) - : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_fastmath : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_fastmath : metal_kq_mvb8_k6_msl_fastmath)))) + let src = (bwidth == 2 ? (fmt == 44 ? MetalKqMvB2Iq4xs_metal_kq_mv_iq4xs_msl : (fmt == 4 ? MetalKqMvB2K4_metal_kq_mv_k4_msl : (fmt == 5 ? MetalKqMvB2K5_metal_kq_mv_k5_msl : MetalKqMvB2K6_metal_kq_mv_k6_msl))) + : (bwidth == 4 ? (fmt == 44 ? MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl : (fmt == 4 ? MetalKqMvB4K4_metal_kq_mv_k4_msl : (fmt == 5 ? MetalKqMvB4K5_metal_kq_mv_k5_msl : MetalKqMvB4K6_metal_kq_mv_k6_msl))) + : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl : metal_kq_mvb8_k6_msl))))) + let entry = (bwidth == 2 ? (fmt == 44 ? MetalKqMvB2Iq4xs_metal_kq_mv_iq4xs_msl_entry : (fmt == 4 ? MetalKqMvB2K4_metal_kq_mv_k4_msl_entry : (fmt == 5 ? MetalKqMvB2K5_metal_kq_mv_k5_msl_entry : MetalKqMvB2K6_metal_kq_mv_k6_msl_entry))) + : (bwidth == 4 ? (fmt == 44 ? MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl_entry : (fmt == 4 ? MetalKqMvB4K4_metal_kq_mv_k4_msl_entry : (fmt == 5 ? MetalKqMvB4K5_metal_kq_mv_k5_msl_entry : MetalKqMvB4K6_metal_kq_mv_k6_msl_entry))) + : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl_entry : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_entry : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_entry : metal_kq_mvb8_k6_msl_entry))))) + let fm = (bwidth == 2 ? (fmt == 44 ? MetalKqMvB2Iq4xs_metal_kq_mv_iq4xs_msl_fastmath : (fmt == 4 ? MetalKqMvB2K4_metal_kq_mv_k4_msl_fastmath : (fmt == 5 ? MetalKqMvB2K5_metal_kq_mv_k5_msl_fastmath : MetalKqMvB2K6_metal_kq_mv_k6_msl_fastmath))) + : (bwidth == 4 ? (fmt == 44 ? MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl_fastmath : (fmt == 4 ? MetalKqMvB4K4_metal_kq_mv_k4_msl_fastmath : (fmt == 5 ? MetalKqMvB4K5_metal_kq_mv_k5_msl_fastmath : MetalKqMvB4K6_metal_kq_mv_k6_msl_fastmath))) + : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl_fastmath : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_fastmath : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_fastmath : metal_kq_mvb8_k6_msl_fastmath))))) var pso = pipeline_from_source(dev, src, entry, fm, err) t |> success(pso != null, "{tag}: pipeline: {err}") return if (pso == null) @@ -160,9 +160,9 @@ def private kq_mvb_gate(t : T?; dev, queue; fmt, bwidth, n, d, nr, ys : int) { metal_set_pipeline(enc, pso) if (bwidth == 8) { metal_set_threadgroup_memory_length(enc, - fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_tgmem : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_tgmem : metal_kq_mvb8_k6_msl_tgmem), 0) + fmt == 44 ? metal_kq_mvb8_iq4xs_msl_tgmem : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_tgmem : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_tgmem : metal_kq_mvb8_k6_msl_tgmem)), 0) } - metal_set_buffer(enc, bks, fmt == 6 ? uint64(nsb * 16) : 0ul, 0) // k6: the f16 d tail + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs: the f16 d tail metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bx, 0ul, 3) @@ -251,7 +251,7 @@ def private moe_kq_gemv_gate(t : T?; dev, queue; fmt, n, d, net, k, nst : int; p bxs = uint((perslot ? k : 1) * n), bys = uint(k * d), hasb = 0u) let ran = with_compute_encoder(queue, err) $(enc : MetalComputeEncoder?) { metal_set_pipeline(enc, pso) - metal_set_buffer(enc, bks, fmt == 6 ? uint64(nsb * 16) : 0ul, 0) // k6: the f16 d tail + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs: the f16 d tail metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bx, 0ul, 3) @@ -721,8 +721,10 @@ def test_metal_kq_gemv_kernels(t : T?) { kq_gemv_gate(t, dev, queue, "k6", 1280, 11) kq_gemv_gate(t, dev, queue, "k4", 512, 30) kq_gemv_gate(t, dev, queue, "k6", 512, 30) + kq_gemv_gate(t, dev, queue, "iq4xs", 1280, 11) + kq_gemv_gate(t, dev, queue, "iq4xs", 512, 30) // the small-batch twins: nr = 2 / 3 (b4 col guard) / 6 (b8 pad), ys > d - for (fmt in [4, 5, 6]) { + for (fmt in [4, 5, 6, 44]) { kq_mvb_gate(t, dev, queue, fmt, 2, 768, 17, 2, 22) kq_mvb_gate(t, dev, queue, fmt, 4, 768, 17, 3, 22) kq_mvb_gate(t, dev, queue, fmt, 8, 768, 17, 6, 22) From 03736402f1aa19effa5b9dbb63a561000f86313d Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Sun, 30 Aug 2026 10:13:46 -0700 Subject: [PATCH 07/61] dasLLAMA: Q3_K native tier, CPU slice - KqFmt.k3 (id 3), the k6-shaped scale row decoded at transcode, [qs][hmask] planes, kernels/repack/tests as k6 twins, every ladder arm; HOW_TO Q3_K note + QUIRKS 16-17 Q3_K is k6's scale structure (16 per-16 signed sub-scales + f16 d, the 18 B row) over a 3-bit quant composed from a 2-bit qs lane and an hmask bit, offset -4 folded like k6's -32. Planes: [qs 64][hmask 32] verbatim (K3_QSB 96); the 12 packed 6-bit scales decode at transcode (q3k_scale6, ggml's kmask unpack) into int8 scale - 32 (K3_SSB 18). dot_k3q8, k3_grp_row_dot, dequant_k3_row_grp, repack_k3_grp (the lanes and mask bits stay in place as 4-byte columns x mr), the k3q8 [tune] family with decline stubs, and every dispatch ladder arm - built by twinning each iq4xs arm of the pilot's CPU-slice commit for k3, which also surfaced two plane-byte accounting sums the pilot had skipped (now counting iq4xs and k3). Gates: test_kqformat 14/14; test_kquant 132/144 interpreted, 140/144 under -jit; the tuner probe's tune-mode family. k3 stays OUT of the probe's test-mode list until its emitter arm lands (QUIRK 16: a stub family fails every perm whose layout companion generates - planes at mr 8, the reference body at mr 4). End to end: Llama-3.2-1B-Instruct-Q3_K_L (Q3_K x64 + Q5_K x48 + Q6_K embd) matches llama.cpp's greedy ids for 52 of 64 tokens on the reference bodies. QUIRK 17: the new [tune] family re-tunes every application sidecar on its first start. Co-Authored-By: Claude Fable 5 --- modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md | 32 ++++ modules/dasLLAMA/LAWS.md | 12 ++ modules/dasLLAMA/dasllama/dasllama_blocks.das | 3 + modules/dasLLAMA/dasllama/dasllama_common.das | 38 ++++- modules/dasLLAMA/dasllama/dasllama_config.das | 3 +- .../dasLLAMA/dasllama/dasllama_convert.das | 50 ++++++ .../dasLLAMA/dasllama/dasllama_gemm_gen.das | 9 ++ .../dasllama/dasllama_gemm_schema.das | 8 +- modules/dasLLAMA/dasllama/dasllama_gguf.das | 39 +++++ .../dasllama/dasllama_gpu_resident.das | 11 +- modules/dasLLAMA/dasllama/dasllama_image.das | 7 +- .../dasLLAMA/dasllama/dasllama_kqformat.das | 19 ++- modules/dasLLAMA/dasllama/dasllama_layout.das | 16 +- modules/dasLLAMA/dasllama/dasllama_load.das | 41 ++++- modules/dasLLAMA/dasllama/dasllama_math.das | 9 +- .../dasllama/dasllama_math_default.das | 82 ++++++++++ .../dasLLAMA/dasllama/dasllama_math_gen.das | 107 ++++++++++++- modules/dasLLAMA/dasllama/dasllama_ple.das | 4 + modules/dasLLAMA/dasllama/dasllama_repack.das | 48 ++++++ modules/dasLLAMA/harness/gen_tune_probe.das | 47 +++++- modules/dasLLAMA/tests/test_kqformat.das | 14 +- modules/dasLLAMA/tests/test_kquant.das | 146 ++++++++++++++++-- 22 files changed, 692 insertions(+), 53 deletions(-) diff --git a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md index 2c07d10406..65ab94f806 100644 --- a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md +++ b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md @@ -343,9 +343,41 @@ where, why it is so today, what unquirked looks like. An empty ledger is a legit minutes minting `run.tune.json` (every family, the confirm pass included) before loading the model - the auto policy, working as designed, but a Mac session's first end-to-end "hang" is that mint. Watch `@tune begin/end` lines in the log. +16. **A stub family fails the probe's test mode on every perm whose layout companion + generates.** `q8q8_layout` is shared and generates for any perm `perm_declines` admits, so + under `=> false` stubs the k3 family's `dot_maddubs_width256_mr8` perm repacked its planes + at mr 8 while the declined tile ran the reference body at `k3q8_layout_gen()` = 4 - + `maxdiff nan`. The lockstep between layout and tile declines IS `perm_declines`, which a + stub does not share. Keep a stub family out of the probe's TEST list (the tune-mode list is + fine: the race gates each perm and mints "reference"); it joins the test list with its + emitter arm. This is why iq4xs entered that list only in Phase B. +17. **A new `[tune]` family re-tunes every application on its first start.** A sidecar with no + entry for a demanded kernel re-tunes the whole scope (the framework's completeness rule), so + `run.das` on the box spent minutes minting before the Q3_K_L load - and every other + sidecar on the box (`chat.das`, the harness probes) does the same on its next run. Budget + for it; do not read the first end-to-end wall time as a load-time regression. ## Per-format notes +### Q3_K (the second format, 2026-08-30) + +Shape: 256-superblock, k6's scale structure exactly (16 per-16 signed sub-scales + f16 d, the +18 B row) over a 3-bit quant composed from a 2-bit lane (byte h*32 + l of `qs`, shift 2j, block +b = 4h + j) and an `hmask` bit (bit b of byte l), offset -4 folded like k6's -32. Disk block +110 B: hmask[32], qs[64], 12 packed 6-bit scales, f16 d. Planes: quants `[qs 64][hmask 32]` +verbatim (K3_QSB 96), the scale row DECODED at transcode (`q3k_scale6` = ggml's kmask unpack, +stored as int8 scale - 32; K3_SSB 18). Ids: `KqFmt.k3` = 7, kernel id 3 (bit width), stream +code 3. Kernels are k6's with a different compose: `dot_k3q8`, `k3_grp_row_dot` +(`isum - 4 * bsum`), `repack_k3_grp` (the 2-bit lanes and mask bits stay in place as 4-byte +columns x mr; k6's scale interleave). The tile reads the PACKED planes (`packed` includes 3) - +no byte-expanded panel. Walk cost: one Python patch twinning every `iq4xs` arm of the pilot's +CPU-slice commit (`63da7571e`) for k3 plus the hand-written codec/kernels/repack; the only +misses the census found were two plane-byte accounting sums the pilot itself had skipped. +Gates: `test_kqformat` 14/14, `test_kquant` 132/144 interpreted and 140/144 under `-jit`, +the probe's tune-mode family (test mode waits for the emitter - QUIRK 16). End to end: +`Llama-3.2-1B-Instruct-Q3_K_L.gguf` (bartowski: Q3_K x64 + Q5_K x48 + Q6_K embd) through `run.das` matches llama.cpp's greedy ids (`simple_ids.exe`, compared through `llama-tokenize --ids` on our text) for 52 of 64 tokens on the reference bodies, gen 22 t/s. +JIT emitter, Vulkan, Metal: pending. + ### IQ4_XS (the pilot, 2026-08-30) Shape: 256-superblock, codebook nibble (`kvalues_iq4nl`, signed, no offset), scale = f16 d x diff --git a/modules/dasLLAMA/LAWS.md b/modules/dasLLAMA/LAWS.md index 62a7e34bf1..c16b435a07 100644 --- a/modules/dasLLAMA/LAWS.md +++ b/modules/dasLLAMA/LAWS.md @@ -75,3 +75,15 @@ module's new `CLAUDE.md` and the root table. Arc ruling the same day: the community i-quants are read as-is ("B for sure. we are not big enough for people to bother with our own fork"), Qwen3.8-27B on a 16 GB card is the Vulkan arc's endgame. + +## 2026-08-30 - HOW_TO_ADD_A_FORMAT.md, followup_general.md (#58) + +Boris, on the IQ4_XS pilot's Metal section and QUIRKS 12-15: "this is exactly the kind of info +we need for that document. once we're done with all formats - we'll unquirk all we can, but +otherwise it'll be like step1-step2-step3 instruction - this is how to add format. like it +should." The QUIRKS ledger stays open and honest through every format of the arc; the unquirk +pass comes AFTER the last format lands, and the surviving document is the numbered walk. Earlier +the same day, on the Metal portion: "we have an option of ssh m1 ... go for it. it feels right +somehow. plus validates the how-to. and we'll ledger M5 pass on new kernels for later" - the M1 +is in the loop for every format from here, and performance work on the new Metal kernels is +the ledgered M5 pass, not this arc. diff --git a/modules/dasLLAMA/dasllama/dasllama_blocks.das b/modules/dasLLAMA/dasllama/dasllama_blocks.das index 63f369b3d4..41cbaba989 100644 --- a/modules/dasLLAMA/dasllama/dasllama_blocks.das +++ b/modules/dasLLAMA/dasllama/dasllama_blocks.das @@ -1403,6 +1403,9 @@ def private kq_bytes_per_weight(f : KqFmt) : float { if (f == KqFmt.k6) { return 212.0 / 256.0 } + if (f == KqFmt.k3) { + return 114.0 / 256.0 // 96B quants + the 18B scale row per 256 + } return 34.0 / 32.0 // q8: 32 quants + an f16 scale per block } diff --git a/modules/dasLLAMA/dasllama/dasllama_common.das b/modules/dasLLAMA/dasllama/dasllama_common.das index 9aab895280..21a2e1dc6a 100644 --- a/modules/dasLLAMA/dasllama/dasllama_common.das +++ b/modules/dasLLAMA/dasllama/dasllama_common.das @@ -838,6 +838,9 @@ struct Model { // the iq4xs tier's plane pair: 128B nibbles in the q40 tiling + a 20B decoded scale row per superblock iq4xsq : array iq4xss : array + // the k3 tier's plane pair: [64 qs][32 hmask] per superblock + the k6-shaped 18B scale row (16 x int8 (sc - 32), f16 d) + k3q : array + k3s : array kquant_native : bool = false // kq stage 4: the kq planes were repacked at load into the active backend's grp layout. // From then on every kq matmul MUST run the backend slots — disk-order portable kernels would @@ -849,6 +852,7 @@ struct Model { kq_repack_mr6 : int64 = 4l kq_repack_mr40 : int64 = 4l kq_repack_mr44 : int64 = 4l + kq_repack_mr3 : int64 = 4l // per-layer weight format tags (KqFmt; empty = all q8). Filled by detect_kq_formats before // layout_offsets so the layout walks each tensor into its format's cursor. wq_fmt : array @@ -1071,6 +1075,7 @@ def model_weights_bytes(t : Model) : int64 { + long_length(t.bf16blob) * 2l + long_length(t.k4q) + long_length(t.k4s) + long_length(t.k5q) + long_length(t.k5s) + long_length(t.k6q) + long_length(t.k6s) + long_length(t.q40q) + long_length(t.q40s) + + long_length(t.iq4xsq) + long_length(t.iq4xss) + long_length(t.k3q) + long_length(t.k3s) + long_length(t.q51q) + long_length(t.q51s) + long_length(t.mblob) + long_length(t.vkblob) + long_length(t.embq) + long_length(t.embs)) @@ -1460,6 +1465,7 @@ def private dlim_cpu_source_impl(var c : DlimCpuConfig) { c.kq_mr6 = active_kq_layout_mr(6) c.kq_mr40 = active_kq_layout_mr(40) c.kq_mr44 = active_kq_layout_mr(44) + c.kq_mr3 = active_kq_layout_mr(3) c.q51_mr = active_q51_layout_mr() // OUTCOME, not request: a backend without s16 twins keeps f32 scale planes (wscale_convert_f16) c.wscale_f16 = g_wscale_f16 && kernel_backend_has_wscale16() @@ -1552,7 +1558,7 @@ def kq_active_mr(t : Model; f : KqFmt) : int64 { if (!t.kq_repacked) { return 1l } - return f == KqFmt.k4 ? t.kq_repack_mr4 : (f == KqFmt.k5 ? t.kq_repack_mr5 : (f == KqFmt.k6 ? t.kq_repack_mr6 : (f == KqFmt.q40 ? t.kq_repack_mr40 : t.kq_repack_mr44))) + return f == KqFmt.k4 ? t.kq_repack_mr4 : (f == KqFmt.k5 ? t.kq_repack_mr5 : (f == KqFmt.k6 ? t.kq_repack_mr6 : (f == KqFmt.q40 ? t.kq_repack_mr40 : (f == KqFmt.iq4xs ? t.kq_repack_mr44 : t.kq_repack_mr3)))) } //! Load a llama-architecture GGUF into the split layout at the requested precision — straight into @@ -1615,8 +1621,8 @@ def footprint(t : Model) : MemFootprint { let q4s = long_length(t.q4scales) * 4l let mx4 = long_length(t.mxq) // uint8 (two nibbles) -> 1 byte each let mx4s = long_length(t.mxs) // raw E8M0 -> 1 byte per 32-block - let kq = long_length(t.k4q) + long_length(t.k5q) + long_length(t.k6q) + long_length(t.q40q) + long_length(t.q51q) - let kqs = long_length(t.k4s) + long_length(t.k5s) + long_length(t.k6s) + long_length(t.q40s) + long_length(t.q51s) + let kq = long_length(t.k4q) + long_length(t.k5q) + long_length(t.k6q) + long_length(t.q40q) + long_length(t.q51q) + long_length(t.iq4xsq) + long_length(t.k3q) + let kqs = long_length(t.k4s) + long_length(t.k5s) + long_length(t.k6s) + long_length(t.q40s) + long_length(t.q51s) + long_length(t.iq4xss) + long_length(t.k3s) return MemFootprint(aux_fp32_bytes = aux, weight_fp32_bytes = wf, q8_bytes = q8, q8_scale_bytes = q8s, q4_bytes = q4, q4_scale_bytes = q4s, mx4_bytes = mx4, mx4_scale_bytes = mx4s, kq_bytes = kq, kq_scale_bytes = kqs, @@ -3290,7 +3296,7 @@ def mm_b_q51_groupn(var y : array; t : Model; offs : array; nregio // KqFmt -> the kernel-layer format id (matmul_kq*/kq_rows_fn/kq_qsb take the int form). q8 maps // to 0, a poison id no kq kernel accepts — kq_rows_for calls this for EVERY tag, so it must stay // total even though q8 branches never read the result. -def private kq_fi(fmt : KqFmt) : int => fmt == KqFmt.k4 ? 4 : (fmt == KqFmt.k5 ? 5 : (fmt == KqFmt.k6 ? 6 : (fmt == KqFmt.q40 ? 40 : (fmt == KqFmt.iq4xs ? 44 : 0)))) +def private kq_fi(fmt : KqFmt) : int => fmt == KqFmt.k4 ? 4 : (fmt == KqFmt.k5 ? 5 : (fmt == KqFmt.k6 ? 6 : (fmt == KqFmt.q40 ? 40 : (fmt == KqFmt.iq4xs ? 44 : (fmt == KqFmt.k3 ? 3 : 0))))) // K-quant GEMV against a PRE-quantized activation (xq/xs/xbs already filled by the bs quantizer). // kq_repacked loads run the backend's stamped cores; disk-order loads run the portable kernels. @@ -3306,6 +3312,8 @@ def mm_at_kq_pre(var y : array; t : Model; fmt : KqFmt; woff : int64; xq matmul_kq_active(40, y, t.q40q, t.q40s, woff, xq, xs, xbs, n, d, yoff) } elif (fmt == KqFmt.iq4xs) { matmul_kq_active(44, y, t.iq4xsq, t.iq4xss, woff, xq, xs, xbs, n, d, yoff) + } elif (fmt == KqFmt.k3) { + matmul_kq_active(3, y, t.k3q, t.k3s, woff, xq, xs, xbs, n, d, yoff) } else { panic("mm_at_kq_pre: '{fmt}' has no kq plane pair") // a new KqFmt must claim its arm, never ride q40's } @@ -3319,6 +3327,8 @@ def mm_at_kq_pre(var y : array; t : Model; fmt : KqFmt; woff : int64; xq matmul_kq(40, y, t.q40q, t.q40s, woff, xq, xs, xbs, n, d, yoff) } elif (fmt == KqFmt.iq4xs) { matmul_kq(44, y, t.iq4xsq, t.iq4xss, woff, xq, xs, xbs, n, d, yoff) + } elif (fmt == KqFmt.k3) { + matmul_kq(3, y, t.k3q, t.k3s, woff, xq, xs, xbs, n, d, yoff) } else { panic("mm_at_kq_pre: '{fmt}' has no kq plane pair") } @@ -3352,6 +3362,8 @@ def mm_b_kq(var s : Session; var y : array; t : Model; fmt : KqFmt; woff matmul_kq_batch(40, y, t.q40q, t.q40s, woff, s.xqb, s.xsb, s.xbsb, n, d, npos) } elif (fmt == KqFmt.iq4xs) { matmul_kq_batch(44, y, t.iq4xsq, t.iq4xss, woff, s.xqb, s.xsb, s.xbsb, n, d, npos) + } elif (fmt == KqFmt.k3) { + matmul_kq_batch(3, y, t.k3q, t.k3s, woff, s.xqb, s.xsb, s.xbsb, n, d, npos) } else { panic("mm_b_kq: '{fmt}' has no kq plane pair") } @@ -3381,6 +3393,8 @@ def mm_at_kq_groupn(var y : array; t : Model; fmt : KqFmt; offs : array; t : Model; fmt : KqFmt; offs : array; t : Model; fmt : KqFmt; offs : array; t : Model; fmt : KqFmt; woff : int64; xq : matmul_kq_batch(40, y, t.q40q, t.q40s, woff, xq, xs, xbs, n, d, ntok) } elif (fmt == KqFmt.iq4xs) { matmul_kq_batch(44, y, t.iq4xsq, t.iq4xss, woff, xq, xs, xbs, n, d, ntok) + } elif (fmt == KqFmt.k3) { + matmul_kq_batch(3, y, t.k3q, t.k3s, woff, xq, xs, xbs, n, d, ntok) } else { panic("mm_b_kq_pre: '{fmt}' has no kq plane pair") } @@ -3445,6 +3465,7 @@ def kq_plane_q(t : Model; fmt : KqFmt; woff : int64) : uint8 const? { if (fmt == KqFmt.k6) return unsafe(addr(t.k6q[sb * K6_QSB])) if (fmt == KqFmt.q40) return unsafe(addr(t.q40q[sb * Q40_QSB])) if (fmt == KqFmt.iq4xs) return unsafe(addr(t.iq4xsq[sb * IQ4XS_QSB])) + if (fmt == KqFmt.k3) return unsafe(addr(t.k3q[sb * K3_QSB])) return null } @@ -3455,6 +3476,7 @@ def kq_plane_s(t : Model; fmt : KqFmt; woff : int64) : uint8 const? { if (fmt == KqFmt.k6) return unsafe(addr(t.k6s[sb * K6_SSB])) if (fmt == KqFmt.q40) return unsafe(addr(t.q40s[sb * Q40_SSB])) if (fmt == KqFmt.iq4xs) return unsafe(addr(t.iq4xss[sb * IQ4XS_SSB])) + if (fmt == KqFmt.k3) return unsafe(addr(t.k3s[sb * K3_SSB])) return null } @@ -3817,6 +3839,8 @@ def embed_row(t : Model; token : int64; var dst : float?) { // nolint:STYLE037 dequant_kq_row_grp(fmt, addr(t.embq[sbg * Q40_QSB]), addr(t.embs[sbg * Q40_SSB]), token % mr, mr, dim, dst) } elif (t.emb_fmt == KqFmt.iq4xs) { dequant_kq_row_grp(fmt, addr(t.embq[sbg * IQ4XS_QSB]), addr(t.embs[sbg * IQ4XS_SSB]), token % mr, mr, dim, dst) + } elif (t.emb_fmt == KqFmt.k3) { + dequant_kq_row_grp(fmt, addr(t.embq[sbg * K3_QSB]), addr(t.embs[sbg * K3_SSB]), token % mr, mr, dim, dst) } else { panic("embed_row: '{t.emb_fmt}' has no kq plane pair") } @@ -3830,6 +3854,8 @@ def embed_row(t : Model; token : int64; var dst : float?) { // nolint:STYLE037 dequant_kq_row_grp(fmt, addr(t.q40q[sbg * Q40_QSB]), addr(t.q40s[sbg * Q40_SSB]), token % mr, mr, dim, dst) } elif (t.emb_fmt == KqFmt.iq4xs) { dequant_kq_row_grp(fmt, addr(t.iq4xsq[sbg * IQ4XS_QSB]), addr(t.iq4xss[sbg * IQ4XS_SSB]), token % mr, mr, dim, dst) + } elif (t.emb_fmt == KqFmt.k3) { + dequant_kq_row_grp(fmt, addr(t.k3q[sbg * K3_QSB]), addr(t.k3s[sbg * K3_SSB]), token % mr, mr, dim, dst) } else { panic("embed_row: '{t.emb_fmt}' has no kq plane pair") } @@ -3854,6 +3880,8 @@ def embed_row(t : Model; token : int64; var dst : float?) { // nolint:STYLE037 dequant_q40_plane_superblock(t.embq, (sb0 + s) * Q40_QSB, t.embs, (sb0 + s) * Q40_SSB, row, s * 256l) } elif (t.emb_fmt == KqFmt.iq4xs) { dequant_iq4xs_plane_superblock(t.embq, (sb0 + s) * IQ4XS_QSB, t.embs, (sb0 + s) * IQ4XS_SSB, row, s * 256l) + } elif (t.emb_fmt == KqFmt.k3) { + dequant_k3_plane_superblock(t.embq, (sb0 + s) * K3_QSB, t.embs, (sb0 + s) * K3_SSB, row, s * 256l) } else { panic("embed_row: '{t.emb_fmt}' has no kq plane pair") } @@ -3871,6 +3899,8 @@ def embed_row(t : Model; token : int64; var dst : float?) { // nolint:STYLE037 dequant_q40_plane_superblock(t.q40q, (sb0 + s) * Q40_QSB, t.q40s, (sb0 + s) * Q40_SSB, row, s * 256l) } elif (t.emb_fmt == KqFmt.iq4xs) { dequant_iq4xs_plane_superblock(t.iq4xsq, (sb0 + s) * IQ4XS_QSB, t.iq4xss, (sb0 + s) * IQ4XS_SSB, row, s * 256l) + } elif (t.emb_fmt == KqFmt.k3) { + dequant_k3_plane_superblock(t.k3q, (sb0 + s) * K3_QSB, t.k3s, (sb0 + s) * K3_SSB, row, s * 256l) } else { panic("embed_row: '{t.emb_fmt}' has no kq plane pair") } diff --git a/modules/dasLLAMA/dasllama/dasllama_config.das b/modules/dasLLAMA/dasllama/dasllama_config.das index 9104fdd776..ef903da633 100644 --- a/modules/dasLLAMA/dasllama/dasllama_config.das +++ b/modules/dasLLAMA/dasllama/dasllama_config.das @@ -28,6 +28,7 @@ struct public DlimCpuConfig { kq_mr6 : int64 kq_mr40 : int64 kq_mr44 : int64 + kq_mr3 : int64 q51_mr : int64 wscale_f16 : bool kquant_native : bool @@ -140,7 +141,7 @@ def public dlim_config_current(quant : string = "q8") : DlimConfiguration { def public dlim_identity(c : DlimConfiguration; image_version : int; tag : string = "") : string { return ("v{image_version}|{c.quant}|{c.cpu.backend}|{c.cpu.wscale_f16 ? "s16" : "s32"}" + "|q8 mr{c.cpu.q8_mr} b{c.cpu.q8_wbias} g{c.cpu.q8_kgroup}" - + "|kq {c.cpu.kq_mr4}/{c.cpu.kq_mr5}/{c.cpu.kq_mr6}/{c.cpu.kq_mr40}/{c.cpu.kq_mr44}" + + "|kq {c.cpu.kq_mr4}/{c.cpu.kq_mr5}/{c.cpu.kq_mr6}/{c.cpu.kq_mr40}/{c.cpu.kq_mr44}/{c.cpu.kq_mr3}" + "|q51 mr{c.cpu.q51_mr}" + "|nat {c.cpu.kquant_native ? 1 : 0}{c.cpu.kq_q40_native ? 1 : 0}{c.cpu.kq_q50_native ? 1 : 0}{c.cpu.kq_q51_native ? 1 : 0}" + (tag != "" ? "|{tag}" : "")) diff --git a/modules/dasLLAMA/dasllama/dasllama_convert.das b/modules/dasLLAMA/dasllama/dasllama_convert.das index f2e3c0368b..ad8dd7d395 100644 --- a/modules/dasLLAMA/dasllama/dasllama_convert.das +++ b/modules/dasLLAMA/dasllama/dasllama_convert.das @@ -663,6 +663,56 @@ def dequant_iq4xs_plane_superblock_at(kq : array | #; kqo : int64; ks : a } } +//! The 6-bit sub-scale i (0..15) of a Q3_K disk block off its 12 packed scale bytes at `so` — +//! ggml's kmask unpack: low nibbles of bytes 0..7 carry scales 0..7, their high nibbles 8..15, +//! bytes 8..11 carry every scale's top two bits at shift 2 * (i / 4). +def q3k_scale6(bytes : array | #; so, i : int64) : int { + let lo = i < 8l ? int(bytes[so + i]) & 15 : int(bytes[so + i - 8l]) >> 4 + let hi = int(bytes[so + 8l + i % 4l]) >> int(2l * (i / 4l)) + return lo | ((hi & 3) << 4) +} + +//! Transcode one Q3_K superblock (110 bytes at `bo`: 32 hmask, 64 qs, 12 packed 6-bit scales, +//! f16 d) into the k3 planes: [qs 64][hmask 32] verbatim to kq[kqo..+96), the 16 sub-scales +//! DECODED to int8 (scale - 32) + the f16 d to ks[kso..+18) — the k6 row shape (exact). +def transcode_q3k_superblock(bytes : array | #; bo : int64; var kq : array; kqo : int64; var ks : array; kso : int64) { + for (i in range64(64l)) { + kq[kqo + i] = bytes[bo + 32l + i] + } + for (i in range64(32l)) { + kq[kqo + 64l + i] = bytes[bo + i] + } + for (i in range64(16l)) { + ks[kso + i] = uint8(q3k_scale6(bytes, bo + 96l, i) - 32) + } + ks[kso + 16l] = bytes[bo + 108l] + ks[kso + 17l] = bytes[bo + 109l] +} + +//! Reference dequant of one k3-plane superblock: w = (d * sc16) * (q - 4) with q = the 2-bit lane +//! of qs byte h*32 + l at shift 2j | hmask bit (4h + j) of byte l << 2, block 4h + j — ggml's +//! own float order (dl = d * (scale - 32), then dl * (q3 - 4)). +def dequant_k3_plane_superblock(kq : array | #; kqo : int64; ks : array | #; kso : int64; var dst : array | #; doff : int64) { + dequant_k3_plane_superblock_at(kq, kqo, ks, kso, kso + 16l, dst, doff) +} + +//! The split-layout twin: `dso` names the f16 d halfword's byte offset independently of the +//! 16-byte sub-scale strip at kso — the k6 split form a metal-blob model keeps. +def dequant_k3_plane_superblock_at(kq : array | #; kqo : int64; ks : array | #; kso, dso : int64; var dst : array | #; doff : int64) { + let d = f16_to_f32(rd_u16(ks, dso)) + for (blk in range64(8l)) { + let qb = kqo + (blk / 4l) * 32l + let sh = int((blk % 4l) * 2l) + let dl0 = d * float(rd_i8(ks, kso + 2l * blk)) + let dl1 = d * float(rd_i8(ks, kso + 2l * blk + 1l)) + for (l in range64(32l)) { + let q2 = (int(kq[qb + l]) >> sh) & 3 + let hb = (int(kq[kqo + 64l + l]) >> int(blk)) & 1 + dst[doff + blk * 32l + l] = (l < 16l ? dl0 : dl1) * float((q2 | (hb << 2)) - 4) + } + } +} + //! Transcode one Q5_1 disk block (24 bytes at `bo`: d f16, m f16, qh u32, 16 nibble bytes) into //! the q51 planes — verbatim splits, exact. The per-block array form the tests drive; the bulk //! loader (gguf_transcode_q51) runs the same split pointerized and threaded. diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das index 1664657274..522d1dce2d 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das @@ -2542,6 +2542,13 @@ def private kq_tile_gen_impl(var gc : LlvmCodeCtx; fmt : int) : bool { def private iq4xs_gemv_gen(var gc : LlvmCodeCtx) : bool => kq_gemv_gen_impl(gc, 44) def private iq4xs_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 44) +// k3: the 2-bit + hmask emitter arm lands with the kernel arc; until then both stamps decline +// (reference bodies serve) — a declined generator is the framework's own fallback path. +[unused_argument(gc)] +def private k3_gemv_gen(var gc : LlvmCodeCtx) : bool => false +[unused_argument(gc)] +def private k3_tile_gen(var gc : LlvmCodeCtx) : bool => false + def private k4_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 4) def private k5_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 5) def private k6_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 6) @@ -2621,5 +2628,7 @@ def public register_dasllama_gemm_generators { register_llvm_code_generator("dasllama_gemm_gen::q40_tile", @@q40_tile_gen) register_llvm_code_generator("dasllama_gemm_gen::iq4xs_gemv", @@iq4xs_gemv_gen) register_llvm_code_generator("dasllama_gemm_gen::iq4xs_tile", @@iq4xs_tile_gen) + register_llvm_code_generator("dasllama_gemm_gen::k3_gemv", @@k3_gemv_gen) + register_llvm_code_generator("dasllama_gemm_gen::k3_tile", @@k3_tile_gen) register_llvm_code_generator("dasllama_gemm_gen::q8q8_witness", @@witness_gen) } diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_schema.das b/modules/dasLLAMA/dasllama/dasllama_gemm_schema.das index b76cfa8b60..b4954f0c94 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_schema.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_schema.das @@ -58,7 +58,7 @@ let Q51_QPB = 20l let Q51_SPB = 4l //! Quant-plane bytes per 256-weight superblock per row for a kq format id (4/5/6 = Q4_K/Q5_K/ -//! Q6_K, 40 = Q4_0, 44 = IQ4_XS). The ONE stride source for every fmt-branched kq walker/ +//! Q6_K, 40 = Q4_0, 44 = IQ4_XS, 3 = Q3_K). The ONE stride source for every fmt-branched kq walker/ //! kernel/repack — an unknown id panics instead of silently walking another format's stride. def kq_qsb(fmt : int) : int64 { if (fmt == 4) return kq_qsb(KqFmt.k4) @@ -66,19 +66,21 @@ def kq_qsb(fmt : int) : int64 { if (fmt == 6) return kq_qsb(KqFmt.k6) if (fmt == 40) return kq_qsb(KqFmt.q40) if (fmt == 44) return kq_qsb(KqFmt.iq4xs) + if (fmt == 3) return kq_qsb(KqFmt.k3) panic("kq_qsb: unknown kq format id {fmt}") return 0l } //! Scale-plane bytes per 256-weight superblock per row (see kq_qsb): k4/k5 20 (16B disk block -//! + 4B pad, decoded in place at repack), k6 18 (native), q40 16 (8 x f16 d, verbatim), -//! iq4xs 20 (f16 d + 8 int8 sub-scales, decoded at transcode, in the k4 row shape). +//! + 4B pad, decoded at repack), k6 18 (native), q40 16 (8 x f16 d), iq4xs 20 (d + 8 int8 +//! sub-scales decoded at transcode, the k4 row shape), k3 18 (16 int8 decoded + d, the k6 row). def kq_ssb(fmt : int) : int64 { if (fmt == 4) return kq_ssb(KqFmt.k4) if (fmt == 5) return kq_ssb(KqFmt.k5) if (fmt == 6) return kq_ssb(KqFmt.k6) if (fmt == 40) return kq_ssb(KqFmt.q40) if (fmt == 44) return kq_ssb(KqFmt.iq4xs) + if (fmt == 3) return kq_ssb(KqFmt.k3) panic("kq_ssb: unknown kq format id {fmt}") return 0l } diff --git a/modules/dasLLAMA/dasllama/dasllama_gguf.das b/modules/dasLLAMA/dasllama/dasllama_gguf.das index af90b089f2..674f979c6a 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gguf.das +++ b/modules/dasLLAMA/dasllama/dasllama_gguf.das @@ -47,6 +47,7 @@ let GGML_TYPE_Q4_0 = 2 let GGML_TYPE_Q5_0 = 6 let GGML_TYPE_Q5_1 = 7 let GGML_TYPE_Q8_0 = 8 +let GGML_TYPE_Q3_K = 11 let GGML_TYPE_Q4_K = 12 let GGML_TYPE_Q5_K = 13 let GGML_TYPE_Q6_K = 14 @@ -745,6 +746,44 @@ def gguf_transcode_iq4xs(m : GGUFMeta; srcbytes : array | #; name : strin } } +//! Transcode a Q3_K tensor into the k3 planes (see gguf_transcode_q4k; strides 96/18, exact): the +//! 110B disk superblock splits into [qs][hmask] verbatim and the k6-shaped scale row with the 16 +//! packed 6-bit sub-scales DECODED to int8 (scale - 32) — transcode_q3k_superblock. +def gguf_transcode_q3k(m : GGUFMeta; srcbytes : array | #; name : string; var kq : array; var ks : array; eloff, expect_n : int64; src_off : int64 = 0l) { + let ti = kq_transcode_check(m, name, GGML_TYPE_Q3_K, "Q3_K", src_off, expect_n) + let nb = expect_n / 256l + if (nb <= 0l) { + return + } + guard_dst(name, "k3 quant plane", (eloff / 256l) * K3_QSB, nb * K3_QSB, long_length(kq)) + guard_dst(name, "k3 scale plane", (eloff / 256l) * K3_SSB, nb * K3_SSB, long_length(ks)) + with_tensor_view(m, srcbytes, ti) $(bytes, tbase) { + let bo = tbase + (src_off / 256l) * 110l + unsafe { + let srcp = addr(bytes[bo]) + var kqp = addr(kq[(eloff / 256l) * K3_QSB]) + var ksp = addr(ks[(eloff / 256l) * K3_SSB]) + maybe_parallel_for(0, int(nb), transcode_jobs(nb, 110l)) $(rb, re) { + unsafe { + for (sb in range64(int64(rb), int64(re))) { + let src = srcp + sb * 110l + var row = ksp + sb * K3_SSB + for (i in range64(16l)) { + let lo = i < 8l ? int(src[96l + i]) & 15 : int(src[96l + i - 8l]) >> 4 + let hi = int(src[104l + i % 4l]) >> int(2l * (i / 4l)) + row[i] = uint8((lo | ((hi & 3) << 4)) - 32) + } + row[16l] = src[108l] + row[17l] = src[109l] + bcopy(kqp + sb * K3_QSB, src + 32l, 64l) + bcopy(kqp + sb * K3_QSB + 64l, src, 32l) + } + } + } + } + } +} + //! Transcode a Q5_1 tensor into the q51 planes (per 32-BLOCK strides 20/4 — see Q51_QB): each //! 24B disk block splits into 2B d + 2B m (scale plane) and 16B nibbles + 4B qh (quant plane), //! all verbatim. Exact; `eloff`/`src_off`/`expect_n` are element offsets, % 32. diff --git a/modules/dasLLAMA/dasllama/dasllama_gpu_resident.das b/modules/dasLLAMA/dasllama/dasllama_gpu_resident.das index 652c4db8a0..f3400651db 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gpu_resident.das +++ b/modules/dasLLAMA/dasllama/dasllama_gpu_resident.das @@ -227,7 +227,7 @@ def private moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, row //! P3 trim (generate-then-free): pack the emb/cls region into embq/embs, mark the model //! trimmed, and FREE the big CPU weight families — the save then writes them as empty planes. //! Arena-only plans qualify (pure resident-driver dense); false = declined, nothing changed. -def trim_model_planes(var t : Model) : bool { +def trim_model_planes(var t : Model) : bool { // nolint:STYLE038 — the flat per-format plane-trim ladder (one arm per KqFmt) is the honest shape if (empty(t.vkplan) || t.config.n_layer_nextn > 0l) { return false } @@ -283,6 +283,11 @@ def trim_model_planes(var t : Model) : bool { t.embs |> reserve_resize(nsb * IQ4XS_SSB) memcpy(addr(t.embq[0]), addr(t.iq4xsq[sb0 * IQ4XS_QSB]), nsb * IQ4XS_QSB) memcpy(addr(t.embs[0]), addr(t.iq4xss[sb0 * IQ4XS_SSB]), nsb * IQ4XS_SSB) + } elif (t.emb_fmt == KqFmt.k3) { + t.embq |> reserve_resize(nsb * K3_QSB) + t.embs |> reserve_resize(nsb * K3_SSB) + memcpy(addr(t.embq[0]), addr(t.k3q[sb0 * K3_QSB]), nsb * K3_QSB) + memcpy(addr(t.embs[0]), addr(t.k3s[sb0 * K3_SSB]), nsb * K3_SSB) } else { to_log(LOG_WARNING, "dasLLAMA trim: emb fmt '{t.emb_fmt}' has no kq plane pair - declining\n") return false @@ -301,6 +306,10 @@ def trim_model_planes(var t : Model) : bool { delete t.k5s delete t.k6q delete t.k6s + delete t.iq4xsq + delete t.iq4xss + delete t.k3q + delete t.k3s delete t.q40q delete t.q40s delete t.q51q diff --git a/modules/dasLLAMA/dasllama/dasllama_image.das b/modules/dasLLAMA/dasllama/dasllama_image.das index 932201fd80..ebec2a5bfa 100644 --- a/modules/dasLLAMA/dasllama/dasllama_image.das +++ b/modules/dasLLAMA/dasllama/dasllama_image.das @@ -38,7 +38,7 @@ require dasllama/dasllama_load // WhisperModel.enc) contribute their planes under dotted names ("enc.fblob"); string-array // fields ride the meta blob via serialize_strings — raw string pointers can't be planes. -let IMAGE_VERSION = 18 // 18: the iq4xs plane pair (IQ4_XS native tier) +let IMAGE_VERSION = 19 // 19: the k3 plane pair (Q3_K native tier); 18: the iq4xs pair //! The metal (blob-only) flavor's identity tag: q8 planes ride the 34B block_q8_0 blob and the //! kq scale planes their GPU forms (convert_model_to_metal_blob) — flavors are per-config and @@ -635,6 +635,7 @@ def serialize_image_meta(var arch : Archive; var t : Model) { arch |> serialize_raw(t.kq_repack_mr6) arch |> serialize_raw(t.kq_repack_mr40) arch |> serialize_raw(t.kq_repack_mr44) + arch |> serialize_raw(t.kq_repack_mr3) arch |> serialize_raw(t.wcls_fmt) arch |> serialize_raw(t.emb_fmt) arch |> serialize_raw(t.ple_emb_fmt) @@ -674,7 +675,7 @@ def serialize_image_meta(var arch : Archive; var t : Model) { } // 67 serialized fields + 3 deliberate skips (blocks, image_map, image_bytes) -let IMAGE_META_FIELDS = 67 + 3 +let IMAGE_META_FIELDS = 68 + 3 //! Count of meta-carried fields of any struct: non-arrays plus string arrays (those cannot //! be raw planes — see serialize_strings). Pair with a per-type field-count constant to @@ -958,7 +959,7 @@ def save_model_image(var t : Model; path : string; tag : string = ""; quant : st // streamed planes size as empty, so their bytes plus a page each are added here; over-reserve is free (dwrite_close truncates) def private stream_extra_bytes(t : Model; jobs : array) : uint64 { var extra = 0ul - for (fname in ["qblob", "mblob", "k4q", "k5q", "k6q", "q40q", "q51q", "iq4xsq"]) { + for (fname in ["qblob", "mblob", "k4q", "k5q", "k6q", "q40q", "q51q", "iq4xsq", "k3q"]) { let sb = stream_plane_bytes(t, jobs, fname) if (sb > 0l) { extra += uint64(sb) + uint64(IMAGE_PAGE) diff --git a/modules/dasLLAMA/dasllama/dasllama_kqformat.das b/modules/dasLLAMA/dasllama/dasllama_kqformat.das index 813bfacaa0..a99f5b6f59 100644 --- a/modules/dasLLAMA/dasllama/dasllama_kqformat.das +++ b/modules/dasLLAMA/dasllama/dasllama_kqformat.das @@ -13,7 +13,7 @@ module dasllama_kqformat shared public require dasllama/dasllama_lint public //! Per-weight storage format under a q8-mode load with native planes: q8 = qblob/qscales; -//! k4/k5/k6/q40/iq4xs = superblock plane pairs (mixed PER TENSOR); q51 = native Q5_1 per-32 +//! k4/k5/k6/q40/iq4xs/k3 = superblock plane pairs (mixed PER TENSOR); q51 = native Q5_1 per-32 //! planes. New members append — the int value is the device stack tag and the image plane id. enum KqFmt : uint8 { q8 @@ -23,18 +23,19 @@ enum KqFmt : uint8 { q40 q51 iq4xs + k3 } //! The superblock-lattice formats (Q8_K-form activations, % 256 rows, repack + stamped kq //! kernels). q51 deliberately fails this: it rides per-32 planes with Q8_0-form activations — //! a `fmt != KqFmt.q8` test does not imply the kq lattice; branch on kq_sb where it does. -def kq_sb(f : KqFmt) : bool => f == KqFmt.k4 || f == KqFmt.k5 || f == KqFmt.k6 || f == KqFmt.q40 || f == KqFmt.iq4xs +def kq_sb(f : KqFmt) : bool => f == KqFmt.k4 || f == KqFmt.k5 || f == KqFmt.k6 || f == KqFmt.q40 || f == KqFmt.iq4xs || f == KqFmt.k3 //! The same predicate over the int id space the GPU drivers carry per stack/plane. -def kq_sb(fmt : int) : bool => fmt == int(KqFmt.k4) || fmt == int(KqFmt.k5) || fmt == int(KqFmt.k6) || fmt == int(KqFmt.q40) || fmt == int(KqFmt.iq4xs) +def kq_sb(fmt : int) : bool => fmt == int(KqFmt.k4) || fmt == int(KqFmt.k5) || fmt == int(KqFmt.k6) || fmt == int(KqFmt.q40) || fmt == int(KqFmt.iq4xs) || fmt == int(KqFmt.k3) //! KqFmt -> the kernel/IR format id space — the ONE bridge between the enum and the int ids the -//! generated kernels take as runtime params. Mnemonic ids: a K-quant is its bit width (4/5/6), +//! generated kernels take as runtime params. Mnemonic ids: a K-quant is its bit width (3/4/5/6), //! Q4_0 is 40, an i-quant is bit width x 10 + a variant digit (IQ4_XS = 44). q8/q51 panic. def kq_schema_id(f : KqFmt) : int { if (f == KqFmt.k4) { @@ -52,6 +53,9 @@ def kq_schema_id(f : KqFmt) : int { if (f == KqFmt.iq4xs) { return 44 } + if (f == KqFmt.k3) { + return 3 + } panic("kq_schema_id: not a kq superblock format") return 0 } @@ -70,6 +74,8 @@ let Q51_QB = 20l // q5_1 quant plane PER 32-BLOCK: 16 nibble bytes (k/k+16 pa let Q51_SB = 4l // q5_1 scale plane PER 32-BLOCK: f16 d + f16 m let IQ4XS_QSB = 128l // IQ4_XS quant plane: 128 nibble bytes, the disk k/k+16 pairing (q40 tiling) let IQ4XS_SSB = 20l // IQ4_XS scale plane: f16 d, 2 pad, 8 x int8 (ls - 32), 8 pad — the k4 decoded-row shape +let K3_QSB = 96l // Q3_K quant plane: 64 qs bytes (four 2-bit lanes each) + 32 hmask bytes, verbatim disk order +let K3_SSB = 18l // Q3_K scale plane: 16 x int8 (6-bit scale - 32, decoded at transcode) + f16 d — the k6 row //! The IQ4_NL / IQ4_XS nibble codebook: weight = scale x IQ4NL_LUT[nibble] (ggml's kvalues_iq4nl). //! Main-context use only (tests, oracles, the emitter's constant bake): a kernel that can run on a @@ -96,6 +102,9 @@ def kq_qsb(f : KqFmt) : int64 { if (f == KqFmt.q51) { return Q51_QB } + if (f == KqFmt.k3) { + return K3_QSB + } panic("kq_qsb: no plane stride for this format") return 0l } @@ -105,7 +114,7 @@ def kq_ssb(f : KqFmt) : int64 { if (f == KqFmt.k4 || f == KqFmt.k5 || f == KqFmt.iq4xs) { return K4_SSB } - if (f == KqFmt.k6) { + if (f == KqFmt.k6 || f == KqFmt.k3) { return K6_SSB } if (f == KqFmt.q40) { diff --git a/modules/dasLLAMA/dasllama/dasllama_layout.das b/modules/dasLLAMA/dasllama/dasllama_layout.das index 5741059610..0eda6b4aab 100644 --- a/modules/dasLLAMA/dasllama/dasllama_layout.das +++ b/modules/dasLLAMA/dasllama/dasllama_layout.das @@ -315,6 +315,10 @@ def private repack_regions(var t : Model; regs : array) { var iq4xssp : uint8? = null if (!empty(t.iq4xsq)) { iq4xsqp = addr(t.iq4xsq[0]) } if (!empty(t.iq4xss)) { iq4xssp = addr(t.iq4xss[0]) } + var k3qp : uint8? = null + var k3sp : uint8? = null + if (!empty(t.k3q)) { k3qp = addr(t.k3q[0]) } + if (!empty(t.k3s)) { k3sp = addr(t.k3s[0]) } if (!empty(t.qblob)) { qbp = addr(t.qblob[0]) } if (!empty(t.qscales)) { qsp = addr(t.qscales[0]) } if (!empty(t.mxq)) { mxqp = addr(t.mxq[0]) } @@ -346,8 +350,8 @@ def private repack_regions(var t : Model; regs : array) { let sb = rp[i].off / 256l let qsb = kq_qsb(f) let ssb = kq_ssb(f) - var kqp = f == 4 ? k4qp : (f == 5 ? k5qp : (f == 6 ? k6qp : (f == 40 ? q40qp : iq4xsqp))) - var ksp = f == 4 ? k4sp : (f == 5 ? k5sp : (f == 6 ? k6sp : (f == 40 ? q40sp : iq4xssp))) + var kqp = f == 4 ? k4qp : (f == 5 ? k5qp : (f == 6 ? k6qp : (f == 40 ? q40qp : (f == 44 ? iq4xsqp : k3qp)))) + var ksp = f == 4 ? k4sp : (f == 5 ? k5sp : (f == 6 ? k6sp : (f == 40 ? q40sp : (f == 44 ? iq4xssp : k3sp)))) invoke(rkq, f, kqp + sb * qsb, ksp + sb * ssb, rp[i].n, rp[i].d) } } @@ -475,6 +479,8 @@ def private push_repack_kq(var regs : array; fmt : KqFmt; woff, n, d push_repack(regs, 40, woff, n, d) } elif (fmt == KqFmt.iq4xs) { push_repack(regs, 44, woff, n, d) + } elif (fmt == KqFmt.k3) { + push_repack(regs, 3, woff, n, d) } } @@ -748,11 +754,13 @@ def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice let qp = (fmt == KqFmt.k4 ? addr(t.k4q[sb0 * kq_qsb(4)]) : (fmt == KqFmt.k5 ? addr(t.k5q[sb0 * kq_qsb(5)]) : (fmt == KqFmt.k6 ? addr(t.k6q[sb0 * kq_qsb(6)]) - : (fmt == KqFmt.q40 ? addr(t.q40q[sb0 * kq_qsb(40)]) : addr(t.iq4xsq[sb0 * kq_qsb(44)]))))) + : (fmt == KqFmt.q40 ? addr(t.q40q[sb0 * kq_qsb(40)]) + : (fmt == KqFmt.iq4xs ? addr(t.iq4xsq[sb0 * kq_qsb(44)]) : addr(t.k3q[sb0 * kq_qsb(3)])))))) let sp = (fmt == KqFmt.k4 ? addr(t.k4s[sb0 * kq_ssb(4)]) : (fmt == KqFmt.k5 ? addr(t.k5s[sb0 * kq_ssb(5)]) : (fmt == KqFmt.k6 ? addr(t.k6s[sb0 * kq_ssb(6)]) - : (fmt == KqFmt.q40 ? addr(t.q40s[sb0 * kq_ssb(40)]) : addr(t.iq4xss[sb0 * kq_ssb(44)]))))) + : (fmt == KqFmt.q40 ? addr(t.q40s[sb0 * kq_ssb(40)]) + : (fmt == KqFmt.iq4xs ? addr(t.iq4xss[sb0 * kq_ssb(44)]) : addr(t.k3s[sb0 * kq_ssb(3)])))))) var wqp = addr(wq[0]) var wsp = addr(ws[0]) let njobs = is_job_que_available() ? min(nslices, 4 * (get_total_hw_jobs() + 1)) : 1 diff --git a/modules/dasLLAMA/dasllama/dasllama_load.das b/modules/dasLLAMA/dasllama/dasllama_load.das index 834afc8d9e..9f0c149d58 100644 --- a/modules/dasLLAMA/dasllama/dasllama_load.das +++ b/modules/dasLLAMA/dasllama/dasllama_load.das @@ -42,6 +42,7 @@ struct private LayoutSizes { q40_n : int64 q51_n : int64 iq4xs_n : int64 + k3_n : int64 pleq8_n : int64 // the dedicated q8 plane holding ONLY the PLE table (fp32/q4 serving) } @@ -55,6 +56,7 @@ struct private KqCursors { q40 : int64 q51 : int64 iq4xs : int64 + k3 : int64 } def private kq_take(var cur : KqCursors; f : KqFmt; n : int64) : int64 { @@ -88,6 +90,11 @@ def private kq_take(var cur : KqCursors; f : KqFmt; n : int64) : int64 { cur.iq4xs += n return o } + if (f == KqFmt.k3) { + let o = cur.k3 + cur.k3 += n + return o + } let o = cur.wo cur.wo += n return o @@ -387,7 +394,7 @@ def private layout_offsets(var t : Model) : LayoutSizes { // nolint:STYLE037,S fo += dim } return LayoutSizes(fblob_n = fo, wblob_n = cur.wo, mx_n = mx, bf16_n = bf16, - k4_n = cur.k4, k5_n = cur.k5, k6_n = cur.k6, q40_n = cur.q40, q51_n = cur.q51, iq4xs_n = cur.iq4xs, pleq8_n = pleq8) + k4_n = cur.k4, k5_n = cur.k5, k6_n = cur.k6, q40_n = cur.q40, q51_n = cur.q51, iq4xs_n = cur.iq4xs, k3_n = cur.k3, pleq8_n = pleq8) } @@ -648,6 +655,8 @@ def stream_field_of(t : Model; fmt : KqFmt) : string { return "q51q" } elif (fmt == KqFmt.iq4xs) { return "iq4xsq" + } elif (fmt == KqFmt.k3) { + return "k3q" } elif (t.quant == QuantMode.q8) { // the blob flavor carries q8 as gguf-native 34B blocks in ONE plane, never the split pair return t.metal_blob ? "mblob" : "qblob" @@ -766,6 +775,8 @@ def fill_stream_plane(m : GGUFMeta; bytes : array | #; var t : Model; fie memcpy(addr(t.q40s[(j.woff / 256l) * Q40_SSB]), addr(temp_s[0]), sh.sb) } elif (j.fmt == KqFmt.iq4xs) { memcpy(addr(t.iq4xss[(j.woff / 256l) * IQ4XS_SSB]), addr(temp_s[0]), sh.sb) + } elif (j.fmt == KqFmt.k3) { + memcpy(addr(t.k3s[(j.woff / 256l) * K3_SSB]), addr(temp_s[0]), sh.sb) } elif (j.fmt == KqFmt.q51) { memcpy(addr(t.q51s[(j.woff / 32l) * Q51_SB]), addr(temp_s[0]), sh.sb) } elif (sh.sb == 0l && j.n > 0l && !t.metal_blob) { @@ -800,6 +811,9 @@ def fill_stream_plane(m : GGUFMeta; bytes : array | #; var t : Model; fie } elif (r.fmt == 44) { invoke(g_stream_repack, 44, addr(temp_q[((r.off - j.woff) / 256l) * IQ4XS_QSB]), addr(t.iq4xss[(r.off / 256l) * IQ4XS_SSB]), r.n, r.d) + } elif (r.fmt == 3) { + invoke(g_stream_repack, 3, addr(temp_q[((r.off - j.woff) / 256l) * K3_QSB]), + addr(t.k3s[(r.off / 256l) * K3_SSB]), r.n, r.d) } } } @@ -1015,6 +1029,9 @@ def private transcode_kq_tensor(m : GGUFMeta; bytes : array | #; name : s } elif (fmt == KqFmt.iq4xs) { gguf_transcode_iq4xs(m, bytes, name, kq, ks, eloff, n, src_off) return "iq4xs transcode (IQ4_XS)" + } elif (fmt == KqFmt.k3) { + gguf_transcode_q3k(m, bytes, name, kq, ks, eloff, n, src_off) + return "k3 transcode (Q3_K)" } gguf_transcode_q51(m, bytes, name, kq, ks, eloff, n, src_off) return "q51 transcode (Q5_1)" @@ -1055,6 +1072,8 @@ def private load_big(m : GGUFMeta; bytes : array | #; name : string; var kind = transcode_kq_tensor(m, bytes, name, fmt, t.q40q, t.q40s, woff, n, src_off, scratch) } elif (fmt == KqFmt.iq4xs) { kind = transcode_kq_tensor(m, bytes, name, fmt, t.iq4xsq, t.iq4xss, woff, n, src_off, scratch) + } elif (fmt == KqFmt.k3) { + kind = transcode_kq_tensor(m, bytes, name, fmt, t.k3q, t.k3s, woff, n, src_off, scratch) } elif (fmt == KqFmt.q51) { kind = transcode_kq_tensor(m, bytes, name, fmt, t.q51q, t.q51s, woff, n, src_off, scratch) } elif (t.quant == QuantMode.q8) { @@ -1154,8 +1173,8 @@ def private log_load_report(t : Model) { to_log(LOG_INFO, "dasLLAMA noisy: backend '{active_kernel_backend()}' | kquant_native {get_kquant_native()} q40_native {get_kq_q40_native()} q50_native {get_kq_q50_native()}\n") // the tier arms at [init], before a --noisy CLI flag can land — restate it here so the flag alone suffices to_log(LOG_INFO, "dasLLAMA noisy: GPU tier installed {moe_gpu_tier_installed()} | want auto {gpu_want_auto()}, moe layers {gpu_want_moe_layers()}, stream {gpu_want_moe_stream()}\n") - to_log(LOG_INFO, "dasLLAMA noisy: format tags - q8 {tally_fmt(t, KqFmt.q8)}, k4 {tally_fmt(t, KqFmt.k4)}, k5 {tally_fmt(t, KqFmt.k5)}, k6 {tally_fmt(t, KqFmt.k6)}, q40 {tally_fmt(t, KqFmt.q40)}, q51 {tally_fmt(t, KqFmt.q51)}, iq4xs {tally_fmt(t, KqFmt.iq4xs)}\n") - to_log(LOG_INFO, "dasLLAMA noisy: plane MB - q8 {float(long_length(t.qblob)) / mb} (+scales {float(long_length(t.qscales) * 4l) / mb}), k4 {float(long_length(t.k4q) + long_length(t.k4s)) / mb}, k5 {float(long_length(t.k5q) + long_length(t.k5s)) / mb}, k6 {float(long_length(t.k6q) + long_length(t.k6s)) / mb}, q40 {float(long_length(t.q40q) + long_length(t.q40s)) / mb}, q51 {float(long_length(t.q51q) + long_length(t.q51s)) / mb}, iq4xs {float(long_length(t.iq4xsq) + long_length(t.iq4xss)) / mb}, f32 {float(long_length(t.wblob) * 4l) / mb}\n") + to_log(LOG_INFO, "dasLLAMA noisy: format tags - q8 {tally_fmt(t, KqFmt.q8)}, k4 {tally_fmt(t, KqFmt.k4)}, k5 {tally_fmt(t, KqFmt.k5)}, k6 {tally_fmt(t, KqFmt.k6)}, q40 {tally_fmt(t, KqFmt.q40)}, q51 {tally_fmt(t, KqFmt.q51)}, iq4xs {tally_fmt(t, KqFmt.iq4xs)}, k3 {tally_fmt(t, KqFmt.k3)}\n") + to_log(LOG_INFO, "dasLLAMA noisy: plane MB - q8 {float(long_length(t.qblob)) / mb} (+scales {float(long_length(t.qscales) * 4l) / mb}), k4 {float(long_length(t.k4q) + long_length(t.k4s)) / mb}, k5 {float(long_length(t.k5q) + long_length(t.k5s)) / mb}, k6 {float(long_length(t.k6q) + long_length(t.k6s)) / mb}, q40 {float(long_length(t.q40q) + long_length(t.q40s)) / mb}, q51 {float(long_length(t.q51q) + long_length(t.q51s)) / mb}, iq4xs {float(long_length(t.iq4xsq) + long_length(t.iq4xss)) / mb}, k3 {float(long_length(t.k3q) + long_length(t.k3s)) / mb}, f32 {float(long_length(t.wblob) * 4l) / mb}\n") } // GGML disk type -> plane format tag (non-K-quant types ride the classic q8 path) @@ -1172,6 +1191,9 @@ def private kq_fmt_of(gt : int) : KqFmt { if (gt == GGML_TYPE_IQ4_XS) { return KqFmt.iq4xs } + if (gt == GGML_TYPE_Q3_K) { + return KqFmt.k3 + } if (gt == GGML_TYPE_Q4_0 && get_kq_q40_native()) { return KqFmt.q40 } @@ -2006,6 +2028,7 @@ def private load_gguf_parsed(var t : Model; m : GGUFMeta; bytes : array | g_stream_plane_total["q40q"] = (sz.q40_n / 256l) * Q40_QSB g_stream_plane_total["q51q"] = (sz.q51_n / 32l) * Q51_QB g_stream_plane_total["iq4xsq"] = (sz.iq4xs_n / 256l) * IQ4XS_QSB + g_stream_plane_total["k3q"] = (sz.k3_n / 256l) * K3_QSB let wb = mode == QuantMode.q8 ? sz.wblob_n : 0l g_stream_plane_total[t.metal_blob ? "mblob" : "qblob"] = (t.metal_blob ? (wb / Q8_BLOCK_ELEMS) * (Q8_QPB + Q8_SPB) : wb) @@ -2058,6 +2081,14 @@ def private load_gguf_parsed(var t : Model; m : GGUFMeta; bytes : array | t.iq4xss |> reserve((sz.iq4xs_n / 256l) * IQ4XS_SSB) t.iq4xss |> resize((sz.iq4xs_n / 256l) * IQ4XS_SSB) } + if (sz.k3_n > 0l) { + if (!stream_q) { + t.k3q |> reserve((sz.k3_n / 256l) * K3_QSB) + t.k3q |> resize((sz.k3_n / 256l) * K3_QSB) + } + t.k3s |> reserve((sz.k3_n / 256l) * K3_SSB) + t.k3s |> resize((sz.k3_n / 256l) * K3_SSB) + } if (mode == QuantMode.q8) { if (!stream_q) { t.qblob |> reserve(sz.wblob_n) @@ -2365,6 +2396,7 @@ def private load_gguf_parsed(var t : Model; m : GGUFMeta; bytes : array | t.kq_repack_mr6 = active_kq_layout_mr(6) t.kq_repack_mr40 = active_kq_layout_mr(40) t.kq_repack_mr44 = active_kq_layout_mr(44) + t.kq_repack_mr3 = active_kq_layout_mr(3) } } else { layout_repack_q8(t) @@ -2382,7 +2414,8 @@ def private load_gguf_parsed(var t : Model; m : GGUFMeta; bytes : array | t.kq_repack_mr6 = active_kq_layout_mr(6) t.kq_repack_mr40 = active_kq_layout_mr(40) t.kq_repack_mr44 = active_kq_layout_mr(44) - to_log(LOG_INFO, "dasLLAMA: K-quant planes repacked to the '{active_kernel_backend()}' layouts (k4 grp{t.kq_repack_mr4}, k5 grp{t.kq_repack_mr5}, k6 grp{t.kq_repack_mr6}, q40 grp{t.kq_repack_mr40}, iq4xs grp{t.kq_repack_mr44})\n") + t.kq_repack_mr3 = active_kq_layout_mr(3) + to_log(LOG_INFO, "dasLLAMA: K-quant planes repacked to the '{active_kernel_backend()}' layouts (k4 grp{t.kq_repack_mr4}, k5 grp{t.kq_repack_mr5}, k6 grp{t.kq_repack_mr6}, q40 grp{t.kq_repack_mr40}, iq4xs grp{t.kq_repack_mr44}, k3 grp{t.kq_repack_mr3})\n") } } } diff --git a/modules/dasLLAMA/dasllama/dasllama_math.das b/modules/dasLLAMA/dasllama/dasllama_math.das index 77384d6c18..58b3699f84 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math.das +++ b/modules/dasLLAMA/dasllama/dasllama_math.das @@ -1085,6 +1085,7 @@ struct KernelBackend { kq_rows_k6 : MatmulKqRowsFn = @@kq_unset_rows kq_rows_q40 : MatmulKqRowsFn = @@kq_unset_rows kq_rows_iq4xs : MatmulKqRowsFn = @@kq_unset_rows + kq_rows_k3 : MatmulKqRowsFn = @@kq_unset_rows kq_batch : MatmulKqBatchFn = @@kq_unset_batch kq_groupn : MatmulKqGroupNFn = @@kq_unset_groupn kq_batch_groupn : MatmulKqBatchGroupNFn = @@kq_unset_batch_groupn @@ -1148,6 +1149,7 @@ var g_kq_rows_k5 = @@kq_unset_rows var g_kq_rows_k6 = @@kq_unset_rows var g_kq_rows_q40 = @@kq_unset_rows var g_kq_rows_iq4xs = @@kq_unset_rows +var g_kq_rows_k3 = @@kq_unset_rows var g_mm_kq_batch = @@kq_unset_batch var g_mm_kq_groupn = @@kq_unset_groupn var g_mm_kq_batch_groupn = @@kq_unset_batch_groupn @@ -1475,7 +1477,7 @@ def active_q8_layout_mr() : int64 => g_bake_cpu_override ? g_bake_cpu.q8_mr : g_ //! family split; the loader freezes these onto the Model at repack time. def active_kq_layout_mr(fmt : int) : int64 { if (g_bake_cpu_override) { - return fmt == 4 ? g_bake_cpu.kq_mr4 : (fmt == 5 ? g_bake_cpu.kq_mr5 : (fmt == 40 ? g_bake_cpu.kq_mr40 : (fmt == 44 ? g_bake_cpu.kq_mr44 : g_bake_cpu.kq_mr6))) + return fmt == 4 ? g_bake_cpu.kq_mr4 : (fmt == 5 ? g_bake_cpu.kq_mr5 : (fmt == 40 ? g_bake_cpu.kq_mr40 : (fmt == 44 ? g_bake_cpu.kq_mr44 : (fmt == 3 ? g_bake_cpu.kq_mr3 : g_bake_cpu.kq_mr6)))) } return int64(invoke(g_active_kq_layout, fmt)) } @@ -1652,6 +1654,7 @@ def private activate(be : KernelBackend) { g_kq_rows_k6 = be.kq_rows_k6 g_kq_rows_q40 = be.kq_rows_q40 g_kq_rows_iq4xs = be.kq_rows_iq4xs + g_kq_rows_k3 = be.kq_rows_k3 g_mm_kq_batch = be.kq_batch g_mm_kq_groupn = be.kq_groupn g_mm_kq_batch_groupn = be.kq_batch_groupn @@ -1855,6 +1858,7 @@ def kq_rows_fn(fmt : int) : MatmulKqRowsFn { if (fmt == 6) return g_kq_rows_k6 if (fmt == 40) return g_kq_rows_q40 if (fmt == 44) return g_kq_rows_iq4xs + if (fmt == 3) return g_kq_rows_k3 return @@kq_unset_rows } @@ -3218,6 +3222,9 @@ def private restore_math_fn_defaults { // nolint:STYLE037,STYLE038 — flat seam if (g_kq_rows_iq4xs == null) { g_kq_rows_iq4xs = @@kq_unset_rows } + if (g_kq_rows_k3 == null) { + g_kq_rows_k3 = @@kq_unset_rows + } if (g_mm_kq_batch == null) { g_mm_kq_batch = @@kq_unset_batch } diff --git a/modules/dasLLAMA/dasllama/dasllama_math_default.das b/modules/dasLLAMA/dasllama/dasllama_math_default.das index ede5add94c..b1fbb8e218 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_default.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_default.das @@ -576,6 +576,46 @@ def dot_iq4xs_q8(kqrow : uint8 const?; ksrow : uint8 const?; xqp : int8 const?; return acc } +//! Q3_K row dot off the DISK-ORDER planes: block b = 4h + j takes the 2-bit lane of qs byte +//! h*32 + l at shift 2j plus hmask bit b of byte l (q in 0..7); k6's fold with the -4 offset: +//! isum = sum sc16 * dot, bsum = sum sc16 * sum a16, then (isum - 4 * bsum) * d * xs per superblock. +[hint(unsafe_range_check, noalias = kqrow, noalias = ksrow, noalias = xqp, noalias = xsp, noalias = xbsp)] +def dot_k3q8(kqrow : uint8 const?; ksrow : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n : int64) : float { + var acc = 0.0 + unsafe { + for (sb in range64(n / 256l)) { + let kqo = sb * 96l + let kso = sb * 18l + let d = f16_to_f32(uint(ksrow[kso + 16l]) | (uint(ksrow[kso + 17l]) << 8u)) + var isum = 0 + var bsum = 0 + for (blk in range64(8l)) { + let qb = kqo + (blk / 4l) * 32l + let hb = kqo + 64l + let sh = int((blk % 4l) * 2l) + let b = sb * 8l + blk + let ab = b * 32l + var i0 = 0 + var i1 = 0 + for (l in range64(16l)) { + let q0 = ((int(kqrow[qb + l]) >> sh) & 3) | (((int(kqrow[hb + l]) >> int(blk)) & 1) << 2) + let q1 = ((int(kqrow[qb + 16l + l]) >> sh) & 3) | (((int(kqrow[hb + 16l + l]) >> int(blk)) & 1) << 2) + i0 += q0 * int(xqp[ab + l]) + i1 += q1 * int(xqp[ab + 16l + l]) + } + let u0 = int(ksrow[kso + 2l * blk]) + let u1 = int(ksrow[kso + 2l * blk + 1l]) + let s0 = u0 < 128 ? u0 : u0 - 256 + let s1 = u1 < 128 ? u1 : u1 - 256 + isum += s0 * i0 + s1 * i1 + bsum += s0 * xbsp[b * 2l] + s1 * xbsp[b * 2l + 1l] + } + acc += float(isum - 4 * bsum) * (d * xsp[sb]) + } + } + return acc +} + // Row-range cores + full GEMVs per format. Not `private`: invoked through hoisted function // pointers from lifted worker lambdas (the fused chains) and the dispatch wrappers in common. def k4_rows_kernel(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, rb, re : int64) { @@ -623,6 +663,15 @@ def iq4xs_rows_kernel(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; x } } +def k3_rows_kernel(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, rb, re : int64) { + let nsb = n / 256l + unsafe { + for (i in range64(rb, re)) { + yp[i] = dot_k3q8(kqp + i * nsb * 96l, ksp + i * nsb * 18l, xqp, xsp, xbsp, n) + } + } +} + def private kq_gemv_kernel(fmt : int; var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d : int64) { var myp = yp maybe_parallel_for(0, int(d), matmul_chunks_gemv(int(d), 1, n * d)) $(rb, re) { @@ -634,6 +683,8 @@ def private kq_gemv_kernel(fmt : int; var yp : float?; kqp : uint8 const?; ksp : q40_rows_kernel(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb), int64(re)) } elif (fmt == 44) { iq4xs_rows_kernel(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb), int64(re)) + } elif (fmt == 3) { + k3_rows_kernel(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb), int64(re)) } else { k6_rows_kernel(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb), int64(re)) } @@ -686,6 +737,31 @@ def private dequant_iq4xs_row_grp(kqg : uint8 const?; ksg : uint8 const?; r, mr, } } +//! k3 grp row dequant: the 2-bit lanes and hmask bits stay in place per (h, column) — w = (d * sc16) * (q - 4). +def private dequant_k3_row_grp(kqg : uint8 const?; ksg : uint8 const?; r, mr, nsb : int64; var dst : float?) { + unsafe { + for (sbi in range64(nsb)) { + let qb = sbi * 96l * mr + let sb = sbi * 18l * mr + let d = f16_to_f32(uint(ksg[sb + 16l * mr + 2l * r]) | (uint(ksg[sb + 16l * mr + 2l * r + 1l]) << 8u)) + for (blk in range64(8l)) { + let kb = sbi * 256l + blk * 32l + let h = blk / 4l + let sh = int((blk % 4l) * 2l) + let s0 = d * float(int(int8(ksg[sb + 2l * blk * mr + r]))) + let s1 = d * float(int(int8(ksg[sb + (2l * blk + 1l) * mr + r]))) + for (l4 in range64(8l)) { + for (t in range64(4l)) { + let q2 = (int(kqg[qb + ((h * 8l + l4) * mr + r) * 4l + t]) >> sh) & 3 + let hb = (int(kqg[qb + 64l * mr + (l4 * mr + r) * 4l + t]) >> int(blk)) & 1 + dst[kb + l4 * 4l + t] = (l4 < 4l ? s0 : s1) * float((q2 | (hb << 2)) - 4) + } + } + } + } + } +} + //! Dequant one row off the grp-REPACKED K-quant planes (repack_k4/k5/k6_grp layout). //! kqg/ksg = row's GROUP plane base (group g = row/mr); r = row % mr. //! embed_row's kq_repacked path — per-token, so scalar is fine. @@ -701,6 +777,10 @@ def dequant_kq_row_grp(fmt : int64; kqg : uint8 const?; ksg : uint8 const?; r, m dequant_iq4xs_row_grp(kqg, ksg, r, mr, nsb, dst) return } + if (fmt == 3l) { + dequant_k3_row_grp(kqg, ksg, r, mr, nsb, dst) + return + } unsafe { for (sbi in range64(nsb)) { let qb = sbi * qsb * mr @@ -797,6 +877,8 @@ def matmul_kq_groupn(fmt : int; var y : array | #; kq : array | #; myp[ii] = dot_q40q8(kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) } elif (fmt == 44) { myp[ii] = dot_iq4xs_q8(kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) + } elif (fmt == 3) { + myp[ii] = dot_k3q8(kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) } else { myp[ii] = dot_k6q8(kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) } diff --git a/modules/dasLLAMA/dasllama/dasllama_math_gen.das b/modules/dasLLAMA/dasllama/dasllama_math_gen.das index 755b70252f..f3a706a3dd 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_gen.das @@ -247,6 +247,10 @@ def iq4xsq8_layout_gen() : int { return q8q8_repack_type(GEMM_REFERENCE_MR).interleave } +def k3q8_layout_gen() : int { + return q8q8_repack_type(GEMM_REFERENCE_MR).interleave +} + // the KernelBackend.kq_layout slot form of kq_layout_of (registered on both gen backends) def private kq_layout_fmt_gen(fmt : int) : int => int(kq_layout_of(fmt)) @@ -265,6 +269,9 @@ def kq_layout_of(fmt : int) : int64 { if (fmt == 44) { return int64(iq4xsq8_layout_gen()) } + if (fmt == 3) { + return int64(k3q8_layout_gen()) + } return int64(k6q8_layout_gen()) } @@ -333,6 +340,48 @@ def iq4xs_grp_row_dot(kqg : uint8 const?; ksg : uint8 const?; r, mr : int64; xqp return acc } +//! One row's dot off the grp k3 planes, scalar — the k3 stubs' reference body and the repack +//! oracle: k6's fold (per-16 signed sub-scales, the -4 offset through the activation sums) over +//! the 2-bit lane + hmask compose. Same float fold order as dot_k3q8. +def k3_grp_row_dot(kqg : uint8 const?; ksg : uint8 const?; r, mr : int64; xqp : int8 const?; xsp : float const?; xbsp : int const?; n : int64) : float { + var acc = 0.0 + let nsb = n / 256l + unsafe { + for (sbi in range64(nsb)) { + let qb = sbi * 96l * mr + let sb = sbi * 18l * mr + var isum = 0 + var bsum = 0 + for (blk in range64(8l)) { + let b = sbi * 8l + blk + let h = blk / 4l + let sh = int((blk % 4l) * 2l) + var ilo = 0 + var ihi = 0 + for (l4 in range64(8l)) { + for (t in range64(4l)) { + let q2 = (int(kqg[qb + ((h * 8l + l4) * mr + r) * 4l + t]) >> sh) & 3 + let hb = (int(kqg[qb + 64l * mr + (l4 * mr + r) * 4l + t]) >> int(blk)) & 1 + let q = q2 | (hb << 2) + if (l4 < 4l) { + ilo += q * int(xqp[b * 32l + l4 * 4l + t]) + } else { + ihi += q * int(xqp[b * 32l + l4 * 4l + t]) + } + } + } + let s0 = int(int8(ksg[sb + 2l * blk * mr + r])) + let s1 = int(int8(ksg[sb + (2l * blk + 1l) * mr + r])) + isum += s0 * ilo + s1 * ihi + bsum += s0 * xbsp[b * 2l] + s1 * xbsp[b * 2l + 1l] + } + let dk = f16_to_f32(uint(ksg[sb + 16l * mr + 2l * r]) | (uint(ksg[sb + 16l * mr + 2l * r + 1l]) << 8u)) + acc += float(isum - 4 * bsum) * (dk * xsp[sbi]) + } + } + return acc +} + //! The K-quant GEMV kernels (kq stage 4): rows [rb, re) of one plane-region pair off the grp //! kq planes (each format's OWN layout companion). Reference bodies = the scalar grp walk above, //! declining in lockstep. Not `private`: they ARE the kq_rows_* slots. @@ -503,6 +552,38 @@ def iq4xsq8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xq } } +[hint(unsafe_range_check, noalias = kqp, noalias = ksp, noalias = xqp, noalias = xsp, noalias = xbsp)] +def k3q8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, rb, re : int64) : void { + let mr = int64(k3q8_layout_gen()) + let nsb = n / 256l + unsafe { + for (i in range64(rb, re)) { + let g = i / mr + yp[i] = k3_grp_row_dot(kqp + g * mr * nsb * 96l, ksp + g * mr * nsb * 18l, i % mr, mr, xqp, xsp, xbsp, n) + } + } +} + +[tune_perm(mr = 4), tune_perm(mr = 4, nrsplit = 2), tune_perm(mr = 8), tune_perm(mr = 8, nrsplit = 2), + tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "maddubs", width = 256, mr = 8, nrsplit = 2, requires = "avx2"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, nrsplit = 2, requires = "avxvnni|avx512vnni"), + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, nrsplit = 2, requires = "avx512vnni,avx512bw"), + tune_companion(fn = "k3q8_gemv_gen", gen = "dasllama_gemm_gen::k3_gemv"), + tune_companion(fn = "k3q8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), + tune(gen = "dasllama_gemm_gen::k3_tile", + fallback = "dot_vpdpbusd_width512_mr16;dot_vpdpbusd_width256_mr8;dot_maddubs_width256_mr8;mr4"), + hint(unsafe_range_check, noalias = kqg, noalias = ksg, noalias = xqp, noalias = xsp, noalias = xbsp)] +def k3q8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d, g, t0 : int64) : void { + let mr = int64(k3q8_layout_gen()) + unsafe { + for (t in range64(4l)) { + for (r in range64(mr)) { + yp[(t0 + t) * d + g * mr + r] = k3_grp_row_dot(kqg, ksg, r, mr, xqp + (t0 + t) * n, xsp + (t0 + t) * (n / 256l), xbsp + (t0 + t) * (n / 16l), n) + } + } + } +} + //! The f16-scale GEMV companion (wscale_f16 rail): q8q8_gemv_gen's twin over the raw binary16 //! group-scale plane. Same rows-range contract; the generated body differs only in the scale //! fold's widen (fcvtl/vcvtph2ps). Not `private`: it IS the mm_rows_s16 slot. @@ -822,6 +903,8 @@ def private repack_kq_gen(fmt : int; var kq : uint8?; var ks : uint8?; n, d : in repack_q40_grp(kq, ks, n, d, kq_layout_of(40)) } elif (fmt == 44) { repack_iq4xs_grp(kq, ks, n, d, kq_layout_of(44)) + } elif (fmt == 3) { + repack_k3_grp(kq, ks, n, d, kq_layout_of(3)) } else { repack_k6_grp(kq, ks, n, d, kq_layout_of(6)) } @@ -836,6 +919,8 @@ def private repack_kq_bake(fmt : int; var kq : uint8?; var ks : uint8?; n, d : i repack_q40_grp(kq, ks, n, d, active_kq_layout_mr(40)) } elif (fmt == 44) { repack_iq4xs_grp(kq, ks, n, d, active_kq_layout_mr(44)) + } elif (fmt == 3) { + repack_k3_grp(kq, ks, n, d, active_kq_layout_mr(3)) } else { repack_k6_grp(kq, ks, n, d, active_kq_layout_mr(6)) } @@ -1348,6 +1433,8 @@ def private kq_kernel_gen(fmt : int; var yp : float?; kqp : uint8 const?; ksp : q40q8_gemv_gen(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb) * mr, int64(re) * mr) } elif (fmt == 44) { iq4xsq8_gemv_gen(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb) * mr, int64(re) * mr) + } elif (fmt == 3) { + k3q8_gemv_gen(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb) * mr, int64(re) * mr) } else { k6q8_gemv_gen(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb) * mr, int64(re) * mr) } @@ -1362,6 +1449,8 @@ def private kq_kernel_gen(fmt : int; var yp : float?; kqp : uint8 const?; ksp : myp[i] = dot_q40q8(kqp + i * nsb * 128l, ksp + i * nsb * 16l, xqp, xsp, xbsp, n) } elif (fmt == 44) { myp[i] = dot_iq4xs_q8(kqp + i * nsb * 128l, ksp + i * nsb * 20l, xqp, xsp, xbsp, n) + } elif (fmt == 3) { + myp[i] = dot_k3q8(kqp + i * nsb * 128l, ksp + i * nsb * 20l, xqp, xsp, xbsp, n) } else { myp[i] = dot_k6q8(kqp + i * nsb * 192l, ksp + i * nsb * 18l, xqp, xsp, xbsp, n) } @@ -1377,7 +1466,7 @@ def private kq_batch_cell_gen(fmt : int; var myp : float?; kqp : uint8 const?; k let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) let nb16 = n / 16l - let packed = fmt == 4 || fmt == 40 || fmt == 44 // pure-nibble planes: the tile reads them directly + let packed = fmt == 4 || fmt == 40 || fmt == 44 || fmt == 3 // packed planes: the tile reads them directly var scratch : array if (!packed) { scratch |> resize(mr * nsb * 256l) @@ -1405,6 +1494,8 @@ def private kq_batch_cell_gen(fmt : int; var myp : float?; kqp : uint8 const?; k q40q8_tile_gen(myp, kqg, ksg, xqp, xsp, xbsp, n, d, g, tk) } elif (fmt == 44) { iq4xsq8_tile_gen(myp, kqg, ksg, xqp, xsp, xbsp, n, d, g, tk) + } elif (fmt == 3) { + k3q8_tile_gen(myp, kqg, ksg, xqp, xsp, xbsp, n, d, g, tk) } else { k6q8_tile_gen(myp, scp, ksg, xqp, xsp, xbsp, n, d, g, tk) } @@ -1419,6 +1510,8 @@ def private kq_batch_cell_gen(fmt : int; var myp : float?; kqp : uint8 const?; k q40q8_gemv_gen(myp + tk * d, kqp, ksp, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n, g * mr, (g + 1l) * mr) } elif (fmt == 44) { iq4xsq8_gemv_gen(myp + tk * d, kqp, ksp, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n, g * mr, (g + 1l) * mr) + } elif (fmt == 3) { + k3q8_gemv_gen(myp + tk * d, kqp, ksp, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n, g * mr, (g + 1l) * mr) } else { k6q8_gemv_gen(myp + tk * d, kqp, ksp, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n, g * mr, (g + 1l) * mr) } @@ -1454,6 +1547,8 @@ def private kq_batch_kernel_gen(fmt : int; var yp : float?; kqp : uint8 const?; myp[tk * d + i] = dot_q40q8(kqp + i * nsb * 128l, ksp + i * nsb * 16l, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * (n / 16l), n) } elif (fmt == 44) { myp[tk * d + i] = dot_iq4xs_q8(kqp + i * nsb * 128l, ksp + i * nsb * 20l, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * (n / 16l), n) + } elif (fmt == 3) { + myp[tk * d + i] = dot_k3q8(kqp + i * nsb * 128l, ksp + i * nsb * 20l, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * (n / 16l), n) } else { myp[tk * d + i] = dot_k6q8(kqp + i * nsb * 192l, ksp + i * nsb * 18l, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * (n / 16l), n) } @@ -1504,6 +1599,8 @@ def private kq_batch_groupn_gen(fmt : int; var yp : float?; kqp : uint8 const?; myp[tk * d + i] = dot_q40q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n) } elif (fmt == 44) { myp[tk * d + i] = dot_iq4xs_q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n) + } elif (fmt == 3) { + myp[tk * d + i] = dot_k3q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n) } else { myp[tk * d + i] = dot_k6q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n) } @@ -1663,6 +1760,8 @@ def kq_groupn_gen(fmt : int; var yp : float?; kqp : uint8 const?; ksp : uint8 co q40q8_gemv_gen(myp + r * d, kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n, gLocal * mr, gEnd * mr) } elif (fmt == 44) { iq4xsq8_gemv_gen(myp + r * d, kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n, gLocal * mr, gEnd * mr) + } elif (fmt == 3) { + k3q8_gemv_gen(myp + r * d, kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n, gLocal * mr, gEnd * mr) } else { k6q8_gemv_gen(myp + r * d, kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n, gLocal * mr, gEnd * mr) } @@ -1683,6 +1782,8 @@ def kq_groupn_gen(fmt : int; var yp : float?; kqp : uint8 const?; ksp : uint8 co myp[r * d + i] = dot_q40q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) } elif (fmt == 44) { myp[r * d + i] = dot_iq4xs_q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) + } elif (fmt == 3) { + myp[r * d + i] = dot_k3q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) } else { myp[r * d + i] = dot_k6q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) } @@ -1976,7 +2077,7 @@ def dasllama_math_gen_register() { // the no-witness stance above covers kq too mm_kq = @@kq_kernel_gen, repack_kq = @@repack_kq_gen, kq_rows_k4 = @@k4q8_gemv_gen, kq_rows_k5 = @@k5q8_gemv_gen, - kq_rows_k6 = @@k6q8_gemv_gen, kq_rows_q40 = @@q40q8_gemv_gen, kq_rows_iq4xs = @@iq4xsq8_gemv_gen, + kq_rows_k6 = @@k6q8_gemv_gen, kq_rows_q40 = @@q40q8_gemv_gen, kq_rows_iq4xs = @@iq4xsq8_gemv_gen, kq_rows_k3 = @@k3q8_gemv_gen, kq_batch = @@kq_batch_kernel_gen, kq_groupn = @@kq_groupn_gen, kq_batch_groupn = @@kq_batch_groupn_gen, // q51 family: tile + GEMV off the grp planes (the tile family's companions stamp @@ -2005,7 +2106,7 @@ def dasllama_math_gen_register() { mm_rows_s16 = @@q8q8_gemv_s16_gen, mm_kq = @@kq_kernel_gen, repack_kq = @@repack_kq_gen, kq_rows_k4 = @@k4q8_gemv_gen, kq_rows_k5 = @@k5q8_gemv_gen, - kq_rows_k6 = @@k6q8_gemv_gen, kq_rows_q40 = @@q40q8_gemv_gen, kq_rows_iq4xs = @@iq4xsq8_gemv_gen, + kq_rows_k6 = @@k6q8_gemv_gen, kq_rows_q40 = @@q40q8_gemv_gen, kq_rows_iq4xs = @@iq4xsq8_gemv_gen, kq_rows_k3 = @@k3q8_gemv_gen, kq_batch = @@kq_batch_kernel_gen, kq_groupn = @@kq_groupn_gen, kq_batch_groupn = @@kq_batch_groupn_gen, groupn_q51 = @@q51q8_groupn_gen, repack_q51 = @@repack_q51_gen, diff --git a/modules/dasLLAMA/dasllama/dasllama_ple.das b/modules/dasLLAMA/dasllama/dasllama_ple.das index 5d24984178..576ff289fb 100644 --- a/modules/dasLLAMA/dasllama/dasllama_ple.das +++ b/modules/dasLLAMA/dasllama/dasllama_ple.das @@ -64,6 +64,8 @@ def ple_check_table(t : Model; origin : string) { have = (long_length(t.q40q) / Q40_QSB) * 256l } elif (t.ple_emb_fmt == KqFmt.iq4xs) { have = (long_length(t.iq4xsq) / IQ4XS_QSB) * 256l + } elif (t.ple_emb_fmt == KqFmt.k3) { + have = (long_length(t.k3q) / K3_QSB) * 256l } if (have < need) { panic("dasLLAMA: the PLE token table is not resident ({origin}): fmt {t.ple_emb_fmt} plane holds {have} of {need} elements - regenerate this carrier (it predates the pinned-table rail)") @@ -99,6 +101,8 @@ def private ple_gather_row(t : Model; row : int64; var dst : array; doff dequant_q40_plane_superblock(t.q40q, sb * Q40_QSB, t.q40s, sb * Q40_SSB, dst, doff + blk * 256l) } elif (t.ple_emb_fmt == KqFmt.iq4xs) { dequant_iq4xs_plane_superblock(t.iq4xsq, sb * IQ4XS_QSB, t.iq4xss, sb * IQ4XS_SSB, dst, doff + blk * 256l) + } elif (t.ple_emb_fmt == KqFmt.k3) { + dequant_k3_plane_superblock(t.k3q, sb * K3_QSB, t.k3s, sb * K3_SSB, dst, doff + blk * 256l) } else { panic("ple_gather_row: '{t.ple_emb_fmt}' has no kq plane pair") } diff --git a/modules/dasLLAMA/dasllama/dasllama_repack.das b/modules/dasLLAMA/dasllama/dasllama_repack.das index 98d3ed70dd..9342856de1 100644 --- a/modules/dasLLAMA/dasllama/dasllama_repack.das +++ b/modules/dasLLAMA/dasllama/dasllama_repack.das @@ -417,6 +417,54 @@ def repack_iq4xs_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { delete tq delete ts } + +//! k3 grp layout: the qs bytes as 16 four-byte columns ([h 0..1][8 l-groups]) x mr rows, then the +//! hmask as 8 columns x mr rows at 64*mr — the 2-bit lanes and mask bits stay in place (the +//! kernels shift per block); scales interleave as k6's row ([16 x mr int8][mr x f16 d]). +def repack_k3_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { + let nsb = n / 256l + let qrow = nsb * 96l + let srow = nsb * 18l + let ng = d / mr + var tq : array + var ts : array + tq |> resize(d * qrow) + ts |> resize(d * srow) + unsafe { + var tqp = addr(tq[0]) + var tsp = addr(ts[0]) + memcpy(tqp, kq, d * qrow) + memcpy(tsp, ks, d * srow) + for (g in range64(ng)) { + for (sbi in range64(nsb)) { + let dq = g * mr * qrow + sbi * 96l * mr + let ds = g * mr * srow + sbi * 18l * mr + for (r in range64(mr)) { + let sq = (g * mr + r) * qrow + sbi * 96l + let ss = (g * mr + r) * srow + sbi * 18l + for (c in range64(16l)) { + for (t in range64(4l)) { + kq[dq + (c * mr + r) * 4l + t] = tqp[sq + c * 4l + t] + } + } + for (c in range64(8l)) { + for (t in range64(4l)) { + kq[dq + 64l * mr + (c * mr + r) * 4l + t] = tqp[sq + 64l + c * 4l + t] + } + } + for (idx in range64(16l)) { + ks[ds + idx * mr + r] = tsp[ss + idx] + } + ks[ds + 16l * mr + 2l * r] = tsp[ss + 16l] + ks[ds + 16l * mr + 2l * r + 1l] = tsp[ss + 17l] + } + } + } + } + delete tq + delete ts +} + // ===== grp panel unpack (the byte-expanded tile form the kq stamps load) ===== //! Unpack ONE group's packed grp k5/k6 quant panel into the BYTE-EXPANDED tile form (wlo/whi diff --git a/modules/dasLLAMA/harness/gen_tune_probe.das b/modules/dasLLAMA/harness/gen_tune_probe.das index 633b47321d..505a2fb709 100644 --- a/modules/dasLLAMA/harness/gen_tune_probe.das +++ b/modules/dasLLAMA/harness/gen_tune_probe.das @@ -440,7 +440,7 @@ struct KqFixture { def kq_qsb(fmt : int64) : int64 => dasllama_gemm_schema::kq_qsb(int(fmt)) def kq_ssb(fmt : int64) : int64 => dasllama_gemm_schema::kq_ssb(int(fmt)) -// the family's sidecar entry = its [tune] function name (the K-quants by bit width, q40/iq4xs by tag) +// the family's sidecar entry = its [tune] function name (the K-quants by bit width - k3 included - q40/iq4xs by tag) def kq_tile_entry(fmt : int64) : string => fmt == 40l ? "q40q8_tile_gen" : (fmt == 44l ? "iq4xsq8_tile_gen" : "k{fmt}q8_tile_gen") // synthetic disk superblocks, varied per (row, superblock) so no two blocks repeat: any byte @@ -486,6 +486,17 @@ def pack_kq_disk_block(fmt : int64; var blkb : array; base : int) { for (i in range(128)) { blkb[8 + i] = uint8((base * 37 + i * 101 + 29) % 256) } + } elif (fmt == 3l) { + // 110B Q3_K disk block: 32 hmask, 64 qs, 12 packed 6-bit scales, f16 d — every byte pattern is legal + for (i in range(96)) { + blkb[i] = uint8((base * 37 + i * 101 + 29) % 256) + } + for (i in range(12)) { + blkb[96 + i] = uint8((base * 29 + i * 71 + 5) % 256) + } + let dbits = f32_to_f16(0.002 + 0.0001 * float(base % 13)) + blkb[108] = uint8(dbits & 0xFF) + blkb[109] = uint8(dbits >> 8u) } elif (fmt == 4l) { pack_kq_scale_header(blkb, base) for (i in range(128)) { @@ -524,6 +535,8 @@ def repack_kq_grp_fmt(fmt : int64; var kq : uint8?; var ks : uint8?; n, d, mr : repack_q40_grp(kq, ks, n, d, mr) } elif (fmt == 44l) { repack_iq4xs_grp(kq, ks, n, d, mr) + } elif (fmt == 3l) { + repack_k3_grp(kq, ks, n, d, mr) } else { repack_k6_grp(kq, ks, n, d, mr) } @@ -536,7 +549,7 @@ def build_kq_fixture(fmt : int64; n, d, ntok : int64) : KqFixture { fx.kq |> resize(d * fx.nsb * qsb) fx.ks |> resize(d * fx.nsb * ssb) var blkb : array - blkb |> resize(fmt == 4l || fmt == 40l ? 144l : (fmt == 5l ? 176l : (fmt == 44l ? 136l : 210l))) + blkb |> resize(fmt == 4l || fmt == 40l ? 144l : (fmt == 5l ? 176l : (fmt == 44l ? 136l : (fmt == 3l ? 110l : 210l)))) for (r in range64(d)) { for (s in range64(fx.nsb)) { let base = int(r * 31l + s * 7l) @@ -550,6 +563,8 @@ def build_kq_fixture(fmt : int64; n, d, ntok : int64) : KqFixture { transcode_q40_superblock(blkb, 0l, fx.kq, sb * qsb, fx.ks, sb * ssb) } elif (fmt == 44l) { transcode_iq4xs_superblock(blkb, 0l, fx.kq, sb * qsb, fx.ks, sb * ssb) + } elif (fmt == 3l) { + transcode_q3k_superblock(blkb, 0l, fx.kq, sb * qsb, fx.ks, sb * ssb) } else { transcode_q6k_superblock(blkb, 0l, fx.kq, sb * qsb, fx.ks, sb * ssb) } @@ -587,8 +602,11 @@ def build_kq_fixture(fmt : int64; n, d, ntok : int64) : KqFixture { : (fmt == 44l ? iq4xs_grp_row_dot(kqg, ksg, r, 4l, addr(fx.xq[0]) + tk * n, addr(fx.xs[0]) + tk * (n / 256l), addr(fx.xbs[0]) + tk * (n / 16l), n) - : kq_grp_row_dot(fmt, kqg, ksg, r, 4l, - addr(fx.xq[0]) + tk * n, addr(fx.xs[0]) + tk * (n / 256l), addr(fx.xbs[0]) + tk * (n / 16l), n))) + : (fmt == 3l + ? k3_grp_row_dot(kqg, ksg, r, 4l, + addr(fx.xq[0]) + tk * n, addr(fx.xs[0]) + tk * (n / 256l), addr(fx.xbs[0]) + tk * (n / 16l), n) + : kq_grp_row_dot(fmt, kqg, ksg, r, 4l, + addr(fx.xq[0]) + tk * n, addr(fx.xs[0]) + tk * (n / 256l), addr(fx.xbs[0]) + tk * (n / 16l), n)))) } } } @@ -613,6 +631,9 @@ def kq_tile_variants(fmt : int64) : array> { if (fmt == 44l) { return <- iq4xsq8_tile_gen_variants() } + if (fmt == 3l) { + return <- k3q8_tile_gen_variants() + } return <- k6q8_tile_gen_variants() } @@ -642,6 +663,12 @@ def kq_gemv_variants_by_suffix(fmt : int64) : table { tab[v._0] = v._1 } delete gvs + } elif (fmt == 3l) { + var gvs <- k3q8_gemv_gen_variants() + for (v in gvs) { + tab[v._0] = v._1 + } + delete gvs } else { var gvs <- k6q8_gemv_gen_variants() for (v in gvs) { @@ -680,6 +707,12 @@ def kq_layout_mrs(fmt : int64) : table { mrs[v._0] = invoke(v._1) } delete lvs + } elif (fmt == 3l) { + var lvs <- k3q8_layout_gen_variants() + for (v in lvs) { + mrs[v._0] = invoke(v._1) + } + delete lvs } else { var lvs <- k6q8_layout_gen_variants() for (v in lvs) { @@ -697,7 +730,7 @@ def kq_layout_mrs(fmt : int64) : table { def run_kq_tile(tile; fx : KqFixture; kq : array; ks : array; mr : int64; var y : array) { let ng = fx.d / mr verify(fx.ntok % 4l == 0l) - let packed = fx.fmt == 4l || fx.fmt == 40l || fx.fmt == 44l // pure-nibble planes: the tile reads them directly + let packed = fx.fmt == 4l || fx.fmt == 40l || fx.fmt == 44l || fx.fmt == 3l // packed planes: the tile reads them directly var panel : array if (!packed) { panel |> resize(int(mr * fx.nsb * 256l)) @@ -1303,6 +1336,8 @@ def test_mode : bool { // nolint:STYLE037,STYLE038 — flat per-variant correctn allok = kq_test_family(6l, kfxs6) && allok allok = kq_test_family(40l, kfxs40) && allok allok = kq_test_family(44l, kfxs44) && allok + // k3 joins this list with its emitter arm: a stub family fails every perm whose layout + // companion generates (planes at mr 8, the reference body at mr 4 - HOW_TO QUIRK 16) delete wtab delete mtab delete gtab @@ -1761,7 +1796,7 @@ def tune_mode_run : bool { // nolint:STYLE037,STYLE038 — flat family-by-family print("aborting mid-run - no sidecar written\n") return false } - for (fmt in fixed_array(4l, 5l, 6l, 40l, 44l)) { + for (fmt in fixed_array(4l, 5l, 6l, 40l, 44l, 3l)) { interrupt_gate("k{fmt}q8_tile_gen") let kq_t0 = ref_time_ticks() let entry = kq_tile_entry(fmt) diff --git a/modules/dasLLAMA/tests/test_kqformat.das b/modules/dasLLAMA/tests/test_kqformat.das index 1ab1eed111..540d96d5a7 100644 --- a/modules/dasLLAMA/tests/test_kqformat.das +++ b/modules/dasLLAMA/tests/test_kqformat.das @@ -21,6 +21,7 @@ def test_kqfmt_enum_values(t : T?) { t |> equal(int(KqFmt.q40), 4) t |> equal(int(KqFmt.q51), 5) t |> equal(int(KqFmt.iq4xs), 6) + t |> equal(int(KqFmt.k3), 7) } } @@ -35,6 +36,8 @@ def test_kqfmt_predicates(t : T?) { t |> success(!kq_sb(KqFmt.q51), "q51 rides per-32 planes") t |> success(kq_sb(KqFmt.iq4xs), "iq4xs is superblock") t |> success(kq_sb(int(KqFmt.iq4xs)), "the int-id predicate agrees for iq4xs") + t |> success(kq_sb(KqFmt.k3), "k3 is superblock") + t |> success(kq_sb(int(KqFmt.k3)), "the int-id predicate agrees for k3") } } @@ -53,6 +56,8 @@ def test_kqfmt_strides(t : T?) { t |> equal(kq_ssb(KqFmt.q51), 4l) // per 32-BLOCK: f16 d + f16 m t |> equal(kq_qsb(KqFmt.iq4xs), 128l) // the q40 tiling (disk k/k+16 pairing) t |> equal(kq_ssb(KqFmt.iq4xs), 20l) // f16 d + 8 int8 sub-scales in the k4 row shape + t |> equal(kq_qsb(KqFmt.k3), 96l) // 64 qs + 32 hmask, verbatim + t |> equal(kq_ssb(KqFmt.k3), 18l) // the k6 row: 16 int8 sub-scales + f16 d } t |> run("named constants agree with the literals") @(t : T?) { t |> equal(K4_QSB, 128l) @@ -67,6 +72,8 @@ def test_kqfmt_strides(t : T?) { t |> equal(Q51_SB, 4l) t |> equal(IQ4XS_QSB, 128l) t |> equal(IQ4XS_SSB, 20l) + t |> equal(K3_QSB, 96l) + t |> equal(K3_SSB, 18l) } t |> run("the IQ4_NL codebook is ggml's kvalues_iq4nl") @(t : T?) { t |> equal(int(IQ4NL_LUT[0]), -127) @@ -81,15 +88,16 @@ def test_kqfmt_strides(t : T?) { [test] def test_kqfmt_id_bridge(t : T?) { - t |> run("kq_schema_id is the one enum->kernel-id bridge (4/5/6/40/44)") @(t : T?) { + t |> run("kq_schema_id is the one enum->kernel-id bridge (3/4/5/6/40/44)") @(t : T?) { t |> equal(kq_schema_id(KqFmt.k4), 4) t |> equal(kq_schema_id(KqFmt.k5), 5) t |> equal(kq_schema_id(KqFmt.k6), 6) t |> equal(kq_schema_id(KqFmt.q40), 40) t |> equal(kq_schema_id(KqFmt.iq4xs), 44) + t |> equal(kq_schema_id(KqFmt.k3), 3) } t |> run("gemm_schema's int-id forms delegate to the same truth") @(t : T?) { - for (f in [KqFmt.k4, KqFmt.k5, KqFmt.k6, KqFmt.q40, KqFmt.iq4xs]) { + for (f in [KqFmt.k4, KqFmt.k5, KqFmt.k6, KqFmt.q40, KqFmt.iq4xs, KqFmt.k3]) { t |> equal(kq_qsb(kq_schema_id(f)), kq_qsb(f)) t |> equal(kq_ssb(kq_schema_id(f)), kq_ssb(f)) } @@ -104,6 +112,7 @@ def test_kqfmt_elems_and_stream(t : T?) { t |> equal(kq_elems(KqFmt.k6), 256l) t |> equal(kq_elems(KqFmt.q40), 256l) t |> equal(kq_elems(KqFmt.iq4xs), 256l) + t |> equal(kq_elems(KqFmt.k3), 256l) t |> equal(kq_elems(KqFmt.q8), 32l) t |> equal(kq_elems(KqFmt.q51), 32l) } @@ -115,5 +124,6 @@ def test_kqfmt_elems_and_stream(t : T?) { t |> equal(kq_stream_code(KqFmt.k6), 6) t |> equal(kq_stream_code(KqFmt.q40), 40) t |> equal(kq_stream_code(KqFmt.iq4xs), 44) + t |> equal(kq_stream_code(KqFmt.k3), 3) } } diff --git a/modules/dasLLAMA/tests/test_kquant.das b/modules/dasLLAMA/tests/test_kquant.das index 35896af6f9..66bdca6735 100644 --- a/modules/dasLLAMA/tests/test_kquant.das +++ b/modules/dasLLAMA/tests/test_kquant.das @@ -1,8 +1,8 @@ options gen2 options stack = 524288 // every dasLLAMA program root takes this budget (options stack does not unify up from libs) options persistent_heap // the layout arm loads a multi-GB model; explicit delete below -options _cyclomatic_complexity = 40 // every gate here is the same flat fmt == 4/5/6/40/44 ladder repeated per stage -options _function_length = 180 // and each gate carries its whole fixture build + oracle inline +options _cyclomatic_complexity = 46 // every gate here is the same flat fmt == 4/5/6/40/44/3 ladder repeated per stage +options _function_length = 200 // and each gate carries its whole fixture build + oracle inline options _dasllama_internal = true require dastest/testing_boost public @@ -27,6 +27,7 @@ require math def private q4_pat(k : int) : int => (k * 7 + 3) % 16 def private q5_pat(k : int) : int => (k * 3 + 7) % 32 def private q6_pat(k : int) : int => (k * 5 + 11) % 64 +def private q3_pat(k : int) : int => (k * 5 + 3) % 8 // The shared synthetic scale/min sets: 6-bit values > 15 exercise the split high-bit arms of // the 6-bit scale/min unpack (j >= 4); d/dmin are f16-exact so expected-value math uses the f32 value. @@ -132,6 +133,49 @@ def private build_iq4xs_block() : array { return <- blkb } +// k3 synthetic superblock: 110B Q3_K disk block — 16 six-bit sub-scales SYNTH_SC[is % 8] packed in +// ggml's kmask form (written in the PACK direction), f16 d = SYNTH_D, the 2-bit lanes + hmask bits +// of q3_pat — weight k = (d * (SYNTH_SC[(k/16) % 8] - 32)) * (q3_pat(k) - 4) +def private build_q3k_block() : array { + var blkb : array + blkb |> resize(110) + var hm : array + var qs : array + var sc : array + hm |> resize(32) + qs |> resize(64) + sc |> resize(12) + for (k in range(256)) { // block b = k/32: qs byte (b/4)*32 + l holds the 2-bit lane at shift 2*(b%4); hmask byte l bit b + let b = k / 32 + let l = k % 32 + let q = q3_pat(k) + qs[(b / 4) * 32 + l] |= (q & 3) << (2 * (b % 4)) + hm[l] |= (q >> 2) << b + } + for (i in range(16)) { // scale i: low nibble in byte i (i < 8) / high nibble of byte i-8, top two bits in byte 8 + i%4 at 2*(i/4) + let v = SYNTH_SC[i % 8] + if (i < 8) { + sc[i] |= v & 15 + } else { + sc[i - 8] |= (v & 15) << 4 + } + sc[8 + i % 4] |= ((v >> 4) & 3) << (2 * (i / 4)) + } + for (i in range(32)) { + blkb[i] = uint8(hm[i]) + } + for (i in range(64)) { + blkb[32 + i] = uint8(qs[i]) + } + for (i in range(12)) { + blkb[96 + i] = uint8(sc[i]) + } + let dbits = f32_to_f16(SYNTH_D) + blkb[108] = uint8(dbits & 0xFF) + blkb[109] = uint8(dbits >> 8u) + return <- blkb +} + // q40 synthetic superblock: 8 x 18B q4_0 disk blocks, per-block f16-exact d = (blk+1)/16 and // the shared nibble pattern — weight k = d(k/32) * (q4_pat(k) - 8) def private q40_d(blk : int) : float => float(blk + 1) * 0.0625 @@ -303,6 +347,24 @@ def test_kq_transcode_planes(t : T?) { t |> success(dst[k] == expected, "iq4xs plane element must match the ggml dequant exactly") } } + t |> run("k3 planes decode the packed 6-bit sub-scales and compose the 3-bit quants exactly") @(t : T?) { + let blkb <- build_q3k_block() + var kq : array + var ks : array + kq |> resize(96) + ks |> resize(18) + transcode_q3k_superblock(blkb, 0l, kq, 0l, ks, 0l) + for (i in range(16)) { + t |> equal(int(int8(ks[i])), SYNTH_SC[i % 8] - 32, "decoded sub-scale is scale - 32, signed") + } + var dst : array + dst |> resize(256) + dequant_k3_plane_superblock(kq, 0l, ks, 0l, dst, 0l) + for (k in range(256)) { + let expected = (SYNTH_D * float(SYNTH_SC[(k / 16) % 8] - 32)) * float(q3_pat(k) - 4) + t |> success(dst[k] == expected, "k3 plane element must match the ggml dequant exactly") + } + } t |> run("q51 planes unpack the hand-packed Q5_1 blocks exactly") @(t : T?) { for (blk in range(4)) { let blkb <- build_q51_block(blk) @@ -549,7 +611,7 @@ def test_q51_kernels(t : T?) { // elementwise dot over the plane dequant (the kernels' declared oracle). def private kq_dot_gate(t : T?; fmt : int; n : int64 = 512l) { let nsb = n / 256l - let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : build_q6k_block()))) + let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : build_q6k_block())))) let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) var kq : array @@ -565,6 +627,8 @@ def private kq_dot_gate(t : T?; fmt : int; n : int64 = 512l) { transcode_q40_superblock(blk1, 0l, kq, s * qsb, ks, s * ssb) } elif (fmt == 44) { transcode_iq4xs_superblock(blk1, 0l, kq, s * qsb, ks, s * ssb) + } elif (fmt == 3) { + transcode_q3k_superblock(blk1, 0l, kq, s * qsb, ks, s * ssb) } else { transcode_q6k_superblock(blk1, 0l, kq, s * qsb, ks, s * ssb) } @@ -586,6 +650,8 @@ def private kq_dot_gate(t : T?; fmt : int; n : int64 = 512l) { dequant_q40_plane_superblock(kq, s * qsb, ks, s * ssb, w, s * 256l) } elif (fmt == 44) { dequant_iq4xs_plane_superblock(kq, s * qsb, ks, s * ssb, w, s * 256l) + } elif (fmt == 3) { + dequant_k3_plane_superblock(kq, s * qsb, ks, s * ssb, w, s * 256l) } else { dequant_k6_plane_superblock(kq, s * qsb, ks, s * ssb, w, s * 256l) } @@ -619,6 +685,8 @@ def private kq_dot_gate(t : T?; fmt : int; n : int64 = 512l) { got = dot_q40q8(addr(kq[0]), addr(ks[0]), addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } elif (fmt == 44) { got = dot_iq4xs_q8(addr(kq[0]), addr(ks[0]), addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + } elif (fmt == 3) { + got = dot_k3q8(addr(kq[0]), addr(ks[0]), addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } else { got = dot_k6q8(addr(kq[0]), addr(ks[0]), addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } @@ -644,9 +712,12 @@ def test_kq_dots(t : T?) { t |> run("dot_iq4xs_q8 matches the fp64 plane-dequant reference") @(t : T?) { kq_dot_gate(t, 44) } + t |> run("dot_k3q8 matches the fp64 plane-dequant reference") @(t : T?) { + kq_dot_gate(t, 3) + } // 122B-class row widths: n=3072 (12 superblocks — qwen35moe-122B gate/up/cls) and n=1024 // (its k5 down_exps). Every earlier kq model kept n at 512..2560. - for (fmt in [4, 5, 6, 40, 44]) { + for (fmt in [4, 5, 6, 40, 44, 3]) { for (nn in [1024l, 3072l]) { t |> run("dot_{kq_tag(fmt)}q8 matches the fp64 reference at n={int(nn)}") @(t : T?) { kq_dot_gate(t, fmt, nn) @@ -665,7 +736,7 @@ def private kq_gemv_rows_gate(t : T?; fmt : int; n : int64 = 512l) { let nsb = n / 256l let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) - let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : build_q6k_block()))) + let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : build_q6k_block())))) var kq : array var ks : array kq |> resize(d * nsb * qsb) @@ -680,6 +751,8 @@ def private kq_gemv_rows_gate(t : T?; fmt : int; n : int64 = 512l) { transcode_q40_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } elif (fmt == 44) { transcode_iq4xs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 3) { + transcode_q3k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } else { transcode_q6k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } @@ -720,6 +793,8 @@ def private kq_gemv_rows_gate(t : T?; fmt : int; n : int64 = 512l) { want = dot_q40q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } elif (fmt == 44) { want = dot_iq4xs_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + } elif (fmt == 3) { + want = dot_k3q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } else { want = dot_k6q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } @@ -755,7 +830,10 @@ def test_kq_gemv_rows(t : T?) { t |> run("portable iq4xs GEMV rows bit-match per-row disk dots") @(t : T?) { kq_gemv_rows_gate(t, 44) } - for (fmt in [4, 5, 6, 40, 44]) { + t |> run("portable k3 GEMV rows bit-match per-row disk dots") @(t : T?) { + kq_gemv_rows_gate(t, 3) + } + for (fmt in [4, 5, 6, 40, 44, 3]) { for (nn in [1024l, 3072l]) { t |> run("portable {kq_tag(fmt)} GEMV rows bit-match per-row disk dots at n={int(nn)}") @(t : T?) { kq_gemv_rows_gate(t, fmt, nn) @@ -775,7 +853,7 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { let nsb = n / 256l let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) - let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : build_q6k_block()))) + let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : build_q6k_block())))) var kq : array var ks : array kq |> resize(d * nsb * qsb) @@ -790,6 +868,8 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { transcode_q40_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } elif (fmt == 44) { transcode_iq4xs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 3) { + transcode_q3k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } else { transcode_q6k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } @@ -829,6 +909,8 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { want[r] = dot_q40q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } elif (fmt == 44) { want[r] = dot_iq4xs_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + } elif (fmt == 3) { + want[r] = dot_k3q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } else { want[r] = dot_k6q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } @@ -841,6 +923,8 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { dequant_q40_plane_superblock(kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb, wrow, r * n + s * 256l) } elif (fmt == 44) { dequant_iq4xs_plane_superblock(kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb, wrow, r * n + s * 256l) + } elif (fmt == 3) { + dequant_k3_plane_superblock(kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb, wrow, r * n + s * 256l) } else { dequant_k6_plane_superblock(kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb, wrow, r * n + s * 256l) } @@ -854,6 +938,8 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { repack_q40_grp(addr(kq[0]), addr(ks[0]), n, d, mr) } elif (fmt == 44) { repack_iq4xs_grp(addr(kq[0]), addr(ks[0]), n, d, mr) + } elif (fmt == 3) { + repack_k3_grp(addr(kq[0]), addr(ks[0]), n, d, mr) } else { repack_k6_grp(addr(kq[0]), addr(ks[0]), n, d, mr) } @@ -869,7 +955,9 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { ? q40_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) : (fmt == 44 ? iq4xs_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) - : kq_grp_row_dot(int64(fmt), kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n))) + : (fmt == 3 + ? k3_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + : kq_grp_row_dot(int64(fmt), kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n)))) if (got != want[r]) { dotbad++ } @@ -896,7 +984,7 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { [test] def test_kq_repack_grp(t : T?) { - for (fmt in [4, 5, 6, 40, 44]) { + for (fmt in [4, 5, 6, 40, 44, 3]) { for (mr in [4l, 8l, 16l]) { t |> run("repack_{kq_tag(fmt)}_grp mr={mr} preserves dots and row dequants") @(t : T?) { kq_repack_gate(t, fmt, mr) @@ -921,7 +1009,7 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { let mr = kq_layout_of(fmt) // the format's OWN layout companion (per-format tune families) let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) - let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : build_q6k_block()))) + let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : build_q6k_block())))) var kq : array var ks : array kq |> resize(d * nsb * qsb) @@ -936,6 +1024,8 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { transcode_q40_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } elif (fmt == 44) { transcode_iq4xs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 3) { + transcode_q3k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } else { transcode_q6k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } @@ -955,6 +1045,8 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { repack_q40_grp(addr(kq[0]), addr(ks[0]), n, d, mr) } elif (fmt == 44) { repack_iq4xs_grp(addr(kq[0]), addr(ks[0]), n, d, mr) + } elif (fmt == 3) { + repack_k3_grp(addr(kq[0]), addr(ks[0]), n, d, mr) } else { repack_k6_grp(addr(kq[0]), addr(ks[0]), n, d, mr) } @@ -983,7 +1075,7 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { // k5/k6 tiles read the byte-expanded panel the batch cell unpacks per (group, token-block) // — the gate covers unpack_kq_panel_grp too (tile-over-panel vs gemv-over-packed planes); // k4/q40 tiles read the packed planes directly - let packed = fmt == 4 || fmt == 40 || fmt == 44 + let packed = fmt == 4 || fmt == 40 || fmt == 44 || fmt == 3 var panel : array if (!packed) { panel |> resize(mr * nsb * 256l) @@ -1006,6 +1098,8 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { q40q8_tile_gen(addr(ytile[0]), kqg, ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) } elif (fmt == 44) { iq4xsq8_tile_gen(addr(ytile[0]), kqg, ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) + } elif (fmt == 3) { + k3q8_tile_gen(addr(ytile[0]), kqg, ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) } else { k6q8_tile_gen(addr(ytile[0]), addr(panel[0]), ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) } @@ -1020,6 +1114,8 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { q40q8_gemv_gen(addr(ytile[tk * d]), addr(kq[0]), addr(ks[0]), addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, g * mr, (g + 1l) * mr) } elif (fmt == 44) { iq4xsq8_gemv_gen(addr(ytile[tk * d]), addr(kq[0]), addr(ks[0]), addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, g * mr, (g + 1l) * mr) + } elif (fmt == 3) { + k3q8_gemv_gen(addr(ytile[tk * d]), addr(kq[0]), addr(ks[0]), addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, g * mr, (g + 1l) * mr) } else { k6q8_gemv_gen(addr(ytile[tk * d]), addr(kq[0]), addr(ks[0]), addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, g * mr, (g + 1l) * mr) } @@ -1035,6 +1131,8 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { q40q8_gemv_gen(addr(ygemv[p * d]), addr(kq[0]), addr(ks[0]), addr(xq[p * n]), addr(xs[p * (n / 256l)]), addr(xbs[p * (n / 16l)]), n, 0l, d) } elif (fmt == 44) { iq4xsq8_gemv_gen(addr(ygemv[p * d]), addr(kq[0]), addr(ks[0]), addr(xq[p * n]), addr(xs[p * (n / 256l)]), addr(xbs[p * (n / 16l)]), n, 0l, d) + } elif (fmt == 3) { + k3q8_gemv_gen(addr(ygemv[p * d]), addr(kq[0]), addr(ks[0]), addr(xq[p * n]), addr(xs[p * (n / 256l)]), addr(xbs[p * (n / 16l)]), n, 0l, d) } else { k6q8_gemv_gen(addr(ygemv[p * d]), addr(kq[0]), addr(ks[0]), addr(xq[p * n]), addr(xs[p * (n / 256l)]), addr(xbs[p * (n / 16l)]), n, 0l, d) } @@ -1060,7 +1158,7 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { [test] def test_kq_tile(t : T?) { - for (fmt in [4, 5, 6, 40, 44]) { + for (fmt in [4, 5, 6, 40, 44, 3]) { t |> run("{kq_tag(fmt)} 4-token tile bit-matches per-token GEMVs") @(t : T?) { kq_tile_gate(t, fmt) } @@ -1081,7 +1179,7 @@ def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; let nsb = n / 256l let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) - let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : build_q6k_block()))) + let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : build_q6k_block())))) var kq : array var ks : array kq |> resize(nreg * d * nsb * qsb) @@ -1096,6 +1194,8 @@ def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; transcode_q40_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } elif (fmt == 44) { transcode_iq4xs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 3) { + transcode_q3k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } else { transcode_q6k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } @@ -1156,6 +1256,9 @@ def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; } elif (fmt == 44) { want_sh[r * d + row] = dot_iq4xs_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) want_pr[r * d + row] = dot_iq4xs_q8(kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n) + } elif (fmt == 3) { + want_sh[r * d + row] = dot_k3q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + want_pr[r * d + row] = dot_k3q8(kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n) } else { want_sh[r * d + row] = dot_k6q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) want_pr[r * d + row] = dot_k6q8(kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n) @@ -1200,6 +1303,8 @@ def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; repack_q40_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) } elif (fmt == 44) { repack_iq4xs_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) + } elif (fmt == 3) { + repack_k3_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) } else { repack_k6_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) } @@ -1219,6 +1324,9 @@ def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; } elif (fmt == 44) { iq4xsq8_gemv_gen(addr(want_sh[r * d]), kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, 0l, d) iq4xsq8_gemv_gen(addr(want_pr[r * d]), kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n, 0l, d) + } elif (fmt == 3) { + k3q8_gemv_gen(addr(want_sh[r * d]), kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, 0l, d) + k3q8_gemv_gen(addr(want_pr[r * d]), kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n, 0l, d) } else { k6q8_gemv_gen(addr(want_sh[r * d]), kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, 0l, d) k6q8_gemv_gen(addr(want_pr[r * d]), kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n, 0l, d) @@ -1254,7 +1362,7 @@ def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; [test] def test_kq_groupn(t : T?) { - for (fmt in [4, 5, 6, 40, 44]) { + for (fmt in [4, 5, 6, 40, 44, 3]) { t |> run("{kq_tag(fmt)} region-list GEMV bit-matches per-row dots (disk + grp slices)") @(t : T?) { kq_groupn_gate(t, fmt) } @@ -1298,7 +1406,7 @@ def private kq_batch_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 for (c in cnts) { nk += c } - let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : build_q6k_block()))) + let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : build_q6k_block())))) var kq : array var ks : array kq |> resize(nreg * d * nsb * qsb) @@ -1313,6 +1421,8 @@ def private kq_batch_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 transcode_q40_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } elif (fmt == 44) { transcode_iq4xs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 3) { + transcode_q3k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } else { transcode_q6k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } @@ -1333,6 +1443,8 @@ def private kq_batch_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 repack_q40_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) } elif (fmt == 44) { repack_iq4xs_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) + } elif (fmt == 3) { + repack_k3_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) } else { repack_k6_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) } @@ -1379,6 +1491,8 @@ def private kq_batch_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 q40q8_gemv_gen(addr(want[tk * d]), kqr, ksr, addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, 0l, d) } elif (fmt == 44) { iq4xsq8_gemv_gen(addr(want[tk * d]), kqr, ksr, addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, 0l, d) + } elif (fmt == 3) { + k3q8_gemv_gen(addr(want[tk * d]), kqr, ksr, addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, 0l, d) } else { k6q8_gemv_gen(addr(want[tk * d]), kqr, ksr, addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, 0l, d) } @@ -1410,7 +1524,7 @@ def private kq_batch_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 [test] def test_kq_batch_groupn(t : T?) { - for (fmt in [4, 5, 6, 40, 44]) { + for (fmt in [4, 5, 6, 40, 44, 3]) { t |> run("{kq_tag(fmt)} batch groupn bit-matches per-token rows-core calls") @(t : T?) { kq_batch_groupn_gate(t, fmt) } From 61d7731f46201127e2baff83248d66a1de539e67 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Sun, 30 Aug 2026 10:36:29 -0700 Subject: [PATCH 08/61] dasLLAMA: Q3_K JIT emitter - a k3 flag through emit_block_kqv2's k6 arms (2-bit lane + hmask compose, offset 4), stamps live, k3 in the probe's test list; HOW_TO note + QUIRK 18 k3 rides emit_block_kqv2 as k6 does (k63 shares the 18 B scale row, the per-16 signed fold and the i16 flush cadence; koff = 4): the compose loads qs column (blk/4)*8 + j (lo) / + 4 (hi), shifts by 2 * (blk % 4), ORs the hmask column j / j + 4 bit blk shifted up two. The decline stubs become kq_gemv_gen_impl / kq_tile_gen_impl at fmt 3; the k3 family joins the probe's test-mode list now that layout and tile decline in lockstep. Gates: the probe 10/10 perms on x64 (dot_maddubs_width256_mr8 live, maxdiff 4.8e-7) and on the M1 Max (sdot mr4/mr8 stamps bit-exact), test_kquant -jit 140/144 on both. The 1B Q3_K_L decodes at 60 t/s after the sidecar re-mint (22 on the reference body); its text flips at token 15 - a near tie: on two real k3 tensors 4957 of 10240 output rows are bit-identical to the reference body and the worst relative difference is 1.7e-4 on a 6e-5 row. Co-Authored-By: Claude Fable 5 --- modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md | 21 +++++- .../dasLLAMA/dasllama/dasllama_gemm_gen.das | 64 +++++++++++-------- modules/dasLLAMA/harness/gen_tune_probe.das | 5 +- 3 files changed, 62 insertions(+), 28 deletions(-) diff --git a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md index 65ab94f806..f47c710899 100644 --- a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md +++ b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md @@ -356,6 +356,13 @@ where, why it is so today, what unquirked looks like. An empty ledger is a legit `run.das` on the box spent minutes minting before the Q3_K_L load - and every other sidecar on the box (`chat.das`, the harness probes) does the same on its next run. Budget for it; do not read the first end-to-end wall time as a load-time regression. +18. **`DAS_TUNE_POLICY=fallback` is not "the reference body".** It walks the family's + `fallback=` chain, and on a box where the chain's first viable perm is the crowned one the + kill switch serves the very stamp under test - the k3 stamped-vs-"reference" rows diff came + back bit-identical for that reason. The original body is reached only by a `"reference"` + sidecar entry for the family (copy the app's sidecar next to the probe as + `.tune.json`, edit the one entry). Unquirked: a policy value that names the reference + tier outright. ## Per-format notes @@ -376,7 +383,19 @@ misses the census found were two plane-byte accounting sums the pilot itself had Gates: `test_kqformat` 14/14, `test_kquant` 132/144 interpreted and 140/144 under `-jit`, the probe's tune-mode family (test mode waits for the emitter - QUIRK 16). End to end: `Llama-3.2-1B-Instruct-Q3_K_L.gguf` (bartowski: Q3_K x64 + Q5_K x48 + Q6_K embd) through `run.das` matches llama.cpp's greedy ids (`simple_ids.exe`, compared through `llama-tokenize --ids` on our text) for 52 of 64 tokens on the reference bodies, gen 22 t/s. -JIT emitter, Vulkan, Metal: pending. + +JIT emitter: no new block body - a `k3` flag through `emit_block_kqv2`'s k6 arms (`k63` +shares the scale row, the per-16 fold and the i16 flush cadence; `koff` = 4 / 32), the compose +being qs column `(blk/4)*8 + j` (lo) / `+ 4` (hi) shifted by `2 * (blk % 4)` OR the hmask +column `j` / `j + 4` bit `blk` shifted up two. The stubs became `kq_*_gen_impl(gc, 3)`, k3 +joined the probe's test list. Gates: the probe 10/10 perms on x64 (`dot_maddubs_width256_mr8` +live, maxdiff 4.8e-7) and on the M1 (sdot mr 4/8 stamps bit-exact), `test_kquant -jit` +140/144 on both. After the sidecar re-mint (`k3q8_tile_gen: dot_maddubs_width256_mr8`, +`verdict=beats`) the 1B decodes at gen 60 t/s (22 on the reference body) and its text flips +at token 15 - a near tie: against the reference body on two real k3 tensors (10240 output rows, +scratchpad `k3_rows.das` with a `"reference"` sidecar entry) 4957 rows are bit-identical and +the worst relative difference is 1.7e-4 on a row of magnitude 6e-5 (fp32 fold-order noise). +Vulkan, Metal: pending. ### IQ4_XS (the pilot, 2026-08-30) diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das index 522d1dce2d..f0e9bc64a2 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das @@ -269,7 +269,7 @@ struct private TileEmit { mx4 : bool // block emitter: mx4 LUT-dequant instead of Q8 loads q51 : bool // block emitter: q51 nibble+qh unpack (32-weight blocks, // per-block f16 d/m scale pairs + xbs min-term) - kq : int // 4/5/6/40 = K-quant superblock emitter, 44 = IQ4_XS (0 = not kq): the + kq : int // 3/4/5/6/40 = K-quant superblock emitter, 44 = IQ4_XS (0 = not kq): the // block unit is a 256-weight superblock over the grp // kq planes; dots run unsigned-q (kq_dot_lane) kqBytes : bool // kq weights are a BYTE-EXPANDED panel (the tile's @@ -926,7 +926,7 @@ def private or_bit_x10(te : TileEmit; var w, bytes, maskv : LLVMOpaqueValue?; na return LLVMBuildOr(b, w, sel, name) } -// One 256-weight SUPERBLOCK, K-quant grp form (te.kq = 4/5/6): weight vectors unpacked +// One 256-weight SUPERBLOCK, K-quant grp form (te.kq = 3/4/5/6/40): weight vectors unpacked // ONCE per (sub-block, dword-group) and dotted against every token — weight-stationary, // bit-exact vs per-token GEMVs (tokCount=1). kq v2 folds integer sub-scales/bsums against Q8_K-form activations. def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f : LLVMOpaqueValue? [8]; tokBase, tokCount : int) { // nolint:STYLE037,STYLE038 — one register-coupled superblock loop nest; the k-quant decode arms share live values @@ -935,10 +935,13 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f let mr = te.interleave let w8 = te.width / 8 let k6 = te.kq == 6 + let k3 = te.kq == 3 + let k63 = k6 || k3 // k3 rides k6's scale row and fold; only the quant compose and the offset differ + let koff = k3 ? 4 : 32 // the signed-lattice offset the bsum fold removes let q40 = te.kq == 40 - let qsb = (te.kq == 4 || q40) ? 128 : (te.kqBytes ? 256 : (te.kq == 5 ? 160 : 192)) + let qsb = (te.kq == 4 || q40) ? 128 : (k3 ? 96 : (te.kqBytes ? 256 : (te.kq == 5 ? 160 : 192))) var wb = LLVMBuildMul(b, sbi, te.types->ConstI64(uint64(mr * qsb)), "wb") - var sb = LLVMBuildMul(b, sbi, te.types->ConstI64(uint64(mr * (k6 ? 18 : (q40 ? 16 : 20)))), "sb") + var sb = LLVMBuildMul(b, sbi, te.types->ConstI64(uint64(mr * (k63 ? 18 : (q40 ? 16 : 20)))), "sb") var xb = LLVMBuildMul(b, sbi, te.types->ConstI64(0x100ul), "xb") var gb = LLVMBuildShl(b, sbi, te.types->ConstI64(3ul), "gb") var maskLoElems <- [for (i in range(w8)); LLVMConstInt(te.types.t_int8, uint64(1 << (i % 4)), 0)] @@ -949,12 +952,12 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f let memBcast = te.dotKind != DOT_SDOT && te.dotKind != DOT_SMMLA let madd16 = memBcast && te.dotKind == DOT_MADDUBS var vri8 = LLVMVectorType(te.types.t_int8, uint(te.rv)) - // weight-side superblock scale rows: k4/k5 f16 d/dmin; k6 f16 d only; q40 has none (per-32 d loads live below) + // weight-side superblock scale rows: k4/k5 f16 d/dmin; k6/k3 f16 d only; q40 has none (per-32 d loads live below) var dv : LLVMOpaqueValue? [2] var mv : LLVMOpaqueValue? [2] if (!q40) { for (qd in range(rq)) { - if (k6) { + if (k63) { dv[qd] = load_f16_vec_at(te, te.sg, LLVMBuildAdd(b, sb, te.types->ConstI64(uint64(16 * mr + 2 * (qd * te.rv))), ""), "d{qd}") } else { dv[qd] = load_f16_vec_at(te, te.sg, LLVMBuildAdd(b, sb, te.types->ConstI64(uint64(2 * (qd * te.rv))), ""), "d{qd}") @@ -971,9 +974,9 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f facc[i] = LLVMConstNull(te.vnf32) } // vector bsums: sdot leg only (x64's vpbroadcastd makes scalars free; k6 needs bs0/bs1 unsummed) - let vecBsums = !memBcast && !k6 + let vecBsums = !memBcast && !k63 // mr>=8 fuses lo+hi into ONE acc (i32-exact — the fold adds them anyway); mr4 keeps split accs for ILP - let fuseAcc = !memBcast && mr >= 8 && !k6 + let fuseAcc = !memBcast && mr >= 8 && !k63 var a32v8 : LLVMOpaqueValue? [8] // [tok * 2 + blk / 4], lane blk % 4 if (vecBsums) { var xb16 = LLVMBuildShl(b, sbi, te.types->ConstI64(4ul), "xb16") @@ -1018,7 +1021,21 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f var noff = LLVMBuildAdd(b, wb, te.types->ConstI64(uint64((blk * 16 + j * 4) * mr + qd * w8)), "") var wlo : LLVMOpaqueValue? var whi : LLVMOpaqueValue? - if (te.kqBytes) { + if (k3) { + // k3: qs column (blk/4)*8 + j (lo) / + 4 (hi), 2-bit lane at shift 2*(blk%4); hmask column j / j+4, bit blk + let c0 = (blk / 4) * 8 + j + let sh = (blk % 4) * 2 + var q3l = load_vec(te, te.vwi8, te.wg, LLVMBuildAdd(b, wb, te.types->ConstI64(uint64(c0 * 4 * mr + qd * w8)), ""), "q3l{blk}_{j * rq + qd}") + var q3h = load_vec(te, te.vwi8, te.wg, LLVMBuildAdd(b, wb, te.types->ConstI64(uint64((c0 + 4) * 4 * mr + qd * w8)), ""), "q3h{blk}_{j * rq + qd}") + var h3l = load_vec(te, te.vwi8, te.wg, LLVMBuildAdd(b, wb, te.types->ConstI64(uint64((64 + j * 4) * mr + qd * w8)), ""), "h3l{blk}_{j * rq + qd}") + var h3h = load_vec(te, te.vwi8, te.wg, LLVMBuildAdd(b, wb, te.types->ConstI64(uint64((64 + (j + 4) * 4) * mr + qd * w8)), ""), "h3h{blk}_{j * rq + qd}") + var lo2 = LLVMBuildAnd(b, LLVMBuildLShr(b, q3l, splat_i8w(te, sh), ""), splat_i8w(te, 3), "") + var hi2 = LLVMBuildAnd(b, LLVMBuildLShr(b, q3h, splat_i8w(te, sh), ""), splat_i8w(te, 3), "") + var lob = LLVMBuildShl(b, LLVMBuildAnd(b, LLVMBuildLShr(b, h3l, splat_i8w(te, blk), ""), splat_i8w(te, 1), ""), splat_i8w(te, 2), "") + var hib = LLVMBuildShl(b, LLVMBuildAnd(b, LLVMBuildLShr(b, h3h, splat_i8w(te, blk), ""), splat_i8w(te, 1), ""), splat_i8w(te, 2), "") + wlo = LLVMBuildOr(b, lo2, lob, "wlo3{blk}_{j * rq + qd}") + whi = LLVMBuildOr(b, hi2, hib, "whi3{blk}_{j * rq + qd}") + } elif (te.kqBytes) { var hoff2 = LLVMBuildAdd(b, noff, te.types->ConstI64(uint64(128 * mr)), "") wlo = load_vec(te, te.vwi8, te.wg, noff, "wlo{blk}_{j * rq + qd}") whi = load_vec(te, te.vwi8, te.wg, hoff2, "whi{blk}_{j * rq + qd}") @@ -1052,7 +1069,7 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f var xhi = bcast_dword(te, te.x[tokBase + i], doff1, "xd{tokBase + i}_{blk}_{j}b") if (madd16) { let k = i * rq + qd - if (k6) { + if (k63) { p16lo[k] = madd16_acc(te, p16lo[k], wlo, xlo) p16hi[k] = madd16_acc(te, p16hi[k], whi, xhi) } else { @@ -1074,12 +1091,12 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f } } } - // i16 chain flushes at the format's overflow bound: k5 after 4 madds, k6 after 2; k4/q40 run the full 8 + // i16 chain flushes at the format's overflow bound: k5 after 4 madds, k6 (and k3, conservatively) after 2; k4/q40 run the full 8 if (madd16 && (j == 1 || j == 3) && te.kq != 4 && !q40) { for (k in range(tokCount * rq)) { a0[k] = madd16_flush(te, a0[k], p16lo[k]) p16lo[k] = null - if (k6) { + if (k63) { a1[k] = madd16_flush(te, a1[k], p16hi[k]) p16hi[k] = null } @@ -1099,7 +1116,7 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f for (qd in range(rq)) { if (q40) { dvb[qd] = load_f16_vec_at(te, te.sg, LLVMBuildAdd(b, sb, te.types->ConstI64(uint64(blk * 2 * mr + 2 * (qd * te.rv))), ""), "d{blk}_{qd}") - } elif (k6) { + } elif (k63) { var sc0p = LLVMBuildGEP2(b, te.types.t_int8, te.sg, LLVMBuildAdd(b, sb, te.types->ConstI64(uint64(2 * blk * mr + qd * te.rv)), ""), "scp{blk}_{qd}a") var sc1p = LLVMBuildGEP2(b, te.types.t_int8, te.sg, LLVMBuildAdd(b, sb, te.types->ConstI64(uint64((2 * blk + 1) * mr + qd * te.rv)), ""), "scp{blk}_{qd}b") scv[qd] = LLVMBuildSExt(b, LLVMBuildLoad2Aligned(b, vri8, sc0p, 1u, ""), te.vni32, "sc{blk}_{qd}a") @@ -1128,7 +1145,7 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f bs0 = LLVMBuildLoad2Aligned(b, te.types.t_int32, bp0, 4u, "bs{tk}_{blk}a") var bp1 = LLVMBuildGEP2(b, te.types.t_int32, te.xbs[tk], bi2b, "bp{tk}_{blk}b") bs1 = LLVMBuildLoad2Aligned(b, te.types.t_int32, bp1, 4u, "bs{tk}_{blk}b") - if (!k6) { + if (!k63) { a32v = splat_i32(te, LLVMBuildAdd(b, bs0, bs1, "a32{tk}_{blk}"), "a32v{tk}_{blk}") } } @@ -1138,7 +1155,7 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f var isub = LLVMBuildSub(b, acc, LLVMBuildMul(b, a32v, splat_i32n(te, 8), ""), "is{tk}_{blk}_{qd}") facc[i * rq + qd] = fold_fma(te, LLVMBuildSIToFP(b, isub, te.vnf32, ""), dvb[qd], facc[i * rq + qd], "fa{tk}_{blk}_{qd}") } - } elif (k6) { + } elif (k63) { var bs0v = splat_i32(te, bs0, "bs0v{tk}_{blk}") var bs1v = splat_i32(te, bs1, "bs1v{tk}_{blk}") for (qd in range(rq)) { @@ -1171,7 +1188,7 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f } } } - // superblock fold: k4/k5 iacc·d8 − bacc·dmin·d8; k6 (iacc−32·bacc)·d·d8 exact int32; q40 facc·d8 + // superblock fold: k4/k5 iacc·d8 − bacc·dmin·d8; k6/k3 (iacc−off·bacc)·d·d8 exact int32 (off 32 / 4); q40 facc·d8 for (i in range(tokCount)) { let tk = tokBase + i var qp = LLVMBuildGEP2(b, te.types.t_float, te.xs[tk], sbi, "qp{tk}") @@ -1183,8 +1200,8 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f continue } var ds = LLVMBuildFMul(b, dv[qd], d8v, "ds{tk}_{qd}") - if (k6) { - var isub = LLVMBuildSub(b, iacc[i * rq + qd], LLVMBuildMul(b, bacc[i * rq + qd], splat_i32n(te, 32), ""), "is{tk}_{qd}") + if (k63) { + var isub = LLVMBuildSub(b, iacc[i * rq + qd], LLVMBuildMul(b, bacc[i * rq + qd], splat_i32n(te, koff), ""), "is{tk}_{qd}") f[i * rq + qd] = fold_fma(te, LLVMBuildSIToFP(b, isub, te.vnf32, ""), ds, f[i * rq + qd], "f{tk}_{qd}s") } else { var ms = LLVMBuildFNeg(b, LLVMBuildFMul(b, mv[qd], d8v, ""), "ms{tk}_{qd}") @@ -2496,7 +2513,7 @@ def private kq_tile_gen_impl(var gc : LlvmCodeCtx; fmt : int) : bool { if (perm_declines(gc, p0)) return false let p = companion_perm(p0) - var te = TileEmit(kq = fmt, kqBytes = fmt == 5 || fmt == 6) // k4/q40/iq4xs tiles read the packed planes + var te = TileEmit(kq = fmt, kqBytes = fmt == 5 || fmt == 6) // k4/q40/iq4xs/k3 tiles read the packed planes if (!setup_tile_emit(te, gc, p, fmt == 44)) return false let b = gc.jit.builder @@ -2542,12 +2559,9 @@ def private kq_tile_gen_impl(var gc : LlvmCodeCtx; fmt : int) : bool { def private iq4xs_gemv_gen(var gc : LlvmCodeCtx) : bool => kq_gemv_gen_impl(gc, 44) def private iq4xs_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 44) -// k3: the 2-bit + hmask emitter arm lands with the kernel arc; until then both stamps decline -// (reference bodies serve) — a declined generator is the framework's own fallback path. -[unused_argument(gc)] -def private k3_gemv_gen(var gc : LlvmCodeCtx) : bool => false -[unused_argument(gc)] -def private k3_tile_gen(var gc : LlvmCodeCtx) : bool => false +// k3 rides the kq walks through emit_block_kqv2's k6 arms (the k3 flag swaps the quant compose and the offset) +def private k3_gemv_gen(var gc : LlvmCodeCtx) : bool => kq_gemv_gen_impl(gc, 3) +def private k3_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 3) def private k4_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 4) def private k5_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 5) diff --git a/modules/dasLLAMA/harness/gen_tune_probe.das b/modules/dasLLAMA/harness/gen_tune_probe.das index 505a2fb709..28b20f1053 100644 --- a/modules/dasLLAMA/harness/gen_tune_probe.das +++ b/modules/dasLLAMA/harness/gen_tune_probe.das @@ -1211,6 +1211,7 @@ def test_mode : bool { // nolint:STYLE037,STYLE038 — flat per-variant correctn var kfxs6 <- [ <- build_kq_fixture(6l, 256l, 32l, 8l), <- build_kq_fixture(6l, 768l, 32l, 8l), <- build_kq_fixture(6l, 2048l, 32l, 8l)] var kfxs40 <- [ <- build_kq_fixture(40l, 256l, 32l, 8l), <- build_kq_fixture(40l, 768l, 32l, 8l), <- build_kq_fixture(40l, 2048l, 32l, 8l)] var kfxs44 <- [ <- build_kq_fixture(44l, 256l, 32l, 8l), <- build_kq_fixture(44l, 768l, 32l, 8l), <- build_kq_fixture(44l, 2048l, 32l, 8l)] + var kfxs3 <- [ <- build_kq_fixture(3l, 256l, 32l, 8l), <- build_kq_fixture(3l, 768l, 32l, 8l), <- build_kq_fixture(3l, 2048l, 32l, 8l)] var vs <- q8q8_tile_gen_variants() var mrs <- variant_mrs() var wbs <- variant_wbias() @@ -1336,8 +1337,7 @@ def test_mode : bool { // nolint:STYLE037,STYLE038 — flat per-variant correctn allok = kq_test_family(6l, kfxs6) && allok allok = kq_test_family(40l, kfxs40) && allok allok = kq_test_family(44l, kfxs44) && allok - // k3 joins this list with its emitter arm: a stub family fails every perm whose layout - // companion generates (planes at mr 8, the reference body at mr 4 - HOW_TO QUIRK 16) + allok = kq_test_family(3l, kfxs3) && allok delete wtab delete mtab delete gtab @@ -1354,6 +1354,7 @@ def test_mode : bool { // nolint:STYLE037,STYLE038 — flat per-variant correctn delete kfxs6 delete kfxs40 delete kfxs44 + delete kfxs3 return allok } From 1cfcda5a30c6f58c1f968d404e7dc61fe7378309 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Sun, 30 Aug 2026 10:44:35 -0700 Subject: [PATCH 09/61] dasLLAMA: Q3_K on the Vulkan tier - KqGemvK3 + KqBatchK3 (k6's classes with the 2-bit + hmask compose, offset 4), the id bridge, the device gather's k3 arms (and iq4xs's missing tail-row arm), six-format cells + a float witness; HOW_TO note + QUIRK 19 vk_kq_schema_id maps 7 -> 3. KqGemvK3 is k6's blk_contrib over k3_dot (6 uint4 per superblock: qs 0..3 shifted by 2*(blk%4), hmask 4..5 bit blk shifted up two) with the offset 4; KqBatchK3 is a child of KqBatchK6 (stage_ws inherited) composing the staged words and folding with 4. Five class ladders gain an arm, the class oracle picks the class, k3_gemv_float_oracle is the witness off the plane bytes. moe_gpu_gather_stack_kq gains k3 arms in the grouped and the tail-row branches - and the tail-row branch now also copies iq4xs planes verbatim instead of re-pairing nibbles and re-decoding scales (latent: every row of the dense 1B is grouped). Gates: test_vulkan_kernels 64/64 with the family cells at six formats; the 1B Q3_K_L on the resident driver matches llama.cpp's greedy ids for 51 of 64 tokens at gen 284 t/s. Co-Authored-By: Claude Fable 5 --- modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md | 19 ++- modules/dasLLAMA/dasllama/dasllama_layout.das | 21 +++- .../dasllama/dasllama_vulkan_classes.das | 112 +++++++++++++++++- .../dasllama/dasllama_vulkan_common.das | 7 +- modules/dasLLAMA/tests/_vkd_oracles.das | 39 ++++++ .../dasLLAMA/tests/test_vulkan_kernels.das | 30 +++-- 6 files changed, 206 insertions(+), 22 deletions(-) diff --git a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md index f47c710899..bb7cdd9dde 100644 --- a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md +++ b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md @@ -363,6 +363,13 @@ where, why it is so today, what unquirked looks like. An empty ledger is a legit sidecar entry for the family (copy the app's sidecar next to the probe as `.tune.json`, edit the one entry). Unquirked: a policy value that names the reference tier outright. +19. **The device gather's tail-row branch is a second ladder.** `moe_gpu_gather_stack_kq` + walks grouped rows (the grp interleave) and ungrouped rows (`d % mr` tails, or an + unrepacked load) through two different per-format ladders; the ungrouped one re-pairs k4 + nibbles and re-decodes the 6-bit scale packing in its `else`, so a format that is already in + the device form (q40, iq4xs, k3) needs its verbatim arm in BOTH. iq4xs had only the grouped + one - latent, because every row of the dense 1B is grouped - and the k3 walk found it. + Unquirked: one per-format "device form" predicate both branches key on. ## Per-format notes @@ -395,7 +402,17 @@ live, maxdiff 4.8e-7) and on the M1 (sdot mr 4/8 stamps bit-exact), `test_kquant at token 15 - a near tie: against the reference body on two real k3 tensors (10240 output rows, scratchpad `k3_rows.das` with a `"reference"` sidecar entry) 4957 rows are bit-identical and the worst relative difference is 1.7e-4 on a row of magnitude 6e-5 (fp32 fold-order noise). -Vulkan, Metal: pending. + +Vulkan: `vk_kq_schema_id` 7 -> 3; `KqGemvK3 : KqGemvBase` (k6's blk_contrib with `k3_dot` - +the lane/mask compose over 6 uint4 per superblock, qs 0..3 and hmask 4..5 - and the offset 4) +and `KqBatchK3 : KqBatchK6` (a child of the k6 class: `stage_ws` inherited, `stage_w` composes +the staged words, `blk_fma` is k6's with 4). A child class must follow its parent in the file +(the class rail resolves parents in order). Five ladders, the class oracle arm, family cells at +six formats, `k3_gemv_float_oracle` as the witness. The device gather +(`moe_gpu_gather_stack_kq`) gained k3 arms in both its grouped and tail-row branches - and the +tail-row branch turned out to lack iq4xs arms too (QUIRK 19). `test_vulkan_kernels` 64/64; +the 1B Q3_K_L on the resident driver matches llama.cpp's greedy ids for 51 of 64 tokens at gen +284 t/s. Metal: pending. ### IQ4_XS (the pilot, 2026-08-30) diff --git a/modules/dasLLAMA/dasllama/dasllama_layout.das b/modules/dasLLAMA/dasllama/dasllama_layout.das index 0eda6b4aab..9796f12ebf 100644 --- a/modules/dasLLAMA/dasllama/dasllama_layout.das +++ b/modules/dasLLAMA/dasllama/dasllama_layout.das @@ -780,8 +780,17 @@ def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice if (grouped) { let gq = qp + sliceQ + g * mr * qrow + sbi * qsb * mr let gs = sp + sliceS + g * mr * srow + sbi * dssb * mr - for (m in range64(128l)) { // nibble region: verbatim per (blk, j, t) - dq[m] = gq[((m / 4l) * mr + r) * 4l + m % 4l] + if (fmt == KqFmt.k3) { // 16 qs columns, then the 8 hmask columns at 64*mr + for (m in range64(64l)) { + dq[m] = gq[((m / 4l) * mr + r) * 4l + m % 4l] + } + for (m in range64(32l)) { + dq[64l + m] = gq[64l * mr + ((m / 4l) * mr + r) * 4l + m % 4l] + } + } else { + for (m in range64(128l)) { // nibble region: verbatim per (blk, j, t) + dq[m] = gq[((m / 4l) * mr + r) * 4l + m % 4l] + } } if (fmt == KqFmt.k5) { for (bj in range64(32l)) { @@ -792,7 +801,7 @@ def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice dq[128l + p] = gq[128l * mr + ((p / 4l) * mr + r) * 4l + p % 4l] } } - if (fmt == KqFmt.k6) { + if (fmt == KqFmt.k6 || fmt == KqFmt.k3) { for (idx in range64(16l)) { dsc[idx] = gs[idx * mr + r] } @@ -821,8 +830,8 @@ def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice } else { let rq = qp + sliceQ + ri * qrow + sbi * qsb let rs = sp + sliceS + ri * srow + sbi * dssb - if (fmt == KqFmt.q40) { - for (m in range64(128l)) { // Q4_0 bytes already pair k / k+16 + if (fmt == KqFmt.q40 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3) { + for (m in range64(qsb)) { // disk bytes already in the device form (k/k+16 pairs; k3's lanes) dq[m] = rq[m] } } else { @@ -848,7 +857,7 @@ def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice dq[128l + p] = rq[128l + p] } } - if (fmt == KqFmt.k6 || fmt == KqFmt.q40) { + if (fmt == KqFmt.k6 || fmt == KqFmt.q40 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3) { // rows already in the decoded device form for (idx in range64(dssb)) { dsc[idx] = rs[idx] } diff --git a/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das b/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das index c541080579..5ca386026c 100644 --- a/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das +++ b/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das @@ -2876,6 +2876,66 @@ class KqBatchK6 : KqBatchBase { } } +// Q3_K tile: k6's split-half fold with the offset 4 over the k3 planes (24 words per superblock: +// 16 qs words, 8 hmask words) — block bu takes qs words (bu/4)*8 + hh*4.. at shift 2*(bu%4) +// OR the hmask words 16 + hh*4.. bit bu, shifted up two +[vk_dispatch(name = "kq_batch_k3_cls", kernel = "run", family = "kq_batch_cls", grid = "wgs", params = "wgs : int64")] +class KqBatchK3 : KqBatchK6 { + def override stage_w(wsb0, wcol, nsb, s, word, grp : uint) { + for (ki in range(2)) { + let k = word + uint(ki) * 8u + let bu = k / 2u + let hh = k % 2u + var wv = uint4(0u) + if (wcol < pa.d) { + let wsb = wsb0 + wcol * nsb + s + let qb4 = wsb * 24u + (bu / 4u) * 8u + hh * 4u + let hb4 = wsb * 24u + 16u + hh * 4u + let shift = (bu % 4u) * 2u + wv = uint4(((wq[qb4] >> shift) & 0x03030303) | (((wq[hb4] >> bu) & 0x01010101) << 2u), + ((wq[qb4 + 1u] >> shift) & 0x03030303) | (((wq[hb4 + 1u] >> bu) & 0x01010101) << 2u), + ((wq[qb4 + 2u] >> shift) & 0x03030303) | (((wq[hb4 + 2u] >> bu) & 0x01010101) << 2u), + ((wq[qb4 + 3u] >> shift) & 0x03030303) | (((wq[hb4 + 3u] >> bu) & 0x01010101) << 2u)) + } + ww4[grp * 17u + k] = wv + } + } + + def override blk_fma(bu, grp, word : uint; xscl : float) : float4 { + let kL = bu * 2u + let kH = bu * 2u + 1u + let xwL = xw4[grp * 17u + kL] + let xwH = xw4[grp * 17u + kH] + let blo = sdot4(xwL.x, 0x01010101) + sdot4(xwL.y, 0x01010101) + sdot4(xwL.z, 0x01010101) + sdot4(xwL.w, 0x01010101) + let bhi = sdot4(xwH.x, 0x01010101) + sdot4(xwH.y, 0x01010101) + sdot4(xwH.z, 0x01010101) + sdot4(xwH.w, 0x01010101) + let wL0 = ww4[word * 17u + kL] + let wL1 = ww4[(word + 8u) * 17u + kL] + let wL2 = ww4[(word + 16u) * 17u + kL] + let wL3 = ww4[(word + 24u) * 17u + kL] + let wH0 = ww4[word * 17u + kH] + let wH1 = ww4[(word + 8u) * 17u + kH] + let wH2 = ww4[(word + 16u) * 17u + kH] + let wH3 = ww4[(word + 24u) * 17u + kH] + let l0 = sdot4(wL0.x, xwL.x) + sdot4(wL0.y, xwL.y) + sdot4(wL0.z, xwL.z) + sdot4(wL0.w, xwL.w) + let l1 = sdot4(wL1.x, xwL.x) + sdot4(wL1.y, xwL.y) + sdot4(wL1.z, xwL.z) + sdot4(wL1.w, xwL.w) + let l2 = sdot4(wL2.x, xwL.x) + sdot4(wL2.y, xwL.y) + sdot4(wL2.z, xwL.z) + sdot4(wL2.w, xwL.w) + let l3 = sdot4(wL3.x, xwL.x) + sdot4(wL3.y, xwL.y) + sdot4(wL3.z, xwL.z) + sdot4(wL3.w, xwL.w) + let h0 = sdot4(wH0.x, xwH.x) + sdot4(wH0.y, xwH.y) + sdot4(wH0.z, xwH.z) + sdot4(wH0.w, xwH.w) + let h1 = sdot4(wH1.x, xwH.x) + sdot4(wH1.y, xwH.y) + sdot4(wH1.z, xwH.z) + sdot4(wH1.w, xwH.w) + let h2 = sdot4(wH2.x, xwH.x) + sdot4(wH2.y, xwH.y) + sdot4(wH2.z, xwH.z) + sdot4(wH2.w, xwH.w) + let h3 = sdot4(wH3.x, xwH.x) + sdot4(wH3.y, xwH.y) + sdot4(wH3.z, xwH.z) + sdot4(wH3.w, xwH.w) + return float4(xscl * (wsw[bu * 32u + word] * float(l0 - 4 * blo) + wsw2[bu * 32u + word] * float(h0 - 4 * bhi)), + xscl * (wsw[bu * 32u + word + 8u] * float(l1 - 4 * blo) + wsw2[bu * 32u + word + 8u] * float(h1 - 4 * bhi)), + xscl * (wsw[bu * 32u + word + 16u] * float(l2 - 4 * blo) + wsw2[bu * 32u + word + 16u] * float(h2 - 4 * bhi)), + xscl * (wsw[bu * 32u + word + 24u] * float(l3 - 4 * blo) + wsw2[bu * 32u + word + 24u] * float(h3 - 4 * bhi))) + } + + [spirv_kernel(local_size_x = 256, name = "kq_batch_k3_cls_spv")] + def override run { + tile_shell() + } +} + // ===== the prefill batch GEMMs — cooperative-matrix variants (DASLLAMA_COOPMAT=f16|int8|mm) ===== let private CML_SST = 20u // mm shared row stride in uints (16 f16-pair data + 4 pad — bank spread) @@ -5789,6 +5849,8 @@ def kq_batch_cls_ensure(fmt : int; q40cm : bool) : bool { return ensure_kq_batch_q40_cls() } elif (fmt == int(KqFmt.iq4xs)) { return ensure_kq_batch_iq4xs_cls() + } elif (fmt == int(KqFmt.k3)) { + return ensure_kq_batch_k3_cls() } panic("dasLLAMA vulkan tier: no kq batch class for format {fmt}") return false @@ -5817,6 +5879,8 @@ def kq_batch_cls_enc_for(fmt : int; q40cm : bool; raw : VkCommandBuffer; var h : enc_kq_batch_q40_cls(raw, h, set_, pc, wgs) } elif (fmt == int(KqFmt.iq4xs)) { enc_kq_batch_iq4xs_cls(raw, h, set_, pc, wgs) + } elif (fmt == int(KqFmt.k3)) { + enc_kq_batch_k3_cls(raw, h, set_, pc, wgs) } else { panic("dasLLAMA vulkan tier: no kq batch class for format {fmt}") } @@ -5839,6 +5903,8 @@ def gemv_cls_ensure(fmt : int) : bool { return ensure_kq_gemv_q40_cls() } elif (fmt == int(KqFmt.iq4xs)) { return ensure_kq_gemv_iq4xs_cls() + } elif (fmt == int(KqFmt.k3)) { + return ensure_kq_gemv_k3_cls() } panic("dasLLAMA vulkan tier: no GEMV class for format {fmt}") return false @@ -5867,14 +5933,16 @@ def gemv_cls_enc(fmt : int; raw : VkCommandBuffer; var h : VkHaz; var set_ : VkD enc_kq_gemv_q40_cls(raw, h, set_, pc, wgs) } elif (fmt == int(KqFmt.iq4xs)) { enc_kq_gemv_iq4xs_cls(raw, h, set_, pc, wgs) + } elif (fmt == int(KqFmt.k3)) { + enc_kq_gemv_k3_cls(raw, h, set_, pc, wgs) } else { panic("dasLLAMA vulkan tier: no GEMV class for format {fmt}") } } // Q4_K: w = d*sc*q - dmin*mn*1, q in [0,15] — per 32-block axs*(d*sc*idot - dmin*mn*bsum). -// The five formats share family=kq_gemv_cls: ONE set layout + ONE set_kq_gemv_cls serves all -// five pipelines (identical inherited members), each leaf keeping its own ensure_/enc_ names. +// The six formats share family=kq_gemv_cls: ONE set layout + ONE set_kq_gemv_cls serves all +// six pipelines (identical inherited members), each leaf keeping its own ensure_/enc_ names. [vk_dispatch(name = "kq_gemv_k4_cls", kernel = "run", family = "kq_gemv_cls", grid = "wgs", params = "wgs : int64")] class KqGemvK4 : KqGemvBase { def override blk_contrib(wsb, blk, xsb : uint) : float { @@ -5937,6 +6005,46 @@ class KqGemvIq4xs : KqGemvBase { } } +// Q3_K: q = 2-bit qs lane | hmask bit << 2, minus 4, per-16 SIGNED sub-scales x d — k6's fold with +// the offset 4 over the k3 planes (6 uint4 per superblock: qs 0..3, hmask 4..5) +[vk_dispatch(name = "kq_gemv_k3_cls", kernel = "run", family = "kq_gemv_cls", grid = "wgs", params = "wgs : int64")] +class KqGemvK3 : KqGemvBase { + // (ilo, ihi, blo, bhi) — an int4, since a shader function cannot return a tuple + def k3_dot(ql, qh, xl, xh, hl, hh : uint4; sh, bit : uint) : int4 { + let m2 = 0x03030303u + let m1 = 0x01010101u + let l0 = ((ql.x >> sh) & m2) | (((hl.x >> bit) & m1) << 2u) + let l1 = ((ql.y >> sh) & m2) | (((hl.y >> bit) & m1) << 2u) + let l2 = ((ql.z >> sh) & m2) | (((hl.z >> bit) & m1) << 2u) + let l3 = ((ql.w >> sh) & m2) | (((hl.w >> bit) & m1) << 2u) + let h0 = ((qh.x >> sh) & m2) | (((hh.x >> bit) & m1) << 2u) + let h1 = ((qh.y >> sh) & m2) | (((hh.y >> bit) & m1) << 2u) + let h2 = ((qh.z >> sh) & m2) | (((hh.z >> bit) & m1) << 2u) + let h3 = ((qh.w >> sh) & m2) | (((hh.w >> bit) & m1) << 2u) + return int4(sdot4(l0, xl.x) + sdot4(l1, xl.y) + sdot4(l2, xl.z) + sdot4(l3, xl.w), + sdot4(h0, xh.x) + sdot4(h1, xh.y) + sdot4(h2, xh.z) + sdot4(h3, xh.w), + sdot4(xl.x, KQC_ONES) + sdot4(xl.y, KQC_ONES) + sdot4(xl.z, KQC_ONES) + sdot4(xl.w, KQC_ONES), + sdot4(xh.x, KQC_ONES) + sdot4(xh.y, KQC_ONES) + sdot4(xh.z, KQC_ONES) + sdot4(xh.w, KQC_ONES)) + } + + def override blk_contrib(wsb, blk, xsb : uint) : float { + let xb4 = xsb * 16u + blk * 2u + let q4 = wsb * 6u + (blk / 4u) * 2u + let dot = k3_dot(wq4[q4], wq4[q4 + 1u], xq4[xb4], xq4[xb4 + 1u], + wq4[wsb * 6u + 4u], wq4[wsb * 6u + 5u], (blk % 4u) * 2u, blk) + let s4 = int4(unpack8(int(wsu[wsb * 5u + blk / 2u]))) // 4 signed sub-scales + let slo = (blk % 2u) == 0u ? s4.x : s4.z + let shi = (blk % 2u) == 0u ? s4.y : s4.w + let dd = unpackHalf2x16(wsu[wsb * 5u + 4u]).x + return axs[xsb] * dd * float(slo * (dot.x - 4 * dot.z) + shi * (dot.y - 4 * dot.w)) + } + + [spirv_kernel(local_size_x = 64, name = "kq_gemv_k3_cls_spv")] + def run { + gemv_shell() + } +} + // Q5_K: the k4 fold with the 5th bit deposited from the block's qh word [vk_dispatch(name = "kq_gemv_k5_cls", kernel = "run", family = "kq_gemv_cls", grid = "wgs", params = "wgs : int64")] class KqGemvK5 : KqGemvBase { diff --git a/modules/dasLLAMA/dasllama/dasllama_vulkan_common.das b/modules/dasLLAMA/dasllama/dasllama_vulkan_common.das index 59c9b847b0..ffbd29e2e9 100644 --- a/modules/dasLLAMA/dasllama/dasllama_vulkan_common.das +++ b/modules/dasLLAMA/dasllama/dasllama_vulkan_common.das @@ -1709,7 +1709,7 @@ struct ArenaSlab { } struct ArenaFmt { - fmt : int // int(KqFmt): 0 = q8, 1/2/3 = k4/k5/k6, 4 = q40, 6 = iq4xs + fmt : int // int(KqFmt): 0 = q8, 1/2/3 = k4/k5/k6, 4 = q40, 6 = iq4xs, 7 = k3 slabs : array cur : int // slab the bump cursor is in slab_cap : int64 // max blocks one slab may carry (msr over both plane strides) @@ -1740,18 +1740,19 @@ def arena_local_blk(blk : int64) : int64 => blk & ((1l << 32l) - 1l) // per-block plane strides — the block is 32 weights for q8 and a 256-weight superblock for kq. // Mirrors stack_plane_bytes: it computes the same totals as count * these strides. -// the tier's stack fmt space is int(KqFmt) — bridge to the gemm_schema kq id space (4/5/6/40/44) +// the tier's stack fmt space is int(KqFmt) — bridge to the gemm_schema kq id space (4/5/6/40/44/3) def vk_kq_schema_id(fmt : int) : int { if (fmt == 1) return 4 if (fmt == 2) return 5 if (fmt == 3) return 6 if (fmt == 4) return 40 if (fmt == 6) return 44 + if (fmt == 7) return 3 panic("vk_kq_schema_id: not a kq stack fmt") return 0 } -// fmt space here = int(KqFmt): 0 q8 (32-elem blocks), 1-4 and 6 the kq lattice (+ q40, iq4xs) +// fmt space here = int(KqFmt): 0 q8 (32-elem blocks), 1-4, 6 and 7 the kq lattice (+ q40, iq4xs, k3) def vk_fmt_b32(fmt : int) : bool => fmt == 0 // meta/activation block unit of a stack format — q8 counts 32-weight blocks, the kq diff --git a/modules/dasLLAMA/tests/_vkd_oracles.das b/modules/dasLLAMA/tests/_vkd_oracles.das index 7d1f87985e..8fc68ce7ab 100644 --- a/modules/dasLLAMA/tests/_vkd_oracles.das +++ b/modules/dasLLAMA/tests/_vkd_oracles.das @@ -171,6 +171,8 @@ def kq_cls_ref(fmt : int; wq, ws, xq : array; xs : array) : KqGemvB k = new KqGemvK5() } elif (fmt == int(KqFmt.iq4xs)) { k = new KqGemvIq4xs() + } elif (fmt == int(KqFmt.k3)) { + k = new KqGemvK3() } else { k = new KqGemvK6() } @@ -243,6 +245,43 @@ def iq4xs_gemv_float_oracle(wq, ws, xq : array; xs : array; return <- y } +// Q3_K GEMV float oracle, INDEPENDENT of the class bit-math: w = d * sc16 * (q - 4) straight off +// the plane bytes (qs lane at shift 2*(blk%4), hmask bit blk, k6's 5-word scale row), x off the +// Q8_K bytes — the witness a wrong lane/mask pick in k3_dot cannot pass while staying self-consistent +def k3_gemv_float_oracle(wq, ws, xq : array; xs : array; + regions : array; n, d, nreg : int) : array { + var y : array + y |> resize(nreg * d) + let nsb = n / 256 + for (r in range(nreg)) { + for (row in range(d)) { + let w0 = int(regions[r * 2]) + row * nsb + let x0 = int(regions[r * 2 + 1]) + var acc = 0.0 + for (s in range(nsb)) { + let sb = w0 + s + let xsb = x0 + s + let dd = half_at(ws, sb * 10 + 8) + for (blk in range(8)) { + let sh = 2 * (blk % 4) + let s0 = byte_s8(ws, sb * 20 + 2 * blk) + let s1 = byte_s8(ws, sb * 20 + 2 * blk + 1) + var bacc = 0.0 + for (l in range(32)) { + let q2 = (byte_u8(wq, sb * 96 + (blk / 4) * 32 + l) >> sh) & 3 + let hb = (byte_u8(wq, sb * 96 + 64 + l) >> blk) & 1 + let q = (q2 | (hb << 2)) - 4 + bacc += float(l < 16 ? s0 : s1) * float(q) * float(byte_s8(xq, xsb * 256 + blk * 32 + l)) + } + acc += xs[xsb] * dd * bacc + } + } + y[r * d + row] = acc + } + } + return <- y +} + // kq batch GEMM oracle over region records — same blk_contrib per (row, col, block) def kq_gemm_oracle(fmt : int; wq, ws, xq : array; xs : array; recs : array; nrec, n, d : int; var y : array) { diff --git a/modules/dasLLAMA/tests/test_vulkan_kernels.das b/modules/dasLLAMA/tests/test_vulkan_kernels.das index fba6cb47c8..13a12348d7 100644 --- a/modules/dasLLAMA/tests/test_vulkan_kernels.das +++ b/modules/dasLLAMA/tests/test_vulkan_kernels.das @@ -379,9 +379,9 @@ def private attn_row_oracle(q : array; qbase : int; kk : array; kb [test] def test_vkd_kq_gemv_family(t0 : T?) { - t0 |> run("kq GEMV class family == the CPU oracle (all 5 formats)") <| @(t : T?) { + t0 |> run("kq GEMV class family == the CPU oracle (all 6 formats)") <| @(t : T?) { static_if (typeinfo builtin_module_exists(vulkan)) { - if (!(ensure_kq_gemv_k4_cls() && ensure_kq_gemv_q40_cls() && ensure_kq_gemv_k5_cls() && ensure_kq_gemv_k6_cls() && ensure_kq_gemv_iq4xs_cls())) { + if (!(ensure_kq_gemv_k4_cls() && ensure_kq_gemv_q40_cls() && ensure_kq_gemv_k5_cls() && ensure_kq_gemv_k6_cls() && ensure_kq_gemv_iq4xs_cls() && ensure_kq_gemv_k3_cls())) { feint("no Vulkan device - skipping\n") return } @@ -389,9 +389,9 @@ def test_vkd_kq_gemv_family(t0 : T?) { let d = 3 // rows per region let nreg = 2 let rows = nreg * d - let nfmt = 5 - let fmts = fixed_array(int(KqFmt.k4), int(KqFmt.q40), int(KqFmt.k5), int(KqFmt.k6), int(KqFmt.iq4xs)) - let stride4s = fixed_array(8, 8, 10, 12, 8) // uint4s per superblock + let nfmt = 6 + let fmts = fixed_array(int(KqFmt.k4), int(KqFmt.q40), int(KqFmt.k5), int(KqFmt.k6), int(KqFmt.iq4xs), int(KqFmt.k3)) + let stride4s = fixed_array(8, 8, 10, 12, 8, 6) // uint4s per superblock for (fi in range(nfmt)) { let fmt = fmts[fi] let stride4 = stride4s[fi] @@ -463,6 +463,8 @@ def test_vkd_kq_gemv_family(t0 : T?) { enc_kq_gemv_k5_cls(raw2, h2, s_cls, pc, wgs) } elif (fmt == int(KqFmt.iq4xs)) { enc_kq_gemv_iq4xs_cls(raw2, h2, s_cls, pc, wgs) + } elif (fmt == int(KqFmt.k3)) { + enc_kq_gemv_k3_cls(raw2, h2, s_cls, pc, wgs) } else { enc_kq_gemv_k6_cls(raw2, h2, s_cls, pc, wgs) } @@ -481,6 +483,12 @@ def test_vkd_kq_gemv_family(t0 : T?) { t |> success(badf == 0, "iq4xs: the class bit-math matches the float dequant oracle ({badf} of {rows} rows off)") delete y_flt } + if (fmt == int(KqFmt.k3)) { // the lane/mask compose has an independent float witness + var y_flt <- k3_gemv_float_oracle(wqh, wsh, xqh, xsh, regs, 512, d, nreg) + let badf = mismatch_approx(y_ref, y_flt) + t |> success(badf == 0, "k3: the class bit-math matches the float dequant oracle ({badf} of {rows} rows off)") + delete y_flt + } delete regs delete y_ref delete wqh @@ -999,9 +1007,9 @@ def test_vkd_q8_batch_tile(t0 : T?) { [test] def test_vkd_kq_batch_family(t0 : T?) { - t0 |> run("kq batch tile class family == the CPU oracle (all 5 formats)") <| @(t : T?) { + t0 |> run("kq batch tile class family == the CPU oracle (all 6 formats)") <| @(t : T?) { static_if (typeinfo builtin_module_exists(vulkan)) { - if (!(ensure_kq_batch_k4_cls() && ensure_kq_batch_q40_cls() && ensure_kq_batch_k5_cls() && ensure_kq_batch_k6_cls() && ensure_kq_batch_iq4xs_cls())) { + if (!(ensure_kq_batch_k4_cls() && ensure_kq_batch_q40_cls() && ensure_kq_batch_k5_cls() && ensure_kq_batch_k6_cls() && ensure_kq_batch_iq4xs_cls() && ensure_kq_batch_k3_cls())) { feint("no Vulkan device - skipping\n") return } @@ -1016,9 +1024,9 @@ def test_vkd_kq_batch_family(t0 : T?) { let wgs1 = ((cnt1 + 31) / 32) * wtiles let wgs = wgs0 + wgs1 let totsb = 2 * d * nsb // superblocks across both regions - let nfmt = 5 - let wstrides = fixed_array(32, 32, 40, 48, 32) // uint words per superblock: k4 q40 k5 k6 iq4xs - let pipes = fixed_array(int(KqFmt.k4), int(KqFmt.q40), int(KqFmt.k5), int(KqFmt.k6), int(KqFmt.iq4xs)) + let nfmt = 6 + let wstrides = fixed_array(32, 32, 40, 48, 32, 24) // uint words per superblock: k4 q40 k5 k6 iq4xs k3 + let pipes = fixed_array(int(KqFmt.k4), int(KqFmt.q40), int(KqFmt.k5), int(KqFmt.k6), int(KqFmt.iq4xs), int(KqFmt.k3)) for (fi in range(nfmt)) { let fmt = pipes[fi] let wstride = wstrides[fi] @@ -1093,6 +1101,8 @@ def test_vkd_kq_batch_family(t0 : T?) { enc_kq_batch_k5_cls(raw2, h2, sc, pc, int64(wgs)) } elif (fmt == int(KqFmt.iq4xs)) { enc_kq_batch_iq4xs_cls(raw2, h2, sc, pc, int64(wgs)) + } elif (fmt == int(KqFmt.k3)) { + enc_kq_batch_k3_cls(raw2, h2, sc, pc, int64(wgs)) } else { enc_kq_batch_k6_cls(raw2, h2, sc, pc, int64(wgs)) } From 0f60e3d74c3c990c8e30154018b281ca43b6cdcd Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Sun, 30 Aug 2026 11:05:56 -0700 Subject: [PATCH 10/61] dasLLAMA: Q3_K on the Metal tier - the k6 split scale form verbatim, MetalKqGemvK3 + the B2/B4/B8 mv twins + a K3 mul_mm arm (k6's kernels with the 2-bit + hmask compose, offset 4), the ladders and the gate; tests at fmt 3; HOW_TO note The k3 scale row is already the k6 form, so the blob transform, kq_scales_of and the alignment rule are k6's arms over t.k3s; quants bind verbatim (96 B per superblock: 16 qs uints, 8 hmask uints). Kernels: MetalKqGemvK3 (k6's lane map - a lane's 4 elements of each of a half's 4 blocks share the qs bytes at shifts 0/2/4/6 and hmask bits 4ip..4ip+3), MetalKqMvK3T B2/B4 and MetalKqMvB8K3 (k6's slot map), a K3 arm in MetalKqMulMmK45T beside SIXBIT (MetalKqMulMmK3). Ladders: enc_kq_gemv / enc_kq_mvb / enc_kq_gemm_mm_b, pf_enc_kq_site_mm (base mul_mm), pf_devw_panel_kq (declines), pf_enc_kq_gemv, moe_site_ok, the sb predicates now kq_sb, kq_fmt_gpu_supported. Fixtures at fmt 3 reuse the k6 split-form fill; kq_row_ref dequantizes through dequant_k3_plane_superblock_at. Gates on the M1 Max: test_metal_gemv_kernels 2/2 and test_metal_gemm_kernels 2/2 with the k3 GEMV / mvb2,4,8 / mul_mm cells; lint clean; the 1B Q3_K_L on the Metal tier decodes a coherent story at gen 223 t/s (the other side of the token-15 near tie). Also ledgers followup_general.md `#59`, the partial mint. Co-Authored-By: Claude Fable 5 --- modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md | 16 +- modules/dasLLAMA/dasllama/dasllama_layout.das | 23 +- .../dasllama/dasllama_metal_common.das | 16 +- .../dasllama/dasllama_metal_kernels.das | 390 ++++++++++++++++-- .../dasllama/dasllama_metal_prefill.das | 8 +- .../dasllama/dasllama_metal_shapes.das | 10 +- modules/dasLLAMA/followup_general.md | 12 + .../dasLLAMA/tests/_metal_kernel_common.das | 11 +- .../tests/test_metal_gemm_kernels.das | 24 +- .../tests/test_metal_gemv_kernels.das | 38 +- 10 files changed, 468 insertions(+), 80 deletions(-) diff --git a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md index bb7cdd9dde..72ec615961 100644 --- a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md +++ b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md @@ -412,7 +412,21 @@ six formats, `k3_gemv_float_oracle` as the witness. The device gather (`moe_gpu_gather_stack_kq`) gained k3 arms in both its grouped and tail-row branches - and the tail-row branch turned out to lack iq4xs arms too (QUIRK 19). `test_vulkan_kernels` 64/64; the 1B Q3_K_L on the resident driver matches llama.cpp's greedy ids for 51 of 64 tokens at gen -284 t/s. Metal: pending. +284 t/s. + +Metal: the k3 scale row IS the k6 form, so `metal_blob_scale_plane("k3s")` is the k6 split +verbatim over `t.k3s`, `kq_scales_of` the k6 arm, `metal_blob_off_ok` the k6 rule; quants bind +verbatim (96 B). Kernels are k6's twins with the compose: `MetalKqGemvK3` (k6's lane map - a +lane's 4 elements of each of a half's 4 blocks share the qs bytes at shifts 0/2/4/6 and the +hmask bits 4ip..4ip+3, exactly k6's `sums[0..3]` shape), `MetalKqMvK3T` B2/B4 + `MetalKqMvB8K3` +(k6's slot map over 24 uints per superblock: qs 0..15, hmask 16..23), a `K3` arm in +`MetalKqMulMmK45T` beside `SIXBIT` (the k4/k5/iq4xs arms re-nested once more - QUIRK 14). A +local named `half3` is a reserved vector type name in daslang; the emitter reports it as a +syntax error at the `let`. Ladders as iq4xs's; fixtures at fmt 3 reuse the k6 split-form fill, +`kq_row_ref` via `dequant_k3_plane_superblock_at`. Gates on the M1 Max: +`test_metal_gemv_kernels` 2/2, `test_metal_gemm_kernels` 2/2; the 1B Q3_K_L on the Metal tier +(`k3_metal_probe.das`, the iq4xs probe with the model swapped) decodes a coherent story at gen 223 +t/s, taking the other side of the same token-15 near tie the CPU stamp took. ### IQ4_XS (the pilot, 2026-08-30) diff --git a/modules/dasLLAMA/dasllama/dasllama_layout.das b/modules/dasLLAMA/dasllama/dasllama_layout.das index 9796f12ebf..bb0b1aa165 100644 --- a/modules/dasLLAMA/dasllama/dasllama_layout.das +++ b/modules/dasLLAMA/dasllama/dasllama_layout.das @@ -29,7 +29,7 @@ require math def private metal_blob_off_ok(off : int64; fmt : KqFmt) : bool { if (off < 0l) return true if (fmt == KqFmt.q8) return off % 256l == 0l - if (fmt == KqFmt.k6 || fmt == KqFmt.iq4xs) return off % 512l == 0l // the split scale form: the (off/256)*2 d-plane bind stays 4B-aligned + if (fmt == KqFmt.k6 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3) return off % 512l == 0l // the split scale form: the (off/256)*2 d-plane bind stays 4B-aligned if (fmt == KqFmt.q51) return off % 128l == 0l // (off/32)*20 and (off/32)*4 both 16B-aligned return true } @@ -188,6 +188,25 @@ def metal_blob_scale_plane(var t : Model; name : string) : bool { } return true } + if (name == "k3s") { + // k3: the CPU row IS the k6 form ([16 int8][f16 d]) — the same split, over t.k3s + if (!empty(t.k3s)) { + var @exact_size k3c : array + let nsb = long_length(t.k3s) / K3_SSB + k3c |> reserve_resize(nsb * K3_SSB) + unsafe { + var dp = addr(k3c[0]) + let sp = addr < uint8 const? >(t.k3s[0]) + for (sb in range64(nsb)) { + memcpy(reinterpret(dp + sb * 16l), reinterpret(sp + sb * 18l), 16) + dp[nsb * 16l + sb * 2l] = sp[sb * 18l + 16l] + dp[nsb * 16l + sb * 2l + 1l] = sp[sb * 18l + 17l] + } + } + t.k3s <- k3c + } + return true + } if (name != "k6s") { return false } @@ -217,7 +236,7 @@ def metal_blob_commit(var t : Model) : bool { if (!metal_blob_eligible(t)) { return false } - for (nm in ["k4s", "k5s", "k6s", "iq4xss"]) { + for (nm in ["k4s", "k5s", "k6s", "iq4xss", "k3s"]) { metal_blob_scale_plane(t, nm) } t.metal_blob = true diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_common.das b/modules/dasLLAMA/dasllama/dasllama_metal_common.das index feb45e2750..c44e1e0950 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_common.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_common.das @@ -245,6 +245,7 @@ var g_pso_kq_k5 : MetalComputePipeline? var g_pso_kq_k5c : MetalComputePipeline? // the select-form nr0=1 twin — n < 3072 / small-d sites var g_pso_kq_k6 : MetalComputePipeline? var g_pso_kq_iq4xs : MetalComputePipeline? +var g_pso_kq_k3 : MetalComputePipeline? var g_pso_kq_mvb2_k4 : MetalComputePipeline? // batch B=2..8: the kq ext small-batch mv twins var g_pso_kq_mvb4_k4 : MetalComputePipeline? var g_pso_kq_mvb2_k5 : MetalComputePipeline? @@ -257,10 +258,14 @@ var g_pso_kq_mvb8_k6 : MetalComputePipeline? var g_pso_kq_mvb2_iq4xs : MetalComputePipeline? var g_pso_kq_mvb4_iq4xs : MetalComputePipeline? var g_pso_kq_mvb8_iq4xs : MetalComputePipeline? +var g_pso_kq_mvb2_k3 : MetalComputePipeline? +var g_pso_kq_mvb4_k3 : MetalComputePipeline? +var g_pso_kq_mvb8_k3 : MetalComputePipeline? var g_pso_kq_mm_b4 : MetalComputePipeline? // batch B>=9: the prefill-owned kq mul_mm twins var g_pso_kq_mm_b5 : MetalComputePipeline? var g_pso_kq_mm_b6 : MetalComputePipeline? -var g_pso_kq_mm_b44 : MetalComputePipeline? // fused W1|W3+swiglu (s16 path) +var g_pso_kq_mm_b44 : MetalComputePipeline? +var g_pso_kq_mm_b3 : MetalComputePipeline? // fused W1|W3+swiglu (s16 path) // the batched-step set (P4): fixed-B GEMVs + the M-pad-32 GEMM twin (f32 X) + row-table kernels var g_pso_gemv_b2 : MetalComputePipeline? var g_pso_gemv_b4 : MetalComputePipeline? @@ -843,6 +848,10 @@ def kq_quants_of(dev : MetalDevice?; t : Model; fmt : KqFmt; off : int64) : tupl return (buf = plane_buffer(dev, addr < void? >(t.iq4xsq[0]), uint64(long_length(t.iq4xsq)), t.image_map != null), qoff = uint64(sb0 * 128l)) } + if (fmt == KqFmt.k3) { + return (buf = plane_buffer(dev, addr < void? >(t.k3q[0]), uint64(long_length(t.k3q)), t.image_map != null), + qoff = uint64(sb0 * 96l)) + } return (buf = plane_buffer(dev, addr < void? >(t.k6q[0]), uint64(long_length(t.k6q)), t.image_map != null), qoff = uint64(sb0 * 192l)) } @@ -864,6 +873,11 @@ def kq_scales_of(dev : MetalDevice?; t : Model; fmt : KqFmt; off : int64) : tupl return (buf = plane_buffer(dev, addr < void? >(t.k4s[0]), uint64(long_length(t.k4s)), t.image_map != null), soff = uint64(sb0 * 16l), doff = 0ul) } + if (fmt == KqFmt.k3) { // the k6 split form over the 18B/sb device plane + let plane_sb = long_length(t.k3s) / 18l + return (buf = plane_buffer(dev, addr < void? >(t.k3s[0]), uint64(long_length(t.k3s)), t.image_map != null), + soff = uint64(sb0 * 16l), doff = uint64(plane_sb * 16l + sb0 * 2l)) + } if (fmt == KqFmt.iq4xs) { // the k6 split form over the 18B/sb device plane let plane_sb = long_length(t.iq4xss) / 18l return (buf = plane_buffer(dev, addr < void? >(t.iq4xss[0]), uint64(long_length(t.iq4xss)), t.image_map != null), diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das index 208445edbd..2e014df246 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das @@ -3641,6 +3641,11 @@ def metal_decode_init : bool { // nolint:STYLE038 — a flat compile_pso per k g_pso_kq_mvb4_iq4xs = compile_pso(MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl, MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl_entry, MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl_fastmath, ok) g_pso_kq_mvb8_iq4xs = compile_pso(metal_kq_mvb8_iq4xs_msl, metal_kq_mvb8_iq4xs_msl_entry, metal_kq_mvb8_iq4xs_msl_fastmath, ok) g_pso_kq_mm_b44 = compile_pso(MetalKqMulMmIq4xs_metal_kq_mulmm_msl, MetalKqMulMmIq4xs_metal_kq_mulmm_msl_entry, MetalKqMulMmIq4xs_metal_kq_mulmm_msl_fastmath, ok) + g_pso_kq_k3 = compile_pso(metal_kq_gemv_k3_msl, metal_kq_gemv_k3_msl_entry, metal_kq_gemv_k3_msl_fastmath, ok) + g_pso_kq_mvb2_k3 = compile_pso(MetalKqMvB2K3_metal_kq_mv_k3_msl, MetalKqMvB2K3_metal_kq_mv_k3_msl_entry, MetalKqMvB2K3_metal_kq_mv_k3_msl_fastmath, ok) + g_pso_kq_mvb4_k3 = compile_pso(MetalKqMvB4K3_metal_kq_mv_k3_msl, MetalKqMvB4K3_metal_kq_mv_k3_msl_entry, MetalKqMvB4K3_metal_kq_mv_k3_msl_fastmath, ok) + g_pso_kq_mvb8_k3 = compile_pso(metal_kq_mvb8_k3_msl, metal_kq_mvb8_k3_msl_entry, metal_kq_mvb8_k3_msl_fastmath, ok) + g_pso_kq_mm_b3 = compile_pso(MetalKqMulMmK3_metal_kq_mulmm_msl, MetalKqMulMmK3_metal_kq_mulmm_msl_entry, MetalKqMulMmK3_metal_kq_mulmm_msl_fastmath, ok) // the batched-step set (P4) g_pso_gemv_b2 = compile_pso(MetalGemvB2_metal_gemv_msl, MetalGemvB2_metal_gemv_msl_entry, MetalGemvB2_metal_gemv_msl_fastmath, ok) g_pso_gemv_b4 = compile_pso(MetalGemvB4_metal_gemv_msl, MetalGemvB4_metal_gemv_msl_entry, MetalGemvB4_metal_gemv_msl_fastmath, ok) @@ -3709,6 +3714,8 @@ def enc_kq_gemv(enc : MetalComputeEncoder?; fmt : KqFmt; bq : MetalBuffer?; qoff rows, n : int64; yoff : uint64 = 0ul; xoff : uint64 = 0ul) { if (fmt == KqFmt.iq4xs) { enc_kq_iq4xs_c(enc, bs, doff, bs, soff, bq, qoff, bx, xoff, by, yoff, bn, bd, rows, n) + } elif (fmt == KqFmt.k3) { + enc_kq_k3_c(enc, bs, doff, bs, soff, bq, qoff, bx, xoff, by, yoff, bn, bd, rows, n) } elif (fmt == KqFmt.k6) { enc_kq_k6_c(enc, bs, doff, bs, soff, bq, qoff, bx, xoff, by, yoff, bn, bd, rows, n) } elif (fmt == KqFmt.k5 && (n < 3072l || rows <= 1024l)) { @@ -3753,6 +3760,14 @@ def enc_kq_mvb(enc : MetalComputeEncoder?; fmt : KqFmt; bq : MetalBuffer?; qoff } else { enc_kq_mvb4_iq4xs_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows, gcols) } + } elif (fmt == KqFmt.k3) { + if (b8) { + enc_kq_mvb8_k3_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows) + } elif (two) { + enc_kq_mvb2_k3_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows) + } else { + enc_kq_mvb4_k3_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows, gcols) + } } elif (fmt == KqFmt.k6) { if (b8) { enc_kq_mvb8_k6_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows) @@ -3787,6 +3802,8 @@ def enc_kq_gemm_mm_b(enc : MetalComputeEncoder?; fmt : KqFmt; bq : MetalBuffer?; bk, bn : MetalBuffer?; mp, rows : int64) { if (fmt == KqFmt.iq4xs) { enc_kq_mm_iq4xs_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, bk, bn, mp, rows) + } elif (fmt == KqFmt.k3) { + enc_kq_mm_k3_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, bk, bn, mp, rows) } elif (fmt == KqFmt.k6) { enc_kq_mm_k6_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, bk, bn, mp, rows) } elif (fmt == KqFmt.k4) { @@ -4218,6 +4235,11 @@ def metal_kernels_release { // nolint:STYLE037,STYLE038 — a flat release_pso release_pso(g_pso_kq_mvb4_iq4xs) release_pso(g_pso_kq_mvb8_iq4xs) release_pso(g_pso_kq_mm_b44) + release_pso(g_pso_kq_k3) + release_pso(g_pso_kq_mvb2_k3) + release_pso(g_pso_kq_mvb4_k3) + release_pso(g_pso_kq_mvb8_k3) + release_pso(g_pso_kq_mm_b3) if (g_pso_rms != null) { metal_release(g_pso_rms) g_pso_rms = null @@ -5444,6 +5466,7 @@ class template MetalKqMulMmK45T : MetalMmTileBase { @template_constant QH : bool = false // q5_K: the 5th-bit overlay plane rides after the nibbles @template_constant SIXBIT : bool = false // q6_K: int8 sub-scales x super-d, ql/qh compose @template_constant IQ4XS : bool = false // iq4xs: the k6 split scale form, the q40 pairing, codebook decode + @template_constant K3 : bool = false // q3_K: the k6 split scale form, 2-bit lane + hmask compose, offset 4 [metal_kernel] def metal_kq_mulmm { // nolint:STYLE038 — MSL kernel body; the mul_mm tiles are simdgroup-barrier coupled @@ -5512,54 +5535,84 @@ class template MetalKqMulMmK45T : MetalMmTileBase { va[k * 4 + 3] = float16(dsc3 * float(qv & 4278190080u) - dmn) } } else { - static_if (IQ4XS) { - // iq4xs: strip byte js is the SIGNED sub-scale, super-d off the d plane (the k6 split - // form); block js = 4 uints of the q40 pairing, il0 picks the nibble half; codebook per element + static_if (K3) { + // q3_K: strip byte si is the SIGNED sub-scale (k6's split form); block js of the + // superblock = qs uints (js/4)*8 + il0*4.. at shift 2*(js%4) | hmask uints 16 + il0*4.. bit js + let hf3 = js / 4u + let gg3 = js % 4u let sv = ks4[blk] - let scw = js < 4u ? sv.x : sv.y - let dsc = float(ksh[blk]) * float(((int(scw >> ((js & 3u) * 8u)) & 255) ^ 128) - 128) - let qb = blk * 32u + js * 4u - let nsh = il0 * 4u + let dall = float(ksh[blk]) + let si = hf3 * 8u + gg3 * 2u + il0 + let scw = si < 8u ? (si < 4u ? sv.x : sv.y) : (si < 12u ? sv.z : sv.w) + let s3 = float(((int(scw >> ((si & 3u) * 8u)) & 255) ^ 128) - 128) + let dsc = dall * s3 + let dmn = dsc * 4.0 + let dsc1 = dsc * 0.00390625 + let dsc2 = dsc * 0.0000152587890625 + let dsc3 = dsc * 0.000000059604644775390625 + let qb = blk * 24u + hf3 * 8u + il0 * 4u + let hb = blk * 24u + 16u + il0 * 4u + let nsh = gg3 * 2u + let hsh = hf3 * 4u + gg3 for [unroll_full] (k in range(4)) { let u = kqu[qb + uint(k)] - for [unroll_full] (c in range(4)) { - va[k * 4 + c] = float16(dsc * iq4_lut((u >> (8u * uint(c) + nsh)) & 15u)) - } + let hu = kqu[hb + uint(k)] + let qv = ((u >> nsh) & 0x03030303) | (((hu >> hsh) & 0x01010101) << 2u) + va[k * 4] = float16(dsc * float(qv & 255u) - dmn) + va[k * 4 + 1] = float16(dsc1 * float(qv & 65280u) - dmn) + va[k * 4 + 2] = float16(dsc2 * float(qv & 16711680u) - dmn) + va[k * 4 + 3] = float16(dsc3 * float(qv & 4278190080u) - dmn) } } else { - let sv = ks4[blk] - var sc = 0u - var mn = 0u - if (js < 4u) { - sc = (sv.y >> (8u * js)) & 63u - mn = (sv.z >> (8u * js)) & 63u - } else { - let j2 = js - 4u - let hi = sv.w >> (8u * j2) - sc = (hi & 15u) | (((sv.y >> (8u * j2 + 6u)) & 3u) << 4u) - mn = ((hi >> 4u) & 15u) | (((sv.z >> (8u * j2 + 6u)) & 3u) << 4u) - } - let dsc = float(ksh[blk * 8u]) * float(sc) - let dmn = float(ksh[blk * 8u + 1u]) * float(mn) - let qb = blk * BLK + (js / 2u) * 8u + il0 * 4u - static_if (QH) { - let hb = blk * BLK + 32u + il0 * 4u - let nsh = (js & 1u) * 4u + static_if (IQ4XS) { + // iq4xs: strip byte js is the SIGNED sub-scale, super-d off the d plane (the k6 split + // form); block js = 4 uints of the q40 pairing, il0 picks the nibble half; codebook per element + let sv = ks4[blk] + let scw = js < 4u ? sv.x : sv.y + let dsc = float(ksh[blk]) * float(((int(scw >> ((js & 3u) * 8u)) & 255) ^ 128) - 128) + let qb = blk * 32u + js * 4u + let nsh = il0 * 4u for [unroll_full] (k in range(4)) { let u = kqu[qb + uint(k)] - let hu = kqu[hb + uint(k)] for [unroll_full] (c in range(4)) { - let q = ((u >> (8u * uint(c) + nsh)) & 15u) | (((hu >> (8u * uint(c) + js)) & 1u) << 4u) - va[k * 4 + c] = float16(dsc * float(q) - dmn) + va[k * 4 + c] = float16(dsc * iq4_lut((u >> (8u * uint(c) + nsh)) & 15u)) } } } else { - let nsh = (js & 1u) * 4u - for [unroll_full] (k in range(4)) { - let u = kqu[qb + uint(k)] - for [unroll_full] (c in range(4)) { - let q = (u >> (8u * uint(c) + nsh)) & 15u - va[k * 4 + c] = float16(dsc * float(q) - dmn) + let sv = ks4[blk] + var sc = 0u + var mn = 0u + if (js < 4u) { + sc = (sv.y >> (8u * js)) & 63u + mn = (sv.z >> (8u * js)) & 63u + } else { + let j2 = js - 4u + let hi = sv.w >> (8u * j2) + sc = (hi & 15u) | (((sv.y >> (8u * j2 + 6u)) & 3u) << 4u) + mn = ((hi >> 4u) & 15u) | (((sv.z >> (8u * j2 + 6u)) & 3u) << 4u) + } + let dsc = float(ksh[blk * 8u]) * float(sc) + let dmn = float(ksh[blk * 8u + 1u]) * float(mn) + let qb = blk * BLK + (js / 2u) * 8u + il0 * 4u + static_if (QH) { + let hb = blk * BLK + 32u + il0 * 4u + let nsh = (js & 1u) * 4u + for [unroll_full] (k in range(4)) { + let u = kqu[qb + uint(k)] + let hu = kqu[hb + uint(k)] + for [unroll_full] (c in range(4)) { + let q = ((u >> (8u * uint(c) + nsh)) & 15u) | (((hu >> (8u * uint(c) + js)) & 1u) << 4u) + va[k * 4 + c] = float16(dsc * float(q) - dmn) + } + } + } else { + let nsh = (js & 1u) * 4u + for [unroll_full] (k in range(4)) { + let u = kqu[qb + uint(k)] + for [unroll_full] (c in range(4)) { + let q = (u >> (8u * uint(c) + nsh)) & 15u + va[k * 4 + c] = float16(dsc * float(q) - dmn) + } } } } @@ -5609,6 +5662,12 @@ class MetalKqMulMmIq4xs : MetalKqMulMmK45T { override IQ4XS = true } +// q3_K rides the K3 arm; the split scale plane binds twice like k6 (d plane at s0off = doff) +[metal_dispatch(name = "enc_kq_mm_k3_c", pso = "g_pso_kq_mm_b3", tgmem = "MetalKqMulMmK3_metal_kq_mulmm_msl_tgmem", tg = 128, grid = "mp/32, rows/64", params = "mp : int64, rows : int64")] +class MetalKqMulMmK3 : MetalKqMulMmK45T { + override K3 = true +} + // Router GEMV over the uploaded fp32 router slab: y[ne] = W[ne x dim] . x per stream. ne <= 256 // rows. Each sg serves NR positions per weight load (slab SLC traffic /NR); BATCHED stamps // clamp tails via ns; singles fold the clamp away and the gate erases ns outright. @@ -6887,6 +6946,263 @@ class MetalKqGemvK6 { } } +// Q3_K: k6's lane map (a lane's 4 elements of each of a half's 4 blocks) over the k3 planes — the +// 2-bit lane of qs byte ip*32 + l at shift 2j | hmask bit 4ip + j of byte l, minus 4; per-16 SIGNED +// sub-scales off the split strips, d off the d plane. +[metal_dispatch(name = "enc_kq_k3_c", pso = "g_pso_kq_k3", tg = 64, grid = "rows/4", params = "rows : int64, n : int64")] +class MetalKqGemvK3 { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d plane — the scale buffer bound at byte nsb*16 (s0off = the caller's doff) + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // 16B sub-scale strips, byte view (same buffer at 0) + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqb : array // k3 quant plane, byte view (qs at 96*blk, hmask at +64) + @ssbo @binding = 3 @role = "read" @off = "xoff" @span = "n*4" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" @span = "rows*4" y : array + @uniform @binding = 5 ndim : uint + @uniform @binding = 6 ddim : uint + + [metal_kernel(name="metal_kq_gemv_k3_msl")] + def metal_kq_gemv_k3 { + let lane = gl_SubgroupInvocationID + let tid = lane / 2u + let ix = lane % 2u + let ip = tid / 8u + let il = tid % 8u + let l0 = il * 4u + let is0 = ip * 8u + l0 / 16u + let hb0 = ip * 4u + let nb = ndim / 256u + let first_row = (gl_WorkGroupID.x * 2u + gl_SubgroupID) * 2u + var yl : float[16] + var sumf : float[2] + var ib = ix + while (ib < nb) { + let yb = ib * 256u + ip * 128u + l0 + for [unroll_full] (l in range(4)) { + yl[4 * l] = x[yb + uint(l)] + yl[4 * l + 1] = x[yb + 32u + uint(l)] + yl[4 * l + 2] = x[yb + 64u + uint(l)] + yl[4 * l + 3] = x[yb + 96u + uint(l)] + } + for [unroll_full] (r in range(2)) { + let blk = (first_row + uint(r)) * nb + ib + let qs0 = blk * 96u + ip * 32u + l0 + let hm0 = blk * 96u + 64u + l0 + let sb = blk * 16u + is0 + var sums : float[4] + for [unroll_full] (l in range(4)) { + let q = uint(kqb[qs0 + uint(l)]) + let h = uint(kqb[hm0 + uint(l)]) >> hb0 + sums[0] += yl[4 * l] * (float((q & 3u) | ((h & 1u) << 2u)) - 4.0) + sums[1] += yl[4 * l + 1] * (float(((q >> 2u) & 3u) | ((h & 2u) << 1u)) - 4.0) + sums[2] += yl[4 * l + 2] * (float(((q >> 4u) & 3u) | (h & 4u)) - 4.0) + sums[3] += yl[4 * l + 3] * (float((q >> 6u) | ((h & 8u) >> 1u)) - 4.0) + } + let s0 = float((int(kscb[sb]) ^ 128) - 128) + let s2 = float((int(kscb[sb + 2u]) ^ 128) - 128) + let s4 = float((int(kscb[sb + 4u]) ^ 128) - 128) + let s6 = float((int(kscb[sb + 6u]) ^ 128) - 128) + let dd = float(kdh[blk]) + sumf[r] += dd * (sums[0] * s0 + sums[1] * s2 + sums[2] * s4 + sums[3] * s6) + } + ib += 2u + } + for [unroll_full] (r in range(2)) { + let s = simd_sum(sumf[r]) + if (lane == 0u && first_row + uint(r) < ddim) { + y[first_row + uint(r)] = s + } + } + } +} + +// The k3 B2/B4 pair: the K6T shell (slot s = block of the half, o4 = its 16-half) with the +// lane + mask compose off the 24-uint superblock (qs 0..15, hmask 16..23), offset 4. +[ |> template_struct_instance] +class template MetalKqMvK3T { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d plane — the scale buffer bound at byte nsb*16 + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // 16B sub-scale strips, byte view (same buffer at 0) + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array // k3 quant plane, uint view (qs at 24*blk, hmask at 24*blk+16) + @ssbo @binding = 3 @role = "read" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" y : array + @uniform @binding = 5 ka : KqMvArgs + @template_constant NR : int = 2 + @template_constant NRU : uint = 2u // NR's uint spelling (colbase math) — override BOTH together + @template_constant TILED : bool = false + + [metal_kernel] + def metal_kq_mv_k3 { // nolint:STYLE038 — the TILED twin loops are byte-identity-driven, not splittable + let lane = gl_SubgroupInvocationID + let tx = lane % 8u + let ty = lane / 8u + let row = gl_WorkGroupID.x * 8u + gl_SubgroupID * 4u + ty + let colbase = gl_WorkGroupID.y * NRU // nolint:LINT002 dead in the TILED=false stamp — measured free + let nb = ka.ndim / 256u + let s = tx / 2u + let o4 = (tx % 2u) * 4u + let shr = 2u * s + let nb4 = ka.ndim / 4u + var sumf : float[NR] + var sb = 0u + while (sb < nb) { + let blk = row * nb + sb + let dd = float(kdh[blk]) + for [unroll_full] (ch in range(2)) { + let ip = uint(ch) + let qb = blk * 24u + ip * 8u + o4 + let hb = blk * 24u + 16u + o4 + let hsh = ip * 4u + s + let g = ip * 8u + s * 2u + (tx % 2u) + let sc = float((int(kscb[blk * 16u + g]) ^ 128) - 128) + let dl = dd * sc + let ml = dl * 4.0 + var w : float[16] + for [unroll_full] (u in range(4)) { + let qu = kqu[qb + uint(u)] + let hu = kqu[hb + uint(u)] + let qv = ((qu >> shr) & 0x03030303) | (((hu >> hsh) & 0x01010101) << 2u) + for [unroll_full] (e in range(4)) { + w[4 * u + e] = float((qv >> (8u * uint(e))) & 255u) * dl - ml + } + } + let cx4 = sb * 64u + ip * 32u + s * 8u + o4 + static_if (TILED) { + for [unroll_full] (b in range(NR)) { + let xb4 = (colbase + uint(b)) * nb4 + cx4 + var acc = 0.0 + for [unroll_full] (u in range(4)) { + let v = x[xb4 + uint(u)] + acc += v.x * w[4 * u] + v.y * w[4 * u + 1] + v.z * w[4 * u + 2] + v.w * w[4 * u + 3] + } + sumf[b] += acc + } + } else { + for [unroll_full] (b in range(NR)) { + let xb4 = uint(b) * nb4 + cx4 + var acc = 0.0 + for [unroll_full] (u in range(4)) { + let v = x[xb4 + uint(u)] + acc += v.x * w[4 * u] + v.y * w[4 * u + 1] + v.z * w[4 * u + 2] + v.w * w[4 * u + 3] + } + sumf[b] += acc + } + } + } + sb++ + } + static_if (TILED) { + for [unroll_full] (b in range(NR)) { + var sm = sumf[b] + sm += simd_shuffle_down(sm, 4u) + sm += simd_shuffle_down(sm, 2u) + sm += simd_shuffle_down(sm, 1u) + if (tx == 0u && row < ka.ddim && colbase + uint(b) < ka.nr) { + y[(colbase + uint(b)) * ka.ys + row] = sm + } + } + } else { + for [unroll_full] (b in range(NR)) { + var sm = sumf[b] + sm += simd_shuffle_down(sm, 4u) + sm += simd_shuffle_down(sm, 2u) + sm += simd_shuffle_down(sm, 1u) + if (tx == 0u && row < ka.ddim && uint(b) < ka.nr) { + y[uint(b) * ka.ys + row] = sm + } + } + } + } +} + +[metal_dispatch(name = "enc_kq_mvb2_k3_c", pso = "g_pso_kq_mvb2_k3", tg = 64, grid = "rows/8", params = "rows : int64")] +class MetalKqMvB2K3 : MetalKqMvK3T { + override NR = 2 + override NRU = 2u + override TILED = false +} + +[metal_dispatch(name = "enc_kq_mvb4_k3_c", pso = "g_pso_kq_mvb4_k3", tg = 64, grid = "rows/8, gcols", params = "rows : int64, gcols : int64")] +class MetalKqMvB4K3 : MetalKqMvK3T { + override NR = 4 + override NRU = 4u + override TILED = true +} + +// the k3 B8 twin: the k6 B8 shell (one superblock's X panel staged per threadgroup) with the +// lane + mask compose +[metal_dispatch(name = "enc_kq_mvb8_k3_c", pso = "g_pso_kq_mvb8_k3", tgmem = "metal_kq_mvb8_k3_msl_tgmem", tg = 64, grid = "rows/8", params = "rows : int64")] +class MetalKqMvB8K3 { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d plane — the scale buffer bound at byte nsb*16 + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // 16B sub-scale strips, byte view (same buffer at 0) + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array // k3 quant plane, uint view (qs at 24*blk, hmask at 24*blk+16) + @ssbo @binding = 3 @role = "read" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" y : array + @uniform @binding = 5 ka : KqMvArgs + @workgroup txp : float4[512] + + [metal_kernel(name="metal_kq_mvb8_k3_msl")] + def metal_kq_mvb8_k3 { + let lid = gl_LocalInvocationID.x + let lane = gl_SubgroupInvocationID + let tx = lane % 8u + let ty = lane / 8u + let row = gl_WorkGroupID.x * 8u + gl_SubgroupID * 4u + ty + let nb = ka.ndim / 256u + let s = tx / 2u + let o4 = (tx % 2u) * 4u + let shr = 2u * s + let nb4 = ka.ndim / 4u + var acc : float4[8] + var sb = 0u + while (sb < nb) { + barrier() + for [unroll_full] (j in range(8)) { + let slot = uint(j) * 64u + lid + txp[slot] = x[(slot / 64u) * nb4 + sb * 64u + (slot % 64u)] + } + barrier() + let blk = row * nb + sb + let dd = float(kdh[blk]) + for [unroll_full] (ch in range(2)) { + let ip = uint(ch) + let qb = blk * 24u + ip * 8u + o4 + let hb = blk * 24u + 16u + o4 + let hsh = ip * 4u + s + var qu : uint[4] + var hu : uint[4] + for [unroll_full] (u in range(4)) { + qu[u] = kqu[qb + uint(u)] + hu[u] = kqu[hb + uint(u)] + } + let g = ip * 8u + s * 2u + (tx % 2u) + let sc = float((int(kscb[blk * 16u + g]) ^ 128) - 128) + let dl = dd * sc + let ml = dl * 4.0 + var w4 : float4[4] + for [unroll_full] (u in range(4)) { + let qvc = ((qu[u] >> shr) & 0x03030303) | (((hu[u] >> hsh) & 0x01010101) << 2u) + w4[u] = float4(float(qvc & 255u), float((qvc >> 8u) & 255u), + float((qvc >> 16u) & 255u), float((qvc >> 24u) & 255u)) * dl - float4(ml) + } + let cxl = ip * 32u + s * 8u + o4 + for [unroll_full] (b in range(8)) { + for [unroll_full] (u in range(4)) { + acc[b] += txp[uint(b) * 64u + cxl + uint(u)] * w4[u] + } + } + } + sb++ + } + for [unroll_full] (b in range(8)) { + var sm = acc[b].x + acc[b].y + acc[b].z + acc[b].w + sm += simd_shuffle_down(sm, 4u) + sm += simd_shuffle_down(sm, 2u) + sm += simd_shuffle_down(sm, 1u) + if (tx == 0u && row < ka.ddim && uint(b) < ka.nr) { + y[uint(b) * ka.ys + row] = sm + } + } + } +} + // IQ4_XS: w = d*sc*LUT[q] with sc the SIGNED (ls - 32) strip byte and no min plane; each lane owns // one 32-block of every 4th superblock (the q40 pairing: byte l -> elems l and l + 16), the scale // plane is the k6 split form (d plane at s0off = doff, 16B sub-scale strips at soff). diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das b/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das index a400bbf51d..965e70410f 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das @@ -4411,7 +4411,7 @@ def private pf_devw_panel(enc : MetalComputeEncoder?; bwblob : MetalBuffer?; wbo //! quant superblocks, plus the k6 2B d tail def private pf_devw_panel_kq(enc : MetalComputeEncoder?; t : Model; fmt : KqFmt; woff : int64; bxh, by, bk, bn : MetalBuffer?; rows, d, kdim : int64; yoff : uint64) : bool { - if (fmt == KqFmt.iq4xs) { // no dev-W dequant kernel for the codebook format yet + if (fmt == KqFmt.iq4xs || fmt == KqFmt.k3) { // no dev-W dequant kernel for these formats yet return false } let dq = fmt == KqFmt.k6 ? g_pf_pso_dq_k6 : (fmt == KqFmt.k4 ? g_pf_pso_dq_k4 : g_pf_pso_dq_k5) @@ -4580,6 +4580,10 @@ def private pf_enc_kq_site_mm(enc : MetalComputeEncoder?; t : Model; fmt : KqFmt enc_kq_mm_iq4xs_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, by, yoff, bk, bn, mp, rows) return } + if (fmt == KqFmt.k3) { // the base mul_mm only - no tensor / tall / dev-W twins yet + enc_kq_mm_k3_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, by, yoff, bk, bn, mp, rows) + return + } //! deep-class arm: a panel dev-W would split past 8 tiles is DRAM-resident - the tall //! in-kernel-dequant stamp reads the quant plane once per 128-row tile instead of //! materializing and re-streaming a 2B/element f16 panel @@ -4685,6 +4689,8 @@ def private pf_enc_kq_gemv(enc : MetalComputeEncoder?; t : Model; fmt : KqFmt; w let bs = kq_scales_of(g_dev, t, fmt, woff) if (fmt == KqFmt.iq4xs) { enc_kq_iq4xs_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, xoff, by, 0ul, bn, bd, rows, n) + } elif (fmt == KqFmt.k3) { + enc_kq_k3_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, xoff, by, 0ul, bn, bd, rows, n) } elif (fmt == KqFmt.k6) { enc_kq_k6_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, xoff, by, 0ul, bn, bd, rows, n) } elif (fmt == KqFmt.k4) { diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_shapes.das b/modules/dasLLAMA/dasllama/dasllama_metal_shapes.das index 45b5eda15b..6994dfa85d 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_shapes.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_shapes.das @@ -180,7 +180,7 @@ def record_needs(var tab : table; missing : MetalNeed) { // formats) must DECLINE kquant_native instead of falling into a wrong-layout kernel branch // (the dispatchers below treat "not k4/k6" as k5) def kq_fmt_gpu_supported(f : KqFmt) : bool { - return f == KqFmt.q8 || f == KqFmt.k4 || f == KqFmt.k5 || f == KqFmt.k6 || f == KqFmt.iq4xs + return f == KqFmt.q8 || f == KqFmt.k4 || f == KqFmt.k5 || f == KqFmt.k6 || f == KqFmt.iq4xs || f == KqFmt.k3 } def private kq_fmts_gpu_supported(a : array) : bool { @@ -243,7 +243,7 @@ def moe_site_ok(fmt : KqFmt; off, ege : int64) : bool { if (fmt == KqFmt.k4 || fmt == KqFmt.k5) { return off >= 0l && (off % 256l) == 0l && (ege % 256l) == 0l } - if (fmt == KqFmt.k6 || fmt == KqFmt.iq4xs) { // the split scale form: off % 512 keeps the d-plane bind 4B-aligned + if (fmt == KqFmt.k6 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3) { // the split scale form: off % 512 keeps the d-plane bind 4B-aligned return off >= 0l && (off % 512l) == 0l && (ege % 256l) == 0l } if (fmt == KqFmt.q51) { // per-32 planes: off % 128 keeps the 20B/4B binds 16B-aligned @@ -324,9 +324,9 @@ def moe_metal_ok(t : Model) : bool { // nolint:STYLE038 — flat per-shape ref let f3 = fmt_at(t.we3_fmt, l) // per-site alignment + reduction dims: superblock kq kernels iterate whole 256-rows, // q51 whole 32-blocks (+ the mul_mm's 64-col output tiles) - let sb1 = f1 == KqFmt.k4 || f1 == KqFmt.k5 || f1 == KqFmt.k6 || f1 == KqFmt.iq4xs - let sb3 = f3 == KqFmt.k4 || f3 == KqFmt.k5 || f3 == KqFmt.k6 || f3 == KqFmt.iq4xs - let sb2 = f2 == KqFmt.k4 || f2 == KqFmt.k5 || f2 == KqFmt.k6 || f2 == KqFmt.iq4xs + let sb1 = kq_sb(f1) + let sb3 = kq_sb(f3) + let sb2 = kq_sb(f2) if (!moe_site_ok(f1, t.we1_offs[l], ege) || !moe_site_ok(f2, t.we2_offs[l], ege) || !moe_site_ok(f3, t.we3_offs[l], ege) || diff --git a/modules/dasLLAMA/followup_general.md b/modules/dasLLAMA/followup_general.md index 2033d89fdd..ca5c0420f6 100644 --- a/modules/dasLLAMA/followup_general.md +++ b/modules/dasLLAMA/followup_general.md @@ -731,3 +731,15 @@ the existing templates, the LUT cost measured against a `constant` table and against a byte-pair decode on the M5, and `bench_metal_gemv_kernels` / `bench_metal_kq_mm_lab` rows for the format beside k4's. + +59. **Partial mint (Boris, 2026-08-30, for after the formats arc): "takes existing mint and + mints new kernels only. at least in debug-only mode."** Today a sidecar missing ANY demanded + kernel re-tunes the whole scope (the completeness rule), so every new `[tune]` family - one + per format in this arc - re-mints every application sidecar on the box on its next start, + minutes each (HOW_TO_ADD_A_FORMAT.md QUIRK 17), and the same rule keeps a stub-era + `"reference"` pin alive after the emitter lands (QUIRK 11). The partial mint keeps the + existing entries and races only the families the sidecar lacks (or names as reference), + validating the merged file as one. Debug-only is the acceptable first form: a + `DAS_TUNE_PARTIAL=1` (or `--tune-partial`) rail that the auto policy does not take, so a + shipped box still mints whole. Done = the rail exists, a fresh family on a minted box + costs one family's race, and the tuner's status line names the partial mint as such. diff --git a/modules/dasLLAMA/tests/_metal_kernel_common.das b/modules/dasLLAMA/tests/_metal_kernel_common.das index 2d4579ba18..de5a06d6f6 100644 --- a/modules/dasLLAMA/tests/_metal_kernel_common.das +++ b/modules/dasLLAMA/tests/_metal_kernel_common.das @@ -172,17 +172,17 @@ let KQ_SAFE_SCALES = fixed_array(0.25, 0.5, 1.0, 0.125, 2.0, 0.75, 1.5, 0.375) // quant plane = hash bytes (any bit pattern is valid); scale plane K4/K5 = [d f16][dmin f16] // [12 packed 6-bit bytes] per superblock, K6 = [16B sub-scales] x nsb + f16 d tail, IQ4_XS (44) = -// the same split form with [8 signed sub-scales in -31..31][8 zero] strips +// the same split form with [8 signed sub-scales in -31..31][8 zero] strips, Q3_K (3) = the k6 form over 96B quants def kq_fill_planes(fmt, nsb : int; var kq : array; var ks : array) { - let qsb = fmt == 4 || fmt == 44 ? 128 : (fmt == 5 ? 160 : 192) + let qsb = fmt == 4 || fmt == 44 ? 128 : (fmt == 5 ? 160 : (fmt == 3 ? 96 : 192)) kq |> resize(nsb * qsb) for (i in range(nsb * qsb)) { kq[i] = kq_hash_byte(i * 3 + fmt) } - if (fmt == 6 || fmt == 44) { + if (fmt == 6 || fmt == 44 || fmt == 3) { ks |> resize(nsb * 18) for (i in range(nsb * 16)) { - ks[i] = fmt == 6 ? kq_hash_byte(i * 7 + 13) : uint8(i % 16 < 8 ? int(kq_hash_byte(i * 7 + 13)) % 63 - 31 : 0) + ks[i] = fmt == 6 || fmt == 3 ? kq_hash_byte(i * 7 + 13) : uint8(i % 16 < 8 ? int(kq_hash_byte(i * 7 + 13)) % 63 - 31 : 0) } for (b in range(nsb)) { let dbits = f32_to_f16(KQ_SAFE_SCALES[b % 8]) @@ -248,6 +248,9 @@ def kq_row_ref(fmt, sb_base, nb, nsb_total : int; kq, ks : array; var wro } elif (fmt == 44) { dequant_iq4xs_plane_superblock_at(kq, int64(sb * 128), ks, int64(sb * 16), int64(nsb_total * 16 + sb * 2), wrow, int64(ib * 256)) + } elif (fmt == 3) { + dequant_k3_plane_superblock_at(kq, int64(sb * 96), ks, int64(sb * 16), + int64(nsb_total * 16 + sb * 2), wrow, int64(ib * 256)) } else { dequant_k6_plane_superblock_at(kq, int64(sb * 192), ks, int64(sb * 16), int64(nsb_total * 16 + sb * 2), wrow, int64(ib * 256)) diff --git a/modules/dasLLAMA/tests/test_metal_gemm_kernels.das b/modules/dasLLAMA/tests/test_metal_gemm_kernels.das index 4eb7e07ae3..2c2f4410ac 100644 --- a/modules/dasLLAMA/tests/test_metal_gemm_kernels.das +++ b/modules/dasLLAMA/tests/test_metal_gemm_kernels.das @@ -36,28 +36,28 @@ def private kq_mulmm_gate(t : T?; dev, queue; fmt : int; tensor : bool; m, kdim, : halfx ? (fmt == 4 ? MetalKqMulMmK4TH_metal_kq_mulmm_t_msl : (fmt == 5 ? MetalKqMulMmK5TH_metal_kq_mulmm_t_msl : MetalKqMulMmK6TH_metal_kq_mulmm_k6_t_msl)) : (fmt == 4 ? MetalKqMulMmK4T_metal_kq_mulmm_t_msl : (fmt == 5 ? MetalKqMulMmK5T_metal_kq_mulmm_t_msl : MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl))) - : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl : MetalKqMulMmK6_metal_kq_mulmm_msl)))) + : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl : MetalKqMulMmK6_metal_kq_mulmm_msl))))) let entry = (tensor ? (tall ? (fmt == 4 ? MetalKqMulMmK4TH128_metal_kq_mulmm_t_msl_entry : (fmt == 5 ? MetalKqMulMmK5TH128_metal_kq_mulmm_t_msl_entry : MetalKqMulMmK6TH128_metal_kq_mulmm_k6_t_msl_entry)) : halfx ? (fmt == 4 ? MetalKqMulMmK4TH_metal_kq_mulmm_t_msl_entry : (fmt == 5 ? MetalKqMulMmK5TH_metal_kq_mulmm_t_msl_entry : MetalKqMulMmK6TH_metal_kq_mulmm_k6_t_msl_entry)) : (fmt == 4 ? MetalKqMulMmK4T_metal_kq_mulmm_t_msl_entry : (fmt == 5 ? MetalKqMulMmK5T_metal_kq_mulmm_t_msl_entry : MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl_entry))) - : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl_entry : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_entry : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_entry : MetalKqMulMmK6_metal_kq_mulmm_msl_entry)))) + : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl_entry : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl_entry : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_entry : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_entry : MetalKqMulMmK6_metal_kq_mulmm_msl_entry))))) let fm = (tensor ? (tall ? (fmt == 4 ? MetalKqMulMmK4TH128_metal_kq_mulmm_t_msl_fastmath : (fmt == 5 ? MetalKqMulMmK5TH128_metal_kq_mulmm_t_msl_fastmath : MetalKqMulMmK6TH128_metal_kq_mulmm_k6_t_msl_fastmath)) : halfx ? (fmt == 4 ? MetalKqMulMmK4TH_metal_kq_mulmm_t_msl_fastmath : (fmt == 5 ? MetalKqMulMmK5TH_metal_kq_mulmm_t_msl_fastmath : MetalKqMulMmK6TH_metal_kq_mulmm_k6_t_msl_fastmath)) : (fmt == 4 ? MetalKqMulMmK4T_metal_kq_mulmm_t_msl_fastmath : (fmt == 5 ? MetalKqMulMmK5T_metal_kq_mulmm_t_msl_fastmath : MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl_fastmath))) - : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl_fastmath : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_fastmath : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_fastmath : MetalKqMulMmK6_metal_kq_mulmm_msl_fastmath)))) + : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl_fastmath : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl_fastmath : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_fastmath : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_fastmath : MetalKqMulMmK6_metal_kq_mulmm_msl_fastmath))))) let tgm = (tensor ? (tall ? (fmt == 4 ? MetalKqMulMmK4TH128_metal_kq_mulmm_t_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5TH128_metal_kq_mulmm_t_msl_tgmem : MetalKqMulMmK6TH128_metal_kq_mulmm_k6_t_msl_tgmem)) : halfx ? (fmt == 4 ? MetalKqMulMmK4TH_metal_kq_mulmm_t_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5TH_metal_kq_mulmm_t_msl_tgmem : MetalKqMulMmK6TH_metal_kq_mulmm_k6_t_msl_tgmem)) : (fmt == 4 ? MetalKqMulMmK4T_metal_kq_mulmm_t_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5T_metal_kq_mulmm_t_msl_tgmem : MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl_tgmem))) - : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl_tgmem : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_tgmem : MetalKqMulMmK6_metal_kq_mulmm_msl_tgmem)))) + : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl_tgmem : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl_tgmem : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_tgmem : MetalKqMulMmK6_metal_kq_mulmm_msl_tgmem))))) var pso = pipeline_from_source(dev, src, entry, fm, err) if (tensor && pso == null && !metal4_tensor_available(dev)) { to_log(LOG_INFO, "{tag}: no Metal-4 tensor toolchain on this box - arm skipped\n") @@ -123,7 +123,7 @@ def private kq_mulmm_gate(t : T?; dev, queue; fmt : int; tensor : bool; m, kdim, let ran = with_compute_encoder(queue, err) $(enc : MetalComputeEncoder?) { metal_set_pipeline(enc, pso) metal_set_threadgroup_memory_length(enc, tgm, 0) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs: the f16 d tail + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs: the f16 d tail metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bx, 0ul, 3) @@ -134,7 +134,7 @@ def private kq_mulmm_gate(t : T?; dev, queue; fmt : int; tensor : bool; m, kdim, if (rem_pso != null) { metal_set_pipeline(enc, rem_pso) metal_set_threadgroup_memory_length(enc, fmt == 4 ? MetalKqMulMmK4TH_metal_kq_mulmm_t_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5TH_metal_kq_mulmm_t_msl_tgmem : MetalKqMulMmK6TH_metal_kq_mulmm_k6_t_msl_tgmem), 0) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 ? uint64(nsb * 16) : 0ul, 0) + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 ? uint64(nsb * 16) : 0ul, 0) metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bx, uint64(r0 * kdim * 2), 3) @@ -484,7 +484,7 @@ def private kq_devw_gate(t : T?; dev, queue; fmt : int; m, kdim, ndim : int) { var dka = CvtArgs(total = uint(nblk)) let ran = with_compute_encoder(queue, err) $(enc : MetalComputeEncoder?) { metal_set_pipeline(enc, dq_pso) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs: the f16 d tail + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs: the f16 d tail metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bwh, 0ul, 3) @@ -1461,7 +1461,7 @@ def private moe_mulmm_kq_gate(t : T?; dev, queue; fmt : int; kdim, ndim : int; c let ran = with_compute_encoder(queue, err) $(enc : MetalComputeEncoder?) { metal_set_pipeline(enc, pso) metal_set_threadgroup_memory_length(enc, tgm, 0) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs: the f16 d tail + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs: the f16 d tail metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bx, 0ul, 3) @@ -1498,7 +1498,7 @@ def private moe_mulmm_kq_gate(t : T?; dev, queue; fmt : int; kdim, ndim : int; c metal_set_threadgroup_memory_length(enc, fmt == 4 ? MetalMoeMulMmK4TH_metal_moe_mulmm_k45_t_msl_tgmem : (fmt == 5 ? MetalMoeMulMmK5TH_metal_moe_mulmm_k45_t_msl_tgmem : MetalMoeMulMmK6TH_metal_moe_mulmm_k6_t_msl_tgmem), 0) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 ? uint64(nsb * 16) : 0ul, 0) + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 ? uint64(nsb * 16) : 0ul, 0) metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bxh, 0ul, 3) @@ -1538,7 +1538,7 @@ def private moe_mulmm_kq_gate(t : T?; dev, queue; fmt : int; kdim, ndim : int; c metal_set_threadgroup_memory_length(enc, fmt == 4 ? MetalMoeMulMmK4TH128_metal_moe_mulmm_k45_t_msl_tgmem : (fmt == 5 ? MetalMoeMulMmK5TH128_metal_moe_mulmm_k45_t_msl_tgmem : MetalMoeMulMmK6TH128_metal_moe_mulmm_k6_t_msl_tgmem), 0) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 ? uint64(nsb * 16) : 0ul, 0) + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 ? uint64(nsb * 16) : 0ul, 0) metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bxh, 0ul, 3) @@ -1551,7 +1551,7 @@ def private moe_mulmm_kq_gate(t : T?; dev, queue; fmt : int; kdim, ndim : int; c metal_set_threadgroup_memory_length(enc, fmt == 4 ? MetalMoeMulMmK4THR_metal_moe_mulmm_k45_t_msl_tgmem : (fmt == 5 ? MetalMoeMulMmK5THR_metal_moe_mulmm_k45_t_msl_tgmem : MetalMoeMulMmK6THR_metal_moe_mulmm_k6_t_msl_tgmem), 0) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 ? uint64(nsb * 16) : 0ul, 0) + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 ? uint64(nsb * 16) : 0ul, 0) metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bxh, 0ul, 3) @@ -2049,6 +2049,8 @@ def test_metal_gemm_kernels(t : T?) { kq_mulmm_gate(t, dev, queue, 4, false, 32, 256, 128) // second shape: 2 col tiles kq_mulmm_gate(t, dev, queue, 44, false, 64, 512, 64) // iq4xs: the base form only (no tensor / tall twins) kq_mulmm_gate(t, dev, queue, 44, false, 32, 256, 128) + kq_mulmm_gate(t, dev, queue, 3, false, 64, 512, 64) // k3: the base form only (no tensor / tall twins) + kq_mulmm_gate(t, dev, queue, 3, false, 32, 256, 128) q8_mulmm_t_gate(t, dev, queue, 64, 256, 64) q8_mulmm_t_gate(t, dev, queue, 64, 256, 64, halfx = true) q8_mulmm_t_gate(t, dev, queue, 64, 160, 64) // kdim % 64 == 32: the bk=64 helper's 32-tail chunk diff --git a/modules/dasLLAMA/tests/test_metal_gemv_kernels.das b/modules/dasLLAMA/tests/test_metal_gemv_kernels.das index 6b58d9dd2d..6b02e13d31 100644 --- a/modules/dasLLAMA/tests/test_metal_gemv_kernels.das +++ b/modules/dasLLAMA/tests/test_metal_gemv_kernels.das @@ -23,12 +23,12 @@ require math // ===== single-stream kq GEMV (MetalKqGemvK4 / K5 / K5C / K6) ===== def private kq_gemv_gate(t : T?; dev, queue; vform : string; n, d : int) { - let fmt = vform == "k4" ? 4 : (vform == "k6" ? 6 : (vform == "iq4xs" ? 44 : 5)) + let fmt = vform == "k4" ? 4 : (vform == "k6" ? 6 : (vform == "iq4xs" ? 44 : (vform == "k3" ? 3 : 5))) let tag = "kq_gemv_{vform} n={n} d={d}" var err : string - let src = vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl : (vform == "k4" ? metal_kq_gemv_k4_msl : (vform == "k5" ? metal_kq_gemv_k5_msl : (vform == "k5c" ? metal_kq_gemv_k5c_msl : metal_kq_gemv_k6_msl))) - let entry = vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl_entry : (vform == "k4" ? metal_kq_gemv_k4_msl_entry : (vform == "k5" ? metal_kq_gemv_k5_msl_entry : (vform == "k5c" ? metal_kq_gemv_k5c_msl_entry : metal_kq_gemv_k6_msl_entry))) - let fm = vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl_fastmath : (vform == "k4" ? metal_kq_gemv_k4_msl_fastmath : (vform == "k5" ? metal_kq_gemv_k5_msl_fastmath : (vform == "k5c" ? metal_kq_gemv_k5c_msl_fastmath : metal_kq_gemv_k6_msl_fastmath))) + let src = vform == "k3" ? metal_kq_gemv_k3_msl : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl : (vform == "k4" ? metal_kq_gemv_k4_msl : (vform == "k5" ? metal_kq_gemv_k5_msl : (vform == "k5c" ? metal_kq_gemv_k5c_msl : metal_kq_gemv_k6_msl)))) + let entry = vform == "k3" ? metal_kq_gemv_k3_msl_entry : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl_entry : (vform == "k4" ? metal_kq_gemv_k4_msl_entry : (vform == "k5" ? metal_kq_gemv_k5_msl_entry : (vform == "k5c" ? metal_kq_gemv_k5c_msl_entry : metal_kq_gemv_k6_msl_entry)))) + let fm = vform == "k3" ? metal_kq_gemv_k3_msl_fastmath : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl_fastmath : (vform == "k4" ? metal_kq_gemv_k4_msl_fastmath : (vform == "k5" ? metal_kq_gemv_k5_msl_fastmath : (vform == "k5c" ? metal_kq_gemv_k5c_msl_fastmath : metal_kq_gemv_k6_msl_fastmath)))) var pso = pipeline_from_source(dev, src, entry, fm, err) t |> success(pso != null, "{tag}: pipeline: {err}") return if (pso == null) @@ -69,7 +69,7 @@ def private kq_gemv_gate(t : T?; dev, queue; vform : string; n, d : int) { let groups = vform == "k5c" ? (d + 1) / 2 : (d + 3) / 4 let ran = with_compute_encoder(queue, err) $(enc : MetalComputeEncoder?) { metal_set_pipeline(enc, pso) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs: the f16 d tail + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs/k3: the f16 d tail metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bx, 0ul, 3) @@ -102,15 +102,15 @@ def private kq_gemv_gate(t : T?; dev, queue; vform : string; n, d : int) { def private kq_mvb_gate(t : T?; dev, queue; fmt, bwidth, n, d, nr, ys : int) { let tag = "kq_mvb{bwidth}_k{fmt} n={n} d={d} nr={nr} ys={ys}" var err : string - let src = (bwidth == 2 ? (fmt == 44 ? MetalKqMvB2Iq4xs_metal_kq_mv_iq4xs_msl : (fmt == 4 ? MetalKqMvB2K4_metal_kq_mv_k4_msl : (fmt == 5 ? MetalKqMvB2K5_metal_kq_mv_k5_msl : MetalKqMvB2K6_metal_kq_mv_k6_msl))) - : (bwidth == 4 ? (fmt == 44 ? MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl : (fmt == 4 ? MetalKqMvB4K4_metal_kq_mv_k4_msl : (fmt == 5 ? MetalKqMvB4K5_metal_kq_mv_k5_msl : MetalKqMvB4K6_metal_kq_mv_k6_msl))) - : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl : metal_kq_mvb8_k6_msl))))) - let entry = (bwidth == 2 ? (fmt == 44 ? MetalKqMvB2Iq4xs_metal_kq_mv_iq4xs_msl_entry : (fmt == 4 ? MetalKqMvB2K4_metal_kq_mv_k4_msl_entry : (fmt == 5 ? MetalKqMvB2K5_metal_kq_mv_k5_msl_entry : MetalKqMvB2K6_metal_kq_mv_k6_msl_entry))) - : (bwidth == 4 ? (fmt == 44 ? MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl_entry : (fmt == 4 ? MetalKqMvB4K4_metal_kq_mv_k4_msl_entry : (fmt == 5 ? MetalKqMvB4K5_metal_kq_mv_k5_msl_entry : MetalKqMvB4K6_metal_kq_mv_k6_msl_entry))) - : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl_entry : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_entry : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_entry : metal_kq_mvb8_k6_msl_entry))))) - let fm = (bwidth == 2 ? (fmt == 44 ? MetalKqMvB2Iq4xs_metal_kq_mv_iq4xs_msl_fastmath : (fmt == 4 ? MetalKqMvB2K4_metal_kq_mv_k4_msl_fastmath : (fmt == 5 ? MetalKqMvB2K5_metal_kq_mv_k5_msl_fastmath : MetalKqMvB2K6_metal_kq_mv_k6_msl_fastmath))) - : (bwidth == 4 ? (fmt == 44 ? MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl_fastmath : (fmt == 4 ? MetalKqMvB4K4_metal_kq_mv_k4_msl_fastmath : (fmt == 5 ? MetalKqMvB4K5_metal_kq_mv_k5_msl_fastmath : MetalKqMvB4K6_metal_kq_mv_k6_msl_fastmath))) - : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl_fastmath : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_fastmath : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_fastmath : metal_kq_mvb8_k6_msl_fastmath))))) + let src = (bwidth == 2 ? (fmt == 3 ? MetalKqMvB2K3_metal_kq_mv_k3_msl : (fmt == 44 ? MetalKqMvB2Iq4xs_metal_kq_mv_iq4xs_msl : (fmt == 4 ? MetalKqMvB2K4_metal_kq_mv_k4_msl : (fmt == 5 ? MetalKqMvB2K5_metal_kq_mv_k5_msl : MetalKqMvB2K6_metal_kq_mv_k6_msl)))) + : (bwidth == 4 ? (fmt == 3 ? MetalKqMvB4K3_metal_kq_mv_k3_msl : (fmt == 44 ? MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl : (fmt == 4 ? MetalKqMvB4K4_metal_kq_mv_k4_msl : (fmt == 5 ? MetalKqMvB4K5_metal_kq_mv_k5_msl : MetalKqMvB4K6_metal_kq_mv_k6_msl)))) + : (fmt == 3 ? metal_kq_mvb8_k3_msl : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl : metal_kq_mvb8_k6_msl)))))) + let entry = (bwidth == 2 ? (fmt == 3 ? MetalKqMvB2K3_metal_kq_mv_k3_msl_entry : (fmt == 44 ? MetalKqMvB2Iq4xs_metal_kq_mv_iq4xs_msl_entry : (fmt == 4 ? MetalKqMvB2K4_metal_kq_mv_k4_msl_entry : (fmt == 5 ? MetalKqMvB2K5_metal_kq_mv_k5_msl_entry : MetalKqMvB2K6_metal_kq_mv_k6_msl_entry)))) + : (bwidth == 4 ? (fmt == 3 ? MetalKqMvB4K3_metal_kq_mv_k3_msl_entry : (fmt == 44 ? MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl_entry : (fmt == 4 ? MetalKqMvB4K4_metal_kq_mv_k4_msl_entry : (fmt == 5 ? MetalKqMvB4K5_metal_kq_mv_k5_msl_entry : MetalKqMvB4K6_metal_kq_mv_k6_msl_entry)))) + : (fmt == 3 ? metal_kq_mvb8_k3_msl_entry : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl_entry : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_entry : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_entry : metal_kq_mvb8_k6_msl_entry)))))) + let fm = (bwidth == 2 ? (fmt == 3 ? MetalKqMvB2K3_metal_kq_mv_k3_msl_fastmath : (fmt == 44 ? MetalKqMvB2Iq4xs_metal_kq_mv_iq4xs_msl_fastmath : (fmt == 4 ? MetalKqMvB2K4_metal_kq_mv_k4_msl_fastmath : (fmt == 5 ? MetalKqMvB2K5_metal_kq_mv_k5_msl_fastmath : MetalKqMvB2K6_metal_kq_mv_k6_msl_fastmath)))) + : (bwidth == 4 ? (fmt == 3 ? MetalKqMvB4K3_metal_kq_mv_k3_msl_fastmath : (fmt == 44 ? MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl_fastmath : (fmt == 4 ? MetalKqMvB4K4_metal_kq_mv_k4_msl_fastmath : (fmt == 5 ? MetalKqMvB4K5_metal_kq_mv_k5_msl_fastmath : MetalKqMvB4K6_metal_kq_mv_k6_msl_fastmath)))) + : (fmt == 3 ? metal_kq_mvb8_k3_msl_fastmath : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl_fastmath : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_fastmath : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_fastmath : metal_kq_mvb8_k6_msl_fastmath)))))) var pso = pipeline_from_source(dev, src, entry, fm, err) t |> success(pso != null, "{tag}: pipeline: {err}") return if (pso == null) @@ -160,9 +160,9 @@ def private kq_mvb_gate(t : T?; dev, queue; fmt, bwidth, n, d, nr, ys : int) { metal_set_pipeline(enc, pso) if (bwidth == 8) { metal_set_threadgroup_memory_length(enc, - fmt == 44 ? metal_kq_mvb8_iq4xs_msl_tgmem : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_tgmem : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_tgmem : metal_kq_mvb8_k6_msl_tgmem)), 0) + fmt == 3 ? metal_kq_mvb8_k3_msl_tgmem : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl_tgmem : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_tgmem : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_tgmem : metal_kq_mvb8_k6_msl_tgmem))), 0) } - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs: the f16 d tail + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs/k3: the f16 d tail metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bx, 0ul, 3) @@ -251,7 +251,7 @@ def private moe_kq_gemv_gate(t : T?; dev, queue; fmt, n, d, net, k, nst : int; p bxs = uint((perslot ? k : 1) * n), bys = uint(k * d), hasb = 0u) let ran = with_compute_encoder(queue, err) $(enc : MetalComputeEncoder?) { metal_set_pipeline(enc, pso) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs: the f16 d tail + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs/k3: the f16 d tail metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bx, 0ul, 3) @@ -723,8 +723,10 @@ def test_metal_kq_gemv_kernels(t : T?) { kq_gemv_gate(t, dev, queue, "k6", 512, 30) kq_gemv_gate(t, dev, queue, "iq4xs", 1280, 11) kq_gemv_gate(t, dev, queue, "iq4xs", 512, 30) + kq_gemv_gate(t, dev, queue, "k3", 1280, 11) + kq_gemv_gate(t, dev, queue, "k3", 512, 30) // the small-batch twins: nr = 2 / 3 (b4 col guard) / 6 (b8 pad), ys > d - for (fmt in [4, 5, 6, 44]) { + for (fmt in [4, 5, 6, 44, 3]) { kq_mvb_gate(t, dev, queue, fmt, 2, 768, 17, 2, 22) kq_mvb_gate(t, dev, queue, fmt, 4, 768, 17, 3, 22) kq_mvb_gate(t, dev, queue, fmt, 8, 768, 17, 6, 22) From a72e4c5087d3d7cde6a1272b294b1d3bcaa4a6b6 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Sun, 30 Aug 2026 11:43:12 -0700 Subject: [PATCH 11/61] dasLLAMA: IQ4_XS Vulkan GEMV at parity - the codebook as four packed uint4 words instead of a fixed_array local (Function storage); HOW_TO QUIRK 20 + the llama.cpp bench tables for both formats iq4_word held kvalues_iq4nl as a 16-entry fixed_array local and indexed it per nibble; the SPIR-V emitter lowers a dynamically indexed local array to Function storage, and the IQ4_XS decode ran at 105.7 t/s tg128 against k3's 372.8. The codebook now sits in four uint4 words (four int8 codes per word) selected by a dynamic vector index plus a byte shift: 338.4 +- 2.7 t/s against llama.cpp b10660's 340.7 (0.99x), pp512 3363 -> 5161; test_vulkan_kernels 64/64 with the float witness. The Metal twin (iq4_lut) rewritten the same way measured flat on the M1 Max (tg128 144.3 vs 138.7, pp512 2802 vs 2953), so it stays on MSL's constant-address array; the Metal IQ4_XS decode gap is the lane map (followup_general `#58`). HOW_TO: QUIRK 20 (the Function-storage trap, SPIR-V only), the IQ4_XS and Q3_K notes carry the per-tier bench tables against llama.cpp (CPU ahead everywhere, Q3_K GPU decode at or above, Vulkan prefill 0.30x for both = no cm2 decode tile); followup_vulkan item 24 carries the measured prefill numbers as the template's second motivation. Co-Authored-By: Claude Fable 5 --- modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md | 45 ++++++++++++++++++- .../dasllama/dasllama_vulkan_classes.das | 18 +++++--- modules/dasLLAMA/followup_vulkan.md | 6 ++- 3 files changed, 61 insertions(+), 8 deletions(-) diff --git a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md index 72ec615961..23037ef592 100644 --- a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md +++ b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md @@ -370,6 +370,19 @@ where, why it is so today, what unquirked looks like. An empty ledger is a legit the device form (q40, iq4xs, k3) needs its verbatim arm in BOTH. iq4xs had only the grouped one - latent, because every row of the dense 1B is grouped - and the k3 walk found it. Unquirked: one per-format "device form" predicate both branches key on. +20. **A `fixed_array` local in a SPIR-V kernel is Function-storage memory.** `iq4_word`'s + first form held `kvalues_iq4nl` as a 16-entry `fixed_array` local and indexed it per + nibble; the emitter lowers a dynamically indexed local array to Function storage - private + memory the driver spills - and the IQ4_XS Vulkan GEMV decoded at 105.7 t/s against k3's + 372.8 on fewer bytes per weight. The form that runs at speed packs the codebook into four + `uint4` words (four int8 codes per word) and selects with a dynamic vector index plus a byte + shift, `(tbl[q >> 2] >> ((q & 3) * 8)) & 0xFF`: 338.4 t/s, 0.99x llama.cpp, kernel suite + 64/64. SPIR-V-only: the Metal twin `iq4_lut` rewritten the same way measured flat (tg128 + 144.3 +- 6.2 against 138.7 +- 5.4, pp512 2802 against 2953), so MSL's constant-address + array stays - the Metal IQ4_XS decode gap (0.56x llama.cpp) is the lane map, not the + lookup, and belongs to followup_general #58. Unquirked: the SPIR-V emitter lowers a `let` + fixed_array of literals to a constant-storage array, or a lint on a dynamically indexed + fixed_array local inside a kernel class. ## Per-format notes @@ -428,6 +441,19 @@ syntax error at the `let`. Ladders as iq4xs's; fixtures at fmt 3 reuse the k6 sp (`k3_metal_probe.das`, the iq4xs probe with the model swapped) decodes a coherent story at gen 223 t/s, taking the other side of the same token-15 near tie the CPU stamp took. +Against llama.cpp b10660 (`lcpp_bench`, das = the debug-jit instrument; zen2 = 16 threads, M1 = +8; Q3_K_L, so a third of the weight bytes are k5/k6): + +| tier | pp512 das / llama.cpp | tg128 das / llama.cpp | +|---|---|---| +| zen2 CPU | 543.6 / 310.8 (1.75x) | 66.4 / 65.1 (1.02x) | +| 5060 Ti Vulkan | 5174 / 17509 (0.30x) | 372.8 / 349.1 (1.07x) | +| M1 CPU | 524.6 / 223.5 (2.35x) | 127.3 / 110.6 (1.15x) | +| M1 Metal | 3316 / 3219 (1.03x) | 200.9 / 193.0 (1.04x) | + +Decode is at or above llama.cpp on every tier; the Vulkan prefill gap is the missing cm2 tile +(followup_vulkan item 24). + ### IQ4_XS (the pilot, 2026-08-30) Shape: 256-superblock, codebook nibble (`kvalues_iq4nl`, signed, no offset), scale = f16 d x @@ -453,4 +479,21 @@ family gate 10/10 perms, live stamp on this box `dot_maddubs_width256_mr8` (mr 8 decodes at 59-60 t/s against 39 t/s on the reference body, same text. The body rides mx4's chunk-load + lane-splat dot path; `emit_block_kqv2`'s x64 `vpbroadcastd` / `madd16` chains are the untried next lever. Vulkan (section 6): `KqGemvIq4xs` + `KqBatchIq4xs`, the kernel suite -64/64 with the five-format family cells and the float witness, and the 1B IQ4_XS model on the resident driver reproduces the CPU text at gen 102 t/s (prefill 38 t/s on the 5-token prompt). Metal: pending. +64/64 with the five-format family cells and the float witness, and the 1B IQ4_XS model on the +resident driver reproduces the CPU text at gen 102 t/s (prefill 38 t/s on the 5-token prompt); +the codebook lookup then cost 3.5x on the GEMV until QUIRK 20's packed-word form. Metal +(section 7): `MetalKqGemvIq4xs`, the `MetalKqMvIq4xsT` B2/B4 + `MetalKqMvB8Iq4xs` twins, the +`IQ4XS` mul_mm arm; gates 2/2 + 2/2 on the M1 Max. + +Against llama.cpp b10660 (`lcpp_bench`, das = the debug-jit instrument, ~8% under the tuned +exe; zen2 = 16 threads, M1 = 8): + +| tier | pp512 das / llama.cpp | tg128 das / llama.cpp | +|---|---|---| +| zen2 CPU | 475.7 / 256.9 (1.85x) | 65.0 / 59.5 (1.09x) | +| 5060 Ti Vulkan | 5161 / 17060 (0.30x) | 338.4 / 340.7 (0.99x) | +| M1 CPU | 796.0 / 263.0 (3.03x) | 132.3 / 128.2 (1.03x) | +| M1 Metal | 2953 / 3565 (0.83x) | 138.7 / 247.3 (0.56x) | + +The Vulkan prefill gap is the missing cm2 decode tile (followup_vulkan item 24: k4 on the same +box does 13144, 0.67x); the Metal decode gap is the per-lane block map (followup_general #58). diff --git a/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das b/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das index 5ca386026c..698c3d49f4 100644 --- a/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das +++ b/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das @@ -2465,13 +2465,19 @@ def private k5_dep(hb : uint) : uint { } // IQ4_XS codebook decode of one word of 4 nibble lanes (bits 7..4 of each byte clear) into 4 -// SIGNED int8 lanes (ggml's kvalues_iq4nl) — the sdot4 operand form; batch and gemv share it +// SIGNED int8 lanes (ggml's kvalues_iq4nl) — the sdot4 operand form; batch and gemv share it. +// The codebook is four packed words picked by a dynamic vector index and byte-extracted: pure +// ALU (a fixed_array local was a Function-storage table, 3.4x slower on the decode GEMV) def private iq4_word(n : uint) : uint { - let lut = fixed_array(-127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113) - let b0 = uint(lut[int(n & 15u)]) & 0xFFu - let b1 = uint(lut[int((n >> 8u) & 15u)]) & 0xFFu - let b2 = uint(lut[int((n >> 16u) & 15u)]) & 0xFFu - let b3 = uint(lut[int((n >> 24u) & 15u)]) & 0xFFu + let tbl = uint4(3215825025u, 4142587343u, 639175937u, 1901675829u) // kvalues_iq4nl, 4 bytes per word + let q0 = n & 15u + let q1 = (n >> 8u) & 15u + let q2 = (n >> 16u) & 15u + let q3 = (n >> 24u) & 15u + let b0 = (tbl[int(q0 >> 2u)] >> ((q0 & 3u) * 8u)) & 0xFFu + let b1 = (tbl[int(q1 >> 2u)] >> ((q1 & 3u) * 8u)) & 0xFFu + let b2 = (tbl[int(q2 >> 2u)] >> ((q2 & 3u) * 8u)) & 0xFFu + let b3 = (tbl[int(q3 >> 2u)] >> ((q3 & 3u) * 8u)) & 0xFFu return b0 | (b1 << 8u) | (b2 << 16u) | (b3 << 24u) } diff --git a/modules/dasLLAMA/followup_vulkan.md b/modules/dasLLAMA/followup_vulkan.md index 4bb0e15c69..2eef4dd896 100644 --- a/modules/dasLLAMA/followup_vulkan.md +++ b/modules/dasLLAMA/followup_vulkan.md @@ -447,7 +447,11 @@ module) is independent and can land any time - it is pure structure. `def override decode_*` per format - the shape `harness/vk_gemm_probe.das`'s `K6PxBase` already proves. Gate: the nine oracle cells in `tests/test_vulkan_kernels.das` stay 0-off, the probe's l/m rows stay within noise. The k5/q40 stamps (item 11's NEXT) land on the - template, not as more copies. + template, not as more copies. Measured 2026-08-30 on the 1B (`lcpp_bench`, 5060 Ti): the + formats with no cm2 tile prefill at 5161 (iq4xs) and 5174 (k3) t/s against k4's 13144 on + the same box (llama.cpp 17060 / 17509 / 19719) while their decode sits at parity, so the + template is also the lever that puts every new format on the tile path; k4's own 0.67x is + the 1B-shape tier gap, a separate item. 25. **Try `VK_NV_cooperative_vector` for decode GEMV on real hardware (Boris, 2026-08-28).** cm2 has no matrix-vector op - its seven feature bits are all tile-shaped, minimum tile 16 - From d08d415b1cc426573a811825a349b9f10271b72d Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Sun, 30 Aug 2026 11:59:38 -0700 Subject: [PATCH 12/61] dasLLAMA: IQ4_XS Metal GEMV at parity - the codebook in threadgroup memory as floats, 16 lanes per superblock, float4 x loads (llama.cpp's kernel_mul_mv_iq4_xs shape) The per-weight constant-array lookup (iq4_lut: two selects, a shift, a mask, the sign fold, the float convert) was the whole gap: MetalKqGemvIq4xs now stages kvalues_iq4nl as 64 floats of threadgroup memory once per group, a 16-lane half of the simdgroup owns one superblock, lane (bu, il) the 8-byte half il of block bu - two uint loads, four float4 x loads, sixteen LUT reads. The gate cell sets the kernel's threadgroup length like the mvb8 cell does. M1 Max, 1B IQ4_XS, lcpp_bench --ngl 99 (das = the debug-jit instrument): tg128 249.4 +- 11.4 against llama.cpp b10660's 249.3 (was 138.7, 0.56x); pp512 unchanged at 2953 (0.83x - the mul_mm arm's turn next). test_metal_gemv_kernels 2/2. Research first (Boris's rule): scratchpad research_llamacpp_iq_kernels.md - llama.cpp keeps every codebook in shared/threadgroup memory pre-converted to float on both backends (PR #10409: IQ4_NL Llama-8B 17.6 -> 59.5 t/s from staging 16 bytes). Co-Authored-By: Claude Fable 5 --- .../dasllama/dasllama_metal_kernels.das | 54 ++++++++++--------- .../tests/test_metal_gemv_kernels.das | 3 ++ 2 files changed, 32 insertions(+), 25 deletions(-) diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das index 2e014df246..bd76d8dde8 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das @@ -7203,51 +7203,55 @@ class MetalKqMvB8K3 { } } -// IQ4_XS: w = d*sc*LUT[q] with sc the SIGNED (ls - 32) strip byte and no min plane; each lane owns -// one 32-block of every 4th superblock (the q40 pairing: byte l -> elems l and l + 16), the scale -// plane is the k6 split form (d plane at s0off = doff, 16B sub-scale strips at soff). -[metal_dispatch(name = "enc_kq_iq4xs_c", pso = "g_pso_kq_iq4xs", tg = 64, grid = "rows/4", params = "rows : int64, n : int64")] +// IQ4_XS: w = d*sc*LUT[q] with sc the SIGNED (ls - 32) strip byte and no min plane. The codebook +// sits in threadgroup memory as floats; a 16-lane half of the simdgroup owns one superblock and +// lane (bu, il) the 8-byte half il of block bu - 8 lo + 8 hi weights from two uint loads against +// four float4 x loads (the q40 pairing: byte l -> elems l and l + 16). The scale plane is the k6 +// split form (d plane at s0off = doff, 16B sub-scale strips at soff). +[metal_dispatch(name = "enc_kq_iq4xs_c", pso = "g_pso_kq_iq4xs", tgmem = "metal_kq_gemv_iq4xs_msl_tgmem", tg = 64, grid = "rows/4", params = "rows : int64, n : int64")] class MetalKqGemvIq4xs { @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d plane — the scale buffer bound at byte nsb*16 (s0off = the caller's doff) @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // 16B strips: 8 signed sub-scales + 8 pad, byte view (same buffer at 0) @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array // iq4xs quant plane, uint view (block bu of blk at 32*blk + 4*bu) - @ssbo @binding = 3 @role = "read" @off = "xoff" @span = "n*4" x : array + @ssbo @binding = 3 @role = "read" @off = "xoff" @span = "n*4" x : array @ssbo @binding = 4 @role = "write" @off = "yoff" @span = "rows*4" y : array @uniform @binding = 5 ndim : uint @uniform @binding = 6 ddim : uint + @workgroup lut : float[64] // kvalues_iq4nl as floats, one slot per thread of the 64-wide group [metal_kernel(name="metal_kq_gemv_iq4xs_msl")] def metal_kq_gemv_iq4xs { + let lid = gl_LocalInvocationID.x + lut[lid] = iq4_lut(lid & 15u) + barrier() let lane = gl_SubgroupInvocationID - let ix = lane / 8u - let bu = lane % 8u + let ix = lane / 16u + let it = lane % 16u + let bu = it / 2u + let il = it % 2u let nb = ndim / 256u let first_row = (gl_WorkGroupID.x * 2u + gl_SubgroupID) * 2u - var yl : float[16] - var yh : float[16] var sumf : float[2] var ib = ix while (ib < nb) { - let yb = ib * 256u + bu * 32u - for [unroll_full] (i in range(16)) { - yl[i] = x[yb + uint(i)] - yh[i] = x[yb + 16u + uint(i)] - } + let yb4 = ib * 64u + bu * 8u + il * 2u + let y0 = x[yb4] + let y1 = x[yb4 + 1u] + let y2 = x[yb4 + 4u] + let y3 = x[yb4 + 5u] for [unroll_full] (r in range(2)) { let blk = (first_row + uint(r)) * nb + ib - let qb = blk * 32u + bu * 4u - var acc = 0.0 - for [unroll_full] (u in range(4)) { - let qv = kqu[qb + uint(u)] - for [unroll_full] (e in range(4)) { - let byv = (qv >> (8u * uint(e))) & 255u - acc += yl[4 * u + e] * iq4_lut(byv & 15u) + yh[4 * u + e] * iq4_lut(byv >> 4u) - } - } + let qb = blk * 32u + bu * 4u + il * 2u + let q0 = kqu[qb] + let q1 = kqu[qb + 1u] + var acc = y0 * float4(lut[q0 & 15u], lut[(q0 >> 8u) & 15u], lut[(q0 >> 16u) & 15u], lut[(q0 >> 24u) & 15u]) + acc += y2 * float4(lut[(q0 >> 4u) & 15u], lut[(q0 >> 12u) & 15u], lut[(q0 >> 20u) & 15u], lut[q0 >> 28u]) + acc += y1 * float4(lut[q1 & 15u], lut[(q1 >> 8u) & 15u], lut[(q1 >> 16u) & 15u], lut[(q1 >> 24u) & 15u]) + acc += y3 * float4(lut[(q1 >> 4u) & 15u], lut[(q1 >> 12u) & 15u], lut[(q1 >> 20u) & 15u], lut[q1 >> 28u]) let sc = float((int(kscb[blk * 16u + bu]) ^ 128) - 128) - sumf[r] += float(kdh[blk]) * sc * acc + sumf[r] += float(kdh[blk]) * sc * (acc.x + acc.y + acc.z + acc.w) } - ib += 4u + ib += 2u } for [unroll_full] (r in range(2)) { let s = simd_sum(sumf[r]) diff --git a/modules/dasLLAMA/tests/test_metal_gemv_kernels.das b/modules/dasLLAMA/tests/test_metal_gemv_kernels.das index 6b02e13d31..fdc1be18be 100644 --- a/modules/dasLLAMA/tests/test_metal_gemv_kernels.das +++ b/modules/dasLLAMA/tests/test_metal_gemv_kernels.das @@ -69,6 +69,9 @@ def private kq_gemv_gate(t : T?; dev, queue; vform : string; n, d : int) { let groups = vform == "k5c" ? (d + 1) / 2 : (d + 3) / 4 let ran = with_compute_encoder(queue, err) $(enc : MetalComputeEncoder?) { metal_set_pipeline(enc, pso) + if (vform == "iq4xs") { + metal_set_threadgroup_memory_length(enc, metal_kq_gemv_iq4xs_msl_tgmem, 0) // the codebook slab + } metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs/k3: the f16 d tail metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) From 749525dc8915afc1228bf015986b1a1efef1f960 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Sun, 30 Aug 2026 12:14:52 -0700 Subject: [PATCH 13/61] dasLLAMA: IQ4_XS Metal prefill above parity - the mul_mm arm reads the codebook off threadgroup memory The IQ4XS arm of MetalKqMulMmK45T decoded through the same per-element constant-array lookup the GEMV just shed; the template now carries a @template_gate = IQ4XS threadgroup float[16], staged by the first 16 threads ahead of the tile loop, and the arm indexes it. M1 Max, 1B IQ4_XS, lcpp_bench --ngl 99: pp512 3611.6 +- 5.1 against llama.cpp b10660's 3574.7 (1.01x, was 2953 = 0.83x); tg128 holds 249.3. test_metal_gemm_kernels 2/2. With the GEMV commit this closes the Metal IQ4_XS gap on both axes. Co-Authored-By: Claude Fable 5 --- modules/dasLLAMA/dasllama/dasllama_metal_kernels.das | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das index bd76d8dde8..19f02f2a94 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das @@ -5467,10 +5467,17 @@ class template MetalKqMulMmK45T : MetalMmTileBase { @template_constant SIXBIT : bool = false // q6_K: int8 sub-scales x super-d, ql/qh compose @template_constant IQ4XS : bool = false // iq4xs: the k6 split scale form, the q40 pairing, codebook decode @template_constant K3 : bool = false // q3_K: the k6 split scale form, 2-bit lane + hmask compose, offset 4 + @workgroup @template_gate = IQ4XS iq4lut : float[16] // kvalues_iq4nl as floats, staged once per threadgroup [metal_kernel] def metal_kq_mulmm { // nolint:STYLE038 — MSL kernel body; the mul_mm tiles are simdgroup-barrier coupled let lid = gl_LocalInvocationID.x + static_if (IQ4XS) { + if (lid < 16u) { + iq4lut[lid] = iq4_lut(lid) + } + barrier() + } let sg = gl_SubgroupID let mBase = gl_WorkGroupID.x * 32u let nBase = gl_WorkGroupID.y * 64u @@ -5566,7 +5573,8 @@ class template MetalKqMulMmK45T : MetalMmTileBase { } else { static_if (IQ4XS) { // iq4xs: strip byte js is the SIGNED sub-scale, super-d off the d plane (the k6 split - // form); block js = 4 uints of the q40 pairing, il0 picks the nibble half; codebook per element + // form); block js = 4 uints of the q40 pairing, il0 picks the nibble half; the codebook + // off threadgroup memory let sv = ks4[blk] let scw = js < 4u ? sv.x : sv.y let dsc = float(ksh[blk]) * float(((int(scw >> ((js & 3u) * 8u)) & 255) ^ 128) - 128) @@ -5575,7 +5583,7 @@ class template MetalKqMulMmK45T : MetalMmTileBase { for [unroll_full] (k in range(4)) { let u = kqu[qb + uint(k)] for [unroll_full] (c in range(4)) { - va[k * 4 + c] = float16(dsc * iq4_lut((u >> (8u * uint(c) + nsh)) & 15u)) + va[k * 4 + c] = float16(dsc * iq4lut[(u >> (8u * uint(c) + nsh)) & 15u]) } } } else { From 05d83f3edd20b858d0647a5686ac0c34dff0a4dd Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Sun, 30 Aug 2026 12:25:05 -0700 Subject: [PATCH 14/61] dasLLAMA: the nine cm2 tiles stamp from one class template (followup_vulkan item 24) KqCm2BatchT carries the whole fast/edge tile body once; the width axis is BN (256/128/32) + STILE (the s column's partial-column fast path, flb rounded to 32, flo Pad) + the BT/ACC/ACCW/ FLO typedefs, the format axis is a template chain - Q8Cm2T/K4Cm2T/K6Cm2T bind BLK/ST/BLKW and author the [spirv_decode] decode - and the nine dispatch classes are eight-line stamps. The bodies are verbatim: the same layouts, the literal no-split loop bounds, the same kernel surface names, so the dispatchers and tests bind unchanged. ~1240 hand-stamped lines fold into ~330; REVIEW_GPU's twin rule stops being violated by this family. Gates on the 5060 Ti: test_vulkan_kernels 63 passed / 1 skipped (the mode-ladder cell, always skipped under DASLLAMA_COOPMAT=cm2) with all six cm2 oracle cells 0-off; the 1B Q4_K_M e2e control row within noise of the hand-stamped baseline - pp512 12997.7 +- 1501.5 against 13144.3 +- 601.2, tg128 358.2 against 360.5. This is the landing pad for the parity work: the iq4xs/k3 (and k5/q40) prefill gap is a missing cm2 tile per format (0.30x llama.cpp on the 1B), and on the template each new format is a decode method, not three kernel copies. Co-Authored-By: Claude Fable 5 --- .../dasllama/dasllama_vulkan_classes.das | 1580 ++--------------- 1 file changed, 172 insertions(+), 1408 deletions(-) diff --git a/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das b/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das index 698c3d49f4..8f8eeb48c6 100644 --- a/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das +++ b/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das @@ -3722,37 +3722,46 @@ class KqQ40CmF16 : MoeCmBase { } } -// ===== the cm2 (NV_cooperative_matrix2) prefill tiles: native fmt-0 planes, decode-in-load ===== +// ===== the cm2 (NV_cooperative_matrix2) prefill tiles: native planes, decode-in-load ===== struct VkQ8Blk { - qs : int16[16] // one fmt-0 q8 block: 32 packed int8 quants as 16-bit lanes; the scale lives in wsh -} - -// the cm2 quant "l" geometry on the NATIVE fmt-0 two-plane layout: 256 threads, one -// 128-weight x 256-token tile per wg, BK=64 (2 blocks) per step, 8x-unrolled k loop. The decode -// METHOD reads the separate scale plane by block index — the reason the class-method decode form -// exists. Binding 4 unused (sparse) -[vk_dispatch(name = "q8_batch_cm2l_cls", grid = "wgs", params = "wgs : int64")] -class Q8Cm2LBatch : MoeCmBase { - @ssbo @binding = 0 wq : array // fmt-0 weight quant blocks (32 bytes each, read as 16-bit lanes) - @ssbo @binding = 1 wsh : array // per-block weight scales (the second plane) - @ssbo @binding = 3 xf16 : array // f16 activation plane - @ssbo @binding = 5 y : array - @workgroup wg_blk0 : uint // the region's block base, staged for the decode method + qs : int16[16] // one fmt-0 q8 block: 32 packed int8 quants as 16-bit lanes; the scale lives in ws +} - [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] - def decode_q8(blk : VkQ8Blk; bc, cib : uint2) : float16 { - let q = unpack8(blk.qs[int((cib.y & 30u) >> 1u)])[int(cib.y & 1u)] - return wsh[wg_blk0 + bc.x * (pa.n >> 5u) + bc.y] * float16(float(int(q))) - } +struct VkK4Blk { + qs : int16[64] // one Q4_K superblock's 128 nibble bytes as 16-bit lanes; its scale row (d|dmin + 8 sc + 8 mn bytes) lives in ws +} + +struct VkK6Blk { + ql : int16[64] // one Q6_K superblock's 128 low-nibble bytes as 16-bit lanes + qh : int16[32] // ... and its 64 high-2-bit bytes; scales (16 int8 + f16 d) live in ws +} + +// The cm2 tile: one class template over the two stamp axes. The token column BN - l 256, m 128, +// s 32 (the expert-bucket geometry: STILE's fast path loads a partial column UNCLAMPED and clamps +// only the store, so every plane it reads carries 32 rows of slack) - and the weight format: BLK +// the 16-bit-lane block struct, ST the scale plane's element, BLKW the block's element width, and +// the format template's decode. 256 threads, one 128-weight x BN-token tile per wg, BK=64 per +// step, an 8x-unrolled k loop; the decode METHOD reads the separate scale plane by block index - +// the reason the class-method decode form exists. Binding 4 unused (sparse) +[ |> template_struct_instance] +class template KqCm2BatchT : MoeCmBase { + @ssbo @binding = 0 @role = "weight" wq : array // the quant plane, read as 16-bit lanes + @ssbo @binding = 1 @role = "weight" ws : array // the scale plane (q8: f16 per block; kq: 5 words per superblock) + @ssbo @binding = 3 xf16 : array // f16 activation plane + @ssbo @binding = 5 y : array + @workgroup wg_blk0 : uint // the region's block base, staged for the decode + @template_constant BN : uint = 256u // the token column + @template_constant STILE : bool = false // the 32-row column's partial-column fast path + @template_constant BLKW : uint = 256u // elements per weight block - [spirv_kernel(local_size_x = 256, name = "q8_batch_cm2l_cls_spv"), arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-tile-pick-and-default")] + [spirv_kernel(local_size_x = 256), arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-tile-pick-and-default")] def run { // nolint:STYLE038 — the fast/edge path pair, barrier- and register-coupled let reg = region_rec() let wblk0 = reg.x let row0 = reg.y let cnt = reg.z - let ttiles = (cnt + 255u) / 256u + let ttiles = (cnt + BN - 1u) / BN // split-k: wgs stack nsplit planes of (wtiles x ttiles); plane ks covers k [k0, k1) and // stores f32 partials at ybase (the reduce sums the planes into y). SINGLE-REGION (dense) // dispatches only — the plane size rides row0 + cnt, so multi-region planes would collide @@ -3768,7 +3777,7 @@ class Q8Cm2LBatch : MoeCmBase { ybase = ks * (row0 + cnt) * pa.d } let k0 = ks * pa.ksplit - let xt = tix % ttiles // token 256-tile (fastest) + let xt = tix % ttiles // token BN-tile (fastest) let wt = tix / ttiles // weight 128-tile if (gl_LocalInvocationID.x == 0u) { wg_blk0 = wblk0 @@ -3777,25 +3786,26 @@ class Q8Cm2LBatch : MoeCmBase { tensorViewCreate(tv) barrier() // wg_blk0 visible before the first decode load var a : coopmatWgA_f16_128x64 - var b : coopmatWgB_f16_64x256 - var acc : coopmatWgAcc_f16_128x256 - let t0 = row0 + xt * 256u + var b : BT + var acc : ACC + let t0 = row0 + xt * BN let m0 = wt * 128u // FAST PATH — every access of this tile provably in bounds and K a whole number of // BK steps: unclamped (clamp-Undefined) layouts, and the B/out strides masked to a // multiple of 8 f16 (identity here — pa.n and pa.d are 32-multiples — but the mask makes - // the alignment PROVABLE to the driver's address analysis via stride &= ~7) - if (m0 + 128u <= pa.d && xt * 256u + 256u <= cnt && (pa.n & 63u) == 0u) { + // the alignment PROVABLE to the driver's address analysis via stride &= ~7). The s column + // gates only the weight tile: its partial token column loads unclamped, its store clamps + if (m0 + 128u <= pa.d && (STILE || xt * BN + BN <= cnt) && (pa.n & 63u) == 0u) { var fla : tensorLayout2D tensorLayoutCreate(fla) - tensorLayoutSetBlockSize(fla, 1u, 32u) + tensorLayoutSetBlockSize(fla, 1u, BLKW) tensorLayoutSetDimension(fla, pa.d, pa.n) - tensorLayoutSetStride(fla, pa.n >> 5u, 1u) + tensorLayoutSetStride(fla, pa.n / BLKW, 1u) var flb : tensorLayout2D tensorLayoutCreate(flb) - tensorLayoutSetDimension(flb, row0 + cnt, pa.n) + tensorLayoutSetDimension(flb, row0 + (STILE ? ((cnt + 31u) & ~31u) : cnt), pa.n) tensorLayoutSetStride(flb, pa.n & ~7u, 1u) - var flo : tensorLayout2D + var flo : FLO tensorLayoutCreate(flo) tensorLayoutSetDimension(flo, row0 + cnt, pa.d) tensorLayoutSetStride(flo, pa.d & ~7u, 1u) @@ -3805,228 +3815,51 @@ class Q8Cm2LBatch : MoeCmBase { var k = 0u for (_i in range(int(pa.n / 512u))) { for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_q8) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 256u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < pa.n) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_q8) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 256u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x256 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, 0u, flo, t0, 256u, m0, 128u, tv) - return - } - var k = k0 - for (_i in range(int((k1 - k0) / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_q8) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 256u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < k1) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_q8) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 256u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x256 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, ybase, flo, t0, 256u, m0, 128u, tv) - return - } - // EDGE PATH — clamp-Constant layouts everywhere. A: the region's [d x n] q8 plane, block - // addressing; layout coords are region-local, so the decode method's scale index is - // wg_blk0 + bc.x * nbb + bc.y off the staged base - var tla : tensorLayout2DPad - tensorLayoutCreate(tla) - tensorLayoutSetBlockSize(tla, 1u, 32u) - tensorLayoutSetDimension(tla, pa.d, pa.n) - tensorLayoutSetStride(tla, pa.n / 32u, 1u) - // B: the f16 activation rows, transpose-viewed; the dimension cuts at the region's end - var tlb : tensorLayout2DPad - tensorLayoutCreate(tlb) - tensorLayoutSetDimension(tlb, row0 + cnt, pa.n) - tensorLayoutSetStride(tlb, pa.n, 1u) - // out: token-major y through the transpose view. MUST be the clamp-Constant layout type: - // only clamped layouts DISCARD out-of-bounds stores from a partial tile - var tlo : tensorLayout2DPad - tensorLayoutCreate(tlo) - tensorLayoutSetDimension(tlo, row0 + cnt, pa.d) - tensorLayoutSetStride(tlo, pa.d, 1u) - // the 8x-unrolled main loop (their UNROLL_K = BK*8 = 512), then the sub-512 tail. - // The hint is honored: a manual 4x expansion A/Bed dead even (2719 +- 20 vs 2749 +- 12 - // on the 3B row), so the loop form stays. Same literal-bound specialization as the - // fast path — the no-split arm never reads k0/k1/ybase - if (pa.ksplit == 0u) { - var k = 0u - for (_i in range(int(pa.n / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_q8) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 256u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < pa.n) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_q8) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 256u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x256 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, 0u, tlo, t0, 256u, m0, 128u, tv) - return - } - var k = k0 - for (_i in range(int((k1 - k0) / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_q8) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 256u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < k1) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_q8) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 256u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x256 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, ybase, tlo, t0, 256u, m0, 128u, tv) - } -} - -// the m-tile sibling (cm2 quant "m": BN=128, same A / BK / thread count); the body mirrors -// Q8Cm2LBatch at 128-token width -[vk_dispatch(name = "q8_batch_cm2m_cls", grid = "wgs", params = "wgs : int64")] -class Q8Cm2MBatch : MoeCmBase { - @ssbo @binding = 0 wq : array // fmt-0 weight quant blocks (32 bytes each, read as 16-bit lanes) - @ssbo @binding = 1 wsh : array // per-block weight scales (the second plane) - @ssbo @binding = 3 xf16 : array // f16 activation plane - @ssbo @binding = 5 y : array - @workgroup wg_blk0 : uint // the region's block base, staged for the decode method - - [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] - def decode_q8(blk : VkQ8Blk; bc, cib : uint2) : float16 { - let q = unpack8(blk.qs[int((cib.y & 30u) >> 1u)])[int(cib.y & 1u)] - return wsh[wg_blk0 + bc.x * (pa.n >> 5u) + bc.y] * float16(float(int(q))) - } - - [spirv_kernel(local_size_x = 256, name = "q8_batch_cm2m_cls_spv"), arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-tile-pick-and-default")] - def run { // nolint:STYLE038 — the fast/edge path pair, barrier- and register-coupled - let reg = region_rec() - let wblk0 = reg.x - let row0 = reg.y - let cnt = reg.z - let ttiles = (cnt + 127u) / 128u - // split-k: wgs stack nsplit planes of (wtiles x ttiles); plane ks covers k [k0, k1) and - // stores f32 partials at ybase (the reduce sums the planes into y). SINGLE-REGION (dense) - // dispatches only — the plane size rides row0 + cnt, so multi-region planes would collide - var tix = reg.w - var ks = 0u - var k1 = pa.n - var ybase = 0u - if (pa.ksplit != 0u) { - let ptiles = ((pa.d + 127u) / 128u) * ttiles - ks = tix / ptiles - tix -= ks * ptiles - k1 = min(pa.n, (ks + 1u) * pa.ksplit) - ybase = ks * (row0 + cnt) * pa.d - } - let k0 = ks * pa.ksplit - let xt = tix % ttiles // token 128-tile (fastest) - let wt = tix / ttiles // weight 128-tile - if (gl_LocalInvocationID.x == 0u) { - wg_blk0 = wblk0 - } - var tv : tensorView2Dt - tensorViewCreate(tv) - barrier() // wg_blk0 visible before the first decode load - var a : coopmatWgA_f16_128x64 - var b : coopmatWgB_f16_64x128 - var acc : coopmatWgAcc_f16_128x128 - let t0 = row0 + xt * 128u - let m0 = wt * 128u - // FAST PATH — see Q8Cm2LBatch; identical rule at the m width, same literal-bound - // no-split specialization (the k0/k1/ybase form in the hot loop measured -27% pp) - if (m0 + 128u <= pa.d && xt * 128u + 128u <= cnt && (pa.n & 63u) == 0u) { - var fla : tensorLayout2D - tensorLayoutCreate(fla) - tensorLayoutSetBlockSize(fla, 1u, 32u) - tensorLayoutSetDimension(fla, pa.d, pa.n) - tensorLayoutSetStride(fla, pa.n >> 5u, 1u) - var flb : tensorLayout2D - tensorLayoutCreate(flb) - tensorLayoutSetDimension(flb, row0 + cnt, pa.n) - tensorLayoutSetStride(flb, pa.n & ~7u, 1u) - var flo : tensorLayout2D - tensorLayoutCreate(flo) - tensorLayoutSetDimension(flo, row0 + cnt, pa.d) - tensorLayoutSetStride(flo, pa.d & ~7u, 1u) - if (pa.ksplit == 0u) { - var k = 0u - for (_i in range(int(pa.n / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_q8) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 128u, k, 64u, tv) + coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode) + coopmatLoadTensor(b, xf16, 0u, flb, t0, BN, k, 64u, tv) acc = coopmatMulAdd(a, b, acc) k += 64u } } while (k < pa.n) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_q8) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 128u, k, 64u, tv) + coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode) + coopmatLoadTensor(b, xf16, 0u, flb, t0, BN, k, 64u, tv) acc = coopmatMulAdd(a, b, acc) k += 64u } coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x128 + var accw : ACCW coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, 0u, flo, t0, 128u, m0, 128u, tv) + coopmatStoreTensor(accw, y, 0u, flo, t0, BN, m0, 128u, tv) return } var k = k0 for (_i in range(int((k1 - k0) / 512u))) { for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_q8) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 128u, k, 64u, tv) + coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode) + coopmatLoadTensor(b, xf16, 0u, flb, t0, BN, k, 64u, tv) acc = coopmatMulAdd(a, b, acc) k += 64u } } while (k < k1) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_q8) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 128u, k, 64u, tv) + coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode) + coopmatLoadTensor(b, xf16, 0u, flb, t0, BN, k, 64u, tv) acc = coopmatMulAdd(a, b, acc) k += 64u } coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x128 + var accw : ACCW coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, ybase, flo, t0, 128u, m0, 128u, tv) + coopmatStoreTensor(accw, y, ybase, flo, t0, BN, m0, 128u, tv) return } // EDGE PATH — clamp-Constant layouts everywhere (store discard load-bearing) var tla : tensorLayout2DPad tensorLayoutCreate(tla) - tensorLayoutSetBlockSize(tla, 1u, 32u) + tensorLayoutSetBlockSize(tla, 1u, BLKW) tensorLayoutSetDimension(tla, pa.d, pa.n) - tensorLayoutSetStride(tla, pa.n / 32u, 1u) + tensorLayoutSetStride(tla, pa.n / BLKW, 1u) var tlb : tensorLayout2DPad tensorLayoutCreate(tlb) tensorLayoutSetDimension(tlb, row0 + cnt, pa.n) @@ -4039,920 +3872,87 @@ class Q8Cm2MBatch : MoeCmBase { var k = 0u for (_i in range(int(pa.n / 512u))) { for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_q8) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 128u, k, 64u, tv) + coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode) + coopmatLoadTensor(b, xf16, 0u, tlb, t0, BN, k, 64u, tv) acc = coopmatMulAdd(a, b, acc) k += 64u } } while (k < pa.n) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_q8) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 128u, k, 64u, tv) + coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode) + coopmatLoadTensor(b, xf16, 0u, tlb, t0, BN, k, 64u, tv) acc = coopmatMulAdd(a, b, acc) k += 64u } coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x128 + var accw : ACCW coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, 0u, tlo, t0, 128u, m0, 128u, tv) + coopmatStoreTensor(accw, y, 0u, tlo, t0, BN, m0, 128u, tv) return } var k = k0 for (_i in range(int((k1 - k0) / 512u))) { for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_q8) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 128u, k, 64u, tv) + coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode) + coopmatLoadTensor(b, xf16, 0u, tlb, t0, BN, k, 64u, tv) acc = coopmatMulAdd(a, b, acc) k += 64u } } while (k < k1) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_q8) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 128u, k, 64u, tv) + coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode) + coopmatLoadTensor(b, xf16, 0u, tlb, t0, BN, k, 64u, tv) acc = coopmatMulAdd(a, b, acc) k += 64u } coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x128 + var accw : ACCW coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, ybase, tlo, t0, 128u, m0, 128u, tv) + coopmatStoreTensor(accw, y, ybase, tlo, t0, BN, m0, 128u, tv) } } -// the s-tile sibling (BN=32): the MoE expert-bucket shape - a region of ~32 rows fills the -// tile instead of padding three quarters of an m column; the body mirrors Q8Cm2MBatch at -// 32-token width -[vk_dispatch(name = "q8_batch_cm2s_cls", grid = "wgs", params = "wgs : int64")] -class Q8Cm2SBatch : MoeCmBase { - @ssbo @binding = 0 wq : array // fmt-0 weight quant blocks (32 bytes each, read as 16-bit lanes) - @ssbo @binding = 1 wsh : array // per-block weight scales (the second plane) - @ssbo @binding = 3 xf16 : array // f16 activation plane - @ssbo @binding = 5 y : array - @workgroup wg_blk0 : uint // the region's block base, staged for the decode method +// the format templates: the block struct, the scale element, the block width and the decode +[ |> template_struct_instance] +class template Q8Cm2T : KqCm2BatchT { + typedef BLK = VkQ8Blk + typedef ST = float16 + override BLKW = 32u [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] - def decode_q8(blk : VkQ8Blk; bc, cib : uint2) : float16 { + def decode(blk : VkQ8Blk; bc, cib : uint2) : float16 { let q = unpack8(blk.qs[int((cib.y & 30u) >> 1u)])[int(cib.y & 1u)] - return wsh[wg_blk0 + bc.x * (pa.n >> 5u) + bc.y] * float16(float(int(q))) - } - - [spirv_kernel(local_size_x = 256, name = "q8_batch_cm2s_cls_spv"), arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-tile-pick-and-default")] - def run { // nolint:STYLE038 — the fast/edge path pair, barrier- and register-coupled - let reg = region_rec() - let wblk0 = reg.x - let row0 = reg.y - let cnt = reg.z - let ttiles = (cnt + 31u) / 32u - // split-k: wgs stack nsplit planes of (wtiles x ttiles); plane ks covers k [k0, k1) and - // stores f32 partials at ybase (the reduce sums the planes into y). SINGLE-REGION (dense) - // dispatches only — the plane size rides row0 + cnt, so multi-region planes would collide - var tix = reg.w - var ks = 0u - var k1 = pa.n - var ybase = 0u - if (pa.ksplit != 0u) { - let ptiles = ((pa.d + 127u) / 128u) * ttiles - ks = tix / ptiles - tix -= ks * ptiles - k1 = min(pa.n, (ks + 1u) * pa.ksplit) - ybase = ks * (row0 + cnt) * pa.d - } - let k0 = ks * pa.ksplit - let xt = tix % ttiles // token 32-tile (fastest) - let wt = tix / ttiles // weight 128-tile - if (gl_LocalInvocationID.x == 0u) { - wg_blk0 = wblk0 - } - var tv : tensorView2Dt - tensorViewCreate(tv) - barrier() // wg_blk0 visible before the first decode load - var a : coopmatWgA_f16_128x64 - var b : coopmatWgB_f16_64x32 - var acc : coopmatWgAcc_f16_128x32 - let t0 = row0 + xt * 32u - let m0 = wt * 128u - // FAST PATH — see Q8Cm2LBatch; identical rule at the s width, same literal-bound - // no-split specialization (the k0/k1/ybase form in the hot loop measured -27% pp) - if (m0 + 128u <= pa.d && (pa.n & 63u) == 0u) { - var fla : tensorLayout2D - tensorLayoutCreate(fla) - tensorLayoutSetBlockSize(fla, 1u, 32u) - tensorLayoutSetDimension(fla, pa.d, pa.n) - tensorLayoutSetStride(fla, pa.n >> 5u, 1u) - var flb : tensorLayout2D - tensorLayoutCreate(flb) - tensorLayoutSetDimension(flb, row0 + ((cnt + 31u) & ~31u), pa.n) - tensorLayoutSetStride(flb, pa.n & ~7u, 1u) - var flo : tensorLayout2DPad - tensorLayoutCreate(flo) - tensorLayoutSetDimension(flo, row0 + cnt, pa.d) - tensorLayoutSetStride(flo, pa.d & ~7u, 1u) - if (pa.ksplit == 0u) { - var k = 0u - for (_i in range(int(pa.n / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_q8) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 32u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < pa.n) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_q8) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 32u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x32 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, 0u, flo, t0, 32u, m0, 128u, tv) - return - } - var k = k0 - for (_i in range(int((k1 - k0) / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_q8) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 32u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < k1) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_q8) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 32u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x32 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, ybase, flo, t0, 32u, m0, 128u, tv) - return - } - // EDGE PATH — clamp-Constant layouts everywhere (store discard load-bearing) - var tla : tensorLayout2DPad - tensorLayoutCreate(tla) - tensorLayoutSetBlockSize(tla, 1u, 32u) - tensorLayoutSetDimension(tla, pa.d, pa.n) - tensorLayoutSetStride(tla, pa.n / 32u, 1u) - var tlb : tensorLayout2DPad - tensorLayoutCreate(tlb) - tensorLayoutSetDimension(tlb, row0 + cnt, pa.n) - tensorLayoutSetStride(tlb, pa.n, 1u) - var tlo : tensorLayout2DPad - tensorLayoutCreate(tlo) - tensorLayoutSetDimension(tlo, row0 + cnt, pa.d) - tensorLayoutSetStride(tlo, pa.d, 1u) - if (pa.ksplit == 0u) { - var k = 0u - for (_i in range(int(pa.n / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_q8) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 32u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < pa.n) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_q8) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 32u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x32 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, 0u, tlo, t0, 32u, m0, 128u, tv) - return - } - var k = k0 - for (_i in range(int((k1 - k0) / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_q8) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 32u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < k1) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_q8) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 32u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x32 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, ybase, tlo, t0, 32u, m0, 128u, tv) + return ws[wg_blk0 + bc.x * (pa.n >> 5u) + bc.y] * float16(float(int(q))) } } -// ===== the cm2 K-quant tiles: Q4_K decode-in-load over the repacked superblock planes ===== - -struct VkK4Blk { - qs : int16[64] // one Q4_K superblock's 128 nibble bytes as 16-bit lanes; its scale row (d|dmin + 8 sc + 8 mn bytes) lives in wsu -} - -// the l-tile Q4_K twin of Q8Cm2LBatch: same geometry, superblock (1, 256) blocks, nibble + per-32-group scale/min decode -[vk_dispatch(name = "kq_batch_k4_cm2l_cls", grid = "wgs", params = "wgs : int64")] -class K4Cm2LBatch : MoeCmBase { - @ssbo @binding = 0 @role = "weight" wq : array // Q4_K quant plane (128-byte superblocks) - @ssbo @binding = 1 @role = "weight" wsu : array // scale plane: 5 words per superblock - @ssbo @binding = 3 xf16 : array // f16 activation plane - @ssbo @binding = 5 y : array - @workgroup wg_blk0 : uint // the region's SUPERBLOCK base, staged for the decode +[ |> template_struct_instance] +class template K4Cm2T : KqCm2BatchT { + typedef BLK = VkK4Blk + typedef ST = uint [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] - def decode_k4(blk : VkK4Blk; bc, cib : uint2) : float16 { + def decode(blk : VkK4Blk; bc, cib : uint2) : float16 { let g = cib.y >> 5u let e = cib.y & 31u let bidx = g * 16u + (e & 15u) let by = uint(int(unpack8(blk.qs[int(bidx >> 1u)])[int(bidx & 1u)])) & 0xFFu let q = (by >> ((e >> 4u) * 4u)) & 0xFu let srow = (wg_blk0 + bc.x * (pa.n >> 8u) + bc.y) * 5u - let dm = unpackHalf2x16(wsu[srow]) - let sc = (wsu[srow + 1u + (g >> 2u)] >> ((g & 3u) * 8u)) & 0xFFu - let mn = (wsu[srow + 3u + (g >> 2u)] >> ((g & 3u) * 8u)) & 0xFFu + let dm = unpackHalf2x16(ws[srow]) + let sc = (ws[srow + 1u + (g >> 2u)] >> ((g & 3u) * 8u)) & 0xFFu + let mn = (ws[srow + 3u + (g >> 2u)] >> ((g & 3u) * 8u)) & 0xFFu return float16(dm.x * float(sc) * float(q) - dm.y * float(mn)) } +} - [spirv_kernel(local_size_x = 256, name = "kq_batch_k4_cm2l_cls_spv"), arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-tile-pick-and-default")] - def run { // nolint:STYLE038 — the fast/edge path pair, barrier- and register-coupled - let reg = region_rec() - let wblk0 = reg.x - let row0 = reg.y - let cnt = reg.z - let ttiles = (cnt + 255u) / 256u - var tix = reg.w - var ks = 0u - var k1 = pa.n - var ybase = 0u - if (pa.ksplit != 0u) { - let ptiles = ((pa.d + 127u) / 128u) * ttiles - ks = tix / ptiles - tix -= ks * ptiles - k1 = min(pa.n, (ks + 1u) * pa.ksplit) - ybase = ks * (row0 + cnt) * pa.d - } - let k0 = ks * pa.ksplit - let xt = tix % ttiles - let wt = tix / ttiles - if (gl_LocalInvocationID.x == 0u) { - wg_blk0 = wblk0 - } - var tv : tensorView2Dt - tensorViewCreate(tv) - barrier() // wg_blk0 visible before the first decode load - var a : coopmatWgA_f16_128x64 - var b : coopmatWgB_f16_64x256 - var acc : coopmatWgAcc_f16_128x256 - let t0 = row0 + xt * 256u - let m0 = wt * 128u - // FAST PATH — see Q8Cm2LBatch (kq n is always a 256-multiple, so only tile bounds gate) - if (m0 + 128u <= pa.d && xt * 256u + 256u <= cnt && (pa.n & 63u) == 0u) { - var fla : tensorLayout2D - tensorLayoutCreate(fla) - tensorLayoutSetBlockSize(fla, 1u, 256u) - tensorLayoutSetDimension(fla, pa.d, pa.n) - tensorLayoutSetStride(fla, pa.n >> 8u, 1u) - var flb : tensorLayout2D - tensorLayoutCreate(flb) - tensorLayoutSetDimension(flb, row0 + cnt, pa.n) - tensorLayoutSetStride(flb, pa.n & ~7u, 1u) - var flo : tensorLayout2D - tensorLayoutCreate(flo) - tensorLayoutSetDimension(flo, row0 + cnt, pa.d) - tensorLayoutSetStride(flo, pa.d & ~7u, 1u) - if (pa.ksplit == 0u) { - var k = 0u - for (_i in range(int(pa.n / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k4) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 256u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < pa.n) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k4) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 256u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x256 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, 0u, flo, t0, 256u, m0, 128u, tv) - return - } - var k = k0 - for (_i in range(int((k1 - k0) / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k4) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 256u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < k1) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k4) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 256u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x256 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, ybase, flo, t0, 256u, m0, 128u, tv) - return - } - // EDGE PATH — clamp-Constant layouts everywhere (store discard load-bearing) - var tla : tensorLayout2DPad - tensorLayoutCreate(tla) - tensorLayoutSetBlockSize(tla, 1u, 256u) - tensorLayoutSetDimension(tla, pa.d, pa.n) - tensorLayoutSetStride(tla, pa.n / 256u, 1u) - var tlb : tensorLayout2DPad - tensorLayoutCreate(tlb) - tensorLayoutSetDimension(tlb, row0 + cnt, pa.n) - tensorLayoutSetStride(tlb, pa.n, 1u) - var tlo : tensorLayout2DPad - tensorLayoutCreate(tlo) - tensorLayoutSetDimension(tlo, row0 + cnt, pa.d) - tensorLayoutSetStride(tlo, pa.d, 1u) - if (pa.ksplit == 0u) { - var k = 0u - for (_i in range(int(pa.n / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k4) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 256u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < pa.n) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k4) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 256u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x256 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, 0u, tlo, t0, 256u, m0, 128u, tv) - return - } - var k = k0 - for (_i in range(int((k1 - k0) / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k4) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 256u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < k1) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k4) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 256u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x256 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, ybase, tlo, t0, 256u, m0, 128u, tv) - } -} - -// the m-tile Q4_K sibling (BN=128) — the k4 decode on the m geometry -[vk_dispatch(name = "kq_batch_k4_cm2m_cls", grid = "wgs", params = "wgs : int64")] -class K4Cm2MBatch : MoeCmBase { - @ssbo @binding = 0 @role = "weight" wq : array - @ssbo @binding = 1 @role = "weight" wsu : array - @ssbo @binding = 3 xf16 : array - @ssbo @binding = 5 y : array - @workgroup wg_blk0 : uint - - [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] - def decode_k4(blk : VkK4Blk; bc, cib : uint2) : float16 { - let g = cib.y >> 5u - let e = cib.y & 31u - let bidx = g * 16u + (e & 15u) - let by = uint(int(unpack8(blk.qs[int(bidx >> 1u)])[int(bidx & 1u)])) & 0xFFu - let q = (by >> ((e >> 4u) * 4u)) & 0xFu - let srow = (wg_blk0 + bc.x * (pa.n >> 8u) + bc.y) * 5u - let dm = unpackHalf2x16(wsu[srow]) - let sc = (wsu[srow + 1u + (g >> 2u)] >> ((g & 3u) * 8u)) & 0xFFu - let mn = (wsu[srow + 3u + (g >> 2u)] >> ((g & 3u) * 8u)) & 0xFFu - return float16(dm.x * float(sc) * float(q) - dm.y * float(mn)) - } - - [spirv_kernel(local_size_x = 256, name = "kq_batch_k4_cm2m_cls_spv"), arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-tile-pick-and-default")] - def run { // nolint:STYLE038 — the fast/edge path pair, barrier- and register-coupled - let reg = region_rec() - let wblk0 = reg.x - let row0 = reg.y - let cnt = reg.z - let ttiles = (cnt + 127u) / 128u - var tix = reg.w - var ks = 0u - var k1 = pa.n - var ybase = 0u - if (pa.ksplit != 0u) { - let ptiles = ((pa.d + 127u) / 128u) * ttiles - ks = tix / ptiles - tix -= ks * ptiles - k1 = min(pa.n, (ks + 1u) * pa.ksplit) - ybase = ks * (row0 + cnt) * pa.d - } - let k0 = ks * pa.ksplit - let xt = tix % ttiles - let wt = tix / ttiles - if (gl_LocalInvocationID.x == 0u) { - wg_blk0 = wblk0 - } - var tv : tensorView2Dt - tensorViewCreate(tv) - barrier() - var a : coopmatWgA_f16_128x64 - var b : coopmatWgB_f16_64x128 - var acc : coopmatWgAcc_f16_128x128 - let t0 = row0 + xt * 128u - let m0 = wt * 128u - if (m0 + 128u <= pa.d && xt * 128u + 128u <= cnt && (pa.n & 63u) == 0u) { - var fla : tensorLayout2D - tensorLayoutCreate(fla) - tensorLayoutSetBlockSize(fla, 1u, 256u) - tensorLayoutSetDimension(fla, pa.d, pa.n) - tensorLayoutSetStride(fla, pa.n >> 8u, 1u) - var flb : tensorLayout2D - tensorLayoutCreate(flb) - tensorLayoutSetDimension(flb, row0 + cnt, pa.n) - tensorLayoutSetStride(flb, pa.n & ~7u, 1u) - var flo : tensorLayout2D - tensorLayoutCreate(flo) - tensorLayoutSetDimension(flo, row0 + cnt, pa.d) - tensorLayoutSetStride(flo, pa.d & ~7u, 1u) - if (pa.ksplit == 0u) { - var k = 0u - for (_i in range(int(pa.n / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k4) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 128u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < pa.n) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k4) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 128u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x128 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, 0u, flo, t0, 128u, m0, 128u, tv) - return - } - var k = k0 - for (_i in range(int((k1 - k0) / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k4) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 128u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < k1) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k4) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 128u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x128 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, ybase, flo, t0, 128u, m0, 128u, tv) - return - } - var tla : tensorLayout2DPad - tensorLayoutCreate(tla) - tensorLayoutSetBlockSize(tla, 1u, 256u) - tensorLayoutSetDimension(tla, pa.d, pa.n) - tensorLayoutSetStride(tla, pa.n / 256u, 1u) - var tlb : tensorLayout2DPad - tensorLayoutCreate(tlb) - tensorLayoutSetDimension(tlb, row0 + cnt, pa.n) - tensorLayoutSetStride(tlb, pa.n, 1u) - var tlo : tensorLayout2DPad - tensorLayoutCreate(tlo) - tensorLayoutSetDimension(tlo, row0 + cnt, pa.d) - tensorLayoutSetStride(tlo, pa.d, 1u) - if (pa.ksplit == 0u) { - var k = 0u - for (_i in range(int(pa.n / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k4) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 128u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < pa.n) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k4) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 128u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x128 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, 0u, tlo, t0, 128u, m0, 128u, tv) - return - } - var k = k0 - for (_i in range(int((k1 - k0) / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k4) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 128u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < k1) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k4) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 128u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x128 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, ybase, tlo, t0, 128u, m0, 128u, tv) - } -} - -// the s-tile Q4_K sibling (BN=32) - the k4 decode on the expert-bucket geometry (see Q8Cm2SBatch) -[vk_dispatch(name = "kq_batch_k4_cm2s_cls", grid = "wgs", params = "wgs : int64")] -class K4Cm2SBatch : MoeCmBase { - @ssbo @binding = 0 @role = "weight" wq : array - @ssbo @binding = 1 @role = "weight" wsu : array - @ssbo @binding = 3 xf16 : array - @ssbo @binding = 5 y : array - @workgroup wg_blk0 : uint - - [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] - def decode_k4(blk : VkK4Blk; bc, cib : uint2) : float16 { - let g = cib.y >> 5u - let e = cib.y & 31u - let bidx = g * 16u + (e & 15u) - let by = uint(int(unpack8(blk.qs[int(bidx >> 1u)])[int(bidx & 1u)])) & 0xFFu - let q = (by >> ((e >> 4u) * 4u)) & 0xFu - let srow = (wg_blk0 + bc.x * (pa.n >> 8u) + bc.y) * 5u - let dm = unpackHalf2x16(wsu[srow]) - let sc = (wsu[srow + 1u + (g >> 2u)] >> ((g & 3u) * 8u)) & 0xFFu - let mn = (wsu[srow + 3u + (g >> 2u)] >> ((g & 3u) * 8u)) & 0xFFu - return float16(dm.x * float(sc) * float(q) - dm.y * float(mn)) - } - - [spirv_kernel(local_size_x = 256, name = "kq_batch_k4_cm2s_cls_spv"), arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-tile-pick-and-default")] - def run { // nolint:STYLE038 — the fast/edge path pair, barrier- and register-coupled - let reg = region_rec() - let wblk0 = reg.x - let row0 = reg.y - let cnt = reg.z - let ttiles = (cnt + 31u) / 32u - var tix = reg.w - var ks = 0u - var k1 = pa.n - var ybase = 0u - if (pa.ksplit != 0u) { - let ptiles = ((pa.d + 127u) / 128u) * ttiles - ks = tix / ptiles - tix -= ks * ptiles - k1 = min(pa.n, (ks + 1u) * pa.ksplit) - ybase = ks * (row0 + cnt) * pa.d - } - let k0 = ks * pa.ksplit - let xt = tix % ttiles - let wt = tix / ttiles - if (gl_LocalInvocationID.x == 0u) { - wg_blk0 = wblk0 - } - var tv : tensorView2Dt - tensorViewCreate(tv) - barrier() - var a : coopmatWgA_f16_128x64 - var b : coopmatWgB_f16_64x32 - var acc : coopmatWgAcc_f16_128x32 - let t0 = row0 + xt * 32u - let m0 = wt * 128u - if (m0 + 128u <= pa.d && (pa.n & 63u) == 0u) { - var fla : tensorLayout2D - tensorLayoutCreate(fla) - tensorLayoutSetBlockSize(fla, 1u, 256u) - tensorLayoutSetDimension(fla, pa.d, pa.n) - tensorLayoutSetStride(fla, pa.n >> 8u, 1u) - var flb : tensorLayout2D - tensorLayoutCreate(flb) - tensorLayoutSetDimension(flb, row0 + ((cnt + 31u) & ~31u), pa.n) - tensorLayoutSetStride(flb, pa.n & ~7u, 1u) - var flo : tensorLayout2DPad - tensorLayoutCreate(flo) - tensorLayoutSetDimension(flo, row0 + cnt, pa.d) - tensorLayoutSetStride(flo, pa.d & ~7u, 1u) - if (pa.ksplit == 0u) { - var k = 0u - for (_i in range(int(pa.n / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k4) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 32u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < pa.n) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k4) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 32u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x32 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, 0u, flo, t0, 32u, m0, 128u, tv) - return - } - var k = k0 - for (_i in range(int((k1 - k0) / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k4) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 32u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < k1) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k4) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 32u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x32 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, ybase, flo, t0, 32u, m0, 128u, tv) - return - } - var tla : tensorLayout2DPad - tensorLayoutCreate(tla) - tensorLayoutSetBlockSize(tla, 1u, 256u) - tensorLayoutSetDimension(tla, pa.d, pa.n) - tensorLayoutSetStride(tla, pa.n / 256u, 1u) - var tlb : tensorLayout2DPad - tensorLayoutCreate(tlb) - tensorLayoutSetDimension(tlb, row0 + cnt, pa.n) - tensorLayoutSetStride(tlb, pa.n, 1u) - var tlo : tensorLayout2DPad - tensorLayoutCreate(tlo) - tensorLayoutSetDimension(tlo, row0 + cnt, pa.d) - tensorLayoutSetStride(tlo, pa.d, 1u) - if (pa.ksplit == 0u) { - var k = 0u - for (_i in range(int(pa.n / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k4) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 32u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < pa.n) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k4) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 32u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x32 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, 0u, tlo, t0, 32u, m0, 128u, tv) - return - } - var k = k0 - for (_i in range(int((k1 - k0) / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k4) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 32u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < k1) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k4) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 32u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x32 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, ybase, tlo, t0, 32u, m0, 128u, tv) - } -} - -struct VkK6Blk { - ql : int16[64] // one Q6_K superblock's 128 low-nibble bytes as 16-bit lanes - qh : int16[32] // ... and its 64 high-2-bit bytes; scales (16 int8 + f16 d) live in wsu -} - -// the l-tile Q6_K twin: 6-bit compose (nibble | qh 2 bits) - 32, per-16 signed sub-scale -[vk_dispatch(name = "kq_batch_k6_cm2l_cls", grid = "wgs", params = "wgs : int64")] -class K6Cm2LBatch : MoeCmBase { - @ssbo @binding = 0 @role = "weight" wq : array // Q6_K quant plane (192-byte superblocks) - @ssbo @binding = 1 @role = "weight" wsu : array // scale plane: 5 words per superblock - @ssbo @binding = 3 xf16 : array // f16 activation plane - @ssbo @binding = 5 y : array - @workgroup wg_blk0 : uint // the region's SUPERBLOCK base, staged for the decode - - [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] - def decode_k6(blk : VkK6Blk; bc, cib : uint2) : float16 { - let e = cib.y - let bu = e >> 5u - let hh = (e >> 4u) & 1u - let j = e & 15u - let lb = bu * 16u + j - let lo = uint(int(unpack8(blk.ql[int(lb >> 1u)])[int(lb & 1u)])) & 0xFFu - let qb = (bu >> 2u) * 32u + hh * 16u + j - let hby = uint(int(unpack8(blk.qh[int(qb >> 1u)])[int(qb & 1u)])) & 0xFFu - let q6 = int(((lo >> (hh * 4u)) & 0xFu) | (((hby >> ((bu & 3u) * 2u)) & 3u) << 4u)) - 32 - let srow = (wg_blk0 + bc.x * (pa.n >> 8u) + bc.y) * 5u - let sidx = e >> 4u - let sc = int(wsu[srow + (sidx >> 2u)] << ((3u - (sidx & 3u)) * 8u)) >> 24 - let d = unpackHalf2x16(wsu[srow + 4u]).x - return float16(d * float(sc) * float(q6)) - } - - [spirv_kernel(local_size_x = 256, name = "kq_batch_k6_cm2l_cls_spv"), arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-tile-pick-and-default")] - def run { // nolint:STYLE038 — the fast/edge path pair, barrier- and register-coupled - let reg = region_rec() - let wblk0 = reg.x - let row0 = reg.y - let cnt = reg.z - let ttiles = (cnt + 255u) / 256u - var tix = reg.w - var ks = 0u - var k1 = pa.n - var ybase = 0u - if (pa.ksplit != 0u) { - let ptiles = ((pa.d + 127u) / 128u) * ttiles - ks = tix / ptiles - tix -= ks * ptiles - k1 = min(pa.n, (ks + 1u) * pa.ksplit) - ybase = ks * (row0 + cnt) * pa.d - } - let k0 = ks * pa.ksplit - let xt = tix % ttiles - let wt = tix / ttiles - if (gl_LocalInvocationID.x == 0u) { - wg_blk0 = wblk0 - } - var tv : tensorView2Dt - tensorViewCreate(tv) - barrier() // wg_blk0 visible before the first decode load - var a : coopmatWgA_f16_128x64 - var b : coopmatWgB_f16_64x256 - var acc : coopmatWgAcc_f16_128x256 - let t0 = row0 + xt * 256u - let m0 = wt * 128u - if (m0 + 128u <= pa.d && xt * 256u + 256u <= cnt && (pa.n & 63u) == 0u) { - var fla : tensorLayout2D - tensorLayoutCreate(fla) - tensorLayoutSetBlockSize(fla, 1u, 256u) - tensorLayoutSetDimension(fla, pa.d, pa.n) - tensorLayoutSetStride(fla, pa.n >> 8u, 1u) - var flb : tensorLayout2D - tensorLayoutCreate(flb) - tensorLayoutSetDimension(flb, row0 + cnt, pa.n) - tensorLayoutSetStride(flb, pa.n & ~7u, 1u) - var flo : tensorLayout2D - tensorLayoutCreate(flo) - tensorLayoutSetDimension(flo, row0 + cnt, pa.d) - tensorLayoutSetStride(flo, pa.d & ~7u, 1u) - if (pa.ksplit == 0u) { - var k = 0u - for (_i in range(int(pa.n / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k6) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 256u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < pa.n) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k6) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 256u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x256 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, 0u, flo, t0, 256u, m0, 128u, tv) - return - } - var k = k0 - for (_i in range(int((k1 - k0) / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k6) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 256u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < k1) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k6) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 256u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x256 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, ybase, flo, t0, 256u, m0, 128u, tv) - return - } - // EDGE PATH — clamp-Constant layouts everywhere (store discard load-bearing) - var tla : tensorLayout2DPad - tensorLayoutCreate(tla) - tensorLayoutSetBlockSize(tla, 1u, 256u) - tensorLayoutSetDimension(tla, pa.d, pa.n) - tensorLayoutSetStride(tla, pa.n / 256u, 1u) - var tlb : tensorLayout2DPad - tensorLayoutCreate(tlb) - tensorLayoutSetDimension(tlb, row0 + cnt, pa.n) - tensorLayoutSetStride(tlb, pa.n, 1u) - var tlo : tensorLayout2DPad - tensorLayoutCreate(tlo) - tensorLayoutSetDimension(tlo, row0 + cnt, pa.d) - tensorLayoutSetStride(tlo, pa.d, 1u) - if (pa.ksplit == 0u) { - var k = 0u - for (_i in range(int(pa.n / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k6) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 256u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < pa.n) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k6) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 256u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x256 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, 0u, tlo, t0, 256u, m0, 128u, tv) - return - } - var k = k0 - for (_i in range(int((k1 - k0) / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k6) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 256u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < k1) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k6) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 256u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x256 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, ybase, tlo, t0, 256u, m0, 128u, tv) - } -} - -// the m-tile Q6_K sibling (BN=128) -[vk_dispatch(name = "kq_batch_k6_cm2m_cls", grid = "wgs", params = "wgs : int64")] -class K6Cm2MBatch : MoeCmBase { - @ssbo @binding = 0 @role = "weight" wq : array - @ssbo @binding = 1 @role = "weight" wsu : array - @ssbo @binding = 3 xf16 : array - @ssbo @binding = 5 y : array - @workgroup wg_blk0 : uint +[ |> template_struct_instance] +class template K6Cm2T : KqCm2BatchT { + typedef BLK = VkK6Blk + typedef ST = uint [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] - def decode_k6(blk : VkK6Blk; bc, cib : uint2) : float16 { + def decode(blk : VkK6Blk; bc, cib : uint2) : float16 { let e = cib.y let bu = e >> 5u let hh = (e >> 4u) & 1u @@ -4964,328 +3964,92 @@ class K6Cm2MBatch : MoeCmBase { let q6 = int(((lo >> (hh * 4u)) & 0xFu) | (((hby >> ((bu & 3u) * 2u)) & 3u) << 4u)) - 32 let srow = (wg_blk0 + bc.x * (pa.n >> 8u) + bc.y) * 5u let sidx = e >> 4u - let sc = int(wsu[srow + (sidx >> 2u)] << ((3u - (sidx & 3u)) * 8u)) >> 24 - let d = unpackHalf2x16(wsu[srow + 4u]).x + let sc = int(ws[srow + (sidx >> 2u)] << ((3u - (sidx & 3u)) * 8u)) >> 24 + let d = unpackHalf2x16(ws[srow + 4u]).x return float16(d * float(sc) * float(q6)) } - - [spirv_kernel(local_size_x = 256, name = "kq_batch_k6_cm2m_cls_spv"), arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-tile-pick-and-default")] - def run { // nolint:STYLE038 — the fast/edge path pair, barrier- and register-coupled - let reg = region_rec() - let wblk0 = reg.x - let row0 = reg.y - let cnt = reg.z - let ttiles = (cnt + 127u) / 128u - var tix = reg.w - var ks = 0u - var k1 = pa.n - var ybase = 0u - if (pa.ksplit != 0u) { - let ptiles = ((pa.d + 127u) / 128u) * ttiles - ks = tix / ptiles - tix -= ks * ptiles - k1 = min(pa.n, (ks + 1u) * pa.ksplit) - ybase = ks * (row0 + cnt) * pa.d - } - let k0 = ks * pa.ksplit - let xt = tix % ttiles - let wt = tix / ttiles - if (gl_LocalInvocationID.x == 0u) { - wg_blk0 = wblk0 - } - var tv : tensorView2Dt - tensorViewCreate(tv) - barrier() - var a : coopmatWgA_f16_128x64 - var b : coopmatWgB_f16_64x128 - var acc : coopmatWgAcc_f16_128x128 - let t0 = row0 + xt * 128u - let m0 = wt * 128u - if (m0 + 128u <= pa.d && xt * 128u + 128u <= cnt && (pa.n & 63u) == 0u) { - var fla : tensorLayout2D - tensorLayoutCreate(fla) - tensorLayoutSetBlockSize(fla, 1u, 256u) - tensorLayoutSetDimension(fla, pa.d, pa.n) - tensorLayoutSetStride(fla, pa.n >> 8u, 1u) - var flb : tensorLayout2D - tensorLayoutCreate(flb) - tensorLayoutSetDimension(flb, row0 + cnt, pa.n) - tensorLayoutSetStride(flb, pa.n & ~7u, 1u) - var flo : tensorLayout2D - tensorLayoutCreate(flo) - tensorLayoutSetDimension(flo, row0 + cnt, pa.d) - tensorLayoutSetStride(flo, pa.d & ~7u, 1u) - if (pa.ksplit == 0u) { - var k = 0u - for (_i in range(int(pa.n / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k6) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 128u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < pa.n) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k6) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 128u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x128 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, 0u, flo, t0, 128u, m0, 128u, tv) - return - } - var k = k0 - for (_i in range(int((k1 - k0) / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k6) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 128u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < k1) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k6) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 128u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x128 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, ybase, flo, t0, 128u, m0, 128u, tv) - return - } - var tla : tensorLayout2DPad - tensorLayoutCreate(tla) - tensorLayoutSetBlockSize(tla, 1u, 256u) - tensorLayoutSetDimension(tla, pa.d, pa.n) - tensorLayoutSetStride(tla, pa.n / 256u, 1u) - var tlb : tensorLayout2DPad - tensorLayoutCreate(tlb) - tensorLayoutSetDimension(tlb, row0 + cnt, pa.n) - tensorLayoutSetStride(tlb, pa.n, 1u) - var tlo : tensorLayout2DPad - tensorLayoutCreate(tlo) - tensorLayoutSetDimension(tlo, row0 + cnt, pa.d) - tensorLayoutSetStride(tlo, pa.d, 1u) - if (pa.ksplit == 0u) { - var k = 0u - for (_i in range(int(pa.n / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k6) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 128u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < pa.n) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k6) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 128u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x128 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, 0u, tlo, t0, 128u, m0, 128u, tv) - return - } - var k = k0 - for (_i in range(int((k1 - k0) / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k6) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 128u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < k1) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k6) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 128u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x128 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, ybase, tlo, t0, 128u, m0, 128u, tv) - } } -// the s-tile Q6_K sibling (BN=32, see Q8Cm2SBatch) -[vk_dispatch(name = "kq_batch_k6_cm2s_cls", grid = "wgs", params = "wgs : int64")] -class K6Cm2SBatch : MoeCmBase { - @ssbo @binding = 0 @role = "weight" wq : array - @ssbo @binding = 1 @role = "weight" wsu : array - @ssbo @binding = 3 xf16 : array - @ssbo @binding = 5 y : array - @workgroup wg_blk0 : uint - - [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] - def decode_k6(blk : VkK6Blk; bc, cib : uint2) : float16 { - let e = cib.y - let bu = e >> 5u - let hh = (e >> 4u) & 1u - let j = e & 15u - let lb = bu * 16u + j - let lo = uint(int(unpack8(blk.ql[int(lb >> 1u)])[int(lb & 1u)])) & 0xFFu - let qb = (bu >> 2u) * 32u + hh * 16u + j - let hby = uint(int(unpack8(blk.qh[int(qb >> 1u)])[int(qb & 1u)])) & 0xFFu - let q6 = int(((lo >> (hh * 4u)) & 0xFu) | (((hby >> ((bu & 3u) * 2u)) & 3u) << 4u)) - 32 - let srow = (wg_blk0 + bc.x * (pa.n >> 8u) + bc.y) * 5u - let sidx = e >> 4u - let sc = int(wsu[srow + (sidx >> 2u)] << ((3u - (sidx & 3u)) * 8u)) >> 24 - let d = unpackHalf2x16(wsu[srow + 4u]).x - return float16(d * float(sc) * float(q6)) - } - - [spirv_kernel(local_size_x = 256, name = "kq_batch_k6_cm2s_cls_spv"), arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-tile-pick-and-default")] - def run { // nolint:STYLE038 — the fast/edge path pair, barrier- and register-coupled - let reg = region_rec() - let wblk0 = reg.x - let row0 = reg.y - let cnt = reg.z - let ttiles = (cnt + 31u) / 32u - var tix = reg.w - var ks = 0u - var k1 = pa.n - var ybase = 0u - if (pa.ksplit != 0u) { - let ptiles = ((pa.d + 127u) / 128u) * ttiles - ks = tix / ptiles - tix -= ks * ptiles - k1 = min(pa.n, (ks + 1u) * pa.ksplit) - ybase = ks * (row0 + cnt) * pa.d - } - let k0 = ks * pa.ksplit - let xt = tix % ttiles - let wt = tix / ttiles - if (gl_LocalInvocationID.x == 0u) { - wg_blk0 = wblk0 - } - var tv : tensorView2Dt - tensorViewCreate(tv) - barrier() - var a : coopmatWgA_f16_128x64 - var b : coopmatWgB_f16_64x32 - var acc : coopmatWgAcc_f16_128x32 - let t0 = row0 + xt * 32u - let m0 = wt * 128u - if (m0 + 128u <= pa.d && (pa.n & 63u) == 0u) { - var fla : tensorLayout2D - tensorLayoutCreate(fla) - tensorLayoutSetBlockSize(fla, 1u, 256u) - tensorLayoutSetDimension(fla, pa.d, pa.n) - tensorLayoutSetStride(fla, pa.n >> 8u, 1u) - var flb : tensorLayout2D - tensorLayoutCreate(flb) - tensorLayoutSetDimension(flb, row0 + ((cnt + 31u) & ~31u), pa.n) - tensorLayoutSetStride(flb, pa.n & ~7u, 1u) - var flo : tensorLayout2DPad - tensorLayoutCreate(flo) - tensorLayoutSetDimension(flo, row0 + cnt, pa.d) - tensorLayoutSetStride(flo, pa.d & ~7u, 1u) - if (pa.ksplit == 0u) { - var k = 0u - for (_i in range(int(pa.n / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k6) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 32u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < pa.n) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k6) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 32u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x32 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, 0u, flo, t0, 32u, m0, 128u, tv) - return - } - var k = k0 - for (_i in range(int((k1 - k0) / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k6) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 32u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < k1) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k6) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 32u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x32 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, ybase, flo, t0, 32u, m0, 128u, tv) - return - } - var tla : tensorLayout2DPad - tensorLayoutCreate(tla) - tensorLayoutSetBlockSize(tla, 1u, 256u) - tensorLayoutSetDimension(tla, pa.d, pa.n) - tensorLayoutSetStride(tla, pa.n / 256u, 1u) - var tlb : tensorLayout2DPad - tensorLayoutCreate(tlb) - tensorLayoutSetDimension(tlb, row0 + cnt, pa.n) - tensorLayoutSetStride(tlb, pa.n, 1u) - var tlo : tensorLayout2DPad - tensorLayoutCreate(tlo) - tensorLayoutSetDimension(tlo, row0 + cnt, pa.d) - tensorLayoutSetStride(tlo, pa.d, 1u) - if (pa.ksplit == 0u) { - var k = 0u - for (_i in range(int(pa.n / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k6) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 32u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < pa.n) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k6) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 32u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x32 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, 0u, tlo, t0, 32u, m0, 128u, tv) - return - } - var k = k0 - for (_i in range(int((k1 - k0) / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k6) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 32u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < k1) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k6) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 32u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x32 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, ybase, tlo, t0, 32u, m0, 128u, tv) - } +// the nine stamps: the three width columns of each format (the dispatcher keys on (fmt, tc)) +[vk_dispatch(name = "q8_batch_cm2l_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Q8Cm2LBatch : Q8Cm2T { + typedef BT = coopmatWgB_f16_64x256 + typedef ACC = coopmatWgAcc_f16_128x256 + typedef ACCW = coopmatWgAcc_f32_128x256 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "q8_batch_cm2m_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Q8Cm2MBatch : Q8Cm2T { + override BN = 128u + typedef BT = coopmatWgB_f16_64x128 + typedef ACC = coopmatWgAcc_f16_128x128 + typedef ACCW = coopmatWgAcc_f32_128x128 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "q8_batch_cm2s_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Q8Cm2SBatch : Q8Cm2T { + override BN = 32u + override STILE = true + typedef BT = coopmatWgB_f16_64x32 + typedef ACC = coopmatWgAcc_f16_128x32 + typedef ACCW = coopmatWgAcc_f32_128x32 + typedef FLO = tensorLayout2DPad +} + +[vk_dispatch(name = "kq_batch_k4_cm2l_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class K4Cm2LBatch : K4Cm2T { + typedef BT = coopmatWgB_f16_64x256 + typedef ACC = coopmatWgAcc_f16_128x256 + typedef ACCW = coopmatWgAcc_f32_128x256 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_k4_cm2m_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class K4Cm2MBatch : K4Cm2T { + override BN = 128u + typedef BT = coopmatWgB_f16_64x128 + typedef ACC = coopmatWgAcc_f16_128x128 + typedef ACCW = coopmatWgAcc_f32_128x128 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_k4_cm2s_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class K4Cm2SBatch : K4Cm2T { + override BN = 32u + override STILE = true + typedef BT = coopmatWgB_f16_64x32 + typedef ACC = coopmatWgAcc_f16_128x32 + typedef ACCW = coopmatWgAcc_f32_128x32 + typedef FLO = tensorLayout2DPad +} + +[vk_dispatch(name = "kq_batch_k6_cm2l_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class K6Cm2LBatch : K6Cm2T { + typedef BT = coopmatWgB_f16_64x256 + typedef ACC = coopmatWgAcc_f16_128x256 + typedef ACCW = coopmatWgAcc_f32_128x256 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_k6_cm2m_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class K6Cm2MBatch : K6Cm2T { + override BN = 128u + typedef BT = coopmatWgB_f16_64x128 + typedef ACC = coopmatWgAcc_f16_128x128 + typedef ACCW = coopmatWgAcc_f32_128x128 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_k6_cm2s_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class K6Cm2SBatch : K6Cm2T { + override BN = 32u + override STILE = true + typedef BT = coopmatWgB_f16_64x32 + typedef ACC = coopmatWgAcc_f16_128x32 + typedef ACCW = coopmatWgAcc_f32_128x32 + typedef FLO = tensorLayout2DPad } // ===== the split-k reduce (sums the cm2 partial planes into y) ===== From bad0b77635ba041314a8e04a1f43fed91ac1a1a4 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Sun, 30 Aug 2026 12:25:06 -0700 Subject: [PATCH 15/61] dasLLAMA: lint - the iq4xs GEMV header trimmed to the comment cap; STYLE037 nolint on the mul_mm shell (the IQ4XS staging branch tipped complexity to 21) Co-Authored-By: Claude Fable 5 --- modules/dasLLAMA/dasllama/dasllama_metal_kernels.das | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das index 19f02f2a94..2f428adee1 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das @@ -5470,7 +5470,7 @@ class template MetalKqMulMmK45T : MetalMmTileBase { @workgroup @template_gate = IQ4XS iq4lut : float[16] // kvalues_iq4nl as floats, staged once per threadgroup [metal_kernel] - def metal_kq_mulmm { // nolint:STYLE038 — MSL kernel body; the mul_mm tiles are simdgroup-barrier coupled + def metal_kq_mulmm { // nolint:STYLE037,STYLE038 — MSL kernel body; the mul_mm tiles are simdgroup-barrier coupled let lid = gl_LocalInvocationID.x static_if (IQ4XS) { if (lid < 16u) { @@ -7211,11 +7211,9 @@ class MetalKqMvB8K3 { } } -// IQ4_XS: w = d*sc*LUT[q] with sc the SIGNED (ls - 32) strip byte and no min plane. The codebook -// sits in threadgroup memory as floats; a 16-lane half of the simdgroup owns one superblock and -// lane (bu, il) the 8-byte half il of block bu - 8 lo + 8 hi weights from two uint loads against -// four float4 x loads (the q40 pairing: byte l -> elems l and l + 16). The scale plane is the k6 -// split form (d plane at s0off = doff, 16B sub-scale strips at soff). +// IQ4_XS: w = d*sc*LUT[q], sc the SIGNED strip byte, no min plane; the codebook in threadgroup +// floats, a 16-lane half-simdgroup per superblock, lane (bu, il) = block bu's 8-byte half il +// (two uint loads, four float4 x loads; the q40 pairing). Scale plane = the k6 split form. [metal_dispatch(name = "enc_kq_iq4xs_c", pso = "g_pso_kq_iq4xs", tgmem = "metal_kq_gemv_iq4xs_msl_tgmem", tg = 64, grid = "rows/4", params = "rows : int64, n : int64")] class MetalKqGemvIq4xs { @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d plane — the scale buffer bound at byte nsb*16 (s0off = the caller's doff) From d19602e57600bbc92fb3a71ea8fdd31d90b20e38 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Sun, 30 Aug 2026 12:53:19 -0700 Subject: [PATCH 16/61] dasLLAMA: iq4xs/k3/k5/q40 join the cm2 f16 feed - four decode methods on the tile template Each format is a template stamp now, not three kernel copies: K5Cm2T (the 5th bit off the packed device qh plane), Q40Cm2T (per-block f16 d, the k/k+16 pairing shared with k4's lane view), Iq4xsCm2T (signed sub-scales x super-d, the codebook staged into a gated @workgroup f16[16] ahead of the tile loop - llama.cpp's init_iq_shmem form, arithmetic only at stage time), K3Cm2T (2-bit lanes + hmask over the k6 scale row). Twelve stamps, the (fmt, tc) dispatch ladders grow their arms, pf_f16_feed admits q8 plus every kq_sb format, and ARCHITECTURE_GPU_VULKAN 2.2k/2.2l now state the seven-format feed. Four device-form CPU oracles and four l/m/s test cells gate it. 5060 Ti, 1B, das = the debug-jit instrument, llama.cpp b10660 Vulkan as the bar: - IQ4_XS pp512 5161 -> 15334 +- 261 (0.90x llama.cpp's 17060, ABOVE the k4 control's 13144); tg128 334.7 holds the GEMV parity (llama.cpp 340.7) - Q3_K_L pp512 5174 -> 14031 +- 86 (0.80x llama.cpp's 17509); tg128 374.5 (llama.cpp 349.1) - suite 72 tests, 71 pass, 1 expected skip (the mode-ladder cell under DASLLAMA_COOPMAT=cm2) The prefill gap for the new formats was the missing cm2 tile (followup_vulkan item 24); what remains against llama.cpp on this 1B shape is the tier-wide small-model gap the k4 control shares. Co-Authored-By: Claude Fable 5 --- modules/dasLLAMA/ARCHITECTURE_GPU_VULKAN.md | 11 +- modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md | 28 +- .../dasllama/dasllama_vulkan_classes.das | 216 +++++++- .../dasllama/dasllama_vulkan_prefill.das | 72 ++- modules/dasLLAMA/followup_vulkan.md | 5 +- modules/dasLLAMA/tests/_vkd_oracles.das | 126 +++++ .../dasLLAMA/tests/test_vulkan_kernels.das | 496 ++++++++++++++++++ 7 files changed, 938 insertions(+), 16 deletions(-) diff --git a/modules/dasLLAMA/ARCHITECTURE_GPU_VULKAN.md b/modules/dasLLAMA/ARCHITECTURE_GPU_VULKAN.md index 3c6f42c045..64138433b9 100644 --- a/modules/dasLLAMA/ARCHITECTURE_GPU_VULKAN.md +++ b/modules/dasLLAMA/ARCHITECTURE_GPU_VULKAN.md @@ -76,8 +76,11 @@ compiler pattern-matches only one spelling into that path: a 16-bit load (`int16 members) followed by `unpack8(w)[i & 1u]` - a byte2 lane select - with sub-fields pulled out by shift and mask. A 32-bit word with a variable shift runs slower; an `unpack8` of a 32-bit word indexed by a runtime value (a byte4 dynamic select) drops the whole kernel off the block-load -path, to about a third of the rate. Every cm2 decode - q8, Q4_K, Q6_K - is spelled the 16-bit -way, which is why the block structs are `int16` arrays over the same bytes. +path, to about a third of the rate. Every cm2 decode - q8 and the six kq superblock formats - +is spelled the 16-bit way, which is why the block structs are `int16` arrays over the same +bytes. The IQ4_XS codebook is the one runtime-indexed read a decode makes: it is staged into a +16-entry `@workgroup` f16 table ahead of the tile loop (llama.cpp's `init_iq_shmem` form), +never selected out of a register vector per element. ### 2.2l The cm2 tile pick and the coopmat default ladder {#cm2-tile-pick-and-default} @@ -98,8 +101,8 @@ and clamps only the store, so every f16 plane the chain feeds it - the gathered image and the hidden plane - is sized with 32 rows of slack past its last region (`ffn_cm2_chunk_rows`). -**The f16 feed admits exactly three weight formats - q8, Q4_K and Q6_K** - the same set the cm2 -decode callbacks cover (sec.2.2k) - and each (format, tile) pair has ONE generated class. The +**The f16 feed admits q8 and every kq superblock format** (`kq_sb`) - the set the cm2 decode +callbacks cover (sec.2.2k) - and each (format, tile) pair has ONE stamped class. The prefill driver reaches them through one dispatcher per stage (`cm2_cls_ensure`, `cm2_cls_set`, `cm2_cls_enc`), all three keyed on the same `(fmt, ml)` pair, so the pipeline a role ensures, the set it binds and the kernel it encodes can never be three different classes. The decode diff --git a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md index 23037ef592..c4a96253ec 100644 --- a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md +++ b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md @@ -185,6 +185,19 @@ admits q8/k4/k6 only (`pf_f16_feed`), q40 and k5 have no cm2 tile either, and `followup_vulkan.md` item 24 rules that new formats land on the one class template, not as three more hand-stamped bodies. IQ4_XS prefill rides the kq batch tile like q40 does. +### 6b. The cm2 prefill tile - a decode method on the template + +On an NV_coopmat2 device the f16 feed serves every kq format through ONE tile template +(`KqCm2BatchT`): a new format is a format template authoring `[spirv_decode] def decode` over +the DEVICE forms (quants as the gather lays them out - k4/k5 re-paired k/k+16, q40/iq4xs/k3 +verbatim; scales the 20 B `KQ_DEV_SSB` row) plus three eight-line width stamps, arms in the +`cm2_cls_ensure/set/enc` ladders, and `pf_f16_feed` admits it via `kq_sb` automatically. A +codebook format raises the `IQLUT` axis - a gated `@workgroup` f16 table staged ahead of the +tile loop (llama.cpp's `init_iq_shmem` form); never select codes out of a register vector per +element inside a decode callback. Gate: a device-form CPU oracle (`f16_gemm_oracle`) and +an l/m/s cell in `tests/test_vulkan_kernels.das`. Payoff on the 1B: iq4xs pp512 5161 -> 15334, +k3 5174 -> 14031 (0.90x / 0.80x llama.cpp's Vulkan, from 0.30x). + ## 7. Metal - `dasllama_metal_kernels.das`, `_common`, `_prefill`, `_shapes`, `dasllama_layout.das` Done for IQ4_XS over ssh on the M1 (the tier compiles MSL and runs only there). The tier is @@ -447,12 +460,12 @@ Against llama.cpp b10660 (`lcpp_bench`, das = the debug-jit instrument; zen2 = 1 | tier | pp512 das / llama.cpp | tg128 das / llama.cpp | |---|---|---| | zen2 CPU | 543.6 / 310.8 (1.75x) | 66.4 / 65.1 (1.02x) | -| 5060 Ti Vulkan | 5174 / 17509 (0.30x) | 372.8 / 349.1 (1.07x) | +| 5060 Ti Vulkan | 14031 / 17509 (0.80x) | 374.5 / 349.1 (1.07x) | | M1 CPU | 524.6 / 223.5 (2.35x) | 127.3 / 110.6 (1.15x) | | M1 Metal | 3316 / 3219 (1.03x) | 200.9 / 193.0 (1.04x) | -Decode is at or above llama.cpp on every tier; the Vulkan prefill gap is the missing cm2 tile -(followup_vulkan item 24). +Decode is at or above llama.cpp on every tier; the Vulkan prefill gap closed with the cm2 +tile (section 6b, 0.30x -> 0.80x). ### IQ4_XS (the pilot, 2026-08-30) @@ -491,9 +504,10 @@ exe; zen2 = 16 threads, M1 = 8): | tier | pp512 das / llama.cpp | tg128 das / llama.cpp | |---|---|---| | zen2 CPU | 475.7 / 256.9 (1.85x) | 65.0 / 59.5 (1.09x) | -| 5060 Ti Vulkan | 5161 / 17060 (0.30x) | 338.4 / 340.7 (0.99x) | +| 5060 Ti Vulkan | 15334 / 17060 (0.90x) | 334.7 / 340.7 (0.98x) | | M1 CPU | 796.0 / 263.0 (3.03x) | 132.3 / 128.2 (1.03x) | -| M1 Metal | 2953 / 3565 (0.83x) | 138.7 / 247.3 (0.56x) | +| M1 Metal | 3612 / 3575 (1.01x) | 249.4 / 249.3 (1.01x) | -The Vulkan prefill gap is the missing cm2 decode tile (followup_vulkan item 24: k4 on the same -box does 13144, 0.67x); the Metal decode gap is the per-lane block map (followup_general #58). +Both gaps closed 2026-08-30: the cm2 decode tile (section 6b) took Vulkan prefill past the k4 +control's 13144, and the threadgroup-LUT GEMV/mul_mm rework took Metal to parity on both axes; +what remains anywhere is the tier-wide small-model prefill gap the k4 control shares. diff --git a/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das b/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das index 8f8eeb48c6..a3eb0064dc 100644 --- a/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das +++ b/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das @@ -3737,6 +3737,16 @@ struct VkK6Blk { qh : int16[32] // ... and its 64 high-2-bit bytes; scales (16 int8 + f16 d) live in ws } +struct VkK5Blk { + qs : int16[64] // one Q5_K superblock's 128 nibble bytes (the k/k+16 device pairing) + qh : int16[16] // ... and its 32 packed 5th-bit bytes: byte g*4+q = bits of elems g*32+4q..+3 (lo) | +16.. (hi) +} + +struct VkK3Blk { + qs : int16[32] // one Q3_K superblock's 64 qs bytes, four 2-bit lanes each, verbatim disk order + hm : int16[16] // ... and its 32 hmask bytes; scales (16 int8 (sc-32) + f16 d) live in ws +} + // The cm2 tile: one class template over the two stamp axes. The token column BN - l 256, m 128, // s 32 (the expert-bucket geometry: STILE's fast path loads a partial column UNCLAMPED and clamps // only the store, so every plane it reads carries 32 rows of slack) - and the weight format: BLK @@ -3754,6 +3764,8 @@ class template KqCm2BatchT : MoeCmBase { @template_constant BN : uint = 256u // the token column @template_constant STILE : bool = false // the 32-row column's partial-column fast path @template_constant BLKW : uint = 256u // elements per weight block + @template_constant IQLUT : bool = false // stage the iq4nl codebook into workgroup memory + @workgroup @template_gate = IQLUT iq4lut : float16[16] // kvalues_iq4nl as f16 (llama.cpp's init_iq_shmem form) [spirv_kernel(local_size_x = 256), arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-tile-pick-and-default")] def run { // nolint:STYLE038 — the fast/edge path pair, barrier- and register-coupled @@ -3784,7 +3796,14 @@ class template KqCm2BatchT : MoeCmBase { } var tv : tensorView2Dt tensorViewCreate(tv) - barrier() // wg_blk0 visible before the first decode load + static_if (IQLUT) { + if (gl_LocalInvocationID.x < 16u) { + let li = gl_LocalInvocationID.x + let tbl = uint4(3215825025u, 4142587343u, 639175937u, 1901675829u) + iq4lut[li] = float16(float((int((tbl[int(li >> 2u)] >> ((li & 3u) * 8u)) & 255u) ^ 128) - 128)) + } + } + barrier() // wg_blk0 (and the gated codebook) visible before the first decode load var a : coopmatWgA_f16_128x64 var b : BT var acc : ACC @@ -3970,7 +3989,92 @@ class template K6Cm2T : KqCm2BatchT { } } -// the nine stamps: the three width columns of each format (the dispatcher keys on (fmt, tc)) +[ |> template_struct_instance] +class template K5Cm2T : KqCm2BatchT { + typedef BLK = VkK5Blk + typedef ST = uint + + [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] + def decode(blk : VkK5Blk; bc, cib : uint2) : float16 { + let g = cib.y >> 5u + let e = cib.y & 31u + let j = e & 15u + let h = e >> 4u + let bidx = g * 16u + j + let by = uint(int(unpack8(blk.qs[int(bidx >> 1u)])[int(bidx & 1u)])) & 0xFFu + let hidx = g * 4u + (j >> 2u) + let hby = uint(int(unpack8(blk.qh[int(hidx >> 1u)])[int(hidx & 1u)])) & 0xFFu + let q = ((by >> (h * 4u)) & 0xFu) | (((hby >> ((j & 3u) + h * 4u)) & 1u) << 4u) + let srow = (wg_blk0 + bc.x * (pa.n >> 8u) + bc.y) * 5u + let dm = unpackHalf2x16(ws[srow]) + let sc = (ws[srow + 1u + (g >> 2u)] >> ((g & 3u) * 8u)) & 0xFFu + let mn = (ws[srow + 3u + (g >> 2u)] >> ((g & 3u) * 8u)) & 0xFFu + return float16(dm.x * float(sc) * float(q) - dm.y * float(mn)) + } +} + +[ |> template_struct_instance] +class template Q40Cm2T : KqCm2BatchT { + typedef BLK = VkK4Blk // the same 128-nibble lane view - q40 rows land in the k/k+16 device pairing too + typedef ST = uint + + [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] + def decode(blk : VkK4Blk; bc, cib : uint2) : float16 { + let g = cib.y >> 5u + let e = cib.y & 31u + let bidx = g * 16u + (e & 15u) + let by = uint(int(unpack8(blk.qs[int(bidx >> 1u)])[int(bidx & 1u)])) & 0xFFu + let q = (by >> ((e >> 4u) * 4u)) & 0xFu + let srow = (wg_blk0 + bc.x * (pa.n >> 8u) + bc.y) * 5u + let dp = unpackHalf2x16(ws[srow + (g >> 1u)]) + let d = (g & 1u) == 0u ? dp.x : dp.y + return float16(d * (float(q) - 8.0)) + } +} + +[ |> template_struct_instance] +class template Iq4xsCm2T : KqCm2BatchT { + typedef BLK = VkK4Blk // the iq4xs plane is the k/k+16 pairing verbatim - the k4 lane view fits + typedef ST = uint + override IQLUT = true + + [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] + def decode(blk : VkK4Blk; bc, cib : uint2) : float16 { + let g = cib.y >> 5u + let e = cib.y & 31u + let bidx = g * 16u + (e & 15u) + let by = uint(int(unpack8(blk.qs[int(bidx >> 1u)])[int(bidx & 1u)])) & 0xFFu + let q = (by >> ((e >> 4u) * 4u)) & 0xFu + let srow = (wg_blk0 + bc.x * (pa.n >> 8u) + bc.y) * 5u + let d = unpackHalf2x16(ws[srow]).x + let sc = int(ws[srow + 1u + (g >> 2u)] << ((3u - (g & 3u)) * 8u)) >> 24 + return float16(d * float(sc) * float(iq4lut[int(q)])) + } +} + +[ |> template_struct_instance] +class template K3Cm2T : KqCm2BatchT { + typedef BLK = VkK3Blk + typedef ST = uint + + [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] + def decode(blk : VkK3Blk; bc, cib : uint2) : float16 { + let e = cib.y + let g = e >> 5u + let l = e & 31u + let qi = (g >> 2u) * 32u + l + let qby = uint(int(unpack8(blk.qs[int(qi >> 1u)])[int(qi & 1u)])) & 0xFFu + let hby = uint(int(unpack8(blk.hm[int(l >> 1u)])[int(l & 1u)])) & 0xFFu + let q3 = int((qby >> ((g & 3u) * 2u)) & 3u) - (((hby >> g) & 1u) == 0u ? 4 : 0) + let srow = (wg_blk0 + bc.x * (pa.n >> 8u) + bc.y) * 5u + let sidx = e >> 4u + let sc = int(ws[srow + (sidx >> 2u)] << ((3u - (sidx & 3u)) * 8u)) >> 24 + let d = unpackHalf2x16(ws[srow + 4u]).x + return float16(d * float(sc) * float(q3)) + } +} + +// the stamps: the three width columns of each format (the dispatcher keys on (fmt, tc)) [vk_dispatch(name = "q8_batch_cm2l_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] class Q8Cm2LBatch : Q8Cm2T { typedef BT = coopmatWgB_f16_64x256 @@ -4052,6 +4156,114 @@ class K6Cm2SBatch : K6Cm2T { typedef FLO = tensorLayout2DPad } +[vk_dispatch(name = "kq_batch_k5_cm2l_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class K5Cm2LBatch : K5Cm2T { + typedef BT = coopmatWgB_f16_64x256 + typedef ACC = coopmatWgAcc_f16_128x256 + typedef ACCW = coopmatWgAcc_f32_128x256 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_k5_cm2m_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class K5Cm2MBatch : K5Cm2T { + override BN = 128u + typedef BT = coopmatWgB_f16_64x128 + typedef ACC = coopmatWgAcc_f16_128x128 + typedef ACCW = coopmatWgAcc_f32_128x128 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_k5_cm2s_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class K5Cm2SBatch : K5Cm2T { + override BN = 32u + override STILE = true + typedef BT = coopmatWgB_f16_64x32 + typedef ACC = coopmatWgAcc_f16_128x32 + typedef ACCW = coopmatWgAcc_f32_128x32 + typedef FLO = tensorLayout2DPad +} + +[vk_dispatch(name = "kq_batch_q40_cm2l_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Q40Cm2LBatch : Q40Cm2T { + typedef BT = coopmatWgB_f16_64x256 + typedef ACC = coopmatWgAcc_f16_128x256 + typedef ACCW = coopmatWgAcc_f32_128x256 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_q40_cm2m_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Q40Cm2MBatch : Q40Cm2T { + override BN = 128u + typedef BT = coopmatWgB_f16_64x128 + typedef ACC = coopmatWgAcc_f16_128x128 + typedef ACCW = coopmatWgAcc_f32_128x128 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_q40_cm2s_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Q40Cm2SBatch : Q40Cm2T { + override BN = 32u + override STILE = true + typedef BT = coopmatWgB_f16_64x32 + typedef ACC = coopmatWgAcc_f16_128x32 + typedef ACCW = coopmatWgAcc_f32_128x32 + typedef FLO = tensorLayout2DPad +} + +[vk_dispatch(name = "kq_batch_iq4xs_cm2l_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Iq4xsCm2LBatch : Iq4xsCm2T { + typedef BT = coopmatWgB_f16_64x256 + typedef ACC = coopmatWgAcc_f16_128x256 + typedef ACCW = coopmatWgAcc_f32_128x256 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_iq4xs_cm2m_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Iq4xsCm2MBatch : Iq4xsCm2T { + override BN = 128u + typedef BT = coopmatWgB_f16_64x128 + typedef ACC = coopmatWgAcc_f16_128x128 + typedef ACCW = coopmatWgAcc_f32_128x128 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_iq4xs_cm2s_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Iq4xsCm2SBatch : Iq4xsCm2T { + override BN = 32u + override STILE = true + typedef BT = coopmatWgB_f16_64x32 + typedef ACC = coopmatWgAcc_f16_128x32 + typedef ACCW = coopmatWgAcc_f32_128x32 + typedef FLO = tensorLayout2DPad +} + +[vk_dispatch(name = "kq_batch_k3_cm2l_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class K3Cm2LBatch : K3Cm2T { + typedef BT = coopmatWgB_f16_64x256 + typedef ACC = coopmatWgAcc_f16_128x256 + typedef ACCW = coopmatWgAcc_f32_128x256 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_k3_cm2m_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class K3Cm2MBatch : K3Cm2T { + override BN = 128u + typedef BT = coopmatWgB_f16_64x128 + typedef ACC = coopmatWgAcc_f16_128x128 + typedef ACCW = coopmatWgAcc_f32_128x128 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_k3_cm2s_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class K3Cm2SBatch : K3Cm2T { + override BN = 32u + override STILE = true + typedef BT = coopmatWgB_f16_64x32 + typedef ACC = coopmatWgAcc_f16_128x32 + typedef ACCW = coopmatWgAcc_f32_128x32 + typedef FLO = tensorLayout2DPad +} + // ===== the split-k reduce (sums the cm2 partial planes into y) ===== struct SkRedArgs { diff --git a/modules/dasLLAMA/dasllama/dasllama_vulkan_prefill.das b/modules/dasLLAMA/dasllama/dasllama_vulkan_prefill.das index a9eef68898..0f104b0155 100644 --- a/modules/dasLLAMA/dasllama/dasllama_vulkan_prefill.das +++ b/modules/dasLLAMA/dasllama/dasllama_vulkan_prefill.das @@ -40,7 +40,7 @@ var private @scratch g_pf_ids_stage : array [arch(at="../ARCHITECTURE_RUNTIME.md#activation-scale-lattice")] def private pf_f16_feed(f : int) : bool { - return (f == int(KqFmt.q8) || f == int(KqFmt.k4) || f == int(KqFmt.k6)) && g_gpu.coopmat_mode == 4 + return (f == int(KqFmt.q8) || kq_sb(f)) && g_gpu.coopmat_mode == 4 } [arch(at="../ARCHITECTURE_RUNTIME.md#activation-scale-lattice")] @@ -56,14 +56,26 @@ let private PF_CHUNK_MAX = 8l // the overlap ramp doubles 1,2,4 then holds her [arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-tile-pick-and-default")] def private cm2_cls_ensure(fmt : int; tc : int64) : bool { - verify(fmt == int(KqFmt.q8) || fmt == int(KqFmt.k4) || fmt == int(KqFmt.k6), "vk prefill: a cm2 tile for a format the f16 feed never admits") + verify(fmt == int(KqFmt.q8) || kq_sb(fmt), "vk prefill: a cm2 tile for a format the f16 feed never admits") verify(tc == 256l || tc == 128l || tc == 32l, "vk prefill: a cm2 tile column the pick never returns") if (fmt == int(KqFmt.k4)) { return tc == 256l ? ensure_kq_batch_k4_cm2l_cls() : (tc == 128l ? ensure_kq_batch_k4_cm2m_cls() : ensure_kq_batch_k4_cm2s_cls()) } + if (fmt == int(KqFmt.k5)) { + return tc == 256l ? ensure_kq_batch_k5_cm2l_cls() : (tc == 128l ? ensure_kq_batch_k5_cm2m_cls() : ensure_kq_batch_k5_cm2s_cls()) + } if (fmt == int(KqFmt.k6)) { return tc == 256l ? ensure_kq_batch_k6_cm2l_cls() : (tc == 128l ? ensure_kq_batch_k6_cm2m_cls() : ensure_kq_batch_k6_cm2s_cls()) } + if (fmt == int(KqFmt.q40)) { + return tc == 256l ? ensure_kq_batch_q40_cm2l_cls() : (tc == 128l ? ensure_kq_batch_q40_cm2m_cls() : ensure_kq_batch_q40_cm2s_cls()) + } + if (fmt == int(KqFmt.iq4xs)) { + return tc == 256l ? ensure_kq_batch_iq4xs_cm2l_cls() : (tc == 128l ? ensure_kq_batch_iq4xs_cm2m_cls() : ensure_kq_batch_iq4xs_cm2s_cls()) + } + if (fmt == int(KqFmt.k3)) { + return tc == 256l ? ensure_kq_batch_k3_cm2l_cls() : (tc == 128l ? ensure_kq_batch_k3_cm2m_cls() : ensure_kq_batch_k3_cm2s_cls()) + } return tc == 256l ? ensure_q8_batch_cm2l_cls() : (tc == 128l ? ensure_q8_batch_cm2m_cls() : ensure_q8_batch_cm2s_cls()) } @@ -75,12 +87,36 @@ def private cm2_cls_set(fmt : int; tc : int64; bufs : uint64 const[5]; sizes : i } return tc == 128l ? set_kq_batch_k4_cm2m_cls(bufs, sizes, gbits) : set_kq_batch_k4_cm2s_cls(bufs, sizes, gbits) } + if (fmt == int(KqFmt.k5)) { + if (tc == 256l) { + return set_kq_batch_k5_cm2l_cls(bufs, sizes, gbits) + } + return tc == 128l ? set_kq_batch_k5_cm2m_cls(bufs, sizes, gbits) : set_kq_batch_k5_cm2s_cls(bufs, sizes, gbits) + } if (fmt == int(KqFmt.k6)) { if (tc == 256l) { return set_kq_batch_k6_cm2l_cls(bufs, sizes, gbits) } return tc == 128l ? set_kq_batch_k6_cm2m_cls(bufs, sizes, gbits) : set_kq_batch_k6_cm2s_cls(bufs, sizes, gbits) } + if (fmt == int(KqFmt.q40)) { + if (tc == 256l) { + return set_kq_batch_q40_cm2l_cls(bufs, sizes, gbits) + } + return tc == 128l ? set_kq_batch_q40_cm2m_cls(bufs, sizes, gbits) : set_kq_batch_q40_cm2s_cls(bufs, sizes, gbits) + } + if (fmt == int(KqFmt.iq4xs)) { + if (tc == 256l) { + return set_kq_batch_iq4xs_cm2l_cls(bufs, sizes, gbits) + } + return tc == 128l ? set_kq_batch_iq4xs_cm2m_cls(bufs, sizes, gbits) : set_kq_batch_iq4xs_cm2s_cls(bufs, sizes, gbits) + } + if (fmt == int(KqFmt.k3)) { + if (tc == 256l) { + return set_kq_batch_k3_cm2l_cls(bufs, sizes, gbits) + } + return tc == 128l ? set_kq_batch_k3_cm2m_cls(bufs, sizes, gbits) : set_kq_batch_k3_cm2s_cls(bufs, sizes, gbits) + } if (tc == 256l) { return set_q8_batch_cm2l_cls(bufs, sizes, gbits) } @@ -97,6 +133,14 @@ def private cm2_cls_enc(fmt : int; tc : int64; raw : VkCommandBuffer; var h : Vk } else { enc_kq_batch_k4_cm2s_cls(raw, h, s, pc, groups) } + } elif (fmt == int(KqFmt.k5)) { + if (tc == 256l) { + enc_kq_batch_k5_cm2l_cls(raw, h, s, pc, groups) + } elif (tc == 128l) { + enc_kq_batch_k5_cm2m_cls(raw, h, s, pc, groups) + } else { + enc_kq_batch_k5_cm2s_cls(raw, h, s, pc, groups) + } } elif (fmt == int(KqFmt.k6)) { if (tc == 256l) { enc_kq_batch_k6_cm2l_cls(raw, h, s, pc, groups) @@ -105,6 +149,30 @@ def private cm2_cls_enc(fmt : int; tc : int64; raw : VkCommandBuffer; var h : Vk } else { enc_kq_batch_k6_cm2s_cls(raw, h, s, pc, groups) } + } elif (fmt == int(KqFmt.q40)) { + if (tc == 256l) { + enc_kq_batch_q40_cm2l_cls(raw, h, s, pc, groups) + } elif (tc == 128l) { + enc_kq_batch_q40_cm2m_cls(raw, h, s, pc, groups) + } else { + enc_kq_batch_q40_cm2s_cls(raw, h, s, pc, groups) + } + } elif (fmt == int(KqFmt.iq4xs)) { + if (tc == 256l) { + enc_kq_batch_iq4xs_cm2l_cls(raw, h, s, pc, groups) + } elif (tc == 128l) { + enc_kq_batch_iq4xs_cm2m_cls(raw, h, s, pc, groups) + } else { + enc_kq_batch_iq4xs_cm2s_cls(raw, h, s, pc, groups) + } + } elif (fmt == int(KqFmt.k3)) { + if (tc == 256l) { + enc_kq_batch_k3_cm2l_cls(raw, h, s, pc, groups) + } elif (tc == 128l) { + enc_kq_batch_k3_cm2m_cls(raw, h, s, pc, groups) + } else { + enc_kq_batch_k3_cm2s_cls(raw, h, s, pc, groups) + } } elif (tc == 256l) { enc_q8_batch_cm2l_cls(raw, h, s, pc, groups) } elif (tc == 128l) { diff --git a/modules/dasLLAMA/followup_vulkan.md b/modules/dasLLAMA/followup_vulkan.md index 2eef4dd896..6f5884ef4b 100644 --- a/modules/dasLLAMA/followup_vulkan.md +++ b/modules/dasLLAMA/followup_vulkan.md @@ -451,7 +451,10 @@ module) is independent and can land any time - it is pure structure. formats with no cm2 tile prefill at 5161 (iq4xs) and 5174 (k3) t/s against k4's 13144 on the same box (llama.cpp 17060 / 17509 / 19719) while their decode sits at parity, so the template is also the lever that puts every new format on the tile path; k4's own 0.67x is - the 1B-shape tier gap, a separate item. + the 1B-shape tier gap, a separate item. DONE 2026-08-30 (the iquant arc): `KqCm2BatchT` + stamps all nine originals (suite + the Q4_K_M e2e row within noise), and k5/q40/iq4xs/k3 + joined as format templates - twelve more stamps, iq4xs pp512 15334 (0.90x llama.cpp, + above the k4 control), k3 14031 (0.80x). Item 11's k5/q40 stamps landed with it. 25. **Try `VK_NV_cooperative_vector` for decode GEMV on real hardware (Boris, 2026-08-28).** cm2 has no matrix-vector op - its seven feature bits are all tile-shaped, minimum tile 16 - diff --git a/modules/dasLLAMA/tests/_vkd_oracles.das b/modules/dasLLAMA/tests/_vkd_oracles.das index 8fc68ce7ab..c00be6134d 100644 --- a/modules/dasLLAMA/tests/_vkd_oracles.das +++ b/modules/dasLLAMA/tests/_vkd_oracles.das @@ -307,3 +307,129 @@ def kq_gemm_oracle(fmt : int; wq, ws, xq : array; xs : array; delete k } } + +// the four remaining kq formats' f16-feed GEMM references, over the same device forms the cm2 +// decode callbacks read (k5 = the k4 row plus the packed 5th-bit plane; q40 = 8 f16 d; iq4xs = +// d + signed sub-scales + the iq4nl codebook; k3 = 2-bit lanes + hmask over the k6 scale row) +def k5f16_gemm_oracle(wq : array; wsu : array; xf : array; + recs : array; nrec, n, d : int; var y : array) { + let nsb = n / 256 + for (rr in range(nrec)) { + let wsb0 = int(recs[rr * 4]) + let row0 = int(recs[rr * 4 + 1]) + let cnt = int(recs[rr * 4 + 2]) + for (r in range(cnt)) { + for (c in range(d)) { + var acc = 0.0 + for (s in range(nsb)) { + let sb = wsb0 + c * nsb + s + let dm = unpackHalf2x16(wsu[sb * 5]) + for (g in range(8)) { + let sc = float(int((wsu[sb * 5 + 1 + g / 4] >> uint((g % 4) * 8)) & 0xFFu)) + let mn = float(int((wsu[sb * 5 + 3 + g / 4] >> uint((g % 4) * 8)) & 0xFFu)) + for (e in range(32)) { + let j = e % 16 + let h = e / 16 + let by = byte_u8(wq, sb * 160 + g * 16 + j) + let hby = byte_u8(wq, sb * 160 + 128 + g * 4 + j / 4) + let q = float(((by >> (h * 4)) & 0xF) | (((hby >> (j % 4 + h * 4)) & 1) << 4)) + let w = float(float16(dm.x * sc * q - dm.y * mn)) + acc += w * half_at(xf, (row0 + r) * n + s * 256 + g * 32 + e) + } + } + } + y[(row0 + r) * d + c] = acc + } + } + } +} + +def q40f16_gemm_oracle(wq : array; wsu : array; xf : array; + recs : array; nrec, n, d : int; var y : array) { + let nsb = n / 256 + for (rr in range(nrec)) { + let wsb0 = int(recs[rr * 4]) + let row0 = int(recs[rr * 4 + 1]) + let cnt = int(recs[rr * 4 + 2]) + for (r in range(cnt)) { + for (c in range(d)) { + var acc = 0.0 + for (s in range(nsb)) { + let sb = wsb0 + c * nsb + s + for (g in range(8)) { + let dp = unpackHalf2x16(wsu[sb * 5 + g / 2]) + let dv = g % 2 == 0 ? dp.x : dp.y + for (e in range(32)) { + let by = byte_u8(wq, sb * 128 + g * 16 + e % 16) + let q = float((by >> ((e / 16) * 4)) & 0xF) + let w = float(float16(dv * (q - 8.0))) + acc += w * half_at(xf, (row0 + r) * n + s * 256 + g * 32 + e) + } + } + } + y[(row0 + r) * d + c] = acc + } + } + } +} + +def iq4xsf16_gemm_oracle(wq : array; wsu : array; xf : array; + recs : array; nrec, n, d : int; var y : array) { + let nsb = n / 256 + let lut = iq4nl_lut() + for (rr in range(nrec)) { + let wsb0 = int(recs[rr * 4]) + let row0 = int(recs[rr * 4 + 1]) + let cnt = int(recs[rr * 4 + 2]) + for (r in range(cnt)) { + for (c in range(d)) { + var acc = 0.0 + for (s in range(nsb)) { + let sb = wsb0 + c * nsb + s + let dv = unpackHalf2x16(wsu[sb * 5]).x + for (g in range(8)) { + let sc = float(int(wsu[sb * 5 + 1 + g / 4] << uint((3 - g % 4) * 8)) >> 24) + for (e in range(32)) { + let by = byte_u8(wq, sb * 128 + g * 16 + e % 16) + let q = (by >> ((e / 16) * 4)) & 0xF + let w = float(float16(dv * sc * float(lut[q]))) + acc += w * half_at(xf, (row0 + r) * n + s * 256 + g * 32 + e) + } + } + } + y[(row0 + r) * d + c] = acc + } + } + } +} + +def k3f16_gemm_oracle(wq : array; wsu : array; xf : array; + recs : array; nrec, n, d : int; var y : array) { + let nsb = n / 256 + for (rr in range(nrec)) { + let wsb0 = int(recs[rr * 4]) + let row0 = int(recs[rr * 4 + 1]) + let cnt = int(recs[rr * 4 + 2]) + for (r in range(cnt)) { + for (c in range(d)) { + var acc = 0.0 + for (s in range(nsb)) { + let sb = wsb0 + c * nsb + s + let dv = unpackHalf2x16(wsu[sb * 5 + 4]).x + for (g in range(8)) { + for (e in range(32)) { + let sidx = g * 2 + e / 16 + let sc = float(int(wsu[sb * 5 + sidx / 4] << uint((3 - sidx % 4) * 8)) >> 24) + let qby = byte_u8(wq, sb * 96 + (g / 4) * 32 + e) + let hby = byte_u8(wq, sb * 96 + 64 + e) + let q3 = float(((qby >> ((g % 4) * 2)) & 3) - (((hby >> g) & 1) == 0 ? 4 : 0)) + let w = float(float16(dv * sc * q3)) + acc += w * half_at(xf, (row0 + r) * n + s * 256 + g * 32 + e) + } + } + } + y[(row0 + r) * d + c] = acc + } + } + } +} diff --git a/modules/dasLLAMA/tests/test_vulkan_kernels.das b/modules/dasLLAMA/tests/test_vulkan_kernels.das index 13a12348d7..82c223a8f3 100644 --- a/modules/dasLLAMA/tests/test_vulkan_kernels.das +++ b/modules/dasLLAMA/tests/test_vulkan_kernels.das @@ -1651,6 +1651,502 @@ def test_vkd_k4_cm2_batch(t0 : T?) { } } +[test] +def test_vkd_k5_cm2_batch(t0 : T?) { + t0 |> run("cm2 Q5_K l/m/s tiles == the CPU oracle (5th-bit compose decode-in-load)") <| @(t : T?) { + static_if (typeinfo builtin_module_exists(vulkan)) { + if (!ensure_q8_batch_cls()) { // engage the device without touching coopmat rails + t |> skip("no Vulkan device") + return + } + if (!(g_gpu.coopmat_mode == 4 && g_gpu.has_coopmat2)) { + t |> skip("the cm2 k5 tiles serve only in mode 4 on an NV_coopmat2 device") + return + } + verify(ensure_kq_batch_k5_cm2l_cls() && ensure_kq_batch_k5_cm2m_cls() && ensure_kq_batch_k5_cm2s_cls(), "k5 cm2 class rails must engage in cm2 mode") + let n = 768 // one 512-unrolled pass + 4 tail steps crossing superblock bounds + let nsb = n / 256 + let d = 160 // one full 128-tile + a 32 edge + let cnt0 = 300 + let cnt1 = 260 + let rows = cnt0 + cnt1 + let totsb = 2 * d * nsb + var wqh : array + var wsuh : array + var xfh : array + wqh |> resize(totsb * 40) + wsuh |> resize(totsb * 5) + xfh |> resize((rows + 32) * n / 2) // + the s tile's 32-row read slack + for (i in range(totsb * 40)) { + wqh[i] = hash_word(uint(i) + 29u) // arbitrary quant bytes + } + for (sb in range(totsb)) { + let dv = 0.0002 * float(1 + sb % 7) // cool scales: the f16 acc must stay far from 65504 + let dmv = 0.00005 * float(1 + sb % 5) + wsuh[sb * 5] = packHalf2x16(float2(dv, dmv)) + for (wi in range(4)) { + wsuh[sb * 5 + 1 + wi] = hash_word(uint(sb * 4 + wi) + 613u) // sc / mn bytes + } + } + for (i in range((rows + 32) * n / 2)) { + xfh[i] = ws_word(uint(i) * 3u + 5u) // f16 activation pairs + } + let wq_bytes = int64(totsb) * 160l + let ws_bytes = int64(length(wsuh)) * 4l + let xf_bytes = int64((rows + 32) * n) * 2l + let y_bytes = int64(rows * d) * 4l + let wqd = make_device_buf(wq_bytes) + let wsd = make_device_buf(ws_bytes) + let xfd = make_device_buf(xf_bytes) + let yd2 = make_device_buf(y_bytes) + var host = make_host_buf(y_bytes, true, [cached = true]) + var y_cls : array + y_cls |> resize(rows * d) + var y_ref : array + y_ref |> resize(rows * d) + for (ml in range(3)) { + let ttile = ml == 0 ? 256 : (ml == 1 ? 128 : 32) + let wtiles = (d + 127) / 128 + let wgs0 = ((cnt0 + ttile - 1) / ttile) * wtiles + let wgs1 = ((cnt1 + ttile - 1) / ttile) * wtiles + let wgs = wgs0 + wgs1 + var sched : array + sched |> resize(2 * 4 + wgs) + sched[0] = 0u + sched[1] = 0u + sched[2] = uint(cnt0) + sched[3] = 0u + sched[4] = uint(d * nsb) + sched[5] = uint(cnt0) + sched[6] = uint(cnt1) + sched[7] = uint(wgs0) + for (w in range(wgs)) { + sched[8 + w] = w < wgs0 ? 0u : 1u + } + unsafe { + upload_region_at(wqd, 0l, addr(wqh[0]), wq_bytes) + upload_region_at(wsd, 0l, addr(wsuh[0]), ws_bytes) + upload_region_at(xfd, 0l, addr(xfh[0]), xf_bytes) + let sc_bytes = int64(2 * 4 + wgs) * 4l + let scd = make_device_buf(sc_bytes) + upload_region_at(scd, 0l, addr(sched[0]), sc_bytes) + var sc = (ml == 0 + ? set_kq_batch_k5_cm2l_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)) + : (ml == 1 + ? set_kq_batch_k5_cm2m_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)) + : set_kq_batch_k5_cm2s_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)))) + var raw2 = alloc_cmd() + let begin = VkCommandBufferBeginInfo() + vk_check(vkBeginCommandBuffer(raw2, begin), null) + var h2 : VkHaz + var pc = BatchArgs(n = uint(n), d = uint(d), map_off = uint(2 * 4)) + if (ml == 0) { + enc_kq_batch_k5_cm2l_cls(raw2, h2, sc, pc, int64(wgs)) + } elif (ml == 1) { + enc_kq_batch_k5_cm2m_cls(raw2, h2, sc, pc, int64(wgs)) + } else { + enc_kq_batch_k5_cm2s_cls(raw2, h2, sc, pc, int64(wgs)) + } + vhz_dep(raw2, h2, 16u, 0u, true) + cmd_copy_whole(raw2, yd2, host.buf, y_bytes) + vk_check(vkEndCommandBuffer(raw2), null) + submit_wait(raw2) + memcpy(addr(y_cls[0]), host.mapped, y_bytes) + } + k5f16_gemm_oracle(wqh, wsuh, xfh, sched, 2, n, d, y_ref) + let bad = mismatch_bars(y_cls, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) + to_log(LOG_INFO, "cm2 k5 {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile batch: {bad} of {rows * d} off the oracle\n") + t |> success(bad == 0, "cm2 k5 {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile matches the CPU oracle ({bad} of {rows * d} off)") + delete sched + } + delete y_ref + delete wqh + delete wsuh + delete xfh + delete y_cls + } else { + t |> skip("dasVulkan not present") + } + } +} + +[test] +def test_vkd_q40_cm2_batch(t0 : T?) { + t0 |> run("cm2 Q4_0 l/m/s tiles == the CPU oracle (per-block f16 d decode-in-load)") <| @(t : T?) { + static_if (typeinfo builtin_module_exists(vulkan)) { + if (!ensure_q8_batch_cls()) { // engage the device without touching coopmat rails + t |> skip("no Vulkan device") + return + } + if (!(g_gpu.coopmat_mode == 4 && g_gpu.has_coopmat2)) { + t |> skip("the cm2 q40 tiles serve only in mode 4 on an NV_coopmat2 device") + return + } + verify(ensure_kq_batch_q40_cm2l_cls() && ensure_kq_batch_q40_cm2m_cls() && ensure_kq_batch_q40_cm2s_cls(), "q40 cm2 class rails must engage in cm2 mode") + let n = 768 // one 512-unrolled pass + 4 tail steps crossing superblock bounds + let nsb = n / 256 + let d = 160 // one full 128-tile + a 32 edge + let cnt0 = 300 + let cnt1 = 260 + let rows = cnt0 + cnt1 + let totsb = 2 * d * nsb + var wqh : array + var wsuh : array + var xfh : array + wqh |> resize(totsb * 32) + wsuh |> resize(totsb * 5) + xfh |> resize((rows + 32) * n / 2) // + the s tile's 32-row read slack + for (i in range(totsb * 32)) { + wqh[i] = hash_word(uint(i) + 29u) // arbitrary quant bytes + } + for (sb in range(totsb)) { + for (wi in range(4)) { // 8 per-block f16 d - cool, the f16 acc must stay far from 65504 + let p = sb * 4 + wi + wsuh[sb * 5 + wi] = packHalf2x16(float2(0.0002 * float(1 + p % 7), 0.00013 * float(1 + p % 5))) + } + wsuh[sb * 5 + 4] = 0u + } + for (i in range((rows + 32) * n / 2)) { + xfh[i] = ws_word(uint(i) * 3u + 5u) // f16 activation pairs + } + let wq_bytes = int64(totsb) * 128l + let ws_bytes = int64(length(wsuh)) * 4l + let xf_bytes = int64((rows + 32) * n) * 2l + let y_bytes = int64(rows * d) * 4l + let wqd = make_device_buf(wq_bytes) + let wsd = make_device_buf(ws_bytes) + let xfd = make_device_buf(xf_bytes) + let yd2 = make_device_buf(y_bytes) + var host = make_host_buf(y_bytes, true, [cached = true]) + var y_cls : array + y_cls |> resize(rows * d) + var y_ref : array + y_ref |> resize(rows * d) + for (ml in range(3)) { + let ttile = ml == 0 ? 256 : (ml == 1 ? 128 : 32) + let wtiles = (d + 127) / 128 + let wgs0 = ((cnt0 + ttile - 1) / ttile) * wtiles + let wgs1 = ((cnt1 + ttile - 1) / ttile) * wtiles + let wgs = wgs0 + wgs1 + var sched : array + sched |> resize(2 * 4 + wgs) + sched[0] = 0u + sched[1] = 0u + sched[2] = uint(cnt0) + sched[3] = 0u + sched[4] = uint(d * nsb) + sched[5] = uint(cnt0) + sched[6] = uint(cnt1) + sched[7] = uint(wgs0) + for (w in range(wgs)) { + sched[8 + w] = w < wgs0 ? 0u : 1u + } + unsafe { + upload_region_at(wqd, 0l, addr(wqh[0]), wq_bytes) + upload_region_at(wsd, 0l, addr(wsuh[0]), ws_bytes) + upload_region_at(xfd, 0l, addr(xfh[0]), xf_bytes) + let sc_bytes = int64(2 * 4 + wgs) * 4l + let scd = make_device_buf(sc_bytes) + upload_region_at(scd, 0l, addr(sched[0]), sc_bytes) + var sc = (ml == 0 + ? set_kq_batch_q40_cm2l_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)) + : (ml == 1 + ? set_kq_batch_q40_cm2m_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)) + : set_kq_batch_q40_cm2s_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)))) + var raw2 = alloc_cmd() + let begin = VkCommandBufferBeginInfo() + vk_check(vkBeginCommandBuffer(raw2, begin), null) + var h2 : VkHaz + var pc = BatchArgs(n = uint(n), d = uint(d), map_off = uint(2 * 4)) + if (ml == 0) { + enc_kq_batch_q40_cm2l_cls(raw2, h2, sc, pc, int64(wgs)) + } elif (ml == 1) { + enc_kq_batch_q40_cm2m_cls(raw2, h2, sc, pc, int64(wgs)) + } else { + enc_kq_batch_q40_cm2s_cls(raw2, h2, sc, pc, int64(wgs)) + } + vhz_dep(raw2, h2, 16u, 0u, true) + cmd_copy_whole(raw2, yd2, host.buf, y_bytes) + vk_check(vkEndCommandBuffer(raw2), null) + submit_wait(raw2) + memcpy(addr(y_cls[0]), host.mapped, y_bytes) + } + q40f16_gemm_oracle(wqh, wsuh, xfh, sched, 2, n, d, y_ref) + let bad = mismatch_bars(y_cls, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) + to_log(LOG_INFO, "cm2 q40 {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile batch: {bad} of {rows * d} off the oracle\n") + t |> success(bad == 0, "cm2 q40 {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile matches the CPU oracle ({bad} of {rows * d} off)") + delete sched + } + delete y_ref + delete wqh + delete wsuh + delete xfh + delete y_cls + } else { + t |> skip("dasVulkan not present") + } + } +} + +[test] +def test_vkd_iq4xs_cm2_batch(t0 : T?) { + t0 |> run("cm2 IQ4_XS l/m/s tiles == the CPU oracle (workgroup-staged codebook decode-in-load)") <| @(t : T?) { + static_if (typeinfo builtin_module_exists(vulkan)) { + if (!ensure_q8_batch_cls()) { // engage the device without touching coopmat rails + t |> skip("no Vulkan device") + return + } + if (!(g_gpu.coopmat_mode == 4 && g_gpu.has_coopmat2)) { + t |> skip("the cm2 iq4xs tiles serve only in mode 4 on an NV_coopmat2 device") + return + } + verify(ensure_kq_batch_iq4xs_cm2l_cls() && ensure_kq_batch_iq4xs_cm2m_cls() && ensure_kq_batch_iq4xs_cm2s_cls(), "iq4xs cm2 class rails must engage in cm2 mode") + let n = 768 // one 512-unrolled pass + 4 tail steps crossing superblock bounds + let nsb = n / 256 + let d = 160 // one full 128-tile + a 32 edge + let cnt0 = 300 + let cnt1 = 260 + let rows = cnt0 + cnt1 + let totsb = 2 * d * nsb + var wqh : array + var wsuh : array + var xfh : array + wqh |> resize(totsb * 32) + wsuh |> resize(totsb * 5) + xfh |> resize((rows + 32) * n / 2) // + the s tile's 32-row read slack + for (i in range(totsb * 32)) { + wqh[i] = hash_word(uint(i) + 29u) // arbitrary quant bytes + } + for (sb in range(totsb)) { + wsuh[sb * 5] = packHalf2x16(float2(0.0002 * float(1 + sb % 7), 0.0)) // cool d + wsuh[sb * 5 + 1] = hash_word(uint(sb * 2) + 613u) // signed sub-scale bytes + wsuh[sb * 5 + 2] = hash_word(uint(sb * 2 + 1) + 613u) + wsuh[sb * 5 + 3] = 0u + wsuh[sb * 5 + 4] = 0u + } + for (i in range((rows + 32) * n / 2)) { + xfh[i] = ws_word(uint(i) * 3u + 5u) // f16 activation pairs + } + let wq_bytes = int64(totsb) * 128l + let ws_bytes = int64(length(wsuh)) * 4l + let xf_bytes = int64((rows + 32) * n) * 2l + let y_bytes = int64(rows * d) * 4l + let wqd = make_device_buf(wq_bytes) + let wsd = make_device_buf(ws_bytes) + let xfd = make_device_buf(xf_bytes) + let yd2 = make_device_buf(y_bytes) + var host = make_host_buf(y_bytes, true, [cached = true]) + var y_cls : array + y_cls |> resize(rows * d) + var y_ref : array + y_ref |> resize(rows * d) + for (ml in range(3)) { + let ttile = ml == 0 ? 256 : (ml == 1 ? 128 : 32) + let wtiles = (d + 127) / 128 + let wgs0 = ((cnt0 + ttile - 1) / ttile) * wtiles + let wgs1 = ((cnt1 + ttile - 1) / ttile) * wtiles + let wgs = wgs0 + wgs1 + var sched : array + sched |> resize(2 * 4 + wgs) + sched[0] = 0u + sched[1] = 0u + sched[2] = uint(cnt0) + sched[3] = 0u + sched[4] = uint(d * nsb) + sched[5] = uint(cnt0) + sched[6] = uint(cnt1) + sched[7] = uint(wgs0) + for (w in range(wgs)) { + sched[8 + w] = w < wgs0 ? 0u : 1u + } + unsafe { + upload_region_at(wqd, 0l, addr(wqh[0]), wq_bytes) + upload_region_at(wsd, 0l, addr(wsuh[0]), ws_bytes) + upload_region_at(xfd, 0l, addr(xfh[0]), xf_bytes) + let sc_bytes = int64(2 * 4 + wgs) * 4l + let scd = make_device_buf(sc_bytes) + upload_region_at(scd, 0l, addr(sched[0]), sc_bytes) + var sc = (ml == 0 + ? set_kq_batch_iq4xs_cm2l_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)) + : (ml == 1 + ? set_kq_batch_iq4xs_cm2m_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)) + : set_kq_batch_iq4xs_cm2s_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)))) + var raw2 = alloc_cmd() + let begin = VkCommandBufferBeginInfo() + vk_check(vkBeginCommandBuffer(raw2, begin), null) + var h2 : VkHaz + var pc = BatchArgs(n = uint(n), d = uint(d), map_off = uint(2 * 4)) + if (ml == 0) { + enc_kq_batch_iq4xs_cm2l_cls(raw2, h2, sc, pc, int64(wgs)) + } elif (ml == 1) { + enc_kq_batch_iq4xs_cm2m_cls(raw2, h2, sc, pc, int64(wgs)) + } else { + enc_kq_batch_iq4xs_cm2s_cls(raw2, h2, sc, pc, int64(wgs)) + } + vhz_dep(raw2, h2, 16u, 0u, true) + cmd_copy_whole(raw2, yd2, host.buf, y_bytes) + vk_check(vkEndCommandBuffer(raw2), null) + submit_wait(raw2) + memcpy(addr(y_cls[0]), host.mapped, y_bytes) + } + iq4xsf16_gemm_oracle(wqh, wsuh, xfh, sched, 2, n, d, y_ref) + let bad = mismatch_bars(y_cls, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) + to_log(LOG_INFO, "cm2 iq4xs {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile batch: {bad} of {rows * d} off the oracle\n") + t |> success(bad == 0, "cm2 iq4xs {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile matches the CPU oracle ({bad} of {rows * d} off)") + delete sched + } + delete y_ref + delete wqh + delete wsuh + delete xfh + delete y_cls + } else { + t |> skip("dasVulkan not present") + } + } +} + +[test] +def test_vkd_k3_cm2_batch(t0 : T?) { + t0 |> run("cm2 Q3_K l/m/s tiles == the CPU oracle (2-bit + hmask compose decode-in-load)") <| @(t : T?) { + static_if (typeinfo builtin_module_exists(vulkan)) { + if (!ensure_q8_batch_cls()) { // engage the device without touching coopmat rails + t |> skip("no Vulkan device") + return + } + if (!(g_gpu.coopmat_mode == 4 && g_gpu.has_coopmat2)) { + t |> skip("the cm2 k3 tiles serve only in mode 4 on an NV_coopmat2 device") + return + } + verify(ensure_kq_batch_k3_cm2l_cls() && ensure_kq_batch_k3_cm2m_cls() && ensure_kq_batch_k3_cm2s_cls(), "k3 cm2 class rails must engage in cm2 mode") + let n = 768 // one 512-unrolled pass + 4 tail steps crossing superblock bounds + let nsb = n / 256 + let d = 160 // one full 128-tile + a 32 edge + let cnt0 = 300 + let cnt1 = 260 + let rows = cnt0 + cnt1 + let totsb = 2 * d * nsb + var wqh : array + var wsuh : array + var xfh : array + wqh |> resize(totsb * 24) + wsuh |> resize(totsb * 5) + xfh |> resize((rows + 32) * n / 2) // + the s tile's 32-row read slack + for (i in range(totsb * 24)) { + wqh[i] = hash_word(uint(i) + 29u) // arbitrary quant bytes + } + for (sb in range(totsb)) { + for (wi in range(4)) { + wsuh[sb * 5 + wi] = hash_word(uint(sb * 4 + wi) + 811u) // signed sub-scale bytes + } + wsuh[sb * 5 + 4] = packHalf2x16(float2(0.0002 * float(1 + sb % 7), 0.0)) // cool d + } + for (i in range((rows + 32) * n / 2)) { + xfh[i] = ws_word(uint(i) * 3u + 5u) // f16 activation pairs + } + let wq_bytes = int64(totsb) * 96l + let ws_bytes = int64(length(wsuh)) * 4l + let xf_bytes = int64((rows + 32) * n) * 2l + let y_bytes = int64(rows * d) * 4l + let wqd = make_device_buf(wq_bytes) + let wsd = make_device_buf(ws_bytes) + let xfd = make_device_buf(xf_bytes) + let yd2 = make_device_buf(y_bytes) + var host = make_host_buf(y_bytes, true, [cached = true]) + var y_cls : array + y_cls |> resize(rows * d) + var y_ref : array + y_ref |> resize(rows * d) + for (ml in range(3)) { + let ttile = ml == 0 ? 256 : (ml == 1 ? 128 : 32) + let wtiles = (d + 127) / 128 + let wgs0 = ((cnt0 + ttile - 1) / ttile) * wtiles + let wgs1 = ((cnt1 + ttile - 1) / ttile) * wtiles + let wgs = wgs0 + wgs1 + var sched : array + sched |> resize(2 * 4 + wgs) + sched[0] = 0u + sched[1] = 0u + sched[2] = uint(cnt0) + sched[3] = 0u + sched[4] = uint(d * nsb) + sched[5] = uint(cnt0) + sched[6] = uint(cnt1) + sched[7] = uint(wgs0) + for (w in range(wgs)) { + sched[8 + w] = w < wgs0 ? 0u : 1u + } + unsafe { + upload_region_at(wqd, 0l, addr(wqh[0]), wq_bytes) + upload_region_at(wsd, 0l, addr(wsuh[0]), ws_bytes) + upload_region_at(xfd, 0l, addr(xfh[0]), xf_bytes) + let sc_bytes = int64(2 * 4 + wgs) * 4l + let scd = make_device_buf(sc_bytes) + upload_region_at(scd, 0l, addr(sched[0]), sc_bytes) + var sc = (ml == 0 + ? set_kq_batch_k3_cm2l_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)) + : (ml == 1 + ? set_kq_batch_k3_cm2m_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)) + : set_kq_batch_k3_cm2s_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)))) + var raw2 = alloc_cmd() + let begin = VkCommandBufferBeginInfo() + vk_check(vkBeginCommandBuffer(raw2, begin), null) + var h2 : VkHaz + var pc = BatchArgs(n = uint(n), d = uint(d), map_off = uint(2 * 4)) + if (ml == 0) { + enc_kq_batch_k3_cm2l_cls(raw2, h2, sc, pc, int64(wgs)) + } elif (ml == 1) { + enc_kq_batch_k3_cm2m_cls(raw2, h2, sc, pc, int64(wgs)) + } else { + enc_kq_batch_k3_cm2s_cls(raw2, h2, sc, pc, int64(wgs)) + } + vhz_dep(raw2, h2, 16u, 0u, true) + cmd_copy_whole(raw2, yd2, host.buf, y_bytes) + vk_check(vkEndCommandBuffer(raw2), null) + submit_wait(raw2) + memcpy(addr(y_cls[0]), host.mapped, y_bytes) + } + k3f16_gemm_oracle(wqh, wsuh, xfh, sched, 2, n, d, y_ref) + let bad = mismatch_bars(y_cls, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) + to_log(LOG_INFO, "cm2 k3 {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile batch: {bad} of {rows * d} off the oracle\n") + t |> success(bad == 0, "cm2 k3 {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile matches the CPU oracle ({bad} of {rows * d} off)") + delete sched + } + delete y_ref + delete wqh + delete wsuh + delete xfh + delete y_cls + } else { + t |> skip("dasVulkan not present") + } + } +} + [test] def test_vkd_k6_cm2_batch(t0 : T?) { t0 |> run("cm2 Q6_K l/m tiles == the CPU oracle (6-bit compose decode-in-load)") <| @(t : T?) { From 6c7ff330ab136c9226853c09b7b510da8845b735 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Sun, 30 Aug 2026 14:34:45 -0700 Subject: [PATCH 17/61] dasLLAMA: IQ3_S joins the kq lattice (CPU slice) - 64/64 greedy ids vs llama.cpp The third format through HOW_TO_ADD_A_FORMAT, and the first grid format: enum iq3s (8), kernel id 33, quant plane [qs 64][qh 8][signs 32] verbatim (104 B), scale row the iq4xs/k4 20 B shape with (1 + 2s) decoded at transcode. The 2 KB iq3s_grid ships as a worker-safe per-call local plus the main-context IQ3S_GRID (the IQ4NL_LUT pattern at 512 entries); signs come off the block's own bytes, so no ksigns table exists anywhere. dot_iq3s_q8 / rows kernel / grp row dequant / repack (26 uniform 4-byte columns) / the ~40 ladders / declined emitter stubs / probe fixtures in both family lists / test arms across every gate (21 mirrored ladders, a pack-direction fixture cell, grid pins). Gates on the worktree binary: test_kqformat 16/16; test_kquant 163 tests, 150 pass, 13 env-gated skips (the batch-groupn class every format shares). E2E: the 1B IQ3_M (IQ3_S x78 + Q4_K x34 + Q6_K embd) matches llama.cpp b10660's greedy ids 64 of 64 on the reference bodies at gen 23 t/s - the first format to hold the whole window. The new family minted verdict=rejected after the whole-scope re-tune, exactly QUIRK 4/16/17's shape. Also: kq_kernel_gen / kq_batch_kernel_gen read the k3 non-repacked rows with k4's strides (128/20 for a 96/18 format) - latent, because every kq load repacks before those branches serve; fixed to 96/18 in passing. Co-Authored-By: Claude Fable 5 --- modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md | 23 +++ modules/dasLLAMA/dasllama/dasllama_blocks.das | 3 + modules/dasLLAMA/dasllama/dasllama_common.das | 38 ++++- modules/dasLLAMA/dasllama/dasllama_config.das | 3 +- .../dasLLAMA/dasllama/dasllama_convert.das | 47 ++++++ .../dasLLAMA/dasllama/dasllama_gemm_gen.das | 9 ++ .../dasllama/dasllama_gemm_schema.das | 7 +- modules/dasLLAMA/dasllama/dasllama_gguf.das | 42 ++++++ .../dasllama/dasllama_gpu_resident.das | 7 + modules/dasLLAMA/dasllama/dasllama_image.das | 7 +- .../dasLLAMA/dasllama/dasllama_kqformat.das | 94 +++++++++++- modules/dasLLAMA/dasllama/dasllama_layout.das | 26 +++- modules/dasLLAMA/dasllama/dasllama_load.das | 41 +++++- modules/dasLLAMA/dasllama/dasllama_math.das | 9 +- .../dasllama/dasllama_math_default.das | 84 +++++++++++ .../dasLLAMA/dasllama/dasllama_math_gen.das | 109 +++++++++++++- modules/dasLLAMA/dasllama/dasllama_ple.das | 4 + modules/dasLLAMA/dasllama/dasllama_repack.das | 45 ++++++ modules/dasLLAMA/harness/gen_tune_probe.das | 49 +++++- modules/dasLLAMA/tests/test_kqformat.das | 25 +++- modules/dasLLAMA/tests/test_kquant.das | 139 +++++++++++++++--- 21 files changed, 752 insertions(+), 59 deletions(-) diff --git a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md index c4a96253ec..c567ffabe1 100644 --- a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md +++ b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md @@ -399,6 +399,29 @@ where, why it is so today, what unquirked looks like. An empty ledger is a legit ## Per-format notes +### IQ3_S (the third format - and the first grid format, 2026-08-30) + +Shape: 256-superblock grid format - an 8-bit grid index plus a qh ninth bit selects +`iq3s_grid[512]` (each uint32 = 4 weight magnitudes), the block's own sign bytes flip them (no +ksigns table anywhere - the disk carries explicit signs), scale = f16 d x (1 + 2 x 4-bit +sub-scale); signed reconstruction, so no `xbsp` term. Disk block 110 B: f16 d, 64 qs, 8 qh, +32 signs, 4 packed scale nibbles. Plane pair: quants [qs 64][qh 8][signs 32] VERBATIM (104 B - +the k3 "disk is already the device form" answer), scale row = the iq4xs/k4 20 B shape with +(1 + 2s) decoded at transcode, so every k4-row consumer serves unchanged. Ids: `KqFmt.iq3s` += 8, kernel id 33, stream code 33. The codebook question a grid format adds: the 2 KB table +ships as `iq3s_grid()` - the per-call-local worker-safe form `IQ4NL_LUT` documented, just 512 +entries now - plus the main-context `IQ3S_GRID`; the repack is 26 uniform 4-byte columns (no +per-region split - simpler than k3's, since nothing shifts per lane). Gates on the worktree +binary: `test_kqformat` 16/16, `test_kquant` 163 tests 150 pass / 13 env-gated skips (the +batch-groupn family class every format shares); the emitter stubs decline (QUIRK 4), the +family minted `verdict=rejected` (QUIRK 16's shape) after the whole-scope re-tune (QUIRK 17), +and the reference bodies serve. End to end: `Llama-3.2-1B-Instruct-IQ3_M.gguf` (bartowski: +IQ3_S x78 + Q4_K x34 + Q6_K embd) through `run.das` matches llama.cpp's greedy ids **64 of +64** at gen 23 t/s - the first format to hold the whole comparison window token-for-token. +JIT emitter, Vulkan, Metal: pending (the per-tier grid-placement answers are pre-researched: +workgroup/threadgroup-staged table, the IQLUT axis on the cm2 template, threadgroup floats +on Metal). + ### Q3_K (the second format, 2026-08-30) Shape: 256-superblock, k6's scale structure exactly (16 per-16 signed sub-scales + f16 d, the diff --git a/modules/dasLLAMA/dasllama/dasllama_blocks.das b/modules/dasLLAMA/dasllama/dasllama_blocks.das index 41cbaba989..40318fd1db 100644 --- a/modules/dasLLAMA/dasllama/dasllama_blocks.das +++ b/modules/dasLLAMA/dasllama/dasllama_blocks.das @@ -1406,6 +1406,9 @@ def private kq_bytes_per_weight(f : KqFmt) : float { if (f == KqFmt.k3) { return 114.0 / 256.0 // 96B quants + the 18B scale row per 256 } + if (f == KqFmt.iq3s) { + return 124.0 / 256.0 // 104B quants + the 20B scale row per 256 + } return 34.0 / 32.0 // q8: 32 quants + an f16 scale per block } diff --git a/modules/dasLLAMA/dasllama/dasllama_common.das b/modules/dasLLAMA/dasllama/dasllama_common.das index 21a2e1dc6a..f1156420f5 100644 --- a/modules/dasLLAMA/dasllama/dasllama_common.das +++ b/modules/dasLLAMA/dasllama/dasllama_common.das @@ -841,6 +841,9 @@ struct Model { // the k3 tier's plane pair: [64 qs][32 hmask] per superblock + the k6-shaped 18B scale row (16 x int8 (sc - 32), f16 d) k3q : array k3s : array + // the iq3s tier's plane pair: [64 qs][8 qh][32 signs] per superblock + the k4-shaped 20B scale row (f16 d, 8 x int8 (1 + 2s)) + iq3sq : array + iq3ss : array kquant_native : bool = false // kq stage 4: the kq planes were repacked at load into the active backend's grp layout. // From then on every kq matmul MUST run the backend slots — disk-order portable kernels would @@ -853,6 +856,7 @@ struct Model { kq_repack_mr40 : int64 = 4l kq_repack_mr44 : int64 = 4l kq_repack_mr3 : int64 = 4l + kq_repack_mr33 : int64 = 4l // per-layer weight format tags (KqFmt; empty = all q8). Filled by detect_kq_formats before // layout_offsets so the layout walks each tensor into its format's cursor. wq_fmt : array @@ -1076,6 +1080,7 @@ def model_weights_bytes(t : Model) : int64 { + long_length(t.k4q) + long_length(t.k4s) + long_length(t.k5q) + long_length(t.k5s) + long_length(t.k6q) + long_length(t.k6s) + long_length(t.q40q) + long_length(t.q40s) + long_length(t.iq4xsq) + long_length(t.iq4xss) + long_length(t.k3q) + long_length(t.k3s) + + long_length(t.iq3sq) + long_length(t.iq3ss) + long_length(t.q51q) + long_length(t.q51s) + long_length(t.mblob) + long_length(t.vkblob) + long_length(t.embq) + long_length(t.embs)) @@ -1466,6 +1471,7 @@ def private dlim_cpu_source_impl(var c : DlimCpuConfig) { c.kq_mr40 = active_kq_layout_mr(40) c.kq_mr44 = active_kq_layout_mr(44) c.kq_mr3 = active_kq_layout_mr(3) + c.kq_mr33 = active_kq_layout_mr(33) c.q51_mr = active_q51_layout_mr() // OUTCOME, not request: a backend without s16 twins keeps f32 scale planes (wscale_convert_f16) c.wscale_f16 = g_wscale_f16 && kernel_backend_has_wscale16() @@ -1558,7 +1564,7 @@ def kq_active_mr(t : Model; f : KqFmt) : int64 { if (!t.kq_repacked) { return 1l } - return f == KqFmt.k4 ? t.kq_repack_mr4 : (f == KqFmt.k5 ? t.kq_repack_mr5 : (f == KqFmt.k6 ? t.kq_repack_mr6 : (f == KqFmt.q40 ? t.kq_repack_mr40 : (f == KqFmt.iq4xs ? t.kq_repack_mr44 : t.kq_repack_mr3)))) + return f == KqFmt.k4 ? t.kq_repack_mr4 : (f == KqFmt.k5 ? t.kq_repack_mr5 : (f == KqFmt.k6 ? t.kq_repack_mr6 : (f == KqFmt.q40 ? t.kq_repack_mr40 : (f == KqFmt.iq4xs ? t.kq_repack_mr44 : (f == KqFmt.k3 ? t.kq_repack_mr3 : t.kq_repack_mr33))))) } //! Load a llama-architecture GGUF into the split layout at the requested precision — straight into @@ -1621,8 +1627,8 @@ def footprint(t : Model) : MemFootprint { let q4s = long_length(t.q4scales) * 4l let mx4 = long_length(t.mxq) // uint8 (two nibbles) -> 1 byte each let mx4s = long_length(t.mxs) // raw E8M0 -> 1 byte per 32-block - let kq = long_length(t.k4q) + long_length(t.k5q) + long_length(t.k6q) + long_length(t.q40q) + long_length(t.q51q) + long_length(t.iq4xsq) + long_length(t.k3q) - let kqs = long_length(t.k4s) + long_length(t.k5s) + long_length(t.k6s) + long_length(t.q40s) + long_length(t.q51s) + long_length(t.iq4xss) + long_length(t.k3s) + let kq = long_length(t.k4q) + long_length(t.k5q) + long_length(t.k6q) + long_length(t.q40q) + long_length(t.q51q) + long_length(t.iq4xsq) + long_length(t.k3q) + long_length(t.iq3sq) + let kqs = long_length(t.k4s) + long_length(t.k5s) + long_length(t.k6s) + long_length(t.q40s) + long_length(t.q51s) + long_length(t.iq4xss) + long_length(t.k3s) + long_length(t.iq3ss) return MemFootprint(aux_fp32_bytes = aux, weight_fp32_bytes = wf, q8_bytes = q8, q8_scale_bytes = q8s, q4_bytes = q4, q4_scale_bytes = q4s, mx4_bytes = mx4, mx4_scale_bytes = mx4s, kq_bytes = kq, kq_scale_bytes = kqs, @@ -3296,7 +3302,7 @@ def mm_b_q51_groupn(var y : array; t : Model; offs : array; nregio // KqFmt -> the kernel-layer format id (matmul_kq*/kq_rows_fn/kq_qsb take the int form). q8 maps // to 0, a poison id no kq kernel accepts — kq_rows_for calls this for EVERY tag, so it must stay // total even though q8 branches never read the result. -def private kq_fi(fmt : KqFmt) : int => fmt == KqFmt.k4 ? 4 : (fmt == KqFmt.k5 ? 5 : (fmt == KqFmt.k6 ? 6 : (fmt == KqFmt.q40 ? 40 : (fmt == KqFmt.iq4xs ? 44 : (fmt == KqFmt.k3 ? 3 : 0))))) +def private kq_fi(fmt : KqFmt) : int => fmt == KqFmt.k4 ? 4 : (fmt == KqFmt.k5 ? 5 : (fmt == KqFmt.k6 ? 6 : (fmt == KqFmt.q40 ? 40 : (fmt == KqFmt.iq4xs ? 44 : (fmt == KqFmt.k3 ? 3 : (fmt == KqFmt.iq3s ? 33 : 0)))))) // K-quant GEMV against a PRE-quantized activation (xq/xs/xbs already filled by the bs quantizer). // kq_repacked loads run the backend's stamped cores; disk-order loads run the portable kernels. @@ -3314,6 +3320,8 @@ def mm_at_kq_pre(var y : array; t : Model; fmt : KqFmt; woff : int64; xq matmul_kq_active(44, y, t.iq4xsq, t.iq4xss, woff, xq, xs, xbs, n, d, yoff) } elif (fmt == KqFmt.k3) { matmul_kq_active(3, y, t.k3q, t.k3s, woff, xq, xs, xbs, n, d, yoff) + } elif (fmt == KqFmt.iq3s) { + matmul_kq_active(33, y, t.iq3sq, t.iq3ss, woff, xq, xs, xbs, n, d, yoff) } else { panic("mm_at_kq_pre: '{fmt}' has no kq plane pair") // a new KqFmt must claim its arm, never ride q40's } @@ -3329,6 +3337,8 @@ def mm_at_kq_pre(var y : array; t : Model; fmt : KqFmt; woff : int64; xq matmul_kq(44, y, t.iq4xsq, t.iq4xss, woff, xq, xs, xbs, n, d, yoff) } elif (fmt == KqFmt.k3) { matmul_kq(3, y, t.k3q, t.k3s, woff, xq, xs, xbs, n, d, yoff) + } elif (fmt == KqFmt.iq3s) { + matmul_kq(33, y, t.iq3sq, t.iq3ss, woff, xq, xs, xbs, n, d, yoff) } else { panic("mm_at_kq_pre: '{fmt}' has no kq plane pair") } @@ -3364,6 +3374,8 @@ def mm_b_kq(var s : Session; var y : array; t : Model; fmt : KqFmt; woff matmul_kq_batch(44, y, t.iq4xsq, t.iq4xss, woff, s.xqb, s.xsb, s.xbsb, n, d, npos) } elif (fmt == KqFmt.k3) { matmul_kq_batch(3, y, t.k3q, t.k3s, woff, s.xqb, s.xsb, s.xbsb, n, d, npos) + } elif (fmt == KqFmt.iq3s) { + matmul_kq_batch(33, y, t.iq3sq, t.iq3ss, woff, s.xqb, s.xsb, s.xbsb, n, d, npos) } else { panic("mm_b_kq: '{fmt}' has no kq plane pair") } @@ -3395,6 +3407,8 @@ def mm_at_kq_groupn(var y : array; t : Model; fmt : KqFmt; offs : array; t : Model; fmt : KqFmt; offs : array; t : Model; fmt : KqFmt; offs : array; t : Model; fmt : KqFmt; woff : int64; xq : matmul_kq_batch(44, y, t.iq4xsq, t.iq4xss, woff, xq, xs, xbs, n, d, ntok) } elif (fmt == KqFmt.k3) { matmul_kq_batch(3, y, t.k3q, t.k3s, woff, xq, xs, xbs, n, d, ntok) + } elif (fmt == KqFmt.iq3s) { + matmul_kq_batch(33, y, t.iq3sq, t.iq3ss, woff, xq, xs, xbs, n, d, ntok) } else { panic("mm_b_kq_pre: '{fmt}' has no kq plane pair") } @@ -3466,6 +3486,7 @@ def kq_plane_q(t : Model; fmt : KqFmt; woff : int64) : uint8 const? { if (fmt == KqFmt.q40) return unsafe(addr(t.q40q[sb * Q40_QSB])) if (fmt == KqFmt.iq4xs) return unsafe(addr(t.iq4xsq[sb * IQ4XS_QSB])) if (fmt == KqFmt.k3) return unsafe(addr(t.k3q[sb * K3_QSB])) + if (fmt == KqFmt.iq3s) return unsafe(addr(t.iq3sq[sb * IQ3S_QSB])) return null } @@ -3477,6 +3498,7 @@ def kq_plane_s(t : Model; fmt : KqFmt; woff : int64) : uint8 const? { if (fmt == KqFmt.q40) return unsafe(addr(t.q40s[sb * Q40_SSB])) if (fmt == KqFmt.iq4xs) return unsafe(addr(t.iq4xss[sb * IQ4XS_SSB])) if (fmt == KqFmt.k3) return unsafe(addr(t.k3s[sb * K3_SSB])) + if (fmt == KqFmt.iq3s) return unsafe(addr(t.iq3ss[sb * IQ3S_SSB])) return null } @@ -3841,6 +3863,8 @@ def embed_row(t : Model; token : int64; var dst : float?) { // nolint:STYLE037 dequant_kq_row_grp(fmt, addr(t.embq[sbg * IQ4XS_QSB]), addr(t.embs[sbg * IQ4XS_SSB]), token % mr, mr, dim, dst) } elif (t.emb_fmt == KqFmt.k3) { dequant_kq_row_grp(fmt, addr(t.embq[sbg * K3_QSB]), addr(t.embs[sbg * K3_SSB]), token % mr, mr, dim, dst) + } elif (t.emb_fmt == KqFmt.iq3s) { + dequant_kq_row_grp(fmt, addr(t.embq[sbg * IQ3S_QSB]), addr(t.embs[sbg * IQ3S_SSB]), token % mr, mr, dim, dst) } else { panic("embed_row: '{t.emb_fmt}' has no kq plane pair") } @@ -3856,6 +3880,8 @@ def embed_row(t : Model; token : int64; var dst : float?) { // nolint:STYLE037 dequant_kq_row_grp(fmt, addr(t.iq4xsq[sbg * IQ4XS_QSB]), addr(t.iq4xss[sbg * IQ4XS_SSB]), token % mr, mr, dim, dst) } elif (t.emb_fmt == KqFmt.k3) { dequant_kq_row_grp(fmt, addr(t.k3q[sbg * K3_QSB]), addr(t.k3s[sbg * K3_SSB]), token % mr, mr, dim, dst) + } elif (t.emb_fmt == KqFmt.iq3s) { + dequant_kq_row_grp(fmt, addr(t.iq3sq[sbg * IQ3S_QSB]), addr(t.iq3ss[sbg * IQ3S_SSB]), token % mr, mr, dim, dst) } else { panic("embed_row: '{t.emb_fmt}' has no kq plane pair") } @@ -3882,6 +3908,8 @@ def embed_row(t : Model; token : int64; var dst : float?) { // nolint:STYLE037 dequant_iq4xs_plane_superblock(t.embq, (sb0 + s) * IQ4XS_QSB, t.embs, (sb0 + s) * IQ4XS_SSB, row, s * 256l) } elif (t.emb_fmt == KqFmt.k3) { dequant_k3_plane_superblock(t.embq, (sb0 + s) * K3_QSB, t.embs, (sb0 + s) * K3_SSB, row, s * 256l) + } elif (t.emb_fmt == KqFmt.iq3s) { + dequant_iq3s_plane_superblock(t.embq, (sb0 + s) * IQ3S_QSB, t.embs, (sb0 + s) * IQ3S_SSB, row, s * 256l) } else { panic("embed_row: '{t.emb_fmt}' has no kq plane pair") } @@ -3901,6 +3929,8 @@ def embed_row(t : Model; token : int64; var dst : float?) { // nolint:STYLE037 dequant_iq4xs_plane_superblock(t.iq4xsq, (sb0 + s) * IQ4XS_QSB, t.iq4xss, (sb0 + s) * IQ4XS_SSB, row, s * 256l) } elif (t.emb_fmt == KqFmt.k3) { dequant_k3_plane_superblock(t.k3q, (sb0 + s) * K3_QSB, t.k3s, (sb0 + s) * K3_SSB, row, s * 256l) + } elif (t.emb_fmt == KqFmt.iq3s) { + dequant_iq3s_plane_superblock(t.iq3sq, (sb0 + s) * IQ3S_QSB, t.iq3ss, (sb0 + s) * IQ3S_SSB, row, s * 256l) } else { panic("embed_row: '{t.emb_fmt}' has no kq plane pair") } diff --git a/modules/dasLLAMA/dasllama/dasllama_config.das b/modules/dasLLAMA/dasllama/dasllama_config.das index ef903da633..f0d9c0501a 100644 --- a/modules/dasLLAMA/dasllama/dasllama_config.das +++ b/modules/dasLLAMA/dasllama/dasllama_config.das @@ -29,6 +29,7 @@ struct public DlimCpuConfig { kq_mr40 : int64 kq_mr44 : int64 kq_mr3 : int64 + kq_mr33 : int64 q51_mr : int64 wscale_f16 : bool kquant_native : bool @@ -141,7 +142,7 @@ def public dlim_config_current(quant : string = "q8") : DlimConfiguration { def public dlim_identity(c : DlimConfiguration; image_version : int; tag : string = "") : string { return ("v{image_version}|{c.quant}|{c.cpu.backend}|{c.cpu.wscale_f16 ? "s16" : "s32"}" + "|q8 mr{c.cpu.q8_mr} b{c.cpu.q8_wbias} g{c.cpu.q8_kgroup}" - + "|kq {c.cpu.kq_mr4}/{c.cpu.kq_mr5}/{c.cpu.kq_mr6}/{c.cpu.kq_mr40}/{c.cpu.kq_mr44}/{c.cpu.kq_mr3}" + + "|kq {c.cpu.kq_mr4}/{c.cpu.kq_mr5}/{c.cpu.kq_mr6}/{c.cpu.kq_mr40}/{c.cpu.kq_mr44}/{c.cpu.kq_mr3}/{c.cpu.kq_mr33}" + "|q51 mr{c.cpu.q51_mr}" + "|nat {c.cpu.kquant_native ? 1 : 0}{c.cpu.kq_q40_native ? 1 : 0}{c.cpu.kq_q50_native ? 1 : 0}{c.cpu.kq_q51_native ? 1 : 0}" + (tag != "" ? "|{tag}" : "")) diff --git a/modules/dasLLAMA/dasllama/dasllama_convert.das b/modules/dasLLAMA/dasllama/dasllama_convert.das index ad8dd7d395..50255a76d4 100644 --- a/modules/dasLLAMA/dasllama/dasllama_convert.das +++ b/modules/dasLLAMA/dasllama/dasllama_convert.das @@ -713,6 +713,53 @@ def dequant_k3_plane_superblock_at(kq : array | #; kqo : int64; ks : arra } } +//! Transcode one IQ3_S superblock (110 bytes at `bo`: f16 d, 64 qs, 8 qh, 32 signs, 4 packed +//! 4-bit scales) into the iq3s planes: [qs][qh][signs] verbatim to kq[kqo..+104), the 8 +//! sub-scales DECODED to int8 (1 + 2s) + the f16 d to ks[kso..+20) — the k4 row shape (exact). +def transcode_iq3s_superblock(bytes : array | #; bo : int64; var kq : array; kqo : int64; var ks : array; kso : int64) { + for (i in range64(104l)) { + kq[kqo + i] = bytes[bo + 2l + i] + } + ks[kso] = bytes[bo] + ks[kso + 1l] = bytes[bo + 1l] + ks[kso + 2l] = uint8(0) + ks[kso + 3l] = uint8(0) + for (b in range64(8l)) { + let nib = b % 2l == 0l ? int(bytes[bo + 106l + b / 2l]) & 15 : int(bytes[bo + 106l + b / 2l]) >> 4 + ks[kso + 4l + b] = uint8(1 + 2 * nib) + } + for (i in range64(12l, 20l)) { + ks[kso + i] = uint8(0) + } +} + +//! Reference dequant of one iq3s-plane superblock: w = (d * (1 + 2s)) * grid_byte * sign, grid +//! entry qs[b*8 + 2l + half] | (qh[b] bit (2l + half) << 8), sign byte signs[b*4 + l] bit j — +//! ggml's own float order (dequantize_row_iq3_s: db per block, then db * grid * +-1). +def dequant_iq3s_plane_superblock(kq : array | #; kqo : int64; ks : array | #; kso : int64; var dst : array | #; doff : int64) { + dequant_iq3s_plane_superblock_at(kq, kqo, ks, kso + 4l, kso, dst, doff) +} + +//! The split-layout twin: `dso` names the f16 d halfword's byte offset independently of the +//! 8-byte sub-scale strip at kso (the metal split form keeps them apart). +def dequant_iq3s_plane_superblock_at(kq : array | #; kqo : int64; ks : array | #; kso, dso : int64; var dst : array | #; doff : int64) { + let d = f16_to_f32(rd_u16(ks, dso)) + let grid = iq3s_grid() + for (b in range64(8l)) { + let db = d * float(int(int8(ks[kso + b]))) + let qh = int(kq[kqo + 64l + b]) + for (l in range64(4l)) { + let sgn = int(kq[kqo + 72l + b * 4l + l]) + let g1 = grid[int(kq[kqo + b * 8l + 2l * l]) | ((qh << int(8l - 2l * l)) & 256)] + let g2 = grid[int(kq[kqo + b * 8l + 2l * l + 1l]) | ((qh << int(7l - 2l * l)) & 256)] + for (j in range64(4l)) { + dst[doff + b * 32l + l * 8l + j] = db * float((g1 >> uint(8l * j)) & 0xFFu) * (((sgn >> int(j)) & 1) != 0 ? -1.0 : 1.0) + dst[doff + b * 32l + l * 8l + 4l + j] = db * float((g2 >> uint(8l * j)) & 0xFFu) * (((sgn >> int(4l + j)) & 1) != 0 ? -1.0 : 1.0) + } + } + } +} + //! Transcode one Q5_1 disk block (24 bytes at `bo`: d f16, m f16, qh u32, 16 nibble bytes) into //! the q51 planes — verbatim splits, exact. The per-block array form the tests drive; the bulk //! loader (gguf_transcode_q51) runs the same split pointerized and threaded. diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das index f0e9bc64a2..e8639946ba 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das @@ -2563,6 +2563,13 @@ def private iq4xs_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, def private k3_gemv_gen(var gc : LlvmCodeCtx) : bool => kq_gemv_gen_impl(gc, 3) def private k3_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 3) +// iq3s: the grid-gather emitter arm lands with its kernel arc; until then both stamps decline +// (reference bodies serve) — a declined generator is the framework's own fallback path. +[unused_argument(gc)] +def private iq3s_gemv_gen(var gc : LlvmCodeCtx) : bool => false +[unused_argument(gc)] +def private iq3s_tile_gen(var gc : LlvmCodeCtx) : bool => false + def private k4_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 4) def private k5_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 5) def private k6_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 6) @@ -2644,5 +2651,7 @@ def public register_dasllama_gemm_generators { register_llvm_code_generator("dasllama_gemm_gen::iq4xs_tile", @@iq4xs_tile_gen) register_llvm_code_generator("dasllama_gemm_gen::k3_gemv", @@k3_gemv_gen) register_llvm_code_generator("dasllama_gemm_gen::k3_tile", @@k3_tile_gen) + register_llvm_code_generator("dasllama_gemm_gen::iq3s_gemv", @@iq3s_gemv_gen) + register_llvm_code_generator("dasllama_gemm_gen::iq3s_tile", @@iq3s_tile_gen) register_llvm_code_generator("dasllama_gemm_gen::q8q8_witness", @@witness_gen) } diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_schema.das b/modules/dasLLAMA/dasllama/dasllama_gemm_schema.das index b4954f0c94..9bd854e9fd 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_schema.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_schema.das @@ -58,7 +58,7 @@ let Q51_QPB = 20l let Q51_SPB = 4l //! Quant-plane bytes per 256-weight superblock per row for a kq format id (4/5/6 = Q4_K/Q5_K/ -//! Q6_K, 40 = Q4_0, 44 = IQ4_XS, 3 = Q3_K). The ONE stride source for every fmt-branched kq walker/ +//! Q6_K, 40 = Q4_0, 44 = IQ4_XS, 3 = Q3_K, 33 = IQ3_S). The ONE stride source for every fmt-branched kq walker/ //! kernel/repack — an unknown id panics instead of silently walking another format's stride. def kq_qsb(fmt : int) : int64 { if (fmt == 4) return kq_qsb(KqFmt.k4) @@ -67,13 +67,15 @@ def kq_qsb(fmt : int) : int64 { if (fmt == 40) return kq_qsb(KqFmt.q40) if (fmt == 44) return kq_qsb(KqFmt.iq4xs) if (fmt == 3) return kq_qsb(KqFmt.k3) + if (fmt == 33) return kq_qsb(KqFmt.iq3s) panic("kq_qsb: unknown kq format id {fmt}") return 0l } //! Scale-plane bytes per 256-weight superblock per row (see kq_qsb): k4/k5 20 (16B disk block //! + 4B pad, decoded at repack), k6 18 (native), q40 16 (8 x f16 d), iq4xs 20 (d + 8 int8 -//! sub-scales decoded at transcode, the k4 row shape), k3 18 (16 int8 decoded + d, the k6 row). +//! sub-scales decoded at transcode, the k4 row shape), k3 18 (16 int8 decoded + d, the k6 row), +//! iq3s 20 (d + 8 int8 (1 + 2s) decoded at transcode, the k4 row shape again). def kq_ssb(fmt : int) : int64 { if (fmt == 4) return kq_ssb(KqFmt.k4) if (fmt == 5) return kq_ssb(KqFmt.k5) @@ -81,6 +83,7 @@ def kq_ssb(fmt : int) : int64 { if (fmt == 40) return kq_ssb(KqFmt.q40) if (fmt == 44) return kq_ssb(KqFmt.iq4xs) if (fmt == 3) return kq_ssb(KqFmt.k3) + if (fmt == 33) return kq_ssb(KqFmt.iq3s) panic("kq_ssb: unknown kq format id {fmt}") return 0l } diff --git a/modules/dasLLAMA/dasllama/dasllama_gguf.das b/modules/dasLLAMA/dasllama/dasllama_gguf.das index 674f979c6a..428d3ccd9d 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gguf.das +++ b/modules/dasLLAMA/dasllama/dasllama_gguf.das @@ -48,6 +48,7 @@ let GGML_TYPE_Q5_0 = 6 let GGML_TYPE_Q5_1 = 7 let GGML_TYPE_Q8_0 = 8 let GGML_TYPE_Q3_K = 11 +let GGML_TYPE_IQ3_S = 21 let GGML_TYPE_Q4_K = 12 let GGML_TYPE_Q5_K = 13 let GGML_TYPE_Q6_K = 14 @@ -784,6 +785,47 @@ def gguf_transcode_q3k(m : GGUFMeta; srcbytes : array | #; name : string; } } +//! Transcode an IQ3_S tensor into the iq3s planes (strides 104/20, exact): the 110B disk +//! superblock splits into [qs 64][qh 8][signs 32] verbatim and the k4-shaped scale row with the +//! 8 packed 4-bit sub-scales DECODED to int8 (1 + 2s) — transcode_iq3s_superblock. +def gguf_transcode_iq3s(m : GGUFMeta; srcbytes : array | #; name : string; var kq : array; var ks : array; eloff, expect_n : int64; src_off : int64 = 0l) { + let ti = kq_transcode_check(m, name, GGML_TYPE_IQ3_S, "IQ3_S", src_off, expect_n) + let nb = expect_n / 256l + if (nb <= 0l) { + return + } + guard_dst(name, "iq3s quant plane", (eloff / 256l) * IQ3S_QSB, nb * IQ3S_QSB, long_length(kq)) + guard_dst(name, "iq3s scale plane", (eloff / 256l) * IQ3S_SSB, nb * IQ3S_SSB, long_length(ks)) + with_tensor_view(m, srcbytes, ti) $(bytes, tbase) { + let bo = tbase + (src_off / 256l) * 110l + unsafe { + let srcp = addr(bytes[bo]) + var kqp = addr(kq[(eloff / 256l) * IQ3S_QSB]) + var ksp = addr(ks[(eloff / 256l) * IQ3S_SSB]) + maybe_parallel_for(0, int(nb), transcode_jobs(nb, 110l)) $(rb, re) { + unsafe { + for (sb in range64(int64(rb), int64(re))) { + let src = srcp + sb * 110l + var row = ksp + sb * IQ3S_SSB + row[0] = src[0] + row[1] = src[1] + row[2] = uint8(0) + row[3] = uint8(0) + for (b in range64(8l)) { + let nib = b % 2l == 0l ? int(src[106l + b / 2l]) & 15 : int(src[106l + b / 2l]) >> 4 + row[4l + b] = uint8(1 + 2 * nib) + } + for (i in range64(12l, 20l)) { + row[i] = uint8(0) + } + bcopy(kqp + sb * IQ3S_QSB, src + 2l, 104l) + } + } + } + } + } +} + //! Transcode a Q5_1 tensor into the q51 planes (per 32-BLOCK strides 20/4 — see Q51_QB): each //! 24B disk block splits into 2B d + 2B m (scale plane) and 16B nibbles + 4B qh (quant plane), //! all verbatim. Exact; `eloff`/`src_off`/`expect_n` are element offsets, % 32. diff --git a/modules/dasLLAMA/dasllama/dasllama_gpu_resident.das b/modules/dasLLAMA/dasllama/dasllama_gpu_resident.das index f3400651db..aff5adb54f 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gpu_resident.das +++ b/modules/dasLLAMA/dasllama/dasllama_gpu_resident.das @@ -288,6 +288,11 @@ def trim_model_planes(var t : Model) : bool { // nolint:STYLE038 — the flat t.embs |> reserve_resize(nsb * K3_SSB) memcpy(addr(t.embq[0]), addr(t.k3q[sb0 * K3_QSB]), nsb * K3_QSB) memcpy(addr(t.embs[0]), addr(t.k3s[sb0 * K3_SSB]), nsb * K3_SSB) + } elif (t.emb_fmt == KqFmt.iq3s) { + t.embq |> reserve_resize(nsb * IQ3S_QSB) + t.embs |> reserve_resize(nsb * IQ3S_SSB) + memcpy(addr(t.embq[0]), addr(t.iq3sq[sb0 * IQ3S_QSB]), nsb * IQ3S_QSB) + memcpy(addr(t.embs[0]), addr(t.iq3ss[sb0 * IQ3S_SSB]), nsb * IQ3S_SSB) } else { to_log(LOG_WARNING, "dasLLAMA trim: emb fmt '{t.emb_fmt}' has no kq plane pair - declining\n") return false @@ -310,6 +315,8 @@ def trim_model_planes(var t : Model) : bool { // nolint:STYLE038 — the flat delete t.iq4xss delete t.k3q delete t.k3s + delete t.iq3sq + delete t.iq3ss delete t.q40q delete t.q40s delete t.q51q diff --git a/modules/dasLLAMA/dasllama/dasllama_image.das b/modules/dasLLAMA/dasllama/dasllama_image.das index ebec2a5bfa..f8e4966091 100644 --- a/modules/dasLLAMA/dasllama/dasllama_image.das +++ b/modules/dasLLAMA/dasllama/dasllama_image.das @@ -38,7 +38,7 @@ require dasllama/dasllama_load // WhisperModel.enc) contribute their planes under dotted names ("enc.fblob"); string-array // fields ride the meta blob via serialize_strings — raw string pointers can't be planes. -let IMAGE_VERSION = 19 // 19: the k3 plane pair (Q3_K native tier); 18: the iq4xs pair +let IMAGE_VERSION = 20 // 20: the iq3s plane pair (IQ3_S native tier); 19: k3; 18: iq4xs //! The metal (blob-only) flavor's identity tag: q8 planes ride the 34B block_q8_0 blob and the //! kq scale planes their GPU forms (convert_model_to_metal_blob) — flavors are per-config and @@ -636,6 +636,7 @@ def serialize_image_meta(var arch : Archive; var t : Model) { arch |> serialize_raw(t.kq_repack_mr40) arch |> serialize_raw(t.kq_repack_mr44) arch |> serialize_raw(t.kq_repack_mr3) + arch |> serialize_raw(t.kq_repack_mr33) arch |> serialize_raw(t.wcls_fmt) arch |> serialize_raw(t.emb_fmt) arch |> serialize_raw(t.ple_emb_fmt) @@ -675,7 +676,7 @@ def serialize_image_meta(var arch : Archive; var t : Model) { } // 67 serialized fields + 3 deliberate skips (blocks, image_map, image_bytes) -let IMAGE_META_FIELDS = 68 + 3 +let IMAGE_META_FIELDS = 69 + 3 //! Count of meta-carried fields of any struct: non-arrays plus string arrays (those cannot //! be raw planes — see serialize_strings). Pair with a per-type field-count constant to @@ -959,7 +960,7 @@ def save_model_image(var t : Model; path : string; tag : string = ""; quant : st // streamed planes size as empty, so their bytes plus a page each are added here; over-reserve is free (dwrite_close truncates) def private stream_extra_bytes(t : Model; jobs : array) : uint64 { var extra = 0ul - for (fname in ["qblob", "mblob", "k4q", "k5q", "k6q", "q40q", "q51q", "iq4xsq", "k3q"]) { + for (fname in ["qblob", "mblob", "k4q", "k5q", "k6q", "q40q", "q51q", "iq4xsq", "k3q", "iq3sq"]) { let sb = stream_plane_bytes(t, jobs, fname) if (sb > 0l) { extra += uint64(sb) + uint64(IMAGE_PAGE) diff --git a/modules/dasLLAMA/dasllama/dasllama_kqformat.das b/modules/dasLLAMA/dasllama/dasllama_kqformat.das index a99f5b6f59..baff68c03f 100644 --- a/modules/dasLLAMA/dasllama/dasllama_kqformat.das +++ b/modules/dasLLAMA/dasllama/dasllama_kqformat.das @@ -13,7 +13,7 @@ module dasllama_kqformat shared public require dasllama/dasllama_lint public //! Per-weight storage format under a q8-mode load with native planes: q8 = qblob/qscales; -//! k4/k5/k6/q40/iq4xs/k3 = superblock plane pairs (mixed PER TENSOR); q51 = native Q5_1 per-32 +//! k4/k5/k6/q40/iq4xs/k3/iq3s = superblock plane pairs (mixed PER TENSOR); q51 = native Q5_1 per-32 //! planes. New members append — the int value is the device stack tag and the image plane id. enum KqFmt : uint8 { q8 @@ -24,19 +24,20 @@ enum KqFmt : uint8 { q51 iq4xs k3 + iq3s } //! The superblock-lattice formats (Q8_K-form activations, % 256 rows, repack + stamped kq //! kernels). q51 deliberately fails this: it rides per-32 planes with Q8_0-form activations — //! a `fmt != KqFmt.q8` test does not imply the kq lattice; branch on kq_sb where it does. -def kq_sb(f : KqFmt) : bool => f == KqFmt.k4 || f == KqFmt.k5 || f == KqFmt.k6 || f == KqFmt.q40 || f == KqFmt.iq4xs || f == KqFmt.k3 +def kq_sb(f : KqFmt) : bool => f == KqFmt.k4 || f == KqFmt.k5 || f == KqFmt.k6 || f == KqFmt.q40 || f == KqFmt.iq4xs || f == KqFmt.k3 || f == KqFmt.iq3s //! The same predicate over the int id space the GPU drivers carry per stack/plane. -def kq_sb(fmt : int) : bool => fmt == int(KqFmt.k4) || fmt == int(KqFmt.k5) || fmt == int(KqFmt.k6) || fmt == int(KqFmt.q40) || fmt == int(KqFmt.iq4xs) || fmt == int(KqFmt.k3) +def kq_sb(fmt : int) : bool => fmt == int(KqFmt.k4) || fmt == int(KqFmt.k5) || fmt == int(KqFmt.k6) || fmt == int(KqFmt.q40) || fmt == int(KqFmt.iq4xs) || fmt == int(KqFmt.k3) || fmt == int(KqFmt.iq3s) //! KqFmt -> the kernel/IR format id space — the ONE bridge between the enum and the int ids the //! generated kernels take as runtime params. Mnemonic ids: a K-quant is its bit width (3/4/5/6), -//! Q4_0 is 40, an i-quant is bit width x 10 + a variant digit (IQ4_XS = 44). q8/q51 panic. +//! Q4_0 is 40, an i-quant is bit width x 10 + a variant digit (IQ4_XS = 44, IQ3_S = 33). q8/q51 panic. def kq_schema_id(f : KqFmt) : int { if (f == KqFmt.k4) { return 4 @@ -56,6 +57,9 @@ def kq_schema_id(f : KqFmt) : int { if (f == KqFmt.k3) { return 3 } + if (f == KqFmt.iq3s) { + return 33 + } panic("kq_schema_id: not a kq superblock format") return 0 } @@ -76,6 +80,8 @@ let IQ4XS_QSB = 128l // IQ4_XS quant plane: 128 nibble bytes, the disk k/k+16 let IQ4XS_SSB = 20l // IQ4_XS scale plane: f16 d, 2 pad, 8 x int8 (ls - 32), 8 pad — the k4 decoded-row shape let K3_QSB = 96l // Q3_K quant plane: 64 qs bytes (four 2-bit lanes each) + 32 hmask bytes, verbatim disk order let K3_SSB = 18l // Q3_K scale plane: 16 x int8 (6-bit scale - 32, decoded at transcode) + f16 d — the k6 row +let IQ3S_QSB = 104l // IQ3_S quant plane: [64 grid-index qs][8 qh ninth bits][32 sign bytes], verbatim disk order +let IQ3S_SSB = 20l // IQ3_S scale plane: f16 d, 2 pad, 8 x int8 (1 + 2s), 8 pad — the iq4xs/k4 row shape //! The IQ4_NL / IQ4_XS nibble codebook: weight = scale x IQ4NL_LUT[nibble] (ggml's kvalues_iq4nl). //! Main-context use only (tests, oracles, the emitter's constant bake): a kernel that can run on a @@ -86,6 +92,81 @@ let IQ4NL_LUT = fixed_array(-127, -104, -83, -65, -49, -35, -22, -10, 1, 1 //! worker-run kernel reads. def iq4nl_lut() : int8[16] => fixed_array(-127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113) +//! The IQ3_S grid (ggml's iq3s_grid): entry qs | (qh bit << 8) packs FOUR weight magnitudes as +//! bytes; the per-element sign rides the block's own sign bytes. Per-call local (2 KB on the +//! caller's frame) - the form every worker-run kernel reads; IQ3S_GRID is the main-context copy. +def iq3s_grid() : uint[512] { + return fixed_array( + 0x01010101u, 0x01010103u, 0x01010105u, 0x0101010bu, 0x0101010fu, 0x01010301u, 0x01010303u, 0x01010305u, + 0x01010309u, 0x0101030du, 0x01010501u, 0x01010503u, 0x0101050bu, 0x01010707u, 0x01010901u, 0x01010905u, + 0x0101090bu, 0x0101090fu, 0x01010b03u, 0x01010b07u, 0x01010d01u, 0x01010d05u, 0x01010f03u, 0x01010f09u, + 0x01010f0fu, 0x01030101u, 0x01030103u, 0x01030105u, 0x01030109u, 0x01030301u, 0x01030303u, 0x0103030bu, + 0x01030501u, 0x01030507u, 0x0103050fu, 0x01030703u, 0x0103070bu, 0x01030909u, 0x01030d03u, 0x01030d0bu, + 0x01030f05u, 0x01050101u, 0x01050103u, 0x0105010bu, 0x0105010fu, 0x01050301u, 0x01050307u, 0x0105030du, + 0x01050503u, 0x0105050bu, 0x01050701u, 0x01050709u, 0x01050905u, 0x0105090bu, 0x0105090fu, 0x01050b03u, + 0x01050b07u, 0x01050f01u, 0x01050f07u, 0x01070107u, 0x01070303u, 0x0107030bu, 0x01070501u, 0x01070505u, + 0x01070703u, 0x01070707u, 0x0107070du, 0x01070909u, 0x01070b01u, 0x01070b05u, 0x01070d0fu, 0x01070f03u, + 0x01070f0bu, 0x01090101u, 0x01090307u, 0x0109030fu, 0x01090503u, 0x01090509u, 0x01090705u, 0x01090901u, + 0x01090907u, 0x01090b03u, 0x01090f01u, 0x010b0105u, 0x010b0109u, 0x010b0501u, 0x010b0505u, 0x010b050du, + 0x010b0707u, 0x010b0903u, 0x010b090bu, 0x010b090fu, 0x010b0d0du, 0x010b0f07u, 0x010d010du, 0x010d0303u, + 0x010d0307u, 0x010d0703u, 0x010d0b05u, 0x010d0f03u, 0x010f0101u, 0x010f0105u, 0x010f0109u, 0x010f0501u, + 0x010f0505u, 0x010f050du, 0x010f0707u, 0x010f0b01u, 0x010f0b09u, 0x03010101u, 0x03010103u, 0x03010105u, + 0x03010109u, 0x03010301u, 0x03010303u, 0x03010307u, 0x0301030bu, 0x0301030fu, 0x03010501u, 0x03010505u, + 0x03010703u, 0x03010709u, 0x0301070du, 0x03010b09u, 0x03010b0du, 0x03010d03u, 0x03010f05u, 0x03030101u, + 0x03030103u, 0x03030107u, 0x0303010du, 0x03030301u, 0x03030309u, 0x03030503u, 0x03030701u, 0x03030707u, + 0x03030903u, 0x03030b01u, 0x03030b05u, 0x03030f01u, 0x03030f0du, 0x03050101u, 0x03050305u, 0x0305030bu, + 0x0305030fu, 0x03050501u, 0x03050509u, 0x03050705u, 0x03050901u, 0x03050907u, 0x03050b0bu, 0x03050d01u, + 0x03050f05u, 0x03070103u, 0x03070109u, 0x0307010fu, 0x03070301u, 0x03070307u, 0x03070503u, 0x0307050fu, + 0x03070701u, 0x03070709u, 0x03070903u, 0x03070d05u, 0x03070f01u, 0x03090107u, 0x0309010bu, 0x03090305u, + 0x03090309u, 0x03090703u, 0x03090707u, 0x03090905u, 0x0309090du, 0x03090b01u, 0x03090b09u, 0x030b0103u, + 0x030b0301u, 0x030b0307u, 0x030b0503u, 0x030b0701u, 0x030b0705u, 0x030b0b03u, 0x030d0501u, 0x030d0509u, + 0x030d050fu, 0x030d0909u, 0x030d090du, 0x030f0103u, 0x030f0107u, 0x030f0301u, 0x030f0305u, 0x030f0503u, + 0x030f070bu, 0x030f0903u, 0x030f0d05u, 0x030f0f01u, 0x05010101u, 0x05010103u, 0x05010107u, 0x0501010bu, + 0x0501010fu, 0x05010301u, 0x05010305u, 0x05010309u, 0x0501030du, 0x05010503u, 0x05010507u, 0x0501050fu, + 0x05010701u, 0x05010705u, 0x05010903u, 0x05010907u, 0x0501090bu, 0x05010b01u, 0x05010b05u, 0x05010d0fu, + 0x05010f01u, 0x05010f07u, 0x05010f0bu, 0x05030101u, 0x05030105u, 0x05030301u, 0x05030307u, 0x0503030fu, + 0x05030505u, 0x0503050bu, 0x05030703u, 0x05030709u, 0x05030905u, 0x05030b03u, 0x05050103u, 0x05050109u, + 0x0505010fu, 0x05050503u, 0x05050507u, 0x05050701u, 0x0505070fu, 0x05050903u, 0x05050b07u, 0x05050b0fu, + 0x05050f03u, 0x05050f09u, 0x05070101u, 0x05070105u, 0x0507010bu, 0x05070303u, 0x05070505u, 0x05070509u, + 0x05070703u, 0x05070707u, 0x05070905u, 0x05070b01u, 0x05070d0du, 0x05090103u, 0x0509010fu, 0x05090501u, + 0x05090507u, 0x05090705u, 0x0509070bu, 0x05090903u, 0x05090f05u, 0x05090f0bu, 0x050b0109u, 0x050b0303u, + 0x050b0505u, 0x050b070fu, 0x050b0901u, 0x050b0b07u, 0x050b0f01u, 0x050d0101u, 0x050d0105u, 0x050d010fu, + 0x050d0503u, 0x050d0b0bu, 0x050d0d03u, 0x050f010bu, 0x050f0303u, 0x050f050du, 0x050f0701u, 0x050f0907u, + 0x050f0b01u, 0x07010105u, 0x07010303u, 0x07010307u, 0x0701030bu, 0x0701030fu, 0x07010505u, 0x07010703u, + 0x07010707u, 0x0701070bu, 0x07010905u, 0x07010909u, 0x0701090fu, 0x07010b03u, 0x07010d07u, 0x07010f03u, + 0x07030103u, 0x07030107u, 0x0703010bu, 0x07030309u, 0x07030503u, 0x07030507u, 0x07030901u, 0x07030d01u, + 0x07030f05u, 0x07030f0du, 0x07050101u, 0x07050305u, 0x07050501u, 0x07050705u, 0x07050709u, 0x07050b01u, + 0x07070103u, 0x07070301u, 0x07070309u, 0x07070503u, 0x07070507u, 0x0707050fu, 0x07070701u, 0x07070903u, + 0x07070907u, 0x0707090fu, 0x07070b0bu, 0x07070f07u, 0x07090107u, 0x07090303u, 0x0709030du, 0x07090505u, + 0x07090703u, 0x07090b05u, 0x07090d01u, 0x07090d09u, 0x070b0103u, 0x070b0301u, 0x070b0305u, 0x070b050bu, + 0x070b0705u, 0x070b0909u, 0x070b0b0du, 0x070b0f07u, 0x070d030du, 0x070d0903u, 0x070f0103u, 0x070f0107u, + 0x070f0501u, 0x070f0505u, 0x070f070bu, 0x09010101u, 0x09010109u, 0x09010305u, 0x09010501u, 0x09010509u, + 0x0901050fu, 0x09010705u, 0x09010903u, 0x09010b01u, 0x09010f01u, 0x09030105u, 0x0903010fu, 0x09030303u, + 0x09030307u, 0x09030505u, 0x09030701u, 0x0903070bu, 0x09030907u, 0x09030b03u, 0x09030b0bu, 0x09050103u, + 0x09050107u, 0x09050301u, 0x0905030bu, 0x09050503u, 0x09050707u, 0x09050901u, 0x09050b0fu, 0x09050d05u, + 0x09050f01u, 0x09070109u, 0x09070303u, 0x09070307u, 0x09070501u, 0x09070505u, 0x09070703u, 0x0907070bu, + 0x09090101u, 0x09090105u, 0x09090509u, 0x0909070fu, 0x09090901u, 0x09090f03u, 0x090b010bu, 0x090b010fu, + 0x090b0503u, 0x090b0d05u, 0x090d0307u, 0x090d0709u, 0x090d0d01u, 0x090f0301u, 0x090f030bu, 0x090f0701u, + 0x090f0907u, 0x090f0b03u, 0x0b010105u, 0x0b010301u, 0x0b010309u, 0x0b010505u, 0x0b010901u, 0x0b010909u, + 0x0b01090fu, 0x0b010b05u, 0x0b010d0du, 0x0b010f09u, 0x0b030103u, 0x0b030107u, 0x0b03010bu, 0x0b030305u, + 0x0b030503u, 0x0b030705u, 0x0b030f05u, 0x0b050101u, 0x0b050303u, 0x0b050507u, 0x0b050701u, 0x0b05070du, + 0x0b050b07u, 0x0b070105u, 0x0b07010fu, 0x0b070301u, 0x0b07050fu, 0x0b070909u, 0x0b070b03u, 0x0b070d0bu, + 0x0b070f07u, 0x0b090103u, 0x0b090109u, 0x0b090501u, 0x0b090705u, 0x0b09090du, 0x0b0b0305u, 0x0b0b050du, + 0x0b0b0b03u, 0x0b0b0b07u, 0x0b0d0905u, 0x0b0f0105u, 0x0b0f0109u, 0x0b0f0505u, 0x0d010303u, 0x0d010307u, + 0x0d01030bu, 0x0d010703u, 0x0d010707u, 0x0d010d01u, 0x0d030101u, 0x0d030501u, 0x0d03050fu, 0x0d030d09u, + 0x0d050305u, 0x0d050709u, 0x0d050905u, 0x0d050b0bu, 0x0d050d05u, 0x0d050f01u, 0x0d070101u, 0x0d070309u, + 0x0d070503u, 0x0d070901u, 0x0d09050bu, 0x0d090907u, 0x0d090d05u, 0x0d0b0101u, 0x0d0b0107u, 0x0d0b0709u, + 0x0d0b0d01u, 0x0d0d010bu, 0x0d0d0901u, 0x0d0f0303u, 0x0d0f0307u, 0x0f010101u, 0x0f010109u, 0x0f01010fu, + 0x0f010501u, 0x0f010505u, 0x0f01070du, 0x0f010901u, 0x0f010b09u, 0x0f010d05u, 0x0f030105u, 0x0f030303u, + 0x0f030509u, 0x0f030907u, 0x0f03090bu, 0x0f050103u, 0x0f050109u, 0x0f050301u, 0x0f05030du, 0x0f050503u, + 0x0f050701u, 0x0f050b03u, 0x0f070105u, 0x0f070705u, 0x0f07070bu, 0x0f070b07u, 0x0f090103u, 0x0f09010bu, + 0x0f090307u, 0x0f090501u, 0x0f090b01u, 0x0f0b0505u, 0x0f0b0905u, 0x0f0d0105u, 0x0f0d0703u, 0x0f0f0101u) +} + +//! Main-context use only (tests, oracles, the emitter's constant bake) - a forked worker reads +//! a module global as zeros; kernels take `iq3s_grid()` instead. +let IQ3S_GRID = iq3s_grid() + //! Quant-plane bytes per stride unit — the typed stride surface. The stride unit is one //! 256-weight superblock for the kq lattice and one 32-weight block for q51 (see kq_elems). //! q8 panics (its planes are byte-per-elem, not stride-walked here). @@ -105,13 +186,16 @@ def kq_qsb(f : KqFmt) : int64 { if (f == KqFmt.k3) { return K3_QSB } + if (f == KqFmt.iq3s) { + return IQ3S_QSB + } panic("kq_qsb: no plane stride for this format") return 0l } //! Scale-plane bytes per stride unit (see kq_qsb for the unit). def kq_ssb(f : KqFmt) : int64 { - if (f == KqFmt.k4 || f == KqFmt.k5 || f == KqFmt.iq4xs) { + if (f == KqFmt.k4 || f == KqFmt.k5 || f == KqFmt.iq4xs || f == KqFmt.iq3s) { return K4_SSB } if (f == KqFmt.k6 || f == KqFmt.k3) { diff --git a/modules/dasLLAMA/dasllama/dasllama_layout.das b/modules/dasLLAMA/dasllama/dasllama_layout.das index bb0b1aa165..9b2bd172b9 100644 --- a/modules/dasLLAMA/dasllama/dasllama_layout.das +++ b/modules/dasLLAMA/dasllama/dasllama_layout.das @@ -338,6 +338,10 @@ def private repack_regions(var t : Model; regs : array) { var k3sp : uint8? = null if (!empty(t.k3q)) { k3qp = addr(t.k3q[0]) } if (!empty(t.k3s)) { k3sp = addr(t.k3s[0]) } + var iq3sqp : uint8? = null + var iq3ssp : uint8? = null + if (!empty(t.iq3sq)) { iq3sqp = addr(t.iq3sq[0]) } + if (!empty(t.iq3ss)) { iq3ssp = addr(t.iq3ss[0]) } if (!empty(t.qblob)) { qbp = addr(t.qblob[0]) } if (!empty(t.qscales)) { qsp = addr(t.qscales[0]) } if (!empty(t.mxq)) { mxqp = addr(t.mxq[0]) } @@ -369,8 +373,8 @@ def private repack_regions(var t : Model; regs : array) { let sb = rp[i].off / 256l let qsb = kq_qsb(f) let ssb = kq_ssb(f) - var kqp = f == 4 ? k4qp : (f == 5 ? k5qp : (f == 6 ? k6qp : (f == 40 ? q40qp : (f == 44 ? iq4xsqp : k3qp)))) - var ksp = f == 4 ? k4sp : (f == 5 ? k5sp : (f == 6 ? k6sp : (f == 40 ? q40sp : (f == 44 ? iq4xssp : k3sp)))) + var kqp = f == 4 ? k4qp : (f == 5 ? k5qp : (f == 6 ? k6qp : (f == 40 ? q40qp : (f == 44 ? iq4xsqp : (f == 3 ? k3qp : iq3sqp))))) + var ksp = f == 4 ? k4sp : (f == 5 ? k5sp : (f == 6 ? k6sp : (f == 40 ? q40sp : (f == 44 ? iq4xssp : (f == 3 ? k3sp : iq3ssp))))) invoke(rkq, f, kqp + sb * qsb, ksp + sb * ssb, rp[i].n, rp[i].d) } } @@ -500,6 +504,8 @@ def private push_repack_kq(var regs : array; fmt : KqFmt; woff, n, d push_repack(regs, 44, woff, n, d) } elif (fmt == KqFmt.k3) { push_repack(regs, 3, woff, n, d) + } elif (fmt == KqFmt.iq3s) { + push_repack(regs, 33, woff, n, d) } } @@ -774,12 +780,14 @@ def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice : (fmt == KqFmt.k5 ? addr(t.k5q[sb0 * kq_qsb(5)]) : (fmt == KqFmt.k6 ? addr(t.k6q[sb0 * kq_qsb(6)]) : (fmt == KqFmt.q40 ? addr(t.q40q[sb0 * kq_qsb(40)]) - : (fmt == KqFmt.iq4xs ? addr(t.iq4xsq[sb0 * kq_qsb(44)]) : addr(t.k3q[sb0 * kq_qsb(3)])))))) + : (fmt == KqFmt.iq4xs ? addr(t.iq4xsq[sb0 * kq_qsb(44)]) + : (fmt == KqFmt.k3 ? addr(t.k3q[sb0 * kq_qsb(3)]) : addr(t.iq3sq[sb0 * kq_qsb(33)]))))))) let sp = (fmt == KqFmt.k4 ? addr(t.k4s[sb0 * kq_ssb(4)]) : (fmt == KqFmt.k5 ? addr(t.k5s[sb0 * kq_ssb(5)]) : (fmt == KqFmt.k6 ? addr(t.k6s[sb0 * kq_ssb(6)]) : (fmt == KqFmt.q40 ? addr(t.q40s[sb0 * kq_ssb(40)]) - : (fmt == KqFmt.iq4xs ? addr(t.iq4xss[sb0 * kq_ssb(44)]) : addr(t.k3s[sb0 * kq_ssb(3)])))))) + : (fmt == KqFmt.iq4xs ? addr(t.iq4xss[sb0 * kq_ssb(44)]) + : (fmt == KqFmt.k3 ? addr(t.k3s[sb0 * kq_ssb(3)]) : addr(t.iq3ss[sb0 * kq_ssb(33)]))))))) var wqp = addr(wq[0]) var wsp = addr(ws[0]) let njobs = is_job_que_available() ? min(nslices, 4 * (get_total_hw_jobs() + 1)) : 1 @@ -799,7 +807,11 @@ def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice if (grouped) { let gq = qp + sliceQ + g * mr * qrow + sbi * qsb * mr let gs = sp + sliceS + g * mr * srow + sbi * dssb * mr - if (fmt == KqFmt.k3) { // 16 qs columns, then the 8 hmask columns at 64*mr + if (fmt == KqFmt.iq3s) { // 26 uniform 4-byte columns over [qs][qh][signs] + for (m in range64(104l)) { + dq[m] = gq[((m / 4l) * mr + r) * 4l + m % 4l] + } + } elif (fmt == KqFmt.k3) { // 16 qs columns, then the 8 hmask columns at 64*mr for (m in range64(64l)) { dq[m] = gq[((m / 4l) * mr + r) * 4l + m % 4l] } @@ -849,7 +861,7 @@ def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice } else { let rq = qp + sliceQ + ri * qrow + sbi * qsb let rs = sp + sliceS + ri * srow + sbi * dssb - if (fmt == KqFmt.q40 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3) { + if (fmt == KqFmt.q40 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s) { for (m in range64(qsb)) { // disk bytes already in the device form (k/k+16 pairs; k3's lanes) dq[m] = rq[m] } @@ -876,7 +888,7 @@ def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice dq[128l + p] = rq[128l + p] } } - if (fmt == KqFmt.k6 || fmt == KqFmt.q40 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3) { // rows already in the decoded device form + if (fmt == KqFmt.k6 || fmt == KqFmt.q40 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s) { // rows already in the decoded device form for (idx in range64(dssb)) { dsc[idx] = rs[idx] } diff --git a/modules/dasLLAMA/dasllama/dasllama_load.das b/modules/dasLLAMA/dasllama/dasllama_load.das index 9f0c149d58..ec8e3790df 100644 --- a/modules/dasLLAMA/dasllama/dasllama_load.das +++ b/modules/dasLLAMA/dasllama/dasllama_load.das @@ -43,6 +43,7 @@ struct private LayoutSizes { q51_n : int64 iq4xs_n : int64 k3_n : int64 + iq3s_n : int64 pleq8_n : int64 // the dedicated q8 plane holding ONLY the PLE table (fp32/q4 serving) } @@ -57,6 +58,7 @@ struct private KqCursors { q51 : int64 iq4xs : int64 k3 : int64 + iq3s : int64 } def private kq_take(var cur : KqCursors; f : KqFmt; n : int64) : int64 { @@ -95,6 +97,11 @@ def private kq_take(var cur : KqCursors; f : KqFmt; n : int64) : int64 { cur.k3 += n return o } + if (f == KqFmt.iq3s) { + let o = cur.iq3s + cur.iq3s += n + return o + } let o = cur.wo cur.wo += n return o @@ -394,7 +401,7 @@ def private layout_offsets(var t : Model) : LayoutSizes { // nolint:STYLE037,S fo += dim } return LayoutSizes(fblob_n = fo, wblob_n = cur.wo, mx_n = mx, bf16_n = bf16, - k4_n = cur.k4, k5_n = cur.k5, k6_n = cur.k6, q40_n = cur.q40, q51_n = cur.q51, iq4xs_n = cur.iq4xs, k3_n = cur.k3, pleq8_n = pleq8) + k4_n = cur.k4, k5_n = cur.k5, k6_n = cur.k6, q40_n = cur.q40, q51_n = cur.q51, iq4xs_n = cur.iq4xs, k3_n = cur.k3, iq3s_n = cur.iq3s, pleq8_n = pleq8) } @@ -657,6 +664,8 @@ def stream_field_of(t : Model; fmt : KqFmt) : string { return "iq4xsq" } elif (fmt == KqFmt.k3) { return "k3q" + } elif (fmt == KqFmt.iq3s) { + return "iq3sq" } elif (t.quant == QuantMode.q8) { // the blob flavor carries q8 as gguf-native 34B blocks in ONE plane, never the split pair return t.metal_blob ? "mblob" : "qblob" @@ -777,6 +786,8 @@ def fill_stream_plane(m : GGUFMeta; bytes : array | #; var t : Model; fie memcpy(addr(t.iq4xss[(j.woff / 256l) * IQ4XS_SSB]), addr(temp_s[0]), sh.sb) } elif (j.fmt == KqFmt.k3) { memcpy(addr(t.k3s[(j.woff / 256l) * K3_SSB]), addr(temp_s[0]), sh.sb) + } elif (j.fmt == KqFmt.iq3s) { + memcpy(addr(t.iq3ss[(j.woff / 256l) * IQ3S_SSB]), addr(temp_s[0]), sh.sb) } elif (j.fmt == KqFmt.q51) { memcpy(addr(t.q51s[(j.woff / 32l) * Q51_SB]), addr(temp_s[0]), sh.sb) } elif (sh.sb == 0l && j.n > 0l && !t.metal_blob) { @@ -814,6 +825,9 @@ def fill_stream_plane(m : GGUFMeta; bytes : array | #; var t : Model; fie } elif (r.fmt == 3) { invoke(g_stream_repack, 3, addr(temp_q[((r.off - j.woff) / 256l) * K3_QSB]), addr(t.k3s[(r.off / 256l) * K3_SSB]), r.n, r.d) + } elif (r.fmt == 33) { + invoke(g_stream_repack, 33, addr(temp_q[((r.off - j.woff) / 256l) * IQ3S_QSB]), + addr(t.iq3ss[(r.off / 256l) * IQ3S_SSB]), r.n, r.d) } } } @@ -1032,6 +1046,9 @@ def private transcode_kq_tensor(m : GGUFMeta; bytes : array | #; name : s } elif (fmt == KqFmt.k3) { gguf_transcode_q3k(m, bytes, name, kq, ks, eloff, n, src_off) return "k3 transcode (Q3_K)" + } elif (fmt == KqFmt.iq3s) { + gguf_transcode_iq3s(m, bytes, name, kq, ks, eloff, n, src_off) + return "iq3s transcode (IQ3_S)" } gguf_transcode_q51(m, bytes, name, kq, ks, eloff, n, src_off) return "q51 transcode (Q5_1)" @@ -1074,6 +1091,8 @@ def private load_big(m : GGUFMeta; bytes : array | #; name : string; var kind = transcode_kq_tensor(m, bytes, name, fmt, t.iq4xsq, t.iq4xss, woff, n, src_off, scratch) } elif (fmt == KqFmt.k3) { kind = transcode_kq_tensor(m, bytes, name, fmt, t.k3q, t.k3s, woff, n, src_off, scratch) + } elif (fmt == KqFmt.iq3s) { + kind = transcode_kq_tensor(m, bytes, name, fmt, t.iq3sq, t.iq3ss, woff, n, src_off, scratch) } elif (fmt == KqFmt.q51) { kind = transcode_kq_tensor(m, bytes, name, fmt, t.q51q, t.q51s, woff, n, src_off, scratch) } elif (t.quant == QuantMode.q8) { @@ -1173,8 +1192,8 @@ def private log_load_report(t : Model) { to_log(LOG_INFO, "dasLLAMA noisy: backend '{active_kernel_backend()}' | kquant_native {get_kquant_native()} q40_native {get_kq_q40_native()} q50_native {get_kq_q50_native()}\n") // the tier arms at [init], before a --noisy CLI flag can land — restate it here so the flag alone suffices to_log(LOG_INFO, "dasLLAMA noisy: GPU tier installed {moe_gpu_tier_installed()} | want auto {gpu_want_auto()}, moe layers {gpu_want_moe_layers()}, stream {gpu_want_moe_stream()}\n") - to_log(LOG_INFO, "dasLLAMA noisy: format tags - q8 {tally_fmt(t, KqFmt.q8)}, k4 {tally_fmt(t, KqFmt.k4)}, k5 {tally_fmt(t, KqFmt.k5)}, k6 {tally_fmt(t, KqFmt.k6)}, q40 {tally_fmt(t, KqFmt.q40)}, q51 {tally_fmt(t, KqFmt.q51)}, iq4xs {tally_fmt(t, KqFmt.iq4xs)}, k3 {tally_fmt(t, KqFmt.k3)}\n") - to_log(LOG_INFO, "dasLLAMA noisy: plane MB - q8 {float(long_length(t.qblob)) / mb} (+scales {float(long_length(t.qscales) * 4l) / mb}), k4 {float(long_length(t.k4q) + long_length(t.k4s)) / mb}, k5 {float(long_length(t.k5q) + long_length(t.k5s)) / mb}, k6 {float(long_length(t.k6q) + long_length(t.k6s)) / mb}, q40 {float(long_length(t.q40q) + long_length(t.q40s)) / mb}, q51 {float(long_length(t.q51q) + long_length(t.q51s)) / mb}, iq4xs {float(long_length(t.iq4xsq) + long_length(t.iq4xss)) / mb}, k3 {float(long_length(t.k3q) + long_length(t.k3s)) / mb}, f32 {float(long_length(t.wblob) * 4l) / mb}\n") + to_log(LOG_INFO, "dasLLAMA noisy: format tags - q8 {tally_fmt(t, KqFmt.q8)}, k4 {tally_fmt(t, KqFmt.k4)}, k5 {tally_fmt(t, KqFmt.k5)}, k6 {tally_fmt(t, KqFmt.k6)}, q40 {tally_fmt(t, KqFmt.q40)}, q51 {tally_fmt(t, KqFmt.q51)}, iq4xs {tally_fmt(t, KqFmt.iq4xs)}, k3 {tally_fmt(t, KqFmt.k3)}, iq3s {tally_fmt(t, KqFmt.iq3s)}\n") + to_log(LOG_INFO, "dasLLAMA noisy: plane MB - q8 {float(long_length(t.qblob)) / mb} (+scales {float(long_length(t.qscales) * 4l) / mb}), k4 {float(long_length(t.k4q) + long_length(t.k4s)) / mb}, k5 {float(long_length(t.k5q) + long_length(t.k5s)) / mb}, k6 {float(long_length(t.k6q) + long_length(t.k6s)) / mb}, q40 {float(long_length(t.q40q) + long_length(t.q40s)) / mb}, q51 {float(long_length(t.q51q) + long_length(t.q51s)) / mb}, iq4xs {float(long_length(t.iq4xsq) + long_length(t.iq4xss)) / mb}, k3 {float(long_length(t.k3q) + long_length(t.k3s)) / mb}, iq3s {float(long_length(t.iq3sq) + long_length(t.iq3ss)) / mb}, f32 {float(long_length(t.wblob) * 4l) / mb}\n") } // GGML disk type -> plane format tag (non-K-quant types ride the classic q8 path) @@ -1194,6 +1213,9 @@ def private kq_fmt_of(gt : int) : KqFmt { if (gt == GGML_TYPE_Q3_K) { return KqFmt.k3 } + if (gt == GGML_TYPE_IQ3_S) { + return KqFmt.iq3s + } if (gt == GGML_TYPE_Q4_0 && get_kq_q40_native()) { return KqFmt.q40 } @@ -2029,6 +2051,7 @@ def private load_gguf_parsed(var t : Model; m : GGUFMeta; bytes : array | g_stream_plane_total["q51q"] = (sz.q51_n / 32l) * Q51_QB g_stream_plane_total["iq4xsq"] = (sz.iq4xs_n / 256l) * IQ4XS_QSB g_stream_plane_total["k3q"] = (sz.k3_n / 256l) * K3_QSB + g_stream_plane_total["iq3sq"] = (sz.iq3s_n / 256l) * IQ3S_QSB let wb = mode == QuantMode.q8 ? sz.wblob_n : 0l g_stream_plane_total[t.metal_blob ? "mblob" : "qblob"] = (t.metal_blob ? (wb / Q8_BLOCK_ELEMS) * (Q8_QPB + Q8_SPB) : wb) @@ -2089,6 +2112,14 @@ def private load_gguf_parsed(var t : Model; m : GGUFMeta; bytes : array | t.k3s |> reserve((sz.k3_n / 256l) * K3_SSB) t.k3s |> resize((sz.k3_n / 256l) * K3_SSB) } + if (sz.iq3s_n > 0l) { + if (!stream_q) { + t.iq3sq |> reserve((sz.iq3s_n / 256l) * IQ3S_QSB) + t.iq3sq |> resize((sz.iq3s_n / 256l) * IQ3S_QSB) + } + t.iq3ss |> reserve((sz.iq3s_n / 256l) * IQ3S_SSB) + t.iq3ss |> resize((sz.iq3s_n / 256l) * IQ3S_SSB) + } if (mode == QuantMode.q8) { if (!stream_q) { t.qblob |> reserve(sz.wblob_n) @@ -2397,6 +2428,7 @@ def private load_gguf_parsed(var t : Model; m : GGUFMeta; bytes : array | t.kq_repack_mr40 = active_kq_layout_mr(40) t.kq_repack_mr44 = active_kq_layout_mr(44) t.kq_repack_mr3 = active_kq_layout_mr(3) + t.kq_repack_mr33 = active_kq_layout_mr(33) } } else { layout_repack_q8(t) @@ -2415,7 +2447,8 @@ def private load_gguf_parsed(var t : Model; m : GGUFMeta; bytes : array | t.kq_repack_mr40 = active_kq_layout_mr(40) t.kq_repack_mr44 = active_kq_layout_mr(44) t.kq_repack_mr3 = active_kq_layout_mr(3) - to_log(LOG_INFO, "dasLLAMA: K-quant planes repacked to the '{active_kernel_backend()}' layouts (k4 grp{t.kq_repack_mr4}, k5 grp{t.kq_repack_mr5}, k6 grp{t.kq_repack_mr6}, q40 grp{t.kq_repack_mr40}, iq4xs grp{t.kq_repack_mr44}, k3 grp{t.kq_repack_mr3})\n") + t.kq_repack_mr33 = active_kq_layout_mr(33) + to_log(LOG_INFO, "dasLLAMA: K-quant planes repacked to the '{active_kernel_backend()}' layouts (k4 grp{t.kq_repack_mr4}, k5 grp{t.kq_repack_mr5}, k6 grp{t.kq_repack_mr6}, q40 grp{t.kq_repack_mr40}, iq4xs grp{t.kq_repack_mr44}, k3 grp{t.kq_repack_mr3}, iq3s grp{t.kq_repack_mr33})\n") } } } diff --git a/modules/dasLLAMA/dasllama/dasllama_math.das b/modules/dasLLAMA/dasllama/dasllama_math.das index 58b3699f84..b3dd2d930c 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math.das +++ b/modules/dasLLAMA/dasllama/dasllama_math.das @@ -1086,6 +1086,7 @@ struct KernelBackend { kq_rows_q40 : MatmulKqRowsFn = @@kq_unset_rows kq_rows_iq4xs : MatmulKqRowsFn = @@kq_unset_rows kq_rows_k3 : MatmulKqRowsFn = @@kq_unset_rows + kq_rows_iq3s : MatmulKqRowsFn = @@kq_unset_rows kq_batch : MatmulKqBatchFn = @@kq_unset_batch kq_groupn : MatmulKqGroupNFn = @@kq_unset_groupn kq_batch_groupn : MatmulKqBatchGroupNFn = @@kq_unset_batch_groupn @@ -1150,6 +1151,7 @@ var g_kq_rows_k6 = @@kq_unset_rows var g_kq_rows_q40 = @@kq_unset_rows var g_kq_rows_iq4xs = @@kq_unset_rows var g_kq_rows_k3 = @@kq_unset_rows +var g_kq_rows_iq3s = @@kq_unset_rows var g_mm_kq_batch = @@kq_unset_batch var g_mm_kq_groupn = @@kq_unset_groupn var g_mm_kq_batch_groupn = @@kq_unset_batch_groupn @@ -1477,7 +1479,7 @@ def active_q8_layout_mr() : int64 => g_bake_cpu_override ? g_bake_cpu.q8_mr : g_ //! family split; the loader freezes these onto the Model at repack time. def active_kq_layout_mr(fmt : int) : int64 { if (g_bake_cpu_override) { - return fmt == 4 ? g_bake_cpu.kq_mr4 : (fmt == 5 ? g_bake_cpu.kq_mr5 : (fmt == 40 ? g_bake_cpu.kq_mr40 : (fmt == 44 ? g_bake_cpu.kq_mr44 : (fmt == 3 ? g_bake_cpu.kq_mr3 : g_bake_cpu.kq_mr6)))) + return fmt == 4 ? g_bake_cpu.kq_mr4 : (fmt == 5 ? g_bake_cpu.kq_mr5 : (fmt == 40 ? g_bake_cpu.kq_mr40 : (fmt == 44 ? g_bake_cpu.kq_mr44 : (fmt == 3 ? g_bake_cpu.kq_mr3 : (fmt == 33 ? g_bake_cpu.kq_mr33 : g_bake_cpu.kq_mr6))))) } return int64(invoke(g_active_kq_layout, fmt)) } @@ -1655,6 +1657,7 @@ def private activate(be : KernelBackend) { g_kq_rows_q40 = be.kq_rows_q40 g_kq_rows_iq4xs = be.kq_rows_iq4xs g_kq_rows_k3 = be.kq_rows_k3 + g_kq_rows_iq3s = be.kq_rows_iq3s g_mm_kq_batch = be.kq_batch g_mm_kq_groupn = be.kq_groupn g_mm_kq_batch_groupn = be.kq_batch_groupn @@ -1859,6 +1862,7 @@ def kq_rows_fn(fmt : int) : MatmulKqRowsFn { if (fmt == 40) return g_kq_rows_q40 if (fmt == 44) return g_kq_rows_iq4xs if (fmt == 3) return g_kq_rows_k3 + if (fmt == 33) return g_kq_rows_iq3s return @@kq_unset_rows } @@ -3225,6 +3229,9 @@ def private restore_math_fn_defaults { // nolint:STYLE037,STYLE038 — flat seam if (g_kq_rows_k3 == null) { g_kq_rows_k3 = @@kq_unset_rows } + if (g_kq_rows_iq3s == null) { + g_kq_rows_iq3s = @@kq_unset_rows + } if (g_mm_kq_batch == null) { g_mm_kq_batch = @@kq_unset_batch } diff --git a/modules/dasLLAMA/dasllama/dasllama_math_default.das b/modules/dasLLAMA/dasllama/dasllama_math_default.das index b1fbb8e218..368dfd4c2a 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_default.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_default.das @@ -616,6 +616,42 @@ def dot_k3q8(kqrow : uint8 const?; ksrow : uint8 const?; xqp : int8 const?; xsp return acc } +//! IQ3_S row dot off the DISK-ORDER planes: per 32-block, grid entry qs[b*8 + 2l + half] | +//! (qh[b] bit << 8) yields four magnitudes, the block's sign bytes flip them, sub-scale +//! (1 + 2s) folds per block; signed weights, so no activation block-sum term. +[unused_argument(xbsp), hint(unsafe_range_check, noalias = kqrow, noalias = ksrow, noalias = xqp, noalias = xsp, noalias = xbsp)] +def dot_iq3s_q8(kqrow : uint8 const?; ksrow : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n : int64) : float { + var acc = 0.0 + let grid = iq3s_grid() + unsafe { + for (sb in range64(n / 256l)) { + let kqo = sb * 104l + let kso = sb * 20l + let d = f16_to_f32(uint(ksrow[kso]) | (uint(ksrow[kso + 1l]) << 8u)) + var isum = 0 + for (blk in range64(8l)) { + let qh = int(kqrow[kqo + 64l + blk]) + let ab = (sb * 8l + blk) * 32l + var ib = 0 + for (l in range64(4l)) { + let sgn = int(kqrow[kqo + 72l + blk * 4l + l]) + let g1 = grid[int(kqrow[kqo + blk * 8l + 2l * l]) | ((qh << int(8l - 2l * l)) & 256)] + let g2 = grid[int(kqrow[kqo + blk * 8l + 2l * l + 1l]) | ((qh << int(7l - 2l * l)) & 256)] + for (j in range64(4l)) { + let w1 = int((g1 >> uint(8l * j)) & 0xFFu) + let w2 = int((g2 >> uint(8l * j)) & 0xFFu) + ib += (((sgn >> int(j)) & 1) != 0 ? -w1 : w1) * int(xqp[ab + l * 8l + j]) + ib += (((sgn >> int(4l + j)) & 1) != 0 ? -w2 : w2) * int(xqp[ab + l * 8l + 4l + j]) + } + } + isum += int(int8(ksrow[kso + 4l + blk])) * ib + } + acc += float(isum) * (d * xsp[sb]) + } + } + return acc +} + // Row-range cores + full GEMVs per format. Not `private`: invoked through hoisted function // pointers from lifted worker lambdas (the fused chains) and the dispatch wrappers in common. def k4_rows_kernel(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, rb, re : int64) { @@ -672,6 +708,15 @@ def k3_rows_kernel(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp } } +def iq3s_rows_kernel(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, rb, re : int64) { + let nsb = n / 256l + unsafe { + for (i in range64(rb, re)) { + yp[i] = dot_iq3s_q8(kqp + i * nsb * 104l, ksp + i * nsb * 20l, xqp, xsp, xbsp, n) + } + } +} + def private kq_gemv_kernel(fmt : int; var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d : int64) { var myp = yp maybe_parallel_for(0, int(d), matmul_chunks_gemv(int(d), 1, n * d)) $(rb, re) { @@ -685,6 +730,8 @@ def private kq_gemv_kernel(fmt : int; var yp : float?; kqp : uint8 const?; ksp : iq4xs_rows_kernel(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb), int64(re)) } elif (fmt == 3) { k3_rows_kernel(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb), int64(re)) + } elif (fmt == 33) { + iq3s_rows_kernel(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb), int64(re)) } else { k6_rows_kernel(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb), int64(re)) } @@ -762,6 +809,37 @@ def private dequant_k3_row_grp(kqg : uint8 const?; ksg : uint8 const?; r, mr, ns } } +//! iq3s grp row dequant: 26 uniform 4-byte columns per superblock row ([qs][qh][signs] in +//! place), the k4-shaped scale interleave — w = (d * (1 + 2s)) * grid_byte * sign. +def private dequant_iq3s_row_grp(kqg : uint8 const?; ksg : uint8 const?; r, mr, nsb : int64; var dst : float?) { + let grid = iq3s_grid() + unsafe { + for (sbi in range64(nsb)) { + let qb = sbi * 104l * mr + let sb = sbi * 20l * mr + let d = f16_to_f32(uint(ksg[sb + 2l * r]) | (uint(ksg[sb + 2l * r + 1l]) << 8u)) + for (blk in range64(8l)) { + let db = d * float(int(int8(ksg[sb + 4l * mr + blk * mr + r]))) + let hp = 64l + blk + let qh = int(kqg[qb + ((hp / 4l) * mr + r) * 4l + hp % 4l]) + for (l in range64(4l)) { + let sp = 72l + blk * 4l + l + let sgn = int(kqg[qb + ((sp / 4l) * mr + r) * 4l + sp % 4l]) + let q1p = blk * 8l + 2l * l + let q2p = q1p + 1l + let g1 = grid[int(kqg[qb + ((q1p / 4l) * mr + r) * 4l + q1p % 4l]) | ((qh << int(8l - 2l * l)) & 256)] + let g2 = grid[int(kqg[qb + ((q2p / 4l) * mr + r) * 4l + q2p % 4l]) | ((qh << int(7l - 2l * l)) & 256)] + let kb = sbi * 256l + blk * 32l + l * 8l + for (j in range64(4l)) { + dst[kb + j] = db * float((g1 >> uint(8l * j)) & 0xFFu) * (((sgn >> int(j)) & 1) != 0 ? -1.0 : 1.0) + dst[kb + 4l + j] = db * float((g2 >> uint(8l * j)) & 0xFFu) * (((sgn >> int(4l + j)) & 1) != 0 ? -1.0 : 1.0) + } + } + } + } + } +} + //! Dequant one row off the grp-REPACKED K-quant planes (repack_k4/k5/k6_grp layout). //! kqg/ksg = row's GROUP plane base (group g = row/mr); r = row % mr. //! embed_row's kq_repacked path — per-token, so scalar is fine. @@ -781,6 +859,10 @@ def dequant_kq_row_grp(fmt : int64; kqg : uint8 const?; ksg : uint8 const?; r, m dequant_k3_row_grp(kqg, ksg, r, mr, nsb, dst) return } + if (fmt == 33l) { + dequant_iq3s_row_grp(kqg, ksg, r, mr, nsb, dst) + return + } unsafe { for (sbi in range64(nsb)) { let qb = sbi * qsb * mr @@ -879,6 +961,8 @@ def matmul_kq_groupn(fmt : int; var y : array | #; kq : array | #; myp[ii] = dot_iq4xs_q8(kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) } elif (fmt == 3) { myp[ii] = dot_k3q8(kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) + } elif (fmt == 33) { + myp[ii] = dot_iq3s_q8(kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) } else { myp[ii] = dot_k6q8(kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) } diff --git a/modules/dasLLAMA/dasllama/dasllama_math_gen.das b/modules/dasLLAMA/dasllama/dasllama_math_gen.das index f3a706a3dd..a0fe82abcc 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_gen.das @@ -251,6 +251,10 @@ def k3q8_layout_gen() : int { return q8q8_repack_type(GEMM_REFERENCE_MR).interleave } +def iq3sq8_layout_gen() : int { + return q8q8_repack_type(GEMM_REFERENCE_MR).interleave +} + // the KernelBackend.kq_layout slot form of kq_layout_of (registered on both gen backends) def private kq_layout_fmt_gen(fmt : int) : int => int(kq_layout_of(fmt)) @@ -272,6 +276,9 @@ def kq_layout_of(fmt : int) : int64 { if (fmt == 3) { return int64(k3q8_layout_gen()) } + if (fmt == 33) { + return int64(iq3sq8_layout_gen()) + } return int64(k6q8_layout_gen()) } @@ -382,6 +389,46 @@ def k3_grp_row_dot(kqg : uint8 const?; ksg : uint8 const?; r, mr : int64; xqp : return acc } +//! One row's dot off the grp iq3s planes, scalar — the iq3s stubs' reference body and the +//! repack oracle: the 26-column layout's per-block grid gather + sign flip, sub-scale (1 + 2s) +//! per block, no activation-sum term. Same integer regroup and float fold as dot_iq3s_q8. +[unused_argument(xbsp)] +def iq3s_grp_row_dot(kqg : uint8 const?; ksg : uint8 const?; r, mr : int64; xqp : int8 const?; xsp : float const?; xbsp : int const?; n : int64) : float { + var acc = 0.0 + let nsb = n / 256l + let grid = iq3s_grid() + unsafe { + for (sbi in range64(nsb)) { + let qb = sbi * 104l * mr + let sb = sbi * 20l * mr + var isum = 0 + for (blk in range64(8l)) { + let hp = 64l + blk + let qh = int(kqg[qb + ((hp / 4l) * mr + r) * 4l + hp % 4l]) + var ib = 0 + for (l in range64(4l)) { + let sp = 72l + blk * 4l + l + let sgn = int(kqg[qb + ((sp / 4l) * mr + r) * 4l + sp % 4l]) + let q1p = blk * 8l + 2l * l + let q2p = q1p + 1l + let g1 = grid[int(kqg[qb + ((q1p / 4l) * mr + r) * 4l + q1p % 4l]) | ((qh << int(8l - 2l * l)) & 256)] + let g2 = grid[int(kqg[qb + ((q2p / 4l) * mr + r) * 4l + q2p % 4l]) | ((qh << int(7l - 2l * l)) & 256)] + for (j in range64(4l)) { + let w1 = int((g1 >> uint(8l * j)) & 0xFFu) + let w2 = int((g2 >> uint(8l * j)) & 0xFFu) + ib += (((sgn >> int(j)) & 1) != 0 ? -w1 : w1) * int(xqp[(sbi * 8l + blk) * 32l + l * 8l + j]) + ib += (((sgn >> int(4l + j)) & 1) != 0 ? -w2 : w2) * int(xqp[(sbi * 8l + blk) * 32l + l * 8l + 4l + j]) + } + } + isum += int(int8(ksg[sb + 4l * mr + blk * mr + r])) * ib + } + let dk = f16_to_f32(uint(ksg[sb + 2l * r]) | (uint(ksg[sb + 2l * r + 1l]) << 8u)) + acc += float(isum) * (dk * xsp[sbi]) + } + } + return acc +} + //! The K-quant GEMV kernels (kq stage 4): rows [rb, re) of one plane-region pair off the grp //! kq planes (each format's OWN layout companion). Reference bodies = the scalar grp walk above, //! declining in lockstep. Not `private`: they ARE the kq_rows_* slots. @@ -584,6 +631,38 @@ def k3q8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp : } } +[hint(unsafe_range_check, noalias = kqp, noalias = ksp, noalias = xqp, noalias = xsp, noalias = xbsp)] +def iq3sq8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, rb, re : int64) : void { + let mr = int64(iq3sq8_layout_gen()) + let nsb = n / 256l + unsafe { + for (i in range64(rb, re)) { + let g = i / mr + yp[i] = iq3s_grp_row_dot(kqp + g * mr * nsb * 104l, ksp + g * mr * nsb * 20l, i % mr, mr, xqp, xsp, xbsp, n) + } + } +} + +[tune_perm(mr = 4), tune_perm(mr = 4, nrsplit = 2), tune_perm(mr = 8), tune_perm(mr = 8, nrsplit = 2), + tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "maddubs", width = 256, mr = 8, nrsplit = 2, requires = "avx2"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, nrsplit = 2, requires = "avxvnni|avx512vnni"), + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, nrsplit = 2, requires = "avx512vnni,avx512bw"), + tune_companion(fn = "iq3sq8_gemv_gen", gen = "dasllama_gemm_gen::iq3s_gemv"), + tune_companion(fn = "iq3sq8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), + tune(gen = "dasllama_gemm_gen::iq3s_tile", + fallback = "dot_vpdpbusd_width512_mr16;dot_vpdpbusd_width256_mr8;dot_maddubs_width256_mr8;mr4"), + hint(unsafe_range_check, noalias = kqg, noalias = ksg, noalias = xqp, noalias = xsp, noalias = xbsp)] +def iq3sq8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d, g, t0 : int64) : void { + let mr = int64(iq3sq8_layout_gen()) + unsafe { + for (t in range64(4l)) { + for (r in range64(mr)) { + yp[(t0 + t) * d + g * mr + r] = iq3s_grp_row_dot(kqg, ksg, r, mr, xqp + (t0 + t) * n, xsp + (t0 + t) * (n / 256l), xbsp + (t0 + t) * (n / 16l), n) + } + } + } +} + //! The f16-scale GEMV companion (wscale_f16 rail): q8q8_gemv_gen's twin over the raw binary16 //! group-scale plane. Same rows-range contract; the generated body differs only in the scale //! fold's widen (fcvtl/vcvtph2ps). Not `private`: it IS the mm_rows_s16 slot. @@ -905,6 +984,8 @@ def private repack_kq_gen(fmt : int; var kq : uint8?; var ks : uint8?; n, d : in repack_iq4xs_grp(kq, ks, n, d, kq_layout_of(44)) } elif (fmt == 3) { repack_k3_grp(kq, ks, n, d, kq_layout_of(3)) + } elif (fmt == 33) { + repack_iq3s_grp(kq, ks, n, d, kq_layout_of(33)) } else { repack_k6_grp(kq, ks, n, d, kq_layout_of(6)) } @@ -921,6 +1002,8 @@ def private repack_kq_bake(fmt : int; var kq : uint8?; var ks : uint8?; n, d : i repack_iq4xs_grp(kq, ks, n, d, active_kq_layout_mr(44)) } elif (fmt == 3) { repack_k3_grp(kq, ks, n, d, active_kq_layout_mr(3)) + } elif (fmt == 33) { + repack_iq3s_grp(kq, ks, n, d, active_kq_layout_mr(33)) } else { repack_k6_grp(kq, ks, n, d, active_kq_layout_mr(6)) } @@ -1435,6 +1518,8 @@ def private kq_kernel_gen(fmt : int; var yp : float?; kqp : uint8 const?; ksp : iq4xsq8_gemv_gen(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb) * mr, int64(re) * mr) } elif (fmt == 3) { k3q8_gemv_gen(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb) * mr, int64(re) * mr) + } elif (fmt == 33) { + iq3sq8_gemv_gen(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb) * mr, int64(re) * mr) } else { k6q8_gemv_gen(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb) * mr, int64(re) * mr) } @@ -1450,7 +1535,9 @@ def private kq_kernel_gen(fmt : int; var yp : float?; kqp : uint8 const?; ksp : } elif (fmt == 44) { myp[i] = dot_iq4xs_q8(kqp + i * nsb * 128l, ksp + i * nsb * 20l, xqp, xsp, xbsp, n) } elif (fmt == 3) { - myp[i] = dot_k3q8(kqp + i * nsb * 128l, ksp + i * nsb * 20l, xqp, xsp, xbsp, n) + myp[i] = dot_k3q8(kqp + i * nsb * 96l, ksp + i * nsb * 18l, xqp, xsp, xbsp, n) + } elif (fmt == 33) { + myp[i] = dot_iq3s_q8(kqp + i * nsb * 104l, ksp + i * nsb * 20l, xqp, xsp, xbsp, n) } else { myp[i] = dot_k6q8(kqp + i * nsb * 192l, ksp + i * nsb * 18l, xqp, xsp, xbsp, n) } @@ -1466,7 +1553,7 @@ def private kq_batch_cell_gen(fmt : int; var myp : float?; kqp : uint8 const?; k let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) let nb16 = n / 16l - let packed = fmt == 4 || fmt == 40 || fmt == 44 || fmt == 3 // packed planes: the tile reads them directly + let packed = fmt == 4 || fmt == 40 || fmt == 44 || fmt == 3 || fmt == 33 // packed planes: the tile reads them directly var scratch : array if (!packed) { scratch |> resize(mr * nsb * 256l) @@ -1496,6 +1583,8 @@ def private kq_batch_cell_gen(fmt : int; var myp : float?; kqp : uint8 const?; k iq4xsq8_tile_gen(myp, kqg, ksg, xqp, xsp, xbsp, n, d, g, tk) } elif (fmt == 3) { k3q8_tile_gen(myp, kqg, ksg, xqp, xsp, xbsp, n, d, g, tk) + } elif (fmt == 33) { + iq3sq8_tile_gen(myp, kqg, ksg, xqp, xsp, xbsp, n, d, g, tk) } else { k6q8_tile_gen(myp, scp, ksg, xqp, xsp, xbsp, n, d, g, tk) } @@ -1512,6 +1601,8 @@ def private kq_batch_cell_gen(fmt : int; var myp : float?; kqp : uint8 const?; k iq4xsq8_gemv_gen(myp + tk * d, kqp, ksp, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n, g * mr, (g + 1l) * mr) } elif (fmt == 3) { k3q8_gemv_gen(myp + tk * d, kqp, ksp, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n, g * mr, (g + 1l) * mr) + } elif (fmt == 33) { + iq3sq8_gemv_gen(myp + tk * d, kqp, ksp, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n, g * mr, (g + 1l) * mr) } else { k6q8_gemv_gen(myp + tk * d, kqp, ksp, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n, g * mr, (g + 1l) * mr) } @@ -1548,7 +1639,9 @@ def private kq_batch_kernel_gen(fmt : int; var yp : float?; kqp : uint8 const?; } elif (fmt == 44) { myp[tk * d + i] = dot_iq4xs_q8(kqp + i * nsb * 128l, ksp + i * nsb * 20l, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * (n / 16l), n) } elif (fmt == 3) { - myp[tk * d + i] = dot_k3q8(kqp + i * nsb * 128l, ksp + i * nsb * 20l, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * (n / 16l), n) + myp[tk * d + i] = dot_k3q8(kqp + i * nsb * 96l, ksp + i * nsb * 18l, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * (n / 16l), n) + } elif (fmt == 33) { + myp[tk * d + i] = dot_iq3s_q8(kqp + i * nsb * 104l, ksp + i * nsb * 20l, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * (n / 16l), n) } else { myp[tk * d + i] = dot_k6q8(kqp + i * nsb * 192l, ksp + i * nsb * 18l, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * (n / 16l), n) } @@ -1601,6 +1694,8 @@ def private kq_batch_groupn_gen(fmt : int; var yp : float?; kqp : uint8 const?; myp[tk * d + i] = dot_iq4xs_q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n) } elif (fmt == 3) { myp[tk * d + i] = dot_k3q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n) + } elif (fmt == 33) { + myp[tk * d + i] = dot_iq3s_q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n) } else { myp[tk * d + i] = dot_k6q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n) } @@ -1762,6 +1857,8 @@ def kq_groupn_gen(fmt : int; var yp : float?; kqp : uint8 const?; ksp : uint8 co iq4xsq8_gemv_gen(myp + r * d, kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n, gLocal * mr, gEnd * mr) } elif (fmt == 3) { k3q8_gemv_gen(myp + r * d, kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n, gLocal * mr, gEnd * mr) + } elif (fmt == 33) { + iq3sq8_gemv_gen(myp + r * d, kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n, gLocal * mr, gEnd * mr) } else { k6q8_gemv_gen(myp + r * d, kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n, gLocal * mr, gEnd * mr) } @@ -1784,6 +1881,8 @@ def kq_groupn_gen(fmt : int; var yp : float?; kqp : uint8 const?; ksp : uint8 co myp[r * d + i] = dot_iq4xs_q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) } elif (fmt == 3) { myp[r * d + i] = dot_k3q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) + } elif (fmt == 33) { + myp[r * d + i] = dot_iq3s_q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) } else { myp[r * d + i] = dot_k6q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) } @@ -2077,7 +2176,7 @@ def dasllama_math_gen_register() { // the no-witness stance above covers kq too mm_kq = @@kq_kernel_gen, repack_kq = @@repack_kq_gen, kq_rows_k4 = @@k4q8_gemv_gen, kq_rows_k5 = @@k5q8_gemv_gen, - kq_rows_k6 = @@k6q8_gemv_gen, kq_rows_q40 = @@q40q8_gemv_gen, kq_rows_iq4xs = @@iq4xsq8_gemv_gen, kq_rows_k3 = @@k3q8_gemv_gen, + kq_rows_k6 = @@k6q8_gemv_gen, kq_rows_q40 = @@q40q8_gemv_gen, kq_rows_iq4xs = @@iq4xsq8_gemv_gen, kq_rows_k3 = @@k3q8_gemv_gen, kq_rows_iq3s = @@iq3sq8_gemv_gen, kq_batch = @@kq_batch_kernel_gen, kq_groupn = @@kq_groupn_gen, kq_batch_groupn = @@kq_batch_groupn_gen, // q51 family: tile + GEMV off the grp planes (the tile family's companions stamp @@ -2106,7 +2205,7 @@ def dasllama_math_gen_register() { mm_rows_s16 = @@q8q8_gemv_s16_gen, mm_kq = @@kq_kernel_gen, repack_kq = @@repack_kq_gen, kq_rows_k4 = @@k4q8_gemv_gen, kq_rows_k5 = @@k5q8_gemv_gen, - kq_rows_k6 = @@k6q8_gemv_gen, kq_rows_q40 = @@q40q8_gemv_gen, kq_rows_iq4xs = @@iq4xsq8_gemv_gen, kq_rows_k3 = @@k3q8_gemv_gen, + kq_rows_k6 = @@k6q8_gemv_gen, kq_rows_q40 = @@q40q8_gemv_gen, kq_rows_iq4xs = @@iq4xsq8_gemv_gen, kq_rows_k3 = @@k3q8_gemv_gen, kq_rows_iq3s = @@iq3sq8_gemv_gen, kq_batch = @@kq_batch_kernel_gen, kq_groupn = @@kq_groupn_gen, kq_batch_groupn = @@kq_batch_groupn_gen, groupn_q51 = @@q51q8_groupn_gen, repack_q51 = @@repack_q51_gen, diff --git a/modules/dasLLAMA/dasllama/dasllama_ple.das b/modules/dasLLAMA/dasllama/dasllama_ple.das index 576ff289fb..4326c00d79 100644 --- a/modules/dasLLAMA/dasllama/dasllama_ple.das +++ b/modules/dasLLAMA/dasllama/dasllama_ple.das @@ -66,6 +66,8 @@ def ple_check_table(t : Model; origin : string) { have = (long_length(t.iq4xsq) / IQ4XS_QSB) * 256l } elif (t.ple_emb_fmt == KqFmt.k3) { have = (long_length(t.k3q) / K3_QSB) * 256l + } elif (t.ple_emb_fmt == KqFmt.iq3s) { + have = (long_length(t.iq3sq) / IQ3S_QSB) * 256l } if (have < need) { panic("dasLLAMA: the PLE token table is not resident ({origin}): fmt {t.ple_emb_fmt} plane holds {have} of {need} elements - regenerate this carrier (it predates the pinned-table rail)") @@ -103,6 +105,8 @@ def private ple_gather_row(t : Model; row : int64; var dst : array; doff dequant_iq4xs_plane_superblock(t.iq4xsq, sb * IQ4XS_QSB, t.iq4xss, sb * IQ4XS_SSB, dst, doff + blk * 256l) } elif (t.ple_emb_fmt == KqFmt.k3) { dequant_k3_plane_superblock(t.k3q, sb * K3_QSB, t.k3s, sb * K3_SSB, dst, doff + blk * 256l) + } elif (t.ple_emb_fmt == KqFmt.iq3s) { + dequant_iq3s_plane_superblock(t.iq3sq, sb * IQ3S_QSB, t.iq3ss, sb * IQ3S_SSB, dst, doff + blk * 256l) } else { panic("ple_gather_row: '{t.ple_emb_fmt}' has no kq plane pair") } diff --git a/modules/dasLLAMA/dasllama/dasllama_repack.das b/modules/dasLLAMA/dasllama/dasllama_repack.das index 9342856de1..fe0de23c4b 100644 --- a/modules/dasLLAMA/dasllama/dasllama_repack.das +++ b/modules/dasLLAMA/dasllama/dasllama_repack.das @@ -465,6 +465,51 @@ def repack_k3_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { delete ts } +//! iq3s grp layout: the whole 104B row as 26 four-byte columns x mr rows — qs, qh and sign +//! bytes stay in place per column (the kernels compose per block); scales interleave as k4's +//! row ([d f16 x mr][pad x mr][8 sc x mr][8 pad x mr]). +def repack_iq3s_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { + let nsb = n / 256l + let qrow = nsb * 104l + let srow = nsb * 20l + let ng = d / mr + var tq : array + var ts : array + tq |> resize(d * qrow) + ts |> resize(d * srow) + unsafe { + var tqp = addr(tq[0]) + var tsp = addr(ts[0]) + memcpy(tqp, kq, d * qrow) + memcpy(tsp, ks, d * srow) + for (g in range64(ng)) { + for (sbi in range64(nsb)) { + let dq = g * mr * qrow + sbi * 104l * mr + let ds = g * mr * srow + sbi * 20l * mr + for (r in range64(mr)) { + let sq = (g * mr + r) * qrow + sbi * 104l + let ss = (g * mr + r) * srow + sbi * 20l + for (c in range64(26l)) { + for (t in range64(4l)) { + kq[dq + (c * mr + r) * 4l + t] = tqp[sq + c * 4l + t] + } + } + ks[ds + 2l * r] = tsp[ss] + ks[ds + 2l * r + 1l] = tsp[ss + 1l] + ks[ds + 2l * mr + 2l * r] = tsp[ss + 2l] + ks[ds + 2l * mr + 2l * r + 1l] = tsp[ss + 3l] + for (b in range64(8l)) { + ks[ds + 4l * mr + b * mr + r] = tsp[ss + 4l + b] + ks[ds + 12l * mr + b * mr + r] = tsp[ss + 12l + b] + } + } + } + } + } + delete tq + delete ts +} + // ===== grp panel unpack (the byte-expanded tile form the kq stamps load) ===== //! Unpack ONE group's packed grp k5/k6 quant panel into the BYTE-EXPANDED tile form (wlo/whi diff --git a/modules/dasLLAMA/harness/gen_tune_probe.das b/modules/dasLLAMA/harness/gen_tune_probe.das index 28b20f1053..69604d2955 100644 --- a/modules/dasLLAMA/harness/gen_tune_probe.das +++ b/modules/dasLLAMA/harness/gen_tune_probe.das @@ -441,7 +441,7 @@ def kq_qsb(fmt : int64) : int64 => dasllama_gemm_schema::kq_qsb(int(fmt)) def kq_ssb(fmt : int64) : int64 => dasllama_gemm_schema::kq_ssb(int(fmt)) // the family's sidecar entry = its [tune] function name (the K-quants by bit width - k3 included - q40/iq4xs by tag) -def kq_tile_entry(fmt : int64) : string => fmt == 40l ? "q40q8_tile_gen" : (fmt == 44l ? "iq4xsq8_tile_gen" : "k{fmt}q8_tile_gen") +def kq_tile_entry(fmt : int64) : string => fmt == 40l ? "q40q8_tile_gen" : (fmt == 44l ? "iq4xsq8_tile_gen" : (fmt == 33l ? "iq3sq8_tile_gen" : "k{fmt}q8_tile_gen")) // synthetic disk superblocks, varied per (row, superblock) so no two blocks repeat: any byte // is a legal quant, and the small d/dmin keep the f16 (s, o) pairs finite and row sums tame @@ -463,7 +463,7 @@ def pack_kq_scale_header(var blkb : array; base : int) { } } -def pack_kq_disk_block(fmt : int64; var blkb : array; base : int) { +def pack_kq_disk_block(fmt : int64; var blkb : array; base : int) { // nolint:STYLE037 — the flat per-format ladder (one arm per kq format) is the honest shape if (fmt == 40l) { for (blk in range(8)) { // 8 x 18B q4_0 disk blocks: f16 d + 16 nibble bytes let dbits = f32_to_f16(0.002 + 0.0001 * float((base + blk) % 13)) @@ -497,6 +497,17 @@ def pack_kq_disk_block(fmt : int64; var blkb : array; base : int) { let dbits = f32_to_f16(0.002 + 0.0001 * float(base % 13)) blkb[108] = uint8(dbits & 0xFF) blkb[109] = uint8(dbits >> 8u) + } elif (fmt == 33l) { + // 110B IQ3_S disk block: f16 d, 64 grid-index qs, 8 qh, 32 signs, 4 packed 4-bit scales — every byte pattern is legal + let dbits = f32_to_f16(0.002 + 0.0001 * float(base % 13)) + blkb[0] = uint8(dbits & 0xFF) + blkb[1] = uint8(dbits >> 8u) + for (i in range(104)) { + blkb[2 + i] = uint8((base * 37 + i * 101 + 29) % 256) + } + for (i in range(4)) { + blkb[106 + i] = uint8((base * 29 + i * 71 + 5) % 256) + } } elif (fmt == 4l) { pack_kq_scale_header(blkb, base) for (i in range(128)) { @@ -537,19 +548,21 @@ def repack_kq_grp_fmt(fmt : int64; var kq : uint8?; var ks : uint8?; n, d, mr : repack_iq4xs_grp(kq, ks, n, d, mr) } elif (fmt == 3l) { repack_k3_grp(kq, ks, n, d, mr) + } elif (fmt == 33l) { + repack_iq3s_grp(kq, ks, n, d, mr) } else { repack_k6_grp(kq, ks, n, d, mr) } } -def build_kq_fixture(fmt : int64; n, d, ntok : int64) : KqFixture { +def build_kq_fixture(fmt : int64; n, d, ntok : int64) : KqFixture { // nolint:STYLE037 — the flat per-format ladder (one arm per kq format) is the honest shape var fx = KqFixture(fmt = fmt, n = n, d = d, ntok = ntok, nsb = n / 256l) let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) fx.kq |> resize(d * fx.nsb * qsb) fx.ks |> resize(d * fx.nsb * ssb) var blkb : array - blkb |> resize(fmt == 4l || fmt == 40l ? 144l : (fmt == 5l ? 176l : (fmt == 44l ? 136l : (fmt == 3l ? 110l : 210l)))) + blkb |> resize(fmt == 4l || fmt == 40l ? 144l : (fmt == 5l ? 176l : (fmt == 44l ? 136l : (fmt == 3l || fmt == 33l ? 110l : 210l)))) for (r in range64(d)) { for (s in range64(fx.nsb)) { let base = int(r * 31l + s * 7l) @@ -565,6 +578,8 @@ def build_kq_fixture(fmt : int64; n, d, ntok : int64) : KqFixture { transcode_iq4xs_superblock(blkb, 0l, fx.kq, sb * qsb, fx.ks, sb * ssb) } elif (fmt == 3l) { transcode_q3k_superblock(blkb, 0l, fx.kq, sb * qsb, fx.ks, sb * ssb) + } elif (fmt == 33l) { + transcode_iq3s_superblock(blkb, 0l, fx.kq, sb * qsb, fx.ks, sb * ssb) } else { transcode_q6k_superblock(blkb, 0l, fx.kq, sb * qsb, fx.ks, sb * ssb) } @@ -605,8 +620,11 @@ def build_kq_fixture(fmt : int64; n, d, ntok : int64) : KqFixture { : (fmt == 3l ? k3_grp_row_dot(kqg, ksg, r, 4l, addr(fx.xq[0]) + tk * n, addr(fx.xs[0]) + tk * (n / 256l), addr(fx.xbs[0]) + tk * (n / 16l), n) - : kq_grp_row_dot(fmt, kqg, ksg, r, 4l, - addr(fx.xq[0]) + tk * n, addr(fx.xs[0]) + tk * (n / 256l), addr(fx.xbs[0]) + tk * (n / 16l), n)))) + : (fmt == 33l + ? iq3s_grp_row_dot(kqg, ksg, r, 4l, + addr(fx.xq[0]) + tk * n, addr(fx.xs[0]) + tk * (n / 256l), addr(fx.xbs[0]) + tk * (n / 16l), n) + : kq_grp_row_dot(fmt, kqg, ksg, r, 4l, + addr(fx.xq[0]) + tk * n, addr(fx.xs[0]) + tk * (n / 256l), addr(fx.xbs[0]) + tk * (n / 16l), n))))) } } } @@ -634,6 +652,9 @@ def kq_tile_variants(fmt : int64) : array> { if (fmt == 3l) { return <- k3q8_tile_gen_variants() } + if (fmt == 33l) { + return <- iq3sq8_tile_gen_variants() + } return <- k6q8_tile_gen_variants() } @@ -669,6 +690,12 @@ def kq_gemv_variants_by_suffix(fmt : int64) : table { tab[v._0] = v._1 } delete gvs + } elif (fmt == 33l) { + var gvs <- iq3sq8_gemv_gen_variants() + for (v in gvs) { + tab[v._0] = v._1 + } + delete gvs } else { var gvs <- k6q8_gemv_gen_variants() for (v in gvs) { @@ -713,6 +740,12 @@ def kq_layout_mrs(fmt : int64) : table { mrs[v._0] = invoke(v._1) } delete lvs + } elif (fmt == 33l) { + var lvs <- iq3sq8_layout_gen_variants() + for (v in lvs) { + mrs[v._0] = invoke(v._1) + } + delete lvs } else { var lvs <- k6q8_layout_gen_variants() for (v in lvs) { @@ -730,7 +763,7 @@ def kq_layout_mrs(fmt : int64) : table { def run_kq_tile(tile; fx : KqFixture; kq : array; ks : array; mr : int64; var y : array) { let ng = fx.d / mr verify(fx.ntok % 4l == 0l) - let packed = fx.fmt == 4l || fx.fmt == 40l || fx.fmt == 44l || fx.fmt == 3l // packed planes: the tile reads them directly + let packed = fx.fmt == 4l || fx.fmt == 40l || fx.fmt == 44l || fx.fmt == 3l || fx.fmt == 33l // packed planes: the tile reads them directly var panel : array if (!packed) { panel |> resize(int(mr * fx.nsb * 256l)) @@ -1797,7 +1830,7 @@ def tune_mode_run : bool { // nolint:STYLE037,STYLE038 — flat family-by-family print("aborting mid-run - no sidecar written\n") return false } - for (fmt in fixed_array(4l, 5l, 6l, 40l, 44l, 3l)) { + for (fmt in fixed_array(4l, 5l, 6l, 40l, 44l, 3l, 33l)) { interrupt_gate("k{fmt}q8_tile_gen") let kq_t0 = ref_time_ticks() let entry = kq_tile_entry(fmt) diff --git a/modules/dasLLAMA/tests/test_kqformat.das b/modules/dasLLAMA/tests/test_kqformat.das index 540d96d5a7..3fa38d5f70 100644 --- a/modules/dasLLAMA/tests/test_kqformat.das +++ b/modules/dasLLAMA/tests/test_kqformat.das @@ -22,6 +22,7 @@ def test_kqfmt_enum_values(t : T?) { t |> equal(int(KqFmt.q51), 5) t |> equal(int(KqFmt.iq4xs), 6) t |> equal(int(KqFmt.k3), 7) + t |> equal(int(KqFmt.iq3s), 8) } } @@ -38,6 +39,8 @@ def test_kqfmt_predicates(t : T?) { t |> success(kq_sb(int(KqFmt.iq4xs)), "the int-id predicate agrees for iq4xs") t |> success(kq_sb(KqFmt.k3), "k3 is superblock") t |> success(kq_sb(int(KqFmt.k3)), "the int-id predicate agrees for k3") + t |> success(kq_sb(KqFmt.iq3s), "iq3s is superblock") + t |> success(kq_sb(int(KqFmt.iq3s)), "the int-id predicate agrees for iq3s") } } @@ -58,6 +61,8 @@ def test_kqfmt_strides(t : T?) { t |> equal(kq_ssb(KqFmt.iq4xs), 20l) // f16 d + 8 int8 sub-scales in the k4 row shape t |> equal(kq_qsb(KqFmt.k3), 96l) // 64 qs + 32 hmask, verbatim t |> equal(kq_ssb(KqFmt.k3), 18l) // the k6 row: 16 int8 sub-scales + f16 d + t |> equal(kq_qsb(KqFmt.iq3s), 104l) // [64 qs][8 qh][32 signs], verbatim + t |> equal(kq_ssb(KqFmt.iq3s), 20l) // the k4 row: f16 d + 8 int8 (1 + 2s) } t |> run("named constants agree with the literals") @(t : T?) { t |> equal(K4_QSB, 128l) @@ -74,6 +79,8 @@ def test_kqfmt_strides(t : T?) { t |> equal(IQ4XS_SSB, 20l) t |> equal(K3_QSB, 96l) t |> equal(K3_SSB, 18l) + t |> equal(IQ3S_QSB, 104l) + t |> equal(IQ3S_SSB, 20l) } t |> run("the IQ4_NL codebook is ggml's kvalues_iq4nl") @(t : T?) { t |> equal(int(IQ4NL_LUT[0]), -127) @@ -86,18 +93,30 @@ def test_kqfmt_strides(t : T?) { } } +[test] +def test_kqfmt_grid(t : T?) { + t |> run("the IQ3_S grid is ggml's iq3s_grid, and the worker-safe local agrees") @(t : T?) { + t |> equal(IQ3S_GRID[0], 0x01010101u) + t |> equal(IQ3S_GRID[511], 0x0f0f0101u) + let lut = iq3s_grid() + t |> equal(IQ3S_GRID[256], lut[256]) + t |> equal(IQ3S_GRID[303], lut[303]) + } +} + [test] def test_kqfmt_id_bridge(t : T?) { - t |> run("kq_schema_id is the one enum->kernel-id bridge (3/4/5/6/40/44)") @(t : T?) { + t |> run("kq_schema_id is the one enum->kernel-id bridge (3/4/5/6/40/44/33)") @(t : T?) { t |> equal(kq_schema_id(KqFmt.k4), 4) t |> equal(kq_schema_id(KqFmt.k5), 5) t |> equal(kq_schema_id(KqFmt.k6), 6) t |> equal(kq_schema_id(KqFmt.q40), 40) t |> equal(kq_schema_id(KqFmt.iq4xs), 44) t |> equal(kq_schema_id(KqFmt.k3), 3) + t |> equal(kq_schema_id(KqFmt.iq3s), 33) } t |> run("gemm_schema's int-id forms delegate to the same truth") @(t : T?) { - for (f in [KqFmt.k4, KqFmt.k5, KqFmt.k6, KqFmt.q40, KqFmt.iq4xs, KqFmt.k3]) { + for (f in [KqFmt.k4, KqFmt.k5, KqFmt.k6, KqFmt.q40, KqFmt.iq4xs, KqFmt.k3, KqFmt.iq3s]) { t |> equal(kq_qsb(kq_schema_id(f)), kq_qsb(f)) t |> equal(kq_ssb(kq_schema_id(f)), kq_ssb(f)) } @@ -113,6 +132,7 @@ def test_kqfmt_elems_and_stream(t : T?) { t |> equal(kq_elems(KqFmt.q40), 256l) t |> equal(kq_elems(KqFmt.iq4xs), 256l) t |> equal(kq_elems(KqFmt.k3), 256l) + t |> equal(kq_elems(KqFmt.iq3s), 256l) t |> equal(kq_elems(KqFmt.q8), 32l) t |> equal(kq_elems(KqFmt.q51), 32l) } @@ -125,5 +145,6 @@ def test_kqfmt_elems_and_stream(t : T?) { t |> equal(kq_stream_code(KqFmt.q40), 40) t |> equal(kq_stream_code(KqFmt.iq4xs), 44) t |> equal(kq_stream_code(KqFmt.k3), 3) + t |> equal(kq_stream_code(KqFmt.iq3s), 33) } } diff --git a/modules/dasLLAMA/tests/test_kquant.das b/modules/dasLLAMA/tests/test_kquant.das index 66bdca6735..4ba8a42a63 100644 --- a/modules/dasLLAMA/tests/test_kquant.das +++ b/modules/dasLLAMA/tests/test_kquant.das @@ -1,8 +1,8 @@ options gen2 options stack = 524288 // every dasLLAMA program root takes this budget (options stack does not unify up from libs) options persistent_heap // the layout arm loads a multi-GB model; explicit delete below -options _cyclomatic_complexity = 46 // every gate here is the same flat fmt == 4/5/6/40/44/3 ladder repeated per stage -options _function_length = 200 // and each gate carries its whole fixture build + oracle inline +options _cyclomatic_complexity = 52 // every gate here is the same flat fmt == 4/5/6/40/44/3 ladder repeated per stage +options _function_length = 220 // and each gate carries its whole fixture build + oracle inline options _dasllama_internal = true require dastest/testing_boost public @@ -105,7 +105,7 @@ def private build_q6k_block() : array { // loop-generated test labels: "k4"/"k5"/"k6" for the K-quants, "q40" for fmt 40, "iq4xs" for // fmt 44 — matches the kernel/repack function names (dot_q40q8, repack_q40_grp) so failures grep 1:1 -def private kq_tag(fmt : int) : string => fmt == 40 ? "q40" : (fmt == 44 ? "iq4xs" : "k{fmt}") +def private kq_tag(fmt : int) : string => fmt == 40 ? "q40" : (fmt == 44 ? "iq4xs" : (fmt == 33 ? "iq3s" : "k{fmt}")) // iq4xs synthetic superblock: 136B disk block — f16 d = SYNTH_D, the 8 six-bit sub-scales are // SYNTH_SC (packed low nibbles into scales_l, top two bits into scales_h), the shared nibble @@ -176,6 +176,35 @@ def private build_q3k_block() : array { return <- blkb } +// iq3s synthetic superblock: 110B IQ3_S disk block — 4-bit sub-scales SYNTH_SC[b] & 15, f16 d = +// SYNTH_D, grid-index/qh/sign byte patterns (written in the PACK direction); the gate recomposes +// each weight through IQ3S_GRID +def private iq3s_qs_pat(i : int) : int => (i * 11 + 3) % 256 +def private iq3s_qh_pat(b : int) : int => (b * 5 + 1) % 256 +def private iq3s_sg_pat(p : int) : int => (p * 7 + 13) % 256 +def private build_iq3s_block() : array { + var blkb : array + blkb |> resize(110) + let dbits = f32_to_f16(SYNTH_D) + blkb[0] = uint8(dbits & 0xFF) + blkb[1] = uint8(dbits >> 8u) + for (i in range(64)) { + blkb[2 + i] = uint8(iq3s_qs_pat(i)) + } + for (b in range(8)) { + blkb[66 + b] = uint8(iq3s_qh_pat(b)) + } + for (p in range(32)) { + blkb[74 + p] = uint8(iq3s_sg_pat(p)) + } + for (b in range(8)) { // scale nibble b: low nibble of byte b/2 (b even), high nibble (b odd) + let v = SYNTH_SC[b] & 15 + let nib = b % 2 == 0 ? v : v << 4 + blkb[106 + b / 2] = uint8(int(blkb[106 + b / 2]) | nib) + } + return <- blkb +} + // q40 synthetic superblock: 8 x 18B q4_0 disk blocks, per-block f16-exact d = (blk+1)/16 and // the shared nibble pattern — weight k = d(k/32) * (q4_pat(k) - 8) def private q40_d(blk : int) : float => float(blk + 1) * 0.0625 @@ -365,6 +394,30 @@ def test_kq_transcode_planes(t : T?) { t |> success(dst[k] == expected, "k3 plane element must match the ggml dequant exactly") } } + t |> run("iq3s planes decode the 4-bit sub-scales and gather the grid exactly") @(t : T?) { + let blkb <- build_iq3s_block() + var kq : array + var ks : array + kq |> resize(104) + ks |> resize(20) + transcode_iq3s_superblock(blkb, 0l, kq, 0l, ks, 0l) + for (b in range(8)) { + t |> equal(int(int8(ks[4 + b])), 1 + 2 * (SYNTH_SC[b] & 15), "decoded sub-scale is 1 + 2s") + } + var dst : array + dst |> resize(256) + dequant_iq3s_plane_superblock(kq, 0l, ks, 0l, dst, 0l) + for (k in range(256)) { + let b = k / 32 + let l = (k % 32) / 8 + let hj = k % 8 + let qi = iq3s_qs_pat(b * 8 + 2 * l + hj / 4) | ((iq3s_qh_pat(b) << (hj < 4 ? 8 - 2 * l : 7 - 2 * l)) & 256) + let g = int((IQ3S_GRID[qi] >> uint(8 * (hj % 4))) & 0xFFu) + let sgn = ((iq3s_sg_pat(b * 4 + l) >> hj) & 1) != 0 ? -1.0 : 1.0 + let expected = (SYNTH_D * float(1 + 2 * (SYNTH_SC[b] & 15))) * float(g) * sgn + t |> success(dst[k] == expected, "iq3s plane element must match the ggml dequant exactly") + } + } t |> run("q51 planes unpack the hand-packed Q5_1 blocks exactly") @(t : T?) { for (blk in range(4)) { let blkb <- build_q51_block(blk) @@ -611,7 +664,7 @@ def test_q51_kernels(t : T?) { // elementwise dot over the plane dequant (the kernels' declared oracle). def private kq_dot_gate(t : T?; fmt : int; n : int64 = 512l) { let nsb = n / 256l - let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : build_q6k_block())))) + let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : build_q6k_block()))))) let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) var kq : array @@ -629,6 +682,8 @@ def private kq_dot_gate(t : T?; fmt : int; n : int64 = 512l) { transcode_iq4xs_superblock(blk1, 0l, kq, s * qsb, ks, s * ssb) } elif (fmt == 3) { transcode_q3k_superblock(blk1, 0l, kq, s * qsb, ks, s * ssb) + } elif (fmt == 33) { + transcode_iq3s_superblock(blk1, 0l, kq, s * qsb, ks, s * ssb) } else { transcode_q6k_superblock(blk1, 0l, kq, s * qsb, ks, s * ssb) } @@ -652,6 +707,8 @@ def private kq_dot_gate(t : T?; fmt : int; n : int64 = 512l) { dequant_iq4xs_plane_superblock(kq, s * qsb, ks, s * ssb, w, s * 256l) } elif (fmt == 3) { dequant_k3_plane_superblock(kq, s * qsb, ks, s * ssb, w, s * 256l) + } elif (fmt == 33) { + dequant_iq3s_plane_superblock(kq, s * qsb, ks, s * ssb, w, s * 256l) } else { dequant_k6_plane_superblock(kq, s * qsb, ks, s * ssb, w, s * 256l) } @@ -687,6 +744,8 @@ def private kq_dot_gate(t : T?; fmt : int; n : int64 = 512l) { got = dot_iq4xs_q8(addr(kq[0]), addr(ks[0]), addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } elif (fmt == 3) { got = dot_k3q8(addr(kq[0]), addr(ks[0]), addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + } elif (fmt == 33) { + got = dot_iq3s_q8(addr(kq[0]), addr(ks[0]), addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } else { got = dot_k6q8(addr(kq[0]), addr(ks[0]), addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } @@ -715,9 +774,12 @@ def test_kq_dots(t : T?) { t |> run("dot_k3q8 matches the fp64 plane-dequant reference") @(t : T?) { kq_dot_gate(t, 3) } + t |> run("dot_iq3s_q8 matches the fp64 plane-dequant reference") @(t : T?) { + kq_dot_gate(t, 33) + } // 122B-class row widths: n=3072 (12 superblocks — qwen35moe-122B gate/up/cls) and n=1024 // (its k5 down_exps). Every earlier kq model kept n at 512..2560. - for (fmt in [4, 5, 6, 40, 44, 3]) { + for (fmt in [4, 5, 6, 40, 44, 3, 33]) { for (nn in [1024l, 3072l]) { t |> run("dot_{kq_tag(fmt)}q8 matches the fp64 reference at n={int(nn)}") @(t : T?) { kq_dot_gate(t, fmt, nn) @@ -736,7 +798,7 @@ def private kq_gemv_rows_gate(t : T?; fmt : int; n : int64 = 512l) { let nsb = n / 256l let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) - let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : build_q6k_block())))) + let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : build_q6k_block()))))) var kq : array var ks : array kq |> resize(d * nsb * qsb) @@ -753,6 +815,8 @@ def private kq_gemv_rows_gate(t : T?; fmt : int; n : int64 = 512l) { transcode_iq4xs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } elif (fmt == 3) { transcode_q3k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 33) { + transcode_iq3s_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } else { transcode_q6k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } @@ -795,6 +859,8 @@ def private kq_gemv_rows_gate(t : T?; fmt : int; n : int64 = 512l) { want = dot_iq4xs_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } elif (fmt == 3) { want = dot_k3q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + } elif (fmt == 33) { + want = dot_iq3s_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } else { want = dot_k6q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } @@ -833,7 +899,10 @@ def test_kq_gemv_rows(t : T?) { t |> run("portable k3 GEMV rows bit-match per-row disk dots") @(t : T?) { kq_gemv_rows_gate(t, 3) } - for (fmt in [4, 5, 6, 40, 44, 3]) { + t |> run("portable iq3s GEMV rows bit-match per-row disk dots") @(t : T?) { + kq_gemv_rows_gate(t, 33) + } + for (fmt in [4, 5, 6, 40, 44, 3, 33]) { for (nn in [1024l, 3072l]) { t |> run("portable {kq_tag(fmt)} GEMV rows bit-match per-row disk dots at n={int(nn)}") @(t : T?) { kq_gemv_rows_gate(t, fmt, nn) @@ -853,7 +922,7 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { let nsb = n / 256l let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) - let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : build_q6k_block())))) + let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : build_q6k_block()))))) var kq : array var ks : array kq |> resize(d * nsb * qsb) @@ -870,6 +939,8 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { transcode_iq4xs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } elif (fmt == 3) { transcode_q3k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 33) { + transcode_iq3s_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } else { transcode_q6k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } @@ -911,6 +982,8 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { want[r] = dot_iq4xs_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } elif (fmt == 3) { want[r] = dot_k3q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + } elif (fmt == 33) { + want[r] = dot_iq3s_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } else { want[r] = dot_k6q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } @@ -925,6 +998,8 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { dequant_iq4xs_plane_superblock(kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb, wrow, r * n + s * 256l) } elif (fmt == 3) { dequant_k3_plane_superblock(kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb, wrow, r * n + s * 256l) + } elif (fmt == 33) { + dequant_iq3s_plane_superblock(kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb, wrow, r * n + s * 256l) } else { dequant_k6_plane_superblock(kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb, wrow, r * n + s * 256l) } @@ -940,6 +1015,8 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { repack_iq4xs_grp(addr(kq[0]), addr(ks[0]), n, d, mr) } elif (fmt == 3) { repack_k3_grp(addr(kq[0]), addr(ks[0]), n, d, mr) + } elif (fmt == 33) { + repack_iq3s_grp(addr(kq[0]), addr(ks[0]), n, d, mr) } else { repack_k6_grp(addr(kq[0]), addr(ks[0]), n, d, mr) } @@ -957,7 +1034,9 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { ? iq4xs_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) : (fmt == 3 ? k3_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) - : kq_grp_row_dot(int64(fmt), kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n)))) + : (fmt == 33 + ? iq3s_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + : kq_grp_row_dot(int64(fmt), kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n))))) if (got != want[r]) { dotbad++ } @@ -984,7 +1063,7 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { [test] def test_kq_repack_grp(t : T?) { - for (fmt in [4, 5, 6, 40, 44, 3]) { + for (fmt in [4, 5, 6, 40, 44, 3, 33]) { for (mr in [4l, 8l, 16l]) { t |> run("repack_{kq_tag(fmt)}_grp mr={mr} preserves dots and row dequants") @(t : T?) { kq_repack_gate(t, fmt, mr) @@ -1009,7 +1088,7 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { let mr = kq_layout_of(fmt) // the format's OWN layout companion (per-format tune families) let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) - let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : build_q6k_block())))) + let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : build_q6k_block()))))) var kq : array var ks : array kq |> resize(d * nsb * qsb) @@ -1026,6 +1105,8 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { transcode_iq4xs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } elif (fmt == 3) { transcode_q3k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 33) { + transcode_iq3s_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } else { transcode_q6k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } @@ -1047,6 +1128,8 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { repack_iq4xs_grp(addr(kq[0]), addr(ks[0]), n, d, mr) } elif (fmt == 3) { repack_k3_grp(addr(kq[0]), addr(ks[0]), n, d, mr) + } elif (fmt == 33) { + repack_iq3s_grp(addr(kq[0]), addr(ks[0]), n, d, mr) } else { repack_k6_grp(addr(kq[0]), addr(ks[0]), n, d, mr) } @@ -1075,7 +1158,7 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { // k5/k6 tiles read the byte-expanded panel the batch cell unpacks per (group, token-block) // — the gate covers unpack_kq_panel_grp too (tile-over-panel vs gemv-over-packed planes); // k4/q40 tiles read the packed planes directly - let packed = fmt == 4 || fmt == 40 || fmt == 44 || fmt == 3 + let packed = fmt == 4 || fmt == 40 || fmt == 44 || fmt == 3 || fmt == 33 var panel : array if (!packed) { panel |> resize(mr * nsb * 256l) @@ -1100,6 +1183,8 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { iq4xsq8_tile_gen(addr(ytile[0]), kqg, ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) } elif (fmt == 3) { k3q8_tile_gen(addr(ytile[0]), kqg, ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) + } elif (fmt == 33) { + iq3sq8_tile_gen(addr(ytile[0]), kqg, ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) } else { k6q8_tile_gen(addr(ytile[0]), addr(panel[0]), ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) } @@ -1116,6 +1201,8 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { iq4xsq8_gemv_gen(addr(ytile[tk * d]), addr(kq[0]), addr(ks[0]), addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, g * mr, (g + 1l) * mr) } elif (fmt == 3) { k3q8_gemv_gen(addr(ytile[tk * d]), addr(kq[0]), addr(ks[0]), addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, g * mr, (g + 1l) * mr) + } elif (fmt == 33) { + iq3sq8_gemv_gen(addr(ytile[tk * d]), addr(kq[0]), addr(ks[0]), addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, g * mr, (g + 1l) * mr) } else { k6q8_gemv_gen(addr(ytile[tk * d]), addr(kq[0]), addr(ks[0]), addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, g * mr, (g + 1l) * mr) } @@ -1133,6 +1220,8 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { iq4xsq8_gemv_gen(addr(ygemv[p * d]), addr(kq[0]), addr(ks[0]), addr(xq[p * n]), addr(xs[p * (n / 256l)]), addr(xbs[p * (n / 16l)]), n, 0l, d) } elif (fmt == 3) { k3q8_gemv_gen(addr(ygemv[p * d]), addr(kq[0]), addr(ks[0]), addr(xq[p * n]), addr(xs[p * (n / 256l)]), addr(xbs[p * (n / 16l)]), n, 0l, d) + } elif (fmt == 33) { + iq3sq8_gemv_gen(addr(ygemv[p * d]), addr(kq[0]), addr(ks[0]), addr(xq[p * n]), addr(xs[p * (n / 256l)]), addr(xbs[p * (n / 16l)]), n, 0l, d) } else { k6q8_gemv_gen(addr(ygemv[p * d]), addr(kq[0]), addr(ks[0]), addr(xq[p * n]), addr(xs[p * (n / 256l)]), addr(xbs[p * (n / 16l)]), n, 0l, d) } @@ -1158,7 +1247,7 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { [test] def test_kq_tile(t : T?) { - for (fmt in [4, 5, 6, 40, 44, 3]) { + for (fmt in [4, 5, 6, 40, 44, 3, 33]) { t |> run("{kq_tag(fmt)} 4-token tile bit-matches per-token GEMVs") @(t : T?) { kq_tile_gate(t, fmt) } @@ -1179,7 +1268,7 @@ def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; let nsb = n / 256l let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) - let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : build_q6k_block())))) + let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : build_q6k_block()))))) var kq : array var ks : array kq |> resize(nreg * d * nsb * qsb) @@ -1196,6 +1285,8 @@ def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; transcode_iq4xs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } elif (fmt == 3) { transcode_q3k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 33) { + transcode_iq3s_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } else { transcode_q6k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } @@ -1259,6 +1350,9 @@ def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; } elif (fmt == 3) { want_sh[r * d + row] = dot_k3q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) want_pr[r * d + row] = dot_k3q8(kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n) + } elif (fmt == 33) { + want_sh[r * d + row] = dot_iq3s_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + want_pr[r * d + row] = dot_iq3s_q8(kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n) } else { want_sh[r * d + row] = dot_k6q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) want_pr[r * d + row] = dot_k6q8(kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n) @@ -1305,6 +1399,8 @@ def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; repack_iq4xs_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) } elif (fmt == 3) { repack_k3_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) + } elif (fmt == 33) { + repack_iq3s_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) } else { repack_k6_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) } @@ -1327,6 +1423,9 @@ def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; } elif (fmt == 3) { k3q8_gemv_gen(addr(want_sh[r * d]), kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, 0l, d) k3q8_gemv_gen(addr(want_pr[r * d]), kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n, 0l, d) + } elif (fmt == 33) { + iq3sq8_gemv_gen(addr(want_sh[r * d]), kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, 0l, d) + iq3sq8_gemv_gen(addr(want_pr[r * d]), kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n, 0l, d) } else { k6q8_gemv_gen(addr(want_sh[r * d]), kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, 0l, d) k6q8_gemv_gen(addr(want_pr[r * d]), kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n, 0l, d) @@ -1362,7 +1461,7 @@ def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; [test] def test_kq_groupn(t : T?) { - for (fmt in [4, 5, 6, 40, 44, 3]) { + for (fmt in [4, 5, 6, 40, 44, 3, 33]) { t |> run("{kq_tag(fmt)} region-list GEMV bit-matches per-row dots (disk + grp slices)") @(t : T?) { kq_groupn_gate(t, fmt) } @@ -1406,7 +1505,7 @@ def private kq_batch_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 for (c in cnts) { nk += c } - let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : build_q6k_block())))) + let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : build_q6k_block()))))) var kq : array var ks : array kq |> resize(nreg * d * nsb * qsb) @@ -1423,6 +1522,8 @@ def private kq_batch_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 transcode_iq4xs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } elif (fmt == 3) { transcode_q3k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 33) { + transcode_iq3s_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } else { transcode_q6k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } @@ -1445,6 +1546,8 @@ def private kq_batch_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 repack_iq4xs_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) } elif (fmt == 3) { repack_k3_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) + } elif (fmt == 33) { + repack_iq3s_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) } else { repack_k6_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) } @@ -1493,6 +1596,8 @@ def private kq_batch_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 iq4xsq8_gemv_gen(addr(want[tk * d]), kqr, ksr, addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, 0l, d) } elif (fmt == 3) { k3q8_gemv_gen(addr(want[tk * d]), kqr, ksr, addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, 0l, d) + } elif (fmt == 33) { + iq3sq8_gemv_gen(addr(want[tk * d]), kqr, ksr, addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, 0l, d) } else { k6q8_gemv_gen(addr(want[tk * d]), kqr, ksr, addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, 0l, d) } @@ -1524,7 +1629,7 @@ def private kq_batch_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 [test] def test_kq_batch_groupn(t : T?) { - for (fmt in [4, 5, 6, 40, 44, 3]) { + for (fmt in [4, 5, 6, 40, 44, 3, 33]) { t |> run("{kq_tag(fmt)} batch groupn bit-matches per-token rows-core calls") @(t : T?) { kq_batch_groupn_gate(t, fmt) } From f063e55548cc77ba47742053b1a498d10a3bdd72 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Sun, 30 Aug 2026 15:46:41 -0700 Subject: [PATCH 18/61] dasLLAMA: the IQ3_S JIT emitter - panel-route tile + gather gemv, pp512 4.93x llama.cpp The first CPU arm written under the llama.cpp side-by-side rule (LAWS 2026-08-30): their kernel's techniques mapped onto our lattice and raced by the probe. The tile rides the panel route - unpack_iq3s_panel_grp expands grid words + signs into the k5/k6 byte panel per group (one i32 store per grid word, sign nibble through a 16-entry mask table, negate (w ^ m) + (m & 0x01010101); magnitudes odd 1..15, no cross-byte carry) - and the gemv gathers each superblock into an alloca panel through an emitted per-row loop (emit_iq3s_gather; the grid and mask live as private [512 x i32]/[16 x i32] module constants). Both ride emit_block_iq4xs's sign-trick lattice with panel loads in place of the nibble+LUT; the reference tile body flips to the panel dot (kq_grp_row_dot_b fmt 33) since the batch cell now hands it the unpacked panel. Also fixed in passing, both found by this arm: - emit_slice recorded the SAVED body/tails block handles as phi incoming blocks; a block-splitting emitter (the gather loop) leaves the builder in its continuation block, so the back edges were malformed. Now captures LLVMGetInsertBlock at the branch points - identical for every non-splitting emitter. - HOW_TO QUIRK 21: an emitter-arm edit does NOT invalidate the JIT DLL cache (generator bodies are not in the hash) - three fix rounds ran stale stamps byte-identically before the cache-hit line gave it away. rm -rf .jitted_scripts after any emitter change. Gates: probe test mode GEN TUNE TEST OK - every k33 perm 1.9e-6 incl the stamped maddubs mr8 pair. Side by side, zen2 16t vs llama.cpp b10660 clean-cpu: pp512 516.9 +- 8.4 vs 104.9 (4.93x - the panel amortizes one gather across the 4-token tile; their nrc=1 kernel re-gathers per token), tg128 52.4 +- 1.0 vs 57.0 (0.92x; the no-panel gemv spelling is followup_general `#61`). Stamped e2e on the 1B IQ3_M: 63/64 greedy ids (the flip is the final token - the stamped near-tie class), gen 44 t/s from 23 on the reference bodies. Co-Authored-By: Claude Fable 5 --- modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md | 36 +++- modules/dasLLAMA/LAWS.md | 9 + .../dasLLAMA/dasllama/dasllama_gemm_gen.das | 175 ++++++++++++++++-- .../dasLLAMA/dasllama/dasllama_math_gen.das | 26 ++- modules/dasLLAMA/dasllama/dasllama_repack.das | 49 +++++ modules/dasLLAMA/followup_general.md | 21 +++ modules/dasLLAMA/harness/gen_tune_probe.das | 5 +- 7 files changed, 287 insertions(+), 34 deletions(-) diff --git a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md index c567ffabe1..45d4e04aea 100644 --- a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md +++ b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md @@ -96,6 +96,11 @@ add the arm next to `q40`'s. The compiler does not find these - a missing arm fa ## 4. CPU kernels - `dasllama_math_default.das`, `dasllama_math_gen.das`, `dasllama_math.das`, `dasllama_repack.das` +- **Read llama.cpp's CPU kernel for the format FIRST** - `ggml-cpu/arch/x86/quants.c` + `ggml_vec_dot__q8_K` (and the arm twin) in the local clone. Map each technique it uses + onto the lattice as its own `[tune_perm]` spelling where it fits; the probe IS the side-by-side + (perms race each other and the reference, per box). Our CPU kernels typically win - keep it + that way by never leaving one of their tricks unmeasured. - `dot__q8` - the portable disk-order row dot (exact integer inner sums, one float fold per superblock), `_rows_kernel`, the `kq_gemv_kernel` arm, the `matmul_kq_groupn` arm. - `dequant__row_grp` - the grp row dequant (own helper; `dequant_kq_row_grp` dispatches). @@ -396,6 +401,15 @@ where, why it is so today, what unquirked looks like. An empty ledger is a legit lookup, and belongs to followup_general #58. Unquirked: the SPIR-V emitter lowers a `let` fixed_array of literals to a constant-storage array, or a lint on a dynamically indexed fixed_array local inside a kernel class. +21. **An emitter-arm edit does NOT invalidate the JIT DLL cache.** The registered code + generators (`dasllama_gemm_gen.das`) run at codegen time and their bodies do not fold into + the cached DLL's hash - after an emitter change, every `-jit` run that hits the cache + executes the OLD stamps with no signal (the iq3s gemv arm "failed" three fix rounds in a + row on byte-identical numbers before the cache-hit line gave it away; the hash was the same + 0xa3a02e12... across every edit). After ANY emitter change: `rm -rf .jitted_scripts` (or + bump `LLVM_JIT_CODEGEN_VERSION`) before trusting a probe or bench. Unquirked: the cache + hash folds the generator bodies (plans/interpreter ledger has the sibling + `jit_dll_semantic_hash` item). ## Per-format notes @@ -418,9 +432,25 @@ family minted `verdict=rejected` (QUIRK 16's shape) after the whole-scope re-tun and the reference bodies serve. End to end: `Llama-3.2-1B-Instruct-IQ3_M.gguf` (bartowski: IQ3_S x78 + Q4_K x34 + Q6_K embd) through `run.das` matches llama.cpp's greedy ids **64 of 64** at gen 23 t/s - the first format to hold the whole comparison window token-for-token. -JIT emitter, Vulkan, Metal: pending (the per-tier grid-placement answers are pre-researched: -workgroup/threadgroup-staged table, the IQLUT axis on the cm2 template, threadgroup floats -on Metal). +JIT emitter (section 5, the first side-by-side under the llama.cpp-kernel rule): the TILE +rides the panel route - `unpack_iq3s_panel_grp` gathers grid words + signs into the k5/k6 +byte-expanded panel per group (one i32 store per grid word, sign nibble through a 16-entry +mask table, negate = `(w ^ m) + (m & 0x01010101)` - magnitudes are odd 1..15, no cross-byte +carry) - and the GEMV gathers each superblock into an alloca panel via an emitted per-row +loop (`emit_iq3s_gather`; grid + mask as private module constants); both then ride +`emit_block_iq4xs`'s sign-trick lattice with panel loads in place of the nibble+LUT. Two +finds along the way: `emit_slice` recorded SAVED block handles as phi incomings - a +block-splitting emitter leaves the builder elsewhere, so back-edges were malformed (fixed: +capture `LLVMGetInsertBlock` at the branch points); and QUIRK 21 ate three fix rounds. +Probe test mode: every k33 perm ok (maddubs mr8 stamped, 1.9e-6); the tuner crowned +`dot_maddubs_width256_mr8` at ~4.9x the reference body. Side by side on the zen2 (16t, +llama.cpp b10660 clean-cpu): pp512 516.9 vs 104.9 (**4.93x** - the panel amortizes the +gather across the tile; their per-row kernel re-gathers per token), tg128 52.4 vs 57.0 +(0.92x - nothing amortizes at one token; the no-panel gemv spelling to close it is +followup_general #61). Stamped e2e: 63/64 greedy ids (the flip is the FINAL token, the +stamped-vs-reference near-tie class), gen 44 t/s. Vulkan, Metal: pending (pre-researched: +the IQLUT axis on the cm2 template takes the 2 KB grid; threadgroup floats + the +base-pointer 9th-bit trick on Metal). ### Q3_K (the second format, 2026-08-30) diff --git a/modules/dasLLAMA/LAWS.md b/modules/dasLLAMA/LAWS.md index c16b435a07..53e63a0d93 100644 --- a/modules/dasLLAMA/LAWS.md +++ b/modules/dasLLAMA/LAWS.md @@ -87,3 +87,12 @@ the same day, on the Metal portion: "we have an option of ssh m1 ... go for it. somehow. plus validates the how-to. and we'll ledger M5 pass on new kernels for later" - the M1 is in the loop for every format from here, and performance work on the new Metal kernels is the ledgered M5 pass, not this arc. + +## 2026-08-30 - HOW_TO_ADD_A_FORMAT.md (section 4), followup_general.md (#60) + +Boris, on the IQ3_S emitter design after reading llama.cpp's AVX2 iq3_s kernel: "our CPU +kernels typically faster - so if they map closer, perhaps worth looking at and doing side by +side?" and then "lets make sure we do for all new CPU kernels, and if we skipped for previous +ones - lets ledger towards the end of this arc." Every new format's CPU kernel work starts by +reading llama.cpp's arch kernel and racing its techniques as tune perms; the formats that +skipped this (IQ4_XS, Q3_K) get the retroactive audit at the end of the arc (#60). diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das index e8639946ba..e5848e5a1c 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das @@ -245,6 +245,11 @@ struct private TileEmit { abs_decl : LLVMOpaqueValue? // x64 sign trick: llvm.abs (VPABSB) abs_ty : LLVMOpaqueType? psign_decl : LLVMOpaqueValue? // x64 width-256 sign apply (VPSIGNB) + iq3s_panel : LLVMOpaqueValue? // iq3s gemv: the per-superblock alloca panel (null = tile mode, the caller unpacked) + iq3s_grid_g : LLVMOpaqueValue? // iq3s gemv: the [512 x i32] grid global + iq3s_smask_g : LLVMOpaqueValue? // iq3s gemv: the [16 x i32] sign-nibble byte-mask global + gfn : LLVMOpaqueValue? // the function under emit (block creation for emitted inner loops) + gctx : LLVMOpaqueContext? psign_ty : LLVMOpaqueType? madd_decl : LLVMOpaqueValue? // x64 maddubs pair (VPMADDUBSW) madd_ty : LLVMOpaqueType? @@ -1212,15 +1217,128 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f } } -// One 256-weight SUPERBLOCK, IQ4_XS grp form (te.kq = 44): mx4's LUT decode (tbl1/pshufb) -// yields SIGNED int8 weights, so the dots ride dot_lane's sign-trick lattice, not kq_dot_lane; -// the fold is k4's integer sub-scale sum with no min term: f += (sum_blk sc * idot) * d * d8. -def private emit_block_iq4xs(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f : LLVMOpaqueValue? [8]; tokBase, tokCount : int) { +// The iq3s gemv gather globals: the grid as a [512 x i32] private constant, the 4-bit sign +// nibble expanded to a byte mask via a [16 x i32] table (negation is (w ^ m) + (m & 0x01010101); +// grid magnitudes are odd 1..15, so no carry crosses a byte lane). +def private iq3s_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { + var grid = LLVMGetNamedGlobal(mod, "dasllama.iq3s.grid") + if (grid == null) { + let g = iq3s_grid() + var elems : array + elems |> reserve(512) + for (i in range(512)) { + elems |> push(LLVMConstInt(te.types.t_int32, uint64(g[i]), 0)) + } + grid = LLVMAddGlobal(mod, LLVMArrayType(te.types.t_int32, 512u), "dasllama.iq3s.grid") + LLVMSetInitializer(grid, LLVMConstArray(te.types.t_int32, array_data_ptr(elems), 512u)) + LLVMSetGlobalConstant(grid, 1) + LLVMSetLinkage(grid, LLVMLinkage.LLVMPrivateLinkage) + LLVMSetAlignment(grid, 64u) + } + te.iq3s_grid_g = grid + var smask = LLVMGetNamedGlobal(mod, "dasllama.iq3s.smask") + if (smask == null) { + var elems2 : array + elems2 |> reserve(16) + for (m in range(16)) { + var v = 0ul + for (t in range(4)) { + if ((m & (1 << t)) != 0) { + v |= 0xFFul << uint64(8 * t) + } + } + elems2 |> push(LLVMConstInt(te.types.t_int32, v, 0)) + } + smask = LLVMAddGlobal(mod, LLVMArrayType(te.types.t_int32, 16u), "dasllama.iq3s.smask") + LLVMSetInitializer(smask, LLVMConstArray(te.types.t_int32, array_data_ptr(elems2), 16u)) + LLVMSetGlobalConstant(smask, 1) + LLVMSetLinkage(smask, LLVMLinkage.LLVMPrivateLinkage) + LLVMSetAlignment(smask, 16u) + } + te.iq3s_smask_g = smask +} + +// iq3s gemv: gather superblock `sbi`'s grid words + signs off the PACKED grp planes into the +// alloca panel — one emitted loop over the mr rows (32 grid words unrolled per iteration); a +// single straight-line loop block, so the phi's back edge is the block itself. +def private emit_iq3s_gather(var te : TileEmit; var sbi : LLVMOpaqueValue?) { + let b = te.builder + let mr = te.interleave + var wbP = LLVMBuildMul(b, sbi, te.types->ConstI64(uint64(mr * 104)), "iq3s.wb") + var pre = LLVMGetInsertBlock(b) + var head = LLVMAppendBasicBlockInContext(te.gctx, te.gfn, "iq3s.gath") + var cont = LLVMAppendBasicBlockInContext(te.gctx, te.gfn, "iq3s.cont") + LLVMBuildBr(b, head) + LLVMPositionBuilderAtEnd(b, head) + var rPhi = LLVMBuildPhi(b, te.types.t_int64, "iq3s.r") + var preVals <- [te.types->ConstI64(0ul)] + var preBlocks <- [pre] + LLVMAddIncoming(rPhi, preVals, preBlocks) + var rT4 = LLVMBuildMul(b, rPhi, te.types->ConstI64(4ul), "iq3s.r4") + var base = LLVMBuildAdd(b, wbP, rT4, "iq3s.base") + let c256 = LLVMConstInt(te.types.t_int32, 256ul, 0) + let c15 = LLVMConstInt(te.types.t_int32, 15ul, 0) + let cneg = LLVMConstInt(te.types.t_int32, 0x01010101ul, 0) + for (blk in range(8)) { + let hp = 64 + blk + var qhp = LLVMBuildGEP2(b, te.types.t_int8, te.wg, LLVMBuildAdd(b, base, te.types->ConstI64(uint64((hp / 4) * mr * 4 + hp % 4)), ""), "") + var qh = LLVMBuildZExt(b, LLVMBuildLoad2Aligned(b, te.types.t_int8, qhp, 1u, ""), te.types.t_int32, "iq3s.qh{blk}") + for (l in range(4)) { + let sp = 72 + blk * 4 + l + var sgp = LLVMBuildGEP2(b, te.types.t_int8, te.wg, LLVMBuildAdd(b, base, te.types->ConstI64(uint64((sp / 4) * mr * 4 + sp % 4)), ""), "") + var sgn = LLVMBuildZExt(b, LLVMBuildLoad2Aligned(b, te.types.t_int8, sgp, 1u, ""), te.types.t_int32, "iq3s.sg{blk}_{l}") + var iv : LLVMOpaqueValue? [2] + for (h in range(2)) { + let qp1 = blk * 8 + 2 * l + h + var qpp = LLVMBuildGEP2(b, te.types.t_int8, te.wg, LLVMBuildAdd(b, base, te.types->ConstI64(uint64((qp1 / 4) * mr * 4 + qp1 % 4)), ""), "") + var qv = LLVMBuildZExt(b, LLVMBuildLoad2Aligned(b, te.types.t_int8, qpp, 1u, ""), te.types.t_int32, "") + var hb = LLVMBuildAnd(b, LLVMBuildShl(b, qh, LLVMConstInt(te.types.t_int32, uint64(8 - h - 2 * l), 0), ""), c256, "") + iv[h] = LLVMBuildOr(b, qv, hb, "iq3s.i{blk}_{l}_{h}") + } + var mv : LLVMOpaqueValue? [2] + mv[0] = LLVMBuildAnd(b, sgn, c15, "") + mv[1] = LLVMBuildAnd(b, LLVMBuildLShr(b, sgn, LLVMConstInt(te.types.t_int32, 4ul, 0), ""), c15, "") + for (h in range(2)) { + var gp = LLVMBuildGEP2(b, te.types.t_int32, te.iq3s_grid_g, iv[h], "") + var gw = LLVMBuildLoad2Aligned(b, te.types.t_int32, gp, 4u, "iq3s.g{blk}_{l}_{h}") + var mp = LLVMBuildGEP2(b, te.types.t_int32, te.iq3s_smask_g, mv[h], "") + var mw = LLVMBuildLoad2Aligned(b, te.types.t_int32, mp, 4u, "") + var w = LLVMBuildAdd(b, LLVMBuildXor(b, gw, mw, ""), LLVMBuildAnd(b, mw, cneg, ""), "iq3s.w{blk}_{l}_{h}") + let e = l * 8 + h * 4 + let dd = (e < 16 ? 0 : 128 * mr) + (blk * 4 + (e % 16) / 4) * mr * 4 + var dp = LLVMBuildGEP2(b, te.types.t_int8, te.iq3s_panel, LLVMBuildAdd(b, rT4, te.types->ConstI64(uint64(dd)), ""), "") + LLVMBuildStore(b, w, dp) + } + } + } + var rNext = LLVMBuildAdd(b, rPhi, te.types->ConstI64(1ul), "iq3s.rn") + var more = LLVMBuildICmp(b, LLVMIntPredicate.LLVMIntSLT, rNext, te.types->ConstI64(uint64(mr)), "iq3s.more") + LLVMBuildCondBr(b, more, head, cont) + var loopVals <- [rNext] + var loopBlocks <- [head] + LLVMAddIncoming(rPhi, loopVals, loopBlocks) + LLVMPositionBuilderAtEnd(b, cont) +} + +// One 256-weight SUPERBLOCK, IQ4_XS/iq3s grp form (te.kq = 44/33): mx4's LUT decode (44) +// or the byte-expanded signed panel (33; the gemv gathers it per superblock, the tile gets the +// runtime unpack) — dot_lane's sign-trick lattice, k4's fold with no min term. +def private emit_block_iq4xs(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f : LLVMOpaqueValue? [8]; tokBase, tokCount : int) { // nolint:STYLE038 — one register-coupled superblock loop nest (the kqv2 emitters' shape) let b = te.builder let rq = te.rq let mr = te.interleave let w8 = te.width / 8 - var wb = LLVMBuildMul(b, sbi, te.types->ConstI64(uint64(mr * 128)), "wb") + let panel = te.kq == 33 // iq3s: the byte-expanded signed panel replaces the nibble+LUT decode; fold identical + let gather = panel && te.iq3s_panel != null // gemv mode: gather this superblock off the packed planes first + var wbase = te.wg + var wb : LLVMOpaqueValue? + if (gather) { + emit_iq3s_gather(te, sbi) + wbase = te.iq3s_panel + wb = te.types->ConstI64(0ul) + } else { + wb = LLVMBuildMul(b, sbi, te.types->ConstI64(uint64(mr * (panel ? 256 : 128))), "wb") + } var sb = LLVMBuildMul(b, sbi, te.types->ConstI64(uint64(mr * 20)), "sb") var xb = LLVMBuildMul(b, sbi, te.types->ConstI64(0x100ul), "xb") var vri8 = LLVMVectorType(te.types.t_int8, uint(te.rv)) @@ -1248,11 +1366,19 @@ def private emit_block_iq4xs(var te : TileEmit; var sbi : LLVMOpaqueValue?; var for (j in range(4)) { for (qd in range(rq)) { var noff = LLVMBuildAdd(b, wb, te.types->ConstI64(uint64((blk * 16 + j * 4) * mr + qd * w8)), "") - var nv = load_vec(te, te.vwi8, te.wg, noff, "nv{blk}_{j * rq + qd}") - var loIdx = LLVMBuildAnd(b, nv, splat_i8w(te, 15), "nib.lo") - var hiIdx = LLVMBuildLShr(b, nv, splat_i8w(te, 4), "nib.hi") - var wlo = lut_lookup(te, loIdx, "wlo{blk}_{j * rq + qd}") - var whi = lut_lookup(te, hiIdx, "whi{blk}_{j * rq + qd}") + var wlo : LLVMOpaqueValue? + var whi : LLVMOpaqueValue? + if (panel) { + var hoff = LLVMBuildAdd(b, noff, te.types->ConstI64(uint64(128 * mr)), "") + wlo = load_vec(te, te.vwi8, wbase, noff, "wlo{blk}_{j * rq + qd}") + whi = load_vec(te, te.vwi8, wbase, hoff, "whi{blk}_{j * rq + qd}") + } else { + var nv = load_vec(te, te.vwi8, te.wg, noff, "nv{blk}_{j * rq + qd}") + var loIdx = LLVMBuildAnd(b, nv, splat_i8w(te, 15), "nib.lo") + var hiIdx = LLVMBuildLShr(b, nv, splat_i8w(te, 4), "nib.hi") + wlo = lut_lookup(te, loIdx, "wlo{blk}_{j * rq + qd}") + whi = lut_lookup(te, hiIdx, "whi{blk}_{j * rq + qd}") + } var ulo = abs_w(te, wlo, "ulo{blk}_{j * rq + qd}") var uhi = abs_w(te, whi, "uhi{blk}_{j * rq + qd}") for (i in range(tokCount)) { @@ -1287,7 +1413,7 @@ def private emit_block_iq4xs(var te : TileEmit; var sbi : LLVMOpaqueValue?; var // token PAIR — single-token slices ride the sdot lattice instead. kq blocks are SUPERBLOCKS // (the kq gemv/tile drive emit_slice with nb = n/256). def private emit_one_block(var te : TileEmit; var bi : LLVMOpaqueValue?; var f : LLVMOpaqueValue? [8]; tokBase, tokCount : int) { - if (te.kq == 44) { + if (te.kq == 44 || te.kq == 33) { emit_block_iq4xs(te, bi, f, tokBase, tokCount) } elif (te.kq != 0) { emit_block_kqv2(te, bi, f, tokBase, tokCount) @@ -1378,13 +1504,15 @@ def private emit_slice(var te : TileEmit; var sa : SliceArgs; var pred : LLVMOpa var bij = LLVMBuildAdd(b, biPhi, te.types->ConstI64(uint64(j)), "") emit_one_block(te, bij, fCur, tokBase, tokCount) } + // the back edge starts wherever the block emitters LEFT the builder (a splitting emitter ends in its continuation block) + var bodyEnd = LLVMGetInsertBlock(b) LLVMBuildBr(b, head) var bodyValsBi <- [biPK] - var bodyBlocks <- [body] + var bodyBlocks <- [bodyEnd] LLVMAddIncoming(biPhi, bodyValsBi, bodyBlocks) for (i in range(nacc)) { var bv <- [fCur[i]] - var bb <- [body] + var bb <- [bodyEnd] LLVMAddIncoming(fPhi[i], bv, bb) } @@ -1414,8 +1542,9 @@ def private emit_slice(var te : TileEmit; var sa : SliceArgs; var pred : LLVMOpa chainBi = LLVMBuildAdd(b, chainBi, te.types->ConstI64(1ul), "") LLVMBuildBr(b, checks[j + 1]) } else { + var tailEnd = LLVMGetInsertBlock(b) // same rule: the tail body may have split its block LLVMBuildBr(b, after) - mergeBlocks |> push(tails[j]) + mergeBlocks |> push(tailEnd) mergeF |> push(chainF) } } @@ -2391,6 +2520,14 @@ def private kq_gemv_gen_impl(var gc : LlvmCodeCtx; fmt : int) : bool { // entry: group range off the row range; strides are disk footprints (quants 128/160/192B, scales 20/20/18B) LLVMPositionBuilderAtEnd(b, entry) + if (fmt == 33) { + // iq3s: the gemv gathers each superblock into an alloca panel (the tile gets the runtime unpack) + te.gfn = gc.impl + te.gctx = gc.jit.ctx + iq3s_emit_globals(te, LLVMGetGlobalParent(gc.impl)) + te.iq3s_panel = LLVMBuildAlloca(b, LLVMArrayType(gc.jit.types.t_int8, uint(256 * te.interleave)), "iq3s.panel") + LLVMSetAlignment(te.iq3s_panel, 32u) + } var sa = SliceArgs(gc_impl = gc.impl, ctx = gc.jit.ctx, kstep = 1) sa.yp = LLVMGetParam(gc.impl, 0u) let kqp = LLVMGetParam(gc.impl, 1u) @@ -2563,12 +2700,10 @@ def private iq4xs_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, def private k3_gemv_gen(var gc : LlvmCodeCtx) : bool => kq_gemv_gen_impl(gc, 3) def private k3_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 3) -// iq3s: the grid-gather emitter arm lands with its kernel arc; until then both stamps decline -// (reference bodies serve) — a declined generator is the framework's own fallback path. -[unused_argument(gc)] -def private iq3s_gemv_gen(var gc : LlvmCodeCtx) : bool => false -[unused_argument(gc)] -def private iq3s_tile_gen(var gc : LlvmCodeCtx) : bool => false +// iq3s: the gemv gathers per superblock into an alloca panel (emit_iq3s_gather); the tile +// reads the runtime-unpacked panel — both then ride the iq4xs block lattice. +def private iq3s_gemv_gen(var gc : LlvmCodeCtx) : bool => kq_gemv_gen_impl(gc, 33) +def private iq3s_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 33) def private k4_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 4) def private k5_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 5) diff --git a/modules/dasLLAMA/dasllama/dasllama_math_gen.das b/modules/dasLLAMA/dasllama/dasllama_math_gen.das index a0fe82abcc..1426c92e72 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_gen.das @@ -657,7 +657,7 @@ def iq3sq8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp unsafe { for (t in range64(4l)) { for (r in range64(mr)) { - yp[(t0 + t) * d + g * mr + r] = iq3s_grp_row_dot(kqg, ksg, r, mr, xqp + (t0 + t) * n, xsp + (t0 + t) * (n / 256l), xbsp + (t0 + t) * (n / 16l), n) + yp[(t0 + t) * d + g * mr + r] = kq_grp_row_dot_b(33l, kqg, ksg, r, mr, xqp + (t0 + t) * n, xsp + (t0 + t) * (n / 256l), xbsp + (t0 + t) * (n / 16l), n) } } } @@ -919,9 +919,9 @@ def private q51q8_groupn_gen(var yp : float?; wq : uint8 const?; ws : uint8 cons -//! kq_grp_row_dot's twin over a BYTE-EXPANDED panel (unpack_kq_panel_grp's output) — the k5/k6 -//! tile wrappers' reference body, bit-exact against the packed-plane dot. fmt picks the scale -//! form only (5 = u8 sc/mn + d/dmin, 6 = signed per-16 sub-scales + d). +//! kq_grp_row_dot's twin over a BYTE-EXPANDED panel (unpack_kq_panel_grp's output) - bit-exact +//! against the packed-plane dot. fmt picks the scale form only: 5 = u8 sc/mn + d/dmin, 6 = +//! signed per-16 sub-scales + d, 33 = SIGNED panel bytes x per-32 (1 + 2s) + d, no bsum term. def kq_grp_row_dot_b(fmt : int64; kqg : uint8 const?; ksg : uint8 const?; r, mr : int64; xqp : int8 const?; xsp : float const?; xbsp : int const?; n : int64) : float { var acc = 0.0 let nsb = n / 256l @@ -938,13 +938,16 @@ def kq_grp_row_dot_b(fmt : int64; kqg : uint8 const?; ksg : uint8 const?; r, mr var ihi = 0 for (j in range64(4l)) { for (t in range64(4l)) { - let qlo = int(kqg[qb + ((blk * 4l + j) * mr + r) * 4l + t]) - let qhi = int(kqg[qb + 128l * mr + ((blk * 4l + j) * mr + r) * 4l + t]) + let qlo = fmt == 33l ? int(int8(kqg[qb + ((blk * 4l + j) * mr + r) * 4l + t])) : int(kqg[qb + ((blk * 4l + j) * mr + r) * 4l + t]) + let qhi = fmt == 33l ? int(int8(kqg[qb + 128l * mr + ((blk * 4l + j) * mr + r) * 4l + t])) : int(kqg[qb + 128l * mr + ((blk * 4l + j) * mr + r) * 4l + t]) ilo += qlo * int(xqp[b * 32l + j * 4l + t]) ihi += qhi * int(xqp[b * 32l + 16l + j * 4l + t]) } } - if (fmt == 6l) { + if (fmt == 33l) { + let sc = int(int8(ksg[sb + 4l * mr + blk * mr + r])) + isum += sc * (ilo + ihi) + } elif (fmt == 6l) { let s0 = int(int8(ksg[sb + 2l * blk * mr + r])) let s1 = int(int8(ksg[sb + (2l * blk + 1l) * mr + r])) isum += s0 * ilo + s1 * ihi @@ -956,7 +959,10 @@ def kq_grp_row_dot_b(fmt : int64; kqg : uint8 const?; ksg : uint8 const?; r, mr bsum += mn * (xbsp[b * 2l] + xbsp[b * 2l + 1l]) } } - if (fmt == 6l) { + if (fmt == 33l) { + let di = f16_to_f32(uint(ksg[sb + 2l * r]) | (uint(ksg[sb + 2l * r + 1l]) << 8u)) + acc += float(isum) * (di * xsp[sbi]) + } elif (fmt == 6l) { let dk6 = f16_to_f32(uint(ksg[sb + 16l * mr + 2l * r]) | (uint(ksg[sb + 16l * mr + 2l * r + 1l]) << 8u)) acc += float(isum - 32 * bsum) * (dk6 * xsp[sbi]) } else { @@ -1553,7 +1559,7 @@ def private kq_batch_cell_gen(fmt : int; var myp : float?; kqp : uint8 const?; k let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) let nb16 = n / 16l - let packed = fmt == 4 || fmt == 40 || fmt == 44 || fmt == 3 || fmt == 33 // packed planes: the tile reads them directly + let packed = fmt == 4 || fmt == 40 || fmt == 44 || fmt == 3 // packed planes: the tile reads them directly (iq3s expands its grid gather into the panel) var scratch : array if (!packed) { scratch |> resize(mr * nsb * 256l) @@ -1584,7 +1590,7 @@ def private kq_batch_cell_gen(fmt : int; var myp : float?; kqp : uint8 const?; k } elif (fmt == 3) { k3q8_tile_gen(myp, kqg, ksg, xqp, xsp, xbsp, n, d, g, tk) } elif (fmt == 33) { - iq3sq8_tile_gen(myp, kqg, ksg, xqp, xsp, xbsp, n, d, g, tk) + iq3sq8_tile_gen(myp, scp, ksg, xqp, xsp, xbsp, n, d, g, tk) } else { k6q8_tile_gen(myp, scp, ksg, xqp, xsp, xbsp, n, d, g, tk) } diff --git a/modules/dasLLAMA/dasllama/dasllama_repack.das b/modules/dasLLAMA/dasllama/dasllama_repack.das index fe0de23c4b..fddd374773 100644 --- a/modules/dasLLAMA/dasllama/dasllama_repack.das +++ b/modules/dasLLAMA/dasllama/dasllama_repack.das @@ -4,6 +4,7 @@ options _dasllama_internal = true module dasllama_repack shared public +require dasllama/dasllama_kqformat // iq3s_grid - the panel unpack gathers through the codebook require dasllama/dasllama_lint public require dasllama/dasllama_math_default // k4_sc_mn — the kq 6-bit sc/mn decoder (stays there: hot in the kq dots, auto-inline is same-module-only) @@ -516,7 +517,55 @@ def repack_iq3s_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { //! kqBytes stamps load verbatim). SWAR over uint64 lanes: k5 deposit uses an add-carry test //! ((x+0x7F)&0x80) to fold the high bit; k6 tops sit at a uniform shift per block. [hint(unsafe_range_check, noalias = kqg, noalias = dst)] +//! iq3s panel unpack: the grid gather + sign apply, packed grp planes -> the byte-expanded +//! signed panel. One grid word = 4 elements = ONE i32 store; the sign nibble expands through +//! the mask table and negates byte-wise (magnitudes are odd 1..15 - no cross-byte carry). +def unpack_iq3s_panel_grp(kqg : uint8 const?; var dst : uint8?; mr, nsb : int64) { + let grid = iq3s_grid() + let smask = fixed_array(0x00000000u, 0x000000FFu, 0x0000FF00u, 0x0000FFFFu, + 0x00FF0000u, 0x00FF00FFu, 0x00FFFF00u, 0x00FFFFFFu, + 0xFF000000u, 0xFF0000FFu, 0xFF00FF00u, 0xFF00FFFFu, + 0xFFFF0000u, 0xFFFF00FFu, 0xFFFFFF00u, 0xFFFFFFFFu) + unsafe { + for (sbi in range64(nsb)) { + let qb = sbi * 104l * mr + let ob = sbi * 256l * mr + for (r in range64(mr)) { + for (blk in range64(8l)) { + let hp = 64l + blk + let qh = int(kqg[qb + ((hp / 4l) * mr + r) * 4l + hp % 4l]) + for (l in range64(4l)) { + let sp = 72l + blk * 4l + l + let sgn = int(kqg[qb + ((sp / 4l) * mr + r) * 4l + sp % 4l]) + let q1p = blk * 8l + 2l * l + let q2p = q1p + 1l + let i1 = int(kqg[qb + ((q1p / 4l) * mr + r) * 4l + q1p % 4l]) | ((qh << int(8l - 2l * l)) & 256) + let i2 = int(kqg[qb + ((q2p / 4l) * mr + r) * 4l + q2p % 4l]) | ((qh << int(7l - 2l * l)) & 256) + let m1 = smask[sgn & 15] + let m2 = smask[(sgn >> 4) & 15] + let w1 = (grid[i1] ^ m1) + (m1 & 0x01010101u) + let w2 = (grid[i2] ^ m2) + (m2 & 0x01010101u) + // element e = l*8 + half*4 + t: e < 16 lands in the lo panel half, else hi + let e1 = l * 8l + let e2 = l * 8l + 4l + let d1 = (e1 < 16l ? 0l : 128l * mr) + ((blk * 4l + (e1 % 16l) / 4l) * mr + r) * 4l + let d2 = (e2 < 16l ? 0l : 128l * mr) + ((blk * 4l + (e2 % 16l) / 4l) * mr + r) * 4l + var wp1 = reinterpret(dst + (ob + d1)) + var wp2 = reinterpret(dst + (ob + d2)) + wp1[0] = w1 + wp2[0] = w2 + } + } + } + } + } +} + def unpack_kq_panel_grp(fmt : int64; kqg : uint8 const?; var dst : uint8?; mr, nsb : int64) { + if (fmt == 33l) { + unpack_iq3s_panel_grp(kqg, dst, mr, nsb) + return + } let LO4 = 0x0F0F0F0F0F0F0F0Ful let SEL = 0x0804020108040201ul // LE lane t (0..3, twice) selects high-bit t let C7F = 0x7F7F7F7F7F7F7F7Ful diff --git a/modules/dasLLAMA/followup_general.md b/modules/dasLLAMA/followup_general.md index ca5c0420f6..bba7498b8b 100644 --- a/modules/dasLLAMA/followup_general.md +++ b/modules/dasLLAMA/followup_general.md @@ -743,3 +743,24 @@ `DAS_TUNE_PARTIAL=1` (or `--tune-partial`) rail that the auto policy does not take, so a shipped box still mints whole. Done = the rail exists, a fresh family on a minted box costs one family's race, and the tuner's status line names the partial mint as such. + +60. **Retroactive llama.cpp CPU-kernel audit for the arc's earlier formats (Boris, + 2026-08-30: "lets make sure we do for all new CPU kernels, and if we skipped for previous + ones - lets ledger towards the end of this arc").** The IQ3_S CPU arm is the first written + against llama.cpp's arch kernel (signs-on-activation, sllv index compose, scalar grid + loads beating hardware gather); IQ4_XS and Q3_K were written from the disk format alone. + Toward the end of the arc: read `ggml_vec_dot_iq4_xs_q8_K` and `ggml_vec_dot_q3_K_q8_K` + (x86 + arm), list every technique ours does not measure, land the missing ones as + `[tune_perm]` spellings and let the probe judge. Done = a per-format note naming what was + raced and what won, beside the existing bench rows. + +61. **IQ3_S CPU decode: race a no-panel gemv spelling (the 0.92x tail).** The stamped gemv + gathers each superblock into an alloca panel and then runs the vector dot - a store/load + round trip per superblock that a single token never amortizes; llama.cpp's per-row form + (grid words composed straight into vectors, signs applied to the ACTIVATION via + shuffle+cmpeq/xor-sub, magnitudes kept unsigned for maddubs) edges it 57.0 vs 52.4 tg128 + on the zen2 (pp512 is ours 4.93x - the panel amortizes across the tile). The counter to + race as a [tune_perm]: compose the gathered words directly into the weight vectors + (insertelement per i32 lane, no panel), and/or the signs-on-activation form that drops + the abs+psign pair. Done = a gemv perm that takes tg128 at or past llama.cpp's, crowned + by the probe. diff --git a/modules/dasLLAMA/harness/gen_tune_probe.das b/modules/dasLLAMA/harness/gen_tune_probe.das index 69604d2955..171f92ebf4 100644 --- a/modules/dasLLAMA/harness/gen_tune_probe.das +++ b/modules/dasLLAMA/harness/gen_tune_probe.das @@ -763,7 +763,7 @@ def kq_layout_mrs(fmt : int64) : table { def run_kq_tile(tile; fx : KqFixture; kq : array; ks : array; mr : int64; var y : array) { let ng = fx.d / mr verify(fx.ntok % 4l == 0l) - let packed = fx.fmt == 4l || fx.fmt == 40l || fx.fmt == 44l || fx.fmt == 3l || fx.fmt == 33l // packed planes: the tile reads them directly + let packed = fx.fmt == 4l || fx.fmt == 40l || fx.fmt == 44l || fx.fmt == 3l // packed planes: the tile reads them directly (iq3s unpacks its panel per group) var panel : array if (!packed) { panel |> resize(int(mr * fx.nsb * 256l)) @@ -1245,6 +1245,7 @@ def test_mode : bool { // nolint:STYLE037,STYLE038 — flat per-variant correctn var kfxs40 <- [ <- build_kq_fixture(40l, 256l, 32l, 8l), <- build_kq_fixture(40l, 768l, 32l, 8l), <- build_kq_fixture(40l, 2048l, 32l, 8l)] var kfxs44 <- [ <- build_kq_fixture(44l, 256l, 32l, 8l), <- build_kq_fixture(44l, 768l, 32l, 8l), <- build_kq_fixture(44l, 2048l, 32l, 8l)] var kfxs3 <- [ <- build_kq_fixture(3l, 256l, 32l, 8l), <- build_kq_fixture(3l, 768l, 32l, 8l), <- build_kq_fixture(3l, 2048l, 32l, 8l)] + var kfxs33 <- [ <- build_kq_fixture(33l, 256l, 32l, 8l), <- build_kq_fixture(33l, 768l, 32l, 8l), <- build_kq_fixture(33l, 2048l, 32l, 8l)] var vs <- q8q8_tile_gen_variants() var mrs <- variant_mrs() var wbs <- variant_wbias() @@ -1371,6 +1372,7 @@ def test_mode : bool { // nolint:STYLE037,STYLE038 — flat per-variant correctn allok = kq_test_family(40l, kfxs40) && allok allok = kq_test_family(44l, kfxs44) && allok allok = kq_test_family(3l, kfxs3) && allok + allok = kq_test_family(33l, kfxs33) && allok delete wtab delete mtab delete gtab @@ -1388,6 +1390,7 @@ def test_mode : bool { // nolint:STYLE037,STYLE038 — flat per-variant correctn delete kfxs40 delete kfxs44 delete kfxs3 + delete kfxs33 return allok } From d90e3a8d119cfdc8a299e931cf36a505bd4eb5c3 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Sun, 30 Aug 2026 16:38:08 -0700 Subject: [PATCH 19/61] dasLLAMA: IQ3_S on the Vulkan tier - workgroup-staged grid, 64/64 ids, tg 0.89x The first grid format on the tier. The 2 KB iq3s grid stages into @workgroup uint[512] once per kernel (llama.cpp's init_iq_shmem form; the source is iq3s_grid_word, a per-index accessor - a shader function cannot return a fixed array - whose QUIRK-20 storage class never sits on a hot path), and the sign nibble expands arithmetically (mask = ((nib * 0x00204081) & 0x01010101) * 255, negate (g ^ mask) + m1). KqGemvIq3s reads the 26-word rows as scalar uints (104 B is not uint4-aligned) with iq4xs's scale fold; KqBatchIq3s : KqBatchIq4xs gathers the grid at stage time and inherits the scale staging and blk_fma verbatim. vk_kq_schema_id 8 -> 33; the batch/gemv ensure/enc ladders grow arms; kq_cls_ref fills the replay's workgroup grid (the CPU replay never runs the kernel head) and the iq3s float witness recomputes rows straight off the plane bytes. Two traps fixed on the way, both now hardened: - pf_f16_feed admitted every kq_sb format, so iq3s prefill hit the cm2 dispatcher whose fall-through silently served the Q8 TILES over iq3s planes - coherent-speed garbage. The feed excludes iq3s until its cm2 tile lands, and all three cm2 ladders verify on any non-q8 fall-through instead of serving the wrong format's pipeline. - the batch family cell's enc ladder lacked the iq3s arm, dispatching the k6 pipeline over iq3s fixtures - 2200/2200 off with byte-stable values across three fix rounds. Gates on the 5060 Ti: test_vulkan_kernels 71/72 (the standard mode-ladder skip) with the seven-format GEMV/batch family cells and the independent iq3s witness; the resident driver matches llama.cpp's greedy ids 64 of 64 at gen 262 t/s. Rows vs llama.cpp b10660 Vulkan: tg128 288.5 +- 0.7 vs 324.2 (0.89x); pp512 6241 +- 9 on the quant feed vs 17865 (0.35x - the cm2 tile on the IQLUT-axis template is the follow-on lever, the same pre-template gap class iq4xs/k3 had at 0.30x). Co-Authored-By: Claude Fable 5 --- modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md | 27 ++- .../dasllama/dasllama_vulkan_classes.das | 185 +++++++++++++++++- .../dasllama/dasllama_vulkan_common.das | 7 +- .../dasllama/dasllama_vulkan_prefill.das | 8 +- modules/dasLLAMA/tests/_vkd_oracles.das | 43 ++++ .../dasLLAMA/tests/test_vulkan_kernels.das | 28 ++- 6 files changed, 280 insertions(+), 18 deletions(-) diff --git a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md index 45d4e04aea..acac8c492e 100644 --- a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md +++ b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md @@ -448,9 +448,30 @@ llama.cpp b10660 clean-cpu): pp512 516.9 vs 104.9 (**4.93x** - the panel amortiz gather across the tile; their per-row kernel re-gathers per token), tg128 52.4 vs 57.0 (0.92x - nothing amortizes at one token; the no-panel gemv spelling to close it is followup_general #61). Stamped e2e: 63/64 greedy ids (the flip is the FINAL token, the -stamped-vs-reference near-tie class), gen 44 t/s. Vulkan, Metal: pending (pre-researched: -the IQLUT axis on the cm2 template takes the 2 KB grid; threadgroup floats + the -base-pointer 9th-bit trick on Metal). +stamped-vs-reference near-tie class), gen 44 t/s. + +Vulkan (section 6, the first grid format on the tier): `vk_kq_schema_id` 8 -> 33; the 2 KB +grid stages into `@workgroup uint[512]` once per kernel (llama.cpp's `init_iq_shmem` form) - +the source is `iq3s_grid_word`, a per-index accessor over a fixed_array local (QUIRK 20's +slow path, off the hot loop; a shader function cannot RETURN a fixed array), and the sign +nibble expands arithmetically (`mask = ((nib * 0x00204081) & 0x01010101) * 255`, negate = +`(g ^ mask) + m1`). `KqGemvIq3s : KqGemvBase` (26-word rows read as scalar uints - 104 B is +not uint4-aligned; iq4xs's scale fold via `iq4_sc`); `KqBatchIq3s : KqBatchIq4xs` (the grid +gather AT STAGE TIME per staged uint4; scale row and blk_fma inherited verbatim). TWO traps +this walk found: a test family cell whose enc ladder lacks the new arm silently dispatches +the ELSE format's pipeline over the new planes (fmt 8 ran the k6 tiles - 2200/2200 off with +byte-stable garbage across "fix" rounds; add the enc arm to EVERY per-fmt ladder in the +cell, not just ensure/strides); and `pf_f16_feed` admitting a format via `kq_sb` before its +cm2 tile exists sent iq3s prefill through the cm2 dispatcher whose fall-through served the +q8 tiles - garbage text at full speed. The feed now excludes iq3s until its tile lands and +all three cm2 ladders `verify` on a non-q8 fall-through. The CPU replay of a class whose +kernel stages workgroup state (the grid) must fill that state in `kq_cls_ref` - the replay +never runs the kernel head. Gates: the suite 71/72 with the seven-format family cells and +the iq3s float witness; the resident driver matches llama.cpp's greedy ids **64 of 64** at +gen 262 t/s. Rows (5060 Ti vs llama.cpp b10660 Vulkan): tg128 288.5 vs 324.2 (0.89x), pp512 +6241 vs 17865 (0.35x - the quant feed; the cm2 tile on the IQLUT-axis template is the +opening lever, exactly iq4xs/k3's pre-template gap class). Metal: pending (threadgroup +floats + the base-pointer 9th-bit trick). ### Q3_K (the second format, 2026-08-30) diff --git a/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das b/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das index a3eb0064dc..9d6f7a8e5b 100644 --- a/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das +++ b/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das @@ -2468,6 +2468,78 @@ def private k5_dep(hb : uint) : uint { // SIGNED int8 lanes (ggml's kvalues_iq4nl) — the sdot4 operand form; batch and gemv share it. // The codebook is four packed words picked by a dynamic vector index and byte-extracted: pure // ALU (a fixed_array local was a Function-storage table, 3.4x slower on the decode GEMV) +// ggml's iq3s_grid, one word per call - the staging loops copy it into workgroup memory once +// per kernel (a few indexed reads per thread), so the local-array storage class (QUIRK 20's +// slow path) never sits on a hot path; a shader function cannot return a fixed array. +def private iq3s_grid_word(i : int) : uint { + let tbl = fixed_array( + 0x01010101u, 0x01010103u, 0x01010105u, 0x0101010bu, 0x0101010fu, 0x01010301u, 0x01010303u, 0x01010305u, + 0x01010309u, 0x0101030du, 0x01010501u, 0x01010503u, 0x0101050bu, 0x01010707u, 0x01010901u, 0x01010905u, + 0x0101090bu, 0x0101090fu, 0x01010b03u, 0x01010b07u, 0x01010d01u, 0x01010d05u, 0x01010f03u, 0x01010f09u, + 0x01010f0fu, 0x01030101u, 0x01030103u, 0x01030105u, 0x01030109u, 0x01030301u, 0x01030303u, 0x0103030bu, + 0x01030501u, 0x01030507u, 0x0103050fu, 0x01030703u, 0x0103070bu, 0x01030909u, 0x01030d03u, 0x01030d0bu, + 0x01030f05u, 0x01050101u, 0x01050103u, 0x0105010bu, 0x0105010fu, 0x01050301u, 0x01050307u, 0x0105030du, + 0x01050503u, 0x0105050bu, 0x01050701u, 0x01050709u, 0x01050905u, 0x0105090bu, 0x0105090fu, 0x01050b03u, + 0x01050b07u, 0x01050f01u, 0x01050f07u, 0x01070107u, 0x01070303u, 0x0107030bu, 0x01070501u, 0x01070505u, + 0x01070703u, 0x01070707u, 0x0107070du, 0x01070909u, 0x01070b01u, 0x01070b05u, 0x01070d0fu, 0x01070f03u, + 0x01070f0bu, 0x01090101u, 0x01090307u, 0x0109030fu, 0x01090503u, 0x01090509u, 0x01090705u, 0x01090901u, + 0x01090907u, 0x01090b03u, 0x01090f01u, 0x010b0105u, 0x010b0109u, 0x010b0501u, 0x010b0505u, 0x010b050du, + 0x010b0707u, 0x010b0903u, 0x010b090bu, 0x010b090fu, 0x010b0d0du, 0x010b0f07u, 0x010d010du, 0x010d0303u, + 0x010d0307u, 0x010d0703u, 0x010d0b05u, 0x010d0f03u, 0x010f0101u, 0x010f0105u, 0x010f0109u, 0x010f0501u, + 0x010f0505u, 0x010f050du, 0x010f0707u, 0x010f0b01u, 0x010f0b09u, 0x03010101u, 0x03010103u, 0x03010105u, + 0x03010109u, 0x03010301u, 0x03010303u, 0x03010307u, 0x0301030bu, 0x0301030fu, 0x03010501u, 0x03010505u, + 0x03010703u, 0x03010709u, 0x0301070du, 0x03010b09u, 0x03010b0du, 0x03010d03u, 0x03010f05u, 0x03030101u, + 0x03030103u, 0x03030107u, 0x0303010du, 0x03030301u, 0x03030309u, 0x03030503u, 0x03030701u, 0x03030707u, + 0x03030903u, 0x03030b01u, 0x03030b05u, 0x03030f01u, 0x03030f0du, 0x03050101u, 0x03050305u, 0x0305030bu, + 0x0305030fu, 0x03050501u, 0x03050509u, 0x03050705u, 0x03050901u, 0x03050907u, 0x03050b0bu, 0x03050d01u, + 0x03050f05u, 0x03070103u, 0x03070109u, 0x0307010fu, 0x03070301u, 0x03070307u, 0x03070503u, 0x0307050fu, + 0x03070701u, 0x03070709u, 0x03070903u, 0x03070d05u, 0x03070f01u, 0x03090107u, 0x0309010bu, 0x03090305u, + 0x03090309u, 0x03090703u, 0x03090707u, 0x03090905u, 0x0309090du, 0x03090b01u, 0x03090b09u, 0x030b0103u, + 0x030b0301u, 0x030b0307u, 0x030b0503u, 0x030b0701u, 0x030b0705u, 0x030b0b03u, 0x030d0501u, 0x030d0509u, + 0x030d050fu, 0x030d0909u, 0x030d090du, 0x030f0103u, 0x030f0107u, 0x030f0301u, 0x030f0305u, 0x030f0503u, + 0x030f070bu, 0x030f0903u, 0x030f0d05u, 0x030f0f01u, 0x05010101u, 0x05010103u, 0x05010107u, 0x0501010bu, + 0x0501010fu, 0x05010301u, 0x05010305u, 0x05010309u, 0x0501030du, 0x05010503u, 0x05010507u, 0x0501050fu, + 0x05010701u, 0x05010705u, 0x05010903u, 0x05010907u, 0x0501090bu, 0x05010b01u, 0x05010b05u, 0x05010d0fu, + 0x05010f01u, 0x05010f07u, 0x05010f0bu, 0x05030101u, 0x05030105u, 0x05030301u, 0x05030307u, 0x0503030fu, + 0x05030505u, 0x0503050bu, 0x05030703u, 0x05030709u, 0x05030905u, 0x05030b03u, 0x05050103u, 0x05050109u, + 0x0505010fu, 0x05050503u, 0x05050507u, 0x05050701u, 0x0505070fu, 0x05050903u, 0x05050b07u, 0x05050b0fu, + 0x05050f03u, 0x05050f09u, 0x05070101u, 0x05070105u, 0x0507010bu, 0x05070303u, 0x05070505u, 0x05070509u, + 0x05070703u, 0x05070707u, 0x05070905u, 0x05070b01u, 0x05070d0du, 0x05090103u, 0x0509010fu, 0x05090501u, + 0x05090507u, 0x05090705u, 0x0509070bu, 0x05090903u, 0x05090f05u, 0x05090f0bu, 0x050b0109u, 0x050b0303u, + 0x050b0505u, 0x050b070fu, 0x050b0901u, 0x050b0b07u, 0x050b0f01u, 0x050d0101u, 0x050d0105u, 0x050d010fu, + 0x050d0503u, 0x050d0b0bu, 0x050d0d03u, 0x050f010bu, 0x050f0303u, 0x050f050du, 0x050f0701u, 0x050f0907u, + 0x050f0b01u, 0x07010105u, 0x07010303u, 0x07010307u, 0x0701030bu, 0x0701030fu, 0x07010505u, 0x07010703u, + 0x07010707u, 0x0701070bu, 0x07010905u, 0x07010909u, 0x0701090fu, 0x07010b03u, 0x07010d07u, 0x07010f03u, + 0x07030103u, 0x07030107u, 0x0703010bu, 0x07030309u, 0x07030503u, 0x07030507u, 0x07030901u, 0x07030d01u, + 0x07030f05u, 0x07030f0du, 0x07050101u, 0x07050305u, 0x07050501u, 0x07050705u, 0x07050709u, 0x07050b01u, + 0x07070103u, 0x07070301u, 0x07070309u, 0x07070503u, 0x07070507u, 0x0707050fu, 0x07070701u, 0x07070903u, + 0x07070907u, 0x0707090fu, 0x07070b0bu, 0x07070f07u, 0x07090107u, 0x07090303u, 0x0709030du, 0x07090505u, + 0x07090703u, 0x07090b05u, 0x07090d01u, 0x07090d09u, 0x070b0103u, 0x070b0301u, 0x070b0305u, 0x070b050bu, + 0x070b0705u, 0x070b0909u, 0x070b0b0du, 0x070b0f07u, 0x070d030du, 0x070d0903u, 0x070f0103u, 0x070f0107u, + 0x070f0501u, 0x070f0505u, 0x070f070bu, 0x09010101u, 0x09010109u, 0x09010305u, 0x09010501u, 0x09010509u, + 0x0901050fu, 0x09010705u, 0x09010903u, 0x09010b01u, 0x09010f01u, 0x09030105u, 0x0903010fu, 0x09030303u, + 0x09030307u, 0x09030505u, 0x09030701u, 0x0903070bu, 0x09030907u, 0x09030b03u, 0x09030b0bu, 0x09050103u, + 0x09050107u, 0x09050301u, 0x0905030bu, 0x09050503u, 0x09050707u, 0x09050901u, 0x09050b0fu, 0x09050d05u, + 0x09050f01u, 0x09070109u, 0x09070303u, 0x09070307u, 0x09070501u, 0x09070505u, 0x09070703u, 0x0907070bu, + 0x09090101u, 0x09090105u, 0x09090509u, 0x0909070fu, 0x09090901u, 0x09090f03u, 0x090b010bu, 0x090b010fu, + 0x090b0503u, 0x090b0d05u, 0x090d0307u, 0x090d0709u, 0x090d0d01u, 0x090f0301u, 0x090f030bu, 0x090f0701u, + 0x090f0907u, 0x090f0b03u, 0x0b010105u, 0x0b010301u, 0x0b010309u, 0x0b010505u, 0x0b010901u, 0x0b010909u, + 0x0b01090fu, 0x0b010b05u, 0x0b010d0du, 0x0b010f09u, 0x0b030103u, 0x0b030107u, 0x0b03010bu, 0x0b030305u, + 0x0b030503u, 0x0b030705u, 0x0b030f05u, 0x0b050101u, 0x0b050303u, 0x0b050507u, 0x0b050701u, 0x0b05070du, + 0x0b050b07u, 0x0b070105u, 0x0b07010fu, 0x0b070301u, 0x0b07050fu, 0x0b070909u, 0x0b070b03u, 0x0b070d0bu, + 0x0b070f07u, 0x0b090103u, 0x0b090109u, 0x0b090501u, 0x0b090705u, 0x0b09090du, 0x0b0b0305u, 0x0b0b050du, + 0x0b0b0b03u, 0x0b0b0b07u, 0x0b0d0905u, 0x0b0f0105u, 0x0b0f0109u, 0x0b0f0505u, 0x0d010303u, 0x0d010307u, + 0x0d01030bu, 0x0d010703u, 0x0d010707u, 0x0d010d01u, 0x0d030101u, 0x0d030501u, 0x0d03050fu, 0x0d030d09u, + 0x0d050305u, 0x0d050709u, 0x0d050905u, 0x0d050b0bu, 0x0d050d05u, 0x0d050f01u, 0x0d070101u, 0x0d070309u, + 0x0d070503u, 0x0d070901u, 0x0d09050bu, 0x0d090907u, 0x0d090d05u, 0x0d0b0101u, 0x0d0b0107u, 0x0d0b0709u, + 0x0d0b0d01u, 0x0d0d010bu, 0x0d0d0901u, 0x0d0f0303u, 0x0d0f0307u, 0x0f010101u, 0x0f010109u, 0x0f01010fu, + 0x0f010501u, 0x0f010505u, 0x0f01070du, 0x0f010901u, 0x0f010b09u, 0x0f010d05u, 0x0f030105u, 0x0f030303u, + 0x0f030509u, 0x0f030907u, 0x0f03090bu, 0x0f050103u, 0x0f050109u, 0x0f050301u, 0x0f05030du, 0x0f050503u, + 0x0f050701u, 0x0f050b03u, 0x0f070105u, 0x0f070705u, 0x0f07070bu, 0x0f070b07u, 0x0f090103u, 0x0f09010bu, + 0x0f090307u, 0x0f090501u, 0x0f090b01u, 0x0f0b0505u, 0x0f0b0905u, 0x0f0d0105u, 0x0f0d0703u, 0x0f0f0101u) + return tbl[i] +} + def private iq4_word(n : uint) : uint { let tbl = uint4(3215825025u, 4142587343u, 639175937u, 1901675829u) // kvalues_iq4nl, 4 bytes per word let q0 = n & 15u @@ -2750,6 +2822,55 @@ class KqBatchIq4xs : KqBatchBase { } } +// IQ3_S tile: the iq4xs staging with the GRID GATHER in stage_w - each staged uint4 is four +// grid words sign-applied (mask = ((nibble * 0x00204081) & 0x01010101) * 255; magnitudes odd +// 1..15, no cross-byte carry); scale row and blk_fma are iq4xs's verbatim (inherited). The 2 KB +// grid stages into workgroup memory once, before the tile loop. +[vk_dispatch(name = "kq_batch_iq3s_cls", kernel = "run", family = "kq_batch_cls", grid = "wgs", params = "wgs : int64")] +class KqBatchIq3s : KqBatchIq4xs { + @workgroup gridw : uint[512] + + def iq3s_signed(g, nib : uint) : uint { + let m1 = (nib * 0x00204081u) & 0x01010101u + let mask = m1 * 255u + return (g ^ mask) + m1 + } + + def override stage_w(wsb0, wcol, nsb, s, word, grp : uint) { + for (ki in range(2)) { + let k = word + uint(ki) * 8u + let bu = k / 2u + let hh = k % 2u + var wv = uint4(0u) + if (wcol < pa.d) { + let wsb = wsb0 + wcol * nsb + s + let qsw = wq[wsb * 26u + bu * 2u + hh] + let qh = (wq[wsb * 26u + 16u + bu / 4u] >> ((bu % 4u) * 8u)) & 255u + let sgw = wq[wsb * 26u + 18u + bu] + let l0 = hh * 2u + let sg0 = (sgw >> (l0 * 8u)) & 255u + let sg1 = (sgw >> (l0 * 8u + 8u)) & 255u + let i0 = (qsw & 255u) | ((qh << (8u - 2u * l0)) & 256u) + let i1 = ((qsw >> 8u) & 255u) | ((qh << (7u - 2u * l0)) & 256u) + let i2 = ((qsw >> 16u) & 255u) | ((qh << (8u - 2u * (l0 + 1u))) & 256u) + let i3 = (qsw >> 24u) | ((qh << (7u - 2u * (l0 + 1u))) & 256u) + wv = uint4(iq3s_signed(gridw[i0], sg0 & 15u), iq3s_signed(gridw[i1], sg0 >> 4u), + iq3s_signed(gridw[i2], sg1 & 15u), iq3s_signed(gridw[i3], sg1 >> 4u)) + } + ww4[grp * 17u + k] = wv + } + } + + [spirv_kernel(local_size_x = 256, name = "kq_batch_iq3s_cls_spv")] + def override run { + let tid = gl_LocalInvocationID.x + gridw[tid] = iq3s_grid_word(int(tid)) + gridw[tid + 256u] = iq3s_grid_word(int(tid + 256u)) + barrier() // the grid visible before the first stage_w + tile_shell() + } +} + // Q5_K tile: the k4 compose with the 5th bit OR'd in from the 40B block's qh words [vk_dispatch(name = "kq_batch_k5_cls", kernel = "run", family = "kq_batch_cls", grid = "wgs", params = "wgs : int64")] class KqBatchK5 : KqBatchBase { @@ -4833,6 +4954,8 @@ def kq_batch_cls_ensure(fmt : int; q40cm : bool) : bool { return ensure_kq_batch_iq4xs_cls() } elif (fmt == int(KqFmt.k3)) { return ensure_kq_batch_k3_cls() + } elif (fmt == int(KqFmt.iq3s)) { + return ensure_kq_batch_iq3s_cls() } panic("dasLLAMA vulkan tier: no kq batch class for format {fmt}") return false @@ -4863,6 +4986,8 @@ def kq_batch_cls_enc_for(fmt : int; q40cm : bool; raw : VkCommandBuffer; var h : enc_kq_batch_iq4xs_cls(raw, h, set_, pc, wgs) } elif (fmt == int(KqFmt.k3)) { enc_kq_batch_k3_cls(raw, h, set_, pc, wgs) + } elif (fmt == int(KqFmt.iq3s)) { + enc_kq_batch_iq3s_cls(raw, h, set_, pc, wgs) } else { panic("dasLLAMA vulkan tier: no kq batch class for format {fmt}") } @@ -4887,6 +5012,8 @@ def gemv_cls_ensure(fmt : int) : bool { return ensure_kq_gemv_iq4xs_cls() } elif (fmt == int(KqFmt.k3)) { return ensure_kq_gemv_k3_cls() + } elif (fmt == int(KqFmt.iq3s)) { + return ensure_kq_gemv_iq3s_cls() } panic("dasLLAMA vulkan tier: no GEMV class for format {fmt}") return false @@ -4917,14 +5044,16 @@ def gemv_cls_enc(fmt : int; raw : VkCommandBuffer; var h : VkHaz; var set_ : VkD enc_kq_gemv_iq4xs_cls(raw, h, set_, pc, wgs) } elif (fmt == int(KqFmt.k3)) { enc_kq_gemv_k3_cls(raw, h, set_, pc, wgs) + } elif (fmt == int(KqFmt.iq3s)) { + enc_kq_gemv_iq3s_cls(raw, h, set_, pc, wgs) } else { panic("dasLLAMA vulkan tier: no GEMV class for format {fmt}") } } // Q4_K: w = d*sc*q - dmin*mn*1, q in [0,15] — per 32-block axs*(d*sc*idot - dmin*mn*bsum). -// The six formats share family=kq_gemv_cls: ONE set layout + ONE set_kq_gemv_cls serves all -// six pipelines (identical inherited members), each leaf keeping its own ensure_/enc_ names. +// The seven formats share family=kq_gemv_cls: ONE set layout + ONE set_kq_gemv_cls serves all +// seven pipelines (identical inherited members), each leaf keeping its own ensure_/enc_ names. [vk_dispatch(name = "kq_gemv_k4_cls", kernel = "run", family = "kq_gemv_cls", grid = "wgs", params = "wgs : int64")] class KqGemvK4 : KqGemvBase { def override blk_contrib(wsb, blk, xsb : uint) : float { @@ -4987,6 +5116,58 @@ class KqGemvIq4xs : KqGemvBase { } } +// IQ3_S: grid gather (8-bit index | qh ninth bit -> 4 magnitudes per word), the block's own +// sign bytes flip them, per-32 signed (1 + 2s) sub-scale x d - iq4xs's fold (no bsum term) +// over the 26-word rows; the 2 KB grid stages into workgroup memory ahead of the shell. +[vk_dispatch(name = "kq_gemv_iq3s_cls", kernel = "run", family = "kq_gemv_cls", grid = "wgs", params = "wgs : int64")] +class KqGemvIq3s : KqGemvBase { + @workgroup gridw : uint[512] + + def iq3s_signed(g, nib : uint) : uint { + let m1 = (nib * 0x00204081u) & 0x01010101u + let mask = m1 * 255u + return (g ^ mask) + m1 + } + + def override blk_contrib(wsb, blk, xsb : uint) : float { + let qb = wsb * 26u + blk * 2u + let qh = (wq[wsb * 26u + 16u + blk / 4u] >> ((blk % 4u) * 8u)) & 255u + let sgw = wq[wsb * 26u + 18u + blk] + let xb4 = xsb * 16u + blk * 2u + let xl = xq4[xb4] + let xh = xq4[xb4 + 1u] + var isum = 0 + for (l in range(2)) { + let qsw = wq[qb + uint(l)] + let l0 = uint(l) * 2u + let sg0 = (sgw >> (l0 * 8u)) & 255u + let sg1 = (sgw >> (l0 * 8u + 8u)) & 255u + let w0 = iq3s_signed(gridw[(qsw & 255u) | ((qh << (8u - 2u * l0)) & 256u)], sg0 & 15u) + let w1 = iq3s_signed(gridw[((qsw >> 8u) & 255u) | ((qh << (7u - 2u * l0)) & 256u)], sg0 >> 4u) + let w2 = iq3s_signed(gridw[((qsw >> 16u) & 255u) | ((qh << (8u - 2u * (l0 + 1u))) & 256u)], sg1 & 15u) + let w3 = iq3s_signed(gridw[(qsw >> 24u) | ((qh << (7u - 2u * (l0 + 1u))) & 256u)], sg1 >> 4u) + if (l == 0) { + isum += sdot4(w0, xl.x) + sdot4(w1, xl.y) + sdot4(w2, xl.z) + sdot4(w3, xl.w) + } else { + isum += sdot4(w0, xh.x) + sdot4(w1, xh.y) + sdot4(w2, xh.z) + sdot4(w3, xh.w) + } + } + let dd = unpackHalf2x16(wsu[wsb * 5u]).x + return axs[xsb] * dd * float(iq4_sc(wsu[wsb * 5u + 1u + blk / 4u], blk) * isum) + } + + [spirv_kernel(local_size_x = 64, name = "kq_gemv_iq3s_cls_spv")] + def run { + let lid = gl_LocalInvocationID.x + for (k in range(8)) { + let idx = lid + uint(k) * 64u + gridw[idx] = iq3s_grid_word(int(idx)) + } + barrier() // the grid visible before the shell's block walk + gemv_shell() + } +} + // Q3_K: q = 2-bit qs lane | hmask bit << 2, minus 4, per-16 SIGNED sub-scales x d — k6's fold with // the offset 4 over the k3 planes (6 uint4 per superblock: qs 0..3, hmask 4..5) [vk_dispatch(name = "kq_gemv_k3_cls", kernel = "run", family = "kq_gemv_cls", grid = "wgs", params = "wgs : int64")] diff --git a/modules/dasLLAMA/dasllama/dasllama_vulkan_common.das b/modules/dasLLAMA/dasllama/dasllama_vulkan_common.das index ffbd29e2e9..e998199030 100644 --- a/modules/dasLLAMA/dasllama/dasllama_vulkan_common.das +++ b/modules/dasLLAMA/dasllama/dasllama_vulkan_common.das @@ -1709,7 +1709,7 @@ struct ArenaSlab { } struct ArenaFmt { - fmt : int // int(KqFmt): 0 = q8, 1/2/3 = k4/k5/k6, 4 = q40, 6 = iq4xs, 7 = k3 + fmt : int // int(KqFmt): 0 = q8, 1/2/3 = k4/k5/k6, 4 = q40, 6 = iq4xs, 7 = k3, 8 = iq3s slabs : array cur : int // slab the bump cursor is in slab_cap : int64 // max blocks one slab may carry (msr over both plane strides) @@ -1740,7 +1740,7 @@ def arena_local_blk(blk : int64) : int64 => blk & ((1l << 32l) - 1l) // per-block plane strides — the block is 32 weights for q8 and a 256-weight superblock for kq. // Mirrors stack_plane_bytes: it computes the same totals as count * these strides. -// the tier's stack fmt space is int(KqFmt) — bridge to the gemm_schema kq id space (4/5/6/40/44/3) +// the tier's stack fmt space is int(KqFmt) — bridge to the gemm_schema kq id space (4/5/6/40/44/3/33) def vk_kq_schema_id(fmt : int) : int { if (fmt == 1) return 4 if (fmt == 2) return 5 @@ -1748,11 +1748,12 @@ def vk_kq_schema_id(fmt : int) : int { if (fmt == 4) return 40 if (fmt == 6) return 44 if (fmt == 7) return 3 + if (fmt == 8) return 33 panic("vk_kq_schema_id: not a kq stack fmt") return 0 } -// fmt space here = int(KqFmt): 0 q8 (32-elem blocks), 1-4, 6 and 7 the kq lattice (+ q40, iq4xs, k3) +// fmt space here = int(KqFmt): 0 q8 (32-elem blocks), 1-4 and 6..8 the kq lattice (+ q40, iq4xs, k3, iq3s) def vk_fmt_b32(fmt : int) : bool => fmt == 0 // meta/activation block unit of a stack format — q8 counts 32-weight blocks, the kq diff --git a/modules/dasLLAMA/dasllama/dasllama_vulkan_prefill.das b/modules/dasLLAMA/dasllama/dasllama_vulkan_prefill.das index 0f104b0155..262def37c1 100644 --- a/modules/dasLLAMA/dasllama/dasllama_vulkan_prefill.das +++ b/modules/dasLLAMA/dasllama/dasllama_vulkan_prefill.das @@ -40,7 +40,8 @@ var private @scratch g_pf_ids_stage : array [arch(at="../ARCHITECTURE_RUNTIME.md#activation-scale-lattice")] def private pf_f16_feed(f : int) : bool { - return (f == int(KqFmt.q8) || kq_sb(f)) && g_gpu.coopmat_mode == 4 + // iq3s stays on the quant feed until its cm2 tile lands (the old fall-through served q8 tiles over iq3s planes) + return (f == int(KqFmt.q8) || (kq_sb(f) && f != int(KqFmt.iq3s))) && g_gpu.coopmat_mode == 4 } [arch(at="../ARCHITECTURE_RUNTIME.md#activation-scale-lattice")] @@ -76,6 +77,7 @@ def private cm2_cls_ensure(fmt : int; tc : int64) : bool { if (fmt == int(KqFmt.k3)) { return tc == 256l ? ensure_kq_batch_k3_cm2l_cls() : (tc == 128l ? ensure_kq_batch_k3_cm2m_cls() : ensure_kq_batch_k3_cm2s_cls()) } + verify(fmt == int(KqFmt.q8), "vk prefill: the cm2 ensure fell through for a kq format - a format the feed admits must own a tile arm") return tc == 256l ? ensure_q8_batch_cm2l_cls() : (tc == 128l ? ensure_q8_batch_cm2m_cls() : ensure_q8_batch_cm2s_cls()) } @@ -117,6 +119,7 @@ def private cm2_cls_set(fmt : int; tc : int64; bufs : uint64 const[5]; sizes : i } return tc == 128l ? set_kq_batch_k3_cm2m_cls(bufs, sizes, gbits) : set_kq_batch_k3_cm2s_cls(bufs, sizes, gbits) } + verify(fmt == int(KqFmt.q8), "vk prefill: the cm2 set fell through for a kq format - a format the feed admits must own a tile arm") if (tc == 256l) { return set_q8_batch_cm2l_cls(bufs, sizes, gbits) } @@ -174,10 +177,13 @@ def private cm2_cls_enc(fmt : int; tc : int64; raw : VkCommandBuffer; var h : Vk enc_kq_batch_k3_cm2s_cls(raw, h, s, pc, groups) } } elif (tc == 256l) { + verify(fmt == int(KqFmt.q8), "vk prefill: the cm2 enc fell through for a kq format - a format the feed admits must own a tile arm") enc_q8_batch_cm2l_cls(raw, h, s, pc, groups) } elif (tc == 128l) { + verify(fmt == int(KqFmt.q8), "vk prefill: the cm2 enc fell through for a kq format - a format the feed admits must own a tile arm") enc_q8_batch_cm2m_cls(raw, h, s, pc, groups) } else { + verify(fmt == int(KqFmt.q8), "vk prefill: the cm2 enc fell through for a kq format - a format the feed admits must own a tile arm") enc_q8_batch_cm2s_cls(raw, h, s, pc, groups) } } diff --git a/modules/dasLLAMA/tests/_vkd_oracles.das b/modules/dasLLAMA/tests/_vkd_oracles.das index c00be6134d..72250c3117 100644 --- a/modules/dasLLAMA/tests/_vkd_oracles.das +++ b/modules/dasLLAMA/tests/_vkd_oracles.das @@ -173,6 +173,12 @@ def kq_cls_ref(fmt : int; wq, ws, xq : array; xs : array) : KqGemvB k = new KqGemvIq4xs() } elif (fmt == int(KqFmt.k3)) { k = new KqGemvK3() + } elif (fmt == int(KqFmt.iq3s)) { + var ki = new KqGemvIq3s() + for (i in range(512)) { // the CPU replay never runs the staging kernel head - fill the workgroup grid here + ki.gridw[i] = IQ3S_GRID[i] + } + k = ki } else { k = new KqGemvK6() } @@ -282,6 +288,43 @@ def k3_gemv_float_oracle(wq, ws, xq : array; xs : array; return <- y } +// IQ3_S GEMV float oracle, INDEPENDENT of the class bit-math: w = d * (1 + 2s) * grid_byte * +// sign straight off the plane bytes ([qs 64][qh 8][signs 32], the k4-shaped scale row), x off +// the Q8_K bytes - the witness a wrong index/sign pick in the gather cannot pass +def iq3s_gemv_float_oracle(wq, ws, xq : array; xs : array; + regions : array; n, d, nreg : int) : array { + var y : array + y |> resize(nreg * d) + let nsb = n / 256 + for (r in range(nreg)) { + for (row in range(d)) { + let w0 = int(regions[r * 2]) + row * nsb + let x0 = int(regions[r * 2 + 1]) + var acc = 0.0 + for (s in range(nsb)) { + let sb = w0 + s + let xsb = x0 + s + let dd = half_at(ws, sb * 10) + for (blk in range(8)) { + let sc = float(byte_s8(ws, sb * 20 + 4 + blk)) // the plane byte IS the signed (1 + 2s) sub-scale + var bacc = 0.0 + for (e in range(32)) { + let l = e / 8 + let hj = e % 8 + let qi = byte_u8(wq, sb * 104 + blk * 8 + 2 * l + hj / 4) | ((byte_u8(wq, sb * 104 + 64 + blk) << (hj < 4 ? 8 - 2 * l : 7 - 2 * l)) & 256) + let g = float(int(IQ3S_GRID[qi] >> uint(8 * (hj % 4))) & 0xFF) + let sgn = ((byte_u8(wq, sb * 104 + 72 + blk * 4 + l) >> hj) & 1) != 0 ? -1.0 : 1.0 + bacc += g * sgn * float(byte_s8(xq, xsb * 256 + blk * 32 + e)) + } + acc += xs[xsb] * dd * sc * bacc + } + } + y[r * d + row] = acc + } + } + return <- y +} + // kq batch GEMM oracle over region records — same blk_contrib per (row, col, block) def kq_gemm_oracle(fmt : int; wq, ws, xq : array; xs : array; recs : array; nrec, n, d : int; var y : array) { diff --git a/modules/dasLLAMA/tests/test_vulkan_kernels.das b/modules/dasLLAMA/tests/test_vulkan_kernels.das index 82c223a8f3..993e6682ad 100644 --- a/modules/dasLLAMA/tests/test_vulkan_kernels.das +++ b/modules/dasLLAMA/tests/test_vulkan_kernels.das @@ -381,7 +381,7 @@ def private attn_row_oracle(q : array; qbase : int; kk : array; kb def test_vkd_kq_gemv_family(t0 : T?) { t0 |> run("kq GEMV class family == the CPU oracle (all 6 formats)") <| @(t : T?) { static_if (typeinfo builtin_module_exists(vulkan)) { - if (!(ensure_kq_gemv_k4_cls() && ensure_kq_gemv_q40_cls() && ensure_kq_gemv_k5_cls() && ensure_kq_gemv_k6_cls() && ensure_kq_gemv_iq4xs_cls() && ensure_kq_gemv_k3_cls())) { + if (!(ensure_kq_gemv_k4_cls() && ensure_kq_gemv_q40_cls() && ensure_kq_gemv_k5_cls() && ensure_kq_gemv_k6_cls() && ensure_kq_gemv_iq4xs_cls() && ensure_kq_gemv_k3_cls() && ensure_kq_gemv_iq3s_cls())) { feint("no Vulkan device - skipping\n") return } @@ -389,14 +389,14 @@ def test_vkd_kq_gemv_family(t0 : T?) { let d = 3 // rows per region let nreg = 2 let rows = nreg * d - let nfmt = 6 - let fmts = fixed_array(int(KqFmt.k4), int(KqFmt.q40), int(KqFmt.k5), int(KqFmt.k6), int(KqFmt.iq4xs), int(KqFmt.k3)) - let stride4s = fixed_array(8, 8, 10, 12, 8, 6) // uint4s per superblock + let nfmt = 7 + let fmts = fixed_array(int(KqFmt.k4), int(KqFmt.q40), int(KqFmt.k5), int(KqFmt.k6), int(KqFmt.iq4xs), int(KqFmt.k3), int(KqFmt.iq3s)) + let stride4s = fixed_array(32, 32, 40, 48, 32, 24, 26) // uint WORDS per superblock (iq3s is not uint4-aligned) for (fi in range(nfmt)) { let fmt = fmts[fi] let stride4 = stride4s[fi] let nwsb = rows * nsb // region r's weights at r*d*nsb - let wq_words = nwsb * stride4 * 4 + let wq_words = nwsb * stride4 let ws_words_n = nwsb * 5 var wqh : array var wsh : array @@ -465,6 +465,8 @@ def test_vkd_kq_gemv_family(t0 : T?) { enc_kq_gemv_iq4xs_cls(raw2, h2, s_cls, pc, wgs) } elif (fmt == int(KqFmt.k3)) { enc_kq_gemv_k3_cls(raw2, h2, s_cls, pc, wgs) + } elif (fmt == int(KqFmt.iq3s)) { + enc_kq_gemv_iq3s_cls(raw2, h2, s_cls, pc, wgs) } else { enc_kq_gemv_k6_cls(raw2, h2, s_cls, pc, wgs) } @@ -489,6 +491,12 @@ def test_vkd_kq_gemv_family(t0 : T?) { t |> success(badf == 0, "k3: the class bit-math matches the float dequant oracle ({badf} of {rows} rows off)") delete y_flt } + if (fmt == int(KqFmt.iq3s)) { // the grid/sign gather has an independent float witness + var y_flt <- iq3s_gemv_float_oracle(wqh, wsh, xqh, xsh, regs, 512, d, nreg) + let badf = mismatch_approx(y_ref, y_flt) + t |> success(badf == 0, "iq3s: the class bit-math matches the float dequant oracle ({badf} of {rows} rows off)") + delete y_flt + } delete regs delete y_ref delete wqh @@ -1009,7 +1017,7 @@ def test_vkd_q8_batch_tile(t0 : T?) { def test_vkd_kq_batch_family(t0 : T?) { t0 |> run("kq batch tile class family == the CPU oracle (all 6 formats)") <| @(t : T?) { static_if (typeinfo builtin_module_exists(vulkan)) { - if (!(ensure_kq_batch_k4_cls() && ensure_kq_batch_q40_cls() && ensure_kq_batch_k5_cls() && ensure_kq_batch_k6_cls() && ensure_kq_batch_iq4xs_cls() && ensure_kq_batch_k3_cls())) { + if (!(ensure_kq_batch_k4_cls() && ensure_kq_batch_q40_cls() && ensure_kq_batch_k5_cls() && ensure_kq_batch_k6_cls() && ensure_kq_batch_iq4xs_cls() && ensure_kq_batch_k3_cls() && ensure_kq_batch_iq3s_cls())) { feint("no Vulkan device - skipping\n") return } @@ -1024,9 +1032,9 @@ def test_vkd_kq_batch_family(t0 : T?) { let wgs1 = ((cnt1 + 31) / 32) * wtiles let wgs = wgs0 + wgs1 let totsb = 2 * d * nsb // superblocks across both regions - let nfmt = 6 - let wstrides = fixed_array(32, 32, 40, 48, 32, 24) // uint words per superblock: k4 q40 k5 k6 iq4xs k3 - let pipes = fixed_array(int(KqFmt.k4), int(KqFmt.q40), int(KqFmt.k5), int(KqFmt.k6), int(KqFmt.iq4xs), int(KqFmt.k3)) + let nfmt = 7 + let wstrides = fixed_array(32, 32, 40, 48, 32, 24, 26) // uint words per superblock: k4 q40 k5 k6 iq4xs k3 iq3s + let pipes = fixed_array(int(KqFmt.k4), int(KqFmt.q40), int(KqFmt.k5), int(KqFmt.k6), int(KqFmt.iq4xs), int(KqFmt.k3), int(KqFmt.iq3s)) for (fi in range(nfmt)) { let fmt = pipes[fi] let wstride = wstrides[fi] @@ -1103,6 +1111,8 @@ def test_vkd_kq_batch_family(t0 : T?) { enc_kq_batch_iq4xs_cls(raw2, h2, sc, pc, int64(wgs)) } elif (fmt == int(KqFmt.k3)) { enc_kq_batch_k3_cls(raw2, h2, sc, pc, int64(wgs)) + } elif (fmt == int(KqFmt.iq3s)) { + enc_kq_batch_iq3s_cls(raw2, h2, sc, pc, int64(wgs)) } else { enc_kq_batch_k6_cls(raw2, h2, sc, pc, int64(wgs)) } From 52d88d2f14002ebfb3265cb8d84523e39b45bccc Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Sun, 30 Aug 2026 16:56:15 -0700 Subject: [PATCH 20/61] dasLLAMA: the IQ3_S cm2 tile - a gated grid axis on the template, pp512 2x to the tier class The first grid format on the f16 feed: an IQ3GRID gated axis on KqCm2BatchT stages the 2 KB iq3s grid into @workgroup uint[512] ahead of the tile loop (iq3s_grid_word feeds it, the IQLUT axis's shape at 512 entries), and Iq3sCm2T's decode gathers per element over the 16-bit-lane block views (VkIq3sBlk: qs/qh/signs as int16 arrays) - index = qs byte | qh ninth bit, sign bit off the block's own sign bytes, iq4xs's d x signed (1 + 2s) fold. Three width stamps, cm2_cls_ensure/set/enc arms, and iq3s re-admitted to pf_f16_feed (the previous commit's exclusion lifted now that the format owns a tile). Gates on the 5060 Ti: the three tiles 0-off against the grid-aware f16-feed oracle (89600 cells each, l/m/s); suite 74 tests, 73 passed, 1 standard skip; the resident e2e holds 63/64 greedy ids vs llama.cpp at gen 277 t/s (the flip is the final token - the f16 feed's fp16 activations move the same near-tie the CPU stamp moved; the quant feed ran 64/64). Rows vs llama.cpp b10660 Vulkan: pp512 6241 -> 12539.6 +- 103 (0.70x their 17865 - AT the tier's shared 1B-shape class, the k4 control is 0.67x); tg128 288.1 holds (0.89x). Co-Authored-By: Claude Fable 5 --- modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md | 14 +- .../dasllama/dasllama_vulkan_classes.das | 66 ++++++++++ .../dasllama/dasllama_vulkan_prefill.das | 20 ++- modules/dasLLAMA/tests/_vkd_oracles.das | 34 +++++ .../dasLLAMA/tests/test_vulkan_kernels.das | 124 ++++++++++++++++++ 5 files changed, 252 insertions(+), 6 deletions(-) diff --git a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md index acac8c492e..00c7f6d2ff 100644 --- a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md +++ b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md @@ -468,10 +468,16 @@ all three cm2 ladders `verify` on a non-q8 fall-through. The CPU replay of a cla kernel stages workgroup state (the grid) must fill that state in `kq_cls_ref` - the replay never runs the kernel head. Gates: the suite 71/72 with the seven-format family cells and the iq3s float witness; the resident driver matches llama.cpp's greedy ids **64 of 64** at -gen 262 t/s. Rows (5060 Ti vs llama.cpp b10660 Vulkan): tg128 288.5 vs 324.2 (0.89x), pp512 -6241 vs 17865 (0.35x - the quant feed; the cm2 tile on the IQLUT-axis template is the -opening lever, exactly iq4xs/k3's pre-template gap class). Metal: pending (threadgroup -floats + the base-pointer 9th-bit trick). +gen 262 t/s. The cm2 tile followed in the same phase: an `IQ3GRID` gated axis on +`KqCm2BatchT` stages the 2 KB grid into a `@workgroup uint[512]` before the tile loop, and +`Iq3sCm2T`'s decode gathers per element over the 16-bit-lane block views (`VkIq3sBlk`: +qs/qh/signs as int16 arrays) with iq4xs's scale fold; three width stamps, the dispatcher +arms, and iq3s re-admitted to `pf_f16_feed`. The three tiles gate 0-off (89600 cells each); +the e2e holds 63/64 on the f16 feed (the final-token near-tie; the quant feed ran 64/64). +Rows (5060 Ti vs llama.cpp b10660 Vulkan): tg128 288.1 vs 324.2 (0.89x), pp512 12540 vs +17865 (0.70x - AT the tier's shared 1B-shape class; the k4 control on this box is 0.67x), +up from 6241 on the quant feed. Metal: pending (threadgroup floats + the base-pointer +9th-bit trick). ### Q3_K (the second format, 2026-08-30) diff --git a/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das b/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das index 9d6f7a8e5b..df2b4de2a2 100644 --- a/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das +++ b/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das @@ -3863,6 +3863,12 @@ struct VkK5Blk { qh : int16[16] // ... and its 32 packed 5th-bit bytes: byte g*4+q = bits of elems g*32+4q..+3 (lo) | +16.. (hi) } +struct VkIq3sBlk { + qs : int16[32] // one IQ3_S superblock's 64 grid-index bytes as 16-bit lanes + qh : int16[4] // ... its 8 ninth-bit bytes + sg : int16[16] // ... and its 32 sign bytes; scales (f16 d + 8 int8 (1 + 2s)) live in ws +} + struct VkK3Blk { qs : int16[32] // one Q3_K superblock's 64 qs bytes, four 2-bit lanes each, verbatim disk order hm : int16[16] // ... and its 32 hmask bytes; scales (16 int8 (sc-32) + f16 d) live in ws @@ -3887,6 +3893,8 @@ class template KqCm2BatchT : MoeCmBase { @template_constant BLKW : uint = 256u // elements per weight block @template_constant IQLUT : bool = false // stage the iq4nl codebook into workgroup memory @workgroup @template_gate = IQLUT iq4lut : float16[16] // kvalues_iq4nl as f16 (llama.cpp's init_iq_shmem form) + @template_constant IQ3GRID : bool = false // stage the 2 KB iq3s grid into workgroup memory + @workgroup @template_gate = IQ3GRID iq3s_gridc : uint[512] [spirv_kernel(local_size_x = 256), arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-tile-pick-and-default")] def run { // nolint:STYLE038 — the fast/edge path pair, barrier- and register-coupled @@ -3924,6 +3932,11 @@ class template KqCm2BatchT : MoeCmBase { iq4lut[li] = float16(float((int((tbl[int(li >> 2u)] >> ((li & 3u) * 8u)) & 255u) ^ 128) - 128)) } } + static_if (IQ3GRID) { + let gi = gl_LocalInvocationID.x + iq3s_gridc[gi] = iq3s_grid_word(int(gi)) + iq3s_gridc[gi + 256u] = iq3s_grid_word(int(gi + 256u)) + } barrier() // wg_blk0 (and the gated codebook) visible before the first decode load var a : coopmatWgA_f16_128x64 var b : BT @@ -4195,6 +4208,32 @@ class template K3Cm2T : KqCm2BatchT { } } +[ |> template_struct_instance] +class template Iq3sCm2T : KqCm2BatchT { + typedef BLK = VkIq3sBlk + typedef ST = uint + override IQ3GRID = true + + [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] + def decode(blk : VkIq3sBlk; bc, cib : uint2) : float16 { + let e = cib.y + let g = e >> 5u + let r = e & 31u + let wj = r >> 2u + let qi = g * 8u + wj + let qb = uint(int(unpack8(blk.qs[int(qi >> 1u)])[int(qi & 1u)])) & 0xFFu + let qh = uint(int(unpack8(blk.qh[int(g >> 1u)])[int(g & 1u)])) & 0xFFu + let si = g * 4u + (r >> 3u) + let sgb = uint(int(unpack8(blk.sg[int(si >> 1u)])[int(si & 1u)])) & 0xFFu + let gw = iq3s_gridc[qb | ((qh << (8u - wj)) & 256u)] + let gb = float((gw >> ((r & 3u) * 8u)) & 0xFFu) + let srow = (wg_blk0 + bc.x * (pa.n >> 8u) + bc.y) * 5u + let d = unpackHalf2x16(ws[srow]).x + let sc = int(ws[srow + 1u + (g >> 2u)] << ((3u - (g & 3u)) * 8u)) >> 24 + return float16(d * float(sc) * (((sgb >> (((r >> 2u) & 1u) * 4u + (r & 3u))) & 1u) != 0u ? -gb : gb)) + } +} + // the stamps: the three width columns of each format (the dispatcher keys on (fmt, tc)) [vk_dispatch(name = "q8_batch_cm2l_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] class Q8Cm2LBatch : Q8Cm2T { @@ -4385,6 +4424,33 @@ class K3Cm2SBatch : K3Cm2T { typedef FLO = tensorLayout2DPad } +[vk_dispatch(name = "kq_batch_iq3s_cm2l_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Iq3sCm2LBatch : Iq3sCm2T { + typedef BT = coopmatWgB_f16_64x256 + typedef ACC = coopmatWgAcc_f16_128x256 + typedef ACCW = coopmatWgAcc_f32_128x256 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_iq3s_cm2m_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Iq3sCm2MBatch : Iq3sCm2T { + override BN = 128u + typedef BT = coopmatWgB_f16_64x128 + typedef ACC = coopmatWgAcc_f16_128x128 + typedef ACCW = coopmatWgAcc_f32_128x128 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_iq3s_cm2s_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Iq3sCm2SBatch : Iq3sCm2T { + override BN = 32u + override STILE = true + typedef BT = coopmatWgB_f16_64x32 + typedef ACC = coopmatWgAcc_f16_128x32 + typedef ACCW = coopmatWgAcc_f32_128x32 + typedef FLO = tensorLayout2DPad +} + // ===== the split-k reduce (sums the cm2 partial planes into y) ===== struct SkRedArgs { diff --git a/modules/dasLLAMA/dasllama/dasllama_vulkan_prefill.das b/modules/dasLLAMA/dasllama/dasllama_vulkan_prefill.das index 262def37c1..e1d50f1c73 100644 --- a/modules/dasLLAMA/dasllama/dasllama_vulkan_prefill.das +++ b/modules/dasLLAMA/dasllama/dasllama_vulkan_prefill.das @@ -40,8 +40,7 @@ var private @scratch g_pf_ids_stage : array [arch(at="../ARCHITECTURE_RUNTIME.md#activation-scale-lattice")] def private pf_f16_feed(f : int) : bool { - // iq3s stays on the quant feed until its cm2 tile lands (the old fall-through served q8 tiles over iq3s planes) - return (f == int(KqFmt.q8) || (kq_sb(f) && f != int(KqFmt.iq3s))) && g_gpu.coopmat_mode == 4 + return (f == int(KqFmt.q8) || kq_sb(f)) && g_gpu.coopmat_mode == 4 } [arch(at="../ARCHITECTURE_RUNTIME.md#activation-scale-lattice")] @@ -77,6 +76,9 @@ def private cm2_cls_ensure(fmt : int; tc : int64) : bool { if (fmt == int(KqFmt.k3)) { return tc == 256l ? ensure_kq_batch_k3_cm2l_cls() : (tc == 128l ? ensure_kq_batch_k3_cm2m_cls() : ensure_kq_batch_k3_cm2s_cls()) } + if (fmt == int(KqFmt.iq3s)) { + return tc == 256l ? ensure_kq_batch_iq3s_cm2l_cls() : (tc == 128l ? ensure_kq_batch_iq3s_cm2m_cls() : ensure_kq_batch_iq3s_cm2s_cls()) + } verify(fmt == int(KqFmt.q8), "vk prefill: the cm2 ensure fell through for a kq format - a format the feed admits must own a tile arm") return tc == 256l ? ensure_q8_batch_cm2l_cls() : (tc == 128l ? ensure_q8_batch_cm2m_cls() : ensure_q8_batch_cm2s_cls()) } @@ -119,6 +121,12 @@ def private cm2_cls_set(fmt : int; tc : int64; bufs : uint64 const[5]; sizes : i } return tc == 128l ? set_kq_batch_k3_cm2m_cls(bufs, sizes, gbits) : set_kq_batch_k3_cm2s_cls(bufs, sizes, gbits) } + if (fmt == int(KqFmt.iq3s)) { + if (tc == 256l) { + return set_kq_batch_iq3s_cm2l_cls(bufs, sizes, gbits) + } + return tc == 128l ? set_kq_batch_iq3s_cm2m_cls(bufs, sizes, gbits) : set_kq_batch_iq3s_cm2s_cls(bufs, sizes, gbits) + } verify(fmt == int(KqFmt.q8), "vk prefill: the cm2 set fell through for a kq format - a format the feed admits must own a tile arm") if (tc == 256l) { return set_q8_batch_cm2l_cls(bufs, sizes, gbits) @@ -176,6 +184,14 @@ def private cm2_cls_enc(fmt : int; tc : int64; raw : VkCommandBuffer; var h : Vk } else { enc_kq_batch_k3_cm2s_cls(raw, h, s, pc, groups) } + } elif (fmt == int(KqFmt.iq3s)) { + if (tc == 256l) { + enc_kq_batch_iq3s_cm2l_cls(raw, h, s, pc, groups) + } elif (tc == 128l) { + enc_kq_batch_iq3s_cm2m_cls(raw, h, s, pc, groups) + } else { + enc_kq_batch_iq3s_cm2s_cls(raw, h, s, pc, groups) + } } elif (tc == 256l) { verify(fmt == int(KqFmt.q8), "vk prefill: the cm2 enc fell through for a kq format - a format the feed admits must own a tile arm") enc_q8_batch_cm2l_cls(raw, h, s, pc, groups) diff --git a/modules/dasLLAMA/tests/_vkd_oracles.das b/modules/dasLLAMA/tests/_vkd_oracles.das index 72250c3117..ef45c266f1 100644 --- a/modules/dasLLAMA/tests/_vkd_oracles.das +++ b/modules/dasLLAMA/tests/_vkd_oracles.das @@ -325,6 +325,40 @@ def iq3s_gemv_float_oracle(wq, ws, xq : array; xs : array; return <- y } +// IQ3_S f16-feed GEMM reference over the device form: grid gather + sign flip per element, +// f16-rounded per weight like its siblings ([qs 64][qh 8][signs 32] rows, the k4-shaped +// scale row with signed (1 + 2s) sub-scales). +def iq3sf16_gemm_oracle(wq : array; wsu : array; xf : array; + recs : array; nrec, n, d : int; var y : array) { + let nsb = n / 256 + for (rr in range(nrec)) { + let wsb0 = int(recs[rr * 4]) + let row0 = int(recs[rr * 4 + 1]) + let cnt = int(recs[rr * 4 + 2]) + for (r in range(cnt)) { + for (c in range(d)) { + var acc = 0.0 + for (s in range(nsb)) { + let sb = wsb0 + c * nsb + s + let dv = unpackHalf2x16(wsu[sb * 5]).x + for (g in range(8)) { + let sc = float(int(wsu[sb * 5 + 1 + g / 4] << uint((3 - g % 4) * 8)) >> 24) + for (e in range(32)) { + let wj = e / 4 + let qi = byte_u8(wq, sb * 104 + g * 8 + wj) | ((byte_u8(wq, sb * 104 + 64 + g) << (8 - wj)) & 256) + let gb = float(int(IQ3S_GRID[qi] >> uint(8 * (e % 4))) & 0xFF) + let sbit = (byte_u8(wq, sb * 104 + 72 + g * 4 + e / 8) >> (((e / 4) % 2) * 4 + e % 4)) & 1 + let w = float(float16(dv * sc * (sbit != 0 ? -gb : gb))) + acc += w * half_at(xf, (row0 + r) * n + s * 256 + g * 32 + e) + } + } + } + y[(row0 + r) * d + c] = acc + } + } + } +} + // kq batch GEMM oracle over region records — same blk_contrib per (row, col, block) def kq_gemm_oracle(fmt : int; wq, ws, xq : array; xs : array; recs : array; nrec, n, d : int; var y : array) { diff --git a/modules/dasLLAMA/tests/test_vulkan_kernels.das b/modules/dasLLAMA/tests/test_vulkan_kernels.das index 993e6682ad..e8ab636220 100644 --- a/modules/dasLLAMA/tests/test_vulkan_kernels.das +++ b/modules/dasLLAMA/tests/test_vulkan_kernels.das @@ -2157,6 +2157,130 @@ def test_vkd_k3_cm2_batch(t0 : T?) { } } +[test] +def test_vkd_iq3s_cm2_batch(t0 : T?) { + t0 |> run("cm2 IQ3_S l/m/s tiles == the CPU oracle (workgroup-staged grid decode-in-load)") <| @(t : T?) { + static_if (typeinfo builtin_module_exists(vulkan)) { + if (!ensure_q8_batch_cls()) { // engage the device without touching coopmat rails + t |> skip("no Vulkan device") + return + } + if (!(g_gpu.coopmat_mode == 4 && g_gpu.has_coopmat2)) { + t |> skip("the cm2 iq3s tiles serve only in mode 4 on an NV_coopmat2 device") + return + } + verify(ensure_kq_batch_iq3s_cm2l_cls() && ensure_kq_batch_iq3s_cm2m_cls() && ensure_kq_batch_iq3s_cm2s_cls(), "iq3s cm2 class rails must engage in cm2 mode") + let n = 768 // one 512-unrolled pass + 4 tail steps crossing superblock bounds + let nsb = n / 256 + let d = 160 // one full 128-tile + a 32 edge + let cnt0 = 300 + let cnt1 = 260 + let rows = cnt0 + cnt1 + let totsb = 2 * d * nsb + var wqh : array + var wsuh : array + var xfh : array + wqh |> resize(totsb * 26) + wsuh |> resize(totsb * 5) + xfh |> resize((rows + 32) * n / 2) // + the s tile's 32-row read slack + for (i in range(totsb * 26)) { + wqh[i] = hash_word(uint(i) + 29u) // arbitrary quant bytes + } + for (sb in range(totsb)) { + wsuh[sb * 5] = packHalf2x16(float2(0.0002 * float(1 + sb % 7), 0.0)) // cool d + wsuh[sb * 5 + 1] = hash_word(uint(sb * 2) + 613u) // signed sub-scale bytes + wsuh[sb * 5 + 2] = hash_word(uint(sb * 2 + 1) + 613u) + wsuh[sb * 5 + 3] = 0u + wsuh[sb * 5 + 4] = 0u + } + for (i in range((rows + 32) * n / 2)) { + xfh[i] = ws_word(uint(i) * 3u + 5u) // f16 activation pairs + } + let wq_bytes = int64(totsb) * 104l + let ws_bytes = int64(length(wsuh)) * 4l + let xf_bytes = int64((rows + 32) * n) * 2l + let y_bytes = int64(rows * d) * 4l + let wqd = make_device_buf(wq_bytes) + let wsd = make_device_buf(ws_bytes) + let xfd = make_device_buf(xf_bytes) + let yd2 = make_device_buf(y_bytes) + var host = make_host_buf(y_bytes, true, [cached = true]) + var y_cls : array + y_cls |> resize(rows * d) + var y_ref : array + y_ref |> resize(rows * d) + for (ml in range(3)) { + let ttile = ml == 0 ? 256 : (ml == 1 ? 128 : 32) + let wtiles = (d + 127) / 128 + let wgs0 = ((cnt0 + ttile - 1) / ttile) * wtiles + let wgs1 = ((cnt1 + ttile - 1) / ttile) * wtiles + let wgs = wgs0 + wgs1 + var sched : array + sched |> resize(2 * 4 + wgs) + sched[0] = 0u + sched[1] = 0u + sched[2] = uint(cnt0) + sched[3] = 0u + sched[4] = uint(d * nsb) + sched[5] = uint(cnt0) + sched[6] = uint(cnt1) + sched[7] = uint(wgs0) + for (w in range(wgs)) { + sched[8 + w] = w < wgs0 ? 0u : 1u + } + unsafe { + upload_region_at(wqd, 0l, addr(wqh[0]), wq_bytes) + upload_region_at(wsd, 0l, addr(wsuh[0]), ws_bytes) + upload_region_at(xfd, 0l, addr(xfh[0]), xf_bytes) + let sc_bytes = int64(2 * 4 + wgs) * 4l + let scd = make_device_buf(sc_bytes) + upload_region_at(scd, 0l, addr(sched[0]), sc_bytes) + var sc = (ml == 0 + ? set_kq_batch_iq3s_cm2l_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)) + : (ml == 1 + ? set_kq_batch_iq3s_cm2m_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)) + : set_kq_batch_iq3s_cm2s_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)))) + var raw2 = alloc_cmd() + let begin = VkCommandBufferBeginInfo() + vk_check(vkBeginCommandBuffer(raw2, begin), null) + var h2 : VkHaz + var pc = BatchArgs(n = uint(n), d = uint(d), map_off = uint(2 * 4)) + if (ml == 0) { + enc_kq_batch_iq3s_cm2l_cls(raw2, h2, sc, pc, int64(wgs)) + } elif (ml == 1) { + enc_kq_batch_iq3s_cm2m_cls(raw2, h2, sc, pc, int64(wgs)) + } else { + enc_kq_batch_iq3s_cm2s_cls(raw2, h2, sc, pc, int64(wgs)) + } + vhz_dep(raw2, h2, 16u, 0u, true) + cmd_copy_whole(raw2, yd2, host.buf, y_bytes) + vk_check(vkEndCommandBuffer(raw2), null) + submit_wait(raw2) + memcpy(addr(y_cls[0]), host.mapped, y_bytes) + } + iq3sf16_gemm_oracle(wqh, wsuh, xfh, sched, 2, n, d, y_ref) + let bad = mismatch_bars(y_cls, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) + to_log(LOG_INFO, "cm2 iq3s {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile batch: {bad} of {rows * d} off the oracle\n") + t |> success(bad == 0, "cm2 iq3s {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile matches the CPU oracle ({bad} of {rows * d} off)") + delete sched + } + delete y_ref + delete wqh + delete wsuh + delete xfh + delete y_cls + } else { + t |> skip("dasVulkan not present") + } + } +} + [test] def test_vkd_k6_cm2_batch(t0 : T?) { t0 |> run("cm2 Q6_K l/m tiles == the CPU oracle (6-bit compose decode-in-load)") <| @(t : T?) { From cf8c3b69fef131e1792aed07cfd87579bde7b5e6 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Sun, 30 Aug 2026 17:39:28 -0700 Subject: [PATCH 21/61] dasMetal: literal fixed-array locals hoist to program-scope constant tables A `let` fixed-array local whose initializer is an all-literal fixed_array(...) now lowers to a program-scope `constant T name[N] = {...};` in the MSL preamble, with the local's reads indexing it by name - llama.cpp's exact shape for quant codebooks and grids, and the form the iq3s 512-word grid needs (the previous rule rejected every initializer, and a 2 KB table has no zero-init-then-fill spelling worth emitting). Same-named tables with different content in two helpers uniquify off the splice counter; identical content dedupes to one emission. Anything else with an initializer stays rejected fail-closed: a `var` (a written table has no constant address space) and any non-literal element, each with its own error text. Census kind stmt.let.const_table; the ConstTab fixture exercises a hoist from a helper body and from the kernel body; _fc_array_init_nonconst pins the non-literal rejection needle. tests/msl: functions 9/9, census, fail_closed all green; the iq3s Metal kernels (the first consumer) gate 0-off in test_metal_gemv/gemm_kernels on the M1 Max. Co-Authored-By: Claude Fable 5 --- modules/dasMetal/metal/msl_emit.das | 82 ++++++++++++++++++- .../_fail_closed/_fc_array_init_nonconst.das | 20 +++++ tests/msl/_msl_common.das | 26 +++++- tests/msl/test_msl_fail_closed.das | 1 + tests/msl/test_msl_functions.das | 16 ++++ 5 files changed, 142 insertions(+), 3 deletions(-) create mode 100644 tests/msl/_fail_closed/_fc_array_init_nonconst.das diff --git a/modules/dasMetal/metal/msl_emit.das b/modules/dasMetal/metal/msl_emit.das index 66315f1d3b..63f0a415db 100644 --- a/modules/dasMetal/metal/msl_emit.das +++ b/modules/dasMetal/metal/msl_emit.das @@ -82,6 +82,7 @@ struct private MslCtx { ustructs : table // @uniform struct name -> its MSL definition (preamble) helpers : table // preamble helper functions referenced by lowered calls gen_helpers : table // GENERATED preamble helpers (tensor ops): mangled name -> text + const_tables : table // hoisted program-scope constant arrays: MSL name -> definition text scan_ptrs : table // scan-time pointer local -> member (write-set resolution) tg_accs : table // staged-GEMM acc local -> "m|n|sgs|kk|at" (decl suppressed on GPU) mpp_used : bool // a tile_tg/rowscale-only kernel still needs the MPP include @@ -2961,6 +2962,77 @@ def private local_msl_name(ctx : MslCtx; name : string) : string { return key_exists(ctx.members, name) ? "{base}_l" : base } +// A `let` fixed-array local with an all-literal `fixed_array(...)` initializer lowers to a +// program-scope `constant` array - the codebook/grid-table shape quant kernels stage from. +// Anything else with an initializer stays rejected: das zero-init is the only local form. +def private hoist_const_table(var ctx : MslCtx; v : VariablePtr) { + let et = v._type.firstType + let n = v._type.fixedDim + if (!v._type.flags.constant) { + err(ctx, v.at, "fixed-array local `{v.name}` initializer hoists to a program-scope constant table - it must be a `let` (a written table has no constant form)") + return + } + if (reject_cpu_only_width(ctx, v.at, et)) { + return + } + let ename = msl_type_name(et) + if (empty(ename)) { + err(ctx, v.at, "local array `{v.name}` element type {describe(et)} has no MSL form") + return + } + var mk : ExprMakeArray? + if (v.init is ExprMakeArray) { + mk = v.init as ExprMakeArray + } elif (v.init is ExprCall) { + let ec = v.init as ExprCall + if (length(ec.arguments) == 1 && ec.arguments[0] is ExprMakeArray) { + mk = ec.arguments[0] as ExprMakeArray + } + } + if (mk == null || length(mk.values) != n) { + err(ctx, v.at, "fixed-array local `{v.name}` initializer hoists to a program-scope constant table - every element must be a literal") + return + } + var vals : array + vals |> reserve(n) + for (el in mk.values) { + if (!(el is ExprConstInt || el is ExprConstUInt || el is ExprConstInt64 || el is ExprConstUInt64 || + el is ExprConstFloat || el is ExprConstDouble || el is ExprConstInt8 || el is ExprConstUInt8 || + el is ExprConstInt16 || el is ExprConstUInt16)) { + err(ctx, v.at, "fixed-array local `{v.name}` initializer hoists to a program-scope constant table - every element must be a literal") + return + } + vals |> push(emit_value(ctx, el)) + } + let vtxt = build_string() $(var w) { + for (i in range(n)) { + if (i % 8 == 0) { + w |> write(" ") + } + w |> write(vals[i]) + if (i != n - 1) { + w |> write(",") + } + if (i % 8 == 7 || i == n - 1) { + w |> write("\n") + } else { + w |> write(" ") + } + } + } + let aname = string(v.name) + var an = local_msl_name(ctx, aname) + var txt = "constant {ename} {an}[{n}] = \{\n{vtxt}\};" + if (key_exists(ctx.const_tables, an) && ctx.const_tables[an] != txt) { + an = "{an}_c{ctx.splice_seq}" // a same-named table with different content in another helper + ctx.splice_seq++ + txt = "constant {ename} {an}[{n}] = \{\n{vtxt}\};" + } + ctx.renames[aname] = an + ctx.const_tables[an] = txt + note(ctx, "stmt.let.const_table") +} + def private emit_stmt(var ctx : MslCtx; e : Expression?; indent : string; var lines : array) { if (e == null) { return @@ -3028,9 +3100,10 @@ def private emit_stmt(var ctx : MslCtx; e : Expression?; indent : string; var li } if (v._type != null && v._type.baseType == Type.tFixedArray && v._type.firstType != null && v._type.firstType.baseType != Type.tFixedArray) { - // single-dim fixed-array local; das zero-init semantics, no initializer form + // single-dim fixed-array local; das zero-init semantics. A `let` with an + // all-literal initializer hoists to a program-scope `constant` table instead. if (v.init != null) { - err(ctx, v.at, "fixed-array local `{v.name}` takes no initializer - das zero-init is the MSL form") + hoist_const_table(ctx, v) continue } let et = v._type.firstType @@ -3669,6 +3742,11 @@ def public generate_msl(fn : FunctionPtr; var errors : array; cfg : MslE hs |> sort() // table iteration order is hash-based — pin the emitted order helper_text = join(hs, "\n") + "\n" } + if (!empty(ctx.const_tables)) { + var cts <- [for (v in values(ctx.const_tables)); v] + cts |> sort() // pinned order, same reason + helper_text = join(cts, "\n") + "\n" + helper_text + } if (!empty(ctx.ustructs)) { // @uniform struct definitions come first — the kernel signature names them. // Sorted: table order is hash-based diff --git a/tests/msl/_fail_closed/_fc_array_init_nonconst.das b/tests/msl/_fail_closed/_fc_array_init_nonconst.das new file mode 100644 index 0000000000..e9c4599cca --- /dev/null +++ b/tests/msl/_fail_closed/_fc_array_init_nonconst.das @@ -0,0 +1,20 @@ +// Fail-closed fixture: a fixed-array local initializer with a non-literal element. Only an +// all-literal `let` hoists to a program-scope constant table; anything else has no MSL form. + +expect 50501 + +options gen2 + +require metal/msl_shader + +class FcArrayInitNonconst { + @ssbo @binding = 0 src : array + @ssbo @binding = 1 dst : array + + [metal_kernel(name="fc_array_init_nonconst_msl")] + def kern { + let i = gl_GlobalInvocationID.x + let tbl = fixed_array(1u, src[i], 3u, 4u) + dst[i] = tbl[i & 3u] + } +} diff --git a/tests/msl/_msl_common.das b/tests/msl/_msl_common.das index bff241bb0a..ab2024d2e5 100644 --- a/tests/msl/_msl_common.das +++ b/tests/msl/_msl_common.das @@ -692,6 +692,29 @@ class UkStride { } } +// ===== hoisted constant tables ===== +// A `let` fixed-array local with all-literal elements lowers to a program-scope `constant` +// array - the codebook/grid shape quant kernels stage from; reads index it by the local's name. + +def private ctb_pick(i : uint) : uint { + let tbl = fixed_array(0x11u, 0x22u, 0x33u, 0x44u, 0x55u, 0x66u, 0x77u, 0x88u) + return tbl[i & 7u] +} + +class ConstTab { + @ssbo @binding = 0 src : array + @ssbo @binding = 1 dst : array + @ssbo @binding = 2 fdst : array + + [metal_kernel(name="consttab_msl"), marker(no_coverage)] + def consttab { + let i = gl_GlobalInvocationID.x + let sgn = fixed_array(1.0, 0.5) + dst[i] = src[i] + ctb_pick(i) + fdst[i] = sgn[i & 1u] * 2.0 + } +} + // ===== @uniform structs (the kargs form) ===== // A kernel with a dozen scalars costs a dozen binds; one struct costs one. Fields are 4-byte // scalars only, so the das layout IS the MSL layout and the host writes its struct straight @@ -1293,6 +1316,7 @@ def public declared_msl_census : table { // nolint:STYLE038 — one fl "cvt.i8.f32", // statements "stmt.let", + "stmt.let.const_table", "stmt.var", "stmt.var.zero", "stmt.assign", @@ -1336,7 +1360,7 @@ def public all_msl_censuses : array { vecarith_msl_census, control_msl_census, loops_msl_census, reduce_msl_census, simd_msl_census, halfarith_msl_census, latconv_msl_census, packed3_msl_census, sgmat_msl_census, sgmat_tiled_msl_census, sgmat_rolled_msl_census, mathcalls_msl_census, userfns_msl_census, - ufxcls_msl_census, ukargs_msl_census, ukstride_msl_census, statearr_msl_census, + ufxcls_msl_census, ukargs_msl_census, ukstride_msl_census, consttab_msl_census, statearr_msl_census, tmmhw_msl_census, tmmhwh_msl_census, tmmq8u_msl_census, tmmq8uh_msl_census, tmmq8uh64_msl_census, tmmq8uh128_msl_census, tmmdeva_msl_census, tmmdevaf_msl_census, tmmnt_msl_census, tmmdevant_msl_census, sgmathalf_msl_census, sgmat_store_tg_msl_census, diff --git a/tests/msl/test_msl_fail_closed.das b/tests/msl/test_msl_fail_closed.das index b3c47f70ff..8929cfc222 100644 --- a/tests/msl/test_msl_fail_closed.das +++ b/tests/msl/test_msl_fail_closed.das @@ -59,6 +59,7 @@ def test_fail_closed_rejections(t : T?) { check_rejects(t, "_fc_uniform_written", "@uniform member `n` is written") check_rejects(t, "_fc_nan_literal", "non-finite float literal has no MSL form") check_rejects(t, "_fc_uniform_array", "array members are @ssbo buffers") + check_rejects(t, "_fc_array_init_nonconst", "every element must be a literal") check_rejects(t, "_fc_return_value", "kernels return void") check_rejects(t, "_fc_recursion", "is recursive - MSL forbids recursion") check_rejects(t, "_fc_ufn_param", "type has no MSL form") diff --git a/tests/msl/test_msl_functions.das b/tests/msl/test_msl_functions.das index 5023e2c97b..cba0ff1d75 100644 --- a/tests/msl/test_msl_functions.das +++ b/tests/msl/test_msl_functions.das @@ -135,3 +135,19 @@ def test_msl_state_and_naming(t : T?) { "the emitted signature carries the derived entry name") } } + +def private hasq(t : T?; needle : string) { + t |> success(find(consttab_msl, needle) >= 0, "emitted MSL carries `{needle}`") + if (find(consttab_msl, needle) < 0) { + to_log(LOG_ERROR, "MISSING `{needle}` in:\n{consttab_msl}\n") + } +} + +[test] +def test_msl_const_tables(t : T?) { + t |> run("literal fixed-array locals hoist to program-scope constant tables") <| @(t : T?) { + hasq(t, "constant uint tbl[8] = \{") // hoisted from the helper's body + hasq(t, "constant float sgn[2] = \{") // hoisted from the kernel body + hasq(t, "tbl[") // reads index the hoisted table by name + } +} From cda7a2c84ebf6898baa10ce9dd153e29a40c3e2e Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Sun, 30 Aug 2026 18:29:39 -0700 Subject: [PATCH 22/61] dasLLAMA: IQ3_S on the Metal tier - constant-table grid, 4-row f4 GEMV, tg 0.95x The last tier for the format. Blob: an "iq3ss" arm mirroring the iq4xs 20->18 split verbatim over t.iq3ss (identical strip/d addressing, so kq_scales_of is the iq4xs arm with the plane swapped), quants at sb0*104, the % 512 off gate joining the split-scale row. Kernels ride the previous commit's MSL constant tables: iq3s_gw() carries the 512-word grid, MetalKqGemvIq3s stages it as a float4[512] MAGNITUDE slab (signs applied by select) at 4 rows per simdgroup (llama.cpp's N_R0_IQ3_S), MetalKqMvIq3sT B2/B4 + MetalKqMvB8Iq3s are the iq4xs shells over the per-32-block lane map with the grid read direct, and MetalKqMulMmK45T grows an IQ3S arm (threadgroup grid under @template_gate; the chain re-nested - `} else static_if` is a parse error). Ladders, prefill gates, and the seven-format test cells follow the iq4xs rows; fixtures at fmt 33 reuse the split-form scale fill with kq_row_ref through dequant_iq3s_plane_superblock_at. The decode GEMV got a full race before shipping: eight forms at n=2048 d=8192 (gather placement x3, gather deleted, signs deleted, llama.cpp's exact geometry, row width, f4 slab) all land in 127-141 GB/s against k4's 204 and k6's 287 in the same harness - the 9-bit-index + per-nibble-sign compose is format-intrinsic on M1, and the f4 slab is the best form and ships. HOW_TO gains the race table and QUIRK 22 (judge Metal kernel levers with the kernel probe, not tg128 e2e - +-8 t/s of e2e noise swallowed the +9% winner twice). Gates on the M1 Max: test_metal_gemv_kernels 2/2, test_metal_gemm_kernels 2/2 (108s corpus), e2e decodes the coherent story at gen 227 t/s. Rows vs llama.cpp b10660 Metal (same-run, IQ3_M): pp512 3237.6 / 3344.3 (0.97x), tg128 199.4 / 209.0 (0.95x, up from 0.82x at the first cut); M1 CPU pp512 886.2 / 433.6 (2.04x), tg128 57.4 / 66.6 (0.86x - the ledgered tg tail class, followup `#61`/`#62`). Co-Authored-By: Claude Fable 5 --- modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md | 44 +- modules/dasLLAMA/dasllama/dasllama_layout.das | 23 +- .../dasllama/dasllama_metal_common.das | 14 + .../dasllama/dasllama_metal_kernels.das | 444 ++++++++++++++++-- .../dasllama/dasllama_metal_prefill.das | 8 +- .../dasllama/dasllama_metal_shapes.das | 4 +- modules/dasLLAMA/followup_general.md | 11 + .../dasLLAMA/tests/_metal_kernel_common.das | 9 +- .../tests/test_metal_gemm_kernels.das | 24 +- .../tests/test_metal_gemv_kernels.das | 43 +- 10 files changed, 549 insertions(+), 75 deletions(-) diff --git a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md index 00c7f6d2ff..0d5a597513 100644 --- a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md +++ b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md @@ -411,6 +411,10 @@ where, why it is so today, what unquirked looks like. An empty ledger is a legit hash folds the generator bodies (plans/interpreter ledger has the sibling `jit_dll_semantic_hash` item). +22. **A Metal kernel lever is judged by a kernel-level probe, never by tg128 e2e.** A tg128 + row on the M1 carries +-8 t/s of noise; the iq3s f4-slab GEMV form (+9% at the kernel) + measured FLAT on e2e twice and was nearly discarded. The dispatch-loop probe (50 dispatches + per encoder, best-of-3 encoders, GB/s off plane bytes) resolves 3% in seconds. ## Per-format notes ### IQ3_S (the third format - and the first grid format, 2026-08-30) @@ -476,8 +480,44 @@ arms, and iq3s re-admitted to `pf_f16_feed`. The three tiles gate 0-off (89600 c the e2e holds 63/64 on the f16 feed (the final-token near-tie; the quant feed ran 64/64). Rows (5060 Ti vs llama.cpp b10660 Vulkan): tg128 288.1 vs 324.2 (0.89x), pp512 12540 vs 17865 (0.70x - AT the tier's shared 1B-shape class; the k4 control on this box is 0.67x), -up from 6241 on the quant feed. Metal: pending (threadgroup floats + the base-pointer -9th-bit trick). +up from 6241 on the quant feed. + +Metal came in five surfaces plus one emitter feature. The blob "iq3ss" arm is the iq4xs +20->18 split VERBATIM over `t.iq3ss` (same strip/d addressing, so `kq_scales_of` is a copy of +the iq4xs arm with the plane swapped); quants bind at `sb0*104`. The MSL emitter grew +constant-table hoisting for the grid (its own dasMetal commit: a `let` fixed-array local with +all-literal elements lowers to a program-scope `constant T name[N] = {...}` - before that, +ANY fixed-array initializer was a compile error, and there is no other way to put 2 KB of +data into a kernel). `iq3s_gw()` carries the 512-word literal; the GEMV and mul_mm stage it +into threadgroup memory, the Mv twins read it direct (unmeasured batch shapes). Kernels: +`MetalKqGemvIq3s` (the iq4xs parity shape but 4 rows/simdgroup - llama.cpp's N_R0_IQ3_S - +with the grid staged as a `float4[512]` MAGNITUDE slab and signs applied by select), +`MetalKqMvIq3sT` B2/B4 + `MetalKqMvB8Iq3s` (iq4xs shells, per-32-block lane map, consecutive +float4 x loads), an `IQ3S` arm in `MetalKqMulMmK45T` (threadgroup grid under +`@template_gate`; re-nest the chain - `} else static_if` is a parse error, QUIRK 14's rule +holds in the MSL emitter too). + +The decode-GEMV gap got the full treatment: at n=2048 d=8192 the shipped kernel raced EIGHT +forms (tg-slab uint gather, constant-table gather, duplicated slab, gather deleted, sign +flip deleted, llama.cpp's exact 1-lane-per-block geometry, 2- vs 4-row, f4 magnitude slab) - +all land in 127-141 GB/s while k4 does 204 and k6 287 in the same harness. The compose chain +(9-bit index + per-nibble signs) is format-intrinsic on M1; the f4 slab is the best form +(+9%) and ships. Gates on the M1 Max: `test_metal_gemv_kernels` 2/2, `test_metal_gemm_kernels` +2/2 (~108s corpus), e2e decodes the coherent story at gen 227 t/s. + +Against llama.cpp b10660 (`lcpp_bench --for-debug-purposes`, das = the debug-jit instrument; +zen2 = 16 threads, M1 = 8; IQ3_M, so attn_v/attn_output/ffn_down are Q4_K and the tied +embedding head is Q6_K - three formats share every decode step): + +| tier | pp512 das / llama.cpp | tg128 das / llama.cpp | +|---|---|---| +| zen2 CPU | 516.9 / 104.9 (4.93x) | 52.4 / 57.0 (0.92x) | +| 5060 Ti Vulkan | 12539.6 / 17865 (0.70x) | 288.1 / 324.2 (0.89x) | +| M1 CPU | 886.2 / 433.6 (2.04x) | 57.4 / 66.6 (0.86x) | +| M1 Metal | 3237.6 / 3344.3 (0.97x) | 199.4 / 209.0 (0.95x) | + +(The M1 Metal ref tg drifts 209-230 across back-to-back rounds - thermal; ratios are +same-run. The tg tails on every tier are the one ledgered class: followup_general #61/#62.) ### Q3_K (the second format, 2026-08-30) diff --git a/modules/dasLLAMA/dasllama/dasllama_layout.das b/modules/dasLLAMA/dasllama/dasllama_layout.das index 9b2bd172b9..f3070a55af 100644 --- a/modules/dasLLAMA/dasllama/dasllama_layout.das +++ b/modules/dasLLAMA/dasllama/dasllama_layout.das @@ -29,7 +29,7 @@ require math def private metal_blob_off_ok(off : int64; fmt : KqFmt) : bool { if (off < 0l) return true if (fmt == KqFmt.q8) return off % 256l == 0l - if (fmt == KqFmt.k6 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3) return off % 512l == 0l // the split scale form: the (off/256)*2 d-plane bind stays 4B-aligned + if (fmt == KqFmt.k6 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s) return off % 512l == 0l // the split scale form: the (off/256)*2 d-plane bind stays 4B-aligned if (fmt == KqFmt.q51) return off % 128l == 0l // (off/32)*20 and (off/32)*4 both 16B-aligned return true } @@ -188,6 +188,25 @@ def metal_blob_scale_plane(var t : Model; name : string) : bool { } return true } + if (name == "iq3ss") { + // iq3s: the row IS the iq4xs 20B shape — the same split (strips = bytes 4..19, d = bytes 0..1), over t.iq3ss + if (!empty(t.iq3ss)) { + var @exact_size i3c : array + let nsb = long_length(t.iq3ss) / IQ3S_SSB + i3c |> reserve_resize(nsb * 18l) + unsafe { + var dp = addr(i3c[0]) + let sp = addr < uint8 const? >(t.iq3ss[0]) + for (sb in range64(nsb)) { + memcpy(reinterpret(dp + sb * 16l), reinterpret(sp + sb * IQ3S_SSB + 4l), 16) + dp[nsb * 16l + sb * 2l] = sp[sb * IQ3S_SSB] + dp[nsb * 16l + sb * 2l + 1l] = sp[sb * IQ3S_SSB + 1l] + } + } + t.iq3ss <- i3c + } + return true + } if (name == "k3s") { // k3: the CPU row IS the k6 form ([16 int8][f16 d]) — the same split, over t.k3s if (!empty(t.k3s)) { @@ -236,7 +255,7 @@ def metal_blob_commit(var t : Model) : bool { if (!metal_blob_eligible(t)) { return false } - for (nm in ["k4s", "k5s", "k6s", "iq4xss", "k3s"]) { + for (nm in ["k4s", "k5s", "k6s", "iq4xss", "k3s", "iq3ss"]) { metal_blob_scale_plane(t, nm) } t.metal_blob = true diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_common.das b/modules/dasLLAMA/dasllama/dasllama_metal_common.das index c44e1e0950..7f944e70bd 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_common.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_common.das @@ -261,6 +261,11 @@ var g_pso_kq_mvb8_iq4xs : MetalComputePipeline? var g_pso_kq_mvb2_k3 : MetalComputePipeline? var g_pso_kq_mvb4_k3 : MetalComputePipeline? var g_pso_kq_mvb8_k3 : MetalComputePipeline? +var g_pso_kq_iq3s : MetalComputePipeline? +var g_pso_kq_mvb2_iq3s : MetalComputePipeline? +var g_pso_kq_mvb4_iq3s : MetalComputePipeline? +var g_pso_kq_mvb8_iq3s : MetalComputePipeline? +var g_pso_kq_mm_b33 : MetalComputePipeline? var g_pso_kq_mm_b4 : MetalComputePipeline? // batch B>=9: the prefill-owned kq mul_mm twins var g_pso_kq_mm_b5 : MetalComputePipeline? var g_pso_kq_mm_b6 : MetalComputePipeline? @@ -852,6 +857,10 @@ def kq_quants_of(dev : MetalDevice?; t : Model; fmt : KqFmt; off : int64) : tupl return (buf = plane_buffer(dev, addr < void? >(t.k3q[0]), uint64(long_length(t.k3q)), t.image_map != null), qoff = uint64(sb0 * 96l)) } + if (fmt == KqFmt.iq3s) { + return (buf = plane_buffer(dev, addr < void? >(t.iq3sq[0]), uint64(long_length(t.iq3sq)), t.image_map != null), + qoff = uint64(sb0 * 104l)) + } return (buf = plane_buffer(dev, addr < void? >(t.k6q[0]), uint64(long_length(t.k6q)), t.image_map != null), qoff = uint64(sb0 * 192l)) } @@ -878,6 +887,11 @@ def kq_scales_of(dev : MetalDevice?; t : Model; fmt : KqFmt; off : int64) : tupl return (buf = plane_buffer(dev, addr < void? >(t.k3s[0]), uint64(long_length(t.k3s)), t.image_map != null), soff = uint64(sb0 * 16l), doff = uint64(plane_sb * 16l + sb0 * 2l)) } + if (fmt == KqFmt.iq3s) { // the k6 split form over the 18B/sb device plane + let plane_sb = long_length(t.iq3ss) / 18l + return (buf = plane_buffer(dev, addr < void? >(t.iq3ss[0]), uint64(long_length(t.iq3ss)), t.image_map != null), + soff = uint64(sb0 * 16l), doff = uint64(plane_sb * 16l + sb0 * 2l)) + } if (fmt == KqFmt.iq4xs) { // the k6 split form over the 18B/sb device plane let plane_sb = long_length(t.iq4xss) / 18l return (buf = plane_buffer(dev, addr < void? >(t.iq4xss[0]), uint64(long_length(t.iq4xss)), t.image_map != null), diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das index 2f428adee1..93965855b9 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das @@ -3646,6 +3646,11 @@ def metal_decode_init : bool { // nolint:STYLE038 — a flat compile_pso per k g_pso_kq_mvb4_k3 = compile_pso(MetalKqMvB4K3_metal_kq_mv_k3_msl, MetalKqMvB4K3_metal_kq_mv_k3_msl_entry, MetalKqMvB4K3_metal_kq_mv_k3_msl_fastmath, ok) g_pso_kq_mvb8_k3 = compile_pso(metal_kq_mvb8_k3_msl, metal_kq_mvb8_k3_msl_entry, metal_kq_mvb8_k3_msl_fastmath, ok) g_pso_kq_mm_b3 = compile_pso(MetalKqMulMmK3_metal_kq_mulmm_msl, MetalKqMulMmK3_metal_kq_mulmm_msl_entry, MetalKqMulMmK3_metal_kq_mulmm_msl_fastmath, ok) + g_pso_kq_iq3s = compile_pso(metal_kq_gemv_iq3s_msl, metal_kq_gemv_iq3s_msl_entry, metal_kq_gemv_iq3s_msl_fastmath, ok) + g_pso_kq_mvb2_iq3s = compile_pso(MetalKqMvB2Iq3s_metal_kq_mv_iq3s_msl, MetalKqMvB2Iq3s_metal_kq_mv_iq3s_msl_entry, MetalKqMvB2Iq3s_metal_kq_mv_iq3s_msl_fastmath, ok) + g_pso_kq_mvb4_iq3s = compile_pso(MetalKqMvB4Iq3s_metal_kq_mv_iq3s_msl, MetalKqMvB4Iq3s_metal_kq_mv_iq3s_msl_entry, MetalKqMvB4Iq3s_metal_kq_mv_iq3s_msl_fastmath, ok) + g_pso_kq_mvb8_iq3s = compile_pso(metal_kq_mvb8_iq3s_msl, metal_kq_mvb8_iq3s_msl_entry, metal_kq_mvb8_iq3s_msl_fastmath, ok) + g_pso_kq_mm_b33 = compile_pso(MetalKqMulMmIq3s_metal_kq_mulmm_msl, MetalKqMulMmIq3s_metal_kq_mulmm_msl_entry, MetalKqMulMmIq3s_metal_kq_mulmm_msl_fastmath, ok) // the batched-step set (P4) g_pso_gemv_b2 = compile_pso(MetalGemvB2_metal_gemv_msl, MetalGemvB2_metal_gemv_msl_entry, MetalGemvB2_metal_gemv_msl_fastmath, ok) g_pso_gemv_b4 = compile_pso(MetalGemvB4_metal_gemv_msl, MetalGemvB4_metal_gemv_msl_entry, MetalGemvB4_metal_gemv_msl_fastmath, ok) @@ -3716,6 +3721,8 @@ def enc_kq_gemv(enc : MetalComputeEncoder?; fmt : KqFmt; bq : MetalBuffer?; qoff enc_kq_iq4xs_c(enc, bs, doff, bs, soff, bq, qoff, bx, xoff, by, yoff, bn, bd, rows, n) } elif (fmt == KqFmt.k3) { enc_kq_k3_c(enc, bs, doff, bs, soff, bq, qoff, bx, xoff, by, yoff, bn, bd, rows, n) + } elif (fmt == KqFmt.iq3s) { + enc_kq_iq3s_c(enc, bs, doff, bs, soff, bq, qoff, bx, xoff, by, yoff, bn, bd, rows, n) } elif (fmt == KqFmt.k6) { enc_kq_k6_c(enc, bs, doff, bs, soff, bq, qoff, bx, xoff, by, yoff, bn, bd, rows, n) } elif (fmt == KqFmt.k5 && (n < 3072l || rows <= 1024l)) { @@ -3768,6 +3775,14 @@ def enc_kq_mvb(enc : MetalComputeEncoder?; fmt : KqFmt; bq : MetalBuffer?; qoff } else { enc_kq_mvb4_k3_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows, gcols) } + } elif (fmt == KqFmt.iq3s) { + if (b8) { + enc_kq_mvb8_iq3s_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows) + } elif (two) { + enc_kq_mvb2_iq3s_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows) + } else { + enc_kq_mvb4_iq3s_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows, gcols) + } } elif (fmt == KqFmt.k6) { if (b8) { enc_kq_mvb8_k6_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows) @@ -3804,6 +3819,8 @@ def enc_kq_gemm_mm_b(enc : MetalComputeEncoder?; fmt : KqFmt; bq : MetalBuffer?; enc_kq_mm_iq4xs_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, bk, bn, mp, rows) } elif (fmt == KqFmt.k3) { enc_kq_mm_k3_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, bk, bn, mp, rows) + } elif (fmt == KqFmt.iq3s) { + enc_kq_mm_iq3s_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, bk, bn, mp, rows) } elif (fmt == KqFmt.k6) { enc_kq_mm_k6_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, bk, bn, mp, rows) } elif (fmt == KqFmt.k4) { @@ -4240,6 +4257,11 @@ def metal_kernels_release { // nolint:STYLE037,STYLE038 — a flat release_pso release_pso(g_pso_kq_mvb4_k3) release_pso(g_pso_kq_mvb8_k3) release_pso(g_pso_kq_mm_b3) + release_pso(g_pso_kq_iq3s) + release_pso(g_pso_kq_mvb2_iq3s) + release_pso(g_pso_kq_mvb4_iq3s) + release_pso(g_pso_kq_mvb8_iq3s) + release_pso(g_pso_kq_mm_b33) if (g_pso_rms != null) { metal_release(g_pso_rms) g_pso_rms = null @@ -5467,7 +5489,9 @@ class template MetalKqMulMmK45T : MetalMmTileBase { @template_constant SIXBIT : bool = false // q6_K: int8 sub-scales x super-d, ql/qh compose @template_constant IQ4XS : bool = false // iq4xs: the k6 split scale form, the q40 pairing, codebook decode @template_constant K3 : bool = false // q3_K: the k6 split scale form, 2-bit lane + hmask compose, offset 4 + @template_constant IQ3S : bool = false // iq3s: the k6 split scale form, threadgroup grid gather + sign flip @workgroup @template_gate = IQ4XS iq4lut : float[16] // kvalues_iq4nl as floats, staged once per threadgroup + @workgroup @template_gate = IQ3S gridw3 : uint[512] // the 2 KB iq3s grid, staged once per threadgroup [metal_kernel] def metal_kq_mulmm { // nolint:STYLE037,STYLE038 — MSL kernel body; the mul_mm tiles are simdgroup-barrier coupled @@ -5478,6 +5502,13 @@ class template MetalKqMulMmK45T : MetalMmTileBase { } barrier() } + static_if (IQ3S) { + for [unroll_full] (gk in range(4)) { + let gi = lid + uint(gk) * 128u + gridw3[gi] = iq3s_gw(int(gi)) + } + barrier() + } let sg = gl_SubgroupID let mBase = gl_WorkGroupID.x * 32u let nBase = gl_WorkGroupID.y * 64u @@ -5571,55 +5602,75 @@ class template MetalKqMulMmK45T : MetalMmTileBase { va[k * 4 + 3] = float16(dsc3 * float(qv & 4278190080u) - dmn) } } else { - static_if (IQ4XS) { - // iq4xs: strip byte js is the SIGNED sub-scale, super-d off the d plane (the k6 split - // form); block js = 4 uints of the q40 pairing, il0 picks the nibble half; the codebook - // off threadgroup memory + static_if (IQ3S) { + // iq3s: strip byte js is the SIGNED sub-scale (the k6 split form); block js's + // qs word (js*2 + il0) = 4 grid words = 16 elems, signs off the block's own bytes let sv = ks4[blk] let scw = js < 4u ? sv.x : sv.y let dsc = float(ksh[blk]) * float(((int(scw >> ((js & 3u) * 8u)) & 255) ^ 128) - 128) - let qb = blk * 32u + js * 4u - let nsh = il0 * 4u + let qsw = kqu[blk * 26u + js * 2u + il0] + let qh3 = (kqu[blk * 26u + 16u + js / 4u] >> ((js % 4u) * 8u)) & 255u + let sgw3 = kqu[blk * 26u + 18u + js] for [unroll_full] (k in range(4)) { - let u = kqu[qb + uint(k)] + let b3 = il0 * 4u + uint(k) + let idx = ((qsw >> (8u * uint(k))) & 255u) | ((qh3 << (8u - b3)) & 256u) + let nib = (sgw3 >> ((b3 / 2u) * 8u + (b3 % 2u) * 4u)) & 15u + let sw = iq3s_sw(gridw3[idx], nib) for [unroll_full] (c in range(4)) { - va[k * 4 + c] = float16(dsc * iq4lut[(u >> (8u * uint(c) + nsh)) & 15u]) + va[k * 4 + c] = float16(dsc * iq3s_sx(sw, uint(c))) } } } else { - let sv = ks4[blk] - var sc = 0u - var mn = 0u - if (js < 4u) { - sc = (sv.y >> (8u * js)) & 63u - mn = (sv.z >> (8u * js)) & 63u - } else { - let j2 = js - 4u - let hi = sv.w >> (8u * j2) - sc = (hi & 15u) | (((sv.y >> (8u * j2 + 6u)) & 3u) << 4u) - mn = ((hi >> 4u) & 15u) | (((sv.z >> (8u * j2 + 6u)) & 3u) << 4u) - } - let dsc = float(ksh[blk * 8u]) * float(sc) - let dmn = float(ksh[blk * 8u + 1u]) * float(mn) - let qb = blk * BLK + (js / 2u) * 8u + il0 * 4u - static_if (QH) { - let hb = blk * BLK + 32u + il0 * 4u - let nsh = (js & 1u) * 4u + static_if (IQ4XS) { + // iq4xs: strip byte js is the SIGNED sub-scale, super-d off the d plane (the k6 split + // form); block js = 4 uints of the q40 pairing, il0 picks the nibble half; the codebook + // off threadgroup memory + let sv = ks4[blk] + let scw = js < 4u ? sv.x : sv.y + let dsc = float(ksh[blk]) * float(((int(scw >> ((js & 3u) * 8u)) & 255) ^ 128) - 128) + let qb = blk * 32u + js * 4u + let nsh = il0 * 4u for [unroll_full] (k in range(4)) { let u = kqu[qb + uint(k)] - let hu = kqu[hb + uint(k)] for [unroll_full] (c in range(4)) { - let q = ((u >> (8u * uint(c) + nsh)) & 15u) | (((hu >> (8u * uint(c) + js)) & 1u) << 4u) - va[k * 4 + c] = float16(dsc * float(q) - dmn) + va[k * 4 + c] = float16(dsc * iq4lut[(u >> (8u * uint(c) + nsh)) & 15u]) } } } else { - let nsh = (js & 1u) * 4u - for [unroll_full] (k in range(4)) { - let u = kqu[qb + uint(k)] - for [unroll_full] (c in range(4)) { - let q = (u >> (8u * uint(c) + nsh)) & 15u - va[k * 4 + c] = float16(dsc * float(q) - dmn) + let sv = ks4[blk] + var sc = 0u + var mn = 0u + if (js < 4u) { + sc = (sv.y >> (8u * js)) & 63u + mn = (sv.z >> (8u * js)) & 63u + } else { + let j2 = js - 4u + let hi = sv.w >> (8u * j2) + sc = (hi & 15u) | (((sv.y >> (8u * j2 + 6u)) & 3u) << 4u) + mn = ((hi >> 4u) & 15u) | (((sv.z >> (8u * j2 + 6u)) & 3u) << 4u) + } + let dsc = float(ksh[blk * 8u]) * float(sc) + let dmn = float(ksh[blk * 8u + 1u]) * float(mn) + let qb = blk * BLK + (js / 2u) * 8u + il0 * 4u + static_if (QH) { + let hb = blk * BLK + 32u + il0 * 4u + let nsh = (js & 1u) * 4u + for [unroll_full] (k in range(4)) { + let u = kqu[qb + uint(k)] + let hu = kqu[hb + uint(k)] + for [unroll_full] (c in range(4)) { + let q = ((u >> (8u * uint(c) + nsh)) & 15u) | (((hu >> (8u * uint(c) + js)) & 1u) << 4u) + va[k * 4 + c] = float16(dsc * float(q) - dmn) + } + } + } else { + let nsh = (js & 1u) * 4u + for [unroll_full] (k in range(4)) { + let u = kqu[qb + uint(k)] + for [unroll_full] (c in range(4)) { + let q = (u >> (8u * uint(c) + nsh)) & 15u + va[k * 4 + c] = float16(dsc * float(q) - dmn) + } } } } @@ -5676,6 +5727,12 @@ class MetalKqMulMmK3 : MetalKqMulMmK45T { override K3 = true } +// iq3s rides the IQ3S arm; the split scale plane binds twice like k6 (d plane at s0off = doff) +[metal_dispatch(name = "enc_kq_mm_iq3s_c", pso = "g_pso_kq_mm_b33", tgmem = "MetalKqMulMmIq3s_metal_kq_mulmm_msl_tgmem", tg = 128, grid = "mp/32, rows/64", params = "mp : int64, rows : int64")] +class MetalKqMulMmIq3s : MetalKqMulMmK45T { + override IQ3S = true +} + // Router GEMV over the uploaded fp32 router slab: y[ne] = W[ne x dim] . x per stream. ne <= 256 // rows. Each sg serves NR positions per weight load (slab SLC traffic /NR); BATCHED stamps // clamp tails via ns; singles fold the clamp away and the gate erases ns outright. @@ -5960,6 +6017,88 @@ def private iq4_lut(q : uint) : float { return float((int((w >> ((q & 3u) * 8u)) & 255u) ^ 128) - 128) } +// ggml's iq3s_grid, one word per call - the GEMV/mul_mm stage it into threadgroup memory once +// per group; the mv twins read it directly (unmeasured batch shapes - followup #58's stance). +def private iq3s_gw(i : int) : uint { + let tbl = fixed_array( + 0x01010101u, 0x01010103u, 0x01010105u, 0x0101010bu, 0x0101010fu, 0x01010301u, 0x01010303u, 0x01010305u, + 0x01010309u, 0x0101030du, 0x01010501u, 0x01010503u, 0x0101050bu, 0x01010707u, 0x01010901u, 0x01010905u, + 0x0101090bu, 0x0101090fu, 0x01010b03u, 0x01010b07u, 0x01010d01u, 0x01010d05u, 0x01010f03u, 0x01010f09u, + 0x01010f0fu, 0x01030101u, 0x01030103u, 0x01030105u, 0x01030109u, 0x01030301u, 0x01030303u, 0x0103030bu, + 0x01030501u, 0x01030507u, 0x0103050fu, 0x01030703u, 0x0103070bu, 0x01030909u, 0x01030d03u, 0x01030d0bu, + 0x01030f05u, 0x01050101u, 0x01050103u, 0x0105010bu, 0x0105010fu, 0x01050301u, 0x01050307u, 0x0105030du, + 0x01050503u, 0x0105050bu, 0x01050701u, 0x01050709u, 0x01050905u, 0x0105090bu, 0x0105090fu, 0x01050b03u, + 0x01050b07u, 0x01050f01u, 0x01050f07u, 0x01070107u, 0x01070303u, 0x0107030bu, 0x01070501u, 0x01070505u, + 0x01070703u, 0x01070707u, 0x0107070du, 0x01070909u, 0x01070b01u, 0x01070b05u, 0x01070d0fu, 0x01070f03u, + 0x01070f0bu, 0x01090101u, 0x01090307u, 0x0109030fu, 0x01090503u, 0x01090509u, 0x01090705u, 0x01090901u, + 0x01090907u, 0x01090b03u, 0x01090f01u, 0x010b0105u, 0x010b0109u, 0x010b0501u, 0x010b0505u, 0x010b050du, + 0x010b0707u, 0x010b0903u, 0x010b090bu, 0x010b090fu, 0x010b0d0du, 0x010b0f07u, 0x010d010du, 0x010d0303u, + 0x010d0307u, 0x010d0703u, 0x010d0b05u, 0x010d0f03u, 0x010f0101u, 0x010f0105u, 0x010f0109u, 0x010f0501u, + 0x010f0505u, 0x010f050du, 0x010f0707u, 0x010f0b01u, 0x010f0b09u, 0x03010101u, 0x03010103u, 0x03010105u, + 0x03010109u, 0x03010301u, 0x03010303u, 0x03010307u, 0x0301030bu, 0x0301030fu, 0x03010501u, 0x03010505u, + 0x03010703u, 0x03010709u, 0x0301070du, 0x03010b09u, 0x03010b0du, 0x03010d03u, 0x03010f05u, 0x03030101u, + 0x03030103u, 0x03030107u, 0x0303010du, 0x03030301u, 0x03030309u, 0x03030503u, 0x03030701u, 0x03030707u, + 0x03030903u, 0x03030b01u, 0x03030b05u, 0x03030f01u, 0x03030f0du, 0x03050101u, 0x03050305u, 0x0305030bu, + 0x0305030fu, 0x03050501u, 0x03050509u, 0x03050705u, 0x03050901u, 0x03050907u, 0x03050b0bu, 0x03050d01u, + 0x03050f05u, 0x03070103u, 0x03070109u, 0x0307010fu, 0x03070301u, 0x03070307u, 0x03070503u, 0x0307050fu, + 0x03070701u, 0x03070709u, 0x03070903u, 0x03070d05u, 0x03070f01u, 0x03090107u, 0x0309010bu, 0x03090305u, + 0x03090309u, 0x03090703u, 0x03090707u, 0x03090905u, 0x0309090du, 0x03090b01u, 0x03090b09u, 0x030b0103u, + 0x030b0301u, 0x030b0307u, 0x030b0503u, 0x030b0701u, 0x030b0705u, 0x030b0b03u, 0x030d0501u, 0x030d0509u, + 0x030d050fu, 0x030d0909u, 0x030d090du, 0x030f0103u, 0x030f0107u, 0x030f0301u, 0x030f0305u, 0x030f0503u, + 0x030f070bu, 0x030f0903u, 0x030f0d05u, 0x030f0f01u, 0x05010101u, 0x05010103u, 0x05010107u, 0x0501010bu, + 0x0501010fu, 0x05010301u, 0x05010305u, 0x05010309u, 0x0501030du, 0x05010503u, 0x05010507u, 0x0501050fu, + 0x05010701u, 0x05010705u, 0x05010903u, 0x05010907u, 0x0501090bu, 0x05010b01u, 0x05010b05u, 0x05010d0fu, + 0x05010f01u, 0x05010f07u, 0x05010f0bu, 0x05030101u, 0x05030105u, 0x05030301u, 0x05030307u, 0x0503030fu, + 0x05030505u, 0x0503050bu, 0x05030703u, 0x05030709u, 0x05030905u, 0x05030b03u, 0x05050103u, 0x05050109u, + 0x0505010fu, 0x05050503u, 0x05050507u, 0x05050701u, 0x0505070fu, 0x05050903u, 0x05050b07u, 0x05050b0fu, + 0x05050f03u, 0x05050f09u, 0x05070101u, 0x05070105u, 0x0507010bu, 0x05070303u, 0x05070505u, 0x05070509u, + 0x05070703u, 0x05070707u, 0x05070905u, 0x05070b01u, 0x05070d0du, 0x05090103u, 0x0509010fu, 0x05090501u, + 0x05090507u, 0x05090705u, 0x0509070bu, 0x05090903u, 0x05090f05u, 0x05090f0bu, 0x050b0109u, 0x050b0303u, + 0x050b0505u, 0x050b070fu, 0x050b0901u, 0x050b0b07u, 0x050b0f01u, 0x050d0101u, 0x050d0105u, 0x050d010fu, + 0x050d0503u, 0x050d0b0bu, 0x050d0d03u, 0x050f010bu, 0x050f0303u, 0x050f050du, 0x050f0701u, 0x050f0907u, + 0x050f0b01u, 0x07010105u, 0x07010303u, 0x07010307u, 0x0701030bu, 0x0701030fu, 0x07010505u, 0x07010703u, + 0x07010707u, 0x0701070bu, 0x07010905u, 0x07010909u, 0x0701090fu, 0x07010b03u, 0x07010d07u, 0x07010f03u, + 0x07030103u, 0x07030107u, 0x0703010bu, 0x07030309u, 0x07030503u, 0x07030507u, 0x07030901u, 0x07030d01u, + 0x07030f05u, 0x07030f0du, 0x07050101u, 0x07050305u, 0x07050501u, 0x07050705u, 0x07050709u, 0x07050b01u, + 0x07070103u, 0x07070301u, 0x07070309u, 0x07070503u, 0x07070507u, 0x0707050fu, 0x07070701u, 0x07070903u, + 0x07070907u, 0x0707090fu, 0x07070b0bu, 0x07070f07u, 0x07090107u, 0x07090303u, 0x0709030du, 0x07090505u, + 0x07090703u, 0x07090b05u, 0x07090d01u, 0x07090d09u, 0x070b0103u, 0x070b0301u, 0x070b0305u, 0x070b050bu, + 0x070b0705u, 0x070b0909u, 0x070b0b0du, 0x070b0f07u, 0x070d030du, 0x070d0903u, 0x070f0103u, 0x070f0107u, + 0x070f0501u, 0x070f0505u, 0x070f070bu, 0x09010101u, 0x09010109u, 0x09010305u, 0x09010501u, 0x09010509u, + 0x0901050fu, 0x09010705u, 0x09010903u, 0x09010b01u, 0x09010f01u, 0x09030105u, 0x0903010fu, 0x09030303u, + 0x09030307u, 0x09030505u, 0x09030701u, 0x0903070bu, 0x09030907u, 0x09030b03u, 0x09030b0bu, 0x09050103u, + 0x09050107u, 0x09050301u, 0x0905030bu, 0x09050503u, 0x09050707u, 0x09050901u, 0x09050b0fu, 0x09050d05u, + 0x09050f01u, 0x09070109u, 0x09070303u, 0x09070307u, 0x09070501u, 0x09070505u, 0x09070703u, 0x0907070bu, + 0x09090101u, 0x09090105u, 0x09090509u, 0x0909070fu, 0x09090901u, 0x09090f03u, 0x090b010bu, 0x090b010fu, + 0x090b0503u, 0x090b0d05u, 0x090d0307u, 0x090d0709u, 0x090d0d01u, 0x090f0301u, 0x090f030bu, 0x090f0701u, + 0x090f0907u, 0x090f0b03u, 0x0b010105u, 0x0b010301u, 0x0b010309u, 0x0b010505u, 0x0b010901u, 0x0b010909u, + 0x0b01090fu, 0x0b010b05u, 0x0b010d0du, 0x0b010f09u, 0x0b030103u, 0x0b030107u, 0x0b03010bu, 0x0b030305u, + 0x0b030503u, 0x0b030705u, 0x0b030f05u, 0x0b050101u, 0x0b050303u, 0x0b050507u, 0x0b050701u, 0x0b05070du, + 0x0b050b07u, 0x0b070105u, 0x0b07010fu, 0x0b070301u, 0x0b07050fu, 0x0b070909u, 0x0b070b03u, 0x0b070d0bu, + 0x0b070f07u, 0x0b090103u, 0x0b090109u, 0x0b090501u, 0x0b090705u, 0x0b09090du, 0x0b0b0305u, 0x0b0b050du, + 0x0b0b0b03u, 0x0b0b0b07u, 0x0b0d0905u, 0x0b0f0105u, 0x0b0f0109u, 0x0b0f0505u, 0x0d010303u, 0x0d010307u, + 0x0d01030bu, 0x0d010703u, 0x0d010707u, 0x0d010d01u, 0x0d030101u, 0x0d030501u, 0x0d03050fu, 0x0d030d09u, + 0x0d050305u, 0x0d050709u, 0x0d050905u, 0x0d050b0bu, 0x0d050d05u, 0x0d050f01u, 0x0d070101u, 0x0d070309u, + 0x0d070503u, 0x0d070901u, 0x0d09050bu, 0x0d090907u, 0x0d090d05u, 0x0d0b0101u, 0x0d0b0107u, 0x0d0b0709u, + 0x0d0b0d01u, 0x0d0d010bu, 0x0d0d0901u, 0x0d0f0303u, 0x0d0f0307u, 0x0f010101u, 0x0f010109u, 0x0f01010fu, + 0x0f010501u, 0x0f010505u, 0x0f01070du, 0x0f010901u, 0x0f010b09u, 0x0f010d05u, 0x0f030105u, 0x0f030303u, + 0x0f030509u, 0x0f030907u, 0x0f03090bu, 0x0f050103u, 0x0f050109u, 0x0f050301u, 0x0f05030du, 0x0f050503u, + 0x0f050701u, 0x0f050b03u, 0x0f070105u, 0x0f070705u, 0x0f07070bu, 0x0f070b07u, 0x0f090103u, 0x0f09010bu, + 0x0f090307u, 0x0f090501u, 0x0f090b01u, 0x0f0b0505u, 0x0f0b0905u, 0x0f0d0105u, 0x0f0d0703u, 0x0f0f0101u) + return tbl[i] +} + +// one signed grid word: the 4-bit sign nibble expands to a byte mask; magnitudes are odd +// 1..15, so the negate's +1 never carries across a byte lane +def private iq3s_sw(g, nib : uint) : uint { + let m1 = (nib * 0x00204081u) & 0x01010101u + return (g ^ (m1 * 255u)) + m1 +} + +def private iq3s_sx(sw, t : uint) : float { + return float((int((sw >> (t * 8u)) & 255u) ^ 128) - 128) +} + // Expert-indexed Q4_K GEMV — MetalKqGemvK4's dot; the superblock index shifts by sel[slot] * esb. // x rides a float4 view: the 32-scalar-x-load form was load-issue-bound at the routed expert // shapes (nfe-row planes) — the vector view is 2.25x there (lab: bench_metal_moe_lab, 142→321 wGB/s). @@ -7268,6 +7407,241 @@ class MetalKqGemvIq4xs { } } +// IQ3_S: w = d*sc*(+-grid byte); the 2 KB grid in threadgroup memory (llama.cpp's shape), a +// 16-lane half-simdgroup per superblock, lane (bu, il) = block bu's half il: one qs word = 4 +// grid words = 16 elems = 4 CONSECUTIVE float4 x loads. Scale plane = the iq4xs/k6 split form. +[metal_dispatch(name = "enc_kq_iq3s_c", pso = "g_pso_kq_iq3s", tgmem = "metal_kq_gemv_iq3s_msl_tgmem", tg = 64, grid = "rows/8", params = "rows : int64, n : int64")] +class MetalKqGemvIq3s { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d plane — the scale buffer bound at byte nsb*16 (s0off = the caller's doff) + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // 16B strips: 8 signed sub-scales + 8 pad, byte view (same buffer at 0) + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array // iq3s quant plane, uint view (26 words per superblock: 16 qs, 2 qh, 8 signs) + @ssbo @binding = 3 @role = "read" @off = "xoff" @span = "n*4" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" @span = "rows*4" y : array + @uniform @binding = 5 ndim : uint + @uniform @binding = 6 ddim : uint + @workgroup gridf : float4[512] // magnitudes pre-expanded at stage time: the inner loop reads one float4 and flips signs, no byte extracts (the best of the eight raced forms - llama.cpp's own geometry included) + + [metal_kernel(name="metal_kq_gemv_iq3s_msl")] + def metal_kq_gemv_iq3s { + let lid = gl_LocalInvocationID.x + for [unroll_full] (k in range(8)) { + let gi = lid + uint(k) * 64u + let gw = iq3s_gw(int(gi)) + gridf[gi] = float4(float(gw & 255u), float((gw >> 8u) & 255u), float((gw >> 16u) & 255u), float(gw >> 24u)) + } + barrier() + let lane = gl_SubgroupInvocationID + let ix = lane / 16u + let it = lane % 16u + let bu = it / 2u + let il = it % 2u + let nb = ndim / 256u + let first_row = (gl_WorkGroupID.x * 2u + gl_SubgroupID) * 4u // 4 rows per simdgroup - y loads and grid gathers amortize x4 (llama.cpp N_R0_IQ3_S) + var sumf : float[4] + var ib = ix + while (ib < nb) { + let yb4 = ib * 64u + bu * 8u + il * 4u + let y0 = x[yb4] + let y1 = x[yb4 + 1u] + let y2 = x[yb4 + 2u] + let y3 = x[yb4 + 3u] + for [unroll_full] (r in range(4)) { + let blk = (first_row + uint(r)) * nb + ib + let qsw = kqu[blk * 26u + bu * 2u + il] + let qh = (kqu[blk * 26u + 16u + bu / 4u] >> ((bu % 4u) * 8u)) & 255u + let sgw = kqu[blk * 26u + 18u + bu] + var accv = float4(0.0) + for [unroll_full] (wi in range(4)) { + let b = il * 4u + uint(wi) + let idx = ((qsw >> (8u * uint(wi))) & 255u) | ((qh << (8u - b)) & 256u) + let nib = (sgw >> ((b / 2u) * 8u + (b % 2u) * 4u)) & 15u + let fs = float4((nib & 1u) != 0u ? -1.0 : 1.0, (nib & 2u) != 0u ? -1.0 : 1.0, + (nib & 4u) != 0u ? -1.0 : 1.0, (nib & 8u) != 0u ? -1.0 : 1.0) + let yv = wi == 0 ? y0 : (wi == 1 ? y1 : (wi == 2 ? y2 : y3)) + accv += yv * (gridf[idx] * fs) + } + let sc = float((int(kscb[blk * 16u + bu]) ^ 128) - 128) + sumf[r] += float(kdh[blk]) * sc * (accv.x + accv.y + accv.z + accv.w) + } + ib += 2u + } + for [unroll_full] (r in range(4)) { + let s = simd_sum(sumf[r]) + if (lane == 0u && first_row + uint(r) < ddim) { + y[first_row + uint(r)] = s + } + } + } +} + +// The iq3s B2/B4 pair: the iq4xs shell over the per-32-block lane map — lane tx owns block tx, +// 8 grid words = 32 consecutive elems as float4s; the grid read direct (unmeasured batch shapes). +[ |> template_struct_instance] +class template MetalKqMvIq3sT { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d plane — the scale buffer bound at byte nsb*16 + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // 16B strips: 8 signed sub-scales + 8 pad (same buffer at 0) + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array // iq3s quant plane, uint view + @ssbo @binding = 3 @role = "read" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" y : array + @uniform @binding = 5 ka : KqMvArgs + @template_constant NR : int = 2 + @template_constant NRU : uint = 2u // NR's uint spelling (colbase math) — override BOTH together + @template_constant TILED : bool = false + + [metal_kernel] + def metal_kq_mv_iq3s { + let lane = gl_SubgroupInvocationID + let tx = lane % 8u + let ty = lane / 8u + let row = gl_WorkGroupID.x * 8u + gl_SubgroupID * 4u + ty + let colbase = gl_WorkGroupID.y * NRU // nolint:LINT002 dead in the TILED=false stamp — measured free + let nb = ka.ndim / 256u + let nb4 = ka.ndim / 4u + var sumf : float[NR] + var sb = 0u + while (sb < nb) { + let blk = row * nb + sb + let dl = float(kdh[blk]) * float((int(kscb[blk * 16u + tx]) ^ 128) - 128) + let qh = (kqu[blk * 26u + 16u + tx / 4u] >> ((tx % 4u) * 8u)) & 255u + let sgw = kqu[blk * 26u + 18u + tx] + var w : float4[8] + for [unroll_full] (h in range(2)) { + let qsw = kqu[blk * 26u + tx * 2u + uint(h)] + for [unroll_full] (wi in range(4)) { + let b = uint(h) * 4u + uint(wi) + let idx = ((qsw >> (8u * uint(wi))) & 255u) | ((qh << (8u - b)) & 256u) + let nib = (sgw >> ((b / 2u) * 8u + (b % 2u) * 4u)) & 15u + let sw = iq3s_sw(iq3s_gw(int(idx)), nib) + w[h * 4 + wi] = float4(iq3s_sx(sw, 0u), iq3s_sx(sw, 1u), iq3s_sx(sw, 2u), iq3s_sx(sw, 3u)) * dl + } + } + let cx4 = sb * 64u + tx * 8u + static_if (TILED) { + for [unroll_full] (b in range(NR)) { + let xb4 = (colbase + uint(b)) * nb4 + cx4 + var acc = 0.0 + for [unroll_full] (u in range(8)) { + let v = x[xb4 + uint(u)] + acc += v.x * w[u].x + v.y * w[u].y + v.z * w[u].z + v.w * w[u].w + } + sumf[b] += acc + } + } else { + for [unroll_full] (b in range(NR)) { + let xb4 = uint(b) * nb4 + cx4 + var acc = 0.0 + for [unroll_full] (u in range(8)) { + let v = x[xb4 + uint(u)] + acc += v.x * w[u].x + v.y * w[u].y + v.z * w[u].z + v.w * w[u].w + } + sumf[b] += acc + } + } + sb++ + } + static_if (TILED) { + for [unroll_full] (b in range(NR)) { + var s = sumf[b] + s += simd_shuffle_down(s, 4u) + s += simd_shuffle_down(s, 2u) + s += simd_shuffle_down(s, 1u) + if (tx == 0u && row < ka.ddim && colbase + uint(b) < ka.nr) { + y[(colbase + uint(b)) * ka.ys + row] = s + } + } + } else { + for [unroll_full] (b in range(NR)) { + var s = sumf[b] + s += simd_shuffle_down(s, 4u) + s += simd_shuffle_down(s, 2u) + s += simd_shuffle_down(s, 1u) + if (tx == 0u && row < ka.ddim && uint(b) < ka.nr) { + y[uint(b) * ka.ys + row] = s + } + } + } + } +} + +[metal_dispatch(name = "enc_kq_mvb2_iq3s_c", pso = "g_pso_kq_mvb2_iq3s", tg = 64, grid = "rows/8", params = "rows : int64")] +class MetalKqMvB2Iq3s : MetalKqMvIq3sT { + override NR = 2 + override NRU = 2u + override TILED = false +} + +[metal_dispatch(name = "enc_kq_mvb4_iq3s_c", pso = "g_pso_kq_mvb4_iq3s", tg = 64, grid = "rows/8, gcols", params = "rows : int64, gcols : int64")] +class MetalKqMvB4Iq3s : MetalKqMvIq3sT { + override NR = 4 + override NRU = 4u + override TILED = true +} + +// the iq3s B8 twin: the iq4xs B8 shell (one superblock's X panel staged per threadgroup) over +// the per-32-block lane map; the grid read direct. +[metal_dispatch(name = "enc_kq_mvb8_iq3s_c", pso = "g_pso_kq_mvb8_iq3s", tgmem = "metal_kq_mvb8_iq3s_msl_tgmem", tg = 64, grid = "rows/8", params = "rows : int64")] +class MetalKqMvB8Iq3s { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d plane — the scale buffer bound at byte nsb*16 + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // 16B strips: 8 signed sub-scales + 8 pad (same buffer at 0) + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array // iq3s quant plane, uint view + @ssbo @binding = 3 @role = "read" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" y : array + @uniform @binding = 5 ka : KqMvArgs + @workgroup txp : float4[512] // [8 streams x 64 float4] one superblock's X panel + + [metal_kernel(name="metal_kq_mvb8_iq3s_msl")] + def metal_kq_mvb8_iq3s { + let lid = gl_LocalInvocationID.x + let lane = gl_SubgroupInvocationID + let tx = lane % 8u + let ty = lane / 8u + let row = gl_WorkGroupID.x * 8u + gl_SubgroupID * 4u + ty + let nb = ka.ndim / 256u + let nb4 = ka.ndim / 4u + var acc : float4[8] + var sb = 0u + while (sb < nb) { + barrier() + for [unroll_full] (j in range(8)) { + let slot = uint(j) * 64u + lid + txp[slot] = x[(slot / 64u) * nb4 + sb * 64u + (slot % 64u)] + } + barrier() + let blk = row * nb + sb + let dl = float(kdh[blk]) * float((int(kscb[blk * 16u + tx]) ^ 128) - 128) + let qh = (kqu[blk * 26u + 16u + tx / 4u] >> ((tx % 4u) * 8u)) & 255u + let sgw = kqu[blk * 26u + 18u + tx] + var w : float4[8] + for [unroll_full] (h in range(2)) { + let qsw = kqu[blk * 26u + tx * 2u + uint(h)] + for [unroll_full] (wi in range(4)) { + let b = uint(h) * 4u + uint(wi) + let idx = ((qsw >> (8u * uint(wi))) & 255u) | ((qh << (8u - b)) & 256u) + let nib = (sgw >> ((b / 2u) * 8u + (b % 2u) * 4u)) & 15u + let sw = iq3s_sw(iq3s_gw(int(idx)), nib) + w[h * 4 + wi] = float4(iq3s_sx(sw, 0u), iq3s_sx(sw, 1u), iq3s_sx(sw, 2u), iq3s_sx(sw, 3u)) * dl + } + } + let cxl = tx * 8u + for [unroll_full] (b in range(8)) { + for [unroll_full] (u in range(8)) { + acc[b] += txp[uint(b) * 64u + cxl + uint(u)] * w[u] + } + } + sb++ + } + for [unroll_full] (b in range(8)) { + var s = acc[b].x + acc[b].y + acc[b].z + acc[b].w + s += simd_shuffle_down(s, 4u) + s += simd_shuffle_down(s, 2u) + s += simd_shuffle_down(s, 1u) + if (tx == 0u && row < ka.ddim && uint(b) < ka.nr) { + y[uint(b) * ka.ys + row] = s + } + } + } +} + // The iq4xs B2/B4 pair: the K4T shell (row/colbase/TILED twins) over the per-32-block lane map of // the GEMV above — lane tx owns block tx of each superblock, 16 lo + 16 hi weights as float4s. [ |> template_struct_instance] diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das b/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das index 965e70410f..db4571a8b9 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das @@ -4411,7 +4411,7 @@ def private pf_devw_panel(enc : MetalComputeEncoder?; bwblob : MetalBuffer?; wbo //! quant superblocks, plus the k6 2B d tail def private pf_devw_panel_kq(enc : MetalComputeEncoder?; t : Model; fmt : KqFmt; woff : int64; bxh, by, bk, bn : MetalBuffer?; rows, d, kdim : int64; yoff : uint64) : bool { - if (fmt == KqFmt.iq4xs || fmt == KqFmt.k3) { // no dev-W dequant kernel for these formats yet + if (fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s) { // no dev-W dequant kernel for these formats yet return false } let dq = fmt == KqFmt.k6 ? g_pf_pso_dq_k6 : (fmt == KqFmt.k4 ? g_pf_pso_dq_k4 : g_pf_pso_dq_k5) @@ -4584,6 +4584,10 @@ def private pf_enc_kq_site_mm(enc : MetalComputeEncoder?; t : Model; fmt : KqFmt enc_kq_mm_k3_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, by, yoff, bk, bn, mp, rows) return } + if (fmt == KqFmt.iq3s) { // the base mul_mm only - no tensor / tall / dev-W twins yet + enc_kq_mm_iq3s_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, by, yoff, bk, bn, mp, rows) + return + } //! deep-class arm: a panel dev-W would split past 8 tiles is DRAM-resident - the tall //! in-kernel-dequant stamp reads the quant plane once per 128-row tile instead of //! materializing and re-streaming a 2B/element f16 panel @@ -4691,6 +4695,8 @@ def private pf_enc_kq_gemv(enc : MetalComputeEncoder?; t : Model; fmt : KqFmt; w enc_kq_iq4xs_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, xoff, by, 0ul, bn, bd, rows, n) } elif (fmt == KqFmt.k3) { enc_kq_k3_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, xoff, by, 0ul, bn, bd, rows, n) + } elif (fmt == KqFmt.iq3s) { + enc_kq_iq3s_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, xoff, by, 0ul, bn, bd, rows, n) } elif (fmt == KqFmt.k6) { enc_kq_k6_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, xoff, by, 0ul, bn, bd, rows, n) } elif (fmt == KqFmt.k4) { diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_shapes.das b/modules/dasLLAMA/dasllama/dasllama_metal_shapes.das index 6994dfa85d..f9068b343f 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_shapes.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_shapes.das @@ -180,7 +180,7 @@ def record_needs(var tab : table; missing : MetalNeed) { // formats) must DECLINE kquant_native instead of falling into a wrong-layout kernel branch // (the dispatchers below treat "not k4/k6" as k5) def kq_fmt_gpu_supported(f : KqFmt) : bool { - return f == KqFmt.q8 || f == KqFmt.k4 || f == KqFmt.k5 || f == KqFmt.k6 || f == KqFmt.iq4xs || f == KqFmt.k3 + return f == KqFmt.q8 || f == KqFmt.k4 || f == KqFmt.k5 || f == KqFmt.k6 || f == KqFmt.iq4xs || f == KqFmt.k3 || f == KqFmt.iq3s } def private kq_fmts_gpu_supported(a : array) : bool { @@ -243,7 +243,7 @@ def moe_site_ok(fmt : KqFmt; off, ege : int64) : bool { if (fmt == KqFmt.k4 || fmt == KqFmt.k5) { return off >= 0l && (off % 256l) == 0l && (ege % 256l) == 0l } - if (fmt == KqFmt.k6 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3) { // the split scale form: off % 512 keeps the d-plane bind 4B-aligned + if (fmt == KqFmt.k6 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s) { // the split scale form: off % 512 keeps the d-plane bind 4B-aligned return off >= 0l && (off % 512l) == 0l && (ege % 256l) == 0l } if (fmt == KqFmt.q51) { // per-32 planes: off % 128 keeps the 20B/4B binds 16B-aligned diff --git a/modules/dasLLAMA/followup_general.md b/modules/dasLLAMA/followup_general.md index bba7498b8b..c42fa72f6c 100644 --- a/modules/dasLLAMA/followup_general.md +++ b/modules/dasLLAMA/followup_general.md @@ -764,3 +764,14 @@ (insertelement per i32 lane, no panel), and/or the signs-on-activation form that drops the abs+psign pair. Done = a gemv perm that takes tg128 at or past llama.cpp's, crowned by the probe. + +62. **IQ3_S Metal decode: the ~140 GB/s compose ceiling (tg 0.95x).** Eight GEMV forms raced + at n=2048 d=8192 - gather placement x3, gather deleted, signs deleted, llama.cpp's exact + 1-lane-per-block geometry, row width, f4 magnitude slab - all land in 127-141 GB/s while + k4 does 204 and k6 287 in the same harness; the 9-bit-index + per-nibble-sign compose is + the cost, not any one instruction class (deleting the gather OR the signs moves it under + 10%). The f4 slab ships (+9%, tg128 0.82x -> 0.95x same-run). Candidates beyond kernel + shape: emitted-MSL diffs vs llama.cpp's compiled kernel (half math, function constants), + or fusing the sign flip into the staged slab per SITE via a second indexed table. Done = + a form that clears 180 GB/s in the dispatch-loop probe (QUIRK 22's harness), or a note + proving the ceiling is shared by llama.cpp's own kernel when isolated the same way. diff --git a/modules/dasLLAMA/tests/_metal_kernel_common.das b/modules/dasLLAMA/tests/_metal_kernel_common.das index de5a06d6f6..921de1a395 100644 --- a/modules/dasLLAMA/tests/_metal_kernel_common.das +++ b/modules/dasLLAMA/tests/_metal_kernel_common.das @@ -174,15 +174,15 @@ let KQ_SAFE_SCALES = fixed_array(0.25, 0.5, 1.0, 0.125, 2.0, 0.75, 1.5, 0.375) // [12 packed 6-bit bytes] per superblock, K6 = [16B sub-scales] x nsb + f16 d tail, IQ4_XS (44) = // the same split form with [8 signed sub-scales in -31..31][8 zero] strips, Q3_K (3) = the k6 form over 96B quants def kq_fill_planes(fmt, nsb : int; var kq : array; var ks : array) { - let qsb = fmt == 4 || fmt == 44 ? 128 : (fmt == 5 ? 160 : (fmt == 3 ? 96 : 192)) + let qsb = fmt == 4 || fmt == 44 ? 128 : (fmt == 5 ? 160 : (fmt == 3 ? 96 : (fmt == 33 ? 104 : 192))) kq |> resize(nsb * qsb) for (i in range(nsb * qsb)) { kq[i] = kq_hash_byte(i * 3 + fmt) } - if (fmt == 6 || fmt == 44 || fmt == 3) { + if (fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33) { ks |> resize(nsb * 18) for (i in range(nsb * 16)) { - ks[i] = fmt == 6 || fmt == 3 ? kq_hash_byte(i * 7 + 13) : uint8(i % 16 < 8 ? int(kq_hash_byte(i * 7 + 13)) % 63 - 31 : 0) + ks[i] = fmt == 6 || fmt == 3 ? kq_hash_byte(i * 7 + 13) : uint8(i % 16 < 8 ? int(kq_hash_byte(i * 7 + 13)) % 63 - 31 : 0) // 44/33: signed strip bytes + zero pad } for (b in range(nsb)) { let dbits = f32_to_f16(KQ_SAFE_SCALES[b % 8]) @@ -251,6 +251,9 @@ def kq_row_ref(fmt, sb_base, nb, nsb_total : int; kq, ks : array; var wro } elif (fmt == 3) { dequant_k3_plane_superblock_at(kq, int64(sb * 96), ks, int64(sb * 16), int64(nsb_total * 16 + sb * 2), wrow, int64(ib * 256)) + } elif (fmt == 33) { + dequant_iq3s_plane_superblock_at(kq, int64(sb * 104), ks, int64(sb * 16), + int64(nsb_total * 16 + sb * 2), wrow, int64(ib * 256)) } else { dequant_k6_plane_superblock_at(kq, int64(sb * 192), ks, int64(sb * 16), int64(nsb_total * 16 + sb * 2), wrow, int64(ib * 256)) diff --git a/modules/dasLLAMA/tests/test_metal_gemm_kernels.das b/modules/dasLLAMA/tests/test_metal_gemm_kernels.das index 2c2f4410ac..ae2fd904a5 100644 --- a/modules/dasLLAMA/tests/test_metal_gemm_kernels.das +++ b/modules/dasLLAMA/tests/test_metal_gemm_kernels.das @@ -36,28 +36,28 @@ def private kq_mulmm_gate(t : T?; dev, queue; fmt : int; tensor : bool; m, kdim, : halfx ? (fmt == 4 ? MetalKqMulMmK4TH_metal_kq_mulmm_t_msl : (fmt == 5 ? MetalKqMulMmK5TH_metal_kq_mulmm_t_msl : MetalKqMulMmK6TH_metal_kq_mulmm_k6_t_msl)) : (fmt == 4 ? MetalKqMulMmK4T_metal_kq_mulmm_t_msl : (fmt == 5 ? MetalKqMulMmK5T_metal_kq_mulmm_t_msl : MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl))) - : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl : MetalKqMulMmK6_metal_kq_mulmm_msl))))) + : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl : MetalKqMulMmK6_metal_kq_mulmm_msl)))))) let entry = (tensor ? (tall ? (fmt == 4 ? MetalKqMulMmK4TH128_metal_kq_mulmm_t_msl_entry : (fmt == 5 ? MetalKqMulMmK5TH128_metal_kq_mulmm_t_msl_entry : MetalKqMulMmK6TH128_metal_kq_mulmm_k6_t_msl_entry)) : halfx ? (fmt == 4 ? MetalKqMulMmK4TH_metal_kq_mulmm_t_msl_entry : (fmt == 5 ? MetalKqMulMmK5TH_metal_kq_mulmm_t_msl_entry : MetalKqMulMmK6TH_metal_kq_mulmm_k6_t_msl_entry)) : (fmt == 4 ? MetalKqMulMmK4T_metal_kq_mulmm_t_msl_entry : (fmt == 5 ? MetalKqMulMmK5T_metal_kq_mulmm_t_msl_entry : MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl_entry))) - : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl_entry : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl_entry : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_entry : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_entry : MetalKqMulMmK6_metal_kq_mulmm_msl_entry))))) + : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl_entry : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl_entry : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl_entry : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_entry : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_entry : MetalKqMulMmK6_metal_kq_mulmm_msl_entry)))))) let fm = (tensor ? (tall ? (fmt == 4 ? MetalKqMulMmK4TH128_metal_kq_mulmm_t_msl_fastmath : (fmt == 5 ? MetalKqMulMmK5TH128_metal_kq_mulmm_t_msl_fastmath : MetalKqMulMmK6TH128_metal_kq_mulmm_k6_t_msl_fastmath)) : halfx ? (fmt == 4 ? MetalKqMulMmK4TH_metal_kq_mulmm_t_msl_fastmath : (fmt == 5 ? MetalKqMulMmK5TH_metal_kq_mulmm_t_msl_fastmath : MetalKqMulMmK6TH_metal_kq_mulmm_k6_t_msl_fastmath)) : (fmt == 4 ? MetalKqMulMmK4T_metal_kq_mulmm_t_msl_fastmath : (fmt == 5 ? MetalKqMulMmK5T_metal_kq_mulmm_t_msl_fastmath : MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl_fastmath))) - : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl_fastmath : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl_fastmath : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_fastmath : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_fastmath : MetalKqMulMmK6_metal_kq_mulmm_msl_fastmath))))) + : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl_fastmath : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl_fastmath : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl_fastmath : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_fastmath : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_fastmath : MetalKqMulMmK6_metal_kq_mulmm_msl_fastmath)))))) let tgm = (tensor ? (tall ? (fmt == 4 ? MetalKqMulMmK4TH128_metal_kq_mulmm_t_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5TH128_metal_kq_mulmm_t_msl_tgmem : MetalKqMulMmK6TH128_metal_kq_mulmm_k6_t_msl_tgmem)) : halfx ? (fmt == 4 ? MetalKqMulMmK4TH_metal_kq_mulmm_t_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5TH_metal_kq_mulmm_t_msl_tgmem : MetalKqMulMmK6TH_metal_kq_mulmm_k6_t_msl_tgmem)) : (fmt == 4 ? MetalKqMulMmK4T_metal_kq_mulmm_t_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5T_metal_kq_mulmm_t_msl_tgmem : MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl_tgmem))) - : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl_tgmem : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl_tgmem : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_tgmem : MetalKqMulMmK6_metal_kq_mulmm_msl_tgmem))))) + : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl_tgmem : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl_tgmem : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl_tgmem : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_tgmem : MetalKqMulMmK6_metal_kq_mulmm_msl_tgmem)))))) var pso = pipeline_from_source(dev, src, entry, fm, err) if (tensor && pso == null && !metal4_tensor_available(dev)) { to_log(LOG_INFO, "{tag}: no Metal-4 tensor toolchain on this box - arm skipped\n") @@ -123,7 +123,7 @@ def private kq_mulmm_gate(t : T?; dev, queue; fmt : int; tensor : bool; m, kdim, let ran = with_compute_encoder(queue, err) $(enc : MetalComputeEncoder?) { metal_set_pipeline(enc, pso) metal_set_threadgroup_memory_length(enc, tgm, 0) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs: the f16 d tail + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs: the f16 d tail metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bx, 0ul, 3) @@ -134,7 +134,7 @@ def private kq_mulmm_gate(t : T?; dev, queue; fmt : int; tensor : bool; m, kdim, if (rem_pso != null) { metal_set_pipeline(enc, rem_pso) metal_set_threadgroup_memory_length(enc, fmt == 4 ? MetalKqMulMmK4TH_metal_kq_mulmm_t_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5TH_metal_kq_mulmm_t_msl_tgmem : MetalKqMulMmK6TH_metal_kq_mulmm_k6_t_msl_tgmem), 0) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 ? uint64(nsb * 16) : 0ul, 0) + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 ? uint64(nsb * 16) : 0ul, 0) metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bx, uint64(r0 * kdim * 2), 3) @@ -484,7 +484,7 @@ def private kq_devw_gate(t : T?; dev, queue; fmt : int; m, kdim, ndim : int) { var dka = CvtArgs(total = uint(nblk)) let ran = with_compute_encoder(queue, err) $(enc : MetalComputeEncoder?) { metal_set_pipeline(enc, dq_pso) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs: the f16 d tail + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs: the f16 d tail metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bwh, 0ul, 3) @@ -1461,7 +1461,7 @@ def private moe_mulmm_kq_gate(t : T?; dev, queue; fmt : int; kdim, ndim : int; c let ran = with_compute_encoder(queue, err) $(enc : MetalComputeEncoder?) { metal_set_pipeline(enc, pso) metal_set_threadgroup_memory_length(enc, tgm, 0) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs: the f16 d tail + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs: the f16 d tail metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bx, 0ul, 3) @@ -1498,7 +1498,7 @@ def private moe_mulmm_kq_gate(t : T?; dev, queue; fmt : int; kdim, ndim : int; c metal_set_threadgroup_memory_length(enc, fmt == 4 ? MetalMoeMulMmK4TH_metal_moe_mulmm_k45_t_msl_tgmem : (fmt == 5 ? MetalMoeMulMmK5TH_metal_moe_mulmm_k45_t_msl_tgmem : MetalMoeMulMmK6TH_metal_moe_mulmm_k6_t_msl_tgmem), 0) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 ? uint64(nsb * 16) : 0ul, 0) + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 ? uint64(nsb * 16) : 0ul, 0) metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bxh, 0ul, 3) @@ -1538,7 +1538,7 @@ def private moe_mulmm_kq_gate(t : T?; dev, queue; fmt : int; kdim, ndim : int; c metal_set_threadgroup_memory_length(enc, fmt == 4 ? MetalMoeMulMmK4TH128_metal_moe_mulmm_k45_t_msl_tgmem : (fmt == 5 ? MetalMoeMulMmK5TH128_metal_moe_mulmm_k45_t_msl_tgmem : MetalMoeMulMmK6TH128_metal_moe_mulmm_k6_t_msl_tgmem), 0) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 ? uint64(nsb * 16) : 0ul, 0) + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 ? uint64(nsb * 16) : 0ul, 0) metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bxh, 0ul, 3) @@ -1551,7 +1551,7 @@ def private moe_mulmm_kq_gate(t : T?; dev, queue; fmt : int; kdim, ndim : int; c metal_set_threadgroup_memory_length(enc, fmt == 4 ? MetalMoeMulMmK4THR_metal_moe_mulmm_k45_t_msl_tgmem : (fmt == 5 ? MetalMoeMulMmK5THR_metal_moe_mulmm_k45_t_msl_tgmem : MetalMoeMulMmK6THR_metal_moe_mulmm_k6_t_msl_tgmem), 0) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 ? uint64(nsb * 16) : 0ul, 0) + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 ? uint64(nsb * 16) : 0ul, 0) metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bxh, 0ul, 3) @@ -2051,6 +2051,8 @@ def test_metal_gemm_kernels(t : T?) { kq_mulmm_gate(t, dev, queue, 44, false, 32, 256, 128) kq_mulmm_gate(t, dev, queue, 3, false, 64, 512, 64) // k3: the base form only (no tensor / tall twins) kq_mulmm_gate(t, dev, queue, 3, false, 32, 256, 128) + kq_mulmm_gate(t, dev, queue, 33, false, 64, 512, 64) // iq3s: the base form only (no tensor / tall twins) + kq_mulmm_gate(t, dev, queue, 33, false, 32, 256, 128) q8_mulmm_t_gate(t, dev, queue, 64, 256, 64) q8_mulmm_t_gate(t, dev, queue, 64, 256, 64, halfx = true) q8_mulmm_t_gate(t, dev, queue, 64, 160, 64) // kdim % 64 == 32: the bk=64 helper's 32-tail chunk diff --git a/modules/dasLLAMA/tests/test_metal_gemv_kernels.das b/modules/dasLLAMA/tests/test_metal_gemv_kernels.das index fdc1be18be..5a5fb13e6b 100644 --- a/modules/dasLLAMA/tests/test_metal_gemv_kernels.das +++ b/modules/dasLLAMA/tests/test_metal_gemv_kernels.das @@ -23,12 +23,12 @@ require math // ===== single-stream kq GEMV (MetalKqGemvK4 / K5 / K5C / K6) ===== def private kq_gemv_gate(t : T?; dev, queue; vform : string; n, d : int) { - let fmt = vform == "k4" ? 4 : (vform == "k6" ? 6 : (vform == "iq4xs" ? 44 : (vform == "k3" ? 3 : 5))) + let fmt = vform == "k4" ? 4 : (vform == "k6" ? 6 : (vform == "iq4xs" ? 44 : (vform == "k3" ? 3 : (vform == "iq3s" ? 33 : 5)))) let tag = "kq_gemv_{vform} n={n} d={d}" var err : string - let src = vform == "k3" ? metal_kq_gemv_k3_msl : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl : (vform == "k4" ? metal_kq_gemv_k4_msl : (vform == "k5" ? metal_kq_gemv_k5_msl : (vform == "k5c" ? metal_kq_gemv_k5c_msl : metal_kq_gemv_k6_msl)))) - let entry = vform == "k3" ? metal_kq_gemv_k3_msl_entry : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl_entry : (vform == "k4" ? metal_kq_gemv_k4_msl_entry : (vform == "k5" ? metal_kq_gemv_k5_msl_entry : (vform == "k5c" ? metal_kq_gemv_k5c_msl_entry : metal_kq_gemv_k6_msl_entry)))) - let fm = vform == "k3" ? metal_kq_gemv_k3_msl_fastmath : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl_fastmath : (vform == "k4" ? metal_kq_gemv_k4_msl_fastmath : (vform == "k5" ? metal_kq_gemv_k5_msl_fastmath : (vform == "k5c" ? metal_kq_gemv_k5c_msl_fastmath : metal_kq_gemv_k6_msl_fastmath)))) + let src = vform == "iq3s" ? metal_kq_gemv_iq3s_msl : (vform == "k3" ? metal_kq_gemv_k3_msl : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl : (vform == "k4" ? metal_kq_gemv_k4_msl : (vform == "k5" ? metal_kq_gemv_k5_msl : (vform == "k5c" ? metal_kq_gemv_k5c_msl : metal_kq_gemv_k6_msl))))) + let entry = vform == "iq3s" ? metal_kq_gemv_iq3s_msl_entry : (vform == "k3" ? metal_kq_gemv_k3_msl_entry : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl_entry : (vform == "k4" ? metal_kq_gemv_k4_msl_entry : (vform == "k5" ? metal_kq_gemv_k5_msl_entry : (vform == "k5c" ? metal_kq_gemv_k5c_msl_entry : metal_kq_gemv_k6_msl_entry))))) + let fm = vform == "iq3s" ? metal_kq_gemv_iq3s_msl_fastmath : (vform == "k3" ? metal_kq_gemv_k3_msl_fastmath : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl_fastmath : (vform == "k4" ? metal_kq_gemv_k4_msl_fastmath : (vform == "k5" ? metal_kq_gemv_k5_msl_fastmath : (vform == "k5c" ? metal_kq_gemv_k5c_msl_fastmath : metal_kq_gemv_k6_msl_fastmath))))) var pso = pipeline_from_source(dev, src, entry, fm, err) t |> success(pso != null, "{tag}: pipeline: {err}") return if (pso == null) @@ -66,13 +66,16 @@ def private kq_gemv_gate(t : T?; dev, queue; vform : string; n, d : int) { var by = buf_fill(dev, d, -1000.0) var bn = buf_u32(dev, uint(n)) var bd = buf_u32(dev, uint(d)) - let groups = vform == "k5c" ? (d + 1) / 2 : (d + 3) / 4 + let groups = vform == "k5c" ? (d + 1) / 2 : (vform == "iq3s" ? (d + 7) / 8 : (d + 3) / 4) let ran = with_compute_encoder(queue, err) $(enc : MetalComputeEncoder?) { metal_set_pipeline(enc, pso) if (vform == "iq4xs") { metal_set_threadgroup_memory_length(enc, metal_kq_gemv_iq4xs_msl_tgmem, 0) // the codebook slab } - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs/k3: the f16 d tail + if (vform == "iq3s") { + metal_set_threadgroup_memory_length(enc, metal_kq_gemv_iq3s_msl_tgmem, 0) // the grid slab + } + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs/k3: the f16 d tail metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bx, 0ul, 3) @@ -105,15 +108,15 @@ def private kq_gemv_gate(t : T?; dev, queue; vform : string; n, d : int) { def private kq_mvb_gate(t : T?; dev, queue; fmt, bwidth, n, d, nr, ys : int) { let tag = "kq_mvb{bwidth}_k{fmt} n={n} d={d} nr={nr} ys={ys}" var err : string - let src = (bwidth == 2 ? (fmt == 3 ? MetalKqMvB2K3_metal_kq_mv_k3_msl : (fmt == 44 ? MetalKqMvB2Iq4xs_metal_kq_mv_iq4xs_msl : (fmt == 4 ? MetalKqMvB2K4_metal_kq_mv_k4_msl : (fmt == 5 ? MetalKqMvB2K5_metal_kq_mv_k5_msl : MetalKqMvB2K6_metal_kq_mv_k6_msl)))) - : (bwidth == 4 ? (fmt == 3 ? MetalKqMvB4K3_metal_kq_mv_k3_msl : (fmt == 44 ? MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl : (fmt == 4 ? MetalKqMvB4K4_metal_kq_mv_k4_msl : (fmt == 5 ? MetalKqMvB4K5_metal_kq_mv_k5_msl : MetalKqMvB4K6_metal_kq_mv_k6_msl)))) - : (fmt == 3 ? metal_kq_mvb8_k3_msl : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl : metal_kq_mvb8_k6_msl)))))) - let entry = (bwidth == 2 ? (fmt == 3 ? MetalKqMvB2K3_metal_kq_mv_k3_msl_entry : (fmt == 44 ? MetalKqMvB2Iq4xs_metal_kq_mv_iq4xs_msl_entry : (fmt == 4 ? MetalKqMvB2K4_metal_kq_mv_k4_msl_entry : (fmt == 5 ? MetalKqMvB2K5_metal_kq_mv_k5_msl_entry : MetalKqMvB2K6_metal_kq_mv_k6_msl_entry)))) - : (bwidth == 4 ? (fmt == 3 ? MetalKqMvB4K3_metal_kq_mv_k3_msl_entry : (fmt == 44 ? MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl_entry : (fmt == 4 ? MetalKqMvB4K4_metal_kq_mv_k4_msl_entry : (fmt == 5 ? MetalKqMvB4K5_metal_kq_mv_k5_msl_entry : MetalKqMvB4K6_metal_kq_mv_k6_msl_entry)))) - : (fmt == 3 ? metal_kq_mvb8_k3_msl_entry : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl_entry : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_entry : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_entry : metal_kq_mvb8_k6_msl_entry)))))) - let fm = (bwidth == 2 ? (fmt == 3 ? MetalKqMvB2K3_metal_kq_mv_k3_msl_fastmath : (fmt == 44 ? MetalKqMvB2Iq4xs_metal_kq_mv_iq4xs_msl_fastmath : (fmt == 4 ? MetalKqMvB2K4_metal_kq_mv_k4_msl_fastmath : (fmt == 5 ? MetalKqMvB2K5_metal_kq_mv_k5_msl_fastmath : MetalKqMvB2K6_metal_kq_mv_k6_msl_fastmath)))) - : (bwidth == 4 ? (fmt == 3 ? MetalKqMvB4K3_metal_kq_mv_k3_msl_fastmath : (fmt == 44 ? MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl_fastmath : (fmt == 4 ? MetalKqMvB4K4_metal_kq_mv_k4_msl_fastmath : (fmt == 5 ? MetalKqMvB4K5_metal_kq_mv_k5_msl_fastmath : MetalKqMvB4K6_metal_kq_mv_k6_msl_fastmath)))) - : (fmt == 3 ? metal_kq_mvb8_k3_msl_fastmath : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl_fastmath : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_fastmath : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_fastmath : metal_kq_mvb8_k6_msl_fastmath)))))) + let src = (bwidth == 2 ? (fmt == 33 ? MetalKqMvB2Iq3s_metal_kq_mv_iq3s_msl : (fmt == 3 ? MetalKqMvB2K3_metal_kq_mv_k3_msl : (fmt == 44 ? MetalKqMvB2Iq4xs_metal_kq_mv_iq4xs_msl : (fmt == 4 ? MetalKqMvB2K4_metal_kq_mv_k4_msl : (fmt == 5 ? MetalKqMvB2K5_metal_kq_mv_k5_msl : MetalKqMvB2K6_metal_kq_mv_k6_msl))))) + : (bwidth == 4 ? (fmt == 33 ? MetalKqMvB4Iq3s_metal_kq_mv_iq3s_msl : (fmt == 3 ? MetalKqMvB4K3_metal_kq_mv_k3_msl : (fmt == 44 ? MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl : (fmt == 4 ? MetalKqMvB4K4_metal_kq_mv_k4_msl : (fmt == 5 ? MetalKqMvB4K5_metal_kq_mv_k5_msl : MetalKqMvB4K6_metal_kq_mv_k6_msl))))) + : (fmt == 33 ? metal_kq_mvb8_iq3s_msl : (fmt == 3 ? metal_kq_mvb8_k3_msl : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl : metal_kq_mvb8_k6_msl))))))) + let entry = (bwidth == 2 ? (fmt == 33 ? MetalKqMvB2Iq3s_metal_kq_mv_iq3s_msl_entry : (fmt == 3 ? MetalKqMvB2K3_metal_kq_mv_k3_msl_entry : (fmt == 44 ? MetalKqMvB2Iq4xs_metal_kq_mv_iq4xs_msl_entry : (fmt == 4 ? MetalKqMvB2K4_metal_kq_mv_k4_msl_entry : (fmt == 5 ? MetalKqMvB2K5_metal_kq_mv_k5_msl_entry : MetalKqMvB2K6_metal_kq_mv_k6_msl_entry))))) + : (bwidth == 4 ? (fmt == 33 ? MetalKqMvB4Iq3s_metal_kq_mv_iq3s_msl_entry : (fmt == 3 ? MetalKqMvB4K3_metal_kq_mv_k3_msl_entry : (fmt == 44 ? MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl_entry : (fmt == 4 ? MetalKqMvB4K4_metal_kq_mv_k4_msl_entry : (fmt == 5 ? MetalKqMvB4K5_metal_kq_mv_k5_msl_entry : MetalKqMvB4K6_metal_kq_mv_k6_msl_entry))))) + : (fmt == 33 ? metal_kq_mvb8_iq3s_msl_entry : (fmt == 3 ? metal_kq_mvb8_k3_msl_entry : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl_entry : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_entry : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_entry : metal_kq_mvb8_k6_msl_entry))))))) + let fm = (bwidth == 2 ? (fmt == 33 ? MetalKqMvB2Iq3s_metal_kq_mv_iq3s_msl_fastmath : (fmt == 3 ? MetalKqMvB2K3_metal_kq_mv_k3_msl_fastmath : (fmt == 44 ? MetalKqMvB2Iq4xs_metal_kq_mv_iq4xs_msl_fastmath : (fmt == 4 ? MetalKqMvB2K4_metal_kq_mv_k4_msl_fastmath : (fmt == 5 ? MetalKqMvB2K5_metal_kq_mv_k5_msl_fastmath : MetalKqMvB2K6_metal_kq_mv_k6_msl_fastmath))))) + : (bwidth == 4 ? (fmt == 33 ? MetalKqMvB4Iq3s_metal_kq_mv_iq3s_msl_fastmath : (fmt == 3 ? MetalKqMvB4K3_metal_kq_mv_k3_msl_fastmath : (fmt == 44 ? MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl_fastmath : (fmt == 4 ? MetalKqMvB4K4_metal_kq_mv_k4_msl_fastmath : (fmt == 5 ? MetalKqMvB4K5_metal_kq_mv_k5_msl_fastmath : MetalKqMvB4K6_metal_kq_mv_k6_msl_fastmath))))) + : (fmt == 33 ? metal_kq_mvb8_iq3s_msl_fastmath : (fmt == 3 ? metal_kq_mvb8_k3_msl_fastmath : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl_fastmath : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_fastmath : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_fastmath : metal_kq_mvb8_k6_msl_fastmath))))))) var pso = pipeline_from_source(dev, src, entry, fm, err) t |> success(pso != null, "{tag}: pipeline: {err}") return if (pso == null) @@ -163,9 +166,9 @@ def private kq_mvb_gate(t : T?; dev, queue; fmt, bwidth, n, d, nr, ys : int) { metal_set_pipeline(enc, pso) if (bwidth == 8) { metal_set_threadgroup_memory_length(enc, - fmt == 3 ? metal_kq_mvb8_k3_msl_tgmem : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl_tgmem : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_tgmem : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_tgmem : metal_kq_mvb8_k6_msl_tgmem))), 0) + fmt == 33 ? metal_kq_mvb8_iq3s_msl_tgmem : (fmt == 3 ? metal_kq_mvb8_k3_msl_tgmem : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl_tgmem : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_tgmem : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_tgmem : metal_kq_mvb8_k6_msl_tgmem)))), 0) } - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs/k3: the f16 d tail + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs/k3: the f16 d tail metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bx, 0ul, 3) @@ -254,7 +257,7 @@ def private moe_kq_gemv_gate(t : T?; dev, queue; fmt, n, d, net, k, nst : int; p bxs = uint((perslot ? k : 1) * n), bys = uint(k * d), hasb = 0u) let ran = with_compute_encoder(queue, err) $(enc : MetalComputeEncoder?) { metal_set_pipeline(enc, pso) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs/k3: the f16 d tail + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs/k3: the f16 d tail metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bx, 0ul, 3) @@ -728,8 +731,10 @@ def test_metal_kq_gemv_kernels(t : T?) { kq_gemv_gate(t, dev, queue, "iq4xs", 512, 30) kq_gemv_gate(t, dev, queue, "k3", 1280, 11) kq_gemv_gate(t, dev, queue, "k3", 512, 30) + kq_gemv_gate(t, dev, queue, "iq3s", 1280, 11) + kq_gemv_gate(t, dev, queue, "iq3s", 512, 30) // the small-batch twins: nr = 2 / 3 (b4 col guard) / 6 (b8 pad), ys > d - for (fmt in [4, 5, 6, 44, 3]) { + for (fmt in [4, 5, 6, 44, 3, 33]) { kq_mvb_gate(t, dev, queue, fmt, 2, 768, 17, 2, 22) kq_mvb_gate(t, dev, queue, fmt, 4, 768, 17, 3, 22) kq_mvb_gate(t, dev, queue, fmt, 8, 768, 17, 6, 22) From c208d0f8216773b55680195c96d4125a1f1d2535 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Sun, 30 Aug 2026 20:03:11 -0700 Subject: [PATCH 23/61] dasLLAMA: IQ3_XXS on the CPU tier - the exact halving fold, the shared iq3s panel form The fourth format of the arc, identity 34, planes 96/20. The trick that shapes everything: ggml folds w = 0.25 * d * (2ls+1) * grid_byte * sign, and every grid level is EVEN (4..62) - so the plane grid stores the bytes halved (2..31) and the transcode halves the stored d (f16_half_bits: an exponent decrement, exact to the subnormal edge). The 0.25 vanishes with no factor left in any kernel: the fold is iq3s's d x strip x byte verbatim (strip = 2ls+1), and the carry-free smask sign trick still holds (it needs nonzero magnitudes, not odd ones). ksigns_iq2xs (bit 7 = even parity) lands in kqformat, shared with the coming iq2 tier. The panel form is iq3s's: unpack_iq3xxs_panel_grp emits the same signed-byte panel, so kq_grp_row_dot_b, the tile ladders and the batch cell widen to fmt == 33 || 34 instead of growing arms. Everything else is the QUIRK-1 ladder walk: transcode (gguf + superblock), dequant refs, scalar dot + rows kernel, grp repack (24 columns), gemv/tile stubs with the full tune-perm grid (both generators decline until the emitter arc - the framework's own fallback), ~40 dispatch arms, plane pair + image v21, probe fixtures (98B disk blocks), and fmt 34 in every test gate loop. Two finds along the way: the test tile gate's `packed` list wrongly held 33 - the iq3s tile (reference and stamped alike) reads the byte-expanded panel, so the gate handed it packed planes and failed on x64 at HEAD; fixed by handing it the panel like k5/k6. And HOW_TO QUIRK 23: census the vehicle GGUF first - the natural i1-IQ3_XXS download quantizes attn to IQ2_S and cannot load until the iq2 tier; the vehicle is bartowski's Qwen2.5-1.5B IQ3_XS (iq3_xxs attn_k/q + ffn, all siblings supported). Gates: test_kqformat 18/18 (tables, the ksigns parity property, the halver edges), test_kquant 181 tests 0 failed. End to end on the vehicle (reference bodies): coherent text at gen 19 t/s, 22/64 greedy ids vs llama.cpp's simple_ids oracle - the fork is a 0.12-logit near-tie whose runner-up IS our token (simple_ids_margin), the same class as iq3s's token-52 flip on a far less noisy mix. Co-Authored-By: Claude Fable 5 --- modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md | 31 ++++ modules/dasLLAMA/dasllama/dasllama_blocks.das | 3 + modules/dasLLAMA/dasllama/dasllama_common.das | 38 ++++- modules/dasLLAMA/dasllama/dasllama_config.das | 3 +- .../dasLLAMA/dasllama/dasllama_convert.das | 45 ++++++ .../dasLLAMA/dasllama/dasllama_gemm_gen.das | 9 ++ .../dasllama/dasllama_gemm_schema.das | 7 +- modules/dasLLAMA/dasllama/dasllama_gguf.das | 43 ++++++ .../dasllama/dasllama_gpu_resident.das | 7 + modules/dasLLAMA/dasllama/dasllama_image.das | 7 +- .../dasLLAMA/dasllama/dasllama_kqformat.das | 89 ++++++++++- modules/dasLLAMA/dasllama/dasllama_layout.das | 22 ++- modules/dasLLAMA/dasllama/dasllama_load.das | 41 +++++- modules/dasLLAMA/dasllama/dasllama_math.das | 9 +- .../dasllama/dasllama_math_default.das | 85 +++++++++++ .../dasLLAMA/dasllama/dasllama_math_gen.das | 116 ++++++++++++++- modules/dasLLAMA/dasllama/dasllama_ple.das | 4 + modules/dasLLAMA/dasllama/dasllama_repack.das | 94 ++++++++++++ modules/dasLLAMA/followup_vulkan.md | 8 + modules/dasLLAMA/harness/gen_tune_probe.das | 42 +++++- modules/dasLLAMA/tests/test_kqformat.das | 39 ++++- modules/dasLLAMA/tests/test_kquant.das | 139 +++++++++++++++--- 22 files changed, 823 insertions(+), 58 deletions(-) diff --git a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md index 0d5a597513..0c6fb5976b 100644 --- a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md +++ b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md @@ -417,6 +417,37 @@ where, why it is so today, what unquirked looks like. An empty ledger is a legit per encoder, best-of-3 encoders, GB/s off plane bytes) resolves 3% in seconds. ## Per-format notes +23. **Census the vehicle GGUF before committing to it.** An ftype is a MIX: the natural + "IQ3_XXS" download (mradermacher i1) quantizes attn_k/q to IQ2_S - unsupported until the + iq2 tier - so the file cannot load end to end. `gguf_census.py` (scratch) reads the tensor + type table in seconds; pick a vehicle whose mix is {supported} + the new format only + (bartowski's Qwen2.5-1.5B IQ3_XS carries iq3_xxs with iq3_s/q4_K/q6_K siblings). +### IQ3_XXS (2026-08-30) + +Shape: 256-superblock, 98B disk block - f16 d, 64 grid-index bytes (`iq3xxs_grid[256]`, one +byte = FOUR magnitudes), 8 x aux32 per superblock (bits 0..27 = four 7-bit `ksigns_iq2xs` +indices, bits 28..31 = the block scale ls; ggml folds w = 0.25 * d * (2ls+1) * grid * sign). +The plane design removes the 0.25 EXACTLY: every ggml grid level is even (4..62), so our +plane grid stores the bytes HALVED (2..31) and the transcode stores d halved +(`f16_half_bits` - an exponent decrement, exact down to the subnormal edge) - the kernel +fold is then iq3s's `d x strip x grid_byte` verbatim with strip = 2ls+1, and the smask sign +trick still holds (it needs g != 0, not odd). Planes 96/20: quants = [qs][aux] verbatim +(24 uniform grp columns), scale = the iq3s row. `ksigns_iq2xs` (bit 7 = even parity) ships +in kqformat and is shared with the coming IQ2 family. Identity 34. + +The panel form is SHARED with iq3s: `unpack_iq3xxs_panel_grp` emits the same signed-byte +panel, so `kq_grp_row_dot_b`, the tile ladders and the batch cell take `fmt == 33 || 34` +conditions rather than new arms. Vehicle: `Qwen2.5-1.5B-Instruct-IQ3_XS` (bartowski - iq3_xxs +on attn_k/q + parts of ffn, everything else already supported); the obvious +mradermacher i1-IQ3_XXS 1B carries IQ2_S attn tensors and must WAIT for the iq2 tier - +census the candidate GGUF first (QUIRK 23). Gates: `test_kqformat` 18/18 (tables, parity +property, the halver edges), `test_kquant` 181 tests 0 failed with fmt 34 in every gate +loop - and the tile gate's `packed` list wrongly held 33 (the iq3s tile reads the +byte-expanded panel), a pre-existing red on x64 fixed in the same change. End to end +(reference bodies - both generators decline until the emitter arc): coherent text, 22/64 +greedy ids vs llama.cpp's `simple_ids.exe` where the fork is a 0.12-logit near-tie whose +top-2 IS our token (`simple_ids_margin`), gen 19 t/s. JIT emitter, Vulkan, Metal: pending. + ### IQ3_S (the third format - and the first grid format, 2026-08-30) Shape: 256-superblock grid format - an 8-bit grid index plus a qh ninth bit selects diff --git a/modules/dasLLAMA/dasllama/dasllama_blocks.das b/modules/dasLLAMA/dasllama/dasllama_blocks.das index 40318fd1db..0bf7d0db87 100644 --- a/modules/dasLLAMA/dasllama/dasllama_blocks.das +++ b/modules/dasLLAMA/dasllama/dasllama_blocks.das @@ -1409,6 +1409,9 @@ def private kq_bytes_per_weight(f : KqFmt) : float { if (f == KqFmt.iq3s) { return 124.0 / 256.0 // 104B quants + the 20B scale row per 256 } + if (f == KqFmt.iq3xxs) { + return 116.0 / 256.0 // 96B quants + the 20B scale row per 256 + } return 34.0 / 32.0 // q8: 32 quants + an f16 scale per block } diff --git a/modules/dasLLAMA/dasllama/dasllama_common.das b/modules/dasLLAMA/dasllama/dasllama_common.das index f1156420f5..91faba104b 100644 --- a/modules/dasLLAMA/dasllama/dasllama_common.das +++ b/modules/dasLLAMA/dasllama/dasllama_common.das @@ -844,6 +844,9 @@ struct Model { // the iq3s tier's plane pair: [64 qs][8 qh][32 signs] per superblock + the k4-shaped 20B scale row (f16 d, 8 x int8 (1 + 2s)) iq3sq : array iq3ss : array + // the iq3xxs tier's plane pair: [64 qs][32 aux] per superblock + the iq3s-shaped 20B scale row (f16 d halved, 8 x uint8 (2ls + 1)) + iq3xxsq : array + iq3xxss : array kquant_native : bool = false // kq stage 4: the kq planes were repacked at load into the active backend's grp layout. // From then on every kq matmul MUST run the backend slots — disk-order portable kernels would @@ -857,6 +860,7 @@ struct Model { kq_repack_mr44 : int64 = 4l kq_repack_mr3 : int64 = 4l kq_repack_mr33 : int64 = 4l + kq_repack_mr34 : int64 = 4l // per-layer weight format tags (KqFmt; empty = all q8). Filled by detect_kq_formats before // layout_offsets so the layout walks each tensor into its format's cursor. wq_fmt : array @@ -1081,6 +1085,7 @@ def model_weights_bytes(t : Model) : int64 { + long_length(t.k6q) + long_length(t.k6s) + long_length(t.q40q) + long_length(t.q40s) + long_length(t.iq4xsq) + long_length(t.iq4xss) + long_length(t.k3q) + long_length(t.k3s) + long_length(t.iq3sq) + long_length(t.iq3ss) + + long_length(t.iq3xxsq) + long_length(t.iq3xxss) + long_length(t.q51q) + long_length(t.q51s) + long_length(t.mblob) + long_length(t.vkblob) + long_length(t.embq) + long_length(t.embs)) @@ -1472,6 +1477,7 @@ def private dlim_cpu_source_impl(var c : DlimCpuConfig) { c.kq_mr44 = active_kq_layout_mr(44) c.kq_mr3 = active_kq_layout_mr(3) c.kq_mr33 = active_kq_layout_mr(33) + c.kq_mr34 = active_kq_layout_mr(34) c.q51_mr = active_q51_layout_mr() // OUTCOME, not request: a backend without s16 twins keeps f32 scale planes (wscale_convert_f16) c.wscale_f16 = g_wscale_f16 && kernel_backend_has_wscale16() @@ -1564,7 +1570,7 @@ def kq_active_mr(t : Model; f : KqFmt) : int64 { if (!t.kq_repacked) { return 1l } - return f == KqFmt.k4 ? t.kq_repack_mr4 : (f == KqFmt.k5 ? t.kq_repack_mr5 : (f == KqFmt.k6 ? t.kq_repack_mr6 : (f == KqFmt.q40 ? t.kq_repack_mr40 : (f == KqFmt.iq4xs ? t.kq_repack_mr44 : (f == KqFmt.k3 ? t.kq_repack_mr3 : t.kq_repack_mr33))))) + return f == KqFmt.k4 ? t.kq_repack_mr4 : (f == KqFmt.k5 ? t.kq_repack_mr5 : (f == KqFmt.k6 ? t.kq_repack_mr6 : (f == KqFmt.q40 ? t.kq_repack_mr40 : (f == KqFmt.iq4xs ? t.kq_repack_mr44 : (f == KqFmt.k3 ? t.kq_repack_mr3 : (f == KqFmt.iq3s ? t.kq_repack_mr33 : t.kq_repack_mr34)))))) } //! Load a llama-architecture GGUF into the split layout at the requested precision — straight into @@ -1627,8 +1633,8 @@ def footprint(t : Model) : MemFootprint { let q4s = long_length(t.q4scales) * 4l let mx4 = long_length(t.mxq) // uint8 (two nibbles) -> 1 byte each let mx4s = long_length(t.mxs) // raw E8M0 -> 1 byte per 32-block - let kq = long_length(t.k4q) + long_length(t.k5q) + long_length(t.k6q) + long_length(t.q40q) + long_length(t.q51q) + long_length(t.iq4xsq) + long_length(t.k3q) + long_length(t.iq3sq) - let kqs = long_length(t.k4s) + long_length(t.k5s) + long_length(t.k6s) + long_length(t.q40s) + long_length(t.q51s) + long_length(t.iq4xss) + long_length(t.k3s) + long_length(t.iq3ss) + let kq = long_length(t.k4q) + long_length(t.k5q) + long_length(t.k6q) + long_length(t.q40q) + long_length(t.q51q) + long_length(t.iq4xsq) + long_length(t.k3q) + long_length(t.iq3sq) + long_length(t.iq3xxsq) + let kqs = long_length(t.k4s) + long_length(t.k5s) + long_length(t.k6s) + long_length(t.q40s) + long_length(t.q51s) + long_length(t.iq4xss) + long_length(t.k3s) + long_length(t.iq3ss) + long_length(t.iq3xxss) return MemFootprint(aux_fp32_bytes = aux, weight_fp32_bytes = wf, q8_bytes = q8, q8_scale_bytes = q8s, q4_bytes = q4, q4_scale_bytes = q4s, mx4_bytes = mx4, mx4_scale_bytes = mx4s, kq_bytes = kq, kq_scale_bytes = kqs, @@ -3302,7 +3308,7 @@ def mm_b_q51_groupn(var y : array; t : Model; offs : array; nregio // KqFmt -> the kernel-layer format id (matmul_kq*/kq_rows_fn/kq_qsb take the int form). q8 maps // to 0, a poison id no kq kernel accepts — kq_rows_for calls this for EVERY tag, so it must stay // total even though q8 branches never read the result. -def private kq_fi(fmt : KqFmt) : int => fmt == KqFmt.k4 ? 4 : (fmt == KqFmt.k5 ? 5 : (fmt == KqFmt.k6 ? 6 : (fmt == KqFmt.q40 ? 40 : (fmt == KqFmt.iq4xs ? 44 : (fmt == KqFmt.k3 ? 3 : (fmt == KqFmt.iq3s ? 33 : 0)))))) +def private kq_fi(fmt : KqFmt) : int => fmt == KqFmt.k4 ? 4 : (fmt == KqFmt.k5 ? 5 : (fmt == KqFmt.k6 ? 6 : (fmt == KqFmt.q40 ? 40 : (fmt == KqFmt.iq4xs ? 44 : (fmt == KqFmt.k3 ? 3 : (fmt == KqFmt.iq3s ? 33 : (fmt == KqFmt.iq3xxs ? 34 : 0))))))) // K-quant GEMV against a PRE-quantized activation (xq/xs/xbs already filled by the bs quantizer). // kq_repacked loads run the backend's stamped cores; disk-order loads run the portable kernels. @@ -3322,6 +3328,8 @@ def mm_at_kq_pre(var y : array; t : Model; fmt : KqFmt; woff : int64; xq matmul_kq_active(3, y, t.k3q, t.k3s, woff, xq, xs, xbs, n, d, yoff) } elif (fmt == KqFmt.iq3s) { matmul_kq_active(33, y, t.iq3sq, t.iq3ss, woff, xq, xs, xbs, n, d, yoff) + } elif (fmt == KqFmt.iq3xxs) { + matmul_kq_active(34, y, t.iq3xxsq, t.iq3xxss, woff, xq, xs, xbs, n, d, yoff) } else { panic("mm_at_kq_pre: '{fmt}' has no kq plane pair") // a new KqFmt must claim its arm, never ride q40's } @@ -3339,6 +3347,8 @@ def mm_at_kq_pre(var y : array; t : Model; fmt : KqFmt; woff : int64; xq matmul_kq(3, y, t.k3q, t.k3s, woff, xq, xs, xbs, n, d, yoff) } elif (fmt == KqFmt.iq3s) { matmul_kq(33, y, t.iq3sq, t.iq3ss, woff, xq, xs, xbs, n, d, yoff) + } elif (fmt == KqFmt.iq3xxs) { + matmul_kq(34, y, t.iq3xxsq, t.iq3xxss, woff, xq, xs, xbs, n, d, yoff) } else { panic("mm_at_kq_pre: '{fmt}' has no kq plane pair") } @@ -3376,6 +3386,8 @@ def mm_b_kq(var s : Session; var y : array; t : Model; fmt : KqFmt; woff matmul_kq_batch(3, y, t.k3q, t.k3s, woff, s.xqb, s.xsb, s.xbsb, n, d, npos) } elif (fmt == KqFmt.iq3s) { matmul_kq_batch(33, y, t.iq3sq, t.iq3ss, woff, s.xqb, s.xsb, s.xbsb, n, d, npos) + } elif (fmt == KqFmt.iq3xxs) { + matmul_kq_batch(34, y, t.iq3xxsq, t.iq3xxss, woff, s.xqb, s.xsb, s.xbsb, n, d, npos) } else { panic("mm_b_kq: '{fmt}' has no kq plane pair") } @@ -3409,6 +3421,8 @@ def mm_at_kq_groupn(var y : array; t : Model; fmt : KqFmt; offs : array; t : Model; fmt : KqFmt; offs : array; t : Model; fmt : KqFmt; offs : array; t : Model; fmt : KqFmt; woff : int64; xq : matmul_kq_batch(3, y, t.k3q, t.k3s, woff, xq, xs, xbs, n, d, ntok) } elif (fmt == KqFmt.iq3s) { matmul_kq_batch(33, y, t.iq3sq, t.iq3ss, woff, xq, xs, xbs, n, d, ntok) + } elif (fmt == KqFmt.iq3xxs) { + matmul_kq_batch(34, y, t.iq3xxsq, t.iq3xxss, woff, xq, xs, xbs, n, d, ntok) } else { panic("mm_b_kq_pre: '{fmt}' has no kq plane pair") } @@ -3487,6 +3507,7 @@ def kq_plane_q(t : Model; fmt : KqFmt; woff : int64) : uint8 const? { if (fmt == KqFmt.iq4xs) return unsafe(addr(t.iq4xsq[sb * IQ4XS_QSB])) if (fmt == KqFmt.k3) return unsafe(addr(t.k3q[sb * K3_QSB])) if (fmt == KqFmt.iq3s) return unsafe(addr(t.iq3sq[sb * IQ3S_QSB])) + if (fmt == KqFmt.iq3xxs) return unsafe(addr(t.iq3xxsq[sb * IQ3XXS_QSB])) return null } @@ -3499,6 +3520,7 @@ def kq_plane_s(t : Model; fmt : KqFmt; woff : int64) : uint8 const? { if (fmt == KqFmt.iq4xs) return unsafe(addr(t.iq4xss[sb * IQ4XS_SSB])) if (fmt == KqFmt.k3) return unsafe(addr(t.k3s[sb * K3_SSB])) if (fmt == KqFmt.iq3s) return unsafe(addr(t.iq3ss[sb * IQ3S_SSB])) + if (fmt == KqFmt.iq3xxs) return unsafe(addr(t.iq3xxss[sb * IQ3XXS_SSB])) return null } @@ -3865,6 +3887,8 @@ def embed_row(t : Model; token : int64; var dst : float?) { // nolint:STYLE037 dequant_kq_row_grp(fmt, addr(t.embq[sbg * K3_QSB]), addr(t.embs[sbg * K3_SSB]), token % mr, mr, dim, dst) } elif (t.emb_fmt == KqFmt.iq3s) { dequant_kq_row_grp(fmt, addr(t.embq[sbg * IQ3S_QSB]), addr(t.embs[sbg * IQ3S_SSB]), token % mr, mr, dim, dst) + } elif (t.emb_fmt == KqFmt.iq3xxs) { + dequant_kq_row_grp(fmt, addr(t.embq[sbg * IQ3XXS_QSB]), addr(t.embs[sbg * IQ3XXS_SSB]), token % mr, mr, dim, dst) } else { panic("embed_row: '{t.emb_fmt}' has no kq plane pair") } @@ -3882,6 +3906,8 @@ def embed_row(t : Model; token : int64; var dst : float?) { // nolint:STYLE037 dequant_kq_row_grp(fmt, addr(t.k3q[sbg * K3_QSB]), addr(t.k3s[sbg * K3_SSB]), token % mr, mr, dim, dst) } elif (t.emb_fmt == KqFmt.iq3s) { dequant_kq_row_grp(fmt, addr(t.iq3sq[sbg * IQ3S_QSB]), addr(t.iq3ss[sbg * IQ3S_SSB]), token % mr, mr, dim, dst) + } elif (t.emb_fmt == KqFmt.iq3xxs) { + dequant_kq_row_grp(fmt, addr(t.iq3xxsq[sbg * IQ3XXS_QSB]), addr(t.iq3xxss[sbg * IQ3XXS_SSB]), token % mr, mr, dim, dst) } else { panic("embed_row: '{t.emb_fmt}' has no kq plane pair") } @@ -3910,6 +3936,8 @@ def embed_row(t : Model; token : int64; var dst : float?) { // nolint:STYLE037 dequant_k3_plane_superblock(t.embq, (sb0 + s) * K3_QSB, t.embs, (sb0 + s) * K3_SSB, row, s * 256l) } elif (t.emb_fmt == KqFmt.iq3s) { dequant_iq3s_plane_superblock(t.embq, (sb0 + s) * IQ3S_QSB, t.embs, (sb0 + s) * IQ3S_SSB, row, s * 256l) + } elif (t.emb_fmt == KqFmt.iq3xxs) { + dequant_iq3xxs_plane_superblock(t.embq, (sb0 + s) * IQ3XXS_QSB, t.embs, (sb0 + s) * IQ3XXS_SSB, row, s * 256l) } else { panic("embed_row: '{t.emb_fmt}' has no kq plane pair") } @@ -3931,6 +3959,8 @@ def embed_row(t : Model; token : int64; var dst : float?) { // nolint:STYLE037 dequant_k3_plane_superblock(t.k3q, (sb0 + s) * K3_QSB, t.k3s, (sb0 + s) * K3_SSB, row, s * 256l) } elif (t.emb_fmt == KqFmt.iq3s) { dequant_iq3s_plane_superblock(t.iq3sq, (sb0 + s) * IQ3S_QSB, t.iq3ss, (sb0 + s) * IQ3S_SSB, row, s * 256l) + } elif (t.emb_fmt == KqFmt.iq3xxs) { + dequant_iq3xxs_plane_superblock(t.iq3xxsq, (sb0 + s) * IQ3XXS_QSB, t.iq3xxss, (sb0 + s) * IQ3XXS_SSB, row, s * 256l) } else { panic("embed_row: '{t.emb_fmt}' has no kq plane pair") } diff --git a/modules/dasLLAMA/dasllama/dasllama_config.das b/modules/dasLLAMA/dasllama/dasllama_config.das index f0d9c0501a..9e1ba72bed 100644 --- a/modules/dasLLAMA/dasllama/dasllama_config.das +++ b/modules/dasLLAMA/dasllama/dasllama_config.das @@ -30,6 +30,7 @@ struct public DlimCpuConfig { kq_mr44 : int64 kq_mr3 : int64 kq_mr33 : int64 + kq_mr34 : int64 q51_mr : int64 wscale_f16 : bool kquant_native : bool @@ -142,7 +143,7 @@ def public dlim_config_current(quant : string = "q8") : DlimConfiguration { def public dlim_identity(c : DlimConfiguration; image_version : int; tag : string = "") : string { return ("v{image_version}|{c.quant}|{c.cpu.backend}|{c.cpu.wscale_f16 ? "s16" : "s32"}" + "|q8 mr{c.cpu.q8_mr} b{c.cpu.q8_wbias} g{c.cpu.q8_kgroup}" - + "|kq {c.cpu.kq_mr4}/{c.cpu.kq_mr5}/{c.cpu.kq_mr6}/{c.cpu.kq_mr40}/{c.cpu.kq_mr44}/{c.cpu.kq_mr3}/{c.cpu.kq_mr33}" + + "|kq {c.cpu.kq_mr4}/{c.cpu.kq_mr5}/{c.cpu.kq_mr6}/{c.cpu.kq_mr40}/{c.cpu.kq_mr44}/{c.cpu.kq_mr3}/{c.cpu.kq_mr33}/{c.cpu.kq_mr34}" + "|q51 mr{c.cpu.q51_mr}" + "|nat {c.cpu.kquant_native ? 1 : 0}{c.cpu.kq_q40_native ? 1 : 0}{c.cpu.kq_q50_native ? 1 : 0}{c.cpu.kq_q51_native ? 1 : 0}" + (tag != "" ? "|{tag}" : "")) diff --git a/modules/dasLLAMA/dasllama/dasllama_convert.das b/modules/dasLLAMA/dasllama/dasllama_convert.das index 50255a76d4..86c9bb2c0f 100644 --- a/modules/dasLLAMA/dasllama/dasllama_convert.das +++ b/modules/dasLLAMA/dasllama/dasllama_convert.das @@ -716,6 +716,51 @@ def dequant_k3_plane_superblock_at(kq : array | #; kqo : int64; ks : arra //! Transcode one IQ3_S superblock (110 bytes at `bo`: f16 d, 64 qs, 8 qh, 32 signs, 4 packed //! 4-bit scales) into the iq3s planes: [qs][qh][signs] verbatim to kq[kqo..+104), the 8 //! sub-scales DECODED to int8 (1 + 2s) + the f16 d to ks[kso..+20) — the k4 row shape (exact). +def transcode_iq3xxs_superblock(bytes : array | #; bo : int64; var kq : array; kqo : int64; var ks : array; kso : int64) { + for (i in range64(96l)) { + kq[kqo + i] = bytes[bo + 2l + i] + } + let dh = f16_half_bits(uint(bytes[bo]) | (uint(bytes[bo + 1l]) << 8u)) + ks[kso] = uint8(dh & 0xFFu) + ks[kso + 1l] = uint8(dh >> 8u) + ks[kso + 2l] = uint8(0) + ks[kso + 3l] = uint8(0) + for (b in range64(8l)) { + ks[kso + 4l + b] = uint8(1 + 2 * (int(bytes[bo + 66l + b * 4l + 3l]) >> 4)) + } + for (i in range64(12l, 20l)) { + ks[kso + i] = uint8(0) + } +} + +//! Reference dequant of one iq3xxs-plane superblock: w = (d_halved * (2ls + 1)) * halved_grid +//! byte * sign — ggml's 0.25 * d * (2ls+1) * grid * sign exactly (both halvings are lossless). +//! Grid entry = one qs byte; sign byte = ksigns_iq2xs[(aux32 >> 7l) & 127], bit j (bit 7 = parity). +def dequant_iq3xxs_plane_superblock(kq : array | #; kqo : int64; ks : array | #; kso : int64; var dst : array | #; doff : int64) { + dequant_iq3xxs_plane_superblock_at(kq, kqo, ks, kso + 4l, kso, dst, doff) +} + +//! The split-layout twin: `dso` names the f16 d halfword's byte offset independently of the +//! 8-byte sub-scale strip at kso (the metal split form keeps them apart). +def dequant_iq3xxs_plane_superblock_at(kq : array | #; kqo : int64; ks : array | #; kso, dso : int64; var dst : array | #; doff : int64) { + let d = f16_to_f32(rd_u16(ks, dso)) + let grid = iq3xxs_grid() + let signs = ksigns_iq2xs() + for (b in range64(8l)) { + let db = d * float(int(ks[kso + b])) + let aux = uint(kq[kqo + 64l + b * 4l]) | (uint(kq[kqo + 64l + b * 4l + 1l]) << 8u) | (uint(kq[kqo + 64l + b * 4l + 2l]) << 16u) | (uint(kq[kqo + 64l + b * 4l + 3l]) << 24u) + for (l in range64(4l)) { + let sgn = int(signs[int((aux >> uint(7l * l)) & 127u)]) + let g1 = grid[int(kq[kqo + b * 8l + 2l * l])] + let g2 = grid[int(kq[kqo + b * 8l + 2l * l + 1l])] + for (j in range64(4l)) { + dst[doff + b * 32l + l * 8l + j] = db * float((g1 >> uint(8l * j)) & 0xFFu) * (((sgn >> int(j)) & 1) != 0 ? -1.0 : 1.0) + dst[doff + b * 32l + l * 8l + 4l + j] = db * float((g2 >> uint(8l * j)) & 0xFFu) * (((sgn >> int(4l + j)) & 1) != 0 ? -1.0 : 1.0) + } + } + } +} + def transcode_iq3s_superblock(bytes : array | #; bo : int64; var kq : array; kqo : int64; var ks : array; kso : int64) { for (i in range64(104l)) { kq[kqo + i] = bytes[bo + 2l + i] diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das index e5848e5a1c..3d835e3b55 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das @@ -2705,6 +2705,13 @@ def private k3_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 3) def private iq3s_gemv_gen(var gc : LlvmCodeCtx) : bool => kq_gemv_gen_impl(gc, 33) def private iq3s_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 33) +// iq3xxs: the grid-gather emitter arm lands with its kernel arc; until then both stamps +// decline (reference bodies serve) — a declined generator is the framework's own fallback path. +[unused_argument(gc)] +def private iq3xxs_gemv_gen(var gc : LlvmCodeCtx) : bool => false +[unused_argument(gc)] +def private iq3xxs_tile_gen(var gc : LlvmCodeCtx) : bool => false + def private k4_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 4) def private k5_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 5) def private k6_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 6) @@ -2788,5 +2795,7 @@ def public register_dasllama_gemm_generators { register_llvm_code_generator("dasllama_gemm_gen::k3_tile", @@k3_tile_gen) register_llvm_code_generator("dasllama_gemm_gen::iq3s_gemv", @@iq3s_gemv_gen) register_llvm_code_generator("dasllama_gemm_gen::iq3s_tile", @@iq3s_tile_gen) + register_llvm_code_generator("dasllama_gemm_gen::iq3xxs_gemv", @@iq3xxs_gemv_gen) + register_llvm_code_generator("dasllama_gemm_gen::iq3xxs_tile", @@iq3xxs_tile_gen) register_llvm_code_generator("dasllama_gemm_gen::q8q8_witness", @@witness_gen) } diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_schema.das b/modules/dasLLAMA/dasllama/dasllama_gemm_schema.das index 9bd854e9fd..46b57af89e 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_schema.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_schema.das @@ -58,7 +58,7 @@ let Q51_QPB = 20l let Q51_SPB = 4l //! Quant-plane bytes per 256-weight superblock per row for a kq format id (4/5/6 = Q4_K/Q5_K/ -//! Q6_K, 40 = Q4_0, 44 = IQ4_XS, 3 = Q3_K, 33 = IQ3_S). The ONE stride source for every fmt-branched kq walker/ +//! Q6_K, 40 = Q4_0, 44 = IQ4_XS, 3 = Q3_K, 33 = IQ3_S, 34 = IQ3_XXS). The ONE stride source for every fmt-branched kq walker/ //! kernel/repack — an unknown id panics instead of silently walking another format's stride. def kq_qsb(fmt : int) : int64 { if (fmt == 4) return kq_qsb(KqFmt.k4) @@ -68,6 +68,7 @@ def kq_qsb(fmt : int) : int64 { if (fmt == 44) return kq_qsb(KqFmt.iq4xs) if (fmt == 3) return kq_qsb(KqFmt.k3) if (fmt == 33) return kq_qsb(KqFmt.iq3s) + if (fmt == 34) return kq_qsb(KqFmt.iq3xxs) panic("kq_qsb: unknown kq format id {fmt}") return 0l } @@ -75,7 +76,8 @@ def kq_qsb(fmt : int) : int64 { //! Scale-plane bytes per 256-weight superblock per row (see kq_qsb): k4/k5 20 (16B disk block //! + 4B pad, decoded at repack), k6 18 (native), q40 16 (8 x f16 d), iq4xs 20 (d + 8 int8 //! sub-scales decoded at transcode, the k4 row shape), k3 18 (16 int8 decoded + d, the k6 row), -//! iq3s 20 (d + 8 int8 (1 + 2s) decoded at transcode, the k4 row shape again). +//! iq3s 20 (d + 8 int8 (1 + 2s) decoded at transcode, the k4 row shape again), iq3xxs 20 (d +//! halved + 8 x (2*ls + 1), same row). def kq_ssb(fmt : int) : int64 { if (fmt == 4) return kq_ssb(KqFmt.k4) if (fmt == 5) return kq_ssb(KqFmt.k5) @@ -84,6 +86,7 @@ def kq_ssb(fmt : int) : int64 { if (fmt == 44) return kq_ssb(KqFmt.iq4xs) if (fmt == 3) return kq_ssb(KqFmt.k3) if (fmt == 33) return kq_ssb(KqFmt.iq3s) + if (fmt == 34) return kq_ssb(KqFmt.iq3xxs) panic("kq_ssb: unknown kq format id {fmt}") return 0l } diff --git a/modules/dasLLAMA/dasllama/dasllama_gguf.das b/modules/dasLLAMA/dasllama/dasllama_gguf.das index 428d3ccd9d..3ed1286ba9 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gguf.das +++ b/modules/dasLLAMA/dasllama/dasllama_gguf.das @@ -49,6 +49,7 @@ let GGML_TYPE_Q5_1 = 7 let GGML_TYPE_Q8_0 = 8 let GGML_TYPE_Q3_K = 11 let GGML_TYPE_IQ3_S = 21 +let GGML_TYPE_IQ3_XXS = 18 let GGML_TYPE_Q4_K = 12 let GGML_TYPE_Q5_K = 13 let GGML_TYPE_Q6_K = 14 @@ -826,6 +827,48 @@ def gguf_transcode_iq3s(m : GGUFMeta; srcbytes : array | #; name : string } } + +//! Transcode an IQ3_XXS tensor into the iq3xxs planes (strides 96/20, exact): qs+aux copy +//! verbatim off the 98B disk block, the iq3s-shaped scale row carries d HALVED (f16_half_bits) +//! and strips (2*ls + 1) — with the halved grid this absorbs ggml's 0.25 exactly. +def gguf_transcode_iq3xxs(m : GGUFMeta; srcbytes : array | #; name : string; var kq : array; var ks : array; eloff, expect_n : int64; src_off : int64 = 0l) { + let ti = kq_transcode_check(m, name, GGML_TYPE_IQ3_XXS, "IQ3_XXS", src_off, expect_n) + let nb = expect_n / 256l + if (nb <= 0l) { + return + } + guard_dst(name, "iq3xxs quant plane", (eloff / 256l) * IQ3XXS_QSB, nb * IQ3XXS_QSB, long_length(kq)) + guard_dst(name, "iq3xxs scale plane", (eloff / 256l) * IQ3XXS_SSB, nb * IQ3XXS_SSB, long_length(ks)) + with_tensor_view(m, srcbytes, ti) $(bytes, tbase) { + let bo = tbase + (src_off / 256l) * 98l + unsafe { + let srcp = addr(bytes[bo]) + var kqp = addr(kq[(eloff / 256l) * IQ3XXS_QSB]) + var ksp = addr(ks[(eloff / 256l) * IQ3XXS_SSB]) + maybe_parallel_for(0, int(nb), transcode_jobs(nb, 98l)) $(rb, re) { + unsafe { + for (sb in range64(int64(rb), int64(re))) { + let src = srcp + sb * 98l + var row = ksp + sb * IQ3XXS_SSB + let dh = f16_half_bits(uint(src[0]) | (uint(src[1]) << 8u)) + row[0] = uint8(dh & 0xFFu) + row[1] = uint8(dh >> 8u) + row[2] = uint8(0) + row[3] = uint8(0) + for (b in range64(8l)) { + row[4l + b] = uint8(1 + 2 * (int(src[66l + b * 4l + 3l]) >> 4)) + } + for (i in range64(12l, 20l)) { + row[i] = uint8(0) + } + bcopy(kqp + sb * IQ3XXS_QSB, src + 2l, 96l) + } + } + } + } + } +} + //! Transcode a Q5_1 tensor into the q51 planes (per 32-BLOCK strides 20/4 — see Q51_QB): each //! 24B disk block splits into 2B d + 2B m (scale plane) and 16B nibbles + 4B qh (quant plane), //! all verbatim. Exact; `eloff`/`src_off`/`expect_n` are element offsets, % 32. diff --git a/modules/dasLLAMA/dasllama/dasllama_gpu_resident.das b/modules/dasLLAMA/dasllama/dasllama_gpu_resident.das index aff5adb54f..5c03dcaa32 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gpu_resident.das +++ b/modules/dasLLAMA/dasllama/dasllama_gpu_resident.das @@ -288,6 +288,11 @@ def trim_model_planes(var t : Model) : bool { // nolint:STYLE038 — the flat t.embs |> reserve_resize(nsb * K3_SSB) memcpy(addr(t.embq[0]), addr(t.k3q[sb0 * K3_QSB]), nsb * K3_QSB) memcpy(addr(t.embs[0]), addr(t.k3s[sb0 * K3_SSB]), nsb * K3_SSB) + } elif (t.emb_fmt == KqFmt.iq3xxs) { + t.embq |> resize(nsb * IQ3XXS_QSB) + t.embs |> resize(nsb * IQ3XXS_SSB) + memcpy(addr(t.embq[0]), addr(t.iq3xxsq[sb0 * IQ3XXS_QSB]), nsb * IQ3XXS_QSB) + memcpy(addr(t.embs[0]), addr(t.iq3xxss[sb0 * IQ3XXS_SSB]), nsb * IQ3XXS_SSB) } elif (t.emb_fmt == KqFmt.iq3s) { t.embq |> reserve_resize(nsb * IQ3S_QSB) t.embs |> reserve_resize(nsb * IQ3S_SSB) @@ -317,6 +322,8 @@ def trim_model_planes(var t : Model) : bool { // nolint:STYLE038 — the flat delete t.k3s delete t.iq3sq delete t.iq3ss + delete t.iq3xxsq + delete t.iq3xxss delete t.q40q delete t.q40s delete t.q51q diff --git a/modules/dasLLAMA/dasllama/dasllama_image.das b/modules/dasLLAMA/dasllama/dasllama_image.das index f8e4966091..469ba1c09b 100644 --- a/modules/dasLLAMA/dasllama/dasllama_image.das +++ b/modules/dasLLAMA/dasllama/dasllama_image.das @@ -38,7 +38,7 @@ require dasllama/dasllama_load // WhisperModel.enc) contribute their planes under dotted names ("enc.fblob"); string-array // fields ride the meta blob via serialize_strings — raw string pointers can't be planes. -let IMAGE_VERSION = 20 // 20: the iq3s plane pair (IQ3_S native tier); 19: k3; 18: iq4xs +let IMAGE_VERSION = 21 // 21: the iq3xxs plane pair (IQ3_XXS native tier); 20: iq3s; 19: k3 //! The metal (blob-only) flavor's identity tag: q8 planes ride the 34B block_q8_0 blob and the //! kq scale planes their GPU forms (convert_model_to_metal_blob) — flavors are per-config and @@ -637,6 +637,7 @@ def serialize_image_meta(var arch : Archive; var t : Model) { arch |> serialize_raw(t.kq_repack_mr44) arch |> serialize_raw(t.kq_repack_mr3) arch |> serialize_raw(t.kq_repack_mr33) + arch |> serialize_raw(t.kq_repack_mr34) arch |> serialize_raw(t.wcls_fmt) arch |> serialize_raw(t.emb_fmt) arch |> serialize_raw(t.ple_emb_fmt) @@ -676,7 +677,7 @@ def serialize_image_meta(var arch : Archive; var t : Model) { } // 67 serialized fields + 3 deliberate skips (blocks, image_map, image_bytes) -let IMAGE_META_FIELDS = 69 + 3 +let IMAGE_META_FIELDS = 70 + 3 //! Count of meta-carried fields of any struct: non-arrays plus string arrays (those cannot //! be raw planes — see serialize_strings). Pair with a per-type field-count constant to @@ -960,7 +961,7 @@ def save_model_image(var t : Model; path : string; tag : string = ""; quant : st // streamed planes size as empty, so their bytes plus a page each are added here; over-reserve is free (dwrite_close truncates) def private stream_extra_bytes(t : Model; jobs : array) : uint64 { var extra = 0ul - for (fname in ["qblob", "mblob", "k4q", "k5q", "k6q", "q40q", "q51q", "iq4xsq", "k3q", "iq3sq"]) { + for (fname in ["qblob", "mblob", "k4q", "k5q", "k6q", "q40q", "q51q", "iq4xsq", "k3q", "iq3sq", "iq3xxsq"]) { let sb = stream_plane_bytes(t, jobs, fname) if (sb > 0l) { extra += uint64(sb) + uint64(IMAGE_PAGE) diff --git a/modules/dasLLAMA/dasllama/dasllama_kqformat.das b/modules/dasLLAMA/dasllama/dasllama_kqformat.das index baff68c03f..e396fbcc4e 100644 --- a/modules/dasLLAMA/dasllama/dasllama_kqformat.das +++ b/modules/dasLLAMA/dasllama/dasllama_kqformat.das @@ -13,7 +13,7 @@ module dasllama_kqformat shared public require dasllama/dasllama_lint public //! Per-weight storage format under a q8-mode load with native planes: q8 = qblob/qscales; -//! k4/k5/k6/q40/iq4xs/k3/iq3s = superblock plane pairs (mixed PER TENSOR); q51 = native Q5_1 per-32 +//! k4/k5/k6/q40/iq4xs/k3/iq3s/iq3xxs = superblock plane pairs (mixed PER TENSOR); q51 = native Q5_1 per-32 //! planes. New members append — the int value is the device stack tag and the image plane id. enum KqFmt : uint8 { q8 @@ -25,15 +25,16 @@ enum KqFmt : uint8 { iq4xs k3 iq3s + iq3xxs } //! The superblock-lattice formats (Q8_K-form activations, % 256 rows, repack + stamped kq //! kernels). q51 deliberately fails this: it rides per-32 planes with Q8_0-form activations — //! a `fmt != KqFmt.q8` test does not imply the kq lattice; branch on kq_sb where it does. -def kq_sb(f : KqFmt) : bool => f == KqFmt.k4 || f == KqFmt.k5 || f == KqFmt.k6 || f == KqFmt.q40 || f == KqFmt.iq4xs || f == KqFmt.k3 || f == KqFmt.iq3s +def kq_sb(f : KqFmt) : bool => f == KqFmt.k4 || f == KqFmt.k5 || f == KqFmt.k6 || f == KqFmt.q40 || f == KqFmt.iq4xs || f == KqFmt.k3 || f == KqFmt.iq3s || f == KqFmt.iq3xxs //! The same predicate over the int id space the GPU drivers carry per stack/plane. -def kq_sb(fmt : int) : bool => fmt == int(KqFmt.k4) || fmt == int(KqFmt.k5) || fmt == int(KqFmt.k6) || fmt == int(KqFmt.q40) || fmt == int(KqFmt.iq4xs) || fmt == int(KqFmt.k3) || fmt == int(KqFmt.iq3s) +def kq_sb(fmt : int) : bool => fmt == int(KqFmt.k4) || fmt == int(KqFmt.k5) || fmt == int(KqFmt.k6) || fmt == int(KqFmt.q40) || fmt == int(KqFmt.iq4xs) || fmt == int(KqFmt.k3) || fmt == int(KqFmt.iq3s) || fmt == int(KqFmt.iq3xxs) //! KqFmt -> the kernel/IR format id space — the ONE bridge between the enum and the int ids the //! generated kernels take as runtime params. Mnemonic ids: a K-quant is its bit width (3/4/5/6), @@ -60,6 +61,9 @@ def kq_schema_id(f : KqFmt) : int { if (f == KqFmt.iq3s) { return 33 } + if (f == KqFmt.iq3xxs) { + return 34 + } panic("kq_schema_id: not a kq superblock format") return 0 } @@ -82,6 +86,8 @@ let K3_QSB = 96l // Q3_K quant plane: 64 qs bytes (four 2-bit lanes each) + 3 let K3_SSB = 18l // Q3_K scale plane: 16 x int8 (6-bit scale - 32, decoded at transcode) + f16 d — the k6 row let IQ3S_QSB = 104l // IQ3_S quant plane: [64 grid-index qs][8 qh ninth bits][32 sign bytes], verbatim disk order let IQ3S_SSB = 20l // IQ3_S scale plane: f16 d, 2 pad, 8 x int8 (1 + 2s), 8 pad — the iq4xs/k4 row shape +let IQ3XXS_QSB = 96l // IQ3_XXS quant plane: [64 grid-index qs][32 aux bytes: per block 4x7-bit sign indices + 4-bit scale], verbatim disk order +let IQ3XXS_SSB = 20l // IQ3_XXS scale plane: f16 d HALVED, 2 pad, 8 x uint8 (2*ls + 1), 8 pad — the iq3s row shape and fold (ggml's global 0.25 = the halved d x the halved grid) //! The IQ4_NL / IQ4_XS nibble codebook: weight = scale x IQ4NL_LUT[nibble] (ggml's kvalues_iq4nl). //! Main-context use only (tests, oracles, the emitter's constant bake): a kernel that can run on a @@ -167,6 +173,78 @@ def iq3s_grid() : uint[512] { //! a module global as zeros; kernels take `iq3s_grid()` instead. let IQ3S_GRID = iq3s_grid() +//! The IQ3_XXS grid, every byte HALVED (ggml levels are even, 4..62 -> 2..31): one qs byte = +//! FOUR magnitudes; the halved grid + halved stored d absorb ggml's 0.25 exactly, so the fold +//! is iq3s's d x strip x byte. Per-call local (1 KB) - the worker-safe form kernels read. +def iq3xxs_grid() : uint[256] { + return fixed_array( + 0x02020202u, 0x0202020au, 0x02020212u, 0x02020606u, 0x0202060eu, 0x0202061fu, 0x02020a02u, 0x02020a0au, + 0x02020e06u, 0x0202120au, 0x02021f0eu, 0x02021f16u, 0x02060206u, 0x0206020eu, 0x02060602u, 0x0206060au, + 0x02060a06u, 0x02060a16u, 0x02060e02u, 0x02060e0au, 0x02061206u, 0x02061612u, 0x02061f02u, 0x020a0202u, + 0x020a020au, 0x020a0212u, 0x020a0606u, 0x020a0a02u, 0x020a0a0au, 0x020a0e06u, 0x020a0e0eu, 0x020a0e1fu, + 0x020a1606u, 0x020a161fu, 0x020a1f16u, 0x020e0206u, 0x020e021fu, 0x020e0602u, 0x020e060au, 0x020e0a16u, + 0x020e1f02u, 0x0212060eu, 0x02120e1fu, 0x02121212u, 0x0212161fu, 0x02121f0eu, 0x02121f16u, 0x02160206u, + 0x0216021fu, 0x02160e0au, 0x0216160au, 0x021a0e16u, 0x021a1a12u, 0x021f0602u, 0x021f0612u, 0x021f061au, + 0x021f120eu, 0x021f1a06u, 0x06020206u, 0x0602020eu, 0x06020602u, 0x0602060au, 0x06020a06u, 0x06020a0eu, + 0x06020e02u, 0x06020e0au, 0x06020e12u, 0x0602121fu, 0x06021602u, 0x06060202u, 0x0606020au, 0x06060606u, + 0x06060a02u, 0x06060a0au, 0x060a0206u, 0x060a020eu, 0x060a0602u, 0x060a060au, 0x060a0a06u, 0x060a0e02u, + 0x060a1f0au, 0x060e0202u, 0x060e020au, 0x060e0a02u, 0x060e0e06u, 0x060e121au, 0x060e1a1au, 0x06120206u, + 0x06120216u, 0x06121602u, 0x06160a02u, 0x06160a12u, 0x0616121au, 0x06161f06u, 0x061a0216u, 0x061f0a0au, + 0x061f1202u, 0x0a020202u, 0x0a02020au, 0x0a020606u, 0x0a02060eu, 0x0a020a02u, 0x0a020a0au, 0x0a020a1au, + 0x0a020e06u, 0x0a02120au, 0x0a060206u, 0x0a06020eu, 0x0a060216u, 0x0a060602u, 0x0a06060au, 0x0a060a06u, + 0x0a060e02u, 0x0a061a0eu, 0x0a061a1fu, 0x0a061f02u, 0x0a0a0202u, 0x0a0a020au, 0x0a0a0606u, 0x0a0a061fu, + 0x0a0a0a02u, 0x0a0a0a0au, 0x0a0a0e1fu, 0x0a0a1202u, 0x0a0a1616u, 0x0a0e0206u, 0x0a0e0602u, 0x0a0e0612u, + 0x0a0e1f02u, 0x0a0e1f12u, 0x0a120e16u, 0x0a12160eu, 0x0a16020eu, 0x0a160a1fu, 0x0a161206u, 0x0a161f12u, + 0x0a1f0206u, 0x0a1f020eu, 0x0a1f061au, 0x0a1f1216u, 0x0e020206u, 0x0e020602u, 0x0e02060au, 0x0e020a06u, + 0x0e020a0eu, 0x0e021602u, 0x0e021a16u, 0x0e021f0au, 0x0e060202u, 0x0e06020au, 0x0e060a02u, 0x0e060e06u, + 0x0e061212u, 0x0e06121au, 0x0e0a0206u, 0x0e0a020eu, 0x0e0a0602u, 0x0e0a0a16u, 0x0e0a160au, 0x0e0a1f0au, + 0x0e0e0606u, 0x0e0e0e0eu, 0x0e120e02u, 0x0e12121fu, 0x0e121f0au, 0x0e160202u, 0x0e16021au, 0x0e160a0au, + 0x0e161616u, 0x0e1a0612u, 0x0e1a0e1au, 0x0e1a1a0eu, 0x0e1f0e0eu, 0x0e1f1a02u, 0x12020212u, 0x1202061fu, + 0x12020e16u, 0x12020e1fu, 0x1202160eu, 0x1202161fu, 0x12061f12u, 0x120a0a02u, 0x120a0e1fu, 0x120a1202u, + 0x120a1a02u, 0x120a1a1au, 0x120e021fu, 0x120e1216u, 0x12120212u, 0x12121606u, 0x12121a12u, 0x12160a16u, + 0x1216120eu, 0x12161f02u, 0x121f0216u, 0x121f0602u, 0x121f060au, 0x121f0e02u, 0x1602060au, 0x16021206u, + 0x16021f02u, 0x16060202u, 0x1606021au, 0x16060a1au, 0x16061616u, 0x160a0612u, 0x160a0e0au, 0x160a1f0au, + 0x160e020au, 0x160e160eu, 0x16120602u, 0x16120a0eu, 0x16120a1fu, 0x16121f0au, 0x1616020au, 0x16160e06u, + 0x161a1602u, 0x161f0a12u, 0x161f120au, 0x1a020a12u, 0x1a021212u, 0x1a02121au, 0x1a021a12u, 0x1a060a06u, + 0x1a061a06u, 0x1a0a061fu, 0x1a0a1a12u, 0x1a0e0e02u, 0x1a0e0e1au, 0x1a121212u, 0x1a160216u, 0x1a16160au, + 0x1a1a0e0eu, 0x1a1f020eu, 0x1a1f0a06u, 0x1f02020eu, 0x1f020216u, 0x1f02021fu, 0x1f020602u, 0x1f020e0au, + 0x1f02160au, 0x1f060a1au, 0x1f061202u, 0x1f0a060au, 0x1f0a1216u, 0x1f0a160au, 0x1f0e0202u, 0x1f0e0616u, + 0x1f0e0e0eu, 0x1f0e1a02u, 0x1f120a06u, 0x1f121206u, 0x1f160202u, 0x1f16020au, 0x1f160a12u, 0x1f1a0e02u) +} + +//! Main-context copy of the halved IQ3_XXS grid (tests, oracles, emitter bakes). +let IQ3XXS_GRID = iq3xxs_grid() + +//! ggml's ksigns_iq2xs: 7-bit sign index -> 8 sign bits (bit 7 = even parity). Shared by the +//! IQ3_XXS and IQ2 families. Per-call local - the worker-safe form. +def ksigns_iq2xs() : uint8[128] { + return fixed_array( + 0x00, 0x81, 0x82, 0x03, 0x84, 0x05, 0x06, 0x87, 0x88, 0x09, 0x0a, 0x8b, 0x0c, 0x8d, 0x8e, 0x0f, + 0x90, 0x11, 0x12, 0x93, 0x14, 0x95, 0x96, 0x17, 0x18, 0x99, 0x9a, 0x1b, 0x9c, 0x1d, 0x1e, 0x9f, + 0xa0, 0x21, 0x22, 0xa3, 0x24, 0xa5, 0xa6, 0x27, 0x28, 0xa9, 0xaa, 0x2b, 0xac, 0x2d, 0x2e, 0xaf, + 0x30, 0xb1, 0xb2, 0x33, 0xb4, 0x35, 0x36, 0xb7, 0xb8, 0x39, 0x3a, 0xbb, 0x3c, 0xbd, 0xbe, 0x3f, + 0xc0, 0x41, 0x42, 0xc3, 0x44, 0xc5, 0xc6, 0x47, 0x48, 0xc9, 0xca, 0x4b, 0xcc, 0x4d, 0x4e, 0xcf, + 0x50, 0xd1, 0xd2, 0x53, 0xd4, 0x55, 0x56, 0xd7, 0xd8, 0x59, 0x5a, 0xdb, 0x5c, 0xdd, 0xde, 0x5f, + 0x60, 0xe1, 0xe2, 0x63, 0xe4, 0x65, 0x66, 0xe7, 0xe8, 0x69, 0x6a, 0xeb, 0x6c, 0xed, 0xee, 0x6f, + 0xf0, 0x71, 0x72, 0xf3, 0x74, 0xf5, 0xf6, 0x77, 0x78, 0xf9, 0xfa, 0x7b, 0xfc, 0x7d, 0x7e, 0xff) +} + +//! Main-context copy of ksigns_iq2xs. +let KSIGNS_IQ2XS = ksigns_iq2xs() + +//! Halve an f16 bit pattern exactly (exponent decrement; subnormal results truncate-halve). +//! The IQ3_XXS transcode folds ggml's 0.25 into the stored d (one half) and the grid (the other). +def f16_half_bits(u : uint) : uint { + let e = (u >> 10u) & 0x1Fu + if (e >= 2u) { + return u - 0x400u + } + if (e == 1u) { + return (u & 0x8000u) | ((0x400u | (u & 0x3FFu)) >> 1u) + } + return (u & 0x8000u) | ((u & 0x3FFu) >> 1u) +} + //! Quant-plane bytes per stride unit — the typed stride surface. The stride unit is one //! 256-weight superblock for the kq lattice and one 32-weight block for q51 (see kq_elems). //! q8 panics (its planes are byte-per-elem, not stride-walked here). @@ -189,13 +267,16 @@ def kq_qsb(f : KqFmt) : int64 { if (f == KqFmt.iq3s) { return IQ3S_QSB } + if (f == KqFmt.iq3xxs) { + return IQ3XXS_QSB + } panic("kq_qsb: no plane stride for this format") return 0l } //! Scale-plane bytes per stride unit (see kq_qsb for the unit). def kq_ssb(f : KqFmt) : int64 { - if (f == KqFmt.k4 || f == KqFmt.k5 || f == KqFmt.iq4xs || f == KqFmt.iq3s) { + if (f == KqFmt.k4 || f == KqFmt.k5 || f == KqFmt.iq4xs || f == KqFmt.iq3s || f == KqFmt.iq3xxs) { return K4_SSB } if (f == KqFmt.k6 || f == KqFmt.k3) { diff --git a/modules/dasLLAMA/dasllama/dasllama_layout.das b/modules/dasLLAMA/dasllama/dasllama_layout.das index f3070a55af..bfc140db57 100644 --- a/modules/dasLLAMA/dasllama/dasllama_layout.das +++ b/modules/dasLLAMA/dasllama/dasllama_layout.das @@ -361,6 +361,10 @@ def private repack_regions(var t : Model; regs : array) { var iq3ssp : uint8? = null if (!empty(t.iq3sq)) { iq3sqp = addr(t.iq3sq[0]) } if (!empty(t.iq3ss)) { iq3ssp = addr(t.iq3ss[0]) } + var iq3xxsqp : uint8? = null + var iq3xxssp : uint8? = null + if (!empty(t.iq3xxsq)) { iq3xxsqp = addr(t.iq3xxsq[0]) } + if (!empty(t.iq3xxss)) { iq3xxssp = addr(t.iq3xxss[0]) } if (!empty(t.qblob)) { qbp = addr(t.qblob[0]) } if (!empty(t.qscales)) { qsp = addr(t.qscales[0]) } if (!empty(t.mxq)) { mxqp = addr(t.mxq[0]) } @@ -392,8 +396,8 @@ def private repack_regions(var t : Model; regs : array) { let sb = rp[i].off / 256l let qsb = kq_qsb(f) let ssb = kq_ssb(f) - var kqp = f == 4 ? k4qp : (f == 5 ? k5qp : (f == 6 ? k6qp : (f == 40 ? q40qp : (f == 44 ? iq4xsqp : (f == 3 ? k3qp : iq3sqp))))) - var ksp = f == 4 ? k4sp : (f == 5 ? k5sp : (f == 6 ? k6sp : (f == 40 ? q40sp : (f == 44 ? iq4xssp : (f == 3 ? k3sp : iq3ssp))))) + var kqp = f == 4 ? k4qp : (f == 5 ? k5qp : (f == 6 ? k6qp : (f == 40 ? q40qp : (f == 44 ? iq4xsqp : (f == 3 ? k3qp : (f == 33 ? iq3sqp : iq3xxsqp)))))) + var ksp = f == 4 ? k4sp : (f == 5 ? k5sp : (f == 6 ? k6sp : (f == 40 ? q40sp : (f == 44 ? iq4xssp : (f == 3 ? k3sp : (f == 33 ? iq3ssp : iq3xxssp)))))) invoke(rkq, f, kqp + sb * qsb, ksp + sb * ssb, rp[i].n, rp[i].d) } } @@ -525,6 +529,8 @@ def private push_repack_kq(var regs : array; fmt : KqFmt; woff, n, d push_repack(regs, 3, woff, n, d) } elif (fmt == KqFmt.iq3s) { push_repack(regs, 33, woff, n, d) + } elif (fmt == KqFmt.iq3xxs) { + push_repack(regs, 34, woff, n, d) } } @@ -800,13 +806,13 @@ def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice : (fmt == KqFmt.k6 ? addr(t.k6q[sb0 * kq_qsb(6)]) : (fmt == KqFmt.q40 ? addr(t.q40q[sb0 * kq_qsb(40)]) : (fmt == KqFmt.iq4xs ? addr(t.iq4xsq[sb0 * kq_qsb(44)]) - : (fmt == KqFmt.k3 ? addr(t.k3q[sb0 * kq_qsb(3)]) : addr(t.iq3sq[sb0 * kq_qsb(33)]))))))) + : (fmt == KqFmt.k3 ? addr(t.k3q[sb0 * kq_qsb(3)]) : (fmt == KqFmt.iq3s ? addr(t.iq3sq[sb0 * kq_qsb(33)]) : addr(t.iq3xxsq[sb0 * kq_qsb(34)])))))))) let sp = (fmt == KqFmt.k4 ? addr(t.k4s[sb0 * kq_ssb(4)]) : (fmt == KqFmt.k5 ? addr(t.k5s[sb0 * kq_ssb(5)]) : (fmt == KqFmt.k6 ? addr(t.k6s[sb0 * kq_ssb(6)]) : (fmt == KqFmt.q40 ? addr(t.q40s[sb0 * kq_ssb(40)]) : (fmt == KqFmt.iq4xs ? addr(t.iq4xss[sb0 * kq_ssb(44)]) - : (fmt == KqFmt.k3 ? addr(t.k3s[sb0 * kq_ssb(3)]) : addr(t.iq3ss[sb0 * kq_ssb(33)]))))))) + : (fmt == KqFmt.k3 ? addr(t.k3s[sb0 * kq_ssb(3)]) : (fmt == KqFmt.iq3s ? addr(t.iq3ss[sb0 * kq_ssb(33)]) : addr(t.iq3xxss[sb0 * kq_ssb(34)])))))))) var wqp = addr(wq[0]) var wsp = addr(ws[0]) let njobs = is_job_que_available() ? min(nslices, 4 * (get_total_hw_jobs() + 1)) : 1 @@ -830,6 +836,10 @@ def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice for (m in range64(104l)) { dq[m] = gq[((m / 4l) * mr + r) * 4l + m % 4l] } + } elif (fmt == KqFmt.iq3xxs) { // 24 uniform 4-byte columns over [qs][aux] + for (m in range64(96l)) { + dq[m] = gq[((m / 4l) * mr + r) * 4l + m % 4l] + } } elif (fmt == KqFmt.k3) { // 16 qs columns, then the 8 hmask columns at 64*mr for (m in range64(64l)) { dq[m] = gq[((m / 4l) * mr + r) * 4l + m % 4l] @@ -880,7 +890,7 @@ def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice } else { let rq = qp + sliceQ + ri * qrow + sbi * qsb let rs = sp + sliceS + ri * srow + sbi * dssb - if (fmt == KqFmt.q40 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s) { + if (fmt == KqFmt.q40 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s || fmt == KqFmt.iq3xxs) { for (m in range64(qsb)) { // disk bytes already in the device form (k/k+16 pairs; k3's lanes) dq[m] = rq[m] } @@ -907,7 +917,7 @@ def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice dq[128l + p] = rq[128l + p] } } - if (fmt == KqFmt.k6 || fmt == KqFmt.q40 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s) { // rows already in the decoded device form + if (fmt == KqFmt.k6 || fmt == KqFmt.q40 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s || fmt == KqFmt.iq3xxs) { // rows already in the decoded device form for (idx in range64(dssb)) { dsc[idx] = rs[idx] } diff --git a/modules/dasLLAMA/dasllama/dasllama_load.das b/modules/dasLLAMA/dasllama/dasllama_load.das index ec8e3790df..03c908c4e7 100644 --- a/modules/dasLLAMA/dasllama/dasllama_load.das +++ b/modules/dasLLAMA/dasllama/dasllama_load.das @@ -44,6 +44,7 @@ struct private LayoutSizes { iq4xs_n : int64 k3_n : int64 iq3s_n : int64 + iq3xxs_n : int64 pleq8_n : int64 // the dedicated q8 plane holding ONLY the PLE table (fp32/q4 serving) } @@ -59,6 +60,7 @@ struct private KqCursors { iq4xs : int64 k3 : int64 iq3s : int64 + iq3xxs : int64 } def private kq_take(var cur : KqCursors; f : KqFmt; n : int64) : int64 { @@ -102,6 +104,11 @@ def private kq_take(var cur : KqCursors; f : KqFmt; n : int64) : int64 { cur.iq3s += n return o } + if (f == KqFmt.iq3xxs) { + let o = cur.iq3xxs + cur.iq3xxs += n + return o + } let o = cur.wo cur.wo += n return o @@ -401,7 +408,7 @@ def private layout_offsets(var t : Model) : LayoutSizes { // nolint:STYLE037,S fo += dim } return LayoutSizes(fblob_n = fo, wblob_n = cur.wo, mx_n = mx, bf16_n = bf16, - k4_n = cur.k4, k5_n = cur.k5, k6_n = cur.k6, q40_n = cur.q40, q51_n = cur.q51, iq4xs_n = cur.iq4xs, k3_n = cur.k3, iq3s_n = cur.iq3s, pleq8_n = pleq8) + k4_n = cur.k4, k5_n = cur.k5, k6_n = cur.k6, q40_n = cur.q40, q51_n = cur.q51, iq4xs_n = cur.iq4xs, k3_n = cur.k3, iq3s_n = cur.iq3s, iq3xxs_n = cur.iq3xxs, pleq8_n = pleq8) } @@ -666,6 +673,8 @@ def stream_field_of(t : Model; fmt : KqFmt) : string { return "k3q" } elif (fmt == KqFmt.iq3s) { return "iq3sq" + } elif (fmt == KqFmt.iq3xxs) { + return "iq3xxsq" } elif (t.quant == QuantMode.q8) { // the blob flavor carries q8 as gguf-native 34B blocks in ONE plane, never the split pair return t.metal_blob ? "mblob" : "qblob" @@ -788,6 +797,8 @@ def fill_stream_plane(m : GGUFMeta; bytes : array | #; var t : Model; fie memcpy(addr(t.k3s[(j.woff / 256l) * K3_SSB]), addr(temp_s[0]), sh.sb) } elif (j.fmt == KqFmt.iq3s) { memcpy(addr(t.iq3ss[(j.woff / 256l) * IQ3S_SSB]), addr(temp_s[0]), sh.sb) + } elif (j.fmt == KqFmt.iq3xxs) { + memcpy(addr(t.iq3xxss[(j.woff / 256l) * IQ3XXS_SSB]), addr(temp_s[0]), sh.sb) } elif (j.fmt == KqFmt.q51) { memcpy(addr(t.q51s[(j.woff / 32l) * Q51_SB]), addr(temp_s[0]), sh.sb) } elif (sh.sb == 0l && j.n > 0l && !t.metal_blob) { @@ -828,6 +839,9 @@ def fill_stream_plane(m : GGUFMeta; bytes : array | #; var t : Model; fie } elif (r.fmt == 33) { invoke(g_stream_repack, 33, addr(temp_q[((r.off - j.woff) / 256l) * IQ3S_QSB]), addr(t.iq3ss[(r.off / 256l) * IQ3S_SSB]), r.n, r.d) + } elif (r.fmt == 34) { + invoke(g_stream_repack, 34, addr(temp_q[((r.off - j.woff) / 256l) * IQ3XXS_QSB]), + addr(t.iq3xxss[(r.off / 256l) * IQ3XXS_SSB]), r.n, r.d) } } } @@ -1049,6 +1063,9 @@ def private transcode_kq_tensor(m : GGUFMeta; bytes : array | #; name : s } elif (fmt == KqFmt.iq3s) { gguf_transcode_iq3s(m, bytes, name, kq, ks, eloff, n, src_off) return "iq3s transcode (IQ3_S)" + } elif (fmt == KqFmt.iq3xxs) { + gguf_transcode_iq3xxs(m, bytes, name, kq, ks, eloff, n, src_off) + return "iq3xxs transcode (IQ3_XXS)" } gguf_transcode_q51(m, bytes, name, kq, ks, eloff, n, src_off) return "q51 transcode (Q5_1)" @@ -1093,6 +1110,8 @@ def private load_big(m : GGUFMeta; bytes : array | #; name : string; var kind = transcode_kq_tensor(m, bytes, name, fmt, t.k3q, t.k3s, woff, n, src_off, scratch) } elif (fmt == KqFmt.iq3s) { kind = transcode_kq_tensor(m, bytes, name, fmt, t.iq3sq, t.iq3ss, woff, n, src_off, scratch) + } elif (fmt == KqFmt.iq3xxs) { + kind = transcode_kq_tensor(m, bytes, name, fmt, t.iq3xxsq, t.iq3xxss, woff, n, src_off, scratch) } elif (fmt == KqFmt.q51) { kind = transcode_kq_tensor(m, bytes, name, fmt, t.q51q, t.q51s, woff, n, src_off, scratch) } elif (t.quant == QuantMode.q8) { @@ -1192,8 +1211,8 @@ def private log_load_report(t : Model) { to_log(LOG_INFO, "dasLLAMA noisy: backend '{active_kernel_backend()}' | kquant_native {get_kquant_native()} q40_native {get_kq_q40_native()} q50_native {get_kq_q50_native()}\n") // the tier arms at [init], before a --noisy CLI flag can land — restate it here so the flag alone suffices to_log(LOG_INFO, "dasLLAMA noisy: GPU tier installed {moe_gpu_tier_installed()} | want auto {gpu_want_auto()}, moe layers {gpu_want_moe_layers()}, stream {gpu_want_moe_stream()}\n") - to_log(LOG_INFO, "dasLLAMA noisy: format tags - q8 {tally_fmt(t, KqFmt.q8)}, k4 {tally_fmt(t, KqFmt.k4)}, k5 {tally_fmt(t, KqFmt.k5)}, k6 {tally_fmt(t, KqFmt.k6)}, q40 {tally_fmt(t, KqFmt.q40)}, q51 {tally_fmt(t, KqFmt.q51)}, iq4xs {tally_fmt(t, KqFmt.iq4xs)}, k3 {tally_fmt(t, KqFmt.k3)}, iq3s {tally_fmt(t, KqFmt.iq3s)}\n") - to_log(LOG_INFO, "dasLLAMA noisy: plane MB - q8 {float(long_length(t.qblob)) / mb} (+scales {float(long_length(t.qscales) * 4l) / mb}), k4 {float(long_length(t.k4q) + long_length(t.k4s)) / mb}, k5 {float(long_length(t.k5q) + long_length(t.k5s)) / mb}, k6 {float(long_length(t.k6q) + long_length(t.k6s)) / mb}, q40 {float(long_length(t.q40q) + long_length(t.q40s)) / mb}, q51 {float(long_length(t.q51q) + long_length(t.q51s)) / mb}, iq4xs {float(long_length(t.iq4xsq) + long_length(t.iq4xss)) / mb}, k3 {float(long_length(t.k3q) + long_length(t.k3s)) / mb}, iq3s {float(long_length(t.iq3sq) + long_length(t.iq3ss)) / mb}, f32 {float(long_length(t.wblob) * 4l) / mb}\n") + to_log(LOG_INFO, "dasLLAMA noisy: format tags - q8 {tally_fmt(t, KqFmt.q8)}, k4 {tally_fmt(t, KqFmt.k4)}, k5 {tally_fmt(t, KqFmt.k5)}, k6 {tally_fmt(t, KqFmt.k6)}, q40 {tally_fmt(t, KqFmt.q40)}, q51 {tally_fmt(t, KqFmt.q51)}, iq4xs {tally_fmt(t, KqFmt.iq4xs)}, k3 {tally_fmt(t, KqFmt.k3)}, iq3s {tally_fmt(t, KqFmt.iq3s)}, iq3xxs {tally_fmt(t, KqFmt.iq3xxs)}\n") + to_log(LOG_INFO, "dasLLAMA noisy: plane MB - q8 {float(long_length(t.qblob)) / mb} (+scales {float(long_length(t.qscales) * 4l) / mb}), k4 {float(long_length(t.k4q) + long_length(t.k4s)) / mb}, k5 {float(long_length(t.k5q) + long_length(t.k5s)) / mb}, k6 {float(long_length(t.k6q) + long_length(t.k6s)) / mb}, q40 {float(long_length(t.q40q) + long_length(t.q40s)) / mb}, q51 {float(long_length(t.q51q) + long_length(t.q51s)) / mb}, iq4xs {float(long_length(t.iq4xsq) + long_length(t.iq4xss)) / mb}, k3 {float(long_length(t.k3q) + long_length(t.k3s)) / mb}, iq3s {float(long_length(t.iq3sq) + long_length(t.iq3ss)) / mb}, iq3xxs {float(long_length(t.iq3xxsq) + long_length(t.iq3xxss)) / mb}, f32 {float(long_length(t.wblob) * 4l) / mb}\n") } // GGML disk type -> plane format tag (non-K-quant types ride the classic q8 path) @@ -1216,6 +1235,9 @@ def private kq_fmt_of(gt : int) : KqFmt { if (gt == GGML_TYPE_IQ3_S) { return KqFmt.iq3s } + if (gt == GGML_TYPE_IQ3_XXS) { + return KqFmt.iq3xxs + } if (gt == GGML_TYPE_Q4_0 && get_kq_q40_native()) { return KqFmt.q40 } @@ -2052,6 +2074,7 @@ def private load_gguf_parsed(var t : Model; m : GGUFMeta; bytes : array | g_stream_plane_total["iq4xsq"] = (sz.iq4xs_n / 256l) * IQ4XS_QSB g_stream_plane_total["k3q"] = (sz.k3_n / 256l) * K3_QSB g_stream_plane_total["iq3sq"] = (sz.iq3s_n / 256l) * IQ3S_QSB + g_stream_plane_total["iq3xxsq"] = (sz.iq3xxs_n / 256l) * IQ3XXS_QSB let wb = mode == QuantMode.q8 ? sz.wblob_n : 0l g_stream_plane_total[t.metal_blob ? "mblob" : "qblob"] = (t.metal_blob ? (wb / Q8_BLOCK_ELEMS) * (Q8_QPB + Q8_SPB) : wb) @@ -2120,6 +2143,14 @@ def private load_gguf_parsed(var t : Model; m : GGUFMeta; bytes : array | t.iq3ss |> reserve((sz.iq3s_n / 256l) * IQ3S_SSB) t.iq3ss |> resize((sz.iq3s_n / 256l) * IQ3S_SSB) } + if (sz.iq3xxs_n > 0l) { + if (!stream_q) { + t.iq3xxsq |> reserve((sz.iq3xxs_n / 256l) * IQ3XXS_QSB) + t.iq3xxsq |> resize((sz.iq3xxs_n / 256l) * IQ3XXS_QSB) + } + t.iq3xxss |> reserve((sz.iq3xxs_n / 256l) * IQ3XXS_SSB) + t.iq3xxss |> resize((sz.iq3xxs_n / 256l) * IQ3XXS_SSB) + } if (mode == QuantMode.q8) { if (!stream_q) { t.qblob |> reserve(sz.wblob_n) @@ -2429,6 +2460,7 @@ def private load_gguf_parsed(var t : Model; m : GGUFMeta; bytes : array | t.kq_repack_mr44 = active_kq_layout_mr(44) t.kq_repack_mr3 = active_kq_layout_mr(3) t.kq_repack_mr33 = active_kq_layout_mr(33) + t.kq_repack_mr34 = active_kq_layout_mr(34) } } else { layout_repack_q8(t) @@ -2448,7 +2480,8 @@ def private load_gguf_parsed(var t : Model; m : GGUFMeta; bytes : array | t.kq_repack_mr44 = active_kq_layout_mr(44) t.kq_repack_mr3 = active_kq_layout_mr(3) t.kq_repack_mr33 = active_kq_layout_mr(33) - to_log(LOG_INFO, "dasLLAMA: K-quant planes repacked to the '{active_kernel_backend()}' layouts (k4 grp{t.kq_repack_mr4}, k5 grp{t.kq_repack_mr5}, k6 grp{t.kq_repack_mr6}, q40 grp{t.kq_repack_mr40}, iq4xs grp{t.kq_repack_mr44}, k3 grp{t.kq_repack_mr3}, iq3s grp{t.kq_repack_mr33})\n") + t.kq_repack_mr34 = active_kq_layout_mr(34) + to_log(LOG_INFO, "dasLLAMA: K-quant planes repacked to the '{active_kernel_backend()}' layouts (k4 grp{t.kq_repack_mr4}, k5 grp{t.kq_repack_mr5}, k6 grp{t.kq_repack_mr6}, q40 grp{t.kq_repack_mr40}, iq4xs grp{t.kq_repack_mr44}, k3 grp{t.kq_repack_mr3}, iq3s grp{t.kq_repack_mr33}, iq3xxs grp{t.kq_repack_mr34})\n") } } } diff --git a/modules/dasLLAMA/dasllama/dasllama_math.das b/modules/dasLLAMA/dasllama/dasllama_math.das index b3dd2d930c..4df5a8b2b9 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math.das +++ b/modules/dasLLAMA/dasllama/dasllama_math.das @@ -1087,6 +1087,7 @@ struct KernelBackend { kq_rows_iq4xs : MatmulKqRowsFn = @@kq_unset_rows kq_rows_k3 : MatmulKqRowsFn = @@kq_unset_rows kq_rows_iq3s : MatmulKqRowsFn = @@kq_unset_rows + kq_rows_iq3xxs : MatmulKqRowsFn = @@kq_unset_rows kq_batch : MatmulKqBatchFn = @@kq_unset_batch kq_groupn : MatmulKqGroupNFn = @@kq_unset_groupn kq_batch_groupn : MatmulKqBatchGroupNFn = @@kq_unset_batch_groupn @@ -1152,6 +1153,7 @@ var g_kq_rows_q40 = @@kq_unset_rows var g_kq_rows_iq4xs = @@kq_unset_rows var g_kq_rows_k3 = @@kq_unset_rows var g_kq_rows_iq3s = @@kq_unset_rows +var g_kq_rows_iq3xxs = @@kq_unset_rows var g_mm_kq_batch = @@kq_unset_batch var g_mm_kq_groupn = @@kq_unset_groupn var g_mm_kq_batch_groupn = @@kq_unset_batch_groupn @@ -1479,7 +1481,7 @@ def active_q8_layout_mr() : int64 => g_bake_cpu_override ? g_bake_cpu.q8_mr : g_ //! family split; the loader freezes these onto the Model at repack time. def active_kq_layout_mr(fmt : int) : int64 { if (g_bake_cpu_override) { - return fmt == 4 ? g_bake_cpu.kq_mr4 : (fmt == 5 ? g_bake_cpu.kq_mr5 : (fmt == 40 ? g_bake_cpu.kq_mr40 : (fmt == 44 ? g_bake_cpu.kq_mr44 : (fmt == 3 ? g_bake_cpu.kq_mr3 : (fmt == 33 ? g_bake_cpu.kq_mr33 : g_bake_cpu.kq_mr6))))) + return fmt == 4 ? g_bake_cpu.kq_mr4 : (fmt == 5 ? g_bake_cpu.kq_mr5 : (fmt == 40 ? g_bake_cpu.kq_mr40 : (fmt == 44 ? g_bake_cpu.kq_mr44 : (fmt == 3 ? g_bake_cpu.kq_mr3 : (fmt == 33 ? g_bake_cpu.kq_mr33 : (fmt == 34 ? g_bake_cpu.kq_mr34 : g_bake_cpu.kq_mr6)))))) } return int64(invoke(g_active_kq_layout, fmt)) } @@ -1658,6 +1660,7 @@ def private activate(be : KernelBackend) { g_kq_rows_iq4xs = be.kq_rows_iq4xs g_kq_rows_k3 = be.kq_rows_k3 g_kq_rows_iq3s = be.kq_rows_iq3s + g_kq_rows_iq3xxs = be.kq_rows_iq3xxs g_mm_kq_batch = be.kq_batch g_mm_kq_groupn = be.kq_groupn g_mm_kq_batch_groupn = be.kq_batch_groupn @@ -1863,6 +1866,7 @@ def kq_rows_fn(fmt : int) : MatmulKqRowsFn { if (fmt == 44) return g_kq_rows_iq4xs if (fmt == 3) return g_kq_rows_k3 if (fmt == 33) return g_kq_rows_iq3s + if (fmt == 34) return g_kq_rows_iq3xxs return @@kq_unset_rows } @@ -3232,6 +3236,9 @@ def private restore_math_fn_defaults { // nolint:STYLE037,STYLE038 — flat seam if (g_kq_rows_iq3s == null) { g_kq_rows_iq3s = @@kq_unset_rows } + if (g_kq_rows_iq3xxs == null) { + g_kq_rows_iq3xxs = @@kq_unset_rows + } if (g_mm_kq_batch == null) { g_mm_kq_batch = @@kq_unset_batch } diff --git a/modules/dasLLAMA/dasllama/dasllama_math_default.das b/modules/dasLLAMA/dasllama/dasllama_math_default.das index 368dfd4c2a..9fa865ce63 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_default.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_default.das @@ -708,6 +708,40 @@ def k3_rows_kernel(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp } } +[unused_argument(xbsp)] +def dot_iq3xxs_q8(kqrow : uint8 const?; ksrow : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n : int64) : float { + var acc = 0.0 + let grid = iq3xxs_grid() + let signs = ksigns_iq2xs() + unsafe { + for (sb in range64(n / 256l)) { + let kqo = sb * 96l + let kso = sb * 20l + let d = f16_to_f32(uint(ksrow[kso]) | (uint(ksrow[kso + 1l]) << 8u)) + var isum = 0 + for (blk in range64(8l)) { + let aux = uint(kqrow[kqo + 64l + blk * 4l]) | (uint(kqrow[kqo + 64l + blk * 4l + 1l]) << 8u) | (uint(kqrow[kqo + 64l + blk * 4l + 2l]) << 16u) | (uint(kqrow[kqo + 64l + blk * 4l + 3l]) << 24u) + let ab = (sb * 8l + blk) * 32l + var ib = 0 + for (l in range64(4l)) { + let sgn = int(signs[int((aux >> uint(7l * l)) & 127u)]) + let g1 = grid[int(kqrow[kqo + blk * 8l + 2l * l])] + let g2 = grid[int(kqrow[kqo + blk * 8l + 2l * l + 1l])] + for (j in range64(4l)) { + let w1 = int((g1 >> uint(8l * j)) & 0xFFu) + let w2 = int((g2 >> uint(8l * j)) & 0xFFu) + ib += (((sgn >> int(j)) & 1) != 0 ? -w1 : w1) * int(xqp[ab + l * 8l + j]) + ib += (((sgn >> int(4l + j)) & 1) != 0 ? -w2 : w2) * int(xqp[ab + l * 8l + 4l + j]) + } + } + isum += int(ksrow[kso + 4l + blk]) * ib + } + acc += float(isum) * (d * xsp[sb]) + } + } + return acc +} + def iq3s_rows_kernel(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, rb, re : int64) { let nsb = n / 256l unsafe { @@ -717,6 +751,15 @@ def iq3s_rows_kernel(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xq } } +def iq3xxs_rows_kernel(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, rb, re : int64) { + let nsb = n / 256l + unsafe { + for (i in range64(rb, re)) { + yp[i] = dot_iq3xxs_q8(kqp + i * nsb * 96l, ksp + i * nsb * 20l, xqp, xsp, xbsp, n) + } + } +} + def private kq_gemv_kernel(fmt : int; var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d : int64) { var myp = yp maybe_parallel_for(0, int(d), matmul_chunks_gemv(int(d), 1, n * d)) $(rb, re) { @@ -732,6 +775,8 @@ def private kq_gemv_kernel(fmt : int; var yp : float?; kqp : uint8 const?; ksp : k3_rows_kernel(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb), int64(re)) } elif (fmt == 33) { iq3s_rows_kernel(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb), int64(re)) + } elif (fmt == 34) { + iq3xxs_rows_kernel(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb), int64(re)) } else { k6_rows_kernel(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb), int64(re)) } @@ -840,6 +885,40 @@ def private dequant_iq3s_row_grp(kqg : uint8 const?; ksg : uint8 const?; r, mr, } } +//! iq3xxs grp row dequant: 24 uniform 4-byte columns per superblock row ([qs][aux] in place), +//! the iq3s scale interleave — w = (d_halved * (2ls + 1)) * halved_grid_byte * sign. +def private dequant_iq3xxs_row_grp(kqg : uint8 const?; ksg : uint8 const?; r, mr, nsb : int64; var dst : float?) { + let grid = iq3xxs_grid() + let signs = ksigns_iq2xs() + unsafe { + for (sbi in range64(nsb)) { + let qb = sbi * 96l * mr + let sb = sbi * 20l * mr + let d = f16_to_f32(uint(ksg[sb + 2l * r]) | (uint(ksg[sb + 2l * r + 1l]) << 8u)) + for (blk in range64(8l)) { + let db = d * float(int(ksg[sb + 4l * mr + blk * mr + r])) + var aux = 0u + for (t in range64(4l)) { + let ap = 64l + blk * 4l + t + aux |= uint(kqg[qb + ((ap / 4l) * mr + r) * 4l + ap % 4l]) << uint(8l * t) + } + for (l in range64(4l)) { + let sgn = int(signs[int((aux >> uint(7l * l)) & 127u)]) + let q1p = blk * 8l + 2l * l + let q2p = q1p + 1l + let g1 = grid[int(kqg[qb + ((q1p / 4l) * mr + r) * 4l + q1p % 4l])] + let g2 = grid[int(kqg[qb + ((q2p / 4l) * mr + r) * 4l + q2p % 4l])] + let kb = sbi * 256l + blk * 32l + l * 8l + for (j in range64(4l)) { + dst[kb + j] = db * float((g1 >> uint(8l * j)) & 0xFFu) * (((sgn >> int(j)) & 1) != 0 ? -1.0 : 1.0) + dst[kb + 4l + j] = db * float((g2 >> uint(8l * j)) & 0xFFu) * (((sgn >> int(4l + j)) & 1) != 0 ? -1.0 : 1.0) + } + } + } + } + } +} + //! Dequant one row off the grp-REPACKED K-quant planes (repack_k4/k5/k6_grp layout). //! kqg/ksg = row's GROUP plane base (group g = row/mr); r = row % mr. //! embed_row's kq_repacked path — per-token, so scalar is fine. @@ -863,6 +942,10 @@ def dequant_kq_row_grp(fmt : int64; kqg : uint8 const?; ksg : uint8 const?; r, m dequant_iq3s_row_grp(kqg, ksg, r, mr, nsb, dst) return } + if (fmt == 34l) { + dequant_iq3xxs_row_grp(kqg, ksg, r, mr, nsb, dst) + return + } unsafe { for (sbi in range64(nsb)) { let qb = sbi * qsb * mr @@ -963,6 +1046,8 @@ def matmul_kq_groupn(fmt : int; var y : array | #; kq : array | #; myp[ii] = dot_k3q8(kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) } elif (fmt == 33) { myp[ii] = dot_iq3s_q8(kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) + } elif (fmt == 34) { + myp[ii] = dot_iq3xxs_q8(kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) } else { myp[ii] = dot_k6q8(kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) } diff --git a/modules/dasLLAMA/dasllama/dasllama_math_gen.das b/modules/dasLLAMA/dasllama/dasllama_math_gen.das index 1426c92e72..8f77809e15 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_gen.das @@ -255,6 +255,10 @@ def iq3sq8_layout_gen() : int { return q8q8_repack_type(GEMM_REFERENCE_MR).interleave } +def iq3xxsq8_layout_gen() : int { + return q8q8_repack_type(GEMM_REFERENCE_MR).interleave +} + // the KernelBackend.kq_layout slot form of kq_layout_of (registered on both gen backends) def private kq_layout_fmt_gen(fmt : int) : int => int(kq_layout_of(fmt)) @@ -279,6 +283,9 @@ def kq_layout_of(fmt : int) : int64 { if (fmt == 33) { return int64(iq3sq8_layout_gen()) } + if (fmt == 34) { + return int64(iq3xxsq8_layout_gen()) + } return int64(k6q8_layout_gen()) } @@ -663,6 +670,81 @@ def iq3sq8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp } } +//! One row's dot off the grp iq3xxs planes, scalar — the iq3xxs stubs' reference body and +//! the tile oracle. d_halved x (2ls + 1) per block, signs off the aux 7-bit ksigns indices, +//! no activation-sum term. Same integer regroup and float fold as dot_iq3xxs_q8. +[unused_argument(xbsp), hint(unsafe_range_check, noalias = kqg, noalias = ksg, noalias = xqp, noalias = xsp, noalias = xbsp)] +def iq3xxs_grp_row_dot(kqg : uint8 const?; ksg : uint8 const?; r, mr : int64; xqp : int8 const?; xsp : float const?; xbsp : int const?; n : int64) : float { + var acc = 0.0 + let nsb = n / 256l + let grid = iq3xxs_grid() + let signs = ksigns_iq2xs() + unsafe { + for (sbi in range64(nsb)) { + let qb = sbi * 96l * mr + let sb = sbi * 20l * mr + var isum = 0 + for (blk in range64(8l)) { + var aux = 0u + for (t in range64(4l)) { + let ap = 64l + blk * 4l + t + aux |= uint(kqg[qb + ((ap / 4l) * mr + r) * 4l + ap % 4l]) << uint(8l * t) + } + var ib = 0 + for (l in range64(4l)) { + let sgn = int(signs[int((aux >> uint(7l * l)) & 127u)]) + let q1p = blk * 8l + 2l * l + let q2p = q1p + 1l + let g1 = grid[int(kqg[qb + ((q1p / 4l) * mr + r) * 4l + q1p % 4l])] + let g2 = grid[int(kqg[qb + ((q2p / 4l) * mr + r) * 4l + q2p % 4l])] + for (j in range64(4l)) { + let w1 = int((g1 >> uint(8l * j)) & 0xFFu) + let w2 = int((g2 >> uint(8l * j)) & 0xFFu) + ib += (((sgn >> int(j)) & 1) != 0 ? -w1 : w1) * int(xqp[(sbi * 8l + blk) * 32l + l * 8l + j]) + ib += (((sgn >> int(4l + j)) & 1) != 0 ? -w2 : w2) * int(xqp[(sbi * 8l + blk) * 32l + l * 8l + 4l + j]) + } + } + isum += int(ksg[sb + 4l * mr + blk * mr + r]) * ib + } + let dk = f16_to_f32(uint(ksg[sb + 2l * r]) | (uint(ksg[sb + 2l * r + 1l]) << 8u)) + acc += float(isum) * (dk * xsp[sbi]) + } + } + return acc +} + +[hint(unsafe_range_check, noalias = kqp, noalias = ksp, noalias = xqp, noalias = xsp, noalias = xbsp)] +def iq3xxsq8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, rb, re : int64) : void { + let mr = int64(iq3xxsq8_layout_gen()) + let nsb = n / 256l + unsafe { + for (i in range64(rb, re)) { + let g = i / mr + yp[i] = iq3xxs_grp_row_dot(kqp + g * mr * nsb * 96l, ksp + g * mr * nsb * 20l, i % mr, mr, xqp, xsp, xbsp, n) + } + } +} + +[tune_perm(mr = 4), tune_perm(mr = 4, nrsplit = 2), tune_perm(mr = 8), tune_perm(mr = 8, nrsplit = 2), + tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "maddubs", width = 256, mr = 8, nrsplit = 2, requires = "avx2"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, nrsplit = 2, requires = "avxvnni|avx512vnni"), + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, nrsplit = 2, requires = "avx512vnni,avx512bw"), + tune_companion(fn = "iq3xxsq8_gemv_gen", gen = "dasllama_gemm_gen::iq3xxs_gemv"), + tune_companion(fn = "iq3xxsq8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), + tune(gen = "dasllama_gemm_gen::iq3xxs_tile", + fallback = "dot_vpdpbusd_width512_mr16;dot_vpdpbusd_width256_mr8;dot_maddubs_width256_mr8;mr4"), + hint(unsafe_range_check, noalias = kqg, noalias = ksg, noalias = xqp, noalias = xsp, noalias = xbsp)] +def iq3xxsq8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d, g, t0 : int64) : void { + let mr = int64(iq3xxsq8_layout_gen()) + unsafe { + for (t in range64(4l)) { + for (r in range64(mr)) { + yp[(t0 + t) * d + g * mr + r] = kq_grp_row_dot_b(34l, kqg, ksg, r, mr, xqp + (t0 + t) * n, xsp + (t0 + t) * (n / 256l), xbsp + (t0 + t) * (n / 16l), n) + } + } + } +} + //! The f16-scale GEMV companion (wscale_f16 rail): q8q8_gemv_gen's twin over the raw binary16 //! group-scale plane. Same rows-range contract; the generated body differs only in the scale //! fold's widen (fcvtl/vcvtph2ps). Not `private`: it IS the mm_rows_s16 slot. @@ -938,13 +1020,13 @@ def kq_grp_row_dot_b(fmt : int64; kqg : uint8 const?; ksg : uint8 const?; r, mr var ihi = 0 for (j in range64(4l)) { for (t in range64(4l)) { - let qlo = fmt == 33l ? int(int8(kqg[qb + ((blk * 4l + j) * mr + r) * 4l + t])) : int(kqg[qb + ((blk * 4l + j) * mr + r) * 4l + t]) - let qhi = fmt == 33l ? int(int8(kqg[qb + 128l * mr + ((blk * 4l + j) * mr + r) * 4l + t])) : int(kqg[qb + 128l * mr + ((blk * 4l + j) * mr + r) * 4l + t]) + let qlo = fmt == 33l || fmt == 34l ? int(int8(kqg[qb + ((blk * 4l + j) * mr + r) * 4l + t])) : int(kqg[qb + ((blk * 4l + j) * mr + r) * 4l + t]) + let qhi = fmt == 33l || fmt == 34l ? int(int8(kqg[qb + 128l * mr + ((blk * 4l + j) * mr + r) * 4l + t])) : int(kqg[qb + 128l * mr + ((blk * 4l + j) * mr + r) * 4l + t]) ilo += qlo * int(xqp[b * 32l + j * 4l + t]) ihi += qhi * int(xqp[b * 32l + 16l + j * 4l + t]) } } - if (fmt == 33l) { + if (fmt == 33l || fmt == 34l) { let sc = int(int8(ksg[sb + 4l * mr + blk * mr + r])) isum += sc * (ilo + ihi) } elif (fmt == 6l) { @@ -959,7 +1041,7 @@ def kq_grp_row_dot_b(fmt : int64; kqg : uint8 const?; ksg : uint8 const?; r, mr bsum += mn * (xbsp[b * 2l] + xbsp[b * 2l + 1l]) } } - if (fmt == 33l) { + if (fmt == 33l || fmt == 34l) { let di = f16_to_f32(uint(ksg[sb + 2l * r]) | (uint(ksg[sb + 2l * r + 1l]) << 8u)) acc += float(isum) * (di * xsp[sbi]) } elif (fmt == 6l) { @@ -992,6 +1074,8 @@ def private repack_kq_gen(fmt : int; var kq : uint8?; var ks : uint8?; n, d : in repack_k3_grp(kq, ks, n, d, kq_layout_of(3)) } elif (fmt == 33) { repack_iq3s_grp(kq, ks, n, d, kq_layout_of(33)) + } elif (fmt == 34) { + repack_iq3xxs_grp(kq, ks, n, d, kq_layout_of(34)) } else { repack_k6_grp(kq, ks, n, d, kq_layout_of(6)) } @@ -1010,6 +1094,8 @@ def private repack_kq_bake(fmt : int; var kq : uint8?; var ks : uint8?; n, d : i repack_k3_grp(kq, ks, n, d, active_kq_layout_mr(3)) } elif (fmt == 33) { repack_iq3s_grp(kq, ks, n, d, active_kq_layout_mr(33)) + } elif (fmt == 34) { + repack_iq3xxs_grp(kq, ks, n, d, active_kq_layout_mr(34)) } else { repack_k6_grp(kq, ks, n, d, active_kq_layout_mr(6)) } @@ -1526,6 +1612,8 @@ def private kq_kernel_gen(fmt : int; var yp : float?; kqp : uint8 const?; ksp : k3q8_gemv_gen(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb) * mr, int64(re) * mr) } elif (fmt == 33) { iq3sq8_gemv_gen(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb) * mr, int64(re) * mr) + } elif (fmt == 34) { + iq3xxsq8_gemv_gen(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb) * mr, int64(re) * mr) } else { k6q8_gemv_gen(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb) * mr, int64(re) * mr) } @@ -1544,6 +1632,8 @@ def private kq_kernel_gen(fmt : int; var yp : float?; kqp : uint8 const?; ksp : myp[i] = dot_k3q8(kqp + i * nsb * 96l, ksp + i * nsb * 18l, xqp, xsp, xbsp, n) } elif (fmt == 33) { myp[i] = dot_iq3s_q8(kqp + i * nsb * 104l, ksp + i * nsb * 20l, xqp, xsp, xbsp, n) + } elif (fmt == 34) { + myp[i] = dot_iq3xxs_q8(kqp + i * nsb * 96l, ksp + i * nsb * 20l, xqp, xsp, xbsp, n) } else { myp[i] = dot_k6q8(kqp + i * nsb * 192l, ksp + i * nsb * 18l, xqp, xsp, xbsp, n) } @@ -1554,7 +1644,7 @@ def private kq_kernel_gen(fmt : int; var yp : float?; kqp : uint8 const?; ksp : // One (group range x token range) cell of the kq batch walk: TB token blocks outer, groups // inner, 4-token tile calls + per-token gemv tails (ts pinned 4 — no TMUL kq tile exists). k5/k6 // tiles unpack a BYTE-EXPANDED scratch panel once per (group, token-block), amortized over TB. -def private kq_batch_cell_gen(fmt : int; var myp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d, mr, TB : int64; ub, ue : int; tb0, tend : int64) { +def private kq_batch_cell_gen(fmt : int; var myp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d, mr, TB : int64; ub, ue : int; tb0, tend : int64) { // nolint:STYLE037 — the flat per-format ladder (one arm per kq format) is the honest shape let nsb = n / 256l let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) @@ -1591,6 +1681,8 @@ def private kq_batch_cell_gen(fmt : int; var myp : float?; kqp : uint8 const?; k k3q8_tile_gen(myp, kqg, ksg, xqp, xsp, xbsp, n, d, g, tk) } elif (fmt == 33) { iq3sq8_tile_gen(myp, scp, ksg, xqp, xsp, xbsp, n, d, g, tk) + } elif (fmt == 34) { + iq3xxsq8_tile_gen(myp, scp, ksg, xqp, xsp, xbsp, n, d, g, tk) } else { k6q8_tile_gen(myp, scp, ksg, xqp, xsp, xbsp, n, d, g, tk) } @@ -1609,6 +1701,8 @@ def private kq_batch_cell_gen(fmt : int; var myp : float?; kqp : uint8 const?; k k3q8_gemv_gen(myp + tk * d, kqp, ksp, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n, g * mr, (g + 1l) * mr) } elif (fmt == 33) { iq3sq8_gemv_gen(myp + tk * d, kqp, ksp, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n, g * mr, (g + 1l) * mr) + } elif (fmt == 34) { + iq3xxsq8_gemv_gen(myp + tk * d, kqp, ksp, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n, g * mr, (g + 1l) * mr) } else { k6q8_gemv_gen(myp + tk * d, kqp, ksp, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n, g * mr, (g + 1l) * mr) } @@ -1648,6 +1742,8 @@ def private kq_batch_kernel_gen(fmt : int; var yp : float?; kqp : uint8 const?; myp[tk * d + i] = dot_k3q8(kqp + i * nsb * 96l, ksp + i * nsb * 18l, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * (n / 16l), n) } elif (fmt == 33) { myp[tk * d + i] = dot_iq3s_q8(kqp + i * nsb * 104l, ksp + i * nsb * 20l, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * (n / 16l), n) + } elif (fmt == 34) { + myp[tk * d + i] = dot_iq3xxs_q8(kqp + i * nsb * 96l, ksp + i * nsb * 20l, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * (n / 16l), n) } else { myp[tk * d + i] = dot_k6q8(kqp + i * nsb * 192l, ksp + i * nsb * 18l, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * (n / 16l), n) } @@ -1702,6 +1798,8 @@ def private kq_batch_groupn_gen(fmt : int; var yp : float?; kqp : uint8 const?; myp[tk * d + i] = dot_k3q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n) } elif (fmt == 33) { myp[tk * d + i] = dot_iq3s_q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n) + } elif (fmt == 34) { + myp[tk * d + i] = dot_iq3xxs_q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n) } else { myp[tk * d + i] = dot_k6q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n) } @@ -1865,6 +1963,8 @@ def kq_groupn_gen(fmt : int; var yp : float?; kqp : uint8 const?; ksp : uint8 co k3q8_gemv_gen(myp + r * d, kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n, gLocal * mr, gEnd * mr) } elif (fmt == 33) { iq3sq8_gemv_gen(myp + r * d, kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n, gLocal * mr, gEnd * mr) + } elif (fmt == 34) { + iq3xxsq8_gemv_gen(myp + r * d, kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n, gLocal * mr, gEnd * mr) } else { k6q8_gemv_gen(myp + r * d, kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n, gLocal * mr, gEnd * mr) } @@ -1889,6 +1989,8 @@ def kq_groupn_gen(fmt : int; var yp : float?; kqp : uint8 const?; ksp : uint8 co myp[r * d + i] = dot_k3q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) } elif (fmt == 33) { myp[r * d + i] = dot_iq3s_q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) + } elif (fmt == 34) { + myp[r * d + i] = dot_iq3xxs_q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) } else { myp[r * d + i] = dot_k6q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) } @@ -2182,7 +2284,7 @@ def dasllama_math_gen_register() { // the no-witness stance above covers kq too mm_kq = @@kq_kernel_gen, repack_kq = @@repack_kq_gen, kq_rows_k4 = @@k4q8_gemv_gen, kq_rows_k5 = @@k5q8_gemv_gen, - kq_rows_k6 = @@k6q8_gemv_gen, kq_rows_q40 = @@q40q8_gemv_gen, kq_rows_iq4xs = @@iq4xsq8_gemv_gen, kq_rows_k3 = @@k3q8_gemv_gen, kq_rows_iq3s = @@iq3sq8_gemv_gen, + kq_rows_k6 = @@k6q8_gemv_gen, kq_rows_q40 = @@q40q8_gemv_gen, kq_rows_iq4xs = @@iq4xsq8_gemv_gen, kq_rows_k3 = @@k3q8_gemv_gen, kq_rows_iq3s = @@iq3sq8_gemv_gen, kq_rows_iq3xxs = @@iq3xxsq8_gemv_gen, kq_batch = @@kq_batch_kernel_gen, kq_groupn = @@kq_groupn_gen, kq_batch_groupn = @@kq_batch_groupn_gen, // q51 family: tile + GEMV off the grp planes (the tile family's companions stamp @@ -2211,7 +2313,7 @@ def dasllama_math_gen_register() { mm_rows_s16 = @@q8q8_gemv_s16_gen, mm_kq = @@kq_kernel_gen, repack_kq = @@repack_kq_gen, kq_rows_k4 = @@k4q8_gemv_gen, kq_rows_k5 = @@k5q8_gemv_gen, - kq_rows_k6 = @@k6q8_gemv_gen, kq_rows_q40 = @@q40q8_gemv_gen, kq_rows_iq4xs = @@iq4xsq8_gemv_gen, kq_rows_k3 = @@k3q8_gemv_gen, kq_rows_iq3s = @@iq3sq8_gemv_gen, + kq_rows_k6 = @@k6q8_gemv_gen, kq_rows_q40 = @@q40q8_gemv_gen, kq_rows_iq4xs = @@iq4xsq8_gemv_gen, kq_rows_k3 = @@k3q8_gemv_gen, kq_rows_iq3s = @@iq3sq8_gemv_gen, kq_rows_iq3xxs = @@iq3xxsq8_gemv_gen, kq_batch = @@kq_batch_kernel_gen, kq_groupn = @@kq_groupn_gen, kq_batch_groupn = @@kq_batch_groupn_gen, groupn_q51 = @@q51q8_groupn_gen, repack_q51 = @@repack_q51_gen, diff --git a/modules/dasLLAMA/dasllama/dasllama_ple.das b/modules/dasLLAMA/dasllama/dasllama_ple.das index 4326c00d79..bd75a4f601 100644 --- a/modules/dasLLAMA/dasllama/dasllama_ple.das +++ b/modules/dasLLAMA/dasllama/dasllama_ple.das @@ -68,6 +68,8 @@ def ple_check_table(t : Model; origin : string) { have = (long_length(t.k3q) / K3_QSB) * 256l } elif (t.ple_emb_fmt == KqFmt.iq3s) { have = (long_length(t.iq3sq) / IQ3S_QSB) * 256l + } elif (t.ple_emb_fmt == KqFmt.iq3xxs) { + have = (long_length(t.iq3xxsq) / IQ3XXS_QSB) * 256l } if (have < need) { panic("dasLLAMA: the PLE token table is not resident ({origin}): fmt {t.ple_emb_fmt} plane holds {have} of {need} elements - regenerate this carrier (it predates the pinned-table rail)") @@ -107,6 +109,8 @@ def private ple_gather_row(t : Model; row : int64; var dst : array; doff dequant_k3_plane_superblock(t.k3q, sb * K3_QSB, t.k3s, sb * K3_SSB, dst, doff + blk * 256l) } elif (t.ple_emb_fmt == KqFmt.iq3s) { dequant_iq3s_plane_superblock(t.iq3sq, sb * IQ3S_QSB, t.iq3ss, sb * IQ3S_SSB, dst, doff + blk * 256l) + } elif (t.ple_emb_fmt == KqFmt.iq3xxs) { + dequant_iq3xxs_plane_superblock(t.iq3xxsq, sb * IQ3XXS_QSB, t.iq3xxss, sb * IQ3XXS_SSB, dst, doff + blk * 256l) } else { panic("ple_gather_row: '{t.ple_emb_fmt}' has no kq plane pair") } diff --git a/modules/dasLLAMA/dasllama/dasllama_repack.das b/modules/dasLLAMA/dasllama/dasllama_repack.das index fddd374773..ed34335eec 100644 --- a/modules/dasLLAMA/dasllama/dasllama_repack.das +++ b/modules/dasLLAMA/dasllama/dasllama_repack.das @@ -561,11 +561,105 @@ def unpack_iq3s_panel_grp(kqg : uint8 const?; var dst : uint8?; mr, nsb : int64) } } +//! iq3xxs grp layout: the whole 96B row as 24 four-byte columns x mr rows — qs and aux move +//! together; the scale row keeps the iq3s interleave (d pairs, pads, strips, pads). +def repack_iq3xxs_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { + let nsb = n / 256l + let qrow = nsb * 96l + let srow = nsb * 20l + let ng = d / mr + var tq : array + var ts : array + tq |> resize(d * qrow) + ts |> resize(d * srow) + unsafe { + var tqp = addr(tq[0]) + var tsp = addr(ts[0]) + memcpy(tqp, kq, d * qrow) + memcpy(tsp, ks, d * srow) + for (g in range64(ng)) { + for (sbi in range64(nsb)) { + let dq = g * mr * qrow + sbi * 96l * mr + let ds = g * mr * srow + sbi * 20l * mr + for (r in range64(mr)) { + let sq = (g * mr + r) * qrow + sbi * 96l + let ss = (g * mr + r) * srow + sbi * 20l + for (c in range64(24l)) { + for (t in range64(4l)) { + kq[dq + (c * mr + r) * 4l + t] = tqp[sq + c * 4l + t] + } + } + ks[ds + 2l * r] = tsp[ss] + ks[ds + 2l * r + 1l] = tsp[ss + 1l] + ks[ds + 2l * mr + 2l * r] = tsp[ss + 2l] + ks[ds + 2l * mr + 2l * r + 1l] = tsp[ss + 3l] + for (b in range64(8l)) { + ks[ds + 4l * mr + b * mr + r] = tsp[ss + 4l + b] + ks[ds + 12l * mr + b * mr + r] = tsp[ss + 12l + b] + } + } + } + } + } + delete tq + delete ts +} + +//! iq3xxs panel unpack: the halved-grid gather + ksigns sign apply, packed grp planes -> the +//! byte-expanded signed panel (the iq3s panel form — the shared fmt-33/34 tile reads it). +def unpack_iq3xxs_panel_grp(kqg : uint8 const?; var dst : uint8?; mr, nsb : int64) { + let grid = iq3xxs_grid() + let ksg = ksigns_iq2xs() + let smask = fixed_array(0x00000000u, 0x000000FFu, 0x0000FF00u, 0x0000FFFFu, + 0x00FF0000u, 0x00FF00FFu, 0x00FFFF00u, 0x00FFFFFFu, + 0xFF000000u, 0xFF0000FFu, 0xFF00FF00u, 0xFF00FFFFu, + 0xFFFF0000u, 0xFFFF00FFu, 0xFFFFFF00u, 0xFFFFFFFFu) + unsafe { + for (sbi in range64(nsb)) { + let qb = sbi * 96l * mr + let ob = sbi * 256l * mr + for (r in range64(mr)) { + for (blk in range64(8l)) { + var aux = 0u + for (t in range64(4l)) { + let ap = 64l + blk * 4l + t + aux |= uint(kqg[qb + ((ap / 4l) * mr + r) * 4l + ap % 4l]) << uint(8l * t) + } + for (l in range64(4l)) { + let sgn = int(ksg[int((aux >> uint(7l * l)) & 127u)]) + let q1p = blk * 8l + 2l * l + let q2p = q1p + 1l + let i1 = int(kqg[qb + ((q1p / 4l) * mr + r) * 4l + q1p % 4l]) + let i2 = int(kqg[qb + ((q2p / 4l) * mr + r) * 4l + q2p % 4l]) + let m1 = smask[sgn & 15] + let m2 = smask[(sgn >> 4) & 15] + let w1 = (grid[i1] ^ m1) + (m1 & 0x01010101u) + let w2 = (grid[i2] ^ m2) + (m2 & 0x01010101u) + // element e = l*8 + half*4 + t: e < 16 lands in the lo panel half, else hi + let e1 = l * 8l + let e2 = l * 8l + 4l + let d1 = (e1 < 16l ? 0l : 128l * mr) + ((blk * 4l + (e1 % 16l) / 4l) * mr + r) * 4l + let d2 = (e2 < 16l ? 0l : 128l * mr) + ((blk * 4l + (e2 % 16l) / 4l) * mr + r) * 4l + var wp1 = reinterpret(dst + (ob + d1)) + var wp2 = reinterpret(dst + (ob + d2)) + wp1[0] = w1 + wp2[0] = w2 + } + } + } + } + } +} + def unpack_kq_panel_grp(fmt : int64; kqg : uint8 const?; var dst : uint8?; mr, nsb : int64) { if (fmt == 33l) { unpack_iq3s_panel_grp(kqg, dst, mr, nsb) return } + if (fmt == 34l) { + unpack_iq3xxs_panel_grp(kqg, dst, mr, nsb) + return + } let LO4 = 0x0F0F0F0F0F0F0F0Ful let SEL = 0x0804020108040201ul // LE lane t (0..3, twice) selects high-bit t let C7F = 0x7F7F7F7F7F7F7F7Ful diff --git a/modules/dasLLAMA/followup_vulkan.md b/modules/dasLLAMA/followup_vulkan.md index 6f5884ef4b..b90a3563f5 100644 --- a/modules/dasLLAMA/followup_vulkan.md +++ b/modules/dasLLAMA/followup_vulkan.md @@ -536,3 +536,11 @@ module) is independent and can land any time - it is pure structure. (`keep_hidden`), plus an override-capability row so a whole-plane consumer can test for it; a cell that embeds through the vulkan override and compares against the CPU pool proves it. + +34. **END-OF-ARC: the pp512 tier class (~0.67-0.70x of llama.cpp on 1B shapes).** Every + sb-format cm2 tile lands in the same band (k4 control 0.67x, iq3s 0.70x on the 5060 Ti) + while tg and the CPU tiers hold parity or better - the gap is the shared batch-GEMM + tier, not any one format's decode. Boris 2026-08-30: this one bothers him at 0.7 - + schedule a dedicated pass at the END of the iquant-formats arc (after the last format + lands), not per-format. Start from the followup 29-32 streamed-layer levers and a + kernel-level probe of the cm2 tile vs llama.cpp's mul_mm_cm2 at matched shapes. diff --git a/modules/dasLLAMA/harness/gen_tune_probe.das b/modules/dasLLAMA/harness/gen_tune_probe.das index 171f92ebf4..49c0fa54ba 100644 --- a/modules/dasLLAMA/harness/gen_tune_probe.das +++ b/modules/dasLLAMA/harness/gen_tune_probe.das @@ -441,7 +441,7 @@ def kq_qsb(fmt : int64) : int64 => dasllama_gemm_schema::kq_qsb(int(fmt)) def kq_ssb(fmt : int64) : int64 => dasllama_gemm_schema::kq_ssb(int(fmt)) // the family's sidecar entry = its [tune] function name (the K-quants by bit width - k3 included - q40/iq4xs by tag) -def kq_tile_entry(fmt : int64) : string => fmt == 40l ? "q40q8_tile_gen" : (fmt == 44l ? "iq4xsq8_tile_gen" : (fmt == 33l ? "iq3sq8_tile_gen" : "k{fmt}q8_tile_gen")) +def kq_tile_entry(fmt : int64) : string => fmt == 40l ? "q40q8_tile_gen" : (fmt == 44l ? "iq4xsq8_tile_gen" : (fmt == 33l ? "iq3sq8_tile_gen" : (fmt == 34l ? "iq3xxsq8_tile_gen" : "k{fmt}q8_tile_gen"))) // synthetic disk superblocks, varied per (row, superblock) so no two blocks repeat: any byte // is a legal quant, and the small d/dmin keep the f16 (s, o) pairs finite and row sums tame @@ -463,7 +463,7 @@ def pack_kq_scale_header(var blkb : array; base : int) { } } -def pack_kq_disk_block(fmt : int64; var blkb : array; base : int) { // nolint:STYLE037 — the flat per-format ladder (one arm per kq format) is the honest shape +def pack_kq_disk_block(fmt : int64; var blkb : array; base : int) { // nolint:STYLE037,STYLE038 — the flat per-format ladder (one arm per kq format) is the honest shape if (fmt == 40l) { for (blk in range(8)) { // 8 x 18B q4_0 disk blocks: f16 d + 16 nibble bytes let dbits = f32_to_f16(0.002 + 0.0001 * float((base + blk) % 13)) @@ -508,6 +508,14 @@ def pack_kq_disk_block(fmt : int64; var blkb : array; base : int) { // for (i in range(4)) { blkb[106 + i] = uint8((base * 29 + i * 71 + 5) % 256) } + } elif (fmt == 34l) { + // 98B IQ3_XXS disk block: f16 d, 64 grid-index qs, 8 x aux32 — every byte pattern is legal + let dbits = f32_to_f16(0.002 + 0.0001 * float(base % 13)) + blkb[0] = uint8(dbits & 0xFF) + blkb[1] = uint8(dbits >> 8u) + for (i in range(96)) { + blkb[2 + i] = uint8((base * 37 + i * 101 + 29) % 256) + } } elif (fmt == 4l) { pack_kq_scale_header(blkb, base) for (i in range(128)) { @@ -550,19 +558,21 @@ def repack_kq_grp_fmt(fmt : int64; var kq : uint8?; var ks : uint8?; n, d, mr : repack_k3_grp(kq, ks, n, d, mr) } elif (fmt == 33l) { repack_iq3s_grp(kq, ks, n, d, mr) + } elif (fmt == 34l) { + repack_iq3xxs_grp(kq, ks, n, d, mr) } else { repack_k6_grp(kq, ks, n, d, mr) } } -def build_kq_fixture(fmt : int64; n, d, ntok : int64) : KqFixture { // nolint:STYLE037 — the flat per-format ladder (one arm per kq format) is the honest shape +def build_kq_fixture(fmt : int64; n, d, ntok : int64) : KqFixture { // nolint:STYLE037,STYLE038 — the flat per-format ladder (one arm per kq format) is the honest shape var fx = KqFixture(fmt = fmt, n = n, d = d, ntok = ntok, nsb = n / 256l) let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) fx.kq |> resize(d * fx.nsb * qsb) fx.ks |> resize(d * fx.nsb * ssb) var blkb : array - blkb |> resize(fmt == 4l || fmt == 40l ? 144l : (fmt == 5l ? 176l : (fmt == 44l ? 136l : (fmt == 3l || fmt == 33l ? 110l : 210l)))) + blkb |> resize(fmt == 4l || fmt == 40l ? 144l : (fmt == 5l ? 176l : (fmt == 44l ? 136l : (fmt == 3l || fmt == 33l ? 110l : (fmt == 34l ? 98l : 210l))))) for (r in range64(d)) { for (s in range64(fx.nsb)) { let base = int(r * 31l + s * 7l) @@ -580,6 +590,8 @@ def build_kq_fixture(fmt : int64; n, d, ntok : int64) : KqFixture { // nolint: transcode_q3k_superblock(blkb, 0l, fx.kq, sb * qsb, fx.ks, sb * ssb) } elif (fmt == 33l) { transcode_iq3s_superblock(blkb, 0l, fx.kq, sb * qsb, fx.ks, sb * ssb) + } elif (fmt == 34l) { + transcode_iq3xxs_superblock(blkb, 0l, fx.kq, sb * qsb, fx.ks, sb * ssb) } else { transcode_q6k_superblock(blkb, 0l, fx.kq, sb * qsb, fx.ks, sb * ssb) } @@ -623,8 +635,11 @@ def build_kq_fixture(fmt : int64; n, d, ntok : int64) : KqFixture { // nolint: : (fmt == 33l ? iq3s_grp_row_dot(kqg, ksg, r, 4l, addr(fx.xq[0]) + tk * n, addr(fx.xs[0]) + tk * (n / 256l), addr(fx.xbs[0]) + tk * (n / 16l), n) - : kq_grp_row_dot(fmt, kqg, ksg, r, 4l, - addr(fx.xq[0]) + tk * n, addr(fx.xs[0]) + tk * (n / 256l), addr(fx.xbs[0]) + tk * (n / 16l), n))))) + : (fmt == 34l + ? iq3xxs_grp_row_dot(kqg, ksg, r, 4l, + addr(fx.xq[0]) + tk * n, addr(fx.xs[0]) + tk * (n / 256l), addr(fx.xbs[0]) + tk * (n / 16l), n) + : kq_grp_row_dot(fmt, kqg, ksg, r, 4l, + addr(fx.xq[0]) + tk * n, addr(fx.xs[0]) + tk * (n / 256l), addr(fx.xbs[0]) + tk * (n / 16l), n)))))) } } } @@ -655,6 +670,9 @@ def kq_tile_variants(fmt : int64) : array> { if (fmt == 33l) { return <- iq3sq8_tile_gen_variants() } + if (fmt == 34l) { + return <- iq3xxsq8_tile_gen_variants() + } return <- k6q8_tile_gen_variants() } @@ -696,6 +714,12 @@ def kq_gemv_variants_by_suffix(fmt : int64) : table { tab[v._0] = v._1 } delete gvs + } elif (fmt == 34l) { + var gvs <- iq3xxsq8_gemv_gen_variants() + for (v in gvs) { + tab[v._0] = v._1 + } + delete gvs } else { var gvs <- k6q8_gemv_gen_variants() for (v in gvs) { @@ -746,6 +770,12 @@ def kq_layout_mrs(fmt : int64) : table { mrs[v._0] = invoke(v._1) } delete lvs + } elif (fmt == 34l) { + var lvs <- iq3xxsq8_layout_gen_variants() + for (v in lvs) { + mrs[v._0] = invoke(v._1) + } + delete lvs } else { var lvs <- k6q8_layout_gen_variants() for (v in lvs) { diff --git a/modules/dasLLAMA/tests/test_kqformat.das b/modules/dasLLAMA/tests/test_kqformat.das index 3fa38d5f70..2ef9e25e2a 100644 --- a/modules/dasLLAMA/tests/test_kqformat.das +++ b/modules/dasLLAMA/tests/test_kqformat.das @@ -23,6 +23,7 @@ def test_kqfmt_enum_values(t : T?) { t |> equal(int(KqFmt.iq4xs), 6) t |> equal(int(KqFmt.k3), 7) t |> equal(int(KqFmt.iq3s), 8) + t |> equal(int(KqFmt.iq3xxs), 9) } } @@ -40,7 +41,9 @@ def test_kqfmt_predicates(t : T?) { t |> success(kq_sb(KqFmt.k3), "k3 is superblock") t |> success(kq_sb(int(KqFmt.k3)), "the int-id predicate agrees for k3") t |> success(kq_sb(KqFmt.iq3s), "iq3s is superblock") + t |> success(kq_sb(KqFmt.iq3xxs), "iq3xxs is superblock") t |> success(kq_sb(int(KqFmt.iq3s)), "the int-id predicate agrees for iq3s") + t |> success(kq_sb(int(KqFmt.iq3xxs)), "the int-id predicate agrees for iq3xxs") } } @@ -62,7 +65,9 @@ def test_kqfmt_strides(t : T?) { t |> equal(kq_qsb(KqFmt.k3), 96l) // 64 qs + 32 hmask, verbatim t |> equal(kq_ssb(KqFmt.k3), 18l) // the k6 row: 16 int8 sub-scales + f16 d t |> equal(kq_qsb(KqFmt.iq3s), 104l) // [64 qs][8 qh][32 signs], verbatim + t |> equal(kq_qsb(KqFmt.iq3xxs), 96l) // [64 qs][32 aux], verbatim t |> equal(kq_ssb(KqFmt.iq3s), 20l) // the k4 row: f16 d + 8 int8 (1 + 2s) + t |> equal(kq_ssb(KqFmt.iq3xxs), 20l) // the same row: f16 d halved + 8 x (2ls + 1) } t |> run("named constants agree with the literals") @(t : T?) { t |> equal(K4_QSB, 128l) @@ -114,9 +119,10 @@ def test_kqfmt_id_bridge(t : T?) { t |> equal(kq_schema_id(KqFmt.iq4xs), 44) t |> equal(kq_schema_id(KqFmt.k3), 3) t |> equal(kq_schema_id(KqFmt.iq3s), 33) + t |> equal(kq_schema_id(KqFmt.iq3xxs), 34) } t |> run("gemm_schema's int-id forms delegate to the same truth") @(t : T?) { - for (f in [KqFmt.k4, KqFmt.k5, KqFmt.k6, KqFmt.q40, KqFmt.iq4xs, KqFmt.k3, KqFmt.iq3s]) { + for (f in [KqFmt.k4, KqFmt.k5, KqFmt.k6, KqFmt.q40, KqFmt.iq4xs, KqFmt.k3, KqFmt.iq3s, KqFmt.iq3xxs]) { t |> equal(kq_qsb(kq_schema_id(f)), kq_qsb(f)) t |> equal(kq_ssb(kq_schema_id(f)), kq_ssb(f)) } @@ -133,6 +139,7 @@ def test_kqfmt_elems_and_stream(t : T?) { t |> equal(kq_elems(KqFmt.iq4xs), 256l) t |> equal(kq_elems(KqFmt.k3), 256l) t |> equal(kq_elems(KqFmt.iq3s), 256l) + t |> equal(kq_elems(KqFmt.iq3xxs), 256l) t |> equal(kq_elems(KqFmt.q8), 32l) t |> equal(kq_elems(KqFmt.q51), 32l) } @@ -146,5 +153,35 @@ def test_kqfmt_elems_and_stream(t : T?) { t |> equal(kq_stream_code(KqFmt.iq4xs), 44) t |> equal(kq_stream_code(KqFmt.k3), 3) t |> equal(kq_stream_code(KqFmt.iq3s), 33) + t |> equal(kq_stream_code(KqFmt.iq3xxs), 34) + } +} + +[test] +def test_iq3xxs_tables(t : T?) { + t |> run("the halved IQ3_XXS grid, ksigns and the f16 halver hold their contracts") @(t : T?) { + let lut = iq3xxs_grid() + for (i in range(256)) { + t |> equal(IQ3XXS_GRID[i], lut[i]) + for (b in range(4)) { + let v = int((lut[i] >> uint(8 * b)) & 0xFFu) + // ggml levels 4..62 halved: even 2..30 plus 31 (the 62 top level) + t |> success(v == 31 || (v >= 2 && v <= 30 && v % 2 == 0), "halved grid byte in range") + } + } + let ks = ksigns_iq2xs() + for (i in range(128)) { + t |> equal(int(KSIGNS_IQ2XS[i]), int(ks[i])) + var parity = 0 + for (b in range(7)) { + parity ^= (i >> b) & 1 + } + t |> equal(int(ks[i]), i | (parity << 7)) // bit 7 = even parity of the 7-bit index + } + // exact halving: a normal f16 halves by exponent decrement (0x3400 = 0.25 -> 0x3000 = + // 0.125), the sign rides along, and the exp==1 edge shifts into the subnormal space + t |> equal(f16_half_bits(0x3400u), 0x3000u) + t |> equal(f16_half_bits(0xB400u), 0xB000u) + t |> equal(f16_half_bits(0x0400u), 0x0200u) } } diff --git a/modules/dasLLAMA/tests/test_kquant.das b/modules/dasLLAMA/tests/test_kquant.das index 4ba8a42a63..1d55b5841f 100644 --- a/modules/dasLLAMA/tests/test_kquant.das +++ b/modules/dasLLAMA/tests/test_kquant.das @@ -105,7 +105,26 @@ def private build_q6k_block() : array { // loop-generated test labels: "k4"/"k5"/"k6" for the K-quants, "q40" for fmt 40, "iq4xs" for // fmt 44 — matches the kernel/repack function names (dot_q40q8, repack_q40_grp) so failures grep 1:1 -def private kq_tag(fmt : int) : string => fmt == 40 ? "q40" : (fmt == 44 ? "iq4xs" : (fmt == 33 ? "iq3s" : "k{fmt}")) +// iq3xxs synthetic superblock: 98B IQ3_XXS disk block — f16 d, 64 grid-index bytes, 8 x aux32 +// (4x7-bit ksigns indices + a 4-bit scale in the top nibble). Any byte pattern is legal. +def private iq3xxs_qs_pat(i : int) : int => (i * 13 + 5) % 256 +def private iq3xxs_aux_pat(p : int) : int => (p * 9 + 17) % 256 +def private build_iq3xxs_block() : array { + var blkb : array + blkb |> resize(98) + let dbits = f32_to_f16(0.002) + blkb[0] = uint8(dbits & 0xFF) + blkb[1] = uint8(dbits >> 8u) + for (i in range(64)) { + blkb[2 + i] = uint8(iq3xxs_qs_pat(i)) + } + for (p in range(32)) { + blkb[66 + p] = uint8(iq3xxs_aux_pat(p)) + } + return <- blkb +} + +def private kq_tag(fmt : int) : string => fmt == 40 ? "q40" : (fmt == 44 ? "iq4xs" : (fmt == 33 ? "iq3s" : (fmt == 34 ? "iq3xxs" : "k{fmt}"))) // iq4xs synthetic superblock: 136B disk block — f16 d = SYNTH_D, the 8 six-bit sub-scales are // SYNTH_SC (packed low nibbles into scales_l, top two bits into scales_h), the shared nibble @@ -664,7 +683,7 @@ def test_q51_kernels(t : T?) { // elementwise dot over the plane dequant (the kernels' declared oracle). def private kq_dot_gate(t : T?; fmt : int; n : int64 = 512l) { let nsb = n / 256l - let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : build_q6k_block()))))) + let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : build_q6k_block())))))) let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) var kq : array @@ -684,6 +703,8 @@ def private kq_dot_gate(t : T?; fmt : int; n : int64 = 512l) { transcode_q3k_superblock(blk1, 0l, kq, s * qsb, ks, s * ssb) } elif (fmt == 33) { transcode_iq3s_superblock(blk1, 0l, kq, s * qsb, ks, s * ssb) + } elif (fmt == 34) { + transcode_iq3xxs_superblock(blk1, 0l, kq, s * qsb, ks, s * ssb) } else { transcode_q6k_superblock(blk1, 0l, kq, s * qsb, ks, s * ssb) } @@ -709,6 +730,8 @@ def private kq_dot_gate(t : T?; fmt : int; n : int64 = 512l) { dequant_k3_plane_superblock(kq, s * qsb, ks, s * ssb, w, s * 256l) } elif (fmt == 33) { dequant_iq3s_plane_superblock(kq, s * qsb, ks, s * ssb, w, s * 256l) + } elif (fmt == 34) { + dequant_iq3xxs_plane_superblock(kq, s * qsb, ks, s * ssb, w, s * 256l) } else { dequant_k6_plane_superblock(kq, s * qsb, ks, s * ssb, w, s * 256l) } @@ -746,6 +769,8 @@ def private kq_dot_gate(t : T?; fmt : int; n : int64 = 512l) { got = dot_k3q8(addr(kq[0]), addr(ks[0]), addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } elif (fmt == 33) { got = dot_iq3s_q8(addr(kq[0]), addr(ks[0]), addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + } elif (fmt == 34) { + got = dot_iq3xxs_q8(addr(kq[0]), addr(ks[0]), addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } else { got = dot_k6q8(addr(kq[0]), addr(ks[0]), addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } @@ -779,7 +804,7 @@ def test_kq_dots(t : T?) { } // 122B-class row widths: n=3072 (12 superblocks — qwen35moe-122B gate/up/cls) and n=1024 // (its k5 down_exps). Every earlier kq model kept n at 512..2560. - for (fmt in [4, 5, 6, 40, 44, 3, 33]) { + for (fmt in [4, 5, 6, 40, 44, 3, 33, 34]) { for (nn in [1024l, 3072l]) { t |> run("dot_{kq_tag(fmt)}q8 matches the fp64 reference at n={int(nn)}") @(t : T?) { kq_dot_gate(t, fmt, nn) @@ -798,7 +823,7 @@ def private kq_gemv_rows_gate(t : T?; fmt : int; n : int64 = 512l) { let nsb = n / 256l let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) - let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : build_q6k_block()))))) + let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : build_q6k_block())))))) var kq : array var ks : array kq |> resize(d * nsb * qsb) @@ -817,6 +842,8 @@ def private kq_gemv_rows_gate(t : T?; fmt : int; n : int64 = 512l) { transcode_q3k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } elif (fmt == 33) { transcode_iq3s_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 34) { + transcode_iq3xxs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } else { transcode_q6k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } @@ -861,6 +888,8 @@ def private kq_gemv_rows_gate(t : T?; fmt : int; n : int64 = 512l) { want = dot_k3q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } elif (fmt == 33) { want = dot_iq3s_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + } elif (fmt == 34) { + want = dot_iq3xxs_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } else { want = dot_k6q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } @@ -902,7 +931,7 @@ def test_kq_gemv_rows(t : T?) { t |> run("portable iq3s GEMV rows bit-match per-row disk dots") @(t : T?) { kq_gemv_rows_gate(t, 33) } - for (fmt in [4, 5, 6, 40, 44, 3, 33]) { + for (fmt in [4, 5, 6, 40, 44, 3, 33, 34]) { for (nn in [1024l, 3072l]) { t |> run("portable {kq_tag(fmt)} GEMV rows bit-match per-row disk dots at n={int(nn)}") @(t : T?) { kq_gemv_rows_gate(t, fmt, nn) @@ -922,7 +951,7 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { let nsb = n / 256l let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) - let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : build_q6k_block()))))) + let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : build_q6k_block())))))) var kq : array var ks : array kq |> resize(d * nsb * qsb) @@ -941,6 +970,8 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { transcode_q3k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } elif (fmt == 33) { transcode_iq3s_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 34) { + transcode_iq3xxs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } else { transcode_q6k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } @@ -984,6 +1015,8 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { want[r] = dot_k3q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } elif (fmt == 33) { want[r] = dot_iq3s_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + } elif (fmt == 34) { + want[r] = dot_iq3xxs_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } else { want[r] = dot_k6q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } @@ -1000,6 +1033,8 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { dequant_k3_plane_superblock(kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb, wrow, r * n + s * 256l) } elif (fmt == 33) { dequant_iq3s_plane_superblock(kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb, wrow, r * n + s * 256l) + } elif (fmt == 34) { + dequant_iq3xxs_plane_superblock(kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb, wrow, r * n + s * 256l) } else { dequant_k6_plane_superblock(kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb, wrow, r * n + s * 256l) } @@ -1017,6 +1052,8 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { repack_k3_grp(addr(kq[0]), addr(ks[0]), n, d, mr) } elif (fmt == 33) { repack_iq3s_grp(addr(kq[0]), addr(ks[0]), n, d, mr) + } elif (fmt == 34) { + repack_iq3xxs_grp(addr(kq[0]), addr(ks[0]), n, d, mr) } else { repack_k6_grp(addr(kq[0]), addr(ks[0]), n, d, mr) } @@ -1034,8 +1071,8 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { ? iq4xs_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) : (fmt == 3 ? k3_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) - : (fmt == 33 - ? iq3s_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + : (fmt == 33 || fmt == 34 + ? (fmt == 33 ? iq3s_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) : iq3xxs_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n)) : kq_grp_row_dot(int64(fmt), kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n))))) if (got != want[r]) { dotbad++ @@ -1063,7 +1100,7 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { [test] def test_kq_repack_grp(t : T?) { - for (fmt in [4, 5, 6, 40, 44, 3, 33]) { + for (fmt in [4, 5, 6, 40, 44, 3, 33, 34]) { for (mr in [4l, 8l, 16l]) { t |> run("repack_{kq_tag(fmt)}_grp mr={mr} preserves dots and row dequants") @(t : T?) { kq_repack_gate(t, fmt, mr) @@ -1081,14 +1118,14 @@ def test_kq_repack_grp(t : T?) { // repacked planes (the tile's per-token fold order IS the GEMV's). Repack at the layout // companion's mr — the same value the stubs/kernels read, so this holds on the reference // bodies (interp) AND the stamped kernels (-jit) alike. -def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { +def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { // nolint:STYLE037 — the flat per-format ladder (one arm per kq format) is the honest shape let d = 32l let ntok = 7l // 4-token tile + a 3-token gemv tail let nsb = n / 256l let mr = kq_layout_of(fmt) // the format's OWN layout companion (per-format tune families) let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) - let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : build_q6k_block()))))) + let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : build_q6k_block())))))) var kq : array var ks : array kq |> resize(d * nsb * qsb) @@ -1107,6 +1144,8 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { transcode_q3k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } elif (fmt == 33) { transcode_iq3s_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 34) { + transcode_iq3xxs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } else { transcode_q6k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } @@ -1130,6 +1169,8 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { repack_k3_grp(addr(kq[0]), addr(ks[0]), n, d, mr) } elif (fmt == 33) { repack_iq3s_grp(addr(kq[0]), addr(ks[0]), n, d, mr) + } elif (fmt == 34) { + repack_iq3xxs_grp(addr(kq[0]), addr(ks[0]), n, d, mr) } else { repack_k6_grp(addr(kq[0]), addr(ks[0]), n, d, mr) } @@ -1158,7 +1199,7 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { // k5/k6 tiles read the byte-expanded panel the batch cell unpacks per (group, token-block) // — the gate covers unpack_kq_panel_grp too (tile-over-panel vs gemv-over-packed planes); // k4/q40 tiles read the packed planes directly - let packed = fmt == 4 || fmt == 40 || fmt == 44 || fmt == 3 || fmt == 33 + let packed = fmt == 4 || fmt == 40 || fmt == 44 || fmt == 3 // iq3s/iq3xxs tiles read the byte-expanded panel (the batch cell's shape) var panel : array if (!packed) { panel |> resize(mr * nsb * 256l) @@ -1184,7 +1225,9 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { } elif (fmt == 3) { k3q8_tile_gen(addr(ytile[0]), kqg, ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) } elif (fmt == 33) { - iq3sq8_tile_gen(addr(ytile[0]), kqg, ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) + iq3sq8_tile_gen(addr(ytile[0]), addr(panel[0]), ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) + } elif (fmt == 34) { + iq3xxsq8_tile_gen(addr(ytile[0]), addr(panel[0]), ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) } else { k6q8_tile_gen(addr(ytile[0]), addr(panel[0]), ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) } @@ -1203,6 +1246,8 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { k3q8_gemv_gen(addr(ytile[tk * d]), addr(kq[0]), addr(ks[0]), addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, g * mr, (g + 1l) * mr) } elif (fmt == 33) { iq3sq8_gemv_gen(addr(ytile[tk * d]), addr(kq[0]), addr(ks[0]), addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, g * mr, (g + 1l) * mr) + } elif (fmt == 34) { + iq3xxsq8_gemv_gen(addr(ytile[tk * d]), addr(kq[0]), addr(ks[0]), addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, g * mr, (g + 1l) * mr) } else { k6q8_gemv_gen(addr(ytile[tk * d]), addr(kq[0]), addr(ks[0]), addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, g * mr, (g + 1l) * mr) } @@ -1222,6 +1267,8 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { k3q8_gemv_gen(addr(ygemv[p * d]), addr(kq[0]), addr(ks[0]), addr(xq[p * n]), addr(xs[p * (n / 256l)]), addr(xbs[p * (n / 16l)]), n, 0l, d) } elif (fmt == 33) { iq3sq8_gemv_gen(addr(ygemv[p * d]), addr(kq[0]), addr(ks[0]), addr(xq[p * n]), addr(xs[p * (n / 256l)]), addr(xbs[p * (n / 16l)]), n, 0l, d) + } elif (fmt == 34) { + iq3xxsq8_gemv_gen(addr(ygemv[p * d]), addr(kq[0]), addr(ks[0]), addr(xq[p * n]), addr(xs[p * (n / 256l)]), addr(xbs[p * (n / 16l)]), n, 0l, d) } else { k6q8_gemv_gen(addr(ygemv[p * d]), addr(kq[0]), addr(ks[0]), addr(xq[p * n]), addr(xs[p * (n / 256l)]), addr(xbs[p * (n / 16l)]), n, 0l, d) } @@ -1247,7 +1294,7 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { [test] def test_kq_tile(t : T?) { - for (fmt in [4, 5, 6, 40, 44, 3, 33]) { + for (fmt in [4, 5, 6, 40, 44, 3, 33, 34]) { t |> run("{kq_tag(fmt)} 4-token tile bit-matches per-token GEMVs") @(t : T?) { kq_tile_gate(t, fmt) } @@ -1264,11 +1311,11 @@ def test_kq_tile(t : T?) { // per-region grp slices, the per-expert-slice repack shape). Regions carry distinct LCG // payloads; both offs forms run — the shared activation image (gate/up, xoff 0) and // per-region activation rows (down, xoff = r*n). -def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; nreg : int64 = 3l) { +def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; nreg : int64 = 3l) { // nolint:STYLE037 — the flat per-format ladder (one arm per kq format) is the honest shape let nsb = n / 256l let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) - let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : build_q6k_block()))))) + let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : build_q6k_block())))))) var kq : array var ks : array kq |> resize(nreg * d * nsb * qsb) @@ -1287,6 +1334,8 @@ def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; transcode_q3k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } elif (fmt == 33) { transcode_iq3s_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 34) { + transcode_iq3xxs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } else { transcode_q6k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } @@ -1353,6 +1402,9 @@ def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; } elif (fmt == 33) { want_sh[r * d + row] = dot_iq3s_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) want_pr[r * d + row] = dot_iq3s_q8(kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n) + } elif (fmt == 34) { + want_sh[r * d + row] = dot_iq3xxs_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + want_pr[r * d + row] = dot_iq3xxs_q8(kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n) } else { want_sh[r * d + row] = dot_k6q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) want_pr[r * d + row] = dot_k6q8(kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n) @@ -1401,6 +1453,8 @@ def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; repack_k3_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) } elif (fmt == 33) { repack_iq3s_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) + } elif (fmt == 34) { + repack_iq3xxs_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) } else { repack_k6_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) } @@ -1426,6 +1480,9 @@ def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; } elif (fmt == 33) { iq3sq8_gemv_gen(addr(want_sh[r * d]), kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, 0l, d) iq3sq8_gemv_gen(addr(want_pr[r * d]), kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n, 0l, d) + } elif (fmt == 34) { + iq3xxsq8_gemv_gen(addr(want_sh[r * d]), kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, 0l, d) + iq3xxsq8_gemv_gen(addr(want_pr[r * d]), kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n, 0l, d) } else { k6q8_gemv_gen(addr(want_sh[r * d]), kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, 0l, d) k6q8_gemv_gen(addr(want_pr[r * d]), kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n, 0l, d) @@ -1461,7 +1518,7 @@ def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; [test] def test_kq_groupn(t : T?) { - for (fmt in [4, 5, 6, 40, 44, 3, 33]) { + for (fmt in [4, 5, 6, 40, 44, 3, 33, 34]) { t |> run("{kq_tag(fmt)} region-list GEMV bit-matches per-row dots (disk + grp slices)") @(t : T?) { kq_groupn_gate(t, fmt) } @@ -1505,7 +1562,7 @@ def private kq_batch_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 for (c in cnts) { nk += c } - let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : build_q6k_block()))))) + let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : build_q6k_block())))))) var kq : array var ks : array kq |> resize(nreg * d * nsb * qsb) @@ -1524,6 +1581,8 @@ def private kq_batch_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 transcode_q3k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } elif (fmt == 33) { transcode_iq3s_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 34) { + transcode_iq3xxs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } else { transcode_q6k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } @@ -1548,6 +1607,8 @@ def private kq_batch_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 repack_k3_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) } elif (fmt == 33) { repack_iq3s_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) + } elif (fmt == 34) { + repack_iq3xxs_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) } else { repack_k6_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) } @@ -1598,6 +1659,8 @@ def private kq_batch_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 k3q8_gemv_gen(addr(want[tk * d]), kqr, ksr, addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, 0l, d) } elif (fmt == 33) { iq3sq8_gemv_gen(addr(want[tk * d]), kqr, ksr, addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, 0l, d) + } elif (fmt == 34) { + iq3xxsq8_gemv_gen(addr(want[tk * d]), kqr, ksr, addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, 0l, d) } else { k6q8_gemv_gen(addr(want[tk * d]), kqr, ksr, addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, 0l, d) } @@ -1629,7 +1692,7 @@ def private kq_batch_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 [test] def test_kq_batch_groupn(t : T?) { - for (fmt in [4, 5, 6, 40, 44, 3, 33]) { + for (fmt in [4, 5, 6, 40, 44, 3, 33, 34]) { t |> run("{kq_tag(fmt)} batch groupn bit-matches per-token rows-core calls") @(t : T?) { kq_batch_groupn_gate(t, fmt) } @@ -1926,3 +1989,41 @@ def private rel_rms(a, b : array) : float { return float(sqrt(num / den)) } + +[test] +def test_iq3xxs_planes(t : T?) { + t |> run("iq3xxs planes fold the 0.25 into d and the grid, and decode exactly") @(t : T?) { + let blkb <- build_iq3xxs_block() + var kq : array + var ks : array + kq |> resize(96) + ks |> resize(20) + transcode_iq3xxs_superblock(blkb, 0l, kq, 0l, ks, 0l) + var dst : array + dst |> resize(256) + dequant_iq3xxs_plane_superblock(kq, 0l, ks, 0l, dst, 0l) + // independent recomputation off the DISK bytes through the identity tables + let dh = f16_to_f32(f16_half_bits(uint(blkb[0]) | (uint(blkb[1]) << 8u))) + let grid = iq3xxs_grid() + let signs = ksigns_iq2xs() + for (k in range(256)) { + let b = k / 32 + let l = (k % 32) / 8 + let hj = k % 8 + var aux = 0u + for (tt in range(4)) { + aux |= uint(iq3xxs_aux_pat(b * 4 + tt)) << uint(8 * tt) + } + let db = dh * float(1 + 2 * int(aux >> 28u)) + let sgnb = int(signs[int((aux >> uint(7 * l)) & 127u)]) + let g = grid[iq3xxs_qs_pat(b * 8 + 2 * l + hj / 4)] + let mag = float((g >> uint(8 * (hj % 4))) & 0xFFu) + let sgn = ((sgnb >> hj) & 1) != 0 ? -1.0 : 1.0 + let expected = db * mag * sgn + t |> success(dst[k] == expected, "iq3xxs plane element must match the table recomputation exactly") + } + delete kq + delete ks + delete dst + } +} From 0e3a8de5b533f5b5aeec7f02c6bdebe85f20bb24 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Sun, 30 Aug 2026 20:26:15 -0700 Subject: [PATCH 24/61] dasLLAMA: the IQ3_XXS JIT emitter - a gather arm on the shared panel route, gen 2.2x The smallest emitter arm of the arc, because Phase A aligned the panel: the tile costs two widened conditions (emit_one_block and emit_block_iq4xs take te.kq == 33 || 34 - the byte-expanded signed panel's stride, scale row and fold are format-agnostic), and only the gemv gather is new code. emit_iq3xxs_gather is the iq3s gather with the aux32 sign word loaded as ONE aligned i32 per block (the four aux bytes sit contiguous in the grp column layout), sign bytes via a dasllama.iq3xxs.ksigns [128 x i32] global feeding the same smask nibble expansion, and the bare qs byte indexing the [256 x i32] HALVED grid - no 9th bit. iq3xxs_emit_globals also shares dasllama.iq3s.smask. The declined stubs flip to kq_gemv_gen_impl/kq_tile_gen_impl(gc, 34). The probe needed the format in TWO lists - the test-mode fixture families and the tune loop's fixed_array - or the gate stamps the perms and silently gates none of them (the first run's log had zero k34 rows next to "stamping 10 permutations"); HOW_TO's Phase-B note carries the trap. Gates on the zen2: 11/11 k34 perms ok vs the scalar oracle (reference, mr4/8, maddubs, vpdpbusd; maxdiff ~2e-6), GEN TUNE TEST OK; the tuner crowns dot_maddubs_width256_mr8 verdict=beats (the board's usual winner, same as iq3s/k3). E2e on the Qwen2.5-1.5B IQ3_XS vehicle: prefill 6 -> 17 t/s, gen 19 -> 41 t/s vs the reference bodies; 17/64 greedy ids vs llama.cpp with the fork ON the margin oracle's top-2 (0.43 logits at step 17 - the stamped float folds move the near-tie flip point from the reference run's step 22, same benign class). Co-Authored-By: Claude Fable 5 --- modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md | 18 ++- .../dasLLAMA/dasllama/dasllama_gemm_gen.das | 152 ++++++++++++++++-- modules/dasLLAMA/harness/gen_tune_probe.das | 5 +- 3 files changed, 158 insertions(+), 17 deletions(-) diff --git a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md index 0c6fb5976b..29872e9a73 100644 --- a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md +++ b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md @@ -446,7 +446,23 @@ loop - and the tile gate's `packed` list wrongly held 33 (the iq3s tile reads th byte-expanded panel), a pre-existing red on x64 fixed in the same change. End to end (reference bodies - both generators decline until the emitter arc): coherent text, 22/64 greedy ids vs llama.cpp's `simple_ids.exe` where the fork is a 0.12-logit near-tie whose -top-2 IS our token (`simple_ids_margin`), gen 19 t/s. JIT emitter, Vulkan, Metal: pending. +top-2 IS our token (`simple_ids_margin`), gen 19 t/s. + +The JIT emitter followed the same day and is the smallest arm of the arc: the TILE is free +(the panel route widens - `emit_one_block`'s and `emit_block_iq4xs`'s `te.kq == 33` become +`33 || 34`; stride, scale row and fold are already the panel's), and only the gemv gather is +new. `emit_iq3xxs_gather` mirrors the iq3s gather with three changes: the aux32 sign word is +ONE aligned i32 column load per block (the four aux bytes of a block land contiguous in the +grp column layout), sign bytes come from a `dasllama.iq3xxs.ksigns` [128 x i32] global +(then the SAME smask nibble expansion), and the grid index is the bare qs byte into the +[256 x i32] HALVED grid - no 9th bit. Two probe lists must both gain the format or the gate +silently skips it: the test-mode fixture families AND the tune loop's +`fixed_array(4l, 5l, ...)` - the first run stamped 10 perms and gated NONE of them (grep +`k34` in the test log to prove coverage). Gates: 11/11 k34 perms ok vs the oracle (maxdiff +~2e-6); the tuner crowns `dot_maddubs_width256_mr8` verdict=beats (the zen2 board's usual +winner, same as iq3s/k3). E2e stamped: prefill 6 -> 17 t/s, gen 19 -> 41 t/s; ids 17/64 +with the fork again ON the margin oracle's top-2 (0.43 logits at step 17 - stamped float +folds move the flip point, they do not change the class). Vulkan, Metal: pending. ### IQ3_S (the third format - and the first grid format, 2026-08-30) diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das index 3d835e3b55..df00e8976d 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das @@ -245,9 +245,10 @@ struct private TileEmit { abs_decl : LLVMOpaqueValue? // x64 sign trick: llvm.abs (VPABSB) abs_ty : LLVMOpaqueType? psign_decl : LLVMOpaqueValue? // x64 width-256 sign apply (VPSIGNB) - iq3s_panel : LLVMOpaqueValue? // iq3s gemv: the per-superblock alloca panel (null = tile mode, the caller unpacked) - iq3s_grid_g : LLVMOpaqueValue? // iq3s gemv: the [512 x i32] grid global - iq3s_smask_g : LLVMOpaqueValue? // iq3s gemv: the [16 x i32] sign-nibble byte-mask global + iq3s_panel : LLVMOpaqueValue? // iq3s/iq3xxs gemv: the per-superblock alloca panel (null = tile mode, the caller unpacked) + iq3s_grid_g : LLVMOpaqueValue? // iq3s gemv: the [512 x i32] grid global (iq3xxs: its [256 x i32] halved grid) + iq3s_smask_g : LLVMOpaqueValue? // iq3s gemv: the [16 x i32] sign-nibble byte-mask global (iq3xxs shares it) + iq3xxs_ksigns_g : LLVMOpaqueValue? // iq3xxs gemv: the [128 x i32] ksigns_iq2xs global (7-bit index -> sign byte) gfn : LLVMOpaqueValue? // the function under emit (block creation for emitted inner loops) gctx : LLVMOpaqueContext? psign_ty : LLVMOpaqueType? @@ -1320,6 +1321,121 @@ def private emit_iq3s_gather(var te : TileEmit; var sbi : LLVMOpaqueValue?) { LLVMPositionBuilderAtEnd(b, cont) } +// The iq3xxs gemv gather globals: the HALVED grid as a [256 x i32] private constant, ksigns +// as a [128 x i32] (7-bit index -> sign byte, bit 7 = parity), plus the shared smask table +// (the same nibble -> byte-mask expansion; magnitudes 2..31 are nonzero, so no carry). +def private iq3xxs_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { + var grid = LLVMGetNamedGlobal(mod, "dasllama.iq3xxs.grid") + if (grid == null) { + let g = iq3xxs_grid() + var elems : array + elems |> reserve(256) + for (i in range(256)) { + elems |> push(LLVMConstInt(te.types.t_int32, uint64(g[i]), 0)) + } + grid = LLVMAddGlobal(mod, LLVMArrayType(te.types.t_int32, 256u), "dasllama.iq3xxs.grid") + LLVMSetInitializer(grid, LLVMConstArray(te.types.t_int32, array_data_ptr(elems), 256u)) + LLVMSetGlobalConstant(grid, 1) + LLVMSetLinkage(grid, LLVMLinkage.LLVMPrivateLinkage) + LLVMSetAlignment(grid, 64u) + } + te.iq3s_grid_g = grid + var ksg = LLVMGetNamedGlobal(mod, "dasllama.iq3xxs.ksigns") + if (ksg == null) { + let kt = ksigns_iq2xs() + var elems3 : array + elems3 |> reserve(128) + for (i in range(128)) { + elems3 |> push(LLVMConstInt(te.types.t_int32, uint64(kt[i]), 0)) + } + ksg = LLVMAddGlobal(mod, LLVMArrayType(te.types.t_int32, 128u), "dasllama.iq3xxs.ksigns") + LLVMSetInitializer(ksg, LLVMConstArray(te.types.t_int32, array_data_ptr(elems3), 128u)) + LLVMSetGlobalConstant(ksg, 1) + LLVMSetLinkage(ksg, LLVMLinkage.LLVMPrivateLinkage) + LLVMSetAlignment(ksg, 16u) + } + te.iq3xxs_ksigns_g = ksg + var smask = LLVMGetNamedGlobal(mod, "dasllama.iq3s.smask") + if (smask == null) { + var elems2 : array + elems2 |> reserve(16) + for (m in range(16)) { + var v = 0ul + for (t in range(4)) { + if ((m & (1 << t)) != 0) { + v |= 0xFFul << uint64(8 * t) + } + } + elems2 |> push(LLVMConstInt(te.types.t_int32, v, 0)) + } + smask = LLVMAddGlobal(mod, LLVMArrayType(te.types.t_int32, 16u), "dasllama.iq3s.smask") + LLVMSetInitializer(smask, LLVMConstArray(te.types.t_int32, array_data_ptr(elems2), 16u)) + LLVMSetGlobalConstant(smask, 1) + LLVMSetLinkage(smask, LLVMLinkage.LLVMPrivateLinkage) + LLVMSetAlignment(smask, 16u) + } + te.iq3s_smask_g = smask +} + +// iq3xxs gemv: gather superblock `sbi` off the PACKED grp planes into the alloca panel - the +// iq3s gather's shape with the aux32 sign word (ONE aligned i32 column load per block) and +// ksigns lookups in place of the explicit sign bytes, and no 9th grid-index bit. +def private emit_iq3xxs_gather(var te : TileEmit; var sbi : LLVMOpaqueValue?) { + let b = te.builder + let mr = te.interleave + var wbP = LLVMBuildMul(b, sbi, te.types->ConstI64(uint64(mr * 96)), "iq3x.wb") + var pre = LLVMGetInsertBlock(b) + var head = LLVMAppendBasicBlockInContext(te.gctx, te.gfn, "iq3x.gath") + var cont = LLVMAppendBasicBlockInContext(te.gctx, te.gfn, "iq3x.cont") + LLVMBuildBr(b, head) + LLVMPositionBuilderAtEnd(b, head) + var rPhi = LLVMBuildPhi(b, te.types.t_int64, "iq3x.r") + var preVals <- [te.types->ConstI64(0ul)] + var preBlocks <- [pre] + LLVMAddIncoming(rPhi, preVals, preBlocks) + var rT4 = LLVMBuildMul(b, rPhi, te.types->ConstI64(4ul), "iq3x.r4") + var base = LLVMBuildAdd(b, wbP, rT4, "iq3x.base") + let c15 = LLVMConstInt(te.types.t_int32, 15ul, 0) + let c127 = LLVMConstInt(te.types.t_int32, 127ul, 0) + let cneg = LLVMConstInt(te.types.t_int32, 0x01010101ul, 0) + for (blk in range(8)) { + var axp = LLVMBuildGEP2(b, te.types.t_int8, te.wg, LLVMBuildAdd(b, base, te.types->ConstI64(uint64((16 + blk) * mr * 4)), ""), "") + var aux = LLVMBuildLoad2Aligned(b, te.types.t_int32, axp, 4u, "iq3x.aux{blk}") + for (l in range(4)) { + var sidx = LLVMBuildAnd(b, l == 0 ? aux : LLVMBuildLShr(b, aux, LLVMConstInt(te.types.t_int32, uint64(7 * l), 0), ""), c127, "") + var ksp2 = LLVMBuildGEP2(b, te.types.t_int32, te.iq3xxs_ksigns_g, sidx, "") + var sgn = LLVMBuildLoad2Aligned(b, te.types.t_int32, ksp2, 4u, "iq3x.sg{blk}_{l}") + var iv : LLVMOpaqueValue? [2] + for (h in range(2)) { + let qp1 = blk * 8 + 2 * l + h + var qpp = LLVMBuildGEP2(b, te.types.t_int8, te.wg, LLVMBuildAdd(b, base, te.types->ConstI64(uint64((qp1 / 4) * mr * 4 + qp1 % 4)), ""), "") + iv[h] = LLVMBuildZExt(b, LLVMBuildLoad2Aligned(b, te.types.t_int8, qpp, 1u, ""), te.types.t_int32, "iq3x.i{blk}_{l}_{h}") + } + var mv : LLVMOpaqueValue? [2] + mv[0] = LLVMBuildAnd(b, sgn, c15, "") + mv[1] = LLVMBuildAnd(b, LLVMBuildLShr(b, sgn, LLVMConstInt(te.types.t_int32, 4ul, 0), ""), c15, "") + for (h in range(2)) { + var gp = LLVMBuildGEP2(b, te.types.t_int32, te.iq3s_grid_g, iv[h], "") + var gw = LLVMBuildLoad2Aligned(b, te.types.t_int32, gp, 4u, "iq3x.g{blk}_{l}_{h}") + var mp = LLVMBuildGEP2(b, te.types.t_int32, te.iq3s_smask_g, mv[h], "") + var mw = LLVMBuildLoad2Aligned(b, te.types.t_int32, mp, 4u, "") + var w = LLVMBuildAdd(b, LLVMBuildXor(b, gw, mw, ""), LLVMBuildAnd(b, mw, cneg, ""), "iq3x.w{blk}_{l}_{h}") + let e = l * 8 + h * 4 + let dd = (e < 16 ? 0 : 128 * mr) + (blk * 4 + (e % 16) / 4) * mr * 4 + var dp = LLVMBuildGEP2(b, te.types.t_int8, te.iq3s_panel, LLVMBuildAdd(b, rT4, te.types->ConstI64(uint64(dd)), ""), "") + LLVMBuildStore(b, w, dp) + } + } + } + var rNext = LLVMBuildAdd(b, rPhi, te.types->ConstI64(1ul), "iq3x.rn") + var more = LLVMBuildICmp(b, LLVMIntPredicate.LLVMIntSLT, rNext, te.types->ConstI64(uint64(mr)), "iq3x.more") + LLVMBuildCondBr(b, more, head, cont) + var loopVals <- [rNext] + var loopBlocks <- [head] + LLVMAddIncoming(rPhi, loopVals, loopBlocks) + LLVMPositionBuilderAtEnd(b, cont) +} + // One 256-weight SUPERBLOCK, IQ4_XS/iq3s grp form (te.kq = 44/33): mx4's LUT decode (44) // or the byte-expanded signed panel (33; the gemv gathers it per superblock, the tile gets the // runtime unpack) — dot_lane's sign-trick lattice, k4's fold with no min term. @@ -1328,12 +1444,16 @@ def private emit_block_iq4xs(var te : TileEmit; var sbi : LLVMOpaqueValue?; var let rq = te.rq let mr = te.interleave let w8 = te.width / 8 - let panel = te.kq == 33 // iq3s: the byte-expanded signed panel replaces the nibble+LUT decode; fold identical + let panel = te.kq == 33 || te.kq == 34 // iq3s/iq3xxs: the byte-expanded signed panel replaces the nibble+LUT decode; fold identical let gather = panel && te.iq3s_panel != null // gemv mode: gather this superblock off the packed planes first var wbase = te.wg var wb : LLVMOpaqueValue? if (gather) { - emit_iq3s_gather(te, sbi) + if (te.kq == 33) { + emit_iq3s_gather(te, sbi) + } else { + emit_iq3xxs_gather(te, sbi) + } wbase = te.iq3s_panel wb = te.types->ConstI64(0ul) } else { @@ -1413,7 +1533,7 @@ def private emit_block_iq4xs(var te : TileEmit; var sbi : LLVMOpaqueValue?; var // token PAIR — single-token slices ride the sdot lattice instead. kq blocks are SUPERBLOCKS // (the kq gemv/tile drive emit_slice with nb = n/256). def private emit_one_block(var te : TileEmit; var bi : LLVMOpaqueValue?; var f : LLVMOpaqueValue? [8]; tokBase, tokCount : int) { - if (te.kq == 44 || te.kq == 33) { + if (te.kq == 44 || te.kq == 33 || te.kq == 34) { emit_block_iq4xs(te, bi, f, tokBase, tokCount) } elif (te.kq != 0) { emit_block_kqv2(te, bi, f, tokBase, tokCount) @@ -2520,11 +2640,15 @@ def private kq_gemv_gen_impl(var gc : LlvmCodeCtx; fmt : int) : bool { // entry: group range off the row range; strides are disk footprints (quants 128/160/192B, scales 20/20/18B) LLVMPositionBuilderAtEnd(b, entry) - if (fmt == 33) { - // iq3s: the gemv gathers each superblock into an alloca panel (the tile gets the runtime unpack) + if (fmt == 33 || fmt == 34) { + // iq3s/iq3xxs: the gemv gathers each superblock into an alloca panel (the tile gets the runtime unpack) te.gfn = gc.impl te.gctx = gc.jit.ctx - iq3s_emit_globals(te, LLVMGetGlobalParent(gc.impl)) + if (fmt == 33) { + iq3s_emit_globals(te, LLVMGetGlobalParent(gc.impl)) + } else { + iq3xxs_emit_globals(te, LLVMGetGlobalParent(gc.impl)) + } te.iq3s_panel = LLVMBuildAlloca(b, LLVMArrayType(gc.jit.types.t_int8, uint(256 * te.interleave)), "iq3s.panel") LLVMSetAlignment(te.iq3s_panel, 32u) } @@ -2705,12 +2829,10 @@ def private k3_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 3) def private iq3s_gemv_gen(var gc : LlvmCodeCtx) : bool => kq_gemv_gen_impl(gc, 33) def private iq3s_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 33) -// iq3xxs: the grid-gather emitter arm lands with its kernel arc; until then both stamps -// decline (reference bodies serve) — a declined generator is the framework's own fallback path. -[unused_argument(gc)] -def private iq3xxs_gemv_gen(var gc : LlvmCodeCtx) : bool => false -[unused_argument(gc)] -def private iq3xxs_tile_gen(var gc : LlvmCodeCtx) : bool => false +// iq3xxs: the gemv gathers per superblock into the alloca panel (emit_iq3xxs_gather - the +// halved grid + ksigns); the tile reads the runtime-unpacked panel - the iq3s route exactly. +def private iq3xxs_gemv_gen(var gc : LlvmCodeCtx) : bool => kq_gemv_gen_impl(gc, 34) +def private iq3xxs_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 34) def private k4_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 4) def private k5_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 5) diff --git a/modules/dasLLAMA/harness/gen_tune_probe.das b/modules/dasLLAMA/harness/gen_tune_probe.das index 49c0fa54ba..62be3a5d74 100644 --- a/modules/dasLLAMA/harness/gen_tune_probe.das +++ b/modules/dasLLAMA/harness/gen_tune_probe.das @@ -1276,6 +1276,7 @@ def test_mode : bool { // nolint:STYLE037,STYLE038 — flat per-variant correctn var kfxs44 <- [ <- build_kq_fixture(44l, 256l, 32l, 8l), <- build_kq_fixture(44l, 768l, 32l, 8l), <- build_kq_fixture(44l, 2048l, 32l, 8l)] var kfxs3 <- [ <- build_kq_fixture(3l, 256l, 32l, 8l), <- build_kq_fixture(3l, 768l, 32l, 8l), <- build_kq_fixture(3l, 2048l, 32l, 8l)] var kfxs33 <- [ <- build_kq_fixture(33l, 256l, 32l, 8l), <- build_kq_fixture(33l, 768l, 32l, 8l), <- build_kq_fixture(33l, 2048l, 32l, 8l)] + var kfxs34 <- [ <- build_kq_fixture(34l, 256l, 32l, 8l), <- build_kq_fixture(34l, 768l, 32l, 8l), <- build_kq_fixture(34l, 2048l, 32l, 8l)] var vs <- q8q8_tile_gen_variants() var mrs <- variant_mrs() var wbs <- variant_wbias() @@ -1403,6 +1404,7 @@ def test_mode : bool { // nolint:STYLE037,STYLE038 — flat per-variant correctn allok = kq_test_family(44l, kfxs44) && allok allok = kq_test_family(3l, kfxs3) && allok allok = kq_test_family(33l, kfxs33) && allok + allok = kq_test_family(34l, kfxs34) && allok delete wtab delete mtab delete gtab @@ -1421,6 +1423,7 @@ def test_mode : bool { // nolint:STYLE037,STYLE038 — flat per-variant correctn delete kfxs44 delete kfxs3 delete kfxs33 + delete kfxs34 return allok } @@ -1863,7 +1866,7 @@ def tune_mode_run : bool { // nolint:STYLE037,STYLE038 — flat family-by-family print("aborting mid-run - no sidecar written\n") return false } - for (fmt in fixed_array(4l, 5l, 6l, 40l, 44l, 3l, 33l)) { + for (fmt in fixed_array(4l, 5l, 6l, 40l, 44l, 3l, 33l, 34l)) { interrupt_gate("k{fmt}q8_tile_gen") let kq_t0 = ref_time_ticks() let entry = kq_tile_entry(fmt) From 0a5add6f56b76ac9f43b51239c3c7a9e360e17ac Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Sun, 30 Aug 2026 21:12:26 -0700 Subject: [PATCH 25/61] dasLLAMA: IQ3_XXS on the Vulkan tier - halved grid, parity signs, no ksigns table The iq3s walk compressed to its minimum. vk_kq_schema_id 9 -> 34. The 1 KB HALVED grid stages into @workgroup uint[256]; the ksigns table disappears entirely - bit 7 of the sign byte is the even parity of the 7-bit index, five ALU ops (the SPIR-V emitter has no `^=`, so the fold spells as plain assignments). KqGemvIq3xxs rides KqGemvBase over the 24-word rows (one aligned aux32 per block, iq3s's iq4_sc scale fold - the (2ls+1) strips read identically); KqBatchIq3xxs : KqBatchIq4xs gathers the halved grid at stage time; Iq3xxsCm2T lands on a new IQ3XGRID gated axis with the aux 7-bit field picked from at most two unpack8 bytes per element. Dispatcher arms, family cells, the independent float witness and the l/m/s cm2 cell mirror the iq3s rows. The vehicle needed care twice (HOW_TO carries both): QUIRK 23's census caught IQ2_S attn in the natural download, and the Qwen fallback proves kernels but cannot arm the resident decode driver (attention QKV bias declines silently) - so the e2e/bench vehicle is a local requant of Llama-3.2-1B with --tensor-type attn_k/q=iq3_xxs overriding llama.cpp's own IQ2_S recipe, quantized with bartowski's imatrix. Gates on the 5060 Ti: the suite 76/76 - the three cm2 tiles 0-off (89600 cells each), the family cells and float witness green. The resident driver arms on the llama-arch vehicle and matches llama.cpp's greedy ids 32/64 with the fork a 0.093-logit near-tie whose runner-up IS our token (the window's smallest margin), gen 202 t/s. Rows vs llama.cpp b10660 Vulkan (same local requant): pp512 12225.7 / 17807.7 (0.69x), tg128 372.1 / 389.9 (0.95x). Co-Authored-By: Claude Fable 5 --- modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md | 23 +- .../dasllama/dasllama_vulkan_classes.das | 225 +++++++++++++++++- .../dasllama/dasllama_vulkan_common.das | 7 +- .../dasllama/dasllama_vulkan_prefill.das | 17 ++ modules/dasLLAMA/tests/_vkd_oracles.das | 78 ++++++ .../dasLLAMA/tests/test_vulkan_kernels.das | 150 +++++++++++- 6 files changed, 486 insertions(+), 14 deletions(-) diff --git a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md index 29872e9a73..cc2f66c194 100644 --- a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md +++ b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md @@ -462,7 +462,28 @@ silently skips it: the test-mode fixture families AND the tune loop's ~2e-6); the tuner crowns `dot_maddubs_width256_mr8` verdict=beats (the zen2 board's usual winner, same as iq3s/k3). E2e stamped: prefill 6 -> 17 t/s, gen 19 -> 41 t/s; ids 17/64 with the fork again ON the margin oracle's top-2 (0.43 logits at step 17 - stamped float -folds move the flip point, they do not change the class). Vulkan, Metal: pending. +folds move the flip point, they do not change the class). + +Vulkan came as the iq3s walk compressed: `vk_kq_schema_id` 9 -> 34; the 1 KB HALVED grid +stages into `@workgroup uint[256]` (one word per thread on the tile, four on the gemv), and +the ksigns TABLE DISAPPEARS - bit 7 of the sign byte is the even parity of the 7-bit index, +five ALU ops (`t = v ^ (v >> 4); t = t ^ (t >> 2); t = t ^ (t >> 1)`; the SPIR-V emitter has +no `^=`, spell the fold as plain assignments). `KqGemvIq3xxs : KqGemvBase` (24-word rows, +one aligned aux word per block, iq3s's fold via `iq4_sc` - the (2ls+1) strips read the same); +`KqBatchIq3xxs : KqBatchIq4xs` (the halved-grid gather at stage time); `Iq3xxsCm2T` on a new +`IQ3XGRID` gated axis (the aux 7-bit field spans at most two bytes - two unpack8 picks per +element). The sdot4 arms, cm2 dispatcher arms and the family/witness/cm2 test cells mirror +iq3s's rows exactly. VEHICLE trap beyond QUIRK 23: the Qwen2.5 vehicle proves kernels but +CANNOT arm the resident decode driver (attention QKV bias - a silent decline), so the +Vulkan e2e/bench vehicle is a LOCAL requant: `llama-quantize --allow-requantize --imatrix + --tensor-type attn_k=iq3_xxs --tensor-type attn_q=iq3_xxs out IQ3_XXS` +- llama.cpp's own ftype recipe puts IQ2_S on attn_k/q, the override keeps the mix inside +the supported set. Gates: the suite 76/76 (the three cm2 tiles 0-off at 89600 cells each, +the float witness in the family cell); the resident driver arms and matches llama.cpp's +greedy ids 32/64 with the fork a 0.093-logit near-tie whose top-2 IS our token (the +smallest margin in the window), gen 202 t/s. Rows (5060 Ti vs llama.cpp b10660 Vulkan, +the local requant): pp512 12225.7 vs 17807.7 (0.69x), tg128 372.1 vs 389.9 (0.95x). +Metal: pending (the constant-table hoisting from the iq3s arc carries the grid). ### IQ3_S (the third format - and the first grid format, 2026-08-30) diff --git a/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das b/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das index df2b4de2a2..42883cdb7f 100644 --- a/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das +++ b/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das @@ -2540,6 +2540,55 @@ def private iq3s_grid_word(i : int) : uint { return tbl[i] } +// the HALVED iq3xxs grid (ggml's iq3xxs_grid with every byte / 2 - the plane convention that +// absorbs the format's 0.25 into the stored d), one word per call; staging loops copy it into +// workgroup memory (QUIRK 20's slow path stays off the hot loop). +def private iq3xxs_grid_word(i : int) : uint { + let tbl = fixed_array( + 0x02020202u, 0x0202020au, 0x02020212u, 0x02020606u, 0x0202060eu, 0x0202061fu, 0x02020a02u, 0x02020a0au, + 0x02020e06u, 0x0202120au, 0x02021f0eu, 0x02021f16u, 0x02060206u, 0x0206020eu, 0x02060602u, 0x0206060au, + 0x02060a06u, 0x02060a16u, 0x02060e02u, 0x02060e0au, 0x02061206u, 0x02061612u, 0x02061f02u, 0x020a0202u, + 0x020a020au, 0x020a0212u, 0x020a0606u, 0x020a0a02u, 0x020a0a0au, 0x020a0e06u, 0x020a0e0eu, 0x020a0e1fu, + 0x020a1606u, 0x020a161fu, 0x020a1f16u, 0x020e0206u, 0x020e021fu, 0x020e0602u, 0x020e060au, 0x020e0a16u, + 0x020e1f02u, 0x0212060eu, 0x02120e1fu, 0x02121212u, 0x0212161fu, 0x02121f0eu, 0x02121f16u, 0x02160206u, + 0x0216021fu, 0x02160e0au, 0x0216160au, 0x021a0e16u, 0x021a1a12u, 0x021f0602u, 0x021f0612u, 0x021f061au, + 0x021f120eu, 0x021f1a06u, 0x06020206u, 0x0602020eu, 0x06020602u, 0x0602060au, 0x06020a06u, 0x06020a0eu, + 0x06020e02u, 0x06020e0au, 0x06020e12u, 0x0602121fu, 0x06021602u, 0x06060202u, 0x0606020au, 0x06060606u, + 0x06060a02u, 0x06060a0au, 0x060a0206u, 0x060a020eu, 0x060a0602u, 0x060a060au, 0x060a0a06u, 0x060a0e02u, + 0x060a1f0au, 0x060e0202u, 0x060e020au, 0x060e0a02u, 0x060e0e06u, 0x060e121au, 0x060e1a1au, 0x06120206u, + 0x06120216u, 0x06121602u, 0x06160a02u, 0x06160a12u, 0x0616121au, 0x06161f06u, 0x061a0216u, 0x061f0a0au, + 0x061f1202u, 0x0a020202u, 0x0a02020au, 0x0a020606u, 0x0a02060eu, 0x0a020a02u, 0x0a020a0au, 0x0a020a1au, + 0x0a020e06u, 0x0a02120au, 0x0a060206u, 0x0a06020eu, 0x0a060216u, 0x0a060602u, 0x0a06060au, 0x0a060a06u, + 0x0a060e02u, 0x0a061a0eu, 0x0a061a1fu, 0x0a061f02u, 0x0a0a0202u, 0x0a0a020au, 0x0a0a0606u, 0x0a0a061fu, + 0x0a0a0a02u, 0x0a0a0a0au, 0x0a0a0e1fu, 0x0a0a1202u, 0x0a0a1616u, 0x0a0e0206u, 0x0a0e0602u, 0x0a0e0612u, + 0x0a0e1f02u, 0x0a0e1f12u, 0x0a120e16u, 0x0a12160eu, 0x0a16020eu, 0x0a160a1fu, 0x0a161206u, 0x0a161f12u, + 0x0a1f0206u, 0x0a1f020eu, 0x0a1f061au, 0x0a1f1216u, 0x0e020206u, 0x0e020602u, 0x0e02060au, 0x0e020a06u, + 0x0e020a0eu, 0x0e021602u, 0x0e021a16u, 0x0e021f0au, 0x0e060202u, 0x0e06020au, 0x0e060a02u, 0x0e060e06u, + 0x0e061212u, 0x0e06121au, 0x0e0a0206u, 0x0e0a020eu, 0x0e0a0602u, 0x0e0a0a16u, 0x0e0a160au, 0x0e0a1f0au, + 0x0e0e0606u, 0x0e0e0e0eu, 0x0e120e02u, 0x0e12121fu, 0x0e121f0au, 0x0e160202u, 0x0e16021au, 0x0e160a0au, + 0x0e161616u, 0x0e1a0612u, 0x0e1a0e1au, 0x0e1a1a0eu, 0x0e1f0e0eu, 0x0e1f1a02u, 0x12020212u, 0x1202061fu, + 0x12020e16u, 0x12020e1fu, 0x1202160eu, 0x1202161fu, 0x12061f12u, 0x120a0a02u, 0x120a0e1fu, 0x120a1202u, + 0x120a1a02u, 0x120a1a1au, 0x120e021fu, 0x120e1216u, 0x12120212u, 0x12121606u, 0x12121a12u, 0x12160a16u, + 0x1216120eu, 0x12161f02u, 0x121f0216u, 0x121f0602u, 0x121f060au, 0x121f0e02u, 0x1602060au, 0x16021206u, + 0x16021f02u, 0x16060202u, 0x1606021au, 0x16060a1au, 0x16061616u, 0x160a0612u, 0x160a0e0au, 0x160a1f0au, + 0x160e020au, 0x160e160eu, 0x16120602u, 0x16120a0eu, 0x16120a1fu, 0x16121f0au, 0x1616020au, 0x16160e06u, + 0x161a1602u, 0x161f0a12u, 0x161f120au, 0x1a020a12u, 0x1a021212u, 0x1a02121au, 0x1a021a12u, 0x1a060a06u, + 0x1a061a06u, 0x1a0a061fu, 0x1a0a1a12u, 0x1a0e0e02u, 0x1a0e0e1au, 0x1a121212u, 0x1a160216u, 0x1a16160au, + 0x1a1a0e0eu, 0x1a1f020eu, 0x1a1f0a06u, 0x1f02020eu, 0x1f020216u, 0x1f02021fu, 0x1f020602u, 0x1f020e0au, + 0x1f02160au, 0x1f060a1au, 0x1f061202u, 0x1f0a060au, 0x1f0a1216u, 0x1f0a160au, 0x1f0e0202u, 0x1f0e0616u, + 0x1f0e0e0eu, 0x1f0e1a02u, 0x1f120a06u, 0x1f121206u, 0x1f160202u, 0x1f16020au, 0x1f160a12u, 0x1f1a0e02u) + return tbl[i] +} + +// ksigns_iq2xs without the table: bit 7 of the sign byte is the EVEN PARITY of the 7-bit +// index, so the byte is idx | (parity << 7) - five ALU ops, nothing staged. +def private ksign7(v : uint) : uint { + var t = v ^ (v >> 4u) + t = t ^ (t >> 2u) + t = t ^ (t >> 1u) + return v | ((t & 1u) << 7u) +} + def private iq4_word(n : uint) : uint { let tbl = uint4(3215825025u, 4142587343u, 639175937u, 1901675829u) // kvalues_iq4nl, 4 bytes per word let q0 = n & 15u @@ -2871,6 +2920,48 @@ class KqBatchIq3s : KqBatchIq4xs { } } +// IQ3_XXS tile: the iq3s staging with the halved-grid gather - bare 8-bit indices (no ninth +// bit), signs from the aux32's four 7-bit ksigns indices via the parity helper (no table); +// scale row and blk_fma are iq4xs's verbatim (inherited). The 1 KB grid stages once. +[vk_dispatch(name = "kq_batch_iq3xxs_cls", kernel = "run", family = "kq_batch_cls", grid = "wgs", params = "wgs : int64")] +class KqBatchIq3xxs : KqBatchIq4xs { + @workgroup gridw : uint[256] + + def iq3s_signed(g, nib : uint) : uint { + let m1 = (nib * 0x00204081u) & 0x01010101u + let mask = m1 * 255u + return (g ^ mask) + m1 + } + + def override stage_w(wsb0, wcol, nsb, s, word, grp : uint) { + for (ki in range(2)) { + let k = word + uint(ki) * 8u + let bu = k / 2u + let hh = k % 2u + var wv = uint4(0u) + if (wcol < pa.d) { + let wsb = wsb0 + wcol * nsb + s + let qsw = wq[wsb * 24u + bu * 2u + hh] + let aux = wq[wsb * 24u + 16u + bu] + let l0 = hh * 2u + let sg0 = ksign7((aux >> (7u * l0)) & 127u) + let sg1 = ksign7((aux >> (7u * l0 + 7u)) & 127u) + wv = uint4(iq3s_signed(gridw[qsw & 255u], sg0 & 15u), iq3s_signed(gridw[(qsw >> 8u) & 255u], sg0 >> 4u), + iq3s_signed(gridw[(qsw >> 16u) & 255u], sg1 & 15u), iq3s_signed(gridw[qsw >> 24u], sg1 >> 4u)) + } + ww4[grp * 17u + k] = wv + } + } + + [spirv_kernel(local_size_x = 256, name = "kq_batch_iq3xxs_cls_spv")] + def override run { + let tid = gl_LocalInvocationID.x + gridw[tid] = iq3xxs_grid_word(int(tid)) + barrier() // the grid visible before the first stage_w + tile_shell() + } +} + // Q5_K tile: the k4 compose with the 5th bit OR'd in from the 40B block's qh words [vk_dispatch(name = "kq_batch_k5_cls", kernel = "run", family = "kq_batch_cls", grid = "wgs", params = "wgs : int64")] class KqBatchK5 : KqBatchBase { @@ -3869,6 +3960,11 @@ struct VkIq3sBlk { sg : int16[16] // ... and its 32 sign bytes; scales (f16 d + 8 int8 (1 + 2s)) live in ws } +struct VkIq3xxsBlk { + qs : int16[32] // one IQ3_XXS superblock's 64 grid-index bytes as 16-bit lanes + ax : int16[16] // ... and its 32 aux bytes (per block: 4x7-bit ksigns indices + 4-bit ls); scales (halved f16 d + 8 x (2ls+1)) live in ws +} + struct VkK3Blk { qs : int16[32] // one Q3_K superblock's 64 qs bytes, four 2-bit lanes each, verbatim disk order hm : int16[16] // ... and its 32 hmask bytes; scales (16 int8 (sc-32) + f16 d) live in ws @@ -3895,6 +3991,8 @@ class template KqCm2BatchT : MoeCmBase { @workgroup @template_gate = IQLUT iq4lut : float16[16] // kvalues_iq4nl as f16 (llama.cpp's init_iq_shmem form) @template_constant IQ3GRID : bool = false // stage the 2 KB iq3s grid into workgroup memory @workgroup @template_gate = IQ3GRID iq3s_gridc : uint[512] + @template_constant IQ3XGRID : bool = false // stage the 1 KB halved iq3xxs grid into workgroup memory + @workgroup @template_gate = IQ3XGRID iq3x_gridc : uint[256] [spirv_kernel(local_size_x = 256), arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-tile-pick-and-default")] def run { // nolint:STYLE038 — the fast/edge path pair, barrier- and register-coupled @@ -3937,6 +4035,9 @@ class template KqCm2BatchT : MoeCmBase { iq3s_gridc[gi] = iq3s_grid_word(int(gi)) iq3s_gridc[gi + 256u] = iq3s_grid_word(int(gi + 256u)) } + static_if (IQ3XGRID) { + iq3x_gridc[gl_LocalInvocationID.x] = iq3xxs_grid_word(int(gl_LocalInvocationID.x)) + } barrier() // wg_blk0 (and the gated codebook) visible before the first decode load var a : coopmatWgA_f16_128x64 var b : BT @@ -4234,6 +4335,39 @@ class template Iq3sCm2T : KqCm2BatchT { } } +[ |> template_struct_instance] +class template Iq3xxsCm2T : KqCm2BatchT { + typedef BLK = VkIq3xxsBlk + typedef ST = uint + override IQ3XGRID = true + + [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] + def decode(blk : VkIq3xxsBlk; bc, cib : uint2) : float16 { + let e = cib.y + let g = e >> 5u + let r = e & 31u + let wj = r >> 2u + let qi = g * 8u + wj + let qb = uint(int(unpack8(blk.qs[int(qi >> 1u)])[int(qi & 1u)])) & 0xFFu + let l = r >> 3u + let bl = 7u * l + let a0i = g * 4u + (bl >> 3u) + let b0 = uint(int(unpack8(blk.ax[int(a0i >> 1u)])[int(a0i & 1u)])) & 0xFFu + let b1 = uint(int(unpack8(blk.ax[int((a0i + 1u) >> 1u)])[int((a0i + 1u) & 1u)])) & 0xFFu + let sidx = ((b0 | (b1 << 8u)) >> (bl & 7u)) & 127u + var tt = sidx ^ (sidx >> 4u) + tt = tt ^ (tt >> 2u) + tt = tt ^ (tt >> 1u) + let sgb = sidx | ((tt & 1u) << 7u) + let gw = iq3x_gridc[qb] + let gb = float((gw >> ((r & 3u) * 8u)) & 0xFFu) + let srow = (wg_blk0 + bc.x * (pa.n >> 8u) + bc.y) * 5u + let d = unpackHalf2x16(ws[srow]).x + let sc = int(ws[srow + 1u + (g >> 2u)] << ((3u - (g & 3u)) * 8u)) >> 24 + return float16(d * float(sc) * (((sgb >> (r & 7u)) & 1u) != 0u ? -gb : gb)) + } +} + // the stamps: the three width columns of each format (the dispatcher keys on (fmt, tc)) [vk_dispatch(name = "q8_batch_cm2l_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] class Q8Cm2LBatch : Q8Cm2T { @@ -4451,6 +4585,33 @@ class Iq3sCm2SBatch : Iq3sCm2T { typedef FLO = tensorLayout2DPad } +[vk_dispatch(name = "kq_batch_iq3xxs_cm2l_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Iq3xxsCm2LBatch : Iq3xxsCm2T { + typedef BT = coopmatWgB_f16_64x256 + typedef ACC = coopmatWgAcc_f16_128x256 + typedef ACCW = coopmatWgAcc_f32_128x256 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_iq3xxs_cm2m_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Iq3xxsCm2MBatch : Iq3xxsCm2T { + override BN = 128u + typedef BT = coopmatWgB_f16_64x128 + typedef ACC = coopmatWgAcc_f16_128x128 + typedef ACCW = coopmatWgAcc_f32_128x128 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_iq3xxs_cm2s_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Iq3xxsCm2SBatch : Iq3xxsCm2T { + override BN = 32u + override STILE = true + typedef BT = coopmatWgB_f16_64x32 + typedef ACC = coopmatWgAcc_f16_128x32 + typedef ACCW = coopmatWgAcc_f32_128x32 + typedef FLO = tensorLayout2DPad +} + // ===== the split-k reduce (sums the cm2 partial planes into y) ===== struct SkRedArgs { @@ -5022,6 +5183,8 @@ def kq_batch_cls_ensure(fmt : int; q40cm : bool) : bool { return ensure_kq_batch_k3_cls() } elif (fmt == int(KqFmt.iq3s)) { return ensure_kq_batch_iq3s_cls() + } elif (fmt == int(KqFmt.iq3xxs)) { + return ensure_kq_batch_iq3xxs_cls() } panic("dasLLAMA vulkan tier: no kq batch class for format {fmt}") return false @@ -5054,6 +5217,8 @@ def kq_batch_cls_enc_for(fmt : int; q40cm : bool; raw : VkCommandBuffer; var h : enc_kq_batch_k3_cls(raw, h, set_, pc, wgs) } elif (fmt == int(KqFmt.iq3s)) { enc_kq_batch_iq3s_cls(raw, h, set_, pc, wgs) + } elif (fmt == int(KqFmt.iq3xxs)) { + enc_kq_batch_iq3xxs_cls(raw, h, set_, pc, wgs) } else { panic("dasLLAMA vulkan tier: no kq batch class for format {fmt}") } @@ -5080,6 +5245,8 @@ def gemv_cls_ensure(fmt : int) : bool { return ensure_kq_gemv_k3_cls() } elif (fmt == int(KqFmt.iq3s)) { return ensure_kq_gemv_iq3s_cls() + } elif (fmt == int(KqFmt.iq3xxs)) { + return ensure_kq_gemv_iq3xxs_cls() } panic("dasLLAMA vulkan tier: no GEMV class for format {fmt}") return false @@ -5112,14 +5279,16 @@ def gemv_cls_enc(fmt : int; raw : VkCommandBuffer; var h : VkHaz; var set_ : VkD enc_kq_gemv_k3_cls(raw, h, set_, pc, wgs) } elif (fmt == int(KqFmt.iq3s)) { enc_kq_gemv_iq3s_cls(raw, h, set_, pc, wgs) + } elif (fmt == int(KqFmt.iq3xxs)) { + enc_kq_gemv_iq3xxs_cls(raw, h, set_, pc, wgs) } else { panic("dasLLAMA vulkan tier: no GEMV class for format {fmt}") } } // Q4_K: w = d*sc*q - dmin*mn*1, q in [0,15] — per 32-block axs*(d*sc*idot - dmin*mn*bsum). -// The seven formats share family=kq_gemv_cls: ONE set layout + ONE set_kq_gemv_cls serves all -// seven pipelines (identical inherited members), each leaf keeping its own ensure_/enc_ names. +// The eight formats share family=kq_gemv_cls: ONE set layout + ONE set_kq_gemv_cls serves all +// eight pipelines (identical inherited members), each leaf keeping its own ensure_/enc_ names. [vk_dispatch(name = "kq_gemv_k4_cls", kernel = "run", family = "kq_gemv_cls", grid = "wgs", params = "wgs : int64")] class KqGemvK4 : KqGemvBase { def override blk_contrib(wsb, blk, xsb : uint) : float { @@ -5234,6 +5403,58 @@ class KqGemvIq3s : KqGemvBase { } } +// IQ3_XXS: halved-grid gather (bare 8-bit index -> 4 magnitudes per word), signs from the +// aux32's four 7-bit ksigns indices (bit 7 = parity, computed - no table), per-32 (2ls + 1) +// sub-scale x halved d - iq3s's fold exactly, over the 24-word rows; the 1 KB grid stages +// into workgroup memory ahead of the shell. +[vk_dispatch(name = "kq_gemv_iq3xxs_cls", kernel = "run", family = "kq_gemv_cls", grid = "wgs", params = "wgs : int64")] +class KqGemvIq3xxs : KqGemvBase { + @workgroup gridw : uint[256] + + def iq3s_signed(g, nib : uint) : uint { + let m1 = (nib * 0x00204081u) & 0x01010101u + let mask = m1 * 255u + return (g ^ mask) + m1 + } + + def override blk_contrib(wsb, blk, xsb : uint) : float { + let qb = wsb * 24u + blk * 2u + let aux = wq[wsb * 24u + 16u + blk] + let xb4 = xsb * 16u + blk * 2u + let xl = xq4[xb4] + let xh = xq4[xb4 + 1u] + var isum = 0 + for (l in range(2)) { + let qsw = wq[qb + uint(l)] + let l0 = uint(l) * 2u + let sg0 = ksign7((aux >> (7u * l0)) & 127u) + let sg1 = ksign7((aux >> (7u * l0 + 7u)) & 127u) + let w0 = iq3s_signed(gridw[qsw & 255u], sg0 & 15u) + let w1 = iq3s_signed(gridw[(qsw >> 8u) & 255u], sg0 >> 4u) + let w2 = iq3s_signed(gridw[(qsw >> 16u) & 255u], sg1 & 15u) + let w3 = iq3s_signed(gridw[qsw >> 24u], sg1 >> 4u) + if (l == 0) { + isum += sdot4(w0, xl.x) + sdot4(w1, xl.y) + sdot4(w2, xl.z) + sdot4(w3, xl.w) + } else { + isum += sdot4(w0, xh.x) + sdot4(w1, xh.y) + sdot4(w2, xh.z) + sdot4(w3, xh.w) + } + } + let dd = unpackHalf2x16(wsu[wsb * 5u]).x + return axs[xsb] * dd * float(iq4_sc(wsu[wsb * 5u + 1u + blk / 4u], blk) * isum) + } + + [spirv_kernel(local_size_x = 64, name = "kq_gemv_iq3xxs_cls_spv")] + def run { + let lid = gl_LocalInvocationID.x + for (k in range(4)) { + let idx = lid + uint(k) * 64u + gridw[idx] = iq3xxs_grid_word(int(idx)) + } + barrier() // the grid visible before the shell's block walk + gemv_shell() + } +} + // Q3_K: q = 2-bit qs lane | hmask bit << 2, minus 4, per-16 SIGNED sub-scales x d — k6's fold with // the offset 4 over the k3 planes (6 uint4 per superblock: qs 0..3, hmask 4..5) [vk_dispatch(name = "kq_gemv_k3_cls", kernel = "run", family = "kq_gemv_cls", grid = "wgs", params = "wgs : int64")] diff --git a/modules/dasLLAMA/dasllama/dasllama_vulkan_common.das b/modules/dasLLAMA/dasllama/dasllama_vulkan_common.das index e998199030..f134b2c868 100644 --- a/modules/dasLLAMA/dasllama/dasllama_vulkan_common.das +++ b/modules/dasLLAMA/dasllama/dasllama_vulkan_common.das @@ -1709,7 +1709,7 @@ struct ArenaSlab { } struct ArenaFmt { - fmt : int // int(KqFmt): 0 = q8, 1/2/3 = k4/k5/k6, 4 = q40, 6 = iq4xs, 7 = k3, 8 = iq3s + fmt : int // int(KqFmt): 0 = q8, 1/2/3 = k4/k5/k6, 4 = q40, 6 = iq4xs, 7 = k3, 8 = iq3s, 9 = iq3xxs slabs : array cur : int // slab the bump cursor is in slab_cap : int64 // max blocks one slab may carry (msr over both plane strides) @@ -1740,7 +1740,7 @@ def arena_local_blk(blk : int64) : int64 => blk & ((1l << 32l) - 1l) // per-block plane strides — the block is 32 weights for q8 and a 256-weight superblock for kq. // Mirrors stack_plane_bytes: it computes the same totals as count * these strides. -// the tier's stack fmt space is int(KqFmt) — bridge to the gemm_schema kq id space (4/5/6/40/44/3/33) +// the tier's stack fmt space is int(KqFmt) — bridge to the gemm_schema kq id space (4/5/6/40/44/3/33/34) def vk_kq_schema_id(fmt : int) : int { if (fmt == 1) return 4 if (fmt == 2) return 5 @@ -1749,11 +1749,12 @@ def vk_kq_schema_id(fmt : int) : int { if (fmt == 6) return 44 if (fmt == 7) return 3 if (fmt == 8) return 33 + if (fmt == 9) return 34 panic("vk_kq_schema_id: not a kq stack fmt") return 0 } -// fmt space here = int(KqFmt): 0 q8 (32-elem blocks), 1-4 and 6..8 the kq lattice (+ q40, iq4xs, k3, iq3s) +// fmt space here = int(KqFmt): 0 q8 (32-elem blocks), 1-4 and 6..9 the kq lattice (+ q40, iq4xs, k3, iq3s, iq3xxs) def vk_fmt_b32(fmt : int) : bool => fmt == 0 // meta/activation block unit of a stack format — q8 counts 32-weight blocks, the kq diff --git a/modules/dasLLAMA/dasllama/dasllama_vulkan_prefill.das b/modules/dasLLAMA/dasllama/dasllama_vulkan_prefill.das index e1d50f1c73..04c70f22fb 100644 --- a/modules/dasLLAMA/dasllama/dasllama_vulkan_prefill.das +++ b/modules/dasLLAMA/dasllama/dasllama_vulkan_prefill.das @@ -79,6 +79,9 @@ def private cm2_cls_ensure(fmt : int; tc : int64) : bool { if (fmt == int(KqFmt.iq3s)) { return tc == 256l ? ensure_kq_batch_iq3s_cm2l_cls() : (tc == 128l ? ensure_kq_batch_iq3s_cm2m_cls() : ensure_kq_batch_iq3s_cm2s_cls()) } + if (fmt == int(KqFmt.iq3xxs)) { + return tc == 256l ? ensure_kq_batch_iq3xxs_cm2l_cls() : (tc == 128l ? ensure_kq_batch_iq3xxs_cm2m_cls() : ensure_kq_batch_iq3xxs_cm2s_cls()) + } verify(fmt == int(KqFmt.q8), "vk prefill: the cm2 ensure fell through for a kq format - a format the feed admits must own a tile arm") return tc == 256l ? ensure_q8_batch_cm2l_cls() : (tc == 128l ? ensure_q8_batch_cm2m_cls() : ensure_q8_batch_cm2s_cls()) } @@ -127,6 +130,12 @@ def private cm2_cls_set(fmt : int; tc : int64; bufs : uint64 const[5]; sizes : i } return tc == 128l ? set_kq_batch_iq3s_cm2m_cls(bufs, sizes, gbits) : set_kq_batch_iq3s_cm2s_cls(bufs, sizes, gbits) } + if (fmt == int(KqFmt.iq3xxs)) { + if (tc == 256l) { + return set_kq_batch_iq3xxs_cm2l_cls(bufs, sizes, gbits) + } + return tc == 128l ? set_kq_batch_iq3xxs_cm2m_cls(bufs, sizes, gbits) : set_kq_batch_iq3xxs_cm2s_cls(bufs, sizes, gbits) + } verify(fmt == int(KqFmt.q8), "vk prefill: the cm2 set fell through for a kq format - a format the feed admits must own a tile arm") if (tc == 256l) { return set_q8_batch_cm2l_cls(bufs, sizes, gbits) @@ -192,6 +201,14 @@ def private cm2_cls_enc(fmt : int; tc : int64; raw : VkCommandBuffer; var h : Vk } else { enc_kq_batch_iq3s_cm2s_cls(raw, h, s, pc, groups) } + } elif (fmt == int(KqFmt.iq3xxs)) { + if (tc == 256l) { + enc_kq_batch_iq3xxs_cm2l_cls(raw, h, s, pc, groups) + } elif (tc == 128l) { + enc_kq_batch_iq3xxs_cm2m_cls(raw, h, s, pc, groups) + } else { + enc_kq_batch_iq3xxs_cm2s_cls(raw, h, s, pc, groups) + } } elif (tc == 256l) { verify(fmt == int(KqFmt.q8), "vk prefill: the cm2 enc fell through for a kq format - a format the feed admits must own a tile arm") enc_q8_batch_cm2l_cls(raw, h, s, pc, groups) diff --git a/modules/dasLLAMA/tests/_vkd_oracles.das b/modules/dasLLAMA/tests/_vkd_oracles.das index ef45c266f1..2f823375ca 100644 --- a/modules/dasLLAMA/tests/_vkd_oracles.das +++ b/modules/dasLLAMA/tests/_vkd_oracles.das @@ -179,6 +179,12 @@ def kq_cls_ref(fmt : int; wq, ws, xq : array; xs : array) : KqGemvB ki.gridw[i] = IQ3S_GRID[i] } k = ki + } elif (fmt == int(KqFmt.iq3xxs)) { + var kx = new KqGemvIq3xxs() + for (i in range(256)) { // same replay rule: the halved grid fills here + kx.gridw[i] = IQ3XXS_GRID[i] + } + k = kx } else { k = new KqGemvK6() } @@ -325,6 +331,44 @@ def iq3s_gemv_float_oracle(wq, ws, xq : array; xs : array; return <- y } +// IQ3_XXS GEMV float oracle, INDEPENDENT of the class bit-math: w = d_halved * (2ls + 1) * +// halved_grid_byte * sign straight off the plane bytes ([qs 64][aux 32], the iq3s-shaped +// scale row), signs through the CPU-side ksigns table. +def iq3xxs_gemv_float_oracle(wq, ws, xq : array; xs : array; + regions : array; n, d, nreg : int) : array { + var y : array + y |> resize(nreg * d) + let nsb = n / 256 + for (r in range(nreg)) { + for (row in range(d)) { + let w0 = int(regions[r * 2]) + row * nsb + let x0 = int(regions[r * 2 + 1]) + var acc = 0.0 + for (s in range(nsb)) { + let sb = w0 + s + let xsb = x0 + s + let dd = half_at(ws, sb * 10) + for (blk in range(8)) { + let sc = float(byte_s8(ws, sb * 20 + 4 + blk)) // the plane byte IS the (2ls + 1) sub-scale + let aux = byte_u8(wq, sb * 96 + 64 + blk * 4) | (byte_u8(wq, sb * 96 + 64 + blk * 4 + 1) << 8) | (byte_u8(wq, sb * 96 + 64 + blk * 4 + 2) << 16) | (byte_u8(wq, sb * 96 + 64 + blk * 4 + 3) << 24) + var bacc = 0.0 + for (e in range(32)) { + let l = e / 8 + let hj = e % 8 + let qi = byte_u8(wq, sb * 96 + blk * 8 + 2 * l + hj / 4) + let g = float(int(IQ3XXS_GRID[qi] >> uint(8 * (hj % 4))) & 0xFF) + let sgn = ((int(KSIGNS_IQ2XS[(aux >> (7 * l)) & 127]) >> hj) & 1) != 0 ? -1.0 : 1.0 + bacc += g * sgn * float(byte_s8(xq, xsb * 256 + blk * 32 + e)) + } + acc += xs[xsb] * dd * sc * bacc + } + } + y[r * d + row] = acc + } + } + return <- y +} + // IQ3_S f16-feed GEMM reference over the device form: grid gather + sign flip per element, // f16-rounded per weight like its siblings ([qs 64][qh 8][signs 32] rows, the k4-shaped // scale row with signed (1 + 2s) sub-scales). @@ -359,6 +403,40 @@ def iq3sf16_gemm_oracle(wq : array; wsu : array; xf : array; } } +// IQ3_XXS f16-feed GEMM reference over the device form: halved-grid gather + ksigns sign per +// element, f16-rounded per weight like its siblings ([qs 64][aux 32] rows, the iq3s-shaped +// scale row with (2ls + 1) sub-scales and the halved d). +def iq3xxsf16_gemm_oracle(wq : array; wsu : array; xf : array; + recs : array; nrec, n, d : int; var y : array) { + let nsb = n / 256 + for (rr in range(nrec)) { + let wsb0 = int(recs[rr * 4]) + let row0 = int(recs[rr * 4 + 1]) + let cnt = int(recs[rr * 4 + 2]) + for (r in range(cnt)) { + for (c in range(d)) { + var acc = 0.0 + for (s in range(nsb)) { + let sb = wsb0 + c * nsb + s + let dv = unpackHalf2x16(wsu[sb * 5]).x + for (g in range(8)) { + let sc = float(int(wsu[sb * 5 + 1 + g / 4] << uint((3 - g % 4) * 8)) >> 24) + let aux = byte_u8(wq, sb * 96 + 64 + g * 4) | (byte_u8(wq, sb * 96 + 64 + g * 4 + 1) << 8) | (byte_u8(wq, sb * 96 + 64 + g * 4 + 2) << 16) | (byte_u8(wq, sb * 96 + 64 + g * 4 + 3) << 24) + for (e in range(32)) { + let qi = byte_u8(wq, sb * 96 + g * 8 + e / 4) + let gb = float(int(IQ3XXS_GRID[qi] >> uint(8 * (e % 4))) & 0xFF) + let sbit = (int(KSIGNS_IQ2XS[(aux >> (7 * (e / 8))) & 127]) >> (e % 8)) & 1 + let w = float(float16(dv * sc * (sbit != 0 ? -gb : gb))) + acc += w * half_at(xf, (row0 + r) * n + s * 256 + g * 32 + e) + } + } + } + y[(row0 + r) * d + c] = acc + } + } + } +} + // kq batch GEMM oracle over region records — same blk_contrib per (row, col, block) def kq_gemm_oracle(fmt : int; wq, ws, xq : array; xs : array; recs : array; nrec, n, d : int; var y : array) { diff --git a/modules/dasLLAMA/tests/test_vulkan_kernels.das b/modules/dasLLAMA/tests/test_vulkan_kernels.das index e8ab636220..f6b7e3746a 100644 --- a/modules/dasLLAMA/tests/test_vulkan_kernels.das +++ b/modules/dasLLAMA/tests/test_vulkan_kernels.das @@ -381,7 +381,7 @@ def private attn_row_oracle(q : array; qbase : int; kk : array; kb def test_vkd_kq_gemv_family(t0 : T?) { t0 |> run("kq GEMV class family == the CPU oracle (all 6 formats)") <| @(t : T?) { static_if (typeinfo builtin_module_exists(vulkan)) { - if (!(ensure_kq_gemv_k4_cls() && ensure_kq_gemv_q40_cls() && ensure_kq_gemv_k5_cls() && ensure_kq_gemv_k6_cls() && ensure_kq_gemv_iq4xs_cls() && ensure_kq_gemv_k3_cls() && ensure_kq_gemv_iq3s_cls())) { + if (!(ensure_kq_gemv_k4_cls() && ensure_kq_gemv_q40_cls() && ensure_kq_gemv_k5_cls() && ensure_kq_gemv_k6_cls() && ensure_kq_gemv_iq4xs_cls() && ensure_kq_gemv_k3_cls() && ensure_kq_gemv_iq3s_cls() && ensure_kq_gemv_iq3xxs_cls())) { feint("no Vulkan device - skipping\n") return } @@ -389,9 +389,9 @@ def test_vkd_kq_gemv_family(t0 : T?) { let d = 3 // rows per region let nreg = 2 let rows = nreg * d - let nfmt = 7 - let fmts = fixed_array(int(KqFmt.k4), int(KqFmt.q40), int(KqFmt.k5), int(KqFmt.k6), int(KqFmt.iq4xs), int(KqFmt.k3), int(KqFmt.iq3s)) - let stride4s = fixed_array(32, 32, 40, 48, 32, 24, 26) // uint WORDS per superblock (iq3s is not uint4-aligned) + let nfmt = 8 + let fmts = fixed_array(int(KqFmt.k4), int(KqFmt.q40), int(KqFmt.k5), int(KqFmt.k6), int(KqFmt.iq4xs), int(KqFmt.k3), int(KqFmt.iq3s), int(KqFmt.iq3xxs)) + let stride4s = fixed_array(32, 32, 40, 48, 32, 24, 26, 24) // uint WORDS per superblock (iq3s/iq3xxs are not uint4-aligned) for (fi in range(nfmt)) { let fmt = fmts[fi] let stride4 = stride4s[fi] @@ -467,6 +467,8 @@ def test_vkd_kq_gemv_family(t0 : T?) { enc_kq_gemv_k3_cls(raw2, h2, s_cls, pc, wgs) } elif (fmt == int(KqFmt.iq3s)) { enc_kq_gemv_iq3s_cls(raw2, h2, s_cls, pc, wgs) + } elif (fmt == int(KqFmt.iq3xxs)) { + enc_kq_gemv_iq3xxs_cls(raw2, h2, s_cls, pc, wgs) } else { enc_kq_gemv_k6_cls(raw2, h2, s_cls, pc, wgs) } @@ -497,6 +499,12 @@ def test_vkd_kq_gemv_family(t0 : T?) { t |> success(badf == 0, "iq3s: the class bit-math matches the float dequant oracle ({badf} of {rows} rows off)") delete y_flt } + if (fmt == int(KqFmt.iq3xxs)) { // the halved-grid/parity-sign gather has its own float witness + var y_flt <- iq3xxs_gemv_float_oracle(wqh, wsh, xqh, xsh, regs, 512, d, nreg) + let badf = mismatch_approx(y_ref, y_flt) + t |> success(badf == 0, "iq3xxs: the class bit-math matches the float dequant oracle ({badf} of {rows} rows off)") + delete y_flt + } delete regs delete y_ref delete wqh @@ -1017,7 +1025,7 @@ def test_vkd_q8_batch_tile(t0 : T?) { def test_vkd_kq_batch_family(t0 : T?) { t0 |> run("kq batch tile class family == the CPU oracle (all 6 formats)") <| @(t : T?) { static_if (typeinfo builtin_module_exists(vulkan)) { - if (!(ensure_kq_batch_k4_cls() && ensure_kq_batch_q40_cls() && ensure_kq_batch_k5_cls() && ensure_kq_batch_k6_cls() && ensure_kq_batch_iq4xs_cls() && ensure_kq_batch_k3_cls() && ensure_kq_batch_iq3s_cls())) { + if (!(ensure_kq_batch_k4_cls() && ensure_kq_batch_q40_cls() && ensure_kq_batch_k5_cls() && ensure_kq_batch_k6_cls() && ensure_kq_batch_iq4xs_cls() && ensure_kq_batch_k3_cls() && ensure_kq_batch_iq3s_cls() && ensure_kq_batch_iq3xxs_cls())) { feint("no Vulkan device - skipping\n") return } @@ -1032,9 +1040,9 @@ def test_vkd_kq_batch_family(t0 : T?) { let wgs1 = ((cnt1 + 31) / 32) * wtiles let wgs = wgs0 + wgs1 let totsb = 2 * d * nsb // superblocks across both regions - let nfmt = 7 - let wstrides = fixed_array(32, 32, 40, 48, 32, 24, 26) // uint words per superblock: k4 q40 k5 k6 iq4xs k3 iq3s - let pipes = fixed_array(int(KqFmt.k4), int(KqFmt.q40), int(KqFmt.k5), int(KqFmt.k6), int(KqFmt.iq4xs), int(KqFmt.k3), int(KqFmt.iq3s)) + let nfmt = 8 + let wstrides = fixed_array(32, 32, 40, 48, 32, 24, 26, 24) // uint words per superblock: k4 q40 k5 k6 iq4xs k3 iq3s iq3xxs + let pipes = fixed_array(int(KqFmt.k4), int(KqFmt.q40), int(KqFmt.k5), int(KqFmt.k6), int(KqFmt.iq4xs), int(KqFmt.k3), int(KqFmt.iq3s), int(KqFmt.iq3xxs)) for (fi in range(nfmt)) { let fmt = pipes[fi] let wstride = wstrides[fi] @@ -1113,6 +1121,8 @@ def test_vkd_kq_batch_family(t0 : T?) { enc_kq_batch_k3_cls(raw2, h2, sc, pc, int64(wgs)) } elif (fmt == int(KqFmt.iq3s)) { enc_kq_batch_iq3s_cls(raw2, h2, sc, pc, int64(wgs)) + } elif (fmt == int(KqFmt.iq3xxs)) { + enc_kq_batch_iq3xxs_cls(raw2, h2, sc, pc, int64(wgs)) } else { enc_kq_batch_k6_cls(raw2, h2, sc, pc, int64(wgs)) } @@ -2281,6 +2291,130 @@ def test_vkd_iq3s_cm2_batch(t0 : T?) { } } +[test] +def test_vkd_iq3xxs_cm2_batch(t0 : T?) { + t0 |> run("cm2 IQ3_XXS l/m/s tiles == the CPU oracle (halved grid + parity signs decode-in-load)") <| @(t : T?) { + static_if (typeinfo builtin_module_exists(vulkan)) { + if (!ensure_q8_batch_cls()) { // engage the device without touching coopmat rails + t |> skip("no Vulkan device") + return + } + if (!(g_gpu.coopmat_mode == 4 && g_gpu.has_coopmat2)) { + t |> skip("the cm2 iq3xxs tiles serve only in mode 4 on an NV_coopmat2 device") + return + } + verify(ensure_kq_batch_iq3xxs_cm2l_cls() && ensure_kq_batch_iq3xxs_cm2m_cls() && ensure_kq_batch_iq3xxs_cm2s_cls(), "iq3xxs cm2 class rails must engage in cm2 mode") + let n = 768 // one 512-unrolled pass + 4 tail steps crossing superblock bounds + let nsb = n / 256 + let d = 160 // one full 128-tile + a 32 edge + let cnt0 = 300 + let cnt1 = 260 + let rows = cnt0 + cnt1 + let totsb = 2 * d * nsb + var wqh : array + var wsuh : array + var xfh : array + wqh |> resize(totsb * 24) + wsuh |> resize(totsb * 5) + xfh |> resize((rows + 32) * n / 2) // + the s tile's 32-row read slack + for (i in range(totsb * 24)) { + wqh[i] = hash_word(uint(i) + 29u) // arbitrary quant bytes + } + for (sb in range(totsb)) { + wsuh[sb * 5] = packHalf2x16(float2(0.0002 * float(1 + sb % 7), 0.0)) // cool d + wsuh[sb * 5 + 1] = hash_word(uint(sb * 2) + 613u) // signed sub-scale bytes + wsuh[sb * 5 + 2] = hash_word(uint(sb * 2 + 1) + 613u) + wsuh[sb * 5 + 3] = 0u + wsuh[sb * 5 + 4] = 0u + } + for (i in range((rows + 32) * n / 2)) { + xfh[i] = ws_word(uint(i) * 3u + 5u) // f16 activation pairs + } + let wq_bytes = int64(totsb) * 96l + let ws_bytes = int64(length(wsuh)) * 4l + let xf_bytes = int64((rows + 32) * n) * 2l + let y_bytes = int64(rows * d) * 4l + let wqd = make_device_buf(wq_bytes) + let wsd = make_device_buf(ws_bytes) + let xfd = make_device_buf(xf_bytes) + let yd2 = make_device_buf(y_bytes) + var host = make_host_buf(y_bytes, true, [cached = true]) + var y_cls : array + y_cls |> resize(rows * d) + var y_ref : array + y_ref |> resize(rows * d) + for (ml in range(3)) { + let ttile = ml == 0 ? 256 : (ml == 1 ? 128 : 32) + let wtiles = (d + 127) / 128 + let wgs0 = ((cnt0 + ttile - 1) / ttile) * wtiles + let wgs1 = ((cnt1 + ttile - 1) / ttile) * wtiles + let wgs = wgs0 + wgs1 + var sched : array + sched |> resize(2 * 4 + wgs) + sched[0] = 0u + sched[1] = 0u + sched[2] = uint(cnt0) + sched[3] = 0u + sched[4] = uint(d * nsb) + sched[5] = uint(cnt0) + sched[6] = uint(cnt1) + sched[7] = uint(wgs0) + for (w in range(wgs)) { + sched[8 + w] = w < wgs0 ? 0u : 1u + } + unsafe { + upload_region_at(wqd, 0l, addr(wqh[0]), wq_bytes) + upload_region_at(wsd, 0l, addr(wsuh[0]), ws_bytes) + upload_region_at(xfd, 0l, addr(xfh[0]), xf_bytes) + let sc_bytes = int64(2 * 4 + wgs) * 4l + let scd = make_device_buf(sc_bytes) + upload_region_at(scd, 0l, addr(sched[0]), sc_bytes) + var sc = (ml == 0 + ? set_kq_batch_iq3xxs_cm2l_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)) + : (ml == 1 + ? set_kq_batch_iq3xxs_cm2m_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)) + : set_kq_batch_iq3xxs_cm2s_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)))) + var raw2 = alloc_cmd() + let begin = VkCommandBufferBeginInfo() + vk_check(vkBeginCommandBuffer(raw2, begin), null) + var h2 : VkHaz + var pc = BatchArgs(n = uint(n), d = uint(d), map_off = uint(2 * 4)) + if (ml == 0) { + enc_kq_batch_iq3xxs_cm2l_cls(raw2, h2, sc, pc, int64(wgs)) + } elif (ml == 1) { + enc_kq_batch_iq3xxs_cm2m_cls(raw2, h2, sc, pc, int64(wgs)) + } else { + enc_kq_batch_iq3xxs_cm2s_cls(raw2, h2, sc, pc, int64(wgs)) + } + vhz_dep(raw2, h2, 16u, 0u, true) + cmd_copy_whole(raw2, yd2, host.buf, y_bytes) + vk_check(vkEndCommandBuffer(raw2), null) + submit_wait(raw2) + memcpy(addr(y_cls[0]), host.mapped, y_bytes) + } + iq3xxsf16_gemm_oracle(wqh, wsuh, xfh, sched, 2, n, d, y_ref) + let bad = mismatch_bars(y_cls, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) + to_log(LOG_INFO, "cm2 iq3xxs {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile batch: {bad} of {rows * d} off the oracle\n") + t |> success(bad == 0, "cm2 iq3xxs {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile matches the CPU oracle ({bad} of {rows * d} off)") + delete sched + } + delete y_ref + delete wqh + delete wsuh + delete xfh + delete y_cls + } else { + t |> skip("dasVulkan not present") + } + } +} + [test] def test_vkd_k6_cm2_batch(t0 : T?) { t0 |> run("cm2 Q6_K l/m tiles == the CPU oracle (6-bit compose decode-in-load)") <| @(t : T?) { From fd1bf95585df96ce66237e4532d19d170e08c1b6 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Sun, 30 Aug 2026 22:04:30 -0700 Subject: [PATCH 26/61] dasLLAMA: IQ3_XXS on the Metal tier - the iq3s shapes over the halved grid, format closed The fourth tier in one day, and the smallest Metal slice of the arc. Blob: the "iq3xxss" arm mirrors the iq4xs 20->18 split verbatim over t.iq3xxss; quants bind at sb0*96; the % 512 off gate and kq_fmt_gpu_supported grow the format. Kernels are the iq3s shapes with three deltas: iq3xxs_gw (the 1 KB HALVED grid through the constant-table hoisting), ksign7m (the parity sign byte - the ksigns table exists on no GPU tier), and 24-word rows with one aux word per block. The GEMV ships directly in the crowned iq3s form - float4 magnitude slab, sign selects, 4 rows per simdgroup, dispatch rows/8 (QUIRK 22's race already ran for this shape); MetalKqMvIq3xxsT B2/B4 + MetalKqMvB8Iq3xxs and the IQ3XXS mul_mm arm (a gated uint[256] grid, the static_if chain re-nested once more) follow their iq3s siblings, with every compile/release/enc ladder and test cell mirrored at fmt 34. One cross-tier fix rode along: the CPU dequant/dot strip reads went SIGNED (int8 cast) to match the GPU kernels' decode - real strips are 1..31 where the readings agree, but the shared synthetic test fills carry high-bit bytes and the Metal row-ref would have diverged from the kernels over them. test_kquant/test_kqformat stay green. Gates on the M1 Max: test_metal_gemv_kernels 2/2, test_metal_gemm_kernels 2/2 (108s corpus); the e2e decodes the SAME story stream as the CPU and Vulkan tiers at gen 218 t/s (ttft 106ms). Rows vs llama.cpp b10660 (same local requant): M1 Metal pp512 3224.0 / 3429.9 (0.94x), tg128 213.5 / 227.3 (0.94x); M1 CPU pp512 906.0 / 410.5 (2.21x), tg128 53.5 / 74.0 (0.72x); zen2 CPU pp512 507.2 / 136.0 (3.73x), tg128 56.7 / 72.6 (0.78x - with the M1 CPU row, the arc's ledgered tg-tail class, followups `#60`/`#61`). Co-Authored-By: Claude Fable 5 --- modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md | 27 +- .../dasLLAMA/dasllama/dasllama_convert.das | 2 +- modules/dasLLAMA/dasllama/dasllama_layout.das | 23 +- .../dasllama/dasllama_math_default.das | 4 +- .../dasLLAMA/dasllama/dasllama_math_gen.das | 2 +- .../dasllama/dasllama_metal_common.das | 14 + .../dasllama/dasllama_metal_kernels.das | 427 ++++++++++++++++-- .../dasllama/dasllama_metal_prefill.das | 8 +- .../dasllama/dasllama_metal_shapes.das | 4 +- .../dasLLAMA/tests/_metal_kernel_common.das | 7 +- .../tests/test_metal_gemm_kernels.das | 24 +- .../tests/test_metal_gemv_kernels.das | 43 +- 12 files changed, 497 insertions(+), 88 deletions(-) diff --git a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md index cc2f66c194..15d89d5e50 100644 --- a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md +++ b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md @@ -483,7 +483,32 @@ the float witness in the family cell); the resident driver arms and matches llam greedy ids 32/64 with the fork a 0.093-logit near-tie whose top-2 IS our token (the smallest margin in the window), gen 202 t/s. Rows (5060 Ti vs llama.cpp b10660 Vulkan, the local requant): pp512 12225.7 vs 17807.7 (0.69x), tg128 372.1 vs 389.9 (0.95x). -Metal: pending (the constant-table hoisting from the iq3s arc carries the grid). +Metal closed the format the same day, and the iq3s walk paid for itself twice: the blob +"iq3xxss" arm is the iq4xs 20->18 split verbatim, `kq_quants_of` binds at sb0*96, and the +kernels are the iq3s shapes with three deltas - `iq3xxs_gw` (the 1 KB halved grid through +the constant-table hoisting), `ksign7m` (the parity sign byte - no ksigns table on Metal +either), and 24-word rows with one aux word per block. The GEMV ships directly in the +crowned iq3s form (float4 magnitude slab + sign selects, 4 rows/simdgroup, dispatch +rows/8 - QUIRK 22's race already ran); the B2/B4/B8 twins and the `IQ3XXS` mul_mm arm (a +gated uint[256], the chain re-nested once more) follow their iq3s siblings. One +cross-tier find: the CPU dequant/dot strip reads went SIGNED (int8) to match the GPU +kernels' decode - real strips are 1..31 either way, but the synthetic test fills carry +high-bit bytes and the split-form fill is shared across formats. Gates on the M1 Max: +`test_metal_gemv_kernels` 2/2, `test_metal_gemm_kernels` 2/2; e2e decodes the SAME story +stream as the CPU/Vulkan tiers at gen 218 t/s (ttft 106ms). + +Against llama.cpp b10660 (`lcpp_bench --for-debug-purposes`; zen2 = 16 threads, M1 = 8; +the local --tensor-type requant, iq3_xxs on attn_k/q + all ffn): + +| tier | pp512 das / llama.cpp | tg128 das / llama.cpp | +|---|---|---| +| zen2 CPU | 507.2 / 136.0 (3.73x) | 56.7 / 72.6 (0.78x) | +| 5060 Ti Vulkan | 12225.7 / 17807.7 (0.69x - the tier class) | 372.1 / 389.9 (0.95x) | +| M1 CPU | 906.0 / 410.5 (2.21x) | 53.5 / 74.0 (0.72x) | +| M1 Metal | 3224.0 / 3429.9 (0.94x) | 213.5 / 227.3 (0.94x) | + +(The tg tails - zen2 0.78x, M1 CPU 0.72x - are the arc's ledgered CPU-decode class, +followup_general #60/#61: the per-superblock gather never amortizes at one token.) ### IQ3_S (the third format - and the first grid format, 2026-08-30) diff --git a/modules/dasLLAMA/dasllama/dasllama_convert.das b/modules/dasLLAMA/dasllama/dasllama_convert.das index 86c9bb2c0f..7cbfe77d3e 100644 --- a/modules/dasLLAMA/dasllama/dasllama_convert.das +++ b/modules/dasLLAMA/dasllama/dasllama_convert.das @@ -747,7 +747,7 @@ def dequant_iq3xxs_plane_superblock_at(kq : array | #; kqo : int64; ks : let grid = iq3xxs_grid() let signs = ksigns_iq2xs() for (b in range64(8l)) { - let db = d * float(int(ks[kso + b])) + let db = d * float(int(int8(ks[kso + b]))) let aux = uint(kq[kqo + 64l + b * 4l]) | (uint(kq[kqo + 64l + b * 4l + 1l]) << 8u) | (uint(kq[kqo + 64l + b * 4l + 2l]) << 16u) | (uint(kq[kqo + 64l + b * 4l + 3l]) << 24u) for (l in range64(4l)) { let sgn = int(signs[int((aux >> uint(7l * l)) & 127u)]) diff --git a/modules/dasLLAMA/dasllama/dasllama_layout.das b/modules/dasLLAMA/dasllama/dasllama_layout.das index bfc140db57..fca803bf69 100644 --- a/modules/dasLLAMA/dasllama/dasllama_layout.das +++ b/modules/dasLLAMA/dasllama/dasllama_layout.das @@ -29,7 +29,7 @@ require math def private metal_blob_off_ok(off : int64; fmt : KqFmt) : bool { if (off < 0l) return true if (fmt == KqFmt.q8) return off % 256l == 0l - if (fmt == KqFmt.k6 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s) return off % 512l == 0l // the split scale form: the (off/256)*2 d-plane bind stays 4B-aligned + if (fmt == KqFmt.k6 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s || fmt == KqFmt.iq3xxs) return off % 512l == 0l // the split scale form: the (off/256)*2 d-plane bind stays 4B-aligned if (fmt == KqFmt.q51) return off % 128l == 0l // (off/32)*20 and (off/32)*4 both 16B-aligned return true } @@ -207,6 +207,25 @@ def metal_blob_scale_plane(var t : Model; name : string) : bool { } return true } + if (name == "iq3xxss") { + // iq3xxs: the row IS the iq4xs 20B shape — the same split (strips = bytes 4..19, d = bytes 0..1), over t.iq3xxss + if (!empty(t.iq3xxss)) { + var @exact_size i3xc : array + let nsb = long_length(t.iq3xxss) / IQ3XXS_SSB + i3xc |> reserve_resize(nsb * 18l) + unsafe { + var dp = addr(i3xc[0]) + let sp = addr < uint8 const? >(t.iq3xxss[0]) + for (sb in range64(nsb)) { + memcpy(reinterpret(dp + sb * 16l), reinterpret(sp + sb * IQ3XXS_SSB + 4l), 16) + dp[nsb * 16l + sb * 2l] = sp[sb * IQ3XXS_SSB] + dp[nsb * 16l + sb * 2l + 1l] = sp[sb * IQ3XXS_SSB + 1l] + } + } + t.iq3xxss <- i3xc + } + return true + } if (name == "k3s") { // k3: the CPU row IS the k6 form ([16 int8][f16 d]) — the same split, over t.k3s if (!empty(t.k3s)) { @@ -255,7 +274,7 @@ def metal_blob_commit(var t : Model) : bool { if (!metal_blob_eligible(t)) { return false } - for (nm in ["k4s", "k5s", "k6s", "iq4xss", "k3s", "iq3ss"]) { + for (nm in ["k4s", "k5s", "k6s", "iq4xss", "k3s", "iq3ss", "iq3xxss"]) { metal_blob_scale_plane(t, nm) } t.metal_blob = true diff --git a/modules/dasLLAMA/dasllama/dasllama_math_default.das b/modules/dasLLAMA/dasllama/dasllama_math_default.das index 9fa865ce63..837a0442a8 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_default.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_default.das @@ -734,7 +734,7 @@ def dot_iq3xxs_q8(kqrow : uint8 const?; ksrow : uint8 const?; xqp : int8 const?; ib += (((sgn >> int(4l + j)) & 1) != 0 ? -w2 : w2) * int(xqp[ab + l * 8l + 4l + j]) } } - isum += int(ksrow[kso + 4l + blk]) * ib + isum += int(int8(ksrow[kso + 4l + blk])) * ib } acc += float(isum) * (d * xsp[sb]) } @@ -896,7 +896,7 @@ def private dequant_iq3xxs_row_grp(kqg : uint8 const?; ksg : uint8 const?; r, mr let sb = sbi * 20l * mr let d = f16_to_f32(uint(ksg[sb + 2l * r]) | (uint(ksg[sb + 2l * r + 1l]) << 8u)) for (blk in range64(8l)) { - let db = d * float(int(ksg[sb + 4l * mr + blk * mr + r])) + let db = d * float(int(int8(ksg[sb + 4l * mr + blk * mr + r]))) var aux = 0u for (t in range64(4l)) { let ap = 64l + blk * 4l + t diff --git a/modules/dasLLAMA/dasllama/dasllama_math_gen.das b/modules/dasLLAMA/dasllama/dasllama_math_gen.das index 8f77809e15..1705e247f0 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_gen.das @@ -704,7 +704,7 @@ def iq3xxs_grp_row_dot(kqg : uint8 const?; ksg : uint8 const?; r, mr : int64; xq ib += (((sgn >> int(4l + j)) & 1) != 0 ? -w2 : w2) * int(xqp[(sbi * 8l + blk) * 32l + l * 8l + 4l + j]) } } - isum += int(ksg[sb + 4l * mr + blk * mr + r]) * ib + isum += int(int8(ksg[sb + 4l * mr + blk * mr + r])) * ib } let dk = f16_to_f32(uint(ksg[sb + 2l * r]) | (uint(ksg[sb + 2l * r + 1l]) << 8u)) acc += float(isum) * (dk * xsp[sbi]) diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_common.das b/modules/dasLLAMA/dasllama/dasllama_metal_common.das index 7f944e70bd..e8f64e80ef 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_common.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_common.das @@ -266,6 +266,11 @@ var g_pso_kq_mvb2_iq3s : MetalComputePipeline? var g_pso_kq_mvb4_iq3s : MetalComputePipeline? var g_pso_kq_mvb8_iq3s : MetalComputePipeline? var g_pso_kq_mm_b33 : MetalComputePipeline? +var g_pso_kq_iq3xxs : MetalComputePipeline? +var g_pso_kq_mvb2_iq3xxs : MetalComputePipeline? +var g_pso_kq_mvb4_iq3xxs : MetalComputePipeline? +var g_pso_kq_mvb8_iq3xxs : MetalComputePipeline? +var g_pso_kq_mm_b34 : MetalComputePipeline? var g_pso_kq_mm_b4 : MetalComputePipeline? // batch B>=9: the prefill-owned kq mul_mm twins var g_pso_kq_mm_b5 : MetalComputePipeline? var g_pso_kq_mm_b6 : MetalComputePipeline? @@ -861,6 +866,10 @@ def kq_quants_of(dev : MetalDevice?; t : Model; fmt : KqFmt; off : int64) : tupl return (buf = plane_buffer(dev, addr < void? >(t.iq3sq[0]), uint64(long_length(t.iq3sq)), t.image_map != null), qoff = uint64(sb0 * 104l)) } + if (fmt == KqFmt.iq3xxs) { + return (buf = plane_buffer(dev, addr < void? >(t.iq3xxsq[0]), uint64(long_length(t.iq3xxsq)), t.image_map != null), + qoff = uint64(sb0 * 96l)) + } return (buf = plane_buffer(dev, addr < void? >(t.k6q[0]), uint64(long_length(t.k6q)), t.image_map != null), qoff = uint64(sb0 * 192l)) } @@ -892,6 +901,11 @@ def kq_scales_of(dev : MetalDevice?; t : Model; fmt : KqFmt; off : int64) : tupl return (buf = plane_buffer(dev, addr < void? >(t.iq3ss[0]), uint64(long_length(t.iq3ss)), t.image_map != null), soff = uint64(sb0 * 16l), doff = uint64(plane_sb * 16l + sb0 * 2l)) } + if (fmt == KqFmt.iq3xxs) { // the k6 split form over the 18B/sb device plane + let plane_sb = long_length(t.iq3xxss) / 18l + return (buf = plane_buffer(dev, addr < void? >(t.iq3xxss[0]), uint64(long_length(t.iq3xxss)), t.image_map != null), + soff = uint64(sb0 * 16l), doff = uint64(plane_sb * 16l + sb0 * 2l)) + } if (fmt == KqFmt.iq4xs) { // the k6 split form over the 18B/sb device plane let plane_sb = long_length(t.iq4xss) / 18l return (buf = plane_buffer(dev, addr < void? >(t.iq4xss[0]), uint64(long_length(t.iq4xss)), t.image_map != null), diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das index 93965855b9..b1880a0cb1 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das @@ -3651,6 +3651,11 @@ def metal_decode_init : bool { // nolint:STYLE038 — a flat compile_pso per k g_pso_kq_mvb4_iq3s = compile_pso(MetalKqMvB4Iq3s_metal_kq_mv_iq3s_msl, MetalKqMvB4Iq3s_metal_kq_mv_iq3s_msl_entry, MetalKqMvB4Iq3s_metal_kq_mv_iq3s_msl_fastmath, ok) g_pso_kq_mvb8_iq3s = compile_pso(metal_kq_mvb8_iq3s_msl, metal_kq_mvb8_iq3s_msl_entry, metal_kq_mvb8_iq3s_msl_fastmath, ok) g_pso_kq_mm_b33 = compile_pso(MetalKqMulMmIq3s_metal_kq_mulmm_msl, MetalKqMulMmIq3s_metal_kq_mulmm_msl_entry, MetalKqMulMmIq3s_metal_kq_mulmm_msl_fastmath, ok) + g_pso_kq_iq3xxs = compile_pso(metal_kq_gemv_iq3xxs_msl, metal_kq_gemv_iq3xxs_msl_entry, metal_kq_gemv_iq3xxs_msl_fastmath, ok) + g_pso_kq_mvb2_iq3xxs = compile_pso(MetalKqMvB2Iq3xxs_metal_kq_mv_iq3xxs_msl, MetalKqMvB2Iq3xxs_metal_kq_mv_iq3xxs_msl_entry, MetalKqMvB2Iq3xxs_metal_kq_mv_iq3xxs_msl_fastmath, ok) + g_pso_kq_mvb4_iq3xxs = compile_pso(MetalKqMvB4Iq3xxs_metal_kq_mv_iq3xxs_msl, MetalKqMvB4Iq3xxs_metal_kq_mv_iq3xxs_msl_entry, MetalKqMvB4Iq3xxs_metal_kq_mv_iq3xxs_msl_fastmath, ok) + g_pso_kq_mvb8_iq3xxs = compile_pso(metal_kq_mvb8_iq3xxs_msl, metal_kq_mvb8_iq3xxs_msl_entry, metal_kq_mvb8_iq3xxs_msl_fastmath, ok) + g_pso_kq_mm_b34 = compile_pso(MetalKqMulMmIq3xxs_metal_kq_mulmm_msl, MetalKqMulMmIq3xxs_metal_kq_mulmm_msl_entry, MetalKqMulMmIq3xxs_metal_kq_mulmm_msl_fastmath, ok) // the batched-step set (P4) g_pso_gemv_b2 = compile_pso(MetalGemvB2_metal_gemv_msl, MetalGemvB2_metal_gemv_msl_entry, MetalGemvB2_metal_gemv_msl_fastmath, ok) g_pso_gemv_b4 = compile_pso(MetalGemvB4_metal_gemv_msl, MetalGemvB4_metal_gemv_msl_entry, MetalGemvB4_metal_gemv_msl_fastmath, ok) @@ -3723,6 +3728,8 @@ def enc_kq_gemv(enc : MetalComputeEncoder?; fmt : KqFmt; bq : MetalBuffer?; qoff enc_kq_k3_c(enc, bs, doff, bs, soff, bq, qoff, bx, xoff, by, yoff, bn, bd, rows, n) } elif (fmt == KqFmt.iq3s) { enc_kq_iq3s_c(enc, bs, doff, bs, soff, bq, qoff, bx, xoff, by, yoff, bn, bd, rows, n) + } elif (fmt == KqFmt.iq3xxs) { + enc_kq_iq3xxs_c(enc, bs, doff, bs, soff, bq, qoff, bx, xoff, by, yoff, bn, bd, rows, n) } elif (fmt == KqFmt.k6) { enc_kq_k6_c(enc, bs, doff, bs, soff, bq, qoff, bx, xoff, by, yoff, bn, bd, rows, n) } elif (fmt == KqFmt.k5 && (n < 3072l || rows <= 1024l)) { @@ -3753,7 +3760,7 @@ def private kq_b8_on : bool { return g_kq_b8_mode == 1 } -def enc_kq_mvb(enc : MetalComputeEncoder?; fmt : KqFmt; bq : MetalBuffer?; qoff : uint64; +def enc_kq_mvb(enc : MetalComputeEncoder?; fmt : KqFmt; bq : MetalBuffer?; qoff : uint64; // nolint:STYLE037 — the flat per-format ladder (one arm per kq format) is the honest shape bs : MetalBuffer?; soff, doff : uint64; bx, by : MetalBuffer?; yoff : uint64; var ka : KqMvArgs; rows, nlive : int64) { let two = nlive == 2l @@ -3783,6 +3790,14 @@ def enc_kq_mvb(enc : MetalComputeEncoder?; fmt : KqFmt; bq : MetalBuffer?; qoff } else { enc_kq_mvb4_iq3s_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows, gcols) } + } elif (fmt == KqFmt.iq3xxs) { + if (b8) { + enc_kq_mvb8_iq3xxs_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows) + } elif (two) { + enc_kq_mvb2_iq3xxs_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows) + } else { + enc_kq_mvb4_iq3xxs_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows, gcols) + } } elif (fmt == KqFmt.k6) { if (b8) { enc_kq_mvb8_k6_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows) @@ -3821,6 +3836,8 @@ def enc_kq_gemm_mm_b(enc : MetalComputeEncoder?; fmt : KqFmt; bq : MetalBuffer?; enc_kq_mm_k3_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, bk, bn, mp, rows) } elif (fmt == KqFmt.iq3s) { enc_kq_mm_iq3s_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, bk, bn, mp, rows) + } elif (fmt == KqFmt.iq3xxs) { + enc_kq_mm_iq3xxs_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, bk, bn, mp, rows) } elif (fmt == KqFmt.k6) { enc_kq_mm_k6_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, bk, bn, mp, rows) } elif (fmt == KqFmt.k4) { @@ -4262,6 +4279,11 @@ def metal_kernels_release { // nolint:STYLE037,STYLE038 — a flat release_pso release_pso(g_pso_kq_mvb4_iq3s) release_pso(g_pso_kq_mvb8_iq3s) release_pso(g_pso_kq_mm_b33) + release_pso(g_pso_kq_iq3xxs) + release_pso(g_pso_kq_mvb2_iq3xxs) + release_pso(g_pso_kq_mvb4_iq3xxs) + release_pso(g_pso_kq_mvb8_iq3xxs) + release_pso(g_pso_kq_mm_b34) if (g_pso_rms != null) { metal_release(g_pso_rms) g_pso_rms = null @@ -5490,8 +5512,10 @@ class template MetalKqMulMmK45T : MetalMmTileBase { @template_constant IQ4XS : bool = false // iq4xs: the k6 split scale form, the q40 pairing, codebook decode @template_constant K3 : bool = false // q3_K: the k6 split scale form, 2-bit lane + hmask compose, offset 4 @template_constant IQ3S : bool = false // iq3s: the k6 split scale form, threadgroup grid gather + sign flip + @template_constant IQ3XXS : bool = false // iq3xxs: the same form over the halved grid + parity signs @workgroup @template_gate = IQ4XS iq4lut : float[16] // kvalues_iq4nl as floats, staged once per threadgroup @workgroup @template_gate = IQ3S gridw3 : uint[512] // the 2 KB iq3s grid, staged once per threadgroup + @workgroup @template_gate = IQ3XXS gridw3x : uint[256] // the 1 KB halved iq3xxs grid, staged once per threadgroup [metal_kernel] def metal_kq_mulmm { // nolint:STYLE037,STYLE038 — MSL kernel body; the mul_mm tiles are simdgroup-barrier coupled @@ -5509,6 +5533,13 @@ class template MetalKqMulMmK45T : MetalMmTileBase { } barrier() } + static_if (IQ3XXS) { + for [unroll_full] (gk in range(2)) { + let gi = lid + uint(gk) * 128u + gridw3x[gi] = iq3xxs_gw(int(gi)) + } + barrier() + } let sg = gl_SubgroupID let mBase = gl_WorkGroupID.x * 32u let nBase = gl_WorkGroupID.y * 64u @@ -5602,74 +5633,93 @@ class template MetalKqMulMmK45T : MetalMmTileBase { va[k * 4 + 3] = float16(dsc3 * float(qv & 4278190080u) - dmn) } } else { - static_if (IQ3S) { - // iq3s: strip byte js is the SIGNED sub-scale (the k6 split form); block js's - // qs word (js*2 + il0) = 4 grid words = 16 elems, signs off the block's own bytes + static_if (IQ3XXS) { + // iq3xxs: the iq3s arm over the halved grid - bare byte index, parity + // signs off the block's aux32 (no table) let sv = ks4[blk] let scw = js < 4u ? sv.x : sv.y let dsc = float(ksh[blk]) * float(((int(scw >> ((js & 3u) * 8u)) & 255) ^ 128) - 128) - let qsw = kqu[blk * 26u + js * 2u + il0] - let qh3 = (kqu[blk * 26u + 16u + js / 4u] >> ((js % 4u) * 8u)) & 255u - let sgw3 = kqu[blk * 26u + 18u + js] + let qsw = kqu[blk * 24u + js * 2u + il0] + let aux3 = kqu[blk * 24u + 16u + js] for [unroll_full] (k in range(4)) { - let b3 = il0 * 4u + uint(k) - let idx = ((qsw >> (8u * uint(k))) & 255u) | ((qh3 << (8u - b3)) & 256u) - let nib = (sgw3 >> ((b3 / 2u) * 8u + (b3 % 2u) * 4u)) & 15u - let sw = iq3s_sw(gridw3[idx], nib) + let q3 = il0 * 2u + uint(k) / 2u + let sgb = ksign7m((aux3 >> (7u * q3)) & 127u) + let nib = uint(k) % 2u == 0u ? sgb & 15u : sgb >> 4u + let sw = iq3s_sw(gridw3x[(qsw >> (8u * uint(k))) & 255u], nib) for [unroll_full] (c in range(4)) { va[k * 4 + c] = float16(dsc * iq3s_sx(sw, uint(c))) } } } else { - static_if (IQ4XS) { - // iq4xs: strip byte js is the SIGNED sub-scale, super-d off the d plane (the k6 split - // form); block js = 4 uints of the q40 pairing, il0 picks the nibble half; the codebook - // off threadgroup memory + static_if (IQ3S) { + // iq3s: strip byte js is the SIGNED sub-scale (the k6 split form); block js's + // qs word (js*2 + il0) = 4 grid words = 16 elems, signs off the block's own bytes let sv = ks4[blk] let scw = js < 4u ? sv.x : sv.y let dsc = float(ksh[blk]) * float(((int(scw >> ((js & 3u) * 8u)) & 255) ^ 128) - 128) - let qb = blk * 32u + js * 4u - let nsh = il0 * 4u + let qsw = kqu[blk * 26u + js * 2u + il0] + let qh3 = (kqu[blk * 26u + 16u + js / 4u] >> ((js % 4u) * 8u)) & 255u + let sgw3 = kqu[blk * 26u + 18u + js] for [unroll_full] (k in range(4)) { - let u = kqu[qb + uint(k)] + let b3 = il0 * 4u + uint(k) + let idx = ((qsw >> (8u * uint(k))) & 255u) | ((qh3 << (8u - b3)) & 256u) + let nib = (sgw3 >> ((b3 / 2u) * 8u + (b3 % 2u) * 4u)) & 15u + let sw = iq3s_sw(gridw3[idx], nib) for [unroll_full] (c in range(4)) { - va[k * 4 + c] = float16(dsc * iq4lut[(u >> (8u * uint(c) + nsh)) & 15u]) + va[k * 4 + c] = float16(dsc * iq3s_sx(sw, uint(c))) } } } else { - let sv = ks4[blk] - var sc = 0u - var mn = 0u - if (js < 4u) { - sc = (sv.y >> (8u * js)) & 63u - mn = (sv.z >> (8u * js)) & 63u - } else { - let j2 = js - 4u - let hi = sv.w >> (8u * j2) - sc = (hi & 15u) | (((sv.y >> (8u * j2 + 6u)) & 3u) << 4u) - mn = ((hi >> 4u) & 15u) | (((sv.z >> (8u * j2 + 6u)) & 3u) << 4u) - } - let dsc = float(ksh[blk * 8u]) * float(sc) - let dmn = float(ksh[blk * 8u + 1u]) * float(mn) - let qb = blk * BLK + (js / 2u) * 8u + il0 * 4u - static_if (QH) { - let hb = blk * BLK + 32u + il0 * 4u - let nsh = (js & 1u) * 4u + static_if (IQ4XS) { + // iq4xs: strip byte js is the SIGNED sub-scale, super-d off the d plane (the k6 split + // form); block js = 4 uints of the q40 pairing, il0 picks the nibble half; the codebook + // off threadgroup memory + let sv = ks4[blk] + let scw = js < 4u ? sv.x : sv.y + let dsc = float(ksh[blk]) * float(((int(scw >> ((js & 3u) * 8u)) & 255) ^ 128) - 128) + let qb = blk * 32u + js * 4u + let nsh = il0 * 4u for [unroll_full] (k in range(4)) { let u = kqu[qb + uint(k)] - let hu = kqu[hb + uint(k)] for [unroll_full] (c in range(4)) { - let q = ((u >> (8u * uint(c) + nsh)) & 15u) | (((hu >> (8u * uint(c) + js)) & 1u) << 4u) - va[k * 4 + c] = float16(dsc * float(q) - dmn) + va[k * 4 + c] = float16(dsc * iq4lut[(u >> (8u * uint(c) + nsh)) & 15u]) } } } else { - let nsh = (js & 1u) * 4u - for [unroll_full] (k in range(4)) { - let u = kqu[qb + uint(k)] - for [unroll_full] (c in range(4)) { - let q = (u >> (8u * uint(c) + nsh)) & 15u - va[k * 4 + c] = float16(dsc * float(q) - dmn) + let sv = ks4[blk] + var sc = 0u + var mn = 0u + if (js < 4u) { + sc = (sv.y >> (8u * js)) & 63u + mn = (sv.z >> (8u * js)) & 63u + } else { + let j2 = js - 4u + let hi = sv.w >> (8u * j2) + sc = (hi & 15u) | (((sv.y >> (8u * j2 + 6u)) & 3u) << 4u) + mn = ((hi >> 4u) & 15u) | (((sv.z >> (8u * j2 + 6u)) & 3u) << 4u) + } + let dsc = float(ksh[blk * 8u]) * float(sc) + let dmn = float(ksh[blk * 8u + 1u]) * float(mn) + let qb = blk * BLK + (js / 2u) * 8u + il0 * 4u + static_if (QH) { + let hb = blk * BLK + 32u + il0 * 4u + let nsh = (js & 1u) * 4u + for [unroll_full] (k in range(4)) { + let u = kqu[qb + uint(k)] + let hu = kqu[hb + uint(k)] + for [unroll_full] (c in range(4)) { + let q = ((u >> (8u * uint(c) + nsh)) & 15u) | (((hu >> (8u * uint(c) + js)) & 1u) << 4u) + va[k * 4 + c] = float16(dsc * float(q) - dmn) + } + } + } else { + let nsh = (js & 1u) * 4u + for [unroll_full] (k in range(4)) { + let u = kqu[qb + uint(k)] + for [unroll_full] (c in range(4)) { + let q = (u >> (8u * uint(c) + nsh)) & 15u + va[k * 4 + c] = float16(dsc * float(q) - dmn) + } } } } @@ -5733,6 +5783,11 @@ class MetalKqMulMmIq3s : MetalKqMulMmK45T { override IQ3S = true } +[metal_dispatch(name = "enc_kq_mm_iq3xxs_c", pso = "g_pso_kq_mm_b34", tgmem = "MetalKqMulMmIq3xxs_metal_kq_mulmm_msl_tgmem", tg = 128, grid = "mp/32, rows/64", params = "mp : int64, rows : int64")] +class MetalKqMulMmIq3xxs : MetalKqMulMmK45T { + override IQ3XXS = true +} + // Router GEMV over the uploaded fp32 router slab: y[ne] = W[ne x dim] . x per stream. ne <= 256 // rows. Each sg serves NR positions per weight load (slab SLC traffic /NR); BATCHED stamps // clamp tails via ns; singles fold the clamp away and the gate erases ns outright. @@ -6099,6 +6154,53 @@ def private iq3s_sx(sw, t : uint) : float { return float((int((sw >> (t * 8u)) & 255u) ^ 128) - 128) } +// the HALVED iq3xxs grid (ggml's iq3xxs_grid, every byte / 2 - the plane convention), one +// word per call; the GEMV and mul_mm stage it, the mv twins read it directly. +def private iq3xxs_gw(i : int) : uint { + let tbl = fixed_array( + 0x02020202u, 0x0202020au, 0x02020212u, 0x02020606u, 0x0202060eu, 0x0202061fu, 0x02020a02u, 0x02020a0au, + 0x02020e06u, 0x0202120au, 0x02021f0eu, 0x02021f16u, 0x02060206u, 0x0206020eu, 0x02060602u, 0x0206060au, + 0x02060a06u, 0x02060a16u, 0x02060e02u, 0x02060e0au, 0x02061206u, 0x02061612u, 0x02061f02u, 0x020a0202u, + 0x020a020au, 0x020a0212u, 0x020a0606u, 0x020a0a02u, 0x020a0a0au, 0x020a0e06u, 0x020a0e0eu, 0x020a0e1fu, + 0x020a1606u, 0x020a161fu, 0x020a1f16u, 0x020e0206u, 0x020e021fu, 0x020e0602u, 0x020e060au, 0x020e0a16u, + 0x020e1f02u, 0x0212060eu, 0x02120e1fu, 0x02121212u, 0x0212161fu, 0x02121f0eu, 0x02121f16u, 0x02160206u, + 0x0216021fu, 0x02160e0au, 0x0216160au, 0x021a0e16u, 0x021a1a12u, 0x021f0602u, 0x021f0612u, 0x021f061au, + 0x021f120eu, 0x021f1a06u, 0x06020206u, 0x0602020eu, 0x06020602u, 0x0602060au, 0x06020a06u, 0x06020a0eu, + 0x06020e02u, 0x06020e0au, 0x06020e12u, 0x0602121fu, 0x06021602u, 0x06060202u, 0x0606020au, 0x06060606u, + 0x06060a02u, 0x06060a0au, 0x060a0206u, 0x060a020eu, 0x060a0602u, 0x060a060au, 0x060a0a06u, 0x060a0e02u, + 0x060a1f0au, 0x060e0202u, 0x060e020au, 0x060e0a02u, 0x060e0e06u, 0x060e121au, 0x060e1a1au, 0x06120206u, + 0x06120216u, 0x06121602u, 0x06160a02u, 0x06160a12u, 0x0616121au, 0x06161f06u, 0x061a0216u, 0x061f0a0au, + 0x061f1202u, 0x0a020202u, 0x0a02020au, 0x0a020606u, 0x0a02060eu, 0x0a020a02u, 0x0a020a0au, 0x0a020a1au, + 0x0a020e06u, 0x0a02120au, 0x0a060206u, 0x0a06020eu, 0x0a060216u, 0x0a060602u, 0x0a06060au, 0x0a060a06u, + 0x0a060e02u, 0x0a061a0eu, 0x0a061a1fu, 0x0a061f02u, 0x0a0a0202u, 0x0a0a020au, 0x0a0a0606u, 0x0a0a061fu, + 0x0a0a0a02u, 0x0a0a0a0au, 0x0a0a0e1fu, 0x0a0a1202u, 0x0a0a1616u, 0x0a0e0206u, 0x0a0e0602u, 0x0a0e0612u, + 0x0a0e1f02u, 0x0a0e1f12u, 0x0a120e16u, 0x0a12160eu, 0x0a16020eu, 0x0a160a1fu, 0x0a161206u, 0x0a161f12u, + 0x0a1f0206u, 0x0a1f020eu, 0x0a1f061au, 0x0a1f1216u, 0x0e020206u, 0x0e020602u, 0x0e02060au, 0x0e020a06u, + 0x0e020a0eu, 0x0e021602u, 0x0e021a16u, 0x0e021f0au, 0x0e060202u, 0x0e06020au, 0x0e060a02u, 0x0e060e06u, + 0x0e061212u, 0x0e06121au, 0x0e0a0206u, 0x0e0a020eu, 0x0e0a0602u, 0x0e0a0a16u, 0x0e0a160au, 0x0e0a1f0au, + 0x0e0e0606u, 0x0e0e0e0eu, 0x0e120e02u, 0x0e12121fu, 0x0e121f0au, 0x0e160202u, 0x0e16021au, 0x0e160a0au, + 0x0e161616u, 0x0e1a0612u, 0x0e1a0e1au, 0x0e1a1a0eu, 0x0e1f0e0eu, 0x0e1f1a02u, 0x12020212u, 0x1202061fu, + 0x12020e16u, 0x12020e1fu, 0x1202160eu, 0x1202161fu, 0x12061f12u, 0x120a0a02u, 0x120a0e1fu, 0x120a1202u, + 0x120a1a02u, 0x120a1a1au, 0x120e021fu, 0x120e1216u, 0x12120212u, 0x12121606u, 0x12121a12u, 0x12160a16u, + 0x1216120eu, 0x12161f02u, 0x121f0216u, 0x121f0602u, 0x121f060au, 0x121f0e02u, 0x1602060au, 0x16021206u, + 0x16021f02u, 0x16060202u, 0x1606021au, 0x16060a1au, 0x16061616u, 0x160a0612u, 0x160a0e0au, 0x160a1f0au, + 0x160e020au, 0x160e160eu, 0x16120602u, 0x16120a0eu, 0x16120a1fu, 0x16121f0au, 0x1616020au, 0x16160e06u, + 0x161a1602u, 0x161f0a12u, 0x161f120au, 0x1a020a12u, 0x1a021212u, 0x1a02121au, 0x1a021a12u, 0x1a060a06u, + 0x1a061a06u, 0x1a0a061fu, 0x1a0a1a12u, 0x1a0e0e02u, 0x1a0e0e1au, 0x1a121212u, 0x1a160216u, 0x1a16160au, + 0x1a1a0e0eu, 0x1a1f020eu, 0x1a1f0a06u, 0x1f02020eu, 0x1f020216u, 0x1f02021fu, 0x1f020602u, 0x1f020e0au, + 0x1f02160au, 0x1f060a1au, 0x1f061202u, 0x1f0a060au, 0x1f0a1216u, 0x1f0a160au, 0x1f0e0202u, 0x1f0e0616u, + 0x1f0e0e0eu, 0x1f0e1a02u, 0x1f120a06u, 0x1f121206u, 0x1f160202u, 0x1f16020au, 0x1f160a12u, 0x1f1a0e02u) + return tbl[i] +} + +// ksigns_iq2xs without the table: bit 7 = the even parity of the 7-bit index. +def private ksign7m(v : uint) : uint { + var t = v ^ (v >> 4u) + t = t ^ (t >> 2u) + t = t ^ (t >> 1u) + return v | ((t & 1u) << 7u) +} + // Expert-indexed Q4_K GEMV — MetalKqGemvK4's dot; the superblock index shifts by sel[slot] * esb. // x rides a float4 view: the 32-scalar-x-load form was load-issue-bound at the routed expert // shapes (nfe-row planes) — the vector view is 2.25x there (lab: bench_metal_moe_lab, 142→321 wGB/s). @@ -7474,6 +7576,239 @@ class MetalKqGemvIq3s { } } +// IQ3_XXS: the iq3s GEMV shape verbatim over the 24-word rows - the 1 KB halved grid as a +// float4 magnitude slab, signs from the block's aux32 through the parity helper (no table), +// the same (2ls+1)-strip x halved-d fold. 4 rows per simdgroup, dispatch rows/8. +[metal_dispatch(name = "enc_kq_iq3xxs_c", pso = "g_pso_kq_iq3xxs", tgmem = "metal_kq_gemv_iq3xxs_msl_tgmem", tg = 64, grid = "rows/8", params = "rows : int64, n : int64")] +class MetalKqGemvIq3xxs { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d plane — the scale buffer bound at byte nsb*16 (s0off = the caller's doff) + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // 16B strips: 8 sub-scale bytes (2ls+1) + 8 pad, byte view (same buffer at 0) + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array // iq3xxs quant plane, uint view (24 words per superblock: 16 qs, 8 aux) + @ssbo @binding = 3 @role = "read" @off = "xoff" @span = "n*4" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" @span = "rows*4" y : array + @uniform @binding = 5 ndim : uint + @uniform @binding = 6 ddim : uint + @workgroup gridf : float4[256] // magnitudes pre-expanded at stage time (the iq3s GEMV's winning form) + + [metal_kernel(name="metal_kq_gemv_iq3xxs_msl")] + def metal_kq_gemv_iq3xxs { + let lid = gl_LocalInvocationID.x + for [unroll_full] (k in range(4)) { + let gi = lid + uint(k) * 64u + let gw = iq3xxs_gw(int(gi)) + gridf[gi] = float4(float(gw & 255u), float((gw >> 8u) & 255u), float((gw >> 16u) & 255u), float(gw >> 24u)) + } + barrier() + let lane = gl_SubgroupInvocationID + let ix = lane / 16u + let it = lane % 16u + let bu = it / 2u + let il = it % 2u + let nb = ndim / 256u + let first_row = (gl_WorkGroupID.x * 2u + gl_SubgroupID) * 4u + var sumf : float[4] + var ib = ix + while (ib < nb) { + let yb4 = ib * 64u + bu * 8u + il * 4u + let y0 = x[yb4] + let y1 = x[yb4 + 1u] + let y2 = x[yb4 + 2u] + let y3 = x[yb4 + 3u] + for [unroll_full] (r in range(4)) { + let blk = (first_row + uint(r)) * nb + ib + let qsw = kqu[blk * 24u + bu * 2u + il] + let aux = kqu[blk * 24u + 16u + bu] + var accv = float4(0.0) + for [unroll_full] (wi in range(4)) { + let q3 = il * 2u + uint(wi) / 2u + let sgb = ksign7m((aux >> (7u * q3)) & 127u) + let nib = uint(wi) % 2u == 0u ? sgb & 15u : sgb >> 4u + let idx = (qsw >> (8u * uint(wi))) & 255u + let fs = float4((nib & 1u) != 0u ? -1.0 : 1.0, (nib & 2u) != 0u ? -1.0 : 1.0, + (nib & 4u) != 0u ? -1.0 : 1.0, (nib & 8u) != 0u ? -1.0 : 1.0) + let yv = wi == 0 ? y0 : (wi == 1 ? y1 : (wi == 2 ? y2 : y3)) + accv += yv * (gridf[idx] * fs) + } + let sc = float((int(kscb[blk * 16u + bu]) ^ 128) - 128) + sumf[r] += float(kdh[blk]) * sc * (accv.x + accv.y + accv.z + accv.w) + } + ib += 2u + } + for [unroll_full] (r in range(4)) { + let s = simd_sum(sumf[r]) + if (lane == 0u && first_row + uint(r) < ddim) { + y[first_row + uint(r)] = s + } + } + } +} + +// The iq3xxs B2/B4 pair: the iq3s shells over the 24-word rows - direct halved-grid reads, +// parity signs (unmeasured batch shapes - followup #58's stance). +[ |> template_struct_instance] +class template MetalKqMvIq3xxsT { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d plane — the scale buffer bound at byte nsb*16 + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // 16B strips: 8 sub-scale bytes (2ls+1) + 8 pad (same buffer at 0) + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array // iq3xxs quant plane, uint view + @ssbo @binding = 3 @role = "read" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" y : array + @uniform @binding = 5 ka : KqMvArgs + @template_constant NR : int = 2 + @template_constant NRU : uint = 2u // NR's uint spelling (colbase math) — override BOTH together + @template_constant TILED : bool = false + + [metal_kernel] + def metal_kq_mv_iq3xxs { + let lane = gl_SubgroupInvocationID + let tx = lane % 8u + let ty = lane / 8u + let row = gl_WorkGroupID.x * 8u + gl_SubgroupID * 4u + ty + let colbase = gl_WorkGroupID.y * NRU // nolint:LINT002 dead in the TILED=false stamp — measured free + let nb = ka.ndim / 256u + let nb4 = ka.ndim / 4u + var sumf : float[NR] + var sb = 0u + while (sb < nb) { + let blk = row * nb + sb + let dl = float(kdh[blk]) * float((int(kscb[blk * 16u + tx]) ^ 128) - 128) + let aux = kqu[blk * 24u + 16u + tx] + var w : float4[8] + for [unroll_full] (h in range(2)) { + let qsw = kqu[blk * 24u + tx * 2u + uint(h)] + for [unroll_full] (wi in range(4)) { + let q3 = uint(h) * 2u + uint(wi) / 2u + let sgb = ksign7m((aux >> (7u * q3)) & 127u) + let nib = uint(wi) % 2u == 0u ? sgb & 15u : sgb >> 4u + let sw = iq3s_sw(iq3xxs_gw(int((qsw >> (8u * uint(wi))) & 255u)), nib) + w[h * 4 + wi] = float4(iq3s_sx(sw, 0u), iq3s_sx(sw, 1u), iq3s_sx(sw, 2u), iq3s_sx(sw, 3u)) * dl + } + } + let cx4 = sb * 64u + tx * 8u + static_if (TILED) { + for [unroll_full] (b in range(NR)) { + let xb4 = (colbase + uint(b)) * nb4 + cx4 + var acc = 0.0 + for [unroll_full] (u in range(8)) { + let v = x[xb4 + uint(u)] + acc += v.x * w[u].x + v.y * w[u].y + v.z * w[u].z + v.w * w[u].w + } + sumf[b] += acc + } + } else { + for [unroll_full] (b in range(NR)) { + let xb4 = uint(b) * nb4 + cx4 + var acc = 0.0 + for [unroll_full] (u in range(8)) { + let v = x[xb4 + uint(u)] + acc += v.x * w[u].x + v.y * w[u].y + v.z * w[u].z + v.w * w[u].w + } + sumf[b] += acc + } + } + sb++ + } + static_if (TILED) { + for [unroll_full] (b in range(NR)) { + var s = sumf[b] + s += simd_shuffle_down(s, 4u) + s += simd_shuffle_down(s, 2u) + s += simd_shuffle_down(s, 1u) + if (tx == 0u && row < ka.ddim && colbase + uint(b) < ka.nr) { + y[(colbase + uint(b)) * ka.ys + row] = s + } + } + } else { + for [unroll_full] (b in range(NR)) { + var s = sumf[b] + s += simd_shuffle_down(s, 4u) + s += simd_shuffle_down(s, 2u) + s += simd_shuffle_down(s, 1u) + if (tx == 0u && row < ka.ddim && uint(b) < ka.nr) { + y[uint(b) * ka.ys + row] = s + } + } + } + } +} + +[metal_dispatch(name = "enc_kq_mvb2_iq3xxs_c", pso = "g_pso_kq_mvb2_iq3xxs", tg = 64, grid = "rows/8", params = "rows : int64")] +class MetalKqMvB2Iq3xxs : MetalKqMvIq3xxsT { + override NR = 2 + override NRU = 2u + override TILED = false +} + +[metal_dispatch(name = "enc_kq_mvb4_iq3xxs_c", pso = "g_pso_kq_mvb4_iq3xxs", tg = 64, grid = "rows/8, gcols", params = "rows : int64, gcols : int64")] +class MetalKqMvB4Iq3xxs : MetalKqMvIq3xxsT { + override NR = 4 + override NRU = 4u + override TILED = true +} + +// the iq3xxs B8 twin: the iq3s B8 shell (one superblock's X panel staged per threadgroup) +// over the 24-word rows; the grid read direct. +[metal_dispatch(name = "enc_kq_mvb8_iq3xxs_c", pso = "g_pso_kq_mvb8_iq3xxs", tgmem = "metal_kq_mvb8_iq3xxs_msl_tgmem", tg = 64, grid = "rows/8", params = "rows : int64")] +class MetalKqMvB8Iq3xxs { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d plane — the scale buffer bound at byte nsb*16 + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // 16B strips: 8 sub-scale bytes (2ls+1) + 8 pad (same buffer at 0) + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array // iq3xxs quant plane, uint view + @ssbo @binding = 3 @role = "read" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" y : array + @uniform @binding = 5 ka : KqMvArgs + @workgroup txp : float4[512] // [8 streams x 64 float4] one superblock's X panel + + [metal_kernel(name="metal_kq_mvb8_iq3xxs_msl")] + def metal_kq_mvb8_iq3xxs { + let lid = gl_LocalInvocationID.x + let lane = gl_SubgroupInvocationID + let tx = lane % 8u + let ty = lane / 8u + let row = gl_WorkGroupID.x * 8u + gl_SubgroupID * 4u + ty + let nb = ka.ndim / 256u + let nb4 = ka.ndim / 4u + var acc : float4[8] + var sb = 0u + while (sb < nb) { + barrier() + for [unroll_full] (j in range(8)) { + let slot = uint(j) * 64u + lid + txp[slot] = x[(slot / 64u) * nb4 + sb * 64u + (slot % 64u)] + } + barrier() + let blk = row * nb + sb + let dl = float(kdh[blk]) * float((int(kscb[blk * 16u + tx]) ^ 128) - 128) + let aux = kqu[blk * 24u + 16u + tx] + var w : float4[8] + for [unroll_full] (h in range(2)) { + let qsw = kqu[blk * 24u + tx * 2u + uint(h)] + for [unroll_full] (wi in range(4)) { + let q3 = uint(h) * 2u + uint(wi) / 2u + let sgb = ksign7m((aux >> (7u * q3)) & 127u) + let nib = uint(wi) % 2u == 0u ? sgb & 15u : sgb >> 4u + let sw = iq3s_sw(iq3xxs_gw(int((qsw >> (8u * uint(wi))) & 255u)), nib) + w[h * 4 + wi] = float4(iq3s_sx(sw, 0u), iq3s_sx(sw, 1u), iq3s_sx(sw, 2u), iq3s_sx(sw, 3u)) * dl + } + } + let cxl = tx * 8u + for [unroll_full] (b in range(8)) { + for [unroll_full] (u in range(8)) { + acc[b] += txp[uint(b) * 64u + cxl + uint(u)] * w[u] + } + } + sb++ + } + for [unroll_full] (b in range(8)) { + var s = acc[b].x + acc[b].y + acc[b].z + acc[b].w + s += simd_shuffle_down(s, 4u) + s += simd_shuffle_down(s, 2u) + s += simd_shuffle_down(s, 1u) + if (tx == 0u && row < ka.ddim && uint(b) < ka.nr) { + y[uint(b) * ka.ys + row] = s + } + } + } +} + // The iq3s B2/B4 pair: the iq4xs shell over the per-32-block lane map — lane tx owns block tx, // 8 grid words = 32 consecutive elems as float4s; the grid read direct (unmeasured batch shapes). [ |> template_struct_instance] diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das b/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das index db4571a8b9..211421ce20 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das @@ -4411,7 +4411,7 @@ def private pf_devw_panel(enc : MetalComputeEncoder?; bwblob : MetalBuffer?; wbo //! quant superblocks, plus the k6 2B d tail def private pf_devw_panel_kq(enc : MetalComputeEncoder?; t : Model; fmt : KqFmt; woff : int64; bxh, by, bk, bn : MetalBuffer?; rows, d, kdim : int64; yoff : uint64) : bool { - if (fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s) { // no dev-W dequant kernel for these formats yet + if (fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s || fmt == KqFmt.iq3xxs) { // no dev-W dequant kernel for these formats yet return false } let dq = fmt == KqFmt.k6 ? g_pf_pso_dq_k6 : (fmt == KqFmt.k4 ? g_pf_pso_dq_k4 : g_pf_pso_dq_k5) @@ -4588,6 +4588,10 @@ def private pf_enc_kq_site_mm(enc : MetalComputeEncoder?; t : Model; fmt : KqFmt enc_kq_mm_iq3s_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, by, yoff, bk, bn, mp, rows) return } + if (fmt == KqFmt.iq3xxs) { // the base mul_mm only - no tensor / tall / dev-W twins yet + enc_kq_mm_iq3xxs_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, by, yoff, bk, bn, mp, rows) + return + } //! deep-class arm: a panel dev-W would split past 8 tiles is DRAM-resident - the tall //! in-kernel-dequant stamp reads the quant plane once per 128-row tile instead of //! materializing and re-streaming a 2B/element f16 panel @@ -4697,6 +4701,8 @@ def private pf_enc_kq_gemv(enc : MetalComputeEncoder?; t : Model; fmt : KqFmt; w enc_kq_k3_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, xoff, by, 0ul, bn, bd, rows, n) } elif (fmt == KqFmt.iq3s) { enc_kq_iq3s_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, xoff, by, 0ul, bn, bd, rows, n) + } elif (fmt == KqFmt.iq3xxs) { + enc_kq_iq3xxs_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, xoff, by, 0ul, bn, bd, rows, n) } elif (fmt == KqFmt.k6) { enc_kq_k6_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, xoff, by, 0ul, bn, bd, rows, n) } elif (fmt == KqFmt.k4) { diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_shapes.das b/modules/dasLLAMA/dasllama/dasllama_metal_shapes.das index f9068b343f..f3619ff909 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_shapes.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_shapes.das @@ -180,7 +180,7 @@ def record_needs(var tab : table; missing : MetalNeed) { // formats) must DECLINE kquant_native instead of falling into a wrong-layout kernel branch // (the dispatchers below treat "not k4/k6" as k5) def kq_fmt_gpu_supported(f : KqFmt) : bool { - return f == KqFmt.q8 || f == KqFmt.k4 || f == KqFmt.k5 || f == KqFmt.k6 || f == KqFmt.iq4xs || f == KqFmt.k3 || f == KqFmt.iq3s + return f == KqFmt.q8 || f == KqFmt.k4 || f == KqFmt.k5 || f == KqFmt.k6 || f == KqFmt.iq4xs || f == KqFmt.k3 || f == KqFmt.iq3s || f == KqFmt.iq3xxs } def private kq_fmts_gpu_supported(a : array) : bool { @@ -243,7 +243,7 @@ def moe_site_ok(fmt : KqFmt; off, ege : int64) : bool { if (fmt == KqFmt.k4 || fmt == KqFmt.k5) { return off >= 0l && (off % 256l) == 0l && (ege % 256l) == 0l } - if (fmt == KqFmt.k6 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s) { // the split scale form: off % 512 keeps the d-plane bind 4B-aligned + if (fmt == KqFmt.k6 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s || fmt == KqFmt.iq3xxs) { // the split scale form: off % 512 keeps the d-plane bind 4B-aligned return off >= 0l && (off % 512l) == 0l && (ege % 256l) == 0l } if (fmt == KqFmt.q51) { // per-32 planes: off % 128 keeps the 20B/4B binds 16B-aligned diff --git a/modules/dasLLAMA/tests/_metal_kernel_common.das b/modules/dasLLAMA/tests/_metal_kernel_common.das index 921de1a395..34cf086fbe 100644 --- a/modules/dasLLAMA/tests/_metal_kernel_common.das +++ b/modules/dasLLAMA/tests/_metal_kernel_common.das @@ -174,12 +174,12 @@ let KQ_SAFE_SCALES = fixed_array(0.25, 0.5, 1.0, 0.125, 2.0, 0.75, 1.5, 0.375) // [12 packed 6-bit bytes] per superblock, K6 = [16B sub-scales] x nsb + f16 d tail, IQ4_XS (44) = // the same split form with [8 signed sub-scales in -31..31][8 zero] strips, Q3_K (3) = the k6 form over 96B quants def kq_fill_planes(fmt, nsb : int; var kq : array; var ks : array) { - let qsb = fmt == 4 || fmt == 44 ? 128 : (fmt == 5 ? 160 : (fmt == 3 ? 96 : (fmt == 33 ? 104 : 192))) + let qsb = fmt == 4 || fmt == 44 ? 128 : (fmt == 5 ? 160 : (fmt == 3 ? 96 : (fmt == 33 ? 104 : (fmt == 34 ? 96 : 192)))) kq |> resize(nsb * qsb) for (i in range(nsb * qsb)) { kq[i] = kq_hash_byte(i * 3 + fmt) } - if (fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33) { + if (fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34) { ks |> resize(nsb * 18) for (i in range(nsb * 16)) { ks[i] = fmt == 6 || fmt == 3 ? kq_hash_byte(i * 7 + 13) : uint8(i % 16 < 8 ? int(kq_hash_byte(i * 7 + 13)) % 63 - 31 : 0) // 44/33: signed strip bytes + zero pad @@ -254,6 +254,9 @@ def kq_row_ref(fmt, sb_base, nb, nsb_total : int; kq, ks : array; var wro } elif (fmt == 33) { dequant_iq3s_plane_superblock_at(kq, int64(sb * 104), ks, int64(sb * 16), int64(nsb_total * 16 + sb * 2), wrow, int64(ib * 256)) + } elif (fmt == 34) { + dequant_iq3xxs_plane_superblock_at(kq, int64(sb * 96), ks, int64(sb * 16), + int64(nsb_total * 16 + sb * 2), wrow, int64(ib * 256)) } else { dequant_k6_plane_superblock_at(kq, int64(sb * 192), ks, int64(sb * 16), int64(nsb_total * 16 + sb * 2), wrow, int64(ib * 256)) diff --git a/modules/dasLLAMA/tests/test_metal_gemm_kernels.das b/modules/dasLLAMA/tests/test_metal_gemm_kernels.das index ae2fd904a5..f2e330efe8 100644 --- a/modules/dasLLAMA/tests/test_metal_gemm_kernels.das +++ b/modules/dasLLAMA/tests/test_metal_gemm_kernels.das @@ -36,28 +36,28 @@ def private kq_mulmm_gate(t : T?; dev, queue; fmt : int; tensor : bool; m, kdim, : halfx ? (fmt == 4 ? MetalKqMulMmK4TH_metal_kq_mulmm_t_msl : (fmt == 5 ? MetalKqMulMmK5TH_metal_kq_mulmm_t_msl : MetalKqMulMmK6TH_metal_kq_mulmm_k6_t_msl)) : (fmt == 4 ? MetalKqMulMmK4T_metal_kq_mulmm_t_msl : (fmt == 5 ? MetalKqMulMmK5T_metal_kq_mulmm_t_msl : MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl))) - : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl : MetalKqMulMmK6_metal_kq_mulmm_msl)))))) + : (fmt == 34 ? MetalKqMulMmIq3xxs_metal_kq_mulmm_msl : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl : MetalKqMulMmK6_metal_kq_mulmm_msl))))))) let entry = (tensor ? (tall ? (fmt == 4 ? MetalKqMulMmK4TH128_metal_kq_mulmm_t_msl_entry : (fmt == 5 ? MetalKqMulMmK5TH128_metal_kq_mulmm_t_msl_entry : MetalKqMulMmK6TH128_metal_kq_mulmm_k6_t_msl_entry)) : halfx ? (fmt == 4 ? MetalKqMulMmK4TH_metal_kq_mulmm_t_msl_entry : (fmt == 5 ? MetalKqMulMmK5TH_metal_kq_mulmm_t_msl_entry : MetalKqMulMmK6TH_metal_kq_mulmm_k6_t_msl_entry)) : (fmt == 4 ? MetalKqMulMmK4T_metal_kq_mulmm_t_msl_entry : (fmt == 5 ? MetalKqMulMmK5T_metal_kq_mulmm_t_msl_entry : MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl_entry))) - : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl_entry : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl_entry : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl_entry : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_entry : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_entry : MetalKqMulMmK6_metal_kq_mulmm_msl_entry)))))) + : (fmt == 34 ? MetalKqMulMmIq3xxs_metal_kq_mulmm_msl_entry : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl_entry : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl_entry : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl_entry : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_entry : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_entry : MetalKqMulMmK6_metal_kq_mulmm_msl_entry))))))) let fm = (tensor ? (tall ? (fmt == 4 ? MetalKqMulMmK4TH128_metal_kq_mulmm_t_msl_fastmath : (fmt == 5 ? MetalKqMulMmK5TH128_metal_kq_mulmm_t_msl_fastmath : MetalKqMulMmK6TH128_metal_kq_mulmm_k6_t_msl_fastmath)) : halfx ? (fmt == 4 ? MetalKqMulMmK4TH_metal_kq_mulmm_t_msl_fastmath : (fmt == 5 ? MetalKqMulMmK5TH_metal_kq_mulmm_t_msl_fastmath : MetalKqMulMmK6TH_metal_kq_mulmm_k6_t_msl_fastmath)) : (fmt == 4 ? MetalKqMulMmK4T_metal_kq_mulmm_t_msl_fastmath : (fmt == 5 ? MetalKqMulMmK5T_metal_kq_mulmm_t_msl_fastmath : MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl_fastmath))) - : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl_fastmath : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl_fastmath : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl_fastmath : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_fastmath : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_fastmath : MetalKqMulMmK6_metal_kq_mulmm_msl_fastmath)))))) + : (fmt == 34 ? MetalKqMulMmIq3xxs_metal_kq_mulmm_msl_fastmath : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl_fastmath : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl_fastmath : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl_fastmath : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_fastmath : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_fastmath : MetalKqMulMmK6_metal_kq_mulmm_msl_fastmath))))))) let tgm = (tensor ? (tall ? (fmt == 4 ? MetalKqMulMmK4TH128_metal_kq_mulmm_t_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5TH128_metal_kq_mulmm_t_msl_tgmem : MetalKqMulMmK6TH128_metal_kq_mulmm_k6_t_msl_tgmem)) : halfx ? (fmt == 4 ? MetalKqMulMmK4TH_metal_kq_mulmm_t_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5TH_metal_kq_mulmm_t_msl_tgmem : MetalKqMulMmK6TH_metal_kq_mulmm_k6_t_msl_tgmem)) : (fmt == 4 ? MetalKqMulMmK4T_metal_kq_mulmm_t_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5T_metal_kq_mulmm_t_msl_tgmem : MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl_tgmem))) - : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl_tgmem : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl_tgmem : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl_tgmem : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_tgmem : MetalKqMulMmK6_metal_kq_mulmm_msl_tgmem)))))) + : (fmt == 34 ? MetalKqMulMmIq3xxs_metal_kq_mulmm_msl_tgmem : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl_tgmem : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl_tgmem : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl_tgmem : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_tgmem : MetalKqMulMmK6_metal_kq_mulmm_msl_tgmem))))))) var pso = pipeline_from_source(dev, src, entry, fm, err) if (tensor && pso == null && !metal4_tensor_available(dev)) { to_log(LOG_INFO, "{tag}: no Metal-4 tensor toolchain on this box - arm skipped\n") @@ -123,7 +123,7 @@ def private kq_mulmm_gate(t : T?; dev, queue; fmt : int; tensor : bool; m, kdim, let ran = with_compute_encoder(queue, err) $(enc : MetalComputeEncoder?) { metal_set_pipeline(enc, pso) metal_set_threadgroup_memory_length(enc, tgm, 0) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs: the f16 d tail + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs: the f16 d tail metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bx, 0ul, 3) @@ -134,7 +134,7 @@ def private kq_mulmm_gate(t : T?; dev, queue; fmt : int; tensor : bool; m, kdim, if (rem_pso != null) { metal_set_pipeline(enc, rem_pso) metal_set_threadgroup_memory_length(enc, fmt == 4 ? MetalKqMulMmK4TH_metal_kq_mulmm_t_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5TH_metal_kq_mulmm_t_msl_tgmem : MetalKqMulMmK6TH_metal_kq_mulmm_k6_t_msl_tgmem), 0) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 ? uint64(nsb * 16) : 0ul, 0) + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 ? uint64(nsb * 16) : 0ul, 0) metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bx, uint64(r0 * kdim * 2), 3) @@ -484,7 +484,7 @@ def private kq_devw_gate(t : T?; dev, queue; fmt : int; m, kdim, ndim : int) { var dka = CvtArgs(total = uint(nblk)) let ran = with_compute_encoder(queue, err) $(enc : MetalComputeEncoder?) { metal_set_pipeline(enc, dq_pso) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs: the f16 d tail + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs: the f16 d tail metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bwh, 0ul, 3) @@ -1461,7 +1461,7 @@ def private moe_mulmm_kq_gate(t : T?; dev, queue; fmt : int; kdim, ndim : int; c let ran = with_compute_encoder(queue, err) $(enc : MetalComputeEncoder?) { metal_set_pipeline(enc, pso) metal_set_threadgroup_memory_length(enc, tgm, 0) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs: the f16 d tail + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs: the f16 d tail metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bx, 0ul, 3) @@ -1498,7 +1498,7 @@ def private moe_mulmm_kq_gate(t : T?; dev, queue; fmt : int; kdim, ndim : int; c metal_set_threadgroup_memory_length(enc, fmt == 4 ? MetalMoeMulMmK4TH_metal_moe_mulmm_k45_t_msl_tgmem : (fmt == 5 ? MetalMoeMulMmK5TH_metal_moe_mulmm_k45_t_msl_tgmem : MetalMoeMulMmK6TH_metal_moe_mulmm_k6_t_msl_tgmem), 0) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 ? uint64(nsb * 16) : 0ul, 0) + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 ? uint64(nsb * 16) : 0ul, 0) metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bxh, 0ul, 3) @@ -1538,7 +1538,7 @@ def private moe_mulmm_kq_gate(t : T?; dev, queue; fmt : int; kdim, ndim : int; c metal_set_threadgroup_memory_length(enc, fmt == 4 ? MetalMoeMulMmK4TH128_metal_moe_mulmm_k45_t_msl_tgmem : (fmt == 5 ? MetalMoeMulMmK5TH128_metal_moe_mulmm_k45_t_msl_tgmem : MetalMoeMulMmK6TH128_metal_moe_mulmm_k6_t_msl_tgmem), 0) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 ? uint64(nsb * 16) : 0ul, 0) + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 ? uint64(nsb * 16) : 0ul, 0) metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bxh, 0ul, 3) @@ -1551,7 +1551,7 @@ def private moe_mulmm_kq_gate(t : T?; dev, queue; fmt : int; kdim, ndim : int; c metal_set_threadgroup_memory_length(enc, fmt == 4 ? MetalMoeMulMmK4THR_metal_moe_mulmm_k45_t_msl_tgmem : (fmt == 5 ? MetalMoeMulMmK5THR_metal_moe_mulmm_k45_t_msl_tgmem : MetalMoeMulMmK6THR_metal_moe_mulmm_k6_t_msl_tgmem), 0) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 ? uint64(nsb * 16) : 0ul, 0) + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 ? uint64(nsb * 16) : 0ul, 0) metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bxh, 0ul, 3) @@ -2053,6 +2053,8 @@ def test_metal_gemm_kernels(t : T?) { kq_mulmm_gate(t, dev, queue, 3, false, 32, 256, 128) kq_mulmm_gate(t, dev, queue, 33, false, 64, 512, 64) // iq3s: the base form only (no tensor / tall twins) kq_mulmm_gate(t, dev, queue, 33, false, 32, 256, 128) + kq_mulmm_gate(t, dev, queue, 34, false, 64, 512, 64) // iq3xxs: the base form only (no tensor / tall twins) + kq_mulmm_gate(t, dev, queue, 34, false, 32, 256, 128) q8_mulmm_t_gate(t, dev, queue, 64, 256, 64) q8_mulmm_t_gate(t, dev, queue, 64, 256, 64, halfx = true) q8_mulmm_t_gate(t, dev, queue, 64, 160, 64) // kdim % 64 == 32: the bk=64 helper's 32-tail chunk diff --git a/modules/dasLLAMA/tests/test_metal_gemv_kernels.das b/modules/dasLLAMA/tests/test_metal_gemv_kernels.das index 5a5fb13e6b..790191b0a8 100644 --- a/modules/dasLLAMA/tests/test_metal_gemv_kernels.das +++ b/modules/dasLLAMA/tests/test_metal_gemv_kernels.das @@ -23,12 +23,12 @@ require math // ===== single-stream kq GEMV (MetalKqGemvK4 / K5 / K5C / K6) ===== def private kq_gemv_gate(t : T?; dev, queue; vform : string; n, d : int) { - let fmt = vform == "k4" ? 4 : (vform == "k6" ? 6 : (vform == "iq4xs" ? 44 : (vform == "k3" ? 3 : (vform == "iq3s" ? 33 : 5)))) + let fmt = vform == "k4" ? 4 : (vform == "k6" ? 6 : (vform == "iq4xs" ? 44 : (vform == "k3" ? 3 : (vform == "iq3s" ? 33 : (vform == "iq3xxs" ? 34 : 5))))) let tag = "kq_gemv_{vform} n={n} d={d}" var err : string - let src = vform == "iq3s" ? metal_kq_gemv_iq3s_msl : (vform == "k3" ? metal_kq_gemv_k3_msl : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl : (vform == "k4" ? metal_kq_gemv_k4_msl : (vform == "k5" ? metal_kq_gemv_k5_msl : (vform == "k5c" ? metal_kq_gemv_k5c_msl : metal_kq_gemv_k6_msl))))) - let entry = vform == "iq3s" ? metal_kq_gemv_iq3s_msl_entry : (vform == "k3" ? metal_kq_gemv_k3_msl_entry : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl_entry : (vform == "k4" ? metal_kq_gemv_k4_msl_entry : (vform == "k5" ? metal_kq_gemv_k5_msl_entry : (vform == "k5c" ? metal_kq_gemv_k5c_msl_entry : metal_kq_gemv_k6_msl_entry))))) - let fm = vform == "iq3s" ? metal_kq_gemv_iq3s_msl_fastmath : (vform == "k3" ? metal_kq_gemv_k3_msl_fastmath : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl_fastmath : (vform == "k4" ? metal_kq_gemv_k4_msl_fastmath : (vform == "k5" ? metal_kq_gemv_k5_msl_fastmath : (vform == "k5c" ? metal_kq_gemv_k5c_msl_fastmath : metal_kq_gemv_k6_msl_fastmath))))) + let src = vform == "iq3xxs" ? metal_kq_gemv_iq3xxs_msl : (vform == "iq3s" ? metal_kq_gemv_iq3s_msl : (vform == "k3" ? metal_kq_gemv_k3_msl : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl : (vform == "k4" ? metal_kq_gemv_k4_msl : (vform == "k5" ? metal_kq_gemv_k5_msl : (vform == "k5c" ? metal_kq_gemv_k5c_msl : metal_kq_gemv_k6_msl)))))) + let entry = vform == "iq3xxs" ? metal_kq_gemv_iq3xxs_msl_entry : (vform == "iq3s" ? metal_kq_gemv_iq3s_msl_entry : (vform == "k3" ? metal_kq_gemv_k3_msl_entry : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl_entry : (vform == "k4" ? metal_kq_gemv_k4_msl_entry : (vform == "k5" ? metal_kq_gemv_k5_msl_entry : (vform == "k5c" ? metal_kq_gemv_k5c_msl_entry : metal_kq_gemv_k6_msl_entry)))))) + let fm = vform == "iq3xxs" ? metal_kq_gemv_iq3xxs_msl_fastmath : (vform == "iq3s" ? metal_kq_gemv_iq3s_msl_fastmath : (vform == "k3" ? metal_kq_gemv_k3_msl_fastmath : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl_fastmath : (vform == "k4" ? metal_kq_gemv_k4_msl_fastmath : (vform == "k5" ? metal_kq_gemv_k5_msl_fastmath : (vform == "k5c" ? metal_kq_gemv_k5c_msl_fastmath : metal_kq_gemv_k6_msl_fastmath)))))) var pso = pipeline_from_source(dev, src, entry, fm, err) t |> success(pso != null, "{tag}: pipeline: {err}") return if (pso == null) @@ -66,7 +66,7 @@ def private kq_gemv_gate(t : T?; dev, queue; vform : string; n, d : int) { var by = buf_fill(dev, d, -1000.0) var bn = buf_u32(dev, uint(n)) var bd = buf_u32(dev, uint(d)) - let groups = vform == "k5c" ? (d + 1) / 2 : (vform == "iq3s" ? (d + 7) / 8 : (d + 3) / 4) + let groups = vform == "k5c" ? (d + 1) / 2 : (vform == "iq3s" || vform == "iq3xxs" ? (d + 7) / 8 : (d + 3) / 4) let ran = with_compute_encoder(queue, err) $(enc : MetalComputeEncoder?) { metal_set_pipeline(enc, pso) if (vform == "iq4xs") { @@ -75,7 +75,10 @@ def private kq_gemv_gate(t : T?; dev, queue; vform : string; n, d : int) { if (vform == "iq3s") { metal_set_threadgroup_memory_length(enc, metal_kq_gemv_iq3s_msl_tgmem, 0) // the grid slab } - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs/k3: the f16 d tail + if (vform == "iq3xxs") { + metal_set_threadgroup_memory_length(enc, metal_kq_gemv_iq3xxs_msl_tgmem, 0) // the halved-grid slab + } + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs/k3: the f16 d tail metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bx, 0ul, 3) @@ -108,15 +111,15 @@ def private kq_gemv_gate(t : T?; dev, queue; vform : string; n, d : int) { def private kq_mvb_gate(t : T?; dev, queue; fmt, bwidth, n, d, nr, ys : int) { let tag = "kq_mvb{bwidth}_k{fmt} n={n} d={d} nr={nr} ys={ys}" var err : string - let src = (bwidth == 2 ? (fmt == 33 ? MetalKqMvB2Iq3s_metal_kq_mv_iq3s_msl : (fmt == 3 ? MetalKqMvB2K3_metal_kq_mv_k3_msl : (fmt == 44 ? MetalKqMvB2Iq4xs_metal_kq_mv_iq4xs_msl : (fmt == 4 ? MetalKqMvB2K4_metal_kq_mv_k4_msl : (fmt == 5 ? MetalKqMvB2K5_metal_kq_mv_k5_msl : MetalKqMvB2K6_metal_kq_mv_k6_msl))))) - : (bwidth == 4 ? (fmt == 33 ? MetalKqMvB4Iq3s_metal_kq_mv_iq3s_msl : (fmt == 3 ? MetalKqMvB4K3_metal_kq_mv_k3_msl : (fmt == 44 ? MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl : (fmt == 4 ? MetalKqMvB4K4_metal_kq_mv_k4_msl : (fmt == 5 ? MetalKqMvB4K5_metal_kq_mv_k5_msl : MetalKqMvB4K6_metal_kq_mv_k6_msl))))) - : (fmt == 33 ? metal_kq_mvb8_iq3s_msl : (fmt == 3 ? metal_kq_mvb8_k3_msl : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl : metal_kq_mvb8_k6_msl))))))) - let entry = (bwidth == 2 ? (fmt == 33 ? MetalKqMvB2Iq3s_metal_kq_mv_iq3s_msl_entry : (fmt == 3 ? MetalKqMvB2K3_metal_kq_mv_k3_msl_entry : (fmt == 44 ? MetalKqMvB2Iq4xs_metal_kq_mv_iq4xs_msl_entry : (fmt == 4 ? MetalKqMvB2K4_metal_kq_mv_k4_msl_entry : (fmt == 5 ? MetalKqMvB2K5_metal_kq_mv_k5_msl_entry : MetalKqMvB2K6_metal_kq_mv_k6_msl_entry))))) - : (bwidth == 4 ? (fmt == 33 ? MetalKqMvB4Iq3s_metal_kq_mv_iq3s_msl_entry : (fmt == 3 ? MetalKqMvB4K3_metal_kq_mv_k3_msl_entry : (fmt == 44 ? MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl_entry : (fmt == 4 ? MetalKqMvB4K4_metal_kq_mv_k4_msl_entry : (fmt == 5 ? MetalKqMvB4K5_metal_kq_mv_k5_msl_entry : MetalKqMvB4K6_metal_kq_mv_k6_msl_entry))))) - : (fmt == 33 ? metal_kq_mvb8_iq3s_msl_entry : (fmt == 3 ? metal_kq_mvb8_k3_msl_entry : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl_entry : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_entry : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_entry : metal_kq_mvb8_k6_msl_entry))))))) - let fm = (bwidth == 2 ? (fmt == 33 ? MetalKqMvB2Iq3s_metal_kq_mv_iq3s_msl_fastmath : (fmt == 3 ? MetalKqMvB2K3_metal_kq_mv_k3_msl_fastmath : (fmt == 44 ? MetalKqMvB2Iq4xs_metal_kq_mv_iq4xs_msl_fastmath : (fmt == 4 ? MetalKqMvB2K4_metal_kq_mv_k4_msl_fastmath : (fmt == 5 ? MetalKqMvB2K5_metal_kq_mv_k5_msl_fastmath : MetalKqMvB2K6_metal_kq_mv_k6_msl_fastmath))))) - : (bwidth == 4 ? (fmt == 33 ? MetalKqMvB4Iq3s_metal_kq_mv_iq3s_msl_fastmath : (fmt == 3 ? MetalKqMvB4K3_metal_kq_mv_k3_msl_fastmath : (fmt == 44 ? MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl_fastmath : (fmt == 4 ? MetalKqMvB4K4_metal_kq_mv_k4_msl_fastmath : (fmt == 5 ? MetalKqMvB4K5_metal_kq_mv_k5_msl_fastmath : MetalKqMvB4K6_metal_kq_mv_k6_msl_fastmath))))) - : (fmt == 33 ? metal_kq_mvb8_iq3s_msl_fastmath : (fmt == 3 ? metal_kq_mvb8_k3_msl_fastmath : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl_fastmath : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_fastmath : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_fastmath : metal_kq_mvb8_k6_msl_fastmath))))))) + let src = (bwidth == 2 ? (fmt == 34 ? MetalKqMvB2Iq3xxs_metal_kq_mv_iq3xxs_msl : (fmt == 33 ? MetalKqMvB2Iq3s_metal_kq_mv_iq3s_msl : (fmt == 3 ? MetalKqMvB2K3_metal_kq_mv_k3_msl : (fmt == 44 ? MetalKqMvB2Iq4xs_metal_kq_mv_iq4xs_msl : (fmt == 4 ? MetalKqMvB2K4_metal_kq_mv_k4_msl : (fmt == 5 ? MetalKqMvB2K5_metal_kq_mv_k5_msl : MetalKqMvB2K6_metal_kq_mv_k6_msl)))))) + : (bwidth == 4 ? (fmt == 34 ? MetalKqMvB4Iq3xxs_metal_kq_mv_iq3xxs_msl : (fmt == 33 ? MetalKqMvB4Iq3s_metal_kq_mv_iq3s_msl : (fmt == 3 ? MetalKqMvB4K3_metal_kq_mv_k3_msl : (fmt == 44 ? MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl : (fmt == 4 ? MetalKqMvB4K4_metal_kq_mv_k4_msl : (fmt == 5 ? MetalKqMvB4K5_metal_kq_mv_k5_msl : MetalKqMvB4K6_metal_kq_mv_k6_msl)))))) + : (fmt == 34 ? metal_kq_mvb8_iq3xxs_msl : (fmt == 33 ? metal_kq_mvb8_iq3s_msl : (fmt == 3 ? metal_kq_mvb8_k3_msl : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl : metal_kq_mvb8_k6_msl)))))))) + let entry = (bwidth == 2 ? (fmt == 34 ? MetalKqMvB2Iq3xxs_metal_kq_mv_iq3xxs_msl_entry : (fmt == 33 ? MetalKqMvB2Iq3s_metal_kq_mv_iq3s_msl_entry : (fmt == 3 ? MetalKqMvB2K3_metal_kq_mv_k3_msl_entry : (fmt == 44 ? MetalKqMvB2Iq4xs_metal_kq_mv_iq4xs_msl_entry : (fmt == 4 ? MetalKqMvB2K4_metal_kq_mv_k4_msl_entry : (fmt == 5 ? MetalKqMvB2K5_metal_kq_mv_k5_msl_entry : MetalKqMvB2K6_metal_kq_mv_k6_msl_entry)))))) + : (bwidth == 4 ? (fmt == 34 ? MetalKqMvB4Iq3xxs_metal_kq_mv_iq3xxs_msl_entry : (fmt == 33 ? MetalKqMvB4Iq3s_metal_kq_mv_iq3s_msl_entry : (fmt == 3 ? MetalKqMvB4K3_metal_kq_mv_k3_msl_entry : (fmt == 44 ? MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl_entry : (fmt == 4 ? MetalKqMvB4K4_metal_kq_mv_k4_msl_entry : (fmt == 5 ? MetalKqMvB4K5_metal_kq_mv_k5_msl_entry : MetalKqMvB4K6_metal_kq_mv_k6_msl_entry)))))) + : (fmt == 34 ? metal_kq_mvb8_iq3xxs_msl_entry : (fmt == 33 ? metal_kq_mvb8_iq3s_msl_entry : (fmt == 3 ? metal_kq_mvb8_k3_msl_entry : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl_entry : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_entry : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_entry : metal_kq_mvb8_k6_msl_entry)))))))) + let fm = (bwidth == 2 ? (fmt == 34 ? MetalKqMvB2Iq3xxs_metal_kq_mv_iq3xxs_msl_fastmath : (fmt == 33 ? MetalKqMvB2Iq3s_metal_kq_mv_iq3s_msl_fastmath : (fmt == 3 ? MetalKqMvB2K3_metal_kq_mv_k3_msl_fastmath : (fmt == 44 ? MetalKqMvB2Iq4xs_metal_kq_mv_iq4xs_msl_fastmath : (fmt == 4 ? MetalKqMvB2K4_metal_kq_mv_k4_msl_fastmath : (fmt == 5 ? MetalKqMvB2K5_metal_kq_mv_k5_msl_fastmath : MetalKqMvB2K6_metal_kq_mv_k6_msl_fastmath)))))) + : (bwidth == 4 ? (fmt == 34 ? MetalKqMvB4Iq3xxs_metal_kq_mv_iq3xxs_msl_fastmath : (fmt == 33 ? MetalKqMvB4Iq3s_metal_kq_mv_iq3s_msl_fastmath : (fmt == 3 ? MetalKqMvB4K3_metal_kq_mv_k3_msl_fastmath : (fmt == 44 ? MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl_fastmath : (fmt == 4 ? MetalKqMvB4K4_metal_kq_mv_k4_msl_fastmath : (fmt == 5 ? MetalKqMvB4K5_metal_kq_mv_k5_msl_fastmath : MetalKqMvB4K6_metal_kq_mv_k6_msl_fastmath)))))) + : (fmt == 34 ? metal_kq_mvb8_iq3xxs_msl_fastmath : (fmt == 33 ? metal_kq_mvb8_iq3s_msl_fastmath : (fmt == 3 ? metal_kq_mvb8_k3_msl_fastmath : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl_fastmath : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_fastmath : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_fastmath : metal_kq_mvb8_k6_msl_fastmath)))))))) var pso = pipeline_from_source(dev, src, entry, fm, err) t |> success(pso != null, "{tag}: pipeline: {err}") return if (pso == null) @@ -166,9 +169,9 @@ def private kq_mvb_gate(t : T?; dev, queue; fmt, bwidth, n, d, nr, ys : int) { metal_set_pipeline(enc, pso) if (bwidth == 8) { metal_set_threadgroup_memory_length(enc, - fmt == 33 ? metal_kq_mvb8_iq3s_msl_tgmem : (fmt == 3 ? metal_kq_mvb8_k3_msl_tgmem : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl_tgmem : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_tgmem : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_tgmem : metal_kq_mvb8_k6_msl_tgmem)))), 0) + fmt == 34 ? metal_kq_mvb8_iq3xxs_msl_tgmem : (fmt == 33 ? metal_kq_mvb8_iq3s_msl_tgmem : (fmt == 3 ? metal_kq_mvb8_k3_msl_tgmem : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl_tgmem : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_tgmem : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_tgmem : metal_kq_mvb8_k6_msl_tgmem))))), 0) } - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs/k3: the f16 d tail + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs/k3: the f16 d tail metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bx, 0ul, 3) @@ -257,7 +260,7 @@ def private moe_kq_gemv_gate(t : T?; dev, queue; fmt, n, d, net, k, nst : int; p bxs = uint((perslot ? k : 1) * n), bys = uint(k * d), hasb = 0u) let ran = with_compute_encoder(queue, err) $(enc : MetalComputeEncoder?) { metal_set_pipeline(enc, pso) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs/k3: the f16 d tail + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs/k3: the f16 d tail metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bx, 0ul, 3) @@ -733,8 +736,10 @@ def test_metal_kq_gemv_kernels(t : T?) { kq_gemv_gate(t, dev, queue, "k3", 512, 30) kq_gemv_gate(t, dev, queue, "iq3s", 1280, 11) kq_gemv_gate(t, dev, queue, "iq3s", 512, 30) + kq_gemv_gate(t, dev, queue, "iq3xxs", 1280, 11) + kq_gemv_gate(t, dev, queue, "iq3xxs", 512, 30) // the small-batch twins: nr = 2 / 3 (b4 col guard) / 6 (b8 pad), ys > d - for (fmt in [4, 5, 6, 44, 3, 33]) { + for (fmt in [4, 5, 6, 44, 3, 33, 34]) { kq_mvb_gate(t, dev, queue, fmt, 2, 768, 17, 2, 22) kq_mvb_gate(t, dev, queue, fmt, 4, 768, 17, 3, 22) kq_mvb_gate(t, dev, queue, fmt, 8, 768, 17, 6, 22) From 757f3ec76ceb29344c9e128f4c229cdf31f78d15 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Sun, 30 Aug 2026 22:49:41 -0700 Subject: [PATCH 27/61] dasLLAMA: IQ4_NL on the CPU tier + JIT - q40's planes, the codebook, one emitter mode The near-free format, identity 45. IQ4_NL's disk bytes ARE Q4_0's (per 32-block f16 d + 16 nibble bytes, the k/k+16 pairing) - only the nibble semantics differ (kvalues_iq4nl, no offset, no bsum). So the planes are Q40_QSB/Q40_SSB verbatim, the transcode is q40's byte split with the type check swapped, the grp repack ROUTES to repack_q40_grp (not copied), and the tile rides the packed lists - no panel, no gather anywhere. The scalar dot is iq4xs's LUT loop with q40's per-block f16-d fold; dequant refs, rows kernel, grp row dot, the ~40 QUIRK-1 ladder arms, plane pair + image v22, probe fixtures (8 x 18B disk blocks) and fmt 45 in every test gate loop follow the walk. The JIT emitter is one `nl` mode on emit_block_iq4xs: per-block d vectors load with the same load_f16_vec_at helper over q40's scale interleave, the fold moves into the block loop (float per-block fma) in place of the sub-scale integer sum, and the LUT bake widens to te.kq == 44 || 45. QUIRK 16 replayed exactly on the pre-emitter gate (10 perms stamped, maddubs rows mismatching on the declined generators at layout mr 8) - the emitter arm turned all 11 k45 rows green (maxdiff ~2e-6) in the same slice, so this commit ships A and B together. Vehicle: a local --allow-requantize (IQ4_NL needs no imatrix; iq4_nl x94 + q5_K/q6_K siblings, census-checked per QUIRK 23). Gates: test_kqformat 18/18, test_kquant 199 tests 0 failed, the tune probe 11/11 k45 perms ok. E2e stamped: coherent text at gen 60 t/s, 11/64 greedy ids vs simple_ids with the fork a 0.042-logit near-tie (the window's smallest margin by two orders) whose runner-up IS our token. zen2 rows vs llama.cpp clean-cpu: pp512 618.3 / 540.6 (1.14x), tg128 64.2 / 62.2 (1.03x). Vulkan, Metal: pending. Co-Authored-By: Claude Fable 5 --- modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md | 25 ++++ modules/dasLLAMA/dasllama/dasllama_blocks.das | 3 + modules/dasLLAMA/dasllama/dasllama_common.das | 38 +++++- modules/dasLLAMA/dasllama/dasllama_config.das | 3 +- .../dasLLAMA/dasllama/dasllama_convert.das | 23 ++++ .../dasLLAMA/dasllama/dasllama_gemm_gen.das | 65 +++++++--- .../dasllama/dasllama_gemm_schema.das | 12 +- modules/dasLLAMA/dasllama/dasllama_gguf.das | 35 +++++ .../dasllama/dasllama_gpu_resident.das | 9 ++ modules/dasLLAMA/dasllama/dasllama_image.das | 7 +- .../dasLLAMA/dasllama/dasllama_kqformat.das | 16 ++- modules/dasLLAMA/dasllama/dasllama_layout.das | 18 ++- modules/dasLLAMA/dasllama/dasllama_load.das | 41 +++++- modules/dasLLAMA/dasllama/dasllama_math.das | 9 +- .../dasllama/dasllama_math_default.das | 65 ++++++++++ .../dasLLAMA/dasllama/dasllama_math_gen.das | 96 +++++++++++++- modules/dasLLAMA/dasllama/dasllama_ple.das | 4 + modules/dasLLAMA/harness/gen_tune_probe.das | 47 ++++++- modules/dasLLAMA/tests/test_kqformat.das | 10 +- modules/dasLLAMA/tests/test_kquant.das | 122 +++++++++++++++--- 20 files changed, 576 insertions(+), 72 deletions(-) diff --git a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md index 15d89d5e50..217f5ee9a9 100644 --- a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md +++ b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md @@ -422,6 +422,31 @@ where, why it is so today, what unquirked looks like. An empty ledger is a legit iq2 tier - so the file cannot load end to end. `gguf_census.py` (scratch) reads the tensor type table in seconds; pick a vehicle whose mix is {supported} + the new format only (bartowski's Qwen2.5-1.5B IQ3_XS carries iq3_xxs with iq3_s/q4_K/q6_K siblings). +### IQ4_NL (the near-free one, 2026-08-30) + +Shape: 32-element blocks, 18B each on disk - f16 d + 16 nibble bytes with the k/k+16 +pairing. That is Q4_0's disk shape BYTE FOR BYTE; only the nibble semantics differ +(`kvalues_iq4nl` - already shipped as `IQ4NL_LUT` - instead of q - 8, so no offset and no +bsum term). The whole CPU story follows from that identity: planes = `Q40_QSB`/`Q40_SSB` +verbatim, transcode = q40's byte split with the type check swapped, grp repack = literally +`repack_q40_grp` (routed, not copied), and the tile rides the PACKED lists - no panel, no +gather. Identity 45. The scalar dot is iq4xs's LUT loop with q40's per-block f16-d fold; +the JIT emitter is one `nl` mode on `emit_block_iq4xs`: the per-block d vectors load with +the same `load_f16_vec_at` helper over q40's scale interleave, and the fold moves INTO the +block loop (float per-block fma) replacing the sub-scale integer sum - plus the LUT-bake +condition widening to `te.kq == 44 || 45`. QUIRK 16 replayed on cue: the pre-emitter gate +stamped 10 perms whose maddubs rows mismatched (declined generators at layout mr 8); the +emitter arm turned all 11 k45 rows green (maxdiff ~2e-6) in the same sitting. + +Vehicle: a local requant again - IQ4_NL needs NO imatrix, just `--allow-requantize` +(mix: iq4_nl x94 + q5_K x18 + q6_K embd, all supported; census first, QUIRK 23). Gates: +`test_kqformat` 18/18, `test_kquant` 199 tests 0 failed, the probe 11/11 k45 perms. E2e +stamped: coherent text at gen 60 t/s, 11/64 greedy ids vs `simple_ids.exe` where the fork +is a 0.042-logit near-tie (the window's smallest margin by 100x) whose top-2 IS our token. +zen2 rows vs llama.cpp clean-cpu: pp512 618.3 vs 540.6 (1.14x), tg128 64.2 vs +62.2 (1.03x). Vulkan, Metal: pending (the generic-dequant class shells + the LUT +already live on both tiers). + ### IQ3_XXS (2026-08-30) Shape: 256-superblock, 98B disk block - f16 d, 64 grid-index bytes (`iq3xxs_grid[256]`, one diff --git a/modules/dasLLAMA/dasllama/dasllama_blocks.das b/modules/dasLLAMA/dasllama/dasllama_blocks.das index 0bf7d0db87..62293ec53e 100644 --- a/modules/dasLLAMA/dasllama/dasllama_blocks.das +++ b/modules/dasLLAMA/dasllama/dasllama_blocks.das @@ -1412,6 +1412,9 @@ def private kq_bytes_per_weight(f : KqFmt) : float { if (f == KqFmt.iq3xxs) { return 116.0 / 256.0 // 96B quants + the 20B scale row per 256 } + if (f == KqFmt.iq4nl) { + return 144.0 / 256.0 // 128B quants + the 16B scale row per 256 (q40's shapes) + } return 34.0 / 32.0 // q8: 32 quants + an f16 scale per block } diff --git a/modules/dasLLAMA/dasllama/dasllama_common.das b/modules/dasLLAMA/dasllama/dasllama_common.das index 91faba104b..88bab0eaaa 100644 --- a/modules/dasLLAMA/dasllama/dasllama_common.das +++ b/modules/dasLLAMA/dasllama/dasllama_common.das @@ -847,6 +847,9 @@ struct Model { // the iq3xxs tier's plane pair: [64 qs][32 aux] per superblock + the iq3s-shaped 20B scale row (f16 d halved, 8 x uint8 (2ls + 1)) iq3xxsq : array iq3xxss : array + // the iq4nl tier's plane pair: q40's shapes exactly (128B nibbles + 8 x f16 d per superblock), LUT decode + iq4nlq : array + iq4nls : array kquant_native : bool = false // kq stage 4: the kq planes were repacked at load into the active backend's grp layout. // From then on every kq matmul MUST run the backend slots — disk-order portable kernels would @@ -861,6 +864,7 @@ struct Model { kq_repack_mr3 : int64 = 4l kq_repack_mr33 : int64 = 4l kq_repack_mr34 : int64 = 4l + kq_repack_mr45 : int64 = 4l // per-layer weight format tags (KqFmt; empty = all q8). Filled by detect_kq_formats before // layout_offsets so the layout walks each tensor into its format's cursor. wq_fmt : array @@ -1086,6 +1090,7 @@ def model_weights_bytes(t : Model) : int64 { + long_length(t.iq4xsq) + long_length(t.iq4xss) + long_length(t.k3q) + long_length(t.k3s) + long_length(t.iq3sq) + long_length(t.iq3ss) + long_length(t.iq3xxsq) + long_length(t.iq3xxss) + + long_length(t.iq4nlq) + long_length(t.iq4nls) + long_length(t.q51q) + long_length(t.q51s) + long_length(t.mblob) + long_length(t.vkblob) + long_length(t.embq) + long_length(t.embs)) @@ -1478,6 +1483,7 @@ def private dlim_cpu_source_impl(var c : DlimCpuConfig) { c.kq_mr3 = active_kq_layout_mr(3) c.kq_mr33 = active_kq_layout_mr(33) c.kq_mr34 = active_kq_layout_mr(34) + c.kq_mr45 = active_kq_layout_mr(45) c.q51_mr = active_q51_layout_mr() // OUTCOME, not request: a backend without s16 twins keeps f32 scale planes (wscale_convert_f16) c.wscale_f16 = g_wscale_f16 && kernel_backend_has_wscale16() @@ -1570,7 +1576,7 @@ def kq_active_mr(t : Model; f : KqFmt) : int64 { if (!t.kq_repacked) { return 1l } - return f == KqFmt.k4 ? t.kq_repack_mr4 : (f == KqFmt.k5 ? t.kq_repack_mr5 : (f == KqFmt.k6 ? t.kq_repack_mr6 : (f == KqFmt.q40 ? t.kq_repack_mr40 : (f == KqFmt.iq4xs ? t.kq_repack_mr44 : (f == KqFmt.k3 ? t.kq_repack_mr3 : (f == KqFmt.iq3s ? t.kq_repack_mr33 : t.kq_repack_mr34)))))) + return f == KqFmt.k4 ? t.kq_repack_mr4 : (f == KqFmt.k5 ? t.kq_repack_mr5 : (f == KqFmt.k6 ? t.kq_repack_mr6 : (f == KqFmt.q40 ? t.kq_repack_mr40 : (f == KqFmt.iq4xs ? t.kq_repack_mr44 : (f == KqFmt.k3 ? t.kq_repack_mr3 : (f == KqFmt.iq3s ? t.kq_repack_mr33 : (f == KqFmt.iq3xxs ? t.kq_repack_mr34 : t.kq_repack_mr45))))))) } //! Load a llama-architecture GGUF into the split layout at the requested precision — straight into @@ -1633,8 +1639,8 @@ def footprint(t : Model) : MemFootprint { let q4s = long_length(t.q4scales) * 4l let mx4 = long_length(t.mxq) // uint8 (two nibbles) -> 1 byte each let mx4s = long_length(t.mxs) // raw E8M0 -> 1 byte per 32-block - let kq = long_length(t.k4q) + long_length(t.k5q) + long_length(t.k6q) + long_length(t.q40q) + long_length(t.q51q) + long_length(t.iq4xsq) + long_length(t.k3q) + long_length(t.iq3sq) + long_length(t.iq3xxsq) - let kqs = long_length(t.k4s) + long_length(t.k5s) + long_length(t.k6s) + long_length(t.q40s) + long_length(t.q51s) + long_length(t.iq4xss) + long_length(t.k3s) + long_length(t.iq3ss) + long_length(t.iq3xxss) + let kq = long_length(t.k4q) + long_length(t.k5q) + long_length(t.k6q) + long_length(t.q40q) + long_length(t.q51q) + long_length(t.iq4xsq) + long_length(t.k3q) + long_length(t.iq3sq) + long_length(t.iq3xxsq) + long_length(t.iq4nlq) + let kqs = long_length(t.k4s) + long_length(t.k5s) + long_length(t.k6s) + long_length(t.q40s) + long_length(t.q51s) + long_length(t.iq4xss) + long_length(t.k3s) + long_length(t.iq3ss) + long_length(t.iq3xxss) + long_length(t.iq4nls) return MemFootprint(aux_fp32_bytes = aux, weight_fp32_bytes = wf, q8_bytes = q8, q8_scale_bytes = q8s, q4_bytes = q4, q4_scale_bytes = q4s, mx4_bytes = mx4, mx4_scale_bytes = mx4s, kq_bytes = kq, kq_scale_bytes = kqs, @@ -3308,7 +3314,7 @@ def mm_b_q51_groupn(var y : array; t : Model; offs : array; nregio // KqFmt -> the kernel-layer format id (matmul_kq*/kq_rows_fn/kq_qsb take the int form). q8 maps // to 0, a poison id no kq kernel accepts — kq_rows_for calls this for EVERY tag, so it must stay // total even though q8 branches never read the result. -def private kq_fi(fmt : KqFmt) : int => fmt == KqFmt.k4 ? 4 : (fmt == KqFmt.k5 ? 5 : (fmt == KqFmt.k6 ? 6 : (fmt == KqFmt.q40 ? 40 : (fmt == KqFmt.iq4xs ? 44 : (fmt == KqFmt.k3 ? 3 : (fmt == KqFmt.iq3s ? 33 : (fmt == KqFmt.iq3xxs ? 34 : 0))))))) +def private kq_fi(fmt : KqFmt) : int => fmt == KqFmt.k4 ? 4 : (fmt == KqFmt.k5 ? 5 : (fmt == KqFmt.k6 ? 6 : (fmt == KqFmt.q40 ? 40 : (fmt == KqFmt.iq4xs ? 44 : (fmt == KqFmt.k3 ? 3 : (fmt == KqFmt.iq3s ? 33 : (fmt == KqFmt.iq3xxs ? 34 : (fmt == KqFmt.iq4nl ? 45 : 0)))))))) // K-quant GEMV against a PRE-quantized activation (xq/xs/xbs already filled by the bs quantizer). // kq_repacked loads run the backend's stamped cores; disk-order loads run the portable kernels. @@ -3330,6 +3336,8 @@ def mm_at_kq_pre(var y : array; t : Model; fmt : KqFmt; woff : int64; xq matmul_kq_active(33, y, t.iq3sq, t.iq3ss, woff, xq, xs, xbs, n, d, yoff) } elif (fmt == KqFmt.iq3xxs) { matmul_kq_active(34, y, t.iq3xxsq, t.iq3xxss, woff, xq, xs, xbs, n, d, yoff) + } elif (fmt == KqFmt.iq4nl) { + matmul_kq_active(45, y, t.iq4nlq, t.iq4nls, woff, xq, xs, xbs, n, d, yoff) } else { panic("mm_at_kq_pre: '{fmt}' has no kq plane pair") // a new KqFmt must claim its arm, never ride q40's } @@ -3349,6 +3357,8 @@ def mm_at_kq_pre(var y : array; t : Model; fmt : KqFmt; woff : int64; xq matmul_kq(33, y, t.iq3sq, t.iq3ss, woff, xq, xs, xbs, n, d, yoff) } elif (fmt == KqFmt.iq3xxs) { matmul_kq(34, y, t.iq3xxsq, t.iq3xxss, woff, xq, xs, xbs, n, d, yoff) + } elif (fmt == KqFmt.iq4nl) { + matmul_kq(45, y, t.iq4nlq, t.iq4nls, woff, xq, xs, xbs, n, d, yoff) } else { panic("mm_at_kq_pre: '{fmt}' has no kq plane pair") } @@ -3388,6 +3398,8 @@ def mm_b_kq(var s : Session; var y : array; t : Model; fmt : KqFmt; woff matmul_kq_batch(33, y, t.iq3sq, t.iq3ss, woff, s.xqb, s.xsb, s.xbsb, n, d, npos) } elif (fmt == KqFmt.iq3xxs) { matmul_kq_batch(34, y, t.iq3xxsq, t.iq3xxss, woff, s.xqb, s.xsb, s.xbsb, n, d, npos) + } elif (fmt == KqFmt.iq4nl) { + matmul_kq_batch(45, y, t.iq4nlq, t.iq4nls, woff, s.xqb, s.xsb, s.xbsb, n, d, npos) } else { panic("mm_b_kq: '{fmt}' has no kq plane pair") } @@ -3423,6 +3435,8 @@ def mm_at_kq_groupn(var y : array; t : Model; fmt : KqFmt; offs : array; t : Model; fmt : KqFmt; offs : array; t : Model; fmt : KqFmt; offs : array; t : Model; fmt : KqFmt; woff : int64; xq : matmul_kq_batch(33, y, t.iq3sq, t.iq3ss, woff, xq, xs, xbs, n, d, ntok) } elif (fmt == KqFmt.iq3xxs) { matmul_kq_batch(34, y, t.iq3xxsq, t.iq3xxss, woff, xq, xs, xbs, n, d, ntok) + } elif (fmt == KqFmt.iq4nl) { + matmul_kq_batch(45, y, t.iq4nlq, t.iq4nls, woff, xq, xs, xbs, n, d, ntok) } else { panic("mm_b_kq_pre: '{fmt}' has no kq plane pair") } @@ -3508,6 +3528,7 @@ def kq_plane_q(t : Model; fmt : KqFmt; woff : int64) : uint8 const? { if (fmt == KqFmt.k3) return unsafe(addr(t.k3q[sb * K3_QSB])) if (fmt == KqFmt.iq3s) return unsafe(addr(t.iq3sq[sb * IQ3S_QSB])) if (fmt == KqFmt.iq3xxs) return unsafe(addr(t.iq3xxsq[sb * IQ3XXS_QSB])) + if (fmt == KqFmt.iq4nl) return unsafe(addr(t.iq4nlq[sb * Q40_QSB])) return null } @@ -3521,6 +3542,7 @@ def kq_plane_s(t : Model; fmt : KqFmt; woff : int64) : uint8 const? { if (fmt == KqFmt.k3) return unsafe(addr(t.k3s[sb * K3_SSB])) if (fmt == KqFmt.iq3s) return unsafe(addr(t.iq3ss[sb * IQ3S_SSB])) if (fmt == KqFmt.iq3xxs) return unsafe(addr(t.iq3xxss[sb * IQ3XXS_SSB])) + if (fmt == KqFmt.iq4nl) return unsafe(addr(t.iq4nls[sb * Q40_SSB])) return null } @@ -3889,6 +3911,8 @@ def embed_row(t : Model; token : int64; var dst : float?) { // nolint:STYLE037 dequant_kq_row_grp(fmt, addr(t.embq[sbg * IQ3S_QSB]), addr(t.embs[sbg * IQ3S_SSB]), token % mr, mr, dim, dst) } elif (t.emb_fmt == KqFmt.iq3xxs) { dequant_kq_row_grp(fmt, addr(t.embq[sbg * IQ3XXS_QSB]), addr(t.embs[sbg * IQ3XXS_SSB]), token % mr, mr, dim, dst) + } elif (t.emb_fmt == KqFmt.iq4nl) { + dequant_kq_row_grp(fmt, addr(t.embq[sbg * Q40_QSB]), addr(t.embs[sbg * Q40_SSB]), token % mr, mr, dim, dst) } else { panic("embed_row: '{t.emb_fmt}' has no kq plane pair") } @@ -3908,6 +3932,8 @@ def embed_row(t : Model; token : int64; var dst : float?) { // nolint:STYLE037 dequant_kq_row_grp(fmt, addr(t.iq3sq[sbg * IQ3S_QSB]), addr(t.iq3ss[sbg * IQ3S_SSB]), token % mr, mr, dim, dst) } elif (t.emb_fmt == KqFmt.iq3xxs) { dequant_kq_row_grp(fmt, addr(t.iq3xxsq[sbg * IQ3XXS_QSB]), addr(t.iq3xxss[sbg * IQ3XXS_SSB]), token % mr, mr, dim, dst) + } elif (t.emb_fmt == KqFmt.iq4nl) { + dequant_kq_row_grp(fmt, addr(t.iq4nlq[sbg * Q40_QSB]), addr(t.iq4nls[sbg * Q40_SSB]), token % mr, mr, dim, dst) } else { panic("embed_row: '{t.emb_fmt}' has no kq plane pair") } @@ -3938,6 +3964,8 @@ def embed_row(t : Model; token : int64; var dst : float?) { // nolint:STYLE037 dequant_iq3s_plane_superblock(t.embq, (sb0 + s) * IQ3S_QSB, t.embs, (sb0 + s) * IQ3S_SSB, row, s * 256l) } elif (t.emb_fmt == KqFmt.iq3xxs) { dequant_iq3xxs_plane_superblock(t.embq, (sb0 + s) * IQ3XXS_QSB, t.embs, (sb0 + s) * IQ3XXS_SSB, row, s * 256l) + } elif (t.emb_fmt == KqFmt.iq4nl) { + dequant_iq4nl_plane_superblock(t.embq, (sb0 + s) * Q40_QSB, t.embs, (sb0 + s) * Q40_SSB, row, s * 256l) } else { panic("embed_row: '{t.emb_fmt}' has no kq plane pair") } @@ -3961,6 +3989,8 @@ def embed_row(t : Model; token : int64; var dst : float?) { // nolint:STYLE037 dequant_iq3s_plane_superblock(t.iq3sq, (sb0 + s) * IQ3S_QSB, t.iq3ss, (sb0 + s) * IQ3S_SSB, row, s * 256l) } elif (t.emb_fmt == KqFmt.iq3xxs) { dequant_iq3xxs_plane_superblock(t.iq3xxsq, (sb0 + s) * IQ3XXS_QSB, t.iq3xxss, (sb0 + s) * IQ3XXS_SSB, row, s * 256l) + } elif (t.emb_fmt == KqFmt.iq4nl) { + dequant_iq4nl_plane_superblock(t.iq4nlq, (sb0 + s) * Q40_QSB, t.iq4nls, (sb0 + s) * Q40_SSB, row, s * 256l) } else { panic("embed_row: '{t.emb_fmt}' has no kq plane pair") } diff --git a/modules/dasLLAMA/dasllama/dasllama_config.das b/modules/dasLLAMA/dasllama/dasllama_config.das index 9e1ba72bed..45d78828a9 100644 --- a/modules/dasLLAMA/dasllama/dasllama_config.das +++ b/modules/dasLLAMA/dasllama/dasllama_config.das @@ -31,6 +31,7 @@ struct public DlimCpuConfig { kq_mr3 : int64 kq_mr33 : int64 kq_mr34 : int64 + kq_mr45 : int64 q51_mr : int64 wscale_f16 : bool kquant_native : bool @@ -143,7 +144,7 @@ def public dlim_config_current(quant : string = "q8") : DlimConfiguration { def public dlim_identity(c : DlimConfiguration; image_version : int; tag : string = "") : string { return ("v{image_version}|{c.quant}|{c.cpu.backend}|{c.cpu.wscale_f16 ? "s16" : "s32"}" + "|q8 mr{c.cpu.q8_mr} b{c.cpu.q8_wbias} g{c.cpu.q8_kgroup}" - + "|kq {c.cpu.kq_mr4}/{c.cpu.kq_mr5}/{c.cpu.kq_mr6}/{c.cpu.kq_mr40}/{c.cpu.kq_mr44}/{c.cpu.kq_mr3}/{c.cpu.kq_mr33}/{c.cpu.kq_mr34}" + + "|kq {c.cpu.kq_mr4}/{c.cpu.kq_mr5}/{c.cpu.kq_mr6}/{c.cpu.kq_mr40}/{c.cpu.kq_mr44}/{c.cpu.kq_mr3}/{c.cpu.kq_mr33}/{c.cpu.kq_mr34}/{c.cpu.kq_mr45}" + "|q51 mr{c.cpu.q51_mr}" + "|nat {c.cpu.kquant_native ? 1 : 0}{c.cpu.kq_q40_native ? 1 : 0}{c.cpu.kq_q50_native ? 1 : 0}{c.cpu.kq_q51_native ? 1 : 0}" + (tag != "" ? "|{tag}" : "")) diff --git a/modules/dasLLAMA/dasllama/dasllama_convert.das b/modules/dasLLAMA/dasllama/dasllama_convert.das index 7cbfe77d3e..22b539ded8 100644 --- a/modules/dasLLAMA/dasllama/dasllama_convert.das +++ b/modules/dasLLAMA/dasllama/dasllama_convert.das @@ -716,6 +716,29 @@ def dequant_k3_plane_superblock_at(kq : array | #; kqo : int64; ks : arra //! Transcode one IQ3_S superblock (110 bytes at `bo`: f16 d, 64 qs, 8 qh, 32 signs, 4 packed //! 4-bit scales) into the iq3s planes: [qs][qh][signs] verbatim to kq[kqo..+104), the 8 //! sub-scales DECODED to int8 (1 + 2s) + the f16 d to ks[kso..+20) — the k4 row shape (exact). +def transcode_iq4nl_superblock(bytes : array | #; bo : int64; var kq : array; kqo : int64; var ks : array; kso : int64) { + for (blk in range64(8l)) { + ks[kso + blk * 2l] = bytes[bo + blk * 18l] + ks[kso + blk * 2l + 1l] = bytes[bo + blk * 18l + 1l] + for (i in range64(16l)) { + kq[kqo + blk * 16l + i] = bytes[bo + blk * 18l + 2l + i] + } + } +} + +//! Reference dequant of one iq4nl-plane superblock: w = d * IQ4NL_LUT[nibble], the q40 k/k+16 +//! pairing, per-32-block f16 d - ggml's dequantize_row_iq4_nl float order. +def dequant_iq4nl_plane_superblock(kq : array | #; kqo : int64; ks : array | #; kso : int64; var dst : array | #; doff : int64) { + for (blk in range64(8l)) { + let d = f16_to_f32(rd_u16(ks, kso + blk * 2l)) + for (l in range64(16l)) { + let b = int(kq[kqo + blk * 16l + l]) + dst[doff + blk * 32l + l] = d * float(int(IQ4NL_LUT[b & 15])) + dst[doff + blk * 32l + 16l + l] = d * float(int(IQ4NL_LUT[b >> 4])) + } + } +} + def transcode_iq3xxs_superblock(bytes : array | #; bo : int64; var kq : array; kqo : int64; var ks : array; kso : int64) { for (i in range64(96l)) { kq[kqo + i] = bytes[bo + 2l + i] diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das index df00e8976d..f327d2fe37 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das @@ -1445,6 +1445,7 @@ def private emit_block_iq4xs(var te : TileEmit; var sbi : LLVMOpaqueValue?; var let mr = te.interleave let w8 = te.width / 8 let panel = te.kq == 33 || te.kq == 34 // iq3s/iq3xxs: the byte-expanded signed panel replaces the nibble+LUT decode; fold identical + let nl = te.kq == 45 // iq4nl: the 44 LUT decode with q40's PER-BLOCK f16 d fold (no sub-scales, no min term) let gather = panel && te.iq3s_panel != null // gemv mode: gather this superblock off the packed planes first var wbase = te.wg var wb : LLVMOpaqueValue? @@ -1459,12 +1460,22 @@ def private emit_block_iq4xs(var te : TileEmit; var sbi : LLVMOpaqueValue?; var } else { wb = LLVMBuildMul(b, sbi, te.types->ConstI64(uint64(mr * (panel ? 256 : 128))), "wb") } - var sb = LLVMBuildMul(b, sbi, te.types->ConstI64(uint64(mr * 20)), "sb") + var sb = LLVMBuildMul(b, sbi, te.types->ConstI64(uint64(mr * (nl ? 16 : 20))), "sb") var xb = LLVMBuildMul(b, sbi, te.types->ConstI64(0x100ul), "xb") var vri8 = LLVMVectorType(te.types.t_int8, uint(te.rv)) var dv : LLVMOpaqueValue? [2] - for (qd in range(rq)) { - dv[qd] = load_f16_vec_at(te, te.sg, LLVMBuildAdd(b, sb, te.types->ConstI64(uint64(2 * (qd * te.rv))), ""), "d{qd}") + if (!nl) { // 44/33/34: one superblock d per row lane; iq4nl loads its per-block d inside the walk + for (qd in range(rq)) { + dv[qd] = load_f16_vec_at(te, te.sg, LLVMBuildAdd(b, sb, te.types->ConstI64(uint64(2 * (qd * te.rv))), ""), "d{qd}") + } + } + var d8nl : LLVMOpaqueValue? [4] + if (nl) { // hoist the per-token activation scales - the per-block fold consumes them 8x + for (i in range(tokCount)) { + let tk = tokBase + i + var qp0 = LLVMBuildGEP2(b, te.types.t_float, te.xs[tk], sbi, "qp{tk}") + d8nl[i] = splat_f32(te, LLVMBuildLoad2Aligned(b, te.types.t_float, qp0, 4u, "d8{tk}"), "d8v{tk}") + } } var iacc : LLVMOpaqueValue? [8] for (i in range(tokCount * rq)) { @@ -1506,17 +1517,34 @@ def private emit_block_iq4xs(var te : TileEmit; var sbi : LLVMOpaqueValue?; var } } } - var scv : LLVMOpaqueValue? [2] - for (qd in range(rq)) { - var scp = LLVMBuildGEP2(b, te.types.t_int8, te.sg, LLVMBuildAdd(b, sb, te.types->ConstI64(uint64(4 * mr + blk * mr + qd * te.rv)), ""), "scp{blk}_{qd}") - scv[qd] = LLVMBuildSExt(b, LLVMBuildLoad2Aligned(b, vri8, scp, 1u, ""), te.vni32, "sc{blk}_{qd}") - } - for (i in range(tokCount)) { + if (nl) { + // per-block fold: f += sitofp(idot) * (d_blk * d8) - q40's shape, straight to float + var dblk : LLVMOpaqueValue? [2] for (qd in range(rq)) { - iacc[i * rq + qd] = LLVMBuildAdd(b, iacc[i * rq + qd], LLVMBuildMul(b, scv[qd], a[i * rq + qd], ""), "ia{tokBase + i}_{blk}_{qd}") + dblk[qd] = load_f16_vec_at(te, te.sg, LLVMBuildAdd(b, sb, te.types->ConstI64(uint64(blk * 2 * mr + 2 * (qd * te.rv))), ""), "dnl{blk}_{qd}") + } + for (i in range(tokCount)) { + for (qd in range(rq)) { + var dsn = LLVMBuildFMul(b, dblk[qd], d8nl[i], "dsn{tokBase + i}_{blk}_{qd}") + f[i * rq + qd] = fold_fma(te, LLVMBuildSIToFP(b, a[i * rq + qd], te.vnf32, ""), dsn, f[i * rq + qd], "f{tokBase + i}_{blk}_{qd}n") + } + } + } else { + var scv : LLVMOpaqueValue? [2] + for (qd in range(rq)) { + var scp = LLVMBuildGEP2(b, te.types.t_int8, te.sg, LLVMBuildAdd(b, sb, te.types->ConstI64(uint64(4 * mr + blk * mr + qd * te.rv)), ""), "scp{blk}_{qd}") + scv[qd] = LLVMBuildSExt(b, LLVMBuildLoad2Aligned(b, vri8, scp, 1u, ""), te.vni32, "sc{blk}_{qd}") + } + for (i in range(tokCount)) { + for (qd in range(rq)) { + iacc[i * rq + qd] = LLVMBuildAdd(b, iacc[i * rq + qd], LLVMBuildMul(b, scv[qd], a[i * rq + qd], ""), "ia{tokBase + i}_{blk}_{qd}") + } } } } + if (nl) { + return // the per-block fold already landed everything in f + } for (i in range(tokCount)) { let tk = tokBase + i var qp = LLVMBuildGEP2(b, te.types.t_float, te.xs[tk], sbi, "qp{tk}") @@ -1533,7 +1561,7 @@ def private emit_block_iq4xs(var te : TileEmit; var sbi : LLVMOpaqueValue?; var // token PAIR — single-token slices ride the sdot lattice instead. kq blocks are SUPERBLOCKS // (the kq gemv/tile drive emit_slice with nb = n/256). def private emit_one_block(var te : TileEmit; var bi : LLVMOpaqueValue?; var f : LLVMOpaqueValue? [8]; tokBase, tokCount : int) { - if (te.kq == 44 || te.kq == 33 || te.kq == 34) { + if (te.kq == 44 || te.kq == 33 || te.kq == 34 || te.kq == 45) { emit_block_iq4xs(te, bi, f, tokBase, tokCount) } elif (te.kq != 0) { emit_block_kqv2(te, bi, f, tokBase, tokCount) @@ -1806,7 +1834,7 @@ def private setup_tile_emit(var te : TileEmit; var gc : LlvmCodeCtx; p : TilePer lutElems |> reserve(16 * lanes) for (_lane in range(lanes)) { for (i in range(16)) { - let v = te.kq == 44 ? int(iq4nl[i]) : e2m1[i] + let v = te.kq == 44 || te.kq == 45 ? int(iq4nl[i]) : e2m1[i] lutElems |> push(LLVMConstInt(te.types.t_int8, uint64(v & 255), 0)) } } @@ -2631,7 +2659,7 @@ def private kq_gemv_gen_impl(var gc : LlvmCodeCtx; fmt : int) : bool { let p = companion_perm(p0) var te = TileEmit(kq = fmt) - if (!setup_tile_emit(te, gc, p, fmt == 44)) return false + if (!setup_tile_emit(te, gc, p, fmt == 44 || fmt == 45)) return false let b = gc.jit.builder var entry = LLVMAppendBasicBlockInContext(gc.jit.ctx, gc.impl, "entry") @@ -2774,8 +2802,8 @@ def private kq_tile_gen_impl(var gc : LlvmCodeCtx; fmt : int) : bool { if (perm_declines(gc, p0)) return false let p = companion_perm(p0) - var te = TileEmit(kq = fmt, kqBytes = fmt == 5 || fmt == 6) // k4/q40/iq4xs/k3 tiles read the packed planes - if (!setup_tile_emit(te, gc, p, fmt == 44)) return false + var te = TileEmit(kq = fmt, kqBytes = fmt == 5 || fmt == 6) // k4/q40/iq4xs/k3/iq4nl tiles read the packed planes + if (!setup_tile_emit(te, gc, p, fmt == 44 || fmt == 45)) return false let b = gc.jit.builder var entry = LLVMAppendBasicBlockInContext(gc.jit.ctx, gc.impl, "entry") @@ -2834,6 +2862,11 @@ def private iq3s_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 3 def private iq3xxs_gemv_gen(var gc : LlvmCodeCtx) : bool => kq_gemv_gen_impl(gc, 34) def private iq3xxs_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 34) +// iq4nl rides the iq4xs LUT block body with q40's per-block f16 d fold (the `nl` mode) over +// the packed grp planes - no panel, no gather, no sub-scales. +def private iq4nl_gemv_gen(var gc : LlvmCodeCtx) : bool => kq_gemv_gen_impl(gc, 45) +def private iq4nl_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 45) + def private k4_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 4) def private k5_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 5) def private k6_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 6) @@ -2919,5 +2952,7 @@ def public register_dasllama_gemm_generators { register_llvm_code_generator("dasllama_gemm_gen::iq3s_tile", @@iq3s_tile_gen) register_llvm_code_generator("dasllama_gemm_gen::iq3xxs_gemv", @@iq3xxs_gemv_gen) register_llvm_code_generator("dasllama_gemm_gen::iq3xxs_tile", @@iq3xxs_tile_gen) + register_llvm_code_generator("dasllama_gemm_gen::iq4nl_gemv", @@iq4nl_gemv_gen) + register_llvm_code_generator("dasllama_gemm_gen::iq4nl_tile", @@iq4nl_tile_gen) register_llvm_code_generator("dasllama_gemm_gen::q8q8_witness", @@witness_gen) } diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_schema.das b/modules/dasLLAMA/dasllama/dasllama_gemm_schema.das index 46b57af89e..956287a78c 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_schema.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_schema.das @@ -58,7 +58,7 @@ let Q51_QPB = 20l let Q51_SPB = 4l //! Quant-plane bytes per 256-weight superblock per row for a kq format id (4/5/6 = Q4_K/Q5_K/ -//! Q6_K, 40 = Q4_0, 44 = IQ4_XS, 3 = Q3_K, 33 = IQ3_S, 34 = IQ3_XXS). The ONE stride source for every fmt-branched kq walker/ +//! Q6_K, 40 = Q4_0, 44 = IQ4_XS, 3 = Q3_K, 33 = IQ3_S, 34 = IQ3_XXS, 45 = IQ4_NL). The ONE stride source for every fmt-branched kq walker/ //! kernel/repack — an unknown id panics instead of silently walking another format's stride. def kq_qsb(fmt : int) : int64 { if (fmt == 4) return kq_qsb(KqFmt.k4) @@ -69,15 +69,14 @@ def kq_qsb(fmt : int) : int64 { if (fmt == 3) return kq_qsb(KqFmt.k3) if (fmt == 33) return kq_qsb(KqFmt.iq3s) if (fmt == 34) return kq_qsb(KqFmt.iq3xxs) + if (fmt == 45) return kq_qsb(KqFmt.iq4nl) panic("kq_qsb: unknown kq format id {fmt}") return 0l } -//! Scale-plane bytes per 256-weight superblock per row (see kq_qsb): k4/k5 20 (16B disk block -//! + 4B pad, decoded at repack), k6 18 (native), q40 16 (8 x f16 d), iq4xs 20 (d + 8 int8 -//! sub-scales decoded at transcode, the k4 row shape), k3 18 (16 int8 decoded + d, the k6 row), -//! iq3s 20 (d + 8 int8 (1 + 2s) decoded at transcode, the k4 row shape again), iq3xxs 20 (d -//! halved + 8 x (2*ls + 1), same row). +//! Scale-plane bytes per 256-weight superblock per row (see kq_qsb): k4/k5 20, k6/k3 18, +//! q40/iq4nl 16 (8 x f16 d), iq4xs/iq3s/iq3xxs 20 (the decoded k4 row shape) - the per-format +//! layouts live on the kq_ssb(KqFmt) arms and the transcode docs. def kq_ssb(fmt : int) : int64 { if (fmt == 4) return kq_ssb(KqFmt.k4) if (fmt == 5) return kq_ssb(KqFmt.k5) @@ -87,6 +86,7 @@ def kq_ssb(fmt : int) : int64 { if (fmt == 3) return kq_ssb(KqFmt.k3) if (fmt == 33) return kq_ssb(KqFmt.iq3s) if (fmt == 34) return kq_ssb(KqFmt.iq3xxs) + if (fmt == 45) return kq_ssb(KqFmt.iq4nl) panic("kq_ssb: unknown kq format id {fmt}") return 0l } diff --git a/modules/dasLLAMA/dasllama/dasllama_gguf.das b/modules/dasLLAMA/dasllama/dasllama_gguf.das index 3ed1286ba9..eedd2dee17 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gguf.das +++ b/modules/dasLLAMA/dasllama/dasllama_gguf.das @@ -50,6 +50,7 @@ let GGML_TYPE_Q8_0 = 8 let GGML_TYPE_Q3_K = 11 let GGML_TYPE_IQ3_S = 21 let GGML_TYPE_IQ3_XXS = 18 +let GGML_TYPE_IQ4_NL = 20 let GGML_TYPE_Q4_K = 12 let GGML_TYPE_Q5_K = 13 let GGML_TYPE_Q6_K = 14 @@ -869,6 +870,40 @@ def gguf_transcode_iq3xxs(m : GGUFMeta; srcbytes : array | #; name : stri } } + +//! Transcode an IQ4_NL tensor into the iq4nl planes (q40's strides 128/16, exact): the disk +//! bytes ARE q40's shape - per 32-block f16 d + 16 nibble bytes, split verbatim; only the +//! nibble semantics differ (IQ4NL_LUT at decode, no -8 offset, no bsum term). +def gguf_transcode_iq4nl(m : GGUFMeta; srcbytes : array | #; name : string; var kq : array; var ks : array; eloff, expect_n : int64; src_off : int64 = 0l) { + let ti = kq_transcode_check(m, name, GGML_TYPE_IQ4_NL, "IQ4_NL", src_off, expect_n) + let nb = expect_n / 256l + if (nb <= 0l) { + return + } + guard_dst(name, "iq4nl quant plane", (eloff / 256l) * Q40_QSB, nb * Q40_QSB, long_length(kq)) + guard_dst(name, "iq4nl scale plane", (eloff / 256l) * Q40_SSB, nb * Q40_SSB, long_length(ks)) + with_tensor_view(m, srcbytes, ti) $(bytes, tbase) { + let bo = tbase + (src_off / 256l) * 144l // 8 x 18B disk blocks per superblock + unsafe { + let srcp = addr(bytes[bo]) + var kqp = addr(kq[(eloff / 256l) * Q40_QSB]) + var ksp = addr(ks[(eloff / 256l) * Q40_SSB]) + maybe_parallel_for(0, int(nb), transcode_jobs(nb, 144l)) $(rb, re) { + unsafe { + for (sb in range64(int64(rb), int64(re))) { + for (blk in range64(8l)) { + let src = srcp + sb * 144l + blk * 18l + ksp[sb * Q40_SSB + blk * 2l] = src[0] // f16 d + ksp[sb * Q40_SSB + blk * 2l + 1l] = src[1] + bcopy(kqp + sb * Q40_QSB + blk * 16l, src + 2l, 16l) + } + } + } + } + } + } +} + //! Transcode a Q5_1 tensor into the q51 planes (per 32-BLOCK strides 20/4 — see Q51_QB): each //! 24B disk block splits into 2B d + 2B m (scale plane) and 16B nibbles + 4B qh (quant plane), //! all verbatim. Exact; `eloff`/`src_off`/`expect_n` are element offsets, % 32. diff --git a/modules/dasLLAMA/dasllama/dasllama_gpu_resident.das b/modules/dasLLAMA/dasllama/dasllama_gpu_resident.das index 5c03dcaa32..96ff4ad00c 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gpu_resident.das +++ b/modules/dasLLAMA/dasllama/dasllama_gpu_resident.das @@ -288,6 +288,13 @@ def trim_model_planes(var t : Model) : bool { // nolint:STYLE038 — the flat t.embs |> reserve_resize(nsb * K3_SSB) memcpy(addr(t.embq[0]), addr(t.k3q[sb0 * K3_QSB]), nsb * K3_QSB) memcpy(addr(t.embs[0]), addr(t.k3s[sb0 * K3_SSB]), nsb * K3_SSB) + } elif (t.emb_fmt == KqFmt.iq4nl) { + t.embq |> reserve(nsb * Q40_QSB) + t.embq |> resize(nsb * Q40_QSB) + t.embs |> reserve(nsb * Q40_SSB) + t.embs |> resize(nsb * Q40_SSB) + memcpy(addr(t.embq[0]), addr(t.iq4nlq[sb0 * Q40_QSB]), nsb * Q40_QSB) + memcpy(addr(t.embs[0]), addr(t.iq4nls[sb0 * Q40_SSB]), nsb * Q40_SSB) } elif (t.emb_fmt == KqFmt.iq3xxs) { t.embq |> resize(nsb * IQ3XXS_QSB) t.embs |> resize(nsb * IQ3XXS_SSB) @@ -324,6 +331,8 @@ def trim_model_planes(var t : Model) : bool { // nolint:STYLE038 — the flat delete t.iq3ss delete t.iq3xxsq delete t.iq3xxss + delete t.iq4nlq + delete t.iq4nls delete t.q40q delete t.q40s delete t.q51q diff --git a/modules/dasLLAMA/dasllama/dasllama_image.das b/modules/dasLLAMA/dasllama/dasllama_image.das index 469ba1c09b..0fe882989e 100644 --- a/modules/dasLLAMA/dasllama/dasllama_image.das +++ b/modules/dasLLAMA/dasllama/dasllama_image.das @@ -38,7 +38,7 @@ require dasllama/dasllama_load // WhisperModel.enc) contribute their planes under dotted names ("enc.fblob"); string-array // fields ride the meta blob via serialize_strings — raw string pointers can't be planes. -let IMAGE_VERSION = 21 // 21: the iq3xxs plane pair (IQ3_XXS native tier); 20: iq3s; 19: k3 +let IMAGE_VERSION = 22 // 22: the iq4nl plane pair (IQ4_NL native tier); 21: iq3xxs; 20: iq3s //! The metal (blob-only) flavor's identity tag: q8 planes ride the 34B block_q8_0 blob and the //! kq scale planes their GPU forms (convert_model_to_metal_blob) — flavors are per-config and @@ -638,6 +638,7 @@ def serialize_image_meta(var arch : Archive; var t : Model) { arch |> serialize_raw(t.kq_repack_mr3) arch |> serialize_raw(t.kq_repack_mr33) arch |> serialize_raw(t.kq_repack_mr34) + arch |> serialize_raw(t.kq_repack_mr45) arch |> serialize_raw(t.wcls_fmt) arch |> serialize_raw(t.emb_fmt) arch |> serialize_raw(t.ple_emb_fmt) @@ -677,7 +678,7 @@ def serialize_image_meta(var arch : Archive; var t : Model) { } // 67 serialized fields + 3 deliberate skips (blocks, image_map, image_bytes) -let IMAGE_META_FIELDS = 70 + 3 +let IMAGE_META_FIELDS = 71 + 3 //! Count of meta-carried fields of any struct: non-arrays plus string arrays (those cannot //! be raw planes — see serialize_strings). Pair with a per-type field-count constant to @@ -961,7 +962,7 @@ def save_model_image(var t : Model; path : string; tag : string = ""; quant : st // streamed planes size as empty, so their bytes plus a page each are added here; over-reserve is free (dwrite_close truncates) def private stream_extra_bytes(t : Model; jobs : array) : uint64 { var extra = 0ul - for (fname in ["qblob", "mblob", "k4q", "k5q", "k6q", "q40q", "q51q", "iq4xsq", "k3q", "iq3sq", "iq3xxsq"]) { + for (fname in ["qblob", "mblob", "k4q", "k5q", "k6q", "q40q", "q51q", "iq4xsq", "k3q", "iq3sq", "iq3xxsq", "iq4nlq"]) { let sb = stream_plane_bytes(t, jobs, fname) if (sb > 0l) { extra += uint64(sb) + uint64(IMAGE_PAGE) diff --git a/modules/dasLLAMA/dasllama/dasllama_kqformat.das b/modules/dasLLAMA/dasllama/dasllama_kqformat.das index e396fbcc4e..5c0de66e7a 100644 --- a/modules/dasLLAMA/dasllama/dasllama_kqformat.das +++ b/modules/dasLLAMA/dasllama/dasllama_kqformat.das @@ -13,7 +13,7 @@ module dasllama_kqformat shared public require dasllama/dasllama_lint public //! Per-weight storage format under a q8-mode load with native planes: q8 = qblob/qscales; -//! k4/k5/k6/q40/iq4xs/k3/iq3s/iq3xxs = superblock plane pairs (mixed PER TENSOR); q51 = native Q5_1 per-32 +//! k4/k5/k6/q40/iq4xs/k3/iq3s/iq3xxs/iq4nl = superblock plane pairs (mixed PER TENSOR); q51 = native Q5_1 per-32 //! planes. New members append — the int value is the device stack tag and the image plane id. enum KqFmt : uint8 { q8 @@ -26,15 +26,16 @@ enum KqFmt : uint8 { k3 iq3s iq3xxs + iq4nl } //! The superblock-lattice formats (Q8_K-form activations, % 256 rows, repack + stamped kq //! kernels). q51 deliberately fails this: it rides per-32 planes with Q8_0-form activations — //! a `fmt != KqFmt.q8` test does not imply the kq lattice; branch on kq_sb where it does. -def kq_sb(f : KqFmt) : bool => f == KqFmt.k4 || f == KqFmt.k5 || f == KqFmt.k6 || f == KqFmt.q40 || f == KqFmt.iq4xs || f == KqFmt.k3 || f == KqFmt.iq3s || f == KqFmt.iq3xxs +def kq_sb(f : KqFmt) : bool => f == KqFmt.k4 || f == KqFmt.k5 || f == KqFmt.k6 || f == KqFmt.q40 || f == KqFmt.iq4xs || f == KqFmt.k3 || f == KqFmt.iq3s || f == KqFmt.iq3xxs || f == KqFmt.iq4nl //! The same predicate over the int id space the GPU drivers carry per stack/plane. -def kq_sb(fmt : int) : bool => fmt == int(KqFmt.k4) || fmt == int(KqFmt.k5) || fmt == int(KqFmt.k6) || fmt == int(KqFmt.q40) || fmt == int(KqFmt.iq4xs) || fmt == int(KqFmt.k3) || fmt == int(KqFmt.iq3s) || fmt == int(KqFmt.iq3xxs) +def kq_sb(fmt : int) : bool => fmt == int(KqFmt.k4) || fmt == int(KqFmt.k5) || fmt == int(KqFmt.k6) || fmt == int(KqFmt.q40) || fmt == int(KqFmt.iq4xs) || fmt == int(KqFmt.k3) || fmt == int(KqFmt.iq3s) || fmt == int(KqFmt.iq3xxs) || fmt == int(KqFmt.iq4nl) //! KqFmt -> the kernel/IR format id space — the ONE bridge between the enum and the int ids the //! generated kernels take as runtime params. Mnemonic ids: a K-quant is its bit width (3/4/5/6), @@ -64,6 +65,9 @@ def kq_schema_id(f : KqFmt) : int { if (f == KqFmt.iq3xxs) { return 34 } + if (f == KqFmt.iq4nl) { + return 45 + } panic("kq_schema_id: not a kq superblock format") return 0 } @@ -88,6 +92,8 @@ let IQ3S_QSB = 104l // IQ3_S quant plane: [64 grid-index qs][8 qh ninth bits][ let IQ3S_SSB = 20l // IQ3_S scale plane: f16 d, 2 pad, 8 x int8 (1 + 2s), 8 pad — the iq4xs/k4 row shape let IQ3XXS_QSB = 96l // IQ3_XXS quant plane: [64 grid-index qs][32 aux bytes: per block 4x7-bit sign indices + 4-bit scale], verbatim disk order let IQ3XXS_SSB = 20l // IQ3_XXS scale plane: f16 d HALVED, 2 pad, 8 x uint8 (2*ls + 1), 8 pad — the iq3s row shape and fold (ggml's global 0.25 = the halved d x the halved grid) +// IQ4_NL reuses Q40_QSB/Q40_SSB outright: the disk bytes are q40's (f16 d + 16 nibble bytes +// per 32-block, the k/k+16 pairing) — only the nibble SEMANTICS differ (IQ4NL_LUT, no -8). //! The IQ4_NL / IQ4_XS nibble codebook: weight = scale x IQ4NL_LUT[nibble] (ggml's kvalues_iq4nl). //! Main-context use only (tests, oracles, the emitter's constant bake): a kernel that can run on a @@ -249,7 +255,7 @@ def f16_half_bits(u : uint) : uint { //! 256-weight superblock for the kq lattice and one 32-weight block for q51 (see kq_elems). //! q8 panics (its planes are byte-per-elem, not stride-walked here). def kq_qsb(f : KqFmt) : int64 { - if (f == KqFmt.k4 || f == KqFmt.q40 || f == KqFmt.iq4xs) { + if (f == KqFmt.k4 || f == KqFmt.q40 || f == KqFmt.iq4xs || f == KqFmt.iq4nl) { return K4_QSB } if (f == KqFmt.k5) { @@ -282,7 +288,7 @@ def kq_ssb(f : KqFmt) : int64 { if (f == KqFmt.k6 || f == KqFmt.k3) { return K6_SSB } - if (f == KqFmt.q40) { + if (f == KqFmt.q40 || f == KqFmt.iq4nl) { return Q40_SSB } if (f == KqFmt.q51) { diff --git a/modules/dasLLAMA/dasllama/dasllama_layout.das b/modules/dasLLAMA/dasllama/dasllama_layout.das index fca803bf69..976eee6534 100644 --- a/modules/dasLLAMA/dasllama/dasllama_layout.das +++ b/modules/dasLLAMA/dasllama/dasllama_layout.das @@ -384,6 +384,10 @@ def private repack_regions(var t : Model; regs : array) { var iq3xxssp : uint8? = null if (!empty(t.iq3xxsq)) { iq3xxsqp = addr(t.iq3xxsq[0]) } if (!empty(t.iq3xxss)) { iq3xxssp = addr(t.iq3xxss[0]) } + var iq4nlqp : uint8? = null + var iq4nlsp : uint8? = null + if (!empty(t.iq4nlq)) { iq4nlqp = addr(t.iq4nlq[0]) } + if (!empty(t.iq4nls)) { iq4nlsp = addr(t.iq4nls[0]) } if (!empty(t.qblob)) { qbp = addr(t.qblob[0]) } if (!empty(t.qscales)) { qsp = addr(t.qscales[0]) } if (!empty(t.mxq)) { mxqp = addr(t.mxq[0]) } @@ -415,8 +419,8 @@ def private repack_regions(var t : Model; regs : array) { let sb = rp[i].off / 256l let qsb = kq_qsb(f) let ssb = kq_ssb(f) - var kqp = f == 4 ? k4qp : (f == 5 ? k5qp : (f == 6 ? k6qp : (f == 40 ? q40qp : (f == 44 ? iq4xsqp : (f == 3 ? k3qp : (f == 33 ? iq3sqp : iq3xxsqp)))))) - var ksp = f == 4 ? k4sp : (f == 5 ? k5sp : (f == 6 ? k6sp : (f == 40 ? q40sp : (f == 44 ? iq4xssp : (f == 3 ? k3sp : (f == 33 ? iq3ssp : iq3xxssp)))))) + var kqp = f == 4 ? k4qp : (f == 5 ? k5qp : (f == 6 ? k6qp : (f == 40 ? q40qp : (f == 44 ? iq4xsqp : (f == 3 ? k3qp : (f == 33 ? iq3sqp : (f == 34 ? iq3xxsqp : iq4nlqp))))))) + var ksp = f == 4 ? k4sp : (f == 5 ? k5sp : (f == 6 ? k6sp : (f == 40 ? q40sp : (f == 44 ? iq4xssp : (f == 3 ? k3sp : (f == 33 ? iq3ssp : (f == 34 ? iq3xxssp : iq4nlsp))))))) invoke(rkq, f, kqp + sb * qsb, ksp + sb * ssb, rp[i].n, rp[i].d) } } @@ -550,6 +554,8 @@ def private push_repack_kq(var regs : array; fmt : KqFmt; woff, n, d push_repack(regs, 33, woff, n, d) } elif (fmt == KqFmt.iq3xxs) { push_repack(regs, 34, woff, n, d) + } elif (fmt == KqFmt.iq4nl) { + push_repack(regs, 45, woff, n, d) } } @@ -825,13 +831,13 @@ def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice : (fmt == KqFmt.k6 ? addr(t.k6q[sb0 * kq_qsb(6)]) : (fmt == KqFmt.q40 ? addr(t.q40q[sb0 * kq_qsb(40)]) : (fmt == KqFmt.iq4xs ? addr(t.iq4xsq[sb0 * kq_qsb(44)]) - : (fmt == KqFmt.k3 ? addr(t.k3q[sb0 * kq_qsb(3)]) : (fmt == KqFmt.iq3s ? addr(t.iq3sq[sb0 * kq_qsb(33)]) : addr(t.iq3xxsq[sb0 * kq_qsb(34)])))))))) + : (fmt == KqFmt.k3 ? addr(t.k3q[sb0 * kq_qsb(3)]) : (fmt == KqFmt.iq3s ? addr(t.iq3sq[sb0 * kq_qsb(33)]) : (fmt == KqFmt.iq3xxs ? addr(t.iq3xxsq[sb0 * kq_qsb(34)]) : addr(t.iq4nlq[sb0 * kq_qsb(45)]))))))))) let sp = (fmt == KqFmt.k4 ? addr(t.k4s[sb0 * kq_ssb(4)]) : (fmt == KqFmt.k5 ? addr(t.k5s[sb0 * kq_ssb(5)]) : (fmt == KqFmt.k6 ? addr(t.k6s[sb0 * kq_ssb(6)]) : (fmt == KqFmt.q40 ? addr(t.q40s[sb0 * kq_ssb(40)]) : (fmt == KqFmt.iq4xs ? addr(t.iq4xss[sb0 * kq_ssb(44)]) - : (fmt == KqFmt.k3 ? addr(t.k3s[sb0 * kq_ssb(3)]) : (fmt == KqFmt.iq3s ? addr(t.iq3ss[sb0 * kq_ssb(33)]) : addr(t.iq3xxss[sb0 * kq_ssb(34)])))))))) + : (fmt == KqFmt.k3 ? addr(t.k3s[sb0 * kq_ssb(3)]) : (fmt == KqFmt.iq3s ? addr(t.iq3ss[sb0 * kq_ssb(33)]) : (fmt == KqFmt.iq3xxs ? addr(t.iq3xxss[sb0 * kq_ssb(34)]) : addr(t.iq4nls[sb0 * kq_ssb(45)]))))))))) var wqp = addr(wq[0]) var wsp = addr(ws[0]) let njobs = is_job_que_available() ? min(nslices, 4 * (get_total_hw_jobs() + 1)) : 1 @@ -909,7 +915,7 @@ def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice } else { let rq = qp + sliceQ + ri * qrow + sbi * qsb let rs = sp + sliceS + ri * srow + sbi * dssb - if (fmt == KqFmt.q40 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s || fmt == KqFmt.iq3xxs) { + if (fmt == KqFmt.q40 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s || fmt == KqFmt.iq3xxs || fmt == KqFmt.iq4nl) { for (m in range64(qsb)) { // disk bytes already in the device form (k/k+16 pairs; k3's lanes) dq[m] = rq[m] } @@ -936,7 +942,7 @@ def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice dq[128l + p] = rq[128l + p] } } - if (fmt == KqFmt.k6 || fmt == KqFmt.q40 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s || fmt == KqFmt.iq3xxs) { // rows already in the decoded device form + if (fmt == KqFmt.k6 || fmt == KqFmt.q40 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s || fmt == KqFmt.iq3xxs || fmt == KqFmt.iq4nl) { // rows already in the decoded device form for (idx in range64(dssb)) { dsc[idx] = rs[idx] } diff --git a/modules/dasLLAMA/dasllama/dasllama_load.das b/modules/dasLLAMA/dasllama/dasllama_load.das index 03c908c4e7..6ed9e421c7 100644 --- a/modules/dasLLAMA/dasllama/dasllama_load.das +++ b/modules/dasLLAMA/dasllama/dasllama_load.das @@ -45,6 +45,7 @@ struct private LayoutSizes { k3_n : int64 iq3s_n : int64 iq3xxs_n : int64 + iq4nl_n : int64 pleq8_n : int64 // the dedicated q8 plane holding ONLY the PLE table (fp32/q4 serving) } @@ -61,6 +62,7 @@ struct private KqCursors { k3 : int64 iq3s : int64 iq3xxs : int64 + iq4nl : int64 } def private kq_take(var cur : KqCursors; f : KqFmt; n : int64) : int64 { @@ -109,6 +111,11 @@ def private kq_take(var cur : KqCursors; f : KqFmt; n : int64) : int64 { cur.iq3xxs += n return o } + if (f == KqFmt.iq4nl) { + let o = cur.iq4nl + cur.iq4nl += n + return o + } let o = cur.wo cur.wo += n return o @@ -408,7 +415,7 @@ def private layout_offsets(var t : Model) : LayoutSizes { // nolint:STYLE037,S fo += dim } return LayoutSizes(fblob_n = fo, wblob_n = cur.wo, mx_n = mx, bf16_n = bf16, - k4_n = cur.k4, k5_n = cur.k5, k6_n = cur.k6, q40_n = cur.q40, q51_n = cur.q51, iq4xs_n = cur.iq4xs, k3_n = cur.k3, iq3s_n = cur.iq3s, iq3xxs_n = cur.iq3xxs, pleq8_n = pleq8) + k4_n = cur.k4, k5_n = cur.k5, k6_n = cur.k6, q40_n = cur.q40, q51_n = cur.q51, iq4xs_n = cur.iq4xs, k3_n = cur.k3, iq3s_n = cur.iq3s, iq3xxs_n = cur.iq3xxs, iq4nl_n = cur.iq4nl, pleq8_n = pleq8) } @@ -675,6 +682,8 @@ def stream_field_of(t : Model; fmt : KqFmt) : string { return "iq3sq" } elif (fmt == KqFmt.iq3xxs) { return "iq3xxsq" + } elif (fmt == KqFmt.iq4nl) { + return "iq4nlq" } elif (t.quant == QuantMode.q8) { // the blob flavor carries q8 as gguf-native 34B blocks in ONE plane, never the split pair return t.metal_blob ? "mblob" : "qblob" @@ -799,6 +808,8 @@ def fill_stream_plane(m : GGUFMeta; bytes : array | #; var t : Model; fie memcpy(addr(t.iq3ss[(j.woff / 256l) * IQ3S_SSB]), addr(temp_s[0]), sh.sb) } elif (j.fmt == KqFmt.iq3xxs) { memcpy(addr(t.iq3xxss[(j.woff / 256l) * IQ3XXS_SSB]), addr(temp_s[0]), sh.sb) + } elif (j.fmt == KqFmt.iq4nl) { + memcpy(addr(t.iq4nls[(j.woff / 256l) * Q40_SSB]), addr(temp_s[0]), sh.sb) } elif (j.fmt == KqFmt.q51) { memcpy(addr(t.q51s[(j.woff / 32l) * Q51_SB]), addr(temp_s[0]), sh.sb) } elif (sh.sb == 0l && j.n > 0l && !t.metal_blob) { @@ -842,6 +853,9 @@ def fill_stream_plane(m : GGUFMeta; bytes : array | #; var t : Model; fie } elif (r.fmt == 34) { invoke(g_stream_repack, 34, addr(temp_q[((r.off - j.woff) / 256l) * IQ3XXS_QSB]), addr(t.iq3xxss[(r.off / 256l) * IQ3XXS_SSB]), r.n, r.d) + } elif (r.fmt == 45) { + invoke(g_stream_repack, 45, addr(temp_q[((r.off - j.woff) / 256l) * Q40_QSB]), + addr(t.iq4nls[(r.off / 256l) * Q40_SSB]), r.n, r.d) } } } @@ -1066,6 +1080,9 @@ def private transcode_kq_tensor(m : GGUFMeta; bytes : array | #; name : s } elif (fmt == KqFmt.iq3xxs) { gguf_transcode_iq3xxs(m, bytes, name, kq, ks, eloff, n, src_off) return "iq3xxs transcode (IQ3_XXS)" + } elif (fmt == KqFmt.iq4nl) { + gguf_transcode_iq4nl(m, bytes, name, kq, ks, eloff, n, src_off) + return "iq4nl transcode (IQ4_NL)" } gguf_transcode_q51(m, bytes, name, kq, ks, eloff, n, src_off) return "q51 transcode (Q5_1)" @@ -1112,6 +1129,8 @@ def private load_big(m : GGUFMeta; bytes : array | #; name : string; var kind = transcode_kq_tensor(m, bytes, name, fmt, t.iq3sq, t.iq3ss, woff, n, src_off, scratch) } elif (fmt == KqFmt.iq3xxs) { kind = transcode_kq_tensor(m, bytes, name, fmt, t.iq3xxsq, t.iq3xxss, woff, n, src_off, scratch) + } elif (fmt == KqFmt.iq4nl) { + kind = transcode_kq_tensor(m, bytes, name, fmt, t.iq4nlq, t.iq4nls, woff, n, src_off, scratch) } elif (fmt == KqFmt.q51) { kind = transcode_kq_tensor(m, bytes, name, fmt, t.q51q, t.q51s, woff, n, src_off, scratch) } elif (t.quant == QuantMode.q8) { @@ -1211,8 +1230,8 @@ def private log_load_report(t : Model) { to_log(LOG_INFO, "dasLLAMA noisy: backend '{active_kernel_backend()}' | kquant_native {get_kquant_native()} q40_native {get_kq_q40_native()} q50_native {get_kq_q50_native()}\n") // the tier arms at [init], before a --noisy CLI flag can land — restate it here so the flag alone suffices to_log(LOG_INFO, "dasLLAMA noisy: GPU tier installed {moe_gpu_tier_installed()} | want auto {gpu_want_auto()}, moe layers {gpu_want_moe_layers()}, stream {gpu_want_moe_stream()}\n") - to_log(LOG_INFO, "dasLLAMA noisy: format tags - q8 {tally_fmt(t, KqFmt.q8)}, k4 {tally_fmt(t, KqFmt.k4)}, k5 {tally_fmt(t, KqFmt.k5)}, k6 {tally_fmt(t, KqFmt.k6)}, q40 {tally_fmt(t, KqFmt.q40)}, q51 {tally_fmt(t, KqFmt.q51)}, iq4xs {tally_fmt(t, KqFmt.iq4xs)}, k3 {tally_fmt(t, KqFmt.k3)}, iq3s {tally_fmt(t, KqFmt.iq3s)}, iq3xxs {tally_fmt(t, KqFmt.iq3xxs)}\n") - to_log(LOG_INFO, "dasLLAMA noisy: plane MB - q8 {float(long_length(t.qblob)) / mb} (+scales {float(long_length(t.qscales) * 4l) / mb}), k4 {float(long_length(t.k4q) + long_length(t.k4s)) / mb}, k5 {float(long_length(t.k5q) + long_length(t.k5s)) / mb}, k6 {float(long_length(t.k6q) + long_length(t.k6s)) / mb}, q40 {float(long_length(t.q40q) + long_length(t.q40s)) / mb}, q51 {float(long_length(t.q51q) + long_length(t.q51s)) / mb}, iq4xs {float(long_length(t.iq4xsq) + long_length(t.iq4xss)) / mb}, k3 {float(long_length(t.k3q) + long_length(t.k3s)) / mb}, iq3s {float(long_length(t.iq3sq) + long_length(t.iq3ss)) / mb}, iq3xxs {float(long_length(t.iq3xxsq) + long_length(t.iq3xxss)) / mb}, f32 {float(long_length(t.wblob) * 4l) / mb}\n") + to_log(LOG_INFO, "dasLLAMA noisy: format tags - q8 {tally_fmt(t, KqFmt.q8)}, k4 {tally_fmt(t, KqFmt.k4)}, k5 {tally_fmt(t, KqFmt.k5)}, k6 {tally_fmt(t, KqFmt.k6)}, q40 {tally_fmt(t, KqFmt.q40)}, q51 {tally_fmt(t, KqFmt.q51)}, iq4xs {tally_fmt(t, KqFmt.iq4xs)}, k3 {tally_fmt(t, KqFmt.k3)}, iq3s {tally_fmt(t, KqFmt.iq3s)}, iq3xxs {tally_fmt(t, KqFmt.iq3xxs)}, iq4nl {tally_fmt(t, KqFmt.iq4nl)}\n") + to_log(LOG_INFO, "dasLLAMA noisy: plane MB - q8 {float(long_length(t.qblob)) / mb} (+scales {float(long_length(t.qscales) * 4l) / mb}), k4 {float(long_length(t.k4q) + long_length(t.k4s)) / mb}, k5 {float(long_length(t.k5q) + long_length(t.k5s)) / mb}, k6 {float(long_length(t.k6q) + long_length(t.k6s)) / mb}, q40 {float(long_length(t.q40q) + long_length(t.q40s)) / mb}, q51 {float(long_length(t.q51q) + long_length(t.q51s)) / mb}, iq4xs {float(long_length(t.iq4xsq) + long_length(t.iq4xss)) / mb}, k3 {float(long_length(t.k3q) + long_length(t.k3s)) / mb}, iq3s {float(long_length(t.iq3sq) + long_length(t.iq3ss)) / mb}, iq3xxs {float(long_length(t.iq3xxsq) + long_length(t.iq3xxss)) / mb}, iq4nl {float(long_length(t.iq4nlq) + long_length(t.iq4nls)) / mb}, f32 {float(long_length(t.wblob) * 4l) / mb}\n") } // GGML disk type -> plane format tag (non-K-quant types ride the classic q8 path) @@ -1238,6 +1257,9 @@ def private kq_fmt_of(gt : int) : KqFmt { if (gt == GGML_TYPE_IQ3_XXS) { return KqFmt.iq3xxs } + if (gt == GGML_TYPE_IQ4_NL) { + return KqFmt.iq4nl + } if (gt == GGML_TYPE_Q4_0 && get_kq_q40_native()) { return KqFmt.q40 } @@ -2075,6 +2097,7 @@ def private load_gguf_parsed(var t : Model; m : GGUFMeta; bytes : array | g_stream_plane_total["k3q"] = (sz.k3_n / 256l) * K3_QSB g_stream_plane_total["iq3sq"] = (sz.iq3s_n / 256l) * IQ3S_QSB g_stream_plane_total["iq3xxsq"] = (sz.iq3xxs_n / 256l) * IQ3XXS_QSB + g_stream_plane_total["iq4nlq"] = (sz.iq4nl_n / 256l) * Q40_QSB let wb = mode == QuantMode.q8 ? sz.wblob_n : 0l g_stream_plane_total[t.metal_blob ? "mblob" : "qblob"] = (t.metal_blob ? (wb / Q8_BLOCK_ELEMS) * (Q8_QPB + Q8_SPB) : wb) @@ -2151,6 +2174,14 @@ def private load_gguf_parsed(var t : Model; m : GGUFMeta; bytes : array | t.iq3xxss |> reserve((sz.iq3xxs_n / 256l) * IQ3XXS_SSB) t.iq3xxss |> resize((sz.iq3xxs_n / 256l) * IQ3XXS_SSB) } + if (sz.iq4nl_n > 0l) { + if (!stream_q) { + t.iq4nlq |> reserve((sz.iq4nl_n / 256l) * Q40_QSB) + t.iq4nlq |> resize((sz.iq4nl_n / 256l) * Q40_QSB) + } + t.iq4nls |> reserve((sz.iq4nl_n / 256l) * Q40_SSB) + t.iq4nls |> resize((sz.iq4nl_n / 256l) * Q40_SSB) + } if (mode == QuantMode.q8) { if (!stream_q) { t.qblob |> reserve(sz.wblob_n) @@ -2461,6 +2492,7 @@ def private load_gguf_parsed(var t : Model; m : GGUFMeta; bytes : array | t.kq_repack_mr3 = active_kq_layout_mr(3) t.kq_repack_mr33 = active_kq_layout_mr(33) t.kq_repack_mr34 = active_kq_layout_mr(34) + t.kq_repack_mr45 = active_kq_layout_mr(45) } } else { layout_repack_q8(t) @@ -2481,7 +2513,8 @@ def private load_gguf_parsed(var t : Model; m : GGUFMeta; bytes : array | t.kq_repack_mr3 = active_kq_layout_mr(3) t.kq_repack_mr33 = active_kq_layout_mr(33) t.kq_repack_mr34 = active_kq_layout_mr(34) - to_log(LOG_INFO, "dasLLAMA: K-quant planes repacked to the '{active_kernel_backend()}' layouts (k4 grp{t.kq_repack_mr4}, k5 grp{t.kq_repack_mr5}, k6 grp{t.kq_repack_mr6}, q40 grp{t.kq_repack_mr40}, iq4xs grp{t.kq_repack_mr44}, k3 grp{t.kq_repack_mr3}, iq3s grp{t.kq_repack_mr33}, iq3xxs grp{t.kq_repack_mr34})\n") + t.kq_repack_mr45 = active_kq_layout_mr(45) + to_log(LOG_INFO, "dasLLAMA: K-quant planes repacked to the '{active_kernel_backend()}' layouts (k4 grp{t.kq_repack_mr4}, k5 grp{t.kq_repack_mr5}, k6 grp{t.kq_repack_mr6}, q40 grp{t.kq_repack_mr40}, iq4xs grp{t.kq_repack_mr44}, k3 grp{t.kq_repack_mr3}, iq3s grp{t.kq_repack_mr33}, iq3xxs grp{t.kq_repack_mr34}, iq4nl grp{t.kq_repack_mr45})\n") } } } diff --git a/modules/dasLLAMA/dasllama/dasllama_math.das b/modules/dasLLAMA/dasllama/dasllama_math.das index 4df5a8b2b9..886bcea91c 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math.das +++ b/modules/dasLLAMA/dasllama/dasllama_math.das @@ -1088,6 +1088,7 @@ struct KernelBackend { kq_rows_k3 : MatmulKqRowsFn = @@kq_unset_rows kq_rows_iq3s : MatmulKqRowsFn = @@kq_unset_rows kq_rows_iq3xxs : MatmulKqRowsFn = @@kq_unset_rows + kq_rows_iq4nl : MatmulKqRowsFn = @@kq_unset_rows kq_batch : MatmulKqBatchFn = @@kq_unset_batch kq_groupn : MatmulKqGroupNFn = @@kq_unset_groupn kq_batch_groupn : MatmulKqBatchGroupNFn = @@kq_unset_batch_groupn @@ -1154,6 +1155,7 @@ var g_kq_rows_iq4xs = @@kq_unset_rows var g_kq_rows_k3 = @@kq_unset_rows var g_kq_rows_iq3s = @@kq_unset_rows var g_kq_rows_iq3xxs = @@kq_unset_rows +var g_kq_rows_iq4nl = @@kq_unset_rows var g_mm_kq_batch = @@kq_unset_batch var g_mm_kq_groupn = @@kq_unset_groupn var g_mm_kq_batch_groupn = @@kq_unset_batch_groupn @@ -1481,7 +1483,7 @@ def active_q8_layout_mr() : int64 => g_bake_cpu_override ? g_bake_cpu.q8_mr : g_ //! family split; the loader freezes these onto the Model at repack time. def active_kq_layout_mr(fmt : int) : int64 { if (g_bake_cpu_override) { - return fmt == 4 ? g_bake_cpu.kq_mr4 : (fmt == 5 ? g_bake_cpu.kq_mr5 : (fmt == 40 ? g_bake_cpu.kq_mr40 : (fmt == 44 ? g_bake_cpu.kq_mr44 : (fmt == 3 ? g_bake_cpu.kq_mr3 : (fmt == 33 ? g_bake_cpu.kq_mr33 : (fmt == 34 ? g_bake_cpu.kq_mr34 : g_bake_cpu.kq_mr6)))))) + return fmt == 4 ? g_bake_cpu.kq_mr4 : (fmt == 5 ? g_bake_cpu.kq_mr5 : (fmt == 40 ? g_bake_cpu.kq_mr40 : (fmt == 44 ? g_bake_cpu.kq_mr44 : (fmt == 3 ? g_bake_cpu.kq_mr3 : (fmt == 33 ? g_bake_cpu.kq_mr33 : (fmt == 34 ? g_bake_cpu.kq_mr34 : (fmt == 45 ? g_bake_cpu.kq_mr45 : g_bake_cpu.kq_mr6))))))) } return int64(invoke(g_active_kq_layout, fmt)) } @@ -1661,6 +1663,7 @@ def private activate(be : KernelBackend) { g_kq_rows_k3 = be.kq_rows_k3 g_kq_rows_iq3s = be.kq_rows_iq3s g_kq_rows_iq3xxs = be.kq_rows_iq3xxs + g_kq_rows_iq4nl = be.kq_rows_iq4nl g_mm_kq_batch = be.kq_batch g_mm_kq_groupn = be.kq_groupn g_mm_kq_batch_groupn = be.kq_batch_groupn @@ -1867,6 +1870,7 @@ def kq_rows_fn(fmt : int) : MatmulKqRowsFn { if (fmt == 3) return g_kq_rows_k3 if (fmt == 33) return g_kq_rows_iq3s if (fmt == 34) return g_kq_rows_iq3xxs + if (fmt == 45) return g_kq_rows_iq4nl return @@kq_unset_rows } @@ -3239,6 +3243,9 @@ def private restore_math_fn_defaults { // nolint:STYLE037,STYLE038 — flat seam if (g_kq_rows_iq3xxs == null) { g_kq_rows_iq3xxs = @@kq_unset_rows } + if (g_kq_rows_iq4nl == null) { + g_kq_rows_iq4nl = @@kq_unset_rows + } if (g_mm_kq_batch == null) { g_mm_kq_batch = @@kq_unset_batch } diff --git a/modules/dasLLAMA/dasllama/dasllama_math_default.das b/modules/dasLLAMA/dasllama/dasllama_math_default.das index 837a0442a8..dcf09ed92f 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_default.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_default.das @@ -742,6 +742,32 @@ def dot_iq3xxs_q8(kqrow : uint8 const?; ksrow : uint8 const?; xqp : int8 const?; return acc } +[unused_argument(xbsp)] +def dot_iq4nl_q8(kqrow : uint8 const?; ksrow : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n : int64) : float { + var acc = 0.0 + let lut = iq4nl_lut() + unsafe { + for (sb in range64(n / 256l)) { + let kqo = sb * 128l + let kso = sb * 16l + var facc = 0.0 + for (blk in range64(8l)) { + let ab = (sb * 8l + blk) * 32l + var idot = 0 + for (l in range64(16l)) { + let q = int(kqrow[kqo + blk * 16l + l]) + idot += int(lut[q & 15]) * int(xqp[ab + l]) + idot += int(lut[q >> 4]) * int(xqp[ab + 16l + l]) + } + let d = f16_to_f32(uint(ksrow[kso + 2l * blk]) | (uint(ksrow[kso + 2l * blk + 1l]) << 8u)) + facc += float(idot) * d + } + acc += facc * xsp[sb] + } + } + return acc +} + def iq3s_rows_kernel(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, rb, re : int64) { let nsb = n / 256l unsafe { @@ -760,6 +786,15 @@ def iq3xxs_rows_kernel(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; } } +def iq4nl_rows_kernel(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, rb, re : int64) { + let nsb = n / 256l + unsafe { + for (i in range64(rb, re)) { + yp[i] = dot_iq4nl_q8(kqp + i * nsb * 128l, ksp + i * nsb * 16l, xqp, xsp, xbsp, n) + } + } +} + def private kq_gemv_kernel(fmt : int; var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d : int64) { var myp = yp maybe_parallel_for(0, int(d), matmul_chunks_gemv(int(d), 1, n * d)) $(rb, re) { @@ -777,6 +812,8 @@ def private kq_gemv_kernel(fmt : int; var yp : float?; kqp : uint8 const?; ksp : iq3s_rows_kernel(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb), int64(re)) } elif (fmt == 34) { iq3xxs_rows_kernel(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb), int64(re)) + } elif (fmt == 45) { + iq4nl_rows_kernel(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb), int64(re)) } else { k6_rows_kernel(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb), int64(re)) } @@ -919,6 +956,28 @@ def private dequant_iq3xxs_row_grp(kqg : uint8 const?; ksg : uint8 const?; r, mr } } +//! iq4nl grp row dequant: q40's tiling and scale interleave, the codebook in place of (q - 8). +def private dequant_iq4nl_row_grp(kqg : uint8 const?; ksg : uint8 const?; r, mr, nsb : int64; var dst : float?) { + let lut = iq4nl_lut() + unsafe { + for (sbi in range64(nsb)) { + let qb = sbi * 128l * mr + let sb = sbi * 16l * mr + for (blk in range64(8l)) { + let kb = sbi * 256l + blk * 32l + let d = f16_to_f32(uint(ksg[sb + blk * 2l * mr + 2l * r]) | (uint(ksg[sb + blk * 2l * mr + 2l * r + 1l]) << 8u)) + for (j in range64(4l)) { + for (t in range64(4l)) { + let nib = uint(kqg[qb + ((blk * 4l + j) * mr + r) * 4l + t]) + dst[kb + j * 4l + t] = d * float(int(lut[int(nib & 15u)])) + dst[kb + 16l + j * 4l + t] = d * float(int(lut[int(nib >> 4u)])) + } + } + } + } + } +} + //! Dequant one row off the grp-REPACKED K-quant planes (repack_k4/k5/k6_grp layout). //! kqg/ksg = row's GROUP plane base (group g = row/mr); r = row % mr. //! embed_row's kq_repacked path — per-token, so scalar is fine. @@ -946,6 +1005,10 @@ def dequant_kq_row_grp(fmt : int64; kqg : uint8 const?; ksg : uint8 const?; r, m dequant_iq3xxs_row_grp(kqg, ksg, r, mr, nsb, dst) return } + if (fmt == 45l) { + dequant_iq4nl_row_grp(kqg, ksg, r, mr, nsb, dst) + return + } unsafe { for (sbi in range64(nsb)) { let qb = sbi * qsb * mr @@ -1048,6 +1111,8 @@ def matmul_kq_groupn(fmt : int; var y : array | #; kq : array | #; myp[ii] = dot_iq3s_q8(kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) } elif (fmt == 34) { myp[ii] = dot_iq3xxs_q8(kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) + } elif (fmt == 45) { + myp[ii] = dot_iq4nl_q8(kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) } else { myp[ii] = dot_k6q8(kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) } diff --git a/modules/dasLLAMA/dasllama/dasllama_math_gen.das b/modules/dasLLAMA/dasllama/dasllama_math_gen.das index 1705e247f0..44ce444832 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_gen.das @@ -259,6 +259,10 @@ def iq3xxsq8_layout_gen() : int { return q8q8_repack_type(GEMM_REFERENCE_MR).interleave } +def iq4nlq8_layout_gen() : int { + return q8q8_repack_type(GEMM_REFERENCE_MR).interleave +} + // the KernelBackend.kq_layout slot form of kq_layout_of (registered on both gen backends) def private kq_layout_fmt_gen(fmt : int) : int => int(kq_layout_of(fmt)) @@ -286,6 +290,9 @@ def kq_layout_of(fmt : int) : int64 { if (fmt == 34) { return int64(iq3xxsq8_layout_gen()) } + if (fmt == 45) { + return int64(iq4nlq8_layout_gen()) + } return int64(k6q8_layout_gen()) } @@ -745,6 +752,69 @@ def iq3xxsq8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; x } } +//! One row's dot off the grp iq4nl planes, scalar - q40's walk with the codebook and no +//! bsum term (the LUT values carry no offset). The iq4nl stubs' reference body and tile oracle. +[unused_argument(xbsp), hint(unsafe_range_check, noalias = kqg, noalias = ksg, noalias = xqp, noalias = xsp, noalias = xbsp)] +def iq4nl_grp_row_dot(kqg : uint8 const?; ksg : uint8 const?; r, mr : int64; xqp : int8 const?; xsp : float const?; xbsp : int const?; n : int64) : float { + var acc = 0.0 + let nsb = n / 256l + let lut = iq4nl_lut() + unsafe { + for (sbi in range64(nsb)) { + let qb = sbi * 128l * mr + let sb = sbi * 16l * mr + var facc = 0.0 + for (blk in range64(8l)) { + let b = sbi * 8l + blk + var idot = 0 + for (j in range64(4l)) { + for (t in range64(4l)) { + let nib = uint(kqg[qb + ((blk * 4l + j) * mr + r) * 4l + t]) + idot += int(lut[int(nib & 15u)]) * int(xqp[b * 32l + j * 4l + t]) + idot += int(lut[int(nib >> 4u)]) * int(xqp[b * 32l + 16l + j * 4l + t]) + } + } + let d = f16_to_f32(uint(ksg[sb + blk * 2l * mr + 2l * r]) | (uint(ksg[sb + blk * 2l * mr + 2l * r + 1l]) << 8u)) + facc += float(idot) * d + } + acc += facc * xsp[sbi] + } + } + return acc +} + +[unused_argument(xbsp), hint(unsafe_range_check, noalias = kqp, noalias = ksp, noalias = xqp, noalias = xsp, noalias = xbsp)] +def iq4nlq8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, rb, re : int64) : void { + let mr = int64(iq4nlq8_layout_gen()) + let nsb = n / 256l + unsafe { + for (i in range64(rb, re)) { + let g = i / mr + yp[i] = iq4nl_grp_row_dot(kqp + g * mr * nsb * 128l, ksp + g * mr * nsb * 16l, i % mr, mr, xqp, xsp, xbsp, n) + } + } +} + +[tune_perm(mr = 4), tune_perm(mr = 4, nrsplit = 2), tune_perm(mr = 8), tune_perm(mr = 8, nrsplit = 2), + tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "maddubs", width = 256, mr = 8, nrsplit = 2, requires = "avx2"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, nrsplit = 2, requires = "avxvnni|avx512vnni"), + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, nrsplit = 2, requires = "avx512vnni,avx512bw"), + tune_companion(fn = "iq4nlq8_gemv_gen", gen = "dasllama_gemm_gen::iq4nl_gemv"), + tune_companion(fn = "iq4nlq8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), + tune(gen = "dasllama_gemm_gen::iq4nl_tile", + fallback = "dot_vpdpbusd_width512_mr16;dot_vpdpbusd_width256_mr8;dot_maddubs_width256_mr8;mr4"), + hint(unsafe_range_check, noalias = kqg, noalias = ksg, noalias = xqp, noalias = xsp, noalias = xbsp)] +def iq4nlq8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d, g, t0 : int64) : void { + let mr = int64(iq4nlq8_layout_gen()) + unsafe { + for (t in range64(4l)) { + for (r in range64(mr)) { + yp[(t0 + t) * d + g * mr + r] = iq4nl_grp_row_dot(kqg, ksg, r, mr, xqp + (t0 + t) * n, xsp + (t0 + t) * (n / 256l), xbsp + (t0 + t) * (n / 16l), n) + } + } + } +} + //! The f16-scale GEMV companion (wscale_f16 rail): q8q8_gemv_gen's twin over the raw binary16 //! group-scale plane. Same rows-range contract; the generated body differs only in the scale //! fold's widen (fcvtl/vcvtph2ps). Not `private`: it IS the mm_rows_s16 slot. @@ -1076,6 +1146,8 @@ def private repack_kq_gen(fmt : int; var kq : uint8?; var ks : uint8?; n, d : in repack_iq3s_grp(kq, ks, n, d, kq_layout_of(33)) } elif (fmt == 34) { repack_iq3xxs_grp(kq, ks, n, d, kq_layout_of(34)) + } elif (fmt == 45) { + repack_q40_grp(kq, ks, n, d, kq_layout_of(45)) // byte-identical plane shapes } else { repack_k6_grp(kq, ks, n, d, kq_layout_of(6)) } @@ -1096,6 +1168,8 @@ def private repack_kq_bake(fmt : int; var kq : uint8?; var ks : uint8?; n, d : i repack_iq3s_grp(kq, ks, n, d, active_kq_layout_mr(33)) } elif (fmt == 34) { repack_iq3xxs_grp(kq, ks, n, d, active_kq_layout_mr(34)) + } elif (fmt == 45) { + repack_q40_grp(kq, ks, n, d, active_kq_layout_mr(45)) // byte-identical plane shapes } else { repack_k6_grp(kq, ks, n, d, active_kq_layout_mr(6)) } @@ -1614,6 +1688,8 @@ def private kq_kernel_gen(fmt : int; var yp : float?; kqp : uint8 const?; ksp : iq3sq8_gemv_gen(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb) * mr, int64(re) * mr) } elif (fmt == 34) { iq3xxsq8_gemv_gen(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb) * mr, int64(re) * mr) + } elif (fmt == 45) { + iq4nlq8_gemv_gen(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb) * mr, int64(re) * mr) } else { k6q8_gemv_gen(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb) * mr, int64(re) * mr) } @@ -1634,6 +1710,8 @@ def private kq_kernel_gen(fmt : int; var yp : float?; kqp : uint8 const?; ksp : myp[i] = dot_iq3s_q8(kqp + i * nsb * 104l, ksp + i * nsb * 20l, xqp, xsp, xbsp, n) } elif (fmt == 34) { myp[i] = dot_iq3xxs_q8(kqp + i * nsb * 96l, ksp + i * nsb * 20l, xqp, xsp, xbsp, n) + } elif (fmt == 45) { + myp[i] = dot_iq4nl_q8(kqp + i * nsb * 128l, ksp + i * nsb * 16l, xqp, xsp, xbsp, n) } else { myp[i] = dot_k6q8(kqp + i * nsb * 192l, ksp + i * nsb * 18l, xqp, xsp, xbsp, n) } @@ -1649,7 +1727,7 @@ def private kq_batch_cell_gen(fmt : int; var myp : float?; kqp : uint8 const?; k let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) let nb16 = n / 16l - let packed = fmt == 4 || fmt == 40 || fmt == 44 || fmt == 3 // packed planes: the tile reads them directly (iq3s expands its grid gather into the panel) + let packed = fmt == 4 || fmt == 40 || fmt == 44 || fmt == 3 || fmt == 45 // packed planes: the tile reads them directly (iq3s/iq3xxs expand their grid gathers into the panel) var scratch : array if (!packed) { scratch |> resize(mr * nsb * 256l) @@ -1683,6 +1761,8 @@ def private kq_batch_cell_gen(fmt : int; var myp : float?; kqp : uint8 const?; k iq3sq8_tile_gen(myp, scp, ksg, xqp, xsp, xbsp, n, d, g, tk) } elif (fmt == 34) { iq3xxsq8_tile_gen(myp, scp, ksg, xqp, xsp, xbsp, n, d, g, tk) + } elif (fmt == 45) { + iq4nlq8_tile_gen(myp, kqg, ksg, xqp, xsp, xbsp, n, d, g, tk) } else { k6q8_tile_gen(myp, scp, ksg, xqp, xsp, xbsp, n, d, g, tk) } @@ -1703,6 +1783,8 @@ def private kq_batch_cell_gen(fmt : int; var myp : float?; kqp : uint8 const?; k iq3sq8_gemv_gen(myp + tk * d, kqp, ksp, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n, g * mr, (g + 1l) * mr) } elif (fmt == 34) { iq3xxsq8_gemv_gen(myp + tk * d, kqp, ksp, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n, g * mr, (g + 1l) * mr) + } elif (fmt == 45) { + iq4nlq8_gemv_gen(myp + tk * d, kqp, ksp, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n, g * mr, (g + 1l) * mr) } else { k6q8_gemv_gen(myp + tk * d, kqp, ksp, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n, g * mr, (g + 1l) * mr) } @@ -1744,6 +1826,8 @@ def private kq_batch_kernel_gen(fmt : int; var yp : float?; kqp : uint8 const?; myp[tk * d + i] = dot_iq3s_q8(kqp + i * nsb * 104l, ksp + i * nsb * 20l, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * (n / 16l), n) } elif (fmt == 34) { myp[tk * d + i] = dot_iq3xxs_q8(kqp + i * nsb * 96l, ksp + i * nsb * 20l, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * (n / 16l), n) + } elif (fmt == 45) { + myp[tk * d + i] = dot_iq4nl_q8(kqp + i * nsb * 128l, ksp + i * nsb * 16l, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * (n / 16l), n) } else { myp[tk * d + i] = dot_k6q8(kqp + i * nsb * 192l, ksp + i * nsb * 18l, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * (n / 16l), n) } @@ -1800,6 +1884,8 @@ def private kq_batch_groupn_gen(fmt : int; var yp : float?; kqp : uint8 const?; myp[tk * d + i] = dot_iq3s_q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n) } elif (fmt == 34) { myp[tk * d + i] = dot_iq3xxs_q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n) + } elif (fmt == 45) { + myp[tk * d + i] = dot_iq4nl_q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n) } else { myp[tk * d + i] = dot_k6q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n) } @@ -1965,6 +2051,8 @@ def kq_groupn_gen(fmt : int; var yp : float?; kqp : uint8 const?; ksp : uint8 co iq3sq8_gemv_gen(myp + r * d, kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n, gLocal * mr, gEnd * mr) } elif (fmt == 34) { iq3xxsq8_gemv_gen(myp + r * d, kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n, gLocal * mr, gEnd * mr) + } elif (fmt == 45) { + iq4nlq8_gemv_gen(myp + r * d, kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n, gLocal * mr, gEnd * mr) } else { k6q8_gemv_gen(myp + r * d, kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n, gLocal * mr, gEnd * mr) } @@ -1991,6 +2079,8 @@ def kq_groupn_gen(fmt : int; var yp : float?; kqp : uint8 const?; ksp : uint8 co myp[r * d + i] = dot_iq3s_q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) } elif (fmt == 34) { myp[r * d + i] = dot_iq3xxs_q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) + } elif (fmt == 45) { + myp[r * d + i] = dot_iq4nl_q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) } else { myp[r * d + i] = dot_k6q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) } @@ -2284,7 +2374,7 @@ def dasllama_math_gen_register() { // the no-witness stance above covers kq too mm_kq = @@kq_kernel_gen, repack_kq = @@repack_kq_gen, kq_rows_k4 = @@k4q8_gemv_gen, kq_rows_k5 = @@k5q8_gemv_gen, - kq_rows_k6 = @@k6q8_gemv_gen, kq_rows_q40 = @@q40q8_gemv_gen, kq_rows_iq4xs = @@iq4xsq8_gemv_gen, kq_rows_k3 = @@k3q8_gemv_gen, kq_rows_iq3s = @@iq3sq8_gemv_gen, kq_rows_iq3xxs = @@iq3xxsq8_gemv_gen, + kq_rows_k6 = @@k6q8_gemv_gen, kq_rows_q40 = @@q40q8_gemv_gen, kq_rows_iq4xs = @@iq4xsq8_gemv_gen, kq_rows_k3 = @@k3q8_gemv_gen, kq_rows_iq3s = @@iq3sq8_gemv_gen, kq_rows_iq3xxs = @@iq3xxsq8_gemv_gen, kq_rows_iq4nl = @@iq4nlq8_gemv_gen, kq_batch = @@kq_batch_kernel_gen, kq_groupn = @@kq_groupn_gen, kq_batch_groupn = @@kq_batch_groupn_gen, // q51 family: tile + GEMV off the grp planes (the tile family's companions stamp @@ -2313,7 +2403,7 @@ def dasllama_math_gen_register() { mm_rows_s16 = @@q8q8_gemv_s16_gen, mm_kq = @@kq_kernel_gen, repack_kq = @@repack_kq_gen, kq_rows_k4 = @@k4q8_gemv_gen, kq_rows_k5 = @@k5q8_gemv_gen, - kq_rows_k6 = @@k6q8_gemv_gen, kq_rows_q40 = @@q40q8_gemv_gen, kq_rows_iq4xs = @@iq4xsq8_gemv_gen, kq_rows_k3 = @@k3q8_gemv_gen, kq_rows_iq3s = @@iq3sq8_gemv_gen, kq_rows_iq3xxs = @@iq3xxsq8_gemv_gen, + kq_rows_k6 = @@k6q8_gemv_gen, kq_rows_q40 = @@q40q8_gemv_gen, kq_rows_iq4xs = @@iq4xsq8_gemv_gen, kq_rows_k3 = @@k3q8_gemv_gen, kq_rows_iq3s = @@iq3sq8_gemv_gen, kq_rows_iq3xxs = @@iq3xxsq8_gemv_gen, kq_rows_iq4nl = @@iq4nlq8_gemv_gen, kq_batch = @@kq_batch_kernel_gen, kq_groupn = @@kq_groupn_gen, kq_batch_groupn = @@kq_batch_groupn_gen, groupn_q51 = @@q51q8_groupn_gen, repack_q51 = @@repack_q51_gen, diff --git a/modules/dasLLAMA/dasllama/dasllama_ple.das b/modules/dasLLAMA/dasllama/dasllama_ple.das index bd75a4f601..6f8ddd0cfe 100644 --- a/modules/dasLLAMA/dasllama/dasllama_ple.das +++ b/modules/dasLLAMA/dasllama/dasllama_ple.das @@ -70,6 +70,8 @@ def ple_check_table(t : Model; origin : string) { have = (long_length(t.iq3sq) / IQ3S_QSB) * 256l } elif (t.ple_emb_fmt == KqFmt.iq3xxs) { have = (long_length(t.iq3xxsq) / IQ3XXS_QSB) * 256l + } elif (t.ple_emb_fmt == KqFmt.iq4nl) { + have = (long_length(t.iq4nlq) / Q40_QSB) * 256l } if (have < need) { panic("dasLLAMA: the PLE token table is not resident ({origin}): fmt {t.ple_emb_fmt} plane holds {have} of {need} elements - regenerate this carrier (it predates the pinned-table rail)") @@ -111,6 +113,8 @@ def private ple_gather_row(t : Model; row : int64; var dst : array; doff dequant_iq3s_plane_superblock(t.iq3sq, sb * IQ3S_QSB, t.iq3ss, sb * IQ3S_SSB, dst, doff + blk * 256l) } elif (t.ple_emb_fmt == KqFmt.iq3xxs) { dequant_iq3xxs_plane_superblock(t.iq3xxsq, sb * IQ3XXS_QSB, t.iq3xxss, sb * IQ3XXS_SSB, dst, doff + blk * 256l) + } elif (t.ple_emb_fmt == KqFmt.iq4nl) { + dequant_iq4nl_plane_superblock(t.iq4nlq, sb * Q40_QSB, t.iq4nls, sb * Q40_SSB, dst, doff + blk * 256l) } else { panic("ple_gather_row: '{t.ple_emb_fmt}' has no kq plane pair") } diff --git a/modules/dasLLAMA/harness/gen_tune_probe.das b/modules/dasLLAMA/harness/gen_tune_probe.das index 62be3a5d74..c250f29cee 100644 --- a/modules/dasLLAMA/harness/gen_tune_probe.das +++ b/modules/dasLLAMA/harness/gen_tune_probe.das @@ -441,7 +441,7 @@ def kq_qsb(fmt : int64) : int64 => dasllama_gemm_schema::kq_qsb(int(fmt)) def kq_ssb(fmt : int64) : int64 => dasllama_gemm_schema::kq_ssb(int(fmt)) // the family's sidecar entry = its [tune] function name (the K-quants by bit width - k3 included - q40/iq4xs by tag) -def kq_tile_entry(fmt : int64) : string => fmt == 40l ? "q40q8_tile_gen" : (fmt == 44l ? "iq4xsq8_tile_gen" : (fmt == 33l ? "iq3sq8_tile_gen" : (fmt == 34l ? "iq3xxsq8_tile_gen" : "k{fmt}q8_tile_gen"))) +def kq_tile_entry(fmt : int64) : string => fmt == 40l ? "q40q8_tile_gen" : (fmt == 44l ? "iq4xsq8_tile_gen" : (fmt == 33l ? "iq3sq8_tile_gen" : (fmt == 34l ? "iq3xxsq8_tile_gen" : (fmt == 45l ? "iq4nlq8_tile_gen" : "k{fmt}q8_tile_gen")))) // synthetic disk superblocks, varied per (row, superblock) so no two blocks repeat: any byte // is a legal quant, and the small d/dmin keep the f16 (s, o) pairs finite and row sums tame @@ -516,6 +516,16 @@ def pack_kq_disk_block(fmt : int64; var blkb : array; base : int) { // for (i in range(96)) { blkb[2 + i] = uint8((base * 37 + i * 101 + 29) % 256) } + } elif (fmt == 45l) { + // 8 x 18B IQ4_NL disk blocks: per block f16 d + 16 nibble bytes — every pattern is legal + for (b in range(8)) { + let dbits = f32_to_f16(0.002 + 0.0001 * float((base + b) % 13)) + blkb[b * 18] = uint8(dbits & 0xFF) + blkb[b * 18 + 1] = uint8(dbits >> 8u) + for (i in range(16)) { + blkb[b * 18 + 2 + i] = uint8((base * 41 + b * 17 + i * 89 + 7) % 256) + } + } } elif (fmt == 4l) { pack_kq_scale_header(blkb, base) for (i in range(128)) { @@ -560,6 +570,8 @@ def repack_kq_grp_fmt(fmt : int64; var kq : uint8?; var ks : uint8?; n, d, mr : repack_iq3s_grp(kq, ks, n, d, mr) } elif (fmt == 34l) { repack_iq3xxs_grp(kq, ks, n, d, mr) + } elif (fmt == 45l) { + repack_q40_grp(kq, ks, n, d, mr) } else { repack_k6_grp(kq, ks, n, d, mr) } @@ -572,7 +584,7 @@ def build_kq_fixture(fmt : int64; n, d, ntok : int64) : KqFixture { // nolint: fx.kq |> resize(d * fx.nsb * qsb) fx.ks |> resize(d * fx.nsb * ssb) var blkb : array - blkb |> resize(fmt == 4l || fmt == 40l ? 144l : (fmt == 5l ? 176l : (fmt == 44l ? 136l : (fmt == 3l || fmt == 33l ? 110l : (fmt == 34l ? 98l : 210l))))) + blkb |> resize(fmt == 4l || fmt == 40l || fmt == 45l ? 144l : (fmt == 5l ? 176l : (fmt == 44l ? 136l : (fmt == 3l || fmt == 33l ? 110l : (fmt == 34l ? 98l : 210l))))) for (r in range64(d)) { for (s in range64(fx.nsb)) { let base = int(r * 31l + s * 7l) @@ -592,6 +604,8 @@ def build_kq_fixture(fmt : int64; n, d, ntok : int64) : KqFixture { // nolint: transcode_iq3s_superblock(blkb, 0l, fx.kq, sb * qsb, fx.ks, sb * ssb) } elif (fmt == 34l) { transcode_iq3xxs_superblock(blkb, 0l, fx.kq, sb * qsb, fx.ks, sb * ssb) + } elif (fmt == 45l) { + transcode_iq4nl_superblock(blkb, 0l, fx.kq, sb * qsb, fx.ks, sb * ssb) } else { transcode_q6k_superblock(blkb, 0l, fx.kq, sb * qsb, fx.ks, sb * ssb) } @@ -638,8 +652,11 @@ def build_kq_fixture(fmt : int64; n, d, ntok : int64) : KqFixture { // nolint: : (fmt == 34l ? iq3xxs_grp_row_dot(kqg, ksg, r, 4l, addr(fx.xq[0]) + tk * n, addr(fx.xs[0]) + tk * (n / 256l), addr(fx.xbs[0]) + tk * (n / 16l), n) - : kq_grp_row_dot(fmt, kqg, ksg, r, 4l, - addr(fx.xq[0]) + tk * n, addr(fx.xs[0]) + tk * (n / 256l), addr(fx.xbs[0]) + tk * (n / 16l), n)))))) + : (fmt == 45l + ? iq4nl_grp_row_dot(kqg, ksg, r, 4l, + addr(fx.xq[0]) + tk * n, addr(fx.xs[0]) + tk * (n / 256l), addr(fx.xbs[0]) + tk * (n / 16l), n) + : kq_grp_row_dot(fmt, kqg, ksg, r, 4l, + addr(fx.xq[0]) + tk * n, addr(fx.xs[0]) + tk * (n / 256l), addr(fx.xbs[0]) + tk * (n / 16l), n))))))) } } } @@ -673,6 +690,9 @@ def kq_tile_variants(fmt : int64) : array> { if (fmt == 34l) { return <- iq3xxsq8_tile_gen_variants() } + if (fmt == 45l) { + return <- iq4nlq8_tile_gen_variants() + } return <- k6q8_tile_gen_variants() } @@ -720,6 +740,12 @@ def kq_gemv_variants_by_suffix(fmt : int64) : table { tab[v._0] = v._1 } delete gvs + } elif (fmt == 45l) { + var gvs <- iq4nlq8_gemv_gen_variants() + for (v in gvs) { + tab[v._0] = v._1 + } + delete gvs } else { var gvs <- k6q8_gemv_gen_variants() for (v in gvs) { @@ -776,6 +802,12 @@ def kq_layout_mrs(fmt : int64) : table { mrs[v._0] = invoke(v._1) } delete lvs + } elif (fmt == 45l) { + var lvs <- iq4nlq8_layout_gen_variants() + for (v in lvs) { + mrs[v._0] = invoke(v._1) + } + delete lvs } else { var lvs <- k6q8_layout_gen_variants() for (v in lvs) { @@ -793,7 +825,7 @@ def kq_layout_mrs(fmt : int64) : table { def run_kq_tile(tile; fx : KqFixture; kq : array; ks : array; mr : int64; var y : array) { let ng = fx.d / mr verify(fx.ntok % 4l == 0l) - let packed = fx.fmt == 4l || fx.fmt == 40l || fx.fmt == 44l || fx.fmt == 3l // packed planes: the tile reads them directly (iq3s unpacks its panel per group) + let packed = fx.fmt == 4l || fx.fmt == 40l || fx.fmt == 44l || fx.fmt == 3l || fx.fmt == 45l // packed planes: the tile reads them directly (iq3s/iq3xxs unpack their panels per group) var panel : array if (!packed) { panel |> resize(int(mr * fx.nsb * 256l)) @@ -1277,6 +1309,7 @@ def test_mode : bool { // nolint:STYLE037,STYLE038 — flat per-variant correctn var kfxs3 <- [ <- build_kq_fixture(3l, 256l, 32l, 8l), <- build_kq_fixture(3l, 768l, 32l, 8l), <- build_kq_fixture(3l, 2048l, 32l, 8l)] var kfxs33 <- [ <- build_kq_fixture(33l, 256l, 32l, 8l), <- build_kq_fixture(33l, 768l, 32l, 8l), <- build_kq_fixture(33l, 2048l, 32l, 8l)] var kfxs34 <- [ <- build_kq_fixture(34l, 256l, 32l, 8l), <- build_kq_fixture(34l, 768l, 32l, 8l), <- build_kq_fixture(34l, 2048l, 32l, 8l)] + var kfxs45 <- [ <- build_kq_fixture(45l, 256l, 32l, 8l), <- build_kq_fixture(45l, 768l, 32l, 8l), <- build_kq_fixture(45l, 2048l, 32l, 8l)] var vs <- q8q8_tile_gen_variants() var mrs <- variant_mrs() var wbs <- variant_wbias() @@ -1405,6 +1438,7 @@ def test_mode : bool { // nolint:STYLE037,STYLE038 — flat per-variant correctn allok = kq_test_family(3l, kfxs3) && allok allok = kq_test_family(33l, kfxs33) && allok allok = kq_test_family(34l, kfxs34) && allok + allok = kq_test_family(45l, kfxs45) && allok delete wtab delete mtab delete gtab @@ -1424,6 +1458,7 @@ def test_mode : bool { // nolint:STYLE037,STYLE038 — flat per-variant correctn delete kfxs3 delete kfxs33 delete kfxs34 + delete kfxs45 return allok } @@ -1866,7 +1901,7 @@ def tune_mode_run : bool { // nolint:STYLE037,STYLE038 — flat family-by-family print("aborting mid-run - no sidecar written\n") return false } - for (fmt in fixed_array(4l, 5l, 6l, 40l, 44l, 3l, 33l, 34l)) { + for (fmt in fixed_array(4l, 5l, 6l, 40l, 44l, 3l, 33l, 34l, 45l)) { interrupt_gate("k{fmt}q8_tile_gen") let kq_t0 = ref_time_ticks() let entry = kq_tile_entry(fmt) diff --git a/modules/dasLLAMA/tests/test_kqformat.das b/modules/dasLLAMA/tests/test_kqformat.das index 2ef9e25e2a..5b15388d94 100644 --- a/modules/dasLLAMA/tests/test_kqformat.das +++ b/modules/dasLLAMA/tests/test_kqformat.das @@ -24,6 +24,7 @@ def test_kqfmt_enum_values(t : T?) { t |> equal(int(KqFmt.k3), 7) t |> equal(int(KqFmt.iq3s), 8) t |> equal(int(KqFmt.iq3xxs), 9) + t |> equal(int(KqFmt.iq4nl), 10) } } @@ -42,8 +43,10 @@ def test_kqfmt_predicates(t : T?) { t |> success(kq_sb(int(KqFmt.k3)), "the int-id predicate agrees for k3") t |> success(kq_sb(KqFmt.iq3s), "iq3s is superblock") t |> success(kq_sb(KqFmt.iq3xxs), "iq3xxs is superblock") + t |> success(kq_sb(KqFmt.iq4nl), "iq4nl is superblock") t |> success(kq_sb(int(KqFmt.iq3s)), "the int-id predicate agrees for iq3s") t |> success(kq_sb(int(KqFmt.iq3xxs)), "the int-id predicate agrees for iq3xxs") + t |> success(kq_sb(int(KqFmt.iq4nl)), "the int-id predicate agrees for iq4nl") } } @@ -66,8 +69,10 @@ def test_kqfmt_strides(t : T?) { t |> equal(kq_ssb(KqFmt.k3), 18l) // the k6 row: 16 int8 sub-scales + f16 d t |> equal(kq_qsb(KqFmt.iq3s), 104l) // [64 qs][8 qh][32 signs], verbatim t |> equal(kq_qsb(KqFmt.iq3xxs), 96l) // [64 qs][32 aux], verbatim + t |> equal(kq_qsb(KqFmt.iq4nl), 128l) // q40's nibble plane, verbatim t |> equal(kq_ssb(KqFmt.iq3s), 20l) // the k4 row: f16 d + 8 int8 (1 + 2s) t |> equal(kq_ssb(KqFmt.iq3xxs), 20l) // the same row: f16 d halved + 8 x (2ls + 1) + t |> equal(kq_ssb(KqFmt.iq4nl), 16l) // q40's row: 8 x f16 d } t |> run("named constants agree with the literals") @(t : T?) { t |> equal(K4_QSB, 128l) @@ -120,9 +125,10 @@ def test_kqfmt_id_bridge(t : T?) { t |> equal(kq_schema_id(KqFmt.k3), 3) t |> equal(kq_schema_id(KqFmt.iq3s), 33) t |> equal(kq_schema_id(KqFmt.iq3xxs), 34) + t |> equal(kq_schema_id(KqFmt.iq4nl), 45) } t |> run("gemm_schema's int-id forms delegate to the same truth") @(t : T?) { - for (f in [KqFmt.k4, KqFmt.k5, KqFmt.k6, KqFmt.q40, KqFmt.iq4xs, KqFmt.k3, KqFmt.iq3s, KqFmt.iq3xxs]) { + for (f in [KqFmt.k4, KqFmt.k5, KqFmt.k6, KqFmt.q40, KqFmt.iq4xs, KqFmt.k3, KqFmt.iq3s, KqFmt.iq3xxs, KqFmt.iq4nl]) { t |> equal(kq_qsb(kq_schema_id(f)), kq_qsb(f)) t |> equal(kq_ssb(kq_schema_id(f)), kq_ssb(f)) } @@ -140,6 +146,7 @@ def test_kqfmt_elems_and_stream(t : T?) { t |> equal(kq_elems(KqFmt.k3), 256l) t |> equal(kq_elems(KqFmt.iq3s), 256l) t |> equal(kq_elems(KqFmt.iq3xxs), 256l) + t |> equal(kq_elems(KqFmt.iq4nl), 256l) t |> equal(kq_elems(KqFmt.q8), 32l) t |> equal(kq_elems(KqFmt.q51), 32l) } @@ -154,6 +161,7 @@ def test_kqfmt_elems_and_stream(t : T?) { t |> equal(kq_stream_code(KqFmt.k3), 3) t |> equal(kq_stream_code(KqFmt.iq3s), 33) t |> equal(kq_stream_code(KqFmt.iq3xxs), 34) + t |> equal(kq_stream_code(KqFmt.iq4nl), 45) } } diff --git a/modules/dasLLAMA/tests/test_kquant.das b/modules/dasLLAMA/tests/test_kquant.das index 1d55b5841f..1abe1188a9 100644 --- a/modules/dasLLAMA/tests/test_kquant.das +++ b/modules/dasLLAMA/tests/test_kquant.das @@ -124,7 +124,24 @@ def private build_iq3xxs_block() : array { return <- blkb } -def private kq_tag(fmt : int) : string => fmt == 40 ? "q40" : (fmt == 44 ? "iq4xs" : (fmt == 33 ? "iq3s" : (fmt == 34 ? "iq3xxs" : "k{fmt}"))) +// iq4nl synthetic superblock: 8 x 18B IQ4_NL disk blocks (per block f16 d + 16 nibble bytes, +// q40's exact disk shape). Any byte pattern is legal. +def private iq4nl_qs_pat(i : int) : int => (i * 19 + 7) % 256 +def private build_iq4nl_block() : array { + var blkb : array + blkb |> resize(144) + for (b in range(8)) { + let dbits = f32_to_f16(0.002 + 0.0003 * float(b)) + blkb[b * 18] = uint8(dbits & 0xFF) + blkb[b * 18 + 1] = uint8(dbits >> 8u) + for (i in range(16)) { + blkb[b * 18 + 2 + i] = uint8(iq4nl_qs_pat(b * 16 + i)) + } + } + return <- blkb +} + +def private kq_tag(fmt : int) : string => fmt == 40 ? "q40" : (fmt == 44 ? "iq4xs" : (fmt == 33 ? "iq3s" : (fmt == 34 ? "iq3xxs" : (fmt == 45 ? "iq4nl" : "k{fmt}")))) // iq4xs synthetic superblock: 136B disk block — f16 d = SYNTH_D, the 8 six-bit sub-scales are // SYNTH_SC (packed low nibbles into scales_l, top two bits into scales_h), the shared nibble @@ -683,7 +700,7 @@ def test_q51_kernels(t : T?) { // elementwise dot over the plane dequant (the kernels' declared oracle). def private kq_dot_gate(t : T?; fmt : int; n : int64 = 512l) { let nsb = n / 256l - let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : build_q6k_block())))))) + let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : (fmt == 45 ? build_iq4nl_block() : build_q6k_block()))))))) let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) var kq : array @@ -705,6 +722,8 @@ def private kq_dot_gate(t : T?; fmt : int; n : int64 = 512l) { transcode_iq3s_superblock(blk1, 0l, kq, s * qsb, ks, s * ssb) } elif (fmt == 34) { transcode_iq3xxs_superblock(blk1, 0l, kq, s * qsb, ks, s * ssb) + } elif (fmt == 45) { + transcode_iq4nl_superblock(blk1, 0l, kq, s * qsb, ks, s * ssb) } else { transcode_q6k_superblock(blk1, 0l, kq, s * qsb, ks, s * ssb) } @@ -732,6 +751,8 @@ def private kq_dot_gate(t : T?; fmt : int; n : int64 = 512l) { dequant_iq3s_plane_superblock(kq, s * qsb, ks, s * ssb, w, s * 256l) } elif (fmt == 34) { dequant_iq3xxs_plane_superblock(kq, s * qsb, ks, s * ssb, w, s * 256l) + } elif (fmt == 45) { + dequant_iq4nl_plane_superblock(kq, s * qsb, ks, s * ssb, w, s * 256l) } else { dequant_k6_plane_superblock(kq, s * qsb, ks, s * ssb, w, s * 256l) } @@ -771,6 +792,8 @@ def private kq_dot_gate(t : T?; fmt : int; n : int64 = 512l) { got = dot_iq3s_q8(addr(kq[0]), addr(ks[0]), addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } elif (fmt == 34) { got = dot_iq3xxs_q8(addr(kq[0]), addr(ks[0]), addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + } elif (fmt == 45) { + got = dot_iq4nl_q8(addr(kq[0]), addr(ks[0]), addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } else { got = dot_k6q8(addr(kq[0]), addr(ks[0]), addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } @@ -804,7 +827,7 @@ def test_kq_dots(t : T?) { } // 122B-class row widths: n=3072 (12 superblocks — qwen35moe-122B gate/up/cls) and n=1024 // (its k5 down_exps). Every earlier kq model kept n at 512..2560. - for (fmt in [4, 5, 6, 40, 44, 3, 33, 34]) { + for (fmt in [4, 5, 6, 40, 44, 3, 33, 34, 45]) { for (nn in [1024l, 3072l]) { t |> run("dot_{kq_tag(fmt)}q8 matches the fp64 reference at n={int(nn)}") @(t : T?) { kq_dot_gate(t, fmt, nn) @@ -823,7 +846,7 @@ def private kq_gemv_rows_gate(t : T?; fmt : int; n : int64 = 512l) { let nsb = n / 256l let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) - let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : build_q6k_block())))))) + let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : (fmt == 45 ? build_iq4nl_block() : build_q6k_block()))))))) var kq : array var ks : array kq |> resize(d * nsb * qsb) @@ -844,6 +867,8 @@ def private kq_gemv_rows_gate(t : T?; fmt : int; n : int64 = 512l) { transcode_iq3s_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } elif (fmt == 34) { transcode_iq3xxs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 45) { + transcode_iq4nl_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } else { transcode_q6k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } @@ -890,6 +915,8 @@ def private kq_gemv_rows_gate(t : T?; fmt : int; n : int64 = 512l) { want = dot_iq3s_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } elif (fmt == 34) { want = dot_iq3xxs_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + } elif (fmt == 45) { + want = dot_iq4nl_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } else { want = dot_k6q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } @@ -931,7 +958,7 @@ def test_kq_gemv_rows(t : T?) { t |> run("portable iq3s GEMV rows bit-match per-row disk dots") @(t : T?) { kq_gemv_rows_gate(t, 33) } - for (fmt in [4, 5, 6, 40, 44, 3, 33, 34]) { + for (fmt in [4, 5, 6, 40, 44, 3, 33, 34, 45]) { for (nn in [1024l, 3072l]) { t |> run("portable {kq_tag(fmt)} GEMV rows bit-match per-row disk dots at n={int(nn)}") @(t : T?) { kq_gemv_rows_gate(t, fmt, nn) @@ -946,12 +973,12 @@ def test_kq_gemv_rows(t : T?) { // grp scalar reference (kq_grp_row_dot — the stamped cores' decline body) and the grp row // dequant to reproduce them BIT-EXACTLY (same block order, exact integer sub-sums — only the // plane layout moved). -def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { +def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { // nolint:STYLE037 — the flat per-format ladder (one arm per kq format) is the honest shape let d = 32l let nsb = n / 256l let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) - let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : build_q6k_block())))))) + let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : (fmt == 45 ? build_iq4nl_block() : build_q6k_block()))))))) var kq : array var ks : array kq |> resize(d * nsb * qsb) @@ -972,6 +999,8 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { transcode_iq3s_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } elif (fmt == 34) { transcode_iq3xxs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 45) { + transcode_iq4nl_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } else { transcode_q6k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } @@ -1017,6 +1046,8 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { want[r] = dot_iq3s_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } elif (fmt == 34) { want[r] = dot_iq3xxs_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + } elif (fmt == 45) { + want[r] = dot_iq4nl_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } else { want[r] = dot_k6q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } @@ -1035,6 +1066,8 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { dequant_iq3s_plane_superblock(kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb, wrow, r * n + s * 256l) } elif (fmt == 34) { dequant_iq3xxs_plane_superblock(kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb, wrow, r * n + s * 256l) + } elif (fmt == 45) { + dequant_iq4nl_plane_superblock(kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb, wrow, r * n + s * 256l) } else { dequant_k6_plane_superblock(kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb, wrow, r * n + s * 256l) } @@ -1054,6 +1087,8 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { repack_iq3s_grp(addr(kq[0]), addr(ks[0]), n, d, mr) } elif (fmt == 34) { repack_iq3xxs_grp(addr(kq[0]), addr(ks[0]), n, d, mr) + } elif (fmt == 45) { + repack_q40_grp(addr(kq[0]), addr(ks[0]), n, d, mr) } else { repack_k6_grp(addr(kq[0]), addr(ks[0]), n, d, mr) } @@ -1071,8 +1106,8 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { ? iq4xs_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) : (fmt == 3 ? k3_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) - : (fmt == 33 || fmt == 34 - ? (fmt == 33 ? iq3s_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) : iq3xxs_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n)) + : (fmt == 33 || fmt == 34 || fmt == 45 + ? (fmt == 33 ? iq3s_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) : (fmt == 34 ? iq3xxs_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) : iq4nl_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n))) : kq_grp_row_dot(int64(fmt), kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n))))) if (got != want[r]) { dotbad++ @@ -1100,7 +1135,7 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { [test] def test_kq_repack_grp(t : T?) { - for (fmt in [4, 5, 6, 40, 44, 3, 33, 34]) { + for (fmt in [4, 5, 6, 40, 44, 3, 33, 34, 45]) { for (mr in [4l, 8l, 16l]) { t |> run("repack_{kq_tag(fmt)}_grp mr={mr} preserves dots and row dequants") @(t : T?) { kq_repack_gate(t, fmt, mr) @@ -1125,7 +1160,7 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { // nolint:STYL let mr = kq_layout_of(fmt) // the format's OWN layout companion (per-format tune families) let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) - let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : build_q6k_block())))))) + let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : (fmt == 45 ? build_iq4nl_block() : build_q6k_block()))))))) var kq : array var ks : array kq |> resize(d * nsb * qsb) @@ -1146,6 +1181,8 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { // nolint:STYL transcode_iq3s_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } elif (fmt == 34) { transcode_iq3xxs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 45) { + transcode_iq4nl_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } else { transcode_q6k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } @@ -1171,6 +1208,8 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { // nolint:STYL repack_iq3s_grp(addr(kq[0]), addr(ks[0]), n, d, mr) } elif (fmt == 34) { repack_iq3xxs_grp(addr(kq[0]), addr(ks[0]), n, d, mr) + } elif (fmt == 45) { + repack_q40_grp(addr(kq[0]), addr(ks[0]), n, d, mr) } else { repack_k6_grp(addr(kq[0]), addr(ks[0]), n, d, mr) } @@ -1199,7 +1238,7 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { // nolint:STYL // k5/k6 tiles read the byte-expanded panel the batch cell unpacks per (group, token-block) // — the gate covers unpack_kq_panel_grp too (tile-over-panel vs gemv-over-packed planes); // k4/q40 tiles read the packed planes directly - let packed = fmt == 4 || fmt == 40 || fmt == 44 || fmt == 3 // iq3s/iq3xxs tiles read the byte-expanded panel (the batch cell's shape) + let packed = fmt == 4 || fmt == 40 || fmt == 44 || fmt == 3 || fmt == 45 // iq3s/iq3xxs tiles read the byte-expanded panel (the batch cell's shape) var panel : array if (!packed) { panel |> resize(mr * nsb * 256l) @@ -1228,6 +1267,8 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { // nolint:STYL iq3sq8_tile_gen(addr(ytile[0]), addr(panel[0]), ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) } elif (fmt == 34) { iq3xxsq8_tile_gen(addr(ytile[0]), addr(panel[0]), ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) + } elif (fmt == 45) { + iq4nlq8_tile_gen(addr(ytile[0]), kqg, ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) } else { k6q8_tile_gen(addr(ytile[0]), addr(panel[0]), ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) } @@ -1248,6 +1289,8 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { // nolint:STYL iq3sq8_gemv_gen(addr(ytile[tk * d]), addr(kq[0]), addr(ks[0]), addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, g * mr, (g + 1l) * mr) } elif (fmt == 34) { iq3xxsq8_gemv_gen(addr(ytile[tk * d]), addr(kq[0]), addr(ks[0]), addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, g * mr, (g + 1l) * mr) + } elif (fmt == 45) { + iq4nlq8_gemv_gen(addr(ytile[tk * d]), addr(kq[0]), addr(ks[0]), addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, g * mr, (g + 1l) * mr) } else { k6q8_gemv_gen(addr(ytile[tk * d]), addr(kq[0]), addr(ks[0]), addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, g * mr, (g + 1l) * mr) } @@ -1269,6 +1312,8 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { // nolint:STYL iq3sq8_gemv_gen(addr(ygemv[p * d]), addr(kq[0]), addr(ks[0]), addr(xq[p * n]), addr(xs[p * (n / 256l)]), addr(xbs[p * (n / 16l)]), n, 0l, d) } elif (fmt == 34) { iq3xxsq8_gemv_gen(addr(ygemv[p * d]), addr(kq[0]), addr(ks[0]), addr(xq[p * n]), addr(xs[p * (n / 256l)]), addr(xbs[p * (n / 16l)]), n, 0l, d) + } elif (fmt == 45) { + iq4nlq8_gemv_gen(addr(ygemv[p * d]), addr(kq[0]), addr(ks[0]), addr(xq[p * n]), addr(xs[p * (n / 256l)]), addr(xbs[p * (n / 16l)]), n, 0l, d) } else { k6q8_gemv_gen(addr(ygemv[p * d]), addr(kq[0]), addr(ks[0]), addr(xq[p * n]), addr(xs[p * (n / 256l)]), addr(xbs[p * (n / 16l)]), n, 0l, d) } @@ -1294,7 +1339,7 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { // nolint:STYL [test] def test_kq_tile(t : T?) { - for (fmt in [4, 5, 6, 40, 44, 3, 33, 34]) { + for (fmt in [4, 5, 6, 40, 44, 3, 33, 34, 45]) { t |> run("{kq_tag(fmt)} 4-token tile bit-matches per-token GEMVs") @(t : T?) { kq_tile_gate(t, fmt) } @@ -1315,7 +1360,7 @@ def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; let nsb = n / 256l let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) - let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : build_q6k_block())))))) + let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : (fmt == 45 ? build_iq4nl_block() : build_q6k_block()))))))) var kq : array var ks : array kq |> resize(nreg * d * nsb * qsb) @@ -1336,6 +1381,8 @@ def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; transcode_iq3s_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } elif (fmt == 34) { transcode_iq3xxs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 45) { + transcode_iq4nl_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } else { transcode_q6k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } @@ -1405,6 +1452,9 @@ def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; } elif (fmt == 34) { want_sh[r * d + row] = dot_iq3xxs_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) want_pr[r * d + row] = dot_iq3xxs_q8(kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n) + } elif (fmt == 45) { + want_sh[r * d + row] = dot_iq4nl_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + want_pr[r * d + row] = dot_iq4nl_q8(kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n) } else { want_sh[r * d + row] = dot_k6q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) want_pr[r * d + row] = dot_k6q8(kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n) @@ -1455,6 +1505,8 @@ def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; repack_iq3s_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) } elif (fmt == 34) { repack_iq3xxs_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) + } elif (fmt == 45) { + repack_q40_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) } else { repack_k6_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) } @@ -1483,6 +1535,9 @@ def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; } elif (fmt == 34) { iq3xxsq8_gemv_gen(addr(want_sh[r * d]), kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, 0l, d) iq3xxsq8_gemv_gen(addr(want_pr[r * d]), kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n, 0l, d) + } elif (fmt == 45) { + iq4nlq8_gemv_gen(addr(want_sh[r * d]), kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, 0l, d) + iq4nlq8_gemv_gen(addr(want_pr[r * d]), kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n, 0l, d) } else { k6q8_gemv_gen(addr(want_sh[r * d]), kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, 0l, d) k6q8_gemv_gen(addr(want_pr[r * d]), kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n, 0l, d) @@ -1518,7 +1573,7 @@ def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; [test] def test_kq_groupn(t : T?) { - for (fmt in [4, 5, 6, 40, 44, 3, 33, 34]) { + for (fmt in [4, 5, 6, 40, 44, 3, 33, 34, 45]) { t |> run("{kq_tag(fmt)} region-list GEMV bit-matches per-row dots (disk + grp slices)") @(t : T?) { kq_groupn_gate(t, fmt) } @@ -1562,7 +1617,7 @@ def private kq_batch_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 for (c in cnts) { nk += c } - let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : build_q6k_block())))))) + let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : (fmt == 45 ? build_iq4nl_block() : build_q6k_block()))))))) var kq : array var ks : array kq |> resize(nreg * d * nsb * qsb) @@ -1583,6 +1638,8 @@ def private kq_batch_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 transcode_iq3s_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } elif (fmt == 34) { transcode_iq3xxs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 45) { + transcode_iq4nl_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } else { transcode_q6k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } @@ -1609,6 +1666,8 @@ def private kq_batch_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 repack_iq3s_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) } elif (fmt == 34) { repack_iq3xxs_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) + } elif (fmt == 45) { + repack_q40_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) } else { repack_k6_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) } @@ -1661,6 +1720,8 @@ def private kq_batch_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 iq3sq8_gemv_gen(addr(want[tk * d]), kqr, ksr, addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, 0l, d) } elif (fmt == 34) { iq3xxsq8_gemv_gen(addr(want[tk * d]), kqr, ksr, addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, 0l, d) + } elif (fmt == 45) { + iq4nlq8_gemv_gen(addr(want[tk * d]), kqr, ksr, addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, 0l, d) } else { k6q8_gemv_gen(addr(want[tk * d]), kqr, ksr, addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, 0l, d) } @@ -1692,7 +1753,7 @@ def private kq_batch_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 [test] def test_kq_batch_groupn(t : T?) { - for (fmt in [4, 5, 6, 40, 44, 3, 33, 34]) { + for (fmt in [4, 5, 6, 40, 44, 3, 33, 34, 45]) { t |> run("{kq_tag(fmt)} batch groupn bit-matches per-token rows-core calls") @(t : T?) { kq_batch_groupn_gate(t, fmt) } @@ -2027,3 +2088,30 @@ def test_iq3xxs_planes(t : T?) { delete dst } } + +[test] +def test_iq4nl_planes(t : T?) { + t |> run("iq4nl planes split q40's bytes and decode through the codebook exactly") @(t : T?) { + let blkb <- build_iq4nl_block() + var kq : array + var ks : array + kq |> resize(128) + ks |> resize(16) + transcode_iq4nl_superblock(blkb, 0l, kq, 0l, ks, 0l) + var dst : array + dst |> resize(256) + dequant_iq4nl_plane_superblock(kq, 0l, ks, 0l, dst, 0l) + for (k in range(256)) { + let b = k / 32 + let l = k % 32 + let d = f16_to_f32(uint(blkb[b * 18]) | (uint(blkb[b * 18 + 1]) << 8u)) + let q = iq4nl_qs_pat(b * 16 + (l % 16)) + let nib = l < 16 ? q & 15 : q >> 4 + let expected = d * float(int(IQ4NL_LUT[nib])) + t |> success(dst[k] == expected, "iq4nl plane element must match the codebook recomputation exactly") + } + delete kq + delete ks + delete dst + } +} From 6db1e2c30dd729b4a62ed4eba3d848ffbb46f900 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Sun, 30 Aug 2026 23:24:58 -0700 Subject: [PATCH 28/61] dasLLAMA: IQ4_NL on the Vulkan tier - pure composition, zero new decodes Every piece already existed; this commit only connects them. KqGemvIq4nl = iq4xs's iq4_dot codebook compose (iq4_word, pure ALU) folded by q40's q40_d per-block f16 d - no sub-scale, no bsum term. KqBatchIq4nl : KqBatchIq4xs overrides ONLY stage_ws with q40's per-block d stage; the LUT stage_w and the no-min blk_fma inherit verbatim. Iq4nlCm2T = VkK4Blk (q40's nibble lane view) + the existing IQLUT workgroup axis, with Q40Cm2T's d pick times iq4lut[q]. vk_kq_schema_id 10 -> 45; the family cells grow to nine formats (stride 32 words); the cm2 cell mirrors q40's with the codebook oracle; kq_cls_ref takes a plain class arm (pure-ALU codebook - no workgroup state to fill on replay). Gates on the 5060 Ti: the suite 78/78 - the three cm2 tiles 0-off (89600 cells each), the nine-format gemv/batch families green. The resident driver arms on the local vehicle and runs gen 245 t/s; greedy ids 11/64 vs simple_ids (the near-tie class on this stream - its step-11 margin is 0.042 logits). Rows vs llama.cpp b10660 Vulkan (same vehicle): pp512 15027.4 / 19177.8 (0.78x), tg128 340.7 / 363.6 (0.94x). Metal: pending. Co-Authored-By: Claude Fable 5 --- modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md | 23 ++- modules/dasLLAMA/dasllama/dasllama_layout.das | 4 +- .../dasllama/dasllama_vulkan_classes.das | 109 ++++++++++++- .../dasllama/dasllama_vulkan_common.das | 7 +- .../dasllama/dasllama_vulkan_prefill.das | 17 +++ modules/dasLLAMA/tests/_vkd_oracles.das | 33 ++++ .../dasLLAMA/tests/test_vulkan_kernels.das | 144 +++++++++++++++++- 7 files changed, 320 insertions(+), 17 deletions(-) diff --git a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md index 217f5ee9a9..8203cef3d1 100644 --- a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md +++ b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md @@ -422,6 +422,14 @@ where, why it is so today, what unquirked looks like. An empty ledger is a legit iq2 tier - so the file cannot load end to end. `gguf_census.py` (scratch) reads the tensor type table in seconds; pick a vehicle whose mix is {supported} + the new format only (bartowski's Qwen2.5-1.5B IQ3_XS carries iq3_xxs with iq3_s/q4_K/q6_K siblings). +24. **A layout/pack edit does NOT invalidate cached `.dlim` images.** The prepared-image + filename hash folds the model and schema, not the packing code: after the grouped-scale + copy fix, the rerun "mapped" the stale image and kept serving the broken device planes - + garbage text and all-zero decode logits survived a correct fix. Delete the model's + `*.dlim` beside the GGUF after ANY `dasllama_layout.das`/pack edit and confirm the next + log line says "baked", not "mapped". Unquirked: fold a pack-code version into the image + hash, the way QUIRK 21's fix would version the JIT DLL cache. + ### IQ4_NL (the near-free one, 2026-08-30) Shape: 32-element blocks, 18B each on disk - f16 d + 16 nibble bytes with the k/k+16 @@ -444,8 +452,19 @@ Vehicle: a local requant again - IQ4_NL needs NO imatrix, just `--allow-requanti stamped: coherent text at gen 60 t/s, 11/64 greedy ids vs `simple_ids.exe` where the fork is a 0.042-logit near-tie (the window's smallest margin by 100x) whose top-2 IS our token. zen2 rows vs llama.cpp clean-cpu: pp512 618.3 vs 540.6 (1.14x), tg128 64.2 vs -62.2 (1.03x). Vulkan, Metal: pending (the generic-dequant class shells + the LUT -already live on both tiers). +62.2 (1.03x). + +Vulkan followed as pure composition - not one new decode was written: `KqGemvIq4nl` = +iq4xs's `iq4_dot` codebook compose folded by q40's `q40_d` per-block d (no sub-scale, no +bsum); `KqBatchIq4nl : KqBatchIq4xs` overrides ONLY `stage_ws` with q40's (the LUT stage_w +and the no-min blk_fma inherit); `Iq4nlCm2T` = `VkK4Blk` + the existing `IQLUT` axis with +Q40Cm2T's d pick times `iq4lut[q]`. Schema fmt 10 -> 45, the nine-format family cells at +stride 32 words, the cm2 cell mirrors q40's with the codebook oracle. Gates: the suite +78/78 (the three cm2 tiles 0-off at 89600 cells each); the resident driver arms and runs +gen 245 t/s, greedy ids 11/64 vs `simple_ids.exe` (the near-tie class - the margin +oracle's step-11 tie is 0.042 logits on this stream). Rows (5060 Ti vs llama.cpp b10660 +Vulkan): pp512 15027.4 vs 19177.8 (0.78x), tg128 340.7 vs 363.6 (0.94x). Metal: +pending (q40's Metal kernels + iq4xs's LUT staging compose the same way). ### IQ3_XXS (2026-08-30) diff --git a/modules/dasLLAMA/dasllama/dasllama_layout.das b/modules/dasLLAMA/dasllama/dasllama_layout.das index 976eee6534..68d65abe23 100644 --- a/modules/dasLLAMA/dasllama/dasllama_layout.das +++ b/modules/dasLLAMA/dasllama/dasllama_layout.das @@ -894,8 +894,8 @@ def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice dsc[17] = gs[16l * mr + 2l * r + 1l] dsc[18] = uint8(0) dsc[19] = uint8(0) - } elif (fmt == KqFmt.q40) { - for (blk in range64(8l)) { // 8 per-block f16 d, mr-interleaved + } elif (fmt == KqFmt.q40 || fmt == KqFmt.iq4nl) { + for (blk in range64(8l)) { // 8 per-block f16 d, mr-interleaved (iq4nl repacks with q40's interleave) dsc[blk * 2l] = gs[blk * 2l * mr + 2l * r] dsc[blk * 2l + 1l] = gs[blk * 2l * mr + 2l * r + 1l] } diff --git a/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das b/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das index 42883cdb7f..c733442996 100644 --- a/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das +++ b/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das @@ -2962,6 +2962,28 @@ class KqBatchIq3xxs : KqBatchIq4xs { } } +// IQ4_NL tile: iq4xs's LUT staging (stage_w and the no-min blk_fma inherit) with q40's +// per-block f16 d staged as the scale - no sub-scales, no bsum term anywhere. +[vk_dispatch(name = "kq_batch_iq4nl_cls", kernel = "run", family = "kq_batch_cls", grid = "wgs", params = "wgs : int64")] +class KqBatchIq4nl : KqBatchIq4xs { + def override stage_ws(wsb0, wt, nsb, s, tid : uint) { + let slane = tid % 32u + let sblk = tid / 32u + var sa = 0.0 + if (wt * 32u + slane < pa.d) { + let wsb = wsb0 + (wt * 32u + slane) * nsb + s + let dp = unpackHalf2x16(wsu[wsb * 5u + sblk / 2u]) + sa = sblk % 2u == 0u ? dp.x : dp.y + } + wsw[sblk * 32u + slane] = sa + } + + [spirv_kernel(local_size_x = 256, name = "kq_batch_iq4nl_cls_spv")] + def override run { + tile_shell() + } +} + // Q5_K tile: the k4 compose with the 5th bit OR'd in from the 40B block's qh words [vk_dispatch(name = "kq_batch_k5_cls", kernel = "run", family = "kq_batch_cls", grid = "wgs", params = "wgs : int64")] class KqBatchK5 : KqBatchBase { @@ -4368,6 +4390,26 @@ class template Iq3xxsCm2T : KqCm2BatchT { } } +[ |> template_struct_instance] +class template Iq4nlCm2T : KqCm2BatchT { + typedef BLK = VkK4Blk // q40's nibble plane verbatim - the k4 lane view fits + typedef ST = uint + override IQLUT = true + + [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] + def decode(blk : VkK4Blk; bc, cib : uint2) : float16 { + let g = cib.y >> 5u + let e = cib.y & 31u + let bidx = g * 16u + (e & 15u) + let by = uint(int(unpack8(blk.qs[int(bidx >> 1u)])[int(bidx & 1u)])) & 0xFFu + let q = (by >> ((e >> 4u) * 4u)) & 0xFu + let srow = (wg_blk0 + bc.x * (pa.n >> 8u) + bc.y) * 5u + let dp = unpackHalf2x16(ws[srow + (g >> 1u)]) + let d = (g & 1u) == 0u ? dp.x : dp.y + return float16(d * float(iq4lut[int(q)])) + } +} + // the stamps: the three width columns of each format (the dispatcher keys on (fmt, tc)) [vk_dispatch(name = "q8_batch_cm2l_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] class Q8Cm2LBatch : Q8Cm2T { @@ -4612,6 +4654,33 @@ class Iq3xxsCm2SBatch : Iq3xxsCm2T { typedef FLO = tensorLayout2DPad } +[vk_dispatch(name = "kq_batch_iq4nl_cm2l_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Iq4nlCm2LBatch : Iq4nlCm2T { + typedef BT = coopmatWgB_f16_64x256 + typedef ACC = coopmatWgAcc_f16_128x256 + typedef ACCW = coopmatWgAcc_f32_128x256 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_iq4nl_cm2m_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Iq4nlCm2MBatch : Iq4nlCm2T { + override BN = 128u + typedef BT = coopmatWgB_f16_64x128 + typedef ACC = coopmatWgAcc_f16_128x128 + typedef ACCW = coopmatWgAcc_f32_128x128 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_iq4nl_cm2s_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Iq4nlCm2SBatch : Iq4nlCm2T { + override BN = 32u + override STILE = true + typedef BT = coopmatWgB_f16_64x32 + typedef ACC = coopmatWgAcc_f16_128x32 + typedef ACCW = coopmatWgAcc_f32_128x32 + typedef FLO = tensorLayout2DPad +} + // ===== the split-k reduce (sums the cm2 partial planes into y) ===== struct SkRedArgs { @@ -5185,6 +5254,8 @@ def kq_batch_cls_ensure(fmt : int; q40cm : bool) : bool { return ensure_kq_batch_iq3s_cls() } elif (fmt == int(KqFmt.iq3xxs)) { return ensure_kq_batch_iq3xxs_cls() + } elif (fmt == int(KqFmt.iq4nl)) { + return ensure_kq_batch_iq4nl_cls() } panic("dasLLAMA vulkan tier: no kq batch class for format {fmt}") return false @@ -5219,6 +5290,8 @@ def kq_batch_cls_enc_for(fmt : int; q40cm : bool; raw : VkCommandBuffer; var h : enc_kq_batch_iq3s_cls(raw, h, set_, pc, wgs) } elif (fmt == int(KqFmt.iq3xxs)) { enc_kq_batch_iq3xxs_cls(raw, h, set_, pc, wgs) + } elif (fmt == int(KqFmt.iq4nl)) { + enc_kq_batch_iq4nl_cls(raw, h, set_, pc, wgs) } else { panic("dasLLAMA vulkan tier: no kq batch class for format {fmt}") } @@ -5247,6 +5320,8 @@ def gemv_cls_ensure(fmt : int) : bool { return ensure_kq_gemv_iq3s_cls() } elif (fmt == int(KqFmt.iq3xxs)) { return ensure_kq_gemv_iq3xxs_cls() + } elif (fmt == int(KqFmt.iq4nl)) { + return ensure_kq_gemv_iq4nl_cls() } panic("dasLLAMA vulkan tier: no GEMV class for format {fmt}") return false @@ -5281,14 +5356,16 @@ def gemv_cls_enc(fmt : int; raw : VkCommandBuffer; var h : VkHaz; var set_ : VkD enc_kq_gemv_iq3s_cls(raw, h, set_, pc, wgs) } elif (fmt == int(KqFmt.iq3xxs)) { enc_kq_gemv_iq3xxs_cls(raw, h, set_, pc, wgs) + } elif (fmt == int(KqFmt.iq4nl)) { + enc_kq_gemv_iq4nl_cls(raw, h, set_, pc, wgs) } else { panic("dasLLAMA vulkan tier: no GEMV class for format {fmt}") } } // Q4_K: w = d*sc*q - dmin*mn*1, q in [0,15] — per 32-block axs*(d*sc*idot - dmin*mn*bsum). -// The eight formats share family=kq_gemv_cls: ONE set layout + ONE set_kq_gemv_cls serves all -// eight pipelines (identical inherited members), each leaf keeping its own ensure_/enc_ names. +// The nine formats share family=kq_gemv_cls: ONE set layout + ONE set_kq_gemv_cls serves all +// nine pipelines (identical inherited members), each leaf keeping its own ensure_/enc_ names. [vk_dispatch(name = "kq_gemv_k4_cls", kernel = "run", family = "kq_gemv_cls", grid = "wgs", params = "wgs : int64")] class KqGemvK4 : KqGemvBase { def override blk_contrib(wsb, blk, xsb : uint) : float { @@ -5455,6 +5532,34 @@ class KqGemvIq3xxs : KqGemvBase { } } +// IQ4_NL: iq4xs's codebook dot (iq4_word, pure ALU) folded by q40's per-block f16 d - no +// sub-scales, no bsum term; the rows are q40's 32-word planes verbatim. +[vk_dispatch(name = "kq_gemv_iq4nl_cls", kernel = "run", family = "kq_gemv_cls", grid = "wgs", params = "wgs : int64")] +class KqGemvIq4nl : KqGemvBase { + def iq4_dot(wv, xl, xh : uint4) : int { + var idot = sdot4(iq4_word(wv.x & KQC_LOW), xl.x) + sdot4(iq4_word(wv.y & KQC_LOW), xl.y) + idot += sdot4(iq4_word(wv.z & KQC_LOW), xl.z) + sdot4(iq4_word(wv.w & KQC_LOW), xl.w) + idot += sdot4(iq4_word((wv.x >> 4u) & KQC_LOW), xh.x) + sdot4(iq4_word((wv.y >> 4u) & KQC_LOW), xh.y) + return idot + sdot4(iq4_word((wv.z >> 4u) & KQC_LOW), xh.z) + sdot4(iq4_word((wv.w >> 4u) & KQC_LOW), xh.w) + } + + def q40_d(wsb, blk : uint) : float { + let dp = unpackHalf2x16(wsu[wsb * 5u + blk / 2u]) + return blk % 2u == 0u ? dp.x : dp.y + } + + def override blk_contrib(wsb, blk, xsb : uint) : float { + let xb4 = xsb * 16u + blk * 2u + let idot = iq4_dot(wq4[wsb * 8u + blk], xq4[xb4], xq4[xb4 + 1u]) + return axs[xsb] * q40_d(wsb, blk) * float(idot) + } + + [spirv_kernel(local_size_x = 64, name = "kq_gemv_iq4nl_cls_spv")] + def run { + gemv_shell() + } +} + // Q3_K: q = 2-bit qs lane | hmask bit << 2, minus 4, per-16 SIGNED sub-scales x d — k6's fold with // the offset 4 over the k3 planes (6 uint4 per superblock: qs 0..3, hmask 4..5) [vk_dispatch(name = "kq_gemv_k3_cls", kernel = "run", family = "kq_gemv_cls", grid = "wgs", params = "wgs : int64")] diff --git a/modules/dasLLAMA/dasllama/dasllama_vulkan_common.das b/modules/dasLLAMA/dasllama/dasllama_vulkan_common.das index f134b2c868..22bd05f67d 100644 --- a/modules/dasLLAMA/dasllama/dasllama_vulkan_common.das +++ b/modules/dasLLAMA/dasllama/dasllama_vulkan_common.das @@ -1709,7 +1709,7 @@ struct ArenaSlab { } struct ArenaFmt { - fmt : int // int(KqFmt): 0 = q8, 1/2/3 = k4/k5/k6, 4 = q40, 6 = iq4xs, 7 = k3, 8 = iq3s, 9 = iq3xxs + fmt : int // int(KqFmt): 0 = q8, 1/2/3 = k4/k5/k6, 4 = q40, 6 = iq4xs, 7 = k3, 8 = iq3s, 9 = iq3xxs, 10 = iq4nl slabs : array cur : int // slab the bump cursor is in slab_cap : int64 // max blocks one slab may carry (msr over both plane strides) @@ -1740,7 +1740,7 @@ def arena_local_blk(blk : int64) : int64 => blk & ((1l << 32l) - 1l) // per-block plane strides — the block is 32 weights for q8 and a 256-weight superblock for kq. // Mirrors stack_plane_bytes: it computes the same totals as count * these strides. -// the tier's stack fmt space is int(KqFmt) — bridge to the gemm_schema kq id space (4/5/6/40/44/3/33/34) +// the tier's stack fmt space is int(KqFmt) — bridge to the gemm_schema kq id space (4/5/6/40/44/3/33/34/45) def vk_kq_schema_id(fmt : int) : int { if (fmt == 1) return 4 if (fmt == 2) return 5 @@ -1750,11 +1750,12 @@ def vk_kq_schema_id(fmt : int) : int { if (fmt == 7) return 3 if (fmt == 8) return 33 if (fmt == 9) return 34 + if (fmt == 10) return 45 panic("vk_kq_schema_id: not a kq stack fmt") return 0 } -// fmt space here = int(KqFmt): 0 q8 (32-elem blocks), 1-4 and 6..9 the kq lattice (+ q40, iq4xs, k3, iq3s, iq3xxs) +// fmt space here = int(KqFmt): 0 q8 (32-elem blocks), 1-4 and 6..10 the kq lattice (+ q40, iq4xs, k3, iq3s, iq3xxs, iq4nl) def vk_fmt_b32(fmt : int) : bool => fmt == 0 // meta/activation block unit of a stack format — q8 counts 32-weight blocks, the kq diff --git a/modules/dasLLAMA/dasllama/dasllama_vulkan_prefill.das b/modules/dasLLAMA/dasllama/dasllama_vulkan_prefill.das index 04c70f22fb..2a027a4549 100644 --- a/modules/dasLLAMA/dasllama/dasllama_vulkan_prefill.das +++ b/modules/dasLLAMA/dasllama/dasllama_vulkan_prefill.das @@ -82,6 +82,9 @@ def private cm2_cls_ensure(fmt : int; tc : int64) : bool { if (fmt == int(KqFmt.iq3xxs)) { return tc == 256l ? ensure_kq_batch_iq3xxs_cm2l_cls() : (tc == 128l ? ensure_kq_batch_iq3xxs_cm2m_cls() : ensure_kq_batch_iq3xxs_cm2s_cls()) } + if (fmt == int(KqFmt.iq4nl)) { + return tc == 256l ? ensure_kq_batch_iq4nl_cm2l_cls() : (tc == 128l ? ensure_kq_batch_iq4nl_cm2m_cls() : ensure_kq_batch_iq4nl_cm2s_cls()) + } verify(fmt == int(KqFmt.q8), "vk prefill: the cm2 ensure fell through for a kq format - a format the feed admits must own a tile arm") return tc == 256l ? ensure_q8_batch_cm2l_cls() : (tc == 128l ? ensure_q8_batch_cm2m_cls() : ensure_q8_batch_cm2s_cls()) } @@ -136,6 +139,12 @@ def private cm2_cls_set(fmt : int; tc : int64; bufs : uint64 const[5]; sizes : i } return tc == 128l ? set_kq_batch_iq3xxs_cm2m_cls(bufs, sizes, gbits) : set_kq_batch_iq3xxs_cm2s_cls(bufs, sizes, gbits) } + if (fmt == int(KqFmt.iq4nl)) { + if (tc == 256l) { + return set_kq_batch_iq4nl_cm2l_cls(bufs, sizes, gbits) + } + return tc == 128l ? set_kq_batch_iq4nl_cm2m_cls(bufs, sizes, gbits) : set_kq_batch_iq4nl_cm2s_cls(bufs, sizes, gbits) + } verify(fmt == int(KqFmt.q8), "vk prefill: the cm2 set fell through for a kq format - a format the feed admits must own a tile arm") if (tc == 256l) { return set_q8_batch_cm2l_cls(bufs, sizes, gbits) @@ -209,6 +218,14 @@ def private cm2_cls_enc(fmt : int; tc : int64; raw : VkCommandBuffer; var h : Vk } else { enc_kq_batch_iq3xxs_cm2s_cls(raw, h, s, pc, groups) } + } elif (fmt == int(KqFmt.iq4nl)) { + if (tc == 256l) { + enc_kq_batch_iq4nl_cm2l_cls(raw, h, s, pc, groups) + } elif (tc == 128l) { + enc_kq_batch_iq4nl_cm2m_cls(raw, h, s, pc, groups) + } else { + enc_kq_batch_iq4nl_cm2s_cls(raw, h, s, pc, groups) + } } elif (tc == 256l) { verify(fmt == int(KqFmt.q8), "vk prefill: the cm2 enc fell through for a kq format - a format the feed admits must own a tile arm") enc_q8_batch_cm2l_cls(raw, h, s, pc, groups) diff --git a/modules/dasLLAMA/tests/_vkd_oracles.das b/modules/dasLLAMA/tests/_vkd_oracles.das index 2f823375ca..c076a9b4ea 100644 --- a/modules/dasLLAMA/tests/_vkd_oracles.das +++ b/modules/dasLLAMA/tests/_vkd_oracles.das @@ -185,6 +185,8 @@ def kq_cls_ref(fmt : int; wq, ws, xq : array; xs : array) : KqGemvB kx.gridw[i] = IQ3XXS_GRID[i] } k = kx + } elif (fmt == int(KqFmt.iq4nl)) { + k = new KqGemvIq4nl() // pure-ALU codebook - no workgroup state to fill } else { k = new KqGemvK6() } @@ -437,6 +439,37 @@ def iq3xxsf16_gemm_oracle(wq : array; wsu : array; xf : array; } } +// IQ4_NL f16-feed GEMM reference over the device form: q40's walk with the codebook in place +// of (q - 8), f16-rounded per weight like its siblings (q40's 32-word rows + 8 f16 d). +def iq4nlf16_gemm_oracle(wq : array; wsu : array; xf : array; + recs : array; nrec, n, d : int; var y : array) { + let nsb = n / 256 + for (rr in range(nrec)) { + let wsb0 = int(recs[rr * 4]) + let row0 = int(recs[rr * 4 + 1]) + let cnt = int(recs[rr * 4 + 2]) + for (r in range(cnt)) { + for (c in range(d)) { + var acc = 0.0 + for (s in range(nsb)) { + let sb = wsb0 + c * nsb + s + for (g in range(8)) { + let dp = unpackHalf2x16(wsu[sb * 5 + g / 2]) + let dv = g % 2 == 0 ? dp.x : dp.y + for (e in range(32)) { + let by = byte_u8(wq, sb * 128 + g * 16 + e % 16) + let q = (by >> ((e / 16) * 4)) & 0xF + let w = float(float16(dv * float(int(IQ4NL_LUT[q])))) + acc += w * half_at(xf, (row0 + r) * n + s * 256 + g * 32 + e) + } + } + } + y[(row0 + r) * d + c] = acc + } + } + } +} + // kq batch GEMM oracle over region records — same blk_contrib per (row, col, block) def kq_gemm_oracle(fmt : int; wq, ws, xq : array; xs : array; recs : array; nrec, n, d : int; var y : array) { diff --git a/modules/dasLLAMA/tests/test_vulkan_kernels.das b/modules/dasLLAMA/tests/test_vulkan_kernels.das index f6b7e3746a..0b57ab49b0 100644 --- a/modules/dasLLAMA/tests/test_vulkan_kernels.das +++ b/modules/dasLLAMA/tests/test_vulkan_kernels.das @@ -381,7 +381,7 @@ def private attn_row_oracle(q : array; qbase : int; kk : array; kb def test_vkd_kq_gemv_family(t0 : T?) { t0 |> run("kq GEMV class family == the CPU oracle (all 6 formats)") <| @(t : T?) { static_if (typeinfo builtin_module_exists(vulkan)) { - if (!(ensure_kq_gemv_k4_cls() && ensure_kq_gemv_q40_cls() && ensure_kq_gemv_k5_cls() && ensure_kq_gemv_k6_cls() && ensure_kq_gemv_iq4xs_cls() && ensure_kq_gemv_k3_cls() && ensure_kq_gemv_iq3s_cls() && ensure_kq_gemv_iq3xxs_cls())) { + if (!(ensure_kq_gemv_k4_cls() && ensure_kq_gemv_q40_cls() && ensure_kq_gemv_k5_cls() && ensure_kq_gemv_k6_cls() && ensure_kq_gemv_iq4xs_cls() && ensure_kq_gemv_k3_cls() && ensure_kq_gemv_iq3s_cls() && ensure_kq_gemv_iq3xxs_cls() && ensure_kq_gemv_iq4nl_cls())) { feint("no Vulkan device - skipping\n") return } @@ -389,9 +389,9 @@ def test_vkd_kq_gemv_family(t0 : T?) { let d = 3 // rows per region let nreg = 2 let rows = nreg * d - let nfmt = 8 - let fmts = fixed_array(int(KqFmt.k4), int(KqFmt.q40), int(KqFmt.k5), int(KqFmt.k6), int(KqFmt.iq4xs), int(KqFmt.k3), int(KqFmt.iq3s), int(KqFmt.iq3xxs)) - let stride4s = fixed_array(32, 32, 40, 48, 32, 24, 26, 24) // uint WORDS per superblock (iq3s/iq3xxs are not uint4-aligned) + let nfmt = 9 + let fmts = fixed_array(int(KqFmt.k4), int(KqFmt.q40), int(KqFmt.k5), int(KqFmt.k6), int(KqFmt.iq4xs), int(KqFmt.k3), int(KqFmt.iq3s), int(KqFmt.iq3xxs), int(KqFmt.iq4nl)) + let stride4s = fixed_array(32, 32, 40, 48, 32, 24, 26, 24, 32) // uint WORDS per superblock (iq3s/iq3xxs are not uint4-aligned) for (fi in range(nfmt)) { let fmt = fmts[fi] let stride4 = stride4s[fi] @@ -469,6 +469,8 @@ def test_vkd_kq_gemv_family(t0 : T?) { enc_kq_gemv_iq3s_cls(raw2, h2, s_cls, pc, wgs) } elif (fmt == int(KqFmt.iq3xxs)) { enc_kq_gemv_iq3xxs_cls(raw2, h2, s_cls, pc, wgs) + } elif (fmt == int(KqFmt.iq4nl)) { + enc_kq_gemv_iq4nl_cls(raw2, h2, s_cls, pc, wgs) } else { enc_kq_gemv_k6_cls(raw2, h2, s_cls, pc, wgs) } @@ -1025,7 +1027,7 @@ def test_vkd_q8_batch_tile(t0 : T?) { def test_vkd_kq_batch_family(t0 : T?) { t0 |> run("kq batch tile class family == the CPU oracle (all 6 formats)") <| @(t : T?) { static_if (typeinfo builtin_module_exists(vulkan)) { - if (!(ensure_kq_batch_k4_cls() && ensure_kq_batch_q40_cls() && ensure_kq_batch_k5_cls() && ensure_kq_batch_k6_cls() && ensure_kq_batch_iq4xs_cls() && ensure_kq_batch_k3_cls() && ensure_kq_batch_iq3s_cls() && ensure_kq_batch_iq3xxs_cls())) { + if (!(ensure_kq_batch_k4_cls() && ensure_kq_batch_q40_cls() && ensure_kq_batch_k5_cls() && ensure_kq_batch_k6_cls() && ensure_kq_batch_iq4xs_cls() && ensure_kq_batch_k3_cls() && ensure_kq_batch_iq3s_cls() && ensure_kq_batch_iq3xxs_cls() && ensure_kq_batch_iq4nl_cls())) { feint("no Vulkan device - skipping\n") return } @@ -1040,9 +1042,9 @@ def test_vkd_kq_batch_family(t0 : T?) { let wgs1 = ((cnt1 + 31) / 32) * wtiles let wgs = wgs0 + wgs1 let totsb = 2 * d * nsb // superblocks across both regions - let nfmt = 8 - let wstrides = fixed_array(32, 32, 40, 48, 32, 24, 26, 24) // uint words per superblock: k4 q40 k5 k6 iq4xs k3 iq3s iq3xxs - let pipes = fixed_array(int(KqFmt.k4), int(KqFmt.q40), int(KqFmt.k5), int(KqFmt.k6), int(KqFmt.iq4xs), int(KqFmt.k3), int(KqFmt.iq3s), int(KqFmt.iq3xxs)) + let nfmt = 9 + let wstrides = fixed_array(32, 32, 40, 48, 32, 24, 26, 24, 32) // uint words per superblock: k4 q40 k5 k6 iq4xs k3 iq3s iq3xxs iq4nl + let pipes = fixed_array(int(KqFmt.k4), int(KqFmt.q40), int(KqFmt.k5), int(KqFmt.k6), int(KqFmt.iq4xs), int(KqFmt.k3), int(KqFmt.iq3s), int(KqFmt.iq3xxs), int(KqFmt.iq4nl)) for (fi in range(nfmt)) { let fmt = pipes[fi] let wstride = wstrides[fi] @@ -1123,6 +1125,8 @@ def test_vkd_kq_batch_family(t0 : T?) { enc_kq_batch_iq3s_cls(raw2, h2, sc, pc, int64(wgs)) } elif (fmt == int(KqFmt.iq3xxs)) { enc_kq_batch_iq3xxs_cls(raw2, h2, sc, pc, int64(wgs)) + } elif (fmt == int(KqFmt.iq4nl)) { + enc_kq_batch_iq4nl_cls(raw2, h2, sc, pc, int64(wgs)) } else { enc_kq_batch_k6_cls(raw2, h2, sc, pc, int64(wgs)) } @@ -1920,6 +1924,130 @@ def test_vkd_q40_cm2_batch(t0 : T?) { } } +[test] +def test_vkd_iq4nl_cm2_batch(t0 : T?) { + t0 |> run("cm2 IQ4_NL l/m/s tiles == the CPU oracle (codebook + per-block f16 d decode-in-load)") <| @(t : T?) { + static_if (typeinfo builtin_module_exists(vulkan)) { + if (!ensure_q8_batch_cls()) { // engage the device without touching coopmat rails + t |> skip("no Vulkan device") + return + } + if (!(g_gpu.coopmat_mode == 4 && g_gpu.has_coopmat2)) { + t |> skip("the cm2 iq4nl tiles serve only in mode 4 on an NV_coopmat2 device") + return + } + verify(ensure_kq_batch_iq4nl_cm2l_cls() && ensure_kq_batch_iq4nl_cm2m_cls() && ensure_kq_batch_iq4nl_cm2s_cls(), "iq4nl cm2 class rails must engage in cm2 mode") + let n = 768 // one 512-unrolled pass + 4 tail steps crossing superblock bounds + let nsb = n / 256 + let d = 160 // one full 128-tile + a 32 edge + let cnt0 = 300 + let cnt1 = 260 + let rows = cnt0 + cnt1 + let totsb = 2 * d * nsb + var wqh : array + var wsuh : array + var xfh : array + wqh |> resize(totsb * 32) + wsuh |> resize(totsb * 5) + xfh |> resize((rows + 32) * n / 2) // + the s tile's 32-row read slack + for (i in range(totsb * 32)) { + wqh[i] = hash_word(uint(i) + 29u) // arbitrary quant bytes + } + for (sb in range(totsb)) { + for (wi in range(4)) { // 8 per-block f16 d - cool, the f16 acc must stay far from 65504 + let p = sb * 4 + wi + wsuh[sb * 5 + wi] = packHalf2x16(float2(0.0002 * float(1 + p % 7), 0.00013 * float(1 + p % 5))) + } + wsuh[sb * 5 + 4] = 0u + } + for (i in range((rows + 32) * n / 2)) { + xfh[i] = ws_word(uint(i) * 3u + 5u) // f16 activation pairs + } + let wq_bytes = int64(totsb) * 128l + let ws_bytes = int64(length(wsuh)) * 4l + let xf_bytes = int64((rows + 32) * n) * 2l + let y_bytes = int64(rows * d) * 4l + let wqd = make_device_buf(wq_bytes) + let wsd = make_device_buf(ws_bytes) + let xfd = make_device_buf(xf_bytes) + let yd2 = make_device_buf(y_bytes) + var host = make_host_buf(y_bytes, true, [cached = true]) + var y_cls : array + y_cls |> resize(rows * d) + var y_ref : array + y_ref |> resize(rows * d) + for (ml in range(3)) { + let ttile = ml == 0 ? 256 : (ml == 1 ? 128 : 32) + let wtiles = (d + 127) / 128 + let wgs0 = ((cnt0 + ttile - 1) / ttile) * wtiles + let wgs1 = ((cnt1 + ttile - 1) / ttile) * wtiles + let wgs = wgs0 + wgs1 + var sched : array + sched |> resize(2 * 4 + wgs) + sched[0] = 0u + sched[1] = 0u + sched[2] = uint(cnt0) + sched[3] = 0u + sched[4] = uint(d * nsb) + sched[5] = uint(cnt0) + sched[6] = uint(cnt1) + sched[7] = uint(wgs0) + for (w in range(wgs)) { + sched[8 + w] = w < wgs0 ? 0u : 1u + } + unsafe { + upload_region_at(wqd, 0l, addr(wqh[0]), wq_bytes) + upload_region_at(wsd, 0l, addr(wsuh[0]), ws_bytes) + upload_region_at(xfd, 0l, addr(xfh[0]), xf_bytes) + let sc_bytes = int64(2 * 4 + wgs) * 4l + let scd = make_device_buf(sc_bytes) + upload_region_at(scd, 0l, addr(sched[0]), sc_bytes) + var sc = (ml == 0 + ? set_kq_batch_iq4nl_cm2l_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)) + : (ml == 1 + ? set_kq_batch_iq4nl_cm2m_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)) + : set_kq_batch_iq4nl_cm2s_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)))) + var raw2 = alloc_cmd() + let begin = VkCommandBufferBeginInfo() + vk_check(vkBeginCommandBuffer(raw2, begin), null) + var h2 : VkHaz + var pc = BatchArgs(n = uint(n), d = uint(d), map_off = uint(2 * 4)) + if (ml == 0) { + enc_kq_batch_iq4nl_cm2l_cls(raw2, h2, sc, pc, int64(wgs)) + } elif (ml == 1) { + enc_kq_batch_iq4nl_cm2m_cls(raw2, h2, sc, pc, int64(wgs)) + } else { + enc_kq_batch_iq4nl_cm2s_cls(raw2, h2, sc, pc, int64(wgs)) + } + vhz_dep(raw2, h2, 16u, 0u, true) + cmd_copy_whole(raw2, yd2, host.buf, y_bytes) + vk_check(vkEndCommandBuffer(raw2), null) + submit_wait(raw2) + memcpy(addr(y_cls[0]), host.mapped, y_bytes) + } + iq4nlf16_gemm_oracle(wqh, wsuh, xfh, sched, 2, n, d, y_ref) + let bad = mismatch_bars(y_cls, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) + to_log(LOG_INFO, "cm2 iq4nl {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile batch: {bad} of {rows * d} off the oracle\n") + t |> success(bad == 0, "cm2 iq4nl {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile matches the CPU oracle ({bad} of {rows * d} off)") + delete sched + } + delete y_ref + delete wqh + delete wsuh + delete xfh + delete y_cls + } else { + t |> skip("dasVulkan not present") + } + } +} + [test] def test_vkd_iq4xs_cm2_batch(t0 : T?) { t0 |> run("cm2 IQ4_XS l/m/s tiles == the CPU oracle (workgroup-staged codebook decode-in-load)") <| @(t : T?) { From a6945e23402d3ca0d62e48654402ba0fbfa74b13 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Mon, 31 Aug 2026 00:18:42 -0700 Subject: [PATCH 29/61] dasLLAMA: IQ4_NL on the Metal tier - the iq4xs kernels with the scale fold swapped, format closed The last tier, and the smallest Metal slice yet: no blob arm, no off-gate row, no split scale plane - q40's 16B/sb plane of 8 f16 d's IS the device form, bound once at soff (kq_scales_of hands doff = 0), and every kernel reads block bu's d at kdh[blk*8+bu] where iq4xs read per-sb d x strip byte. MetalKqGemvIq4nl, MetalKqMvIq4nlT B2/B4 + MetalKqMvB8Iq4nl are the iq4xs shells over the threadgroup codebook; MetalKqMulMmK45T grows an IQ4NL constant that rides the IQ4XS arm flipping only the dsc read. Ladders, prefill gates, the shapes gate, and the test cells follow at fmt 45 (fixtures: 16B of f16 per-block d's, kq_row_ref through dequant_iq4nl_plane_superblock; the five g_pso_* decls live in dasllama_metal_common.das - the [metal_dispatch] pso= attribute only names them). Gates on the M1 Max: test_metal_gemv_kernels 2/2, test_metal_gemm_kernels 2/2 (116 s corpus); the metal-blob e2e decodes the same stream, gen 285 t/s, greedy ids 11/64 vs simple_ids with the fork = the format's 0.042-logit near-tie at step 11 on every tier. Rows vs llama.cpp b10660 (same requant): Metal pp512 3599.5 / 3774.6 (0.95x), tg128 248.4 / 250.2 (0.99x); M1 CPU pp512 727.1 / 724.1 (1.00x), tg128 134.4 / 133.7 (1.01x). IQ4_NL = done on all four tiers. Co-Authored-By: Claude Fable 5 --- modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md | 28 +- .../dasllama/dasllama_metal_common.das | 13 + .../dasllama/dasllama_metal_kernels.das | 260 +++++++++++++++++- .../dasllama/dasllama_metal_prefill.das | 8 +- .../dasllama/dasllama_metal_shapes.das | 2 +- .../dasLLAMA/tests/_metal_kernel_common.das | 14 +- .../tests/test_metal_gemm_kernels.das | 10 +- .../tests/test_metal_gemv_kernels.das | 35 ++- 8 files changed, 341 insertions(+), 29 deletions(-) diff --git a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md index 8203cef3d1..cc0f45c57b 100644 --- a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md +++ b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md @@ -463,8 +463,32 @@ stride 32 words, the cm2 cell mirrors q40's with the codebook oracle. Gates: the 78/78 (the three cm2 tiles 0-off at 89600 cells each); the resident driver arms and runs gen 245 t/s, greedy ids 11/64 vs `simple_ids.exe` (the near-tie class - the margin oracle's step-11 tie is 0.042 logits on this stream). Rows (5060 Ti vs llama.cpp b10660 -Vulkan): pp512 15027.4 vs 19177.8 (0.78x), tg128 340.7 vs 363.6 (0.94x). Metal: -pending (q40's Metal kernels + iq4xs's LUT staging compose the same way). +Vulkan): pp512 15027.4 vs 19177.8 (0.78x), tg128 340.7 vs 363.6 (0.94x). + +Metal closed the format as the promised compose - the iq4xs kernels with the scale fold +swapped. No blob arm, no off-gate row, no split plane: q40's 16B/sb plane of 8 f16 d's IS +the device form, bound once at soff (kq_scales_of hands doff = 0), and every kernel reads +block bu's d at `kdh[blk*8+bu]` where iq4xs read per-sb d x strip byte. `MetalKqGemvIq4nl`, +`MetalKqMvIq4nlT` B2/B4 + `MetalKqMvB8Iq4nl` (the iq4xs shells over the threadgroup +codebook), and an `IQ4NL` constant on `MetalKqMulMmK45T` riding the IQ4XS arm with only the +dsc read flipped. One checklist line this walk misses easily: the `g_pso_*` globals are +DECLARED in dasllama_metal_common.das - the `[metal_dispatch]` pso= attribute only NAMES +them - so five new kernels = five decl lines there, or every reference errors 30838. Gates +on the M1 Max: test_metal_gemv_kernels 2/2, test_metal_gemm_kernels 2/2 (116 s corpus); the +metal-blob e2e decodes the same stream at gen 285 t/s, ids 11/64 with the fork = the +format's 0.042-logit near-tie at step 11 on every tier. + +Where IQ4_NL landed (vs llama.cpp b10660, the local requant): + +| tier | pp512 (ours / theirs) | tg128 (ours / theirs) | +|---|---|---| +| zen2 CPU | 618.3 / 540.6 (1.14x) | 64.2 / 62.2 (1.03x) | +| M1 CPU | 727.1 / 724.1 (1.00x) | 134.4 / 133.7 (1.01x) | +| 5060 Ti Vulkan | 15027.4 / 19177.8 (0.78x) | 340.7 / 363.6 (0.94x) | +| M1 Metal | 3599.5 / 3774.6 (0.95x) | 248.4 / 250.2 (0.99x) | + +The nearly-free prediction held: not one new decode on any tier, no CPU tg tail (no +gather), and the only real bug of the format was the layout copy arm QUIRK 24 caught. ### IQ3_XXS (2026-08-30) diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_common.das b/modules/dasLLAMA/dasllama/dasllama_metal_common.das index e8f64e80ef..efe619070e 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_common.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_common.das @@ -267,6 +267,11 @@ var g_pso_kq_mvb4_iq3s : MetalComputePipeline? var g_pso_kq_mvb8_iq3s : MetalComputePipeline? var g_pso_kq_mm_b33 : MetalComputePipeline? var g_pso_kq_iq3xxs : MetalComputePipeline? +var g_pso_kq_iq4nl : MetalComputePipeline? +var g_pso_kq_mvb2_iq4nl : MetalComputePipeline? +var g_pso_kq_mvb4_iq4nl : MetalComputePipeline? +var g_pso_kq_mvb8_iq4nl : MetalComputePipeline? +var g_pso_kq_mm_b45 : MetalComputePipeline? var g_pso_kq_mvb2_iq3xxs : MetalComputePipeline? var g_pso_kq_mvb4_iq3xxs : MetalComputePipeline? var g_pso_kq_mvb8_iq3xxs : MetalComputePipeline? @@ -870,6 +875,10 @@ def kq_quants_of(dev : MetalDevice?; t : Model; fmt : KqFmt; off : int64) : tupl return (buf = plane_buffer(dev, addr < void? >(t.iq3xxsq[0]), uint64(long_length(t.iq3xxsq)), t.image_map != null), qoff = uint64(sb0 * 96l)) } + if (fmt == KqFmt.iq4nl) { + return (buf = plane_buffer(dev, addr < void? >(t.iq4nlq[0]), uint64(long_length(t.iq4nlq)), t.image_map != null), + qoff = uint64(sb0 * 128l)) + } return (buf = plane_buffer(dev, addr < void? >(t.k6q[0]), uint64(long_length(t.k6q)), t.image_map != null), qoff = uint64(sb0 * 192l)) } @@ -911,6 +920,10 @@ def kq_scales_of(dev : MetalDevice?; t : Model; fmt : KqFmt; off : int64) : tupl return (buf = plane_buffer(dev, addr < void? >(t.iq4xss[0]), uint64(long_length(t.iq4xss)), t.image_map != null), soff = uint64(sb0 * 16l), doff = uint64(plane_sb * 16l + sb0 * 2l)) } + if (fmt == KqFmt.iq4nl) { // q40's 16B/sb plane of 8 f16 d - verbatim, one bind (doff unused) + return (buf = plane_buffer(dev, addr < void? >(t.iq4nls[0]), uint64(long_length(t.iq4nls)), t.image_map != null), + soff = uint64(sb0 * 16l), doff = 0ul) + } return (buf = plane_buffer(dev, addr < void? >(t.k5s[0]), uint64(long_length(t.k5s)), t.image_map != null), soff = uint64(sb0 * 16l), doff = 0ul) } diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das index b1880a0cb1..2f928268ca 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das @@ -3656,6 +3656,11 @@ def metal_decode_init : bool { // nolint:STYLE038 — a flat compile_pso per k g_pso_kq_mvb4_iq3xxs = compile_pso(MetalKqMvB4Iq3xxs_metal_kq_mv_iq3xxs_msl, MetalKqMvB4Iq3xxs_metal_kq_mv_iq3xxs_msl_entry, MetalKqMvB4Iq3xxs_metal_kq_mv_iq3xxs_msl_fastmath, ok) g_pso_kq_mvb8_iq3xxs = compile_pso(metal_kq_mvb8_iq3xxs_msl, metal_kq_mvb8_iq3xxs_msl_entry, metal_kq_mvb8_iq3xxs_msl_fastmath, ok) g_pso_kq_mm_b34 = compile_pso(MetalKqMulMmIq3xxs_metal_kq_mulmm_msl, MetalKqMulMmIq3xxs_metal_kq_mulmm_msl_entry, MetalKqMulMmIq3xxs_metal_kq_mulmm_msl_fastmath, ok) + g_pso_kq_iq4nl = compile_pso(metal_kq_gemv_iq4nl_msl, metal_kq_gemv_iq4nl_msl_entry, metal_kq_gemv_iq4nl_msl_fastmath, ok) + g_pso_kq_mvb2_iq4nl = compile_pso(MetalKqMvB2Iq4nl_metal_kq_mv_iq4nl_msl, MetalKqMvB2Iq4nl_metal_kq_mv_iq4nl_msl_entry, MetalKqMvB2Iq4nl_metal_kq_mv_iq4nl_msl_fastmath, ok) + g_pso_kq_mvb4_iq4nl = compile_pso(MetalKqMvB4Iq4nl_metal_kq_mv_iq4nl_msl, MetalKqMvB4Iq4nl_metal_kq_mv_iq4nl_msl_entry, MetalKqMvB4Iq4nl_metal_kq_mv_iq4nl_msl_fastmath, ok) + g_pso_kq_mvb8_iq4nl = compile_pso(metal_kq_mvb8_iq4nl_msl, metal_kq_mvb8_iq4nl_msl_entry, metal_kq_mvb8_iq4nl_msl_fastmath, ok) + g_pso_kq_mm_b45 = compile_pso(MetalKqMulMmIq4nl_metal_kq_mulmm_msl, MetalKqMulMmIq4nl_metal_kq_mulmm_msl_entry, MetalKqMulMmIq4nl_metal_kq_mulmm_msl_fastmath, ok) // the batched-step set (P4) g_pso_gemv_b2 = compile_pso(MetalGemvB2_metal_gemv_msl, MetalGemvB2_metal_gemv_msl_entry, MetalGemvB2_metal_gemv_msl_fastmath, ok) g_pso_gemv_b4 = compile_pso(MetalGemvB4_metal_gemv_msl, MetalGemvB4_metal_gemv_msl_entry, MetalGemvB4_metal_gemv_msl_fastmath, ok) @@ -3730,6 +3735,8 @@ def enc_kq_gemv(enc : MetalComputeEncoder?; fmt : KqFmt; bq : MetalBuffer?; qoff enc_kq_iq3s_c(enc, bs, doff, bs, soff, bq, qoff, bx, xoff, by, yoff, bn, bd, rows, n) } elif (fmt == KqFmt.iq3xxs) { enc_kq_iq3xxs_c(enc, bs, doff, bs, soff, bq, qoff, bx, xoff, by, yoff, bn, bd, rows, n) + } elif (fmt == KqFmt.iq4nl) { + enc_kq_iq4nl_c(enc, bs, soff, bs, soff, bq, qoff, bx, xoff, by, yoff, bn, bd, rows, n) } elif (fmt == KqFmt.k6) { enc_kq_k6_c(enc, bs, doff, bs, soff, bq, qoff, bx, xoff, by, yoff, bn, bd, rows, n) } elif (fmt == KqFmt.k5 && (n < 3072l || rows <= 1024l)) { @@ -3798,6 +3805,14 @@ def enc_kq_mvb(enc : MetalComputeEncoder?; fmt : KqFmt; bq : MetalBuffer?; qoff } else { enc_kq_mvb4_iq3xxs_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows, gcols) } + } elif (fmt == KqFmt.iq4nl) { + if (b8) { + enc_kq_mvb8_iq4nl_c(enc, bs, soff, bs, soff, bq, qoff, bx, by, yoff, ka, rows) + } elif (two) { + enc_kq_mvb2_iq4nl_c(enc, bs, soff, bs, soff, bq, qoff, bx, by, yoff, ka, rows) + } else { + enc_kq_mvb4_iq4nl_c(enc, bs, soff, bs, soff, bq, qoff, bx, by, yoff, ka, rows, gcols) + } } elif (fmt == KqFmt.k6) { if (b8) { enc_kq_mvb8_k6_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows) @@ -3838,6 +3853,8 @@ def enc_kq_gemm_mm_b(enc : MetalComputeEncoder?; fmt : KqFmt; bq : MetalBuffer?; enc_kq_mm_iq3s_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, bk, bn, mp, rows) } elif (fmt == KqFmt.iq3xxs) { enc_kq_mm_iq3xxs_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, bk, bn, mp, rows) + } elif (fmt == KqFmt.iq4nl) { + enc_kq_mm_iq4nl_c(enc, bs, soff, bs, soff, bq, qoff, bx, by, yoff, bk, bn, mp, rows) } elif (fmt == KqFmt.k6) { enc_kq_mm_k6_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, bk, bn, mp, rows) } elif (fmt == KqFmt.k4) { @@ -4284,6 +4301,11 @@ def metal_kernels_release { // nolint:STYLE037,STYLE038 — a flat release_pso release_pso(g_pso_kq_mvb4_iq3xxs) release_pso(g_pso_kq_mvb8_iq3xxs) release_pso(g_pso_kq_mm_b34) + release_pso(g_pso_kq_iq4nl) + release_pso(g_pso_kq_mvb2_iq4nl) + release_pso(g_pso_kq_mvb4_iq4nl) + release_pso(g_pso_kq_mvb8_iq4nl) + release_pso(g_pso_kq_mm_b45) if (g_pso_rms != null) { metal_release(g_pso_rms) g_pso_rms = null @@ -5513,6 +5535,7 @@ class template MetalKqMulMmK45T : MetalMmTileBase { @template_constant K3 : bool = false // q3_K: the k6 split scale form, 2-bit lane + hmask compose, offset 4 @template_constant IQ3S : bool = false // iq3s: the k6 split scale form, threadgroup grid gather + sign flip @template_constant IQ3XXS : bool = false // iq3xxs: the same form over the halved grid + parity signs + @template_constant IQ4NL : bool = false // iq4nl: the IQ4XS nibble walk with the per-32-block f16 d plane (no sub-scale; s0off = soff) @workgroup @template_gate = IQ4XS iq4lut : float[16] // kvalues_iq4nl as floats, staged once per threadgroup @workgroup @template_gate = IQ3S gridw3 : uint[512] // the 2 KB iq3s grid, staged once per threadgroup @workgroup @template_gate = IQ3XXS gridw3x : uint[256] // the 1 KB halved iq3xxs grid, staged once per threadgroup @@ -5673,10 +5696,15 @@ class template MetalKqMulMmK45T : MetalMmTileBase { static_if (IQ4XS) { // iq4xs: strip byte js is the SIGNED sub-scale, super-d off the d plane (the k6 split // form); block js = 4 uints of the q40 pairing, il0 picks the nibble half; the codebook - // off threadgroup memory - let sv = ks4[blk] - let scw = js < 4u ? sv.x : sv.y - let dsc = float(ksh[blk]) * float(((int(scw >> ((js & 3u) * 8u)) & 255) ^ 128) - 128) + // off threadgroup memory. iq4nl: same walk, dsc = block js's own f16 d + var dsc = 0.0 + static_if (IQ4NL) { + dsc = float(ksh[blk * 8u + js]) + } else { + let sv = ks4[blk] + let scw = js < 4u ? sv.x : sv.y + dsc = float(ksh[blk]) * float(((int(scw >> ((js & 3u) * 8u)) & 255) ^ 128) - 128) + } let qb = blk * 32u + js * 4u let nsh = il0 * 4u for [unroll_full] (k in range(4)) { @@ -5788,6 +5816,13 @@ class MetalKqMulMmIq3xxs : MetalKqMulMmK45T { override IQ3XXS = true } +// iq4nl rides the IQ4XS arm with IQ4NL flipping the scale read to q40's per-32-block d plane +[metal_dispatch(name = "enc_kq_mm_iq4nl_c", pso = "g_pso_kq_mm_b45", tgmem = "MetalKqMulMmIq4nl_metal_kq_mulmm_msl_tgmem", tg = 128, grid = "mp/32, rows/64", params = "mp : int64, rows : int64")] +class MetalKqMulMmIq4nl : MetalKqMulMmK45T { + override IQ4XS = true + override IQ4NL = true +} + // Router GEMV over the uploaded fp32 router slab: y[ne] = W[ne x dim] . x per stream. ne <= 256 // rows. Each sg serves NR positions per weight load (slab SLC traffic /NR); BATCHED stamps // clamp tails via ns; singles fold the clamp away and the gate erases ns outright. @@ -7509,6 +7544,62 @@ class MetalKqGemvIq4xs { } } +// IQ4_NL: w = d*LUT[q] with a PER-32-BLOCK f16 d (q40's 16B/sb scale plane verbatim, one bind, +// no sub-scale, no split plane); the quant walk is the iq4xs GEMV's q40 pairing over the same +// threadgroup codebook. kscb keeps the shared kq binding shape (iq4nl has no strip bytes). +[metal_dispatch(name = "enc_kq_iq4nl_c", pso = "g_pso_kq_iq4nl", tgmem = "metal_kq_gemv_iq4nl_msl_tgmem", tg = 64, grid = "rows/4", params = "rows : int64, n : int64")] +class MetalKqGemvIq4nl { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the 16B/sb d plane, half view - block bu's d at blk*8+bu + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // the same plane, byte view (binding-shape twin; unused) + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array // iq4nl quant plane, uint view (q40's 128B/sb nibble tiling) + @ssbo @binding = 3 @role = "read" @off = "xoff" @span = "n*4" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" @span = "rows*4" y : array + @uniform @binding = 5 ndim : uint + @uniform @binding = 6 ddim : uint + @workgroup lut : float[64] // kvalues_iq4nl as floats, one slot per thread of the 64-wide group + + [metal_kernel(name="metal_kq_gemv_iq4nl_msl")] + def metal_kq_gemv_iq4nl { + let lid = gl_LocalInvocationID.x + lut[lid] = iq4_lut(lid & 15u) + barrier() + let lane = gl_SubgroupInvocationID + let ix = lane / 16u + let it = lane % 16u + let bu = it / 2u + let il = it % 2u + let nb = ndim / 256u + let first_row = (gl_WorkGroupID.x * 2u + gl_SubgroupID) * 2u + var sumf : float[2] + var ib = ix + while (ib < nb) { + let yb4 = ib * 64u + bu * 8u + il * 2u + let y0 = x[yb4] + let y1 = x[yb4 + 1u] + let y2 = x[yb4 + 4u] + let y3 = x[yb4 + 5u] + for [unroll_full] (r in range(2)) { + let blk = (first_row + uint(r)) * nb + ib + let qb = blk * 32u + bu * 4u + il * 2u + let q0 = kqu[qb] + let q1 = kqu[qb + 1u] + var acc = y0 * float4(lut[q0 & 15u], lut[(q0 >> 8u) & 15u], lut[(q0 >> 16u) & 15u], lut[(q0 >> 24u) & 15u]) + acc += y2 * float4(lut[(q0 >> 4u) & 15u], lut[(q0 >> 12u) & 15u], lut[(q0 >> 20u) & 15u], lut[q0 >> 28u]) + acc += y1 * float4(lut[q1 & 15u], lut[(q1 >> 8u) & 15u], lut[(q1 >> 16u) & 15u], lut[(q1 >> 24u) & 15u]) + acc += y3 * float4(lut[(q1 >> 4u) & 15u], lut[(q1 >> 12u) & 15u], lut[(q1 >> 20u) & 15u], lut[q1 >> 28u]) + sumf[r] += float(kdh[blk * 8u + bu]) * (acc.x + acc.y + acc.z + acc.w) + } + ib += 2u + } + for [unroll_full] (r in range(2)) { + let s = simd_sum(sumf[r]) + if (lane == 0u && first_row + uint(r) < ddim) { + y[first_row + uint(r)] = s + } + } + } +} + // IQ3_S: w = d*sc*(+-grid byte); the 2 KB grid in threadgroup memory (llama.cpp's shape), a // 16-lane half-simdgroup per superblock, lane (bu, il) = block bu's half il: one qs word = 4 // grid words = 16 elems = 4 CONSECUTIVE float4 x loads. Scale plane = the iq4xs/k6 split form. @@ -8139,6 +8230,167 @@ class MetalKqMvB8Iq4xs { } } +// The iq4nl B2/B4 pair: the iq4xs shells with the per-32-block d fold (no strip byte). +[ |> template_struct_instance] +class template MetalKqMvIq4nlT { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the 16B/sb d plane, half view + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // binding-shape twin (unused) + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array // iq4nl quant plane, uint view + @ssbo @binding = 3 @role = "read" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" y : array + @uniform @binding = 5 ka : KqMvArgs + @template_constant NR : int = 2 + @template_constant NRU : uint = 2u // NR's uint spelling (colbase math) — override BOTH together + @template_constant TILED : bool = false + + [metal_kernel] + def metal_kq_mv_iq4nl { + let lane = gl_SubgroupInvocationID + let tx = lane % 8u + let ty = lane / 8u + let row = gl_WorkGroupID.x * 8u + gl_SubgroupID * 4u + ty + let colbase = gl_WorkGroupID.y * NRU // nolint:LINT002 dead in the TILED=false stamp — measured free + let nb = ka.ndim / 256u + let nb4 = ka.ndim / 4u + var sumf : float[NR] + var sb = 0u + while (sb < nb) { + let blk = row * nb + sb + let dl = float(kdh[blk * 8u + tx]) + let qb = blk * 32u + tx * 4u + var wl : float4[4] + var wh : float4[4] + for [unroll_full] (u in range(4)) { + let qv = kqu[qb + uint(u)] + wl[u] = float4(iq4_lut(qv & 15u), iq4_lut((qv >> 8u) & 15u), iq4_lut((qv >> 16u) & 15u), iq4_lut((qv >> 24u) & 15u)) * dl + wh[u] = float4(iq4_lut((qv >> 4u) & 15u), iq4_lut((qv >> 12u) & 15u), iq4_lut((qv >> 20u) & 15u), iq4_lut(qv >> 28u)) * dl + } + let cx4 = sb * 64u + tx * 8u + static_if (TILED) { + for [unroll_full] (b in range(NR)) { + let xb4 = (colbase + uint(b)) * nb4 + cx4 + var acc = 0.0 + for [unroll_full] (u in range(4)) { + let v = x[xb4 + uint(u)] + let vh = x[xb4 + 4u + uint(u)] + acc += v.x * wl[u].x + v.y * wl[u].y + v.z * wl[u].z + v.w * wl[u].w + acc += vh.x * wh[u].x + vh.y * wh[u].y + vh.z * wh[u].z + vh.w * wh[u].w + } + sumf[b] += acc + } + } else { + for [unroll_full] (b in range(NR)) { + let xb4 = uint(b) * nb4 + cx4 + var acc = 0.0 + for [unroll_full] (u in range(4)) { + let v = x[xb4 + uint(u)] + let vh = x[xb4 + 4u + uint(u)] + acc += v.x * wl[u].x + v.y * wl[u].y + v.z * wl[u].z + v.w * wl[u].w + acc += vh.x * wh[u].x + vh.y * wh[u].y + vh.z * wh[u].z + vh.w * wh[u].w + } + sumf[b] += acc + } + } + sb++ + } + static_if (TILED) { + for [unroll_full] (b in range(NR)) { + var s = sumf[b] + s += simd_shuffle_down(s, 4u) + s += simd_shuffle_down(s, 2u) + s += simd_shuffle_down(s, 1u) + if (tx == 0u && row < ka.ddim && colbase + uint(b) < ka.nr) { + y[(colbase + uint(b)) * ka.ys + row] = s + } + } + } else { + for [unroll_full] (b in range(NR)) { + var s = sumf[b] + s += simd_shuffle_down(s, 4u) + s += simd_shuffle_down(s, 2u) + s += simd_shuffle_down(s, 1u) + if (tx == 0u && row < ka.ddim && uint(b) < ka.nr) { + y[uint(b) * ka.ys + row] = s + } + } + } + } +} + +[metal_dispatch(name = "enc_kq_mvb2_iq4nl_c", pso = "g_pso_kq_mvb2_iq4nl", tg = 64, grid = "rows/8", params = "rows : int64")] +class MetalKqMvB2Iq4nl : MetalKqMvIq4nlT { + override NR = 2 + override NRU = 2u + override TILED = false +} + +[metal_dispatch(name = "enc_kq_mvb4_iq4nl_c", pso = "g_pso_kq_mvb4_iq4nl", tg = 64, grid = "rows/8, gcols", params = "rows : int64, gcols : int64")] +class MetalKqMvB4Iq4nl : MetalKqMvIq4nlT { + override NR = 4 + override NRU = 4u + override TILED = true +} + +// the iq4nl B8 twin: the iq4xs B8 shell (one superblock's X panel staged per threadgroup) with +// the per-32-block d fold +[metal_dispatch(name = "enc_kq_mvb8_iq4nl_c", pso = "g_pso_kq_mvb8_iq4nl", tgmem = "metal_kq_mvb8_iq4nl_msl_tgmem", tg = 64, grid = "rows/8", params = "rows : int64")] +class MetalKqMvB8Iq4nl { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the 16B/sb d plane, half view + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // binding-shape twin (unused) + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array // iq4nl quant plane, uint view + @ssbo @binding = 3 @role = "read" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" y : array + @uniform @binding = 5 ka : KqMvArgs + @workgroup txp : float4[512] // [8 streams x 64 float4] one superblock's X panel + + [metal_kernel(name="metal_kq_mvb8_iq4nl_msl")] + def metal_kq_mvb8_iq4nl { + let lid = gl_LocalInvocationID.x + let lane = gl_SubgroupInvocationID + let tx = lane % 8u + let ty = lane / 8u + let row = gl_WorkGroupID.x * 8u + gl_SubgroupID * 4u + ty + let nb = ka.ndim / 256u + let nb4 = ka.ndim / 4u + var acc : float4[8] + var sb = 0u + while (sb < nb) { + barrier() + for [unroll_full] (j in range(8)) { + let slot = uint(j) * 64u + lid + txp[slot] = x[(slot / 64u) * nb4 + sb * 64u + (slot % 64u)] + } + barrier() + let blk = row * nb + sb + let dl = float(kdh[blk * 8u + tx]) + let qb = blk * 32u + tx * 4u + var wl : float4[4] + var wh : float4[4] + for [unroll_full] (u in range(4)) { + let qv = kqu[qb + uint(u)] + wl[u] = float4(iq4_lut(qv & 15u), iq4_lut((qv >> 8u) & 15u), iq4_lut((qv >> 16u) & 15u), iq4_lut((qv >> 24u) & 15u)) * dl + wh[u] = float4(iq4_lut((qv >> 4u) & 15u), iq4_lut((qv >> 12u) & 15u), iq4_lut((qv >> 20u) & 15u), iq4_lut(qv >> 28u)) * dl + } + let cxl = tx * 8u + for [unroll_full] (b in range(8)) { + for [unroll_full] (u in range(4)) { + acc[b] += txp[uint(b) * 64u + cxl + uint(u)] * wl[u] + txp[uint(b) * 64u + cxl + 4u + uint(u)] * wh[u] + } + } + sb++ + } + for [unroll_full] (b in range(8)) { + var s = acc[b].x + acc[b].y + acc[b].z + acc[b].w + s += simd_shuffle_down(s, 4u) + s += simd_shuffle_down(s, 2u) + s += simd_shuffle_down(s, 1u) + if (tx == 0u && row < ka.ddim && uint(b) < ka.nr) { + y[uint(b) * ka.ys + row] = s + } + } + } +} + // The B2/B4 width pair as one template per format (the B8 trio below is a DIFFERENT algorithm and // stays hand-written). TILED branch-duplicates the b-loop/writeback because a LIVE colbase on B2 // costs +2% (k4) / +0.5% (k6) at cls (bench_metal_gemv_kernels KqMv round); B4 stamps byte-identical. diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das b/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das index 211421ce20..1e6cbf4f62 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das @@ -4411,7 +4411,7 @@ def private pf_devw_panel(enc : MetalComputeEncoder?; bwblob : MetalBuffer?; wbo //! quant superblocks, plus the k6 2B d tail def private pf_devw_panel_kq(enc : MetalComputeEncoder?; t : Model; fmt : KqFmt; woff : int64; bxh, by, bk, bn : MetalBuffer?; rows, d, kdim : int64; yoff : uint64) : bool { - if (fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s || fmt == KqFmt.iq3xxs) { // no dev-W dequant kernel for these formats yet + if (fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s || fmt == KqFmt.iq3xxs || fmt == KqFmt.iq4nl) { // no dev-W dequant kernel for these formats yet return false } let dq = fmt == KqFmt.k6 ? g_pf_pso_dq_k6 : (fmt == KqFmt.k4 ? g_pf_pso_dq_k4 : g_pf_pso_dq_k5) @@ -4592,6 +4592,10 @@ def private pf_enc_kq_site_mm(enc : MetalComputeEncoder?; t : Model; fmt : KqFmt enc_kq_mm_iq3xxs_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, by, yoff, bk, bn, mp, rows) return } + if (fmt == KqFmt.iq4nl) { // the base mul_mm only - no tensor / tall / dev-W twins yet + enc_kq_mm_iq4nl_c(enc, bs.buf, bs.soff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, by, yoff, bk, bn, mp, rows) + return + } //! deep-class arm: a panel dev-W would split past 8 tiles is DRAM-resident - the tall //! in-kernel-dequant stamp reads the quant plane once per 128-row tile instead of //! materializing and re-streaming a 2B/element f16 panel @@ -4703,6 +4707,8 @@ def private pf_enc_kq_gemv(enc : MetalComputeEncoder?; t : Model; fmt : KqFmt; w enc_kq_iq3s_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, xoff, by, 0ul, bn, bd, rows, n) } elif (fmt == KqFmt.iq3xxs) { enc_kq_iq3xxs_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, xoff, by, 0ul, bn, bd, rows, n) + } elif (fmt == KqFmt.iq4nl) { + enc_kq_iq4nl_c(enc, bs.buf, bs.soff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, xoff, by, 0ul, bn, bd, rows, n) } elif (fmt == KqFmt.k6) { enc_kq_k6_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, xoff, by, 0ul, bn, bd, rows, n) } elif (fmt == KqFmt.k4) { diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_shapes.das b/modules/dasLLAMA/dasllama/dasllama_metal_shapes.das index f3619ff909..a2e6bc91d9 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_shapes.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_shapes.das @@ -180,7 +180,7 @@ def record_needs(var tab : table; missing : MetalNeed) { // formats) must DECLINE kquant_native instead of falling into a wrong-layout kernel branch // (the dispatchers below treat "not k4/k6" as k5) def kq_fmt_gpu_supported(f : KqFmt) : bool { - return f == KqFmt.q8 || f == KqFmt.k4 || f == KqFmt.k5 || f == KqFmt.k6 || f == KqFmt.iq4xs || f == KqFmt.k3 || f == KqFmt.iq3s || f == KqFmt.iq3xxs + return f == KqFmt.q8 || f == KqFmt.k4 || f == KqFmt.k5 || f == KqFmt.k6 || f == KqFmt.iq4xs || f == KqFmt.k3 || f == KqFmt.iq3s || f == KqFmt.iq3xxs || f == KqFmt.iq4nl } def private kq_fmts_gpu_supported(a : array) : bool { diff --git a/modules/dasLLAMA/tests/_metal_kernel_common.das b/modules/dasLLAMA/tests/_metal_kernel_common.das index 34cf086fbe..4360c8b4ed 100644 --- a/modules/dasLLAMA/tests/_metal_kernel_common.das +++ b/modules/dasLLAMA/tests/_metal_kernel_common.das @@ -174,12 +174,20 @@ let KQ_SAFE_SCALES = fixed_array(0.25, 0.5, 1.0, 0.125, 2.0, 0.75, 1.5, 0.375) // [12 packed 6-bit bytes] per superblock, K6 = [16B sub-scales] x nsb + f16 d tail, IQ4_XS (44) = // the same split form with [8 signed sub-scales in -31..31][8 zero] strips, Q3_K (3) = the k6 form over 96B quants def kq_fill_planes(fmt, nsb : int; var kq : array; var ks : array) { - let qsb = fmt == 4 || fmt == 44 ? 128 : (fmt == 5 ? 160 : (fmt == 3 ? 96 : (fmt == 33 ? 104 : (fmt == 34 ? 96 : 192)))) + let qsb = fmt == 4 || fmt == 44 || fmt == 45 ? 128 : (fmt == 5 ? 160 : (fmt == 3 ? 96 : (fmt == 33 ? 104 : (fmt == 34 ? 96 : 192)))) kq |> resize(nsb * qsb) for (i in range(nsb * qsb)) { kq[i] = kq_hash_byte(i * 3 + fmt) } - if (fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34) { + if (fmt == 45) { + // iq4nl: 16B/sb = 8 per-32-block f16 d's, verbatim (no strips) + ks |> resize(nsb * 16) + for (b in range(nsb * 8)) { + let dbits = f32_to_f16(KQ_SAFE_SCALES[(b + (b / 8) * 3) % 8]) + ks[b * 2] = uint8(dbits & 0xFFu) + ks[b * 2 + 1] = uint8(dbits >> 8u) + } + } elif (fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34) { ks |> resize(nsb * 18) for (i in range(nsb * 16)) { ks[i] = fmt == 6 || fmt == 3 ? kq_hash_byte(i * 7 + 13) : uint8(i % 16 < 8 ? int(kq_hash_byte(i * 7 + 13)) % 63 - 31 : 0) // 44/33: signed strip bytes + zero pad @@ -257,6 +265,8 @@ def kq_row_ref(fmt, sb_base, nb, nsb_total : int; kq, ks : array; var wro } elif (fmt == 34) { dequant_iq3xxs_plane_superblock_at(kq, int64(sb * 96), ks, int64(sb * 16), int64(nsb_total * 16 + sb * 2), wrow, int64(ib * 256)) + } elif (fmt == 45) { + dequant_iq4nl_plane_superblock(kq, int64(sb * 128), ks, int64(sb * 16), wrow, int64(ib * 256)) } else { dequant_k6_plane_superblock_at(kq, int64(sb * 192), ks, int64(sb * 16), int64(nsb_total * 16 + sb * 2), wrow, int64(ib * 256)) diff --git a/modules/dasLLAMA/tests/test_metal_gemm_kernels.das b/modules/dasLLAMA/tests/test_metal_gemm_kernels.das index f2e330efe8..7b6dd23994 100644 --- a/modules/dasLLAMA/tests/test_metal_gemm_kernels.das +++ b/modules/dasLLAMA/tests/test_metal_gemm_kernels.das @@ -36,28 +36,28 @@ def private kq_mulmm_gate(t : T?; dev, queue; fmt : int; tensor : bool; m, kdim, : halfx ? (fmt == 4 ? MetalKqMulMmK4TH_metal_kq_mulmm_t_msl : (fmt == 5 ? MetalKqMulMmK5TH_metal_kq_mulmm_t_msl : MetalKqMulMmK6TH_metal_kq_mulmm_k6_t_msl)) : (fmt == 4 ? MetalKqMulMmK4T_metal_kq_mulmm_t_msl : (fmt == 5 ? MetalKqMulMmK5T_metal_kq_mulmm_t_msl : MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl))) - : (fmt == 34 ? MetalKqMulMmIq3xxs_metal_kq_mulmm_msl : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl : MetalKqMulMmK6_metal_kq_mulmm_msl))))))) + : (fmt == 45 ? MetalKqMulMmIq4nl_metal_kq_mulmm_msl : (fmt == 34 ? MetalKqMulMmIq3xxs_metal_kq_mulmm_msl : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl : MetalKqMulMmK6_metal_kq_mulmm_msl)))))))) let entry = (tensor ? (tall ? (fmt == 4 ? MetalKqMulMmK4TH128_metal_kq_mulmm_t_msl_entry : (fmt == 5 ? MetalKqMulMmK5TH128_metal_kq_mulmm_t_msl_entry : MetalKqMulMmK6TH128_metal_kq_mulmm_k6_t_msl_entry)) : halfx ? (fmt == 4 ? MetalKqMulMmK4TH_metal_kq_mulmm_t_msl_entry : (fmt == 5 ? MetalKqMulMmK5TH_metal_kq_mulmm_t_msl_entry : MetalKqMulMmK6TH_metal_kq_mulmm_k6_t_msl_entry)) : (fmt == 4 ? MetalKqMulMmK4T_metal_kq_mulmm_t_msl_entry : (fmt == 5 ? MetalKqMulMmK5T_metal_kq_mulmm_t_msl_entry : MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl_entry))) - : (fmt == 34 ? MetalKqMulMmIq3xxs_metal_kq_mulmm_msl_entry : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl_entry : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl_entry : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl_entry : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_entry : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_entry : MetalKqMulMmK6_metal_kq_mulmm_msl_entry))))))) + : (fmt == 45 ? MetalKqMulMmIq4nl_metal_kq_mulmm_msl_entry : (fmt == 34 ? MetalKqMulMmIq3xxs_metal_kq_mulmm_msl_entry : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl_entry : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl_entry : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl_entry : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_entry : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_entry : MetalKqMulMmK6_metal_kq_mulmm_msl_entry)))))))) let fm = (tensor ? (tall ? (fmt == 4 ? MetalKqMulMmK4TH128_metal_kq_mulmm_t_msl_fastmath : (fmt == 5 ? MetalKqMulMmK5TH128_metal_kq_mulmm_t_msl_fastmath : MetalKqMulMmK6TH128_metal_kq_mulmm_k6_t_msl_fastmath)) : halfx ? (fmt == 4 ? MetalKqMulMmK4TH_metal_kq_mulmm_t_msl_fastmath : (fmt == 5 ? MetalKqMulMmK5TH_metal_kq_mulmm_t_msl_fastmath : MetalKqMulMmK6TH_metal_kq_mulmm_k6_t_msl_fastmath)) : (fmt == 4 ? MetalKqMulMmK4T_metal_kq_mulmm_t_msl_fastmath : (fmt == 5 ? MetalKqMulMmK5T_metal_kq_mulmm_t_msl_fastmath : MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl_fastmath))) - : (fmt == 34 ? MetalKqMulMmIq3xxs_metal_kq_mulmm_msl_fastmath : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl_fastmath : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl_fastmath : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl_fastmath : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_fastmath : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_fastmath : MetalKqMulMmK6_metal_kq_mulmm_msl_fastmath))))))) + : (fmt == 45 ? MetalKqMulMmIq4nl_metal_kq_mulmm_msl_fastmath : (fmt == 34 ? MetalKqMulMmIq3xxs_metal_kq_mulmm_msl_fastmath : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl_fastmath : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl_fastmath : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl_fastmath : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_fastmath : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_fastmath : MetalKqMulMmK6_metal_kq_mulmm_msl_fastmath)))))))) let tgm = (tensor ? (tall ? (fmt == 4 ? MetalKqMulMmK4TH128_metal_kq_mulmm_t_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5TH128_metal_kq_mulmm_t_msl_tgmem : MetalKqMulMmK6TH128_metal_kq_mulmm_k6_t_msl_tgmem)) : halfx ? (fmt == 4 ? MetalKqMulMmK4TH_metal_kq_mulmm_t_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5TH_metal_kq_mulmm_t_msl_tgmem : MetalKqMulMmK6TH_metal_kq_mulmm_k6_t_msl_tgmem)) : (fmt == 4 ? MetalKqMulMmK4T_metal_kq_mulmm_t_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5T_metal_kq_mulmm_t_msl_tgmem : MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl_tgmem))) - : (fmt == 34 ? MetalKqMulMmIq3xxs_metal_kq_mulmm_msl_tgmem : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl_tgmem : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl_tgmem : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl_tgmem : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_tgmem : MetalKqMulMmK6_metal_kq_mulmm_msl_tgmem))))))) + : (fmt == 45 ? MetalKqMulMmIq4nl_metal_kq_mulmm_msl_tgmem : (fmt == 34 ? MetalKqMulMmIq3xxs_metal_kq_mulmm_msl_tgmem : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl_tgmem : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl_tgmem : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl_tgmem : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_tgmem : MetalKqMulMmK6_metal_kq_mulmm_msl_tgmem)))))))) var pso = pipeline_from_source(dev, src, entry, fm, err) if (tensor && pso == null && !metal4_tensor_available(dev)) { to_log(LOG_INFO, "{tag}: no Metal-4 tensor toolchain on this box - arm skipped\n") @@ -2055,6 +2055,8 @@ def test_metal_gemm_kernels(t : T?) { kq_mulmm_gate(t, dev, queue, 33, false, 32, 256, 128) kq_mulmm_gate(t, dev, queue, 34, false, 64, 512, 64) // iq3xxs: the base form only (no tensor / tall twins) kq_mulmm_gate(t, dev, queue, 34, false, 32, 256, 128) + kq_mulmm_gate(t, dev, queue, 45, false, 64, 512, 64) // iq4nl: the base form only (no tensor / tall twins) + kq_mulmm_gate(t, dev, queue, 45, false, 32, 256, 128) q8_mulmm_t_gate(t, dev, queue, 64, 256, 64) q8_mulmm_t_gate(t, dev, queue, 64, 256, 64, halfx = true) q8_mulmm_t_gate(t, dev, queue, 64, 160, 64) // kdim % 64 == 32: the bk=64 helper's 32-tail chunk diff --git a/modules/dasLLAMA/tests/test_metal_gemv_kernels.das b/modules/dasLLAMA/tests/test_metal_gemv_kernels.das index 790191b0a8..ba37c2b828 100644 --- a/modules/dasLLAMA/tests/test_metal_gemv_kernels.das +++ b/modules/dasLLAMA/tests/test_metal_gemv_kernels.das @@ -23,12 +23,12 @@ require math // ===== single-stream kq GEMV (MetalKqGemvK4 / K5 / K5C / K6) ===== def private kq_gemv_gate(t : T?; dev, queue; vform : string; n, d : int) { - let fmt = vform == "k4" ? 4 : (vform == "k6" ? 6 : (vform == "iq4xs" ? 44 : (vform == "k3" ? 3 : (vform == "iq3s" ? 33 : (vform == "iq3xxs" ? 34 : 5))))) + let fmt = vform == "k4" ? 4 : (vform == "k6" ? 6 : (vform == "iq4xs" ? 44 : (vform == "k3" ? 3 : (vform == "iq3s" ? 33 : (vform == "iq3xxs" ? 34 : (vform == "iq4nl" ? 45 : 5)))))) let tag = "kq_gemv_{vform} n={n} d={d}" var err : string - let src = vform == "iq3xxs" ? metal_kq_gemv_iq3xxs_msl : (vform == "iq3s" ? metal_kq_gemv_iq3s_msl : (vform == "k3" ? metal_kq_gemv_k3_msl : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl : (vform == "k4" ? metal_kq_gemv_k4_msl : (vform == "k5" ? metal_kq_gemv_k5_msl : (vform == "k5c" ? metal_kq_gemv_k5c_msl : metal_kq_gemv_k6_msl)))))) - let entry = vform == "iq3xxs" ? metal_kq_gemv_iq3xxs_msl_entry : (vform == "iq3s" ? metal_kq_gemv_iq3s_msl_entry : (vform == "k3" ? metal_kq_gemv_k3_msl_entry : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl_entry : (vform == "k4" ? metal_kq_gemv_k4_msl_entry : (vform == "k5" ? metal_kq_gemv_k5_msl_entry : (vform == "k5c" ? metal_kq_gemv_k5c_msl_entry : metal_kq_gemv_k6_msl_entry)))))) - let fm = vform == "iq3xxs" ? metal_kq_gemv_iq3xxs_msl_fastmath : (vform == "iq3s" ? metal_kq_gemv_iq3s_msl_fastmath : (vform == "k3" ? metal_kq_gemv_k3_msl_fastmath : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl_fastmath : (vform == "k4" ? metal_kq_gemv_k4_msl_fastmath : (vform == "k5" ? metal_kq_gemv_k5_msl_fastmath : (vform == "k5c" ? metal_kq_gemv_k5c_msl_fastmath : metal_kq_gemv_k6_msl_fastmath)))))) + let src = vform == "iq4nl" ? metal_kq_gemv_iq4nl_msl : (vform == "iq3xxs" ? metal_kq_gemv_iq3xxs_msl : (vform == "iq3s" ? metal_kq_gemv_iq3s_msl : (vform == "k3" ? metal_kq_gemv_k3_msl : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl : (vform == "k4" ? metal_kq_gemv_k4_msl : (vform == "k5" ? metal_kq_gemv_k5_msl : (vform == "k5c" ? metal_kq_gemv_k5c_msl : metal_kq_gemv_k6_msl))))))) + let entry = vform == "iq4nl" ? metal_kq_gemv_iq4nl_msl_entry : (vform == "iq3xxs" ? metal_kq_gemv_iq3xxs_msl_entry : (vform == "iq3s" ? metal_kq_gemv_iq3s_msl_entry : (vform == "k3" ? metal_kq_gemv_k3_msl_entry : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl_entry : (vform == "k4" ? metal_kq_gemv_k4_msl_entry : (vform == "k5" ? metal_kq_gemv_k5_msl_entry : (vform == "k5c" ? metal_kq_gemv_k5c_msl_entry : metal_kq_gemv_k6_msl_entry))))))) + let fm = vform == "iq4nl" ? metal_kq_gemv_iq4nl_msl_fastmath : (vform == "iq3xxs" ? metal_kq_gemv_iq3xxs_msl_fastmath : (vform == "iq3s" ? metal_kq_gemv_iq3s_msl_fastmath : (vform == "k3" ? metal_kq_gemv_k3_msl_fastmath : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl_fastmath : (vform == "k4" ? metal_kq_gemv_k4_msl_fastmath : (vform == "k5" ? metal_kq_gemv_k5_msl_fastmath : (vform == "k5c" ? metal_kq_gemv_k5c_msl_fastmath : metal_kq_gemv_k6_msl_fastmath))))))) var pso = pipeline_from_source(dev, src, entry, fm, err) t |> success(pso != null, "{tag}: pipeline: {err}") return if (pso == null) @@ -78,6 +78,9 @@ def private kq_gemv_gate(t : T?; dev, queue; vform : string; n, d : int) { if (vform == "iq3xxs") { metal_set_threadgroup_memory_length(enc, metal_kq_gemv_iq3xxs_msl_tgmem, 0) // the halved-grid slab } + if (vform == "iq4nl") { + metal_set_threadgroup_memory_length(enc, metal_kq_gemv_iq4nl_msl_tgmem, 0) // the codebook slab + } metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs/k3: the f16 d tail metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) @@ -111,15 +114,15 @@ def private kq_gemv_gate(t : T?; dev, queue; vform : string; n, d : int) { def private kq_mvb_gate(t : T?; dev, queue; fmt, bwidth, n, d, nr, ys : int) { let tag = "kq_mvb{bwidth}_k{fmt} n={n} d={d} nr={nr} ys={ys}" var err : string - let src = (bwidth == 2 ? (fmt == 34 ? MetalKqMvB2Iq3xxs_metal_kq_mv_iq3xxs_msl : (fmt == 33 ? MetalKqMvB2Iq3s_metal_kq_mv_iq3s_msl : (fmt == 3 ? MetalKqMvB2K3_metal_kq_mv_k3_msl : (fmt == 44 ? MetalKqMvB2Iq4xs_metal_kq_mv_iq4xs_msl : (fmt == 4 ? MetalKqMvB2K4_metal_kq_mv_k4_msl : (fmt == 5 ? MetalKqMvB2K5_metal_kq_mv_k5_msl : MetalKqMvB2K6_metal_kq_mv_k6_msl)))))) - : (bwidth == 4 ? (fmt == 34 ? MetalKqMvB4Iq3xxs_metal_kq_mv_iq3xxs_msl : (fmt == 33 ? MetalKqMvB4Iq3s_metal_kq_mv_iq3s_msl : (fmt == 3 ? MetalKqMvB4K3_metal_kq_mv_k3_msl : (fmt == 44 ? MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl : (fmt == 4 ? MetalKqMvB4K4_metal_kq_mv_k4_msl : (fmt == 5 ? MetalKqMvB4K5_metal_kq_mv_k5_msl : MetalKqMvB4K6_metal_kq_mv_k6_msl)))))) - : (fmt == 34 ? metal_kq_mvb8_iq3xxs_msl : (fmt == 33 ? metal_kq_mvb8_iq3s_msl : (fmt == 3 ? metal_kq_mvb8_k3_msl : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl : metal_kq_mvb8_k6_msl)))))))) - let entry = (bwidth == 2 ? (fmt == 34 ? MetalKqMvB2Iq3xxs_metal_kq_mv_iq3xxs_msl_entry : (fmt == 33 ? MetalKqMvB2Iq3s_metal_kq_mv_iq3s_msl_entry : (fmt == 3 ? MetalKqMvB2K3_metal_kq_mv_k3_msl_entry : (fmt == 44 ? MetalKqMvB2Iq4xs_metal_kq_mv_iq4xs_msl_entry : (fmt == 4 ? MetalKqMvB2K4_metal_kq_mv_k4_msl_entry : (fmt == 5 ? MetalKqMvB2K5_metal_kq_mv_k5_msl_entry : MetalKqMvB2K6_metal_kq_mv_k6_msl_entry)))))) - : (bwidth == 4 ? (fmt == 34 ? MetalKqMvB4Iq3xxs_metal_kq_mv_iq3xxs_msl_entry : (fmt == 33 ? MetalKqMvB4Iq3s_metal_kq_mv_iq3s_msl_entry : (fmt == 3 ? MetalKqMvB4K3_metal_kq_mv_k3_msl_entry : (fmt == 44 ? MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl_entry : (fmt == 4 ? MetalKqMvB4K4_metal_kq_mv_k4_msl_entry : (fmt == 5 ? MetalKqMvB4K5_metal_kq_mv_k5_msl_entry : MetalKqMvB4K6_metal_kq_mv_k6_msl_entry)))))) - : (fmt == 34 ? metal_kq_mvb8_iq3xxs_msl_entry : (fmt == 33 ? metal_kq_mvb8_iq3s_msl_entry : (fmt == 3 ? metal_kq_mvb8_k3_msl_entry : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl_entry : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_entry : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_entry : metal_kq_mvb8_k6_msl_entry)))))))) - let fm = (bwidth == 2 ? (fmt == 34 ? MetalKqMvB2Iq3xxs_metal_kq_mv_iq3xxs_msl_fastmath : (fmt == 33 ? MetalKqMvB2Iq3s_metal_kq_mv_iq3s_msl_fastmath : (fmt == 3 ? MetalKqMvB2K3_metal_kq_mv_k3_msl_fastmath : (fmt == 44 ? MetalKqMvB2Iq4xs_metal_kq_mv_iq4xs_msl_fastmath : (fmt == 4 ? MetalKqMvB2K4_metal_kq_mv_k4_msl_fastmath : (fmt == 5 ? MetalKqMvB2K5_metal_kq_mv_k5_msl_fastmath : MetalKqMvB2K6_metal_kq_mv_k6_msl_fastmath)))))) - : (bwidth == 4 ? (fmt == 34 ? MetalKqMvB4Iq3xxs_metal_kq_mv_iq3xxs_msl_fastmath : (fmt == 33 ? MetalKqMvB4Iq3s_metal_kq_mv_iq3s_msl_fastmath : (fmt == 3 ? MetalKqMvB4K3_metal_kq_mv_k3_msl_fastmath : (fmt == 44 ? MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl_fastmath : (fmt == 4 ? MetalKqMvB4K4_metal_kq_mv_k4_msl_fastmath : (fmt == 5 ? MetalKqMvB4K5_metal_kq_mv_k5_msl_fastmath : MetalKqMvB4K6_metal_kq_mv_k6_msl_fastmath)))))) - : (fmt == 34 ? metal_kq_mvb8_iq3xxs_msl_fastmath : (fmt == 33 ? metal_kq_mvb8_iq3s_msl_fastmath : (fmt == 3 ? metal_kq_mvb8_k3_msl_fastmath : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl_fastmath : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_fastmath : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_fastmath : metal_kq_mvb8_k6_msl_fastmath)))))))) + let src = (bwidth == 2 ? (fmt == 45 ? MetalKqMvB2Iq4nl_metal_kq_mv_iq4nl_msl : (fmt == 34 ? MetalKqMvB2Iq3xxs_metal_kq_mv_iq3xxs_msl : (fmt == 33 ? MetalKqMvB2Iq3s_metal_kq_mv_iq3s_msl : (fmt == 3 ? MetalKqMvB2K3_metal_kq_mv_k3_msl : (fmt == 44 ? MetalKqMvB2Iq4xs_metal_kq_mv_iq4xs_msl : (fmt == 4 ? MetalKqMvB2K4_metal_kq_mv_k4_msl : (fmt == 5 ? MetalKqMvB2K5_metal_kq_mv_k5_msl : MetalKqMvB2K6_metal_kq_mv_k6_msl))))))) + : (bwidth == 4 ? (fmt == 45 ? MetalKqMvB4Iq4nl_metal_kq_mv_iq4nl_msl : (fmt == 34 ? MetalKqMvB4Iq3xxs_metal_kq_mv_iq3xxs_msl : (fmt == 33 ? MetalKqMvB4Iq3s_metal_kq_mv_iq3s_msl : (fmt == 3 ? MetalKqMvB4K3_metal_kq_mv_k3_msl : (fmt == 44 ? MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl : (fmt == 4 ? MetalKqMvB4K4_metal_kq_mv_k4_msl : (fmt == 5 ? MetalKqMvB4K5_metal_kq_mv_k5_msl : MetalKqMvB4K6_metal_kq_mv_k6_msl))))))) + : (fmt == 45 ? metal_kq_mvb8_iq4nl_msl : (fmt == 34 ? metal_kq_mvb8_iq3xxs_msl : (fmt == 33 ? metal_kq_mvb8_iq3s_msl : (fmt == 3 ? metal_kq_mvb8_k3_msl : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl : metal_kq_mvb8_k6_msl))))))))) + let entry = (bwidth == 2 ? (fmt == 45 ? MetalKqMvB2Iq4nl_metal_kq_mv_iq4nl_msl_entry : (fmt == 34 ? MetalKqMvB2Iq3xxs_metal_kq_mv_iq3xxs_msl_entry : (fmt == 33 ? MetalKqMvB2Iq3s_metal_kq_mv_iq3s_msl_entry : (fmt == 3 ? MetalKqMvB2K3_metal_kq_mv_k3_msl_entry : (fmt == 44 ? MetalKqMvB2Iq4xs_metal_kq_mv_iq4xs_msl_entry : (fmt == 4 ? MetalKqMvB2K4_metal_kq_mv_k4_msl_entry : (fmt == 5 ? MetalKqMvB2K5_metal_kq_mv_k5_msl_entry : MetalKqMvB2K6_metal_kq_mv_k6_msl_entry))))))) + : (bwidth == 4 ? (fmt == 45 ? MetalKqMvB4Iq4nl_metal_kq_mv_iq4nl_msl_entry : (fmt == 34 ? MetalKqMvB4Iq3xxs_metal_kq_mv_iq3xxs_msl_entry : (fmt == 33 ? MetalKqMvB4Iq3s_metal_kq_mv_iq3s_msl_entry : (fmt == 3 ? MetalKqMvB4K3_metal_kq_mv_k3_msl_entry : (fmt == 44 ? MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl_entry : (fmt == 4 ? MetalKqMvB4K4_metal_kq_mv_k4_msl_entry : (fmt == 5 ? MetalKqMvB4K5_metal_kq_mv_k5_msl_entry : MetalKqMvB4K6_metal_kq_mv_k6_msl_entry))))))) + : (fmt == 45 ? metal_kq_mvb8_iq4nl_msl_entry : (fmt == 34 ? metal_kq_mvb8_iq3xxs_msl_entry : (fmt == 33 ? metal_kq_mvb8_iq3s_msl_entry : (fmt == 3 ? metal_kq_mvb8_k3_msl_entry : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl_entry : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_entry : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_entry : metal_kq_mvb8_k6_msl_entry))))))))) + let fm = (bwidth == 2 ? (fmt == 45 ? MetalKqMvB2Iq4nl_metal_kq_mv_iq4nl_msl_fastmath : (fmt == 34 ? MetalKqMvB2Iq3xxs_metal_kq_mv_iq3xxs_msl_fastmath : (fmt == 33 ? MetalKqMvB2Iq3s_metal_kq_mv_iq3s_msl_fastmath : (fmt == 3 ? MetalKqMvB2K3_metal_kq_mv_k3_msl_fastmath : (fmt == 44 ? MetalKqMvB2Iq4xs_metal_kq_mv_iq4xs_msl_fastmath : (fmt == 4 ? MetalKqMvB2K4_metal_kq_mv_k4_msl_fastmath : (fmt == 5 ? MetalKqMvB2K5_metal_kq_mv_k5_msl_fastmath : MetalKqMvB2K6_metal_kq_mv_k6_msl_fastmath))))))) + : (bwidth == 4 ? (fmt == 45 ? MetalKqMvB4Iq4nl_metal_kq_mv_iq4nl_msl_fastmath : (fmt == 34 ? MetalKqMvB4Iq3xxs_metal_kq_mv_iq3xxs_msl_fastmath : (fmt == 33 ? MetalKqMvB4Iq3s_metal_kq_mv_iq3s_msl_fastmath : (fmt == 3 ? MetalKqMvB4K3_metal_kq_mv_k3_msl_fastmath : (fmt == 44 ? MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl_fastmath : (fmt == 4 ? MetalKqMvB4K4_metal_kq_mv_k4_msl_fastmath : (fmt == 5 ? MetalKqMvB4K5_metal_kq_mv_k5_msl_fastmath : MetalKqMvB4K6_metal_kq_mv_k6_msl_fastmath))))))) + : (fmt == 45 ? metal_kq_mvb8_iq4nl_msl_fastmath : (fmt == 34 ? metal_kq_mvb8_iq3xxs_msl_fastmath : (fmt == 33 ? metal_kq_mvb8_iq3s_msl_fastmath : (fmt == 3 ? metal_kq_mvb8_k3_msl_fastmath : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl_fastmath : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_fastmath : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_fastmath : metal_kq_mvb8_k6_msl_fastmath))))))))) var pso = pipeline_from_source(dev, src, entry, fm, err) t |> success(pso != null, "{tag}: pipeline: {err}") return if (pso == null) @@ -169,7 +172,7 @@ def private kq_mvb_gate(t : T?; dev, queue; fmt, bwidth, n, d, nr, ys : int) { metal_set_pipeline(enc, pso) if (bwidth == 8) { metal_set_threadgroup_memory_length(enc, - fmt == 34 ? metal_kq_mvb8_iq3xxs_msl_tgmem : (fmt == 33 ? metal_kq_mvb8_iq3s_msl_tgmem : (fmt == 3 ? metal_kq_mvb8_k3_msl_tgmem : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl_tgmem : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_tgmem : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_tgmem : metal_kq_mvb8_k6_msl_tgmem))))), 0) + fmt == 45 ? metal_kq_mvb8_iq4nl_msl_tgmem : (fmt == 34 ? metal_kq_mvb8_iq3xxs_msl_tgmem : (fmt == 33 ? metal_kq_mvb8_iq3s_msl_tgmem : (fmt == 3 ? metal_kq_mvb8_k3_msl_tgmem : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl_tgmem : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_tgmem : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_tgmem : metal_kq_mvb8_k6_msl_tgmem)))))), 0) } metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs/k3: the f16 d tail metal_set_buffer(enc, bks, 0ul, 1) @@ -738,8 +741,10 @@ def test_metal_kq_gemv_kernels(t : T?) { kq_gemv_gate(t, dev, queue, "iq3s", 512, 30) kq_gemv_gate(t, dev, queue, "iq3xxs", 1280, 11) kq_gemv_gate(t, dev, queue, "iq3xxs", 512, 30) + kq_gemv_gate(t, dev, queue, "iq4nl", 1280, 11) + kq_gemv_gate(t, dev, queue, "iq4nl", 512, 30) // the small-batch twins: nr = 2 / 3 (b4 col guard) / 6 (b8 pad), ys > d - for (fmt in [4, 5, 6, 44, 3, 33, 34]) { + for (fmt in [4, 5, 6, 44, 3, 33, 34, 45]) { kq_mvb_gate(t, dev, queue, fmt, 2, 768, 17, 2, 22) kq_mvb_gate(t, dev, queue, fmt, 4, 768, 17, 3, 22) kq_mvb_gate(t, dev, queue, fmt, 8, 768, 17, 6, 22) From 95ec35ffac1c3d7e0e0341eef1bb91f0a796e9c9 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Mon, 31 Aug 2026 01:05:24 -0700 Subject: [PATCH 30/61] dasLLAMA: Q2_K native tier, CPU slice - KqFmt.k2 (id 2, stream code 20), the k4/k5 fold at per-16 granularity, every ladder arm; QUIRK 25 Q2_K is the k4/k5 scale STRUCTURE at k6's granularity: 16 per-16-element sc/min nibble pairs folded as (d*sc)*q - (dmin*mn) - unsigned 2-bit lanes, so the min term rides the activation 16-sums (xbsp), exactly dot_k4q8's shape. Disk block 84B: [16 sc/min bytes][64 qs][f16 d] [f16 dmin]; planes: qs verbatim (K2_QSB 64, k3's lanes minus the hmask), the scale row reordered header-first [d][dmin][16 sc/min bytes] (K2_SSB 20). dot_k2q8, k2_grp_row_dot, dequant_k2_row_grp, repack_k2_grp (16 four-byte columns x mr; [16 sc x mr][mr x 4B header]), the k2q8 [tune] family with decline stubs, and every dispatch ladder arm - built by twinning the k3 Phase A walk over the current tree's iq4nl-terminated ladders. THE TRAP (QUIRK 25): the mnemonic kernel id 2 IS q51's stream-region tag - the first load crashed with k2 regions dispatched down the q51 repack arm (index out of range on the empty q51s plane). k2 streams under code 20 (kq_stream_code), translated back to kernel id 2 at the two dispatch boundaries (stream_repack_one's arm, repack_regions' fk). The stream-code space carries q8=0, mx4=1, q51=2 - a new format's kernel id must dodge all three. Gates: test_kqformat 18/18, test_kquant 216 (200 pass, 16 env-gated skips), lint 0 on the 20-file changed set. E2e: Llama-3.2-1B-Instruct-Q2_K-local.gguf (local requant: Q2_K x64 + Q3_K x32 + Q4_K x16 + Q6_K embd - every sibling supported) decodes a coherent stream at gen 30 t/s reference bodies; greedy ids 19/64 vs simple_ids with the fork a 0.153-logit near-tie (top2 IS our token; the lossiest format of the arc diverges earliest, as expected). Unlike QUIRK 17's precedent no whole-scope re-mint fired on the first e2e - watch at Phase B. JIT emitter, Vulkan, Metal: pending. Co-Authored-By: Claude Fable 5 --- modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md | 30 ++++- modules/dasLLAMA/dasllama/dasllama_blocks.das | 3 + modules/dasLLAMA/dasllama/dasllama_common.das | 43 +++++- modules/dasLLAMA/dasllama/dasllama_config.das | 3 +- .../dasLLAMA/dasllama/dasllama_convert.das | 34 +++++ .../dasLLAMA/dasllama/dasllama_gemm_gen.das | 11 +- .../dasllama/dasllama_gemm_schema.das | 2 + modules/dasLLAMA/dasllama/dasllama_gguf.das | 36 ++++++ .../dasllama/dasllama_gpu_resident.das | 7 + modules/dasLLAMA/dasllama/dasllama_image.das | 7 +- .../dasLLAMA/dasllama/dasllama_kqformat.das | 20 ++- modules/dasLLAMA/dasllama/dasllama_layout.das | 23 +++- modules/dasLLAMA/dasllama/dasllama_load.das | 41 +++++- modules/dasLLAMA/dasllama/dasllama_math.das | 9 +- .../dasllama/dasllama_math_default.das | 80 +++++++++++- .../dasLLAMA/dasllama/dasllama_math_gen.das | 101 ++++++++++++++- modules/dasLLAMA/dasllama/dasllama_ple.das | 4 + modules/dasLLAMA/dasllama/dasllama_repack.das | 43 ++++++ modules/dasLLAMA/harness/gen_tune_probe.das | 43 +++++- modules/dasLLAMA/tests/test_kqformat.das | 12 +- modules/dasLLAMA/tests/test_kquant.das | 122 +++++++++++++++--- 21 files changed, 617 insertions(+), 57 deletions(-) diff --git a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md index cc0f45c57b..846312bb6e 100644 --- a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md +++ b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md @@ -415,8 +415,6 @@ where, why it is so today, what unquirked looks like. An empty ledger is a legit row on the M1 carries +-8 t/s of noise; the iq3s f4-slab GEMV form (+9% at the kernel) measured FLAT on e2e twice and was nearly discarded. The dispatch-loop probe (50 dispatches per encoder, best-of-3 encoders, GB/s off plane bytes) resolves 3% in seconds. -## Per-format notes - 23. **Census the vehicle GGUF before committing to it.** An ftype is a MIX: the natural "IQ3_XXS" download (mradermacher i1) quantizes attn_k/q to IQ2_S - unsupported until the iq2 tier - so the file cannot load end to end. `gguf_census.py` (scratch) reads the tensor @@ -430,6 +428,34 @@ where, why it is so today, what unquirked looks like. An empty ledger is a legit log line says "baked", not "mapped". Unquirked: fold a pack-code version into the image hash, the way QUIRK 21's fix would version the JIT DLL cache. +25. **The stream-code space is NOT the kernel-id space: q51 squats on 2.** Stream region + tags are q8=0, mx4=1, q51=2, then the kq kernel ids - Q2_K's mnemonic id 2 collided and + the first load dispatched k2 regions down the q51 repack arm (index out of range on the + empty q51s plane). k2 streams under code 20 (kq_stream_code), translated back to kernel + id 2 at the two dispatch boundaries (stream_repack_one's arm, repack_regions' fk). A new + format's kernel id must dodge 0/1/2 in the stream space or claim a distinct code the + same way. + +## Per-format notes + +### Q2_K Phase A (CPU, 2026-08-31) + +Shape: 256-superblock, the k4/k5 scale STRUCTURE at k6's granularity - 16 per-16-element +sc/min nibble-pair bytes folded as (d*sc)*q - (dmin*mn); the 2-bit lanes are unsigned, so +the min term rides the activation 16-sums (xbsp), exactly dot_k4q8's shape. Disk 84B: +[16 sc/min][64 qs][f16 d][f16 dmin]. Planes: qs verbatim (K2_QSB 64 - k3's lanes minus the +hmask), the scale row reordered header-first [d][dmin][16 sc/min] (K2_SSB 20). Ids: +KqFmt.k2 = 11, kernel id 2, stream code 20 (QUIRK 25). Kernels: dot_k2q8 (dot_k4q8's fold +over 16 groups), k2_grp_row_dot, repack_k2_grp (16 four-byte columns x mr; scale +[16 sc x mr][mr x 4B header]); the tile rides the packed planes. Gates: test_kqformat +18/18, test_kquant 216 (200 pass, 16 env-gated skips), lint 0. E2e: the local requant +(Q2_K x64 + Q3_K/Q4_K/Q6_K siblings, all supported - llama-quantize with the imatrix, no +--tensor-type override needed) decodes coherently at gen 30 t/s reference bodies, greedy +ids 19/64 with the fork a 0.153-logit near-tie (top2 IS our token; the lossiest format +diverges earliest). No whole-scope re-mint fired on the first e2e (zero "@tune begin" +lines), unlike QUIRK 17's precedent - watch at Phase B. JIT emitter, Vulkan, Metal: +pending. + ### IQ4_NL (the near-free one, 2026-08-30) Shape: 32-element blocks, 18B each on disk - f16 d + 16 nibble bytes with the k/k+16 diff --git a/modules/dasLLAMA/dasllama/dasllama_blocks.das b/modules/dasLLAMA/dasllama/dasllama_blocks.das index 62293ec53e..84b631b712 100644 --- a/modules/dasLLAMA/dasllama/dasllama_blocks.das +++ b/modules/dasLLAMA/dasllama/dasllama_blocks.das @@ -1415,6 +1415,9 @@ def private kq_bytes_per_weight(f : KqFmt) : float { if (f == KqFmt.iq4nl) { return 144.0 / 256.0 // 128B quants + the 16B scale row per 256 (q40's shapes) } + if (f == KqFmt.k2) { + return 84.0 / 256.0 // 64B quants + the 20B scale row per 256 + } return 34.0 / 32.0 // q8: 32 quants + an f16 scale per block } diff --git a/modules/dasLLAMA/dasllama/dasllama_common.das b/modules/dasLLAMA/dasllama/dasllama_common.das index 88bab0eaaa..0a0c5f7ff7 100644 --- a/modules/dasLLAMA/dasllama/dasllama_common.das +++ b/modules/dasLLAMA/dasllama/dasllama_common.das @@ -850,6 +850,9 @@ struct Model { // the iq4nl tier's plane pair: q40's shapes exactly (128B nibbles + 8 x f16 d per superblock), LUT decode iq4nlq : array iq4nls : array + // the k2 tier's plane pair: 64 qs bytes per superblock + the 20B scale row [f16 d][f16 dmin][16 sc/min bytes] + k2q : array + k2s : array kquant_native : bool = false // kq stage 4: the kq planes were repacked at load into the active backend's grp layout. // From then on every kq matmul MUST run the backend slots — disk-order portable kernels would @@ -865,6 +868,7 @@ struct Model { kq_repack_mr33 : int64 = 4l kq_repack_mr34 : int64 = 4l kq_repack_mr45 : int64 = 4l + kq_repack_mr2 : int64 = 4l // per-layer weight format tags (KqFmt; empty = all q8). Filled by detect_kq_formats before // layout_offsets so the layout walks each tensor into its format's cursor. wq_fmt : array @@ -1090,7 +1094,7 @@ def model_weights_bytes(t : Model) : int64 { + long_length(t.iq4xsq) + long_length(t.iq4xss) + long_length(t.k3q) + long_length(t.k3s) + long_length(t.iq3sq) + long_length(t.iq3ss) + long_length(t.iq3xxsq) + long_length(t.iq3xxss) - + long_length(t.iq4nlq) + long_length(t.iq4nls) + + long_length(t.iq4nlq) + long_length(t.iq4nls) + long_length(t.k2q) + long_length(t.k2s) + long_length(t.q51q) + long_length(t.q51s) + long_length(t.mblob) + long_length(t.vkblob) + long_length(t.embq) + long_length(t.embs)) @@ -1484,6 +1488,7 @@ def private dlim_cpu_source_impl(var c : DlimCpuConfig) { c.kq_mr33 = active_kq_layout_mr(33) c.kq_mr34 = active_kq_layout_mr(34) c.kq_mr45 = active_kq_layout_mr(45) + c.kq_mr2 = active_kq_layout_mr(2) c.q51_mr = active_q51_layout_mr() // OUTCOME, not request: a backend without s16 twins keeps f32 scale planes (wscale_convert_f16) c.wscale_f16 = g_wscale_f16 && kernel_backend_has_wscale16() @@ -1576,7 +1581,7 @@ def kq_active_mr(t : Model; f : KqFmt) : int64 { if (!t.kq_repacked) { return 1l } - return f == KqFmt.k4 ? t.kq_repack_mr4 : (f == KqFmt.k5 ? t.kq_repack_mr5 : (f == KqFmt.k6 ? t.kq_repack_mr6 : (f == KqFmt.q40 ? t.kq_repack_mr40 : (f == KqFmt.iq4xs ? t.kq_repack_mr44 : (f == KqFmt.k3 ? t.kq_repack_mr3 : (f == KqFmt.iq3s ? t.kq_repack_mr33 : (f == KqFmt.iq3xxs ? t.kq_repack_mr34 : t.kq_repack_mr45))))))) + return f == KqFmt.k4 ? t.kq_repack_mr4 : (f == KqFmt.k5 ? t.kq_repack_mr5 : (f == KqFmt.k6 ? t.kq_repack_mr6 : (f == KqFmt.q40 ? t.kq_repack_mr40 : (f == KqFmt.iq4xs ? t.kq_repack_mr44 : (f == KqFmt.k3 ? t.kq_repack_mr3 : (f == KqFmt.iq3s ? t.kq_repack_mr33 : (f == KqFmt.iq3xxs ? t.kq_repack_mr34 : (f == KqFmt.iq4nl ? t.kq_repack_mr45 : t.kq_repack_mr2)))))))) } //! Load a llama-architecture GGUF into the split layout at the requested precision — straight into @@ -1639,8 +1644,8 @@ def footprint(t : Model) : MemFootprint { let q4s = long_length(t.q4scales) * 4l let mx4 = long_length(t.mxq) // uint8 (two nibbles) -> 1 byte each let mx4s = long_length(t.mxs) // raw E8M0 -> 1 byte per 32-block - let kq = long_length(t.k4q) + long_length(t.k5q) + long_length(t.k6q) + long_length(t.q40q) + long_length(t.q51q) + long_length(t.iq4xsq) + long_length(t.k3q) + long_length(t.iq3sq) + long_length(t.iq3xxsq) + long_length(t.iq4nlq) - let kqs = long_length(t.k4s) + long_length(t.k5s) + long_length(t.k6s) + long_length(t.q40s) + long_length(t.q51s) + long_length(t.iq4xss) + long_length(t.k3s) + long_length(t.iq3ss) + long_length(t.iq3xxss) + long_length(t.iq4nls) + let kq = long_length(t.k4q) + long_length(t.k5q) + long_length(t.k6q) + long_length(t.q40q) + long_length(t.q51q) + long_length(t.iq4xsq) + long_length(t.k3q) + long_length(t.iq3sq) + long_length(t.iq3xxsq) + long_length(t.iq4nlq) + long_length(t.k2q) + let kqs = long_length(t.k4s) + long_length(t.k5s) + long_length(t.k6s) + long_length(t.q40s) + long_length(t.q51s) + long_length(t.iq4xss) + long_length(t.k3s) + long_length(t.iq3ss) + long_length(t.iq3xxss) + long_length(t.iq4nls) + long_length(t.k2s) return MemFootprint(aux_fp32_bytes = aux, weight_fp32_bytes = wf, q8_bytes = q8, q8_scale_bytes = q8s, q4_bytes = q4, q4_scale_bytes = q4s, mx4_bytes = mx4, mx4_scale_bytes = mx4s, kq_bytes = kq, kq_scale_bytes = kqs, @@ -3314,11 +3319,11 @@ def mm_b_q51_groupn(var y : array; t : Model; offs : array; nregio // KqFmt -> the kernel-layer format id (matmul_kq*/kq_rows_fn/kq_qsb take the int form). q8 maps // to 0, a poison id no kq kernel accepts — kq_rows_for calls this for EVERY tag, so it must stay // total even though q8 branches never read the result. -def private kq_fi(fmt : KqFmt) : int => fmt == KqFmt.k4 ? 4 : (fmt == KqFmt.k5 ? 5 : (fmt == KqFmt.k6 ? 6 : (fmt == KqFmt.q40 ? 40 : (fmt == KqFmt.iq4xs ? 44 : (fmt == KqFmt.k3 ? 3 : (fmt == KqFmt.iq3s ? 33 : (fmt == KqFmt.iq3xxs ? 34 : (fmt == KqFmt.iq4nl ? 45 : 0)))))))) +def private kq_fi(fmt : KqFmt) : int => fmt == KqFmt.k4 ? 4 : (fmt == KqFmt.k5 ? 5 : (fmt == KqFmt.k6 ? 6 : (fmt == KqFmt.q40 ? 40 : (fmt == KqFmt.iq4xs ? 44 : (fmt == KqFmt.k3 ? 3 : (fmt == KqFmt.iq3s ? 33 : (fmt == KqFmt.iq3xxs ? 34 : (fmt == KqFmt.iq4nl ? 45 : (fmt == KqFmt.k2 ? 2 : 0))))))))) // K-quant GEMV against a PRE-quantized activation (xq/xs/xbs already filled by the bs quantizer). // kq_repacked loads run the backend's stamped cores; disk-order loads run the portable kernels. -def mm_at_kq_pre(var y : array; t : Model; fmt : KqFmt; woff : int64; xq : array; xs : array; xbs : array; n, d : int64; yoff : int64 = 0l) { +def mm_at_kq_pre(var y : array; t : Model; fmt : KqFmt; woff : int64; xq : array; xs : array; xbs : array; n, d : int64; yoff : int64 = 0l) { // nolint:STYLE037 — the flat per-format ladder (one arm per kq format) is the honest shape if (t.kq_repacked) { if (fmt == KqFmt.k4) { matmul_kq_active(4, y, t.k4q, t.k4s, woff, xq, xs, xbs, n, d, yoff) @@ -3338,6 +3343,8 @@ def mm_at_kq_pre(var y : array; t : Model; fmt : KqFmt; woff : int64; xq matmul_kq_active(34, y, t.iq3xxsq, t.iq3xxss, woff, xq, xs, xbs, n, d, yoff) } elif (fmt == KqFmt.iq4nl) { matmul_kq_active(45, y, t.iq4nlq, t.iq4nls, woff, xq, xs, xbs, n, d, yoff) + } elif (fmt == KqFmt.k2) { + matmul_kq_active(2, y, t.k2q, t.k2s, woff, xq, xs, xbs, n, d, yoff) } else { panic("mm_at_kq_pre: '{fmt}' has no kq plane pair") // a new KqFmt must claim its arm, never ride q40's } @@ -3359,6 +3366,8 @@ def mm_at_kq_pre(var y : array; t : Model; fmt : KqFmt; woff : int64; xq matmul_kq(34, y, t.iq3xxsq, t.iq3xxss, woff, xq, xs, xbs, n, d, yoff) } elif (fmt == KqFmt.iq4nl) { matmul_kq(45, y, t.iq4nlq, t.iq4nls, woff, xq, xs, xbs, n, d, yoff) + } elif (fmt == KqFmt.k2) { + matmul_kq(2, y, t.k2q, t.k2s, woff, xq, xs, xbs, n, d, yoff) } else { panic("mm_at_kq_pre: '{fmt}' has no kq plane pair") } @@ -3400,6 +3409,8 @@ def mm_b_kq(var s : Session; var y : array; t : Model; fmt : KqFmt; woff matmul_kq_batch(34, y, t.iq3xxsq, t.iq3xxss, woff, s.xqb, s.xsb, s.xbsb, n, d, npos) } elif (fmt == KqFmt.iq4nl) { matmul_kq_batch(45, y, t.iq4nlq, t.iq4nls, woff, s.xqb, s.xsb, s.xbsb, n, d, npos) + } elif (fmt == KqFmt.k2) { + matmul_kq_batch(2, y, t.k2q, t.k2s, woff, s.xqb, s.xsb, s.xbsb, n, d, npos) } else { panic("mm_b_kq: '{fmt}' has no kq plane pair") } @@ -3417,7 +3428,7 @@ def mm_b_kq(var s : Session; var y : array; t : Model; fmt : KqFmt; woff // Region-list kq GEMV over the model's planes (the MoE expert dispatch's kq twin of // mm_at_q8_groupn; offs = (weight, activation) element-offset pairs within fmt's plane). // Repacked loads run the backend's kq_groupn; disk-order loads run the portable region walk. -def mm_at_kq_groupn(var y : array; t : Model; fmt : KqFmt; offs : array; nregions : int64; xq : array; xs : array; xbs : array; n, d : int64) { +def mm_at_kq_groupn(var y : array; t : Model; fmt : KqFmt; offs : array; nregions : int64; xq : array; xs : array; xbs : array; n, d : int64) { // nolint:STYLE037 — the flat per-format ladder (one arm per kq format) is the honest shape if (t.kq_repacked) { if (fmt == KqFmt.k4) { matmul_kq_groupn_active(4, y, t.k4q, t.k4s, offs, nregions, xq, xs, xbs, n, d) @@ -3437,6 +3448,8 @@ def mm_at_kq_groupn(var y : array; t : Model; fmt : KqFmt; offs : array; t : Model; fmt : KqFmt; offs : array; t : Model; fmt : KqFmt; offs : array; t : Model; fmt : KqFmt; woff : int64; xq : matmul_kq_batch(34, y, t.iq3xxsq, t.iq3xxss, woff, xq, xs, xbs, n, d, ntok) } elif (fmt == KqFmt.iq4nl) { matmul_kq_batch(45, y, t.iq4nlq, t.iq4nls, woff, xq, xs, xbs, n, d, ntok) + } elif (fmt == KqFmt.k2) { + matmul_kq_batch(2, y, t.k2q, t.k2s, woff, xq, xs, xbs, n, d, ntok) } else { panic("mm_b_kq_pre: '{fmt}' has no kq plane pair") } @@ -3529,6 +3548,7 @@ def kq_plane_q(t : Model; fmt : KqFmt; woff : int64) : uint8 const? { if (fmt == KqFmt.iq3s) return unsafe(addr(t.iq3sq[sb * IQ3S_QSB])) if (fmt == KqFmt.iq3xxs) return unsafe(addr(t.iq3xxsq[sb * IQ3XXS_QSB])) if (fmt == KqFmt.iq4nl) return unsafe(addr(t.iq4nlq[sb * Q40_QSB])) + if (fmt == KqFmt.k2) return unsafe(addr(t.k2q[sb * K2_QSB])) return null } @@ -3543,6 +3563,7 @@ def kq_plane_s(t : Model; fmt : KqFmt; woff : int64) : uint8 const? { if (fmt == KqFmt.iq3s) return unsafe(addr(t.iq3ss[sb * IQ3S_SSB])) if (fmt == KqFmt.iq3xxs) return unsafe(addr(t.iq3xxss[sb * IQ3XXS_SSB])) if (fmt == KqFmt.iq4nl) return unsafe(addr(t.iq4nls[sb * Q40_SSB])) + if (fmt == KqFmt.k2) return unsafe(addr(t.k2s[sb * K2_SSB])) return null } @@ -3913,6 +3934,8 @@ def embed_row(t : Model; token : int64; var dst : float?) { // nolint:STYLE037 dequant_kq_row_grp(fmt, addr(t.embq[sbg * IQ3XXS_QSB]), addr(t.embs[sbg * IQ3XXS_SSB]), token % mr, mr, dim, dst) } elif (t.emb_fmt == KqFmt.iq4nl) { dequant_kq_row_grp(fmt, addr(t.embq[sbg * Q40_QSB]), addr(t.embs[sbg * Q40_SSB]), token % mr, mr, dim, dst) + } elif (t.emb_fmt == KqFmt.k2) { + dequant_kq_row_grp(fmt, addr(t.embq[sbg * K2_QSB]), addr(t.embs[sbg * K2_SSB]), token % mr, mr, dim, dst) } else { panic("embed_row: '{t.emb_fmt}' has no kq plane pair") } @@ -3934,6 +3957,8 @@ def embed_row(t : Model; token : int64; var dst : float?) { // nolint:STYLE037 dequant_kq_row_grp(fmt, addr(t.iq3xxsq[sbg * IQ3XXS_QSB]), addr(t.iq3xxss[sbg * IQ3XXS_SSB]), token % mr, mr, dim, dst) } elif (t.emb_fmt == KqFmt.iq4nl) { dequant_kq_row_grp(fmt, addr(t.iq4nlq[sbg * Q40_QSB]), addr(t.iq4nls[sbg * Q40_SSB]), token % mr, mr, dim, dst) + } elif (t.emb_fmt == KqFmt.k2) { + dequant_kq_row_grp(fmt, addr(t.k2q[sbg * K2_QSB]), addr(t.k2s[sbg * K2_SSB]), token % mr, mr, dim, dst) } else { panic("embed_row: '{t.emb_fmt}' has no kq plane pair") } @@ -3966,6 +3991,8 @@ def embed_row(t : Model; token : int64; var dst : float?) { // nolint:STYLE037 dequant_iq3xxs_plane_superblock(t.embq, (sb0 + s) * IQ3XXS_QSB, t.embs, (sb0 + s) * IQ3XXS_SSB, row, s * 256l) } elif (t.emb_fmt == KqFmt.iq4nl) { dequant_iq4nl_plane_superblock(t.embq, (sb0 + s) * Q40_QSB, t.embs, (sb0 + s) * Q40_SSB, row, s * 256l) + } elif (t.emb_fmt == KqFmt.k2) { + dequant_k2_plane_superblock(t.embq, (sb0 + s) * K2_QSB, t.embs, (sb0 + s) * K2_SSB, row, s * 256l) } else { panic("embed_row: '{t.emb_fmt}' has no kq plane pair") } @@ -3991,6 +4018,8 @@ def embed_row(t : Model; token : int64; var dst : float?) { // nolint:STYLE037 dequant_iq3xxs_plane_superblock(t.iq3xxsq, (sb0 + s) * IQ3XXS_QSB, t.iq3xxss, (sb0 + s) * IQ3XXS_SSB, row, s * 256l) } elif (t.emb_fmt == KqFmt.iq4nl) { dequant_iq4nl_plane_superblock(t.iq4nlq, (sb0 + s) * Q40_QSB, t.iq4nls, (sb0 + s) * Q40_SSB, row, s * 256l) + } elif (t.emb_fmt == KqFmt.k2) { + dequant_k2_plane_superblock(t.k2q, (sb0 + s) * K2_QSB, t.k2s, (sb0 + s) * K2_SSB, row, s * 256l) } else { panic("embed_row: '{t.emb_fmt}' has no kq plane pair") } diff --git a/modules/dasLLAMA/dasllama/dasllama_config.das b/modules/dasLLAMA/dasllama/dasllama_config.das index 45d78828a9..d8f0afb8a2 100644 --- a/modules/dasLLAMA/dasllama/dasllama_config.das +++ b/modules/dasLLAMA/dasllama/dasllama_config.das @@ -32,6 +32,7 @@ struct public DlimCpuConfig { kq_mr33 : int64 kq_mr34 : int64 kq_mr45 : int64 + kq_mr2 : int64 q51_mr : int64 wscale_f16 : bool kquant_native : bool @@ -144,7 +145,7 @@ def public dlim_config_current(quant : string = "q8") : DlimConfiguration { def public dlim_identity(c : DlimConfiguration; image_version : int; tag : string = "") : string { return ("v{image_version}|{c.quant}|{c.cpu.backend}|{c.cpu.wscale_f16 ? "s16" : "s32"}" + "|q8 mr{c.cpu.q8_mr} b{c.cpu.q8_wbias} g{c.cpu.q8_kgroup}" - + "|kq {c.cpu.kq_mr4}/{c.cpu.kq_mr5}/{c.cpu.kq_mr6}/{c.cpu.kq_mr40}/{c.cpu.kq_mr44}/{c.cpu.kq_mr3}/{c.cpu.kq_mr33}/{c.cpu.kq_mr34}/{c.cpu.kq_mr45}" + + "|kq {c.cpu.kq_mr4}/{c.cpu.kq_mr5}/{c.cpu.kq_mr6}/{c.cpu.kq_mr40}/{c.cpu.kq_mr44}/{c.cpu.kq_mr3}/{c.cpu.kq_mr33}/{c.cpu.kq_mr34}/{c.cpu.kq_mr45}/{c.cpu.kq_mr2}" + "|q51 mr{c.cpu.q51_mr}" + "|nat {c.cpu.kquant_native ? 1 : 0}{c.cpu.kq_q40_native ? 1 : 0}{c.cpu.kq_q50_native ? 1 : 0}{c.cpu.kq_q51_native ? 1 : 0}" + (tag != "" ? "|{tag}" : "")) diff --git a/modules/dasLLAMA/dasllama/dasllama_convert.das b/modules/dasLLAMA/dasllama/dasllama_convert.das index 22b539ded8..5a696a3557 100644 --- a/modules/dasLLAMA/dasllama/dasllama_convert.das +++ b/modules/dasLLAMA/dasllama/dasllama_convert.das @@ -663,6 +663,40 @@ def dequant_iq4xs_plane_superblock_at(kq : array | #; kqo : int64; ks : a } } +//! Transcode one Q2_K superblock (84 bytes at `bo`: 16 sc/min nibble-pair bytes, 64 qs, f16 d, +//! f16 dmin) into the k2 planes: qs verbatim to kq[kqo..+64), the scale row REORDERED to +//! [f16 d][f16 dmin][16 sc/min bytes] at ks[kso..+20) - all bytes verbatim (exact). +def transcode_q2k_superblock(bytes : array | #; bo : int64; var kq : array; kqo : int64; var ks : array; kso : int64) { + ks[kso] = bytes[bo + 80l] + ks[kso + 1l] = bytes[bo + 81l] + ks[kso + 2l] = bytes[bo + 82l] + ks[kso + 3l] = bytes[bo + 83l] + for (i in range64(16l)) { + ks[kso + 4l + i] = bytes[bo + i] + } + for (i in range64(64l)) { + kq[kqo + i] = bytes[bo + 16l + i] + } +} + +//! Reference dequant of one k2-plane superblock: w = (d * sc) * q - (dmin * mn) per 16-elem group +//! g (sc/mn = the group's nibble pair); q = the 2-bit lane of qs byte (g/8)*32 + (g%2)*16 + l at +//! shift 2*((g%8)/2) - ggml's dequantize_row_q2_K float order (dl*q - ml). +def dequant_k2_plane_superblock(kq : array | #; kqo : int64; ks : array | #; kso : int64; var dst : array | #; doff : int64) { + let d = f16_to_f32(rd_u16(ks, kso)) + let dmin = f16_to_f32(rd_u16(ks, kso + 2l)) + for (g in range64(16l)) { + let scb = int(ks[kso + 4l + g]) + let dl = d * float(scb & 15) + let ml = dmin * float(scb >> 4) + let qb = kqo + (g / 8l) * 32l + (g % 2l) * 16l + let sh = int(((g % 8l) / 2l) * 2l) + for (l in range64(16l)) { + dst[doff + g * 16l + l] = dl * float((int(kq[qb + l]) >> sh) & 3) - ml + } + } +} + //! The 6-bit sub-scale i (0..15) of a Q3_K disk block off its 12 packed scale bytes at `so` — //! ggml's kmask unpack: low nibbles of bytes 0..7 carry scales 0..7, their high nibbles 8..15, //! bytes 8..11 carry every scale's top two bits at shift 2 * (i / 4). diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das index f327d2fe37..1d6c0e705a 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das @@ -1439,7 +1439,7 @@ def private emit_iq3xxs_gather(var te : TileEmit; var sbi : LLVMOpaqueValue?) { // One 256-weight SUPERBLOCK, IQ4_XS/iq3s grp form (te.kq = 44/33): mx4's LUT decode (44) // or the byte-expanded signed panel (33; the gemv gathers it per superblock, the tile gets the // runtime unpack) — dot_lane's sign-trick lattice, k4's fold with no min term. -def private emit_block_iq4xs(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f : LLVMOpaqueValue? [8]; tokBase, tokCount : int) { // nolint:STYLE038 — one register-coupled superblock loop nest (the kqv2 emitters' shape) +def private emit_block_iq4xs(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f : LLVMOpaqueValue? [8]; tokBase, tokCount : int) { // nolint:STYLE037,STYLE038 — one register-coupled superblock loop nest (the kqv2 emitters' shape) let b = te.builder let rq = te.rq let mr = te.interleave @@ -2867,6 +2867,13 @@ def private iq3xxs_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, def private iq4nl_gemv_gen(var gc : LlvmCodeCtx) : bool => kq_gemv_gen_impl(gc, 45) def private iq4nl_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 45) +// k2: the 2-bit + min-fold emitter arm lands with Phase B; until then both stamps decline +// (reference bodies serve) - a declined generator is the framework's own fallback path. +[unused_argument(gc)] +def private k2_gemv_gen(var gc : LlvmCodeCtx) : bool => false +[unused_argument(gc)] +def private k2_tile_gen(var gc : LlvmCodeCtx) : bool => false + def private k4_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 4) def private k5_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 5) def private k6_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 6) @@ -2954,5 +2961,7 @@ def public register_dasllama_gemm_generators { register_llvm_code_generator("dasllama_gemm_gen::iq3xxs_tile", @@iq3xxs_tile_gen) register_llvm_code_generator("dasllama_gemm_gen::iq4nl_gemv", @@iq4nl_gemv_gen) register_llvm_code_generator("dasllama_gemm_gen::iq4nl_tile", @@iq4nl_tile_gen) + register_llvm_code_generator("dasllama_gemm_gen::k2_gemv", @@k2_gemv_gen) + register_llvm_code_generator("dasllama_gemm_gen::k2_tile", @@k2_tile_gen) register_llvm_code_generator("dasllama_gemm_gen::q8q8_witness", @@witness_gen) } diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_schema.das b/modules/dasLLAMA/dasllama/dasllama_gemm_schema.das index 956287a78c..b1700f91cb 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_schema.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_schema.das @@ -70,6 +70,7 @@ def kq_qsb(fmt : int) : int64 { if (fmt == 33) return kq_qsb(KqFmt.iq3s) if (fmt == 34) return kq_qsb(KqFmt.iq3xxs) if (fmt == 45) return kq_qsb(KqFmt.iq4nl) + if (fmt == 2) return kq_qsb(KqFmt.k2) panic("kq_qsb: unknown kq format id {fmt}") return 0l } @@ -87,6 +88,7 @@ def kq_ssb(fmt : int) : int64 { if (fmt == 33) return kq_ssb(KqFmt.iq3s) if (fmt == 34) return kq_ssb(KqFmt.iq3xxs) if (fmt == 45) return kq_ssb(KqFmt.iq4nl) + if (fmt == 2) return kq_ssb(KqFmt.k2) panic("kq_ssb: unknown kq format id {fmt}") return 0l } diff --git a/modules/dasLLAMA/dasllama/dasllama_gguf.das b/modules/dasLLAMA/dasllama/dasllama_gguf.das index eedd2dee17..61458c59f3 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gguf.das +++ b/modules/dasLLAMA/dasllama/dasllama_gguf.das @@ -48,6 +48,7 @@ let GGML_TYPE_Q5_0 = 6 let GGML_TYPE_Q5_1 = 7 let GGML_TYPE_Q8_0 = 8 let GGML_TYPE_Q3_K = 11 +let GGML_TYPE_Q2_K = 10 let GGML_TYPE_IQ3_S = 21 let GGML_TYPE_IQ3_XXS = 18 let GGML_TYPE_IQ4_NL = 20 @@ -749,6 +750,41 @@ def gguf_transcode_iq4xs(m : GGUFMeta; srcbytes : array | #; name : strin } } +//! Transcode a Q2_K tensor into the k2 planes (strides 64/20, exact): the 84B disk superblock +//! splits into qs verbatim and the 20B scale row [f16 d][f16 dmin][16 sc/min bytes] - all +//! bytes verbatim, header moved first (transcode_q2k_superblock). +def gguf_transcode_q2k(m : GGUFMeta; srcbytes : array | #; name : string; var kq : array; var ks : array; eloff, expect_n : int64; src_off : int64 = 0l) { + let ti = kq_transcode_check(m, name, GGML_TYPE_Q2_K, "Q2_K", src_off, expect_n) + let nb = expect_n / 256l + if (nb <= 0l) { + return + } + guard_dst(name, "k2 quant plane", (eloff / 256l) * K2_QSB, nb * K2_QSB, long_length(kq)) + guard_dst(name, "k2 scale plane", (eloff / 256l) * K2_SSB, nb * K2_SSB, long_length(ks)) + with_tensor_view(m, srcbytes, ti) $(bytes, tbase) { + let bo = tbase + (src_off / 256l) * 84l + unsafe { + let srcp = addr(bytes[bo]) + var kqp = addr(kq[(eloff / 256l) * K2_QSB]) + var ksp = addr(ks[(eloff / 256l) * K2_SSB]) + maybe_parallel_for(0, int(nb), transcode_jobs(nb, 84l)) $(rb, re) { + unsafe { + for (sb in range64(int64(rb), int64(re))) { + let src = srcp + sb * 84l + var row = ksp + sb * K2_SSB + row[0l] = src[80l] + row[1l] = src[81l] + row[2l] = src[82l] + row[3l] = src[83l] + bcopy(row + 4l, src, 16l) + bcopy(kqp + sb * K2_QSB, src + 16l, 64l) + } + } + } + } + } +} + //! Transcode a Q3_K tensor into the k3 planes (see gguf_transcode_q4k; strides 96/18, exact): the //! 110B disk superblock splits into [qs][hmask] verbatim and the k6-shaped scale row with the 16 //! packed 6-bit sub-scales DECODED to int8 (scale - 32) — transcode_q3k_superblock. diff --git a/modules/dasLLAMA/dasllama/dasllama_gpu_resident.das b/modules/dasLLAMA/dasllama/dasllama_gpu_resident.das index 96ff4ad00c..529c1b7b56 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gpu_resident.das +++ b/modules/dasLLAMA/dasllama/dasllama_gpu_resident.das @@ -295,6 +295,11 @@ def trim_model_planes(var t : Model) : bool { // nolint:STYLE038 — the flat t.embs |> resize(nsb * Q40_SSB) memcpy(addr(t.embq[0]), addr(t.iq4nlq[sb0 * Q40_QSB]), nsb * Q40_QSB) memcpy(addr(t.embs[0]), addr(t.iq4nls[sb0 * Q40_SSB]), nsb * Q40_SSB) + } elif (t.emb_fmt == KqFmt.k2) { + t.embq |> reserve_resize(nsb * K2_QSB) + t.embs |> reserve_resize(nsb * K2_SSB) + memcpy(addr(t.embq[0]), addr(t.k2q[sb0 * K2_QSB]), nsb * K2_QSB) + memcpy(addr(t.embs[0]), addr(t.k2s[sb0 * K2_SSB]), nsb * K2_SSB) } elif (t.emb_fmt == KqFmt.iq3xxs) { t.embq |> resize(nsb * IQ3XXS_QSB) t.embs |> resize(nsb * IQ3XXS_SSB) @@ -333,6 +338,8 @@ def trim_model_planes(var t : Model) : bool { // nolint:STYLE038 — the flat delete t.iq3xxss delete t.iq4nlq delete t.iq4nls + delete t.k2q + delete t.k2s delete t.q40q delete t.q40s delete t.q51q diff --git a/modules/dasLLAMA/dasllama/dasllama_image.das b/modules/dasLLAMA/dasllama/dasllama_image.das index 0fe882989e..ca4bc95bfc 100644 --- a/modules/dasLLAMA/dasllama/dasllama_image.das +++ b/modules/dasLLAMA/dasllama/dasllama_image.das @@ -38,7 +38,7 @@ require dasllama/dasllama_load // WhisperModel.enc) contribute their planes under dotted names ("enc.fblob"); string-array // fields ride the meta blob via serialize_strings — raw string pointers can't be planes. -let IMAGE_VERSION = 22 // 22: the iq4nl plane pair (IQ4_NL native tier); 21: iq3xxs; 20: iq3s +let IMAGE_VERSION = 23 // 23: the k2 plane pair (Q2_K native tier); 22: iq4nl; 21: iq3xxs //! The metal (blob-only) flavor's identity tag: q8 planes ride the 34B block_q8_0 blob and the //! kq scale planes their GPU forms (convert_model_to_metal_blob) — flavors are per-config and @@ -639,6 +639,7 @@ def serialize_image_meta(var arch : Archive; var t : Model) { arch |> serialize_raw(t.kq_repack_mr33) arch |> serialize_raw(t.kq_repack_mr34) arch |> serialize_raw(t.kq_repack_mr45) + arch |> serialize_raw(t.kq_repack_mr2) arch |> serialize_raw(t.wcls_fmt) arch |> serialize_raw(t.emb_fmt) arch |> serialize_raw(t.ple_emb_fmt) @@ -678,7 +679,7 @@ def serialize_image_meta(var arch : Archive; var t : Model) { } // 67 serialized fields + 3 deliberate skips (blocks, image_map, image_bytes) -let IMAGE_META_FIELDS = 71 + 3 +let IMAGE_META_FIELDS = 72 + 3 //! Count of meta-carried fields of any struct: non-arrays plus string arrays (those cannot //! be raw planes — see serialize_strings). Pair with a per-type field-count constant to @@ -962,7 +963,7 @@ def save_model_image(var t : Model; path : string; tag : string = ""; quant : st // streamed planes size as empty, so their bytes plus a page each are added here; over-reserve is free (dwrite_close truncates) def private stream_extra_bytes(t : Model; jobs : array) : uint64 { var extra = 0ul - for (fname in ["qblob", "mblob", "k4q", "k5q", "k6q", "q40q", "q51q", "iq4xsq", "k3q", "iq3sq", "iq3xxsq", "iq4nlq"]) { + for (fname in ["qblob", "mblob", "k4q", "k5q", "k6q", "q40q", "q51q", "iq4xsq", "k3q", "iq3sq", "iq3xxsq", "iq4nlq", "k2q"]) { let sb = stream_plane_bytes(t, jobs, fname) if (sb > 0l) { extra += uint64(sb) + uint64(IMAGE_PAGE) diff --git a/modules/dasLLAMA/dasllama/dasllama_kqformat.das b/modules/dasLLAMA/dasllama/dasllama_kqformat.das index 5c0de66e7a..d559082013 100644 --- a/modules/dasLLAMA/dasllama/dasllama_kqformat.das +++ b/modules/dasLLAMA/dasllama/dasllama_kqformat.das @@ -13,7 +13,7 @@ module dasllama_kqformat shared public require dasllama/dasllama_lint public //! Per-weight storage format under a q8-mode load with native planes: q8 = qblob/qscales; -//! k4/k5/k6/q40/iq4xs/k3/iq3s/iq3xxs/iq4nl = superblock plane pairs (mixed PER TENSOR); q51 = native Q5_1 per-32 +//! k4/k5/k6/q40/iq4xs/k3/iq3s/iq3xxs/iq4nl/k2 = superblock plane pairs (mixed PER TENSOR); q51 = native Q5_1 per-32 //! planes. New members append — the int value is the device stack tag and the image plane id. enum KqFmt : uint8 { q8 @@ -27,15 +27,16 @@ enum KqFmt : uint8 { iq3s iq3xxs iq4nl + k2 } //! The superblock-lattice formats (Q8_K-form activations, % 256 rows, repack + stamped kq //! kernels). q51 deliberately fails this: it rides per-32 planes with Q8_0-form activations — //! a `fmt != KqFmt.q8` test does not imply the kq lattice; branch on kq_sb where it does. -def kq_sb(f : KqFmt) : bool => f == KqFmt.k4 || f == KqFmt.k5 || f == KqFmt.k6 || f == KqFmt.q40 || f == KqFmt.iq4xs || f == KqFmt.k3 || f == KqFmt.iq3s || f == KqFmt.iq3xxs || f == KqFmt.iq4nl +def kq_sb(f : KqFmt) : bool => f == KqFmt.k4 || f == KqFmt.k5 || f == KqFmt.k6 || f == KqFmt.q40 || f == KqFmt.iq4xs || f == KqFmt.k3 || f == KqFmt.iq3s || f == KqFmt.iq3xxs || f == KqFmt.iq4nl || f == KqFmt.k2 //! The same predicate over the int id space the GPU drivers carry per stack/plane. -def kq_sb(fmt : int) : bool => fmt == int(KqFmt.k4) || fmt == int(KqFmt.k5) || fmt == int(KqFmt.k6) || fmt == int(KqFmt.q40) || fmt == int(KqFmt.iq4xs) || fmt == int(KqFmt.k3) || fmt == int(KqFmt.iq3s) || fmt == int(KqFmt.iq3xxs) || fmt == int(KqFmt.iq4nl) +def kq_sb(fmt : int) : bool => fmt == int(KqFmt.k4) || fmt == int(KqFmt.k5) || fmt == int(KqFmt.k6) || fmt == int(KqFmt.q40) || fmt == int(KqFmt.iq4xs) || fmt == int(KqFmt.k3) || fmt == int(KqFmt.iq3s) || fmt == int(KqFmt.iq3xxs) || fmt == int(KqFmt.iq4nl) || fmt == int(KqFmt.k2) //! KqFmt -> the kernel/IR format id space — the ONE bridge between the enum and the int ids the //! generated kernels take as runtime params. Mnemonic ids: a K-quant is its bit width (3/4/5/6), @@ -68,6 +69,9 @@ def kq_schema_id(f : KqFmt) : int { if (f == KqFmt.iq4nl) { return 45 } + if (f == KqFmt.k2) { + return 2 + } panic("kq_schema_id: not a kq superblock format") return 0 } @@ -92,6 +96,8 @@ let IQ3S_QSB = 104l // IQ3_S quant plane: [64 grid-index qs][8 qh ninth bits][ let IQ3S_SSB = 20l // IQ3_S scale plane: f16 d, 2 pad, 8 x int8 (1 + 2s), 8 pad — the iq4xs/k4 row shape let IQ3XXS_QSB = 96l // IQ3_XXS quant plane: [64 grid-index qs][32 aux bytes: per block 4x7-bit sign indices + 4-bit scale], verbatim disk order let IQ3XXS_SSB = 20l // IQ3_XXS scale plane: f16 d HALVED, 2 pad, 8 x uint8 (2*ls + 1), 8 pad — the iq3s row shape and fold (ggml's global 0.25 = the halved d x the halved grid) +let K2_QSB = 64l // Q2_K quant plane: 64 qs bytes (four 2-bit lanes each), verbatim disk order +let K2_SSB = 20l // Q2_K scale plane: [f16 d][f16 dmin][16 sc/min nibble-pair bytes] - the disk scale block, header first // IQ4_NL reuses Q40_QSB/Q40_SSB outright: the disk bytes are q40's (f16 d + 16 nibble bytes // per 32-block, the k/k+16 pairing) — only the nibble SEMANTICS differ (IQ4NL_LUT, no -8). @@ -276,13 +282,16 @@ def kq_qsb(f : KqFmt) : int64 { if (f == KqFmt.iq3xxs) { return IQ3XXS_QSB } + if (f == KqFmt.k2) { + return K2_QSB + } panic("kq_qsb: no plane stride for this format") return 0l } //! Scale-plane bytes per stride unit (see kq_qsb for the unit). def kq_ssb(f : KqFmt) : int64 { - if (f == KqFmt.k4 || f == KqFmt.k5 || f == KqFmt.iq4xs || f == KqFmt.iq3s || f == KqFmt.iq3xxs) { + if (f == KqFmt.k4 || f == KqFmt.k5 || f == KqFmt.iq4xs || f == KqFmt.iq3s || f == KqFmt.iq3xxs || f == KqFmt.k2) { return K4_SSB } if (f == KqFmt.k6 || f == KqFmt.k3) { @@ -310,5 +319,8 @@ def kq_stream_code(f : KqFmt) : int { if (f == KqFmt.q51) { return 2 } + if (f == KqFmt.k2) { + return 20 // kernel id 2 IS q51's stream tag - k2 streams under 20, translated back at the repack dispatchers + } return kq_schema_id(f) } diff --git a/modules/dasLLAMA/dasllama/dasllama_layout.das b/modules/dasLLAMA/dasllama/dasllama_layout.das index 68d65abe23..d666577656 100644 --- a/modules/dasLLAMA/dasllama/dasllama_layout.das +++ b/modules/dasLLAMA/dasllama/dasllama_layout.das @@ -388,6 +388,10 @@ def private repack_regions(var t : Model; regs : array) { var iq4nlsp : uint8? = null if (!empty(t.iq4nlq)) { iq4nlqp = addr(t.iq4nlq[0]) } if (!empty(t.iq4nls)) { iq4nlsp = addr(t.iq4nls[0]) } + var k2qp : uint8? = null + var k2sp : uint8? = null + if (!empty(t.k2q)) { k2qp = addr(t.k2q[0]) } + if (!empty(t.k2s)) { k2sp = addr(t.k2s[0]) } if (!empty(t.qblob)) { qbp = addr(t.qblob[0]) } if (!empty(t.qscales)) { qsp = addr(t.qscales[0]) } if (!empty(t.mxq)) { mxqp = addr(t.mxq[0]) } @@ -417,11 +421,12 @@ def private repack_regions(var t : Model; regs : array) { invoke(rq51, q51qp + (rp[i].off / 32l) * Q51_QPB, q51sp + (rp[i].off / 32l) * Q51_SPB, rp[i].n, rp[i].d) } else { let sb = rp[i].off / 256l - let qsb = kq_qsb(f) - let ssb = kq_ssb(f) - var kqp = f == 4 ? k4qp : (f == 5 ? k5qp : (f == 6 ? k6qp : (f == 40 ? q40qp : (f == 44 ? iq4xsqp : (f == 3 ? k3qp : (f == 33 ? iq3sqp : (f == 34 ? iq3xxsqp : iq4nlqp))))))) - var ksp = f == 4 ? k4sp : (f == 5 ? k5sp : (f == 6 ? k6sp : (f == 40 ? q40sp : (f == 44 ? iq4xssp : (f == 3 ? k3sp : (f == 33 ? iq3ssp : (f == 34 ? iq3xxssp : iq4nlsp))))))) - invoke(rkq, f, kqp + sb * qsb, ksp + sb * ssb, rp[i].n, rp[i].d) + let fk = f == 20 ? 2 : f // k2 streams under 20 (kernel id 2 = q51's stream tag) + let qsb = kq_qsb(fk) + let ssb = kq_ssb(fk) + var kqp = f == 4 ? k4qp : (f == 5 ? k5qp : (f == 6 ? k6qp : (f == 40 ? q40qp : (f == 44 ? iq4xsqp : (f == 3 ? k3qp : (f == 33 ? iq3sqp : (f == 34 ? iq3xxsqp : (f == 45 ? iq4nlqp : k2qp)))))))) + var ksp = f == 4 ? k4sp : (f == 5 ? k5sp : (f == 6 ? k6sp : (f == 40 ? q40sp : (f == 44 ? iq4xssp : (f == 3 ? k3sp : (f == 33 ? iq3ssp : (f == 34 ? iq3xxssp : (f == 45 ? iq4nlsp : k2sp)))))))) + invoke(rkq, fk, kqp + sb * qsb, ksp + sb * ssb, rp[i].n, rp[i].d) } } } @@ -556,6 +561,8 @@ def private push_repack_kq(var regs : array; fmt : KqFmt; woff, n, d push_repack(regs, 34, woff, n, d) } elif (fmt == KqFmt.iq4nl) { push_repack(regs, 45, woff, n, d) + } elif (fmt == KqFmt.k2) { + push_repack(regs, 20, woff, n, d) } } @@ -691,6 +698,8 @@ def private stream_repack_one(fmt : int; qp, sp : void?; n, d : int64) { invoke(active_repack_q8q8(), reinterpret(qp), reinterpret(sp), n, d) } elif (fmt == 2) { invoke(active_repack_q51(), reinterpret(qp), reinterpret(sp), n, d) + } elif (fmt == 20) { // k2's stream tag - the kq repack runs under its kernel id 2 + invoke(active_repack_kq(), 2, reinterpret(qp), reinterpret(sp), n, d) } else { invoke(active_repack_kq(), fmt, reinterpret(qp), reinterpret(sp), n, d) } @@ -831,13 +840,13 @@ def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice : (fmt == KqFmt.k6 ? addr(t.k6q[sb0 * kq_qsb(6)]) : (fmt == KqFmt.q40 ? addr(t.q40q[sb0 * kq_qsb(40)]) : (fmt == KqFmt.iq4xs ? addr(t.iq4xsq[sb0 * kq_qsb(44)]) - : (fmt == KqFmt.k3 ? addr(t.k3q[sb0 * kq_qsb(3)]) : (fmt == KqFmt.iq3s ? addr(t.iq3sq[sb0 * kq_qsb(33)]) : (fmt == KqFmt.iq3xxs ? addr(t.iq3xxsq[sb0 * kq_qsb(34)]) : addr(t.iq4nlq[sb0 * kq_qsb(45)]))))))))) + : (fmt == KqFmt.k3 ? addr(t.k3q[sb0 * kq_qsb(3)]) : (fmt == KqFmt.iq3s ? addr(t.iq3sq[sb0 * kq_qsb(33)]) : (fmt == KqFmt.iq3xxs ? addr(t.iq3xxsq[sb0 * kq_qsb(34)]) : (fmt == KqFmt.iq4nl ? addr(t.iq4nlq[sb0 * kq_qsb(45)]) : addr(t.k2q[sb0 * kq_qsb(2)])))))))))) let sp = (fmt == KqFmt.k4 ? addr(t.k4s[sb0 * kq_ssb(4)]) : (fmt == KqFmt.k5 ? addr(t.k5s[sb0 * kq_ssb(5)]) : (fmt == KqFmt.k6 ? addr(t.k6s[sb0 * kq_ssb(6)]) : (fmt == KqFmt.q40 ? addr(t.q40s[sb0 * kq_ssb(40)]) : (fmt == KqFmt.iq4xs ? addr(t.iq4xss[sb0 * kq_ssb(44)]) - : (fmt == KqFmt.k3 ? addr(t.k3s[sb0 * kq_ssb(3)]) : (fmt == KqFmt.iq3s ? addr(t.iq3ss[sb0 * kq_ssb(33)]) : (fmt == KqFmt.iq3xxs ? addr(t.iq3xxss[sb0 * kq_ssb(34)]) : addr(t.iq4nls[sb0 * kq_ssb(45)]))))))))) + : (fmt == KqFmt.k3 ? addr(t.k3s[sb0 * kq_ssb(3)]) : (fmt == KqFmt.iq3s ? addr(t.iq3ss[sb0 * kq_ssb(33)]) : (fmt == KqFmt.iq3xxs ? addr(t.iq3xxss[sb0 * kq_ssb(34)]) : (fmt == KqFmt.iq4nl ? addr(t.iq4nls[sb0 * kq_ssb(45)]) : addr(t.k2s[sb0 * kq_ssb(2)])))))))))) var wqp = addr(wq[0]) var wsp = addr(ws[0]) let njobs = is_job_que_available() ? min(nslices, 4 * (get_total_hw_jobs() + 1)) : 1 diff --git a/modules/dasLLAMA/dasllama/dasllama_load.das b/modules/dasLLAMA/dasllama/dasllama_load.das index 6ed9e421c7..3e01982942 100644 --- a/modules/dasLLAMA/dasllama/dasllama_load.das +++ b/modules/dasLLAMA/dasllama/dasllama_load.das @@ -46,6 +46,7 @@ struct private LayoutSizes { iq3s_n : int64 iq3xxs_n : int64 iq4nl_n : int64 + k2_n : int64 pleq8_n : int64 // the dedicated q8 plane holding ONLY the PLE table (fp32/q4 serving) } @@ -63,6 +64,7 @@ struct private KqCursors { iq3s : int64 iq3xxs : int64 iq4nl : int64 + k2 : int64 } def private kq_take(var cur : KqCursors; f : KqFmt; n : int64) : int64 { @@ -116,6 +118,11 @@ def private kq_take(var cur : KqCursors; f : KqFmt; n : int64) : int64 { cur.iq4nl += n return o } + if (f == KqFmt.k2) { + let o = cur.k2 + cur.k2 += n + return o + } let o = cur.wo cur.wo += n return o @@ -415,7 +422,7 @@ def private layout_offsets(var t : Model) : LayoutSizes { // nolint:STYLE037,S fo += dim } return LayoutSizes(fblob_n = fo, wblob_n = cur.wo, mx_n = mx, bf16_n = bf16, - k4_n = cur.k4, k5_n = cur.k5, k6_n = cur.k6, q40_n = cur.q40, q51_n = cur.q51, iq4xs_n = cur.iq4xs, k3_n = cur.k3, iq3s_n = cur.iq3s, iq3xxs_n = cur.iq3xxs, iq4nl_n = cur.iq4nl, pleq8_n = pleq8) + k4_n = cur.k4, k5_n = cur.k5, k6_n = cur.k6, q40_n = cur.q40, q51_n = cur.q51, iq4xs_n = cur.iq4xs, k3_n = cur.k3, iq3s_n = cur.iq3s, iq3xxs_n = cur.iq3xxs, iq4nl_n = cur.iq4nl, k2_n = cur.k2, pleq8_n = pleq8) } @@ -684,6 +691,8 @@ def stream_field_of(t : Model; fmt : KqFmt) : string { return "iq3xxsq" } elif (fmt == KqFmt.iq4nl) { return "iq4nlq" + } elif (fmt == KqFmt.k2) { + return "k2q" } elif (t.quant == QuantMode.q8) { // the blob flavor carries q8 as gguf-native 34B blocks in ONE plane, never the split pair return t.metal_blob ? "mblob" : "qblob" @@ -810,6 +819,8 @@ def fill_stream_plane(m : GGUFMeta; bytes : array | #; var t : Model; fie memcpy(addr(t.iq3xxss[(j.woff / 256l) * IQ3XXS_SSB]), addr(temp_s[0]), sh.sb) } elif (j.fmt == KqFmt.iq4nl) { memcpy(addr(t.iq4nls[(j.woff / 256l) * Q40_SSB]), addr(temp_s[0]), sh.sb) + } elif (j.fmt == KqFmt.k2) { + memcpy(addr(t.k2s[(j.woff / 256l) * K2_SSB]), addr(temp_s[0]), sh.sb) } elif (j.fmt == KqFmt.q51) { memcpy(addr(t.q51s[(j.woff / 32l) * Q51_SB]), addr(temp_s[0]), sh.sb) } elif (sh.sb == 0l && j.n > 0l && !t.metal_blob) { @@ -856,6 +867,9 @@ def fill_stream_plane(m : GGUFMeta; bytes : array | #; var t : Model; fie } elif (r.fmt == 45) { invoke(g_stream_repack, 45, addr(temp_q[((r.off - j.woff) / 256l) * Q40_QSB]), addr(t.iq4nls[(r.off / 256l) * Q40_SSB]), r.n, r.d) + } elif (r.fmt == 20) { + invoke(g_stream_repack, 20, addr(temp_q[((r.off - j.woff) / 256l) * K2_QSB]), + addr(t.k2s[(r.off / 256l) * K2_SSB]), r.n, r.d) } } } @@ -1083,6 +1097,9 @@ def private transcode_kq_tensor(m : GGUFMeta; bytes : array | #; name : s } elif (fmt == KqFmt.iq4nl) { gguf_transcode_iq4nl(m, bytes, name, kq, ks, eloff, n, src_off) return "iq4nl transcode (IQ4_NL)" + } elif (fmt == KqFmt.k2) { + gguf_transcode_q2k(m, bytes, name, kq, ks, eloff, n, src_off) + return "k2 transcode (Q2_K)" } gguf_transcode_q51(m, bytes, name, kq, ks, eloff, n, src_off) return "q51 transcode (Q5_1)" @@ -1131,6 +1148,8 @@ def private load_big(m : GGUFMeta; bytes : array | #; name : string; var kind = transcode_kq_tensor(m, bytes, name, fmt, t.iq3xxsq, t.iq3xxss, woff, n, src_off, scratch) } elif (fmt == KqFmt.iq4nl) { kind = transcode_kq_tensor(m, bytes, name, fmt, t.iq4nlq, t.iq4nls, woff, n, src_off, scratch) + } elif (fmt == KqFmt.k2) { + kind = transcode_kq_tensor(m, bytes, name, fmt, t.k2q, t.k2s, woff, n, src_off, scratch) } elif (fmt == KqFmt.q51) { kind = transcode_kq_tensor(m, bytes, name, fmt, t.q51q, t.q51s, woff, n, src_off, scratch) } elif (t.quant == QuantMode.q8) { @@ -1230,8 +1249,8 @@ def private log_load_report(t : Model) { to_log(LOG_INFO, "dasLLAMA noisy: backend '{active_kernel_backend()}' | kquant_native {get_kquant_native()} q40_native {get_kq_q40_native()} q50_native {get_kq_q50_native()}\n") // the tier arms at [init], before a --noisy CLI flag can land — restate it here so the flag alone suffices to_log(LOG_INFO, "dasLLAMA noisy: GPU tier installed {moe_gpu_tier_installed()} | want auto {gpu_want_auto()}, moe layers {gpu_want_moe_layers()}, stream {gpu_want_moe_stream()}\n") - to_log(LOG_INFO, "dasLLAMA noisy: format tags - q8 {tally_fmt(t, KqFmt.q8)}, k4 {tally_fmt(t, KqFmt.k4)}, k5 {tally_fmt(t, KqFmt.k5)}, k6 {tally_fmt(t, KqFmt.k6)}, q40 {tally_fmt(t, KqFmt.q40)}, q51 {tally_fmt(t, KqFmt.q51)}, iq4xs {tally_fmt(t, KqFmt.iq4xs)}, k3 {tally_fmt(t, KqFmt.k3)}, iq3s {tally_fmt(t, KqFmt.iq3s)}, iq3xxs {tally_fmt(t, KqFmt.iq3xxs)}, iq4nl {tally_fmt(t, KqFmt.iq4nl)}\n") - to_log(LOG_INFO, "dasLLAMA noisy: plane MB - q8 {float(long_length(t.qblob)) / mb} (+scales {float(long_length(t.qscales) * 4l) / mb}), k4 {float(long_length(t.k4q) + long_length(t.k4s)) / mb}, k5 {float(long_length(t.k5q) + long_length(t.k5s)) / mb}, k6 {float(long_length(t.k6q) + long_length(t.k6s)) / mb}, q40 {float(long_length(t.q40q) + long_length(t.q40s)) / mb}, q51 {float(long_length(t.q51q) + long_length(t.q51s)) / mb}, iq4xs {float(long_length(t.iq4xsq) + long_length(t.iq4xss)) / mb}, k3 {float(long_length(t.k3q) + long_length(t.k3s)) / mb}, iq3s {float(long_length(t.iq3sq) + long_length(t.iq3ss)) / mb}, iq3xxs {float(long_length(t.iq3xxsq) + long_length(t.iq3xxss)) / mb}, iq4nl {float(long_length(t.iq4nlq) + long_length(t.iq4nls)) / mb}, f32 {float(long_length(t.wblob) * 4l) / mb}\n") + to_log(LOG_INFO, "dasLLAMA noisy: format tags - q8 {tally_fmt(t, KqFmt.q8)}, k4 {tally_fmt(t, KqFmt.k4)}, k5 {tally_fmt(t, KqFmt.k5)}, k6 {tally_fmt(t, KqFmt.k6)}, q40 {tally_fmt(t, KqFmt.q40)}, q51 {tally_fmt(t, KqFmt.q51)}, iq4xs {tally_fmt(t, KqFmt.iq4xs)}, k3 {tally_fmt(t, KqFmt.k3)}, iq3s {tally_fmt(t, KqFmt.iq3s)}, iq3xxs {tally_fmt(t, KqFmt.iq3xxs)}, iq4nl {tally_fmt(t, KqFmt.iq4nl)}, k2 {tally_fmt(t, KqFmt.k2)}\n") + to_log(LOG_INFO, "dasLLAMA noisy: plane MB - q8 {float(long_length(t.qblob)) / mb} (+scales {float(long_length(t.qscales) * 4l) / mb}), k4 {float(long_length(t.k4q) + long_length(t.k4s)) / mb}, k5 {float(long_length(t.k5q) + long_length(t.k5s)) / mb}, k6 {float(long_length(t.k6q) + long_length(t.k6s)) / mb}, q40 {float(long_length(t.q40q) + long_length(t.q40s)) / mb}, q51 {float(long_length(t.q51q) + long_length(t.q51s)) / mb}, iq4xs {float(long_length(t.iq4xsq) + long_length(t.iq4xss)) / mb}, k3 {float(long_length(t.k3q) + long_length(t.k3s)) / mb}, iq3s {float(long_length(t.iq3sq) + long_length(t.iq3ss)) / mb}, iq3xxs {float(long_length(t.iq3xxsq) + long_length(t.iq3xxss)) / mb}, iq4nl {float(long_length(t.iq4nlq) + long_length(t.iq4nls)) / mb}, k2 {float(long_length(t.k2q) + long_length(t.k2s)) / mb}, f32 {float(long_length(t.wblob) * 4l) / mb}\n") } // GGML disk type -> plane format tag (non-K-quant types ride the classic q8 path) @@ -1260,6 +1279,9 @@ def private kq_fmt_of(gt : int) : KqFmt { if (gt == GGML_TYPE_IQ4_NL) { return KqFmt.iq4nl } + if (gt == GGML_TYPE_Q2_K) { + return KqFmt.k2 + } if (gt == GGML_TYPE_Q4_0 && get_kq_q40_native()) { return KqFmt.q40 } @@ -2098,6 +2120,7 @@ def private load_gguf_parsed(var t : Model; m : GGUFMeta; bytes : array | g_stream_plane_total["iq3sq"] = (sz.iq3s_n / 256l) * IQ3S_QSB g_stream_plane_total["iq3xxsq"] = (sz.iq3xxs_n / 256l) * IQ3XXS_QSB g_stream_plane_total["iq4nlq"] = (sz.iq4nl_n / 256l) * Q40_QSB + g_stream_plane_total["k2q"] = (sz.k2_n / 256l) * K2_QSB let wb = mode == QuantMode.q8 ? sz.wblob_n : 0l g_stream_plane_total[t.metal_blob ? "mblob" : "qblob"] = (t.metal_blob ? (wb / Q8_BLOCK_ELEMS) * (Q8_QPB + Q8_SPB) : wb) @@ -2182,6 +2205,14 @@ def private load_gguf_parsed(var t : Model; m : GGUFMeta; bytes : array | t.iq4nls |> reserve((sz.iq4nl_n / 256l) * Q40_SSB) t.iq4nls |> resize((sz.iq4nl_n / 256l) * Q40_SSB) } + if (sz.k2_n > 0l) { + if (!stream_q) { + t.k2q |> reserve((sz.k2_n / 256l) * K2_QSB) + t.k2q |> resize((sz.k2_n / 256l) * K2_QSB) + } + t.k2s |> reserve((sz.k2_n / 256l) * K2_SSB) + t.k2s |> resize((sz.k2_n / 256l) * K2_SSB) + } if (mode == QuantMode.q8) { if (!stream_q) { t.qblob |> reserve(sz.wblob_n) @@ -2493,6 +2524,7 @@ def private load_gguf_parsed(var t : Model; m : GGUFMeta; bytes : array | t.kq_repack_mr33 = active_kq_layout_mr(33) t.kq_repack_mr34 = active_kq_layout_mr(34) t.kq_repack_mr45 = active_kq_layout_mr(45) + t.kq_repack_mr2 = active_kq_layout_mr(2) } } else { layout_repack_q8(t) @@ -2514,7 +2546,8 @@ def private load_gguf_parsed(var t : Model; m : GGUFMeta; bytes : array | t.kq_repack_mr33 = active_kq_layout_mr(33) t.kq_repack_mr34 = active_kq_layout_mr(34) t.kq_repack_mr45 = active_kq_layout_mr(45) - to_log(LOG_INFO, "dasLLAMA: K-quant planes repacked to the '{active_kernel_backend()}' layouts (k4 grp{t.kq_repack_mr4}, k5 grp{t.kq_repack_mr5}, k6 grp{t.kq_repack_mr6}, q40 grp{t.kq_repack_mr40}, iq4xs grp{t.kq_repack_mr44}, k3 grp{t.kq_repack_mr3}, iq3s grp{t.kq_repack_mr33}, iq3xxs grp{t.kq_repack_mr34}, iq4nl grp{t.kq_repack_mr45})\n") + t.kq_repack_mr2 = active_kq_layout_mr(2) + to_log(LOG_INFO, "dasLLAMA: K-quant planes repacked to the '{active_kernel_backend()}' layouts (k4 grp{t.kq_repack_mr4}, k5 grp{t.kq_repack_mr5}, k6 grp{t.kq_repack_mr6}, q40 grp{t.kq_repack_mr40}, iq4xs grp{t.kq_repack_mr44}, k3 grp{t.kq_repack_mr3}, iq3s grp{t.kq_repack_mr33}, iq3xxs grp{t.kq_repack_mr34}, iq4nl grp{t.kq_repack_mr45}, k2 grp{t.kq_repack_mr2})\n") } } } diff --git a/modules/dasLLAMA/dasllama/dasllama_math.das b/modules/dasLLAMA/dasllama/dasllama_math.das index 886bcea91c..71255d2064 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math.das +++ b/modules/dasLLAMA/dasllama/dasllama_math.das @@ -1089,6 +1089,7 @@ struct KernelBackend { kq_rows_iq3s : MatmulKqRowsFn = @@kq_unset_rows kq_rows_iq3xxs : MatmulKqRowsFn = @@kq_unset_rows kq_rows_iq4nl : MatmulKqRowsFn = @@kq_unset_rows + kq_rows_k2 : MatmulKqRowsFn = @@kq_unset_rows kq_batch : MatmulKqBatchFn = @@kq_unset_batch kq_groupn : MatmulKqGroupNFn = @@kq_unset_groupn kq_batch_groupn : MatmulKqBatchGroupNFn = @@kq_unset_batch_groupn @@ -1156,6 +1157,7 @@ var g_kq_rows_k3 = @@kq_unset_rows var g_kq_rows_iq3s = @@kq_unset_rows var g_kq_rows_iq3xxs = @@kq_unset_rows var g_kq_rows_iq4nl = @@kq_unset_rows +var g_kq_rows_k2 = @@kq_unset_rows var g_mm_kq_batch = @@kq_unset_batch var g_mm_kq_groupn = @@kq_unset_groupn var g_mm_kq_batch_groupn = @@kq_unset_batch_groupn @@ -1483,7 +1485,7 @@ def active_q8_layout_mr() : int64 => g_bake_cpu_override ? g_bake_cpu.q8_mr : g_ //! family split; the loader freezes these onto the Model at repack time. def active_kq_layout_mr(fmt : int) : int64 { if (g_bake_cpu_override) { - return fmt == 4 ? g_bake_cpu.kq_mr4 : (fmt == 5 ? g_bake_cpu.kq_mr5 : (fmt == 40 ? g_bake_cpu.kq_mr40 : (fmt == 44 ? g_bake_cpu.kq_mr44 : (fmt == 3 ? g_bake_cpu.kq_mr3 : (fmt == 33 ? g_bake_cpu.kq_mr33 : (fmt == 34 ? g_bake_cpu.kq_mr34 : (fmt == 45 ? g_bake_cpu.kq_mr45 : g_bake_cpu.kq_mr6))))))) + return fmt == 4 ? g_bake_cpu.kq_mr4 : (fmt == 5 ? g_bake_cpu.kq_mr5 : (fmt == 40 ? g_bake_cpu.kq_mr40 : (fmt == 44 ? g_bake_cpu.kq_mr44 : (fmt == 3 ? g_bake_cpu.kq_mr3 : (fmt == 33 ? g_bake_cpu.kq_mr33 : (fmt == 34 ? g_bake_cpu.kq_mr34 : (fmt == 45 ? g_bake_cpu.kq_mr45 : (fmt == 2 ? g_bake_cpu.kq_mr2 : g_bake_cpu.kq_mr6)))))))) } return int64(invoke(g_active_kq_layout, fmt)) } @@ -1664,6 +1666,7 @@ def private activate(be : KernelBackend) { g_kq_rows_iq3s = be.kq_rows_iq3s g_kq_rows_iq3xxs = be.kq_rows_iq3xxs g_kq_rows_iq4nl = be.kq_rows_iq4nl + g_kq_rows_k2 = be.kq_rows_k2 g_mm_kq_batch = be.kq_batch g_mm_kq_groupn = be.kq_groupn g_mm_kq_batch_groupn = be.kq_batch_groupn @@ -1871,6 +1874,7 @@ def kq_rows_fn(fmt : int) : MatmulKqRowsFn { if (fmt == 33) return g_kq_rows_iq3s if (fmt == 34) return g_kq_rows_iq3xxs if (fmt == 45) return g_kq_rows_iq4nl + if (fmt == 2) return g_kq_rows_k2 return @@kq_unset_rows } @@ -3246,6 +3250,9 @@ def private restore_math_fn_defaults { // nolint:STYLE037,STYLE038 — flat seam if (g_kq_rows_iq4nl == null) { g_kq_rows_iq4nl = @@kq_unset_rows } + if (g_kq_rows_k2 == null) { + g_kq_rows_k2 = @@kq_unset_rows + } if (g_mm_kq_batch == null) { g_mm_kq_batch = @@kq_unset_batch } diff --git a/modules/dasLLAMA/dasllama/dasllama_math_default.das b/modules/dasLLAMA/dasllama/dasllama_math_default.das index dcf09ed92f..632537abb1 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_default.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_default.das @@ -795,6 +795,49 @@ def iq4nl_rows_kernel(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; x } } +//! Q2_K row dot off the DISK-ORDER planes: per 16-elem group g, sc/mn = the group's nibble +//! pair, q = the unsigned 2-bit lane; the k4/k5 fold - isum on d*sc, the min term through the +//! activation 16-sums (bsum on dmin*mn). +[hint(unsafe_range_check, noalias = kqrow, noalias = ksrow, noalias = xqp, noalias = xsp, noalias = xbsp)] +def dot_k2q8(kqrow : uint8 const?; ksrow : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n : int64) : float { + var acc = 0.0 + unsafe { + for (sb in range64(n / 256l)) { + let kqo = sb * 64l + let kso = sb * 20l + let dd = f16_to_f32(uint(ksrow[kso]) | (uint(ksrow[kso + 1l]) << 8u)) + let dm = f16_to_f32(uint(ksrow[kso + 2l]) | (uint(ksrow[kso + 3l]) << 8u)) + var isum = 0 + var bsum = 0 + for (g in range64(16l)) { + let scb = int(ksrow[kso + 4l + g]) + let qb = kqo + (g / 8l) * 32l + (g % 2l) * 16l + let sh = int(((g % 8l) / 2l) * 2l) + let ab = sb * 256l + g * 16l + var ig = 0 + for (l in range64(16l)) { + ig += ((int(kqrow[qb + l]) >> sh) & 3) * int(xqp[ab + l]) + } + isum += (scb & 15) * ig + bsum += (scb >> 4) * xbsp[sb * 16l + g] + } + let d8 = xsp[sb] + acc += float(isum) * (dd * d8) + acc -= float(bsum) * (dm * d8) + } + } + return acc +} + +def k2_rows_kernel(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, rb, re : int64) { + let nsb = n / 256l + unsafe { + for (i in range64(rb, re)) { + yp[i] = dot_k2q8(kqp + i * nsb * 64l, ksp + i * nsb * 20l, xqp, xsp, xbsp, n) + } + } +} + def private kq_gemv_kernel(fmt : int; var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d : int64) { var myp = yp maybe_parallel_for(0, int(d), matmul_chunks_gemv(int(d), 1, n * d)) $(rb, re) { @@ -814,6 +857,8 @@ def private kq_gemv_kernel(fmt : int; var yp : float?; kqp : uint8 const?; ksp : iq3xxs_rows_kernel(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb), int64(re)) } elif (fmt == 45) { iq4nl_rows_kernel(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb), int64(re)) + } elif (fmt == 2) { + k2_rows_kernel(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb), int64(re)) } else { k6_rows_kernel(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb), int64(re)) } @@ -978,10 +1023,37 @@ def private dequant_iq4nl_row_grp(kqg : uint8 const?; ksg : uint8 const?; r, mr, } } +//! k2 grp row dequant: the 2-bit lanes stay in place as 16 four-byte columns x mr; scale row = +//! [16 sc/min bytes interleaved x mr][mr x 4B d+dmin header]. w = (d*sc)*q - (dmin*mn). +def private dequant_k2_row_grp(kqg : uint8 const?; ksg : uint8 const?; r, mr, nsb : int64; var dst : float?) { + unsafe { + for (sbi in range64(nsb)) { + let qb = sbi * 64l * mr + let sb = sbi * 20l * mr + let hd = sb + 16l * mr + 4l * r + let d = f16_to_f32(uint(ksg[hd]) | (uint(ksg[hd + 1l]) << 8u)) + let dmin = f16_to_f32(uint(ksg[hd + 2l]) | (uint(ksg[hd + 3l]) << 8u)) + for (g in range64(16l)) { + let scb = int(ksg[sb + g * mr + r]) + let dl = d * float(scb & 15) + let ml = dmin * float(scb >> 4) + let base = (g / 8l) * 8l + (g % 2l) * 4l + let sh = int(((g % 8l) / 2l) * 2l) + for (c4 in range64(4l)) { + for (t in range64(4l)) { + let q = (int(kqg[qb + ((base + c4) * mr + r) * 4l + t]) >> sh) & 3 + dst[sbi * 256l + g * 16l + c4 * 4l + t] = dl * float(q) - ml + } + } + } + } + } +} + //! Dequant one row off the grp-REPACKED K-quant planes (repack_k4/k5/k6_grp layout). //! kqg/ksg = row's GROUP plane base (group g = row/mr); r = row % mr. //! embed_row's kq_repacked path — per-token, so scalar is fine. -def dequant_kq_row_grp(fmt : int64; kqg : uint8 const?; ksg : uint8 const?; r, mr, n : int64; var dst : float?) { +def dequant_kq_row_grp(fmt : int64; kqg : uint8 const?; ksg : uint8 const?; r, mr, n : int64; var dst : float?) { // nolint:STYLE038 — the flat per-format dispatch (one arm per kq format) is the honest shape let nsb = n / 256l let qsb = kq_qsb(int(fmt)) let ssb = kq_ssb(int(fmt)) @@ -1009,6 +1081,10 @@ def dequant_kq_row_grp(fmt : int64; kqg : uint8 const?; ksg : uint8 const?; r, m dequant_iq4nl_row_grp(kqg, ksg, r, mr, nsb, dst) return } + if (fmt == 2l) { + dequant_k2_row_grp(kqg, ksg, r, mr, nsb, dst) + return + } unsafe { for (sbi in range64(nsb)) { let qb = sbi * qsb * mr @@ -1113,6 +1189,8 @@ def matmul_kq_groupn(fmt : int; var y : array | #; kq : array | #; myp[ii] = dot_iq3xxs_q8(kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) } elif (fmt == 45) { myp[ii] = dot_iq4nl_q8(kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) + } elif (fmt == 2) { + myp[ii] = dot_k2q8(kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) } else { myp[ii] = dot_k6q8(kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) } diff --git a/modules/dasLLAMA/dasllama/dasllama_math_gen.das b/modules/dasLLAMA/dasllama/dasllama_math_gen.das index 44ce444832..c6135c0aec 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_gen.das @@ -263,6 +263,10 @@ def iq4nlq8_layout_gen() : int { return q8q8_repack_type(GEMM_REFERENCE_MR).interleave } +def k2q8_layout_gen() : int { + return q8q8_repack_type(GEMM_REFERENCE_MR).interleave +} + // the KernelBackend.kq_layout slot form of kq_layout_of (registered on both gen backends) def private kq_layout_fmt_gen(fmt : int) : int => int(kq_layout_of(fmt)) @@ -293,6 +297,9 @@ def kq_layout_of(fmt : int) : int64 { if (fmt == 45) { return int64(iq4nlq8_layout_gen()) } + if (fmt == 2) { + return int64(k2q8_layout_gen()) + } return int64(k6q8_layout_gen()) } @@ -815,6 +822,74 @@ def iq4nlq8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xq } } +//! One row's dot off the grp k2 planes, scalar - the k2 stubs' reference body and the +//! repack oracle: the k4/k5 fold (per-16 nibble sc/min pairs, the min term through the +//! activation sums) over the in-place 2-bit lanes. Same float fold order as dot_k2q8. +def k2_grp_row_dot(kqg : uint8 const?; ksg : uint8 const?; r, mr : int64; xqp : int8 const?; xsp : float const?; xbsp : int const?; n : int64) : float { + var acc = 0.0 + let nsb = n / 256l + unsafe { + for (sbi in range64(nsb)) { + let qb = sbi * 64l * mr + let sb = sbi * 20l * mr + var isum = 0 + var bsum = 0 + for (g in range64(16l)) { + let scb = int(ksg[sb + g * mr + r]) + let base = (g / 8l) * 8l + (g % 2l) * 4l + let sh = int(((g % 8l) / 2l) * 2l) + var ig = 0 + for (c4 in range64(4l)) { + for (t in range64(4l)) { + let q = (int(kqg[qb + ((base + c4) * mr + r) * 4l + t]) >> sh) & 3 + ig += q * int(xqp[sbi * 256l + g * 16l + c4 * 4l + t]) + } + } + isum += (scb & 15) * ig + bsum += (scb >> 4) * xbsp[sbi * 16l + g] + } + let hd = sb + 16l * mr + 4l * r + let dk = f16_to_f32(uint(ksg[hd]) | (uint(ksg[hd + 1l]) << 8u)) + let dm = f16_to_f32(uint(ksg[hd + 2l]) | (uint(ksg[hd + 3l]) << 8u)) + acc += float(isum) * (dk * xsp[sbi]) + acc -= float(bsum) * (dm * xsp[sbi]) + } + } + return acc +} + +[hint(unsafe_range_check, noalias = kqp, noalias = ksp, noalias = xqp, noalias = xsp, noalias = xbsp)] +def k2q8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, rb, re : int64) : void { + let mr = int64(k2q8_layout_gen()) + let nsb = n / 256l + unsafe { + for (i in range64(rb, re)) { + let g = i / mr + yp[i] = k2_grp_row_dot(kqp + g * mr * nsb * 64l, ksp + g * mr * nsb * 20l, i % mr, mr, xqp, xsp, xbsp, n) + } + } +} + +[tune_perm(mr = 4), tune_perm(mr = 4, nrsplit = 2), tune_perm(mr = 8), tune_perm(mr = 8, nrsplit = 2), + tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "maddubs", width = 256, mr = 8, nrsplit = 2, requires = "avx2"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, nrsplit = 2, requires = "avxvnni|avx512vnni"), + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, nrsplit = 2, requires = "avx512vnni,avx512bw"), + tune_companion(fn = "k2q8_gemv_gen", gen = "dasllama_gemm_gen::k2_gemv"), + tune_companion(fn = "k2q8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), + tune(gen = "dasllama_gemm_gen::k2_tile", + fallback = "dot_vpdpbusd_width512_mr16;dot_vpdpbusd_width256_mr8;dot_maddubs_width256_mr8;mr4"), + hint(unsafe_range_check, noalias = kqg, noalias = ksg, noalias = xqp, noalias = xsp, noalias = xbsp)] +def k2q8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d, g, t0 : int64) : void { + let mr = int64(k2q8_layout_gen()) + unsafe { + for (t in range64(4l)) { + for (r in range64(mr)) { + yp[(t0 + t) * d + g * mr + r] = k2_grp_row_dot(kqg, ksg, r, mr, xqp + (t0 + t) * n, xsp + (t0 + t) * (n / 256l), xbsp + (t0 + t) * (n / 16l), n) + } + } + } +} + //! The f16-scale GEMV companion (wscale_f16 rail): q8q8_gemv_gen's twin over the raw binary16 //! group-scale plane. Same rows-range contract; the generated body differs only in the scale //! fold's widen (fcvtl/vcvtph2ps). Not `private`: it IS the mm_rows_s16 slot. @@ -1148,6 +1223,8 @@ def private repack_kq_gen(fmt : int; var kq : uint8?; var ks : uint8?; n, d : in repack_iq3xxs_grp(kq, ks, n, d, kq_layout_of(34)) } elif (fmt == 45) { repack_q40_grp(kq, ks, n, d, kq_layout_of(45)) // byte-identical plane shapes + } elif (fmt == 2) { + repack_k2_grp(kq, ks, n, d, kq_layout_of(2)) } else { repack_k6_grp(kq, ks, n, d, kq_layout_of(6)) } @@ -1170,6 +1247,8 @@ def private repack_kq_bake(fmt : int; var kq : uint8?; var ks : uint8?; n, d : i repack_iq3xxs_grp(kq, ks, n, d, active_kq_layout_mr(34)) } elif (fmt == 45) { repack_q40_grp(kq, ks, n, d, active_kq_layout_mr(45)) // byte-identical plane shapes + } elif (fmt == 2) { + repack_k2_grp(kq, ks, n, d, active_kq_layout_mr(2)) } else { repack_k6_grp(kq, ks, n, d, active_kq_layout_mr(6)) } @@ -1690,6 +1769,8 @@ def private kq_kernel_gen(fmt : int; var yp : float?; kqp : uint8 const?; ksp : iq3xxsq8_gemv_gen(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb) * mr, int64(re) * mr) } elif (fmt == 45) { iq4nlq8_gemv_gen(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb) * mr, int64(re) * mr) + } elif (fmt == 2) { + k2q8_gemv_gen(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb) * mr, int64(re) * mr) } else { k6q8_gemv_gen(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb) * mr, int64(re) * mr) } @@ -1712,6 +1793,8 @@ def private kq_kernel_gen(fmt : int; var yp : float?; kqp : uint8 const?; ksp : myp[i] = dot_iq3xxs_q8(kqp + i * nsb * 96l, ksp + i * nsb * 20l, xqp, xsp, xbsp, n) } elif (fmt == 45) { myp[i] = dot_iq4nl_q8(kqp + i * nsb * 128l, ksp + i * nsb * 16l, xqp, xsp, xbsp, n) + } elif (fmt == 2) { + myp[i] = dot_k2q8(kqp + i * nsb * 64l, ksp + i * nsb * 20l, xqp, xsp, xbsp, n) } else { myp[i] = dot_k6q8(kqp + i * nsb * 192l, ksp + i * nsb * 18l, xqp, xsp, xbsp, n) } @@ -1727,7 +1810,7 @@ def private kq_batch_cell_gen(fmt : int; var myp : float?; kqp : uint8 const?; k let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) let nb16 = n / 16l - let packed = fmt == 4 || fmt == 40 || fmt == 44 || fmt == 3 || fmt == 45 // packed planes: the tile reads them directly (iq3s/iq3xxs expand their grid gathers into the panel) + let packed = fmt == 4 || fmt == 40 || fmt == 44 || fmt == 3 || fmt == 45 || fmt == 2 // packed planes: the tile reads them directly (iq3s/iq3xxs expand their grid gathers into the panel) var scratch : array if (!packed) { scratch |> resize(mr * nsb * 256l) @@ -1763,6 +1846,8 @@ def private kq_batch_cell_gen(fmt : int; var myp : float?; kqp : uint8 const?; k iq3xxsq8_tile_gen(myp, scp, ksg, xqp, xsp, xbsp, n, d, g, tk) } elif (fmt == 45) { iq4nlq8_tile_gen(myp, kqg, ksg, xqp, xsp, xbsp, n, d, g, tk) + } elif (fmt == 2) { + k2q8_tile_gen(myp, kqg, ksg, xqp, xsp, xbsp, n, d, g, tk) } else { k6q8_tile_gen(myp, scp, ksg, xqp, xsp, xbsp, n, d, g, tk) } @@ -1785,6 +1870,8 @@ def private kq_batch_cell_gen(fmt : int; var myp : float?; kqp : uint8 const?; k iq3xxsq8_gemv_gen(myp + tk * d, kqp, ksp, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n, g * mr, (g + 1l) * mr) } elif (fmt == 45) { iq4nlq8_gemv_gen(myp + tk * d, kqp, ksp, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n, g * mr, (g + 1l) * mr) + } elif (fmt == 2) { + k2q8_gemv_gen(myp + tk * d, kqp, ksp, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n, g * mr, (g + 1l) * mr) } else { k6q8_gemv_gen(myp + tk * d, kqp, ksp, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n, g * mr, (g + 1l) * mr) } @@ -1828,6 +1915,8 @@ def private kq_batch_kernel_gen(fmt : int; var yp : float?; kqp : uint8 const?; myp[tk * d + i] = dot_iq3xxs_q8(kqp + i * nsb * 96l, ksp + i * nsb * 20l, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * (n / 16l), n) } elif (fmt == 45) { myp[tk * d + i] = dot_iq4nl_q8(kqp + i * nsb * 128l, ksp + i * nsb * 16l, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * (n / 16l), n) + } elif (fmt == 2) { + myp[tk * d + i] = dot_k2q8(kqp + i * nsb * 64l, ksp + i * nsb * 20l, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * (n / 16l), n) } else { myp[tk * d + i] = dot_k6q8(kqp + i * nsb * 192l, ksp + i * nsb * 18l, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * (n / 16l), n) } @@ -1886,6 +1975,8 @@ def private kq_batch_groupn_gen(fmt : int; var yp : float?; kqp : uint8 const?; myp[tk * d + i] = dot_iq3xxs_q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n) } elif (fmt == 45) { myp[tk * d + i] = dot_iq4nl_q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n) + } elif (fmt == 2) { + myp[tk * d + i] = dot_k2q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n) } else { myp[tk * d + i] = dot_k6q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n) } @@ -2053,6 +2144,8 @@ def kq_groupn_gen(fmt : int; var yp : float?; kqp : uint8 const?; ksp : uint8 co iq3xxsq8_gemv_gen(myp + r * d, kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n, gLocal * mr, gEnd * mr) } elif (fmt == 45) { iq4nlq8_gemv_gen(myp + r * d, kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n, gLocal * mr, gEnd * mr) + } elif (fmt == 2) { + k2q8_gemv_gen(myp + r * d, kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n, gLocal * mr, gEnd * mr) } else { k6q8_gemv_gen(myp + r * d, kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n, gLocal * mr, gEnd * mr) } @@ -2081,6 +2174,8 @@ def kq_groupn_gen(fmt : int; var yp : float?; kqp : uint8 const?; ksp : uint8 co myp[r * d + i] = dot_iq3xxs_q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) } elif (fmt == 45) { myp[r * d + i] = dot_iq4nl_q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) + } elif (fmt == 2) { + myp[r * d + i] = dot_k2q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) } else { myp[r * d + i] = dot_k6q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) } @@ -2374,7 +2469,7 @@ def dasllama_math_gen_register() { // the no-witness stance above covers kq too mm_kq = @@kq_kernel_gen, repack_kq = @@repack_kq_gen, kq_rows_k4 = @@k4q8_gemv_gen, kq_rows_k5 = @@k5q8_gemv_gen, - kq_rows_k6 = @@k6q8_gemv_gen, kq_rows_q40 = @@q40q8_gemv_gen, kq_rows_iq4xs = @@iq4xsq8_gemv_gen, kq_rows_k3 = @@k3q8_gemv_gen, kq_rows_iq3s = @@iq3sq8_gemv_gen, kq_rows_iq3xxs = @@iq3xxsq8_gemv_gen, kq_rows_iq4nl = @@iq4nlq8_gemv_gen, + kq_rows_k6 = @@k6q8_gemv_gen, kq_rows_q40 = @@q40q8_gemv_gen, kq_rows_iq4xs = @@iq4xsq8_gemv_gen, kq_rows_k3 = @@k3q8_gemv_gen, kq_rows_iq3s = @@iq3sq8_gemv_gen, kq_rows_iq3xxs = @@iq3xxsq8_gemv_gen, kq_rows_iq4nl = @@iq4nlq8_gemv_gen, kq_rows_k2 = @@k2q8_gemv_gen, kq_batch = @@kq_batch_kernel_gen, kq_groupn = @@kq_groupn_gen, kq_batch_groupn = @@kq_batch_groupn_gen, // q51 family: tile + GEMV off the grp planes (the tile family's companions stamp @@ -2403,7 +2498,7 @@ def dasllama_math_gen_register() { mm_rows_s16 = @@q8q8_gemv_s16_gen, mm_kq = @@kq_kernel_gen, repack_kq = @@repack_kq_gen, kq_rows_k4 = @@k4q8_gemv_gen, kq_rows_k5 = @@k5q8_gemv_gen, - kq_rows_k6 = @@k6q8_gemv_gen, kq_rows_q40 = @@q40q8_gemv_gen, kq_rows_iq4xs = @@iq4xsq8_gemv_gen, kq_rows_k3 = @@k3q8_gemv_gen, kq_rows_iq3s = @@iq3sq8_gemv_gen, kq_rows_iq3xxs = @@iq3xxsq8_gemv_gen, kq_rows_iq4nl = @@iq4nlq8_gemv_gen, + kq_rows_k6 = @@k6q8_gemv_gen, kq_rows_q40 = @@q40q8_gemv_gen, kq_rows_iq4xs = @@iq4xsq8_gemv_gen, kq_rows_k3 = @@k3q8_gemv_gen, kq_rows_iq3s = @@iq3sq8_gemv_gen, kq_rows_iq3xxs = @@iq3xxsq8_gemv_gen, kq_rows_iq4nl = @@iq4nlq8_gemv_gen, kq_rows_k2 = @@k2q8_gemv_gen, kq_batch = @@kq_batch_kernel_gen, kq_groupn = @@kq_groupn_gen, kq_batch_groupn = @@kq_batch_groupn_gen, groupn_q51 = @@q51q8_groupn_gen, repack_q51 = @@repack_q51_gen, diff --git a/modules/dasLLAMA/dasllama/dasllama_ple.das b/modules/dasLLAMA/dasllama/dasllama_ple.das index 6f8ddd0cfe..34e4c840ca 100644 --- a/modules/dasLLAMA/dasllama/dasllama_ple.das +++ b/modules/dasLLAMA/dasllama/dasllama_ple.das @@ -72,6 +72,8 @@ def ple_check_table(t : Model; origin : string) { have = (long_length(t.iq3xxsq) / IQ3XXS_QSB) * 256l } elif (t.ple_emb_fmt == KqFmt.iq4nl) { have = (long_length(t.iq4nlq) / Q40_QSB) * 256l + } elif (t.ple_emb_fmt == KqFmt.k2) { + have = (long_length(t.k2q) / K2_QSB) * 256l } if (have < need) { panic("dasLLAMA: the PLE token table is not resident ({origin}): fmt {t.ple_emb_fmt} plane holds {have} of {need} elements - regenerate this carrier (it predates the pinned-table rail)") @@ -115,6 +117,8 @@ def private ple_gather_row(t : Model; row : int64; var dst : array; doff dequant_iq3xxs_plane_superblock(t.iq3xxsq, sb * IQ3XXS_QSB, t.iq3xxss, sb * IQ3XXS_SSB, dst, doff + blk * 256l) } elif (t.ple_emb_fmt == KqFmt.iq4nl) { dequant_iq4nl_plane_superblock(t.iq4nlq, sb * Q40_QSB, t.iq4nls, sb * Q40_SSB, dst, doff + blk * 256l) + } elif (t.ple_emb_fmt == KqFmt.k2) { + dequant_k2_plane_superblock(t.k2q, sb * K2_QSB, t.k2s, sb * K2_SSB, dst, doff + blk * 256l) } else { panic("ple_gather_row: '{t.ple_emb_fmt}' has no kq plane pair") } diff --git a/modules/dasLLAMA/dasllama/dasllama_repack.das b/modules/dasLLAMA/dasllama/dasllama_repack.das index ed34335eec..ae4823a294 100644 --- a/modules/dasLLAMA/dasllama/dasllama_repack.das +++ b/modules/dasLLAMA/dasllama/dasllama_repack.das @@ -511,6 +511,49 @@ def repack_iq3s_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { delete ts } +//! k2 grp layout: the qs bytes as 16 four-byte columns x mr rows (the 2-bit lanes stay in +//! place - the kernels shift per group); scale row interleaves as [16 sc/min bytes x mr] +//! [mr x 4B d+dmin header]. +def repack_k2_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { + let nsb = n / 256l + let qrow = nsb * 64l + let srow = nsb * 20l + let ng = d / mr + var tq : array + var ts : array + tq |> resize(d * qrow) + ts |> resize(d * srow) + unsafe { + var tqp = addr(tq[0]) + var tsp = addr(ts[0]) + memcpy(tqp, kq, d * qrow) + memcpy(tsp, ks, d * srow) + for (g in range64(ng)) { + for (sbi in range64(nsb)) { + let dq = g * mr * qrow + sbi * 64l * mr + let ds = g * mr * srow + sbi * 20l * mr + for (r in range64(mr)) { + let sq = (g * mr + r) * qrow + sbi * 64l + let ss = (g * mr + r) * srow + sbi * 20l + for (c in range64(16l)) { + for (t in range64(4l)) { + kq[dq + (c * mr + r) * 4l + t] = tqp[sq + c * 4l + t] + } + } + for (idx in range64(16l)) { + ks[ds + idx * mr + r] = tsp[ss + 4l + idx] + } + for (h in range64(4l)) { + ks[ds + 16l * mr + 4l * r + h] = tsp[ss + h] + } + } + } + } + } + delete tq + delete ts +} + // ===== grp panel unpack (the byte-expanded tile form the kq stamps load) ===== //! Unpack ONE group's packed grp k5/k6 quant panel into the BYTE-EXPANDED tile form (wlo/whi diff --git a/modules/dasLLAMA/harness/gen_tune_probe.das b/modules/dasLLAMA/harness/gen_tune_probe.das index c250f29cee..db0348dfc7 100644 --- a/modules/dasLLAMA/harness/gen_tune_probe.das +++ b/modules/dasLLAMA/harness/gen_tune_probe.das @@ -526,6 +526,17 @@ def pack_kq_disk_block(fmt : int64; var blkb : array; base : int) { // blkb[b * 18 + 2 + i] = uint8((base * 41 + b * 17 + i * 89 + 7) % 256) } } + } elif (fmt == 2l) { + // 84B Q2_K disk block: 16 sc/min bytes, 64 qs, f16 d, f16 dmin - every byte pattern is legal + for (i in range(80)) { + blkb[i] = uint8((base * 37 + i * 101 + 29) % 256) + } + let dbits = f32_to_f16(0.002 + 0.0001 * float(base % 13)) + blkb[80] = uint8(dbits & 0xFF) + blkb[81] = uint8(dbits >> 8u) + let mbits = f32_to_f16(0.001 + 0.0001 * float((base + 5) % 11)) + blkb[82] = uint8(mbits & 0xFF) + blkb[83] = uint8(mbits >> 8u) } elif (fmt == 4l) { pack_kq_scale_header(blkb, base) for (i in range(128)) { @@ -572,6 +583,8 @@ def repack_kq_grp_fmt(fmt : int64; var kq : uint8?; var ks : uint8?; n, d, mr : repack_iq3xxs_grp(kq, ks, n, d, mr) } elif (fmt == 45l) { repack_q40_grp(kq, ks, n, d, mr) + } elif (fmt == 2l) { + repack_k2_grp(kq, ks, n, d, mr) } else { repack_k6_grp(kq, ks, n, d, mr) } @@ -584,7 +597,7 @@ def build_kq_fixture(fmt : int64; n, d, ntok : int64) : KqFixture { // nolint: fx.kq |> resize(d * fx.nsb * qsb) fx.ks |> resize(d * fx.nsb * ssb) var blkb : array - blkb |> resize(fmt == 4l || fmt == 40l || fmt == 45l ? 144l : (fmt == 5l ? 176l : (fmt == 44l ? 136l : (fmt == 3l || fmt == 33l ? 110l : (fmt == 34l ? 98l : 210l))))) + blkb |> resize(fmt == 2l ? 84l : (fmt == 4l || fmt == 40l || fmt == 45l ? 144l : (fmt == 5l ? 176l : (fmt == 44l ? 136l : (fmt == 3l || fmt == 33l ? 110l : (fmt == 34l ? 98l : 210l)))))) for (r in range64(d)) { for (s in range64(fx.nsb)) { let base = int(r * 31l + s * 7l) @@ -606,6 +619,8 @@ def build_kq_fixture(fmt : int64; n, d, ntok : int64) : KqFixture { // nolint: transcode_iq3xxs_superblock(blkb, 0l, fx.kq, sb * qsb, fx.ks, sb * ssb) } elif (fmt == 45l) { transcode_iq4nl_superblock(blkb, 0l, fx.kq, sb * qsb, fx.ks, sb * ssb) + } elif (fmt == 2l) { + transcode_q2k_superblock(blkb, 0l, fx.kq, sb * qsb, fx.ks, sb * ssb) } else { transcode_q6k_superblock(blkb, 0l, fx.kq, sb * qsb, fx.ks, sb * ssb) } @@ -655,8 +670,11 @@ def build_kq_fixture(fmt : int64; n, d, ntok : int64) : KqFixture { // nolint: : (fmt == 45l ? iq4nl_grp_row_dot(kqg, ksg, r, 4l, addr(fx.xq[0]) + tk * n, addr(fx.xs[0]) + tk * (n / 256l), addr(fx.xbs[0]) + tk * (n / 16l), n) - : kq_grp_row_dot(fmt, kqg, ksg, r, 4l, - addr(fx.xq[0]) + tk * n, addr(fx.xs[0]) + tk * (n / 256l), addr(fx.xbs[0]) + tk * (n / 16l), n))))))) + : (fmt == 2l + ? k2_grp_row_dot(kqg, ksg, r, 4l, + addr(fx.xq[0]) + tk * n, addr(fx.xs[0]) + tk * (n / 256l), addr(fx.xbs[0]) + tk * (n / 16l), n) + : kq_grp_row_dot(fmt, kqg, ksg, r, 4l, + addr(fx.xq[0]) + tk * n, addr(fx.xs[0]) + tk * (n / 256l), addr(fx.xbs[0]) + tk * (n / 16l), n)))))))) } } } @@ -693,6 +711,9 @@ def kq_tile_variants(fmt : int64) : array> { if (fmt == 45l) { return <- iq4nlq8_tile_gen_variants() } + if (fmt == 2l) { + return <- k2q8_tile_gen_variants() + } return <- k6q8_tile_gen_variants() } @@ -746,6 +767,12 @@ def kq_gemv_variants_by_suffix(fmt : int64) : table { tab[v._0] = v._1 } delete gvs + } elif (fmt == 2l) { + var gvs <- k2q8_gemv_gen_variants() + for (v in gvs) { + tab[v._0] = v._1 + } + delete gvs } else { var gvs <- k6q8_gemv_gen_variants() for (v in gvs) { @@ -808,6 +835,12 @@ def kq_layout_mrs(fmt : int64) : table { mrs[v._0] = invoke(v._1) } delete lvs + } elif (fmt == 2l) { + var lvs <- k2q8_layout_gen_variants() + for (v in lvs) { + mrs[v._0] = invoke(v._1) + } + delete lvs } else { var lvs <- k6q8_layout_gen_variants() for (v in lvs) { @@ -825,7 +858,7 @@ def kq_layout_mrs(fmt : int64) : table { def run_kq_tile(tile; fx : KqFixture; kq : array; ks : array; mr : int64; var y : array) { let ng = fx.d / mr verify(fx.ntok % 4l == 0l) - let packed = fx.fmt == 4l || fx.fmt == 40l || fx.fmt == 44l || fx.fmt == 3l || fx.fmt == 45l // packed planes: the tile reads them directly (iq3s/iq3xxs unpack their panels per group) + let packed = fx.fmt == 4l || fx.fmt == 40l || fx.fmt == 44l || fx.fmt == 3l || fx.fmt == 45l || fx.fmt == 2l // packed planes: the tile reads them directly (iq3s/iq3xxs unpack their panels per group) var panel : array if (!packed) { panel |> resize(int(mr * fx.nsb * 256l)) @@ -1901,7 +1934,7 @@ def tune_mode_run : bool { // nolint:STYLE037,STYLE038 — flat family-by-family print("aborting mid-run - no sidecar written\n") return false } - for (fmt in fixed_array(4l, 5l, 6l, 40l, 44l, 3l, 33l, 34l, 45l)) { + for (fmt in fixed_array(4l, 5l, 6l, 40l, 44l, 3l, 33l, 34l, 45l, 2l)) { interrupt_gate("k{fmt}q8_tile_gen") let kq_t0 = ref_time_ticks() let entry = kq_tile_entry(fmt) diff --git a/modules/dasLLAMA/tests/test_kqformat.das b/modules/dasLLAMA/tests/test_kqformat.das index 5b15388d94..7d842bb574 100644 --- a/modules/dasLLAMA/tests/test_kqformat.das +++ b/modules/dasLLAMA/tests/test_kqformat.das @@ -25,6 +25,7 @@ def test_kqfmt_enum_values(t : T?) { t |> equal(int(KqFmt.iq3s), 8) t |> equal(int(KqFmt.iq3xxs), 9) t |> equal(int(KqFmt.iq4nl), 10) + t |> equal(int(KqFmt.k2), 11) } } @@ -44,9 +45,11 @@ def test_kqfmt_predicates(t : T?) { t |> success(kq_sb(KqFmt.iq3s), "iq3s is superblock") t |> success(kq_sb(KqFmt.iq3xxs), "iq3xxs is superblock") t |> success(kq_sb(KqFmt.iq4nl), "iq4nl is superblock") + t |> success(kq_sb(KqFmt.k2), "k2 is superblock") t |> success(kq_sb(int(KqFmt.iq3s)), "the int-id predicate agrees for iq3s") t |> success(kq_sb(int(KqFmt.iq3xxs)), "the int-id predicate agrees for iq3xxs") t |> success(kq_sb(int(KqFmt.iq4nl)), "the int-id predicate agrees for iq4nl") + t |> success(kq_sb(int(KqFmt.k2)), "the int-id predicate agrees for k2") } } @@ -73,6 +76,8 @@ def test_kqfmt_strides(t : T?) { t |> equal(kq_ssb(KqFmt.iq3s), 20l) // the k4 row: f16 d + 8 int8 (1 + 2s) t |> equal(kq_ssb(KqFmt.iq3xxs), 20l) // the same row: f16 d halved + 8 x (2ls + 1) t |> equal(kq_ssb(KqFmt.iq4nl), 16l) // q40's row: 8 x f16 d + t |> equal(kq_qsb(KqFmt.k2), 64l) // 64 qs bytes, verbatim + t |> equal(kq_ssb(KqFmt.k2), 20l) // [f16 d][f16 dmin][16 sc/min bytes] } t |> run("named constants agree with the literals") @(t : T?) { t |> equal(K4_QSB, 128l) @@ -89,6 +94,8 @@ def test_kqfmt_strides(t : T?) { t |> equal(IQ4XS_SSB, 20l) t |> equal(K3_QSB, 96l) t |> equal(K3_SSB, 18l) + t |> equal(K2_QSB, 64l) + t |> equal(K2_SSB, 20l) t |> equal(IQ3S_QSB, 104l) t |> equal(IQ3S_SSB, 20l) } @@ -126,9 +133,10 @@ def test_kqfmt_id_bridge(t : T?) { t |> equal(kq_schema_id(KqFmt.iq3s), 33) t |> equal(kq_schema_id(KqFmt.iq3xxs), 34) t |> equal(kq_schema_id(KqFmt.iq4nl), 45) + t |> equal(kq_schema_id(KqFmt.k2), 2) } t |> run("gemm_schema's int-id forms delegate to the same truth") @(t : T?) { - for (f in [KqFmt.k4, KqFmt.k5, KqFmt.k6, KqFmt.q40, KqFmt.iq4xs, KqFmt.k3, KqFmt.iq3s, KqFmt.iq3xxs, KqFmt.iq4nl]) { + for (f in [KqFmt.k4, KqFmt.k5, KqFmt.k6, KqFmt.q40, KqFmt.iq4xs, KqFmt.k3, KqFmt.iq3s, KqFmt.iq3xxs, KqFmt.iq4nl, KqFmt.k2]) { t |> equal(kq_qsb(kq_schema_id(f)), kq_qsb(f)) t |> equal(kq_ssb(kq_schema_id(f)), kq_ssb(f)) } @@ -147,6 +155,7 @@ def test_kqfmt_elems_and_stream(t : T?) { t |> equal(kq_elems(KqFmt.iq3s), 256l) t |> equal(kq_elems(KqFmt.iq3xxs), 256l) t |> equal(kq_elems(KqFmt.iq4nl), 256l) + t |> equal(kq_elems(KqFmt.k2), 256l) t |> equal(kq_elems(KqFmt.q8), 32l) t |> equal(kq_elems(KqFmt.q51), 32l) } @@ -162,6 +171,7 @@ def test_kqfmt_elems_and_stream(t : T?) { t |> equal(kq_stream_code(KqFmt.iq3s), 33) t |> equal(kq_stream_code(KqFmt.iq3xxs), 34) t |> equal(kq_stream_code(KqFmt.iq4nl), 45) + t |> equal(kq_stream_code(KqFmt.k2), 20) // kernel id 2 = q51's stream tag } } diff --git a/modules/dasLLAMA/tests/test_kquant.das b/modules/dasLLAMA/tests/test_kquant.das index 1abe1188a9..cdbc29d94e 100644 --- a/modules/dasLLAMA/tests/test_kquant.das +++ b/modules/dasLLAMA/tests/test_kquant.das @@ -1,8 +1,8 @@ options gen2 options stack = 524288 // every dasLLAMA program root takes this budget (options stack does not unify up from libs) options persistent_heap // the layout arm loads a multi-GB model; explicit delete below -options _cyclomatic_complexity = 52 // every gate here is the same flat fmt == 4/5/6/40/44/3 ladder repeated per stage -options _function_length = 220 // and each gate carries its whole fixture build + oracle inline +options _cyclomatic_complexity = 58 // every gate here is the same flat fmt == 4/5/6/40/44/3 ladder repeated per stage +options _function_length = 240 // and each gate carries its whole fixture build + oracle inline options _dasllama_internal = true require dastest/testing_boost public @@ -28,6 +28,7 @@ def private q4_pat(k : int) : int => (k * 7 + 3) % 16 def private q5_pat(k : int) : int => (k * 3 + 7) % 32 def private q6_pat(k : int) : int => (k * 5 + 11) % 64 def private q3_pat(k : int) : int => (k * 5 + 3) % 8 +def private q2_pat(k : int) : int => (k * 3 + 1) % 4 // The shared synthetic scale/min sets: 6-bit values > 15 exercise the split high-bit arms of // the 6-bit scale/min unpack (j >= 4); d/dmin are f16-exact so expected-value math uses the f32 value. @@ -241,6 +242,33 @@ def private build_iq3s_block() : array { return <- blkb } +// k2 synthetic superblock: 84B Q2_K disk block - per-group sc/min nibble pairs (SYNTH_SC & 15, +// SYNTH_MN & 15), f16 d/dmin, the 2-bit lanes of q2_pat (written in the PACK direction) - +// weight k = SYNTH_D * (SYNTH_SC[g % 8] & 15) * q2_pat(k) - SYNTH_DMIN * (SYNTH_MN[g % 8] & 15), g = k / 16 +def private build_q2k_block() : array { + var blkb : array + blkb |> resize(84) + var qs : array + qs |> resize(64) + for (k in range(256)) { // group g = k/16: qs byte (g/8)*32 + (g%2)*16 + k%16 holds the 2-bit lane at shift 2*((g%8)/2) + let g = k / 16 + qs[(g / 8) * 32 + (g % 2) * 16 + k % 16] |= q2_pat(k) << (2 * ((g % 8) / 2)) + } + for (g in range(16)) { + blkb[g] = uint8((SYNTH_SC[g % 8] & 15) | ((SYNTH_MN[g % 8] & 15) << 4)) + } + for (i in range(64)) { + blkb[16 + i] = uint8(qs[i]) + } + let dbits = f32_to_f16(SYNTH_D) + blkb[80] = uint8(dbits & 0xFF) + blkb[81] = uint8(dbits >> 8u) + let mbits = f32_to_f16(SYNTH_DMIN) + blkb[82] = uint8(mbits & 0xFF) + blkb[83] = uint8(mbits >> 8u) + return <- blkb +} + // q40 synthetic superblock: 8 x 18B q4_0 disk blocks, per-block f16-exact d = (blk+1)/16 and // the shared nibble pattern — weight k = d(k/32) * (q4_pat(k) - 8) def private q40_d(blk : int) : float => float(blk + 1) * 0.0625 @@ -454,6 +482,22 @@ def test_kq_transcode_planes(t : T?) { t |> success(dst[k] == expected, "iq3s plane element must match the ggml dequant exactly") } } + t |> run("k2 planes reorder the header and compose the 2-bit quants exactly") @(t : T?) { + let blkb <- build_q2k_block() + var kq : array + var ks : array + kq |> resize(64) + ks |> resize(20) + transcode_q2k_superblock(blkb, 0l, kq, 0l, ks, 0l) + var dst : array + dst |> resize(256) + dequant_k2_plane_superblock(kq, 0l, ks, 0l, dst, 0l) + for (k in range(256)) { + let g = k / 16 + let expected = SYNTH_D * float(SYNTH_SC[g % 8] & 15) * float(q2_pat(k)) - SYNTH_DMIN * float(SYNTH_MN[g % 8] & 15) + t |> success(dst[k] == expected, "k2 plane element must match the ggml dequant exactly") + } + } t |> run("q51 planes unpack the hand-packed Q5_1 blocks exactly") @(t : T?) { for (blk in range(4)) { let blkb <- build_q51_block(blk) @@ -700,7 +744,7 @@ def test_q51_kernels(t : T?) { // elementwise dot over the plane dequant (the kernels' declared oracle). def private kq_dot_gate(t : T?; fmt : int; n : int64 = 512l) { let nsb = n / 256l - let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : (fmt == 45 ? build_iq4nl_block() : build_q6k_block()))))))) + let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : (fmt == 45 ? build_iq4nl_block() : (fmt == 2 ? build_q2k_block() : build_q6k_block())))))))) let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) var kq : array @@ -724,6 +768,8 @@ def private kq_dot_gate(t : T?; fmt : int; n : int64 = 512l) { transcode_iq3xxs_superblock(blk1, 0l, kq, s * qsb, ks, s * ssb) } elif (fmt == 45) { transcode_iq4nl_superblock(blk1, 0l, kq, s * qsb, ks, s * ssb) + } elif (fmt == 2) { + transcode_q2k_superblock(blk1, 0l, kq, s * qsb, ks, s * ssb) } else { transcode_q6k_superblock(blk1, 0l, kq, s * qsb, ks, s * ssb) } @@ -753,6 +799,8 @@ def private kq_dot_gate(t : T?; fmt : int; n : int64 = 512l) { dequant_iq3xxs_plane_superblock(kq, s * qsb, ks, s * ssb, w, s * 256l) } elif (fmt == 45) { dequant_iq4nl_plane_superblock(kq, s * qsb, ks, s * ssb, w, s * 256l) + } elif (fmt == 2) { + dequant_k2_plane_superblock(kq, s * qsb, ks, s * ssb, w, s * 256l) } else { dequant_k6_plane_superblock(kq, s * qsb, ks, s * ssb, w, s * 256l) } @@ -794,6 +842,8 @@ def private kq_dot_gate(t : T?; fmt : int; n : int64 = 512l) { got = dot_iq3xxs_q8(addr(kq[0]), addr(ks[0]), addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } elif (fmt == 45) { got = dot_iq4nl_q8(addr(kq[0]), addr(ks[0]), addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + } elif (fmt == 2) { + got = dot_k2q8(addr(kq[0]), addr(ks[0]), addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } else { got = dot_k6q8(addr(kq[0]), addr(ks[0]), addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } @@ -827,7 +877,7 @@ def test_kq_dots(t : T?) { } // 122B-class row widths: n=3072 (12 superblocks — qwen35moe-122B gate/up/cls) and n=1024 // (its k5 down_exps). Every earlier kq model kept n at 512..2560. - for (fmt in [4, 5, 6, 40, 44, 3, 33, 34, 45]) { + for (fmt in [4, 5, 6, 40, 44, 3, 33, 34, 45, 2]) { for (nn in [1024l, 3072l]) { t |> run("dot_{kq_tag(fmt)}q8 matches the fp64 reference at n={int(nn)}") @(t : T?) { kq_dot_gate(t, fmt, nn) @@ -846,7 +896,7 @@ def private kq_gemv_rows_gate(t : T?; fmt : int; n : int64 = 512l) { let nsb = n / 256l let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) - let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : (fmt == 45 ? build_iq4nl_block() : build_q6k_block()))))))) + let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : (fmt == 45 ? build_iq4nl_block() : (fmt == 2 ? build_q2k_block() : build_q6k_block())))))))) var kq : array var ks : array kq |> resize(d * nsb * qsb) @@ -869,6 +919,8 @@ def private kq_gemv_rows_gate(t : T?; fmt : int; n : int64 = 512l) { transcode_iq3xxs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } elif (fmt == 45) { transcode_iq4nl_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 2) { + transcode_q2k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } else { transcode_q6k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } @@ -917,6 +969,8 @@ def private kq_gemv_rows_gate(t : T?; fmt : int; n : int64 = 512l) { want = dot_iq3xxs_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } elif (fmt == 45) { want = dot_iq4nl_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + } elif (fmt == 2) { + want = dot_k2q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } else { want = dot_k6q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } @@ -958,7 +1012,7 @@ def test_kq_gemv_rows(t : T?) { t |> run("portable iq3s GEMV rows bit-match per-row disk dots") @(t : T?) { kq_gemv_rows_gate(t, 33) } - for (fmt in [4, 5, 6, 40, 44, 3, 33, 34, 45]) { + for (fmt in [4, 5, 6, 40, 44, 3, 33, 34, 45, 2]) { for (nn in [1024l, 3072l]) { t |> run("portable {kq_tag(fmt)} GEMV rows bit-match per-row disk dots at n={int(nn)}") @(t : T?) { kq_gemv_rows_gate(t, fmt, nn) @@ -978,7 +1032,7 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { let nsb = n / 256l let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) - let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : (fmt == 45 ? build_iq4nl_block() : build_q6k_block()))))))) + let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : (fmt == 45 ? build_iq4nl_block() : (fmt == 2 ? build_q2k_block() : build_q6k_block())))))))) var kq : array var ks : array kq |> resize(d * nsb * qsb) @@ -1001,6 +1055,8 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { transcode_iq3xxs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } elif (fmt == 45) { transcode_iq4nl_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 2) { + transcode_q2k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } else { transcode_q6k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } @@ -1048,6 +1104,8 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { want[r] = dot_iq3xxs_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } elif (fmt == 45) { want[r] = dot_iq4nl_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + } elif (fmt == 2) { + want[r] = dot_k2q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } else { want[r] = dot_k6q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } @@ -1068,6 +1126,8 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { dequant_iq3xxs_plane_superblock(kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb, wrow, r * n + s * 256l) } elif (fmt == 45) { dequant_iq4nl_plane_superblock(kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb, wrow, r * n + s * 256l) + } elif (fmt == 2) { + dequant_k2_plane_superblock(kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb, wrow, r * n + s * 256l) } else { dequant_k6_plane_superblock(kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb, wrow, r * n + s * 256l) } @@ -1089,6 +1149,8 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { repack_iq3xxs_grp(addr(kq[0]), addr(ks[0]), n, d, mr) } elif (fmt == 45) { repack_q40_grp(addr(kq[0]), addr(ks[0]), n, d, mr) + } elif (fmt == 2) { + repack_k2_grp(addr(kq[0]), addr(ks[0]), n, d, mr) } else { repack_k6_grp(addr(kq[0]), addr(ks[0]), n, d, mr) } @@ -1106,8 +1168,8 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { ? iq4xs_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) : (fmt == 3 ? k3_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) - : (fmt == 33 || fmt == 34 || fmt == 45 - ? (fmt == 33 ? iq3s_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) : (fmt == 34 ? iq3xxs_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) : iq4nl_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n))) + : (fmt == 33 || fmt == 34 || fmt == 45 || fmt == 2 + ? (fmt == 33 ? iq3s_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) : (fmt == 34 ? iq3xxs_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) : (fmt == 45 ? iq4nl_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) : k2_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n)))) : kq_grp_row_dot(int64(fmt), kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n))))) if (got != want[r]) { dotbad++ @@ -1135,7 +1197,7 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { [test] def test_kq_repack_grp(t : T?) { - for (fmt in [4, 5, 6, 40, 44, 3, 33, 34, 45]) { + for (fmt in [4, 5, 6, 40, 44, 3, 33, 34, 45, 2]) { for (mr in [4l, 8l, 16l]) { t |> run("repack_{kq_tag(fmt)}_grp mr={mr} preserves dots and row dequants") @(t : T?) { kq_repack_gate(t, fmt, mr) @@ -1160,7 +1222,7 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { // nolint:STYL let mr = kq_layout_of(fmt) // the format's OWN layout companion (per-format tune families) let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) - let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : (fmt == 45 ? build_iq4nl_block() : build_q6k_block()))))))) + let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : (fmt == 45 ? build_iq4nl_block() : (fmt == 2 ? build_q2k_block() : build_q6k_block())))))))) var kq : array var ks : array kq |> resize(d * nsb * qsb) @@ -1183,6 +1245,8 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { // nolint:STYL transcode_iq3xxs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } elif (fmt == 45) { transcode_iq4nl_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 2) { + transcode_q2k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } else { transcode_q6k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } @@ -1210,6 +1274,8 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { // nolint:STYL repack_iq3xxs_grp(addr(kq[0]), addr(ks[0]), n, d, mr) } elif (fmt == 45) { repack_q40_grp(addr(kq[0]), addr(ks[0]), n, d, mr) + } elif (fmt == 2) { + repack_k2_grp(addr(kq[0]), addr(ks[0]), n, d, mr) } else { repack_k6_grp(addr(kq[0]), addr(ks[0]), n, d, mr) } @@ -1238,7 +1304,7 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { // nolint:STYL // k5/k6 tiles read the byte-expanded panel the batch cell unpacks per (group, token-block) // — the gate covers unpack_kq_panel_grp too (tile-over-panel vs gemv-over-packed planes); // k4/q40 tiles read the packed planes directly - let packed = fmt == 4 || fmt == 40 || fmt == 44 || fmt == 3 || fmt == 45 // iq3s/iq3xxs tiles read the byte-expanded panel (the batch cell's shape) + let packed = fmt == 4 || fmt == 40 || fmt == 44 || fmt == 3 || fmt == 45 || fmt == 2 // iq3s/iq3xxs tiles read the byte-expanded panel (the batch cell's shape) var panel : array if (!packed) { panel |> resize(mr * nsb * 256l) @@ -1269,6 +1335,8 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { // nolint:STYL iq3xxsq8_tile_gen(addr(ytile[0]), addr(panel[0]), ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) } elif (fmt == 45) { iq4nlq8_tile_gen(addr(ytile[0]), kqg, ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) + } elif (fmt == 2) { + k2q8_tile_gen(addr(ytile[0]), kqg, ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) } else { k6q8_tile_gen(addr(ytile[0]), addr(panel[0]), ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) } @@ -1291,6 +1359,8 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { // nolint:STYL iq3xxsq8_gemv_gen(addr(ytile[tk * d]), addr(kq[0]), addr(ks[0]), addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, g * mr, (g + 1l) * mr) } elif (fmt == 45) { iq4nlq8_gemv_gen(addr(ytile[tk * d]), addr(kq[0]), addr(ks[0]), addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, g * mr, (g + 1l) * mr) + } elif (fmt == 2) { + k2q8_gemv_gen(addr(ytile[tk * d]), addr(kq[0]), addr(ks[0]), addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, g * mr, (g + 1l) * mr) } else { k6q8_gemv_gen(addr(ytile[tk * d]), addr(kq[0]), addr(ks[0]), addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, g * mr, (g + 1l) * mr) } @@ -1314,6 +1384,8 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { // nolint:STYL iq3xxsq8_gemv_gen(addr(ygemv[p * d]), addr(kq[0]), addr(ks[0]), addr(xq[p * n]), addr(xs[p * (n / 256l)]), addr(xbs[p * (n / 16l)]), n, 0l, d) } elif (fmt == 45) { iq4nlq8_gemv_gen(addr(ygemv[p * d]), addr(kq[0]), addr(ks[0]), addr(xq[p * n]), addr(xs[p * (n / 256l)]), addr(xbs[p * (n / 16l)]), n, 0l, d) + } elif (fmt == 2) { + k2q8_gemv_gen(addr(ygemv[p * d]), addr(kq[0]), addr(ks[0]), addr(xq[p * n]), addr(xs[p * (n / 256l)]), addr(xbs[p * (n / 16l)]), n, 0l, d) } else { k6q8_gemv_gen(addr(ygemv[p * d]), addr(kq[0]), addr(ks[0]), addr(xq[p * n]), addr(xs[p * (n / 256l)]), addr(xbs[p * (n / 16l)]), n, 0l, d) } @@ -1339,7 +1411,7 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { // nolint:STYL [test] def test_kq_tile(t : T?) { - for (fmt in [4, 5, 6, 40, 44, 3, 33, 34, 45]) { + for (fmt in [4, 5, 6, 40, 44, 3, 33, 34, 45, 2]) { t |> run("{kq_tag(fmt)} 4-token tile bit-matches per-token GEMVs") @(t : T?) { kq_tile_gate(t, fmt) } @@ -1360,7 +1432,7 @@ def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; let nsb = n / 256l let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) - let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : (fmt == 45 ? build_iq4nl_block() : build_q6k_block()))))))) + let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : (fmt == 45 ? build_iq4nl_block() : (fmt == 2 ? build_q2k_block() : build_q6k_block())))))))) var kq : array var ks : array kq |> resize(nreg * d * nsb * qsb) @@ -1383,6 +1455,8 @@ def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; transcode_iq3xxs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } elif (fmt == 45) { transcode_iq4nl_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 2) { + transcode_q2k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } else { transcode_q6k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } @@ -1455,6 +1529,9 @@ def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; } elif (fmt == 45) { want_sh[r * d + row] = dot_iq4nl_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) want_pr[r * d + row] = dot_iq4nl_q8(kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n) + } elif (fmt == 2) { + want_sh[r * d + row] = dot_k2q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + want_pr[r * d + row] = dot_k2q8(kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n) } else { want_sh[r * d + row] = dot_k6q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) want_pr[r * d + row] = dot_k6q8(kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n) @@ -1507,6 +1584,8 @@ def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; repack_iq3xxs_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) } elif (fmt == 45) { repack_q40_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) + } elif (fmt == 2) { + repack_k2_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) } else { repack_k6_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) } @@ -1538,6 +1617,9 @@ def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; } elif (fmt == 45) { iq4nlq8_gemv_gen(addr(want_sh[r * d]), kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, 0l, d) iq4nlq8_gemv_gen(addr(want_pr[r * d]), kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n, 0l, d) + } elif (fmt == 2) { + k2q8_gemv_gen(addr(want_sh[r * d]), kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, 0l, d) + k2q8_gemv_gen(addr(want_pr[r * d]), kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n, 0l, d) } else { k6q8_gemv_gen(addr(want_sh[r * d]), kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, 0l, d) k6q8_gemv_gen(addr(want_pr[r * d]), kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n, 0l, d) @@ -1573,7 +1655,7 @@ def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; [test] def test_kq_groupn(t : T?) { - for (fmt in [4, 5, 6, 40, 44, 3, 33, 34, 45]) { + for (fmt in [4, 5, 6, 40, 44, 3, 33, 34, 45, 2]) { t |> run("{kq_tag(fmt)} region-list GEMV bit-matches per-row dots (disk + grp slices)") @(t : T?) { kq_groupn_gate(t, fmt) } @@ -1617,7 +1699,7 @@ def private kq_batch_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 for (c in cnts) { nk += c } - let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : (fmt == 45 ? build_iq4nl_block() : build_q6k_block()))))))) + let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : (fmt == 45 ? build_iq4nl_block() : (fmt == 2 ? build_q2k_block() : build_q6k_block())))))))) var kq : array var ks : array kq |> resize(nreg * d * nsb * qsb) @@ -1640,6 +1722,8 @@ def private kq_batch_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 transcode_iq3xxs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } elif (fmt == 45) { transcode_iq4nl_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 2) { + transcode_q2k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } else { transcode_q6k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } @@ -1668,6 +1752,8 @@ def private kq_batch_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 repack_iq3xxs_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) } elif (fmt == 45) { repack_q40_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) + } elif (fmt == 2) { + repack_k2_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) } else { repack_k6_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) } @@ -1722,6 +1808,8 @@ def private kq_batch_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 iq3xxsq8_gemv_gen(addr(want[tk * d]), kqr, ksr, addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, 0l, d) } elif (fmt == 45) { iq4nlq8_gemv_gen(addr(want[tk * d]), kqr, ksr, addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, 0l, d) + } elif (fmt == 2) { + k2q8_gemv_gen(addr(want[tk * d]), kqr, ksr, addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, 0l, d) } else { k6q8_gemv_gen(addr(want[tk * d]), kqr, ksr, addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, 0l, d) } @@ -1753,7 +1841,7 @@ def private kq_batch_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 [test] def test_kq_batch_groupn(t : T?) { - for (fmt in [4, 5, 6, 40, 44, 3, 33, 34, 45]) { + for (fmt in [4, 5, 6, 40, 44, 3, 33, 34, 45, 2]) { t |> run("{kq_tag(fmt)} batch groupn bit-matches per-token rows-core calls") @(t : T?) { kq_batch_groupn_gate(t, fmt) } From 89059a561c5026b5cd5877bc0cb36b709a3075b5 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Mon, 31 Aug 2026 01:48:32 -0700 Subject: [PATCH 31/61] dasLLAMA: Q2_K JIT emitter - a fourth kqv2 arm (per-16 nibble scales, k4/k5 epilogue), zen2 at parity The k2 flag through emit_block_kqv2: k3's column compose minus the hmask (the 2-bit lane at shift 2*(blk%4) off 16 uniform four-byte columns), per-16 nibble scale extracts (ZExt the pair bytes to i32 lanes, then &15 / >>4 - scv/mnv carry the two groups' sc, new mnlo/mnhi their mins), the k63 split lo/hi accumulator machinery generalized behind a per16 flag (vecBsums/fuseAcc/madd16 split/flush/bs0-bs1), and the k4/k5 two-fma epilogue folding bacc on dmin. Prerequisite: the k2 grp scale header went FIELD-MAJOR ([16 sc x mr][mr x f16 d] [mr x f16 dmin]) so load_f16_vec_at serves the d/dmin vectors - repack_k2_grp and both grp readers moved together, re-gated bit-exact. The stubs became kq_*_gen_impl(gc, 2) and the probe's TEST list gains the family. Phase A's no-mint oddity resolved itself: a declined-stub family demands no sidecar entry, so the whole-scope mint fired on this phase's first run instead. Gates: probe test mode 11/11 k2 perms (dot_maddubs_width256_mr8 stamps at 4.8e-7 on zen2, the vpdpbusd perms decline here like every format), test_kquant -jit 212/216 (4 model skips). The re-mint crowned dot_maddubs_width256_mr8; the stamped e2e runs gen 75 t/s (30 reference), greedy ids 19/64 with the same 0.153-logit token-19 near-tie as the reference bodies. zen2 16t vs llama.cpp clean-cpu b10660: pp512 418.6 / 412.0 (1.02x), tg128 77.1 / 81.1 (0.95x) - pp above parity, tg in class. Co-Authored-By: Claude Fable 5 --- modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md | 18 +++++- .../dasLLAMA/dasllama/dasllama_gemm_gen.das | 59 +++++++++++++++---- .../dasllama/dasllama_math_default.das | 7 ++- .../dasLLAMA/dasllama/dasllama_math_gen.das | 5 +- modules/dasLLAMA/dasllama/dasllama_repack.das | 11 ++-- modules/dasLLAMA/harness/gen_tune_probe.das | 3 + 6 files changed, 78 insertions(+), 25 deletions(-) diff --git a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md index 846312bb6e..b25b4bc969 100644 --- a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md +++ b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md @@ -453,8 +453,22 @@ over 16 groups), k2_grp_row_dot, repack_k2_grp (16 four-byte columns x mr; scale --tensor-type override needed) decodes coherently at gen 30 t/s reference bodies, greedy ids 19/64 with the fork a 0.153-logit near-tie (top2 IS our token; the lossiest format diverges earliest). No whole-scope re-mint fired on the first e2e (zero "@tune begin" -lines), unlike QUIRK 17's precedent - watch at Phase B. JIT emitter, Vulkan, Metal: -pending. +lines), unlike QUIRK 17's precedent - resolved at Phase B: a declined-stub family demands +no sidecar entry; the mint fires once real generators exist. + +Phase B (JIT emitter, 2026-08-31): a fourth arm through emit_block_kqv2 - the k2 flag +swaps the quant compose to k3's column walk minus the hmask (2-bit lane at shift +2*(blk%4)), the scale reads to per-16 nibble extracts (ZExt the pair bytes to i32 lanes, +then &15 / >>4 - no i8-width splat needed; scv/mnv carry the two groups' sc, mnlo/mnhi +their mins), keeps the k63 split lo/hi accumulators behind a per16 flag +(vecBsums/fuseAcc/madd16/flush/bs0-bs1), and takes the k4/k5 two-fma epilogue (bacc folds +on dmin). Prerequisite refactor: the grp scale header went FIELD-MAJOR +([16 sc x mr][mr x f16 d][mr x f16 dmin]) so load_f16_vec_at serves the d/dmin vectors - +repack_k2_grp and both grp readers moved together, re-gated bit-exact. Gates: probe test +mode 11/11 k2 perms (maddubs mr8 stamps at 4.8e-7; vpdpbusd declines on zen2 as every +format), test_kquant -jit 212/216. Crowned dot_maddubs_width256_mr8; stamped e2e gen +30 -> 75 t/s, ids 19/64 with the same token-19 near-tie. zen2 16t vs llama.cpp clean-cpu +b10660: pp512 418.6 / 412.0 (1.02x), tg128 77.1 / 81.1 (0.95x). Vulkan, Metal: pending. ### IQ4_NL (the near-free one, 2026-08-30) diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das index 1d6c0e705a..80ccfe3b5f 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das @@ -942,10 +942,12 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f let w8 = te.width / 8 let k6 = te.kq == 6 let k3 = te.kq == 3 + let k2 = te.kq == 2 let k63 = k6 || k3 // k3 rides k6's scale row and fold; only the quant compose and the offset differ + let per16 = k63 || k2 // per-16 scale granularity: split lo/hi accumulators, unsummed bs0/bs1 let koff = k3 ? 4 : 32 // the signed-lattice offset the bsum fold removes let q40 = te.kq == 40 - let qsb = (te.kq == 4 || q40) ? 128 : (k3 ? 96 : (te.kqBytes ? 256 : (te.kq == 5 ? 160 : 192))) + let qsb = (te.kq == 4 || q40) ? 128 : (k3 ? 96 : (k2 ? 64 : (te.kqBytes ? 256 : (te.kq == 5 ? 160 : 192)))) var wb = LLVMBuildMul(b, sbi, te.types->ConstI64(uint64(mr * qsb)), "wb") var sb = LLVMBuildMul(b, sbi, te.types->ConstI64(uint64(mr * (k63 ? 18 : (q40 ? 16 : 20)))), "sb") var xb = LLVMBuildMul(b, sbi, te.types->ConstI64(0x100ul), "xb") @@ -963,8 +965,11 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f var mv : LLVMOpaqueValue? [2] if (!q40) { for (qd in range(rq)) { - if (k63) { + if (k63 || k2) { dv[qd] = load_f16_vec_at(te, te.sg, LLVMBuildAdd(b, sb, te.types->ConstI64(uint64(16 * mr + 2 * (qd * te.rv))), ""), "d{qd}") + if (k2) { + mv[qd] = load_f16_vec_at(te, te.sg, LLVMBuildAdd(b, sb, te.types->ConstI64(uint64(18 * mr + 2 * (qd * te.rv))), ""), "dm{qd}") + } } else { dv[qd] = load_f16_vec_at(te, te.sg, LLVMBuildAdd(b, sb, te.types->ConstI64(uint64(2 * (qd * te.rv))), ""), "d{qd}") mv[qd] = load_f16_vec_at(te, te.sg, LLVMBuildAdd(b, sb, te.types->ConstI64(uint64(2 * mr + 2 * (qd * te.rv))), ""), "dm{qd}") @@ -980,9 +985,9 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f facc[i] = LLVMConstNull(te.vnf32) } // vector bsums: sdot leg only (x64's vpbroadcastd makes scalars free; k6 needs bs0/bs1 unsummed) - let vecBsums = !memBcast && !k63 + let vecBsums = !memBcast && !per16 // mr>=8 fuses lo+hi into ONE acc (i32-exact — the fold adds them anyway); mr4 keeps split accs for ILP - let fuseAcc = !memBcast && mr >= 8 && !k63 + let fuseAcc = !memBcast && mr >= 8 && !per16 var a32v8 : LLVMOpaqueValue? [8] // [tok * 2 + blk / 4], lane blk % 4 if (vecBsums) { var xb16 = LLVMBuildShl(b, sbi, te.types->ConstI64(4ul), "xb16") @@ -1041,6 +1046,14 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f var hib = LLVMBuildShl(b, LLVMBuildAnd(b, LLVMBuildLShr(b, h3h, splat_i8w(te, blk), ""), splat_i8w(te, 1), ""), splat_i8w(te, 2), "") wlo = LLVMBuildOr(b, lo2, lob, "wlo3{blk}_{j * rq + qd}") whi = LLVMBuildOr(b, hi2, hib, "whi3{blk}_{j * rq + qd}") + } elif (k2) { + // k2: qs column (blk/4)*8 + j (lo 16) / + 4 (hi 16), 2-bit lane at shift 2*(blk%4) - k3's walk minus the hmask + let c0 = (blk / 4) * 8 + j + let sh = (blk % 4) * 2 + var q2l = load_vec(te, te.vwi8, te.wg, LLVMBuildAdd(b, wb, te.types->ConstI64(uint64(c0 * 4 * mr + qd * w8)), ""), "q2l{blk}_{j * rq + qd}") + var q2h = load_vec(te, te.vwi8, te.wg, LLVMBuildAdd(b, wb, te.types->ConstI64(uint64((c0 + 4) * 4 * mr + qd * w8)), ""), "q2h{blk}_{j * rq + qd}") + wlo = LLVMBuildAnd(b, LLVMBuildLShr(b, q2l, splat_i8w(te, sh), ""), splat_i8w(te, 3), "wlo2{blk}_{j * rq + qd}") + whi = LLVMBuildAnd(b, LLVMBuildLShr(b, q2h, splat_i8w(te, sh), ""), splat_i8w(te, 3), "whi2{blk}_{j * rq + qd}") } elif (te.kqBytes) { var hoff2 = LLVMBuildAdd(b, noff, te.types->ConstI64(uint64(128 * mr)), "") wlo = load_vec(te, te.vwi8, te.wg, noff, "wlo{blk}_{j * rq + qd}") @@ -1075,7 +1088,7 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f var xhi = bcast_dword(te, te.x[tokBase + i], doff1, "xd{tokBase + i}_{blk}_{j}b") if (madd16) { let k = i * rq + qd - if (k63) { + if (per16) { p16lo[k] = madd16_acc(te, p16lo[k], wlo, xlo) p16hi[k] = madd16_acc(te, p16hi[k], whi, xhi) } else { @@ -1102,7 +1115,7 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f for (k in range(tokCount * rq)) { a0[k] = madd16_flush(te, a0[k], p16lo[k]) p16lo[k] = null - if (k63) { + if (per16) { a1[k] = madd16_flush(te, a1[k], p16hi[k]) p16hi[k] = null } @@ -1118,6 +1131,8 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f // the integer fold — no float ops per block, except q40 (its per-32 f16 d admits no cross-block integer fold) var scv : LLVMOpaqueValue? [2] var mnv : LLVMOpaqueValue? [2] + var mnlo : LLVMOpaqueValue? [2] // k2: the two groups' min nibbles (scv/mnv carry their sc nibbles) + var mnhi : LLVMOpaqueValue? [2] var dvb : LLVMOpaqueValue? [2] for (qd in range(rq)) { if (q40) { @@ -1127,6 +1142,16 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f var sc1p = LLVMBuildGEP2(b, te.types.t_int8, te.sg, LLVMBuildAdd(b, sb, te.types->ConstI64(uint64((2 * blk + 1) * mr + qd * te.rv)), ""), "scp{blk}_{qd}b") scv[qd] = LLVMBuildSExt(b, LLVMBuildLoad2Aligned(b, vri8, sc0p, 1u, ""), te.vni32, "sc{blk}_{qd}a") mnv[qd] = LLVMBuildSExt(b, LLVMBuildLoad2Aligned(b, vri8, sc1p, 1u, ""), te.vni32, "sc{blk}_{qd}b") + } elif (k2) { + // per-16 nibble pairs: byte 2blk = the lo group's sc|mn<<4, byte 2blk+1 = the hi group's + var b0p = LLVMBuildGEP2(b, te.types.t_int8, te.sg, LLVMBuildAdd(b, sb, te.types->ConstI64(uint64(2 * blk * mr + qd * te.rv)), ""), "k2p{blk}_{qd}a") + var b1p = LLVMBuildGEP2(b, te.types.t_int8, te.sg, LLVMBuildAdd(b, sb, te.types->ConstI64(uint64((2 * blk + 1) * mr + qd * te.rv)), ""), "k2p{blk}_{qd}b") + var z0 = LLVMBuildZExt(b, LLVMBuildLoad2Aligned(b, vri8, b0p, 1u, ""), te.vni32, "k2z{blk}_{qd}a") + var z1 = LLVMBuildZExt(b, LLVMBuildLoad2Aligned(b, vri8, b1p, 1u, ""), te.vni32, "k2z{blk}_{qd}b") + scv[qd] = LLVMBuildAnd(b, z0, splat_i32n(te, 15), "sc2{blk}_{qd}a") + mnv[qd] = LLVMBuildAnd(b, z1, splat_i32n(te, 15), "sc2{blk}_{qd}b") + mnlo[qd] = LLVMBuildLShr(b, z0, splat_i32n(te, 4), "mn2{blk}_{qd}a") + mnhi[qd] = LLVMBuildLShr(b, z1, splat_i32n(te, 4), "mn2{blk}_{qd}b") } else { var scp = LLVMBuildGEP2(b, te.types.t_int8, te.sg, LLVMBuildAdd(b, sb, te.types->ConstI64(uint64(4 * mr + blk * mr + qd * te.rv)), ""), "scp{blk}_{qd}") scv[qd] = LLVMBuildZExt(b, LLVMBuildLoad2Aligned(b, vri8, scp, 1u, ""), te.vni32, "sc{blk}_{qd}") @@ -1151,7 +1176,7 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f bs0 = LLVMBuildLoad2Aligned(b, te.types.t_int32, bp0, 4u, "bs{tk}_{blk}a") var bp1 = LLVMBuildGEP2(b, te.types.t_int32, te.xbs[tk], bi2b, "bp{tk}_{blk}b") bs1 = LLVMBuildLoad2Aligned(b, te.types.t_int32, bp1, 4u, "bs{tk}_{blk}b") - if (!k63) { + if (!per16) { a32v = splat_i32(te, LLVMBuildAdd(b, bs0, bs1, "a32{tk}_{blk}"), "a32v{tk}_{blk}") } } @@ -1170,6 +1195,16 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f iacc[i * rq + qd] = LLVMBuildAdd(b, iacc[i * rq + qd], ia, "ia{tk}_{blk}_{qd}") bacc[i * rq + qd] = LLVMBuildAdd(b, bacc[i * rq + qd], ba, "ba{tk}_{blk}_{qd}") } + } elif (k2) { + // ia = sc_lo*dot_lo + sc_hi*dot_hi; ba = mn_lo*bs0 + mn_hi*bs1 - the k4/k5 epilogue folds bacc on dmin + var bs0v = splat_i32(te, bs0, "bs0v{tk}_{blk}") + var bs1v = splat_i32(te, bs1, "bs1v{tk}_{blk}") + for (qd in range(rq)) { + var ia = LLVMBuildAdd(b, LLVMBuildMul(b, scv[qd], a0[i * rq + qd], ""), LLVMBuildMul(b, mnv[qd], a1[i * rq + qd], ""), "ia2{tk}_{blk}_{qd}") + var ba = LLVMBuildAdd(b, LLVMBuildMul(b, mnlo[qd], bs0v, ""), LLVMBuildMul(b, mnhi[qd], bs1v, ""), "ba2{tk}_{blk}_{qd}") + iacc[i * rq + qd] = LLVMBuildAdd(b, iacc[i * rq + qd], ia, "ia{tk}_{blk}_{qd}k2") + bacc[i * rq + qd] = LLVMBuildAdd(b, bacc[i * rq + qd], ba, "ba{tk}_{blk}_{qd}k2") + } } else { for (qd in range(rq)) { var acc = fuseAcc ? a0[i * rq + qd] : LLVMBuildAdd(b, a0[i * rq + qd], a1[i * rq + qd], "acc{tk}_{blk}_{qd}") @@ -2867,12 +2902,10 @@ def private iq3xxs_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, def private iq4nl_gemv_gen(var gc : LlvmCodeCtx) : bool => kq_gemv_gen_impl(gc, 45) def private iq4nl_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 45) -// k2: the 2-bit + min-fold emitter arm lands with Phase B; until then both stamps decline -// (reference bodies serve) - a declined generator is the framework's own fallback path. -[unused_argument(gc)] -def private k2_gemv_gen(var gc : LlvmCodeCtx) : bool => false -[unused_argument(gc)] -def private k2_tile_gen(var gc : LlvmCodeCtx) : bool => false +// k2 rides the kq walks through emit_block_kqv2's per-16 arms (the k2 flag swaps the 2-bit +// compose, the nibble scale extracts, and the epilogue to k4/k5's dmin fold) +def private k2_gemv_gen(var gc : LlvmCodeCtx) : bool => kq_gemv_gen_impl(gc, 2) +def private k2_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 2) def private k4_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 4) def private k5_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 5) diff --git a/modules/dasLLAMA/dasllama/dasllama_math_default.das b/modules/dasLLAMA/dasllama/dasllama_math_default.das index 632537abb1..baa1f2230e 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_default.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_default.das @@ -1024,15 +1024,16 @@ def private dequant_iq4nl_row_grp(kqg : uint8 const?; ksg : uint8 const?; r, mr, } //! k2 grp row dequant: the 2-bit lanes stay in place as 16 four-byte columns x mr; scale row = -//! [16 sc/min bytes interleaved x mr][mr x 4B d+dmin header]. w = (d*sc)*q - (dmin*mn). +//! [16 sc/min bytes interleaved x mr][mr x f16 d][mr x f16 dmin]. w = (d*sc)*q - (dmin*mn). def private dequant_k2_row_grp(kqg : uint8 const?; ksg : uint8 const?; r, mr, nsb : int64; var dst : float?) { unsafe { for (sbi in range64(nsb)) { let qb = sbi * 64l * mr let sb = sbi * 20l * mr - let hd = sb + 16l * mr + 4l * r + let hd = sb + 16l * mr + 2l * r let d = f16_to_f32(uint(ksg[hd]) | (uint(ksg[hd + 1l]) << 8u)) - let dmin = f16_to_f32(uint(ksg[hd + 2l]) | (uint(ksg[hd + 3l]) << 8u)) + let hm = sb + 18l * mr + 2l * r + let dmin = f16_to_f32(uint(ksg[hm]) | (uint(ksg[hm + 1l]) << 8u)) for (g in range64(16l)) { let scb = int(ksg[sb + g * mr + r]) let dl = d * float(scb & 15) diff --git a/modules/dasLLAMA/dasllama/dasllama_math_gen.das b/modules/dasLLAMA/dasllama/dasllama_math_gen.das index c6135c0aec..a72a2464de 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_gen.das @@ -848,9 +848,10 @@ def k2_grp_row_dot(kqg : uint8 const?; ksg : uint8 const?; r, mr : int64; xqp : isum += (scb & 15) * ig bsum += (scb >> 4) * xbsp[sbi * 16l + g] } - let hd = sb + 16l * mr + 4l * r + let hd = sb + 16l * mr + 2l * r let dk = f16_to_f32(uint(ksg[hd]) | (uint(ksg[hd + 1l]) << 8u)) - let dm = f16_to_f32(uint(ksg[hd + 2l]) | (uint(ksg[hd + 3l]) << 8u)) + let hm = sb + 18l * mr + 2l * r + let dm = f16_to_f32(uint(ksg[hm]) | (uint(ksg[hm + 1l]) << 8u)) acc += float(isum) * (dk * xsp[sbi]) acc -= float(bsum) * (dm * xsp[sbi]) } diff --git a/modules/dasLLAMA/dasllama/dasllama_repack.das b/modules/dasLLAMA/dasllama/dasllama_repack.das index ae4823a294..bcdce451f0 100644 --- a/modules/dasLLAMA/dasllama/dasllama_repack.das +++ b/modules/dasLLAMA/dasllama/dasllama_repack.das @@ -512,8 +512,8 @@ def repack_iq3s_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { } //! k2 grp layout: the qs bytes as 16 four-byte columns x mr rows (the 2-bit lanes stay in -//! place - the kernels shift per group); scale row interleaves as [16 sc/min bytes x mr] -//! [mr x 4B d+dmin header]. +//! place - the kernels shift per group); scale row interleaves FIELD-MAJOR as +//! [16 sc/min bytes x mr][mr x f16 d][mr x f16 dmin] (contiguous d rows for the emitter's f16 vector loads). def repack_k2_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { let nsb = n / 256l let qrow = nsb * 64l @@ -543,9 +543,10 @@ def repack_k2_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { for (idx in range64(16l)) { ks[ds + idx * mr + r] = tsp[ss + 4l + idx] } - for (h in range64(4l)) { - ks[ds + 16l * mr + 4l * r + h] = tsp[ss + h] - } + ks[ds + 16l * mr + 2l * r] = tsp[ss] + ks[ds + 16l * mr + 2l * r + 1l] = tsp[ss + 1l] + ks[ds + 18l * mr + 2l * r] = tsp[ss + 2l] + ks[ds + 18l * mr + 2l * r + 1l] = tsp[ss + 3l] } } } diff --git a/modules/dasLLAMA/harness/gen_tune_probe.das b/modules/dasLLAMA/harness/gen_tune_probe.das index db0348dfc7..172bb7c532 100644 --- a/modules/dasLLAMA/harness/gen_tune_probe.das +++ b/modules/dasLLAMA/harness/gen_tune_probe.das @@ -1343,6 +1343,7 @@ def test_mode : bool { // nolint:STYLE037,STYLE038 — flat per-variant correctn var kfxs33 <- [ <- build_kq_fixture(33l, 256l, 32l, 8l), <- build_kq_fixture(33l, 768l, 32l, 8l), <- build_kq_fixture(33l, 2048l, 32l, 8l)] var kfxs34 <- [ <- build_kq_fixture(34l, 256l, 32l, 8l), <- build_kq_fixture(34l, 768l, 32l, 8l), <- build_kq_fixture(34l, 2048l, 32l, 8l)] var kfxs45 <- [ <- build_kq_fixture(45l, 256l, 32l, 8l), <- build_kq_fixture(45l, 768l, 32l, 8l), <- build_kq_fixture(45l, 2048l, 32l, 8l)] + var kfxs2 <- [ <- build_kq_fixture(2l, 256l, 32l, 8l), <- build_kq_fixture(2l, 768l, 32l, 8l), <- build_kq_fixture(2l, 2048l, 32l, 8l)] var vs <- q8q8_tile_gen_variants() var mrs <- variant_mrs() var wbs <- variant_wbias() @@ -1472,6 +1473,7 @@ def test_mode : bool { // nolint:STYLE037,STYLE038 — flat per-variant correctn allok = kq_test_family(33l, kfxs33) && allok allok = kq_test_family(34l, kfxs34) && allok allok = kq_test_family(45l, kfxs45) && allok + allok = kq_test_family(2l, kfxs2) && allok delete wtab delete mtab delete gtab @@ -1492,6 +1494,7 @@ def test_mode : bool { // nolint:STYLE037,STYLE038 — flat per-variant correctn delete kfxs33 delete kfxs34 delete kfxs45 + delete kfxs2 return allok } From a82599eb9d562e0d6d651b19b04f42bb80f700eb Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Mon, 31 Aug 2026 02:08:20 -0700 Subject: [PATCH 32/61] dasLLAMA: Q2_K on the Vulkan tier - pair-byte nibble scales on the k3/k6 shells, tg above parity KqGemvK2 rides k3's dot minus the hmask (its int4 already carries the two half sums) with the pair-byte nibble scales and the k4 d|dmin fold: dm.x*(sc_lo*ilo + sc_hi*ihi) - dm.y*(mn_lo*blo + mn_hi*bhi). KqBatchK2 : KqBatchK6 stages the unsigned 2-bit lanes (k3's staging minus the hmask), FOUR scale planes (d*sc and dmin*mn per 16-half - two new workgroup arrays), and a split-half fma folding the min sides on the half sums. K2Cm2T = VkK2Blk (int16[32] lanes) with K4Cm2T's srow walk over the pair bytes; three cm2 stamps. The device scale row is the CPU 20B row VERBATIM - both gather paths copy through (the grouped path re-packs the field-major grp header); vk_kq_schema_id 11 -> 2; the family cells grow to ten formats (stride 16 words); k2f16_gemm_oracle + the cm2 cell + the kq_cls_ref arm land with them. TRAP replayed from k3: a vk class child must FOLLOW its parent in the file - KqBatchK2 first landed above KqBatchK6 and died at "parent structure not found". Gates on the 5060 Ti: the suite 80/80 - the three k2 cm2 tiles 0-off (89600 cells each), the ten-format gemv/batch families green. The resident driver arms on the local vehicle (fresh image bake) and runs gen 284 t/s; greedy ids 19/64 vs simple_ids with the same 0.153-logit token-19 near-tie as the CPU tiers. Rows vs llama.cpp b10660 build-vulkan: pp512 14544.0 / 16752.7 (0.87x - above the 0.69-0.78 tier class), tg128 442.8 / 424.1 (1.04x). Metal: pending. Co-Authored-By: Claude Fable 5 --- modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md | 17 +- modules/dasLLAMA/dasllama/dasllama_layout.das | 16 +- .../dasllama/dasllama_vulkan_classes.das | 186 +++++++++++++++++- .../dasllama/dasllama_vulkan_common.das | 7 +- .../dasllama/dasllama_vulkan_prefill.das | 17 ++ modules/dasLLAMA/tests/_vkd_oracles.das | 34 ++++ .../dasLLAMA/tests/test_vulkan_kernels.das | 143 +++++++++++++- 7 files changed, 404 insertions(+), 16 deletions(-) diff --git a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md index b25b4bc969..4e2f724006 100644 --- a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md +++ b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md @@ -468,7 +468,22 @@ repack_k2_grp and both grp readers moved together, re-gated bit-exact. Gates: pr mode 11/11 k2 perms (maddubs mr8 stamps at 4.8e-7; vpdpbusd declines on zen2 as every format), test_kquant -jit 212/216. Crowned dot_maddubs_width256_mr8; stamped e2e gen 30 -> 75 t/s, ids 19/64 with the same token-19 near-tie. zen2 16t vs llama.cpp clean-cpu -b10660: pp512 418.6 / 412.0 (1.02x), tg128 77.1 / 81.1 (0.95x). Vulkan, Metal: pending. +b10660: pp512 418.6 / 412.0 (1.02x), tg128 77.1 / 81.1 (0.95x). + +Phase C (Vulkan, 2026-08-31): KqGemvK2 = k3's dot minus the hmask (its int4 (ilo, ihi, +blo, bhi) already carries the half sums) folded by the pair-byte nibble scales and the k4 +d|dmin word: dm.x*(sc_lo*ilo + sc_hi*ihi) - dm.y*(mn_lo*blo + mn_hi*bhi). KqBatchK2 : +KqBatchK6 stages the unsigned lanes (k3's staging minus the hmask), FOUR scale planes +(d*sc and dmin*mn per 16-half - two new @workgroup arrays), and a split-half fma folding +the min sides on the half sums. K2Cm2T = VkK2Blk (int16[32]) with K4Cm2T's srow walk over +the pair bytes. The device scale row is the CPU 20B row VERBATIM - both gather paths copy +through (the grouped path re-packs the field-major grp header). vk_kq_schema_id 11 -> 2; +ten-format family cells at stride 16 words. TRAP replayed from k3: a vk class child must +FOLLOW its parent in the file (KqBatchK2 first landed above KqBatchK6 - "parent structure +not found"). Gates: the suite 80/80, the three k2 cm2 tiles 0-off; resident e2e armed +(fresh bake), gen 284 t/s, ids 19/64 with the same token-19 near-tie. Rows (5060 Ti vs +llama.cpp b10660 build-vulkan): pp512 14544.0 / 16752.7 (0.87x - above the 0.69-0.78 tier +class), tg128 442.8 / 424.1 (1.04x). Metal: pending. ### IQ4_NL (the near-free one, 2026-08-30) diff --git a/modules/dasLLAMA/dasllama/dasllama_layout.das b/modules/dasLLAMA/dasllama/dasllama_layout.das index d666577656..00db3c70a6 100644 --- a/modules/dasLLAMA/dasllama/dasllama_layout.das +++ b/modules/dasLLAMA/dasllama/dasllama_layout.das @@ -881,6 +881,10 @@ def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice for (m in range64(32l)) { dq[64l + m] = gq[64l * mr + ((m / 4l) * mr + r) * 4l + m % 4l] } + } elif (fmt == KqFmt.k2) { // 16 qs columns, nothing else + for (m in range64(64l)) { + dq[m] = gq[((m / 4l) * mr + r) * 4l + m % 4l] + } } else { for (m in range64(128l)) { // nibble region: verbatim per (blk, j, t) dq[m] = gq[((m / 4l) * mr + r) * 4l + m % 4l] @@ -911,6 +915,14 @@ def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice for (idx in range64(16l, 20l)) { dsc[idx] = uint8(0) } + } elif (fmt == KqFmt.k2) { // the field-major grp header re-packs to [d][dmin][16 pair bytes] + dsc[0] = gs[16l * mr + 2l * r] + dsc[1] = gs[16l * mr + 2l * r + 1l] + dsc[2] = gs[18l * mr + 2l * r] + dsc[3] = gs[18l * mr + 2l * r + 1l] + for (idx in range64(16l)) { + dsc[4l + idx] = gs[idx * mr + r] + } } else { dsc[0] = gs[2l * r] dsc[1] = gs[2l * r + 1l] @@ -924,7 +936,7 @@ def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice } else { let rq = qp + sliceQ + ri * qrow + sbi * qsb let rs = sp + sliceS + ri * srow + sbi * dssb - if (fmt == KqFmt.q40 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s || fmt == KqFmt.iq3xxs || fmt == KqFmt.iq4nl) { + if (fmt == KqFmt.q40 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s || fmt == KqFmt.iq3xxs || fmt == KqFmt.iq4nl || fmt == KqFmt.k2) { for (m in range64(qsb)) { // disk bytes already in the device form (k/k+16 pairs; k3's lanes) dq[m] = rq[m] } @@ -951,7 +963,7 @@ def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice dq[128l + p] = rq[128l + p] } } - if (fmt == KqFmt.k6 || fmt == KqFmt.q40 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s || fmt == KqFmt.iq3xxs || fmt == KqFmt.iq4nl) { // rows already in the decoded device form + if (fmt == KqFmt.k6 || fmt == KqFmt.q40 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s || fmt == KqFmt.iq3xxs || fmt == KqFmt.iq4nl || fmt == KqFmt.k2) { // rows already in the decoded device form for (idx in range64(dssb)) { dsc[idx] = rs[idx] } diff --git a/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das b/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das index c733442996..66ccdd25fb 100644 --- a/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das +++ b/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das @@ -3176,6 +3176,90 @@ class KqBatchK3 : KqBatchK6 { } } +// Q2_K tile: k3's staging minus the hmask (unsigned 0..3 lanes), FOUR scale planes - d*sc and +// dmin*mn per 16-half - and a split-half fma folding the min sides against the half sums. +[vk_dispatch(name = "kq_batch_k2_cls", kernel = "run", family = "kq_batch_cls", grid = "wgs", params = "wgs : int64")] +class KqBatchK2 : KqBatchK6 { + @workgroup wsw3 : float[256] // dmin*mn, lo half + @workgroup wsw4 : float[256] // dmin*mn, hi half + + def override stage_w(wsb0, wcol, nsb, s, word, grp : uint) { + for (ki in range(2)) { + let k = word + uint(ki) * 8u + let bu = k / 2u + let hh = k % 2u + var wv = uint4(0u) + if (wcol < pa.d) { + let wsb = wsb0 + wcol * nsb + s + let qb4 = wsb * 16u + (bu / 4u) * 8u + hh * 4u + let shift = (bu % 4u) * 2u + wv = uint4((wq[qb4] >> shift) & 0x03030303, (wq[qb4 + 1u] >> shift) & 0x03030303, + (wq[qb4 + 2u] >> shift) & 0x03030303, (wq[qb4 + 3u] >> shift) & 0x03030303) + } + ww4[grp * 17u + k] = wv + } + } + + def override stage_ws(wsb0, wt, nsb, s, tid : uint) { + let slane = tid % 32u + let sblk = tid / 32u + var sa = 0.0 + var sbv = 0.0 + var ma = 0.0 + var mb = 0.0 + if (wt * 32u + slane < pa.d) { + let wsb = wsb0 + (wt * 32u + slane) * nsb + s + let w = wsu[wsb * 5u + 1u + sblk / 2u] + let b0 = (w >> ((sblk % 2u) * 16u)) & 255u + let b1 = (w >> ((sblk % 2u) * 16u + 8u)) & 255u + let dm = unpackHalf2x16(wsu[wsb * 5u]) + sa = dm.x * float(b0 & 15u) + sbv = dm.x * float(b1 & 15u) + ma = dm.y * float(b0 >> 4u) + mb = dm.y * float(b1 >> 4u) + } + wsw[sblk * 32u + slane] = sa + wsw2[sblk * 32u + slane] = sbv + wsw3[sblk * 32u + slane] = ma + wsw4[sblk * 32u + slane] = mb + } + + // the split-half fma with the min planes: xscl * (sc_lo*l + sc_hi*h - mn_lo*blo - mn_hi*bhi) + def override blk_fma(bu, grp, word : uint; xscl : float) : float4 { + let kL = bu * 2u + let kH = bu * 2u + 1u + let xwL = xw4[grp * 17u + kL] + let xwH = xw4[grp * 17u + kH] + let blo = sdot4(xwL.x, 0x01010101) + sdot4(xwL.y, 0x01010101) + sdot4(xwL.z, 0x01010101) + sdot4(xwL.w, 0x01010101) + let bhi = sdot4(xwH.x, 0x01010101) + sdot4(xwH.y, 0x01010101) + sdot4(xwH.z, 0x01010101) + sdot4(xwH.w, 0x01010101) + let wL0 = ww4[word * 17u + kL] + let wL1 = ww4[(word + 8u) * 17u + kL] + let wL2 = ww4[(word + 16u) * 17u + kL] + let wL3 = ww4[(word + 24u) * 17u + kL] + let wH0 = ww4[word * 17u + kH] + let wH1 = ww4[(word + 8u) * 17u + kH] + let wH2 = ww4[(word + 16u) * 17u + kH] + let wH3 = ww4[(word + 24u) * 17u + kH] + let l0 = sdot4(wL0.x, xwL.x) + sdot4(wL0.y, xwL.y) + sdot4(wL0.z, xwL.z) + sdot4(wL0.w, xwL.w) + let l1 = sdot4(wL1.x, xwL.x) + sdot4(wL1.y, xwL.y) + sdot4(wL1.z, xwL.z) + sdot4(wL1.w, xwL.w) + let l2 = sdot4(wL2.x, xwL.x) + sdot4(wL2.y, xwL.y) + sdot4(wL2.z, xwL.z) + sdot4(wL2.w, xwL.w) + let l3 = sdot4(wL3.x, xwL.x) + sdot4(wL3.y, xwL.y) + sdot4(wL3.z, xwL.z) + sdot4(wL3.w, xwL.w) + let h0 = sdot4(wH0.x, xwH.x) + sdot4(wH0.y, xwH.y) + sdot4(wH0.z, xwH.z) + sdot4(wH0.w, xwH.w) + let h1 = sdot4(wH1.x, xwH.x) + sdot4(wH1.y, xwH.y) + sdot4(wH1.z, xwH.z) + sdot4(wH1.w, xwH.w) + let h2 = sdot4(wH2.x, xwH.x) + sdot4(wH2.y, xwH.y) + sdot4(wH2.z, xwH.z) + sdot4(wH2.w, xwH.w) + let h3 = sdot4(wH3.x, xwH.x) + sdot4(wH3.y, xwH.y) + sdot4(wH3.z, xwH.z) + sdot4(wH3.w, xwH.w) + return float4(xscl * (wsw[bu * 32u + word] * float(l0) + wsw2[bu * 32u + word] * float(h0) - wsw3[bu * 32u + word] * float(blo) - wsw4[bu * 32u + word] * float(bhi)), + xscl * (wsw[bu * 32u + word + 8u] * float(l1) + wsw2[bu * 32u + word + 8u] * float(h1) - wsw3[bu * 32u + word + 8u] * float(blo) - wsw4[bu * 32u + word + 8u] * float(bhi)), + xscl * (wsw[bu * 32u + word + 16u] * float(l2) + wsw2[bu * 32u + word + 16u] * float(h2) - wsw3[bu * 32u + word + 16u] * float(blo) - wsw4[bu * 32u + word + 16u] * float(bhi)), + xscl * (wsw[bu * 32u + word + 24u] * float(l3) + wsw2[bu * 32u + word + 24u] * float(h3) - wsw3[bu * 32u + word + 24u] * float(blo) - wsw4[bu * 32u + word + 24u] * float(bhi))) + } + + [spirv_kernel(local_size_x = 256, name = "kq_batch_k2_cls_spv")] + def override run { + tile_shell() + } +} + // ===== the prefill batch GEMMs — cooperative-matrix variants (DASLLAMA_COOPMAT=f16|int8|mm) ===== let private CML_SST = 20u // mm shared row stride in uints (16 f16-pair data + 4 pad — bank spread) @@ -3992,6 +4076,10 @@ struct VkK3Blk { hm : int16[16] // ... and its 32 hmask bytes; scales (16 int8 (sc-32) + f16 d) live in ws } +struct VkK2Blk { + qs : int16[32] // one Q2_K superblock's 64 qs bytes, four 2-bit lanes each; scales ([f16 d][f16 dmin][16 sc/min pair bytes]) live in ws +} + // The cm2 tile: one class template over the two stamp axes. The token column BN - l 256, m 128, // s 32 (the expert-bucket geometry: STILE's fast path loads a partial column UNCLAMPED and clamps // only the store, so every plane it reads carries 32 rows of slack) - and the weight format: BLK @@ -4331,6 +4419,26 @@ class template K3Cm2T : KqCm2BatchT { } } +[ |> template_struct_instance] +class template K2Cm2T : KqCm2BatchT { + typedef BLK = VkK2Blk + typedef ST = uint + + [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] + def decode(blk : VkK2Blk; bc, cib : uint2) : float16 { + let e = cib.y + let g = e >> 4u + let l = e & 15u + let qi = (g >> 3u) * 32u + (g & 1u) * 16u + l + let qby = uint(int(unpack8(blk.qs[int(qi >> 1u)])[int(qi & 1u)])) & 0xFFu + let q = (qby >> (((g >> 1u) & 3u) * 2u)) & 3u + let srow = (wg_blk0 + bc.x * (pa.n >> 8u) + bc.y) * 5u + let dm = unpackHalf2x16(ws[srow]) + let pb = (ws[srow + 1u + (g >> 2u)] >> ((g & 3u) * 8u)) & 0xFFu + return float16(dm.x * float(pb & 15u) * float(q) - dm.y * float(pb >> 4u)) + } +} + [ |> template_struct_instance] class template Iq3sCm2T : KqCm2BatchT { typedef BLK = VkIq3sBlk @@ -4681,6 +4789,33 @@ class Iq4nlCm2SBatch : Iq4nlCm2T { typedef FLO = tensorLayout2DPad } +[vk_dispatch(name = "kq_batch_k2_cm2l_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class K2Cm2LBatch : K2Cm2T { + typedef BT = coopmatWgB_f16_64x256 + typedef ACC = coopmatWgAcc_f16_128x256 + typedef ACCW = coopmatWgAcc_f32_128x256 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_k2_cm2m_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class K2Cm2MBatch : K2Cm2T { + override BN = 128u + typedef BT = coopmatWgB_f16_64x128 + typedef ACC = coopmatWgAcc_f16_128x128 + typedef ACCW = coopmatWgAcc_f32_128x128 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_k2_cm2s_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class K2Cm2SBatch : K2Cm2T { + override BN = 32u + override STILE = true + typedef BT = coopmatWgB_f16_64x32 + typedef ACC = coopmatWgAcc_f16_128x32 + typedef ACCW = coopmatWgAcc_f32_128x32 + typedef FLO = tensorLayout2DPad +} + // ===== the split-k reduce (sums the cm2 partial planes into y) ===== struct SkRedArgs { @@ -5256,6 +5391,8 @@ def kq_batch_cls_ensure(fmt : int; q40cm : bool) : bool { return ensure_kq_batch_iq3xxs_cls() } elif (fmt == int(KqFmt.iq4nl)) { return ensure_kq_batch_iq4nl_cls() + } elif (fmt == int(KqFmt.k2)) { + return ensure_kq_batch_k2_cls() } panic("dasLLAMA vulkan tier: no kq batch class for format {fmt}") return false @@ -5292,6 +5429,8 @@ def kq_batch_cls_enc_for(fmt : int; q40cm : bool; raw : VkCommandBuffer; var h : enc_kq_batch_iq3xxs_cls(raw, h, set_, pc, wgs) } elif (fmt == int(KqFmt.iq4nl)) { enc_kq_batch_iq4nl_cls(raw, h, set_, pc, wgs) + } elif (fmt == int(KqFmt.k2)) { + enc_kq_batch_k2_cls(raw, h, set_, pc, wgs) } else { panic("dasLLAMA vulkan tier: no kq batch class for format {fmt}") } @@ -5322,6 +5461,8 @@ def gemv_cls_ensure(fmt : int) : bool { return ensure_kq_gemv_iq3xxs_cls() } elif (fmt == int(KqFmt.iq4nl)) { return ensure_kq_gemv_iq4nl_cls() + } elif (fmt == int(KqFmt.k2)) { + return ensure_kq_gemv_k2_cls() } panic("dasLLAMA vulkan tier: no GEMV class for format {fmt}") return false @@ -5358,14 +5499,16 @@ def gemv_cls_enc(fmt : int; raw : VkCommandBuffer; var h : VkHaz; var set_ : VkD enc_kq_gemv_iq3xxs_cls(raw, h, set_, pc, wgs) } elif (fmt == int(KqFmt.iq4nl)) { enc_kq_gemv_iq4nl_cls(raw, h, set_, pc, wgs) + } elif (fmt == int(KqFmt.k2)) { + enc_kq_gemv_k2_cls(raw, h, set_, pc, wgs) } else { panic("dasLLAMA vulkan tier: no GEMV class for format {fmt}") } } // Q4_K: w = d*sc*q - dmin*mn*1, q in [0,15] — per 32-block axs*(d*sc*idot - dmin*mn*bsum). -// The nine formats share family=kq_gemv_cls: ONE set layout + ONE set_kq_gemv_cls serves all -// nine pipelines (identical inherited members), each leaf keeping its own ensure_/enc_ names. +// The ten formats share family=kq_gemv_cls: ONE set layout + ONE set_kq_gemv_cls serves all +// ten pipelines (identical inherited members), each leaf keeping its own ensure_/enc_ names. [vk_dispatch(name = "kq_gemv_k4_cls", kernel = "run", family = "kq_gemv_cls", grid = "wgs", params = "wgs : int64")] class KqGemvK4 : KqGemvBase { def override blk_contrib(wsb, blk, xsb : uint) : float { @@ -5600,6 +5743,45 @@ class KqGemvK3 : KqGemvBase { } } +// Q2_K: q = the unsigned 2-bit qs lane, per-16 sc/min NIBBLE pairs x d/dmin - the k4 d|dmin +// fold at k6's granularity over the 16-word rows (4 uint4 of qs; scale row [d|dmin][16 pair bytes]) +[vk_dispatch(name = "kq_gemv_k2_cls", kernel = "run", family = "kq_gemv_cls", grid = "wgs", params = "wgs : int64")] +class KqGemvK2 : KqGemvBase { + // (ilo, ihi, blo, bhi) - an int4, since a shader function cannot return a tuple + def k2_dot(ql, qh, xl, xh : uint4; sh : uint) : int4 { + let m2 = 0x03030303u + let l0 = (ql.x >> sh) & m2 + let l1 = (ql.y >> sh) & m2 + let l2 = (ql.z >> sh) & m2 + let l3 = (ql.w >> sh) & m2 + let h0 = (qh.x >> sh) & m2 + let h1 = (qh.y >> sh) & m2 + let h2 = (qh.z >> sh) & m2 + let h3 = (qh.w >> sh) & m2 + return int4(sdot4(l0, xl.x) + sdot4(l1, xl.y) + sdot4(l2, xl.z) + sdot4(l3, xl.w), + sdot4(h0, xh.x) + sdot4(h1, xh.y) + sdot4(h2, xh.z) + sdot4(h3, xh.w), + sdot4(xl.x, KQC_ONES) + sdot4(xl.y, KQC_ONES) + sdot4(xl.z, KQC_ONES) + sdot4(xl.w, KQC_ONES), + sdot4(xh.x, KQC_ONES) + sdot4(xh.y, KQC_ONES) + sdot4(xh.z, KQC_ONES) + sdot4(xh.w, KQC_ONES)) + } + + def override blk_contrib(wsb, blk, xsb : uint) : float { + let xb4 = xsb * 16u + blk * 2u + let q4 = wsb * 4u + (blk / 4u) * 2u + let dot = k2_dot(wq4[q4], wq4[q4 + 1u], xq4[xb4], xq4[xb4 + 1u], (blk % 4u) * 2u) + let w = wsu[wsb * 5u + 1u + blk / 2u] + let b0 = (w >> ((blk % 2u) * 16u)) & 255u + let b1 = (w >> ((blk % 2u) * 16u + 8u)) & 255u + let dm = unpackHalf2x16(wsu[wsb * 5u]) + return axs[xsb] * (dm.x * float(int(b0 & 15u) * dot.x + int(b1 & 15u) * dot.y) + - dm.y * float(int(b0 >> 4u) * dot.z + int(b1 >> 4u) * dot.w)) + } + + [spirv_kernel(local_size_x = 64, name = "kq_gemv_k2_cls_spv")] + def run { + gemv_shell() + } +} + // Q5_K: the k4 fold with the 5th bit deposited from the block's qh word [vk_dispatch(name = "kq_gemv_k5_cls", kernel = "run", family = "kq_gemv_cls", grid = "wgs", params = "wgs : int64")] class KqGemvK5 : KqGemvBase { diff --git a/modules/dasLLAMA/dasllama/dasllama_vulkan_common.das b/modules/dasLLAMA/dasllama/dasllama_vulkan_common.das index 22bd05f67d..d83b7f48b7 100644 --- a/modules/dasLLAMA/dasllama/dasllama_vulkan_common.das +++ b/modules/dasLLAMA/dasllama/dasllama_vulkan_common.das @@ -1709,7 +1709,7 @@ struct ArenaSlab { } struct ArenaFmt { - fmt : int // int(KqFmt): 0 = q8, 1/2/3 = k4/k5/k6, 4 = q40, 6 = iq4xs, 7 = k3, 8 = iq3s, 9 = iq3xxs, 10 = iq4nl + fmt : int // int(KqFmt): 0 = q8, 1/2/3 = k4/k5/k6, 4 = q40, 6 = iq4xs, 7 = k3, 8 = iq3s, 9 = iq3xxs, 10 = iq4nl, 11 = k2 slabs : array cur : int // slab the bump cursor is in slab_cap : int64 // max blocks one slab may carry (msr over both plane strides) @@ -1740,7 +1740,7 @@ def arena_local_blk(blk : int64) : int64 => blk & ((1l << 32l) - 1l) // per-block plane strides — the block is 32 weights for q8 and a 256-weight superblock for kq. // Mirrors stack_plane_bytes: it computes the same totals as count * these strides. -// the tier's stack fmt space is int(KqFmt) — bridge to the gemm_schema kq id space (4/5/6/40/44/3/33/34/45) +// the tier's stack fmt space is int(KqFmt) — bridge to the gemm_schema kq id space (4/5/6/40/44/3/33/34/45/2) def vk_kq_schema_id(fmt : int) : int { if (fmt == 1) return 4 if (fmt == 2) return 5 @@ -1751,11 +1751,12 @@ def vk_kq_schema_id(fmt : int) : int { if (fmt == 8) return 33 if (fmt == 9) return 34 if (fmt == 10) return 45 + if (fmt == 11) return 2 panic("vk_kq_schema_id: not a kq stack fmt") return 0 } -// fmt space here = int(KqFmt): 0 q8 (32-elem blocks), 1-4 and 6..10 the kq lattice (+ q40, iq4xs, k3, iq3s, iq3xxs, iq4nl) +// fmt space here = int(KqFmt): 0 q8 (32-elem blocks), 1-4 and 6..11 the kq lattice (+ q40, iq4xs, k3, iq3s, iq3xxs, iq4nl, k2) def vk_fmt_b32(fmt : int) : bool => fmt == 0 // meta/activation block unit of a stack format — q8 counts 32-weight blocks, the kq diff --git a/modules/dasLLAMA/dasllama/dasllama_vulkan_prefill.das b/modules/dasLLAMA/dasllama/dasllama_vulkan_prefill.das index 2a027a4549..d88d3df400 100644 --- a/modules/dasLLAMA/dasllama/dasllama_vulkan_prefill.das +++ b/modules/dasLLAMA/dasllama/dasllama_vulkan_prefill.das @@ -85,6 +85,9 @@ def private cm2_cls_ensure(fmt : int; tc : int64) : bool { if (fmt == int(KqFmt.iq4nl)) { return tc == 256l ? ensure_kq_batch_iq4nl_cm2l_cls() : (tc == 128l ? ensure_kq_batch_iq4nl_cm2m_cls() : ensure_kq_batch_iq4nl_cm2s_cls()) } + if (fmt == int(KqFmt.k2)) { + return tc == 256l ? ensure_kq_batch_k2_cm2l_cls() : (tc == 128l ? ensure_kq_batch_k2_cm2m_cls() : ensure_kq_batch_k2_cm2s_cls()) + } verify(fmt == int(KqFmt.q8), "vk prefill: the cm2 ensure fell through for a kq format - a format the feed admits must own a tile arm") return tc == 256l ? ensure_q8_batch_cm2l_cls() : (tc == 128l ? ensure_q8_batch_cm2m_cls() : ensure_q8_batch_cm2s_cls()) } @@ -145,6 +148,12 @@ def private cm2_cls_set(fmt : int; tc : int64; bufs : uint64 const[5]; sizes : i } return tc == 128l ? set_kq_batch_iq4nl_cm2m_cls(bufs, sizes, gbits) : set_kq_batch_iq4nl_cm2s_cls(bufs, sizes, gbits) } + if (fmt == int(KqFmt.k2)) { + if (tc == 256l) { + return set_kq_batch_k2_cm2l_cls(bufs, sizes, gbits) + } + return tc == 128l ? set_kq_batch_k2_cm2m_cls(bufs, sizes, gbits) : set_kq_batch_k2_cm2s_cls(bufs, sizes, gbits) + } verify(fmt == int(KqFmt.q8), "vk prefill: the cm2 set fell through for a kq format - a format the feed admits must own a tile arm") if (tc == 256l) { return set_q8_batch_cm2l_cls(bufs, sizes, gbits) @@ -226,6 +235,14 @@ def private cm2_cls_enc(fmt : int; tc : int64; raw : VkCommandBuffer; var h : Vk } else { enc_kq_batch_iq4nl_cm2s_cls(raw, h, s, pc, groups) } + } elif (fmt == int(KqFmt.k2)) { + if (tc == 256l) { + enc_kq_batch_k2_cm2l_cls(raw, h, s, pc, groups) + } elif (tc == 128l) { + enc_kq_batch_k2_cm2m_cls(raw, h, s, pc, groups) + } else { + enc_kq_batch_k2_cm2s_cls(raw, h, s, pc, groups) + } } elif (tc == 256l) { verify(fmt == int(KqFmt.q8), "vk prefill: the cm2 enc fell through for a kq format - a format the feed admits must own a tile arm") enc_q8_batch_cm2l_cls(raw, h, s, pc, groups) diff --git a/modules/dasLLAMA/tests/_vkd_oracles.das b/modules/dasLLAMA/tests/_vkd_oracles.das index c076a9b4ea..ab189c1c78 100644 --- a/modules/dasLLAMA/tests/_vkd_oracles.das +++ b/modules/dasLLAMA/tests/_vkd_oracles.das @@ -187,6 +187,8 @@ def kq_cls_ref(fmt : int; wq, ws, xq : array; xs : array) : KqGemvB k = kx } elif (fmt == int(KqFmt.iq4nl)) { k = new KqGemvIq4nl() // pure-ALU codebook - no workgroup state to fill + } elif (fmt == int(KqFmt.k2)) { + k = new KqGemvK2() // pure ALU - no workgroup state to fill } else { k = new KqGemvK6() } @@ -470,6 +472,38 @@ def iq4nlf16_gemm_oracle(wq : array; wsu : array; xf : array; } } +// Q2_K f16-feed GEMM reference over the device form: per-16 sc/min nibble pairs x d/dmin, +// f16-rounded per weight like its siblings (16-word rows + [d|dmin][16 pair bytes] scales). +def k2f16_gemm_oracle(wq : array; wsu : array; xf : array; + recs : array; nrec, n, d : int; var y : array) { + let nsb = n / 256 + for (rr in range(nrec)) { + let wsb0 = int(recs[rr * 4]) + let row0 = int(recs[rr * 4 + 1]) + let cnt = int(recs[rr * 4 + 2]) + for (r in range(cnt)) { + for (c in range(d)) { + var acc = 0.0 + for (s in range(nsb)) { + let sb = wsb0 + c * nsb + s + let dm = unpackHalf2x16(wsu[sb * 5]) + for (g in range(16)) { + let pb = (int(wsu[sb * 5 + 1 + g / 4]) >> ((g % 4) * 8)) & 0xFF + for (l in range(16)) { + let qi = (g / 8) * 32 + (g % 2) * 16 + l + let by = byte_u8(wq, sb * 64 + qi) + let q = (by >> (((g / 2) % 4) * 2)) & 3 + let w = float(float16(dm.x * float(pb & 0xF) * float(q) - dm.y * float(pb >> 4))) + acc += w * half_at(xf, (row0 + r) * n + s * 256 + g * 16 + l) + } + } + } + y[(row0 + r) * d + c] = acc + } + } + } +} + // kq batch GEMM oracle over region records — same blk_contrib per (row, col, block) def kq_gemm_oracle(fmt : int; wq, ws, xq : array; xs : array; recs : array; nrec, n, d : int; var y : array) { diff --git a/modules/dasLLAMA/tests/test_vulkan_kernels.das b/modules/dasLLAMA/tests/test_vulkan_kernels.das index 0b57ab49b0..47c84070c8 100644 --- a/modules/dasLLAMA/tests/test_vulkan_kernels.das +++ b/modules/dasLLAMA/tests/test_vulkan_kernels.das @@ -381,7 +381,7 @@ def private attn_row_oracle(q : array; qbase : int; kk : array; kb def test_vkd_kq_gemv_family(t0 : T?) { t0 |> run("kq GEMV class family == the CPU oracle (all 6 formats)") <| @(t : T?) { static_if (typeinfo builtin_module_exists(vulkan)) { - if (!(ensure_kq_gemv_k4_cls() && ensure_kq_gemv_q40_cls() && ensure_kq_gemv_k5_cls() && ensure_kq_gemv_k6_cls() && ensure_kq_gemv_iq4xs_cls() && ensure_kq_gemv_k3_cls() && ensure_kq_gemv_iq3s_cls() && ensure_kq_gemv_iq3xxs_cls() && ensure_kq_gemv_iq4nl_cls())) { + if (!(ensure_kq_gemv_k4_cls() && ensure_kq_gemv_q40_cls() && ensure_kq_gemv_k5_cls() && ensure_kq_gemv_k6_cls() && ensure_kq_gemv_iq4xs_cls() && ensure_kq_gemv_k3_cls() && ensure_kq_gemv_iq3s_cls() && ensure_kq_gemv_iq3xxs_cls() && ensure_kq_gemv_iq4nl_cls() && ensure_kq_gemv_k2_cls())) { feint("no Vulkan device - skipping\n") return } @@ -389,9 +389,9 @@ def test_vkd_kq_gemv_family(t0 : T?) { let d = 3 // rows per region let nreg = 2 let rows = nreg * d - let nfmt = 9 - let fmts = fixed_array(int(KqFmt.k4), int(KqFmt.q40), int(KqFmt.k5), int(KqFmt.k6), int(KqFmt.iq4xs), int(KqFmt.k3), int(KqFmt.iq3s), int(KqFmt.iq3xxs), int(KqFmt.iq4nl)) - let stride4s = fixed_array(32, 32, 40, 48, 32, 24, 26, 24, 32) // uint WORDS per superblock (iq3s/iq3xxs are not uint4-aligned) + let nfmt = 10 + let fmts = fixed_array(int(KqFmt.k4), int(KqFmt.q40), int(KqFmt.k5), int(KqFmt.k6), int(KqFmt.iq4xs), int(KqFmt.k3), int(KqFmt.iq3s), int(KqFmt.iq3xxs), int(KqFmt.iq4nl), int(KqFmt.k2)) + let stride4s = fixed_array(32, 32, 40, 48, 32, 24, 26, 24, 32, 16) // uint WORDS per superblock (iq3s/iq3xxs are not uint4-aligned) for (fi in range(nfmt)) { let fmt = fmts[fi] let stride4 = stride4s[fi] @@ -471,6 +471,8 @@ def test_vkd_kq_gemv_family(t0 : T?) { enc_kq_gemv_iq3xxs_cls(raw2, h2, s_cls, pc, wgs) } elif (fmt == int(KqFmt.iq4nl)) { enc_kq_gemv_iq4nl_cls(raw2, h2, s_cls, pc, wgs) + } elif (fmt == int(KqFmt.k2)) { + enc_kq_gemv_k2_cls(raw2, h2, s_cls, pc, wgs) } else { enc_kq_gemv_k6_cls(raw2, h2, s_cls, pc, wgs) } @@ -1027,7 +1029,7 @@ def test_vkd_q8_batch_tile(t0 : T?) { def test_vkd_kq_batch_family(t0 : T?) { t0 |> run("kq batch tile class family == the CPU oracle (all 6 formats)") <| @(t : T?) { static_if (typeinfo builtin_module_exists(vulkan)) { - if (!(ensure_kq_batch_k4_cls() && ensure_kq_batch_q40_cls() && ensure_kq_batch_k5_cls() && ensure_kq_batch_k6_cls() && ensure_kq_batch_iq4xs_cls() && ensure_kq_batch_k3_cls() && ensure_kq_batch_iq3s_cls() && ensure_kq_batch_iq3xxs_cls() && ensure_kq_batch_iq4nl_cls())) { + if (!(ensure_kq_batch_k4_cls() && ensure_kq_batch_q40_cls() && ensure_kq_batch_k5_cls() && ensure_kq_batch_k6_cls() && ensure_kq_batch_iq4xs_cls() && ensure_kq_batch_k3_cls() && ensure_kq_batch_iq3s_cls() && ensure_kq_batch_iq3xxs_cls() && ensure_kq_batch_iq4nl_cls() && ensure_kq_batch_k2_cls())) { feint("no Vulkan device - skipping\n") return } @@ -1042,9 +1044,9 @@ def test_vkd_kq_batch_family(t0 : T?) { let wgs1 = ((cnt1 + 31) / 32) * wtiles let wgs = wgs0 + wgs1 let totsb = 2 * d * nsb // superblocks across both regions - let nfmt = 9 - let wstrides = fixed_array(32, 32, 40, 48, 32, 24, 26, 24, 32) // uint words per superblock: k4 q40 k5 k6 iq4xs k3 iq3s iq3xxs iq4nl - let pipes = fixed_array(int(KqFmt.k4), int(KqFmt.q40), int(KqFmt.k5), int(KqFmt.k6), int(KqFmt.iq4xs), int(KqFmt.k3), int(KqFmt.iq3s), int(KqFmt.iq3xxs), int(KqFmt.iq4nl)) + let nfmt = 10 + let wstrides = fixed_array(32, 32, 40, 48, 32, 24, 26, 24, 32, 16) // uint words per superblock: k4 q40 k5 k6 iq4xs k3 iq3s iq3xxs iq4nl k2 + let pipes = fixed_array(int(KqFmt.k4), int(KqFmt.q40), int(KqFmt.k5), int(KqFmt.k6), int(KqFmt.iq4xs), int(KqFmt.k3), int(KqFmt.iq3s), int(KqFmt.iq3xxs), int(KqFmt.iq4nl), int(KqFmt.k2)) for (fi in range(nfmt)) { let fmt = pipes[fi] let wstride = wstrides[fi] @@ -1127,6 +1129,8 @@ def test_vkd_kq_batch_family(t0 : T?) { enc_kq_batch_iq3xxs_cls(raw2, h2, sc, pc, int64(wgs)) } elif (fmt == int(KqFmt.iq4nl)) { enc_kq_batch_iq4nl_cls(raw2, h2, sc, pc, int64(wgs)) + } elif (fmt == int(KqFmt.k2)) { + enc_kq_batch_k2_cls(raw2, h2, sc, pc, int64(wgs)) } else { enc_kq_batch_k6_cls(raw2, h2, sc, pc, int64(wgs)) } @@ -1924,6 +1928,129 @@ def test_vkd_q40_cm2_batch(t0 : T?) { } } +[test] +def test_vkd_k2_cm2_batch(t0 : T?) { + t0 |> run("cm2 Q2_K l/m/s tiles == the CPU oracle (pair-byte nibble scales decode-in-load)") <| @(t : T?) { + static_if (typeinfo builtin_module_exists(vulkan)) { + if (!ensure_q8_batch_cls()) { // engage the device without touching coopmat rails + t |> skip("no Vulkan device") + return + } + if (!(g_gpu.coopmat_mode == 4 && g_gpu.has_coopmat2)) { + t |> skip("the cm2 k2 tiles serve only in mode 4 on an NV_coopmat2 device") + return + } + verify(ensure_kq_batch_k2_cm2l_cls() && ensure_kq_batch_k2_cm2m_cls() && ensure_kq_batch_k2_cm2s_cls(), "k2 cm2 class rails must engage in cm2 mode") + let n = 768 // one 512-unrolled pass + 4 tail steps crossing superblock bounds + let nsb = n / 256 + let d = 160 // one full 128-tile + a 32 edge + let cnt0 = 300 + let cnt1 = 260 + let rows = cnt0 + cnt1 + let totsb = 2 * d * nsb + var wqh : array + var wsuh : array + var xfh : array + wqh |> resize(totsb * 16) + wsuh |> resize(totsb * 5) + xfh |> resize((rows + 32) * n / 2) // + the s tile's 32-row read slack + for (i in range(totsb * 16)) { + wqh[i] = hash_word(uint(i) + 31u) // arbitrary quant bytes + } + for (sb in range(totsb)) { + wsuh[sb * 5] = packHalf2x16(float2(0.0004 * float(1 + sb % 7), 0.0003 * float(1 + sb % 5))) + for (wi in range(4)) { // 16 sc/min nibble-pair bytes + wsuh[sb * 5 + 1 + wi] = hash_word(uint(sb * 4 + wi) + 91u) + } + } + for (i in range((rows + 32) * n / 2)) { + xfh[i] = ws_word(uint(i) * 3u + 7u) // f16 activation pairs + } + let wq_bytes = int64(totsb) * 64l + let ws_bytes = int64(length(wsuh)) * 4l + let xf_bytes = int64((rows + 32) * n) * 2l + let y_bytes = int64(rows * d) * 4l + let wqd = make_device_buf(wq_bytes) + let wsd = make_device_buf(ws_bytes) + let xfd = make_device_buf(xf_bytes) + let yd2 = make_device_buf(y_bytes) + var host = make_host_buf(y_bytes, true, [cached = true]) + var y_cls : array + y_cls |> resize(rows * d) + var y_ref : array + y_ref |> resize(rows * d) + for (ml in range(3)) { + let ttile = ml == 0 ? 256 : (ml == 1 ? 128 : 32) + let wtiles = (d + 127) / 128 + let wgs0 = ((cnt0 + ttile - 1) / ttile) * wtiles + let wgs1 = ((cnt1 + ttile - 1) / ttile) * wtiles + let wgs = wgs0 + wgs1 + var sched : array + sched |> resize(2 * 4 + wgs) + sched[0] = 0u + sched[1] = 0u + sched[2] = uint(cnt0) + sched[3] = 0u + sched[4] = uint(d * nsb) + sched[5] = uint(cnt0) + sched[6] = uint(cnt1) + sched[7] = uint(wgs0) + for (w in range(wgs)) { + sched[8 + w] = w < wgs0 ? 0u : 1u + } + unsafe { + upload_region_at(wqd, 0l, addr(wqh[0]), wq_bytes) + upload_region_at(wsd, 0l, addr(wsuh[0]), ws_bytes) + upload_region_at(xfd, 0l, addr(xfh[0]), xf_bytes) + let sc_bytes = int64(2 * 4 + wgs) * 4l + let scd = make_device_buf(sc_bytes) + upload_region_at(scd, 0l, addr(sched[0]), sc_bytes) + var sc = (ml == 0 + ? set_kq_batch_k2_cm2l_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)) + : (ml == 1 + ? set_kq_batch_k2_cm2m_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)) + : set_kq_batch_k2_cm2s_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)))) + var raw2 = alloc_cmd() + let begin = VkCommandBufferBeginInfo() + vk_check(vkBeginCommandBuffer(raw2, begin), null) + var h2 : VkHaz + var pc = BatchArgs(n = uint(n), d = uint(d), map_off = uint(2 * 4)) + if (ml == 0) { + enc_kq_batch_k2_cm2l_cls(raw2, h2, sc, pc, int64(wgs)) + } elif (ml == 1) { + enc_kq_batch_k2_cm2m_cls(raw2, h2, sc, pc, int64(wgs)) + } else { + enc_kq_batch_k2_cm2s_cls(raw2, h2, sc, pc, int64(wgs)) + } + vhz_dep(raw2, h2, 16u, 0u, true) + cmd_copy_whole(raw2, yd2, host.buf, y_bytes) + vk_check(vkEndCommandBuffer(raw2), null) + submit_wait(raw2) + memcpy(addr(y_cls[0]), host.mapped, y_bytes) + } + k2f16_gemm_oracle(wqh, wsuh, xfh, sched, 2, n, d, y_ref) + let bad = mismatch_bars(y_cls, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) + to_log(LOG_INFO, "cm2 k2 {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile batch: {bad} of {rows * d} off the oracle\n") + t |> success(bad == 0, "cm2 k2 {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile matches the CPU oracle ({bad} of {rows * d} off)") + delete sched + } + delete y_ref + delete wqh + delete wsuh + delete xfh + delete y_cls + } else { + t |> skip("dasVulkan not present") + } + } +} + [test] def test_vkd_iq4nl_cm2_batch(t0 : T?) { t0 |> run("cm2 IQ4_NL l/m/s tiles == the CPU oracle (codebook + per-block f16 d decode-in-load)") <| @(t : T?) { From 6f8e45d44cd199edd8f26872141f94ed8bcd808b Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Mon, 31 Aug 2026 03:02:06 -0700 Subject: [PATCH 33/61] dasLLAMA: Q2_K on the Metal tier - the k3 shells minus the hmask, format closed on all four tiers Blob: the "k2s" arm splits the 20B scale row k6-style into [16 pair-byte strips x nsb] [4B d+dmin tails x nsb] - a verbatim 20B bind would need off % 1024, the split keeps every bind aligned for free (no off_ok arm); dequant_k2_plane_superblock grew its _at twin for the split read. Kernels are the k3 shells minus the hmask: MetalKqGemvK2 (the k3 lane map - sc scales the dot, mn folds on the lane's per-block x sums against dmin off the 4B tail), MetalKqMvK2T B2/B4 + MetalKqMvB8K2 (w = q*d*sc - dmin*mn per element), and a K2 arm on MetalKqMulMmK45T reading the pair byte from the superblock's one uint4 strip. Every compile/release/enc ladder, the prefill gates, the shapes gate, and the test cells follow at fmt 2 (fixtures: the split 20B form, kq_row_ref through the _at twin). TRAP: the K2 mul_mm arm wraps the k4/k5 default - a hand brace-balance close landed one line early, the default ran unconditionally after the arm, and the emitted MSL died on "redefinition of sv"; count the balance from INSIDE the else. Gates on the M1 Max: test_metal_gemv_kernels 2/2, test_metal_gemm_kernels 2/2; the metal-blob e2e decodes the same stream at gen 246 t/s, greedy ids 19/64 with the same 0.153-logit token-19 near-tie as every tier. Rows vs llama.cpp b10660 (same requant): Metal pp512 3577.6 / 3568.1 (1.00x), tg128 213.7 / 237.1 (0.90x, both refs +-11 thermal); M1 CPU pp512 543.4 / 210.0 (2.59x), tg128 155.8 / 128.2 (1.21x). With zen2 1.02x/0.95x and Vulkan 0.87x/1.04x, Q2_K = done on all four tiers. Co-Authored-By: Claude Fable 5 --- modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md | 23 +- .../dasLLAMA/dasllama/dasllama_convert.das | 12 +- modules/dasLLAMA/dasllama/dasllama_layout.das | 22 +- .../dasllama/dasllama_metal_common.das | 14 + .../dasllama/dasllama_metal_kernels.das | 305 ++++++++++++++++++ .../dasllama/dasllama_metal_prefill.das | 8 +- .../dasllama/dasllama_metal_shapes.das | 2 +- .../dasLLAMA/tests/_metal_kernel_common.das | 21 +- .../tests/test_metal_gemm_kernels.das | 24 +- .../tests/test_metal_gemv_kernels.das | 38 +-- 10 files changed, 431 insertions(+), 38 deletions(-) diff --git a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md index 4e2f724006..d4ec754bd0 100644 --- a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md +++ b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md @@ -483,7 +483,28 @@ FOLLOW its parent in the file (KqBatchK2 first landed above KqBatchK6 - "parent not found"). Gates: the suite 80/80, the three k2 cm2 tiles 0-off; resident e2e armed (fresh bake), gen 284 t/s, ids 19/64 with the same token-19 near-tie. Rows (5060 Ti vs llama.cpp b10660 build-vulkan): pp512 14544.0 / 16752.7 (0.87x - above the 0.69-0.78 tier -class), tg128 442.8 / 424.1 (1.04x). Metal: pending. +class), tg128 442.8 / 424.1 (1.04x). + +Phase D (Metal, 2026-08-31): the "k2s" blob arm splits the 20B row k6-style into [16 +pair-byte strips x nsb][4B d+dmin tails x nsb] (a verbatim 20B bind would need off % 1024; +the split keeps every bind aligned free - no off_ok arm). dequant_k2_plane_superblock grew +its _at twin for the split read. Kernels are the k3 shells minus the hmask: MetalKqGemvK2 +(the k3 lane map; sc scales the dot, mn folds on per-block x sums against dmin off the 4B +tail), MetalKqMvK2T B2/B4 + MetalKqMvB8K2 (w = q*d*sc - dmin*mn per element), and a K2 arm +on MetalKqMulMmK45T (the pair byte from the one uint4 strip; wrapped around the k4/k5 +default - a hand brace-balance close landed one line early and the default ran after the +arm, "redefinition of sv" in the emitted MSL; balance from INSIDE the else). Gates on the +M1 Max: gemv 2/2, gemm 2/2; the metal-blob e2e decodes the same stream at gen 246 t/s, +ids 19/64 with the same token-19 near-tie. + +Where Q2_K landed (vs llama.cpp b10660, the local requant): + +| tier | pp512 (ours / theirs) | tg128 (ours / theirs) | +|---|---|---| +| zen2 CPU | 418.6 / 412.0 (1.02x) | 77.1 / 81.1 (0.95x) | +| M1 CPU | 543.4 / 210.0 (2.59x) | 155.8 / 128.2 (1.21x) | +| 5060 Ti Vulkan | 14544.0 / 16752.7 (0.87x) | 442.8 / 424.1 (1.04x) | +| M1 Metal | 3577.6 / 3568.1 (1.00x) | 213.7 / 237.1 (0.90x, both refs drift +-11 thermally) | ### IQ4_NL (the near-free one, 2026-08-30) diff --git a/modules/dasLLAMA/dasllama/dasllama_convert.das b/modules/dasLLAMA/dasllama/dasllama_convert.das index 5a696a3557..2f3b526dd7 100644 --- a/modules/dasLLAMA/dasllama/dasllama_convert.das +++ b/modules/dasLLAMA/dasllama/dasllama_convert.das @@ -683,10 +683,16 @@ def transcode_q2k_superblock(bytes : array | #; bo : int64; var kq : arra //! g (sc/mn = the group's nibble pair); q = the 2-bit lane of qs byte (g/8)*32 + (g%2)*16 + l at //! shift 2*((g%8)/2) - ggml's dequantize_row_q2_K float order (dl*q - ml). def dequant_k2_plane_superblock(kq : array | #; kqo : int64; ks : array | #; kso : int64; var dst : array | #; doff : int64) { - let d = f16_to_f32(rd_u16(ks, kso)) - let dmin = f16_to_f32(rd_u16(ks, kso + 2l)) + dequant_k2_plane_superblock_at(kq, kqo, ks, kso + 4l, kso, dst, doff) +} + +//! The split-layout twin: `sso` names the 16 pair-byte strip, `dso` the 4B d+dmin header - +//! independent bases, so the metal-blob split plane reads through the same math. +def dequant_k2_plane_superblock_at(kq : array | #; kqo : int64; ks : array | #; sso, dso : int64; var dst : array | #; doff : int64) { + let d = f16_to_f32(rd_u16(ks, dso)) + let dmin = f16_to_f32(rd_u16(ks, dso + 2l)) for (g in range64(16l)) { - let scb = int(ks[kso + 4l + g]) + let scb = int(ks[sso + g]) let dl = d * float(scb & 15) let ml = dmin * float(scb >> 4) let qb = kqo + (g / 8l) * 32l + (g % 2l) * 16l diff --git a/modules/dasLLAMA/dasllama/dasllama_layout.das b/modules/dasLLAMA/dasllama/dasllama_layout.das index 00db3c70a6..472e65bbff 100644 --- a/modules/dasLLAMA/dasllama/dasllama_layout.das +++ b/modules/dasLLAMA/dasllama/dasllama_layout.das @@ -226,6 +226,26 @@ def metal_blob_scale_plane(var t : Model; name : string) : bool { } return true } + if (name == "k2s") { + // k2: the 20B row splits like k6 - the 16 pair-byte strips first, the 4B d+dmin headers as the tail + if (!empty(t.k2s)) { + var @exact_size k2c : array + let nsb = long_length(t.k2s) / K2_SSB + k2c |> reserve_resize(nsb * K2_SSB) + unsafe { + var dp = addr(k2c[0]) + let sp = addr < uint8 const? >(t.k2s[0]) + for (sb in range64(nsb)) { + memcpy(reinterpret(dp + sb * 16l), reinterpret(sp + sb * K2_SSB + 4l), 16) + for (h in range64(4l)) { + dp[nsb * 16l + sb * 4l + h] = sp[sb * K2_SSB + h] + } + } + } + t.k2s <- k2c + } + return true + } if (name == "k3s") { // k3: the CPU row IS the k6 form ([16 int8][f16 d]) — the same split, over t.k3s if (!empty(t.k3s)) { @@ -274,7 +294,7 @@ def metal_blob_commit(var t : Model) : bool { if (!metal_blob_eligible(t)) { return false } - for (nm in ["k4s", "k5s", "k6s", "iq4xss", "k3s", "iq3ss", "iq3xxss"]) { + for (nm in ["k4s", "k5s", "k6s", "iq4xss", "k3s", "iq3ss", "iq3xxss", "k2s"]) { metal_blob_scale_plane(t, nm) } t.metal_blob = true diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_common.das b/modules/dasLLAMA/dasllama/dasllama_metal_common.das index efe619070e..678e89d282 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_common.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_common.das @@ -268,6 +268,11 @@ var g_pso_kq_mvb8_iq3s : MetalComputePipeline? var g_pso_kq_mm_b33 : MetalComputePipeline? var g_pso_kq_iq3xxs : MetalComputePipeline? var g_pso_kq_iq4nl : MetalComputePipeline? +var g_pso_kq_k2 : MetalComputePipeline? +var g_pso_kq_mvb2_k2 : MetalComputePipeline? +var g_pso_kq_mvb4_k2 : MetalComputePipeline? +var g_pso_kq_mvb8_k2 : MetalComputePipeline? +var g_pso_kq_mm_b2 : MetalComputePipeline? var g_pso_kq_mvb2_iq4nl : MetalComputePipeline? var g_pso_kq_mvb4_iq4nl : MetalComputePipeline? var g_pso_kq_mvb8_iq4nl : MetalComputePipeline? @@ -879,6 +884,10 @@ def kq_quants_of(dev : MetalDevice?; t : Model; fmt : KqFmt; off : int64) : tupl return (buf = plane_buffer(dev, addr < void? >(t.iq4nlq[0]), uint64(long_length(t.iq4nlq)), t.image_map != null), qoff = uint64(sb0 * 128l)) } + if (fmt == KqFmt.k2) { + return (buf = plane_buffer(dev, addr < void? >(t.k2q[0]), uint64(long_length(t.k2q)), t.image_map != null), + qoff = uint64(sb0 * 64l)) + } return (buf = plane_buffer(dev, addr < void? >(t.k6q[0]), uint64(long_length(t.k6q)), t.image_map != null), qoff = uint64(sb0 * 192l)) } @@ -924,6 +933,11 @@ def kq_scales_of(dev : MetalDevice?; t : Model; fmt : KqFmt; off : int64) : tupl return (buf = plane_buffer(dev, addr < void? >(t.iq4nls[0]), uint64(long_length(t.iq4nls)), t.image_map != null), soff = uint64(sb0 * 16l), doff = 0ul) } + if (fmt == KqFmt.k2) { // the k6 split shape with a 4B d+dmin tail per superblock + let plane_sb = long_length(t.k2s) / 20l + return (buf = plane_buffer(dev, addr < void? >(t.k2s[0]), uint64(long_length(t.k2s)), t.image_map != null), + soff = uint64(sb0 * 16l), doff = uint64(plane_sb * 16l + sb0 * 4l)) + } return (buf = plane_buffer(dev, addr < void? >(t.k5s[0]), uint64(long_length(t.k5s)), t.image_map != null), soff = uint64(sb0 * 16l), doff = 0ul) } diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das index 2f928268ca..ae7aa4543e 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das @@ -3661,6 +3661,11 @@ def metal_decode_init : bool { // nolint:STYLE038 — a flat compile_pso per k g_pso_kq_mvb4_iq4nl = compile_pso(MetalKqMvB4Iq4nl_metal_kq_mv_iq4nl_msl, MetalKqMvB4Iq4nl_metal_kq_mv_iq4nl_msl_entry, MetalKqMvB4Iq4nl_metal_kq_mv_iq4nl_msl_fastmath, ok) g_pso_kq_mvb8_iq4nl = compile_pso(metal_kq_mvb8_iq4nl_msl, metal_kq_mvb8_iq4nl_msl_entry, metal_kq_mvb8_iq4nl_msl_fastmath, ok) g_pso_kq_mm_b45 = compile_pso(MetalKqMulMmIq4nl_metal_kq_mulmm_msl, MetalKqMulMmIq4nl_metal_kq_mulmm_msl_entry, MetalKqMulMmIq4nl_metal_kq_mulmm_msl_fastmath, ok) + g_pso_kq_k2 = compile_pso(metal_kq_gemv_k2_msl, metal_kq_gemv_k2_msl_entry, metal_kq_gemv_k2_msl_fastmath, ok) + g_pso_kq_mvb2_k2 = compile_pso(MetalKqMvB2K2_metal_kq_mv_k2_msl, MetalKqMvB2K2_metal_kq_mv_k2_msl_entry, MetalKqMvB2K2_metal_kq_mv_k2_msl_fastmath, ok) + g_pso_kq_mvb4_k2 = compile_pso(MetalKqMvB4K2_metal_kq_mv_k2_msl, MetalKqMvB4K2_metal_kq_mv_k2_msl_entry, MetalKqMvB4K2_metal_kq_mv_k2_msl_fastmath, ok) + g_pso_kq_mvb8_k2 = compile_pso(metal_kq_mvb8_k2_msl, metal_kq_mvb8_k2_msl_entry, metal_kq_mvb8_k2_msl_fastmath, ok) + g_pso_kq_mm_b2 = compile_pso(MetalKqMulMmK2_metal_kq_mulmm_msl, MetalKqMulMmK2_metal_kq_mulmm_msl_entry, MetalKqMulMmK2_metal_kq_mulmm_msl_fastmath, ok) // the batched-step set (P4) g_pso_gemv_b2 = compile_pso(MetalGemvB2_metal_gemv_msl, MetalGemvB2_metal_gemv_msl_entry, MetalGemvB2_metal_gemv_msl_fastmath, ok) g_pso_gemv_b4 = compile_pso(MetalGemvB4_metal_gemv_msl, MetalGemvB4_metal_gemv_msl_entry, MetalGemvB4_metal_gemv_msl_fastmath, ok) @@ -3737,6 +3742,8 @@ def enc_kq_gemv(enc : MetalComputeEncoder?; fmt : KqFmt; bq : MetalBuffer?; qoff enc_kq_iq3xxs_c(enc, bs, doff, bs, soff, bq, qoff, bx, xoff, by, yoff, bn, bd, rows, n) } elif (fmt == KqFmt.iq4nl) { enc_kq_iq4nl_c(enc, bs, soff, bs, soff, bq, qoff, bx, xoff, by, yoff, bn, bd, rows, n) + } elif (fmt == KqFmt.k2) { + enc_kq_k2_c(enc, bs, doff, bs, soff, bq, qoff, bx, xoff, by, yoff, bn, bd, rows, n) } elif (fmt == KqFmt.k6) { enc_kq_k6_c(enc, bs, doff, bs, soff, bq, qoff, bx, xoff, by, yoff, bn, bd, rows, n) } elif (fmt == KqFmt.k5 && (n < 3072l || rows <= 1024l)) { @@ -3813,6 +3820,14 @@ def enc_kq_mvb(enc : MetalComputeEncoder?; fmt : KqFmt; bq : MetalBuffer?; qoff } else { enc_kq_mvb4_iq4nl_c(enc, bs, soff, bs, soff, bq, qoff, bx, by, yoff, ka, rows, gcols) } + } elif (fmt == KqFmt.k2) { + if (b8) { + enc_kq_mvb8_k2_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows) + } elif (two) { + enc_kq_mvb2_k2_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows) + } else { + enc_kq_mvb4_k2_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows, gcols) + } } elif (fmt == KqFmt.k6) { if (b8) { enc_kq_mvb8_k6_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows) @@ -3855,6 +3870,8 @@ def enc_kq_gemm_mm_b(enc : MetalComputeEncoder?; fmt : KqFmt; bq : MetalBuffer?; enc_kq_mm_iq3xxs_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, bk, bn, mp, rows) } elif (fmt == KqFmt.iq4nl) { enc_kq_mm_iq4nl_c(enc, bs, soff, bs, soff, bq, qoff, bx, by, yoff, bk, bn, mp, rows) + } elif (fmt == KqFmt.k2) { + enc_kq_mm_k2_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, bk, bn, mp, rows) } elif (fmt == KqFmt.k6) { enc_kq_mm_k6_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, bk, bn, mp, rows) } elif (fmt == KqFmt.k4) { @@ -4306,6 +4323,11 @@ def metal_kernels_release { // nolint:STYLE037,STYLE038 — a flat release_pso release_pso(g_pso_kq_mvb4_iq4nl) release_pso(g_pso_kq_mvb8_iq4nl) release_pso(g_pso_kq_mm_b45) + release_pso(g_pso_kq_k2) + release_pso(g_pso_kq_mvb2_k2) + release_pso(g_pso_kq_mvb4_k2) + release_pso(g_pso_kq_mvb8_k2) + release_pso(g_pso_kq_mm_b2) if (g_pso_rms != null) { metal_release(g_pso_rms) g_pso_rms = null @@ -5536,6 +5558,7 @@ class template MetalKqMulMmK45T : MetalMmTileBase { @template_constant IQ3S : bool = false // iq3s: the k6 split scale form, threadgroup grid gather + sign flip @template_constant IQ3XXS : bool = false // iq3xxs: the same form over the halved grid + parity signs @template_constant IQ4NL : bool = false // iq4nl: the IQ4XS nibble walk with the per-32-block f16 d plane (no sub-scale; s0off = soff) + @template_constant K2 : bool = false // q2_K: per-16 sc/min nibble pairs (one uint4 strip), the split 4B d+dmin tail at s0off @workgroup @template_gate = IQ4XS iq4lut : float[16] // kvalues_iq4nl as floats, staged once per threadgroup @workgroup @template_gate = IQ3S gridw3 : uint[512] // the 2 KB iq3s grid, staged once per threadgroup @workgroup @template_gate = IQ3XXS gridw3x : uint[256] // the 1 KB halved iq3xxs grid, staged once per threadgroup @@ -5714,6 +5737,30 @@ class template MetalKqMulMmK45T : MetalMmTileBase { } } } else { + static_if (K2) { + // q2_K: pair byte si = the group's sc|mn nibbles; d/dmin off the split tail + let hf3 = js / 4u + let gg3 = js % 4u + let sv = ks4[blk] + let si = hf3 * 8u + gg3 * 2u + il0 + let pw = si < 8u ? (si < 4u ? sv.x : sv.y) : (si < 12u ? sv.z : sv.w) + let pb = (pw >> ((si & 3u) * 8u)) & 255u + let dsc = float(ksh[blk * 2u]) * float(pb & 15u) + let dmn = float(ksh[blk * 2u + 1u]) * float(pb >> 4u) + let dsc1 = dsc * 0.00390625 + let dsc2 = dsc * 0.0000152587890625 + let dsc3 = dsc * 0.000000059604644775390625 + let qb = blk * 16u + hf3 * 8u + il0 * 4u + let nsh = gg3 * 2u + for [unroll_full] (k in range(4)) { + let u = kqu[qb + uint(k)] + let qv = (u >> nsh) & 0x03030303 + va[k * 4] = float16(dsc * float(qv & 255u) - dmn) + va[k * 4 + 1] = float16(dsc1 * float(qv & 65280u) - dmn) + va[k * 4 + 2] = float16(dsc2 * float(qv & 16711680u) - dmn) + va[k * 4 + 3] = float16(dsc3 * float(qv & 4278190080u) - dmn) + } + } else { let sv = ks4[blk] var sc = 0u var mn = 0u @@ -5752,6 +5799,7 @@ class template MetalKqMulMmK45T : MetalMmTileBase { } } } + } } } } @@ -5816,6 +5864,12 @@ class MetalKqMulMmIq3xxs : MetalKqMulMmK45T { override IQ3XXS = true } +// q2_K rides its own K2 arm; the split scale plane binds twice like k6 (the 4B tail at s0off = doff) +[metal_dispatch(name = "enc_kq_mm_k2_c", pso = "g_pso_kq_mm_b2", tgmem = "MetalKqMulMmK2_metal_kq_mulmm_msl_tgmem", tg = 128, grid = "mp/32, rows/64", params = "mp : int64, rows : int64")] +class MetalKqMulMmK2 : MetalKqMulMmK45T { + override K2 = true +} + // iq4nl rides the IQ4XS arm with IQ4NL flipping the scale read to q40's per-32-block d plane [metal_dispatch(name = "enc_kq_mm_iq4nl_c", pso = "g_pso_kq_mm_b45", tgmem = "MetalKqMulMmIq4nl_metal_kq_mulmm_msl_tgmem", tg = 128, grid = "mp/32, rows/64", params = "mp : int64, rows : int64")] class MetalKqMulMmIq4nl : MetalKqMulMmK45T { @@ -7230,6 +7284,257 @@ class MetalKqGemvK6 { } } +// Q2_K: the k3 GEMV's lane map minus the hmask - unsigned 2-bit lanes, per-16 sc/min NIBBLE +// pairs off the split strips (sc scales the dot, mn folds on the lane's x sums against dmin), +// d + dmin off the 4B tail plane. +[metal_dispatch(name = "enc_kq_k2_c", pso = "g_pso_kq_k2", tg = 64, grid = "rows/4", params = "rows : int64, n : int64")] +class MetalKqGemvK2 { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d+dmin tail - the scale buffer bound at byte nsb*16 (s0off = the caller's doff) + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // 16B pair-byte strips, byte view (same buffer at 0) + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqb : array // k2 quant plane, byte view (qs at 64*blk) + @ssbo @binding = 3 @role = "read" @off = "xoff" @span = "n*4" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" @span = "rows*4" y : array + @uniform @binding = 5 ndim : uint + @uniform @binding = 6 ddim : uint + + [metal_kernel(name="metal_kq_gemv_k2_msl")] + def metal_kq_gemv_k2 { + let lane = gl_SubgroupInvocationID + let tid = lane / 2u + let ix = lane % 2u + let ip = tid / 8u + let il = tid % 8u + let l0 = il * 4u + let is0 = ip * 8u + l0 / 16u + let nb = ndim / 256u + let first_row = (gl_WorkGroupID.x * 2u + gl_SubgroupID) * 2u + var yl : float[16] + var sumf : float[2] + var ib = ix + while (ib < nb) { + let yb = ib * 256u + ip * 128u + l0 + var ysum : float[4] + for [unroll_full] (l in range(4)) { + yl[4 * l] = x[yb + uint(l)] + yl[4 * l + 1] = x[yb + 32u + uint(l)] + yl[4 * l + 2] = x[yb + 64u + uint(l)] + yl[4 * l + 3] = x[yb + 96u + uint(l)] + ysum[0] += yl[4 * l] + ysum[1] += yl[4 * l + 1] + ysum[2] += yl[4 * l + 2] + ysum[3] += yl[4 * l + 3] + } + for [unroll_full] (r in range(2)) { + let blk = (first_row + uint(r)) * nb + ib + let qs0 = blk * 64u + ip * 32u + l0 + let sb = blk * 16u + is0 + var sums : float[4] + for [unroll_full] (l in range(4)) { + let q = uint(kqb[qs0 + uint(l)]) + sums[0] += yl[4 * l] * float(q & 3u) + sums[1] += yl[4 * l + 1] * float((q >> 2u) & 3u) + sums[2] += yl[4 * l + 2] * float((q >> 4u) & 3u) + sums[3] += yl[4 * l + 3] * float(q >> 6u) + } + let b0 = uint(kscb[sb]) + let b2 = uint(kscb[sb + 2u]) + let b4 = uint(kscb[sb + 4u]) + let b6 = uint(kscb[sb + 6u]) + let dd = float(kdh[blk * 2u]) + let dmn = float(kdh[blk * 2u + 1u]) + sumf[r] += dd * (sums[0] * float(b0 & 15u) + sums[1] * float(b2 & 15u) + sums[2] * float(b4 & 15u) + sums[3] * float(b6 & 15u)) + sumf[r] -= dmn * (ysum[0] * float(b0 >> 4u) + ysum[1] * float(b2 >> 4u) + ysum[2] * float(b4 >> 4u) + ysum[3] * float(b6 >> 4u)) + } + ib += 2u + } + for [unroll_full] (r in range(2)) { + let s = simd_sum(sumf[r]) + if (lane == 0u && first_row + uint(r) < ddim) { + y[first_row + uint(r)] = s + } + } + } +} + +// The k2 B2/B4 pair: the K3T shell minus the hmask - w = q*d*sc - dmin*mn per element, the +// pair byte picked per (slot, half). +[ |> template_struct_instance] +class template MetalKqMvK2T { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d+dmin tail - the scale buffer bound at byte nsb*16 + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // 16B pair-byte strips, byte view (same buffer at 0) + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array // k2 quant plane, uint view (16 qs words per superblock) + @ssbo @binding = 3 @role = "read" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" y : array + @uniform @binding = 5 ka : KqMvArgs + @template_constant NR : int = 2 + @template_constant NRU : uint = 2u // NR's uint spelling (colbase math) — override BOTH together + @template_constant TILED : bool = false + + [metal_kernel] + def metal_kq_mv_k2 { // nolint:STYLE038 — the TILED twin loops are byte-identity-driven, not splittable + let lane = gl_SubgroupInvocationID + let tx = lane % 8u + let ty = lane / 8u + let row = gl_WorkGroupID.x * 8u + gl_SubgroupID * 4u + ty + let colbase = gl_WorkGroupID.y * NRU // nolint:LINT002 dead in the TILED=false stamp — measured free + let nb = ka.ndim / 256u + let s = tx / 2u + let o4 = (tx % 2u) * 4u + let shr = 2u * s + let nb4 = ka.ndim / 4u + var sumf : float[NR] + var sb = 0u + while (sb < nb) { + let blk = row * nb + sb + let dd = float(kdh[blk * 2u]) + let dmn = float(kdh[blk * 2u + 1u]) + for [unroll_full] (ch in range(2)) { + let ip = uint(ch) + let qb = blk * 16u + ip * 8u + o4 + let g = ip * 8u + s * 2u + (tx % 2u) + let pb = uint(kscb[blk * 16u + g]) + let dl = dd * float(pb & 15u) + let ml = dmn * float(pb >> 4u) + var w : float[16] + for [unroll_full] (u in range(4)) { + let qu = kqu[qb + uint(u)] + let qv = (qu >> shr) & 0x03030303 + for [unroll_full] (e in range(4)) { + w[4 * u + e] = float((qv >> (8u * uint(e))) & 255u) * dl - ml + } + } + let cx4 = sb * 64u + ip * 32u + s * 8u + o4 + static_if (TILED) { + for [unroll_full] (b in range(NR)) { + let xb4 = (colbase + uint(b)) * nb4 + cx4 + var acc = 0.0 + for [unroll_full] (u in range(4)) { + let v = x[xb4 + uint(u)] + acc += v.x * w[4 * u] + v.y * w[4 * u + 1] + v.z * w[4 * u + 2] + v.w * w[4 * u + 3] + } + sumf[b] += acc + } + } else { + for [unroll_full] (b in range(NR)) { + let xb4 = uint(b) * nb4 + cx4 + var acc = 0.0 + for [unroll_full] (u in range(4)) { + let v = x[xb4 + uint(u)] + acc += v.x * w[4 * u] + v.y * w[4 * u + 1] + v.z * w[4 * u + 2] + v.w * w[4 * u + 3] + } + sumf[b] += acc + } + } + } + sb++ + } + static_if (TILED) { + for [unroll_full] (b in range(NR)) { + var sm = sumf[b] + sm += simd_shuffle_down(sm, 4u) + sm += simd_shuffle_down(sm, 2u) + sm += simd_shuffle_down(sm, 1u) + if (tx == 0u && row < ka.ddim && colbase + uint(b) < ka.nr) { + y[(colbase + uint(b)) * ka.ys + row] = sm + } + } + } else { + for [unroll_full] (b in range(NR)) { + var sm = sumf[b] + sm += simd_shuffle_down(sm, 4u) + sm += simd_shuffle_down(sm, 2u) + sm += simd_shuffle_down(sm, 1u) + if (tx == 0u && row < ka.ddim && uint(b) < ka.nr) { + y[uint(b) * ka.ys + row] = sm + } + } + } + } +} + +[metal_dispatch(name = "enc_kq_mvb2_k2_c", pso = "g_pso_kq_mvb2_k2", tg = 64, grid = "rows/8", params = "rows : int64")] +class MetalKqMvB2K2 : MetalKqMvK2T { + override NR = 2 + override NRU = 2u + override TILED = false +} + +[metal_dispatch(name = "enc_kq_mvb4_k2_c", pso = "g_pso_kq_mvb4_k2", tg = 64, grid = "rows/8, gcols", params = "rows : int64, gcols : int64")] +class MetalKqMvB4K2 : MetalKqMvK2T { + override NR = 4 + override NRU = 4u + override TILED = true +} + +// the k2 B8 twin: the k3 B8 shell minus the hmask, per-element w = q*d*sc - dmin*mn +[metal_dispatch(name = "enc_kq_mvb8_k2_c", pso = "g_pso_kq_mvb8_k2", tgmem = "metal_kq_mvb8_k2_msl_tgmem", tg = 64, grid = "rows/8", params = "rows : int64")] +class MetalKqMvB8K2 { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d+dmin tail - the scale buffer bound at byte nsb*16 + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // 16B pair-byte strips, byte view (same buffer at 0) + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array // k2 quant plane, uint view + @ssbo @binding = 3 @role = "read" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" y : array + @uniform @binding = 5 ka : KqMvArgs + @workgroup txp : float4[512] + + [metal_kernel(name="metal_kq_mvb8_k2_msl")] + def metal_kq_mvb8_k2 { + let lid = gl_LocalInvocationID.x + let lane = gl_SubgroupInvocationID + let tx = lane % 8u + let ty = lane / 8u + let row = gl_WorkGroupID.x * 8u + gl_SubgroupID * 4u + ty + let nb = ka.ndim / 256u + let s = tx / 2u + let o4 = (tx % 2u) * 4u + let shr = 2u * s + let nb4 = ka.ndim / 4u + var acc : float4[8] + var sb = 0u + while (sb < nb) { + barrier() + for [unroll_full] (j in range(8)) { + let slot = uint(j) * 64u + lid + txp[slot] = x[(slot / 64u) * nb4 + sb * 64u + (slot % 64u)] + } + barrier() + let blk = row * nb + sb + let dd = float(kdh[blk * 2u]) + let dmn = float(kdh[blk * 2u + 1u]) + for [unroll_full] (ch in range(2)) { + let ip = uint(ch) + let qb = blk * 16u + ip * 8u + o4 + let g = ip * 8u + s * 2u + (tx % 2u) + let pb = uint(kscb[blk * 16u + g]) + let dl = dd * float(pb & 15u) + let ml = dmn * float(pb >> 4u) + var w4 : float4[4] + for [unroll_full] (u in range(4)) { + let qvc = (kqu[qb + uint(u)] >> shr) & 0x03030303 + w4[u] = float4(float(qvc & 255u), float((qvc >> 8u) & 255u), + float((qvc >> 16u) & 255u), float((qvc >> 24u) & 255u)) * dl - float4(ml) + } + let cxl = ip * 32u + s * 8u + o4 + for [unroll_full] (b in range(8)) { + for [unroll_full] (u in range(4)) { + acc[b] += txp[uint(b) * 64u + cxl + uint(u)] * w4[u] + } + } + } + sb++ + } + for [unroll_full] (b in range(8)) { + var sm = acc[b].x + acc[b].y + acc[b].z + acc[b].w + sm += simd_shuffle_down(sm, 4u) + sm += simd_shuffle_down(sm, 2u) + sm += simd_shuffle_down(sm, 1u) + if (tx == 0u && row < ka.ddim && uint(b) < ka.nr) { + y[uint(b) * ka.ys + row] = sm + } + } + } +} + // Q3_K: k6's lane map (a lane's 4 elements of each of a half's 4 blocks) over the k3 planes — the // 2-bit lane of qs byte ip*32 + l at shift 2j | hmask bit 4ip + j of byte l, minus 4; per-16 SIGNED // sub-scales off the split strips, d off the d plane. diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das b/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das index 1e6cbf4f62..98e7dcbbf6 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das @@ -4411,7 +4411,7 @@ def private pf_devw_panel(enc : MetalComputeEncoder?; bwblob : MetalBuffer?; wbo //! quant superblocks, plus the k6 2B d tail def private pf_devw_panel_kq(enc : MetalComputeEncoder?; t : Model; fmt : KqFmt; woff : int64; bxh, by, bk, bn : MetalBuffer?; rows, d, kdim : int64; yoff : uint64) : bool { - if (fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s || fmt == KqFmt.iq3xxs || fmt == KqFmt.iq4nl) { // no dev-W dequant kernel for these formats yet + if (fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s || fmt == KqFmt.iq3xxs || fmt == KqFmt.iq4nl || fmt == KqFmt.k2) { // no dev-W dequant kernel for these formats yet return false } let dq = fmt == KqFmt.k6 ? g_pf_pso_dq_k6 : (fmt == KqFmt.k4 ? g_pf_pso_dq_k4 : g_pf_pso_dq_k5) @@ -4596,6 +4596,10 @@ def private pf_enc_kq_site_mm(enc : MetalComputeEncoder?; t : Model; fmt : KqFmt enc_kq_mm_iq4nl_c(enc, bs.buf, bs.soff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, by, yoff, bk, bn, mp, rows) return } + if (fmt == KqFmt.k2) { // the base mul_mm only - no tensor / tall / dev-W twins yet + enc_kq_mm_k2_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, by, yoff, bk, bn, mp, rows) + return + } //! deep-class arm: a panel dev-W would split past 8 tiles is DRAM-resident - the tall //! in-kernel-dequant stamp reads the quant plane once per 128-row tile instead of //! materializing and re-streaming a 2B/element f16 panel @@ -4709,6 +4713,8 @@ def private pf_enc_kq_gemv(enc : MetalComputeEncoder?; t : Model; fmt : KqFmt; w enc_kq_iq3xxs_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, xoff, by, 0ul, bn, bd, rows, n) } elif (fmt == KqFmt.iq4nl) { enc_kq_iq4nl_c(enc, bs.buf, bs.soff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, xoff, by, 0ul, bn, bd, rows, n) + } elif (fmt == KqFmt.k2) { + enc_kq_k2_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, xoff, by, 0ul, bn, bd, rows, n) } elif (fmt == KqFmt.k6) { enc_kq_k6_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, xoff, by, 0ul, bn, bd, rows, n) } elif (fmt == KqFmt.k4) { diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_shapes.das b/modules/dasLLAMA/dasllama/dasllama_metal_shapes.das index a2e6bc91d9..1c6f5d5d96 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_shapes.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_shapes.das @@ -180,7 +180,7 @@ def record_needs(var tab : table; missing : MetalNeed) { // formats) must DECLINE kquant_native instead of falling into a wrong-layout kernel branch // (the dispatchers below treat "not k4/k6" as k5) def kq_fmt_gpu_supported(f : KqFmt) : bool { - return f == KqFmt.q8 || f == KqFmt.k4 || f == KqFmt.k5 || f == KqFmt.k6 || f == KqFmt.iq4xs || f == KqFmt.k3 || f == KqFmt.iq3s || f == KqFmt.iq3xxs || f == KqFmt.iq4nl + return f == KqFmt.q8 || f == KqFmt.k4 || f == KqFmt.k5 || f == KqFmt.k6 || f == KqFmt.iq4xs || f == KqFmt.k3 || f == KqFmt.iq3s || f == KqFmt.iq3xxs || f == KqFmt.iq4nl || f == KqFmt.k2 } def private kq_fmts_gpu_supported(a : array) : bool { diff --git a/modules/dasLLAMA/tests/_metal_kernel_common.das b/modules/dasLLAMA/tests/_metal_kernel_common.das index 4360c8b4ed..090dbc52a9 100644 --- a/modules/dasLLAMA/tests/_metal_kernel_common.das +++ b/modules/dasLLAMA/tests/_metal_kernel_common.das @@ -174,12 +174,26 @@ let KQ_SAFE_SCALES = fixed_array(0.25, 0.5, 1.0, 0.125, 2.0, 0.75, 1.5, 0.375) // [12 packed 6-bit bytes] per superblock, K6 = [16B sub-scales] x nsb + f16 d tail, IQ4_XS (44) = // the same split form with [8 signed sub-scales in -31..31][8 zero] strips, Q3_K (3) = the k6 form over 96B quants def kq_fill_planes(fmt, nsb : int; var kq : array; var ks : array) { - let qsb = fmt == 4 || fmt == 44 || fmt == 45 ? 128 : (fmt == 5 ? 160 : (fmt == 3 ? 96 : (fmt == 33 ? 104 : (fmt == 34 ? 96 : 192)))) + let qsb = fmt == 2 ? 64 : (fmt == 4 || fmt == 44 || fmt == 45 ? 128 : (fmt == 5 ? 160 : (fmt == 3 ? 96 : (fmt == 33 ? 104 : (fmt == 34 ? 96 : 192))))) kq |> resize(nsb * qsb) for (i in range(nsb * qsb)) { kq[i] = kq_hash_byte(i * 3 + fmt) } - if (fmt == 45) { + if (fmt == 2) { + // k2: the metal split form - [16 pair-byte strips x nsb][4B d+dmin tail x nsb] + ks |> resize(nsb * 20) + for (i in range(nsb * 16)) { + ks[i] = kq_hash_byte(i * 7 + 13) + } + for (b in range(nsb)) { + let dbits = f32_to_f16(KQ_SAFE_SCALES[b % 8]) + let mbits = f32_to_f16(KQ_SAFE_SCALES[(b + 3) % 8]) + ks[nsb * 16 + b * 4] = uint8(dbits & 0xFFu) + ks[nsb * 16 + b * 4 + 1] = uint8(dbits >> 8u) + ks[nsb * 16 + b * 4 + 2] = uint8(mbits & 0xFFu) + ks[nsb * 16 + b * 4 + 3] = uint8(mbits >> 8u) + } + } elif (fmt == 45) { // iq4nl: 16B/sb = 8 per-32-block f16 d's, verbatim (no strips) ks |> resize(nsb * 16) for (b in range(nsb * 8)) { @@ -267,6 +281,9 @@ def kq_row_ref(fmt, sb_base, nb, nsb_total : int; kq, ks : array; var wro int64(nsb_total * 16 + sb * 2), wrow, int64(ib * 256)) } elif (fmt == 45) { dequant_iq4nl_plane_superblock(kq, int64(sb * 128), ks, int64(sb * 16), wrow, int64(ib * 256)) + } elif (fmt == 2) { + dequant_k2_plane_superblock_at(kq, int64(sb * 64), ks, int64(sb * 16), + int64(nsb_total * 16 + sb * 4), wrow, int64(ib * 256)) } else { dequant_k6_plane_superblock_at(kq, int64(sb * 192), ks, int64(sb * 16), int64(nsb_total * 16 + sb * 2), wrow, int64(ib * 256)) diff --git a/modules/dasLLAMA/tests/test_metal_gemm_kernels.das b/modules/dasLLAMA/tests/test_metal_gemm_kernels.das index 7b6dd23994..8af7395c8f 100644 --- a/modules/dasLLAMA/tests/test_metal_gemm_kernels.das +++ b/modules/dasLLAMA/tests/test_metal_gemm_kernels.das @@ -36,28 +36,28 @@ def private kq_mulmm_gate(t : T?; dev, queue; fmt : int; tensor : bool; m, kdim, : halfx ? (fmt == 4 ? MetalKqMulMmK4TH_metal_kq_mulmm_t_msl : (fmt == 5 ? MetalKqMulMmK5TH_metal_kq_mulmm_t_msl : MetalKqMulMmK6TH_metal_kq_mulmm_k6_t_msl)) : (fmt == 4 ? MetalKqMulMmK4T_metal_kq_mulmm_t_msl : (fmt == 5 ? MetalKqMulMmK5T_metal_kq_mulmm_t_msl : MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl))) - : (fmt == 45 ? MetalKqMulMmIq4nl_metal_kq_mulmm_msl : (fmt == 34 ? MetalKqMulMmIq3xxs_metal_kq_mulmm_msl : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl : MetalKqMulMmK6_metal_kq_mulmm_msl)))))))) + : (fmt == 45 ? MetalKqMulMmIq4nl_metal_kq_mulmm_msl : (fmt == 2 ? MetalKqMulMmK2_metal_kq_mulmm_msl : (fmt == 34 ? MetalKqMulMmIq3xxs_metal_kq_mulmm_msl : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl : MetalKqMulMmK6_metal_kq_mulmm_msl))))))))) let entry = (tensor ? (tall ? (fmt == 4 ? MetalKqMulMmK4TH128_metal_kq_mulmm_t_msl_entry : (fmt == 5 ? MetalKqMulMmK5TH128_metal_kq_mulmm_t_msl_entry : MetalKqMulMmK6TH128_metal_kq_mulmm_k6_t_msl_entry)) : halfx ? (fmt == 4 ? MetalKqMulMmK4TH_metal_kq_mulmm_t_msl_entry : (fmt == 5 ? MetalKqMulMmK5TH_metal_kq_mulmm_t_msl_entry : MetalKqMulMmK6TH_metal_kq_mulmm_k6_t_msl_entry)) : (fmt == 4 ? MetalKqMulMmK4T_metal_kq_mulmm_t_msl_entry : (fmt == 5 ? MetalKqMulMmK5T_metal_kq_mulmm_t_msl_entry : MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl_entry))) - : (fmt == 45 ? MetalKqMulMmIq4nl_metal_kq_mulmm_msl_entry : (fmt == 34 ? MetalKqMulMmIq3xxs_metal_kq_mulmm_msl_entry : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl_entry : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl_entry : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl_entry : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_entry : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_entry : MetalKqMulMmK6_metal_kq_mulmm_msl_entry)))))))) + : (fmt == 45 ? MetalKqMulMmIq4nl_metal_kq_mulmm_msl_entry : (fmt == 2 ? MetalKqMulMmK2_metal_kq_mulmm_msl_entry : (fmt == 34 ? MetalKqMulMmIq3xxs_metal_kq_mulmm_msl_entry : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl_entry : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl_entry : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl_entry : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_entry : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_entry : MetalKqMulMmK6_metal_kq_mulmm_msl_entry))))))))) let fm = (tensor ? (tall ? (fmt == 4 ? MetalKqMulMmK4TH128_metal_kq_mulmm_t_msl_fastmath : (fmt == 5 ? MetalKqMulMmK5TH128_metal_kq_mulmm_t_msl_fastmath : MetalKqMulMmK6TH128_metal_kq_mulmm_k6_t_msl_fastmath)) : halfx ? (fmt == 4 ? MetalKqMulMmK4TH_metal_kq_mulmm_t_msl_fastmath : (fmt == 5 ? MetalKqMulMmK5TH_metal_kq_mulmm_t_msl_fastmath : MetalKqMulMmK6TH_metal_kq_mulmm_k6_t_msl_fastmath)) : (fmt == 4 ? MetalKqMulMmK4T_metal_kq_mulmm_t_msl_fastmath : (fmt == 5 ? MetalKqMulMmK5T_metal_kq_mulmm_t_msl_fastmath : MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl_fastmath))) - : (fmt == 45 ? MetalKqMulMmIq4nl_metal_kq_mulmm_msl_fastmath : (fmt == 34 ? MetalKqMulMmIq3xxs_metal_kq_mulmm_msl_fastmath : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl_fastmath : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl_fastmath : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl_fastmath : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_fastmath : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_fastmath : MetalKqMulMmK6_metal_kq_mulmm_msl_fastmath)))))))) + : (fmt == 45 ? MetalKqMulMmIq4nl_metal_kq_mulmm_msl_fastmath : (fmt == 2 ? MetalKqMulMmK2_metal_kq_mulmm_msl_fastmath : (fmt == 34 ? MetalKqMulMmIq3xxs_metal_kq_mulmm_msl_fastmath : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl_fastmath : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl_fastmath : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl_fastmath : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_fastmath : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_fastmath : MetalKqMulMmK6_metal_kq_mulmm_msl_fastmath))))))))) let tgm = (tensor ? (tall ? (fmt == 4 ? MetalKqMulMmK4TH128_metal_kq_mulmm_t_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5TH128_metal_kq_mulmm_t_msl_tgmem : MetalKqMulMmK6TH128_metal_kq_mulmm_k6_t_msl_tgmem)) : halfx ? (fmt == 4 ? MetalKqMulMmK4TH_metal_kq_mulmm_t_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5TH_metal_kq_mulmm_t_msl_tgmem : MetalKqMulMmK6TH_metal_kq_mulmm_k6_t_msl_tgmem)) : (fmt == 4 ? MetalKqMulMmK4T_metal_kq_mulmm_t_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5T_metal_kq_mulmm_t_msl_tgmem : MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl_tgmem))) - : (fmt == 45 ? MetalKqMulMmIq4nl_metal_kq_mulmm_msl_tgmem : (fmt == 34 ? MetalKqMulMmIq3xxs_metal_kq_mulmm_msl_tgmem : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl_tgmem : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl_tgmem : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl_tgmem : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_tgmem : MetalKqMulMmK6_metal_kq_mulmm_msl_tgmem)))))))) + : (fmt == 45 ? MetalKqMulMmIq4nl_metal_kq_mulmm_msl_tgmem : (fmt == 2 ? MetalKqMulMmK2_metal_kq_mulmm_msl_tgmem : (fmt == 34 ? MetalKqMulMmIq3xxs_metal_kq_mulmm_msl_tgmem : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl_tgmem : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl_tgmem : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl_tgmem : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_tgmem : MetalKqMulMmK6_metal_kq_mulmm_msl_tgmem))))))))) var pso = pipeline_from_source(dev, src, entry, fm, err) if (tensor && pso == null && !metal4_tensor_available(dev)) { to_log(LOG_INFO, "{tag}: no Metal-4 tensor toolchain on this box - arm skipped\n") @@ -123,7 +123,7 @@ def private kq_mulmm_gate(t : T?; dev, queue; fmt : int; tensor : bool; m, kdim, let ran = with_compute_encoder(queue, err) $(enc : MetalComputeEncoder?) { metal_set_pipeline(enc, pso) metal_set_threadgroup_memory_length(enc, tgm, 0) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs: the f16 d tail + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs: the f16 d tail metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bx, 0ul, 3) @@ -134,7 +134,7 @@ def private kq_mulmm_gate(t : T?; dev, queue; fmt : int; tensor : bool; m, kdim, if (rem_pso != null) { metal_set_pipeline(enc, rem_pso) metal_set_threadgroup_memory_length(enc, fmt == 4 ? MetalKqMulMmK4TH_metal_kq_mulmm_t_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5TH_metal_kq_mulmm_t_msl_tgmem : MetalKqMulMmK6TH_metal_kq_mulmm_k6_t_msl_tgmem), 0) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 ? uint64(nsb * 16) : 0ul, 0) + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 ? uint64(nsb * 16) : 0ul, 0) metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bx, uint64(r0 * kdim * 2), 3) @@ -484,7 +484,7 @@ def private kq_devw_gate(t : T?; dev, queue; fmt : int; m, kdim, ndim : int) { var dka = CvtArgs(total = uint(nblk)) let ran = with_compute_encoder(queue, err) $(enc : MetalComputeEncoder?) { metal_set_pipeline(enc, dq_pso) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs: the f16 d tail + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs: the f16 d tail metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bwh, 0ul, 3) @@ -1461,7 +1461,7 @@ def private moe_mulmm_kq_gate(t : T?; dev, queue; fmt : int; kdim, ndim : int; c let ran = with_compute_encoder(queue, err) $(enc : MetalComputeEncoder?) { metal_set_pipeline(enc, pso) metal_set_threadgroup_memory_length(enc, tgm, 0) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs: the f16 d tail + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs: the f16 d tail metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bx, 0ul, 3) @@ -1498,7 +1498,7 @@ def private moe_mulmm_kq_gate(t : T?; dev, queue; fmt : int; kdim, ndim : int; c metal_set_threadgroup_memory_length(enc, fmt == 4 ? MetalMoeMulMmK4TH_metal_moe_mulmm_k45_t_msl_tgmem : (fmt == 5 ? MetalMoeMulMmK5TH_metal_moe_mulmm_k45_t_msl_tgmem : MetalMoeMulMmK6TH_metal_moe_mulmm_k6_t_msl_tgmem), 0) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 ? uint64(nsb * 16) : 0ul, 0) + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 ? uint64(nsb * 16) : 0ul, 0) metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bxh, 0ul, 3) @@ -1538,7 +1538,7 @@ def private moe_mulmm_kq_gate(t : T?; dev, queue; fmt : int; kdim, ndim : int; c metal_set_threadgroup_memory_length(enc, fmt == 4 ? MetalMoeMulMmK4TH128_metal_moe_mulmm_k45_t_msl_tgmem : (fmt == 5 ? MetalMoeMulMmK5TH128_metal_moe_mulmm_k45_t_msl_tgmem : MetalMoeMulMmK6TH128_metal_moe_mulmm_k6_t_msl_tgmem), 0) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 ? uint64(nsb * 16) : 0ul, 0) + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 ? uint64(nsb * 16) : 0ul, 0) metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bxh, 0ul, 3) @@ -1551,7 +1551,7 @@ def private moe_mulmm_kq_gate(t : T?; dev, queue; fmt : int; kdim, ndim : int; c metal_set_threadgroup_memory_length(enc, fmt == 4 ? MetalMoeMulMmK4THR_metal_moe_mulmm_k45_t_msl_tgmem : (fmt == 5 ? MetalMoeMulMmK5THR_metal_moe_mulmm_k45_t_msl_tgmem : MetalMoeMulMmK6THR_metal_moe_mulmm_k6_t_msl_tgmem), 0) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 ? uint64(nsb * 16) : 0ul, 0) + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 ? uint64(nsb * 16) : 0ul, 0) metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bxh, 0ul, 3) @@ -2057,6 +2057,8 @@ def test_metal_gemm_kernels(t : T?) { kq_mulmm_gate(t, dev, queue, 34, false, 32, 256, 128) kq_mulmm_gate(t, dev, queue, 45, false, 64, 512, 64) // iq4nl: the base form only (no tensor / tall twins) kq_mulmm_gate(t, dev, queue, 45, false, 32, 256, 128) + kq_mulmm_gate(t, dev, queue, 2, false, 64, 512, 64) // k2: the base form only (no tensor / tall twins) + kq_mulmm_gate(t, dev, queue, 2, false, 32, 256, 128) q8_mulmm_t_gate(t, dev, queue, 64, 256, 64) q8_mulmm_t_gate(t, dev, queue, 64, 256, 64, halfx = true) q8_mulmm_t_gate(t, dev, queue, 64, 160, 64) // kdim % 64 == 32: the bk=64 helper's 32-tail chunk diff --git a/modules/dasLLAMA/tests/test_metal_gemv_kernels.das b/modules/dasLLAMA/tests/test_metal_gemv_kernels.das index ba37c2b828..23c4946f4c 100644 --- a/modules/dasLLAMA/tests/test_metal_gemv_kernels.das +++ b/modules/dasLLAMA/tests/test_metal_gemv_kernels.das @@ -23,12 +23,12 @@ require math // ===== single-stream kq GEMV (MetalKqGemvK4 / K5 / K5C / K6) ===== def private kq_gemv_gate(t : T?; dev, queue; vform : string; n, d : int) { - let fmt = vform == "k4" ? 4 : (vform == "k6" ? 6 : (vform == "iq4xs" ? 44 : (vform == "k3" ? 3 : (vform == "iq3s" ? 33 : (vform == "iq3xxs" ? 34 : (vform == "iq4nl" ? 45 : 5)))))) + let fmt = vform == "k4" ? 4 : (vform == "k6" ? 6 : (vform == "iq4xs" ? 44 : (vform == "k3" ? 3 : (vform == "iq3s" ? 33 : (vform == "iq3xxs" ? 34 : (vform == "iq4nl" ? 45 : (vform == "k2" ? 2 : 5))))))) let tag = "kq_gemv_{vform} n={n} d={d}" var err : string - let src = vform == "iq4nl" ? metal_kq_gemv_iq4nl_msl : (vform == "iq3xxs" ? metal_kq_gemv_iq3xxs_msl : (vform == "iq3s" ? metal_kq_gemv_iq3s_msl : (vform == "k3" ? metal_kq_gemv_k3_msl : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl : (vform == "k4" ? metal_kq_gemv_k4_msl : (vform == "k5" ? metal_kq_gemv_k5_msl : (vform == "k5c" ? metal_kq_gemv_k5c_msl : metal_kq_gemv_k6_msl))))))) - let entry = vform == "iq4nl" ? metal_kq_gemv_iq4nl_msl_entry : (vform == "iq3xxs" ? metal_kq_gemv_iq3xxs_msl_entry : (vform == "iq3s" ? metal_kq_gemv_iq3s_msl_entry : (vform == "k3" ? metal_kq_gemv_k3_msl_entry : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl_entry : (vform == "k4" ? metal_kq_gemv_k4_msl_entry : (vform == "k5" ? metal_kq_gemv_k5_msl_entry : (vform == "k5c" ? metal_kq_gemv_k5c_msl_entry : metal_kq_gemv_k6_msl_entry))))))) - let fm = vform == "iq4nl" ? metal_kq_gemv_iq4nl_msl_fastmath : (vform == "iq3xxs" ? metal_kq_gemv_iq3xxs_msl_fastmath : (vform == "iq3s" ? metal_kq_gemv_iq3s_msl_fastmath : (vform == "k3" ? metal_kq_gemv_k3_msl_fastmath : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl_fastmath : (vform == "k4" ? metal_kq_gemv_k4_msl_fastmath : (vform == "k5" ? metal_kq_gemv_k5_msl_fastmath : (vform == "k5c" ? metal_kq_gemv_k5c_msl_fastmath : metal_kq_gemv_k6_msl_fastmath))))))) + let src = vform == "k2" ? metal_kq_gemv_k2_msl : (vform == "iq4nl" ? metal_kq_gemv_iq4nl_msl : (vform == "iq3xxs" ? metal_kq_gemv_iq3xxs_msl : (vform == "iq3s" ? metal_kq_gemv_iq3s_msl : (vform == "k3" ? metal_kq_gemv_k3_msl : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl : (vform == "k4" ? metal_kq_gemv_k4_msl : (vform == "k5" ? metal_kq_gemv_k5_msl : (vform == "k5c" ? metal_kq_gemv_k5c_msl : metal_kq_gemv_k6_msl)))))))) + let entry = vform == "k2" ? metal_kq_gemv_k2_msl_entry : (vform == "iq4nl" ? metal_kq_gemv_iq4nl_msl_entry : (vform == "iq3xxs" ? metal_kq_gemv_iq3xxs_msl_entry : (vform == "iq3s" ? metal_kq_gemv_iq3s_msl_entry : (vform == "k3" ? metal_kq_gemv_k3_msl_entry : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl_entry : (vform == "k4" ? metal_kq_gemv_k4_msl_entry : (vform == "k5" ? metal_kq_gemv_k5_msl_entry : (vform == "k5c" ? metal_kq_gemv_k5c_msl_entry : metal_kq_gemv_k6_msl_entry)))))))) + let fm = vform == "k2" ? metal_kq_gemv_k2_msl_fastmath : (vform == "iq4nl" ? metal_kq_gemv_iq4nl_msl_fastmath : (vform == "iq3xxs" ? metal_kq_gemv_iq3xxs_msl_fastmath : (vform == "iq3s" ? metal_kq_gemv_iq3s_msl_fastmath : (vform == "k3" ? metal_kq_gemv_k3_msl_fastmath : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl_fastmath : (vform == "k4" ? metal_kq_gemv_k4_msl_fastmath : (vform == "k5" ? metal_kq_gemv_k5_msl_fastmath : (vform == "k5c" ? metal_kq_gemv_k5c_msl_fastmath : metal_kq_gemv_k6_msl_fastmath)))))))) var pso = pipeline_from_source(dev, src, entry, fm, err) t |> success(pso != null, "{tag}: pipeline: {err}") return if (pso == null) @@ -81,7 +81,7 @@ def private kq_gemv_gate(t : T?; dev, queue; vform : string; n, d : int) { if (vform == "iq4nl") { metal_set_threadgroup_memory_length(enc, metal_kq_gemv_iq4nl_msl_tgmem, 0) // the codebook slab } - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs/k3: the f16 d tail + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs/k3: the f16 d tail metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bx, 0ul, 3) @@ -114,15 +114,15 @@ def private kq_gemv_gate(t : T?; dev, queue; vform : string; n, d : int) { def private kq_mvb_gate(t : T?; dev, queue; fmt, bwidth, n, d, nr, ys : int) { let tag = "kq_mvb{bwidth}_k{fmt} n={n} d={d} nr={nr} ys={ys}" var err : string - let src = (bwidth == 2 ? (fmt == 45 ? MetalKqMvB2Iq4nl_metal_kq_mv_iq4nl_msl : (fmt == 34 ? MetalKqMvB2Iq3xxs_metal_kq_mv_iq3xxs_msl : (fmt == 33 ? MetalKqMvB2Iq3s_metal_kq_mv_iq3s_msl : (fmt == 3 ? MetalKqMvB2K3_metal_kq_mv_k3_msl : (fmt == 44 ? MetalKqMvB2Iq4xs_metal_kq_mv_iq4xs_msl : (fmt == 4 ? MetalKqMvB2K4_metal_kq_mv_k4_msl : (fmt == 5 ? MetalKqMvB2K5_metal_kq_mv_k5_msl : MetalKqMvB2K6_metal_kq_mv_k6_msl))))))) - : (bwidth == 4 ? (fmt == 45 ? MetalKqMvB4Iq4nl_metal_kq_mv_iq4nl_msl : (fmt == 34 ? MetalKqMvB4Iq3xxs_metal_kq_mv_iq3xxs_msl : (fmt == 33 ? MetalKqMvB4Iq3s_metal_kq_mv_iq3s_msl : (fmt == 3 ? MetalKqMvB4K3_metal_kq_mv_k3_msl : (fmt == 44 ? MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl : (fmt == 4 ? MetalKqMvB4K4_metal_kq_mv_k4_msl : (fmt == 5 ? MetalKqMvB4K5_metal_kq_mv_k5_msl : MetalKqMvB4K6_metal_kq_mv_k6_msl))))))) - : (fmt == 45 ? metal_kq_mvb8_iq4nl_msl : (fmt == 34 ? metal_kq_mvb8_iq3xxs_msl : (fmt == 33 ? metal_kq_mvb8_iq3s_msl : (fmt == 3 ? metal_kq_mvb8_k3_msl : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl : metal_kq_mvb8_k6_msl))))))))) - let entry = (bwidth == 2 ? (fmt == 45 ? MetalKqMvB2Iq4nl_metal_kq_mv_iq4nl_msl_entry : (fmt == 34 ? MetalKqMvB2Iq3xxs_metal_kq_mv_iq3xxs_msl_entry : (fmt == 33 ? MetalKqMvB2Iq3s_metal_kq_mv_iq3s_msl_entry : (fmt == 3 ? MetalKqMvB2K3_metal_kq_mv_k3_msl_entry : (fmt == 44 ? MetalKqMvB2Iq4xs_metal_kq_mv_iq4xs_msl_entry : (fmt == 4 ? MetalKqMvB2K4_metal_kq_mv_k4_msl_entry : (fmt == 5 ? MetalKqMvB2K5_metal_kq_mv_k5_msl_entry : MetalKqMvB2K6_metal_kq_mv_k6_msl_entry))))))) - : (bwidth == 4 ? (fmt == 45 ? MetalKqMvB4Iq4nl_metal_kq_mv_iq4nl_msl_entry : (fmt == 34 ? MetalKqMvB4Iq3xxs_metal_kq_mv_iq3xxs_msl_entry : (fmt == 33 ? MetalKqMvB4Iq3s_metal_kq_mv_iq3s_msl_entry : (fmt == 3 ? MetalKqMvB4K3_metal_kq_mv_k3_msl_entry : (fmt == 44 ? MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl_entry : (fmt == 4 ? MetalKqMvB4K4_metal_kq_mv_k4_msl_entry : (fmt == 5 ? MetalKqMvB4K5_metal_kq_mv_k5_msl_entry : MetalKqMvB4K6_metal_kq_mv_k6_msl_entry))))))) - : (fmt == 45 ? metal_kq_mvb8_iq4nl_msl_entry : (fmt == 34 ? metal_kq_mvb8_iq3xxs_msl_entry : (fmt == 33 ? metal_kq_mvb8_iq3s_msl_entry : (fmt == 3 ? metal_kq_mvb8_k3_msl_entry : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl_entry : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_entry : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_entry : metal_kq_mvb8_k6_msl_entry))))))))) - let fm = (bwidth == 2 ? (fmt == 45 ? MetalKqMvB2Iq4nl_metal_kq_mv_iq4nl_msl_fastmath : (fmt == 34 ? MetalKqMvB2Iq3xxs_metal_kq_mv_iq3xxs_msl_fastmath : (fmt == 33 ? MetalKqMvB2Iq3s_metal_kq_mv_iq3s_msl_fastmath : (fmt == 3 ? MetalKqMvB2K3_metal_kq_mv_k3_msl_fastmath : (fmt == 44 ? MetalKqMvB2Iq4xs_metal_kq_mv_iq4xs_msl_fastmath : (fmt == 4 ? MetalKqMvB2K4_metal_kq_mv_k4_msl_fastmath : (fmt == 5 ? MetalKqMvB2K5_metal_kq_mv_k5_msl_fastmath : MetalKqMvB2K6_metal_kq_mv_k6_msl_fastmath))))))) - : (bwidth == 4 ? (fmt == 45 ? MetalKqMvB4Iq4nl_metal_kq_mv_iq4nl_msl_fastmath : (fmt == 34 ? MetalKqMvB4Iq3xxs_metal_kq_mv_iq3xxs_msl_fastmath : (fmt == 33 ? MetalKqMvB4Iq3s_metal_kq_mv_iq3s_msl_fastmath : (fmt == 3 ? MetalKqMvB4K3_metal_kq_mv_k3_msl_fastmath : (fmt == 44 ? MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl_fastmath : (fmt == 4 ? MetalKqMvB4K4_metal_kq_mv_k4_msl_fastmath : (fmt == 5 ? MetalKqMvB4K5_metal_kq_mv_k5_msl_fastmath : MetalKqMvB4K6_metal_kq_mv_k6_msl_fastmath))))))) - : (fmt == 45 ? metal_kq_mvb8_iq4nl_msl_fastmath : (fmt == 34 ? metal_kq_mvb8_iq3xxs_msl_fastmath : (fmt == 33 ? metal_kq_mvb8_iq3s_msl_fastmath : (fmt == 3 ? metal_kq_mvb8_k3_msl_fastmath : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl_fastmath : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_fastmath : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_fastmath : metal_kq_mvb8_k6_msl_fastmath))))))))) + let src = (bwidth == 2 ? (fmt == 45 ? MetalKqMvB2Iq4nl_metal_kq_mv_iq4nl_msl : (fmt == 2 ? MetalKqMvB2K2_metal_kq_mv_k2_msl : (fmt == 34 ? MetalKqMvB2Iq3xxs_metal_kq_mv_iq3xxs_msl : (fmt == 33 ? MetalKqMvB2Iq3s_metal_kq_mv_iq3s_msl : (fmt == 3 ? MetalKqMvB2K3_metal_kq_mv_k3_msl : (fmt == 44 ? MetalKqMvB2Iq4xs_metal_kq_mv_iq4xs_msl : (fmt == 4 ? MetalKqMvB2K4_metal_kq_mv_k4_msl : (fmt == 5 ? MetalKqMvB2K5_metal_kq_mv_k5_msl : MetalKqMvB2K6_metal_kq_mv_k6_msl)))))))) + : (bwidth == 4 ? (fmt == 45 ? MetalKqMvB4Iq4nl_metal_kq_mv_iq4nl_msl : (fmt == 2 ? MetalKqMvB4K2_metal_kq_mv_k2_msl : (fmt == 34 ? MetalKqMvB4Iq3xxs_metal_kq_mv_iq3xxs_msl : (fmt == 33 ? MetalKqMvB4Iq3s_metal_kq_mv_iq3s_msl : (fmt == 3 ? MetalKqMvB4K3_metal_kq_mv_k3_msl : (fmt == 44 ? MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl : (fmt == 4 ? MetalKqMvB4K4_metal_kq_mv_k4_msl : (fmt == 5 ? MetalKqMvB4K5_metal_kq_mv_k5_msl : MetalKqMvB4K6_metal_kq_mv_k6_msl)))))))) + : (fmt == 45 ? metal_kq_mvb8_iq4nl_msl : (fmt == 2 ? metal_kq_mvb8_k2_msl : (fmt == 34 ? metal_kq_mvb8_iq3xxs_msl : (fmt == 33 ? metal_kq_mvb8_iq3s_msl : (fmt == 3 ? metal_kq_mvb8_k3_msl : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl : metal_kq_mvb8_k6_msl)))))))))) + let entry = (bwidth == 2 ? (fmt == 45 ? MetalKqMvB2Iq4nl_metal_kq_mv_iq4nl_msl_entry : (fmt == 2 ? MetalKqMvB2K2_metal_kq_mv_k2_msl_entry : (fmt == 34 ? MetalKqMvB2Iq3xxs_metal_kq_mv_iq3xxs_msl_entry : (fmt == 33 ? MetalKqMvB2Iq3s_metal_kq_mv_iq3s_msl_entry : (fmt == 3 ? MetalKqMvB2K3_metal_kq_mv_k3_msl_entry : (fmt == 44 ? MetalKqMvB2Iq4xs_metal_kq_mv_iq4xs_msl_entry : (fmt == 4 ? MetalKqMvB2K4_metal_kq_mv_k4_msl_entry : (fmt == 5 ? MetalKqMvB2K5_metal_kq_mv_k5_msl_entry : MetalKqMvB2K6_metal_kq_mv_k6_msl_entry)))))))) + : (bwidth == 4 ? (fmt == 45 ? MetalKqMvB4Iq4nl_metal_kq_mv_iq4nl_msl_entry : (fmt == 2 ? MetalKqMvB4K2_metal_kq_mv_k2_msl_entry : (fmt == 34 ? MetalKqMvB4Iq3xxs_metal_kq_mv_iq3xxs_msl_entry : (fmt == 33 ? MetalKqMvB4Iq3s_metal_kq_mv_iq3s_msl_entry : (fmt == 3 ? MetalKqMvB4K3_metal_kq_mv_k3_msl_entry : (fmt == 44 ? MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl_entry : (fmt == 4 ? MetalKqMvB4K4_metal_kq_mv_k4_msl_entry : (fmt == 5 ? MetalKqMvB4K5_metal_kq_mv_k5_msl_entry : MetalKqMvB4K6_metal_kq_mv_k6_msl_entry)))))))) + : (fmt == 45 ? metal_kq_mvb8_iq4nl_msl_entry : (fmt == 2 ? metal_kq_mvb8_k2_msl_entry : (fmt == 34 ? metal_kq_mvb8_iq3xxs_msl_entry : (fmt == 33 ? metal_kq_mvb8_iq3s_msl_entry : (fmt == 3 ? metal_kq_mvb8_k3_msl_entry : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl_entry : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_entry : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_entry : metal_kq_mvb8_k6_msl_entry)))))))))) + let fm = (bwidth == 2 ? (fmt == 45 ? MetalKqMvB2Iq4nl_metal_kq_mv_iq4nl_msl_fastmath : (fmt == 2 ? MetalKqMvB2K2_metal_kq_mv_k2_msl_fastmath : (fmt == 34 ? MetalKqMvB2Iq3xxs_metal_kq_mv_iq3xxs_msl_fastmath : (fmt == 33 ? MetalKqMvB2Iq3s_metal_kq_mv_iq3s_msl_fastmath : (fmt == 3 ? MetalKqMvB2K3_metal_kq_mv_k3_msl_fastmath : (fmt == 44 ? MetalKqMvB2Iq4xs_metal_kq_mv_iq4xs_msl_fastmath : (fmt == 4 ? MetalKqMvB2K4_metal_kq_mv_k4_msl_fastmath : (fmt == 5 ? MetalKqMvB2K5_metal_kq_mv_k5_msl_fastmath : MetalKqMvB2K6_metal_kq_mv_k6_msl_fastmath)))))))) + : (bwidth == 4 ? (fmt == 45 ? MetalKqMvB4Iq4nl_metal_kq_mv_iq4nl_msl_fastmath : (fmt == 2 ? MetalKqMvB4K2_metal_kq_mv_k2_msl_fastmath : (fmt == 34 ? MetalKqMvB4Iq3xxs_metal_kq_mv_iq3xxs_msl_fastmath : (fmt == 33 ? MetalKqMvB4Iq3s_metal_kq_mv_iq3s_msl_fastmath : (fmt == 3 ? MetalKqMvB4K3_metal_kq_mv_k3_msl_fastmath : (fmt == 44 ? MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl_fastmath : (fmt == 4 ? MetalKqMvB4K4_metal_kq_mv_k4_msl_fastmath : (fmt == 5 ? MetalKqMvB4K5_metal_kq_mv_k5_msl_fastmath : MetalKqMvB4K6_metal_kq_mv_k6_msl_fastmath)))))))) + : (fmt == 45 ? metal_kq_mvb8_iq4nl_msl_fastmath : (fmt == 2 ? metal_kq_mvb8_k2_msl_fastmath : (fmt == 34 ? metal_kq_mvb8_iq3xxs_msl_fastmath : (fmt == 33 ? metal_kq_mvb8_iq3s_msl_fastmath : (fmt == 3 ? metal_kq_mvb8_k3_msl_fastmath : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl_fastmath : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_fastmath : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_fastmath : metal_kq_mvb8_k6_msl_fastmath)))))))))) var pso = pipeline_from_source(dev, src, entry, fm, err) t |> success(pso != null, "{tag}: pipeline: {err}") return if (pso == null) @@ -172,9 +172,9 @@ def private kq_mvb_gate(t : T?; dev, queue; fmt, bwidth, n, d, nr, ys : int) { metal_set_pipeline(enc, pso) if (bwidth == 8) { metal_set_threadgroup_memory_length(enc, - fmt == 45 ? metal_kq_mvb8_iq4nl_msl_tgmem : (fmt == 34 ? metal_kq_mvb8_iq3xxs_msl_tgmem : (fmt == 33 ? metal_kq_mvb8_iq3s_msl_tgmem : (fmt == 3 ? metal_kq_mvb8_k3_msl_tgmem : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl_tgmem : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_tgmem : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_tgmem : metal_kq_mvb8_k6_msl_tgmem)))))), 0) + fmt == 2 ? metal_kq_mvb8_k2_msl_tgmem : (fmt == 45 ? metal_kq_mvb8_iq4nl_msl_tgmem : (fmt == 34 ? metal_kq_mvb8_iq3xxs_msl_tgmem : (fmt == 33 ? metal_kq_mvb8_iq3s_msl_tgmem : (fmt == 3 ? metal_kq_mvb8_k3_msl_tgmem : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl_tgmem : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_tgmem : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_tgmem : metal_kq_mvb8_k6_msl_tgmem))))))), 0) } - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs/k3: the f16 d tail + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs/k3: the f16 d tail metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bx, 0ul, 3) @@ -263,7 +263,7 @@ def private moe_kq_gemv_gate(t : T?; dev, queue; fmt, n, d, net, k, nst : int; p bxs = uint((perslot ? k : 1) * n), bys = uint(k * d), hasb = 0u) let ran = with_compute_encoder(queue, err) $(enc : MetalComputeEncoder?) { metal_set_pipeline(enc, pso) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs/k3: the f16 d tail + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs/k3: the f16 d tail metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bx, 0ul, 3) @@ -743,8 +743,10 @@ def test_metal_kq_gemv_kernels(t : T?) { kq_gemv_gate(t, dev, queue, "iq3xxs", 512, 30) kq_gemv_gate(t, dev, queue, "iq4nl", 1280, 11) kq_gemv_gate(t, dev, queue, "iq4nl", 512, 30) + kq_gemv_gate(t, dev, queue, "k2", 1280, 11) + kq_gemv_gate(t, dev, queue, "k2", 512, 30) // the small-batch twins: nr = 2 / 3 (b4 col guard) / 6 (b8 pad), ys > d - for (fmt in [4, 5, 6, 44, 3, 33, 34, 45]) { + for (fmt in [4, 5, 6, 44, 3, 33, 34, 45, 2]) { kq_mvb_gate(t, dev, queue, fmt, 2, 768, 17, 2, 22) kq_mvb_gate(t, dev, queue, fmt, 4, 768, 17, 3, 22) kq_mvb_gate(t, dev, queue, fmt, 8, 768, 17, 6, 22) From d70a4afa8b017df2de5de402fd0f608702029202 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Mon, 31 Aug 2026 03:03:09 -0700 Subject: [PATCH 34/61] dasLLAMA: drop the stale STYLE038 tag on metal_kq_mv_k2 (the hmask-free body fits the limit) Co-Authored-By: Claude Fable 5 --- modules/dasLLAMA/dasllama/dasllama_metal_kernels.das | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das index ae7aa4543e..41ffb6040f 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das @@ -7371,7 +7371,7 @@ class template MetalKqMvK2T { @template_constant TILED : bool = false [metal_kernel] - def metal_kq_mv_k2 { // nolint:STYLE038 — the TILED twin loops are byte-identity-driven, not splittable + def metal_kq_mv_k2 { let lane = gl_SubgroupInvocationID let tx = lane % 8u let ty = lane / 8u From 0c651658b86e07eab6115b320132db0b254a62f6 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Mon, 31 Aug 2026 03:34:42 -0700 Subject: [PATCH 35/61] dasLLAMA: IQ2_S joins the kq lattice (CPU slice) - the u64-grid tier opens, and with it the first community i1 file The first u64-grid format: enum iq2s (12), kernel id 23, quant plane [32 grid-index bytes][32 sign bytes][8 qh] verbatim (IQ2S_QSB 72), scale row the k4 20B STRIDE with PER-16 strips - f16 d EIGHTH-ed at transcode (three exact exponent decrements: ggml's (0.5+ls)*0.25 fold becomes (2ls+1) x d/8) + 16 x (1+2s) filling the row with no pad. The 8 KB iq2s_grid ships as the worker-safe per-call local iq2s_grid2() (1024 u64 entries as low/high uint pairs - a 10-bit qs|qh index selects EIGHT magnitudes {8, 25, 43}) plus the main-context IQ2S_GRID; signs are the block's own bytes (bit j), the iq3s pattern with a wider gather. dot_iq2s_q8 / rows kernel / grp row dequant / repack (18 uniform 4-byte columns; the k4 scale interleave with 16 strips) / every dispatch ladder arm / declined emitter stubs / probe fixtures (tune list only - QUIRK 16) / test arms across every gate, built as the k2 walk's twin over the current ladders. Gates on the worktree binary: test_kqformat 18/18; test_kquant 233 tests, 216 pass, 17 env-gated skips; lint 0 on the 20-file changed set. End to end: the mradermacher Llama-3.2-1B-Instruct.i1-IQ3_XXS.gguf - the community file that has sat on disk since the iq3xxs phase, blocked on its IQ2_S attn_k/q x32 - now LOADS and decodes a coherent story through FIVE formats in one graph (IQ2_S + IQ3_XXS + IQ3_S + Q4_K + Q5_K), matching llama.cpp's greedy ids 10/64 with the fork a 0.040-logit near-tie (the arc's tightest; top2 IS our token) at gen 41 t/s reference bodies. JIT emitter, Vulkan, Metal: pending. Co-Authored-By: Claude Fable 5 --- modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md | 17 ++ modules/dasLLAMA/dasllama/dasllama_blocks.das | 3 + modules/dasLLAMA/dasllama/dasllama_common.das | 39 ++- modules/dasLLAMA/dasllama/dasllama_config.das | 3 +- .../dasLLAMA/dasllama/dasllama_convert.das | 45 +++ .../dasLLAMA/dasllama/dasllama_gemm_gen.das | 9 + .../dasllama/dasllama_gemm_schema.das | 2 + modules/dasLLAMA/dasllama/dasllama_gguf.das | 40 +++ .../dasllama/dasllama_gpu_resident.das | 7 + modules/dasLLAMA/dasllama/dasllama_image.das | 7 +- .../dasLLAMA/dasllama/dasllama_kqformat.das | 285 +++++++++++++++++- modules/dasLLAMA/dasllama/dasllama_layout.das | 22 +- modules/dasLLAMA/dasllama/dasllama_load.das | 41 ++- modules/dasLLAMA/dasllama/dasllama_math.das | 9 +- .../dasllama/dasllama_math_default.das | 87 +++++- .../dasLLAMA/dasllama/dasllama_math_gen.das | 105 ++++++- modules/dasLLAMA/dasllama/dasllama_ple.das | 4 + modules/dasLLAMA/dasllama/dasllama_repack.das | 44 +++ modules/dasLLAMA/harness/gen_tune_probe.das | 46 ++- modules/dasLLAMA/tests/test_kqformat.das | 12 +- modules/dasLLAMA/tests/test_kquant.das | 135 +++++++-- 21 files changed, 907 insertions(+), 55 deletions(-) diff --git a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md index d4ec754bd0..f865174b0a 100644 --- a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md +++ b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md @@ -438,6 +438,23 @@ where, why it is so today, what unquirked looks like. An empty ledger is a legit ## Per-format notes +### IQ2_S Phase A (CPU, 2026-08-31) - the u64-grid tier + +Shape: 256-superblock grid format, the first with a u64 grid - a 10-bit index (qs byte | +qh 2 bits) selects EIGHT magnitudes ({8, 25, 43}) from iq2s_grid[1024]; the block's own +sign bytes flip them (bit j - no ksigns table), scales are PER-16 nibbles folded as +(0.5+ls)*0.25 = (2ls+1) x d/8. Disk 82B: [f16 d][32 idx][32 signs][8 qh][8 scale-nibble +bytes]. Planes: [idx][signs][qh] verbatim (IQ2S_QSB 72), scale row the k4 20B STRIDE with +d EIGHTH-ed at transcode (three f16_half_bits) + 16 strips (1+2s) - the row fills exactly, +no pad. Ids: KqFmt.iq2s = 12, kernel id 23, stream code 23 (clears QUIRK 25's 0/1/2). The +grid ships as iq2s_grid2() - the per-call local at 8 KB (2048 uint words, low/high pairs) - +plus the main-context IQ2S_GRID. Repack: 18 uniform 4-byte columns; the k4 scale interleave +with 16 strips. Gates: test_kqformat 18/18, test_kquant 233 (216 pass, 17 env skips), +lint 0. E2e: the mradermacher i1-IQ3_XXS vehicle (on disk since the iq3xxs phase, blocked +on its IQ2_S attn x32) now loads and decodes FIVE formats in one graph - ids 10/64 vs +llama.cpp with the fork a 0.040-logit near-tie (the arc's tightest; top2 IS our token), +gen 41 t/s reference bodies. JIT emitter, Vulkan, Metal: pending. + ### Q2_K Phase A (CPU, 2026-08-31) Shape: 256-superblock, the k4/k5 scale STRUCTURE at k6's granularity - 16 per-16-element diff --git a/modules/dasLLAMA/dasllama/dasllama_blocks.das b/modules/dasLLAMA/dasllama/dasllama_blocks.das index 84b631b712..0310519500 100644 --- a/modules/dasLLAMA/dasllama/dasllama_blocks.das +++ b/modules/dasLLAMA/dasllama/dasllama_blocks.das @@ -1418,6 +1418,9 @@ def private kq_bytes_per_weight(f : KqFmt) : float { if (f == KqFmt.k2) { return 84.0 / 256.0 // 64B quants + the 20B scale row per 256 } + if (f == KqFmt.iq2s) { + return 92.0 / 256.0 // 72B quants + the 20B scale row per 256 + } return 34.0 / 32.0 // q8: 32 quants + an f16 scale per block } diff --git a/modules/dasLLAMA/dasllama/dasllama_common.das b/modules/dasLLAMA/dasllama/dasllama_common.das index 0a0c5f7ff7..1ee57818cc 100644 --- a/modules/dasLLAMA/dasllama/dasllama_common.das +++ b/modules/dasLLAMA/dasllama/dasllama_common.das @@ -853,6 +853,9 @@ struct Model { // the k2 tier's plane pair: 64 qs bytes per superblock + the 20B scale row [f16 d][f16 dmin][16 sc/min bytes] k2q : array k2s : array + // the iq2s tier's plane pair: [32 idx][32 signs][8 qh] per superblock + the 20B row (f16 d/8, 16 x (1 + 2s)) + iq2sq : array + iq2ss : array kquant_native : bool = false // kq stage 4: the kq planes were repacked at load into the active backend's grp layout. // From then on every kq matmul MUST run the backend slots — disk-order portable kernels would @@ -869,6 +872,7 @@ struct Model { kq_repack_mr34 : int64 = 4l kq_repack_mr45 : int64 = 4l kq_repack_mr2 : int64 = 4l + kq_repack_mr23 : int64 = 4l // per-layer weight format tags (KqFmt; empty = all q8). Filled by detect_kq_formats before // layout_offsets so the layout walks each tensor into its format's cursor. wq_fmt : array @@ -1094,7 +1098,7 @@ def model_weights_bytes(t : Model) : int64 { + long_length(t.iq4xsq) + long_length(t.iq4xss) + long_length(t.k3q) + long_length(t.k3s) + long_length(t.iq3sq) + long_length(t.iq3ss) + long_length(t.iq3xxsq) + long_length(t.iq3xxss) - + long_length(t.iq4nlq) + long_length(t.iq4nls) + long_length(t.k2q) + long_length(t.k2s) + + long_length(t.iq4nlq) + long_length(t.iq4nls) + long_length(t.k2q) + long_length(t.k2s) + long_length(t.iq2sq) + long_length(t.iq2ss) + long_length(t.q51q) + long_length(t.q51s) + long_length(t.mblob) + long_length(t.vkblob) + long_length(t.embq) + long_length(t.embs)) @@ -1489,6 +1493,7 @@ def private dlim_cpu_source_impl(var c : DlimCpuConfig) { c.kq_mr34 = active_kq_layout_mr(34) c.kq_mr45 = active_kq_layout_mr(45) c.kq_mr2 = active_kq_layout_mr(2) + c.kq_mr23 = active_kq_layout_mr(23) c.q51_mr = active_q51_layout_mr() // OUTCOME, not request: a backend without s16 twins keeps f32 scale planes (wscale_convert_f16) c.wscale_f16 = g_wscale_f16 && kernel_backend_has_wscale16() @@ -1581,7 +1586,7 @@ def kq_active_mr(t : Model; f : KqFmt) : int64 { if (!t.kq_repacked) { return 1l } - return f == KqFmt.k4 ? t.kq_repack_mr4 : (f == KqFmt.k5 ? t.kq_repack_mr5 : (f == KqFmt.k6 ? t.kq_repack_mr6 : (f == KqFmt.q40 ? t.kq_repack_mr40 : (f == KqFmt.iq4xs ? t.kq_repack_mr44 : (f == KqFmt.k3 ? t.kq_repack_mr3 : (f == KqFmt.iq3s ? t.kq_repack_mr33 : (f == KqFmt.iq3xxs ? t.kq_repack_mr34 : (f == KqFmt.iq4nl ? t.kq_repack_mr45 : t.kq_repack_mr2)))))))) + return f == KqFmt.k4 ? t.kq_repack_mr4 : (f == KqFmt.k5 ? t.kq_repack_mr5 : (f == KqFmt.k6 ? t.kq_repack_mr6 : (f == KqFmt.q40 ? t.kq_repack_mr40 : (f == KqFmt.iq4xs ? t.kq_repack_mr44 : (f == KqFmt.k3 ? t.kq_repack_mr3 : (f == KqFmt.iq3s ? t.kq_repack_mr33 : (f == KqFmt.iq3xxs ? t.kq_repack_mr34 : (f == KqFmt.iq4nl ? t.kq_repack_mr45 : (f == KqFmt.k2 ? t.kq_repack_mr2 : t.kq_repack_mr23))))))))) } //! Load a llama-architecture GGUF into the split layout at the requested precision — straight into @@ -1644,8 +1649,8 @@ def footprint(t : Model) : MemFootprint { let q4s = long_length(t.q4scales) * 4l let mx4 = long_length(t.mxq) // uint8 (two nibbles) -> 1 byte each let mx4s = long_length(t.mxs) // raw E8M0 -> 1 byte per 32-block - let kq = long_length(t.k4q) + long_length(t.k5q) + long_length(t.k6q) + long_length(t.q40q) + long_length(t.q51q) + long_length(t.iq4xsq) + long_length(t.k3q) + long_length(t.iq3sq) + long_length(t.iq3xxsq) + long_length(t.iq4nlq) + long_length(t.k2q) - let kqs = long_length(t.k4s) + long_length(t.k5s) + long_length(t.k6s) + long_length(t.q40s) + long_length(t.q51s) + long_length(t.iq4xss) + long_length(t.k3s) + long_length(t.iq3ss) + long_length(t.iq3xxss) + long_length(t.iq4nls) + long_length(t.k2s) + let kq = long_length(t.k4q) + long_length(t.k5q) + long_length(t.k6q) + long_length(t.q40q) + long_length(t.q51q) + long_length(t.iq4xsq) + long_length(t.k3q) + long_length(t.iq3sq) + long_length(t.iq3xxsq) + long_length(t.iq4nlq) + long_length(t.k2q) + long_length(t.iq2sq) + let kqs = long_length(t.k4s) + long_length(t.k5s) + long_length(t.k6s) + long_length(t.q40s) + long_length(t.q51s) + long_length(t.iq4xss) + long_length(t.k3s) + long_length(t.iq3ss) + long_length(t.iq3xxss) + long_length(t.iq4nls) + long_length(t.k2s) + long_length(t.iq2ss) return MemFootprint(aux_fp32_bytes = aux, weight_fp32_bytes = wf, q8_bytes = q8, q8_scale_bytes = q8s, q4_bytes = q4, q4_scale_bytes = q4s, mx4_bytes = mx4, mx4_scale_bytes = mx4s, kq_bytes = kq, kq_scale_bytes = kqs, @@ -3319,7 +3324,7 @@ def mm_b_q51_groupn(var y : array; t : Model; offs : array; nregio // KqFmt -> the kernel-layer format id (matmul_kq*/kq_rows_fn/kq_qsb take the int form). q8 maps // to 0, a poison id no kq kernel accepts — kq_rows_for calls this for EVERY tag, so it must stay // total even though q8 branches never read the result. -def private kq_fi(fmt : KqFmt) : int => fmt == KqFmt.k4 ? 4 : (fmt == KqFmt.k5 ? 5 : (fmt == KqFmt.k6 ? 6 : (fmt == KqFmt.q40 ? 40 : (fmt == KqFmt.iq4xs ? 44 : (fmt == KqFmt.k3 ? 3 : (fmt == KqFmt.iq3s ? 33 : (fmt == KqFmt.iq3xxs ? 34 : (fmt == KqFmt.iq4nl ? 45 : (fmt == KqFmt.k2 ? 2 : 0))))))))) +def private kq_fi(fmt : KqFmt) : int => fmt == KqFmt.k4 ? 4 : (fmt == KqFmt.k5 ? 5 : (fmt == KqFmt.k6 ? 6 : (fmt == KqFmt.q40 ? 40 : (fmt == KqFmt.iq4xs ? 44 : (fmt == KqFmt.k3 ? 3 : (fmt == KqFmt.iq3s ? 33 : (fmt == KqFmt.iq3xxs ? 34 : (fmt == KqFmt.iq4nl ? 45 : (fmt == KqFmt.k2 ? 2 : (fmt == KqFmt.iq2s ? 23 : 0)))))))))) // K-quant GEMV against a PRE-quantized activation (xq/xs/xbs already filled by the bs quantizer). // kq_repacked loads run the backend's stamped cores; disk-order loads run the portable kernels. @@ -3345,6 +3350,8 @@ def mm_at_kq_pre(var y : array; t : Model; fmt : KqFmt; woff : int64; xq matmul_kq_active(45, y, t.iq4nlq, t.iq4nls, woff, xq, xs, xbs, n, d, yoff) } elif (fmt == KqFmt.k2) { matmul_kq_active(2, y, t.k2q, t.k2s, woff, xq, xs, xbs, n, d, yoff) + } elif (fmt == KqFmt.iq2s) { + matmul_kq_active(23, y, t.iq2sq, t.iq2ss, woff, xq, xs, xbs, n, d, yoff) } else { panic("mm_at_kq_pre: '{fmt}' has no kq plane pair") // a new KqFmt must claim its arm, never ride q40's } @@ -3368,6 +3375,8 @@ def mm_at_kq_pre(var y : array; t : Model; fmt : KqFmt; woff : int64; xq matmul_kq(45, y, t.iq4nlq, t.iq4nls, woff, xq, xs, xbs, n, d, yoff) } elif (fmt == KqFmt.k2) { matmul_kq(2, y, t.k2q, t.k2s, woff, xq, xs, xbs, n, d, yoff) + } elif (fmt == KqFmt.iq2s) { + matmul_kq(23, y, t.iq2sq, t.iq2ss, woff, xq, xs, xbs, n, d, yoff) } else { panic("mm_at_kq_pre: '{fmt}' has no kq plane pair") } @@ -3411,6 +3420,8 @@ def mm_b_kq(var s : Session; var y : array; t : Model; fmt : KqFmt; woff matmul_kq_batch(45, y, t.iq4nlq, t.iq4nls, woff, s.xqb, s.xsb, s.xbsb, n, d, npos) } elif (fmt == KqFmt.k2) { matmul_kq_batch(2, y, t.k2q, t.k2s, woff, s.xqb, s.xsb, s.xbsb, n, d, npos) + } elif (fmt == KqFmt.iq2s) { + matmul_kq_batch(23, y, t.iq2sq, t.iq2ss, woff, s.xqb, s.xsb, s.xbsb, n, d, npos) } else { panic("mm_b_kq: '{fmt}' has no kq plane pair") } @@ -3450,6 +3461,8 @@ def mm_at_kq_groupn(var y : array; t : Model; fmt : KqFmt; offs : array; t : Model; fmt : KqFmt; offs : array; t : Model; fmt : KqFmt; offs : array; t : Model; fmt : KqFmt; woff : int64; xq : matmul_kq_batch(45, y, t.iq4nlq, t.iq4nls, woff, xq, xs, xbs, n, d, ntok) } elif (fmt == KqFmt.k2) { matmul_kq_batch(2, y, t.k2q, t.k2s, woff, xq, xs, xbs, n, d, ntok) + } elif (fmt == KqFmt.iq2s) { + matmul_kq_batch(23, y, t.iq2sq, t.iq2ss, woff, xq, xs, xbs, n, d, ntok) } else { panic("mm_b_kq_pre: '{fmt}' has no kq plane pair") } @@ -3549,6 +3568,7 @@ def kq_plane_q(t : Model; fmt : KqFmt; woff : int64) : uint8 const? { if (fmt == KqFmt.iq3xxs) return unsafe(addr(t.iq3xxsq[sb * IQ3XXS_QSB])) if (fmt == KqFmt.iq4nl) return unsafe(addr(t.iq4nlq[sb * Q40_QSB])) if (fmt == KqFmt.k2) return unsafe(addr(t.k2q[sb * K2_QSB])) + if (fmt == KqFmt.iq2s) return unsafe(addr(t.iq2sq[sb * IQ2S_QSB])) return null } @@ -3564,6 +3584,7 @@ def kq_plane_s(t : Model; fmt : KqFmt; woff : int64) : uint8 const? { if (fmt == KqFmt.iq3xxs) return unsafe(addr(t.iq3xxss[sb * IQ3XXS_SSB])) if (fmt == KqFmt.iq4nl) return unsafe(addr(t.iq4nls[sb * Q40_SSB])) if (fmt == KqFmt.k2) return unsafe(addr(t.k2s[sb * K2_SSB])) + if (fmt == KqFmt.iq2s) return unsafe(addr(t.iq2ss[sb * IQ2S_SSB])) return null } @@ -3936,6 +3957,8 @@ def embed_row(t : Model; token : int64; var dst : float?) { // nolint:STYLE037 dequant_kq_row_grp(fmt, addr(t.embq[sbg * Q40_QSB]), addr(t.embs[sbg * Q40_SSB]), token % mr, mr, dim, dst) } elif (t.emb_fmt == KqFmt.k2) { dequant_kq_row_grp(fmt, addr(t.embq[sbg * K2_QSB]), addr(t.embs[sbg * K2_SSB]), token % mr, mr, dim, dst) + } elif (t.emb_fmt == KqFmt.iq2s) { + dequant_kq_row_grp(fmt, addr(t.embq[sbg * IQ2S_QSB]), addr(t.embs[sbg * IQ2S_SSB]), token % mr, mr, dim, dst) } else { panic("embed_row: '{t.emb_fmt}' has no kq plane pair") } @@ -3959,6 +3982,8 @@ def embed_row(t : Model; token : int64; var dst : float?) { // nolint:STYLE037 dequant_kq_row_grp(fmt, addr(t.iq4nlq[sbg * Q40_QSB]), addr(t.iq4nls[sbg * Q40_SSB]), token % mr, mr, dim, dst) } elif (t.emb_fmt == KqFmt.k2) { dequant_kq_row_grp(fmt, addr(t.k2q[sbg * K2_QSB]), addr(t.k2s[sbg * K2_SSB]), token % mr, mr, dim, dst) + } elif (t.emb_fmt == KqFmt.iq2s) { + dequant_kq_row_grp(fmt, addr(t.iq2sq[sbg * IQ2S_QSB]), addr(t.iq2ss[sbg * IQ2S_SSB]), token % mr, mr, dim, dst) } else { panic("embed_row: '{t.emb_fmt}' has no kq plane pair") } @@ -3993,6 +4018,8 @@ def embed_row(t : Model; token : int64; var dst : float?) { // nolint:STYLE037 dequant_iq4nl_plane_superblock(t.embq, (sb0 + s) * Q40_QSB, t.embs, (sb0 + s) * Q40_SSB, row, s * 256l) } elif (t.emb_fmt == KqFmt.k2) { dequant_k2_plane_superblock(t.embq, (sb0 + s) * K2_QSB, t.embs, (sb0 + s) * K2_SSB, row, s * 256l) + } elif (t.emb_fmt == KqFmt.iq2s) { + dequant_iq2s_plane_superblock(t.embq, (sb0 + s) * IQ2S_QSB, t.embs, (sb0 + s) * IQ2S_SSB, row, s * 256l) } else { panic("embed_row: '{t.emb_fmt}' has no kq plane pair") } @@ -4020,6 +4047,8 @@ def embed_row(t : Model; token : int64; var dst : float?) { // nolint:STYLE037 dequant_iq4nl_plane_superblock(t.iq4nlq, (sb0 + s) * Q40_QSB, t.iq4nls, (sb0 + s) * Q40_SSB, row, s * 256l) } elif (t.emb_fmt == KqFmt.k2) { dequant_k2_plane_superblock(t.k2q, (sb0 + s) * K2_QSB, t.k2s, (sb0 + s) * K2_SSB, row, s * 256l) + } elif (t.emb_fmt == KqFmt.iq2s) { + dequant_iq2s_plane_superblock(t.iq2sq, (sb0 + s) * IQ2S_QSB, t.iq2ss, (sb0 + s) * IQ2S_SSB, row, s * 256l) } else { panic("embed_row: '{t.emb_fmt}' has no kq plane pair") } diff --git a/modules/dasLLAMA/dasllama/dasllama_config.das b/modules/dasLLAMA/dasllama/dasllama_config.das index d8f0afb8a2..1347dae14b 100644 --- a/modules/dasLLAMA/dasllama/dasllama_config.das +++ b/modules/dasLLAMA/dasllama/dasllama_config.das @@ -33,6 +33,7 @@ struct public DlimCpuConfig { kq_mr34 : int64 kq_mr45 : int64 kq_mr2 : int64 + kq_mr23 : int64 q51_mr : int64 wscale_f16 : bool kquant_native : bool @@ -145,7 +146,7 @@ def public dlim_config_current(quant : string = "q8") : DlimConfiguration { def public dlim_identity(c : DlimConfiguration; image_version : int; tag : string = "") : string { return ("v{image_version}|{c.quant}|{c.cpu.backend}|{c.cpu.wscale_f16 ? "s16" : "s32"}" + "|q8 mr{c.cpu.q8_mr} b{c.cpu.q8_wbias} g{c.cpu.q8_kgroup}" - + "|kq {c.cpu.kq_mr4}/{c.cpu.kq_mr5}/{c.cpu.kq_mr6}/{c.cpu.kq_mr40}/{c.cpu.kq_mr44}/{c.cpu.kq_mr3}/{c.cpu.kq_mr33}/{c.cpu.kq_mr34}/{c.cpu.kq_mr45}/{c.cpu.kq_mr2}" + + "|kq {c.cpu.kq_mr4}/{c.cpu.kq_mr5}/{c.cpu.kq_mr6}/{c.cpu.kq_mr40}/{c.cpu.kq_mr44}/{c.cpu.kq_mr3}/{c.cpu.kq_mr33}/{c.cpu.kq_mr34}/{c.cpu.kq_mr45}/{c.cpu.kq_mr2}/{c.cpu.kq_mr23}" + "|q51 mr{c.cpu.q51_mr}" + "|nat {c.cpu.kquant_native ? 1 : 0}{c.cpu.kq_q40_native ? 1 : 0}{c.cpu.kq_q50_native ? 1 : 0}{c.cpu.kq_q51_native ? 1 : 0}" + (tag != "" ? "|{tag}" : "")) diff --git a/modules/dasLLAMA/dasllama/dasllama_convert.das b/modules/dasLLAMA/dasllama/dasllama_convert.das index 2f3b526dd7..0f210f44a3 100644 --- a/modules/dasLLAMA/dasllama/dasllama_convert.das +++ b/modules/dasLLAMA/dasllama/dasllama_convert.das @@ -868,6 +868,51 @@ def dequant_iq3s_plane_superblock_at(kq : array | #; kqo : int64; ks : ar } } +//! Transcode one IQ2_S superblock (82 B at `bo`: f16 d, 32 idx + 32 sign bytes, 8 qh, 8 packed +//! scale nibbles): [idx][signs][qh] verbatim to kq[kqo..+72), d EIGHTH-ed (three exact exponent +//! decrements - ggml's (0.5+ls)*0.25 becomes (2ls+1) x d/8) + 16 strips (1+2s) to ks[kso..+20). +def transcode_iq2s_superblock(bytes : array | #; bo : int64; var kq : array; kqo : int64; var ks : array; kso : int64) { + for (i in range64(72l)) { + kq[kqo + i] = bytes[bo + 2l + i] + } + let d8 = f16_half_bits(f16_half_bits(f16_half_bits(uint(bytes[bo]) | (uint(bytes[bo + 1l]) << 8u)))) + ks[kso] = uint8(d8 & 0xFFu) + ks[kso + 1l] = uint8(d8 >> 8u) + ks[kso + 2l] = uint8(0) + ks[kso + 3l] = uint8(0) + for (g in range64(16l)) { + let nib = g % 2l == 0l ? int(bytes[bo + 74l + g / 2l]) & 15 : int(bytes[bo + 74l + g / 2l]) >> 4 + ks[kso + 4l + g] = uint8(1 + 2 * nib) + } +} + +//! Reference dequant of one iq2s-plane superblock: w = (d8 * (1 + 2s)) * grid_byte * sign - +//! grid entry idx[b*4+l] | ((qh[b] << (8-2l)) & 0x300) is TWO uint words (8 magnitudes), sign +//! byte signs[b*4+l] bit j, strip = the 16-chunk 2b + l/2. ggml's dequantize_row_iq2_s order. +def dequant_iq2s_plane_superblock(kq : array | #; kqo : int64; ks : array | #; kso : int64; var dst : array | #; doff : int64) { + dequant_iq2s_plane_superblock_at(kq, kqo, ks, kso + 4l, kso, dst, doff) +} + +//! The split-layout twin: `sso` names the 16-strip base, `dso` the f16 d8 halfword. +def dequant_iq2s_plane_superblock_at(kq : array | #; kqo : int64; ks : array | #; sso, dso : int64; var dst : array | #; doff : int64) { + let d8 = f16_to_f32(rd_u16(ks, dso)) + let grid = iq2s_grid2() + for (b in range64(8l)) { + let qh = int(kq[kqo + 64l + b]) + for (l in range64(4l)) { + let db = d8 * float(int(ks[sso + 2l * b + l / 2l])) + let sgn = int(kq[kqo + 32l + b * 4l + l]) + let idx = int(kq[kqo + b * 4l + l]) | ((qh << int(8l - 2l * l)) & 0x300) + let g1 = grid[idx * 2] + let g2 = grid[idx * 2 + 1] + for (j in range64(4l)) { + dst[doff + b * 32l + l * 8l + j] = db * float((g1 >> uint(8l * j)) & 0xFFu) * (((sgn >> int(j)) & 1) != 0 ? -1.0 : 1.0) + dst[doff + b * 32l + l * 8l + 4l + j] = db * float((g2 >> uint(8l * j)) & 0xFFu) * (((sgn >> int(4l + j)) & 1) != 0 ? -1.0 : 1.0) + } + } + } +} + //! Transcode one Q5_1 disk block (24 bytes at `bo`: d f16, m f16, qh u32, 16 nibble bytes) into //! the q51 planes — verbatim splits, exact. The per-block array form the tests drive; the bulk //! loader (gguf_transcode_q51) runs the same split pointerized and threaded. diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das index 80ccfe3b5f..1e87e9be3a 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das @@ -2907,6 +2907,13 @@ def private iq4nl_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, def private k2_gemv_gen(var gc : LlvmCodeCtx) : bool => kq_gemv_gen_impl(gc, 2) def private k2_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 2) +// iq2s: the grid-gather emitter arm lands with Phase B; until then both stamps decline +// (reference bodies serve) - a declined generator is the framework's own fallback path. +[unused_argument(gc)] +def private iq2s_gemv_gen(var gc : LlvmCodeCtx) : bool => false +[unused_argument(gc)] +def private iq2s_tile_gen(var gc : LlvmCodeCtx) : bool => false + def private k4_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 4) def private k5_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 5) def private k6_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 6) @@ -2996,5 +3003,7 @@ def public register_dasllama_gemm_generators { register_llvm_code_generator("dasllama_gemm_gen::iq4nl_tile", @@iq4nl_tile_gen) register_llvm_code_generator("dasllama_gemm_gen::k2_gemv", @@k2_gemv_gen) register_llvm_code_generator("dasllama_gemm_gen::k2_tile", @@k2_tile_gen) + register_llvm_code_generator("dasllama_gemm_gen::iq2s_gemv", @@iq2s_gemv_gen) + register_llvm_code_generator("dasllama_gemm_gen::iq2s_tile", @@iq2s_tile_gen) register_llvm_code_generator("dasllama_gemm_gen::q8q8_witness", @@witness_gen) } diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_schema.das b/modules/dasLLAMA/dasllama/dasllama_gemm_schema.das index b1700f91cb..5526e86288 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_schema.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_schema.das @@ -71,6 +71,7 @@ def kq_qsb(fmt : int) : int64 { if (fmt == 34) return kq_qsb(KqFmt.iq3xxs) if (fmt == 45) return kq_qsb(KqFmt.iq4nl) if (fmt == 2) return kq_qsb(KqFmt.k2) + if (fmt == 23) return kq_qsb(KqFmt.iq2s) panic("kq_qsb: unknown kq format id {fmt}") return 0l } @@ -89,6 +90,7 @@ def kq_ssb(fmt : int) : int64 { if (fmt == 34) return kq_ssb(KqFmt.iq3xxs) if (fmt == 45) return kq_ssb(KqFmt.iq4nl) if (fmt == 2) return kq_ssb(KqFmt.k2) + if (fmt == 23) return kq_ssb(KqFmt.iq2s) panic("kq_ssb: unknown kq format id {fmt}") return 0l } diff --git a/modules/dasLLAMA/dasllama/dasllama_gguf.das b/modules/dasLLAMA/dasllama/dasllama_gguf.das index 61458c59f3..a3335c8ca9 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gguf.das +++ b/modules/dasLLAMA/dasllama/dasllama_gguf.das @@ -50,6 +50,7 @@ let GGML_TYPE_Q8_0 = 8 let GGML_TYPE_Q3_K = 11 let GGML_TYPE_Q2_K = 10 let GGML_TYPE_IQ3_S = 21 +let GGML_TYPE_IQ2_S = 22 let GGML_TYPE_IQ3_XXS = 18 let GGML_TYPE_IQ4_NL = 20 let GGML_TYPE_Q4_K = 12 @@ -940,6 +941,45 @@ def gguf_transcode_iq4nl(m : GGUFMeta; srcbytes : array | #; name : strin } } +//! Transcode an IQ2_S tensor into the iq2s planes (strides 72/20, exact): the 82B disk +//! superblock splits into [idx][signs][qh] verbatim, d eighth-ed, the 16 strips decoded to +//! (1 + 2s) - transcode_iq2s_superblock. +def gguf_transcode_iq2s(m : GGUFMeta; srcbytes : array | #; name : string; var kq : array; var ks : array; eloff, expect_n : int64; src_off : int64 = 0l) { + let ti = kq_transcode_check(m, name, GGML_TYPE_IQ2_S, "IQ2_S", src_off, expect_n) + let nb = expect_n / 256l + if (nb <= 0l) { + return + } + guard_dst(name, "iq2s quant plane", (eloff / 256l) * IQ2S_QSB, nb * IQ2S_QSB, long_length(kq)) + guard_dst(name, "iq2s scale plane", (eloff / 256l) * IQ2S_SSB, nb * IQ2S_SSB, long_length(ks)) + with_tensor_view(m, srcbytes, ti) $(bytes, tbase) { + let bo = tbase + (src_off / 256l) * 82l + unsafe { + let srcp = addr(bytes[bo]) + var kqp = addr(kq[(eloff / 256l) * IQ2S_QSB]) + var ksp = addr(ks[(eloff / 256l) * IQ2S_SSB]) + maybe_parallel_for(0, int(nb), transcode_jobs(nb, 82l)) $(rb, re) { + unsafe { + for (sb in range64(int64(rb), int64(re))) { + let src = srcp + sb * 82l + var row = ksp + sb * IQ2S_SSB + let d8 = f16_half_bits(f16_half_bits(f16_half_bits(uint(src[0l]) | (uint(src[1l]) << 8u)))) + row[0l] = uint8(d8 & 0xFFu) + row[1l] = uint8(d8 >> 8u) + row[2l] = uint8(0) + row[3l] = uint8(0) + for (g in range64(16l)) { + let nib = g % 2l == 0l ? int(src[74l + g / 2l]) & 15 : int(src[74l + g / 2l]) >> 4 + row[4l + g] = uint8(1 + 2 * nib) + } + bcopy(kqp + sb * IQ2S_QSB, src + 2l, 72l) + } + } + } + } + } +} + //! Transcode a Q5_1 tensor into the q51 planes (per 32-BLOCK strides 20/4 — see Q51_QB): each //! 24B disk block splits into 2B d + 2B m (scale plane) and 16B nibbles + 4B qh (quant plane), //! all verbatim. Exact; `eloff`/`src_off`/`expect_n` are element offsets, % 32. diff --git a/modules/dasLLAMA/dasllama/dasllama_gpu_resident.das b/modules/dasLLAMA/dasllama/dasllama_gpu_resident.das index 529c1b7b56..ce46a9a7f9 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gpu_resident.das +++ b/modules/dasLLAMA/dasllama/dasllama_gpu_resident.das @@ -300,6 +300,11 @@ def trim_model_planes(var t : Model) : bool { // nolint:STYLE038 — the flat t.embs |> reserve_resize(nsb * K2_SSB) memcpy(addr(t.embq[0]), addr(t.k2q[sb0 * K2_QSB]), nsb * K2_QSB) memcpy(addr(t.embs[0]), addr(t.k2s[sb0 * K2_SSB]), nsb * K2_SSB) + } elif (t.emb_fmt == KqFmt.iq2s) { + t.embq |> reserve_resize(nsb * IQ2S_QSB) + t.embs |> reserve_resize(nsb * IQ2S_SSB) + memcpy(addr(t.embq[0]), addr(t.iq2sq[sb0 * IQ2S_QSB]), nsb * IQ2S_QSB) + memcpy(addr(t.embs[0]), addr(t.iq2ss[sb0 * IQ2S_SSB]), nsb * IQ2S_SSB) } elif (t.emb_fmt == KqFmt.iq3xxs) { t.embq |> resize(nsb * IQ3XXS_QSB) t.embs |> resize(nsb * IQ3XXS_SSB) @@ -340,6 +345,8 @@ def trim_model_planes(var t : Model) : bool { // nolint:STYLE038 — the flat delete t.iq4nls delete t.k2q delete t.k2s + delete t.iq2sq + delete t.iq2ss delete t.q40q delete t.q40s delete t.q51q diff --git a/modules/dasLLAMA/dasllama/dasllama_image.das b/modules/dasLLAMA/dasllama/dasllama_image.das index ca4bc95bfc..44ecb3c7f8 100644 --- a/modules/dasLLAMA/dasllama/dasllama_image.das +++ b/modules/dasLLAMA/dasllama/dasllama_image.das @@ -38,7 +38,7 @@ require dasllama/dasllama_load // WhisperModel.enc) contribute their planes under dotted names ("enc.fblob"); string-array // fields ride the meta blob via serialize_strings — raw string pointers can't be planes. -let IMAGE_VERSION = 23 // 23: the k2 plane pair (Q2_K native tier); 22: iq4nl; 21: iq3xxs +let IMAGE_VERSION = 24 // 24: the iq2s plane pair (IQ2_S native tier); 23: k2; 22: iq4nl //! The metal (blob-only) flavor's identity tag: q8 planes ride the 34B block_q8_0 blob and the //! kq scale planes their GPU forms (convert_model_to_metal_blob) — flavors are per-config and @@ -640,6 +640,7 @@ def serialize_image_meta(var arch : Archive; var t : Model) { arch |> serialize_raw(t.kq_repack_mr34) arch |> serialize_raw(t.kq_repack_mr45) arch |> serialize_raw(t.kq_repack_mr2) + arch |> serialize_raw(t.kq_repack_mr23) arch |> serialize_raw(t.wcls_fmt) arch |> serialize_raw(t.emb_fmt) arch |> serialize_raw(t.ple_emb_fmt) @@ -679,7 +680,7 @@ def serialize_image_meta(var arch : Archive; var t : Model) { } // 67 serialized fields + 3 deliberate skips (blocks, image_map, image_bytes) -let IMAGE_META_FIELDS = 72 + 3 +let IMAGE_META_FIELDS = 73 + 3 //! Count of meta-carried fields of any struct: non-arrays plus string arrays (those cannot //! be raw planes — see serialize_strings). Pair with a per-type field-count constant to @@ -963,7 +964,7 @@ def save_model_image(var t : Model; path : string; tag : string = ""; quant : st // streamed planes size as empty, so their bytes plus a page each are added here; over-reserve is free (dwrite_close truncates) def private stream_extra_bytes(t : Model; jobs : array) : uint64 { var extra = 0ul - for (fname in ["qblob", "mblob", "k4q", "k5q", "k6q", "q40q", "q51q", "iq4xsq", "k3q", "iq3sq", "iq3xxsq", "iq4nlq", "k2q"]) { + for (fname in ["qblob", "mblob", "k4q", "k5q", "k6q", "q40q", "q51q", "iq4xsq", "k3q", "iq3sq", "iq3xxsq", "iq4nlq", "k2q", "iq2sq"]) { let sb = stream_plane_bytes(t, jobs, fname) if (sb > 0l) { extra += uint64(sb) + uint64(IMAGE_PAGE) diff --git a/modules/dasLLAMA/dasllama/dasllama_kqformat.das b/modules/dasLLAMA/dasllama/dasllama_kqformat.das index d559082013..43607ffb0b 100644 --- a/modules/dasLLAMA/dasllama/dasllama_kqformat.das +++ b/modules/dasLLAMA/dasllama/dasllama_kqformat.das @@ -13,7 +13,7 @@ module dasllama_kqformat shared public require dasllama/dasllama_lint public //! Per-weight storage format under a q8-mode load with native planes: q8 = qblob/qscales; -//! k4/k5/k6/q40/iq4xs/k3/iq3s/iq3xxs/iq4nl/k2 = superblock plane pairs (mixed PER TENSOR); q51 = native Q5_1 per-32 +//! k4/k5/k6/q40/iq4xs/k3/iq3s/iq3xxs/iq4nl/k2/iq2s = superblock plane pairs (mixed PER TENSOR); q51 = native Q5_1 per-32 //! planes. New members append — the int value is the device stack tag and the image plane id. enum KqFmt : uint8 { q8 @@ -28,15 +28,16 @@ enum KqFmt : uint8 { iq3xxs iq4nl k2 + iq2s } //! The superblock-lattice formats (Q8_K-form activations, % 256 rows, repack + stamped kq //! kernels). q51 deliberately fails this: it rides per-32 planes with Q8_0-form activations — //! a `fmt != KqFmt.q8` test does not imply the kq lattice; branch on kq_sb where it does. -def kq_sb(f : KqFmt) : bool => f == KqFmt.k4 || f == KqFmt.k5 || f == KqFmt.k6 || f == KqFmt.q40 || f == KqFmt.iq4xs || f == KqFmt.k3 || f == KqFmt.iq3s || f == KqFmt.iq3xxs || f == KqFmt.iq4nl || f == KqFmt.k2 +def kq_sb(f : KqFmt) : bool => f == KqFmt.k4 || f == KqFmt.k5 || f == KqFmt.k6 || f == KqFmt.q40 || f == KqFmt.iq4xs || f == KqFmt.k3 || f == KqFmt.iq3s || f == KqFmt.iq3xxs || f == KqFmt.iq4nl || f == KqFmt.k2 || f == KqFmt.iq2s //! The same predicate over the int id space the GPU drivers carry per stack/plane. -def kq_sb(fmt : int) : bool => fmt == int(KqFmt.k4) || fmt == int(KqFmt.k5) || fmt == int(KqFmt.k6) || fmt == int(KqFmt.q40) || fmt == int(KqFmt.iq4xs) || fmt == int(KqFmt.k3) || fmt == int(KqFmt.iq3s) || fmt == int(KqFmt.iq3xxs) || fmt == int(KqFmt.iq4nl) || fmt == int(KqFmt.k2) +def kq_sb(fmt : int) : bool => fmt == int(KqFmt.k4) || fmt == int(KqFmt.k5) || fmt == int(KqFmt.k6) || fmt == int(KqFmt.q40) || fmt == int(KqFmt.iq4xs) || fmt == int(KqFmt.k3) || fmt == int(KqFmt.iq3s) || fmt == int(KqFmt.iq3xxs) || fmt == int(KqFmt.iq4nl) || fmt == int(KqFmt.k2) || fmt == int(KqFmt.iq2s) //! KqFmt -> the kernel/IR format id space — the ONE bridge between the enum and the int ids the //! generated kernels take as runtime params. Mnemonic ids: a K-quant is its bit width (3/4/5/6), @@ -72,6 +73,9 @@ def kq_schema_id(f : KqFmt) : int { if (f == KqFmt.k2) { return 2 } + if (f == KqFmt.iq2s) { + return 23 + } panic("kq_schema_id: not a kq superblock format") return 0 } @@ -98,6 +102,8 @@ let IQ3XXS_QSB = 96l // IQ3_XXS quant plane: [64 grid-index qs][32 aux bytes: p let IQ3XXS_SSB = 20l // IQ3_XXS scale plane: f16 d HALVED, 2 pad, 8 x uint8 (2*ls + 1), 8 pad — the iq3s row shape and fold (ggml's global 0.25 = the halved d x the halved grid) let K2_QSB = 64l // Q2_K quant plane: 64 qs bytes (four 2-bit lanes each), verbatim disk order let K2_SSB = 20l // Q2_K scale plane: [f16 d][f16 dmin][16 sc/min nibble-pair bytes] - the disk scale block, header first +let IQ2S_QSB = 72l // IQ2_S quant plane: [32 grid-index low bytes][32 sign bytes][8 qh], verbatim disk order +let IQ2S_SSB = 20l // IQ2_S scale plane: f16 d EIGHTH-ed (ggml's (0.5+ls)*0.25 = (2ls+1)/8), 2 pad, 16 x uint8 (2ls+1) - per-16 strips in the 20B row // IQ4_NL reuses Q40_QSB/Q40_SSB outright: the disk bytes are q40's (f16 d + 16 nibble bytes // per 32-block, the k/k+16 pairing) — only the nibble SEMANTICS differ (IQ4NL_LUT, no -8). @@ -257,6 +263,274 @@ def f16_half_bits(u : uint) : uint { return (u & 0x8000u) | ((u & 0x3FFu) >> 1u) } + +//! The IQ2_S grid (ggml's iq2s_grid, 1024 u64 entries as low/high uint pairs): a 10-bit index +//! (qs byte | qh 2 bits) selects EIGHT magnitudes ({8, 25, 43}); signs ride the block's own +//! sign bytes. Per-call local (8 KB) - the worker-safe form; IQ2S_GRID is the main-context copy. +def iq2s_grid2() : uint[2048] { // nolint:STYLE038 - a 2048-word data table, not splittable + return fixed_array( + 0x08080808, 0x08080808, 0x0808082b, 0x08080808, 0x08081919, 0x08080808, 0x08082b08, 0x08080808, + 0x08082b2b, 0x08080808, 0x08190819, 0x08080808, 0x08191908, 0x08080808, 0x0819192b, 0x08080808, + 0x08192b19, 0x08080808, 0x082b0808, 0x08080808, 0x082b082b, 0x08080808, 0x082b1919, 0x08080808, + 0x082b2b08, 0x08080808, 0x19080819, 0x08080808, 0x19081908, 0x08080808, 0x1908192b, 0x08080808, + 0x19082b19, 0x08080808, 0x19190808, 0x08080808, 0x1919082b, 0x08080808, 0x19191919, 0x08080808, + 0x19192b08, 0x08080808, 0x192b0819, 0x08080808, 0x192b1908, 0x08080808, 0x192b192b, 0x08080808, + 0x192b2b19, 0x08080808, 0x2b080808, 0x08080808, 0x2b08082b, 0x08080808, 0x2b081919, 0x08080808, + 0x2b082b08, 0x08080808, 0x2b190819, 0x08080808, 0x2b191908, 0x08080808, 0x2b2b0808, 0x08080808, + 0x2b2b1919, 0x08080808, 0x2b2b2b2b, 0x08080808, 0x08080819, 0x08080819, 0x08081908, 0x08080819, + 0x0808192b, 0x08080819, 0x08082b19, 0x08080819, 0x08190808, 0x08080819, 0x0819082b, 0x08080819, + 0x08191919, 0x08080819, 0x08192b08, 0x08080819, 0x082b0819, 0x08080819, 0x082b1908, 0x08080819, + 0x19080808, 0x08080819, 0x1908082b, 0x08080819, 0x19081919, 0x08080819, 0x19082b08, 0x08080819, + 0x19190819, 0x08080819, 0x19191908, 0x08080819, 0x1919192b, 0x08080819, 0x19192b19, 0x08080819, + 0x192b0808, 0x08080819, 0x192b1919, 0x08080819, 0x192b2b08, 0x08080819, 0x2b080819, 0x08080819, + 0x2b081908, 0x08080819, 0x2b190808, 0x08080819, 0x2b19082b, 0x08080819, 0x2b191919, 0x08080819, + 0x2b2b0819, 0x08080819, 0x2b2b1908, 0x08080819, 0x08080808, 0x0808082b, 0x0808082b, 0x0808082b, + 0x08081919, 0x0808082b, 0x08082b08, 0x0808082b, 0x08190819, 0x0808082b, 0x08191908, 0x0808082b, + 0x082b0808, 0x0808082b, 0x082b2b2b, 0x0808082b, 0x19080819, 0x0808082b, 0x19081908, 0x0808082b, + 0x1908192b, 0x0808082b, 0x19082b19, 0x0808082b, 0x19190808, 0x0808082b, 0x19191919, 0x0808082b, + 0x2b080808, 0x0808082b, 0x2b081919, 0x0808082b, 0x2b082b2b, 0x0808082b, 0x2b191908, 0x0808082b, + 0x2b2b082b, 0x0808082b, 0x08080819, 0x08081908, 0x08081908, 0x08081908, 0x0808192b, 0x08081908, + 0x08082b19, 0x08081908, 0x08190808, 0x08081908, 0x0819082b, 0x08081908, 0x08191919, 0x08081908, + 0x08192b08, 0x08081908, 0x082b0819, 0x08081908, 0x082b1908, 0x08081908, 0x082b192b, 0x08081908, + 0x082b2b19, 0x08081908, 0x19080808, 0x08081908, 0x1908082b, 0x08081908, 0x19081919, 0x08081908, + 0x19082b08, 0x08081908, 0x19082b2b, 0x08081908, 0x19190819, 0x08081908, 0x19191908, 0x08081908, + 0x1919192b, 0x08081908, 0x19192b19, 0x08081908, 0x192b0808, 0x08081908, 0x192b082b, 0x08081908, + 0x192b1919, 0x08081908, 0x2b080819, 0x08081908, 0x2b081908, 0x08081908, 0x2b08192b, 0x08081908, + 0x2b082b19, 0x08081908, 0x2b190808, 0x08081908, 0x2b191919, 0x08081908, 0x2b192b08, 0x08081908, + 0x2b2b0819, 0x08081908, 0x2b2b1908, 0x08081908, 0x08080808, 0x08081919, 0x0808082b, 0x08081919, + 0x08081919, 0x08081919, 0x08082b08, 0x08081919, 0x08082b2b, 0x08081919, 0x08190819, 0x08081919, + 0x08191908, 0x08081919, 0x0819192b, 0x08081919, 0x08192b19, 0x08081919, 0x082b0808, 0x08081919, + 0x082b1919, 0x08081919, 0x082b2b08, 0x08081919, 0x19080819, 0x08081919, 0x19081908, 0x08081919, + 0x1908192b, 0x08081919, 0x19082b19, 0x08081919, 0x19190808, 0x08081919, 0x1919082b, 0x08081919, + 0x19191919, 0x08081919, 0x19192b08, 0x08081919, 0x192b0819, 0x08081919, 0x192b1908, 0x08081919, + 0x2b080808, 0x08081919, 0x2b08082b, 0x08081919, 0x2b081919, 0x08081919, 0x2b082b08, 0x08081919, + 0x2b190819, 0x08081919, 0x2b191908, 0x08081919, 0x2b2b0808, 0x08081919, 0x08080819, 0x0808192b, + 0x08081908, 0x0808192b, 0x0808192b, 0x0808192b, 0x08082b19, 0x0808192b, 0x08190808, 0x0808192b, + 0x08191919, 0x0808192b, 0x19080808, 0x0808192b, 0x19081919, 0x0808192b, 0x19082b08, 0x0808192b, + 0x19190819, 0x0808192b, 0x19191908, 0x0808192b, 0x192b0808, 0x0808192b, 0x2b080819, 0x0808192b, + 0x2b081908, 0x0808192b, 0x2b190808, 0x0808192b, 0x08080808, 0x08082b08, 0x0808082b, 0x08082b08, + 0x08081919, 0x08082b08, 0x08082b08, 0x08082b08, 0x08190819, 0x08082b08, 0x08191908, 0x08082b08, + 0x0819192b, 0x08082b08, 0x08192b19, 0x08082b08, 0x082b0808, 0x08082b08, 0x082b1919, 0x08082b08, + 0x082b2b2b, 0x08082b08, 0x19080819, 0x08082b08, 0x19081908, 0x08082b08, 0x1908192b, 0x08082b08, + 0x19082b19, 0x08082b08, 0x19190808, 0x08082b08, 0x1919082b, 0x08082b08, 0x19191919, 0x08082b08, + 0x19192b08, 0x08082b08, 0x192b0819, 0x08082b08, 0x192b1908, 0x08082b08, 0x2b080808, 0x08082b08, + 0x2b081919, 0x08082b08, 0x2b191908, 0x08082b08, 0x2b2b2b2b, 0x08082b08, 0x08080819, 0x08082b19, + 0x08081908, 0x08082b19, 0x08190808, 0x08082b19, 0x0819082b, 0x08082b19, 0x08191919, 0x08082b19, + 0x08192b08, 0x08082b19, 0x082b0819, 0x08082b19, 0x19080808, 0x08082b19, 0x19081919, 0x08082b19, + 0x19082b08, 0x08082b19, 0x19190819, 0x08082b19, 0x19191908, 0x08082b19, 0x192b0808, 0x08082b19, + 0x2b080819, 0x08082b19, 0x2b190808, 0x08082b19, 0x08080808, 0x08082b2b, 0x08190819, 0x08082b2b, + 0x08191908, 0x08082b2b, 0x082b082b, 0x08082b2b, 0x082b2b08, 0x08082b2b, 0x082b2b2b, 0x08082b2b, + 0x19190808, 0x08082b2b, 0x2b192b19, 0x08082b2b, 0x08080819, 0x08190808, 0x08081908, 0x08190808, + 0x0808192b, 0x08190808, 0x08082b19, 0x08190808, 0x08190808, 0x08190808, 0x0819082b, 0x08190808, + 0x08191919, 0x08190808, 0x08192b08, 0x08190808, 0x082b0819, 0x08190808, 0x082b1908, 0x08190808, + 0x082b192b, 0x08190808, 0x19080808, 0x08190808, 0x1908082b, 0x08190808, 0x19081919, 0x08190808, + 0x19082b08, 0x08190808, 0x19190819, 0x08190808, 0x19191908, 0x08190808, 0x1919192b, 0x08190808, + 0x19192b19, 0x08190808, 0x192b0808, 0x08190808, 0x192b082b, 0x08190808, 0x192b1919, 0x08190808, + 0x192b2b08, 0x08190808, 0x2b080819, 0x08190808, 0x2b081908, 0x08190808, 0x2b08192b, 0x08190808, + 0x2b190808, 0x08190808, 0x2b191919, 0x08190808, 0x2b192b08, 0x08190808, 0x2b2b0819, 0x08190808, + 0x2b2b1908, 0x08190808, 0x08080808, 0x08190819, 0x0808082b, 0x08190819, 0x08081919, 0x08190819, + 0x08082b08, 0x08190819, 0x08082b2b, 0x08190819, 0x08190819, 0x08190819, 0x08191908, 0x08190819, + 0x0819192b, 0x08190819, 0x08192b19, 0x08190819, 0x082b0808, 0x08190819, 0x082b082b, 0x08190819, + 0x082b1919, 0x08190819, 0x082b2b08, 0x08190819, 0x19080819, 0x08190819, 0x19081908, 0x08190819, + 0x1908192b, 0x08190819, 0x19082b19, 0x08190819, 0x19190808, 0x08190819, 0x1919082b, 0x08190819, + 0x19191919, 0x08190819, 0x19192b08, 0x08190819, 0x192b0819, 0x08190819, 0x192b1908, 0x08190819, + 0x2b080808, 0x08190819, 0x2b08082b, 0x08190819, 0x2b081919, 0x08190819, 0x2b082b08, 0x08190819, + 0x2b190819, 0x08190819, 0x2b191908, 0x08190819, 0x08080819, 0x0819082b, 0x08081908, 0x0819082b, + 0x08082b19, 0x0819082b, 0x08190808, 0x0819082b, 0x08191919, 0x0819082b, 0x082b0819, 0x0819082b, + 0x082b1908, 0x0819082b, 0x19080808, 0x0819082b, 0x19081919, 0x0819082b, 0x19190819, 0x0819082b, + 0x19191908, 0x0819082b, 0x2b080819, 0x0819082b, 0x2b081908, 0x0819082b, 0x2b190808, 0x0819082b, + 0x08080808, 0x08191908, 0x0808082b, 0x08191908, 0x08081919, 0x08191908, 0x08082b08, 0x08191908, + 0x08190819, 0x08191908, 0x08191908, 0x08191908, 0x0819192b, 0x08191908, 0x08192b19, 0x08191908, + 0x082b0808, 0x08191908, 0x082b1919, 0x08191908, 0x082b2b08, 0x08191908, 0x19080819, 0x08191908, + 0x19081908, 0x08191908, 0x1908192b, 0x08191908, 0x19082b19, 0x08191908, 0x19190808, 0x08191908, + 0x1919082b, 0x08191908, 0x19191919, 0x08191908, 0x19192b08, 0x08191908, 0x192b0819, 0x08191908, + 0x192b1908, 0x08191908, 0x2b080808, 0x08191908, 0x2b08082b, 0x08191908, 0x2b081919, 0x08191908, + 0x2b082b08, 0x08191908, 0x2b190819, 0x08191908, 0x2b191908, 0x08191908, 0x2b2b0808, 0x08191908, + 0x08080819, 0x08191919, 0x08081908, 0x08191919, 0x0808192b, 0x08191919, 0x08082b19, 0x08191919, + 0x08190808, 0x08191919, 0x0819082b, 0x08191919, 0x08191919, 0x08191919, 0x08192b08, 0x08191919, + 0x082b0819, 0x08191919, 0x082b1908, 0x08191919, 0x19080808, 0x08191919, 0x1908082b, 0x08191919, + 0x19081919, 0x08191919, 0x19082b08, 0x08191919, 0x19190819, 0x08191919, 0x19191908, 0x08191919, + 0x192b0808, 0x08191919, 0x2b080819, 0x08191919, 0x2b081908, 0x08191919, 0x2b190808, 0x08191919, + 0x08080808, 0x0819192b, 0x08081919, 0x0819192b, 0x08082b08, 0x0819192b, 0x08190819, 0x0819192b, + 0x08191908, 0x0819192b, 0x082b0808, 0x0819192b, 0x19080819, 0x0819192b, 0x19081908, 0x0819192b, + 0x19190808, 0x0819192b, 0x2b080808, 0x0819192b, 0x2b2b2b2b, 0x0819192b, 0x08080819, 0x08192b08, + 0x08081908, 0x08192b08, 0x0808192b, 0x08192b08, 0x08082b19, 0x08192b08, 0x08190808, 0x08192b08, + 0x08191919, 0x08192b08, 0x08192b08, 0x08192b08, 0x082b0819, 0x08192b08, 0x19080808, 0x08192b08, + 0x1908082b, 0x08192b08, 0x19081919, 0x08192b08, 0x19082b08, 0x08192b08, 0x19190819, 0x08192b08, + 0x19191908, 0x08192b08, 0x192b0808, 0x08192b08, 0x2b080819, 0x08192b08, 0x2b081908, 0x08192b08, + 0x08080808, 0x08192b19, 0x0808082b, 0x08192b19, 0x08081919, 0x08192b19, 0x08082b08, 0x08192b19, + 0x08190819, 0x08192b19, 0x08191908, 0x08192b19, 0x082b0808, 0x08192b19, 0x19080819, 0x08192b19, + 0x19081908, 0x08192b19, 0x19190808, 0x08192b19, 0x192b2b19, 0x08192b19, 0x2b2b082b, 0x08192b19, + 0x08081908, 0x08192b2b, 0x08190808, 0x08192b2b, 0x19080808, 0x08192b2b, 0x1919192b, 0x08192b2b, + 0x08080808, 0x082b0808, 0x0808082b, 0x082b0808, 0x08081919, 0x082b0808, 0x08082b08, 0x082b0808, + 0x08190819, 0x082b0808, 0x08191908, 0x082b0808, 0x0819192b, 0x082b0808, 0x08192b19, 0x082b0808, + 0x082b0808, 0x082b0808, 0x082b1919, 0x082b0808, 0x082b2b2b, 0x082b0808, 0x19080819, 0x082b0808, + 0x19081908, 0x082b0808, 0x19190808, 0x082b0808, 0x1919082b, 0x082b0808, 0x19191919, 0x082b0808, + 0x192b1908, 0x082b0808, 0x2b080808, 0x082b0808, 0x2b082b2b, 0x082b0808, 0x2b191908, 0x082b0808, + 0x2b2b2b2b, 0x082b0808, 0x08080819, 0x082b0819, 0x08081908, 0x082b0819, 0x08190808, 0x082b0819, + 0x0819082b, 0x082b0819, 0x08191919, 0x082b0819, 0x082b0819, 0x082b0819, 0x19080808, 0x082b0819, + 0x1908082b, 0x082b0819, 0x19081919, 0x082b0819, 0x19190819, 0x082b0819, 0x19191908, 0x082b0819, + 0x192b0808, 0x082b0819, 0x2b080819, 0x082b0819, 0x2b081908, 0x082b0819, 0x2b190808, 0x082b0819, + 0x08080808, 0x082b082b, 0x08082b2b, 0x082b082b, 0x082b082b, 0x082b082b, 0x082b2b08, 0x082b082b, + 0x082b2b2b, 0x082b082b, 0x19081908, 0x082b082b, 0x19190808, 0x082b082b, 0x2b082b08, 0x082b082b, + 0x2b082b2b, 0x082b082b, 0x2b2b2b08, 0x082b082b, 0x08080819, 0x082b1908, 0x08081908, 0x082b1908, + 0x0808192b, 0x082b1908, 0x08082b19, 0x082b1908, 0x08190808, 0x082b1908, 0x08191919, 0x082b1908, + 0x08192b08, 0x082b1908, 0x082b0819, 0x082b1908, 0x082b1908, 0x082b1908, 0x19080808, 0x082b1908, + 0x1908082b, 0x082b1908, 0x19081919, 0x082b1908, 0x19082b08, 0x082b1908, 0x19190819, 0x082b1908, + 0x19191908, 0x082b1908, 0x192b0808, 0x082b1908, 0x2b080819, 0x082b1908, 0x2b081908, 0x082b1908, + 0x2b190808, 0x082b1908, 0x08080808, 0x082b1919, 0x08081919, 0x082b1919, 0x08082b08, 0x082b1919, + 0x08190819, 0x082b1919, 0x08191908, 0x082b1919, 0x082b0808, 0x082b1919, 0x19080819, 0x082b1919, + 0x19081908, 0x082b1919, 0x19190808, 0x082b1919, 0x192b192b, 0x082b1919, 0x2b080808, 0x082b1919, + 0x08080819, 0x082b192b, 0x08081908, 0x082b192b, 0x08190808, 0x082b192b, 0x19080808, 0x082b192b, + 0x19192b19, 0x082b192b, 0x08080808, 0x082b2b08, 0x08081919, 0x082b2b08, 0x08190819, 0x082b2b08, + 0x08191908, 0x082b2b08, 0x19080819, 0x082b2b08, 0x19081908, 0x082b2b08, 0x19190808, 0x082b2b08, + 0x2b082b2b, 0x082b2b08, 0x2b2b2b2b, 0x082b2b08, 0x08080819, 0x082b2b19, 0x08081908, 0x082b2b19, + 0x08190808, 0x082b2b19, 0x2b191919, 0x082b2b19, 0x08082b2b, 0x082b2b2b, 0x082b082b, 0x082b2b2b, + 0x192b1908, 0x082b2b2b, 0x2b082b08, 0x082b2b2b, 0x2b082b2b, 0x082b2b2b, 0x08080819, 0x19080808, + 0x08081908, 0x19080808, 0x0808192b, 0x19080808, 0x08082b19, 0x19080808, 0x08190808, 0x19080808, + 0x0819082b, 0x19080808, 0x08191919, 0x19080808, 0x08192b08, 0x19080808, 0x08192b2b, 0x19080808, + 0x082b0819, 0x19080808, 0x082b1908, 0x19080808, 0x082b192b, 0x19080808, 0x19080808, 0x19080808, + 0x1908082b, 0x19080808, 0x19081919, 0x19080808, 0x19082b08, 0x19080808, 0x19082b2b, 0x19080808, + 0x19190819, 0x19080808, 0x19191908, 0x19080808, 0x1919192b, 0x19080808, 0x19192b19, 0x19080808, + 0x192b0808, 0x19080808, 0x192b082b, 0x19080808, 0x192b1919, 0x19080808, 0x2b080819, 0x19080808, + 0x2b081908, 0x19080808, 0x2b190808, 0x19080808, 0x2b191919, 0x19080808, 0x2b192b08, 0x19080808, + 0x2b2b0819, 0x19080808, 0x2b2b1908, 0x19080808, 0x08080808, 0x19080819, 0x0808082b, 0x19080819, + 0x08081919, 0x19080819, 0x08082b08, 0x19080819, 0x08190819, 0x19080819, 0x08191908, 0x19080819, + 0x0819192b, 0x19080819, 0x08192b19, 0x19080819, 0x082b0808, 0x19080819, 0x082b082b, 0x19080819, + 0x082b1919, 0x19080819, 0x19080819, 0x19080819, 0x19081908, 0x19080819, 0x1908192b, 0x19080819, + 0x19082b19, 0x19080819, 0x19190808, 0x19080819, 0x1919082b, 0x19080819, 0x19191919, 0x19080819, + 0x19192b08, 0x19080819, 0x192b0819, 0x19080819, 0x192b1908, 0x19080819, 0x2b080808, 0x19080819, + 0x2b08082b, 0x19080819, 0x2b081919, 0x19080819, 0x2b082b08, 0x19080819, 0x2b190819, 0x19080819, + 0x2b191908, 0x19080819, 0x2b2b0808, 0x19080819, 0x08080819, 0x1908082b, 0x08081908, 0x1908082b, + 0x08190808, 0x1908082b, 0x0819082b, 0x1908082b, 0x08191919, 0x1908082b, 0x08192b08, 0x1908082b, + 0x082b1908, 0x1908082b, 0x19080808, 0x1908082b, 0x19081919, 0x1908082b, 0x19082b08, 0x1908082b, + 0x19190819, 0x1908082b, 0x19191908, 0x1908082b, 0x192b0808, 0x1908082b, 0x2b080819, 0x1908082b, + 0x2b081908, 0x1908082b, 0x08080808, 0x19081908, 0x0808082b, 0x19081908, 0x08081919, 0x19081908, + 0x08082b08, 0x19081908, 0x08082b2b, 0x19081908, 0x08190819, 0x19081908, 0x08191908, 0x19081908, + 0x0819192b, 0x19081908, 0x08192b19, 0x19081908, 0x082b0808, 0x19081908, 0x082b082b, 0x19081908, + 0x082b1919, 0x19081908, 0x082b2b08, 0x19081908, 0x19080819, 0x19081908, 0x19081908, 0x19081908, + 0x1908192b, 0x19081908, 0x19082b19, 0x19081908, 0x19190808, 0x19081908, 0x1919082b, 0x19081908, + 0x19191919, 0x19081908, 0x19192b08, 0x19081908, 0x192b0819, 0x19081908, 0x192b1908, 0x19081908, + 0x2b080808, 0x19081908, 0x2b08082b, 0x19081908, 0x2b081919, 0x19081908, 0x2b082b08, 0x19081908, + 0x2b190819, 0x19081908, 0x2b191908, 0x19081908, 0x2b2b0808, 0x19081908, 0x08080819, 0x19081919, + 0x08081908, 0x19081919, 0x0808192b, 0x19081919, 0x08082b19, 0x19081919, 0x08190808, 0x19081919, + 0x0819082b, 0x19081919, 0x08191919, 0x19081919, 0x08192b08, 0x19081919, 0x082b0819, 0x19081919, + 0x082b1908, 0x19081919, 0x19080808, 0x19081919, 0x1908082b, 0x19081919, 0x19081919, 0x19081919, + 0x19082b08, 0x19081919, 0x19190819, 0x19081919, 0x19191908, 0x19081919, 0x192b0808, 0x19081919, + 0x192b2b2b, 0x19081919, 0x2b080819, 0x19081919, 0x2b081908, 0x19081919, 0x2b190808, 0x19081919, + 0x08080808, 0x1908192b, 0x0808082b, 0x1908192b, 0x08081919, 0x1908192b, 0x08082b08, 0x1908192b, + 0x08190819, 0x1908192b, 0x08191908, 0x1908192b, 0x082b0808, 0x1908192b, 0x19080819, 0x1908192b, + 0x19081908, 0x1908192b, 0x19190808, 0x1908192b, 0x2b080808, 0x1908192b, 0x2b2b1919, 0x1908192b, + 0x08080819, 0x19082b08, 0x08081908, 0x19082b08, 0x08082b19, 0x19082b08, 0x08190808, 0x19082b08, + 0x0819082b, 0x19082b08, 0x08191919, 0x19082b08, 0x08192b08, 0x19082b08, 0x082b0819, 0x19082b08, + 0x082b1908, 0x19082b08, 0x19080808, 0x19082b08, 0x1908082b, 0x19082b08, 0x19081919, 0x19082b08, + 0x19082b08, 0x19082b08, 0x19190819, 0x19082b08, 0x19191908, 0x19082b08, 0x192b0808, 0x19082b08, + 0x2b081908, 0x19082b08, 0x2b190808, 0x19082b08, 0x08080808, 0x19082b19, 0x0808082b, 0x19082b19, + 0x08081919, 0x19082b19, 0x08082b08, 0x19082b19, 0x08190819, 0x19082b19, 0x08191908, 0x19082b19, + 0x082b0808, 0x19082b19, 0x19080819, 0x19082b19, 0x19081908, 0x19082b19, 0x19190808, 0x19082b19, + 0x2b080808, 0x19082b19, 0x2b19192b, 0x19082b19, 0x08080819, 0x19082b2b, 0x08081908, 0x19082b2b, + 0x08190808, 0x19082b2b, 0x19080808, 0x19082b2b, 0x08080808, 0x19190808, 0x0808082b, 0x19190808, + 0x08081919, 0x19190808, 0x08082b08, 0x19190808, 0x08190819, 0x19190808, 0x08191908, 0x19190808, + 0x0819192b, 0x19190808, 0x08192b19, 0x19190808, 0x082b0808, 0x19190808, 0x082b082b, 0x19190808, + 0x082b1919, 0x19190808, 0x082b2b08, 0x19190808, 0x19080819, 0x19190808, 0x19081908, 0x19190808, + 0x1908192b, 0x19190808, 0x19082b19, 0x19190808, 0x19190808, 0x19190808, 0x1919082b, 0x19190808, + 0x19191919, 0x19190808, 0x19192b08, 0x19190808, 0x192b0819, 0x19190808, 0x192b1908, 0x19190808, + 0x2b080808, 0x19190808, 0x2b08082b, 0x19190808, 0x2b081919, 0x19190808, 0x2b082b08, 0x19190808, + 0x2b190819, 0x19190808, 0x2b191908, 0x19190808, 0x08080819, 0x19190819, 0x08081908, 0x19190819, + 0x0808192b, 0x19190819, 0x08082b19, 0x19190819, 0x08190808, 0x19190819, 0x0819082b, 0x19190819, + 0x08191919, 0x19190819, 0x08192b08, 0x19190819, 0x082b0819, 0x19190819, 0x082b1908, 0x19190819, + 0x19080808, 0x19190819, 0x1908082b, 0x19190819, 0x19081919, 0x19190819, 0x19082b08, 0x19190819, + 0x19190819, 0x19190819, 0x19191908, 0x19190819, 0x192b0808, 0x19190819, 0x2b080819, 0x19190819, + 0x2b081908, 0x19190819, 0x2b190808, 0x19190819, 0x08080808, 0x1919082b, 0x08081919, 0x1919082b, + 0x08082b08, 0x1919082b, 0x08190819, 0x1919082b, 0x08191908, 0x1919082b, 0x082b0808, 0x1919082b, + 0x19080819, 0x1919082b, 0x19081908, 0x1919082b, 0x19190808, 0x1919082b, 0x192b2b19, 0x1919082b, + 0x2b080808, 0x1919082b, 0x08080819, 0x19191908, 0x08081908, 0x19191908, 0x0808192b, 0x19191908, + 0x08082b19, 0x19191908, 0x08190808, 0x19191908, 0x0819082b, 0x19191908, 0x08191919, 0x19191908, + 0x08192b08, 0x19191908, 0x082b0819, 0x19191908, 0x082b1908, 0x19191908, 0x19080808, 0x19191908, + 0x1908082b, 0x19191908, 0x19081919, 0x19191908, 0x19082b08, 0x19191908, 0x19190819, 0x19191908, + 0x19191908, 0x19191908, 0x192b0808, 0x19191908, 0x2b080819, 0x19191908, 0x2b081908, 0x19191908, + 0x2b190808, 0x19191908, 0x08080808, 0x19191919, 0x0808082b, 0x19191919, 0x08081919, 0x19191919, + 0x08082b08, 0x19191919, 0x08190819, 0x19191919, 0x08191908, 0x19191919, 0x082b0808, 0x19191919, + 0x19080819, 0x19191919, 0x19081908, 0x19191919, 0x19190808, 0x19191919, 0x2b080808, 0x19191919, + 0x08080819, 0x1919192b, 0x08081908, 0x1919192b, 0x08190808, 0x1919192b, 0x082b192b, 0x1919192b, + 0x19080808, 0x1919192b, 0x08080808, 0x19192b08, 0x0808082b, 0x19192b08, 0x08081919, 0x19192b08, + 0x08082b08, 0x19192b08, 0x08190819, 0x19192b08, 0x08191908, 0x19192b08, 0x082b0808, 0x19192b08, + 0x19080819, 0x19192b08, 0x19081908, 0x19192b08, 0x19190808, 0x19192b08, 0x19192b2b, 0x19192b08, + 0x2b080808, 0x19192b08, 0x08080819, 0x19192b19, 0x08081908, 0x19192b19, 0x08190808, 0x19192b19, + 0x19080808, 0x19192b19, 0x08080808, 0x19192b2b, 0x08192b19, 0x19192b2b, 0x2b081919, 0x19192b2b, + 0x2b2b2b08, 0x19192b2b, 0x08080819, 0x192b0808, 0x08081908, 0x192b0808, 0x0808192b, 0x192b0808, + 0x08190808, 0x192b0808, 0x0819082b, 0x192b0808, 0x08191919, 0x192b0808, 0x08192b08, 0x192b0808, + 0x082b0819, 0x192b0808, 0x082b1908, 0x192b0808, 0x19080808, 0x192b0808, 0x19081919, 0x192b0808, + 0x19082b08, 0x192b0808, 0x19190819, 0x192b0808, 0x19191908, 0x192b0808, 0x192b0808, 0x192b0808, + 0x2b081908, 0x192b0808, 0x2b190808, 0x192b0808, 0x08080808, 0x192b0819, 0x0808082b, 0x192b0819, + 0x08081919, 0x192b0819, 0x08082b08, 0x192b0819, 0x08190819, 0x192b0819, 0x08191908, 0x192b0819, + 0x082b0808, 0x192b0819, 0x19080819, 0x192b0819, 0x19081908, 0x192b0819, 0x19190808, 0x192b0819, + 0x2b080808, 0x192b0819, 0x2b192b19, 0x192b0819, 0x08081908, 0x192b082b, 0x08190808, 0x192b082b, + 0x19080808, 0x192b082b, 0x1919192b, 0x192b082b, 0x2b2b0819, 0x192b082b, 0x08080808, 0x192b1908, + 0x08081919, 0x192b1908, 0x08082b08, 0x192b1908, 0x08190819, 0x192b1908, 0x08191908, 0x192b1908, + 0x082b0808, 0x192b1908, 0x19080819, 0x192b1908, 0x19081908, 0x192b1908, 0x19190808, 0x192b1908, + 0x2b080808, 0x192b1908, 0x08080819, 0x192b1919, 0x08081908, 0x192b1919, 0x08190808, 0x192b1919, + 0x19080808, 0x192b1919, 0x19082b2b, 0x192b1919, 0x192b2b08, 0x192b1919, 0x2b19082b, 0x192b1919, + 0x08080808, 0x192b192b, 0x2b191908, 0x192b192b, 0x08080819, 0x192b2b08, 0x08081908, 0x192b2b08, + 0x08190808, 0x192b2b08, 0x192b1919, 0x192b2b08, 0x2b192b08, 0x192b2b08, 0x08080808, 0x192b2b19, + 0x082b2b2b, 0x192b2b19, 0x1908082b, 0x192b2b2b, 0x2b2b0819, 0x192b2b2b, 0x08080808, 0x2b080808, + 0x0808082b, 0x2b080808, 0x08081919, 0x2b080808, 0x08082b08, 0x2b080808, 0x08190819, 0x2b080808, + 0x08191908, 0x2b080808, 0x08192b19, 0x2b080808, 0x082b0808, 0x2b080808, 0x082b1919, 0x2b080808, + 0x19080819, 0x2b080808, 0x19081908, 0x2b080808, 0x19190808, 0x2b080808, 0x1919082b, 0x2b080808, + 0x19191919, 0x2b080808, 0x19192b08, 0x2b080808, 0x192b0819, 0x2b080808, 0x2b080808, 0x2b080808, + 0x2b081919, 0x2b080808, 0x2b190819, 0x2b080808, 0x2b191908, 0x2b080808, 0x08080819, 0x2b080819, + 0x08081908, 0x2b080819, 0x08082b19, 0x2b080819, 0x08190808, 0x2b080819, 0x0819082b, 0x2b080819, + 0x08191919, 0x2b080819, 0x08192b08, 0x2b080819, 0x082b0819, 0x2b080819, 0x082b1908, 0x2b080819, + 0x19080808, 0x2b080819, 0x1908082b, 0x2b080819, 0x19081919, 0x2b080819, 0x19082b08, 0x2b080819, + 0x19190819, 0x2b080819, 0x19191908, 0x2b080819, 0x2b080819, 0x2b080819, 0x2b081908, 0x2b080819, + 0x2b190808, 0x2b080819, 0x2b2b2b19, 0x2b080819, 0x08080808, 0x2b08082b, 0x08081919, 0x2b08082b, + 0x08082b2b, 0x2b08082b, 0x08190819, 0x2b08082b, 0x08191908, 0x2b08082b, 0x19080819, 0x2b08082b, + 0x19081908, 0x2b08082b, 0x19190808, 0x2b08082b, 0x08080819, 0x2b081908, 0x08081908, 0x2b081908, + 0x0808192b, 0x2b081908, 0x08082b19, 0x2b081908, 0x08190808, 0x2b081908, 0x0819082b, 0x2b081908, + 0x08191919, 0x2b081908, 0x08192b08, 0x2b081908, 0x082b0819, 0x2b081908, 0x19080808, 0x2b081908, + 0x1908082b, 0x2b081908, 0x19081919, 0x2b081908, 0x19082b08, 0x2b081908, 0x19190819, 0x2b081908, + 0x19191908, 0x2b081908, 0x192b0808, 0x2b081908, 0x2b080819, 0x2b081908, 0x2b081908, 0x2b081908, + 0x2b190808, 0x2b081908, 0x08080808, 0x2b081919, 0x0808082b, 0x2b081919, 0x08081919, 0x2b081919, + 0x08082b08, 0x2b081919, 0x08190819, 0x2b081919, 0x08191908, 0x2b081919, 0x082b0808, 0x2b081919, + 0x19080819, 0x2b081919, 0x19081908, 0x2b081919, 0x19190808, 0x2b081919, 0x2b080808, 0x2b081919, + 0x2b082b2b, 0x2b081919, 0x08080819, 0x2b08192b, 0x08081908, 0x2b08192b, 0x08190808, 0x2b08192b, + 0x082b2b19, 0x2b08192b, 0x19080808, 0x2b08192b, 0x08080808, 0x2b082b08, 0x08081919, 0x2b082b08, + 0x08190819, 0x2b082b08, 0x08191908, 0x2b082b08, 0x19080819, 0x2b082b08, 0x19081908, 0x2b082b08, + 0x19190808, 0x2b082b08, 0x2b2b082b, 0x2b082b08, 0x08080819, 0x2b082b19, 0x08081908, 0x2b082b19, + 0x19080808, 0x2b082b19, 0x192b1919, 0x2b082b19, 0x082b082b, 0x2b082b2b, 0x19192b08, 0x2b082b2b, + 0x19192b2b, 0x2b082b2b, 0x2b08082b, 0x2b082b2b, 0x2b2b082b, 0x2b082b2b, 0x08080819, 0x2b190808, + 0x08081908, 0x2b190808, 0x08082b19, 0x2b190808, 0x08190808, 0x2b190808, 0x0819082b, 0x2b190808, + 0x08191919, 0x2b190808, 0x08192b08, 0x2b190808, 0x082b1908, 0x2b190808, 0x19080808, 0x2b190808, + 0x1908082b, 0x2b190808, 0x19081919, 0x2b190808, 0x19082b08, 0x2b190808, 0x19190819, 0x2b190808, + 0x19191908, 0x2b190808, 0x192b0808, 0x2b190808, 0x2b080819, 0x2b190808, 0x2b081908, 0x2b190808, + 0x2b190808, 0x2b190808, 0x08080808, 0x2b190819, 0x08081919, 0x2b190819, 0x08190819, 0x2b190819, + 0x08191908, 0x2b190819, 0x19080819, 0x2b190819, 0x19081908, 0x2b190819, 0x19190808, 0x2b190819, + 0x19192b2b, 0x2b190819, 0x08080819, 0x2b19082b, 0x08081908, 0x2b19082b, 0x08190808, 0x2b19082b, + 0x19080808, 0x2b19082b, 0x2b2b192b, 0x2b19082b, 0x08080808, 0x2b191908, 0x0808082b, 0x2b191908, + 0x08081919, 0x2b191908, 0x08082b08, 0x2b191908, 0x08190819, 0x2b191908, 0x08191908, 0x2b191908, + 0x082b0808, 0x2b191908, 0x19080819, 0x2b191908, 0x19081908, 0x2b191908, 0x19190808, 0x2b191908, + 0x2b080808, 0x2b191908, 0x2b19192b, 0x2b191908, 0x08080819, 0x2b191919, 0x08081908, 0x2b191919, + 0x08190808, 0x2b191919, 0x19080808, 0x2b191919, 0x2b192b08, 0x2b191919, 0x2b2b0819, 0x2b191919, + 0x08080808, 0x2b19192b, 0x1908192b, 0x2b19192b, 0x192b1908, 0x2b19192b, 0x08080819, 0x2b192b08, + 0x08081908, 0x2b192b08, 0x08190808, 0x2b192b08, 0x082b192b, 0x2b192b08, 0x19080808, 0x2b192b08, + 0x2b2b2b19, 0x2b192b08, 0x08080808, 0x2b192b19, 0x19082b19, 0x2b192b19, 0x1919082b, 0x2b192b19, + 0x2b190808, 0x2b192b2b, 0x08080808, 0x2b2b0808, 0x08081919, 0x2b2b0808, 0x08082b2b, 0x2b2b0808, + 0x08191908, 0x2b2b0808, 0x082b082b, 0x2b2b0808, 0x082b2b2b, 0x2b2b0808, 0x19080819, 0x2b2b0808, + 0x19081908, 0x2b2b0808, 0x19190808, 0x2b2b0808, 0x2b2b082b, 0x2b2b0808, 0x2b2b2b2b, 0x2b2b0808, + 0x19080808, 0x2b2b0819, 0x192b1919, 0x2b2b0819, 0x0808082b, 0x2b2b082b, 0x08082b2b, 0x2b2b082b, + 0x082b082b, 0x2b2b082b, 0x082b2b08, 0x2b2b082b, 0x082b2b2b, 0x2b2b082b, 0x2b08082b, 0x2b2b082b, + 0x2b082b08, 0x2b2b082b, 0x2b082b2b, 0x2b2b082b, 0x2b2b2b08, 0x2b2b082b, 0x08080819, 0x2b2b1908, + 0x08081908, 0x2b2b1908, 0x08190808, 0x2b2b1908, 0x19080808, 0x2b2b1908, 0x2b082b19, 0x2b2b1908, + 0x2b2b1908, 0x2b2b1908, 0x08080808, 0x2b2b1919, 0x08192b19, 0x2b2b1919, 0x19190819, 0x2b2b192b, + 0x08082b2b, 0x2b2b2b08, 0x082b2b08, 0x2b2b2b08, 0x2b2b082b, 0x2b2b2b08, 0x19191908, 0x2b2b2b19, + 0x2b08192b, 0x2b2b2b19, 0x08082b08, 0x2b2b2b2b, 0x08082b2b, 0x2b2b2b2b, 0x082b0808, 0x2b2b2b2b, + 0x082b082b, 0x2b2b2b2b, 0x082b2b08, 0x2b2b2b2b, 0x2b082b08, 0x2b2b2b2b, 0x2b2b2b2b, 0x2b2b2b2b) +} + +//! Main-context use only (tests, oracles, the emitter's constant bake) - a forked worker reads +//! a module global as zeros; kernels take `iq2s_grid2()` instead. +let IQ2S_GRID = iq2s_grid2() + //! Quant-plane bytes per stride unit — the typed stride surface. The stride unit is one //! 256-weight superblock for the kq lattice and one 32-weight block for q51 (see kq_elems). //! q8 panics (its planes are byte-per-elem, not stride-walked here). @@ -285,13 +559,16 @@ def kq_qsb(f : KqFmt) : int64 { if (f == KqFmt.k2) { return K2_QSB } + if (f == KqFmt.iq2s) { + return IQ2S_QSB + } panic("kq_qsb: no plane stride for this format") return 0l } //! Scale-plane bytes per stride unit (see kq_qsb for the unit). def kq_ssb(f : KqFmt) : int64 { - if (f == KqFmt.k4 || f == KqFmt.k5 || f == KqFmt.iq4xs || f == KqFmt.iq3s || f == KqFmt.iq3xxs || f == KqFmt.k2) { + if (f == KqFmt.k4 || f == KqFmt.k5 || f == KqFmt.iq4xs || f == KqFmt.iq3s || f == KqFmt.iq3xxs || f == KqFmt.k2 || f == KqFmt.iq2s) { return K4_SSB } if (f == KqFmt.k6 || f == KqFmt.k3) { diff --git a/modules/dasLLAMA/dasllama/dasllama_layout.das b/modules/dasLLAMA/dasllama/dasllama_layout.das index 472e65bbff..696ea89104 100644 --- a/modules/dasLLAMA/dasllama/dasllama_layout.das +++ b/modules/dasLLAMA/dasllama/dasllama_layout.das @@ -412,6 +412,10 @@ def private repack_regions(var t : Model; regs : array) { var k2sp : uint8? = null if (!empty(t.k2q)) { k2qp = addr(t.k2q[0]) } if (!empty(t.k2s)) { k2sp = addr(t.k2s[0]) } + var iq2sqp : uint8? = null + var iq2ssp : uint8? = null + if (!empty(t.iq2sq)) { iq2sqp = addr(t.iq2sq[0]) } + if (!empty(t.iq2ss)) { iq2ssp = addr(t.iq2ss[0]) } if (!empty(t.qblob)) { qbp = addr(t.qblob[0]) } if (!empty(t.qscales)) { qsp = addr(t.qscales[0]) } if (!empty(t.mxq)) { mxqp = addr(t.mxq[0]) } @@ -444,8 +448,8 @@ def private repack_regions(var t : Model; regs : array) { let fk = f == 20 ? 2 : f // k2 streams under 20 (kernel id 2 = q51's stream tag) let qsb = kq_qsb(fk) let ssb = kq_ssb(fk) - var kqp = f == 4 ? k4qp : (f == 5 ? k5qp : (f == 6 ? k6qp : (f == 40 ? q40qp : (f == 44 ? iq4xsqp : (f == 3 ? k3qp : (f == 33 ? iq3sqp : (f == 34 ? iq3xxsqp : (f == 45 ? iq4nlqp : k2qp)))))))) - var ksp = f == 4 ? k4sp : (f == 5 ? k5sp : (f == 6 ? k6sp : (f == 40 ? q40sp : (f == 44 ? iq4xssp : (f == 3 ? k3sp : (f == 33 ? iq3ssp : (f == 34 ? iq3xxssp : (f == 45 ? iq4nlsp : k2sp)))))))) + var kqp = f == 4 ? k4qp : (f == 5 ? k5qp : (f == 6 ? k6qp : (f == 40 ? q40qp : (f == 44 ? iq4xsqp : (f == 3 ? k3qp : (f == 33 ? iq3sqp : (f == 34 ? iq3xxsqp : (f == 45 ? iq4nlqp : (f == 2 ? k2qp : iq2sqp))))))))) + var ksp = f == 4 ? k4sp : (f == 5 ? k5sp : (f == 6 ? k6sp : (f == 40 ? q40sp : (f == 44 ? iq4xssp : (f == 3 ? k3sp : (f == 33 ? iq3ssp : (f == 34 ? iq3xxssp : (f == 45 ? iq4nlsp : (f == 2 ? k2sp : iq2ssp))))))))) invoke(rkq, fk, kqp + sb * qsb, ksp + sb * ssb, rp[i].n, rp[i].d) } } @@ -583,6 +587,8 @@ def private push_repack_kq(var regs : array; fmt : KqFmt; woff, n, d push_repack(regs, 45, woff, n, d) } elif (fmt == KqFmt.k2) { push_repack(regs, 20, woff, n, d) + } elif (fmt == KqFmt.iq2s) { + push_repack(regs, 23, woff, n, d) } } @@ -860,13 +866,13 @@ def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice : (fmt == KqFmt.k6 ? addr(t.k6q[sb0 * kq_qsb(6)]) : (fmt == KqFmt.q40 ? addr(t.q40q[sb0 * kq_qsb(40)]) : (fmt == KqFmt.iq4xs ? addr(t.iq4xsq[sb0 * kq_qsb(44)]) - : (fmt == KqFmt.k3 ? addr(t.k3q[sb0 * kq_qsb(3)]) : (fmt == KqFmt.iq3s ? addr(t.iq3sq[sb0 * kq_qsb(33)]) : (fmt == KqFmt.iq3xxs ? addr(t.iq3xxsq[sb0 * kq_qsb(34)]) : (fmt == KqFmt.iq4nl ? addr(t.iq4nlq[sb0 * kq_qsb(45)]) : addr(t.k2q[sb0 * kq_qsb(2)])))))))))) + : (fmt == KqFmt.k3 ? addr(t.k3q[sb0 * kq_qsb(3)]) : (fmt == KqFmt.iq3s ? addr(t.iq3sq[sb0 * kq_qsb(33)]) : (fmt == KqFmt.iq3xxs ? addr(t.iq3xxsq[sb0 * kq_qsb(34)]) : (fmt == KqFmt.iq4nl ? addr(t.iq4nlq[sb0 * kq_qsb(45)]) : (fmt == KqFmt.k2 ? addr(t.k2q[sb0 * kq_qsb(2)]) : addr(t.iq2sq[sb0 * kq_qsb(23)]))))))))))) let sp = (fmt == KqFmt.k4 ? addr(t.k4s[sb0 * kq_ssb(4)]) : (fmt == KqFmt.k5 ? addr(t.k5s[sb0 * kq_ssb(5)]) : (fmt == KqFmt.k6 ? addr(t.k6s[sb0 * kq_ssb(6)]) : (fmt == KqFmt.q40 ? addr(t.q40s[sb0 * kq_ssb(40)]) : (fmt == KqFmt.iq4xs ? addr(t.iq4xss[sb0 * kq_ssb(44)]) - : (fmt == KqFmt.k3 ? addr(t.k3s[sb0 * kq_ssb(3)]) : (fmt == KqFmt.iq3s ? addr(t.iq3ss[sb0 * kq_ssb(33)]) : (fmt == KqFmt.iq3xxs ? addr(t.iq3xxss[sb0 * kq_ssb(34)]) : (fmt == KqFmt.iq4nl ? addr(t.iq4nls[sb0 * kq_ssb(45)]) : addr(t.k2s[sb0 * kq_ssb(2)])))))))))) + : (fmt == KqFmt.k3 ? addr(t.k3s[sb0 * kq_ssb(3)]) : (fmt == KqFmt.iq3s ? addr(t.iq3ss[sb0 * kq_ssb(33)]) : (fmt == KqFmt.iq3xxs ? addr(t.iq3xxss[sb0 * kq_ssb(34)]) : (fmt == KqFmt.iq4nl ? addr(t.iq4nls[sb0 * kq_ssb(45)]) : (fmt == KqFmt.k2 ? addr(t.k2s[sb0 * kq_ssb(2)]) : addr(t.iq2ss[sb0 * kq_ssb(23)]))))))))))) var wqp = addr(wq[0]) var wsp = addr(ws[0]) let njobs = is_job_que_available() ? min(nslices, 4 * (get_total_hw_jobs() + 1)) : 1 @@ -905,6 +911,10 @@ def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice for (m in range64(64l)) { dq[m] = gq[((m / 4l) * mr + r) * 4l + m % 4l] } + } elif (fmt == KqFmt.iq2s) { // 18 uniform 4-byte columns over [idx][signs][qh] + for (m in range64(72l)) { + dq[m] = gq[((m / 4l) * mr + r) * 4l + m % 4l] + } } else { for (m in range64(128l)) { // nibble region: verbatim per (blk, j, t) dq[m] = gq[((m / 4l) * mr + r) * 4l + m % 4l] @@ -956,7 +966,7 @@ def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice } else { let rq = qp + sliceQ + ri * qrow + sbi * qsb let rs = sp + sliceS + ri * srow + sbi * dssb - if (fmt == KqFmt.q40 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s || fmt == KqFmt.iq3xxs || fmt == KqFmt.iq4nl || fmt == KqFmt.k2) { + if (fmt == KqFmt.q40 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s || fmt == KqFmt.iq3xxs || fmt == KqFmt.iq4nl || fmt == KqFmt.k2 || fmt == KqFmt.iq2s) { for (m in range64(qsb)) { // disk bytes already in the device form (k/k+16 pairs; k3's lanes) dq[m] = rq[m] } @@ -983,7 +993,7 @@ def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice dq[128l + p] = rq[128l + p] } } - if (fmt == KqFmt.k6 || fmt == KqFmt.q40 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s || fmt == KqFmt.iq3xxs || fmt == KqFmt.iq4nl || fmt == KqFmt.k2) { // rows already in the decoded device form + if (fmt == KqFmt.k6 || fmt == KqFmt.q40 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s || fmt == KqFmt.iq3xxs || fmt == KqFmt.iq4nl || fmt == KqFmt.k2 || fmt == KqFmt.iq2s) { // rows already in the decoded device form for (idx in range64(dssb)) { dsc[idx] = rs[idx] } diff --git a/modules/dasLLAMA/dasllama/dasllama_load.das b/modules/dasLLAMA/dasllama/dasllama_load.das index 3e01982942..7e0312fc13 100644 --- a/modules/dasLLAMA/dasllama/dasllama_load.das +++ b/modules/dasLLAMA/dasllama/dasllama_load.das @@ -47,6 +47,7 @@ struct private LayoutSizes { iq3xxs_n : int64 iq4nl_n : int64 k2_n : int64 + iq2s_n : int64 pleq8_n : int64 // the dedicated q8 plane holding ONLY the PLE table (fp32/q4 serving) } @@ -65,6 +66,7 @@ struct private KqCursors { iq3xxs : int64 iq4nl : int64 k2 : int64 + iq2s : int64 } def private kq_take(var cur : KqCursors; f : KqFmt; n : int64) : int64 { @@ -123,6 +125,11 @@ def private kq_take(var cur : KqCursors; f : KqFmt; n : int64) : int64 { cur.k2 += n return o } + if (f == KqFmt.iq2s) { + let o = cur.iq2s + cur.iq2s += n + return o + } let o = cur.wo cur.wo += n return o @@ -422,7 +429,7 @@ def private layout_offsets(var t : Model) : LayoutSizes { // nolint:STYLE037,S fo += dim } return LayoutSizes(fblob_n = fo, wblob_n = cur.wo, mx_n = mx, bf16_n = bf16, - k4_n = cur.k4, k5_n = cur.k5, k6_n = cur.k6, q40_n = cur.q40, q51_n = cur.q51, iq4xs_n = cur.iq4xs, k3_n = cur.k3, iq3s_n = cur.iq3s, iq3xxs_n = cur.iq3xxs, iq4nl_n = cur.iq4nl, k2_n = cur.k2, pleq8_n = pleq8) + k4_n = cur.k4, k5_n = cur.k5, k6_n = cur.k6, q40_n = cur.q40, q51_n = cur.q51, iq4xs_n = cur.iq4xs, k3_n = cur.k3, iq3s_n = cur.iq3s, iq3xxs_n = cur.iq3xxs, iq4nl_n = cur.iq4nl, k2_n = cur.k2, iq2s_n = cur.iq2s, pleq8_n = pleq8) } @@ -693,6 +700,8 @@ def stream_field_of(t : Model; fmt : KqFmt) : string { return "iq4nlq" } elif (fmt == KqFmt.k2) { return "k2q" + } elif (fmt == KqFmt.iq2s) { + return "iq2sq" } elif (t.quant == QuantMode.q8) { // the blob flavor carries q8 as gguf-native 34B blocks in ONE plane, never the split pair return t.metal_blob ? "mblob" : "qblob" @@ -821,6 +830,8 @@ def fill_stream_plane(m : GGUFMeta; bytes : array | #; var t : Model; fie memcpy(addr(t.iq4nls[(j.woff / 256l) * Q40_SSB]), addr(temp_s[0]), sh.sb) } elif (j.fmt == KqFmt.k2) { memcpy(addr(t.k2s[(j.woff / 256l) * K2_SSB]), addr(temp_s[0]), sh.sb) + } elif (j.fmt == KqFmt.iq2s) { + memcpy(addr(t.iq2ss[(j.woff / 256l) * IQ2S_SSB]), addr(temp_s[0]), sh.sb) } elif (j.fmt == KqFmt.q51) { memcpy(addr(t.q51s[(j.woff / 32l) * Q51_SB]), addr(temp_s[0]), sh.sb) } elif (sh.sb == 0l && j.n > 0l && !t.metal_blob) { @@ -870,6 +881,9 @@ def fill_stream_plane(m : GGUFMeta; bytes : array | #; var t : Model; fie } elif (r.fmt == 20) { invoke(g_stream_repack, 20, addr(temp_q[((r.off - j.woff) / 256l) * K2_QSB]), addr(t.k2s[(r.off / 256l) * K2_SSB]), r.n, r.d) + } elif (r.fmt == 23) { + invoke(g_stream_repack, 23, addr(temp_q[((r.off - j.woff) / 256l) * IQ2S_QSB]), + addr(t.iq2ss[(r.off / 256l) * IQ2S_SSB]), r.n, r.d) } } } @@ -1100,6 +1114,9 @@ def private transcode_kq_tensor(m : GGUFMeta; bytes : array | #; name : s } elif (fmt == KqFmt.k2) { gguf_transcode_q2k(m, bytes, name, kq, ks, eloff, n, src_off) return "k2 transcode (Q2_K)" + } elif (fmt == KqFmt.iq2s) { + gguf_transcode_iq2s(m, bytes, name, kq, ks, eloff, n, src_off) + return "iq2s transcode (IQ2_S)" } gguf_transcode_q51(m, bytes, name, kq, ks, eloff, n, src_off) return "q51 transcode (Q5_1)" @@ -1150,6 +1167,8 @@ def private load_big(m : GGUFMeta; bytes : array | #; name : string; var kind = transcode_kq_tensor(m, bytes, name, fmt, t.iq4nlq, t.iq4nls, woff, n, src_off, scratch) } elif (fmt == KqFmt.k2) { kind = transcode_kq_tensor(m, bytes, name, fmt, t.k2q, t.k2s, woff, n, src_off, scratch) + } elif (fmt == KqFmt.iq2s) { + kind = transcode_kq_tensor(m, bytes, name, fmt, t.iq2sq, t.iq2ss, woff, n, src_off, scratch) } elif (fmt == KqFmt.q51) { kind = transcode_kq_tensor(m, bytes, name, fmt, t.q51q, t.q51s, woff, n, src_off, scratch) } elif (t.quant == QuantMode.q8) { @@ -1249,8 +1268,8 @@ def private log_load_report(t : Model) { to_log(LOG_INFO, "dasLLAMA noisy: backend '{active_kernel_backend()}' | kquant_native {get_kquant_native()} q40_native {get_kq_q40_native()} q50_native {get_kq_q50_native()}\n") // the tier arms at [init], before a --noisy CLI flag can land — restate it here so the flag alone suffices to_log(LOG_INFO, "dasLLAMA noisy: GPU tier installed {moe_gpu_tier_installed()} | want auto {gpu_want_auto()}, moe layers {gpu_want_moe_layers()}, stream {gpu_want_moe_stream()}\n") - to_log(LOG_INFO, "dasLLAMA noisy: format tags - q8 {tally_fmt(t, KqFmt.q8)}, k4 {tally_fmt(t, KqFmt.k4)}, k5 {tally_fmt(t, KqFmt.k5)}, k6 {tally_fmt(t, KqFmt.k6)}, q40 {tally_fmt(t, KqFmt.q40)}, q51 {tally_fmt(t, KqFmt.q51)}, iq4xs {tally_fmt(t, KqFmt.iq4xs)}, k3 {tally_fmt(t, KqFmt.k3)}, iq3s {tally_fmt(t, KqFmt.iq3s)}, iq3xxs {tally_fmt(t, KqFmt.iq3xxs)}, iq4nl {tally_fmt(t, KqFmt.iq4nl)}, k2 {tally_fmt(t, KqFmt.k2)}\n") - to_log(LOG_INFO, "dasLLAMA noisy: plane MB - q8 {float(long_length(t.qblob)) / mb} (+scales {float(long_length(t.qscales) * 4l) / mb}), k4 {float(long_length(t.k4q) + long_length(t.k4s)) / mb}, k5 {float(long_length(t.k5q) + long_length(t.k5s)) / mb}, k6 {float(long_length(t.k6q) + long_length(t.k6s)) / mb}, q40 {float(long_length(t.q40q) + long_length(t.q40s)) / mb}, q51 {float(long_length(t.q51q) + long_length(t.q51s)) / mb}, iq4xs {float(long_length(t.iq4xsq) + long_length(t.iq4xss)) / mb}, k3 {float(long_length(t.k3q) + long_length(t.k3s)) / mb}, iq3s {float(long_length(t.iq3sq) + long_length(t.iq3ss)) / mb}, iq3xxs {float(long_length(t.iq3xxsq) + long_length(t.iq3xxss)) / mb}, iq4nl {float(long_length(t.iq4nlq) + long_length(t.iq4nls)) / mb}, k2 {float(long_length(t.k2q) + long_length(t.k2s)) / mb}, f32 {float(long_length(t.wblob) * 4l) / mb}\n") + to_log(LOG_INFO, "dasLLAMA noisy: format tags - q8 {tally_fmt(t, KqFmt.q8)}, k4 {tally_fmt(t, KqFmt.k4)}, k5 {tally_fmt(t, KqFmt.k5)}, k6 {tally_fmt(t, KqFmt.k6)}, q40 {tally_fmt(t, KqFmt.q40)}, q51 {tally_fmt(t, KqFmt.q51)}, iq4xs {tally_fmt(t, KqFmt.iq4xs)}, k3 {tally_fmt(t, KqFmt.k3)}, iq3s {tally_fmt(t, KqFmt.iq3s)}, iq3xxs {tally_fmt(t, KqFmt.iq3xxs)}, iq4nl {tally_fmt(t, KqFmt.iq4nl)}, k2 {tally_fmt(t, KqFmt.k2)}, iq2s {tally_fmt(t, KqFmt.iq2s)}\n") + to_log(LOG_INFO, "dasLLAMA noisy: plane MB - q8 {float(long_length(t.qblob)) / mb} (+scales {float(long_length(t.qscales) * 4l) / mb}), k4 {float(long_length(t.k4q) + long_length(t.k4s)) / mb}, k5 {float(long_length(t.k5q) + long_length(t.k5s)) / mb}, k6 {float(long_length(t.k6q) + long_length(t.k6s)) / mb}, q40 {float(long_length(t.q40q) + long_length(t.q40s)) / mb}, q51 {float(long_length(t.q51q) + long_length(t.q51s)) / mb}, iq4xs {float(long_length(t.iq4xsq) + long_length(t.iq4xss)) / mb}, k3 {float(long_length(t.k3q) + long_length(t.k3s)) / mb}, iq3s {float(long_length(t.iq3sq) + long_length(t.iq3ss)) / mb}, iq3xxs {float(long_length(t.iq3xxsq) + long_length(t.iq3xxss)) / mb}, iq4nl {float(long_length(t.iq4nlq) + long_length(t.iq4nls)) / mb}, k2 {float(long_length(t.k2q) + long_length(t.k2s)) / mb}, iq2s {float(long_length(t.iq2sq) + long_length(t.iq2ss)) / mb}, f32 {float(long_length(t.wblob) * 4l) / mb}\n") } // GGML disk type -> plane format tag (non-K-quant types ride the classic q8 path) @@ -1282,6 +1301,9 @@ def private kq_fmt_of(gt : int) : KqFmt { if (gt == GGML_TYPE_Q2_K) { return KqFmt.k2 } + if (gt == GGML_TYPE_IQ2_S) { + return KqFmt.iq2s + } if (gt == GGML_TYPE_Q4_0 && get_kq_q40_native()) { return KqFmt.q40 } @@ -2121,6 +2143,7 @@ def private load_gguf_parsed(var t : Model; m : GGUFMeta; bytes : array | g_stream_plane_total["iq3xxsq"] = (sz.iq3xxs_n / 256l) * IQ3XXS_QSB g_stream_plane_total["iq4nlq"] = (sz.iq4nl_n / 256l) * Q40_QSB g_stream_plane_total["k2q"] = (sz.k2_n / 256l) * K2_QSB + g_stream_plane_total["iq2sq"] = (sz.iq2s_n / 256l) * IQ2S_QSB let wb = mode == QuantMode.q8 ? sz.wblob_n : 0l g_stream_plane_total[t.metal_blob ? "mblob" : "qblob"] = (t.metal_blob ? (wb / Q8_BLOCK_ELEMS) * (Q8_QPB + Q8_SPB) : wb) @@ -2213,6 +2236,14 @@ def private load_gguf_parsed(var t : Model; m : GGUFMeta; bytes : array | t.k2s |> reserve((sz.k2_n / 256l) * K2_SSB) t.k2s |> resize((sz.k2_n / 256l) * K2_SSB) } + if (sz.iq2s_n > 0l) { + if (!stream_q) { + t.iq2sq |> reserve((sz.iq2s_n / 256l) * IQ2S_QSB) + t.iq2sq |> resize((sz.iq2s_n / 256l) * IQ2S_QSB) + } + t.iq2ss |> reserve((sz.iq2s_n / 256l) * IQ2S_SSB) + t.iq2ss |> resize((sz.iq2s_n / 256l) * IQ2S_SSB) + } if (mode == QuantMode.q8) { if (!stream_q) { t.qblob |> reserve(sz.wblob_n) @@ -2525,6 +2556,7 @@ def private load_gguf_parsed(var t : Model; m : GGUFMeta; bytes : array | t.kq_repack_mr34 = active_kq_layout_mr(34) t.kq_repack_mr45 = active_kq_layout_mr(45) t.kq_repack_mr2 = active_kq_layout_mr(2) + t.kq_repack_mr23 = active_kq_layout_mr(23) } } else { layout_repack_q8(t) @@ -2547,7 +2579,8 @@ def private load_gguf_parsed(var t : Model; m : GGUFMeta; bytes : array | t.kq_repack_mr34 = active_kq_layout_mr(34) t.kq_repack_mr45 = active_kq_layout_mr(45) t.kq_repack_mr2 = active_kq_layout_mr(2) - to_log(LOG_INFO, "dasLLAMA: K-quant planes repacked to the '{active_kernel_backend()}' layouts (k4 grp{t.kq_repack_mr4}, k5 grp{t.kq_repack_mr5}, k6 grp{t.kq_repack_mr6}, q40 grp{t.kq_repack_mr40}, iq4xs grp{t.kq_repack_mr44}, k3 grp{t.kq_repack_mr3}, iq3s grp{t.kq_repack_mr33}, iq3xxs grp{t.kq_repack_mr34}, iq4nl grp{t.kq_repack_mr45}, k2 grp{t.kq_repack_mr2})\n") + t.kq_repack_mr23 = active_kq_layout_mr(23) + to_log(LOG_INFO, "dasLLAMA: K-quant planes repacked to the '{active_kernel_backend()}' layouts (k4 grp{t.kq_repack_mr4}, k5 grp{t.kq_repack_mr5}, k6 grp{t.kq_repack_mr6}, q40 grp{t.kq_repack_mr40}, iq4xs grp{t.kq_repack_mr44}, k3 grp{t.kq_repack_mr3}, iq3s grp{t.kq_repack_mr33}, iq3xxs grp{t.kq_repack_mr34}, iq4nl grp{t.kq_repack_mr45}, k2 grp{t.kq_repack_mr2}, iq2s grp{t.kq_repack_mr23})\n") } } } diff --git a/modules/dasLLAMA/dasllama/dasllama_math.das b/modules/dasLLAMA/dasllama/dasllama_math.das index 71255d2064..4ba609527b 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math.das +++ b/modules/dasLLAMA/dasllama/dasllama_math.das @@ -1090,6 +1090,7 @@ struct KernelBackend { kq_rows_iq3xxs : MatmulKqRowsFn = @@kq_unset_rows kq_rows_iq4nl : MatmulKqRowsFn = @@kq_unset_rows kq_rows_k2 : MatmulKqRowsFn = @@kq_unset_rows + kq_rows_iq2s : MatmulKqRowsFn = @@kq_unset_rows kq_batch : MatmulKqBatchFn = @@kq_unset_batch kq_groupn : MatmulKqGroupNFn = @@kq_unset_groupn kq_batch_groupn : MatmulKqBatchGroupNFn = @@kq_unset_batch_groupn @@ -1158,6 +1159,7 @@ var g_kq_rows_iq3s = @@kq_unset_rows var g_kq_rows_iq3xxs = @@kq_unset_rows var g_kq_rows_iq4nl = @@kq_unset_rows var g_kq_rows_k2 = @@kq_unset_rows +var g_kq_rows_iq2s = @@kq_unset_rows var g_mm_kq_batch = @@kq_unset_batch var g_mm_kq_groupn = @@kq_unset_groupn var g_mm_kq_batch_groupn = @@kq_unset_batch_groupn @@ -1485,7 +1487,7 @@ def active_q8_layout_mr() : int64 => g_bake_cpu_override ? g_bake_cpu.q8_mr : g_ //! family split; the loader freezes these onto the Model at repack time. def active_kq_layout_mr(fmt : int) : int64 { if (g_bake_cpu_override) { - return fmt == 4 ? g_bake_cpu.kq_mr4 : (fmt == 5 ? g_bake_cpu.kq_mr5 : (fmt == 40 ? g_bake_cpu.kq_mr40 : (fmt == 44 ? g_bake_cpu.kq_mr44 : (fmt == 3 ? g_bake_cpu.kq_mr3 : (fmt == 33 ? g_bake_cpu.kq_mr33 : (fmt == 34 ? g_bake_cpu.kq_mr34 : (fmt == 45 ? g_bake_cpu.kq_mr45 : (fmt == 2 ? g_bake_cpu.kq_mr2 : g_bake_cpu.kq_mr6)))))))) + return fmt == 4 ? g_bake_cpu.kq_mr4 : (fmt == 5 ? g_bake_cpu.kq_mr5 : (fmt == 40 ? g_bake_cpu.kq_mr40 : (fmt == 44 ? g_bake_cpu.kq_mr44 : (fmt == 3 ? g_bake_cpu.kq_mr3 : (fmt == 33 ? g_bake_cpu.kq_mr33 : (fmt == 34 ? g_bake_cpu.kq_mr34 : (fmt == 45 ? g_bake_cpu.kq_mr45 : (fmt == 2 ? g_bake_cpu.kq_mr2 : (fmt == 23 ? g_bake_cpu.kq_mr23 : g_bake_cpu.kq_mr6))))))))) } return int64(invoke(g_active_kq_layout, fmt)) } @@ -1667,6 +1669,7 @@ def private activate(be : KernelBackend) { g_kq_rows_iq3xxs = be.kq_rows_iq3xxs g_kq_rows_iq4nl = be.kq_rows_iq4nl g_kq_rows_k2 = be.kq_rows_k2 + g_kq_rows_iq2s = be.kq_rows_iq2s g_mm_kq_batch = be.kq_batch g_mm_kq_groupn = be.kq_groupn g_mm_kq_batch_groupn = be.kq_batch_groupn @@ -1875,6 +1878,7 @@ def kq_rows_fn(fmt : int) : MatmulKqRowsFn { if (fmt == 34) return g_kq_rows_iq3xxs if (fmt == 45) return g_kq_rows_iq4nl if (fmt == 2) return g_kq_rows_k2 + if (fmt == 23) return g_kq_rows_iq2s return @@kq_unset_rows } @@ -3253,6 +3257,9 @@ def private restore_math_fn_defaults { // nolint:STYLE037,STYLE038 — flat seam if (g_kq_rows_k2 == null) { g_kq_rows_k2 = @@kq_unset_rows } + if (g_kq_rows_iq2s == null) { + g_kq_rows_iq2s = @@kq_unset_rows + } if (g_mm_kq_batch == null) { g_mm_kq_batch = @@kq_unset_batch } diff --git a/modules/dasLLAMA/dasllama/dasllama_math_default.das b/modules/dasLLAMA/dasllama/dasllama_math_default.das index baa1f2230e..e90eacd735 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_default.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_default.das @@ -652,6 +652,43 @@ def dot_iq3s_q8(kqrow : uint8 const?; ksrow : uint8 const?; xqp : int8 const?; x return acc } +//! IQ2_S row dot off the DISK-ORDER planes: per 32-block, grid entry idx | qh<<8 bits yields +//! EIGHT magnitudes (two uint words), the block's sign bytes flip them, per-16 strips (1 + 2s) +//! fold on the eighth-ed d; signed weights, so no activation block-sum term. +[unused_argument(xbsp), hint(unsafe_range_check, noalias = kqrow, noalias = ksrow, noalias = xqp, noalias = xsp, noalias = xbsp)] +def dot_iq2s_q8(kqrow : uint8 const?; ksrow : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n : int64) : float { + var acc = 0.0 + let grid = iq2s_grid2() + unsafe { + for (sb in range64(n / 256l)) { + let kqo = sb * 72l + let kso = sb * 20l + let d8 = f16_to_f32(uint(ksrow[kso]) | (uint(ksrow[kso + 1l]) << 8u)) + var isum = 0 + for (b in range64(8l)) { + let qh = int(kqrow[kqo + 64l + b]) + let ab = (sb * 8l + b) * 32l + for (l in range64(4l)) { + let sgn = int(kqrow[kqo + 32l + b * 4l + l]) + let idx = int(kqrow[kqo + b * 4l + l]) | ((qh << int(8l - 2l * l)) & 0x300) + let g1 = grid[idx * 2] + let g2 = grid[idx * 2 + 1] + var ig = 0 + for (j in range64(4l)) { + let w1 = int((g1 >> uint(8l * j)) & 0xFFu) + let w2 = int((g2 >> uint(8l * j)) & 0xFFu) + ig += (((sgn >> int(j)) & 1) != 0 ? -w1 : w1) * int(xqp[ab + l * 8l + j]) + ig += (((sgn >> int(4l + j)) & 1) != 0 ? -w2 : w2) * int(xqp[ab + l * 8l + 4l + j]) + } + isum += int(ksrow[kso + 4l + 2l * b + l / 2l]) * ig + } + } + acc += float(isum) * (d8 * xsp[sb]) + } + } + return acc +} + // Row-range cores + full GEMVs per format. Not `private`: invoked through hoisted function // pointers from lifted worker lambdas (the fused chains) and the dispatch wrappers in common. def k4_rows_kernel(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, rb, re : int64) { @@ -838,6 +875,15 @@ def k2_rows_kernel(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp } } +def iq2s_rows_kernel(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, rb, re : int64) { + let nsb = n / 256l + unsafe { + for (i in range64(rb, re)) { + yp[i] = dot_iq2s_q8(kqp + i * nsb * 72l, ksp + i * nsb * 20l, xqp, xsp, xbsp, n) + } + } +} + def private kq_gemv_kernel(fmt : int; var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d : int64) { var myp = yp maybe_parallel_for(0, int(d), matmul_chunks_gemv(int(d), 1, n * d)) $(rb, re) { @@ -859,6 +905,8 @@ def private kq_gemv_kernel(fmt : int; var yp : float?; kqp : uint8 const?; ksp : iq4nl_rows_kernel(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb), int64(re)) } elif (fmt == 2) { k2_rows_kernel(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb), int64(re)) + } elif (fmt == 23) { + iq2s_rows_kernel(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb), int64(re)) } else { k6_rows_kernel(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb), int64(re)) } @@ -1051,10 +1099,41 @@ def private dequant_k2_row_grp(kqg : uint8 const?; ksg : uint8 const?; r, mr, ns } } +//! iq2s grp row dequant: 18 uniform 4-byte columns per superblock row ([idx][signs][qh] in +//! place), the k4-shaped scale interleave with 16 strips - w = (d8 * (1 + 2s)) * grid * sign. +def private dequant_iq2s_row_grp(kqg : uint8 const?; ksg : uint8 const?; r, mr, nsb : int64; var dst : float?) { + let grid = iq2s_grid2() + unsafe { + for (sbi in range64(nsb)) { + let qb = sbi * 72l * mr + let sb = sbi * 20l * mr + let d8 = f16_to_f32(uint(ksg[sb + 2l * r]) | (uint(ksg[sb + 2l * r + 1l]) << 8u)) + for (b in range64(8l)) { + let hp = 64l + b + let qh = int(kqg[qb + ((hp / 4l) * mr + r) * 4l + hp % 4l]) + for (l in range64(4l)) { + let db = d8 * float(int(ksg[sb + 4l * mr + (2l * b + l / 2l) * mr + r])) + let sp = 32l + b * 4l + l + let sgn = int(kqg[qb + ((sp / 4l) * mr + r) * 4l + sp % 4l]) + let ip = b * 4l + l + let idx = int(kqg[qb + ((ip / 4l) * mr + r) * 4l + ip % 4l]) | ((qh << int(8l - 2l * l)) & 0x300) + let g1 = grid[idx * 2] + let g2 = grid[idx * 2 + 1] + let kb = sbi * 256l + b * 32l + l * 8l + for (j in range64(4l)) { + dst[kb + j] = db * float((g1 >> uint(8l * j)) & 0xFFu) * (((sgn >> int(j)) & 1) != 0 ? -1.0 : 1.0) + dst[kb + 4l + j] = db * float((g2 >> uint(8l * j)) & 0xFFu) * (((sgn >> int(4l + j)) & 1) != 0 ? -1.0 : 1.0) + } + } + } + } + } +} + //! Dequant one row off the grp-REPACKED K-quant planes (repack_k4/k5/k6_grp layout). //! kqg/ksg = row's GROUP plane base (group g = row/mr); r = row % mr. //! embed_row's kq_repacked path — per-token, so scalar is fine. -def dequant_kq_row_grp(fmt : int64; kqg : uint8 const?; ksg : uint8 const?; r, mr, n : int64; var dst : float?) { // nolint:STYLE038 — the flat per-format dispatch (one arm per kq format) is the honest shape +def dequant_kq_row_grp(fmt : int64; kqg : uint8 const?; ksg : uint8 const?; r, mr, n : int64; var dst : float?) { // nolint:STYLE037,STYLE038 — the flat per-format dispatch (one arm per kq format) is the honest shape let nsb = n / 256l let qsb = kq_qsb(int(fmt)) let ssb = kq_ssb(int(fmt)) @@ -1086,6 +1165,10 @@ def dequant_kq_row_grp(fmt : int64; kqg : uint8 const?; ksg : uint8 const?; r, m dequant_k2_row_grp(kqg, ksg, r, mr, nsb, dst) return } + if (fmt == 23l) { + dequant_iq2s_row_grp(kqg, ksg, r, mr, nsb, dst) + return + } unsafe { for (sbi in range64(nsb)) { let qb = sbi * qsb * mr @@ -1192,6 +1275,8 @@ def matmul_kq_groupn(fmt : int; var y : array | #; kq : array | #; myp[ii] = dot_iq4nl_q8(kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) } elif (fmt == 2) { myp[ii] = dot_k2q8(kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) + } elif (fmt == 23) { + myp[ii] = dot_iq2s_q8(kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) } else { myp[ii] = dot_k6q8(kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) } diff --git a/modules/dasLLAMA/dasllama/dasllama_math_gen.das b/modules/dasLLAMA/dasllama/dasllama_math_gen.das index a72a2464de..c7356f9e71 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_gen.das @@ -267,6 +267,10 @@ def k2q8_layout_gen() : int { return q8q8_repack_type(GEMM_REFERENCE_MR).interleave } +def iq2sq8_layout_gen() : int { + return q8q8_repack_type(GEMM_REFERENCE_MR).interleave +} + // the KernelBackend.kq_layout slot form of kq_layout_of (registered on both gen backends) def private kq_layout_fmt_gen(fmt : int) : int => int(kq_layout_of(fmt)) @@ -300,6 +304,9 @@ def kq_layout_of(fmt : int) : int64 { if (fmt == 2) { return int64(k2q8_layout_gen()) } + if (fmt == 23) { + return int64(iq2sq8_layout_gen()) + } return int64(k6q8_layout_gen()) } @@ -891,6 +898,78 @@ def k2q8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp : } } +//! One row's dot off the grp iq2s planes, scalar - the iq2s stubs' reference body and the +//! repack oracle: the 18-column layout's per-block grid gather + sign flip, per-16 strips +//! (1 + 2s) on the eighth-ed d, no activation-sum term. Same fold order as dot_iq2s_q8. +[unused_argument(xbsp)] +def iq2s_grp_row_dot(kqg : uint8 const?; ksg : uint8 const?; r, mr : int64; xqp : int8 const?; xsp : float const?; xbsp : int const?; n : int64) : float { + var acc = 0.0 + let nsb = n / 256l + let grid = iq2s_grid2() + unsafe { + for (sbi in range64(nsb)) { + let qb = sbi * 72l * mr + let sb = sbi * 20l * mr + var isum = 0 + for (b in range64(8l)) { + let hp = 64l + b + let qh = int(kqg[qb + ((hp / 4l) * mr + r) * 4l + hp % 4l]) + for (l in range64(4l)) { + let sp = 32l + b * 4l + l + let sgn = int(kqg[qb + ((sp / 4l) * mr + r) * 4l + sp % 4l]) + let ip = b * 4l + l + let idx = int(kqg[qb + ((ip / 4l) * mr + r) * 4l + ip % 4l]) | ((qh << int(8l - 2l * l)) & 0x300) + let g1 = grid[idx * 2] + let g2 = grid[idx * 2 + 1] + var ig = 0 + for (j in range64(4l)) { + let w1 = int((g1 >> uint(8l * j)) & 0xFFu) + let w2 = int((g2 >> uint(8l * j)) & 0xFFu) + ig += (((sgn >> int(j)) & 1) != 0 ? -w1 : w1) * int(xqp[(sbi * 8l + b) * 32l + l * 8l + j]) + ig += (((sgn >> int(4l + j)) & 1) != 0 ? -w2 : w2) * int(xqp[(sbi * 8l + b) * 32l + l * 8l + 4l + j]) + } + isum += int(ksg[sb + 4l * mr + (2l * b + l / 2l) * mr + r]) * ig + } + } + let dk = f16_to_f32(uint(ksg[sb + 2l * r]) | (uint(ksg[sb + 2l * r + 1l]) << 8u)) + acc += float(isum) * (dk * xsp[sbi]) + } + } + return acc +} + +[hint(unsafe_range_check, noalias = kqp, noalias = ksp, noalias = xqp, noalias = xsp, noalias = xbsp)] +def iq2sq8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, rb, re : int64) : void { + let mr = int64(iq2sq8_layout_gen()) + let nsb = n / 256l + unsafe { + for (i in range64(rb, re)) { + let g = i / mr + yp[i] = iq2s_grp_row_dot(kqp + g * mr * nsb * 72l, ksp + g * mr * nsb * 20l, i % mr, mr, xqp, xsp, xbsp, n) + } + } +} + +[tune_perm(mr = 4), tune_perm(mr = 4, nrsplit = 2), tune_perm(mr = 8), tune_perm(mr = 8, nrsplit = 2), + tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "maddubs", width = 256, mr = 8, nrsplit = 2, requires = "avx2"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, nrsplit = 2, requires = "avxvnni|avx512vnni"), + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, nrsplit = 2, requires = "avx512vnni,avx512bw"), + tune_companion(fn = "iq2sq8_gemv_gen", gen = "dasllama_gemm_gen::iq2s_gemv"), + tune_companion(fn = "iq2sq8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), + tune(gen = "dasllama_gemm_gen::iq2s_tile", + fallback = "dot_vpdpbusd_width512_mr16;dot_vpdpbusd_width256_mr8;dot_maddubs_width256_mr8;mr4"), + hint(unsafe_range_check, noalias = kqg, noalias = ksg, noalias = xqp, noalias = xsp, noalias = xbsp)] +def iq2sq8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d, g, t0 : int64) : void { + let mr = int64(iq2sq8_layout_gen()) + unsafe { + for (t in range64(4l)) { + for (r in range64(mr)) { + yp[(t0 + t) * d + g * mr + r] = iq2s_grp_row_dot(kqg, ksg, r, mr, xqp + (t0 + t) * n, xsp + (t0 + t) * (n / 256l), xbsp + (t0 + t) * (n / 16l), n) + } + } + } +} + //! The f16-scale GEMV companion (wscale_f16 rail): q8q8_gemv_gen's twin over the raw binary16 //! group-scale plane. Same rows-range contract; the generated body differs only in the scale //! fold's widen (fcvtl/vcvtph2ps). Not `private`: it IS the mm_rows_s16 slot. @@ -1226,6 +1305,8 @@ def private repack_kq_gen(fmt : int; var kq : uint8?; var ks : uint8?; n, d : in repack_q40_grp(kq, ks, n, d, kq_layout_of(45)) // byte-identical plane shapes } elif (fmt == 2) { repack_k2_grp(kq, ks, n, d, kq_layout_of(2)) + } elif (fmt == 23) { + repack_iq2s_grp(kq, ks, n, d, kq_layout_of(23)) } else { repack_k6_grp(kq, ks, n, d, kq_layout_of(6)) } @@ -1250,6 +1331,8 @@ def private repack_kq_bake(fmt : int; var kq : uint8?; var ks : uint8?; n, d : i repack_q40_grp(kq, ks, n, d, active_kq_layout_mr(45)) // byte-identical plane shapes } elif (fmt == 2) { repack_k2_grp(kq, ks, n, d, active_kq_layout_mr(2)) + } elif (fmt == 23) { + repack_iq2s_grp(kq, ks, n, d, active_kq_layout_mr(23)) } else { repack_k6_grp(kq, ks, n, d, active_kq_layout_mr(6)) } @@ -1772,6 +1855,8 @@ def private kq_kernel_gen(fmt : int; var yp : float?; kqp : uint8 const?; ksp : iq4nlq8_gemv_gen(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb) * mr, int64(re) * mr) } elif (fmt == 2) { k2q8_gemv_gen(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb) * mr, int64(re) * mr) + } elif (fmt == 23) { + iq2sq8_gemv_gen(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb) * mr, int64(re) * mr) } else { k6q8_gemv_gen(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb) * mr, int64(re) * mr) } @@ -1796,6 +1881,8 @@ def private kq_kernel_gen(fmt : int; var yp : float?; kqp : uint8 const?; ksp : myp[i] = dot_iq4nl_q8(kqp + i * nsb * 128l, ksp + i * nsb * 16l, xqp, xsp, xbsp, n) } elif (fmt == 2) { myp[i] = dot_k2q8(kqp + i * nsb * 64l, ksp + i * nsb * 20l, xqp, xsp, xbsp, n) + } elif (fmt == 23) { + myp[i] = dot_iq2s_q8(kqp + i * nsb * 72l, ksp + i * nsb * 20l, xqp, xsp, xbsp, n) } else { myp[i] = dot_k6q8(kqp + i * nsb * 192l, ksp + i * nsb * 18l, xqp, xsp, xbsp, n) } @@ -1811,7 +1898,7 @@ def private kq_batch_cell_gen(fmt : int; var myp : float?; kqp : uint8 const?; k let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) let nb16 = n / 16l - let packed = fmt == 4 || fmt == 40 || fmt == 44 || fmt == 3 || fmt == 45 || fmt == 2 // packed planes: the tile reads them directly (iq3s/iq3xxs expand their grid gathers into the panel) + let packed = fmt == 4 || fmt == 40 || fmt == 44 || fmt == 3 || fmt == 45 || fmt == 2 || fmt == 23 // packed planes: the tile reads them directly (iq3s/iq3xxs expand their grid gathers into the panel) var scratch : array if (!packed) { scratch |> resize(mr * nsb * 256l) @@ -1849,6 +1936,8 @@ def private kq_batch_cell_gen(fmt : int; var myp : float?; kqp : uint8 const?; k iq4nlq8_tile_gen(myp, kqg, ksg, xqp, xsp, xbsp, n, d, g, tk) } elif (fmt == 2) { k2q8_tile_gen(myp, kqg, ksg, xqp, xsp, xbsp, n, d, g, tk) + } elif (fmt == 23) { + iq2sq8_tile_gen(myp, kqg, ksg, xqp, xsp, xbsp, n, d, g, tk) } else { k6q8_tile_gen(myp, scp, ksg, xqp, xsp, xbsp, n, d, g, tk) } @@ -1873,6 +1962,8 @@ def private kq_batch_cell_gen(fmt : int; var myp : float?; kqp : uint8 const?; k iq4nlq8_gemv_gen(myp + tk * d, kqp, ksp, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n, g * mr, (g + 1l) * mr) } elif (fmt == 2) { k2q8_gemv_gen(myp + tk * d, kqp, ksp, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n, g * mr, (g + 1l) * mr) + } elif (fmt == 23) { + iq2sq8_gemv_gen(myp + tk * d, kqp, ksp, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n, g * mr, (g + 1l) * mr) } else { k6q8_gemv_gen(myp + tk * d, kqp, ksp, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n, g * mr, (g + 1l) * mr) } @@ -1918,6 +2009,8 @@ def private kq_batch_kernel_gen(fmt : int; var yp : float?; kqp : uint8 const?; myp[tk * d + i] = dot_iq4nl_q8(kqp + i * nsb * 128l, ksp + i * nsb * 16l, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * (n / 16l), n) } elif (fmt == 2) { myp[tk * d + i] = dot_k2q8(kqp + i * nsb * 64l, ksp + i * nsb * 20l, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * (n / 16l), n) + } elif (fmt == 23) { + myp[tk * d + i] = dot_iq2s_q8(kqp + i * nsb * 72l, ksp + i * nsb * 20l, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * (n / 16l), n) } else { myp[tk * d + i] = dot_k6q8(kqp + i * nsb * 192l, ksp + i * nsb * 18l, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * (n / 16l), n) } @@ -1978,6 +2071,8 @@ def private kq_batch_groupn_gen(fmt : int; var yp : float?; kqp : uint8 const?; myp[tk * d + i] = dot_iq4nl_q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n) } elif (fmt == 2) { myp[tk * d + i] = dot_k2q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n) + } elif (fmt == 23) { + myp[tk * d + i] = dot_iq2s_q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n) } else { myp[tk * d + i] = dot_k6q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n) } @@ -2147,6 +2242,8 @@ def kq_groupn_gen(fmt : int; var yp : float?; kqp : uint8 const?; ksp : uint8 co iq4nlq8_gemv_gen(myp + r * d, kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n, gLocal * mr, gEnd * mr) } elif (fmt == 2) { k2q8_gemv_gen(myp + r * d, kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n, gLocal * mr, gEnd * mr) + } elif (fmt == 23) { + iq2sq8_gemv_gen(myp + r * d, kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n, gLocal * mr, gEnd * mr) } else { k6q8_gemv_gen(myp + r * d, kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n, gLocal * mr, gEnd * mr) } @@ -2177,6 +2274,8 @@ def kq_groupn_gen(fmt : int; var yp : float?; kqp : uint8 const?; ksp : uint8 co myp[r * d + i] = dot_iq4nl_q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) } elif (fmt == 2) { myp[r * d + i] = dot_k2q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) + } elif (fmt == 23) { + myp[r * d + i] = dot_iq2s_q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) } else { myp[r * d + i] = dot_k6q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) } @@ -2470,7 +2569,7 @@ def dasllama_math_gen_register() { // the no-witness stance above covers kq too mm_kq = @@kq_kernel_gen, repack_kq = @@repack_kq_gen, kq_rows_k4 = @@k4q8_gemv_gen, kq_rows_k5 = @@k5q8_gemv_gen, - kq_rows_k6 = @@k6q8_gemv_gen, kq_rows_q40 = @@q40q8_gemv_gen, kq_rows_iq4xs = @@iq4xsq8_gemv_gen, kq_rows_k3 = @@k3q8_gemv_gen, kq_rows_iq3s = @@iq3sq8_gemv_gen, kq_rows_iq3xxs = @@iq3xxsq8_gemv_gen, kq_rows_iq4nl = @@iq4nlq8_gemv_gen, kq_rows_k2 = @@k2q8_gemv_gen, + kq_rows_k6 = @@k6q8_gemv_gen, kq_rows_q40 = @@q40q8_gemv_gen, kq_rows_iq4xs = @@iq4xsq8_gemv_gen, kq_rows_k3 = @@k3q8_gemv_gen, kq_rows_iq3s = @@iq3sq8_gemv_gen, kq_rows_iq3xxs = @@iq3xxsq8_gemv_gen, kq_rows_iq4nl = @@iq4nlq8_gemv_gen, kq_rows_k2 = @@k2q8_gemv_gen, kq_rows_iq2s = @@iq2sq8_gemv_gen, kq_batch = @@kq_batch_kernel_gen, kq_groupn = @@kq_groupn_gen, kq_batch_groupn = @@kq_batch_groupn_gen, // q51 family: tile + GEMV off the grp planes (the tile family's companions stamp @@ -2499,7 +2598,7 @@ def dasllama_math_gen_register() { mm_rows_s16 = @@q8q8_gemv_s16_gen, mm_kq = @@kq_kernel_gen, repack_kq = @@repack_kq_gen, kq_rows_k4 = @@k4q8_gemv_gen, kq_rows_k5 = @@k5q8_gemv_gen, - kq_rows_k6 = @@k6q8_gemv_gen, kq_rows_q40 = @@q40q8_gemv_gen, kq_rows_iq4xs = @@iq4xsq8_gemv_gen, kq_rows_k3 = @@k3q8_gemv_gen, kq_rows_iq3s = @@iq3sq8_gemv_gen, kq_rows_iq3xxs = @@iq3xxsq8_gemv_gen, kq_rows_iq4nl = @@iq4nlq8_gemv_gen, kq_rows_k2 = @@k2q8_gemv_gen, + kq_rows_k6 = @@k6q8_gemv_gen, kq_rows_q40 = @@q40q8_gemv_gen, kq_rows_iq4xs = @@iq4xsq8_gemv_gen, kq_rows_k3 = @@k3q8_gemv_gen, kq_rows_iq3s = @@iq3sq8_gemv_gen, kq_rows_iq3xxs = @@iq3xxsq8_gemv_gen, kq_rows_iq4nl = @@iq4nlq8_gemv_gen, kq_rows_k2 = @@k2q8_gemv_gen, kq_rows_iq2s = @@iq2sq8_gemv_gen, kq_batch = @@kq_batch_kernel_gen, kq_groupn = @@kq_groupn_gen, kq_batch_groupn = @@kq_batch_groupn_gen, groupn_q51 = @@q51q8_groupn_gen, repack_q51 = @@repack_q51_gen, diff --git a/modules/dasLLAMA/dasllama/dasllama_ple.das b/modules/dasLLAMA/dasllama/dasllama_ple.das index 34e4c840ca..a49124a8a2 100644 --- a/modules/dasLLAMA/dasllama/dasllama_ple.das +++ b/modules/dasLLAMA/dasllama/dasllama_ple.das @@ -74,6 +74,8 @@ def ple_check_table(t : Model; origin : string) { have = (long_length(t.iq4nlq) / Q40_QSB) * 256l } elif (t.ple_emb_fmt == KqFmt.k2) { have = (long_length(t.k2q) / K2_QSB) * 256l + } elif (t.ple_emb_fmt == KqFmt.iq2s) { + have = (long_length(t.iq2sq) / IQ2S_QSB) * 256l } if (have < need) { panic("dasLLAMA: the PLE token table is not resident ({origin}): fmt {t.ple_emb_fmt} plane holds {have} of {need} elements - regenerate this carrier (it predates the pinned-table rail)") @@ -119,6 +121,8 @@ def private ple_gather_row(t : Model; row : int64; var dst : array; doff dequant_iq4nl_plane_superblock(t.iq4nlq, sb * Q40_QSB, t.iq4nls, sb * Q40_SSB, dst, doff + blk * 256l) } elif (t.ple_emb_fmt == KqFmt.k2) { dequant_k2_plane_superblock(t.k2q, sb * K2_QSB, t.k2s, sb * K2_SSB, dst, doff + blk * 256l) + } elif (t.ple_emb_fmt == KqFmt.iq2s) { + dequant_iq2s_plane_superblock(t.iq2sq, sb * IQ2S_QSB, t.iq2ss, sb * IQ2S_SSB, dst, doff + blk * 256l) } else { panic("ple_gather_row: '{t.ple_emb_fmt}' has no kq plane pair") } diff --git a/modules/dasLLAMA/dasllama/dasllama_repack.das b/modules/dasLLAMA/dasllama/dasllama_repack.das index bcdce451f0..06ef439543 100644 --- a/modules/dasLLAMA/dasllama/dasllama_repack.das +++ b/modules/dasLLAMA/dasllama/dasllama_repack.das @@ -555,6 +555,50 @@ def repack_k2_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { delete ts } +//! iq2s grp layout: the whole 72B row as 18 four-byte columns x mr rows - idx, sign and qh +//! bytes stay in place per column; scales interleave as k4's row with 16 strips +//! ([d8 f16 x mr][pad x mr][16 strips x mr]). +def repack_iq2s_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { + let nsb = n / 256l + let qrow = nsb * 72l + let srow = nsb * 20l + let ng = d / mr + var tq : array + var ts : array + tq |> resize(d * qrow) + ts |> resize(d * srow) + unsafe { + var tqp = addr(tq[0]) + var tsp = addr(ts[0]) + memcpy(tqp, kq, d * qrow) + memcpy(tsp, ks, d * srow) + for (g in range64(ng)) { + for (sbi in range64(nsb)) { + let dq = g * mr * qrow + sbi * 72l * mr + let ds = g * mr * srow + sbi * 20l * mr + for (r in range64(mr)) { + let sq = (g * mr + r) * qrow + sbi * 72l + let ss = (g * mr + r) * srow + sbi * 20l + for (c in range64(18l)) { + for (t in range64(4l)) { + kq[dq + (c * mr + r) * 4l + t] = tqp[sq + c * 4l + t] + } + } + ks[ds + 2l * r] = tsp[ss] + ks[ds + 2l * r + 1l] = tsp[ss + 1l] + ks[ds + 2l * mr + 2l * r] = tsp[ss + 2l] + ks[ds + 2l * mr + 2l * r + 1l] = tsp[ss + 3l] + for (g16 in range64(16l)) { + ks[ds + 4l * mr + g16 * mr + r] = tsp[ss + 4l + g16] + } + } + } + } + } + delete tq + delete ts +} + // ===== grp panel unpack (the byte-expanded tile form the kq stamps load) ===== //! Unpack ONE group's packed grp k5/k6 quant panel into the BYTE-EXPANDED tile form (wlo/whi diff --git a/modules/dasLLAMA/harness/gen_tune_probe.das b/modules/dasLLAMA/harness/gen_tune_probe.das index 172bb7c532..7038c1e5e6 100644 --- a/modules/dasLLAMA/harness/gen_tune_probe.das +++ b/modules/dasLLAMA/harness/gen_tune_probe.das @@ -441,7 +441,7 @@ def kq_qsb(fmt : int64) : int64 => dasllama_gemm_schema::kq_qsb(int(fmt)) def kq_ssb(fmt : int64) : int64 => dasllama_gemm_schema::kq_ssb(int(fmt)) // the family's sidecar entry = its [tune] function name (the K-quants by bit width - k3 included - q40/iq4xs by tag) -def kq_tile_entry(fmt : int64) : string => fmt == 40l ? "q40q8_tile_gen" : (fmt == 44l ? "iq4xsq8_tile_gen" : (fmt == 33l ? "iq3sq8_tile_gen" : (fmt == 34l ? "iq3xxsq8_tile_gen" : (fmt == 45l ? "iq4nlq8_tile_gen" : "k{fmt}q8_tile_gen")))) +def kq_tile_entry(fmt : int64) : string => fmt == 40l ? "q40q8_tile_gen" : (fmt == 44l ? "iq4xsq8_tile_gen" : (fmt == 33l ? "iq3sq8_tile_gen" : (fmt == 34l ? "iq3xxsq8_tile_gen" : (fmt == 45l ? "iq4nlq8_tile_gen" : (fmt == 23l ? "iq2sq8_tile_gen" : "k{fmt}q8_tile_gen"))))) // synthetic disk superblocks, varied per (row, superblock) so no two blocks repeat: any byte // is a legal quant, and the small d/dmin keep the f16 (s, o) pairs finite and row sums tame @@ -537,6 +537,14 @@ def pack_kq_disk_block(fmt : int64; var blkb : array; base : int) { // let mbits = f32_to_f16(0.001 + 0.0001 * float((base + 5) % 11)) blkb[82] = uint8(mbits & 0xFF) blkb[83] = uint8(mbits >> 8u) + } elif (fmt == 23l) { + // 82B IQ2_S disk block: f16 d, 32 idx + 32 sign bytes, 8 qh, 8 packed scale nibbles - every byte pattern is legal + let dbits = f32_to_f16(0.002 + 0.0001 * float(base % 13)) + blkb[0] = uint8(dbits & 0xFF) + blkb[1] = uint8(dbits >> 8u) + for (i in range(80)) { + blkb[2 + i] = uint8((base * 37 + i * 101 + 29) % 256) + } } elif (fmt == 4l) { pack_kq_scale_header(blkb, base) for (i in range(128)) { @@ -585,6 +593,8 @@ def repack_kq_grp_fmt(fmt : int64; var kq : uint8?; var ks : uint8?; n, d, mr : repack_q40_grp(kq, ks, n, d, mr) } elif (fmt == 2l) { repack_k2_grp(kq, ks, n, d, mr) + } elif (fmt == 23l) { + repack_iq2s_grp(kq, ks, n, d, mr) } else { repack_k6_grp(kq, ks, n, d, mr) } @@ -597,7 +607,7 @@ def build_kq_fixture(fmt : int64; n, d, ntok : int64) : KqFixture { // nolint: fx.kq |> resize(d * fx.nsb * qsb) fx.ks |> resize(d * fx.nsb * ssb) var blkb : array - blkb |> resize(fmt == 2l ? 84l : (fmt == 4l || fmt == 40l || fmt == 45l ? 144l : (fmt == 5l ? 176l : (fmt == 44l ? 136l : (fmt == 3l || fmt == 33l ? 110l : (fmt == 34l ? 98l : 210l)))))) + blkb |> resize(fmt == 2l ? 84l : (fmt == 23l ? 82l : (fmt == 4l || fmt == 40l || fmt == 45l ? 144l : (fmt == 5l ? 176l : (fmt == 44l ? 136l : (fmt == 3l || fmt == 33l ? 110l : (fmt == 34l ? 98l : 210l))))))) for (r in range64(d)) { for (s in range64(fx.nsb)) { let base = int(r * 31l + s * 7l) @@ -621,6 +631,8 @@ def build_kq_fixture(fmt : int64; n, d, ntok : int64) : KqFixture { // nolint: transcode_iq4nl_superblock(blkb, 0l, fx.kq, sb * qsb, fx.ks, sb * ssb) } elif (fmt == 2l) { transcode_q2k_superblock(blkb, 0l, fx.kq, sb * qsb, fx.ks, sb * ssb) + } elif (fmt == 23l) { + transcode_iq2s_superblock(blkb, 0l, fx.kq, sb * qsb, fx.ks, sb * ssb) } else { transcode_q6k_superblock(blkb, 0l, fx.kq, sb * qsb, fx.ks, sb * ssb) } @@ -670,11 +682,14 @@ def build_kq_fixture(fmt : int64; n, d, ntok : int64) : KqFixture { // nolint: : (fmt == 45l ? iq4nl_grp_row_dot(kqg, ksg, r, 4l, addr(fx.xq[0]) + tk * n, addr(fx.xs[0]) + tk * (n / 256l), addr(fx.xbs[0]) + tk * (n / 16l), n) - : (fmt == 2l + : (fmt == 23l + ? iq2s_grp_row_dot(kqg, ksg, r, 4l, + addr(fx.xq[0]) + tk * n, addr(fx.xs[0]) + tk * (n / 256l), addr(fx.xbs[0]) + tk * (n / 16l), n) + : (fmt == 2l ? k2_grp_row_dot(kqg, ksg, r, 4l, addr(fx.xq[0]) + tk * n, addr(fx.xs[0]) + tk * (n / 256l), addr(fx.xbs[0]) + tk * (n / 16l), n) : kq_grp_row_dot(fmt, kqg, ksg, r, 4l, - addr(fx.xq[0]) + tk * n, addr(fx.xs[0]) + tk * (n / 256l), addr(fx.xbs[0]) + tk * (n / 16l), n)))))))) + addr(fx.xq[0]) + tk * n, addr(fx.xs[0]) + tk * (n / 256l), addr(fx.xbs[0]) + tk * (n / 16l), n))))))))) } } } @@ -714,10 +729,13 @@ def kq_tile_variants(fmt : int64) : array> { if (fmt == 2l) { return <- k2q8_tile_gen_variants() } + if (fmt == 23l) { + return <- iq2sq8_tile_gen_variants() + } return <- k6q8_tile_gen_variants() } -def kq_gemv_variants_by_suffix(fmt : int64) : table { +def kq_gemv_variants_by_suffix(fmt : int64) : table { // nolint:STYLE037 — the flat per-format ladder (one arm per kq format) is the honest shape var tab : table if (fmt == 4l) { var gvs <- k4q8_gemv_gen_variants() @@ -767,6 +785,12 @@ def kq_gemv_variants_by_suffix(fmt : int64) : table { tab[v._0] = v._1 } delete gvs + } elif (fmt == 23l) { + var gvs <- iq2sq8_gemv_gen_variants() + for (v in gvs) { + tab[v._0] = v._1 + } + delete gvs } elif (fmt == 2l) { var gvs <- k2q8_gemv_gen_variants() for (v in gvs) { @@ -785,7 +809,7 @@ def kq_gemv_variants_by_suffix(fmt : int64) : table { // suffix -> plane interleave off the family's OWN layout companion registry (per-format // layouts since the family split — declined rows report the reference grp4) -def kq_layout_mrs(fmt : int64) : table { +def kq_layout_mrs(fmt : int64) : table { // nolint:STYLE037 — the flat per-format ladder (one arm per kq format) is the honest shape var mrs : table if (fmt == 4l) { var lvs <- k4q8_layout_gen_variants() @@ -835,6 +859,12 @@ def kq_layout_mrs(fmt : int64) : table { mrs[v._0] = invoke(v._1) } delete lvs + } elif (fmt == 23l) { + var lvs <- iq2sq8_layout_gen_variants() + for (v in lvs) { + mrs[v._0] = invoke(v._1) + } + delete lvs } elif (fmt == 2l) { var lvs <- k2q8_layout_gen_variants() for (v in lvs) { @@ -858,7 +888,7 @@ def kq_layout_mrs(fmt : int64) : table { def run_kq_tile(tile; fx : KqFixture; kq : array; ks : array; mr : int64; var y : array) { let ng = fx.d / mr verify(fx.ntok % 4l == 0l) - let packed = fx.fmt == 4l || fx.fmt == 40l || fx.fmt == 44l || fx.fmt == 3l || fx.fmt == 45l || fx.fmt == 2l // packed planes: the tile reads them directly (iq3s/iq3xxs unpack their panels per group) + let packed = fx.fmt == 4l || fx.fmt == 40l || fx.fmt == 44l || fx.fmt == 3l || fx.fmt == 45l || fx.fmt == 2l || fx.fmt == 23l // packed planes: the tile reads them directly (iq3s/iq3xxs unpack their panels per group) var panel : array if (!packed) { panel |> resize(int(mr * fx.nsb * 256l)) @@ -1937,7 +1967,7 @@ def tune_mode_run : bool { // nolint:STYLE037,STYLE038 — flat family-by-family print("aborting mid-run - no sidecar written\n") return false } - for (fmt in fixed_array(4l, 5l, 6l, 40l, 44l, 3l, 33l, 34l, 45l, 2l)) { + for (fmt in fixed_array(4l, 5l, 6l, 40l, 44l, 3l, 33l, 34l, 45l, 2l, 23l)) { interrupt_gate("k{fmt}q8_tile_gen") let kq_t0 = ref_time_ticks() let entry = kq_tile_entry(fmt) diff --git a/modules/dasLLAMA/tests/test_kqformat.das b/modules/dasLLAMA/tests/test_kqformat.das index 7d842bb574..fa6edec061 100644 --- a/modules/dasLLAMA/tests/test_kqformat.das +++ b/modules/dasLLAMA/tests/test_kqformat.das @@ -26,6 +26,7 @@ def test_kqfmt_enum_values(t : T?) { t |> equal(int(KqFmt.iq3xxs), 9) t |> equal(int(KqFmt.iq4nl), 10) t |> equal(int(KqFmt.k2), 11) + t |> equal(int(KqFmt.iq2s), 12) } } @@ -50,6 +51,8 @@ def test_kqfmt_predicates(t : T?) { t |> success(kq_sb(int(KqFmt.iq3xxs)), "the int-id predicate agrees for iq3xxs") t |> success(kq_sb(int(KqFmt.iq4nl)), "the int-id predicate agrees for iq4nl") t |> success(kq_sb(int(KqFmt.k2)), "the int-id predicate agrees for k2") + t |> success(kq_sb(KqFmt.iq2s), "iq2s is superblock") + t |> success(kq_sb(int(KqFmt.iq2s)), "the int-id predicate agrees for iq2s") } } @@ -78,6 +81,8 @@ def test_kqfmt_strides(t : T?) { t |> equal(kq_ssb(KqFmt.iq4nl), 16l) // q40's row: 8 x f16 d t |> equal(kq_qsb(KqFmt.k2), 64l) // 64 qs bytes, verbatim t |> equal(kq_ssb(KqFmt.k2), 20l) // [f16 d][f16 dmin][16 sc/min bytes] + t |> equal(kq_qsb(KqFmt.iq2s), 72l) // [32 idx][32 signs][8 qh], verbatim + t |> equal(kq_ssb(KqFmt.iq2s), 20l) // f16 d/8 + 16 x (1 + 2s) strips } t |> run("named constants agree with the literals") @(t : T?) { t |> equal(K4_QSB, 128l) @@ -96,6 +101,8 @@ def test_kqfmt_strides(t : T?) { t |> equal(K3_SSB, 18l) t |> equal(K2_QSB, 64l) t |> equal(K2_SSB, 20l) + t |> equal(IQ2S_QSB, 72l) + t |> equal(IQ2S_SSB, 20l) t |> equal(IQ3S_QSB, 104l) t |> equal(IQ3S_SSB, 20l) } @@ -134,9 +141,10 @@ def test_kqfmt_id_bridge(t : T?) { t |> equal(kq_schema_id(KqFmt.iq3xxs), 34) t |> equal(kq_schema_id(KqFmt.iq4nl), 45) t |> equal(kq_schema_id(KqFmt.k2), 2) + t |> equal(kq_schema_id(KqFmt.iq2s), 23) } t |> run("gemm_schema's int-id forms delegate to the same truth") @(t : T?) { - for (f in [KqFmt.k4, KqFmt.k5, KqFmt.k6, KqFmt.q40, KqFmt.iq4xs, KqFmt.k3, KqFmt.iq3s, KqFmt.iq3xxs, KqFmt.iq4nl, KqFmt.k2]) { + for (f in [KqFmt.k4, KqFmt.k5, KqFmt.k6, KqFmt.q40, KqFmt.iq4xs, KqFmt.k3, KqFmt.iq3s, KqFmt.iq3xxs, KqFmt.iq4nl, KqFmt.k2, KqFmt.iq2s]) { t |> equal(kq_qsb(kq_schema_id(f)), kq_qsb(f)) t |> equal(kq_ssb(kq_schema_id(f)), kq_ssb(f)) } @@ -156,6 +164,7 @@ def test_kqfmt_elems_and_stream(t : T?) { t |> equal(kq_elems(KqFmt.iq3xxs), 256l) t |> equal(kq_elems(KqFmt.iq4nl), 256l) t |> equal(kq_elems(KqFmt.k2), 256l) + t |> equal(kq_elems(KqFmt.iq2s), 256l) t |> equal(kq_elems(KqFmt.q8), 32l) t |> equal(kq_elems(KqFmt.q51), 32l) } @@ -172,6 +181,7 @@ def test_kqfmt_elems_and_stream(t : T?) { t |> equal(kq_stream_code(KqFmt.iq3xxs), 34) t |> equal(kq_stream_code(KqFmt.iq4nl), 45) t |> equal(kq_stream_code(KqFmt.k2), 20) // kernel id 2 = q51's stream tag + t |> equal(kq_stream_code(KqFmt.iq2s), 23) } } diff --git a/modules/dasLLAMA/tests/test_kquant.das b/modules/dasLLAMA/tests/test_kquant.das index cdbc29d94e..9652df004d 100644 --- a/modules/dasLLAMA/tests/test_kquant.das +++ b/modules/dasLLAMA/tests/test_kquant.das @@ -1,8 +1,8 @@ options gen2 options stack = 524288 // every dasLLAMA program root takes this budget (options stack does not unify up from libs) options persistent_heap // the layout arm loads a multi-GB model; explicit delete below -options _cyclomatic_complexity = 58 // every gate here is the same flat fmt == 4/5/6/40/44/3 ladder repeated per stage -options _function_length = 240 // and each gate carries its whole fixture build + oracle inline +options _cyclomatic_complexity = 64 // every gate here is the same flat fmt == 4/5/6/40/44/3 ladder repeated per stage +options _function_length = 260 // and each gate carries its whole fixture build + oracle inline options _dasllama_internal = true require dastest/testing_boost public @@ -142,7 +142,7 @@ def private build_iq4nl_block() : array { return <- blkb } -def private kq_tag(fmt : int) : string => fmt == 40 ? "q40" : (fmt == 44 ? "iq4xs" : (fmt == 33 ? "iq3s" : (fmt == 34 ? "iq3xxs" : (fmt == 45 ? "iq4nl" : "k{fmt}")))) +def private kq_tag(fmt : int) : string => fmt == 40 ? "q40" : (fmt == 44 ? "iq4xs" : (fmt == 33 ? "iq3s" : (fmt == 34 ? "iq3xxs" : (fmt == 45 ? "iq4nl" : (fmt == 23 ? "iq2s" : "k{fmt}"))))) // iq4xs synthetic superblock: 136B disk block — f16 d = SYNTH_D, the 8 six-bit sub-scales are // SYNTH_SC (packed low nibbles into scales_l, top two bits into scales_h), the shared nibble @@ -269,6 +269,35 @@ def private build_q2k_block() : array { return <- blkb } +// iq2s synthetic superblock: 82B IQ2_S disk block - per-16 scale nibbles SYNTH_SC & 15, f16 d, +// idx/qh/sign byte patterns (written in the PACK direction); the gate recomposes each weight +// through IQ2S_GRID +def private iq2s_ix_pat(i : int) : int => (i * 13 + 5) % 256 +def private iq2s_qh_pat(b : int) : int => (b * 3 + 1) % 256 +def private iq2s_sg_pat(p : int) : int => (p * 7 + 11) % 256 +def private build_iq2s_block() : array { + var blkb : array + blkb |> resize(82) + let dbits = f32_to_f16(SYNTH_D) + blkb[0] = uint8(dbits & 0xFF) + blkb[1] = uint8(dbits >> 8u) + for (i in range(32)) { + blkb[2 + i] = uint8(iq2s_ix_pat(i)) + } + for (p in range(32)) { + blkb[34 + p] = uint8(iq2s_sg_pat(p)) + } + for (b in range(8)) { + blkb[66 + b] = uint8(iq2s_qh_pat(b)) + } + for (g in range(16)) { // scale nibble g: low nibble of byte g/2 (g even), high (g odd) + let v = SYNTH_SC[g % 8] & 15 + let nib = g % 2 == 0 ? v : v << 4 + blkb[74 + g / 2] = uint8(int(blkb[74 + g / 2]) | nib) + } + return <- blkb +} + // q40 synthetic superblock: 8 x 18B q4_0 disk blocks, per-block f16-exact d = (blk+1)/16 and // the shared nibble pattern — weight k = d(k/32) * (q4_pat(k) - 8) def private q40_d(blk : int) : float => float(blk + 1) * 0.0625 @@ -498,6 +527,32 @@ def test_kq_transcode_planes(t : T?) { t |> success(dst[k] == expected, "k2 plane element must match the ggml dequant exactly") } } + t |> run("iq2s planes eighth the d, decode the strips, and gather the u64 grid exactly") @(t : T?) { + let blkb <- build_iq2s_block() + var kq : array + var ks : array + kq |> resize(72) + ks |> resize(20) + transcode_iq2s_superblock(blkb, 0l, kq, 0l, ks, 0l) + for (g in range(16)) { + t |> equal(int(ks[4 + g]), 1 + 2 * (SYNTH_SC[g % 8] & 15), "decoded strip is 1 + 2s") + } + var dst : array + dst |> resize(256) + dequant_iq2s_plane_superblock(kq, 0l, ks, 0l, dst, 0l) + let d8 = SYNTH_D * 0.125 + for (k in range(256)) { + let b = k / 32 + let l = (k % 32) / 8 + let j = k % 8 + let idx = iq2s_ix_pat(b * 4 + l) | ((iq2s_qh_pat(b) << (8 - 2 * l)) & 0x300) + let gw = j < 4 ? IQ2S_GRID[idx * 2] : IQ2S_GRID[idx * 2 + 1] + let g = int((gw >> uint(8 * (j % 4))) & 0xFFu) + let sgn = ((iq2s_sg_pat(b * 4 + l) >> j) & 1) != 0 ? -1.0 : 1.0 + let expected = (d8 * float(1 + 2 * (SYNTH_SC[(2 * b + l / 2) % 8] & 15))) * float(g) * sgn + t |> success(dst[k] == expected, "iq2s plane element must match the ggml dequant exactly") + } + } t |> run("q51 planes unpack the hand-packed Q5_1 blocks exactly") @(t : T?) { for (blk in range(4)) { let blkb <- build_q51_block(blk) @@ -744,7 +799,7 @@ def test_q51_kernels(t : T?) { // elementwise dot over the plane dequant (the kernels' declared oracle). def private kq_dot_gate(t : T?; fmt : int; n : int64 = 512l) { let nsb = n / 256l - let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : (fmt == 45 ? build_iq4nl_block() : (fmt == 2 ? build_q2k_block() : build_q6k_block())))))))) + let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : (fmt == 45 ? build_iq4nl_block() : (fmt == 2 ? build_q2k_block() : (fmt == 23 ? build_iq2s_block() : build_q6k_block()))))))))) let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) var kq : array @@ -770,6 +825,8 @@ def private kq_dot_gate(t : T?; fmt : int; n : int64 = 512l) { transcode_iq4nl_superblock(blk1, 0l, kq, s * qsb, ks, s * ssb) } elif (fmt == 2) { transcode_q2k_superblock(blk1, 0l, kq, s * qsb, ks, s * ssb) + } elif (fmt == 23) { + transcode_iq2s_superblock(blk1, 0l, kq, s * qsb, ks, s * ssb) } else { transcode_q6k_superblock(blk1, 0l, kq, s * qsb, ks, s * ssb) } @@ -801,6 +858,8 @@ def private kq_dot_gate(t : T?; fmt : int; n : int64 = 512l) { dequant_iq4nl_plane_superblock(kq, s * qsb, ks, s * ssb, w, s * 256l) } elif (fmt == 2) { dequant_k2_plane_superblock(kq, s * qsb, ks, s * ssb, w, s * 256l) + } elif (fmt == 23) { + dequant_iq2s_plane_superblock(kq, s * qsb, ks, s * ssb, w, s * 256l) } else { dequant_k6_plane_superblock(kq, s * qsb, ks, s * ssb, w, s * 256l) } @@ -844,6 +903,8 @@ def private kq_dot_gate(t : T?; fmt : int; n : int64 = 512l) { got = dot_iq4nl_q8(addr(kq[0]), addr(ks[0]), addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } elif (fmt == 2) { got = dot_k2q8(addr(kq[0]), addr(ks[0]), addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + } elif (fmt == 23) { + got = dot_iq2s_q8(addr(kq[0]), addr(ks[0]), addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } else { got = dot_k6q8(addr(kq[0]), addr(ks[0]), addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } @@ -877,7 +938,7 @@ def test_kq_dots(t : T?) { } // 122B-class row widths: n=3072 (12 superblocks — qwen35moe-122B gate/up/cls) and n=1024 // (its k5 down_exps). Every earlier kq model kept n at 512..2560. - for (fmt in [4, 5, 6, 40, 44, 3, 33, 34, 45, 2]) { + for (fmt in [4, 5, 6, 40, 44, 3, 33, 34, 45, 2, 23]) { for (nn in [1024l, 3072l]) { t |> run("dot_{kq_tag(fmt)}q8 matches the fp64 reference at n={int(nn)}") @(t : T?) { kq_dot_gate(t, fmt, nn) @@ -896,7 +957,7 @@ def private kq_gemv_rows_gate(t : T?; fmt : int; n : int64 = 512l) { let nsb = n / 256l let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) - let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : (fmt == 45 ? build_iq4nl_block() : (fmt == 2 ? build_q2k_block() : build_q6k_block())))))))) + let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : (fmt == 45 ? build_iq4nl_block() : (fmt == 2 ? build_q2k_block() : (fmt == 23 ? build_iq2s_block() : build_q6k_block()))))))))) var kq : array var ks : array kq |> resize(d * nsb * qsb) @@ -921,6 +982,8 @@ def private kq_gemv_rows_gate(t : T?; fmt : int; n : int64 = 512l) { transcode_iq4nl_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } elif (fmt == 2) { transcode_q2k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 23) { + transcode_iq2s_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } else { transcode_q6k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } @@ -971,6 +1034,8 @@ def private kq_gemv_rows_gate(t : T?; fmt : int; n : int64 = 512l) { want = dot_iq4nl_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } elif (fmt == 2) { want = dot_k2q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + } elif (fmt == 23) { + want = dot_iq2s_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } else { want = dot_k6q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } @@ -1012,7 +1077,7 @@ def test_kq_gemv_rows(t : T?) { t |> run("portable iq3s GEMV rows bit-match per-row disk dots") @(t : T?) { kq_gemv_rows_gate(t, 33) } - for (fmt in [4, 5, 6, 40, 44, 3, 33, 34, 45, 2]) { + for (fmt in [4, 5, 6, 40, 44, 3, 33, 34, 45, 2, 23]) { for (nn in [1024l, 3072l]) { t |> run("portable {kq_tag(fmt)} GEMV rows bit-match per-row disk dots at n={int(nn)}") @(t : T?) { kq_gemv_rows_gate(t, fmt, nn) @@ -1032,7 +1097,7 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { let nsb = n / 256l let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) - let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : (fmt == 45 ? build_iq4nl_block() : (fmt == 2 ? build_q2k_block() : build_q6k_block())))))))) + let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : (fmt == 45 ? build_iq4nl_block() : (fmt == 2 ? build_q2k_block() : (fmt == 23 ? build_iq2s_block() : build_q6k_block()))))))))) var kq : array var ks : array kq |> resize(d * nsb * qsb) @@ -1057,6 +1122,8 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { transcode_iq4nl_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } elif (fmt == 2) { transcode_q2k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 23) { + transcode_iq2s_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } else { transcode_q6k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } @@ -1106,6 +1173,8 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { want[r] = dot_iq4nl_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } elif (fmt == 2) { want[r] = dot_k2q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + } elif (fmt == 23) { + want[r] = dot_iq2s_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } else { want[r] = dot_k6q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } @@ -1128,6 +1197,8 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { dequant_iq4nl_plane_superblock(kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb, wrow, r * n + s * 256l) } elif (fmt == 2) { dequant_k2_plane_superblock(kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb, wrow, r * n + s * 256l) + } elif (fmt == 23) { + dequant_iq2s_plane_superblock(kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb, wrow, r * n + s * 256l) } else { dequant_k6_plane_superblock(kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb, wrow, r * n + s * 256l) } @@ -1151,6 +1222,8 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { repack_q40_grp(addr(kq[0]), addr(ks[0]), n, d, mr) } elif (fmt == 2) { repack_k2_grp(addr(kq[0]), addr(ks[0]), n, d, mr) + } elif (fmt == 23) { + repack_iq2s_grp(addr(kq[0]), addr(ks[0]), n, d, mr) } else { repack_k6_grp(addr(kq[0]), addr(ks[0]), n, d, mr) } @@ -1168,8 +1241,8 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { ? iq4xs_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) : (fmt == 3 ? k3_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) - : (fmt == 33 || fmt == 34 || fmt == 45 || fmt == 2 - ? (fmt == 33 ? iq3s_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) : (fmt == 34 ? iq3xxs_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) : (fmt == 45 ? iq4nl_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) : k2_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n)))) + : (fmt == 33 || fmt == 34 || fmt == 45 || fmt == 2 || fmt == 23 + ? (fmt == 33 ? iq3s_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) : (fmt == 34 ? iq3xxs_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) : (fmt == 45 ? iq4nl_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) : (fmt == 2 ? k2_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) : iq2s_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n))))) : kq_grp_row_dot(int64(fmt), kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n))))) if (got != want[r]) { dotbad++ @@ -1197,7 +1270,7 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { [test] def test_kq_repack_grp(t : T?) { - for (fmt in [4, 5, 6, 40, 44, 3, 33, 34, 45, 2]) { + for (fmt in [4, 5, 6, 40, 44, 3, 33, 34, 45, 2, 23]) { for (mr in [4l, 8l, 16l]) { t |> run("repack_{kq_tag(fmt)}_grp mr={mr} preserves dots and row dequants") @(t : T?) { kq_repack_gate(t, fmt, mr) @@ -1222,7 +1295,7 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { // nolint:STYL let mr = kq_layout_of(fmt) // the format's OWN layout companion (per-format tune families) let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) - let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : (fmt == 45 ? build_iq4nl_block() : (fmt == 2 ? build_q2k_block() : build_q6k_block())))))))) + let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : (fmt == 45 ? build_iq4nl_block() : (fmt == 2 ? build_q2k_block() : (fmt == 23 ? build_iq2s_block() : build_q6k_block()))))))))) var kq : array var ks : array kq |> resize(d * nsb * qsb) @@ -1247,6 +1320,8 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { // nolint:STYL transcode_iq4nl_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } elif (fmt == 2) { transcode_q2k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 23) { + transcode_iq2s_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } else { transcode_q6k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } @@ -1276,6 +1351,8 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { // nolint:STYL repack_q40_grp(addr(kq[0]), addr(ks[0]), n, d, mr) } elif (fmt == 2) { repack_k2_grp(addr(kq[0]), addr(ks[0]), n, d, mr) + } elif (fmt == 23) { + repack_iq2s_grp(addr(kq[0]), addr(ks[0]), n, d, mr) } else { repack_k6_grp(addr(kq[0]), addr(ks[0]), n, d, mr) } @@ -1304,7 +1381,7 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { // nolint:STYL // k5/k6 tiles read the byte-expanded panel the batch cell unpacks per (group, token-block) // — the gate covers unpack_kq_panel_grp too (tile-over-panel vs gemv-over-packed planes); // k4/q40 tiles read the packed planes directly - let packed = fmt == 4 || fmt == 40 || fmt == 44 || fmt == 3 || fmt == 45 || fmt == 2 // iq3s/iq3xxs tiles read the byte-expanded panel (the batch cell's shape) + let packed = fmt == 4 || fmt == 40 || fmt == 44 || fmt == 3 || fmt == 45 || fmt == 2 || fmt == 23 // iq3s/iq3xxs tiles read the byte-expanded panel (the batch cell's shape) var panel : array if (!packed) { panel |> resize(mr * nsb * 256l) @@ -1337,6 +1414,8 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { // nolint:STYL iq4nlq8_tile_gen(addr(ytile[0]), kqg, ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) } elif (fmt == 2) { k2q8_tile_gen(addr(ytile[0]), kqg, ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) + } elif (fmt == 23) { + iq2sq8_tile_gen(addr(ytile[0]), kqg, ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) } else { k6q8_tile_gen(addr(ytile[0]), addr(panel[0]), ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) } @@ -1361,6 +1440,8 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { // nolint:STYL iq4nlq8_gemv_gen(addr(ytile[tk * d]), addr(kq[0]), addr(ks[0]), addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, g * mr, (g + 1l) * mr) } elif (fmt == 2) { k2q8_gemv_gen(addr(ytile[tk * d]), addr(kq[0]), addr(ks[0]), addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, g * mr, (g + 1l) * mr) + } elif (fmt == 23) { + iq2sq8_gemv_gen(addr(ytile[tk * d]), addr(kq[0]), addr(ks[0]), addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, g * mr, (g + 1l) * mr) } else { k6q8_gemv_gen(addr(ytile[tk * d]), addr(kq[0]), addr(ks[0]), addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, g * mr, (g + 1l) * mr) } @@ -1386,6 +1467,8 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { // nolint:STYL iq4nlq8_gemv_gen(addr(ygemv[p * d]), addr(kq[0]), addr(ks[0]), addr(xq[p * n]), addr(xs[p * (n / 256l)]), addr(xbs[p * (n / 16l)]), n, 0l, d) } elif (fmt == 2) { k2q8_gemv_gen(addr(ygemv[p * d]), addr(kq[0]), addr(ks[0]), addr(xq[p * n]), addr(xs[p * (n / 256l)]), addr(xbs[p * (n / 16l)]), n, 0l, d) + } elif (fmt == 23) { + iq2sq8_gemv_gen(addr(ygemv[p * d]), addr(kq[0]), addr(ks[0]), addr(xq[p * n]), addr(xs[p * (n / 256l)]), addr(xbs[p * (n / 16l)]), n, 0l, d) } else { k6q8_gemv_gen(addr(ygemv[p * d]), addr(kq[0]), addr(ks[0]), addr(xq[p * n]), addr(xs[p * (n / 256l)]), addr(xbs[p * (n / 16l)]), n, 0l, d) } @@ -1411,7 +1494,7 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { // nolint:STYL [test] def test_kq_tile(t : T?) { - for (fmt in [4, 5, 6, 40, 44, 3, 33, 34, 45, 2]) { + for (fmt in [4, 5, 6, 40, 44, 3, 33, 34, 45, 2, 23]) { t |> run("{kq_tag(fmt)} 4-token tile bit-matches per-token GEMVs") @(t : T?) { kq_tile_gate(t, fmt) } @@ -1432,7 +1515,7 @@ def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; let nsb = n / 256l let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) - let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : (fmt == 45 ? build_iq4nl_block() : (fmt == 2 ? build_q2k_block() : build_q6k_block())))))))) + let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : (fmt == 45 ? build_iq4nl_block() : (fmt == 2 ? build_q2k_block() : (fmt == 23 ? build_iq2s_block() : build_q6k_block()))))))))) var kq : array var ks : array kq |> resize(nreg * d * nsb * qsb) @@ -1457,6 +1540,8 @@ def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; transcode_iq4nl_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } elif (fmt == 2) { transcode_q2k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 23) { + transcode_iq2s_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } else { transcode_q6k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } @@ -1532,6 +1617,9 @@ def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; } elif (fmt == 2) { want_sh[r * d + row] = dot_k2q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) want_pr[r * d + row] = dot_k2q8(kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n) + } elif (fmt == 23) { + want_sh[r * d + row] = dot_iq2s_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + want_pr[r * d + row] = dot_iq2s_q8(kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n) } else { want_sh[r * d + row] = dot_k6q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) want_pr[r * d + row] = dot_k6q8(kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n) @@ -1586,6 +1674,8 @@ def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; repack_q40_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) } elif (fmt == 2) { repack_k2_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) + } elif (fmt == 23) { + repack_iq2s_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) } else { repack_k6_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) } @@ -1620,6 +1710,9 @@ def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; } elif (fmt == 2) { k2q8_gemv_gen(addr(want_sh[r * d]), kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, 0l, d) k2q8_gemv_gen(addr(want_pr[r * d]), kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n, 0l, d) + } elif (fmt == 23) { + iq2sq8_gemv_gen(addr(want_sh[r * d]), kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, 0l, d) + iq2sq8_gemv_gen(addr(want_pr[r * d]), kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n, 0l, d) } else { k6q8_gemv_gen(addr(want_sh[r * d]), kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, 0l, d) k6q8_gemv_gen(addr(want_pr[r * d]), kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n, 0l, d) @@ -1655,7 +1748,7 @@ def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; [test] def test_kq_groupn(t : T?) { - for (fmt in [4, 5, 6, 40, 44, 3, 33, 34, 45, 2]) { + for (fmt in [4, 5, 6, 40, 44, 3, 33, 34, 45, 2, 23]) { t |> run("{kq_tag(fmt)} region-list GEMV bit-matches per-row dots (disk + grp slices)") @(t : T?) { kq_groupn_gate(t, fmt) } @@ -1699,7 +1792,7 @@ def private kq_batch_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 for (c in cnts) { nk += c } - let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : (fmt == 45 ? build_iq4nl_block() : (fmt == 2 ? build_q2k_block() : build_q6k_block())))))))) + let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : (fmt == 45 ? build_iq4nl_block() : (fmt == 2 ? build_q2k_block() : (fmt == 23 ? build_iq2s_block() : build_q6k_block()))))))))) var kq : array var ks : array kq |> resize(nreg * d * nsb * qsb) @@ -1724,6 +1817,8 @@ def private kq_batch_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 transcode_iq4nl_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } elif (fmt == 2) { transcode_q2k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 23) { + transcode_iq2s_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } else { transcode_q6k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } @@ -1754,6 +1849,8 @@ def private kq_batch_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 repack_q40_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) } elif (fmt == 2) { repack_k2_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) + } elif (fmt == 23) { + repack_iq2s_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) } else { repack_k6_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) } @@ -1810,6 +1907,8 @@ def private kq_batch_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 iq4nlq8_gemv_gen(addr(want[tk * d]), kqr, ksr, addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, 0l, d) } elif (fmt == 2) { k2q8_gemv_gen(addr(want[tk * d]), kqr, ksr, addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, 0l, d) + } elif (fmt == 23) { + iq2sq8_gemv_gen(addr(want[tk * d]), kqr, ksr, addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, 0l, d) } else { k6q8_gemv_gen(addr(want[tk * d]), kqr, ksr, addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, 0l, d) } @@ -1841,7 +1940,7 @@ def private kq_batch_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 [test] def test_kq_batch_groupn(t : T?) { - for (fmt in [4, 5, 6, 40, 44, 3, 33, 34, 45, 2]) { + for (fmt in [4, 5, 6, 40, 44, 3, 33, 34, 45, 2, 23]) { t |> run("{kq_tag(fmt)} batch groupn bit-matches per-token rows-core calls") @(t : T?) { kq_batch_groupn_gate(t, fmt) } From 9aa0becbeb362d89482eaec294ecb1d71fd747a0 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Mon, 31 Aug 2026 04:50:43 -0700 Subject: [PATCH 36/61] dasLLAMA: IQ2_S Phase B - the JIT emitter rides the panel route, the first u64-grid format generated iq2s joins the fmt-33/34 panel form end to end: the tile leaves the packed lists and reads the byte-expanded panel (new unpack_iq2s_panel_grp - the 10-bit qs|qh index doubled into the u64 grid's low/high word pair, signs through the shared smask expansion), kq_grp_row_dot_b grows a 23 arm (SIGNED panel bytes x per-16 UNSIGNED strips under the 33/34 d fold - d pre-eighth-ed at transcode), and the gemv gathers per superblock via emit_iq2s_gather off a [2048 x i32] private grid constant. In emit_block_iq4xs the iq2 flag splits the lo/hi accumulators and loads two ZExt strip vectors per block - per-16 strips cannot share the fused 33/34 dot. Probe 11/11 k23 perms (maddubs 8.3e-7); the tuner crowns dot_maddubs_width256_mr8 verdict=beats. QUIRK 15 rerun as written: run.tune.json still pinned iq2sq8_tile_gen "reference" from Phase A, cleared by run.das --tune. Gates: test_kquant -jit 229 pass, interp 216/17 skips, kqformat 18/18, M1 probe + kquant -jit 232/1, lint 0. E2e stamped: gen 41 -> 47 t/s; the fork vs llama.cpp moves step 10 -> 5, a 0.211-logit top-2 near-tie (stamped folds move the flip point, not the class). zen2 16t vs clean-cpu, the i1 vehicle: pp512 501.5 vs 138.5 (3.62x), tg128 55.8 vs 73.5 (0.76x - the ledgered `#60`/`#61` CPU-decode tail). Co-Authored-By: Claude Fable 5 --- modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md | 21 ++- .../dasLLAMA/dasllama/dasllama_gemm_gen.das | 148 ++++++++++++++++-- .../dasLLAMA/dasllama/dasllama_math_gen.das | 20 ++- modules/dasLLAMA/dasllama/dasllama_repack.das | 46 ++++++ modules/dasLLAMA/harness/gen_tune_probe.das | 5 +- modules/dasLLAMA/tests/test_kquant.das | 4 +- 6 files changed, 220 insertions(+), 24 deletions(-) diff --git a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md index f865174b0a..6a000ee708 100644 --- a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md +++ b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md @@ -453,7 +453,26 @@ with 16 strips. Gates: test_kqformat 18/18, test_kquant 233 (216 pass, 17 env sk lint 0. E2e: the mradermacher i1-IQ3_XXS vehicle (on disk since the iq3xxs phase, blocked on its IQ2_S attn x32) now loads and decodes FIVE formats in one graph - ids 10/64 vs llama.cpp with the fork a 0.040-logit near-tie (the arc's tightest; top2 IS our token), -gen 41 t/s reference bodies. JIT emitter, Vulkan, Metal: pending. +gen 41 t/s reference bodies. + +Phase B (JIT emitter, 2026-08-31): iq2s joins the fmt-33/34 PANEL route end to end - the +tile drops off the packed lists (probe, batch cell, test gate) and reads the byte-expanded +panel via a new `unpack_iq2s_panel_grp` (the 10-bit qs|qh index - `(qh << (8-2l)) & 0x300` - +doubled into the u64 grid's low/high word pair, sign bytes through the same smask +expansion), with `kq_grp_row_dot_b` growing a 23 arm: SIGNED panel bytes but per-16 +UNSIGNED strips (s0*ilo + s1*ihi) under the 33/34 d fold (d pre-eighth-ed at transcode). +The gemv gathers per superblock via `emit_iq2s_gather` off `iq2s_emit_globals`' [2048 x +i32] private grid + the shared smask; in `emit_block_iq4xs` the iq2 flag SPLITS the lo/hi +accumulators (a/a1) and loads two ZExt strip vectors per block, since per-16 strips cannot +share the fused 33/34 dot. Probe: 11/11 k23 perms ok (maddubs 8.3e-7); the tuner crowns +`dot_maddubs_width256_mr8` verdict=beats (the zen2 usual). QUIRK 15 reran exactly as +written: run.tune.json still pinned `iq2sq8_tile_gen : "reference"` from Phase A, cleared +with `run.das -- --tune`. Gates: test_kquant -jit 229 pass, interp 216/17 skips, +kqformat 18/18, M1 probe + kquant -jit 232/1, lint 0. E2e stamped: gen 41 -> 47 t/s; the +fork vs llama.cpp moves from step 10 to step 5 (0.211-logit top-2 near-tie - stamped folds +move the flip point, not the class; the iq3xxs precedent). zen2 16t vs clean-cpu on the i1 +vehicle: pp512 501.5 vs 138.5 (3.62x), tg128 55.8 vs 73.5 (0.76x - the ledgered #60/#61 +CPU-decode tail). Vulkan, Metal: pending. ### Q2_K Phase A (CPU, 2026-08-31) diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das index 1e87e9be3a..0b53e39d53 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das @@ -1471,6 +1471,108 @@ def private emit_iq3xxs_gather(var te : TileEmit; var sbi : LLVMOpaqueValue?) { LLVMPositionBuilderAtEnd(b, cont) } +// The iq2s gemv gather globals: the u64 grid as a [2048 x i32] private constant (1024 entries +// as low/high word pairs), plus the shared smask table (magnitudes {8, 25, 43} are nonzero - +// no carry crosses a byte lane on negation). +def private iq2s_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { + var grid = LLVMGetNamedGlobal(mod, "dasllama.iq2s.grid") + if (grid == null) { + let g = iq2s_grid2() + var elems : array + elems |> reserve(2048) + for (i in range(2048)) { + elems |> push(LLVMConstInt(te.types.t_int32, uint64(g[i]), 0)) + } + grid = LLVMAddGlobal(mod, LLVMArrayType(te.types.t_int32, 2048u), "dasllama.iq2s.grid") + LLVMSetInitializer(grid, LLVMConstArray(te.types.t_int32, array_data_ptr(elems), 2048u)) + LLVMSetGlobalConstant(grid, 1) + LLVMSetLinkage(grid, LLVMLinkage.LLVMPrivateLinkage) + LLVMSetAlignment(grid, 64u) + } + te.iq3s_grid_g = grid + var smask = LLVMGetNamedGlobal(mod, "dasllama.iq3s.smask") + if (smask == null) { + var elems2 : array + elems2 |> reserve(16) + for (m in range(16)) { + var v = 0ul + for (t in range(4)) { + if ((m & (1 << t)) != 0) { + v |= 0xFFul << uint64(8 * t) + } + } + elems2 |> push(LLVMConstInt(te.types.t_int32, v, 0)) + } + smask = LLVMAddGlobal(mod, LLVMArrayType(te.types.t_int32, 16u), "dasllama.iq3s.smask") + LLVMSetInitializer(smask, LLVMConstArray(te.types.t_int32, array_data_ptr(elems2), 16u)) + LLVMSetGlobalConstant(smask, 1) + LLVMSetLinkage(smask, LLVMLinkage.LLVMPrivateLinkage) + LLVMSetAlignment(smask, 16u) + } + te.iq3s_smask_g = smask +} + +// iq2s gemv: gather superblock `sbi` off the PACKED grp planes into the alloca panel - the +// iq3s gather's shape with the 10-bit index (qs byte | qh's 0x300 window) doubled into the +// u64 grid's low/high word pair; the sign byte is explicit like iq3s's. +def private emit_iq2s_gather(var te : TileEmit; var sbi : LLVMOpaqueValue?) { + let b = te.builder + let mr = te.interleave + var wbP = LLVMBuildMul(b, sbi, te.types->ConstI64(uint64(mr * 72)), "iq2s.wb") + var pre = LLVMGetInsertBlock(b) + var head = LLVMAppendBasicBlockInContext(te.gctx, te.gfn, "iq2s.gath") + var cont = LLVMAppendBasicBlockInContext(te.gctx, te.gfn, "iq2s.cont") + LLVMBuildBr(b, head) + LLVMPositionBuilderAtEnd(b, head) + var rPhi = LLVMBuildPhi(b, te.types.t_int64, "iq2s.r") + var preVals <- [te.types->ConstI64(0ul)] + var preBlocks <- [pre] + LLVMAddIncoming(rPhi, preVals, preBlocks) + var rT4 = LLVMBuildMul(b, rPhi, te.types->ConstI64(4ul), "iq2s.r4") + var base = LLVMBuildAdd(b, wbP, rT4, "iq2s.base") + let c768 = LLVMConstInt(te.types.t_int32, 0x300ul, 0) + let c15 = LLVMConstInt(te.types.t_int32, 15ul, 0) + let cneg = LLVMConstInt(te.types.t_int32, 0x01010101ul, 0) + for (blk in range(8)) { + let hp = 64 + blk + var qhp = LLVMBuildGEP2(b, te.types.t_int8, te.wg, LLVMBuildAdd(b, base, te.types->ConstI64(uint64((hp / 4) * mr * 4 + hp % 4)), ""), "") + var qh = LLVMBuildZExt(b, LLVMBuildLoad2Aligned(b, te.types.t_int8, qhp, 1u, ""), te.types.t_int32, "iq2s.qh{blk}") + for (l in range(4)) { + let sp = 32 + blk * 4 + l + var sgp = LLVMBuildGEP2(b, te.types.t_int8, te.wg, LLVMBuildAdd(b, base, te.types->ConstI64(uint64((sp / 4) * mr * 4 + sp % 4)), ""), "") + var sgn = LLVMBuildZExt(b, LLVMBuildLoad2Aligned(b, te.types.t_int8, sgp, 1u, ""), te.types.t_int32, "iq2s.sg{blk}_{l}") + let ip = blk * 4 + l + var ipp = LLVMBuildGEP2(b, te.types.t_int8, te.wg, LLVMBuildAdd(b, base, te.types->ConstI64(uint64((ip / 4) * mr * 4 + ip % 4)), ""), "") + var iv0 = LLVMBuildZExt(b, LLVMBuildLoad2Aligned(b, te.types.t_int8, ipp, 1u, ""), te.types.t_int32, "") + var hb = LLVMBuildAnd(b, LLVMBuildShl(b, qh, LLVMConstInt(te.types.t_int32, uint64(8 - 2 * l), 0), ""), c768, "") + var idx = LLVMBuildOr(b, iv0, hb, "iq2s.i{blk}_{l}") + var idx2 = LLVMBuildShl(b, idx, LLVMConstInt(te.types.t_int32, 1ul, 0), "") + var mv : LLVMOpaqueValue? [2] + mv[0] = LLVMBuildAnd(b, sgn, c15, "") + mv[1] = LLVMBuildAnd(b, LLVMBuildLShr(b, sgn, LLVMConstInt(te.types.t_int32, 4ul, 0), ""), c15, "") + for (h in range(2)) { + var gi = h == 0 ? idx2 : LLVMBuildAdd(b, idx2, LLVMConstInt(te.types.t_int32, 1ul, 0), "") + var gp = LLVMBuildGEP2(b, te.types.t_int32, te.iq3s_grid_g, gi, "") + var gw = LLVMBuildLoad2Aligned(b, te.types.t_int32, gp, 4u, "iq2s.g{blk}_{l}_{h}") + var mp = LLVMBuildGEP2(b, te.types.t_int32, te.iq3s_smask_g, mv[h], "") + var mw = LLVMBuildLoad2Aligned(b, te.types.t_int32, mp, 4u, "") + var w = LLVMBuildAdd(b, LLVMBuildXor(b, gw, mw, ""), LLVMBuildAnd(b, mw, cneg, ""), "iq2s.w{blk}_{l}_{h}") + let e = l * 8 + h * 4 + let dd = (e < 16 ? 0 : 128 * mr) + (blk * 4 + (e % 16) / 4) * mr * 4 + var dp = LLVMBuildGEP2(b, te.types.t_int8, te.iq3s_panel, LLVMBuildAdd(b, rT4, te.types->ConstI64(uint64(dd)), ""), "") + LLVMBuildStore(b, w, dp) + } + } + } + var rNext = LLVMBuildAdd(b, rPhi, te.types->ConstI64(1ul), "iq2s.rn") + var more = LLVMBuildICmp(b, LLVMIntPredicate.LLVMIntSLT, rNext, te.types->ConstI64(uint64(mr)), "iq2s.more") + LLVMBuildCondBr(b, more, head, cont) + var loopVals <- [rNext] + var loopBlocks <- [head] + LLVMAddIncoming(rPhi, loopVals, loopBlocks) + LLVMPositionBuilderAtEnd(b, cont) +} + // One 256-weight SUPERBLOCK, IQ4_XS/iq3s grp form (te.kq = 44/33): mx4's LUT decode (44) // or the byte-expanded signed panel (33; the gemv gathers it per superblock, the tile gets the // runtime unpack) — dot_lane's sign-trick lattice, k4's fold with no min term. @@ -1479,7 +1581,8 @@ def private emit_block_iq4xs(var te : TileEmit; var sbi : LLVMOpaqueValue?; var let rq = te.rq let mr = te.interleave let w8 = te.width / 8 - let panel = te.kq == 33 || te.kq == 34 // iq3s/iq3xxs: the byte-expanded signed panel replaces the nibble+LUT decode; fold identical + let panel = te.kq == 33 || te.kq == 34 || te.kq == 23 // iq3s/iq3xxs/iq2s: the byte-expanded signed panel replaces the nibble+LUT decode + let iq2 = te.kq == 23 // iq2s: per-16 UNSIGNED strips - lo/hi halves keep split accumulators let nl = te.kq == 45 // iq4nl: the 44 LUT decode with q40's PER-BLOCK f16 d fold (no sub-scales, no min term) let gather = panel && te.iq3s_panel != null // gemv mode: gather this superblock off the packed planes first var wbase = te.wg @@ -1487,6 +1590,8 @@ def private emit_block_iq4xs(var te : TileEmit; var sbi : LLVMOpaqueValue?; var if (gather) { if (te.kq == 33) { emit_iq3s_gather(te, sbi) + } elif (te.kq == 23) { + emit_iq2s_gather(te, sbi) } else { emit_iq3xxs_gather(te, sbi) } @@ -1526,8 +1631,10 @@ def private emit_block_iq4xs(var te : TileEmit; var sbi : LLVMOpaqueValue?; var xv1[i] = load_v16i8(te, te.x[tokBase + i], xoff1, "x{tokBase + i}_{blk}hi") } var a : LLVMOpaqueValue? [8] // [token * rq + qd]; tokCount*rq <= 8 (nrsplit*vq rail) + var a1 : LLVMOpaqueValue? [8] // iq2s: the hi-16 half's own accumulator (per-16 strips) for (i in range(tokCount * rq)) { a[i] = LLVMConstNull(te.vni32) + a1[i] = LLVMConstNull(te.vni32) } for (j in range(4)) { for (qd in range(rq)) { @@ -1548,7 +1655,12 @@ def private emit_block_iq4xs(var te : TileEmit; var sbi : LLVMOpaqueValue?; var var ulo = abs_w(te, wlo, "ulo{blk}_{j * rq + qd}") var uhi = abs_w(te, whi, "uhi{blk}_{j * rq + qd}") for (i in range(tokCount)) { - a[i * rq + qd] = dot_lane(te, dot_lane(te, a[i * rq + qd], wlo, ulo, xv0[i], j), whi, uhi, xv1[i], j) + if (iq2) { + a[i * rq + qd] = dot_lane(te, a[i * rq + qd], wlo, ulo, xv0[i], j) + a1[i * rq + qd] = dot_lane(te, a1[i * rq + qd], whi, uhi, xv1[i], j) + } else { + a[i * rq + qd] = dot_lane(te, dot_lane(te, a[i * rq + qd], wlo, ulo, xv0[i], j), whi, uhi, xv1[i], j) + } } } } @@ -1566,13 +1678,25 @@ def private emit_block_iq4xs(var te : TileEmit; var sbi : LLVMOpaqueValue?; var } } else { var scv : LLVMOpaqueValue? [2] + var scv1 : LLVMOpaqueValue? [2] // iq2s: the hi half's strip for (qd in range(rq)) { + if (iq2) { + var s0p = LLVMBuildGEP2(b, te.types.t_int8, te.sg, LLVMBuildAdd(b, sb, te.types->ConstI64(uint64(4 * mr + 2 * blk * mr + qd * te.rv)), ""), "s0p{blk}_{qd}") + var s1p = LLVMBuildGEP2(b, te.types.t_int8, te.sg, LLVMBuildAdd(b, sb, te.types->ConstI64(uint64(4 * mr + (2 * blk + 1) * mr + qd * te.rv)), ""), "s1p{blk}_{qd}") + scv[qd] = LLVMBuildZExt(b, LLVMBuildLoad2Aligned(b, vri8, s0p, 1u, ""), te.vni32, "sc{blk}_{qd}a") + scv1[qd] = LLVMBuildZExt(b, LLVMBuildLoad2Aligned(b, vri8, s1p, 1u, ""), te.vni32, "sc{blk}_{qd}b") + continue + } var scp = LLVMBuildGEP2(b, te.types.t_int8, te.sg, LLVMBuildAdd(b, sb, te.types->ConstI64(uint64(4 * mr + blk * mr + qd * te.rv)), ""), "scp{blk}_{qd}") scv[qd] = LLVMBuildSExt(b, LLVMBuildLoad2Aligned(b, vri8, scp, 1u, ""), te.vni32, "sc{blk}_{qd}") } for (i in range(tokCount)) { for (qd in range(rq)) { - iacc[i * rq + qd] = LLVMBuildAdd(b, iacc[i * rq + qd], LLVMBuildMul(b, scv[qd], a[i * rq + qd], ""), "ia{tokBase + i}_{blk}_{qd}") + var ia = LLVMBuildMul(b, scv[qd], a[i * rq + qd], "") + if (iq2) { + ia = LLVMBuildAdd(b, ia, LLVMBuildMul(b, scv1[qd], a1[i * rq + qd], ""), "") + } + iacc[i * rq + qd] = LLVMBuildAdd(b, iacc[i * rq + qd], ia, "ia{tokBase + i}_{blk}_{qd}") } } } @@ -1596,7 +1720,7 @@ def private emit_block_iq4xs(var te : TileEmit; var sbi : LLVMOpaqueValue?; var // token PAIR — single-token slices ride the sdot lattice instead. kq blocks are SUPERBLOCKS // (the kq gemv/tile drive emit_slice with nb = n/256). def private emit_one_block(var te : TileEmit; var bi : LLVMOpaqueValue?; var f : LLVMOpaqueValue? [8]; tokBase, tokCount : int) { - if (te.kq == 44 || te.kq == 33 || te.kq == 34 || te.kq == 45) { + if (te.kq == 44 || te.kq == 33 || te.kq == 34 || te.kq == 45 || te.kq == 23) { emit_block_iq4xs(te, bi, f, tokBase, tokCount) } elif (te.kq != 0) { emit_block_kqv2(te, bi, f, tokBase, tokCount) @@ -2703,12 +2827,14 @@ def private kq_gemv_gen_impl(var gc : LlvmCodeCtx; fmt : int) : bool { // entry: group range off the row range; strides are disk footprints (quants 128/160/192B, scales 20/20/18B) LLVMPositionBuilderAtEnd(b, entry) - if (fmt == 33 || fmt == 34) { - // iq3s/iq3xxs: the gemv gathers each superblock into an alloca panel (the tile gets the runtime unpack) + if (fmt == 33 || fmt == 34 || fmt == 23) { + // iq3s/iq3xxs/iq2s: the gemv gathers each superblock into an alloca panel (the tile gets the runtime unpack) te.gfn = gc.impl te.gctx = gc.jit.ctx if (fmt == 33) { iq3s_emit_globals(te, LLVMGetGlobalParent(gc.impl)) + } elif (fmt == 23) { + iq2s_emit_globals(te, LLVMGetGlobalParent(gc.impl)) } else { iq3xxs_emit_globals(te, LLVMGetGlobalParent(gc.impl)) } @@ -2907,12 +3033,10 @@ def private iq4nl_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, def private k2_gemv_gen(var gc : LlvmCodeCtx) : bool => kq_gemv_gen_impl(gc, 2) def private k2_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 2) -// iq2s: the grid-gather emitter arm lands with Phase B; until then both stamps decline -// (reference bodies serve) - a declined generator is the framework's own fallback path. -[unused_argument(gc)] -def private iq2s_gemv_gen(var gc : LlvmCodeCtx) : bool => false -[unused_argument(gc)] -def private iq2s_tile_gen(var gc : LlvmCodeCtx) : bool => false +// iq2s rides the iq4xs walk through the panel route: emit_iq2s_gather feeds the shared +// byte-expanded panel; per-16 UNSIGNED strips keep split lo/hi accumulators (the iq2 flag) +def private iq2s_gemv_gen(var gc : LlvmCodeCtx) : bool => kq_gemv_gen_impl(gc, 23) +def private iq2s_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 23) def private k4_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 4) def private k5_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 5) diff --git a/modules/dasLLAMA/dasllama/dasllama_math_gen.das b/modules/dasLLAMA/dasllama/dasllama_math_gen.das index c7356f9e71..f62f39071c 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_gen.das @@ -964,7 +964,7 @@ def iq2sq8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp unsafe { for (t in range64(4l)) { for (r in range64(mr)) { - yp[(t0 + t) * d + g * mr + r] = iq2s_grp_row_dot(kqg, ksg, r, mr, xqp + (t0 + t) * n, xsp + (t0 + t) * (n / 256l), xbsp + (t0 + t) * (n / 16l), n) + yp[(t0 + t) * d + g * mr + r] = kq_grp_row_dot_b(23l, kqg, ksg, r, mr, xqp + (t0 + t) * n, xsp + (t0 + t) * (n / 256l), xbsp + (t0 + t) * (n / 16l), n) } } } @@ -1227,8 +1227,8 @@ def private q51q8_groupn_gen(var yp : float?; wq : uint8 const?; ws : uint8 cons //! kq_grp_row_dot's twin over a BYTE-EXPANDED panel (unpack_kq_panel_grp's output) - bit-exact -//! against the packed-plane dot. fmt picks the scale form only: 5 = u8 sc/mn + d/dmin, 6 = -//! signed per-16 sub-scales + d, 33 = SIGNED panel bytes x per-32 (1 + 2s) + d, no bsum term. +//! against the packed-plane dot. fmt picks the scale form only: 5 = u8 sc/mn + d/dmin, 6 = signed +//! per-16 sub-scales + d, 33/34 = per-32 (1 + 2s) + d, 23 = per-16 UNSIGNED strips + d/8; no bsum for 33/34/23. def kq_grp_row_dot_b(fmt : int64; kqg : uint8 const?; ksg : uint8 const?; r, mr : int64; xqp : int8 const?; xsp : float const?; xbsp : int const?; n : int64) : float { var acc = 0.0 let nsb = n / 256l @@ -1245,8 +1245,8 @@ def kq_grp_row_dot_b(fmt : int64; kqg : uint8 const?; ksg : uint8 const?; r, mr var ihi = 0 for (j in range64(4l)) { for (t in range64(4l)) { - let qlo = fmt == 33l || fmt == 34l ? int(int8(kqg[qb + ((blk * 4l + j) * mr + r) * 4l + t])) : int(kqg[qb + ((blk * 4l + j) * mr + r) * 4l + t]) - let qhi = fmt == 33l || fmt == 34l ? int(int8(kqg[qb + 128l * mr + ((blk * 4l + j) * mr + r) * 4l + t])) : int(kqg[qb + 128l * mr + ((blk * 4l + j) * mr + r) * 4l + t]) + let qlo = fmt == 33l || fmt == 34l || fmt == 23l ? int(int8(kqg[qb + ((blk * 4l + j) * mr + r) * 4l + t])) : int(kqg[qb + ((blk * 4l + j) * mr + r) * 4l + t]) + let qhi = fmt == 33l || fmt == 34l || fmt == 23l ? int(int8(kqg[qb + 128l * mr + ((blk * 4l + j) * mr + r) * 4l + t])) : int(kqg[qb + 128l * mr + ((blk * 4l + j) * mr + r) * 4l + t]) ilo += qlo * int(xqp[b * 32l + j * 4l + t]) ihi += qhi * int(xqp[b * 32l + 16l + j * 4l + t]) } @@ -1254,6 +1254,10 @@ def kq_grp_row_dot_b(fmt : int64; kqg : uint8 const?; ksg : uint8 const?; r, mr if (fmt == 33l || fmt == 34l) { let sc = int(int8(ksg[sb + 4l * mr + blk * mr + r])) isum += sc * (ilo + ihi) + } elif (fmt == 23l) { + let s0 = int(ksg[sb + 4l * mr + 2l * blk * mr + r]) + let s1 = int(ksg[sb + 4l * mr + (2l * blk + 1l) * mr + r]) + isum += s0 * ilo + s1 * ihi } elif (fmt == 6l) { let s0 = int(int8(ksg[sb + 2l * blk * mr + r])) let s1 = int(int8(ksg[sb + (2l * blk + 1l) * mr + r])) @@ -1266,7 +1270,7 @@ def kq_grp_row_dot_b(fmt : int64; kqg : uint8 const?; ksg : uint8 const?; r, mr bsum += mn * (xbsp[b * 2l] + xbsp[b * 2l + 1l]) } } - if (fmt == 33l || fmt == 34l) { + if (fmt == 33l || fmt == 34l || fmt == 23l) { let di = f16_to_f32(uint(ksg[sb + 2l * r]) | (uint(ksg[sb + 2l * r + 1l]) << 8u)) acc += float(isum) * (di * xsp[sbi]) } elif (fmt == 6l) { @@ -1898,7 +1902,7 @@ def private kq_batch_cell_gen(fmt : int; var myp : float?; kqp : uint8 const?; k let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) let nb16 = n / 16l - let packed = fmt == 4 || fmt == 40 || fmt == 44 || fmt == 3 || fmt == 45 || fmt == 2 || fmt == 23 // packed planes: the tile reads them directly (iq3s/iq3xxs expand their grid gathers into the panel) + let packed = fmt == 4 || fmt == 40 || fmt == 44 || fmt == 3 || fmt == 45 || fmt == 2 // packed planes: the tile reads them directly (iq3s/iq3xxs/iq2s expand their grid gathers into the panel) var scratch : array if (!packed) { scratch |> resize(mr * nsb * 256l) @@ -1937,7 +1941,7 @@ def private kq_batch_cell_gen(fmt : int; var myp : float?; kqp : uint8 const?; k } elif (fmt == 2) { k2q8_tile_gen(myp, kqg, ksg, xqp, xsp, xbsp, n, d, g, tk) } elif (fmt == 23) { - iq2sq8_tile_gen(myp, kqg, ksg, xqp, xsp, xbsp, n, d, g, tk) + iq2sq8_tile_gen(myp, scp, ksg, xqp, xsp, xbsp, n, d, g, tk) } else { k6q8_tile_gen(myp, scp, ksg, xqp, xsp, xbsp, n, d, g, tk) } diff --git a/modules/dasLLAMA/dasllama/dasllama_repack.das b/modules/dasLLAMA/dasllama/dasllama_repack.das index 06ef439543..4d23ccdf62 100644 --- a/modules/dasLLAMA/dasllama/dasllama_repack.das +++ b/modules/dasLLAMA/dasllama/dasllama_repack.das @@ -739,6 +739,48 @@ def unpack_iq3xxs_panel_grp(kqg : uint8 const?; var dst : uint8?; mr, nsb : int6 } } +//! iq2s panel unpack: the 10-bit qs|qh gather off the u64 grid (TWO words per index - eight +//! magnitudes {8, 25, 43}) + the block's own sign bytes, packed grp planes -> the shared +//! byte-expanded signed panel (the fmt-33/34 panel form; d is pre-eighth-ed at transcode). +def unpack_iq2s_panel_grp(kqg : uint8 const?; var dst : uint8?; mr, nsb : int64) { + let grid = iq2s_grid2() + let smask = fixed_array(0x00000000u, 0x000000FFu, 0x0000FF00u, 0x0000FFFFu, + 0x00FF0000u, 0x00FF00FFu, 0x00FFFF00u, 0x00FFFFFFu, + 0xFF000000u, 0xFF0000FFu, 0xFF00FF00u, 0xFF00FFFFu, + 0xFFFF0000u, 0xFFFF00FFu, 0xFFFFFF00u, 0xFFFFFFFFu) + unsafe { + for (sbi in range64(nsb)) { + let qb = sbi * 72l * mr + let ob = sbi * 256l * mr + for (r in range64(mr)) { + for (blk in range64(8l)) { + let hp = 64l + blk + let qh = int(kqg[qb + ((hp / 4l) * mr + r) * 4l + hp % 4l]) + for (l in range64(4l)) { + let sp = 32l + blk * 4l + l + let sgn = int(kqg[qb + ((sp / 4l) * mr + r) * 4l + sp % 4l]) + let ip = blk * 4l + l + let ix = (int(kqg[qb + ((ip / 4l) * mr + r) * 4l + ip % 4l]) | ((qh << int(8l - 2l * l)) & 0x300)) * 2 + let m1 = smask[sgn & 15] + let m2 = smask[(sgn >> 4) & 15] + let w1 = (grid[ix] ^ m1) + (m1 & 0x01010101u) + let w2 = (grid[ix + 1] ^ m2) + (m2 & 0x01010101u) + // element e = l*8 + half*4 + t: e < 16 lands in the lo panel half, else hi + let e1 = l * 8l + let e2 = l * 8l + 4l + let d1 = (e1 < 16l ? 0l : 128l * mr) + ((blk * 4l + (e1 % 16l) / 4l) * mr + r) * 4l + let d2 = (e2 < 16l ? 0l : 128l * mr) + ((blk * 4l + (e2 % 16l) / 4l) * mr + r) * 4l + var wp1 = reinterpret(dst + (ob + d1)) + var wp2 = reinterpret(dst + (ob + d2)) + wp1[0] = w1 + wp2[0] = w2 + } + } + } + } + } +} + def unpack_kq_panel_grp(fmt : int64; kqg : uint8 const?; var dst : uint8?; mr, nsb : int64) { if (fmt == 33l) { unpack_iq3s_panel_grp(kqg, dst, mr, nsb) @@ -748,6 +790,10 @@ def unpack_kq_panel_grp(fmt : int64; kqg : uint8 const?; var dst : uint8?; mr, n unpack_iq3xxs_panel_grp(kqg, dst, mr, nsb) return } + if (fmt == 23l) { + unpack_iq2s_panel_grp(kqg, dst, mr, nsb) + return + } let LO4 = 0x0F0F0F0F0F0F0F0Ful let SEL = 0x0804020108040201ul // LE lane t (0..3, twice) selects high-bit t let C7F = 0x7F7F7F7F7F7F7F7Ful diff --git a/modules/dasLLAMA/harness/gen_tune_probe.das b/modules/dasLLAMA/harness/gen_tune_probe.das index 7038c1e5e6..55b5d73eb6 100644 --- a/modules/dasLLAMA/harness/gen_tune_probe.das +++ b/modules/dasLLAMA/harness/gen_tune_probe.das @@ -888,7 +888,7 @@ def kq_layout_mrs(fmt : int64) : table { // nolint:STYLE037 — t def run_kq_tile(tile; fx : KqFixture; kq : array; ks : array; mr : int64; var y : array) { let ng = fx.d / mr verify(fx.ntok % 4l == 0l) - let packed = fx.fmt == 4l || fx.fmt == 40l || fx.fmt == 44l || fx.fmt == 3l || fx.fmt == 45l || fx.fmt == 2l || fx.fmt == 23l // packed planes: the tile reads them directly (iq3s/iq3xxs unpack their panels per group) + let packed = fx.fmt == 4l || fx.fmt == 40l || fx.fmt == 44l || fx.fmt == 3l || fx.fmt == 45l || fx.fmt == 2l // packed planes: the tile reads them directly (iq3s/iq3xxs/iq2s unpack their panels per group) var panel : array if (!packed) { panel |> resize(int(mr * fx.nsb * 256l)) @@ -1374,6 +1374,7 @@ def test_mode : bool { // nolint:STYLE037,STYLE038 — flat per-variant correctn var kfxs34 <- [ <- build_kq_fixture(34l, 256l, 32l, 8l), <- build_kq_fixture(34l, 768l, 32l, 8l), <- build_kq_fixture(34l, 2048l, 32l, 8l)] var kfxs45 <- [ <- build_kq_fixture(45l, 256l, 32l, 8l), <- build_kq_fixture(45l, 768l, 32l, 8l), <- build_kq_fixture(45l, 2048l, 32l, 8l)] var kfxs2 <- [ <- build_kq_fixture(2l, 256l, 32l, 8l), <- build_kq_fixture(2l, 768l, 32l, 8l), <- build_kq_fixture(2l, 2048l, 32l, 8l)] + var kfxs23 <- [ <- build_kq_fixture(23l, 256l, 32l, 8l), <- build_kq_fixture(23l, 768l, 32l, 8l), <- build_kq_fixture(23l, 2048l, 32l, 8l)] var vs <- q8q8_tile_gen_variants() var mrs <- variant_mrs() var wbs <- variant_wbias() @@ -1504,6 +1505,7 @@ def test_mode : bool { // nolint:STYLE037,STYLE038 — flat per-variant correctn allok = kq_test_family(34l, kfxs34) && allok allok = kq_test_family(45l, kfxs45) && allok allok = kq_test_family(2l, kfxs2) && allok + allok = kq_test_family(23l, kfxs23) && allok delete wtab delete mtab delete gtab @@ -1525,6 +1527,7 @@ def test_mode : bool { // nolint:STYLE037,STYLE038 — flat per-variant correctn delete kfxs34 delete kfxs45 delete kfxs2 + delete kfxs23 return allok } diff --git a/modules/dasLLAMA/tests/test_kquant.das b/modules/dasLLAMA/tests/test_kquant.das index 9652df004d..29588e5730 100644 --- a/modules/dasLLAMA/tests/test_kquant.das +++ b/modules/dasLLAMA/tests/test_kquant.das @@ -1381,7 +1381,7 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { // nolint:STYL // k5/k6 tiles read the byte-expanded panel the batch cell unpacks per (group, token-block) // — the gate covers unpack_kq_panel_grp too (tile-over-panel vs gemv-over-packed planes); // k4/q40 tiles read the packed planes directly - let packed = fmt == 4 || fmt == 40 || fmt == 44 || fmt == 3 || fmt == 45 || fmt == 2 || fmt == 23 // iq3s/iq3xxs tiles read the byte-expanded panel (the batch cell's shape) + let packed = fmt == 4 || fmt == 40 || fmt == 44 || fmt == 3 || fmt == 45 || fmt == 2 // iq3s/iq3xxs/iq2s tiles read the byte-expanded panel (the batch cell's shape) var panel : array if (!packed) { panel |> resize(mr * nsb * 256l) @@ -1415,7 +1415,7 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { // nolint:STYL } elif (fmt == 2) { k2q8_tile_gen(addr(ytile[0]), kqg, ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) } elif (fmt == 23) { - iq2sq8_tile_gen(addr(ytile[0]), kqg, ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) + iq2sq8_tile_gen(addr(ytile[0]), addr(panel[0]), ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) } else { k6q8_tile_gen(addr(ytile[0]), addr(panel[0]), ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) } From 48f851699c89117ce78f59f21b9cbfa59a0ccf26 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Mon, 31 Aug 2026 05:24:27 -0700 Subject: [PATCH 37/61] dasLLAMA: IQ2_S Phase C - the u64 grid crosses to Vulkan, the first 8 KB workgroup-staged codebook KqGemvIq2s widens the iq3s shell to the u64 grid: @workgroup uint[2048] staged 32 words per thread, TWO grid words per 10-bit qs|qh index, the block's own sign bytes through the shared smask negate, per-16 UNSIGNED strips folded k2-style (split lo/hi dots, d pre-eighth-ed at transcode). KqBatchIq2s : KqBatchK6 puts the iq3s gather in stage_w and the strip pair on the wsw/wsw2 planes with a plain split fma (no offset, no min term); Iq2sCm2T decodes VkIq2sBlk ([qs][sg][qh] 16-bit lanes) on a new IQ2SGRID gated axis. The device scale row is the CPU 20B row verbatim - the grouped gather's k4/k5 else arm already emits [d][2B hole][16 strips], no layout edit. vk_kq_schema_id 12 -> 23, eleven-format family cells (stride 18), a dedicated float witness + an iq2sf16 cm2 oracle. Gates: test_vulkan_kernels 82/82 (the three iq2s cm2 tiles 0-off at 89600 cells each), lint 0. The resident driver arms on the i1 vehicle: gen 233 t/s, the fork vs llama.cpp at step 5 = the same 0.211-logit top-2 near-tie as the CPU-JIT tier. 5060 Ti vs llama.cpp b10660 Vulkan -ngl 99: pp512 12099.3 vs 17377.5 (0.70x - the tier class), tg128 292.4 vs 362.5 (0.81x; the gemv re-stages the 8 KB grid per 2-row workgroup - the known tg lever). Co-Authored-By: Claude Fable 5 --- modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md | 21 +- .../dasllama/dasllama_vulkan_classes.das | 492 +++++++++++++++++- .../dasllama/dasllama_vulkan_common.das | 1 + .../dasllama/dasllama_vulkan_prefill.das | 17 + modules/dasLLAMA/tests/_vkd_oracles.das | 83 +++ .../dasLLAMA/tests/test_vulkan_kernels.das | 150 +++++- 6 files changed, 754 insertions(+), 10 deletions(-) diff --git a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md index 6a000ee708..242efe7ff1 100644 --- a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md +++ b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md @@ -472,7 +472,26 @@ kqformat 18/18, M1 probe + kquant -jit 232/1, lint 0. E2e stamped: gen 41 -> 47 fork vs llama.cpp moves from step 10 to step 5 (0.211-logit top-2 near-tie - stamped folds move the flip point, not the class; the iq3xxs precedent). zen2 16t vs clean-cpu on the i1 vehicle: pp512 501.5 vs 138.5 (3.62x), tg128 55.8 vs 73.5 (0.76x - the ledgered #60/#61 -CPU-decode tail). Vulkan, Metal: pending. +CPU-decode tail). + +Phase C (Vulkan, 2026-08-31): the iq3s walk widened to the u64 grid. `iq2s_grid_word` ships +the 8 KB table as low/high word pairs (word 2i / 2i+1); `KqGemvIq2s` = the iq3s shell with +`@workgroup uint[2048]` (32 words staged per thread at local_size 64), TWO grid words per +10-bit qs|qh index (`(qh << (8-2l)) & 0x300`), the block's own sign bytes through the same +smask negate, and the per-16 UNSIGNED strips folded k2-style off wsu words 1..4 (split +lo/hi dots, d pre-eighth-ed). `KqBatchIq2s : KqBatchK6` - the iq3s gather in stage_w, the +strip pair on the wsw/wsw2 planes, and a plain split fma (no offset, no min term). +`Iq2sCm2T` = `VkIq2sBlk` ([qs 16][sg 16][qh 4] int16 lanes) on a new `IQ2SGRID` gated axis +(8 words per thread at 256); the device scale row is the CPU 20B row VERBATIM (d + 2B hole ++ 16 strips - the grouped gather's k4/k5 else arm already emits exactly that, no layout +edit). vk_kq_schema_id 12 -> 23; eleven-format family cells at stride 18 words; a dedicated +float witness (`iq2s_gemv_float_oracle`) plus `iq2sf16_gemm_oracle` for the cm2 tiles. +Gates: the suite 82/82 (the three iq2s cm2 tiles 0-off at 89600 cells each); lint 0. The +resident driver arms on the i1 vehicle: gen 233 t/s, the fork vs llama.cpp at step 5 = the +SAME 0.211-logit top-2 near-tie as the CPU-JIT tier. Rows (5060 Ti vs llama.cpp b10660 +build-vulkan -ngl 99): pp512 12099.3 / 17377.5 (0.70x - the tier class), tg128 292.4 / +362.5 (0.81x; the gemv re-stages the 8 KB grid per 2-row workgroup - the obvious tg lever, +same ledger class as the pp tier). Metal: pending. ### Q2_K Phase A (CPU, 2026-08-31) diff --git a/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das b/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das index 66ccdd25fb..2ff271c0d1 100644 --- a/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das +++ b/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das @@ -2540,6 +2540,270 @@ def private iq3s_grid_word(i : int) : uint { return tbl[i] } +// ggml's iq2s_grid as low/high word pairs - 1024 u64 entries, one uint word per call (word 2i = +// magnitudes 0..3 of entry i, word 2i+1 = 4..7); the staging loops copy it into workgroup +// memory once per kernel, so the local-array storage class never sits on a hot path. +def private iq2s_grid_word(i : int) : uint { // nolint:STYLE038 - a 2048-word data table, not splittable + let tbl = fixed_array( + 0x08080808, 0x08080808, 0x0808082b, 0x08080808, 0x08081919, 0x08080808, 0x08082b08, 0x08080808, + 0x08082b2b, 0x08080808, 0x08190819, 0x08080808, 0x08191908, 0x08080808, 0x0819192b, 0x08080808, + 0x08192b19, 0x08080808, 0x082b0808, 0x08080808, 0x082b082b, 0x08080808, 0x082b1919, 0x08080808, + 0x082b2b08, 0x08080808, 0x19080819, 0x08080808, 0x19081908, 0x08080808, 0x1908192b, 0x08080808, + 0x19082b19, 0x08080808, 0x19190808, 0x08080808, 0x1919082b, 0x08080808, 0x19191919, 0x08080808, + 0x19192b08, 0x08080808, 0x192b0819, 0x08080808, 0x192b1908, 0x08080808, 0x192b192b, 0x08080808, + 0x192b2b19, 0x08080808, 0x2b080808, 0x08080808, 0x2b08082b, 0x08080808, 0x2b081919, 0x08080808, + 0x2b082b08, 0x08080808, 0x2b190819, 0x08080808, 0x2b191908, 0x08080808, 0x2b2b0808, 0x08080808, + 0x2b2b1919, 0x08080808, 0x2b2b2b2b, 0x08080808, 0x08080819, 0x08080819, 0x08081908, 0x08080819, + 0x0808192b, 0x08080819, 0x08082b19, 0x08080819, 0x08190808, 0x08080819, 0x0819082b, 0x08080819, + 0x08191919, 0x08080819, 0x08192b08, 0x08080819, 0x082b0819, 0x08080819, 0x082b1908, 0x08080819, + 0x19080808, 0x08080819, 0x1908082b, 0x08080819, 0x19081919, 0x08080819, 0x19082b08, 0x08080819, + 0x19190819, 0x08080819, 0x19191908, 0x08080819, 0x1919192b, 0x08080819, 0x19192b19, 0x08080819, + 0x192b0808, 0x08080819, 0x192b1919, 0x08080819, 0x192b2b08, 0x08080819, 0x2b080819, 0x08080819, + 0x2b081908, 0x08080819, 0x2b190808, 0x08080819, 0x2b19082b, 0x08080819, 0x2b191919, 0x08080819, + 0x2b2b0819, 0x08080819, 0x2b2b1908, 0x08080819, 0x08080808, 0x0808082b, 0x0808082b, 0x0808082b, + 0x08081919, 0x0808082b, 0x08082b08, 0x0808082b, 0x08190819, 0x0808082b, 0x08191908, 0x0808082b, + 0x082b0808, 0x0808082b, 0x082b2b2b, 0x0808082b, 0x19080819, 0x0808082b, 0x19081908, 0x0808082b, + 0x1908192b, 0x0808082b, 0x19082b19, 0x0808082b, 0x19190808, 0x0808082b, 0x19191919, 0x0808082b, + 0x2b080808, 0x0808082b, 0x2b081919, 0x0808082b, 0x2b082b2b, 0x0808082b, 0x2b191908, 0x0808082b, + 0x2b2b082b, 0x0808082b, 0x08080819, 0x08081908, 0x08081908, 0x08081908, 0x0808192b, 0x08081908, + 0x08082b19, 0x08081908, 0x08190808, 0x08081908, 0x0819082b, 0x08081908, 0x08191919, 0x08081908, + 0x08192b08, 0x08081908, 0x082b0819, 0x08081908, 0x082b1908, 0x08081908, 0x082b192b, 0x08081908, + 0x082b2b19, 0x08081908, 0x19080808, 0x08081908, 0x1908082b, 0x08081908, 0x19081919, 0x08081908, + 0x19082b08, 0x08081908, 0x19082b2b, 0x08081908, 0x19190819, 0x08081908, 0x19191908, 0x08081908, + 0x1919192b, 0x08081908, 0x19192b19, 0x08081908, 0x192b0808, 0x08081908, 0x192b082b, 0x08081908, + 0x192b1919, 0x08081908, 0x2b080819, 0x08081908, 0x2b081908, 0x08081908, 0x2b08192b, 0x08081908, + 0x2b082b19, 0x08081908, 0x2b190808, 0x08081908, 0x2b191919, 0x08081908, 0x2b192b08, 0x08081908, + 0x2b2b0819, 0x08081908, 0x2b2b1908, 0x08081908, 0x08080808, 0x08081919, 0x0808082b, 0x08081919, + 0x08081919, 0x08081919, 0x08082b08, 0x08081919, 0x08082b2b, 0x08081919, 0x08190819, 0x08081919, + 0x08191908, 0x08081919, 0x0819192b, 0x08081919, 0x08192b19, 0x08081919, 0x082b0808, 0x08081919, + 0x082b1919, 0x08081919, 0x082b2b08, 0x08081919, 0x19080819, 0x08081919, 0x19081908, 0x08081919, + 0x1908192b, 0x08081919, 0x19082b19, 0x08081919, 0x19190808, 0x08081919, 0x1919082b, 0x08081919, + 0x19191919, 0x08081919, 0x19192b08, 0x08081919, 0x192b0819, 0x08081919, 0x192b1908, 0x08081919, + 0x2b080808, 0x08081919, 0x2b08082b, 0x08081919, 0x2b081919, 0x08081919, 0x2b082b08, 0x08081919, + 0x2b190819, 0x08081919, 0x2b191908, 0x08081919, 0x2b2b0808, 0x08081919, 0x08080819, 0x0808192b, + 0x08081908, 0x0808192b, 0x0808192b, 0x0808192b, 0x08082b19, 0x0808192b, 0x08190808, 0x0808192b, + 0x08191919, 0x0808192b, 0x19080808, 0x0808192b, 0x19081919, 0x0808192b, 0x19082b08, 0x0808192b, + 0x19190819, 0x0808192b, 0x19191908, 0x0808192b, 0x192b0808, 0x0808192b, 0x2b080819, 0x0808192b, + 0x2b081908, 0x0808192b, 0x2b190808, 0x0808192b, 0x08080808, 0x08082b08, 0x0808082b, 0x08082b08, + 0x08081919, 0x08082b08, 0x08082b08, 0x08082b08, 0x08190819, 0x08082b08, 0x08191908, 0x08082b08, + 0x0819192b, 0x08082b08, 0x08192b19, 0x08082b08, 0x082b0808, 0x08082b08, 0x082b1919, 0x08082b08, + 0x082b2b2b, 0x08082b08, 0x19080819, 0x08082b08, 0x19081908, 0x08082b08, 0x1908192b, 0x08082b08, + 0x19082b19, 0x08082b08, 0x19190808, 0x08082b08, 0x1919082b, 0x08082b08, 0x19191919, 0x08082b08, + 0x19192b08, 0x08082b08, 0x192b0819, 0x08082b08, 0x192b1908, 0x08082b08, 0x2b080808, 0x08082b08, + 0x2b081919, 0x08082b08, 0x2b191908, 0x08082b08, 0x2b2b2b2b, 0x08082b08, 0x08080819, 0x08082b19, + 0x08081908, 0x08082b19, 0x08190808, 0x08082b19, 0x0819082b, 0x08082b19, 0x08191919, 0x08082b19, + 0x08192b08, 0x08082b19, 0x082b0819, 0x08082b19, 0x19080808, 0x08082b19, 0x19081919, 0x08082b19, + 0x19082b08, 0x08082b19, 0x19190819, 0x08082b19, 0x19191908, 0x08082b19, 0x192b0808, 0x08082b19, + 0x2b080819, 0x08082b19, 0x2b190808, 0x08082b19, 0x08080808, 0x08082b2b, 0x08190819, 0x08082b2b, + 0x08191908, 0x08082b2b, 0x082b082b, 0x08082b2b, 0x082b2b08, 0x08082b2b, 0x082b2b2b, 0x08082b2b, + 0x19190808, 0x08082b2b, 0x2b192b19, 0x08082b2b, 0x08080819, 0x08190808, 0x08081908, 0x08190808, + 0x0808192b, 0x08190808, 0x08082b19, 0x08190808, 0x08190808, 0x08190808, 0x0819082b, 0x08190808, + 0x08191919, 0x08190808, 0x08192b08, 0x08190808, 0x082b0819, 0x08190808, 0x082b1908, 0x08190808, + 0x082b192b, 0x08190808, 0x19080808, 0x08190808, 0x1908082b, 0x08190808, 0x19081919, 0x08190808, + 0x19082b08, 0x08190808, 0x19190819, 0x08190808, 0x19191908, 0x08190808, 0x1919192b, 0x08190808, + 0x19192b19, 0x08190808, 0x192b0808, 0x08190808, 0x192b082b, 0x08190808, 0x192b1919, 0x08190808, + 0x192b2b08, 0x08190808, 0x2b080819, 0x08190808, 0x2b081908, 0x08190808, 0x2b08192b, 0x08190808, + 0x2b190808, 0x08190808, 0x2b191919, 0x08190808, 0x2b192b08, 0x08190808, 0x2b2b0819, 0x08190808, + 0x2b2b1908, 0x08190808, 0x08080808, 0x08190819, 0x0808082b, 0x08190819, 0x08081919, 0x08190819, + 0x08082b08, 0x08190819, 0x08082b2b, 0x08190819, 0x08190819, 0x08190819, 0x08191908, 0x08190819, + 0x0819192b, 0x08190819, 0x08192b19, 0x08190819, 0x082b0808, 0x08190819, 0x082b082b, 0x08190819, + 0x082b1919, 0x08190819, 0x082b2b08, 0x08190819, 0x19080819, 0x08190819, 0x19081908, 0x08190819, + 0x1908192b, 0x08190819, 0x19082b19, 0x08190819, 0x19190808, 0x08190819, 0x1919082b, 0x08190819, + 0x19191919, 0x08190819, 0x19192b08, 0x08190819, 0x192b0819, 0x08190819, 0x192b1908, 0x08190819, + 0x2b080808, 0x08190819, 0x2b08082b, 0x08190819, 0x2b081919, 0x08190819, 0x2b082b08, 0x08190819, + 0x2b190819, 0x08190819, 0x2b191908, 0x08190819, 0x08080819, 0x0819082b, 0x08081908, 0x0819082b, + 0x08082b19, 0x0819082b, 0x08190808, 0x0819082b, 0x08191919, 0x0819082b, 0x082b0819, 0x0819082b, + 0x082b1908, 0x0819082b, 0x19080808, 0x0819082b, 0x19081919, 0x0819082b, 0x19190819, 0x0819082b, + 0x19191908, 0x0819082b, 0x2b080819, 0x0819082b, 0x2b081908, 0x0819082b, 0x2b190808, 0x0819082b, + 0x08080808, 0x08191908, 0x0808082b, 0x08191908, 0x08081919, 0x08191908, 0x08082b08, 0x08191908, + 0x08190819, 0x08191908, 0x08191908, 0x08191908, 0x0819192b, 0x08191908, 0x08192b19, 0x08191908, + 0x082b0808, 0x08191908, 0x082b1919, 0x08191908, 0x082b2b08, 0x08191908, 0x19080819, 0x08191908, + 0x19081908, 0x08191908, 0x1908192b, 0x08191908, 0x19082b19, 0x08191908, 0x19190808, 0x08191908, + 0x1919082b, 0x08191908, 0x19191919, 0x08191908, 0x19192b08, 0x08191908, 0x192b0819, 0x08191908, + 0x192b1908, 0x08191908, 0x2b080808, 0x08191908, 0x2b08082b, 0x08191908, 0x2b081919, 0x08191908, + 0x2b082b08, 0x08191908, 0x2b190819, 0x08191908, 0x2b191908, 0x08191908, 0x2b2b0808, 0x08191908, + 0x08080819, 0x08191919, 0x08081908, 0x08191919, 0x0808192b, 0x08191919, 0x08082b19, 0x08191919, + 0x08190808, 0x08191919, 0x0819082b, 0x08191919, 0x08191919, 0x08191919, 0x08192b08, 0x08191919, + 0x082b0819, 0x08191919, 0x082b1908, 0x08191919, 0x19080808, 0x08191919, 0x1908082b, 0x08191919, + 0x19081919, 0x08191919, 0x19082b08, 0x08191919, 0x19190819, 0x08191919, 0x19191908, 0x08191919, + 0x192b0808, 0x08191919, 0x2b080819, 0x08191919, 0x2b081908, 0x08191919, 0x2b190808, 0x08191919, + 0x08080808, 0x0819192b, 0x08081919, 0x0819192b, 0x08082b08, 0x0819192b, 0x08190819, 0x0819192b, + 0x08191908, 0x0819192b, 0x082b0808, 0x0819192b, 0x19080819, 0x0819192b, 0x19081908, 0x0819192b, + 0x19190808, 0x0819192b, 0x2b080808, 0x0819192b, 0x2b2b2b2b, 0x0819192b, 0x08080819, 0x08192b08, + 0x08081908, 0x08192b08, 0x0808192b, 0x08192b08, 0x08082b19, 0x08192b08, 0x08190808, 0x08192b08, + 0x08191919, 0x08192b08, 0x08192b08, 0x08192b08, 0x082b0819, 0x08192b08, 0x19080808, 0x08192b08, + 0x1908082b, 0x08192b08, 0x19081919, 0x08192b08, 0x19082b08, 0x08192b08, 0x19190819, 0x08192b08, + 0x19191908, 0x08192b08, 0x192b0808, 0x08192b08, 0x2b080819, 0x08192b08, 0x2b081908, 0x08192b08, + 0x08080808, 0x08192b19, 0x0808082b, 0x08192b19, 0x08081919, 0x08192b19, 0x08082b08, 0x08192b19, + 0x08190819, 0x08192b19, 0x08191908, 0x08192b19, 0x082b0808, 0x08192b19, 0x19080819, 0x08192b19, + 0x19081908, 0x08192b19, 0x19190808, 0x08192b19, 0x192b2b19, 0x08192b19, 0x2b2b082b, 0x08192b19, + 0x08081908, 0x08192b2b, 0x08190808, 0x08192b2b, 0x19080808, 0x08192b2b, 0x1919192b, 0x08192b2b, + 0x08080808, 0x082b0808, 0x0808082b, 0x082b0808, 0x08081919, 0x082b0808, 0x08082b08, 0x082b0808, + 0x08190819, 0x082b0808, 0x08191908, 0x082b0808, 0x0819192b, 0x082b0808, 0x08192b19, 0x082b0808, + 0x082b0808, 0x082b0808, 0x082b1919, 0x082b0808, 0x082b2b2b, 0x082b0808, 0x19080819, 0x082b0808, + 0x19081908, 0x082b0808, 0x19190808, 0x082b0808, 0x1919082b, 0x082b0808, 0x19191919, 0x082b0808, + 0x192b1908, 0x082b0808, 0x2b080808, 0x082b0808, 0x2b082b2b, 0x082b0808, 0x2b191908, 0x082b0808, + 0x2b2b2b2b, 0x082b0808, 0x08080819, 0x082b0819, 0x08081908, 0x082b0819, 0x08190808, 0x082b0819, + 0x0819082b, 0x082b0819, 0x08191919, 0x082b0819, 0x082b0819, 0x082b0819, 0x19080808, 0x082b0819, + 0x1908082b, 0x082b0819, 0x19081919, 0x082b0819, 0x19190819, 0x082b0819, 0x19191908, 0x082b0819, + 0x192b0808, 0x082b0819, 0x2b080819, 0x082b0819, 0x2b081908, 0x082b0819, 0x2b190808, 0x082b0819, + 0x08080808, 0x082b082b, 0x08082b2b, 0x082b082b, 0x082b082b, 0x082b082b, 0x082b2b08, 0x082b082b, + 0x082b2b2b, 0x082b082b, 0x19081908, 0x082b082b, 0x19190808, 0x082b082b, 0x2b082b08, 0x082b082b, + 0x2b082b2b, 0x082b082b, 0x2b2b2b08, 0x082b082b, 0x08080819, 0x082b1908, 0x08081908, 0x082b1908, + 0x0808192b, 0x082b1908, 0x08082b19, 0x082b1908, 0x08190808, 0x082b1908, 0x08191919, 0x082b1908, + 0x08192b08, 0x082b1908, 0x082b0819, 0x082b1908, 0x082b1908, 0x082b1908, 0x19080808, 0x082b1908, + 0x1908082b, 0x082b1908, 0x19081919, 0x082b1908, 0x19082b08, 0x082b1908, 0x19190819, 0x082b1908, + 0x19191908, 0x082b1908, 0x192b0808, 0x082b1908, 0x2b080819, 0x082b1908, 0x2b081908, 0x082b1908, + 0x2b190808, 0x082b1908, 0x08080808, 0x082b1919, 0x08081919, 0x082b1919, 0x08082b08, 0x082b1919, + 0x08190819, 0x082b1919, 0x08191908, 0x082b1919, 0x082b0808, 0x082b1919, 0x19080819, 0x082b1919, + 0x19081908, 0x082b1919, 0x19190808, 0x082b1919, 0x192b192b, 0x082b1919, 0x2b080808, 0x082b1919, + 0x08080819, 0x082b192b, 0x08081908, 0x082b192b, 0x08190808, 0x082b192b, 0x19080808, 0x082b192b, + 0x19192b19, 0x082b192b, 0x08080808, 0x082b2b08, 0x08081919, 0x082b2b08, 0x08190819, 0x082b2b08, + 0x08191908, 0x082b2b08, 0x19080819, 0x082b2b08, 0x19081908, 0x082b2b08, 0x19190808, 0x082b2b08, + 0x2b082b2b, 0x082b2b08, 0x2b2b2b2b, 0x082b2b08, 0x08080819, 0x082b2b19, 0x08081908, 0x082b2b19, + 0x08190808, 0x082b2b19, 0x2b191919, 0x082b2b19, 0x08082b2b, 0x082b2b2b, 0x082b082b, 0x082b2b2b, + 0x192b1908, 0x082b2b2b, 0x2b082b08, 0x082b2b2b, 0x2b082b2b, 0x082b2b2b, 0x08080819, 0x19080808, + 0x08081908, 0x19080808, 0x0808192b, 0x19080808, 0x08082b19, 0x19080808, 0x08190808, 0x19080808, + 0x0819082b, 0x19080808, 0x08191919, 0x19080808, 0x08192b08, 0x19080808, 0x08192b2b, 0x19080808, + 0x082b0819, 0x19080808, 0x082b1908, 0x19080808, 0x082b192b, 0x19080808, 0x19080808, 0x19080808, + 0x1908082b, 0x19080808, 0x19081919, 0x19080808, 0x19082b08, 0x19080808, 0x19082b2b, 0x19080808, + 0x19190819, 0x19080808, 0x19191908, 0x19080808, 0x1919192b, 0x19080808, 0x19192b19, 0x19080808, + 0x192b0808, 0x19080808, 0x192b082b, 0x19080808, 0x192b1919, 0x19080808, 0x2b080819, 0x19080808, + 0x2b081908, 0x19080808, 0x2b190808, 0x19080808, 0x2b191919, 0x19080808, 0x2b192b08, 0x19080808, + 0x2b2b0819, 0x19080808, 0x2b2b1908, 0x19080808, 0x08080808, 0x19080819, 0x0808082b, 0x19080819, + 0x08081919, 0x19080819, 0x08082b08, 0x19080819, 0x08190819, 0x19080819, 0x08191908, 0x19080819, + 0x0819192b, 0x19080819, 0x08192b19, 0x19080819, 0x082b0808, 0x19080819, 0x082b082b, 0x19080819, + 0x082b1919, 0x19080819, 0x19080819, 0x19080819, 0x19081908, 0x19080819, 0x1908192b, 0x19080819, + 0x19082b19, 0x19080819, 0x19190808, 0x19080819, 0x1919082b, 0x19080819, 0x19191919, 0x19080819, + 0x19192b08, 0x19080819, 0x192b0819, 0x19080819, 0x192b1908, 0x19080819, 0x2b080808, 0x19080819, + 0x2b08082b, 0x19080819, 0x2b081919, 0x19080819, 0x2b082b08, 0x19080819, 0x2b190819, 0x19080819, + 0x2b191908, 0x19080819, 0x2b2b0808, 0x19080819, 0x08080819, 0x1908082b, 0x08081908, 0x1908082b, + 0x08190808, 0x1908082b, 0x0819082b, 0x1908082b, 0x08191919, 0x1908082b, 0x08192b08, 0x1908082b, + 0x082b1908, 0x1908082b, 0x19080808, 0x1908082b, 0x19081919, 0x1908082b, 0x19082b08, 0x1908082b, + 0x19190819, 0x1908082b, 0x19191908, 0x1908082b, 0x192b0808, 0x1908082b, 0x2b080819, 0x1908082b, + 0x2b081908, 0x1908082b, 0x08080808, 0x19081908, 0x0808082b, 0x19081908, 0x08081919, 0x19081908, + 0x08082b08, 0x19081908, 0x08082b2b, 0x19081908, 0x08190819, 0x19081908, 0x08191908, 0x19081908, + 0x0819192b, 0x19081908, 0x08192b19, 0x19081908, 0x082b0808, 0x19081908, 0x082b082b, 0x19081908, + 0x082b1919, 0x19081908, 0x082b2b08, 0x19081908, 0x19080819, 0x19081908, 0x19081908, 0x19081908, + 0x1908192b, 0x19081908, 0x19082b19, 0x19081908, 0x19190808, 0x19081908, 0x1919082b, 0x19081908, + 0x19191919, 0x19081908, 0x19192b08, 0x19081908, 0x192b0819, 0x19081908, 0x192b1908, 0x19081908, + 0x2b080808, 0x19081908, 0x2b08082b, 0x19081908, 0x2b081919, 0x19081908, 0x2b082b08, 0x19081908, + 0x2b190819, 0x19081908, 0x2b191908, 0x19081908, 0x2b2b0808, 0x19081908, 0x08080819, 0x19081919, + 0x08081908, 0x19081919, 0x0808192b, 0x19081919, 0x08082b19, 0x19081919, 0x08190808, 0x19081919, + 0x0819082b, 0x19081919, 0x08191919, 0x19081919, 0x08192b08, 0x19081919, 0x082b0819, 0x19081919, + 0x082b1908, 0x19081919, 0x19080808, 0x19081919, 0x1908082b, 0x19081919, 0x19081919, 0x19081919, + 0x19082b08, 0x19081919, 0x19190819, 0x19081919, 0x19191908, 0x19081919, 0x192b0808, 0x19081919, + 0x192b2b2b, 0x19081919, 0x2b080819, 0x19081919, 0x2b081908, 0x19081919, 0x2b190808, 0x19081919, + 0x08080808, 0x1908192b, 0x0808082b, 0x1908192b, 0x08081919, 0x1908192b, 0x08082b08, 0x1908192b, + 0x08190819, 0x1908192b, 0x08191908, 0x1908192b, 0x082b0808, 0x1908192b, 0x19080819, 0x1908192b, + 0x19081908, 0x1908192b, 0x19190808, 0x1908192b, 0x2b080808, 0x1908192b, 0x2b2b1919, 0x1908192b, + 0x08080819, 0x19082b08, 0x08081908, 0x19082b08, 0x08082b19, 0x19082b08, 0x08190808, 0x19082b08, + 0x0819082b, 0x19082b08, 0x08191919, 0x19082b08, 0x08192b08, 0x19082b08, 0x082b0819, 0x19082b08, + 0x082b1908, 0x19082b08, 0x19080808, 0x19082b08, 0x1908082b, 0x19082b08, 0x19081919, 0x19082b08, + 0x19082b08, 0x19082b08, 0x19190819, 0x19082b08, 0x19191908, 0x19082b08, 0x192b0808, 0x19082b08, + 0x2b081908, 0x19082b08, 0x2b190808, 0x19082b08, 0x08080808, 0x19082b19, 0x0808082b, 0x19082b19, + 0x08081919, 0x19082b19, 0x08082b08, 0x19082b19, 0x08190819, 0x19082b19, 0x08191908, 0x19082b19, + 0x082b0808, 0x19082b19, 0x19080819, 0x19082b19, 0x19081908, 0x19082b19, 0x19190808, 0x19082b19, + 0x2b080808, 0x19082b19, 0x2b19192b, 0x19082b19, 0x08080819, 0x19082b2b, 0x08081908, 0x19082b2b, + 0x08190808, 0x19082b2b, 0x19080808, 0x19082b2b, 0x08080808, 0x19190808, 0x0808082b, 0x19190808, + 0x08081919, 0x19190808, 0x08082b08, 0x19190808, 0x08190819, 0x19190808, 0x08191908, 0x19190808, + 0x0819192b, 0x19190808, 0x08192b19, 0x19190808, 0x082b0808, 0x19190808, 0x082b082b, 0x19190808, + 0x082b1919, 0x19190808, 0x082b2b08, 0x19190808, 0x19080819, 0x19190808, 0x19081908, 0x19190808, + 0x1908192b, 0x19190808, 0x19082b19, 0x19190808, 0x19190808, 0x19190808, 0x1919082b, 0x19190808, + 0x19191919, 0x19190808, 0x19192b08, 0x19190808, 0x192b0819, 0x19190808, 0x192b1908, 0x19190808, + 0x2b080808, 0x19190808, 0x2b08082b, 0x19190808, 0x2b081919, 0x19190808, 0x2b082b08, 0x19190808, + 0x2b190819, 0x19190808, 0x2b191908, 0x19190808, 0x08080819, 0x19190819, 0x08081908, 0x19190819, + 0x0808192b, 0x19190819, 0x08082b19, 0x19190819, 0x08190808, 0x19190819, 0x0819082b, 0x19190819, + 0x08191919, 0x19190819, 0x08192b08, 0x19190819, 0x082b0819, 0x19190819, 0x082b1908, 0x19190819, + 0x19080808, 0x19190819, 0x1908082b, 0x19190819, 0x19081919, 0x19190819, 0x19082b08, 0x19190819, + 0x19190819, 0x19190819, 0x19191908, 0x19190819, 0x192b0808, 0x19190819, 0x2b080819, 0x19190819, + 0x2b081908, 0x19190819, 0x2b190808, 0x19190819, 0x08080808, 0x1919082b, 0x08081919, 0x1919082b, + 0x08082b08, 0x1919082b, 0x08190819, 0x1919082b, 0x08191908, 0x1919082b, 0x082b0808, 0x1919082b, + 0x19080819, 0x1919082b, 0x19081908, 0x1919082b, 0x19190808, 0x1919082b, 0x192b2b19, 0x1919082b, + 0x2b080808, 0x1919082b, 0x08080819, 0x19191908, 0x08081908, 0x19191908, 0x0808192b, 0x19191908, + 0x08082b19, 0x19191908, 0x08190808, 0x19191908, 0x0819082b, 0x19191908, 0x08191919, 0x19191908, + 0x08192b08, 0x19191908, 0x082b0819, 0x19191908, 0x082b1908, 0x19191908, 0x19080808, 0x19191908, + 0x1908082b, 0x19191908, 0x19081919, 0x19191908, 0x19082b08, 0x19191908, 0x19190819, 0x19191908, + 0x19191908, 0x19191908, 0x192b0808, 0x19191908, 0x2b080819, 0x19191908, 0x2b081908, 0x19191908, + 0x2b190808, 0x19191908, 0x08080808, 0x19191919, 0x0808082b, 0x19191919, 0x08081919, 0x19191919, + 0x08082b08, 0x19191919, 0x08190819, 0x19191919, 0x08191908, 0x19191919, 0x082b0808, 0x19191919, + 0x19080819, 0x19191919, 0x19081908, 0x19191919, 0x19190808, 0x19191919, 0x2b080808, 0x19191919, + 0x08080819, 0x1919192b, 0x08081908, 0x1919192b, 0x08190808, 0x1919192b, 0x082b192b, 0x1919192b, + 0x19080808, 0x1919192b, 0x08080808, 0x19192b08, 0x0808082b, 0x19192b08, 0x08081919, 0x19192b08, + 0x08082b08, 0x19192b08, 0x08190819, 0x19192b08, 0x08191908, 0x19192b08, 0x082b0808, 0x19192b08, + 0x19080819, 0x19192b08, 0x19081908, 0x19192b08, 0x19190808, 0x19192b08, 0x19192b2b, 0x19192b08, + 0x2b080808, 0x19192b08, 0x08080819, 0x19192b19, 0x08081908, 0x19192b19, 0x08190808, 0x19192b19, + 0x19080808, 0x19192b19, 0x08080808, 0x19192b2b, 0x08192b19, 0x19192b2b, 0x2b081919, 0x19192b2b, + 0x2b2b2b08, 0x19192b2b, 0x08080819, 0x192b0808, 0x08081908, 0x192b0808, 0x0808192b, 0x192b0808, + 0x08190808, 0x192b0808, 0x0819082b, 0x192b0808, 0x08191919, 0x192b0808, 0x08192b08, 0x192b0808, + 0x082b0819, 0x192b0808, 0x082b1908, 0x192b0808, 0x19080808, 0x192b0808, 0x19081919, 0x192b0808, + 0x19082b08, 0x192b0808, 0x19190819, 0x192b0808, 0x19191908, 0x192b0808, 0x192b0808, 0x192b0808, + 0x2b081908, 0x192b0808, 0x2b190808, 0x192b0808, 0x08080808, 0x192b0819, 0x0808082b, 0x192b0819, + 0x08081919, 0x192b0819, 0x08082b08, 0x192b0819, 0x08190819, 0x192b0819, 0x08191908, 0x192b0819, + 0x082b0808, 0x192b0819, 0x19080819, 0x192b0819, 0x19081908, 0x192b0819, 0x19190808, 0x192b0819, + 0x2b080808, 0x192b0819, 0x2b192b19, 0x192b0819, 0x08081908, 0x192b082b, 0x08190808, 0x192b082b, + 0x19080808, 0x192b082b, 0x1919192b, 0x192b082b, 0x2b2b0819, 0x192b082b, 0x08080808, 0x192b1908, + 0x08081919, 0x192b1908, 0x08082b08, 0x192b1908, 0x08190819, 0x192b1908, 0x08191908, 0x192b1908, + 0x082b0808, 0x192b1908, 0x19080819, 0x192b1908, 0x19081908, 0x192b1908, 0x19190808, 0x192b1908, + 0x2b080808, 0x192b1908, 0x08080819, 0x192b1919, 0x08081908, 0x192b1919, 0x08190808, 0x192b1919, + 0x19080808, 0x192b1919, 0x19082b2b, 0x192b1919, 0x192b2b08, 0x192b1919, 0x2b19082b, 0x192b1919, + 0x08080808, 0x192b192b, 0x2b191908, 0x192b192b, 0x08080819, 0x192b2b08, 0x08081908, 0x192b2b08, + 0x08190808, 0x192b2b08, 0x192b1919, 0x192b2b08, 0x2b192b08, 0x192b2b08, 0x08080808, 0x192b2b19, + 0x082b2b2b, 0x192b2b19, 0x1908082b, 0x192b2b2b, 0x2b2b0819, 0x192b2b2b, 0x08080808, 0x2b080808, + 0x0808082b, 0x2b080808, 0x08081919, 0x2b080808, 0x08082b08, 0x2b080808, 0x08190819, 0x2b080808, + 0x08191908, 0x2b080808, 0x08192b19, 0x2b080808, 0x082b0808, 0x2b080808, 0x082b1919, 0x2b080808, + 0x19080819, 0x2b080808, 0x19081908, 0x2b080808, 0x19190808, 0x2b080808, 0x1919082b, 0x2b080808, + 0x19191919, 0x2b080808, 0x19192b08, 0x2b080808, 0x192b0819, 0x2b080808, 0x2b080808, 0x2b080808, + 0x2b081919, 0x2b080808, 0x2b190819, 0x2b080808, 0x2b191908, 0x2b080808, 0x08080819, 0x2b080819, + 0x08081908, 0x2b080819, 0x08082b19, 0x2b080819, 0x08190808, 0x2b080819, 0x0819082b, 0x2b080819, + 0x08191919, 0x2b080819, 0x08192b08, 0x2b080819, 0x082b0819, 0x2b080819, 0x082b1908, 0x2b080819, + 0x19080808, 0x2b080819, 0x1908082b, 0x2b080819, 0x19081919, 0x2b080819, 0x19082b08, 0x2b080819, + 0x19190819, 0x2b080819, 0x19191908, 0x2b080819, 0x2b080819, 0x2b080819, 0x2b081908, 0x2b080819, + 0x2b190808, 0x2b080819, 0x2b2b2b19, 0x2b080819, 0x08080808, 0x2b08082b, 0x08081919, 0x2b08082b, + 0x08082b2b, 0x2b08082b, 0x08190819, 0x2b08082b, 0x08191908, 0x2b08082b, 0x19080819, 0x2b08082b, + 0x19081908, 0x2b08082b, 0x19190808, 0x2b08082b, 0x08080819, 0x2b081908, 0x08081908, 0x2b081908, + 0x0808192b, 0x2b081908, 0x08082b19, 0x2b081908, 0x08190808, 0x2b081908, 0x0819082b, 0x2b081908, + 0x08191919, 0x2b081908, 0x08192b08, 0x2b081908, 0x082b0819, 0x2b081908, 0x19080808, 0x2b081908, + 0x1908082b, 0x2b081908, 0x19081919, 0x2b081908, 0x19082b08, 0x2b081908, 0x19190819, 0x2b081908, + 0x19191908, 0x2b081908, 0x192b0808, 0x2b081908, 0x2b080819, 0x2b081908, 0x2b081908, 0x2b081908, + 0x2b190808, 0x2b081908, 0x08080808, 0x2b081919, 0x0808082b, 0x2b081919, 0x08081919, 0x2b081919, + 0x08082b08, 0x2b081919, 0x08190819, 0x2b081919, 0x08191908, 0x2b081919, 0x082b0808, 0x2b081919, + 0x19080819, 0x2b081919, 0x19081908, 0x2b081919, 0x19190808, 0x2b081919, 0x2b080808, 0x2b081919, + 0x2b082b2b, 0x2b081919, 0x08080819, 0x2b08192b, 0x08081908, 0x2b08192b, 0x08190808, 0x2b08192b, + 0x082b2b19, 0x2b08192b, 0x19080808, 0x2b08192b, 0x08080808, 0x2b082b08, 0x08081919, 0x2b082b08, + 0x08190819, 0x2b082b08, 0x08191908, 0x2b082b08, 0x19080819, 0x2b082b08, 0x19081908, 0x2b082b08, + 0x19190808, 0x2b082b08, 0x2b2b082b, 0x2b082b08, 0x08080819, 0x2b082b19, 0x08081908, 0x2b082b19, + 0x19080808, 0x2b082b19, 0x192b1919, 0x2b082b19, 0x082b082b, 0x2b082b2b, 0x19192b08, 0x2b082b2b, + 0x19192b2b, 0x2b082b2b, 0x2b08082b, 0x2b082b2b, 0x2b2b082b, 0x2b082b2b, 0x08080819, 0x2b190808, + 0x08081908, 0x2b190808, 0x08082b19, 0x2b190808, 0x08190808, 0x2b190808, 0x0819082b, 0x2b190808, + 0x08191919, 0x2b190808, 0x08192b08, 0x2b190808, 0x082b1908, 0x2b190808, 0x19080808, 0x2b190808, + 0x1908082b, 0x2b190808, 0x19081919, 0x2b190808, 0x19082b08, 0x2b190808, 0x19190819, 0x2b190808, + 0x19191908, 0x2b190808, 0x192b0808, 0x2b190808, 0x2b080819, 0x2b190808, 0x2b081908, 0x2b190808, + 0x2b190808, 0x2b190808, 0x08080808, 0x2b190819, 0x08081919, 0x2b190819, 0x08190819, 0x2b190819, + 0x08191908, 0x2b190819, 0x19080819, 0x2b190819, 0x19081908, 0x2b190819, 0x19190808, 0x2b190819, + 0x19192b2b, 0x2b190819, 0x08080819, 0x2b19082b, 0x08081908, 0x2b19082b, 0x08190808, 0x2b19082b, + 0x19080808, 0x2b19082b, 0x2b2b192b, 0x2b19082b, 0x08080808, 0x2b191908, 0x0808082b, 0x2b191908, + 0x08081919, 0x2b191908, 0x08082b08, 0x2b191908, 0x08190819, 0x2b191908, 0x08191908, 0x2b191908, + 0x082b0808, 0x2b191908, 0x19080819, 0x2b191908, 0x19081908, 0x2b191908, 0x19190808, 0x2b191908, + 0x2b080808, 0x2b191908, 0x2b19192b, 0x2b191908, 0x08080819, 0x2b191919, 0x08081908, 0x2b191919, + 0x08190808, 0x2b191919, 0x19080808, 0x2b191919, 0x2b192b08, 0x2b191919, 0x2b2b0819, 0x2b191919, + 0x08080808, 0x2b19192b, 0x1908192b, 0x2b19192b, 0x192b1908, 0x2b19192b, 0x08080819, 0x2b192b08, + 0x08081908, 0x2b192b08, 0x08190808, 0x2b192b08, 0x082b192b, 0x2b192b08, 0x19080808, 0x2b192b08, + 0x2b2b2b19, 0x2b192b08, 0x08080808, 0x2b192b19, 0x19082b19, 0x2b192b19, 0x1919082b, 0x2b192b19, + 0x2b190808, 0x2b192b2b, 0x08080808, 0x2b2b0808, 0x08081919, 0x2b2b0808, 0x08082b2b, 0x2b2b0808, + 0x08191908, 0x2b2b0808, 0x082b082b, 0x2b2b0808, 0x082b2b2b, 0x2b2b0808, 0x19080819, 0x2b2b0808, + 0x19081908, 0x2b2b0808, 0x19190808, 0x2b2b0808, 0x2b2b082b, 0x2b2b0808, 0x2b2b2b2b, 0x2b2b0808, + 0x19080808, 0x2b2b0819, 0x192b1919, 0x2b2b0819, 0x0808082b, 0x2b2b082b, 0x08082b2b, 0x2b2b082b, + 0x082b082b, 0x2b2b082b, 0x082b2b08, 0x2b2b082b, 0x082b2b2b, 0x2b2b082b, 0x2b08082b, 0x2b2b082b, + 0x2b082b08, 0x2b2b082b, 0x2b082b2b, 0x2b2b082b, 0x2b2b2b08, 0x2b2b082b, 0x08080819, 0x2b2b1908, + 0x08081908, 0x2b2b1908, 0x08190808, 0x2b2b1908, 0x19080808, 0x2b2b1908, 0x2b082b19, 0x2b2b1908, + 0x2b2b1908, 0x2b2b1908, 0x08080808, 0x2b2b1919, 0x08192b19, 0x2b2b1919, 0x19190819, 0x2b2b192b, + 0x08082b2b, 0x2b2b2b08, 0x082b2b08, 0x2b2b2b08, 0x2b2b082b, 0x2b2b2b08, 0x19191908, 0x2b2b2b19, + 0x2b08192b, 0x2b2b2b19, 0x08082b08, 0x2b2b2b2b, 0x08082b2b, 0x2b2b2b2b, 0x082b0808, 0x2b2b2b2b, + 0x082b082b, 0x2b2b2b2b, 0x082b2b08, 0x2b2b2b2b, 0x2b082b08, 0x2b2b2b2b, 0x2b2b2b2b, 0x2b2b2b2b) + return tbl[i] +} + // the HALVED iq3xxs grid (ggml's iq3xxs_grid with every byte / 2 - the plane convention that // absorbs the format's 0.25 into the stored d), one word per call; staging loops copy it into // workgroup memory (QUIRK 20's slow path stays off the hot loop). @@ -3260,6 +3524,100 @@ class KqBatchK2 : KqBatchK6 { } } +// IQ2_S tile: the iq3s grid gather widened to the u64 grid (a 10-bit qs|qh index picks TWO +// words - eight magnitudes) with the block's own sign bytes; the per-16 UNSIGNED strips ride +// k6's wsw/wsw2 planes with a plain split fma (no offset, no min term). The 8 KB grid stages +// once, 8 words per thread. +[vk_dispatch(name = "kq_batch_iq2s_cls", kernel = "run", family = "kq_batch_cls", grid = "wgs", params = "wgs : int64")] +class KqBatchIq2s : KqBatchK6 { + @workgroup gridw : uint[2048] + + def iq3s_signed(g, nib : uint) : uint { + let m1 = (nib * 0x00204081u) & 0x01010101u + let mask = m1 * 255u + return (g ^ mask) + m1 + } + + def override stage_w(wsb0, wcol, nsb, s, word, grp : uint) { + for (ki in range(2)) { + let k = word + uint(ki) * 8u + let bu = k / 2u + let hh = k % 2u + var wv = uint4(0u) + if (wcol < pa.d) { + let wsb = wsb0 + wcol * nsb + s + let qsw = wq[wsb * 18u + bu] + let sgw = wq[wsb * 18u + 8u + bu] + let qh = (wq[wsb * 18u + 16u + bu / 4u] >> ((bu % 4u) * 8u)) & 255u + let l0 = hh * 2u + let sg0 = (sgw >> (l0 * 8u)) & 255u + let sg1 = (sgw >> (l0 * 8u + 8u)) & 255u + let i0 = (((qsw >> (l0 * 8u)) & 255u) | ((qh << (8u - 2u * l0)) & 0x300u)) * 2u + let i1 = (((qsw >> (l0 * 8u + 8u)) & 255u) | ((qh << (8u - 2u * (l0 + 1u))) & 0x300u)) * 2u + wv = uint4(iq3s_signed(gridw[i0], sg0 & 15u), iq3s_signed(gridw[i0 + 1u], sg0 >> 4u), + iq3s_signed(gridw[i1], sg1 & 15u), iq3s_signed(gridw[i1 + 1u], sg1 >> 4u)) + } + ww4[grp * 17u + k] = wv + } + } + + def override stage_ws(wsb0, wt, nsb, s, tid : uint) { + let slane = tid % 32u + let sblk = tid / 32u + var sa = 0.0 + var sbv = 0.0 + if (wt * 32u + slane < pa.d) { + let wsb = wsb0 + (wt * 32u + slane) * nsb + s + let w = wsu[wsb * 5u + 1u + sblk / 2u] + let s0 = (w >> ((sblk % 2u) * 16u)) & 255u + let s1 = (w >> ((sblk % 2u) * 16u + 8u)) & 255u + let dd = unpackHalf2x16(wsu[wsb * 5u]).x + sa = dd * float(s0) + sbv = dd * float(s1) + } + wsw[sblk * 32u + slane] = sa + wsw2[sblk * 32u + slane] = sbv + } + + // the split-half fma with UNSIGNED strips: xscl * (d*s0*l + d*s1*h) - no offset, no min + def override blk_fma(bu, grp, word : uint; xscl : float) : float4 { + let kL = bu * 2u + let kH = bu * 2u + 1u + let xwL = xw4[grp * 17u + kL] + let xwH = xw4[grp * 17u + kH] + let wL0 = ww4[word * 17u + kL] + let wL1 = ww4[(word + 8u) * 17u + kL] + let wL2 = ww4[(word + 16u) * 17u + kL] + let wL3 = ww4[(word + 24u) * 17u + kL] + let wH0 = ww4[word * 17u + kH] + let wH1 = ww4[(word + 8u) * 17u + kH] + let wH2 = ww4[(word + 16u) * 17u + kH] + let wH3 = ww4[(word + 24u) * 17u + kH] + let l0 = sdot4(wL0.x, xwL.x) + sdot4(wL0.y, xwL.y) + sdot4(wL0.z, xwL.z) + sdot4(wL0.w, xwL.w) + let l1 = sdot4(wL1.x, xwL.x) + sdot4(wL1.y, xwL.y) + sdot4(wL1.z, xwL.z) + sdot4(wL1.w, xwL.w) + let l2 = sdot4(wL2.x, xwL.x) + sdot4(wL2.y, xwL.y) + sdot4(wL2.z, xwL.z) + sdot4(wL2.w, xwL.w) + let l3 = sdot4(wL3.x, xwL.x) + sdot4(wL3.y, xwL.y) + sdot4(wL3.z, xwL.z) + sdot4(wL3.w, xwL.w) + let h0 = sdot4(wH0.x, xwH.x) + sdot4(wH0.y, xwH.y) + sdot4(wH0.z, xwH.z) + sdot4(wH0.w, xwH.w) + let h1 = sdot4(wH1.x, xwH.x) + sdot4(wH1.y, xwH.y) + sdot4(wH1.z, xwH.z) + sdot4(wH1.w, xwH.w) + let h2 = sdot4(wH2.x, xwH.x) + sdot4(wH2.y, xwH.y) + sdot4(wH2.z, xwH.z) + sdot4(wH2.w, xwH.w) + let h3 = sdot4(wH3.x, xwH.x) + sdot4(wH3.y, xwH.y) + sdot4(wH3.z, xwH.z) + sdot4(wH3.w, xwH.w) + return float4(xscl * (wsw[bu * 32u + word] * float(l0) + wsw2[bu * 32u + word] * float(h0)), + xscl * (wsw[bu * 32u + word + 8u] * float(l1) + wsw2[bu * 32u + word + 8u] * float(h1)), + xscl * (wsw[bu * 32u + word + 16u] * float(l2) + wsw2[bu * 32u + word + 16u] * float(h2)), + xscl * (wsw[bu * 32u + word + 24u] * float(l3) + wsw2[bu * 32u + word + 24u] * float(h3))) + } + + [spirv_kernel(local_size_x = 256, name = "kq_batch_iq2s_cls_spv")] + def override run { + let tid = gl_LocalInvocationID.x + for (k in range(8)) { + gridw[tid + uint(k) * 256u] = iq2s_grid_word(int(tid + uint(k) * 256u)) + } + barrier() // the grid visible before the first stage_w + tile_shell() + } +} + // ===== the prefill batch GEMMs — cooperative-matrix variants (DASLLAMA_COOPMAT=f16|int8|mm) ===== let private CML_SST = 20u // mm shared row stride in uints (16 f16-pair data + 4 pad — bank spread) @@ -4080,6 +4438,12 @@ struct VkK2Blk { qs : int16[32] // one Q2_K superblock's 64 qs bytes, four 2-bit lanes each; scales ([f16 d][f16 dmin][16 sc/min pair bytes]) live in ws } +struct VkIq2sBlk { + qs : int16[16] // one IQ2_S superblock's 32 grid-index bytes as 16-bit lanes + sg : int16[16] // ... its 32 sign bytes; scales (f16 d8 + 16 unsigned (1 + 2s) strips) live in ws + qh : int16[4] // ... and its 8 qh bytes (two index high bits per (block, l) window) +} + // The cm2 tile: one class template over the two stamp axes. The token column BN - l 256, m 128, // s 32 (the expert-bucket geometry: STILE's fast path loads a partial column UNCLAMPED and clamps // only the store, so every plane it reads carries 32 rows of slack) - and the weight format: BLK @@ -4103,9 +4467,11 @@ class template KqCm2BatchT : MoeCmBase { @workgroup @template_gate = IQ3GRID iq3s_gridc : uint[512] @template_constant IQ3XGRID : bool = false // stage the 1 KB halved iq3xxs grid into workgroup memory @workgroup @template_gate = IQ3XGRID iq3x_gridc : uint[256] + @template_constant IQ2SGRID : bool = false // stage the 8 KB iq2s u64 grid into workgroup memory + @workgroup @template_gate = IQ2SGRID iq2s_gridc : uint[2048] [spirv_kernel(local_size_x = 256), arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-tile-pick-and-default")] - def run { // nolint:STYLE038 — the fast/edge path pair, barrier- and register-coupled + def run { // nolint:STYLE037,STYLE038 — the fast/edge path pair, barrier- and register-coupled let reg = region_rec() let wblk0 = reg.x let row0 = reg.y @@ -4148,6 +4514,12 @@ class template KqCm2BatchT : MoeCmBase { static_if (IQ3XGRID) { iq3x_gridc[gl_LocalInvocationID.x] = iq3xxs_grid_word(int(gl_LocalInvocationID.x)) } + static_if (IQ2SGRID) { + let g2 = gl_LocalInvocationID.x + for (k in range(8)) { + iq2s_gridc[g2 + uint(k) * 256u] = iq2s_grid_word(int(g2 + uint(k) * 256u)) + } + } barrier() // wg_blk0 (and the gated codebook) visible before the first decode load var a : coopmatWgA_f16_128x64 var b : BT @@ -4518,6 +4890,33 @@ class template Iq4nlCm2T : KqCm2BatchT { } } +[ |> template_struct_instance] +class template Iq2sCm2T : KqCm2BatchT { + typedef BLK = VkIq2sBlk + typedef ST = uint + override IQ2SGRID = true + + [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] + def decode(blk : VkIq2sBlk; bc, cib : uint2) : float16 { + let e = cib.y + let g = e >> 5u + let r = e & 31u + let l = r >> 3u + let j = r & 7u + let qi = g * 4u + l + let qb = uint(int(unpack8(blk.qs[int(qi >> 1u)])[int(qi & 1u)])) & 0xFFu + let qh = uint(int(unpack8(blk.qh[int(g >> 1u)])[int(g & 1u)])) & 0xFFu + let sgb = uint(int(unpack8(blk.sg[int(qi >> 1u)])[int(qi & 1u)])) & 0xFFu + let gw = iq2s_gridc[(qb | ((qh << (8u - 2u * l)) & 0x300u)) * 2u + (j >> 2u)] + let gb = float((gw >> ((j & 3u) * 8u)) & 0xFFu) + let srow = (wg_blk0 + bc.x * (pa.n >> 8u) + bc.y) * 5u + let d = unpackHalf2x16(ws[srow]).x + let si = g * 2u + (r >> 4u) + let sc = (ws[srow + 1u + (si >> 2u)] >> ((si & 3u) * 8u)) & 0xFFu + return float16(d * float(sc) * (((sgb >> j) & 1u) != 0u ? -gb : gb)) + } +} + // the stamps: the three width columns of each format (the dispatcher keys on (fmt, tc)) [vk_dispatch(name = "q8_batch_cm2l_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] class Q8Cm2LBatch : Q8Cm2T { @@ -4816,6 +5215,33 @@ class K2Cm2SBatch : K2Cm2T { typedef FLO = tensorLayout2DPad } +[vk_dispatch(name = "kq_batch_iq2s_cm2l_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Iq2sCm2LBatch : Iq2sCm2T { + typedef BT = coopmatWgB_f16_64x256 + typedef ACC = coopmatWgAcc_f16_128x256 + typedef ACCW = coopmatWgAcc_f32_128x256 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_iq2s_cm2m_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Iq2sCm2MBatch : Iq2sCm2T { + override BN = 128u + typedef BT = coopmatWgB_f16_64x128 + typedef ACC = coopmatWgAcc_f16_128x128 + typedef ACCW = coopmatWgAcc_f32_128x128 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_iq2s_cm2s_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Iq2sCm2SBatch : Iq2sCm2T { + override BN = 32u + override STILE = true + typedef BT = coopmatWgB_f16_64x32 + typedef ACC = coopmatWgAcc_f16_128x32 + typedef ACCW = coopmatWgAcc_f32_128x32 + typedef FLO = tensorLayout2DPad +} + // ===== the split-k reduce (sums the cm2 partial planes into y) ===== struct SkRedArgs { @@ -5393,6 +5819,8 @@ def kq_batch_cls_ensure(fmt : int; q40cm : bool) : bool { return ensure_kq_batch_iq4nl_cls() } elif (fmt == int(KqFmt.k2)) { return ensure_kq_batch_k2_cls() + } elif (fmt == int(KqFmt.iq2s)) { + return ensure_kq_batch_iq2s_cls() } panic("dasLLAMA vulkan tier: no kq batch class for format {fmt}") return false @@ -5431,6 +5859,8 @@ def kq_batch_cls_enc_for(fmt : int; q40cm : bool; raw : VkCommandBuffer; var h : enc_kq_batch_iq4nl_cls(raw, h, set_, pc, wgs) } elif (fmt == int(KqFmt.k2)) { enc_kq_batch_k2_cls(raw, h, set_, pc, wgs) + } elif (fmt == int(KqFmt.iq2s)) { + enc_kq_batch_iq2s_cls(raw, h, set_, pc, wgs) } else { panic("dasLLAMA vulkan tier: no kq batch class for format {fmt}") } @@ -5463,6 +5893,8 @@ def gemv_cls_ensure(fmt : int) : bool { return ensure_kq_gemv_iq4nl_cls() } elif (fmt == int(KqFmt.k2)) { return ensure_kq_gemv_k2_cls() + } elif (fmt == int(KqFmt.iq2s)) { + return ensure_kq_gemv_iq2s_cls() } panic("dasLLAMA vulkan tier: no GEMV class for format {fmt}") return false @@ -5501,6 +5933,8 @@ def gemv_cls_enc(fmt : int; raw : VkCommandBuffer; var h : VkHaz; var set_ : VkD enc_kq_gemv_iq4nl_cls(raw, h, set_, pc, wgs) } elif (fmt == int(KqFmt.k2)) { enc_kq_gemv_k2_cls(raw, h, set_, pc, wgs) + } elif (fmt == int(KqFmt.iq2s)) { + enc_kq_gemv_iq2s_cls(raw, h, set_, pc, wgs) } else { panic("dasLLAMA vulkan tier: no GEMV class for format {fmt}") } @@ -5852,3 +6286,59 @@ class KqGemvK6 : KqGemvBase { gemv_shell() } } + +// IQ2_S: the iq3s gather widened to the u64 grid - a 10-bit qs|qh index picks TWO grid words +// (eight magnitudes {8, 25, 43}), the block's own sign bytes flip them; per-16 UNSIGNED +// strips (1 + 2s) keep split lo/hi dots under the eighth-ed d. The 8 KB grid stages ahead. +[vk_dispatch(name = "kq_gemv_iq2s_cls", kernel = "run", family = "kq_gemv_cls", grid = "wgs", params = "wgs : int64")] +class KqGemvIq2s : KqGemvBase { + @workgroup gridw : uint[2048] + + def iq3s_signed(g, nib : uint) : uint { + let m1 = (nib * 0x00204081u) & 0x01010101u + let mask = m1 * 255u + return (g ^ mask) + m1 + } + + def override blk_contrib(wsb, blk, xsb : uint) : float { + let qw = wq[wsb * 18u + blk] + let sgw = wq[wsb * 18u + 8u + blk] + let qh = (wq[wsb * 18u + 16u + blk / 4u] >> ((blk % 4u) * 8u)) & 255u + let xb4 = xsb * 16u + blk * 2u + let xl = xq4[xb4] + let xh = xq4[xb4 + 1u] + let sw = wsu[wsb * 5u + 1u + blk / 2u] + let s0 = int((sw >> ((blk % 2u) * 16u)) & 255u) + let s1 = int((sw >> ((blk % 2u) * 16u + 8u)) & 255u) + var isum = 0 + for (h in range(2)) { + let l0 = uint(h) * 2u + let sg0 = (sgw >> (l0 * 8u)) & 255u + let sg1 = (sgw >> (l0 * 8u + 8u)) & 255u + let i0 = (((qw >> (l0 * 8u)) & 255u) | ((qh << (8u - 2u * l0)) & 0x300u)) * 2u + let i1 = (((qw >> (l0 * 8u + 8u)) & 255u) | ((qh << (8u - 2u * (l0 + 1u))) & 0x300u)) * 2u + let w0 = iq3s_signed(gridw[i0], sg0 & 15u) + let w1 = iq3s_signed(gridw[i0 + 1u], sg0 >> 4u) + let w2 = iq3s_signed(gridw[i1], sg1 & 15u) + let w3 = iq3s_signed(gridw[i1 + 1u], sg1 >> 4u) + if (h == 0) { + isum += s0 * (sdot4(w0, xl.x) + sdot4(w1, xl.y) + sdot4(w2, xl.z) + sdot4(w3, xl.w)) + } else { + isum += s1 * (sdot4(w0, xh.x) + sdot4(w1, xh.y) + sdot4(w2, xh.z) + sdot4(w3, xh.w)) + } + } + let dd = unpackHalf2x16(wsu[wsb * 5u]).x + return axs[xsb] * dd * float(isum) + } + + [spirv_kernel(local_size_x = 64, name = "kq_gemv_iq2s_cls_spv")] + def run { + let lid = gl_LocalInvocationID.x + for (k in range(32)) { + let idx = lid + uint(k) * 64u + gridw[idx] = iq2s_grid_word(int(idx)) + } + barrier() // the grid visible before the shell's block walk + gemv_shell() + } +} diff --git a/modules/dasLLAMA/dasllama/dasllama_vulkan_common.das b/modules/dasLLAMA/dasllama/dasllama_vulkan_common.das index d83b7f48b7..edaba85d16 100644 --- a/modules/dasLLAMA/dasllama/dasllama_vulkan_common.das +++ b/modules/dasLLAMA/dasllama/dasllama_vulkan_common.das @@ -1752,6 +1752,7 @@ def vk_kq_schema_id(fmt : int) : int { if (fmt == 9) return 34 if (fmt == 10) return 45 if (fmt == 11) return 2 + if (fmt == 12) return 23 panic("vk_kq_schema_id: not a kq stack fmt") return 0 } diff --git a/modules/dasLLAMA/dasllama/dasllama_vulkan_prefill.das b/modules/dasLLAMA/dasllama/dasllama_vulkan_prefill.das index d88d3df400..df065fadac 100644 --- a/modules/dasLLAMA/dasllama/dasllama_vulkan_prefill.das +++ b/modules/dasLLAMA/dasllama/dasllama_vulkan_prefill.das @@ -88,6 +88,9 @@ def private cm2_cls_ensure(fmt : int; tc : int64) : bool { if (fmt == int(KqFmt.k2)) { return tc == 256l ? ensure_kq_batch_k2_cm2l_cls() : (tc == 128l ? ensure_kq_batch_k2_cm2m_cls() : ensure_kq_batch_k2_cm2s_cls()) } + if (fmt == int(KqFmt.iq2s)) { + return tc == 256l ? ensure_kq_batch_iq2s_cm2l_cls() : (tc == 128l ? ensure_kq_batch_iq2s_cm2m_cls() : ensure_kq_batch_iq2s_cm2s_cls()) + } verify(fmt == int(KqFmt.q8), "vk prefill: the cm2 ensure fell through for a kq format - a format the feed admits must own a tile arm") return tc == 256l ? ensure_q8_batch_cm2l_cls() : (tc == 128l ? ensure_q8_batch_cm2m_cls() : ensure_q8_batch_cm2s_cls()) } @@ -154,6 +157,12 @@ def private cm2_cls_set(fmt : int; tc : int64; bufs : uint64 const[5]; sizes : i } return tc == 128l ? set_kq_batch_k2_cm2m_cls(bufs, sizes, gbits) : set_kq_batch_k2_cm2s_cls(bufs, sizes, gbits) } + if (fmt == int(KqFmt.iq2s)) { + if (tc == 256l) { + return set_kq_batch_iq2s_cm2l_cls(bufs, sizes, gbits) + } + return tc == 128l ? set_kq_batch_iq2s_cm2m_cls(bufs, sizes, gbits) : set_kq_batch_iq2s_cm2s_cls(bufs, sizes, gbits) + } verify(fmt == int(KqFmt.q8), "vk prefill: the cm2 set fell through for a kq format - a format the feed admits must own a tile arm") if (tc == 256l) { return set_q8_batch_cm2l_cls(bufs, sizes, gbits) @@ -243,6 +252,14 @@ def private cm2_cls_enc(fmt : int; tc : int64; raw : VkCommandBuffer; var h : Vk } else { enc_kq_batch_k2_cm2s_cls(raw, h, s, pc, groups) } + } elif (fmt == int(KqFmt.iq2s)) { + if (tc == 256l) { + enc_kq_batch_iq2s_cm2l_cls(raw, h, s, pc, groups) + } elif (tc == 128l) { + enc_kq_batch_iq2s_cm2m_cls(raw, h, s, pc, groups) + } else { + enc_kq_batch_iq2s_cm2s_cls(raw, h, s, pc, groups) + } } elif (tc == 256l) { verify(fmt == int(KqFmt.q8), "vk prefill: the cm2 enc fell through for a kq format - a format the feed admits must own a tile arm") enc_q8_batch_cm2l_cls(raw, h, s, pc, groups) diff --git a/modules/dasLLAMA/tests/_vkd_oracles.das b/modules/dasLLAMA/tests/_vkd_oracles.das index ab189c1c78..2e4737f90e 100644 --- a/modules/dasLLAMA/tests/_vkd_oracles.das +++ b/modules/dasLLAMA/tests/_vkd_oracles.das @@ -189,6 +189,12 @@ def kq_cls_ref(fmt : int; wq, ws, xq : array; xs : array) : KqGemvB k = new KqGemvIq4nl() // pure-ALU codebook - no workgroup state to fill } elif (fmt == int(KqFmt.k2)) { k = new KqGemvK2() // pure ALU - no workgroup state to fill + } elif (fmt == int(KqFmt.iq2s)) { + var k2s = new KqGemvIq2s() + for (i in range(2048)) { // same replay rule: the u64 grid's word pairs fill here + k2s.gridw[i] = IQ2S_GRID[i] + } + k = k2s } else { k = new KqGemvK6() } @@ -373,6 +379,47 @@ def iq3xxs_gemv_float_oracle(wq, ws, xq : array; xs : array; return <- y } +// IQ2_S GEMV float oracle, INDEPENDENT of the class bit-math: w = d8 * (1 + 2s) * grid_byte * +// sign straight off the plane bytes ([idx 32][signs 32][qh 8], the k4-shaped scale row with +// UNSIGNED per-16 strips and the pre-eighth-ed d). +def iq2s_gemv_float_oracle(wq, ws, xq : array; xs : array; + regions : array; n, d, nreg : int) : array { + var y : array + y |> resize(nreg * d) + let nsb = n / 256 + for (r in range(nreg)) { + for (row in range(d)) { + let w0 = int(regions[r * 2]) + row * nsb + let x0 = int(regions[r * 2 + 1]) + var acc = 0.0 + for (s in range(nsb)) { + let sb = w0 + s + let xsb = x0 + s + let dd = half_at(ws, sb * 10) + for (blk in range(8)) { + let qh = byte_u8(wq, sb * 72 + 64 + blk) + for (hh in range(2)) { + let sc = float(byte_u8(ws, sb * 20 + 4 + blk * 2 + hh)) + var bacc = 0.0 + for (e in range(16)) { + let ee = hh * 16 + e + let l = ee / 8 + let hj = ee % 8 + let qi = (byte_u8(wq, sb * 72 + blk * 4 + l) | ((qh << (8 - 2 * l)) & 0x300)) * 2 + hj / 4 + let g = float(int(IQ2S_GRID[qi] >> uint(8 * (hj % 4))) & 0xFF) + let sgn = ((byte_u8(wq, sb * 72 + 32 + blk * 4 + l) >> hj) & 1) != 0 ? -1.0 : 1.0 + bacc += g * sgn * float(byte_s8(xq, xsb * 256 + blk * 32 + ee)) + } + acc += xs[xsb] * dd * sc * bacc + } + } + } + y[r * d + row] = acc + } + } + return <- y +} + // IQ3_S f16-feed GEMM reference over the device form: grid gather + sign flip per element, // f16-rounded per weight like its siblings ([qs 64][qh 8][signs 32] rows, the k4-shaped // scale row with signed (1 + 2s) sub-scales). @@ -441,6 +488,42 @@ def iq3xxsf16_gemm_oracle(wq : array; wsu : array; xf : array; } } +// IQ2_S f16-feed GEMM reference over the device form: u64-grid gather (two words per 10-bit +// index) + the block's own sign bytes, f16-rounded per weight like its siblings ([idx 32] +// [signs 32][qh 8] rows, the k4-shaped scale row with UNSIGNED per-16 strips, d pre-eighth-ed). +def iq2sf16_gemm_oracle(wq : array; wsu : array; xf : array; + recs : array; nrec, n, d : int; var y : array) { + let nsb = n / 256 + for (rr in range(nrec)) { + let wsb0 = int(recs[rr * 4]) + let row0 = int(recs[rr * 4 + 1]) + let cnt = int(recs[rr * 4 + 2]) + for (r in range(cnt)) { + for (c in range(d)) { + var acc = 0.0 + for (s in range(nsb)) { + let sb = wsb0 + c * nsb + s + let dv = unpackHalf2x16(wsu[sb * 5]).x + for (g in range(8)) { + let qh = byte_u8(wq, sb * 72 + 64 + g) + for (e in range(32)) { + let l = e / 8 + let si = g * 2 + e / 16 + let sc = float((wsu[sb * 5 + 1 + si / 4] >> uint((si % 4) * 8)) & 0xFFu) + let qi = (byte_u8(wq, sb * 72 + g * 4 + l) | ((qh << (8 - 2 * l)) & 0x300)) * 2 + (e % 8) / 4 + let gb = float(int(IQ2S_GRID[qi] >> uint(8 * (e % 4))) & 0xFF) + let sbit = (byte_u8(wq, sb * 72 + 32 + g * 4 + l) >> (e % 8)) & 1 + let w = float(float16(dv * sc * (sbit != 0 ? -gb : gb))) + acc += w * half_at(xf, (row0 + r) * n + s * 256 + g * 32 + e) + } + } + } + y[(row0 + r) * d + c] = acc + } + } + } +} + // IQ4_NL f16-feed GEMM reference over the device form: q40's walk with the codebook in place // of (q - 8), f16-rounded per weight like its siblings (q40's 32-word rows + 8 f16 d). def iq4nlf16_gemm_oracle(wq : array; wsu : array; xf : array; diff --git a/modules/dasLLAMA/tests/test_vulkan_kernels.das b/modules/dasLLAMA/tests/test_vulkan_kernels.das index 47c84070c8..b4daf4e4a0 100644 --- a/modules/dasLLAMA/tests/test_vulkan_kernels.das +++ b/modules/dasLLAMA/tests/test_vulkan_kernels.das @@ -381,7 +381,7 @@ def private attn_row_oracle(q : array; qbase : int; kk : array; kb def test_vkd_kq_gemv_family(t0 : T?) { t0 |> run("kq GEMV class family == the CPU oracle (all 6 formats)") <| @(t : T?) { static_if (typeinfo builtin_module_exists(vulkan)) { - if (!(ensure_kq_gemv_k4_cls() && ensure_kq_gemv_q40_cls() && ensure_kq_gemv_k5_cls() && ensure_kq_gemv_k6_cls() && ensure_kq_gemv_iq4xs_cls() && ensure_kq_gemv_k3_cls() && ensure_kq_gemv_iq3s_cls() && ensure_kq_gemv_iq3xxs_cls() && ensure_kq_gemv_iq4nl_cls() && ensure_kq_gemv_k2_cls())) { + if (!(ensure_kq_gemv_k4_cls() && ensure_kq_gemv_q40_cls() && ensure_kq_gemv_k5_cls() && ensure_kq_gemv_k6_cls() && ensure_kq_gemv_iq4xs_cls() && ensure_kq_gemv_k3_cls() && ensure_kq_gemv_iq3s_cls() && ensure_kq_gemv_iq3xxs_cls() && ensure_kq_gemv_iq4nl_cls() && ensure_kq_gemv_k2_cls() && ensure_kq_gemv_iq2s_cls())) { feint("no Vulkan device - skipping\n") return } @@ -389,9 +389,9 @@ def test_vkd_kq_gemv_family(t0 : T?) { let d = 3 // rows per region let nreg = 2 let rows = nreg * d - let nfmt = 10 - let fmts = fixed_array(int(KqFmt.k4), int(KqFmt.q40), int(KqFmt.k5), int(KqFmt.k6), int(KqFmt.iq4xs), int(KqFmt.k3), int(KqFmt.iq3s), int(KqFmt.iq3xxs), int(KqFmt.iq4nl), int(KqFmt.k2)) - let stride4s = fixed_array(32, 32, 40, 48, 32, 24, 26, 24, 32, 16) // uint WORDS per superblock (iq3s/iq3xxs are not uint4-aligned) + let nfmt = 11 + let fmts = fixed_array(int(KqFmt.k4), int(KqFmt.q40), int(KqFmt.k5), int(KqFmt.k6), int(KqFmt.iq4xs), int(KqFmt.k3), int(KqFmt.iq3s), int(KqFmt.iq3xxs), int(KqFmt.iq4nl), int(KqFmt.k2), int(KqFmt.iq2s)) + let stride4s = fixed_array(32, 32, 40, 48, 32, 24, 26, 24, 32, 16, 18) // uint WORDS per superblock (iq3s/iq3xxs/iq2s are not uint4-aligned) for (fi in range(nfmt)) { let fmt = fmts[fi] let stride4 = stride4s[fi] @@ -473,6 +473,8 @@ def test_vkd_kq_gemv_family(t0 : T?) { enc_kq_gemv_iq4nl_cls(raw2, h2, s_cls, pc, wgs) } elif (fmt == int(KqFmt.k2)) { enc_kq_gemv_k2_cls(raw2, h2, s_cls, pc, wgs) + } elif (fmt == int(KqFmt.iq2s)) { + enc_kq_gemv_iq2s_cls(raw2, h2, s_cls, pc, wgs) } else { enc_kq_gemv_k6_cls(raw2, h2, s_cls, pc, wgs) } @@ -509,6 +511,12 @@ def test_vkd_kq_gemv_family(t0 : T?) { t |> success(badf == 0, "iq3xxs: the class bit-math matches the float dequant oracle ({badf} of {rows} rows off)") delete y_flt } + if (fmt == int(KqFmt.iq2s)) { // the u64-grid/sign-byte gather has its own float witness + var y_flt <- iq2s_gemv_float_oracle(wqh, wsh, xqh, xsh, regs, 512, d, nreg) + let badf = mismatch_approx(y_ref, y_flt) + t |> success(badf == 0, "iq2s: the class bit-math matches the float dequant oracle ({badf} of {rows} rows off)") + delete y_flt + } delete regs delete y_ref delete wqh @@ -1029,7 +1037,7 @@ def test_vkd_q8_batch_tile(t0 : T?) { def test_vkd_kq_batch_family(t0 : T?) { t0 |> run("kq batch tile class family == the CPU oracle (all 6 formats)") <| @(t : T?) { static_if (typeinfo builtin_module_exists(vulkan)) { - if (!(ensure_kq_batch_k4_cls() && ensure_kq_batch_q40_cls() && ensure_kq_batch_k5_cls() && ensure_kq_batch_k6_cls() && ensure_kq_batch_iq4xs_cls() && ensure_kq_batch_k3_cls() && ensure_kq_batch_iq3s_cls() && ensure_kq_batch_iq3xxs_cls() && ensure_kq_batch_iq4nl_cls() && ensure_kq_batch_k2_cls())) { + if (!(ensure_kq_batch_k4_cls() && ensure_kq_batch_q40_cls() && ensure_kq_batch_k5_cls() && ensure_kq_batch_k6_cls() && ensure_kq_batch_iq4xs_cls() && ensure_kq_batch_k3_cls() && ensure_kq_batch_iq3s_cls() && ensure_kq_batch_iq3xxs_cls() && ensure_kq_batch_iq4nl_cls() && ensure_kq_batch_k2_cls() && ensure_kq_batch_iq2s_cls())) { feint("no Vulkan device - skipping\n") return } @@ -1044,9 +1052,9 @@ def test_vkd_kq_batch_family(t0 : T?) { let wgs1 = ((cnt1 + 31) / 32) * wtiles let wgs = wgs0 + wgs1 let totsb = 2 * d * nsb // superblocks across both regions - let nfmt = 10 - let wstrides = fixed_array(32, 32, 40, 48, 32, 24, 26, 24, 32, 16) // uint words per superblock: k4 q40 k5 k6 iq4xs k3 iq3s iq3xxs iq4nl k2 - let pipes = fixed_array(int(KqFmt.k4), int(KqFmt.q40), int(KqFmt.k5), int(KqFmt.k6), int(KqFmt.iq4xs), int(KqFmt.k3), int(KqFmt.iq3s), int(KqFmt.iq3xxs), int(KqFmt.iq4nl), int(KqFmt.k2)) + let nfmt = 11 + let wstrides = fixed_array(32, 32, 40, 48, 32, 24, 26, 24, 32, 16, 18) // uint words per superblock: k4 q40 k5 k6 iq4xs k3 iq3s iq3xxs iq4nl k2 iq2s + let pipes = fixed_array(int(KqFmt.k4), int(KqFmt.q40), int(KqFmt.k5), int(KqFmt.k6), int(KqFmt.iq4xs), int(KqFmt.k3), int(KqFmt.iq3s), int(KqFmt.iq3xxs), int(KqFmt.iq4nl), int(KqFmt.k2), int(KqFmt.iq2s)) for (fi in range(nfmt)) { let fmt = pipes[fi] let wstride = wstrides[fi] @@ -1131,6 +1139,8 @@ def test_vkd_kq_batch_family(t0 : T?) { enc_kq_batch_iq4nl_cls(raw2, h2, sc, pc, int64(wgs)) } elif (fmt == int(KqFmt.k2)) { enc_kq_batch_k2_cls(raw2, h2, sc, pc, int64(wgs)) + } elif (fmt == int(KqFmt.iq2s)) { + enc_kq_batch_iq2s_cls(raw2, h2, sc, pc, int64(wgs)) } else { enc_kq_batch_k6_cls(raw2, h2, sc, pc, int64(wgs)) } @@ -2546,6 +2556,130 @@ def test_vkd_iq3s_cm2_batch(t0 : T?) { } } +[test] +def test_vkd_iq2s_cm2_batch(t0 : T?) { + t0 |> run("cm2 IQ2_S l/m/s tiles == the CPU oracle (workgroup-staged u64 grid decode-in-load)") <| @(t : T?) { + static_if (typeinfo builtin_module_exists(vulkan)) { + if (!ensure_q8_batch_cls()) { // engage the device without touching coopmat rails + t |> skip("no Vulkan device") + return + } + if (!(g_gpu.coopmat_mode == 4 && g_gpu.has_coopmat2)) { + t |> skip("the cm2 iq2s tiles serve only in mode 4 on an NV_coopmat2 device") + return + } + verify(ensure_kq_batch_iq2s_cm2l_cls() && ensure_kq_batch_iq2s_cm2m_cls() && ensure_kq_batch_iq2s_cm2s_cls(), "iq2s cm2 class rails must engage in cm2 mode") + let n = 768 // one 512-unrolled pass + 4 tail steps crossing superblock bounds + let nsb = n / 256 + let d = 160 // one full 128-tile + a 32 edge + let cnt0 = 300 + let cnt1 = 260 + let rows = cnt0 + cnt1 + let totsb = 2 * d * nsb + var wqh : array + var wsuh : array + var xfh : array + wqh |> resize(totsb * 18) + wsuh |> resize(totsb * 5) + xfh |> resize((rows + 32) * n / 2) // + the s tile's 32-row read slack + for (i in range(totsb * 18)) { + wqh[i] = hash_word(uint(i) + 29u) // arbitrary quant bytes + } + for (sb in range(totsb)) { + wsuh[sb * 5] = packHalf2x16(float2(0.0002 * float(1 + sb % 7), 0.0)) // cool d8 + wsuh[sb * 5 + 1] = hash_word(uint(sb * 4) + 613u) // 16 unsigned strip bytes + wsuh[sb * 5 + 2] = hash_word(uint(sb * 4 + 1) + 613u) + wsuh[sb * 5 + 3] = hash_word(uint(sb * 4 + 2) + 613u) + wsuh[sb * 5 + 4] = hash_word(uint(sb * 4 + 3) + 613u) + } + for (i in range((rows + 32) * n / 2)) { + xfh[i] = ws_word(uint(i) * 3u + 5u) // f16 activation pairs + } + let wq_bytes = int64(totsb) * 72l + let ws_bytes = int64(length(wsuh)) * 4l + let xf_bytes = int64((rows + 32) * n) * 2l + let y_bytes = int64(rows * d) * 4l + let wqd = make_device_buf(wq_bytes) + let wsd = make_device_buf(ws_bytes) + let xfd = make_device_buf(xf_bytes) + let yd2 = make_device_buf(y_bytes) + var host = make_host_buf(y_bytes, true, [cached = true]) + var y_cls : array + y_cls |> resize(rows * d) + var y_ref : array + y_ref |> resize(rows * d) + for (ml in range(3)) { + let ttile = ml == 0 ? 256 : (ml == 1 ? 128 : 32) + let wtiles = (d + 127) / 128 + let wgs0 = ((cnt0 + ttile - 1) / ttile) * wtiles + let wgs1 = ((cnt1 + ttile - 1) / ttile) * wtiles + let wgs = wgs0 + wgs1 + var sched : array + sched |> resize(2 * 4 + wgs) + sched[0] = 0u + sched[1] = 0u + sched[2] = uint(cnt0) + sched[3] = 0u + sched[4] = uint(d * nsb) + sched[5] = uint(cnt0) + sched[6] = uint(cnt1) + sched[7] = uint(wgs0) + for (w in range(wgs)) { + sched[8 + w] = w < wgs0 ? 0u : 1u + } + unsafe { + upload_region_at(wqd, 0l, addr(wqh[0]), wq_bytes) + upload_region_at(wsd, 0l, addr(wsuh[0]), ws_bytes) + upload_region_at(xfd, 0l, addr(xfh[0]), xf_bytes) + let sc_bytes = int64(2 * 4 + wgs) * 4l + let scd = make_device_buf(sc_bytes) + upload_region_at(scd, 0l, addr(sched[0]), sc_bytes) + var sc = (ml == 0 + ? set_kq_batch_iq2s_cm2l_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)) + : (ml == 1 + ? set_kq_batch_iq2s_cm2m_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)) + : set_kq_batch_iq2s_cm2s_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)))) + var raw2 = alloc_cmd() + let begin = VkCommandBufferBeginInfo() + vk_check(vkBeginCommandBuffer(raw2, begin), null) + var h2 : VkHaz + var pc = BatchArgs(n = uint(n), d = uint(d), map_off = uint(2 * 4)) + if (ml == 0) { + enc_kq_batch_iq2s_cm2l_cls(raw2, h2, sc, pc, int64(wgs)) + } elif (ml == 1) { + enc_kq_batch_iq2s_cm2m_cls(raw2, h2, sc, pc, int64(wgs)) + } else { + enc_kq_batch_iq2s_cm2s_cls(raw2, h2, sc, pc, int64(wgs)) + } + vhz_dep(raw2, h2, 16u, 0u, true) + cmd_copy_whole(raw2, yd2, host.buf, y_bytes) + vk_check(vkEndCommandBuffer(raw2), null) + submit_wait(raw2) + memcpy(addr(y_cls[0]), host.mapped, y_bytes) + } + iq2sf16_gemm_oracle(wqh, wsuh, xfh, sched, 2, n, d, y_ref) + let bad = mismatch_bars(y_cls, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) + to_log(LOG_INFO, "cm2 iq2s {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile batch: {bad} of {rows * d} off the oracle\n") + t |> success(bad == 0, "cm2 iq2s {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile matches the CPU oracle ({bad} of {rows * d} off)") + delete sched + } + delete y_ref + delete wqh + delete wsuh + delete xfh + delete y_cls + } else { + t |> skip("dasVulkan not present") + } + } +} + [test] def test_vkd_iq3xxs_cm2_batch(t0 : T?) { t0 |> run("cm2 IQ3_XXS l/m/s tiles == the CPU oracle (halved grid + parity signs decode-in-load)") <| @(t : T?) { From d99ceda21d3c8426a3692e7d03c693c296b1491e Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Mon, 31 Aug 2026 06:41:27 -0700 Subject: [PATCH 38/61] dasLLAMA: IQ2_S Phase D - Metal closes the format, the 8 KB grid rides the constant-table hoisting The u64 grid ships as iq2s_gw - an all-literal fixed_array local the 363654b0d hoisting lowers to a program-scope constant table - so every kernel reads it DIRECT: no threadgroup slab anywhere (an f4-expanded slab of 1024 entries would be 32 KB, past the budget; the iq3s gridf form does not scale to u64 grids). MetalKqGemvIq2s = the iq3s GEMV geometry with per-word iq3s_sw/iq3s_sx compose and per-16 UNSIGNED strips (plain byte reads, no ^128 sign trick); the B2/B4/B8 twins split dl per half; the mul_mm rides a new IQ2S arm. The "iq2ss" blob arm is iq3ss's verbatim twin ([16 strips][2B d8 tail], 18B/sb); ladders across shapes/common/prefill/ layout; fill/rowref arms in the shared test helpers. Gates on the M1 Max: test_metal_gemv_kernels 2/2, test_metal_gemm_kernels 2/2, lint 0. E2e (parity --ngl 99, resident Metal decode): ids 40/64 vs llama.cpp - the fork at step 40 is a 0.0654-logit top-2 near-tie, the arc's deepest match, on the same story stream as the CPU-JIT tier. Four-tier table (vs llama.cpp b10660, the i1 vehicle): zen2 3.62x/0.76x, 5060 Vulkan 0.70x/0.81x, M1 CPU 2.14x/0.73x, M1 Metal 0.93x/0.93x - the CPU/Vulkan tg tails are the ledgered `#60`/`#61` class. Co-Authored-By: Claude Fable 5 --- modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md | 32 +- modules/dasLLAMA/dasllama/dasllama_layout.das | 23 +- .../dasllama/dasllama_metal_common.das | 14 + .../dasllama/dasllama_metal_kernels.das | 536 ++++++++++++++++++ .../dasllama/dasllama_metal_prefill.das | 8 +- .../dasllama/dasllama_metal_shapes.das | 4 +- .../dasLLAMA/tests/_metal_kernel_common.das | 11 +- .../tests/test_metal_gemm_kernels.das | 24 +- .../tests/test_metal_gemv_kernels.das | 40 +- 9 files changed, 652 insertions(+), 40 deletions(-) diff --git a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md index 242efe7ff1..29020be3f2 100644 --- a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md +++ b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md @@ -491,7 +491,37 @@ resident driver arms on the i1 vehicle: gen 233 t/s, the fork vs llama.cpp at st SAME 0.211-logit top-2 near-tie as the CPU-JIT tier. Rows (5060 Ti vs llama.cpp b10660 build-vulkan -ngl 99): pp512 12099.3 / 17377.5 (0.70x - the tier class), tg128 292.4 / 362.5 (0.81x; the gemv re-stages the 8 KB grid per 2-row workgroup - the obvious tg lever, -same ledger class as the pp tier). Metal: pending. +same ledger class as the pp tier). + +Phase D (Metal, 2026-08-31): the constant-table commit pays off - `iq2s_gw` ships the 8 KB +grid as low/high word pairs through the 363654b0d hoisting (an all-literal fixed_array +local lowers to a program-scope constant table), so EVERY kernel reads it DIRECT: no +threadgroup slab anywhere (an f4-expanded slab of 1024 entries would be 32 KB, past the tg +budget - the iq3s gridf form does not scale to u64 grids). `MetalKqGemvIq2s` = the iq3s +GEMV geometry (4 rows/simdgroup, dispatch rows/8) with per-word `iq3s_sw`/`iq3s_sx` +compose and per-16 UNSIGNED strips (plain byte reads, no ^128 sign trick); the B2/B4/B8 +twins split dl into dl0/dl1 per half; the mul_mm rides a new IQ2S arm (strip byte js*2+il0 +ZExt, two va words per idx byte). The "iq2ss" blob arm is iq3ss's verbatim twin ([16 +strips][2B d8 tail], 18B/sb); ladders: kq_fmt_gpu_supported, moe_site_ok / blob_off_ok % +512, the 72B quant bind, the k6-split scale bind. Gates on the M1 Max: +test_metal_gemv_kernels 2/2, test_metal_gemm_kernels 2/2 (iq2s gemv/mv/mul_mm rows + the +shared fill/rowref arms); lint 0. E2e (parity --ngl 99, resident Metal decode): ids 40/64 - +the fork at step 40 is a 0.0654-logit top-2 near-tie (the arc's DEEPEST match), on the same +Sophia stream as the CPU-JIT tier. TRAP from the walk: a paren-safe ternary-ladder insert +must land INSIDE the bwidth arm, not before it (three b4 lines rebuilt). + +Against llama.cpp b10660 (`lcpp_bench --for-debug-purposes`; zen2 = 16 threads, M1 = 8; +the mradermacher i1 vehicle - IQ2_S attn x32 + IQ3_XXS/IQ3_S/Q4_K/Q5_K): + +| tier | pp512 das / llama.cpp | tg128 das / llama.cpp | +|---|---|---| +| zen2 CPU | 501.5 / 138.5 (3.62x) | 55.8 / 73.5 (0.76x) | +| 5060 Ti Vulkan | 12099.3 / 17377.5 (0.70x - the tier class) | 292.4 / 362.5 (0.81x) | +| M1 CPU | 883.3 / 413.4 (2.14x) | 53.7 / 73.9 (0.73x) | +| M1 Metal | 3170.5 / 3427.2 (0.93x) | 205.8 / 220.7 (0.93x) | + +(The CPU tg tails - zen2 0.76x, M1 0.73x - are the ledgered #60/#61 class; the Vulkan tg +0.81x adds the per-workgroup 8 KB grid re-stage to the same ledger.) ### Q2_K Phase A (CPU, 2026-08-31) diff --git a/modules/dasLLAMA/dasllama/dasllama_layout.das b/modules/dasLLAMA/dasllama/dasllama_layout.das index 696ea89104..7d2d130ee7 100644 --- a/modules/dasLLAMA/dasllama/dasllama_layout.das +++ b/modules/dasLLAMA/dasllama/dasllama_layout.das @@ -29,7 +29,7 @@ require math def private metal_blob_off_ok(off : int64; fmt : KqFmt) : bool { if (off < 0l) return true if (fmt == KqFmt.q8) return off % 256l == 0l - if (fmt == KqFmt.k6 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s || fmt == KqFmt.iq3xxs) return off % 512l == 0l // the split scale form: the (off/256)*2 d-plane bind stays 4B-aligned + if (fmt == KqFmt.k6 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s || fmt == KqFmt.iq3xxs || fmt == KqFmt.iq2s) return off % 512l == 0l // the split scale form: the (off/256)*2 d-plane bind stays 4B-aligned if (fmt == KqFmt.q51) return off % 128l == 0l // (off/32)*20 and (off/32)*4 both 16B-aligned return true } @@ -226,6 +226,25 @@ def metal_blob_scale_plane(var t : Model; name : string) : bool { } return true } + if (name == "iq2ss") { + // iq2s: the row IS the iq4xs 20B shape - the same split (strips = bytes 4..19, d8 = bytes 0..1), over t.iq2ss + if (!empty(t.iq2ss)) { + var @exact_size i2c : array + let nsb = long_length(t.iq2ss) / IQ2S_SSB + i2c |> reserve_resize(nsb * 18l) + unsafe { + var dp = addr(i2c[0]) + let sp = addr < uint8 const? >(t.iq2ss[0]) + for (sb in range64(nsb)) { + memcpy(reinterpret(dp + sb * 16l), reinterpret(sp + sb * IQ2S_SSB + 4l), 16) + dp[nsb * 16l + sb * 2l] = sp[sb * IQ2S_SSB] + dp[nsb * 16l + sb * 2l + 1l] = sp[sb * IQ2S_SSB + 1l] + } + } + t.iq2ss <- i2c + } + return true + } if (name == "k2s") { // k2: the 20B row splits like k6 - the 16 pair-byte strips first, the 4B d+dmin headers as the tail if (!empty(t.k2s)) { @@ -294,7 +313,7 @@ def metal_blob_commit(var t : Model) : bool { if (!metal_blob_eligible(t)) { return false } - for (nm in ["k4s", "k5s", "k6s", "iq4xss", "k3s", "iq3ss", "iq3xxss", "k2s"]) { + for (nm in ["k4s", "k5s", "k6s", "iq4xss", "k3s", "iq3ss", "iq3xxss", "k2s", "iq2ss"]) { metal_blob_scale_plane(t, nm) } t.metal_blob = true diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_common.das b/modules/dasLLAMA/dasllama/dasllama_metal_common.das index 678e89d282..df9003ba1e 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_common.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_common.das @@ -277,6 +277,11 @@ var g_pso_kq_mvb2_iq4nl : MetalComputePipeline? var g_pso_kq_mvb4_iq4nl : MetalComputePipeline? var g_pso_kq_mvb8_iq4nl : MetalComputePipeline? var g_pso_kq_mm_b45 : MetalComputePipeline? +var g_pso_kq_iq2s : MetalComputePipeline? +var g_pso_kq_mvb2_iq2s : MetalComputePipeline? +var g_pso_kq_mvb4_iq2s : MetalComputePipeline? +var g_pso_kq_mvb8_iq2s : MetalComputePipeline? +var g_pso_kq_mm_b23 : MetalComputePipeline? var g_pso_kq_mvb2_iq3xxs : MetalComputePipeline? var g_pso_kq_mvb4_iq3xxs : MetalComputePipeline? var g_pso_kq_mvb8_iq3xxs : MetalComputePipeline? @@ -888,6 +893,10 @@ def kq_quants_of(dev : MetalDevice?; t : Model; fmt : KqFmt; off : int64) : tupl return (buf = plane_buffer(dev, addr < void? >(t.k2q[0]), uint64(long_length(t.k2q)), t.image_map != null), qoff = uint64(sb0 * 64l)) } + if (fmt == KqFmt.iq2s) { + return (buf = plane_buffer(dev, addr < void? >(t.iq2sq[0]), uint64(long_length(t.iq2sq)), t.image_map != null), + qoff = uint64(sb0 * 72l)) + } return (buf = plane_buffer(dev, addr < void? >(t.k6q[0]), uint64(long_length(t.k6q)), t.image_map != null), qoff = uint64(sb0 * 192l)) } @@ -938,6 +947,11 @@ def kq_scales_of(dev : MetalDevice?; t : Model; fmt : KqFmt; off : int64) : tupl return (buf = plane_buffer(dev, addr < void? >(t.k2s[0]), uint64(long_length(t.k2s)), t.image_map != null), soff = uint64(sb0 * 16l), doff = uint64(plane_sb * 16l + sb0 * 4l)) } + if (fmt == KqFmt.iq2s) { // the k6 split form over the 18B/sb device plane (d8 tail) + let plane_sb = long_length(t.iq2ss) / 18l + return (buf = plane_buffer(dev, addr < void? >(t.iq2ss[0]), uint64(long_length(t.iq2ss)), t.image_map != null), + soff = uint64(sb0 * 16l), doff = uint64(plane_sb * 16l + sb0 * 2l)) + } return (buf = plane_buffer(dev, addr < void? >(t.k5s[0]), uint64(long_length(t.k5s)), t.image_map != null), soff = uint64(sb0 * 16l), doff = 0ul) } diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das index 41ffb6040f..03569bac69 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das @@ -3660,6 +3660,11 @@ def metal_decode_init : bool { // nolint:STYLE038 — a flat compile_pso per k g_pso_kq_mvb2_iq4nl = compile_pso(MetalKqMvB2Iq4nl_metal_kq_mv_iq4nl_msl, MetalKqMvB2Iq4nl_metal_kq_mv_iq4nl_msl_entry, MetalKqMvB2Iq4nl_metal_kq_mv_iq4nl_msl_fastmath, ok) g_pso_kq_mvb4_iq4nl = compile_pso(MetalKqMvB4Iq4nl_metal_kq_mv_iq4nl_msl, MetalKqMvB4Iq4nl_metal_kq_mv_iq4nl_msl_entry, MetalKqMvB4Iq4nl_metal_kq_mv_iq4nl_msl_fastmath, ok) g_pso_kq_mvb8_iq4nl = compile_pso(metal_kq_mvb8_iq4nl_msl, metal_kq_mvb8_iq4nl_msl_entry, metal_kq_mvb8_iq4nl_msl_fastmath, ok) + g_pso_kq_iq2s = compile_pso(metal_kq_gemv_iq2s_msl, metal_kq_gemv_iq2s_msl_entry, metal_kq_gemv_iq2s_msl_fastmath, ok) + g_pso_kq_mvb2_iq2s = compile_pso(MetalKqMvB2Iq2s_metal_kq_mv_iq2s_msl, MetalKqMvB2Iq2s_metal_kq_mv_iq2s_msl_entry, MetalKqMvB2Iq2s_metal_kq_mv_iq2s_msl_fastmath, ok) + g_pso_kq_mvb4_iq2s = compile_pso(MetalKqMvB4Iq2s_metal_kq_mv_iq2s_msl, MetalKqMvB4Iq2s_metal_kq_mv_iq2s_msl_entry, MetalKqMvB4Iq2s_metal_kq_mv_iq2s_msl_fastmath, ok) + g_pso_kq_mvb8_iq2s = compile_pso(metal_kq_mvb8_iq2s_msl, metal_kq_mvb8_iq2s_msl_entry, metal_kq_mvb8_iq2s_msl_fastmath, ok) + g_pso_kq_mm_b23 = compile_pso(MetalKqMulMmIq2s_metal_kq_mulmm_msl, MetalKqMulMmIq2s_metal_kq_mulmm_msl_entry, MetalKqMulMmIq2s_metal_kq_mulmm_msl_fastmath, ok) g_pso_kq_mm_b45 = compile_pso(MetalKqMulMmIq4nl_metal_kq_mulmm_msl, MetalKqMulMmIq4nl_metal_kq_mulmm_msl_entry, MetalKqMulMmIq4nl_metal_kq_mulmm_msl_fastmath, ok) g_pso_kq_k2 = compile_pso(metal_kq_gemv_k2_msl, metal_kq_gemv_k2_msl_entry, metal_kq_gemv_k2_msl_fastmath, ok) g_pso_kq_mvb2_k2 = compile_pso(MetalKqMvB2K2_metal_kq_mv_k2_msl, MetalKqMvB2K2_metal_kq_mv_k2_msl_entry, MetalKqMvB2K2_metal_kq_mv_k2_msl_fastmath, ok) @@ -3744,6 +3749,8 @@ def enc_kq_gemv(enc : MetalComputeEncoder?; fmt : KqFmt; bq : MetalBuffer?; qoff enc_kq_iq4nl_c(enc, bs, soff, bs, soff, bq, qoff, bx, xoff, by, yoff, bn, bd, rows, n) } elif (fmt == KqFmt.k2) { enc_kq_k2_c(enc, bs, doff, bs, soff, bq, qoff, bx, xoff, by, yoff, bn, bd, rows, n) + } elif (fmt == KqFmt.iq2s) { + enc_kq_iq2s_c(enc, bs, doff, bs, soff, bq, qoff, bx, xoff, by, yoff, bn, bd, rows, n) } elif (fmt == KqFmt.k6) { enc_kq_k6_c(enc, bs, doff, bs, soff, bq, qoff, bx, xoff, by, yoff, bn, bd, rows, n) } elif (fmt == KqFmt.k5 && (n < 3072l || rows <= 1024l)) { @@ -3828,6 +3835,14 @@ def enc_kq_mvb(enc : MetalComputeEncoder?; fmt : KqFmt; bq : MetalBuffer?; qoff } else { enc_kq_mvb4_k2_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows, gcols) } + } elif (fmt == KqFmt.iq2s) { + if (b8) { + enc_kq_mvb8_iq2s_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows) + } elif (two) { + enc_kq_mvb2_iq2s_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows) + } else { + enc_kq_mvb4_iq2s_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows, gcols) + } } elif (fmt == KqFmt.k6) { if (b8) { enc_kq_mvb8_k6_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows) @@ -3872,6 +3887,8 @@ def enc_kq_gemm_mm_b(enc : MetalComputeEncoder?; fmt : KqFmt; bq : MetalBuffer?; enc_kq_mm_iq4nl_c(enc, bs, soff, bs, soff, bq, qoff, bx, by, yoff, bk, bn, mp, rows) } elif (fmt == KqFmt.k2) { enc_kq_mm_k2_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, bk, bn, mp, rows) + } elif (fmt == KqFmt.iq2s) { + enc_kq_mm_iq2s_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, bk, bn, mp, rows) } elif (fmt == KqFmt.k6) { enc_kq_mm_k6_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, bk, bn, mp, rows) } elif (fmt == KqFmt.k4) { @@ -4319,6 +4336,11 @@ def metal_kernels_release { // nolint:STYLE037,STYLE038 — a flat release_pso release_pso(g_pso_kq_mvb8_iq3xxs) release_pso(g_pso_kq_mm_b34) release_pso(g_pso_kq_iq4nl) + release_pso(g_pso_kq_iq2s) + release_pso(g_pso_kq_mvb2_iq2s) + release_pso(g_pso_kq_mvb4_iq2s) + release_pso(g_pso_kq_mvb8_iq2s) + release_pso(g_pso_kq_mm_b23) release_pso(g_pso_kq_mvb2_iq4nl) release_pso(g_pso_kq_mvb4_iq4nl) release_pso(g_pso_kq_mvb8_iq4nl) @@ -5559,6 +5581,7 @@ class template MetalKqMulMmK45T : MetalMmTileBase { @template_constant IQ3XXS : bool = false // iq3xxs: the same form over the halved grid + parity signs @template_constant IQ4NL : bool = false // iq4nl: the IQ4XS nibble walk with the per-32-block f16 d plane (no sub-scale; s0off = soff) @template_constant K2 : bool = false // q2_K: per-16 sc/min nibble pairs (one uint4 strip), the split 4B d+dmin tail at s0off + @template_constant IQ2S : bool = false // iq2s: the u64-grid gather DIRECT off the hoisted constant table, per-16 UNSIGNED strips @workgroup @template_gate = IQ4XS iq4lut : float[16] // kvalues_iq4nl as floats, staged once per threadgroup @workgroup @template_gate = IQ3S gridw3 : uint[512] // the 2 KB iq3s grid, staged once per threadgroup @workgroup @template_gate = IQ3XXS gridw3x : uint[256] // the 1 KB halved iq3xxs grid, staged once per threadgroup @@ -5679,6 +5702,27 @@ class template MetalKqMulMmK45T : MetalMmTileBase { va[k * 4 + 3] = float16(dsc3 * float(qv & 4278190080u) - dmn) } } else { + static_if (IQ2S) { + // iq2s: strip byte js*2+il0 is the UNSIGNED per-16 sub-scale; idx byte l of + // block js doubles into the u64 grid's word pair, signs off the block's bytes + let sv = ks4[blk] + let si2 = js * 2u + il0 + let scw = si2 < 8u ? (si2 < 4u ? sv.x : sv.y) : (si2 < 12u ? sv.z : sv.w) + let dsc = float(ksh[blk]) * float((scw >> ((si2 & 3u) * 8u)) & 255u) + let qw2 = kqu[blk * 18u + js] + let sgw2 = kqu[blk * 18u + 8u + js] + let qh2 = (kqu[blk * 18u + 16u + js / 4u] >> ((js % 4u) * 8u)) & 255u + for [unroll_full] (k in range(4)) { + let l2 = il0 * 2u + uint(k) / 2u + let idx2 = (((qw2 >> (8u * l2)) & 255u) | ((qh2 << (8u - 2u * l2)) & 0x300u)) * 2u + uint(k) % 2u + let sgb2 = (sgw2 >> (8u * l2)) & 255u + let nib = uint(k) % 2u == 0u ? sgb2 & 15u : sgb2 >> 4u + let sw = iq3s_sw(iq2s_gw(int(idx2)), nib) + for [unroll_full] (c in range(4)) { + va[k * 4 + c] = float16(dsc * iq3s_sx(sw, uint(c))) + } + } + } else { static_if (IQ3XXS) { // iq3xxs: the iq3s arm over the halved grid - bare byte index, parity // signs off the block's aux32 (no table) @@ -5801,6 +5845,7 @@ class template MetalKqMulMmK45T : MetalMmTileBase { } } } + } } } barrier() @@ -5870,6 +5915,12 @@ class MetalKqMulMmK2 : MetalKqMulMmK45T { override K2 = true } +// iq2s rides its own IQ2S arm; the split scale plane binds twice like k6 (d8 plane at s0off = doff) +[metal_dispatch(name = "enc_kq_mm_iq2s_c", pso = "g_pso_kq_mm_b23", tgmem = "MetalKqMulMmIq2s_metal_kq_mulmm_msl_tgmem", tg = 128, grid = "mp/32, rows/64", params = "mp : int64, rows : int64")] +class MetalKqMulMmIq2s : MetalKqMulMmK45T { + override IQ2S = true +} + // iq4nl rides the IQ4XS arm with IQ4NL flipping the scale read to q40's per-32-block d plane [metal_dispatch(name = "enc_kq_mm_iq4nl_c", pso = "g_pso_kq_mm_b45", tgmem = "MetalKqMulMmIq4nl_metal_kq_mulmm_msl_tgmem", tg = 128, grid = "mp/32, rows/64", params = "mp : int64, rows : int64")] class MetalKqMulMmIq4nl : MetalKqMulMmK45T { @@ -6290,6 +6341,270 @@ def private ksign7m(v : uint) : uint { return v | ((t & 1u) << 7u) } +// ggml's iq2s_grid as low/high word pairs (word 2i = magnitudes 0..3 of u64 entry i, word 2i+1 +// = 4..7); an all-literal fixed_array local, so it lowers to a program-scope constant table - +// every kernel reads it DIRECT (an f4 slab of 1024 entries would be 32 KB, past the tg budget). +def private iq2s_gw(i : int) : uint { // nolint:STYLE038 - a 2048-word data table, not splittable + let tbl = fixed_array( + 0x08080808, 0x08080808, 0x0808082b, 0x08080808, 0x08081919, 0x08080808, 0x08082b08, 0x08080808, + 0x08082b2b, 0x08080808, 0x08190819, 0x08080808, 0x08191908, 0x08080808, 0x0819192b, 0x08080808, + 0x08192b19, 0x08080808, 0x082b0808, 0x08080808, 0x082b082b, 0x08080808, 0x082b1919, 0x08080808, + 0x082b2b08, 0x08080808, 0x19080819, 0x08080808, 0x19081908, 0x08080808, 0x1908192b, 0x08080808, + 0x19082b19, 0x08080808, 0x19190808, 0x08080808, 0x1919082b, 0x08080808, 0x19191919, 0x08080808, + 0x19192b08, 0x08080808, 0x192b0819, 0x08080808, 0x192b1908, 0x08080808, 0x192b192b, 0x08080808, + 0x192b2b19, 0x08080808, 0x2b080808, 0x08080808, 0x2b08082b, 0x08080808, 0x2b081919, 0x08080808, + 0x2b082b08, 0x08080808, 0x2b190819, 0x08080808, 0x2b191908, 0x08080808, 0x2b2b0808, 0x08080808, + 0x2b2b1919, 0x08080808, 0x2b2b2b2b, 0x08080808, 0x08080819, 0x08080819, 0x08081908, 0x08080819, + 0x0808192b, 0x08080819, 0x08082b19, 0x08080819, 0x08190808, 0x08080819, 0x0819082b, 0x08080819, + 0x08191919, 0x08080819, 0x08192b08, 0x08080819, 0x082b0819, 0x08080819, 0x082b1908, 0x08080819, + 0x19080808, 0x08080819, 0x1908082b, 0x08080819, 0x19081919, 0x08080819, 0x19082b08, 0x08080819, + 0x19190819, 0x08080819, 0x19191908, 0x08080819, 0x1919192b, 0x08080819, 0x19192b19, 0x08080819, + 0x192b0808, 0x08080819, 0x192b1919, 0x08080819, 0x192b2b08, 0x08080819, 0x2b080819, 0x08080819, + 0x2b081908, 0x08080819, 0x2b190808, 0x08080819, 0x2b19082b, 0x08080819, 0x2b191919, 0x08080819, + 0x2b2b0819, 0x08080819, 0x2b2b1908, 0x08080819, 0x08080808, 0x0808082b, 0x0808082b, 0x0808082b, + 0x08081919, 0x0808082b, 0x08082b08, 0x0808082b, 0x08190819, 0x0808082b, 0x08191908, 0x0808082b, + 0x082b0808, 0x0808082b, 0x082b2b2b, 0x0808082b, 0x19080819, 0x0808082b, 0x19081908, 0x0808082b, + 0x1908192b, 0x0808082b, 0x19082b19, 0x0808082b, 0x19190808, 0x0808082b, 0x19191919, 0x0808082b, + 0x2b080808, 0x0808082b, 0x2b081919, 0x0808082b, 0x2b082b2b, 0x0808082b, 0x2b191908, 0x0808082b, + 0x2b2b082b, 0x0808082b, 0x08080819, 0x08081908, 0x08081908, 0x08081908, 0x0808192b, 0x08081908, + 0x08082b19, 0x08081908, 0x08190808, 0x08081908, 0x0819082b, 0x08081908, 0x08191919, 0x08081908, + 0x08192b08, 0x08081908, 0x082b0819, 0x08081908, 0x082b1908, 0x08081908, 0x082b192b, 0x08081908, + 0x082b2b19, 0x08081908, 0x19080808, 0x08081908, 0x1908082b, 0x08081908, 0x19081919, 0x08081908, + 0x19082b08, 0x08081908, 0x19082b2b, 0x08081908, 0x19190819, 0x08081908, 0x19191908, 0x08081908, + 0x1919192b, 0x08081908, 0x19192b19, 0x08081908, 0x192b0808, 0x08081908, 0x192b082b, 0x08081908, + 0x192b1919, 0x08081908, 0x2b080819, 0x08081908, 0x2b081908, 0x08081908, 0x2b08192b, 0x08081908, + 0x2b082b19, 0x08081908, 0x2b190808, 0x08081908, 0x2b191919, 0x08081908, 0x2b192b08, 0x08081908, + 0x2b2b0819, 0x08081908, 0x2b2b1908, 0x08081908, 0x08080808, 0x08081919, 0x0808082b, 0x08081919, + 0x08081919, 0x08081919, 0x08082b08, 0x08081919, 0x08082b2b, 0x08081919, 0x08190819, 0x08081919, + 0x08191908, 0x08081919, 0x0819192b, 0x08081919, 0x08192b19, 0x08081919, 0x082b0808, 0x08081919, + 0x082b1919, 0x08081919, 0x082b2b08, 0x08081919, 0x19080819, 0x08081919, 0x19081908, 0x08081919, + 0x1908192b, 0x08081919, 0x19082b19, 0x08081919, 0x19190808, 0x08081919, 0x1919082b, 0x08081919, + 0x19191919, 0x08081919, 0x19192b08, 0x08081919, 0x192b0819, 0x08081919, 0x192b1908, 0x08081919, + 0x2b080808, 0x08081919, 0x2b08082b, 0x08081919, 0x2b081919, 0x08081919, 0x2b082b08, 0x08081919, + 0x2b190819, 0x08081919, 0x2b191908, 0x08081919, 0x2b2b0808, 0x08081919, 0x08080819, 0x0808192b, + 0x08081908, 0x0808192b, 0x0808192b, 0x0808192b, 0x08082b19, 0x0808192b, 0x08190808, 0x0808192b, + 0x08191919, 0x0808192b, 0x19080808, 0x0808192b, 0x19081919, 0x0808192b, 0x19082b08, 0x0808192b, + 0x19190819, 0x0808192b, 0x19191908, 0x0808192b, 0x192b0808, 0x0808192b, 0x2b080819, 0x0808192b, + 0x2b081908, 0x0808192b, 0x2b190808, 0x0808192b, 0x08080808, 0x08082b08, 0x0808082b, 0x08082b08, + 0x08081919, 0x08082b08, 0x08082b08, 0x08082b08, 0x08190819, 0x08082b08, 0x08191908, 0x08082b08, + 0x0819192b, 0x08082b08, 0x08192b19, 0x08082b08, 0x082b0808, 0x08082b08, 0x082b1919, 0x08082b08, + 0x082b2b2b, 0x08082b08, 0x19080819, 0x08082b08, 0x19081908, 0x08082b08, 0x1908192b, 0x08082b08, + 0x19082b19, 0x08082b08, 0x19190808, 0x08082b08, 0x1919082b, 0x08082b08, 0x19191919, 0x08082b08, + 0x19192b08, 0x08082b08, 0x192b0819, 0x08082b08, 0x192b1908, 0x08082b08, 0x2b080808, 0x08082b08, + 0x2b081919, 0x08082b08, 0x2b191908, 0x08082b08, 0x2b2b2b2b, 0x08082b08, 0x08080819, 0x08082b19, + 0x08081908, 0x08082b19, 0x08190808, 0x08082b19, 0x0819082b, 0x08082b19, 0x08191919, 0x08082b19, + 0x08192b08, 0x08082b19, 0x082b0819, 0x08082b19, 0x19080808, 0x08082b19, 0x19081919, 0x08082b19, + 0x19082b08, 0x08082b19, 0x19190819, 0x08082b19, 0x19191908, 0x08082b19, 0x192b0808, 0x08082b19, + 0x2b080819, 0x08082b19, 0x2b190808, 0x08082b19, 0x08080808, 0x08082b2b, 0x08190819, 0x08082b2b, + 0x08191908, 0x08082b2b, 0x082b082b, 0x08082b2b, 0x082b2b08, 0x08082b2b, 0x082b2b2b, 0x08082b2b, + 0x19190808, 0x08082b2b, 0x2b192b19, 0x08082b2b, 0x08080819, 0x08190808, 0x08081908, 0x08190808, + 0x0808192b, 0x08190808, 0x08082b19, 0x08190808, 0x08190808, 0x08190808, 0x0819082b, 0x08190808, + 0x08191919, 0x08190808, 0x08192b08, 0x08190808, 0x082b0819, 0x08190808, 0x082b1908, 0x08190808, + 0x082b192b, 0x08190808, 0x19080808, 0x08190808, 0x1908082b, 0x08190808, 0x19081919, 0x08190808, + 0x19082b08, 0x08190808, 0x19190819, 0x08190808, 0x19191908, 0x08190808, 0x1919192b, 0x08190808, + 0x19192b19, 0x08190808, 0x192b0808, 0x08190808, 0x192b082b, 0x08190808, 0x192b1919, 0x08190808, + 0x192b2b08, 0x08190808, 0x2b080819, 0x08190808, 0x2b081908, 0x08190808, 0x2b08192b, 0x08190808, + 0x2b190808, 0x08190808, 0x2b191919, 0x08190808, 0x2b192b08, 0x08190808, 0x2b2b0819, 0x08190808, + 0x2b2b1908, 0x08190808, 0x08080808, 0x08190819, 0x0808082b, 0x08190819, 0x08081919, 0x08190819, + 0x08082b08, 0x08190819, 0x08082b2b, 0x08190819, 0x08190819, 0x08190819, 0x08191908, 0x08190819, + 0x0819192b, 0x08190819, 0x08192b19, 0x08190819, 0x082b0808, 0x08190819, 0x082b082b, 0x08190819, + 0x082b1919, 0x08190819, 0x082b2b08, 0x08190819, 0x19080819, 0x08190819, 0x19081908, 0x08190819, + 0x1908192b, 0x08190819, 0x19082b19, 0x08190819, 0x19190808, 0x08190819, 0x1919082b, 0x08190819, + 0x19191919, 0x08190819, 0x19192b08, 0x08190819, 0x192b0819, 0x08190819, 0x192b1908, 0x08190819, + 0x2b080808, 0x08190819, 0x2b08082b, 0x08190819, 0x2b081919, 0x08190819, 0x2b082b08, 0x08190819, + 0x2b190819, 0x08190819, 0x2b191908, 0x08190819, 0x08080819, 0x0819082b, 0x08081908, 0x0819082b, + 0x08082b19, 0x0819082b, 0x08190808, 0x0819082b, 0x08191919, 0x0819082b, 0x082b0819, 0x0819082b, + 0x082b1908, 0x0819082b, 0x19080808, 0x0819082b, 0x19081919, 0x0819082b, 0x19190819, 0x0819082b, + 0x19191908, 0x0819082b, 0x2b080819, 0x0819082b, 0x2b081908, 0x0819082b, 0x2b190808, 0x0819082b, + 0x08080808, 0x08191908, 0x0808082b, 0x08191908, 0x08081919, 0x08191908, 0x08082b08, 0x08191908, + 0x08190819, 0x08191908, 0x08191908, 0x08191908, 0x0819192b, 0x08191908, 0x08192b19, 0x08191908, + 0x082b0808, 0x08191908, 0x082b1919, 0x08191908, 0x082b2b08, 0x08191908, 0x19080819, 0x08191908, + 0x19081908, 0x08191908, 0x1908192b, 0x08191908, 0x19082b19, 0x08191908, 0x19190808, 0x08191908, + 0x1919082b, 0x08191908, 0x19191919, 0x08191908, 0x19192b08, 0x08191908, 0x192b0819, 0x08191908, + 0x192b1908, 0x08191908, 0x2b080808, 0x08191908, 0x2b08082b, 0x08191908, 0x2b081919, 0x08191908, + 0x2b082b08, 0x08191908, 0x2b190819, 0x08191908, 0x2b191908, 0x08191908, 0x2b2b0808, 0x08191908, + 0x08080819, 0x08191919, 0x08081908, 0x08191919, 0x0808192b, 0x08191919, 0x08082b19, 0x08191919, + 0x08190808, 0x08191919, 0x0819082b, 0x08191919, 0x08191919, 0x08191919, 0x08192b08, 0x08191919, + 0x082b0819, 0x08191919, 0x082b1908, 0x08191919, 0x19080808, 0x08191919, 0x1908082b, 0x08191919, + 0x19081919, 0x08191919, 0x19082b08, 0x08191919, 0x19190819, 0x08191919, 0x19191908, 0x08191919, + 0x192b0808, 0x08191919, 0x2b080819, 0x08191919, 0x2b081908, 0x08191919, 0x2b190808, 0x08191919, + 0x08080808, 0x0819192b, 0x08081919, 0x0819192b, 0x08082b08, 0x0819192b, 0x08190819, 0x0819192b, + 0x08191908, 0x0819192b, 0x082b0808, 0x0819192b, 0x19080819, 0x0819192b, 0x19081908, 0x0819192b, + 0x19190808, 0x0819192b, 0x2b080808, 0x0819192b, 0x2b2b2b2b, 0x0819192b, 0x08080819, 0x08192b08, + 0x08081908, 0x08192b08, 0x0808192b, 0x08192b08, 0x08082b19, 0x08192b08, 0x08190808, 0x08192b08, + 0x08191919, 0x08192b08, 0x08192b08, 0x08192b08, 0x082b0819, 0x08192b08, 0x19080808, 0x08192b08, + 0x1908082b, 0x08192b08, 0x19081919, 0x08192b08, 0x19082b08, 0x08192b08, 0x19190819, 0x08192b08, + 0x19191908, 0x08192b08, 0x192b0808, 0x08192b08, 0x2b080819, 0x08192b08, 0x2b081908, 0x08192b08, + 0x08080808, 0x08192b19, 0x0808082b, 0x08192b19, 0x08081919, 0x08192b19, 0x08082b08, 0x08192b19, + 0x08190819, 0x08192b19, 0x08191908, 0x08192b19, 0x082b0808, 0x08192b19, 0x19080819, 0x08192b19, + 0x19081908, 0x08192b19, 0x19190808, 0x08192b19, 0x192b2b19, 0x08192b19, 0x2b2b082b, 0x08192b19, + 0x08081908, 0x08192b2b, 0x08190808, 0x08192b2b, 0x19080808, 0x08192b2b, 0x1919192b, 0x08192b2b, + 0x08080808, 0x082b0808, 0x0808082b, 0x082b0808, 0x08081919, 0x082b0808, 0x08082b08, 0x082b0808, + 0x08190819, 0x082b0808, 0x08191908, 0x082b0808, 0x0819192b, 0x082b0808, 0x08192b19, 0x082b0808, + 0x082b0808, 0x082b0808, 0x082b1919, 0x082b0808, 0x082b2b2b, 0x082b0808, 0x19080819, 0x082b0808, + 0x19081908, 0x082b0808, 0x19190808, 0x082b0808, 0x1919082b, 0x082b0808, 0x19191919, 0x082b0808, + 0x192b1908, 0x082b0808, 0x2b080808, 0x082b0808, 0x2b082b2b, 0x082b0808, 0x2b191908, 0x082b0808, + 0x2b2b2b2b, 0x082b0808, 0x08080819, 0x082b0819, 0x08081908, 0x082b0819, 0x08190808, 0x082b0819, + 0x0819082b, 0x082b0819, 0x08191919, 0x082b0819, 0x082b0819, 0x082b0819, 0x19080808, 0x082b0819, + 0x1908082b, 0x082b0819, 0x19081919, 0x082b0819, 0x19190819, 0x082b0819, 0x19191908, 0x082b0819, + 0x192b0808, 0x082b0819, 0x2b080819, 0x082b0819, 0x2b081908, 0x082b0819, 0x2b190808, 0x082b0819, + 0x08080808, 0x082b082b, 0x08082b2b, 0x082b082b, 0x082b082b, 0x082b082b, 0x082b2b08, 0x082b082b, + 0x082b2b2b, 0x082b082b, 0x19081908, 0x082b082b, 0x19190808, 0x082b082b, 0x2b082b08, 0x082b082b, + 0x2b082b2b, 0x082b082b, 0x2b2b2b08, 0x082b082b, 0x08080819, 0x082b1908, 0x08081908, 0x082b1908, + 0x0808192b, 0x082b1908, 0x08082b19, 0x082b1908, 0x08190808, 0x082b1908, 0x08191919, 0x082b1908, + 0x08192b08, 0x082b1908, 0x082b0819, 0x082b1908, 0x082b1908, 0x082b1908, 0x19080808, 0x082b1908, + 0x1908082b, 0x082b1908, 0x19081919, 0x082b1908, 0x19082b08, 0x082b1908, 0x19190819, 0x082b1908, + 0x19191908, 0x082b1908, 0x192b0808, 0x082b1908, 0x2b080819, 0x082b1908, 0x2b081908, 0x082b1908, + 0x2b190808, 0x082b1908, 0x08080808, 0x082b1919, 0x08081919, 0x082b1919, 0x08082b08, 0x082b1919, + 0x08190819, 0x082b1919, 0x08191908, 0x082b1919, 0x082b0808, 0x082b1919, 0x19080819, 0x082b1919, + 0x19081908, 0x082b1919, 0x19190808, 0x082b1919, 0x192b192b, 0x082b1919, 0x2b080808, 0x082b1919, + 0x08080819, 0x082b192b, 0x08081908, 0x082b192b, 0x08190808, 0x082b192b, 0x19080808, 0x082b192b, + 0x19192b19, 0x082b192b, 0x08080808, 0x082b2b08, 0x08081919, 0x082b2b08, 0x08190819, 0x082b2b08, + 0x08191908, 0x082b2b08, 0x19080819, 0x082b2b08, 0x19081908, 0x082b2b08, 0x19190808, 0x082b2b08, + 0x2b082b2b, 0x082b2b08, 0x2b2b2b2b, 0x082b2b08, 0x08080819, 0x082b2b19, 0x08081908, 0x082b2b19, + 0x08190808, 0x082b2b19, 0x2b191919, 0x082b2b19, 0x08082b2b, 0x082b2b2b, 0x082b082b, 0x082b2b2b, + 0x192b1908, 0x082b2b2b, 0x2b082b08, 0x082b2b2b, 0x2b082b2b, 0x082b2b2b, 0x08080819, 0x19080808, + 0x08081908, 0x19080808, 0x0808192b, 0x19080808, 0x08082b19, 0x19080808, 0x08190808, 0x19080808, + 0x0819082b, 0x19080808, 0x08191919, 0x19080808, 0x08192b08, 0x19080808, 0x08192b2b, 0x19080808, + 0x082b0819, 0x19080808, 0x082b1908, 0x19080808, 0x082b192b, 0x19080808, 0x19080808, 0x19080808, + 0x1908082b, 0x19080808, 0x19081919, 0x19080808, 0x19082b08, 0x19080808, 0x19082b2b, 0x19080808, + 0x19190819, 0x19080808, 0x19191908, 0x19080808, 0x1919192b, 0x19080808, 0x19192b19, 0x19080808, + 0x192b0808, 0x19080808, 0x192b082b, 0x19080808, 0x192b1919, 0x19080808, 0x2b080819, 0x19080808, + 0x2b081908, 0x19080808, 0x2b190808, 0x19080808, 0x2b191919, 0x19080808, 0x2b192b08, 0x19080808, + 0x2b2b0819, 0x19080808, 0x2b2b1908, 0x19080808, 0x08080808, 0x19080819, 0x0808082b, 0x19080819, + 0x08081919, 0x19080819, 0x08082b08, 0x19080819, 0x08190819, 0x19080819, 0x08191908, 0x19080819, + 0x0819192b, 0x19080819, 0x08192b19, 0x19080819, 0x082b0808, 0x19080819, 0x082b082b, 0x19080819, + 0x082b1919, 0x19080819, 0x19080819, 0x19080819, 0x19081908, 0x19080819, 0x1908192b, 0x19080819, + 0x19082b19, 0x19080819, 0x19190808, 0x19080819, 0x1919082b, 0x19080819, 0x19191919, 0x19080819, + 0x19192b08, 0x19080819, 0x192b0819, 0x19080819, 0x192b1908, 0x19080819, 0x2b080808, 0x19080819, + 0x2b08082b, 0x19080819, 0x2b081919, 0x19080819, 0x2b082b08, 0x19080819, 0x2b190819, 0x19080819, + 0x2b191908, 0x19080819, 0x2b2b0808, 0x19080819, 0x08080819, 0x1908082b, 0x08081908, 0x1908082b, + 0x08190808, 0x1908082b, 0x0819082b, 0x1908082b, 0x08191919, 0x1908082b, 0x08192b08, 0x1908082b, + 0x082b1908, 0x1908082b, 0x19080808, 0x1908082b, 0x19081919, 0x1908082b, 0x19082b08, 0x1908082b, + 0x19190819, 0x1908082b, 0x19191908, 0x1908082b, 0x192b0808, 0x1908082b, 0x2b080819, 0x1908082b, + 0x2b081908, 0x1908082b, 0x08080808, 0x19081908, 0x0808082b, 0x19081908, 0x08081919, 0x19081908, + 0x08082b08, 0x19081908, 0x08082b2b, 0x19081908, 0x08190819, 0x19081908, 0x08191908, 0x19081908, + 0x0819192b, 0x19081908, 0x08192b19, 0x19081908, 0x082b0808, 0x19081908, 0x082b082b, 0x19081908, + 0x082b1919, 0x19081908, 0x082b2b08, 0x19081908, 0x19080819, 0x19081908, 0x19081908, 0x19081908, + 0x1908192b, 0x19081908, 0x19082b19, 0x19081908, 0x19190808, 0x19081908, 0x1919082b, 0x19081908, + 0x19191919, 0x19081908, 0x19192b08, 0x19081908, 0x192b0819, 0x19081908, 0x192b1908, 0x19081908, + 0x2b080808, 0x19081908, 0x2b08082b, 0x19081908, 0x2b081919, 0x19081908, 0x2b082b08, 0x19081908, + 0x2b190819, 0x19081908, 0x2b191908, 0x19081908, 0x2b2b0808, 0x19081908, 0x08080819, 0x19081919, + 0x08081908, 0x19081919, 0x0808192b, 0x19081919, 0x08082b19, 0x19081919, 0x08190808, 0x19081919, + 0x0819082b, 0x19081919, 0x08191919, 0x19081919, 0x08192b08, 0x19081919, 0x082b0819, 0x19081919, + 0x082b1908, 0x19081919, 0x19080808, 0x19081919, 0x1908082b, 0x19081919, 0x19081919, 0x19081919, + 0x19082b08, 0x19081919, 0x19190819, 0x19081919, 0x19191908, 0x19081919, 0x192b0808, 0x19081919, + 0x192b2b2b, 0x19081919, 0x2b080819, 0x19081919, 0x2b081908, 0x19081919, 0x2b190808, 0x19081919, + 0x08080808, 0x1908192b, 0x0808082b, 0x1908192b, 0x08081919, 0x1908192b, 0x08082b08, 0x1908192b, + 0x08190819, 0x1908192b, 0x08191908, 0x1908192b, 0x082b0808, 0x1908192b, 0x19080819, 0x1908192b, + 0x19081908, 0x1908192b, 0x19190808, 0x1908192b, 0x2b080808, 0x1908192b, 0x2b2b1919, 0x1908192b, + 0x08080819, 0x19082b08, 0x08081908, 0x19082b08, 0x08082b19, 0x19082b08, 0x08190808, 0x19082b08, + 0x0819082b, 0x19082b08, 0x08191919, 0x19082b08, 0x08192b08, 0x19082b08, 0x082b0819, 0x19082b08, + 0x082b1908, 0x19082b08, 0x19080808, 0x19082b08, 0x1908082b, 0x19082b08, 0x19081919, 0x19082b08, + 0x19082b08, 0x19082b08, 0x19190819, 0x19082b08, 0x19191908, 0x19082b08, 0x192b0808, 0x19082b08, + 0x2b081908, 0x19082b08, 0x2b190808, 0x19082b08, 0x08080808, 0x19082b19, 0x0808082b, 0x19082b19, + 0x08081919, 0x19082b19, 0x08082b08, 0x19082b19, 0x08190819, 0x19082b19, 0x08191908, 0x19082b19, + 0x082b0808, 0x19082b19, 0x19080819, 0x19082b19, 0x19081908, 0x19082b19, 0x19190808, 0x19082b19, + 0x2b080808, 0x19082b19, 0x2b19192b, 0x19082b19, 0x08080819, 0x19082b2b, 0x08081908, 0x19082b2b, + 0x08190808, 0x19082b2b, 0x19080808, 0x19082b2b, 0x08080808, 0x19190808, 0x0808082b, 0x19190808, + 0x08081919, 0x19190808, 0x08082b08, 0x19190808, 0x08190819, 0x19190808, 0x08191908, 0x19190808, + 0x0819192b, 0x19190808, 0x08192b19, 0x19190808, 0x082b0808, 0x19190808, 0x082b082b, 0x19190808, + 0x082b1919, 0x19190808, 0x082b2b08, 0x19190808, 0x19080819, 0x19190808, 0x19081908, 0x19190808, + 0x1908192b, 0x19190808, 0x19082b19, 0x19190808, 0x19190808, 0x19190808, 0x1919082b, 0x19190808, + 0x19191919, 0x19190808, 0x19192b08, 0x19190808, 0x192b0819, 0x19190808, 0x192b1908, 0x19190808, + 0x2b080808, 0x19190808, 0x2b08082b, 0x19190808, 0x2b081919, 0x19190808, 0x2b082b08, 0x19190808, + 0x2b190819, 0x19190808, 0x2b191908, 0x19190808, 0x08080819, 0x19190819, 0x08081908, 0x19190819, + 0x0808192b, 0x19190819, 0x08082b19, 0x19190819, 0x08190808, 0x19190819, 0x0819082b, 0x19190819, + 0x08191919, 0x19190819, 0x08192b08, 0x19190819, 0x082b0819, 0x19190819, 0x082b1908, 0x19190819, + 0x19080808, 0x19190819, 0x1908082b, 0x19190819, 0x19081919, 0x19190819, 0x19082b08, 0x19190819, + 0x19190819, 0x19190819, 0x19191908, 0x19190819, 0x192b0808, 0x19190819, 0x2b080819, 0x19190819, + 0x2b081908, 0x19190819, 0x2b190808, 0x19190819, 0x08080808, 0x1919082b, 0x08081919, 0x1919082b, + 0x08082b08, 0x1919082b, 0x08190819, 0x1919082b, 0x08191908, 0x1919082b, 0x082b0808, 0x1919082b, + 0x19080819, 0x1919082b, 0x19081908, 0x1919082b, 0x19190808, 0x1919082b, 0x192b2b19, 0x1919082b, + 0x2b080808, 0x1919082b, 0x08080819, 0x19191908, 0x08081908, 0x19191908, 0x0808192b, 0x19191908, + 0x08082b19, 0x19191908, 0x08190808, 0x19191908, 0x0819082b, 0x19191908, 0x08191919, 0x19191908, + 0x08192b08, 0x19191908, 0x082b0819, 0x19191908, 0x082b1908, 0x19191908, 0x19080808, 0x19191908, + 0x1908082b, 0x19191908, 0x19081919, 0x19191908, 0x19082b08, 0x19191908, 0x19190819, 0x19191908, + 0x19191908, 0x19191908, 0x192b0808, 0x19191908, 0x2b080819, 0x19191908, 0x2b081908, 0x19191908, + 0x2b190808, 0x19191908, 0x08080808, 0x19191919, 0x0808082b, 0x19191919, 0x08081919, 0x19191919, + 0x08082b08, 0x19191919, 0x08190819, 0x19191919, 0x08191908, 0x19191919, 0x082b0808, 0x19191919, + 0x19080819, 0x19191919, 0x19081908, 0x19191919, 0x19190808, 0x19191919, 0x2b080808, 0x19191919, + 0x08080819, 0x1919192b, 0x08081908, 0x1919192b, 0x08190808, 0x1919192b, 0x082b192b, 0x1919192b, + 0x19080808, 0x1919192b, 0x08080808, 0x19192b08, 0x0808082b, 0x19192b08, 0x08081919, 0x19192b08, + 0x08082b08, 0x19192b08, 0x08190819, 0x19192b08, 0x08191908, 0x19192b08, 0x082b0808, 0x19192b08, + 0x19080819, 0x19192b08, 0x19081908, 0x19192b08, 0x19190808, 0x19192b08, 0x19192b2b, 0x19192b08, + 0x2b080808, 0x19192b08, 0x08080819, 0x19192b19, 0x08081908, 0x19192b19, 0x08190808, 0x19192b19, + 0x19080808, 0x19192b19, 0x08080808, 0x19192b2b, 0x08192b19, 0x19192b2b, 0x2b081919, 0x19192b2b, + 0x2b2b2b08, 0x19192b2b, 0x08080819, 0x192b0808, 0x08081908, 0x192b0808, 0x0808192b, 0x192b0808, + 0x08190808, 0x192b0808, 0x0819082b, 0x192b0808, 0x08191919, 0x192b0808, 0x08192b08, 0x192b0808, + 0x082b0819, 0x192b0808, 0x082b1908, 0x192b0808, 0x19080808, 0x192b0808, 0x19081919, 0x192b0808, + 0x19082b08, 0x192b0808, 0x19190819, 0x192b0808, 0x19191908, 0x192b0808, 0x192b0808, 0x192b0808, + 0x2b081908, 0x192b0808, 0x2b190808, 0x192b0808, 0x08080808, 0x192b0819, 0x0808082b, 0x192b0819, + 0x08081919, 0x192b0819, 0x08082b08, 0x192b0819, 0x08190819, 0x192b0819, 0x08191908, 0x192b0819, + 0x082b0808, 0x192b0819, 0x19080819, 0x192b0819, 0x19081908, 0x192b0819, 0x19190808, 0x192b0819, + 0x2b080808, 0x192b0819, 0x2b192b19, 0x192b0819, 0x08081908, 0x192b082b, 0x08190808, 0x192b082b, + 0x19080808, 0x192b082b, 0x1919192b, 0x192b082b, 0x2b2b0819, 0x192b082b, 0x08080808, 0x192b1908, + 0x08081919, 0x192b1908, 0x08082b08, 0x192b1908, 0x08190819, 0x192b1908, 0x08191908, 0x192b1908, + 0x082b0808, 0x192b1908, 0x19080819, 0x192b1908, 0x19081908, 0x192b1908, 0x19190808, 0x192b1908, + 0x2b080808, 0x192b1908, 0x08080819, 0x192b1919, 0x08081908, 0x192b1919, 0x08190808, 0x192b1919, + 0x19080808, 0x192b1919, 0x19082b2b, 0x192b1919, 0x192b2b08, 0x192b1919, 0x2b19082b, 0x192b1919, + 0x08080808, 0x192b192b, 0x2b191908, 0x192b192b, 0x08080819, 0x192b2b08, 0x08081908, 0x192b2b08, + 0x08190808, 0x192b2b08, 0x192b1919, 0x192b2b08, 0x2b192b08, 0x192b2b08, 0x08080808, 0x192b2b19, + 0x082b2b2b, 0x192b2b19, 0x1908082b, 0x192b2b2b, 0x2b2b0819, 0x192b2b2b, 0x08080808, 0x2b080808, + 0x0808082b, 0x2b080808, 0x08081919, 0x2b080808, 0x08082b08, 0x2b080808, 0x08190819, 0x2b080808, + 0x08191908, 0x2b080808, 0x08192b19, 0x2b080808, 0x082b0808, 0x2b080808, 0x082b1919, 0x2b080808, + 0x19080819, 0x2b080808, 0x19081908, 0x2b080808, 0x19190808, 0x2b080808, 0x1919082b, 0x2b080808, + 0x19191919, 0x2b080808, 0x19192b08, 0x2b080808, 0x192b0819, 0x2b080808, 0x2b080808, 0x2b080808, + 0x2b081919, 0x2b080808, 0x2b190819, 0x2b080808, 0x2b191908, 0x2b080808, 0x08080819, 0x2b080819, + 0x08081908, 0x2b080819, 0x08082b19, 0x2b080819, 0x08190808, 0x2b080819, 0x0819082b, 0x2b080819, + 0x08191919, 0x2b080819, 0x08192b08, 0x2b080819, 0x082b0819, 0x2b080819, 0x082b1908, 0x2b080819, + 0x19080808, 0x2b080819, 0x1908082b, 0x2b080819, 0x19081919, 0x2b080819, 0x19082b08, 0x2b080819, + 0x19190819, 0x2b080819, 0x19191908, 0x2b080819, 0x2b080819, 0x2b080819, 0x2b081908, 0x2b080819, + 0x2b190808, 0x2b080819, 0x2b2b2b19, 0x2b080819, 0x08080808, 0x2b08082b, 0x08081919, 0x2b08082b, + 0x08082b2b, 0x2b08082b, 0x08190819, 0x2b08082b, 0x08191908, 0x2b08082b, 0x19080819, 0x2b08082b, + 0x19081908, 0x2b08082b, 0x19190808, 0x2b08082b, 0x08080819, 0x2b081908, 0x08081908, 0x2b081908, + 0x0808192b, 0x2b081908, 0x08082b19, 0x2b081908, 0x08190808, 0x2b081908, 0x0819082b, 0x2b081908, + 0x08191919, 0x2b081908, 0x08192b08, 0x2b081908, 0x082b0819, 0x2b081908, 0x19080808, 0x2b081908, + 0x1908082b, 0x2b081908, 0x19081919, 0x2b081908, 0x19082b08, 0x2b081908, 0x19190819, 0x2b081908, + 0x19191908, 0x2b081908, 0x192b0808, 0x2b081908, 0x2b080819, 0x2b081908, 0x2b081908, 0x2b081908, + 0x2b190808, 0x2b081908, 0x08080808, 0x2b081919, 0x0808082b, 0x2b081919, 0x08081919, 0x2b081919, + 0x08082b08, 0x2b081919, 0x08190819, 0x2b081919, 0x08191908, 0x2b081919, 0x082b0808, 0x2b081919, + 0x19080819, 0x2b081919, 0x19081908, 0x2b081919, 0x19190808, 0x2b081919, 0x2b080808, 0x2b081919, + 0x2b082b2b, 0x2b081919, 0x08080819, 0x2b08192b, 0x08081908, 0x2b08192b, 0x08190808, 0x2b08192b, + 0x082b2b19, 0x2b08192b, 0x19080808, 0x2b08192b, 0x08080808, 0x2b082b08, 0x08081919, 0x2b082b08, + 0x08190819, 0x2b082b08, 0x08191908, 0x2b082b08, 0x19080819, 0x2b082b08, 0x19081908, 0x2b082b08, + 0x19190808, 0x2b082b08, 0x2b2b082b, 0x2b082b08, 0x08080819, 0x2b082b19, 0x08081908, 0x2b082b19, + 0x19080808, 0x2b082b19, 0x192b1919, 0x2b082b19, 0x082b082b, 0x2b082b2b, 0x19192b08, 0x2b082b2b, + 0x19192b2b, 0x2b082b2b, 0x2b08082b, 0x2b082b2b, 0x2b2b082b, 0x2b082b2b, 0x08080819, 0x2b190808, + 0x08081908, 0x2b190808, 0x08082b19, 0x2b190808, 0x08190808, 0x2b190808, 0x0819082b, 0x2b190808, + 0x08191919, 0x2b190808, 0x08192b08, 0x2b190808, 0x082b1908, 0x2b190808, 0x19080808, 0x2b190808, + 0x1908082b, 0x2b190808, 0x19081919, 0x2b190808, 0x19082b08, 0x2b190808, 0x19190819, 0x2b190808, + 0x19191908, 0x2b190808, 0x192b0808, 0x2b190808, 0x2b080819, 0x2b190808, 0x2b081908, 0x2b190808, + 0x2b190808, 0x2b190808, 0x08080808, 0x2b190819, 0x08081919, 0x2b190819, 0x08190819, 0x2b190819, + 0x08191908, 0x2b190819, 0x19080819, 0x2b190819, 0x19081908, 0x2b190819, 0x19190808, 0x2b190819, + 0x19192b2b, 0x2b190819, 0x08080819, 0x2b19082b, 0x08081908, 0x2b19082b, 0x08190808, 0x2b19082b, + 0x19080808, 0x2b19082b, 0x2b2b192b, 0x2b19082b, 0x08080808, 0x2b191908, 0x0808082b, 0x2b191908, + 0x08081919, 0x2b191908, 0x08082b08, 0x2b191908, 0x08190819, 0x2b191908, 0x08191908, 0x2b191908, + 0x082b0808, 0x2b191908, 0x19080819, 0x2b191908, 0x19081908, 0x2b191908, 0x19190808, 0x2b191908, + 0x2b080808, 0x2b191908, 0x2b19192b, 0x2b191908, 0x08080819, 0x2b191919, 0x08081908, 0x2b191919, + 0x08190808, 0x2b191919, 0x19080808, 0x2b191919, 0x2b192b08, 0x2b191919, 0x2b2b0819, 0x2b191919, + 0x08080808, 0x2b19192b, 0x1908192b, 0x2b19192b, 0x192b1908, 0x2b19192b, 0x08080819, 0x2b192b08, + 0x08081908, 0x2b192b08, 0x08190808, 0x2b192b08, 0x082b192b, 0x2b192b08, 0x19080808, 0x2b192b08, + 0x2b2b2b19, 0x2b192b08, 0x08080808, 0x2b192b19, 0x19082b19, 0x2b192b19, 0x1919082b, 0x2b192b19, + 0x2b190808, 0x2b192b2b, 0x08080808, 0x2b2b0808, 0x08081919, 0x2b2b0808, 0x08082b2b, 0x2b2b0808, + 0x08191908, 0x2b2b0808, 0x082b082b, 0x2b2b0808, 0x082b2b2b, 0x2b2b0808, 0x19080819, 0x2b2b0808, + 0x19081908, 0x2b2b0808, 0x19190808, 0x2b2b0808, 0x2b2b082b, 0x2b2b0808, 0x2b2b2b2b, 0x2b2b0808, + 0x19080808, 0x2b2b0819, 0x192b1919, 0x2b2b0819, 0x0808082b, 0x2b2b082b, 0x08082b2b, 0x2b2b082b, + 0x082b082b, 0x2b2b082b, 0x082b2b08, 0x2b2b082b, 0x082b2b2b, 0x2b2b082b, 0x2b08082b, 0x2b2b082b, + 0x2b082b08, 0x2b2b082b, 0x2b082b2b, 0x2b2b082b, 0x2b2b2b08, 0x2b2b082b, 0x08080819, 0x2b2b1908, + 0x08081908, 0x2b2b1908, 0x08190808, 0x2b2b1908, 0x19080808, 0x2b2b1908, 0x2b082b19, 0x2b2b1908, + 0x2b2b1908, 0x2b2b1908, 0x08080808, 0x2b2b1919, 0x08192b19, 0x2b2b1919, 0x19190819, 0x2b2b192b, + 0x08082b2b, 0x2b2b2b08, 0x082b2b08, 0x2b2b2b08, 0x2b2b082b, 0x2b2b2b08, 0x19191908, 0x2b2b2b19, + 0x2b08192b, 0x2b2b2b19, 0x08082b08, 0x2b2b2b2b, 0x08082b2b, 0x2b2b2b2b, 0x082b0808, 0x2b2b2b2b, + 0x082b082b, 0x2b2b2b2b, 0x082b2b08, 0x2b2b2b2b, 0x2b082b08, 0x2b2b2b2b, 0x2b2b2b2b, 0x2b2b2b2b) + return tbl[i] +} + // Expert-indexed Q4_K GEMV — MetalKqGemvK4's dot; the superblock index shifts by sel[slot] * esb. // x rides a float4 view: the 32-scalar-x-load form was load-issue-bound at the routed expert // shapes (nfe-row planes) — the vector view is 2.25x there (lab: bench_metal_moe_lab, 142→321 wGB/s). @@ -7972,6 +8287,67 @@ class MetalKqGemvIq3s { } } +// IQ2_S: the iq3s GEMV geometry (4 rows per simdgroup, dispatch rows/8) over the u64 grid read +// DIRECT off the hoisted constant table; per-16 UNSIGNED strips scale each 16-element half, +// d pre-eighth-ed at transcode - the fold is d8 * strip * signed_grid_byte. +[metal_dispatch(name = "enc_kq_iq2s_c", pso = "g_pso_kq_iq2s", tg = 64, grid = "rows/8", params = "rows : int64, n : int64")] +class MetalKqGemvIq2s { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d8 plane - the scale buffer bound at byte nsb*16 (s0off = the caller's doff) + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // 16B strips: 16 UNSIGNED per-16 sub-scales, byte view (same buffer at 0) + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array // iq2s quant plane, uint view (18 words per superblock: 8 idx, 8 signs, 2 qh) + @ssbo @binding = 3 @role = "read" @off = "xoff" @span = "n*4" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" @span = "rows*4" y : array + @uniform @binding = 5 ndim : uint + @uniform @binding = 6 ddim : uint + + [metal_kernel(name="metal_kq_gemv_iq2s_msl")] + def metal_kq_gemv_iq2s { + let lane = gl_SubgroupInvocationID + let ix = lane / 16u + let it = lane % 16u + let bu = it / 2u + let il = it % 2u + let nb = ndim / 256u + let first_row = (gl_WorkGroupID.x * 2u + gl_SubgroupID) * 4u + var sumf : float[4] + var ib = ix + while (ib < nb) { + let yb4 = ib * 64u + bu * 8u + il * 4u + let y0 = x[yb4] + let y1 = x[yb4 + 1u] + let y2 = x[yb4 + 2u] + let y3 = x[yb4 + 3u] + for [unroll_full] (r in range(4)) { + let blk = (first_row + uint(r)) * nb + ib + let qw = kqu[blk * 18u + bu] + let sgw = kqu[blk * 18u + 8u + bu] + let qh = (kqu[blk * 18u + 16u + bu / 4u] >> ((bu % 4u) * 8u)) & 255u + var accv = float4(0.0) + for [unroll_full] (wi in range(2)) { + let l = il * 2u + uint(wi) + let idx2 = (((qw >> (8u * l)) & 255u) | ((qh << (8u - 2u * l)) & 0x300u)) * 2u + let sgn = (sgw >> (8u * l)) & 255u + let w0 = iq3s_sw(iq2s_gw(int(idx2)), sgn & 15u) + let w1 = iq3s_sw(iq2s_gw(int(idx2 + 1u)), sgn >> 4u) + let ya = wi == 0 ? y0 : y2 + let yb = wi == 0 ? y1 : y3 + accv += ya * float4(iq3s_sx(w0, 0u), iq3s_sx(w0, 1u), iq3s_sx(w0, 2u), iq3s_sx(w0, 3u)) + accv += yb * float4(iq3s_sx(w1, 0u), iq3s_sx(w1, 1u), iq3s_sx(w1, 2u), iq3s_sx(w1, 3u)) + } + let sc = float(kscb[blk * 16u + bu * 2u + il]) + sumf[r] += float(kdh[blk]) * sc * (accv.x + accv.y + accv.z + accv.w) + } + ib += 2u + } + for [unroll_full] (r in range(4)) { + let s = simd_sum(sumf[r]) + if (lane == 0u && first_row + uint(r) < ddim) { + y[first_row + uint(r)] = s + } + } + } +} + // IQ3_XXS: the iq3s GEMV shape verbatim over the 24-word rows - the 1 KB halved grid as a // float4 magnitude slab, signs from the block's aux32 through the parity helper (no table), // the same (2ls+1)-strip x halved-d fold. 4 rows per simdgroup, dispatch rows/8. @@ -8373,6 +8749,166 @@ class MetalKqMvB8Iq3s { } } +// The iq2s B2/B4 pair: the iq3s shell over the u64 grid read direct - lane tx owns block tx, +// TWO grid words per 10-bit index; per-16 UNSIGNED strips give each half its own d8*strip. +[ |> template_struct_instance] +class template MetalKqMvIq2sT { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d8 plane - the scale buffer bound at byte nsb*16 + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // 16B strips: 16 UNSIGNED per-16 sub-scales (same buffer at 0) + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array // iq2s quant plane, uint view + @ssbo @binding = 3 @role = "read" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" y : array + @uniform @binding = 5 ka : KqMvArgs + @template_constant NR : int = 2 + @template_constant NRU : uint = 2u // NR's uint spelling (colbase math) - override BOTH together + @template_constant TILED : bool = false + + [metal_kernel] + def metal_kq_mv_iq2s { + let lane = gl_SubgroupInvocationID + let tx = lane % 8u + let ty = lane / 8u + let row = gl_WorkGroupID.x * 8u + gl_SubgroupID * 4u + ty + let colbase = gl_WorkGroupID.y * NRU // nolint:LINT002 dead in the TILED=false stamp - measured free + let nb = ka.ndim / 256u + let nb4 = ka.ndim / 4u + var sumf : float[NR] + var sb = 0u + while (sb < nb) { + let blk = row * nb + sb + let dl0 = float(kdh[blk]) * float(kscb[blk * 16u + tx * 2u]) + let dl1 = float(kdh[blk]) * float(kscb[blk * 16u + tx * 2u + 1u]) + let qw = kqu[blk * 18u + tx] + let sgw = kqu[blk * 18u + 8u + tx] + let qh = (kqu[blk * 18u + 16u + tx / 4u] >> ((tx % 4u) * 8u)) & 255u + var w : float4[8] + for [unroll_full] (l in range(4)) { + let lu = uint(l) + let idx2 = (((qw >> (8u * lu)) & 255u) | ((qh << (8u - 2u * lu)) & 0x300u)) * 2u + let sgn = (sgw >> (8u * lu)) & 255u + let dl = l < 2 ? dl0 : dl1 + let sw0 = iq3s_sw(iq2s_gw(int(idx2)), sgn & 15u) + let sw1 = iq3s_sw(iq2s_gw(int(idx2 + 1u)), sgn >> 4u) + w[l * 2] = float4(iq3s_sx(sw0, 0u), iq3s_sx(sw0, 1u), iq3s_sx(sw0, 2u), iq3s_sx(sw0, 3u)) * dl + w[l * 2 + 1] = float4(iq3s_sx(sw1, 0u), iq3s_sx(sw1, 1u), iq3s_sx(sw1, 2u), iq3s_sx(sw1, 3u)) * dl + } + let cx4 = sb * 64u + tx * 8u + static_if (TILED) { + for [unroll_full] (b in range(NR)) { + let xb4 = (colbase + uint(b)) * nb4 + cx4 + var acc = 0.0 + for [unroll_full] (u in range(8)) { + let v = x[xb4 + uint(u)] + acc += v.x * w[u].x + v.y * w[u].y + v.z * w[u].z + v.w * w[u].w + } + sumf[b] += acc + } + } else { + for [unroll_full] (b in range(NR)) { + let xb4 = uint(b) * nb4 + cx4 + var acc = 0.0 + for [unroll_full] (u in range(8)) { + let v = x[xb4 + uint(u)] + acc += v.x * w[u].x + v.y * w[u].y + v.z * w[u].z + v.w * w[u].w + } + sumf[b] += acc + } + } + sb++ + } + for [unroll_full] (b in range(NR)) { + var s = sumf[b] + s += simd_shuffle_down(s, 4u) + s += simd_shuffle_down(s, 2u) + s += simd_shuffle_down(s, 1u) + if (tx == 0u && row < ka.ddim && uint(b) < ka.nr) { + y[uint(b) * ka.ys + row] = s + } + } + } +} + +[metal_dispatch(name = "enc_kq_mvb2_iq2s_c", pso = "g_pso_kq_mvb2_iq2s", tg = 64, grid = "rows/8", params = "rows : int64")] +class MetalKqMvB2Iq2s : MetalKqMvIq2sT { + override NR = 2 + override NRU = 2u + override TILED = false +} + +[metal_dispatch(name = "enc_kq_mvb4_iq2s_c", pso = "g_pso_kq_mvb4_iq2s", tg = 64, grid = "rows/8, gcols", params = "rows : int64, gcols : int64")] +class MetalKqMvB4Iq2s : MetalKqMvIq2sT { + override NR = 4 + override NRU = 4u + override TILED = true +} + +// the iq2s B8 twin: the iq3s B8 shell (one superblock's X panel staged per threadgroup) with +// the u64 two-word compose; the grid read direct. +[metal_dispatch(name = "enc_kq_mvb8_iq2s_c", pso = "g_pso_kq_mvb8_iq2s", tgmem = "metal_kq_mvb8_iq2s_msl_tgmem", tg = 64, grid = "rows/8", params = "rows : int64")] +class MetalKqMvB8Iq2s { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d8 plane - the scale buffer bound at byte nsb*16 + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // 16B strips: 16 UNSIGNED per-16 sub-scales (same buffer at 0) + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array // iq2s quant plane, uint view + @ssbo @binding = 3 @role = "read" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" y : array + @uniform @binding = 5 ka : KqMvArgs + @workgroup txp : float4[512] // [8 streams x 64 float4] one superblock's X panel + + [metal_kernel(name="metal_kq_mvb8_iq2s_msl")] + def metal_kq_mvb8_iq2s { + let lid = gl_LocalInvocationID.x + let lane = gl_SubgroupInvocationID + let tx = lane % 8u + let ty = lane / 8u + let row = gl_WorkGroupID.x * 8u + gl_SubgroupID * 4u + ty + let nb = ka.ndim / 256u + let nb4 = ka.ndim / 4u + var acc : float4[8] + var sb = 0u + while (sb < nb) { + barrier() + for [unroll_full] (j in range(8)) { + let slot = uint(j) * 64u + lid + txp[slot] = x[(slot / 64u) * nb4 + sb * 64u + (slot % 64u)] + } + barrier() + let blk = row * nb + sb + let dl0 = float(kdh[blk]) * float(kscb[blk * 16u + tx * 2u]) + let dl1 = float(kdh[blk]) * float(kscb[blk * 16u + tx * 2u + 1u]) + let qw = kqu[blk * 18u + tx] + let sgw = kqu[blk * 18u + 8u + tx] + let qh = (kqu[blk * 18u + 16u + tx / 4u] >> ((tx % 4u) * 8u)) & 255u + var w : float4[8] + for [unroll_full] (l in range(4)) { + let lu = uint(l) + let idx2 = (((qw >> (8u * lu)) & 255u) | ((qh << (8u - 2u * lu)) & 0x300u)) * 2u + let sgn = (sgw >> (8u * lu)) & 255u + let dl = l < 2 ? dl0 : dl1 + let sw0 = iq3s_sw(iq2s_gw(int(idx2)), sgn & 15u) + let sw1 = iq3s_sw(iq2s_gw(int(idx2 + 1u)), sgn >> 4u) + w[l * 2] = float4(iq3s_sx(sw0, 0u), iq3s_sx(sw0, 1u), iq3s_sx(sw0, 2u), iq3s_sx(sw0, 3u)) * dl + w[l * 2 + 1] = float4(iq3s_sx(sw1, 0u), iq3s_sx(sw1, 1u), iq3s_sx(sw1, 2u), iq3s_sx(sw1, 3u)) * dl + } + let cxl = tx * 8u + for [unroll_full] (b in range(8)) { + for [unroll_full] (u in range(8)) { + acc[b] += txp[uint(b) * 64u + cxl + uint(u)] * w[u] + } + } + sb++ + } + for [unroll_full] (b in range(8)) { + var s = acc[b].x + acc[b].y + acc[b].z + acc[b].w + s += simd_shuffle_down(s, 4u) + s += simd_shuffle_down(s, 2u) + s += simd_shuffle_down(s, 1u) + if (tx == 0u && row < ka.ddim && uint(b) < ka.nr) { + y[uint(b) * ka.ys + row] = s + } + } + } +} + // The iq4xs B2/B4 pair: the K4T shell (row/colbase/TILED twins) over the per-32-block lane map of // the GEMV above — lane tx owns block tx of each superblock, 16 lo + 16 hi weights as float4s. [ |> template_struct_instance] diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das b/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das index 98e7dcbbf6..dad20b25dd 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das @@ -4411,7 +4411,7 @@ def private pf_devw_panel(enc : MetalComputeEncoder?; bwblob : MetalBuffer?; wbo //! quant superblocks, plus the k6 2B d tail def private pf_devw_panel_kq(enc : MetalComputeEncoder?; t : Model; fmt : KqFmt; woff : int64; bxh, by, bk, bn : MetalBuffer?; rows, d, kdim : int64; yoff : uint64) : bool { - if (fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s || fmt == KqFmt.iq3xxs || fmt == KqFmt.iq4nl || fmt == KqFmt.k2) { // no dev-W dequant kernel for these formats yet + if (fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s || fmt == KqFmt.iq3xxs || fmt == KqFmt.iq4nl || fmt == KqFmt.k2 || fmt == KqFmt.iq2s) { // no dev-W dequant kernel for these formats yet return false } let dq = fmt == KqFmt.k6 ? g_pf_pso_dq_k6 : (fmt == KqFmt.k4 ? g_pf_pso_dq_k4 : g_pf_pso_dq_k5) @@ -4600,6 +4600,10 @@ def private pf_enc_kq_site_mm(enc : MetalComputeEncoder?; t : Model; fmt : KqFmt enc_kq_mm_k2_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, by, yoff, bk, bn, mp, rows) return } + if (fmt == KqFmt.iq2s) { // the base mul_mm only - no tensor / tall / dev-W twins yet + enc_kq_mm_iq2s_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, by, yoff, bk, bn, mp, rows) + return + } //! deep-class arm: a panel dev-W would split past 8 tiles is DRAM-resident - the tall //! in-kernel-dequant stamp reads the quant plane once per 128-row tile instead of //! materializing and re-streaming a 2B/element f16 panel @@ -4711,6 +4715,8 @@ def private pf_enc_kq_gemv(enc : MetalComputeEncoder?; t : Model; fmt : KqFmt; w enc_kq_iq3s_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, xoff, by, 0ul, bn, bd, rows, n) } elif (fmt == KqFmt.iq3xxs) { enc_kq_iq3xxs_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, xoff, by, 0ul, bn, bd, rows, n) + } elif (fmt == KqFmt.iq2s) { + enc_kq_iq2s_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, xoff, by, 0ul, bn, bd, rows, n) } elif (fmt == KqFmt.iq4nl) { enc_kq_iq4nl_c(enc, bs.buf, bs.soff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, xoff, by, 0ul, bn, bd, rows, n) } elif (fmt == KqFmt.k2) { diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_shapes.das b/modules/dasLLAMA/dasllama/dasllama_metal_shapes.das index 1c6f5d5d96..569a74b44e 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_shapes.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_shapes.das @@ -180,7 +180,7 @@ def record_needs(var tab : table; missing : MetalNeed) { // formats) must DECLINE kquant_native instead of falling into a wrong-layout kernel branch // (the dispatchers below treat "not k4/k6" as k5) def kq_fmt_gpu_supported(f : KqFmt) : bool { - return f == KqFmt.q8 || f == KqFmt.k4 || f == KqFmt.k5 || f == KqFmt.k6 || f == KqFmt.iq4xs || f == KqFmt.k3 || f == KqFmt.iq3s || f == KqFmt.iq3xxs || f == KqFmt.iq4nl || f == KqFmt.k2 + return f == KqFmt.q8 || f == KqFmt.k4 || f == KqFmt.k5 || f == KqFmt.k6 || f == KqFmt.iq4xs || f == KqFmt.k3 || f == KqFmt.iq3s || f == KqFmt.iq3xxs || f == KqFmt.iq4nl || f == KqFmt.k2 || f == KqFmt.iq2s } def private kq_fmts_gpu_supported(a : array) : bool { @@ -243,7 +243,7 @@ def moe_site_ok(fmt : KqFmt; off, ege : int64) : bool { if (fmt == KqFmt.k4 || fmt == KqFmt.k5) { return off >= 0l && (off % 256l) == 0l && (ege % 256l) == 0l } - if (fmt == KqFmt.k6 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s || fmt == KqFmt.iq3xxs) { // the split scale form: off % 512 keeps the d-plane bind 4B-aligned + if (fmt == KqFmt.k6 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s || fmt == KqFmt.iq3xxs || fmt == KqFmt.iq2s) { // the split scale form: off % 512 keeps the d-plane bind 4B-aligned return off >= 0l && (off % 512l) == 0l && (ege % 256l) == 0l } if (fmt == KqFmt.q51) { // per-32 planes: off % 128 keeps the 20B/4B binds 16B-aligned diff --git a/modules/dasLLAMA/tests/_metal_kernel_common.das b/modules/dasLLAMA/tests/_metal_kernel_common.das index 090dbc52a9..6ebb73dcc9 100644 --- a/modules/dasLLAMA/tests/_metal_kernel_common.das +++ b/modules/dasLLAMA/tests/_metal_kernel_common.das @@ -173,8 +173,8 @@ let KQ_SAFE_SCALES = fixed_array(0.25, 0.5, 1.0, 0.125, 2.0, 0.75, 1.5, 0.375) // quant plane = hash bytes (any bit pattern is valid); scale plane K4/K5 = [d f16][dmin f16] // [12 packed 6-bit bytes] per superblock, K6 = [16B sub-scales] x nsb + f16 d tail, IQ4_XS (44) = // the same split form with [8 signed sub-scales in -31..31][8 zero] strips, Q3_K (3) = the k6 form over 96B quants -def kq_fill_planes(fmt, nsb : int; var kq : array; var ks : array) { - let qsb = fmt == 2 ? 64 : (fmt == 4 || fmt == 44 || fmt == 45 ? 128 : (fmt == 5 ? 160 : (fmt == 3 ? 96 : (fmt == 33 ? 104 : (fmt == 34 ? 96 : 192))))) +def kq_fill_planes(fmt, nsb : int; var kq : array; var ks : array) { // nolint:STYLE037 - the flat per-format fill ladder is the honest shape + let qsb = fmt == 2 ? 64 : (fmt == 23 ? 72 : (fmt == 4 || fmt == 44 || fmt == 45 ? 128 : (fmt == 5 ? 160 : (fmt == 3 ? 96 : (fmt == 33 ? 104 : (fmt == 34 ? 96 : 192)))))) kq |> resize(nsb * qsb) for (i in range(nsb * qsb)) { kq[i] = kq_hash_byte(i * 3 + fmt) @@ -201,10 +201,10 @@ def kq_fill_planes(fmt, nsb : int; var kq : array; var ks : array) ks[b * 2] = uint8(dbits & 0xFFu) ks[b * 2 + 1] = uint8(dbits >> 8u) } - } elif (fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34) { + } elif (fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 23) { ks |> resize(nsb * 18) for (i in range(nsb * 16)) { - ks[i] = fmt == 6 || fmt == 3 ? kq_hash_byte(i * 7 + 13) : uint8(i % 16 < 8 ? int(kq_hash_byte(i * 7 + 13)) % 63 - 31 : 0) // 44/33: signed strip bytes + zero pad + ks[i] = fmt == 6 || fmt == 3 || fmt == 23 ? kq_hash_byte(i * 7 + 13) : uint8(i % 16 < 8 ? int(kq_hash_byte(i * 7 + 13)) % 63 - 31 : 0) // 44/33: signed strip bytes + zero pad; 23: 16 unsigned strips } for (b in range(nsb)) { let dbits = f32_to_f16(KQ_SAFE_SCALES[b % 8]) @@ -284,6 +284,9 @@ def kq_row_ref(fmt, sb_base, nb, nsb_total : int; kq, ks : array; var wro } elif (fmt == 2) { dequant_k2_plane_superblock_at(kq, int64(sb * 64), ks, int64(sb * 16), int64(nsb_total * 16 + sb * 4), wrow, int64(ib * 256)) + } elif (fmt == 23) { + dequant_iq2s_plane_superblock_at(kq, int64(sb * 72), ks, int64(sb * 16), + int64(nsb_total * 16 + sb * 2), wrow, int64(ib * 256)) } else { dequant_k6_plane_superblock_at(kq, int64(sb * 192), ks, int64(sb * 16), int64(nsb_total * 16 + sb * 2), wrow, int64(ib * 256)) diff --git a/modules/dasLLAMA/tests/test_metal_gemm_kernels.das b/modules/dasLLAMA/tests/test_metal_gemm_kernels.das index 8af7395c8f..2db7885687 100644 --- a/modules/dasLLAMA/tests/test_metal_gemm_kernels.das +++ b/modules/dasLLAMA/tests/test_metal_gemm_kernels.das @@ -36,28 +36,28 @@ def private kq_mulmm_gate(t : T?; dev, queue; fmt : int; tensor : bool; m, kdim, : halfx ? (fmt == 4 ? MetalKqMulMmK4TH_metal_kq_mulmm_t_msl : (fmt == 5 ? MetalKqMulMmK5TH_metal_kq_mulmm_t_msl : MetalKqMulMmK6TH_metal_kq_mulmm_k6_t_msl)) : (fmt == 4 ? MetalKqMulMmK4T_metal_kq_mulmm_t_msl : (fmt == 5 ? MetalKqMulMmK5T_metal_kq_mulmm_t_msl : MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl))) - : (fmt == 45 ? MetalKqMulMmIq4nl_metal_kq_mulmm_msl : (fmt == 2 ? MetalKqMulMmK2_metal_kq_mulmm_msl : (fmt == 34 ? MetalKqMulMmIq3xxs_metal_kq_mulmm_msl : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl : MetalKqMulMmK6_metal_kq_mulmm_msl))))))))) + : (fmt == 23 ? MetalKqMulMmIq2s_metal_kq_mulmm_msl : (fmt == 45 ? MetalKqMulMmIq4nl_metal_kq_mulmm_msl : (fmt == 2 ? MetalKqMulMmK2_metal_kq_mulmm_msl : (fmt == 34 ? MetalKqMulMmIq3xxs_metal_kq_mulmm_msl : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl : MetalKqMulMmK6_metal_kq_mulmm_msl)))))))))) let entry = (tensor ? (tall ? (fmt == 4 ? MetalKqMulMmK4TH128_metal_kq_mulmm_t_msl_entry : (fmt == 5 ? MetalKqMulMmK5TH128_metal_kq_mulmm_t_msl_entry : MetalKqMulMmK6TH128_metal_kq_mulmm_k6_t_msl_entry)) : halfx ? (fmt == 4 ? MetalKqMulMmK4TH_metal_kq_mulmm_t_msl_entry : (fmt == 5 ? MetalKqMulMmK5TH_metal_kq_mulmm_t_msl_entry : MetalKqMulMmK6TH_metal_kq_mulmm_k6_t_msl_entry)) : (fmt == 4 ? MetalKqMulMmK4T_metal_kq_mulmm_t_msl_entry : (fmt == 5 ? MetalKqMulMmK5T_metal_kq_mulmm_t_msl_entry : MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl_entry))) - : (fmt == 45 ? MetalKqMulMmIq4nl_metal_kq_mulmm_msl_entry : (fmt == 2 ? MetalKqMulMmK2_metal_kq_mulmm_msl_entry : (fmt == 34 ? MetalKqMulMmIq3xxs_metal_kq_mulmm_msl_entry : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl_entry : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl_entry : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl_entry : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_entry : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_entry : MetalKqMulMmK6_metal_kq_mulmm_msl_entry))))))))) + : (fmt == 23 ? MetalKqMulMmIq2s_metal_kq_mulmm_msl_entry : (fmt == 45 ? MetalKqMulMmIq4nl_metal_kq_mulmm_msl_entry : (fmt == 2 ? MetalKqMulMmK2_metal_kq_mulmm_msl_entry : (fmt == 34 ? MetalKqMulMmIq3xxs_metal_kq_mulmm_msl_entry : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl_entry : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl_entry : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl_entry : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_entry : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_entry : MetalKqMulMmK6_metal_kq_mulmm_msl_entry)))))))))) let fm = (tensor ? (tall ? (fmt == 4 ? MetalKqMulMmK4TH128_metal_kq_mulmm_t_msl_fastmath : (fmt == 5 ? MetalKqMulMmK5TH128_metal_kq_mulmm_t_msl_fastmath : MetalKqMulMmK6TH128_metal_kq_mulmm_k6_t_msl_fastmath)) : halfx ? (fmt == 4 ? MetalKqMulMmK4TH_metal_kq_mulmm_t_msl_fastmath : (fmt == 5 ? MetalKqMulMmK5TH_metal_kq_mulmm_t_msl_fastmath : MetalKqMulMmK6TH_metal_kq_mulmm_k6_t_msl_fastmath)) : (fmt == 4 ? MetalKqMulMmK4T_metal_kq_mulmm_t_msl_fastmath : (fmt == 5 ? MetalKqMulMmK5T_metal_kq_mulmm_t_msl_fastmath : MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl_fastmath))) - : (fmt == 45 ? MetalKqMulMmIq4nl_metal_kq_mulmm_msl_fastmath : (fmt == 2 ? MetalKqMulMmK2_metal_kq_mulmm_msl_fastmath : (fmt == 34 ? MetalKqMulMmIq3xxs_metal_kq_mulmm_msl_fastmath : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl_fastmath : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl_fastmath : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl_fastmath : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_fastmath : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_fastmath : MetalKqMulMmK6_metal_kq_mulmm_msl_fastmath))))))))) + : (fmt == 23 ? MetalKqMulMmIq2s_metal_kq_mulmm_msl_fastmath : (fmt == 45 ? MetalKqMulMmIq4nl_metal_kq_mulmm_msl_fastmath : (fmt == 2 ? MetalKqMulMmK2_metal_kq_mulmm_msl_fastmath : (fmt == 34 ? MetalKqMulMmIq3xxs_metal_kq_mulmm_msl_fastmath : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl_fastmath : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl_fastmath : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl_fastmath : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_fastmath : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_fastmath : MetalKqMulMmK6_metal_kq_mulmm_msl_fastmath)))))))))) let tgm = (tensor ? (tall ? (fmt == 4 ? MetalKqMulMmK4TH128_metal_kq_mulmm_t_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5TH128_metal_kq_mulmm_t_msl_tgmem : MetalKqMulMmK6TH128_metal_kq_mulmm_k6_t_msl_tgmem)) : halfx ? (fmt == 4 ? MetalKqMulMmK4TH_metal_kq_mulmm_t_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5TH_metal_kq_mulmm_t_msl_tgmem : MetalKqMulMmK6TH_metal_kq_mulmm_k6_t_msl_tgmem)) : (fmt == 4 ? MetalKqMulMmK4T_metal_kq_mulmm_t_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5T_metal_kq_mulmm_t_msl_tgmem : MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl_tgmem))) - : (fmt == 45 ? MetalKqMulMmIq4nl_metal_kq_mulmm_msl_tgmem : (fmt == 2 ? MetalKqMulMmK2_metal_kq_mulmm_msl_tgmem : (fmt == 34 ? MetalKqMulMmIq3xxs_metal_kq_mulmm_msl_tgmem : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl_tgmem : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl_tgmem : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl_tgmem : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_tgmem : MetalKqMulMmK6_metal_kq_mulmm_msl_tgmem))))))))) + : (fmt == 23 ? MetalKqMulMmIq2s_metal_kq_mulmm_msl_tgmem : (fmt == 45 ? MetalKqMulMmIq4nl_metal_kq_mulmm_msl_tgmem : (fmt == 2 ? MetalKqMulMmK2_metal_kq_mulmm_msl_tgmem : (fmt == 34 ? MetalKqMulMmIq3xxs_metal_kq_mulmm_msl_tgmem : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl_tgmem : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl_tgmem : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl_tgmem : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_tgmem : MetalKqMulMmK6_metal_kq_mulmm_msl_tgmem)))))))))) var pso = pipeline_from_source(dev, src, entry, fm, err) if (tensor && pso == null && !metal4_tensor_available(dev)) { to_log(LOG_INFO, "{tag}: no Metal-4 tensor toolchain on this box - arm skipped\n") @@ -123,7 +123,7 @@ def private kq_mulmm_gate(t : T?; dev, queue; fmt : int; tensor : bool; m, kdim, let ran = with_compute_encoder(queue, err) $(enc : MetalComputeEncoder?) { metal_set_pipeline(enc, pso) metal_set_threadgroup_memory_length(enc, tgm, 0) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs: the f16 d tail + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs: the f16 d tail metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bx, 0ul, 3) @@ -134,7 +134,7 @@ def private kq_mulmm_gate(t : T?; dev, queue; fmt : int; tensor : bool; m, kdim, if (rem_pso != null) { metal_set_pipeline(enc, rem_pso) metal_set_threadgroup_memory_length(enc, fmt == 4 ? MetalKqMulMmK4TH_metal_kq_mulmm_t_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5TH_metal_kq_mulmm_t_msl_tgmem : MetalKqMulMmK6TH_metal_kq_mulmm_k6_t_msl_tgmem), 0) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 ? uint64(nsb * 16) : 0ul, 0) + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 ? uint64(nsb * 16) : 0ul, 0) metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bx, uint64(r0 * kdim * 2), 3) @@ -484,7 +484,7 @@ def private kq_devw_gate(t : T?; dev, queue; fmt : int; m, kdim, ndim : int) { var dka = CvtArgs(total = uint(nblk)) let ran = with_compute_encoder(queue, err) $(enc : MetalComputeEncoder?) { metal_set_pipeline(enc, dq_pso) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs: the f16 d tail + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs: the f16 d tail metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bwh, 0ul, 3) @@ -1461,7 +1461,7 @@ def private moe_mulmm_kq_gate(t : T?; dev, queue; fmt : int; kdim, ndim : int; c let ran = with_compute_encoder(queue, err) $(enc : MetalComputeEncoder?) { metal_set_pipeline(enc, pso) metal_set_threadgroup_memory_length(enc, tgm, 0) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs: the f16 d tail + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs: the f16 d tail metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bx, 0ul, 3) @@ -1498,7 +1498,7 @@ def private moe_mulmm_kq_gate(t : T?; dev, queue; fmt : int; kdim, ndim : int; c metal_set_threadgroup_memory_length(enc, fmt == 4 ? MetalMoeMulMmK4TH_metal_moe_mulmm_k45_t_msl_tgmem : (fmt == 5 ? MetalMoeMulMmK5TH_metal_moe_mulmm_k45_t_msl_tgmem : MetalMoeMulMmK6TH_metal_moe_mulmm_k6_t_msl_tgmem), 0) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 ? uint64(nsb * 16) : 0ul, 0) + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 ? uint64(nsb * 16) : 0ul, 0) metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bxh, 0ul, 3) @@ -1538,7 +1538,7 @@ def private moe_mulmm_kq_gate(t : T?; dev, queue; fmt : int; kdim, ndim : int; c metal_set_threadgroup_memory_length(enc, fmt == 4 ? MetalMoeMulMmK4TH128_metal_moe_mulmm_k45_t_msl_tgmem : (fmt == 5 ? MetalMoeMulMmK5TH128_metal_moe_mulmm_k45_t_msl_tgmem : MetalMoeMulMmK6TH128_metal_moe_mulmm_k6_t_msl_tgmem), 0) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 ? uint64(nsb * 16) : 0ul, 0) + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 ? uint64(nsb * 16) : 0ul, 0) metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bxh, 0ul, 3) @@ -1551,7 +1551,7 @@ def private moe_mulmm_kq_gate(t : T?; dev, queue; fmt : int; kdim, ndim : int; c metal_set_threadgroup_memory_length(enc, fmt == 4 ? MetalMoeMulMmK4THR_metal_moe_mulmm_k45_t_msl_tgmem : (fmt == 5 ? MetalMoeMulMmK5THR_metal_moe_mulmm_k45_t_msl_tgmem : MetalMoeMulMmK6THR_metal_moe_mulmm_k6_t_msl_tgmem), 0) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 ? uint64(nsb * 16) : 0ul, 0) + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 ? uint64(nsb * 16) : 0ul, 0) metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bxh, 0ul, 3) @@ -2059,6 +2059,8 @@ def test_metal_gemm_kernels(t : T?) { kq_mulmm_gate(t, dev, queue, 45, false, 32, 256, 128) kq_mulmm_gate(t, dev, queue, 2, false, 64, 512, 64) // k2: the base form only (no tensor / tall twins) kq_mulmm_gate(t, dev, queue, 2, false, 32, 256, 128) + kq_mulmm_gate(t, dev, queue, 23, false, 64, 512, 64) // iq2s: the base form only (no tensor / tall twins) + kq_mulmm_gate(t, dev, queue, 23, false, 32, 256, 128) q8_mulmm_t_gate(t, dev, queue, 64, 256, 64) q8_mulmm_t_gate(t, dev, queue, 64, 256, 64, halfx = true) q8_mulmm_t_gate(t, dev, queue, 64, 160, 64) // kdim % 64 == 32: the bk=64 helper's 32-tail chunk diff --git a/modules/dasLLAMA/tests/test_metal_gemv_kernels.das b/modules/dasLLAMA/tests/test_metal_gemv_kernels.das index 23c4946f4c..699c8a9f2a 100644 --- a/modules/dasLLAMA/tests/test_metal_gemv_kernels.das +++ b/modules/dasLLAMA/tests/test_metal_gemv_kernels.das @@ -23,12 +23,12 @@ require math // ===== single-stream kq GEMV (MetalKqGemvK4 / K5 / K5C / K6) ===== def private kq_gemv_gate(t : T?; dev, queue; vform : string; n, d : int) { - let fmt = vform == "k4" ? 4 : (vform == "k6" ? 6 : (vform == "iq4xs" ? 44 : (vform == "k3" ? 3 : (vform == "iq3s" ? 33 : (vform == "iq3xxs" ? 34 : (vform == "iq4nl" ? 45 : (vform == "k2" ? 2 : 5))))))) + let fmt = vform == "k4" ? 4 : (vform == "k6" ? 6 : (vform == "iq4xs" ? 44 : (vform == "k3" ? 3 : (vform == "iq3s" ? 33 : (vform == "iq3xxs" ? 34 : (vform == "iq4nl" ? 45 : (vform == "iq2s" ? 23 : (vform == "k2" ? 2 : 5)))))))) let tag = "kq_gemv_{vform} n={n} d={d}" var err : string - let src = vform == "k2" ? metal_kq_gemv_k2_msl : (vform == "iq4nl" ? metal_kq_gemv_iq4nl_msl : (vform == "iq3xxs" ? metal_kq_gemv_iq3xxs_msl : (vform == "iq3s" ? metal_kq_gemv_iq3s_msl : (vform == "k3" ? metal_kq_gemv_k3_msl : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl : (vform == "k4" ? metal_kq_gemv_k4_msl : (vform == "k5" ? metal_kq_gemv_k5_msl : (vform == "k5c" ? metal_kq_gemv_k5c_msl : metal_kq_gemv_k6_msl)))))))) - let entry = vform == "k2" ? metal_kq_gemv_k2_msl_entry : (vform == "iq4nl" ? metal_kq_gemv_iq4nl_msl_entry : (vform == "iq3xxs" ? metal_kq_gemv_iq3xxs_msl_entry : (vform == "iq3s" ? metal_kq_gemv_iq3s_msl_entry : (vform == "k3" ? metal_kq_gemv_k3_msl_entry : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl_entry : (vform == "k4" ? metal_kq_gemv_k4_msl_entry : (vform == "k5" ? metal_kq_gemv_k5_msl_entry : (vform == "k5c" ? metal_kq_gemv_k5c_msl_entry : metal_kq_gemv_k6_msl_entry)))))))) - let fm = vform == "k2" ? metal_kq_gemv_k2_msl_fastmath : (vform == "iq4nl" ? metal_kq_gemv_iq4nl_msl_fastmath : (vform == "iq3xxs" ? metal_kq_gemv_iq3xxs_msl_fastmath : (vform == "iq3s" ? metal_kq_gemv_iq3s_msl_fastmath : (vform == "k3" ? metal_kq_gemv_k3_msl_fastmath : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl_fastmath : (vform == "k4" ? metal_kq_gemv_k4_msl_fastmath : (vform == "k5" ? metal_kq_gemv_k5_msl_fastmath : (vform == "k5c" ? metal_kq_gemv_k5c_msl_fastmath : metal_kq_gemv_k6_msl_fastmath)))))))) + let src = vform == "iq2s" ? metal_kq_gemv_iq2s_msl : (vform == "k2" ? metal_kq_gemv_k2_msl : (vform == "iq4nl" ? metal_kq_gemv_iq4nl_msl : (vform == "iq3xxs" ? metal_kq_gemv_iq3xxs_msl : (vform == "iq3s" ? metal_kq_gemv_iq3s_msl : (vform == "k3" ? metal_kq_gemv_k3_msl : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl : (vform == "k4" ? metal_kq_gemv_k4_msl : (vform == "k5" ? metal_kq_gemv_k5_msl : (vform == "k5c" ? metal_kq_gemv_k5c_msl : metal_kq_gemv_k6_msl))))))))) + let entry = vform == "iq2s" ? metal_kq_gemv_iq2s_msl_entry : (vform == "k2" ? metal_kq_gemv_k2_msl_entry : (vform == "iq4nl" ? metal_kq_gemv_iq4nl_msl_entry : (vform == "iq3xxs" ? metal_kq_gemv_iq3xxs_msl_entry : (vform == "iq3s" ? metal_kq_gemv_iq3s_msl_entry : (vform == "k3" ? metal_kq_gemv_k3_msl_entry : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl_entry : (vform == "k4" ? metal_kq_gemv_k4_msl_entry : (vform == "k5" ? metal_kq_gemv_k5_msl_entry : (vform == "k5c" ? metal_kq_gemv_k5c_msl_entry : metal_kq_gemv_k6_msl_entry))))))))) + let fm = vform == "iq2s" ? metal_kq_gemv_iq2s_msl_fastmath : (vform == "k2" ? metal_kq_gemv_k2_msl_fastmath : (vform == "iq4nl" ? metal_kq_gemv_iq4nl_msl_fastmath : (vform == "iq3xxs" ? metal_kq_gemv_iq3xxs_msl_fastmath : (vform == "iq3s" ? metal_kq_gemv_iq3s_msl_fastmath : (vform == "k3" ? metal_kq_gemv_k3_msl_fastmath : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl_fastmath : (vform == "k4" ? metal_kq_gemv_k4_msl_fastmath : (vform == "k5" ? metal_kq_gemv_k5_msl_fastmath : (vform == "k5c" ? metal_kq_gemv_k5c_msl_fastmath : metal_kq_gemv_k6_msl_fastmath))))))))) var pso = pipeline_from_source(dev, src, entry, fm, err) t |> success(pso != null, "{tag}: pipeline: {err}") return if (pso == null) @@ -66,7 +66,7 @@ def private kq_gemv_gate(t : T?; dev, queue; vform : string; n, d : int) { var by = buf_fill(dev, d, -1000.0) var bn = buf_u32(dev, uint(n)) var bd = buf_u32(dev, uint(d)) - let groups = vform == "k5c" ? (d + 1) / 2 : (vform == "iq3s" || vform == "iq3xxs" ? (d + 7) / 8 : (d + 3) / 4) + let groups = vform == "k5c" ? (d + 1) / 2 : (vform == "iq3s" || vform == "iq3xxs" || vform == "iq2s" ? (d + 7) / 8 : (d + 3) / 4) let ran = with_compute_encoder(queue, err) $(enc : MetalComputeEncoder?) { metal_set_pipeline(enc, pso) if (vform == "iq4xs") { @@ -81,7 +81,7 @@ def private kq_gemv_gate(t : T?; dev, queue; vform : string; n, d : int) { if (vform == "iq4nl") { metal_set_threadgroup_memory_length(enc, metal_kq_gemv_iq4nl_msl_tgmem, 0) // the codebook slab } - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs/k3: the f16 d tail + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs/k3: the f16 d tail metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bx, 0ul, 3) @@ -114,15 +114,15 @@ def private kq_gemv_gate(t : T?; dev, queue; vform : string; n, d : int) { def private kq_mvb_gate(t : T?; dev, queue; fmt, bwidth, n, d, nr, ys : int) { let tag = "kq_mvb{bwidth}_k{fmt} n={n} d={d} nr={nr} ys={ys}" var err : string - let src = (bwidth == 2 ? (fmt == 45 ? MetalKqMvB2Iq4nl_metal_kq_mv_iq4nl_msl : (fmt == 2 ? MetalKqMvB2K2_metal_kq_mv_k2_msl : (fmt == 34 ? MetalKqMvB2Iq3xxs_metal_kq_mv_iq3xxs_msl : (fmt == 33 ? MetalKqMvB2Iq3s_metal_kq_mv_iq3s_msl : (fmt == 3 ? MetalKqMvB2K3_metal_kq_mv_k3_msl : (fmt == 44 ? MetalKqMvB2Iq4xs_metal_kq_mv_iq4xs_msl : (fmt == 4 ? MetalKqMvB2K4_metal_kq_mv_k4_msl : (fmt == 5 ? MetalKqMvB2K5_metal_kq_mv_k5_msl : MetalKqMvB2K6_metal_kq_mv_k6_msl)))))))) - : (bwidth == 4 ? (fmt == 45 ? MetalKqMvB4Iq4nl_metal_kq_mv_iq4nl_msl : (fmt == 2 ? MetalKqMvB4K2_metal_kq_mv_k2_msl : (fmt == 34 ? MetalKqMvB4Iq3xxs_metal_kq_mv_iq3xxs_msl : (fmt == 33 ? MetalKqMvB4Iq3s_metal_kq_mv_iq3s_msl : (fmt == 3 ? MetalKqMvB4K3_metal_kq_mv_k3_msl : (fmt == 44 ? MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl : (fmt == 4 ? MetalKqMvB4K4_metal_kq_mv_k4_msl : (fmt == 5 ? MetalKqMvB4K5_metal_kq_mv_k5_msl : MetalKqMvB4K6_metal_kq_mv_k6_msl)))))))) - : (fmt == 45 ? metal_kq_mvb8_iq4nl_msl : (fmt == 2 ? metal_kq_mvb8_k2_msl : (fmt == 34 ? metal_kq_mvb8_iq3xxs_msl : (fmt == 33 ? metal_kq_mvb8_iq3s_msl : (fmt == 3 ? metal_kq_mvb8_k3_msl : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl : metal_kq_mvb8_k6_msl)))))))))) - let entry = (bwidth == 2 ? (fmt == 45 ? MetalKqMvB2Iq4nl_metal_kq_mv_iq4nl_msl_entry : (fmt == 2 ? MetalKqMvB2K2_metal_kq_mv_k2_msl_entry : (fmt == 34 ? MetalKqMvB2Iq3xxs_metal_kq_mv_iq3xxs_msl_entry : (fmt == 33 ? MetalKqMvB2Iq3s_metal_kq_mv_iq3s_msl_entry : (fmt == 3 ? MetalKqMvB2K3_metal_kq_mv_k3_msl_entry : (fmt == 44 ? MetalKqMvB2Iq4xs_metal_kq_mv_iq4xs_msl_entry : (fmt == 4 ? MetalKqMvB2K4_metal_kq_mv_k4_msl_entry : (fmt == 5 ? MetalKqMvB2K5_metal_kq_mv_k5_msl_entry : MetalKqMvB2K6_metal_kq_mv_k6_msl_entry)))))))) - : (bwidth == 4 ? (fmt == 45 ? MetalKqMvB4Iq4nl_metal_kq_mv_iq4nl_msl_entry : (fmt == 2 ? MetalKqMvB4K2_metal_kq_mv_k2_msl_entry : (fmt == 34 ? MetalKqMvB4Iq3xxs_metal_kq_mv_iq3xxs_msl_entry : (fmt == 33 ? MetalKqMvB4Iq3s_metal_kq_mv_iq3s_msl_entry : (fmt == 3 ? MetalKqMvB4K3_metal_kq_mv_k3_msl_entry : (fmt == 44 ? MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl_entry : (fmt == 4 ? MetalKqMvB4K4_metal_kq_mv_k4_msl_entry : (fmt == 5 ? MetalKqMvB4K5_metal_kq_mv_k5_msl_entry : MetalKqMvB4K6_metal_kq_mv_k6_msl_entry)))))))) - : (fmt == 45 ? metal_kq_mvb8_iq4nl_msl_entry : (fmt == 2 ? metal_kq_mvb8_k2_msl_entry : (fmt == 34 ? metal_kq_mvb8_iq3xxs_msl_entry : (fmt == 33 ? metal_kq_mvb8_iq3s_msl_entry : (fmt == 3 ? metal_kq_mvb8_k3_msl_entry : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl_entry : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_entry : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_entry : metal_kq_mvb8_k6_msl_entry)))))))))) - let fm = (bwidth == 2 ? (fmt == 45 ? MetalKqMvB2Iq4nl_metal_kq_mv_iq4nl_msl_fastmath : (fmt == 2 ? MetalKqMvB2K2_metal_kq_mv_k2_msl_fastmath : (fmt == 34 ? MetalKqMvB2Iq3xxs_metal_kq_mv_iq3xxs_msl_fastmath : (fmt == 33 ? MetalKqMvB2Iq3s_metal_kq_mv_iq3s_msl_fastmath : (fmt == 3 ? MetalKqMvB2K3_metal_kq_mv_k3_msl_fastmath : (fmt == 44 ? MetalKqMvB2Iq4xs_metal_kq_mv_iq4xs_msl_fastmath : (fmt == 4 ? MetalKqMvB2K4_metal_kq_mv_k4_msl_fastmath : (fmt == 5 ? MetalKqMvB2K5_metal_kq_mv_k5_msl_fastmath : MetalKqMvB2K6_metal_kq_mv_k6_msl_fastmath)))))))) - : (bwidth == 4 ? (fmt == 45 ? MetalKqMvB4Iq4nl_metal_kq_mv_iq4nl_msl_fastmath : (fmt == 2 ? MetalKqMvB4K2_metal_kq_mv_k2_msl_fastmath : (fmt == 34 ? MetalKqMvB4Iq3xxs_metal_kq_mv_iq3xxs_msl_fastmath : (fmt == 33 ? MetalKqMvB4Iq3s_metal_kq_mv_iq3s_msl_fastmath : (fmt == 3 ? MetalKqMvB4K3_metal_kq_mv_k3_msl_fastmath : (fmt == 44 ? MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl_fastmath : (fmt == 4 ? MetalKqMvB4K4_metal_kq_mv_k4_msl_fastmath : (fmt == 5 ? MetalKqMvB4K5_metal_kq_mv_k5_msl_fastmath : MetalKqMvB4K6_metal_kq_mv_k6_msl_fastmath)))))))) - : (fmt == 45 ? metal_kq_mvb8_iq4nl_msl_fastmath : (fmt == 2 ? metal_kq_mvb8_k2_msl_fastmath : (fmt == 34 ? metal_kq_mvb8_iq3xxs_msl_fastmath : (fmt == 33 ? metal_kq_mvb8_iq3s_msl_fastmath : (fmt == 3 ? metal_kq_mvb8_k3_msl_fastmath : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl_fastmath : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_fastmath : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_fastmath : metal_kq_mvb8_k6_msl_fastmath)))))))))) + let src = (bwidth == 2 ? (fmt == 23 ? MetalKqMvB2Iq2s_metal_kq_mv_iq2s_msl : (fmt == 45 ? MetalKqMvB2Iq4nl_metal_kq_mv_iq4nl_msl : (fmt == 2 ? MetalKqMvB2K2_metal_kq_mv_k2_msl : (fmt == 34 ? MetalKqMvB2Iq3xxs_metal_kq_mv_iq3xxs_msl : (fmt == 33 ? MetalKqMvB2Iq3s_metal_kq_mv_iq3s_msl : (fmt == 3 ? MetalKqMvB2K3_metal_kq_mv_k3_msl : (fmt == 44 ? MetalKqMvB2Iq4xs_metal_kq_mv_iq4xs_msl : (fmt == 4 ? MetalKqMvB2K4_metal_kq_mv_k4_msl : (fmt == 5 ? MetalKqMvB2K5_metal_kq_mv_k5_msl : MetalKqMvB2K6_metal_kq_mv_k6_msl))))))))) + : (bwidth == 4 ? (fmt == 23 ? MetalKqMvB4Iq2s_metal_kq_mv_iq2s_msl : (fmt == 45 ? MetalKqMvB4Iq4nl_metal_kq_mv_iq4nl_msl : (fmt == 2 ? MetalKqMvB4K2_metal_kq_mv_k2_msl : (fmt == 34 ? MetalKqMvB4Iq3xxs_metal_kq_mv_iq3xxs_msl : (fmt == 33 ? MetalKqMvB4Iq3s_metal_kq_mv_iq3s_msl : (fmt == 3 ? MetalKqMvB4K3_metal_kq_mv_k3_msl : (fmt == 44 ? MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl : (fmt == 4 ? MetalKqMvB4K4_metal_kq_mv_k4_msl : (fmt == 5 ? MetalKqMvB4K5_metal_kq_mv_k5_msl : MetalKqMvB4K6_metal_kq_mv_k6_msl))))))))) + : (fmt == 23 ? metal_kq_mvb8_iq2s_msl : (fmt == 45 ? metal_kq_mvb8_iq4nl_msl : (fmt == 2 ? metal_kq_mvb8_k2_msl : (fmt == 34 ? metal_kq_mvb8_iq3xxs_msl : (fmt == 33 ? metal_kq_mvb8_iq3s_msl : (fmt == 3 ? metal_kq_mvb8_k3_msl : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl : metal_kq_mvb8_k6_msl))))))))))) + let entry = (bwidth == 2 ? (fmt == 23 ? MetalKqMvB2Iq2s_metal_kq_mv_iq2s_msl_entry : (fmt == 45 ? MetalKqMvB2Iq4nl_metal_kq_mv_iq4nl_msl_entry : (fmt == 2 ? MetalKqMvB2K2_metal_kq_mv_k2_msl_entry : (fmt == 34 ? MetalKqMvB2Iq3xxs_metal_kq_mv_iq3xxs_msl_entry : (fmt == 33 ? MetalKqMvB2Iq3s_metal_kq_mv_iq3s_msl_entry : (fmt == 3 ? MetalKqMvB2K3_metal_kq_mv_k3_msl_entry : (fmt == 44 ? MetalKqMvB2Iq4xs_metal_kq_mv_iq4xs_msl_entry : (fmt == 4 ? MetalKqMvB2K4_metal_kq_mv_k4_msl_entry : (fmt == 5 ? MetalKqMvB2K5_metal_kq_mv_k5_msl_entry : MetalKqMvB2K6_metal_kq_mv_k6_msl_entry))))))))) + : (bwidth == 4 ? (fmt == 23 ? MetalKqMvB4Iq2s_metal_kq_mv_iq2s_msl_entry : (fmt == 45 ? MetalKqMvB4Iq4nl_metal_kq_mv_iq4nl_msl_entry : (fmt == 2 ? MetalKqMvB4K2_metal_kq_mv_k2_msl_entry : (fmt == 34 ? MetalKqMvB4Iq3xxs_metal_kq_mv_iq3xxs_msl_entry : (fmt == 33 ? MetalKqMvB4Iq3s_metal_kq_mv_iq3s_msl_entry : (fmt == 3 ? MetalKqMvB4K3_metal_kq_mv_k3_msl_entry : (fmt == 44 ? MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl_entry : (fmt == 4 ? MetalKqMvB4K4_metal_kq_mv_k4_msl_entry : (fmt == 5 ? MetalKqMvB4K5_metal_kq_mv_k5_msl_entry : MetalKqMvB4K6_metal_kq_mv_k6_msl_entry))))))))) + : (fmt == 23 ? metal_kq_mvb8_iq2s_msl_entry : (fmt == 45 ? metal_kq_mvb8_iq4nl_msl_entry : (fmt == 2 ? metal_kq_mvb8_k2_msl_entry : (fmt == 34 ? metal_kq_mvb8_iq3xxs_msl_entry : (fmt == 33 ? metal_kq_mvb8_iq3s_msl_entry : (fmt == 3 ? metal_kq_mvb8_k3_msl_entry : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl_entry : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_entry : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_entry : metal_kq_mvb8_k6_msl_entry))))))))))) + let fm = (bwidth == 2 ? (fmt == 23 ? MetalKqMvB2Iq2s_metal_kq_mv_iq2s_msl_fastmath : (fmt == 45 ? MetalKqMvB2Iq4nl_metal_kq_mv_iq4nl_msl_fastmath : (fmt == 2 ? MetalKqMvB2K2_metal_kq_mv_k2_msl_fastmath : (fmt == 34 ? MetalKqMvB2Iq3xxs_metal_kq_mv_iq3xxs_msl_fastmath : (fmt == 33 ? MetalKqMvB2Iq3s_metal_kq_mv_iq3s_msl_fastmath : (fmt == 3 ? MetalKqMvB2K3_metal_kq_mv_k3_msl_fastmath : (fmt == 44 ? MetalKqMvB2Iq4xs_metal_kq_mv_iq4xs_msl_fastmath : (fmt == 4 ? MetalKqMvB2K4_metal_kq_mv_k4_msl_fastmath : (fmt == 5 ? MetalKqMvB2K5_metal_kq_mv_k5_msl_fastmath : MetalKqMvB2K6_metal_kq_mv_k6_msl_fastmath))))))))) + : (bwidth == 4 ? (fmt == 23 ? MetalKqMvB4Iq2s_metal_kq_mv_iq2s_msl_fastmath : (fmt == 45 ? MetalKqMvB4Iq4nl_metal_kq_mv_iq4nl_msl_fastmath : (fmt == 2 ? MetalKqMvB4K2_metal_kq_mv_k2_msl_fastmath : (fmt == 34 ? MetalKqMvB4Iq3xxs_metal_kq_mv_iq3xxs_msl_fastmath : (fmt == 33 ? MetalKqMvB4Iq3s_metal_kq_mv_iq3s_msl_fastmath : (fmt == 3 ? MetalKqMvB4K3_metal_kq_mv_k3_msl_fastmath : (fmt == 44 ? MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl_fastmath : (fmt == 4 ? MetalKqMvB4K4_metal_kq_mv_k4_msl_fastmath : (fmt == 5 ? MetalKqMvB4K5_metal_kq_mv_k5_msl_fastmath : MetalKqMvB4K6_metal_kq_mv_k6_msl_fastmath))))))))) + : (fmt == 23 ? metal_kq_mvb8_iq2s_msl_fastmath : (fmt == 45 ? metal_kq_mvb8_iq4nl_msl_fastmath : (fmt == 2 ? metal_kq_mvb8_k2_msl_fastmath : (fmt == 34 ? metal_kq_mvb8_iq3xxs_msl_fastmath : (fmt == 33 ? metal_kq_mvb8_iq3s_msl_fastmath : (fmt == 3 ? metal_kq_mvb8_k3_msl_fastmath : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl_fastmath : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_fastmath : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_fastmath : metal_kq_mvb8_k6_msl_fastmath))))))))))) var pso = pipeline_from_source(dev, src, entry, fm, err) t |> success(pso != null, "{tag}: pipeline: {err}") return if (pso == null) @@ -172,9 +172,9 @@ def private kq_mvb_gate(t : T?; dev, queue; fmt, bwidth, n, d, nr, ys : int) { metal_set_pipeline(enc, pso) if (bwidth == 8) { metal_set_threadgroup_memory_length(enc, - fmt == 2 ? metal_kq_mvb8_k2_msl_tgmem : (fmt == 45 ? metal_kq_mvb8_iq4nl_msl_tgmem : (fmt == 34 ? metal_kq_mvb8_iq3xxs_msl_tgmem : (fmt == 33 ? metal_kq_mvb8_iq3s_msl_tgmem : (fmt == 3 ? metal_kq_mvb8_k3_msl_tgmem : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl_tgmem : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_tgmem : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_tgmem : metal_kq_mvb8_k6_msl_tgmem))))))), 0) + fmt == 23 ? metal_kq_mvb8_iq2s_msl_tgmem : (fmt == 2 ? metal_kq_mvb8_k2_msl_tgmem : (fmt == 45 ? metal_kq_mvb8_iq4nl_msl_tgmem : (fmt == 34 ? metal_kq_mvb8_iq3xxs_msl_tgmem : (fmt == 33 ? metal_kq_mvb8_iq3s_msl_tgmem : (fmt == 3 ? metal_kq_mvb8_k3_msl_tgmem : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl_tgmem : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_tgmem : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_tgmem : metal_kq_mvb8_k6_msl_tgmem)))))))), 0) } - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs/k3: the f16 d tail + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs/k3: the f16 d tail metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bx, 0ul, 3) @@ -263,7 +263,7 @@ def private moe_kq_gemv_gate(t : T?; dev, queue; fmt, n, d, net, k, nst : int; p bxs = uint((perslot ? k : 1) * n), bys = uint(k * d), hasb = 0u) let ran = with_compute_encoder(queue, err) $(enc : MetalComputeEncoder?) { metal_set_pipeline(enc, pso) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs/k3: the f16 d tail + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs/k3: the f16 d tail metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bx, 0ul, 3) @@ -745,8 +745,10 @@ def test_metal_kq_gemv_kernels(t : T?) { kq_gemv_gate(t, dev, queue, "iq4nl", 512, 30) kq_gemv_gate(t, dev, queue, "k2", 1280, 11) kq_gemv_gate(t, dev, queue, "k2", 512, 30) + kq_gemv_gate(t, dev, queue, "iq2s", 1280, 11) + kq_gemv_gate(t, dev, queue, "iq2s", 512, 30) // the small-batch twins: nr = 2 / 3 (b4 col guard) / 6 (b8 pad), ys > d - for (fmt in [4, 5, 6, 44, 3, 33, 34, 45, 2]) { + for (fmt in [4, 5, 6, 44, 3, 33, 34, 45, 2, 23]) { kq_mvb_gate(t, dev, queue, fmt, 2, 768, 17, 2, 22) kq_mvb_gate(t, dev, queue, fmt, 4, 768, 17, 3, 22) kq_mvb_gate(t, dev, queue, fmt, 8, 768, 17, 6, 22) From 068b9788c2a030dcd750b4ff27e453b551d2f058 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Mon, 31 Aug 2026 09:43:41 -0700 Subject: [PATCH 39/61] dasLLAMA: IQ2_XS joins the kq lattice (CPU slice) - and the dead k2 repack-pointer arm falls The ksigns u64 tier: enum iq2xs (13), kernel/stream id 24. Each u16 qs word = a 9-bit index into iq2xs_grid[512] (u64 entries, magnitudes {8,25,43}) + a 7-bit KSIGNS_IQ2XS index - the iq3xxs sign machinery over the iq2s grid form. Scale row = IQ2_S's exactly (d EIGHTH-ed at transcode, 16 x (1+2s) per-16 strips); quant plane = the 64B qs region verbatim (16 uniform grp columns; a u16's bytes always share a column). Full CPU walk: transcode/dequant/dot/rows/grp dot/repack, every dispatch ladder, declined emitter stubs, tune-list probe rows, test arms across every gate (21 twinned) and a synthetic plane gate recomposing through grid + ksigns. The vehicle (a local Q8_0 -> IQ2_XS requant; llama.cpp's ftype puts q2_K on blk.0/1 ffn_down) exposed a LATENT bug from the iq2s walk: the big-load repack-pointer ternaries key the region's STREAM code, where `f == 2` is dead (q51 owns 2) and k2 arrives as 20 - since the iq2s walk every k2 region fell through to the newest format's planes, and here repack_k2_grp(mr 8) ground over the iq2xs planes (NaN logits, argmax token 0 forever, every serial gate green). The arms key `fk` now (QUIRK 26). Boris's team-lane global-init rule landed as QUIRK 27, with GEMM_REFERENCE_MR turned into a function as defense. Gates: test_kqformat 18/18, test_kquant 250 (232 pass, 18 env skips), probe GEN TUNE TEST OK, lint 0 (20 files). E2e: ids 64/64 vs llama.cpp - the arc's FIRST full-match greedy stream, no fork in the window - gen 23 t/s reference bodies. JIT emitter, Vulkan, Metal: pending. Co-Authored-By: Claude Fable 5 --- modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md | 43 +++++ modules/dasLLAMA/dasllama/dasllama_blocks.das | 3 + modules/dasLLAMA/dasllama/dasllama_common.das | 39 ++++- modules/dasLLAMA/dasllama/dasllama_config.das | 3 +- .../dasLLAMA/dasllama/dasllama_convert.das | 46 +++++ .../dasLLAMA/dasllama/dasllama_gemm_gen.das | 11 +- .../dasllama/dasllama_gemm_schema.das | 4 +- modules/dasLLAMA/dasllama/dasllama_gguf.das | 40 +++++ .../dasllama/dasllama_gpu_resident.das | 7 + modules/dasLLAMA/dasllama/dasllama_image.das | 7 +- .../dasLLAMA/dasllama/dasllama_kqformat.das | 157 +++++++++++++++++- modules/dasLLAMA/dasllama/dasllama_layout.das | 24 ++- modules/dasLLAMA/dasllama/dasllama_load.das | 41 ++++- modules/dasLLAMA/dasllama/dasllama_math.das | 9 +- .../dasllama/dasllama_math_default.das | 85 ++++++++++ .../dasLLAMA/dasllama/dasllama_math_gen.das | 132 +++++++++++++-- modules/dasLLAMA/dasllama/dasllama_ple.das | 4 + modules/dasLLAMA/dasllama/dasllama_repack.das | 44 +++++ modules/dasLLAMA/harness/gen_tune_probe.das | 44 ++++- modules/dasLLAMA/tests/test_kqformat.das | 12 +- modules/dasLLAMA/tests/test_kquant.das | 130 +++++++++++++-- 21 files changed, 815 insertions(+), 70 deletions(-) diff --git a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md index 29020be3f2..16a5ffeae9 100644 --- a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md +++ b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md @@ -436,8 +436,51 @@ where, why it is so today, what unquirked looks like. An empty ledger is a legit format's kernel id must dodge 0/1/2 in the stream space or claim a distinct code the same way. +26. **Region `f` is the STREAM code at the repack-pointer ternaries too - an `f == 2` arm + there is DEAD (q51 intercepts 2 upstream), and k2 arrives as f = 20.** QUIRK 25's third + boundary: the big-load repack pointer pick (layout.das, the kqp/ksp ternaries beside the + `fk` translation) originally ended `... : k2qp`, so k2's 20 landed on the ELSE by + position. The iq2s walk turned that else into a keyed chain (`f == 2 ? k2qp : iq2sqp`) - + from that day every k2 region fell through to the NEWEST format's planes, latent until + the first vehicle mixing Q2_K with an i-quant (the IQ2_XS requant: llama.cpp's ftype + puts q2_K on blk.0/1 ffn_down). repack_k2_grp(mr 8) ground over the iq2xs planes: + k2-shaped scale headers, scattered garbage/NaN f16 scale reads, logits all NaN, argmax + token 0 forever. Every serial gate passes (fixtures repack through the fn directly) - + the corruption exists only in the LOADED model, and the first rows can look clean. The + arms key `fk` now. When a walk touches any region/stream ladder, check stream-code vs + kernel-id keying at EVERY boundary, and e2e a MIXED vehicle, not just a pure one. + +27. **Team-mode job lanes never run global init - a `let` module global reads ZERO there.** + (Boris's standing rule, stated during the same hunt.) Nothing reachable from a + team-lane kernel may read a das module global: tables ride per-call builders + (`iq2s_grid2()`), constants ride functions - `GEMM_REFERENCE_MR` is a function for this + reason. Audit a new format's whole rows/tile/dot call graph for global reads; a global + that happens to const-fold today is one refactor away from a lane read of zero. + ## Per-format notes +### IQ2_XS Phase A (CPU, 2026-08-31) - the ksigns u64 tier + +Shape: 256-superblock grid format - each of the 32 u16 qs words carries a 9-bit index into +iq2xs_grid[512] (u64 entries, magnitudes {8, 25, 43}) and a 7-bit KSIGNS_IQ2XS index (bit j +flips element j) - the iq3xxs sign machinery over the iq2s-style u64 grid. Scales are the +IQ2_S row EXACTLY: per-16 nibbles folded (0.5+ls)*0.25 = (2ls+1) x d/8. Disk 74B: [f16 d] +[32 u16 qs][8 scale nibble bytes]. Planes: the 64B qs region verbatim (IQ2XS_QSB 64 - 16 +uniform grp columns; a u16's two bytes always share a column), the 20B eighth-ed-d strip +row (IQ2XS_SSB 20). Ids: KqFmt.iq2xs = 13, kernel id 24, stream code 24. Grid: +iq2xs_grid2() (4 KB per-call local, 1024 words as low/high pairs) + IQ2XS_GRID. Vehicle: a +local requant (llama-quantize --allow-requantize --imatrix, Q8_0 -> IQ2_XS) - census CLEAN: +iq2_xs x94 + q4_K x16 (attn_v) + q2_K x2 (blk.0/1 ffn_down) + q5_K embd, every sibling +already supported. + +The walk surfaced TWO buried defects, now QUIRKs 26/27: the DEAD `f == 2` repack-pointer +arm (k2 regions fell through to the newest format's planes since the iq2s walk - this +vehicle's q2_K ffn_down ground repack_k2_grp over the iq2xs planes; the arms key `fk` now), +and Boris's team-lane global-init rule (GEMM_REFERENCE_MR is a function now). Gates: +test_kqformat 18/18, test_kquant 250 (232 pass, 18 env skips), probe GEN TUNE TEST OK, +lint 0. E2e: ids 64/64 vs llama.cpp - the arc's FIRST full-match greedy stream, no fork +anywhere in the window - gen 23 t/s reference bodies. JIT emitter, Vulkan, Metal: pending. + ### IQ2_S Phase A (CPU, 2026-08-31) - the u64-grid tier Shape: 256-superblock grid format, the first with a u64 grid - a 10-bit index (qs byte | diff --git a/modules/dasLLAMA/dasllama/dasllama_blocks.das b/modules/dasLLAMA/dasllama/dasllama_blocks.das index 0310519500..c0a9d860c7 100644 --- a/modules/dasLLAMA/dasllama/dasllama_blocks.das +++ b/modules/dasLLAMA/dasllama/dasllama_blocks.das @@ -1421,6 +1421,9 @@ def private kq_bytes_per_weight(f : KqFmt) : float { if (f == KqFmt.iq2s) { return 92.0 / 256.0 // 72B quants + the 20B scale row per 256 } + if (f == KqFmt.iq2xs) { + return 84.0 / 256.0 // 64B quants + the 20B scale row per 256 + } return 34.0 / 32.0 // q8: 32 quants + an f16 scale per block } diff --git a/modules/dasLLAMA/dasllama/dasllama_common.das b/modules/dasLLAMA/dasllama/dasllama_common.das index 1ee57818cc..d52b01f196 100644 --- a/modules/dasLLAMA/dasllama/dasllama_common.das +++ b/modules/dasLLAMA/dasllama/dasllama_common.das @@ -856,6 +856,9 @@ struct Model { // the iq2s tier's plane pair: [32 idx][32 signs][8 qh] per superblock + the 20B row (f16 d/8, 16 x (1 + 2s)) iq2sq : array iq2ss : array + // the iq2xs tier's plane pair: 32 u16 qs words per superblock + the 20B row (f16 d/8, 16 x (1 + 2s)) + iq2xsq : array + iq2xss : array kquant_native : bool = false // kq stage 4: the kq planes were repacked at load into the active backend's grp layout. // From then on every kq matmul MUST run the backend slots — disk-order portable kernels would @@ -873,6 +876,7 @@ struct Model { kq_repack_mr45 : int64 = 4l kq_repack_mr2 : int64 = 4l kq_repack_mr23 : int64 = 4l + kq_repack_mr24 : int64 = 4l // per-layer weight format tags (KqFmt; empty = all q8). Filled by detect_kq_formats before // layout_offsets so the layout walks each tensor into its format's cursor. wq_fmt : array @@ -1098,7 +1102,7 @@ def model_weights_bytes(t : Model) : int64 { + long_length(t.iq4xsq) + long_length(t.iq4xss) + long_length(t.k3q) + long_length(t.k3s) + long_length(t.iq3sq) + long_length(t.iq3ss) + long_length(t.iq3xxsq) + long_length(t.iq3xxss) - + long_length(t.iq4nlq) + long_length(t.iq4nls) + long_length(t.k2q) + long_length(t.k2s) + long_length(t.iq2sq) + long_length(t.iq2ss) + + long_length(t.iq4nlq) + long_length(t.iq4nls) + long_length(t.k2q) + long_length(t.k2s) + long_length(t.iq2sq) + long_length(t.iq2ss) + long_length(t.iq2xsq) + long_length(t.iq2xss) + long_length(t.q51q) + long_length(t.q51s) + long_length(t.mblob) + long_length(t.vkblob) + long_length(t.embq) + long_length(t.embs)) @@ -1494,6 +1498,7 @@ def private dlim_cpu_source_impl(var c : DlimCpuConfig) { c.kq_mr45 = active_kq_layout_mr(45) c.kq_mr2 = active_kq_layout_mr(2) c.kq_mr23 = active_kq_layout_mr(23) + c.kq_mr24 = active_kq_layout_mr(24) c.q51_mr = active_q51_layout_mr() // OUTCOME, not request: a backend without s16 twins keeps f32 scale planes (wscale_convert_f16) c.wscale_f16 = g_wscale_f16 && kernel_backend_has_wscale16() @@ -1586,7 +1591,7 @@ def kq_active_mr(t : Model; f : KqFmt) : int64 { if (!t.kq_repacked) { return 1l } - return f == KqFmt.k4 ? t.kq_repack_mr4 : (f == KqFmt.k5 ? t.kq_repack_mr5 : (f == KqFmt.k6 ? t.kq_repack_mr6 : (f == KqFmt.q40 ? t.kq_repack_mr40 : (f == KqFmt.iq4xs ? t.kq_repack_mr44 : (f == KqFmt.k3 ? t.kq_repack_mr3 : (f == KqFmt.iq3s ? t.kq_repack_mr33 : (f == KqFmt.iq3xxs ? t.kq_repack_mr34 : (f == KqFmt.iq4nl ? t.kq_repack_mr45 : (f == KqFmt.k2 ? t.kq_repack_mr2 : t.kq_repack_mr23))))))))) + return f == KqFmt.k4 ? t.kq_repack_mr4 : (f == KqFmt.k5 ? t.kq_repack_mr5 : (f == KqFmt.k6 ? t.kq_repack_mr6 : (f == KqFmt.q40 ? t.kq_repack_mr40 : (f == KqFmt.iq4xs ? t.kq_repack_mr44 : (f == KqFmt.k3 ? t.kq_repack_mr3 : (f == KqFmt.iq3s ? t.kq_repack_mr33 : (f == KqFmt.iq3xxs ? t.kq_repack_mr34 : (f == KqFmt.iq4nl ? t.kq_repack_mr45 : (f == KqFmt.k2 ? t.kq_repack_mr2 : (f == KqFmt.iq2s ? t.kq_repack_mr23 : t.kq_repack_mr24)))))))))) } //! Load a llama-architecture GGUF into the split layout at the requested precision — straight into @@ -1649,8 +1654,8 @@ def footprint(t : Model) : MemFootprint { let q4s = long_length(t.q4scales) * 4l let mx4 = long_length(t.mxq) // uint8 (two nibbles) -> 1 byte each let mx4s = long_length(t.mxs) // raw E8M0 -> 1 byte per 32-block - let kq = long_length(t.k4q) + long_length(t.k5q) + long_length(t.k6q) + long_length(t.q40q) + long_length(t.q51q) + long_length(t.iq4xsq) + long_length(t.k3q) + long_length(t.iq3sq) + long_length(t.iq3xxsq) + long_length(t.iq4nlq) + long_length(t.k2q) + long_length(t.iq2sq) - let kqs = long_length(t.k4s) + long_length(t.k5s) + long_length(t.k6s) + long_length(t.q40s) + long_length(t.q51s) + long_length(t.iq4xss) + long_length(t.k3s) + long_length(t.iq3ss) + long_length(t.iq3xxss) + long_length(t.iq4nls) + long_length(t.k2s) + long_length(t.iq2ss) + let kq = long_length(t.k4q) + long_length(t.k5q) + long_length(t.k6q) + long_length(t.q40q) + long_length(t.q51q) + long_length(t.iq4xsq) + long_length(t.k3q) + long_length(t.iq3sq) + long_length(t.iq3xxsq) + long_length(t.iq4nlq) + long_length(t.k2q) + long_length(t.iq2sq) + long_length(t.iq2xsq) + let kqs = long_length(t.k4s) + long_length(t.k5s) + long_length(t.k6s) + long_length(t.q40s) + long_length(t.q51s) + long_length(t.iq4xss) + long_length(t.k3s) + long_length(t.iq3ss) + long_length(t.iq3xxss) + long_length(t.iq4nls) + long_length(t.k2s) + long_length(t.iq2ss) + long_length(t.iq2xss) return MemFootprint(aux_fp32_bytes = aux, weight_fp32_bytes = wf, q8_bytes = q8, q8_scale_bytes = q8s, q4_bytes = q4, q4_scale_bytes = q4s, mx4_bytes = mx4, mx4_scale_bytes = mx4s, kq_bytes = kq, kq_scale_bytes = kqs, @@ -3324,7 +3329,7 @@ def mm_b_q51_groupn(var y : array; t : Model; offs : array; nregio // KqFmt -> the kernel-layer format id (matmul_kq*/kq_rows_fn/kq_qsb take the int form). q8 maps // to 0, a poison id no kq kernel accepts — kq_rows_for calls this for EVERY tag, so it must stay // total even though q8 branches never read the result. -def private kq_fi(fmt : KqFmt) : int => fmt == KqFmt.k4 ? 4 : (fmt == KqFmt.k5 ? 5 : (fmt == KqFmt.k6 ? 6 : (fmt == KqFmt.q40 ? 40 : (fmt == KqFmt.iq4xs ? 44 : (fmt == KqFmt.k3 ? 3 : (fmt == KqFmt.iq3s ? 33 : (fmt == KqFmt.iq3xxs ? 34 : (fmt == KqFmt.iq4nl ? 45 : (fmt == KqFmt.k2 ? 2 : (fmt == KqFmt.iq2s ? 23 : 0)))))))))) +def private kq_fi(fmt : KqFmt) : int => fmt == KqFmt.k4 ? 4 : (fmt == KqFmt.k5 ? 5 : (fmt == KqFmt.k6 ? 6 : (fmt == KqFmt.q40 ? 40 : (fmt == KqFmt.iq4xs ? 44 : (fmt == KqFmt.k3 ? 3 : (fmt == KqFmt.iq3s ? 33 : (fmt == KqFmt.iq3xxs ? 34 : (fmt == KqFmt.iq4nl ? 45 : (fmt == KqFmt.k2 ? 2 : (fmt == KqFmt.iq2s ? 23 : (fmt == KqFmt.iq2xs ? 24 : 0))))))))))) // K-quant GEMV against a PRE-quantized activation (xq/xs/xbs already filled by the bs quantizer). // kq_repacked loads run the backend's stamped cores; disk-order loads run the portable kernels. @@ -3352,6 +3357,8 @@ def mm_at_kq_pre(var y : array; t : Model; fmt : KqFmt; woff : int64; xq matmul_kq_active(2, y, t.k2q, t.k2s, woff, xq, xs, xbs, n, d, yoff) } elif (fmt == KqFmt.iq2s) { matmul_kq_active(23, y, t.iq2sq, t.iq2ss, woff, xq, xs, xbs, n, d, yoff) + } elif (fmt == KqFmt.iq2xs) { + matmul_kq_active(24, y, t.iq2xsq, t.iq2xss, woff, xq, xs, xbs, n, d, yoff) } else { panic("mm_at_kq_pre: '{fmt}' has no kq plane pair") // a new KqFmt must claim its arm, never ride q40's } @@ -3377,6 +3384,8 @@ def mm_at_kq_pre(var y : array; t : Model; fmt : KqFmt; woff : int64; xq matmul_kq(2, y, t.k2q, t.k2s, woff, xq, xs, xbs, n, d, yoff) } elif (fmt == KqFmt.iq2s) { matmul_kq(23, y, t.iq2sq, t.iq2ss, woff, xq, xs, xbs, n, d, yoff) + } elif (fmt == KqFmt.iq2xs) { + matmul_kq(24, y, t.iq2xsq, t.iq2xss, woff, xq, xs, xbs, n, d, yoff) } else { panic("mm_at_kq_pre: '{fmt}' has no kq plane pair") } @@ -3422,6 +3431,8 @@ def mm_b_kq(var s : Session; var y : array; t : Model; fmt : KqFmt; woff matmul_kq_batch(2, y, t.k2q, t.k2s, woff, s.xqb, s.xsb, s.xbsb, n, d, npos) } elif (fmt == KqFmt.iq2s) { matmul_kq_batch(23, y, t.iq2sq, t.iq2ss, woff, s.xqb, s.xsb, s.xbsb, n, d, npos) + } elif (fmt == KqFmt.iq2xs) { + matmul_kq_batch(24, y, t.iq2xsq, t.iq2xss, woff, s.xqb, s.xsb, s.xbsb, n, d, npos) } else { panic("mm_b_kq: '{fmt}' has no kq plane pair") } @@ -3463,6 +3474,8 @@ def mm_at_kq_groupn(var y : array; t : Model; fmt : KqFmt; offs : array; t : Model; fmt : KqFmt; offs : array; t : Model; fmt : KqFmt; offs : array; t : Model; fmt : KqFmt; woff : int64; xq : matmul_kq_batch(2, y, t.k2q, t.k2s, woff, xq, xs, xbs, n, d, ntok) } elif (fmt == KqFmt.iq2s) { matmul_kq_batch(23, y, t.iq2sq, t.iq2ss, woff, xq, xs, xbs, n, d, ntok) + } elif (fmt == KqFmt.iq2xs) { + matmul_kq_batch(24, y, t.iq2xsq, t.iq2xss, woff, xq, xs, xbs, n, d, ntok) } else { panic("mm_b_kq_pre: '{fmt}' has no kq plane pair") } @@ -3569,6 +3588,7 @@ def kq_plane_q(t : Model; fmt : KqFmt; woff : int64) : uint8 const? { if (fmt == KqFmt.iq4nl) return unsafe(addr(t.iq4nlq[sb * Q40_QSB])) if (fmt == KqFmt.k2) return unsafe(addr(t.k2q[sb * K2_QSB])) if (fmt == KqFmt.iq2s) return unsafe(addr(t.iq2sq[sb * IQ2S_QSB])) + if (fmt == KqFmt.iq2xs) return unsafe(addr(t.iq2xsq[sb * IQ2XS_QSB])) return null } @@ -3585,6 +3605,7 @@ def kq_plane_s(t : Model; fmt : KqFmt; woff : int64) : uint8 const? { if (fmt == KqFmt.iq4nl) return unsafe(addr(t.iq4nls[sb * Q40_SSB])) if (fmt == KqFmt.k2) return unsafe(addr(t.k2s[sb * K2_SSB])) if (fmt == KqFmt.iq2s) return unsafe(addr(t.iq2ss[sb * IQ2S_SSB])) + if (fmt == KqFmt.iq2xs) return unsafe(addr(t.iq2xss[sb * IQ2XS_SSB])) return null } @@ -3959,6 +3980,8 @@ def embed_row(t : Model; token : int64; var dst : float?) { // nolint:STYLE037 dequant_kq_row_grp(fmt, addr(t.embq[sbg * K2_QSB]), addr(t.embs[sbg * K2_SSB]), token % mr, mr, dim, dst) } elif (t.emb_fmt == KqFmt.iq2s) { dequant_kq_row_grp(fmt, addr(t.embq[sbg * IQ2S_QSB]), addr(t.embs[sbg * IQ2S_SSB]), token % mr, mr, dim, dst) + } elif (t.emb_fmt == KqFmt.iq2xs) { + dequant_kq_row_grp(fmt, addr(t.embq[sbg * IQ2XS_QSB]), addr(t.embs[sbg * IQ2XS_SSB]), token % mr, mr, dim, dst) } else { panic("embed_row: '{t.emb_fmt}' has no kq plane pair") } @@ -3984,6 +4007,8 @@ def embed_row(t : Model; token : int64; var dst : float?) { // nolint:STYLE037 dequant_kq_row_grp(fmt, addr(t.k2q[sbg * K2_QSB]), addr(t.k2s[sbg * K2_SSB]), token % mr, mr, dim, dst) } elif (t.emb_fmt == KqFmt.iq2s) { dequant_kq_row_grp(fmt, addr(t.iq2sq[sbg * IQ2S_QSB]), addr(t.iq2ss[sbg * IQ2S_SSB]), token % mr, mr, dim, dst) + } elif (t.emb_fmt == KqFmt.iq2xs) { + dequant_kq_row_grp(fmt, addr(t.iq2xsq[sbg * IQ2XS_QSB]), addr(t.iq2xss[sbg * IQ2XS_SSB]), token % mr, mr, dim, dst) } else { panic("embed_row: '{t.emb_fmt}' has no kq plane pair") } @@ -4020,6 +4045,8 @@ def embed_row(t : Model; token : int64; var dst : float?) { // nolint:STYLE037 dequant_k2_plane_superblock(t.embq, (sb0 + s) * K2_QSB, t.embs, (sb0 + s) * K2_SSB, row, s * 256l) } elif (t.emb_fmt == KqFmt.iq2s) { dequant_iq2s_plane_superblock(t.embq, (sb0 + s) * IQ2S_QSB, t.embs, (sb0 + s) * IQ2S_SSB, row, s * 256l) + } elif (t.emb_fmt == KqFmt.iq2xs) { + dequant_iq2xs_plane_superblock(t.embq, (sb0 + s) * IQ2XS_QSB, t.embs, (sb0 + s) * IQ2XS_SSB, row, s * 256l) } else { panic("embed_row: '{t.emb_fmt}' has no kq plane pair") } @@ -4049,6 +4076,8 @@ def embed_row(t : Model; token : int64; var dst : float?) { // nolint:STYLE037 dequant_k2_plane_superblock(t.k2q, (sb0 + s) * K2_QSB, t.k2s, (sb0 + s) * K2_SSB, row, s * 256l) } elif (t.emb_fmt == KqFmt.iq2s) { dequant_iq2s_plane_superblock(t.iq2sq, (sb0 + s) * IQ2S_QSB, t.iq2ss, (sb0 + s) * IQ2S_SSB, row, s * 256l) + } elif (t.emb_fmt == KqFmt.iq2xs) { + dequant_iq2xs_plane_superblock(t.iq2xsq, (sb0 + s) * IQ2XS_QSB, t.iq2xss, (sb0 + s) * IQ2XS_SSB, row, s * 256l) } else { panic("embed_row: '{t.emb_fmt}' has no kq plane pair") } diff --git a/modules/dasLLAMA/dasllama/dasllama_config.das b/modules/dasLLAMA/dasllama/dasllama_config.das index 1347dae14b..b141cdf027 100644 --- a/modules/dasLLAMA/dasllama/dasllama_config.das +++ b/modules/dasLLAMA/dasllama/dasllama_config.das @@ -34,6 +34,7 @@ struct public DlimCpuConfig { kq_mr45 : int64 kq_mr2 : int64 kq_mr23 : int64 + kq_mr24 : int64 q51_mr : int64 wscale_f16 : bool kquant_native : bool @@ -146,7 +147,7 @@ def public dlim_config_current(quant : string = "q8") : DlimConfiguration { def public dlim_identity(c : DlimConfiguration; image_version : int; tag : string = "") : string { return ("v{image_version}|{c.quant}|{c.cpu.backend}|{c.cpu.wscale_f16 ? "s16" : "s32"}" + "|q8 mr{c.cpu.q8_mr} b{c.cpu.q8_wbias} g{c.cpu.q8_kgroup}" - + "|kq {c.cpu.kq_mr4}/{c.cpu.kq_mr5}/{c.cpu.kq_mr6}/{c.cpu.kq_mr40}/{c.cpu.kq_mr44}/{c.cpu.kq_mr3}/{c.cpu.kq_mr33}/{c.cpu.kq_mr34}/{c.cpu.kq_mr45}/{c.cpu.kq_mr2}/{c.cpu.kq_mr23}" + + "|kq {c.cpu.kq_mr4}/{c.cpu.kq_mr5}/{c.cpu.kq_mr6}/{c.cpu.kq_mr40}/{c.cpu.kq_mr44}/{c.cpu.kq_mr3}/{c.cpu.kq_mr33}/{c.cpu.kq_mr34}/{c.cpu.kq_mr45}/{c.cpu.kq_mr2}/{c.cpu.kq_mr23}/{c.cpu.kq_mr24}" + "|q51 mr{c.cpu.q51_mr}" + "|nat {c.cpu.kquant_native ? 1 : 0}{c.cpu.kq_q40_native ? 1 : 0}{c.cpu.kq_q50_native ? 1 : 0}{c.cpu.kq_q51_native ? 1 : 0}" + (tag != "" ? "|{tag}" : "")) diff --git a/modules/dasLLAMA/dasllama/dasllama_convert.das b/modules/dasLLAMA/dasllama/dasllama_convert.das index 0f210f44a3..0f8ca8efb6 100644 --- a/modules/dasLLAMA/dasllama/dasllama_convert.das +++ b/modules/dasLLAMA/dasllama/dasllama_convert.das @@ -913,6 +913,52 @@ def dequant_iq2s_plane_superblock_at(kq : array | #; kqo : int64; ks : ar } } +//! Transcode one IQ2_XS superblock (74 bytes at `bo`: f16 d, 32 u16 qs words, 8 packed scale +//! nibbles) into the iq2xs planes: the 64B qs region verbatim to kq[kqo..+64), d EIGHTH-ed +//! (ggml's (0.5+ls)*0.25 fold becomes (2ls+1) x d/8) + the 16 strips (1 + 2s) to ks[kso..+20). +def transcode_iq2xs_superblock(bytes : array | #; bo : int64; var kq : array; kqo : int64; var ks : array; kso : int64) { + for (i in range64(64l)) { + kq[kqo + i] = bytes[bo + 2l + i] + } + let d8 = f16_half_bits(f16_half_bits(f16_half_bits(uint(bytes[bo]) | (uint(bytes[bo + 1l]) << 8u)))) + ks[kso] = uint8(d8 & 0xFFu) + ks[kso + 1l] = uint8(d8 >> 8u) + ks[kso + 2l] = uint8(0) + ks[kso + 3l] = uint8(0) + for (g in range64(16l)) { + let nib = g % 2l == 0l ? int(bytes[bo + 66l + g / 2l]) & 15 : int(bytes[bo + 66l + g / 2l]) >> 4 + ks[kso + 4l + g] = uint8(1 + 2 * nib) + } +} + +//! Reference dequant of one iq2xs-plane superblock: w = (d8 * (1 + 2s)) * grid_byte * sign - +//! u16 word (b*4+l)'s low 9 bits pick TWO grid words (8 magnitudes), its high 7 bits pick the +//! ksigns sign byte (bit j), strip = the 16-chunk 2b + l/2. ggml's dequantize_row_iq2_xs order. +def dequant_iq2xs_plane_superblock(kq : array | #; kqo : int64; ks : array | #; kso : int64; var dst : array | #; doff : int64) { + dequant_iq2xs_plane_superblock_at(kq, kqo, ks, kso + 4l, kso, dst, doff) +} + +//! The split-layout twin: `sso` names the 16-strip base, `dso` the f16 d8 halfword. +def dequant_iq2xs_plane_superblock_at(kq : array | #; kqo : int64; ks : array | #; sso, dso : int64; var dst : array | #; doff : int64) { + let d8 = f16_to_f32(rd_u16(ks, dso)) + let grid = iq2xs_grid2() + let ksg = ksigns_iq2xs() + for (b in range64(8l)) { + for (l in range64(4l)) { + let db = d8 * float(int(ks[sso + 2l * b + l / 2l])) + let w16 = uint(kq[kqo + (b * 4l + l) * 2l]) | (uint(kq[kqo + (b * 4l + l) * 2l + 1l]) << 8u) + let sgn = int(ksg[int(w16 >> 9u)]) + let idx = int(w16 & 511u) + let g1 = grid[idx * 2] + let g2 = grid[idx * 2 + 1] + for (j in range64(4l)) { + dst[doff + b * 32l + l * 8l + j] = db * float((g1 >> uint(8l * j)) & 0xFFu) * (((sgn >> int(j)) & 1) != 0 ? -1.0 : 1.0) + dst[doff + b * 32l + l * 8l + 4l + j] = db * float((g2 >> uint(8l * j)) & 0xFFu) * (((sgn >> int(4l + j)) & 1) != 0 ? -1.0 : 1.0) + } + } + } +} + //! Transcode one Q5_1 disk block (24 bytes at `bo`: d f16, m f16, qh u32, 16 nibble bytes) into //! the q51 planes — verbatim splits, exact. The per-block array form the tests drive; the bulk //! loader (gguf_transcode_q51) runs the same split pointerized and threaded. diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das index 0b53e39d53..ce28b7858b 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das @@ -87,7 +87,7 @@ def private kgroup_of(p : TilePerm) : int => p.dotPrim == "smmla" ? 8 : 4 struct private TilePerm { kstep : int = 2 nrsplit : int = 4 - mr : int = GEMM_REFERENCE_MR + mr : int = GEMM_REFERENCE_MR() gkstep : int = 1 dotPrim : string = "sdot" width : int = 128 @@ -3038,6 +3038,13 @@ def private k2_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 2) def private iq2s_gemv_gen(var gc : LlvmCodeCtx) : bool => kq_gemv_gen_impl(gc, 23) def private iq2s_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 23) +// iq2xs: the grid-gather emitter arm lands with its Phase B; until then both stamps decline +// (reference bodies serve) - a declined generator is the framework's own fallback path. +[unused_argument(gc)] +def private iq2xs_gemv_gen(var gc : LlvmCodeCtx) : bool => false +[unused_argument(gc)] +def private iq2xs_tile_gen(var gc : LlvmCodeCtx) : bool => false + def private k4_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 4) def private k5_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 5) def private k6_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 6) @@ -3129,5 +3136,7 @@ def public register_dasllama_gemm_generators { register_llvm_code_generator("dasllama_gemm_gen::k2_tile", @@k2_tile_gen) register_llvm_code_generator("dasllama_gemm_gen::iq2s_gemv", @@iq2s_gemv_gen) register_llvm_code_generator("dasllama_gemm_gen::iq2s_tile", @@iq2s_tile_gen) + register_llvm_code_generator("dasllama_gemm_gen::iq2xs_gemv", @@iq2xs_gemv_gen) + register_llvm_code_generator("dasllama_gemm_gen::iq2xs_tile", @@iq2xs_tile_gen) register_llvm_code_generator("dasllama_gemm_gen::q8q8_witness", @@witness_gen) } diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_schema.das b/modules/dasLLAMA/dasllama/dasllama_gemm_schema.das index 5526e86288..02fb985946 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_schema.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_schema.das @@ -40,7 +40,7 @@ def q8q8_repack_type(mr : int; wbias : int = 0; kgroup : int = 4) : Q8RepackType //! The mr of the REFERENCE tier's layout (q8q8_repack_type(4), the old hand arm64-laneq layout //! the reference stub bodies read). Non-reference mr runs the generated grp kernel/layout pair, //! stamped from one manifest entry so both decline in lockstep (M4). -let GEMM_REFERENCE_MR = 4 +def GEMM_REFERENCE_MR : int => 4 // a FUNCTION, not a let global: team-mode job lanes never run global init, so a let here reads 0 on the lanes the unstamped reference layout/repack run on //! GPU-tier device-plane shape of the Q8 rail: 32-weight blocks, 32B quants + one f16 scale //! halfword per block. Every q8 plane sizing (gathers, arena/stack math) derives from these. @@ -72,6 +72,7 @@ def kq_qsb(fmt : int) : int64 { if (fmt == 45) return kq_qsb(KqFmt.iq4nl) if (fmt == 2) return kq_qsb(KqFmt.k2) if (fmt == 23) return kq_qsb(KqFmt.iq2s) + if (fmt == 24) return kq_qsb(KqFmt.iq2xs) panic("kq_qsb: unknown kq format id {fmt}") return 0l } @@ -91,6 +92,7 @@ def kq_ssb(fmt : int) : int64 { if (fmt == 45) return kq_ssb(KqFmt.iq4nl) if (fmt == 2) return kq_ssb(KqFmt.k2) if (fmt == 23) return kq_ssb(KqFmt.iq2s) + if (fmt == 24) return kq_ssb(KqFmt.iq2xs) panic("kq_ssb: unknown kq format id {fmt}") return 0l } diff --git a/modules/dasLLAMA/dasllama/dasllama_gguf.das b/modules/dasLLAMA/dasllama/dasllama_gguf.das index a3335c8ca9..bc72f907c4 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gguf.das +++ b/modules/dasLLAMA/dasllama/dasllama_gguf.das @@ -51,6 +51,7 @@ let GGML_TYPE_Q3_K = 11 let GGML_TYPE_Q2_K = 10 let GGML_TYPE_IQ3_S = 21 let GGML_TYPE_IQ2_S = 22 +let GGML_TYPE_IQ2_XS = 17 let GGML_TYPE_IQ3_XXS = 18 let GGML_TYPE_IQ4_NL = 20 let GGML_TYPE_Q4_K = 12 @@ -980,6 +981,45 @@ def gguf_transcode_iq2s(m : GGUFMeta; srcbytes : array | #; name : string } } +//! Transcode an IQ2_XS tensor into the iq2xs planes (strides 64/20, exact): the 74B disk +//! superblock's 32 u16 qs words go verbatim, d eighth-ed, the 16 strips decoded to (1 + 2s) - +//! transcode_iq2xs_superblock. +def gguf_transcode_iq2xs(m : GGUFMeta; srcbytes : array | #; name : string; var kq : array; var ks : array; eloff, expect_n : int64; src_off : int64 = 0l) { + let ti = kq_transcode_check(m, name, GGML_TYPE_IQ2_XS, "IQ2_XS", src_off, expect_n) + let nb = expect_n / 256l + if (nb <= 0l) { + return + } + guard_dst(name, "iq2xs quant plane", (eloff / 256l) * IQ2XS_QSB, nb * IQ2XS_QSB, long_length(kq)) + guard_dst(name, "iq2xs scale plane", (eloff / 256l) * IQ2XS_SSB, nb * IQ2XS_SSB, long_length(ks)) + with_tensor_view(m, srcbytes, ti) $(bytes, tbase) { + let bo = tbase + (src_off / 256l) * 74l + unsafe { + let srcp = addr(bytes[bo]) + var kqp = addr(kq[(eloff / 256l) * IQ2XS_QSB]) + var ksp = addr(ks[(eloff / 256l) * IQ2XS_SSB]) + maybe_parallel_for(0, int(nb), transcode_jobs(nb, 74l)) $(rb, re) { + unsafe { + for (sb in range64(int64(rb), int64(re))) { + let src = srcp + sb * 74l + var row = ksp + sb * IQ2XS_SSB + let d8 = f16_half_bits(f16_half_bits(f16_half_bits(uint(src[0l]) | (uint(src[1l]) << 8u)))) + row[0l] = uint8(d8 & 0xFFu) + row[1l] = uint8(d8 >> 8u) + row[2l] = uint8(0) + row[3l] = uint8(0) + for (g in range64(16l)) { + let nib = g % 2l == 0l ? int(src[66l + g / 2l]) & 15 : int(src[66l + g / 2l]) >> 4 + row[4l + g] = uint8(1 + 2 * nib) + } + bcopy(kqp + sb * IQ2XS_QSB, src + 2l, 64l) + } + } + } + } + } +} + //! Transcode a Q5_1 tensor into the q51 planes (per 32-BLOCK strides 20/4 — see Q51_QB): each //! 24B disk block splits into 2B d + 2B m (scale plane) and 16B nibbles + 4B qh (quant plane), //! all verbatim. Exact; `eloff`/`src_off`/`expect_n` are element offsets, % 32. diff --git a/modules/dasLLAMA/dasllama/dasllama_gpu_resident.das b/modules/dasLLAMA/dasllama/dasllama_gpu_resident.das index ce46a9a7f9..a114e91329 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gpu_resident.das +++ b/modules/dasLLAMA/dasllama/dasllama_gpu_resident.das @@ -305,6 +305,11 @@ def trim_model_planes(var t : Model) : bool { // nolint:STYLE038 — the flat t.embs |> reserve_resize(nsb * IQ2S_SSB) memcpy(addr(t.embq[0]), addr(t.iq2sq[sb0 * IQ2S_QSB]), nsb * IQ2S_QSB) memcpy(addr(t.embs[0]), addr(t.iq2ss[sb0 * IQ2S_SSB]), nsb * IQ2S_SSB) + } elif (t.emb_fmt == KqFmt.iq2xs) { + t.embq |> reserve_resize(nsb * IQ2XS_QSB) + t.embs |> reserve_resize(nsb * IQ2XS_SSB) + memcpy(addr(t.embq[0]), addr(t.iq2xsq[sb0 * IQ2XS_QSB]), nsb * IQ2XS_QSB) + memcpy(addr(t.embs[0]), addr(t.iq2xss[sb0 * IQ2XS_SSB]), nsb * IQ2XS_SSB) } elif (t.emb_fmt == KqFmt.iq3xxs) { t.embq |> resize(nsb * IQ3XXS_QSB) t.embs |> resize(nsb * IQ3XXS_SSB) @@ -347,6 +352,8 @@ def trim_model_planes(var t : Model) : bool { // nolint:STYLE038 — the flat delete t.k2s delete t.iq2sq delete t.iq2ss + delete t.iq2xsq + delete t.iq2xss delete t.q40q delete t.q40s delete t.q51q diff --git a/modules/dasLLAMA/dasllama/dasllama_image.das b/modules/dasLLAMA/dasllama/dasllama_image.das index 44ecb3c7f8..c54ed4a90e 100644 --- a/modules/dasLLAMA/dasllama/dasllama_image.das +++ b/modules/dasLLAMA/dasllama/dasllama_image.das @@ -38,7 +38,7 @@ require dasllama/dasllama_load // WhisperModel.enc) contribute their planes under dotted names ("enc.fblob"); string-array // fields ride the meta blob via serialize_strings — raw string pointers can't be planes. -let IMAGE_VERSION = 24 // 24: the iq2s plane pair (IQ2_S native tier); 23: k2; 22: iq4nl +let IMAGE_VERSION = 25 // 25: the iq2xs plane pair (IQ2_XS native tier); 24: iq2s; 23: k2 //! The metal (blob-only) flavor's identity tag: q8 planes ride the 34B block_q8_0 blob and the //! kq scale planes their GPU forms (convert_model_to_metal_blob) — flavors are per-config and @@ -641,6 +641,7 @@ def serialize_image_meta(var arch : Archive; var t : Model) { arch |> serialize_raw(t.kq_repack_mr45) arch |> serialize_raw(t.kq_repack_mr2) arch |> serialize_raw(t.kq_repack_mr23) + arch |> serialize_raw(t.kq_repack_mr24) arch |> serialize_raw(t.wcls_fmt) arch |> serialize_raw(t.emb_fmt) arch |> serialize_raw(t.ple_emb_fmt) @@ -680,7 +681,7 @@ def serialize_image_meta(var arch : Archive; var t : Model) { } // 67 serialized fields + 3 deliberate skips (blocks, image_map, image_bytes) -let IMAGE_META_FIELDS = 73 + 3 +let IMAGE_META_FIELDS = 74 + 3 //! Count of meta-carried fields of any struct: non-arrays plus string arrays (those cannot //! be raw planes — see serialize_strings). Pair with a per-type field-count constant to @@ -964,7 +965,7 @@ def save_model_image(var t : Model; path : string; tag : string = ""; quant : st // streamed planes size as empty, so their bytes plus a page each are added here; over-reserve is free (dwrite_close truncates) def private stream_extra_bytes(t : Model; jobs : array) : uint64 { var extra = 0ul - for (fname in ["qblob", "mblob", "k4q", "k5q", "k6q", "q40q", "q51q", "iq4xsq", "k3q", "iq3sq", "iq3xxsq", "iq4nlq", "k2q", "iq2sq"]) { + for (fname in ["qblob", "mblob", "k4q", "k5q", "k6q", "q40q", "q51q", "iq4xsq", "k3q", "iq3sq", "iq3xxsq", "iq4nlq", "k2q", "iq2sq", "iq2xsq"]) { let sb = stream_plane_bytes(t, jobs, fname) if (sb > 0l) { extra += uint64(sb) + uint64(IMAGE_PAGE) diff --git a/modules/dasLLAMA/dasllama/dasllama_kqformat.das b/modules/dasLLAMA/dasllama/dasllama_kqformat.das index 43607ffb0b..826901d2bc 100644 --- a/modules/dasLLAMA/dasllama/dasllama_kqformat.das +++ b/modules/dasLLAMA/dasllama/dasllama_kqformat.das @@ -13,7 +13,7 @@ module dasllama_kqformat shared public require dasllama/dasllama_lint public //! Per-weight storage format under a q8-mode load with native planes: q8 = qblob/qscales; -//! k4/k5/k6/q40/iq4xs/k3/iq3s/iq3xxs/iq4nl/k2/iq2s = superblock plane pairs (mixed PER TENSOR); q51 = native Q5_1 per-32 +//! k4/k5/k6/q40/iq4xs/k3/iq3s/iq3xxs/iq4nl/k2/iq2s/iq2xs = superblock plane pairs (mixed PER TENSOR); q51 = native Q5_1 per-32 //! planes. New members append — the int value is the device stack tag and the image plane id. enum KqFmt : uint8 { q8 @@ -29,15 +29,16 @@ enum KqFmt : uint8 { iq4nl k2 iq2s + iq2xs } //! The superblock-lattice formats (Q8_K-form activations, % 256 rows, repack + stamped kq //! kernels). q51 deliberately fails this: it rides per-32 planes with Q8_0-form activations — //! a `fmt != KqFmt.q8` test does not imply the kq lattice; branch on kq_sb where it does. -def kq_sb(f : KqFmt) : bool => f == KqFmt.k4 || f == KqFmt.k5 || f == KqFmt.k6 || f == KqFmt.q40 || f == KqFmt.iq4xs || f == KqFmt.k3 || f == KqFmt.iq3s || f == KqFmt.iq3xxs || f == KqFmt.iq4nl || f == KqFmt.k2 || f == KqFmt.iq2s +def kq_sb(f : KqFmt) : bool => f == KqFmt.k4 || f == KqFmt.k5 || f == KqFmt.k6 || f == KqFmt.q40 || f == KqFmt.iq4xs || f == KqFmt.k3 || f == KqFmt.iq3s || f == KqFmt.iq3xxs || f == KqFmt.iq4nl || f == KqFmt.k2 || f == KqFmt.iq2s || f == KqFmt.iq2xs //! The same predicate over the int id space the GPU drivers carry per stack/plane. -def kq_sb(fmt : int) : bool => fmt == int(KqFmt.k4) || fmt == int(KqFmt.k5) || fmt == int(KqFmt.k6) || fmt == int(KqFmt.q40) || fmt == int(KqFmt.iq4xs) || fmt == int(KqFmt.k3) || fmt == int(KqFmt.iq3s) || fmt == int(KqFmt.iq3xxs) || fmt == int(KqFmt.iq4nl) || fmt == int(KqFmt.k2) || fmt == int(KqFmt.iq2s) +def kq_sb(fmt : int) : bool => fmt == int(KqFmt.k4) || fmt == int(KqFmt.k5) || fmt == int(KqFmt.k6) || fmt == int(KqFmt.q40) || fmt == int(KqFmt.iq4xs) || fmt == int(KqFmt.k3) || fmt == int(KqFmt.iq3s) || fmt == int(KqFmt.iq3xxs) || fmt == int(KqFmt.iq4nl) || fmt == int(KqFmt.k2) || fmt == int(KqFmt.iq2s) || fmt == int(KqFmt.iq2xs) //! KqFmt -> the kernel/IR format id space — the ONE bridge between the enum and the int ids the //! generated kernels take as runtime params. Mnemonic ids: a K-quant is its bit width (3/4/5/6), @@ -76,6 +77,9 @@ def kq_schema_id(f : KqFmt) : int { if (f == KqFmt.iq2s) { return 23 } + if (f == KqFmt.iq2xs) { + return 24 + } panic("kq_schema_id: not a kq superblock format") return 0 } @@ -104,6 +108,8 @@ let K2_QSB = 64l // Q2_K quant plane: 64 qs bytes (four 2-bit lanes each), ve let K2_SSB = 20l // Q2_K scale plane: [f16 d][f16 dmin][16 sc/min nibble-pair bytes] - the disk scale block, header first let IQ2S_QSB = 72l // IQ2_S quant plane: [32 grid-index low bytes][32 sign bytes][8 qh], verbatim disk order let IQ2S_SSB = 20l // IQ2_S scale plane: f16 d EIGHTH-ed (ggml's (0.5+ls)*0.25 = (2ls+1)/8), 2 pad, 16 x uint8 (2ls+1) - per-16 strips in the 20B row +let IQ2XS_QSB = 64l // IQ2_XS quant plane: 32 u16 words ([9-bit grid index][7-bit ksigns index]), verbatim disk order +let IQ2XS_SSB = 20l // IQ2_XS scale plane: the iq2s row exactly - f16 d EIGHTH-ed, 2 pad, 16 x uint8 (2ls+1) per-16 strips // IQ4_NL reuses Q40_QSB/Q40_SSB outright: the disk bytes are q40's (f16 d + 16 nibble bytes // per 32-block, the k/k+16 pairing) — only the nibble SEMANTICS differ (IQ4NL_LUT, no -8). @@ -531,6 +537,146 @@ def iq2s_grid2() : uint[2048] { // nolint:STYLE038 - a 2048-word data table, n //! a module global as zeros; kernels take `iq2s_grid2()` instead. let IQ2S_GRID = iq2s_grid2() + +//! The IQ2_XS grid (ggml's iq2xs_grid, 512 u64 entries as low/high uint pairs): the u16 word's +//! low 9 bits select EIGHT magnitudes ({8, 25, 43}), its high 7 bits the KSIGNS_IQ2XS sign +//! byte (bit j flips element j). Per-call local (4 KB), worker-safe; IQ2XS_GRID = main-context copy. +def iq2xs_grid2() : uint[1024] { // nolint:STYLE038 - a 1024-word data table, not splittable + return fixed_array( + 0x08080808, 0x08080808, 0x0808082b, 0x08080808, 0x08081919, 0x08080808, 0x08082b08, 0x08080808, + 0x08082b2b, 0x08080808, 0x08190819, 0x08080808, 0x08191908, 0x08080808, 0x0819192b, 0x08080808, + 0x08192b19, 0x08080808, 0x082b0808, 0x08080808, 0x082b082b, 0x08080808, 0x082b1919, 0x08080808, + 0x082b2b08, 0x08080808, 0x19080819, 0x08080808, 0x19081908, 0x08080808, 0x1908192b, 0x08080808, + 0x19082b19, 0x08080808, 0x19190808, 0x08080808, 0x1919082b, 0x08080808, 0x19191919, 0x08080808, + 0x19192b08, 0x08080808, 0x192b0819, 0x08080808, 0x192b1908, 0x08080808, 0x2b080808, 0x08080808, + 0x2b08082b, 0x08080808, 0x2b081919, 0x08080808, 0x2b082b08, 0x08080808, 0x2b190819, 0x08080808, + 0x2b191908, 0x08080808, 0x2b192b19, 0x08080808, 0x2b2b0808, 0x08080808, 0x08080819, 0x08080819, + 0x08081908, 0x08080819, 0x0808192b, 0x08080819, 0x08082b19, 0x08080819, 0x08190808, 0x08080819, + 0x0819082b, 0x08080819, 0x08191919, 0x08080819, 0x08192b08, 0x08080819, 0x08192b2b, 0x08080819, + 0x082b0819, 0x08080819, 0x082b1908, 0x08080819, 0x19080808, 0x08080819, 0x1908082b, 0x08080819, + 0x19081919, 0x08080819, 0x19082b08, 0x08080819, 0x19190819, 0x08080819, 0x19191908, 0x08080819, + 0x192b0808, 0x08080819, 0x192b2b08, 0x08080819, 0x2b080819, 0x08080819, 0x2b081908, 0x08080819, + 0x2b190808, 0x08080819, 0x08080808, 0x0808082b, 0x0808082b, 0x0808082b, 0x08081919, 0x0808082b, + 0x08082b08, 0x0808082b, 0x08190819, 0x0808082b, 0x08191908, 0x0808082b, 0x082b0808, 0x0808082b, + 0x19080819, 0x0808082b, 0x19081908, 0x0808082b, 0x19190808, 0x0808082b, 0x19191919, 0x0808082b, + 0x2b080808, 0x0808082b, 0x2b082b2b, 0x0808082b, 0x08080819, 0x08081908, 0x08081908, 0x08081908, + 0x0808192b, 0x08081908, 0x08082b19, 0x08081908, 0x08190808, 0x08081908, 0x0819082b, 0x08081908, + 0x08191919, 0x08081908, 0x08192b08, 0x08081908, 0x082b0819, 0x08081908, 0x082b1908, 0x08081908, + 0x19080808, 0x08081908, 0x1908082b, 0x08081908, 0x19081919, 0x08081908, 0x19082b08, 0x08081908, + 0x19190819, 0x08081908, 0x19191908, 0x08081908, 0x1919192b, 0x08081908, 0x192b0808, 0x08081908, + 0x2b080819, 0x08081908, 0x2b081908, 0x08081908, 0x2b190808, 0x08081908, 0x08080808, 0x08081919, + 0x0808082b, 0x08081919, 0x08081919, 0x08081919, 0x08082b08, 0x08081919, 0x08190819, 0x08081919, + 0x08191908, 0x08081919, 0x082b0808, 0x08081919, 0x19080819, 0x08081919, 0x19081908, 0x08081919, + 0x19190808, 0x08081919, 0x192b0819, 0x08081919, 0x2b080808, 0x08081919, 0x08080819, 0x0808192b, + 0x08081908, 0x0808192b, 0x08190808, 0x0808192b, 0x082b192b, 0x0808192b, 0x19080808, 0x0808192b, + 0x1908082b, 0x0808192b, 0x2b081908, 0x0808192b, 0x08080808, 0x08082b08, 0x0808082b, 0x08082b08, + 0x08081919, 0x08082b08, 0x08082b08, 0x08082b08, 0x08082b2b, 0x08082b08, 0x08190819, 0x08082b08, + 0x08191908, 0x08082b08, 0x082b0808, 0x08082b08, 0x082b1919, 0x08082b08, 0x19080819, 0x08082b08, + 0x19081908, 0x08082b08, 0x19190808, 0x08082b08, 0x19192b08, 0x08082b08, 0x2b080808, 0x08082b08, + 0x2b2b0808, 0x08082b08, 0x2b2b2b2b, 0x08082b08, 0x08080819, 0x08082b19, 0x08081908, 0x08082b19, + 0x08190808, 0x08082b19, 0x19080808, 0x08082b19, 0x2b080819, 0x08082b19, 0x2b082b19, 0x08082b19, + 0x08080808, 0x08082b2b, 0x082b0808, 0x08082b2b, 0x082b2b08, 0x08082b2b, 0x2b19192b, 0x08082b2b, + 0x2b2b0808, 0x08082b2b, 0x08080819, 0x08190808, 0x08081908, 0x08190808, 0x0808192b, 0x08190808, + 0x08082b19, 0x08190808, 0x08190808, 0x08190808, 0x0819082b, 0x08190808, 0x08191919, 0x08190808, + 0x08192b08, 0x08190808, 0x082b0819, 0x08190808, 0x082b1908, 0x08190808, 0x19080808, 0x08190808, + 0x1908082b, 0x08190808, 0x19081919, 0x08190808, 0x19082b08, 0x08190808, 0x19190819, 0x08190808, + 0x19191908, 0x08190808, 0x192b0808, 0x08190808, 0x192b2b2b, 0x08190808, 0x2b080819, 0x08190808, + 0x2b081908, 0x08190808, 0x2b190808, 0x08190808, 0x08080808, 0x08190819, 0x0808082b, 0x08190819, + 0x08081919, 0x08190819, 0x08082b08, 0x08190819, 0x08190819, 0x08190819, 0x08191908, 0x08190819, + 0x082b0808, 0x08190819, 0x19080819, 0x08190819, 0x19081908, 0x08190819, 0x19190808, 0x08190819, + 0x2b080808, 0x08190819, 0x2b191908, 0x08190819, 0x2b19192b, 0x08190819, 0x08080819, 0x0819082b, + 0x08081908, 0x0819082b, 0x0808192b, 0x0819082b, 0x08190808, 0x0819082b, 0x19080808, 0x0819082b, + 0x192b0808, 0x0819082b, 0x08080808, 0x08191908, 0x0808082b, 0x08191908, 0x08081919, 0x08191908, + 0x08082b08, 0x08191908, 0x08190819, 0x08191908, 0x08191908, 0x08191908, 0x082b0808, 0x08191908, + 0x19080819, 0x08191908, 0x19081908, 0x08191908, 0x19082b19, 0x08191908, 0x19190808, 0x08191908, + 0x192b1908, 0x08191908, 0x2b080808, 0x08191908, 0x08080819, 0x08191919, 0x08081908, 0x08191919, + 0x08190808, 0x08191919, 0x19080808, 0x08191919, 0x08080808, 0x0819192b, 0x08191908, 0x0819192b, + 0x19082b19, 0x0819192b, 0x08080819, 0x08192b08, 0x08081908, 0x08192b08, 0x08190808, 0x08192b08, + 0x0819082b, 0x08192b08, 0x19080808, 0x08192b08, 0x19191908, 0x08192b08, 0x2b08192b, 0x08192b08, + 0x08080808, 0x08192b19, 0x08081919, 0x08192b19, 0x192b192b, 0x08192b19, 0x19190819, 0x08192b2b, + 0x2b2b2b19, 0x08192b2b, 0x08080808, 0x082b0808, 0x0808082b, 0x082b0808, 0x08081919, 0x082b0808, + 0x08082b08, 0x082b0808, 0x08082b2b, 0x082b0808, 0x08190819, 0x082b0808, 0x08191908, 0x082b0808, + 0x082b0808, 0x082b0808, 0x19080819, 0x082b0808, 0x19081908, 0x082b0808, 0x19190808, 0x082b0808, + 0x2b080808, 0x082b0808, 0x2b2b0808, 0x082b0808, 0x08080819, 0x082b0819, 0x08081908, 0x082b0819, + 0x08190808, 0x082b0819, 0x19080808, 0x082b0819, 0x19082b08, 0x082b0819, 0x192b1919, 0x082b0819, + 0x08080808, 0x082b082b, 0x082b082b, 0x082b082b, 0x2b080808, 0x082b082b, 0x2b2b2b08, 0x082b082b, + 0x08080819, 0x082b1908, 0x08081908, 0x082b1908, 0x08190808, 0x082b1908, 0x082b2b19, 0x082b1908, + 0x19080808, 0x082b1908, 0x08080808, 0x082b1919, 0x19080819, 0x082b1919, 0x1919082b, 0x082b1919, + 0x2b192b19, 0x082b1919, 0x08080819, 0x082b192b, 0x08192b2b, 0x082b192b, 0x2b2b192b, 0x082b192b, + 0x08080808, 0x082b2b08, 0x08082b08, 0x082b2b08, 0x08082b2b, 0x082b2b08, 0x082b0808, 0x082b2b08, + 0x19191919, 0x082b2b08, 0x2b082b08, 0x082b2b08, 0x2b2b082b, 0x082b2b08, 0x192b2b08, 0x082b2b19, + 0x2b190808, 0x082b2b19, 0x08082b08, 0x082b2b2b, 0x082b0808, 0x082b2b2b, 0x2b08082b, 0x082b2b2b, + 0x2b082b08, 0x082b2b2b, 0x2b082b2b, 0x082b2b2b, 0x08080819, 0x19080808, 0x08081908, 0x19080808, + 0x0808192b, 0x19080808, 0x08082b19, 0x19080808, 0x08190808, 0x19080808, 0x0819082b, 0x19080808, + 0x08191919, 0x19080808, 0x08192b08, 0x19080808, 0x082b0819, 0x19080808, 0x082b1908, 0x19080808, + 0x19080808, 0x19080808, 0x1908082b, 0x19080808, 0x19081919, 0x19080808, 0x19082b08, 0x19080808, + 0x19082b2b, 0x19080808, 0x19190819, 0x19080808, 0x19191908, 0x19080808, 0x192b0808, 0x19080808, + 0x192b1919, 0x19080808, 0x2b080819, 0x19080808, 0x2b081908, 0x19080808, 0x2b190808, 0x19080808, + 0x08080808, 0x19080819, 0x0808082b, 0x19080819, 0x08081919, 0x19080819, 0x08082b08, 0x19080819, + 0x08190819, 0x19080819, 0x08191908, 0x19080819, 0x082b0808, 0x19080819, 0x19080819, 0x19080819, + 0x19081908, 0x19080819, 0x19190808, 0x19080819, 0x2b080808, 0x19080819, 0x2b081919, 0x19080819, + 0x2b2b082b, 0x19080819, 0x08080819, 0x1908082b, 0x08081908, 0x1908082b, 0x08190808, 0x1908082b, + 0x0819082b, 0x1908082b, 0x082b2b19, 0x1908082b, 0x19080808, 0x1908082b, 0x08080808, 0x19081908, + 0x0808082b, 0x19081908, 0x08081919, 0x19081908, 0x08082b08, 0x19081908, 0x08190819, 0x19081908, + 0x08191908, 0x19081908, 0x08192b19, 0x19081908, 0x082b0808, 0x19081908, 0x19080819, 0x19081908, + 0x19081908, 0x19081908, 0x19190808, 0x19081908, 0x2b080808, 0x19081908, 0x2b191908, 0x19081908, + 0x08080819, 0x19081919, 0x08081908, 0x19081919, 0x08190808, 0x19081919, 0x082b1908, 0x19081919, + 0x19080808, 0x19081919, 0x2b192b2b, 0x19081919, 0x08080808, 0x1908192b, 0x08082b2b, 0x1908192b, + 0x19081908, 0x1908192b, 0x19190808, 0x1908192b, 0x08080819, 0x19082b08, 0x08081908, 0x19082b08, + 0x08190808, 0x19082b08, 0x19080808, 0x19082b08, 0x19081919, 0x19082b08, 0x19191908, 0x19082b08, + 0x192b082b, 0x19082b08, 0x08080808, 0x19082b19, 0x08190819, 0x19082b19, 0x19081908, 0x19082b19, + 0x19190808, 0x19082b19, 0x192b2b19, 0x19082b19, 0x08081908, 0x19082b2b, 0x08080808, 0x19190808, + 0x0808082b, 0x19190808, 0x08081919, 0x19190808, 0x08082b08, 0x19190808, 0x08190819, 0x19190808, + 0x08191908, 0x19190808, 0x082b0808, 0x19190808, 0x082b2b08, 0x19190808, 0x19080819, 0x19190808, + 0x19081908, 0x19190808, 0x19190808, 0x19190808, 0x2b080808, 0x19190808, 0x08080819, 0x19190819, + 0x08081908, 0x19190819, 0x08190808, 0x19190819, 0x08191919, 0x19190819, 0x19080808, 0x19190819, + 0x1908082b, 0x19190819, 0x08080808, 0x1919082b, 0x19081908, 0x1919082b, 0x2b2b2b2b, 0x1919082b, + 0x08080819, 0x19191908, 0x08081908, 0x19191908, 0x08190808, 0x19191908, 0x082b0819, 0x19191908, + 0x19080808, 0x19191908, 0x192b0808, 0x19191908, 0x2b080819, 0x19191908, 0x2b2b0819, 0x19191908, + 0x08080808, 0x19191919, 0x08082b08, 0x19191919, 0x2b080808, 0x19191919, 0x2b082b08, 0x19191919, + 0x082b0819, 0x1919192b, 0x192b2b08, 0x1919192b, 0x2b2b0819, 0x1919192b, 0x08080808, 0x19192b08, + 0x08191908, 0x19192b08, 0x19080819, 0x19192b08, 0x19190808, 0x19192b08, 0x2b192b19, 0x19192b08, + 0x08192b2b, 0x19192b19, 0x19080808, 0x19192b19, 0x1908082b, 0x19192b19, 0x2b081919, 0x19192b2b, + 0x08080819, 0x192b0808, 0x08081908, 0x192b0808, 0x08190808, 0x192b0808, 0x19080808, 0x192b0808, + 0x19191908, 0x192b0808, 0x192b082b, 0x192b0808, 0x2b08192b, 0x192b0808, 0x2b2b2b19, 0x192b0808, + 0x08080808, 0x192b0819, 0x082b1908, 0x192b082b, 0x19082b2b, 0x192b082b, 0x2b19082b, 0x192b082b, + 0x08080808, 0x192b1908, 0x0819192b, 0x192b1908, 0x08190808, 0x192b1919, 0x19080808, 0x192b1919, + 0x19081919, 0x192b1919, 0x2b2b1908, 0x192b1919, 0x08080819, 0x192b2b08, 0x192b2b2b, 0x192b2b08, + 0x082b1919, 0x192b2b19, 0x0808192b, 0x192b2b2b, 0x19191908, 0x192b2b2b, 0x192b082b, 0x192b2b2b, + 0x08080808, 0x2b080808, 0x0808082b, 0x2b080808, 0x08081919, 0x2b080808, 0x08082b08, 0x2b080808, + 0x08190819, 0x2b080808, 0x08191908, 0x2b080808, 0x082b0808, 0x2b080808, 0x082b2b2b, 0x2b080808, + 0x19080819, 0x2b080808, 0x19081908, 0x2b080808, 0x19190808, 0x2b080808, 0x2b080808, 0x2b080808, + 0x2b08082b, 0x2b080808, 0x2b2b2b08, 0x2b080808, 0x2b2b2b2b, 0x2b080808, 0x08080819, 0x2b080819, + 0x08081908, 0x2b080819, 0x0808192b, 0x2b080819, 0x08190808, 0x2b080819, 0x19080808, 0x2b080819, + 0x19190819, 0x2b080819, 0x19192b19, 0x2b080819, 0x08080808, 0x2b08082b, 0x082b0808, 0x2b08082b, + 0x2b080808, 0x2b08082b, 0x2b08082b, 0x2b08082b, 0x2b2b0808, 0x2b08082b, 0x2b2b2b08, 0x2b08082b, + 0x08080819, 0x2b081908, 0x08081908, 0x2b081908, 0x08190808, 0x2b081908, 0x0819082b, 0x2b081908, + 0x08191919, 0x2b081908, 0x19080808, 0x2b081908, 0x192b0808, 0x2b081908, 0x2b082b19, 0x2b081908, + 0x08080808, 0x2b081919, 0x19081908, 0x2b081919, 0x2b2b1919, 0x2b081919, 0x08192b08, 0x2b08192b, + 0x192b2b2b, 0x2b08192b, 0x08080808, 0x2b082b08, 0x08082b08, 0x2b082b08, 0x082b1919, 0x2b082b08, + 0x19192b2b, 0x2b082b08, 0x2b080808, 0x2b082b08, 0x2b08082b, 0x2b082b08, 0x2b2b2b08, 0x2b082b08, + 0x0808192b, 0x2b082b19, 0x082b082b, 0x2b082b2b, 0x2b080808, 0x2b082b2b, 0x2b082b08, 0x2b082b2b, + 0x2b19192b, 0x2b082b2b, 0x2b2b2b08, 0x2b082b2b, 0x08080819, 0x2b190808, 0x08081908, 0x2b190808, + 0x08190808, 0x2b190808, 0x19080808, 0x2b190808, 0x1919192b, 0x2b190808, 0x2b081908, 0x2b190808, + 0x08080808, 0x2b190819, 0x082b082b, 0x2b190819, 0x192b1908, 0x2b190819, 0x1919192b, 0x2b19082b, + 0x2b082b19, 0x2b19082b, 0x08080808, 0x2b191908, 0x08081919, 0x2b191908, 0x19081908, 0x2b191908, + 0x19190808, 0x2b191908, 0x19192b08, 0x2b191908, 0x082b2b19, 0x2b191919, 0x2b190808, 0x2b191919, + 0x2b19082b, 0x2b191919, 0x19080819, 0x2b19192b, 0x19190819, 0x2b192b08, 0x2b2b192b, 0x2b192b08, + 0x19082b19, 0x2b192b19, 0x08191919, 0x2b192b2b, 0x192b0808, 0x2b192b2b, 0x08080808, 0x2b2b0808, + 0x0808082b, 0x2b2b0808, 0x08082b08, 0x2b2b0808, 0x08082b2b, 0x2b2b0808, 0x082b0808, 0x2b2b0808, + 0x082b2b2b, 0x2b2b0808, 0x2b2b0808, 0x2b2b0808, 0x19190819, 0x2b2b0819, 0x19192b19, 0x2b2b0819, + 0x2b2b192b, 0x2b2b0819, 0x08080808, 0x2b2b082b, 0x0808082b, 0x2b2b082b, 0x08082b08, 0x2b2b082b, + 0x082b2b2b, 0x2b2b082b, 0x2b080808, 0x2b2b082b, 0x2b2b0808, 0x2b2b082b, 0x19080808, 0x2b2b1908, + 0x2b191919, 0x2b2b1908, 0x192b1919, 0x2b2b192b, 0x2b192b08, 0x2b2b192b, 0x08082b2b, 0x2b2b2b08, + 0x082b0808, 0x2b2b2b08, 0x082b082b, 0x2b2b2b08, 0x082b2b08, 0x2b2b2b08, 0x2b2b0808, 0x2b2b2b08, + 0x2b2b2b08, 0x2b2b2b08, 0x08081908, 0x2b2b2b19, 0x2b081908, 0x2b2b2b19, 0x2b08192b, 0x2b2b2b19, + 0x082b2b08, 0x2b2b2b2b, 0x082b2b2b, 0x2b2b2b2b, 0x2b190819, 0x2b2b2b2b, 0x2b2b2b2b, 0x2b2b2b2b) +} + +//! Main-context use only (tests, oracles, the emitter's constant bake) - a forked worker reads +//! a module global as zeros; kernels take `iq2xs_grid2()` instead. +let IQ2XS_GRID = iq2xs_grid2() + //! Quant-plane bytes per stride unit — the typed stride surface. The stride unit is one //! 256-weight superblock for the kq lattice and one 32-weight block for q51 (see kq_elems). //! q8 panics (its planes are byte-per-elem, not stride-walked here). @@ -562,13 +708,16 @@ def kq_qsb(f : KqFmt) : int64 { if (f == KqFmt.iq2s) { return IQ2S_QSB } + if (f == KqFmt.iq2xs) { + return IQ2XS_QSB + } panic("kq_qsb: no plane stride for this format") return 0l } //! Scale-plane bytes per stride unit (see kq_qsb for the unit). def kq_ssb(f : KqFmt) : int64 { - if (f == KqFmt.k4 || f == KqFmt.k5 || f == KqFmt.iq4xs || f == KqFmt.iq3s || f == KqFmt.iq3xxs || f == KqFmt.k2 || f == KqFmt.iq2s) { + if (f == KqFmt.k4 || f == KqFmt.k5 || f == KqFmt.iq4xs || f == KqFmt.iq3s || f == KqFmt.iq3xxs || f == KqFmt.k2 || f == KqFmt.iq2s || f == KqFmt.iq2xs) { return K4_SSB } if (f == KqFmt.k6 || f == KqFmt.k3) { diff --git a/modules/dasLLAMA/dasllama/dasllama_layout.das b/modules/dasLLAMA/dasllama/dasllama_layout.das index 7d2d130ee7..2e87580458 100644 --- a/modules/dasLLAMA/dasllama/dasllama_layout.das +++ b/modules/dasLLAMA/dasllama/dasllama_layout.das @@ -435,6 +435,10 @@ def private repack_regions(var t : Model; regs : array) { var iq2ssp : uint8? = null if (!empty(t.iq2sq)) { iq2sqp = addr(t.iq2sq[0]) } if (!empty(t.iq2ss)) { iq2ssp = addr(t.iq2ss[0]) } + var iq2xsqp : uint8? = null + var iq2xssp : uint8? = null + if (!empty(t.iq2xsq)) { iq2xsqp = addr(t.iq2xsq[0]) } + if (!empty(t.iq2xss)) { iq2xssp = addr(t.iq2xss[0]) } if (!empty(t.qblob)) { qbp = addr(t.qblob[0]) } if (!empty(t.qscales)) { qsp = addr(t.qscales[0]) } if (!empty(t.mxq)) { mxqp = addr(t.mxq[0]) } @@ -464,11 +468,11 @@ def private repack_regions(var t : Model; regs : array) { invoke(rq51, q51qp + (rp[i].off / 32l) * Q51_QPB, q51sp + (rp[i].off / 32l) * Q51_SPB, rp[i].n, rp[i].d) } else { let sb = rp[i].off / 256l - let fk = f == 20 ? 2 : f // k2 streams under 20 (kernel id 2 = q51's stream tag) + let fk = f == 20 ? 2 : f let qsb = kq_qsb(fk) let ssb = kq_ssb(fk) - var kqp = f == 4 ? k4qp : (f == 5 ? k5qp : (f == 6 ? k6qp : (f == 40 ? q40qp : (f == 44 ? iq4xsqp : (f == 3 ? k3qp : (f == 33 ? iq3sqp : (f == 34 ? iq3xxsqp : (f == 45 ? iq4nlqp : (f == 2 ? k2qp : iq2sqp))))))))) - var ksp = f == 4 ? k4sp : (f == 5 ? k5sp : (f == 6 ? k6sp : (f == 40 ? q40sp : (f == 44 ? iq4xssp : (f == 3 ? k3sp : (f == 33 ? iq3ssp : (f == 34 ? iq3xxssp : (f == 45 ? iq4nlsp : (f == 2 ? k2sp : iq2ssp))))))))) + var kqp = f == 4 ? k4qp : (f == 5 ? k5qp : (f == 6 ? k6qp : (f == 40 ? q40qp : (f == 44 ? iq4xsqp : (f == 3 ? k3qp : (f == 33 ? iq3sqp : (f == 34 ? iq3xxsqp : (f == 45 ? iq4nlqp : (fk == 2 ? k2qp : (f == 23 ? iq2sqp : iq2xsqp)))))))))) + var ksp = f == 4 ? k4sp : (f == 5 ? k5sp : (f == 6 ? k6sp : (f == 40 ? q40sp : (f == 44 ? iq4xssp : (f == 3 ? k3sp : (f == 33 ? iq3ssp : (f == 34 ? iq3xxssp : (f == 45 ? iq4nlsp : (fk == 2 ? k2sp : (f == 23 ? iq2ssp : iq2xssp)))))))))) invoke(rkq, fk, kqp + sb * qsb, ksp + sb * ssb, rp[i].n, rp[i].d) } } @@ -608,6 +612,8 @@ def private push_repack_kq(var regs : array; fmt : KqFmt; woff, n, d push_repack(regs, 20, woff, n, d) } elif (fmt == KqFmt.iq2s) { push_repack(regs, 23, woff, n, d) + } elif (fmt == KqFmt.iq2xs) { + push_repack(regs, 24, woff, n, d) } } @@ -885,13 +891,13 @@ def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice : (fmt == KqFmt.k6 ? addr(t.k6q[sb0 * kq_qsb(6)]) : (fmt == KqFmt.q40 ? addr(t.q40q[sb0 * kq_qsb(40)]) : (fmt == KqFmt.iq4xs ? addr(t.iq4xsq[sb0 * kq_qsb(44)]) - : (fmt == KqFmt.k3 ? addr(t.k3q[sb0 * kq_qsb(3)]) : (fmt == KqFmt.iq3s ? addr(t.iq3sq[sb0 * kq_qsb(33)]) : (fmt == KqFmt.iq3xxs ? addr(t.iq3xxsq[sb0 * kq_qsb(34)]) : (fmt == KqFmt.iq4nl ? addr(t.iq4nlq[sb0 * kq_qsb(45)]) : (fmt == KqFmt.k2 ? addr(t.k2q[sb0 * kq_qsb(2)]) : addr(t.iq2sq[sb0 * kq_qsb(23)]))))))))))) + : (fmt == KqFmt.k3 ? addr(t.k3q[sb0 * kq_qsb(3)]) : (fmt == KqFmt.iq3s ? addr(t.iq3sq[sb0 * kq_qsb(33)]) : (fmt == KqFmt.iq3xxs ? addr(t.iq3xxsq[sb0 * kq_qsb(34)]) : (fmt == KqFmt.iq4nl ? addr(t.iq4nlq[sb0 * kq_qsb(45)]) : (fmt == KqFmt.k2 ? addr(t.k2q[sb0 * kq_qsb(2)]) : (fmt == KqFmt.iq2s ? addr(t.iq2sq[sb0 * kq_qsb(23)]) : addr(t.iq2xsq[sb0 * kq_qsb(24)])))))))))))) let sp = (fmt == KqFmt.k4 ? addr(t.k4s[sb0 * kq_ssb(4)]) : (fmt == KqFmt.k5 ? addr(t.k5s[sb0 * kq_ssb(5)]) : (fmt == KqFmt.k6 ? addr(t.k6s[sb0 * kq_ssb(6)]) : (fmt == KqFmt.q40 ? addr(t.q40s[sb0 * kq_ssb(40)]) : (fmt == KqFmt.iq4xs ? addr(t.iq4xss[sb0 * kq_ssb(44)]) - : (fmt == KqFmt.k3 ? addr(t.k3s[sb0 * kq_ssb(3)]) : (fmt == KqFmt.iq3s ? addr(t.iq3ss[sb0 * kq_ssb(33)]) : (fmt == KqFmt.iq3xxs ? addr(t.iq3xxss[sb0 * kq_ssb(34)]) : (fmt == KqFmt.iq4nl ? addr(t.iq4nls[sb0 * kq_ssb(45)]) : (fmt == KqFmt.k2 ? addr(t.k2s[sb0 * kq_ssb(2)]) : addr(t.iq2ss[sb0 * kq_ssb(23)]))))))))))) + : (fmt == KqFmt.k3 ? addr(t.k3s[sb0 * kq_ssb(3)]) : (fmt == KqFmt.iq3s ? addr(t.iq3ss[sb0 * kq_ssb(33)]) : (fmt == KqFmt.iq3xxs ? addr(t.iq3xxss[sb0 * kq_ssb(34)]) : (fmt == KqFmt.iq4nl ? addr(t.iq4nls[sb0 * kq_ssb(45)]) : (fmt == KqFmt.k2 ? addr(t.k2s[sb0 * kq_ssb(2)]) : (fmt == KqFmt.iq2s ? addr(t.iq2ss[sb0 * kq_ssb(23)]) : addr(t.iq2xss[sb0 * kq_ssb(24)])))))))))))) var wqp = addr(wq[0]) var wsp = addr(ws[0]) let njobs = is_job_que_available() ? min(nslices, 4 * (get_total_hw_jobs() + 1)) : 1 @@ -934,6 +940,10 @@ def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice for (m in range64(72l)) { dq[m] = gq[((m / 4l) * mr + r) * 4l + m % 4l] } + } elif (fmt == KqFmt.iq2xs) { // 16 uniform 4-byte columns over the u16 qs words + for (m in range64(64l)) { + dq[m] = gq[((m / 4l) * mr + r) * 4l + m % 4l] + } } else { for (m in range64(128l)) { // nibble region: verbatim per (blk, j, t) dq[m] = gq[((m / 4l) * mr + r) * 4l + m % 4l] @@ -985,7 +995,7 @@ def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice } else { let rq = qp + sliceQ + ri * qrow + sbi * qsb let rs = sp + sliceS + ri * srow + sbi * dssb - if (fmt == KqFmt.q40 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s || fmt == KqFmt.iq3xxs || fmt == KqFmt.iq4nl || fmt == KqFmt.k2 || fmt == KqFmt.iq2s) { + if (fmt == KqFmt.q40 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s || fmt == KqFmt.iq3xxs || fmt == KqFmt.iq4nl || fmt == KqFmt.k2 || fmt == KqFmt.iq2s || fmt == KqFmt.iq2xs) { for (m in range64(qsb)) { // disk bytes already in the device form (k/k+16 pairs; k3's lanes) dq[m] = rq[m] } @@ -1012,7 +1022,7 @@ def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice dq[128l + p] = rq[128l + p] } } - if (fmt == KqFmt.k6 || fmt == KqFmt.q40 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s || fmt == KqFmt.iq3xxs || fmt == KqFmt.iq4nl || fmt == KqFmt.k2 || fmt == KqFmt.iq2s) { // rows already in the decoded device form + if (fmt == KqFmt.k6 || fmt == KqFmt.q40 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s || fmt == KqFmt.iq3xxs || fmt == KqFmt.iq4nl || fmt == KqFmt.k2 || fmt == KqFmt.iq2s || fmt == KqFmt.iq2xs) { // rows already in the decoded device form for (idx in range64(dssb)) { dsc[idx] = rs[idx] } diff --git a/modules/dasLLAMA/dasllama/dasllama_load.das b/modules/dasLLAMA/dasllama/dasllama_load.das index 7e0312fc13..91379c37fd 100644 --- a/modules/dasLLAMA/dasllama/dasllama_load.das +++ b/modules/dasLLAMA/dasllama/dasllama_load.das @@ -48,6 +48,7 @@ struct private LayoutSizes { iq4nl_n : int64 k2_n : int64 iq2s_n : int64 + iq2xs_n : int64 pleq8_n : int64 // the dedicated q8 plane holding ONLY the PLE table (fp32/q4 serving) } @@ -67,6 +68,7 @@ struct private KqCursors { iq4nl : int64 k2 : int64 iq2s : int64 + iq2xs : int64 } def private kq_take(var cur : KqCursors; f : KqFmt; n : int64) : int64 { @@ -130,6 +132,11 @@ def private kq_take(var cur : KqCursors; f : KqFmt; n : int64) : int64 { cur.iq2s += n return o } + if (f == KqFmt.iq2xs) { + let o = cur.iq2xs + cur.iq2xs += n + return o + } let o = cur.wo cur.wo += n return o @@ -429,7 +436,7 @@ def private layout_offsets(var t : Model) : LayoutSizes { // nolint:STYLE037,S fo += dim } return LayoutSizes(fblob_n = fo, wblob_n = cur.wo, mx_n = mx, bf16_n = bf16, - k4_n = cur.k4, k5_n = cur.k5, k6_n = cur.k6, q40_n = cur.q40, q51_n = cur.q51, iq4xs_n = cur.iq4xs, k3_n = cur.k3, iq3s_n = cur.iq3s, iq3xxs_n = cur.iq3xxs, iq4nl_n = cur.iq4nl, k2_n = cur.k2, iq2s_n = cur.iq2s, pleq8_n = pleq8) + k4_n = cur.k4, k5_n = cur.k5, k6_n = cur.k6, q40_n = cur.q40, q51_n = cur.q51, iq4xs_n = cur.iq4xs, k3_n = cur.k3, iq3s_n = cur.iq3s, iq3xxs_n = cur.iq3xxs, iq4nl_n = cur.iq4nl, k2_n = cur.k2, iq2s_n = cur.iq2s, iq2xs_n = cur.iq2xs, pleq8_n = pleq8) } @@ -702,6 +709,8 @@ def stream_field_of(t : Model; fmt : KqFmt) : string { return "k2q" } elif (fmt == KqFmt.iq2s) { return "iq2sq" + } elif (fmt == KqFmt.iq2xs) { + return "iq2xsq" } elif (t.quant == QuantMode.q8) { // the blob flavor carries q8 as gguf-native 34B blocks in ONE plane, never the split pair return t.metal_blob ? "mblob" : "qblob" @@ -832,6 +841,8 @@ def fill_stream_plane(m : GGUFMeta; bytes : array | #; var t : Model; fie memcpy(addr(t.k2s[(j.woff / 256l) * K2_SSB]), addr(temp_s[0]), sh.sb) } elif (j.fmt == KqFmt.iq2s) { memcpy(addr(t.iq2ss[(j.woff / 256l) * IQ2S_SSB]), addr(temp_s[0]), sh.sb) + } elif (j.fmt == KqFmt.iq2xs) { + memcpy(addr(t.iq2xss[(j.woff / 256l) * IQ2XS_SSB]), addr(temp_s[0]), sh.sb) } elif (j.fmt == KqFmt.q51) { memcpy(addr(t.q51s[(j.woff / 32l) * Q51_SB]), addr(temp_s[0]), sh.sb) } elif (sh.sb == 0l && j.n > 0l && !t.metal_blob) { @@ -884,6 +895,9 @@ def fill_stream_plane(m : GGUFMeta; bytes : array | #; var t : Model; fie } elif (r.fmt == 23) { invoke(g_stream_repack, 23, addr(temp_q[((r.off - j.woff) / 256l) * IQ2S_QSB]), addr(t.iq2ss[(r.off / 256l) * IQ2S_SSB]), r.n, r.d) + } elif (r.fmt == 24) { + invoke(g_stream_repack, 24, addr(temp_q[((r.off - j.woff) / 256l) * IQ2XS_QSB]), + addr(t.iq2xss[(r.off / 256l) * IQ2XS_SSB]), r.n, r.d) } } } @@ -1117,6 +1131,9 @@ def private transcode_kq_tensor(m : GGUFMeta; bytes : array | #; name : s } elif (fmt == KqFmt.iq2s) { gguf_transcode_iq2s(m, bytes, name, kq, ks, eloff, n, src_off) return "iq2s transcode (IQ2_S)" + } elif (fmt == KqFmt.iq2xs) { + gguf_transcode_iq2xs(m, bytes, name, kq, ks, eloff, n, src_off) + return "iq2xs transcode (IQ2_XS)" } gguf_transcode_q51(m, bytes, name, kq, ks, eloff, n, src_off) return "q51 transcode (Q5_1)" @@ -1169,6 +1186,8 @@ def private load_big(m : GGUFMeta; bytes : array | #; name : string; var kind = transcode_kq_tensor(m, bytes, name, fmt, t.k2q, t.k2s, woff, n, src_off, scratch) } elif (fmt == KqFmt.iq2s) { kind = transcode_kq_tensor(m, bytes, name, fmt, t.iq2sq, t.iq2ss, woff, n, src_off, scratch) + } elif (fmt == KqFmt.iq2xs) { + kind = transcode_kq_tensor(m, bytes, name, fmt, t.iq2xsq, t.iq2xss, woff, n, src_off, scratch) } elif (fmt == KqFmt.q51) { kind = transcode_kq_tensor(m, bytes, name, fmt, t.q51q, t.q51s, woff, n, src_off, scratch) } elif (t.quant == QuantMode.q8) { @@ -1268,8 +1287,8 @@ def private log_load_report(t : Model) { to_log(LOG_INFO, "dasLLAMA noisy: backend '{active_kernel_backend()}' | kquant_native {get_kquant_native()} q40_native {get_kq_q40_native()} q50_native {get_kq_q50_native()}\n") // the tier arms at [init], before a --noisy CLI flag can land — restate it here so the flag alone suffices to_log(LOG_INFO, "dasLLAMA noisy: GPU tier installed {moe_gpu_tier_installed()} | want auto {gpu_want_auto()}, moe layers {gpu_want_moe_layers()}, stream {gpu_want_moe_stream()}\n") - to_log(LOG_INFO, "dasLLAMA noisy: format tags - q8 {tally_fmt(t, KqFmt.q8)}, k4 {tally_fmt(t, KqFmt.k4)}, k5 {tally_fmt(t, KqFmt.k5)}, k6 {tally_fmt(t, KqFmt.k6)}, q40 {tally_fmt(t, KqFmt.q40)}, q51 {tally_fmt(t, KqFmt.q51)}, iq4xs {tally_fmt(t, KqFmt.iq4xs)}, k3 {tally_fmt(t, KqFmt.k3)}, iq3s {tally_fmt(t, KqFmt.iq3s)}, iq3xxs {tally_fmt(t, KqFmt.iq3xxs)}, iq4nl {tally_fmt(t, KqFmt.iq4nl)}, k2 {tally_fmt(t, KqFmt.k2)}, iq2s {tally_fmt(t, KqFmt.iq2s)}\n") - to_log(LOG_INFO, "dasLLAMA noisy: plane MB - q8 {float(long_length(t.qblob)) / mb} (+scales {float(long_length(t.qscales) * 4l) / mb}), k4 {float(long_length(t.k4q) + long_length(t.k4s)) / mb}, k5 {float(long_length(t.k5q) + long_length(t.k5s)) / mb}, k6 {float(long_length(t.k6q) + long_length(t.k6s)) / mb}, q40 {float(long_length(t.q40q) + long_length(t.q40s)) / mb}, q51 {float(long_length(t.q51q) + long_length(t.q51s)) / mb}, iq4xs {float(long_length(t.iq4xsq) + long_length(t.iq4xss)) / mb}, k3 {float(long_length(t.k3q) + long_length(t.k3s)) / mb}, iq3s {float(long_length(t.iq3sq) + long_length(t.iq3ss)) / mb}, iq3xxs {float(long_length(t.iq3xxsq) + long_length(t.iq3xxss)) / mb}, iq4nl {float(long_length(t.iq4nlq) + long_length(t.iq4nls)) / mb}, k2 {float(long_length(t.k2q) + long_length(t.k2s)) / mb}, iq2s {float(long_length(t.iq2sq) + long_length(t.iq2ss)) / mb}, f32 {float(long_length(t.wblob) * 4l) / mb}\n") + to_log(LOG_INFO, "dasLLAMA noisy: format tags - q8 {tally_fmt(t, KqFmt.q8)}, k4 {tally_fmt(t, KqFmt.k4)}, k5 {tally_fmt(t, KqFmt.k5)}, k6 {tally_fmt(t, KqFmt.k6)}, q40 {tally_fmt(t, KqFmt.q40)}, q51 {tally_fmt(t, KqFmt.q51)}, iq4xs {tally_fmt(t, KqFmt.iq4xs)}, k3 {tally_fmt(t, KqFmt.k3)}, iq3s {tally_fmt(t, KqFmt.iq3s)}, iq3xxs {tally_fmt(t, KqFmt.iq3xxs)}, iq4nl {tally_fmt(t, KqFmt.iq4nl)}, k2 {tally_fmt(t, KqFmt.k2)}, iq2s {tally_fmt(t, KqFmt.iq2s)}, iq2xs {tally_fmt(t, KqFmt.iq2xs)}\n") + to_log(LOG_INFO, "dasLLAMA noisy: plane MB - q8 {float(long_length(t.qblob)) / mb} (+scales {float(long_length(t.qscales) * 4l) / mb}), k4 {float(long_length(t.k4q) + long_length(t.k4s)) / mb}, k5 {float(long_length(t.k5q) + long_length(t.k5s)) / mb}, k6 {float(long_length(t.k6q) + long_length(t.k6s)) / mb}, q40 {float(long_length(t.q40q) + long_length(t.q40s)) / mb}, q51 {float(long_length(t.q51q) + long_length(t.q51s)) / mb}, iq4xs {float(long_length(t.iq4xsq) + long_length(t.iq4xss)) / mb}, k3 {float(long_length(t.k3q) + long_length(t.k3s)) / mb}, iq3s {float(long_length(t.iq3sq) + long_length(t.iq3ss)) / mb}, iq3xxs {float(long_length(t.iq3xxsq) + long_length(t.iq3xxss)) / mb}, iq4nl {float(long_length(t.iq4nlq) + long_length(t.iq4nls)) / mb}, k2 {float(long_length(t.k2q) + long_length(t.k2s)) / mb}, iq2s {float(long_length(t.iq2sq) + long_length(t.iq2ss)) / mb}, iq2xs {float(long_length(t.iq2xsq) + long_length(t.iq2xss)) / mb}, f32 {float(long_length(t.wblob) * 4l) / mb}\n") } // GGML disk type -> plane format tag (non-K-quant types ride the classic q8 path) @@ -1304,6 +1323,9 @@ def private kq_fmt_of(gt : int) : KqFmt { if (gt == GGML_TYPE_IQ2_S) { return KqFmt.iq2s } + if (gt == GGML_TYPE_IQ2_XS) { + return KqFmt.iq2xs + } if (gt == GGML_TYPE_Q4_0 && get_kq_q40_native()) { return KqFmt.q40 } @@ -2144,6 +2166,7 @@ def private load_gguf_parsed(var t : Model; m : GGUFMeta; bytes : array | g_stream_plane_total["iq4nlq"] = (sz.iq4nl_n / 256l) * Q40_QSB g_stream_plane_total["k2q"] = (sz.k2_n / 256l) * K2_QSB g_stream_plane_total["iq2sq"] = (sz.iq2s_n / 256l) * IQ2S_QSB + g_stream_plane_total["iq2xsq"] = (sz.iq2xs_n / 256l) * IQ2XS_QSB let wb = mode == QuantMode.q8 ? sz.wblob_n : 0l g_stream_plane_total[t.metal_blob ? "mblob" : "qblob"] = (t.metal_blob ? (wb / Q8_BLOCK_ELEMS) * (Q8_QPB + Q8_SPB) : wb) @@ -2244,6 +2267,14 @@ def private load_gguf_parsed(var t : Model; m : GGUFMeta; bytes : array | t.iq2ss |> reserve((sz.iq2s_n / 256l) * IQ2S_SSB) t.iq2ss |> resize((sz.iq2s_n / 256l) * IQ2S_SSB) } + if (sz.iq2xs_n > 0l) { + if (!stream_q) { + t.iq2xsq |> reserve((sz.iq2xs_n / 256l) * IQ2XS_QSB) + t.iq2xsq |> resize((sz.iq2xs_n / 256l) * IQ2XS_QSB) + } + t.iq2xss |> reserve((sz.iq2xs_n / 256l) * IQ2XS_SSB) + t.iq2xss |> resize((sz.iq2xs_n / 256l) * IQ2XS_SSB) + } if (mode == QuantMode.q8) { if (!stream_q) { t.qblob |> reserve(sz.wblob_n) @@ -2557,6 +2588,7 @@ def private load_gguf_parsed(var t : Model; m : GGUFMeta; bytes : array | t.kq_repack_mr45 = active_kq_layout_mr(45) t.kq_repack_mr2 = active_kq_layout_mr(2) t.kq_repack_mr23 = active_kq_layout_mr(23) + t.kq_repack_mr24 = active_kq_layout_mr(24) } } else { layout_repack_q8(t) @@ -2580,7 +2612,8 @@ def private load_gguf_parsed(var t : Model; m : GGUFMeta; bytes : array | t.kq_repack_mr45 = active_kq_layout_mr(45) t.kq_repack_mr2 = active_kq_layout_mr(2) t.kq_repack_mr23 = active_kq_layout_mr(23) - to_log(LOG_INFO, "dasLLAMA: K-quant planes repacked to the '{active_kernel_backend()}' layouts (k4 grp{t.kq_repack_mr4}, k5 grp{t.kq_repack_mr5}, k6 grp{t.kq_repack_mr6}, q40 grp{t.kq_repack_mr40}, iq4xs grp{t.kq_repack_mr44}, k3 grp{t.kq_repack_mr3}, iq3s grp{t.kq_repack_mr33}, iq3xxs grp{t.kq_repack_mr34}, iq4nl grp{t.kq_repack_mr45}, k2 grp{t.kq_repack_mr2}, iq2s grp{t.kq_repack_mr23})\n") + t.kq_repack_mr24 = active_kq_layout_mr(24) + to_log(LOG_INFO, "dasLLAMA: K-quant planes repacked to the '{active_kernel_backend()}' layouts (k4 grp{t.kq_repack_mr4}, k5 grp{t.kq_repack_mr5}, k6 grp{t.kq_repack_mr6}, q40 grp{t.kq_repack_mr40}, iq4xs grp{t.kq_repack_mr44}, k3 grp{t.kq_repack_mr3}, iq3s grp{t.kq_repack_mr33}, iq3xxs grp{t.kq_repack_mr34}, iq4nl grp{t.kq_repack_mr45}, k2 grp{t.kq_repack_mr2}, iq2s grp{t.kq_repack_mr23}, iq2xs grp{t.kq_repack_mr24})\n") } } } diff --git a/modules/dasLLAMA/dasllama/dasllama_math.das b/modules/dasLLAMA/dasllama/dasllama_math.das index 4ba609527b..39216c4e08 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math.das +++ b/modules/dasLLAMA/dasllama/dasllama_math.das @@ -1091,6 +1091,7 @@ struct KernelBackend { kq_rows_iq4nl : MatmulKqRowsFn = @@kq_unset_rows kq_rows_k2 : MatmulKqRowsFn = @@kq_unset_rows kq_rows_iq2s : MatmulKqRowsFn = @@kq_unset_rows + kq_rows_iq2xs : MatmulKqRowsFn = @@kq_unset_rows kq_batch : MatmulKqBatchFn = @@kq_unset_batch kq_groupn : MatmulKqGroupNFn = @@kq_unset_groupn kq_batch_groupn : MatmulKqBatchGroupNFn = @@kq_unset_batch_groupn @@ -1160,6 +1161,7 @@ var g_kq_rows_iq3xxs = @@kq_unset_rows var g_kq_rows_iq4nl = @@kq_unset_rows var g_kq_rows_k2 = @@kq_unset_rows var g_kq_rows_iq2s = @@kq_unset_rows +var g_kq_rows_iq2xs = @@kq_unset_rows var g_mm_kq_batch = @@kq_unset_batch var g_mm_kq_groupn = @@kq_unset_groupn var g_mm_kq_batch_groupn = @@kq_unset_batch_groupn @@ -1487,7 +1489,7 @@ def active_q8_layout_mr() : int64 => g_bake_cpu_override ? g_bake_cpu.q8_mr : g_ //! family split; the loader freezes these onto the Model at repack time. def active_kq_layout_mr(fmt : int) : int64 { if (g_bake_cpu_override) { - return fmt == 4 ? g_bake_cpu.kq_mr4 : (fmt == 5 ? g_bake_cpu.kq_mr5 : (fmt == 40 ? g_bake_cpu.kq_mr40 : (fmt == 44 ? g_bake_cpu.kq_mr44 : (fmt == 3 ? g_bake_cpu.kq_mr3 : (fmt == 33 ? g_bake_cpu.kq_mr33 : (fmt == 34 ? g_bake_cpu.kq_mr34 : (fmt == 45 ? g_bake_cpu.kq_mr45 : (fmt == 2 ? g_bake_cpu.kq_mr2 : (fmt == 23 ? g_bake_cpu.kq_mr23 : g_bake_cpu.kq_mr6))))))))) + return fmt == 4 ? g_bake_cpu.kq_mr4 : (fmt == 5 ? g_bake_cpu.kq_mr5 : (fmt == 40 ? g_bake_cpu.kq_mr40 : (fmt == 44 ? g_bake_cpu.kq_mr44 : (fmt == 3 ? g_bake_cpu.kq_mr3 : (fmt == 33 ? g_bake_cpu.kq_mr33 : (fmt == 34 ? g_bake_cpu.kq_mr34 : (fmt == 45 ? g_bake_cpu.kq_mr45 : (fmt == 2 ? g_bake_cpu.kq_mr2 : (fmt == 23 ? g_bake_cpu.kq_mr23 : (fmt == 24 ? g_bake_cpu.kq_mr24 : g_bake_cpu.kq_mr6)))))))))) } return int64(invoke(g_active_kq_layout, fmt)) } @@ -1670,6 +1672,7 @@ def private activate(be : KernelBackend) { g_kq_rows_iq4nl = be.kq_rows_iq4nl g_kq_rows_k2 = be.kq_rows_k2 g_kq_rows_iq2s = be.kq_rows_iq2s + g_kq_rows_iq2xs = be.kq_rows_iq2xs g_mm_kq_batch = be.kq_batch g_mm_kq_groupn = be.kq_groupn g_mm_kq_batch_groupn = be.kq_batch_groupn @@ -1879,6 +1882,7 @@ def kq_rows_fn(fmt : int) : MatmulKqRowsFn { if (fmt == 45) return g_kq_rows_iq4nl if (fmt == 2) return g_kq_rows_k2 if (fmt == 23) return g_kq_rows_iq2s + if (fmt == 24) return g_kq_rows_iq2xs return @@kq_unset_rows } @@ -3260,6 +3264,9 @@ def private restore_math_fn_defaults { // nolint:STYLE037,STYLE038 — flat seam if (g_kq_rows_iq2s == null) { g_kq_rows_iq2s = @@kq_unset_rows } + if (g_kq_rows_iq2xs == null) { + g_kq_rows_iq2xs = @@kq_unset_rows + } if (g_mm_kq_batch == null) { g_mm_kq_batch = @@kq_unset_batch } diff --git a/modules/dasLLAMA/dasllama/dasllama_math_default.das b/modules/dasLLAMA/dasllama/dasllama_math_default.das index e90eacd735..1e0e1c35ca 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_default.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_default.das @@ -689,6 +689,44 @@ def dot_iq2s_q8(kqrow : uint8 const?; ksrow : uint8 const?; xqp : int8 const?; x return acc } +//! IQ2_XS row dot off the DISK-ORDER planes: each u16 qs word's low 9 bits pick a u64 grid +//! entry (EIGHT magnitudes, two uint words), its high 7 bits pick the ksigns sign byte; per-16 +//! strips (1 + 2s) fold on the eighth-ed d; signed weights, so no activation block-sum term. +[unused_argument(xbsp), hint(unsafe_range_check, noalias = kqrow, noalias = ksrow, noalias = xqp, noalias = xsp, noalias = xbsp)] +def dot_iq2xs_q8(kqrow : uint8 const?; ksrow : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n : int64) : float { + var acc = 0.0 + let grid = iq2xs_grid2() + let ksg = ksigns_iq2xs() + unsafe { + for (sb in range64(n / 256l)) { + let kqo = sb * 64l + let kso = sb * 20l + let d8 = f16_to_f32(uint(ksrow[kso]) | (uint(ksrow[kso + 1l]) << 8u)) + var isum = 0 + for (b in range64(8l)) { + let ab = (sb * 8l + b) * 32l + for (l in range64(4l)) { + let w16 = uint(kqrow[kqo + (b * 4l + l) * 2l]) | (uint(kqrow[kqo + (b * 4l + l) * 2l + 1l]) << 8u) + let sgn = int(ksg[int(w16 >> 9u)]) + let idx = int(w16 & 511u) + let g1 = grid[idx * 2] + let g2 = grid[idx * 2 + 1] + var ig = 0 + for (j in range64(4l)) { + let w1 = int((g1 >> uint(8l * j)) & 0xFFu) + let w2 = int((g2 >> uint(8l * j)) & 0xFFu) + ig += (((sgn >> int(j)) & 1) != 0 ? -w1 : w1) * int(xqp[ab + l * 8l + j]) + ig += (((sgn >> int(4l + j)) & 1) != 0 ? -w2 : w2) * int(xqp[ab + l * 8l + 4l + j]) + } + isum += int(ksrow[kso + 4l + 2l * b + l / 2l]) * ig + } + } + acc += float(isum) * (d8 * xsp[sb]) + } + } + return acc +} + // Row-range cores + full GEMVs per format. Not `private`: invoked through hoisted function // pointers from lifted worker lambdas (the fused chains) and the dispatch wrappers in common. def k4_rows_kernel(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, rb, re : int64) { @@ -884,6 +922,15 @@ def iq2s_rows_kernel(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xq } } +def iq2xs_rows_kernel(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, rb, re : int64) { + let nsb = n / 256l + unsafe { + for (i in range64(rb, re)) { + yp[i] = dot_iq2xs_q8(kqp + i * nsb * 64l, ksp + i * nsb * 20l, xqp, xsp, xbsp, n) + } + } +} + def private kq_gemv_kernel(fmt : int; var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d : int64) { var myp = yp maybe_parallel_for(0, int(d), matmul_chunks_gemv(int(d), 1, n * d)) $(rb, re) { @@ -907,6 +954,8 @@ def private kq_gemv_kernel(fmt : int; var yp : float?; kqp : uint8 const?; ksp : k2_rows_kernel(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb), int64(re)) } elif (fmt == 23) { iq2s_rows_kernel(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb), int64(re)) + } elif (fmt == 24) { + iq2xs_rows_kernel(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb), int64(re)) } else { k6_rows_kernel(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb), int64(re)) } @@ -1130,6 +1179,36 @@ def private dequant_iq2s_row_grp(kqg : uint8 const?; ksg : uint8 const?; r, mr, } } +//! iq2xs grp row dequant: 16 uniform 4-byte columns per superblock row (the u16 qs words in +//! place), the k4-shaped scale interleave with 16 strips - w = (d8 * (1 + 2s)) * grid * sign. +def private dequant_iq2xs_row_grp(kqg : uint8 const?; ksg : uint8 const?; r, mr, nsb : int64; var dst : float?) { + let grid = iq2xs_grid2() + let ksgn = ksigns_iq2xs() + unsafe { + for (sbi in range64(nsb)) { + let qb = sbi * 64l * mr + let sb = sbi * 20l * mr + let d8 = f16_to_f32(uint(ksg[sb + 2l * r]) | (uint(ksg[sb + 2l * r + 1l]) << 8u)) + for (b in range64(8l)) { + for (l in range64(4l)) { + let db = d8 * float(int(ksg[sb + 4l * mr + (2l * b + l / 2l) * mr + r])) + let lp = (b * 4l + l) * 2l + let w16 = uint(kqg[qb + ((lp / 4l) * mr + r) * 4l + lp % 4l]) | (uint(kqg[qb + ((lp / 4l) * mr + r) * 4l + lp % 4l + 1l]) << 8u) + let sgn = int(ksgn[int(w16 >> 9u)]) + let idx = int(w16 & 511u) + let g1 = grid[idx * 2] + let g2 = grid[idx * 2 + 1] + let kb = sbi * 256l + b * 32l + l * 8l + for (j in range64(4l)) { + dst[kb + j] = db * float((g1 >> uint(8l * j)) & 0xFFu) * (((sgn >> int(j)) & 1) != 0 ? -1.0 : 1.0) + dst[kb + 4l + j] = db * float((g2 >> uint(8l * j)) & 0xFFu) * (((sgn >> int(4l + j)) & 1) != 0 ? -1.0 : 1.0) + } + } + } + } + } +} + //! Dequant one row off the grp-REPACKED K-quant planes (repack_k4/k5/k6_grp layout). //! kqg/ksg = row's GROUP plane base (group g = row/mr); r = row % mr. //! embed_row's kq_repacked path — per-token, so scalar is fine. @@ -1169,6 +1248,10 @@ def dequant_kq_row_grp(fmt : int64; kqg : uint8 const?; ksg : uint8 const?; r, m dequant_iq2s_row_grp(kqg, ksg, r, mr, nsb, dst) return } + if (fmt == 24l) { + dequant_iq2xs_row_grp(kqg, ksg, r, mr, nsb, dst) + return + } unsafe { for (sbi in range64(nsb)) { let qb = sbi * qsb * mr @@ -1277,6 +1360,8 @@ def matmul_kq_groupn(fmt : int; var y : array | #; kq : array | #; myp[ii] = dot_k2q8(kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) } elif (fmt == 23) { myp[ii] = dot_iq2s_q8(kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) + } elif (fmt == 24) { + myp[ii] = dot_iq2xs_q8(kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) } else { myp[ii] = dot_k6q8(kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) } diff --git a/modules/dasLLAMA/dasllama/dasllama_math_gen.das b/modules/dasLLAMA/dasllama/dasllama_math_gen.das index f62f39071c..a89eca3962 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_gen.das @@ -63,21 +63,21 @@ struct private DasllamaTuneScope {} //! Reference body = the reference tile's grp4; generator declines in lockstep with the tile //! generator, so a perm the tile can't emit keeps the grp4 reference pair on both sides. def q8q8_layout_gen() : int { - return q8q8_repack_type(GEMM_REFERENCE_MR).interleave + return q8q8_repack_type(GEMM_REFERENCE_MR()).interleave } //! The wbias companion (slice H): additive bias baked into the grp plane's GROUP-row bytes — //! 128 for bias128 vpdpbusd perms (w^0x80, −128·Σx acc-init correction), 0 otherwise. Reference //! body = the unbiased grp4 pair, declining in lockstep so plane bytes and kernels never disagree. def q8q8_wbias_gen() : int { - return q8q8_repack_type(GEMM_REFERENCE_MR).wbias + return q8q8_repack_type(GEMM_REFERENCE_MR()).wbias } //! The kgroup companion (slice J): k-bytes per row per interleave group — 8 for smmla perms //! (row-pair × 8-k MMA operands), 4 for every vector-dot perm. Reference body = the kg4 grp4 //! pair, declining in lockstep so plane byte order and kernels never disagree. def q8q8_kgroup_gen() : int { - return q8q8_repack_type(GEMM_REFERENCE_MR).kgroup + return q8q8_repack_type(GEMM_REFERENCE_MR()).kgroup } //! The tokstep companion (slice I): tokens one stamped tile call covers — 16·nrsplit for amx @@ -228,47 +228,51 @@ def kq_grp_row_dot(fmt : int64; kqg : uint8 const?; ksg : uint8 const?; r, mr : //! read. Each format tunes separately and takes its own winner's mr — k5/k6 tiles prefer grp4 //! while q8 crowns grp8 (M1, 2026-07-12). Reference bodies = grp4, declining in lockstep. def k4q8_layout_gen() : int { - return q8q8_repack_type(GEMM_REFERENCE_MR).interleave + return q8q8_repack_type(GEMM_REFERENCE_MR()).interleave } def k5q8_layout_gen() : int { - return q8q8_repack_type(GEMM_REFERENCE_MR).interleave + return q8q8_repack_type(GEMM_REFERENCE_MR()).interleave } def k6q8_layout_gen() : int { - return q8q8_repack_type(GEMM_REFERENCE_MR).interleave + return q8q8_repack_type(GEMM_REFERENCE_MR()).interleave } def q40q8_layout_gen() : int { - return q8q8_repack_type(GEMM_REFERENCE_MR).interleave + return q8q8_repack_type(GEMM_REFERENCE_MR()).interleave } def iq4xsq8_layout_gen() : int { - return q8q8_repack_type(GEMM_REFERENCE_MR).interleave + return q8q8_repack_type(GEMM_REFERENCE_MR()).interleave } def k3q8_layout_gen() : int { - return q8q8_repack_type(GEMM_REFERENCE_MR).interleave + return q8q8_repack_type(GEMM_REFERENCE_MR()).interleave } def iq3sq8_layout_gen() : int { - return q8q8_repack_type(GEMM_REFERENCE_MR).interleave + return q8q8_repack_type(GEMM_REFERENCE_MR()).interleave } def iq3xxsq8_layout_gen() : int { - return q8q8_repack_type(GEMM_REFERENCE_MR).interleave + return q8q8_repack_type(GEMM_REFERENCE_MR()).interleave } def iq4nlq8_layout_gen() : int { - return q8q8_repack_type(GEMM_REFERENCE_MR).interleave + return q8q8_repack_type(GEMM_REFERENCE_MR()).interleave } def k2q8_layout_gen() : int { - return q8q8_repack_type(GEMM_REFERENCE_MR).interleave + return q8q8_repack_type(GEMM_REFERENCE_MR()).interleave } def iq2sq8_layout_gen() : int { - return q8q8_repack_type(GEMM_REFERENCE_MR).interleave + return q8q8_repack_type(GEMM_REFERENCE_MR()).interleave +} + +def iq2xsq8_layout_gen() : int { + return q8q8_repack_type(GEMM_REFERENCE_MR()).interleave } // the KernelBackend.kq_layout slot form of kq_layout_of (registered on both gen backends) @@ -307,6 +311,9 @@ def kq_layout_of(fmt : int) : int64 { if (fmt == 23) { return int64(iq2sq8_layout_gen()) } + if (fmt == 24) { + return int64(iq2xsq8_layout_gen()) + } return int64(k6q8_layout_gen()) } @@ -970,6 +977,77 @@ def iq2sq8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp } } +//! One row's dot off the grp iq2xs planes, scalar - the iq2xs stubs' reference body and +//! the repack oracle: per u16 qs word the 9-bit grid gather + ksigns sign flip, per-16 strips +//! (1 + 2s) on the eighth-ed d, no activation-sum term. Same fold order as dot_iq2xs_q8. +[unused_argument(xbsp)] +def iq2xs_grp_row_dot(kqg : uint8 const?; ksg : uint8 const?; r, mr : int64; xqp : int8 const?; xsp : float const?; xbsp : int const?; n : int64) : float { + var acc = 0.0 + let nsb = n / 256l + let grid = iq2xs_grid2() + let ksgn = ksigns_iq2xs() + unsafe { + for (sbi in range64(nsb)) { + let qb = sbi * 64l * mr + let sb = sbi * 20l * mr + var isum = 0 + for (b in range64(8l)) { + for (l in range64(4l)) { + let lp = (b * 4l + l) * 2l + let w16 = uint(kqg[qb + ((lp / 4l) * mr + r) * 4l + lp % 4l]) | (uint(kqg[qb + ((lp / 4l) * mr + r) * 4l + lp % 4l + 1l]) << 8u) + let sgn = int(ksgn[int(w16 >> 9u)]) + let idx = int(w16 & 511u) + let g1 = grid[idx * 2] + let g2 = grid[idx * 2 + 1] + var ig = 0 + for (j in range64(4l)) { + let w1 = int((g1 >> uint(8l * j)) & 0xFFu) + let w2 = int((g2 >> uint(8l * j)) & 0xFFu) + ig += (((sgn >> int(j)) & 1) != 0 ? -w1 : w1) * int(xqp[(sbi * 8l + b) * 32l + l * 8l + j]) + ig += (((sgn >> int(4l + j)) & 1) != 0 ? -w2 : w2) * int(xqp[(sbi * 8l + b) * 32l + l * 8l + 4l + j]) + } + isum += int(ksg[sb + 4l * mr + (2l * b + l / 2l) * mr + r]) * ig + } + } + let dk = f16_to_f32(uint(ksg[sb + 2l * r]) | (uint(ksg[sb + 2l * r + 1l]) << 8u)) + acc += float(isum) * (dk * xsp[sbi]) + } + } + return acc +} + +[hint(unsafe_range_check, noalias = kqp, noalias = ksp, noalias = xqp, noalias = xsp, noalias = xbsp)] +def iq2xsq8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, rb, re : int64) : void { + let mr = int64(iq2xsq8_layout_gen()) + let nsb = n / 256l + unsafe { + for (i in range64(rb, re)) { + let g = i / mr + yp[i] = iq2xs_grp_row_dot(kqp + g * mr * nsb * 64l, ksp + g * mr * nsb * 20l, i % mr, mr, xqp, xsp, xbsp, n) + } + } +} + +[tune_perm(mr = 4), tune_perm(mr = 4, nrsplit = 2), tune_perm(mr = 8), tune_perm(mr = 8, nrsplit = 2), + tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "maddubs", width = 256, mr = 8, nrsplit = 2, requires = "avx2"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, nrsplit = 2, requires = "avxvnni|avx512vnni"), + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, nrsplit = 2, requires = "avx512vnni,avx512bw"), + tune_companion(fn = "iq2xsq8_gemv_gen", gen = "dasllama_gemm_gen::iq2xs_gemv"), + tune_companion(fn = "iq2xsq8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), + tune(gen = "dasllama_gemm_gen::iq2xs_tile", + fallback = "dot_vpdpbusd_width512_mr16;dot_vpdpbusd_width256_mr8;dot_maddubs_width256_mr8;mr4"), + hint(unsafe_range_check, noalias = kqg, noalias = ksg, noalias = xqp, noalias = xsp, noalias = xbsp)] +def iq2xsq8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d, g, t0 : int64) : void { + let mr = int64(iq2xsq8_layout_gen()) + unsafe { + for (t in range64(4l)) { + for (r in range64(mr)) { + yp[(t0 + t) * d + g * mr + r] = iq2xs_grp_row_dot(kqg, ksg, r, mr, xqp + (t0 + t) * n, xsp + (t0 + t) * (n / 256l), xbsp + (t0 + t) * (n / 16l), n) + } + } + } +} + //! The f16-scale GEMV companion (wscale_f16 rail): q8q8_gemv_gen's twin over the raw binary16 //! group-scale plane. Same rows-range contract; the generated body differs only in the scale //! fold's widen (fcvtl/vcvtph2ps). Not `private`: it IS the mm_rows_s16 slot. @@ -1311,6 +1389,8 @@ def private repack_kq_gen(fmt : int; var kq : uint8?; var ks : uint8?; n, d : in repack_k2_grp(kq, ks, n, d, kq_layout_of(2)) } elif (fmt == 23) { repack_iq2s_grp(kq, ks, n, d, kq_layout_of(23)) + } elif (fmt == 24) { + repack_iq2xs_grp(kq, ks, n, d, kq_layout_of(24)) } else { repack_k6_grp(kq, ks, n, d, kq_layout_of(6)) } @@ -1337,6 +1417,8 @@ def private repack_kq_bake(fmt : int; var kq : uint8?; var ks : uint8?; n, d : i repack_k2_grp(kq, ks, n, d, active_kq_layout_mr(2)) } elif (fmt == 23) { repack_iq2s_grp(kq, ks, n, d, active_kq_layout_mr(23)) + } elif (fmt == 24) { + repack_iq2xs_grp(kq, ks, n, d, active_kq_layout_mr(24)) } else { repack_k6_grp(kq, ks, n, d, active_kq_layout_mr(6)) } @@ -1861,6 +1943,8 @@ def private kq_kernel_gen(fmt : int; var yp : float?; kqp : uint8 const?; ksp : k2q8_gemv_gen(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb) * mr, int64(re) * mr) } elif (fmt == 23) { iq2sq8_gemv_gen(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb) * mr, int64(re) * mr) + } elif (fmt == 24) { + iq2xsq8_gemv_gen(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb) * mr, int64(re) * mr) } else { k6q8_gemv_gen(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb) * mr, int64(re) * mr) } @@ -1887,6 +1971,8 @@ def private kq_kernel_gen(fmt : int; var yp : float?; kqp : uint8 const?; ksp : myp[i] = dot_k2q8(kqp + i * nsb * 64l, ksp + i * nsb * 20l, xqp, xsp, xbsp, n) } elif (fmt == 23) { myp[i] = dot_iq2s_q8(kqp + i * nsb * 72l, ksp + i * nsb * 20l, xqp, xsp, xbsp, n) + } elif (fmt == 24) { + myp[i] = dot_iq2xs_q8(kqp + i * nsb * 64l, ksp + i * nsb * 20l, xqp, xsp, xbsp, n) } else { myp[i] = dot_k6q8(kqp + i * nsb * 192l, ksp + i * nsb * 18l, xqp, xsp, xbsp, n) } @@ -1902,7 +1988,7 @@ def private kq_batch_cell_gen(fmt : int; var myp : float?; kqp : uint8 const?; k let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) let nb16 = n / 16l - let packed = fmt == 4 || fmt == 40 || fmt == 44 || fmt == 3 || fmt == 45 || fmt == 2 // packed planes: the tile reads them directly (iq3s/iq3xxs/iq2s expand their grid gathers into the panel) + let packed = fmt == 4 || fmt == 40 || fmt == 44 || fmt == 3 || fmt == 45 || fmt == 2 || fmt == 24 // packed planes: the tile reads them directly (iq3s/iq3xxs/iq2s expand their grid gathers into the panel) var scratch : array if (!packed) { scratch |> resize(mr * nsb * 256l) @@ -1942,6 +2028,8 @@ def private kq_batch_cell_gen(fmt : int; var myp : float?; kqp : uint8 const?; k k2q8_tile_gen(myp, kqg, ksg, xqp, xsp, xbsp, n, d, g, tk) } elif (fmt == 23) { iq2sq8_tile_gen(myp, scp, ksg, xqp, xsp, xbsp, n, d, g, tk) + } elif (fmt == 24) { + iq2xsq8_tile_gen(myp, kqg, ksg, xqp, xsp, xbsp, n, d, g, tk) } else { k6q8_tile_gen(myp, scp, ksg, xqp, xsp, xbsp, n, d, g, tk) } @@ -1968,6 +2056,8 @@ def private kq_batch_cell_gen(fmt : int; var myp : float?; kqp : uint8 const?; k k2q8_gemv_gen(myp + tk * d, kqp, ksp, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n, g * mr, (g + 1l) * mr) } elif (fmt == 23) { iq2sq8_gemv_gen(myp + tk * d, kqp, ksp, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n, g * mr, (g + 1l) * mr) + } elif (fmt == 24) { + iq2xsq8_gemv_gen(myp + tk * d, kqp, ksp, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n, g * mr, (g + 1l) * mr) } else { k6q8_gemv_gen(myp + tk * d, kqp, ksp, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n, g * mr, (g + 1l) * mr) } @@ -2015,6 +2105,8 @@ def private kq_batch_kernel_gen(fmt : int; var yp : float?; kqp : uint8 const?; myp[tk * d + i] = dot_k2q8(kqp + i * nsb * 64l, ksp + i * nsb * 20l, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * (n / 16l), n) } elif (fmt == 23) { myp[tk * d + i] = dot_iq2s_q8(kqp + i * nsb * 72l, ksp + i * nsb * 20l, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * (n / 16l), n) + } elif (fmt == 24) { + myp[tk * d + i] = dot_iq2xs_q8(kqp + i * nsb * 64l, ksp + i * nsb * 20l, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * (n / 16l), n) } else { myp[tk * d + i] = dot_k6q8(kqp + i * nsb * 192l, ksp + i * nsb * 18l, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * (n / 16l), n) } @@ -2077,6 +2169,8 @@ def private kq_batch_groupn_gen(fmt : int; var yp : float?; kqp : uint8 const?; myp[tk * d + i] = dot_k2q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n) } elif (fmt == 23) { myp[tk * d + i] = dot_iq2s_q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n) + } elif (fmt == 24) { + myp[tk * d + i] = dot_iq2xs_q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n) } else { myp[tk * d + i] = dot_k6q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n) } @@ -2248,6 +2342,8 @@ def kq_groupn_gen(fmt : int; var yp : float?; kqp : uint8 const?; ksp : uint8 co k2q8_gemv_gen(myp + r * d, kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n, gLocal * mr, gEnd * mr) } elif (fmt == 23) { iq2sq8_gemv_gen(myp + r * d, kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n, gLocal * mr, gEnd * mr) + } elif (fmt == 24) { + iq2xsq8_gemv_gen(myp + r * d, kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n, gLocal * mr, gEnd * mr) } else { k6q8_gemv_gen(myp + r * d, kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n, gLocal * mr, gEnd * mr) } @@ -2280,6 +2376,8 @@ def kq_groupn_gen(fmt : int; var yp : float?; kqp : uint8 const?; ksp : uint8 co myp[r * d + i] = dot_k2q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) } elif (fmt == 23) { myp[r * d + i] = dot_iq2s_q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) + } elif (fmt == 24) { + myp[r * d + i] = dot_iq2xs_q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) } else { myp[r * d + i] = dot_k6q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) } @@ -2573,7 +2671,7 @@ def dasllama_math_gen_register() { // the no-witness stance above covers kq too mm_kq = @@kq_kernel_gen, repack_kq = @@repack_kq_gen, kq_rows_k4 = @@k4q8_gemv_gen, kq_rows_k5 = @@k5q8_gemv_gen, - kq_rows_k6 = @@k6q8_gemv_gen, kq_rows_q40 = @@q40q8_gemv_gen, kq_rows_iq4xs = @@iq4xsq8_gemv_gen, kq_rows_k3 = @@k3q8_gemv_gen, kq_rows_iq3s = @@iq3sq8_gemv_gen, kq_rows_iq3xxs = @@iq3xxsq8_gemv_gen, kq_rows_iq4nl = @@iq4nlq8_gemv_gen, kq_rows_k2 = @@k2q8_gemv_gen, kq_rows_iq2s = @@iq2sq8_gemv_gen, + kq_rows_k6 = @@k6q8_gemv_gen, kq_rows_q40 = @@q40q8_gemv_gen, kq_rows_iq4xs = @@iq4xsq8_gemv_gen, kq_rows_k3 = @@k3q8_gemv_gen, kq_rows_iq3s = @@iq3sq8_gemv_gen, kq_rows_iq3xxs = @@iq3xxsq8_gemv_gen, kq_rows_iq4nl = @@iq4nlq8_gemv_gen, kq_rows_k2 = @@k2q8_gemv_gen, kq_rows_iq2s = @@iq2sq8_gemv_gen, kq_rows_iq2xs = @@iq2xsq8_gemv_gen, kq_batch = @@kq_batch_kernel_gen, kq_groupn = @@kq_groupn_gen, kq_batch_groupn = @@kq_batch_groupn_gen, // q51 family: tile + GEMV off the grp planes (the tile family's companions stamp @@ -2602,7 +2700,7 @@ def dasllama_math_gen_register() { mm_rows_s16 = @@q8q8_gemv_s16_gen, mm_kq = @@kq_kernel_gen, repack_kq = @@repack_kq_gen, kq_rows_k4 = @@k4q8_gemv_gen, kq_rows_k5 = @@k5q8_gemv_gen, - kq_rows_k6 = @@k6q8_gemv_gen, kq_rows_q40 = @@q40q8_gemv_gen, kq_rows_iq4xs = @@iq4xsq8_gemv_gen, kq_rows_k3 = @@k3q8_gemv_gen, kq_rows_iq3s = @@iq3sq8_gemv_gen, kq_rows_iq3xxs = @@iq3xxsq8_gemv_gen, kq_rows_iq4nl = @@iq4nlq8_gemv_gen, kq_rows_k2 = @@k2q8_gemv_gen, kq_rows_iq2s = @@iq2sq8_gemv_gen, + kq_rows_k6 = @@k6q8_gemv_gen, kq_rows_q40 = @@q40q8_gemv_gen, kq_rows_iq4xs = @@iq4xsq8_gemv_gen, kq_rows_k3 = @@k3q8_gemv_gen, kq_rows_iq3s = @@iq3sq8_gemv_gen, kq_rows_iq3xxs = @@iq3xxsq8_gemv_gen, kq_rows_iq4nl = @@iq4nlq8_gemv_gen, kq_rows_k2 = @@k2q8_gemv_gen, kq_rows_iq2s = @@iq2sq8_gemv_gen, kq_rows_iq2xs = @@iq2xsq8_gemv_gen, kq_batch = @@kq_batch_kernel_gen, kq_groupn = @@kq_groupn_gen, kq_batch_groupn = @@kq_batch_groupn_gen, groupn_q51 = @@q51q8_groupn_gen, repack_q51 = @@repack_q51_gen, diff --git a/modules/dasLLAMA/dasllama/dasllama_ple.das b/modules/dasLLAMA/dasllama/dasllama_ple.das index a49124a8a2..36a44acccb 100644 --- a/modules/dasLLAMA/dasllama/dasllama_ple.das +++ b/modules/dasLLAMA/dasllama/dasllama_ple.das @@ -76,6 +76,8 @@ def ple_check_table(t : Model; origin : string) { have = (long_length(t.k2q) / K2_QSB) * 256l } elif (t.ple_emb_fmt == KqFmt.iq2s) { have = (long_length(t.iq2sq) / IQ2S_QSB) * 256l + } elif (t.ple_emb_fmt == KqFmt.iq2xs) { + have = (long_length(t.iq2xsq) / IQ2XS_QSB) * 256l } if (have < need) { panic("dasLLAMA: the PLE token table is not resident ({origin}): fmt {t.ple_emb_fmt} plane holds {have} of {need} elements - regenerate this carrier (it predates the pinned-table rail)") @@ -123,6 +125,8 @@ def private ple_gather_row(t : Model; row : int64; var dst : array; doff dequant_k2_plane_superblock(t.k2q, sb * K2_QSB, t.k2s, sb * K2_SSB, dst, doff + blk * 256l) } elif (t.ple_emb_fmt == KqFmt.iq2s) { dequant_iq2s_plane_superblock(t.iq2sq, sb * IQ2S_QSB, t.iq2ss, sb * IQ2S_SSB, dst, doff + blk * 256l) + } elif (t.ple_emb_fmt == KqFmt.iq2xs) { + dequant_iq2xs_plane_superblock(t.iq2xsq, sb * IQ2XS_QSB, t.iq2xss, sb * IQ2XS_SSB, dst, doff + blk * 256l) } else { panic("ple_gather_row: '{t.ple_emb_fmt}' has no kq plane pair") } diff --git a/modules/dasLLAMA/dasllama/dasllama_repack.das b/modules/dasLLAMA/dasllama/dasllama_repack.das index 4d23ccdf62..beeda51210 100644 --- a/modules/dasLLAMA/dasllama/dasllama_repack.das +++ b/modules/dasLLAMA/dasllama/dasllama_repack.das @@ -599,6 +599,50 @@ def repack_iq2s_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { delete ts } +//! iq2xs grp layout: the whole 64B row as 16 four-byte columns x mr rows - the u16 qs words +//! stay in place per column (each word's two bytes share a column); scales interleave as k4's +//! row with 16 strips ([d8 f16 x mr][pad x mr][16 strips x mr]). +def repack_iq2xs_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { + let nsb = n / 256l + let qrow = nsb * 64l + let srow = nsb * 20l + let ng = d / mr + var tq : array + var ts : array + tq |> resize(d * qrow) + ts |> resize(d * srow) + unsafe { + var tqp = addr(tq[0]) + var tsp = addr(ts[0]) + memcpy(tqp, kq, d * qrow) + memcpy(tsp, ks, d * srow) + for (g in range64(ng)) { + for (sbi in range64(nsb)) { + let dq = g * mr * qrow + sbi * 64l * mr + let ds = g * mr * srow + sbi * 20l * mr + for (r in range64(mr)) { + let sq = (g * mr + r) * qrow + sbi * 64l + let ss = (g * mr + r) * srow + sbi * 20l + for (c in range64(16l)) { + for (t in range64(4l)) { + kq[dq + (c * mr + r) * 4l + t] = tqp[sq + c * 4l + t] + } + } + ks[ds + 2l * r] = tsp[ss] + ks[ds + 2l * r + 1l] = tsp[ss + 1l] + ks[ds + 2l * mr + 2l * r] = tsp[ss + 2l] + ks[ds + 2l * mr + 2l * r + 1l] = tsp[ss + 3l] + for (g16 in range64(16l)) { + ks[ds + 4l * mr + g16 * mr + r] = tsp[ss + 4l + g16] + } + } + } + } + } + delete tq + delete ts +} + // ===== grp panel unpack (the byte-expanded tile form the kq stamps load) ===== //! Unpack ONE group's packed grp k5/k6 quant panel into the BYTE-EXPANDED tile form (wlo/whi diff --git a/modules/dasLLAMA/harness/gen_tune_probe.das b/modules/dasLLAMA/harness/gen_tune_probe.das index 55b5d73eb6..ae3e53f1f7 100644 --- a/modules/dasLLAMA/harness/gen_tune_probe.das +++ b/modules/dasLLAMA/harness/gen_tune_probe.das @@ -441,7 +441,7 @@ def kq_qsb(fmt : int64) : int64 => dasllama_gemm_schema::kq_qsb(int(fmt)) def kq_ssb(fmt : int64) : int64 => dasllama_gemm_schema::kq_ssb(int(fmt)) // the family's sidecar entry = its [tune] function name (the K-quants by bit width - k3 included - q40/iq4xs by tag) -def kq_tile_entry(fmt : int64) : string => fmt == 40l ? "q40q8_tile_gen" : (fmt == 44l ? "iq4xsq8_tile_gen" : (fmt == 33l ? "iq3sq8_tile_gen" : (fmt == 34l ? "iq3xxsq8_tile_gen" : (fmt == 45l ? "iq4nlq8_tile_gen" : (fmt == 23l ? "iq2sq8_tile_gen" : "k{fmt}q8_tile_gen"))))) +def kq_tile_entry(fmt : int64) : string => fmt == 40l ? "q40q8_tile_gen" : (fmt == 44l ? "iq4xsq8_tile_gen" : (fmt == 33l ? "iq3sq8_tile_gen" : (fmt == 34l ? "iq3xxsq8_tile_gen" : (fmt == 45l ? "iq4nlq8_tile_gen" : (fmt == 23l ? "iq2sq8_tile_gen" : (fmt == 24l ? "iq2xsq8_tile_gen" : "k{fmt}q8_tile_gen")))))) // synthetic disk superblocks, varied per (row, superblock) so no two blocks repeat: any byte // is a legal quant, and the small d/dmin keep the f16 (s, o) pairs finite and row sums tame @@ -537,6 +537,14 @@ def pack_kq_disk_block(fmt : int64; var blkb : array; base : int) { // let mbits = f32_to_f16(0.001 + 0.0001 * float((base + 5) % 11)) blkb[82] = uint8(mbits & 0xFF) blkb[83] = uint8(mbits >> 8u) + } elif (fmt == 24l) { + // 74B IQ2_XS disk block: f16 d, 32 u16 qs words, 8 packed scale nibbles - every byte pattern is legal + let dbits = f32_to_f16(0.002 + 0.0001 * float(base % 13)) + blkb[0] = uint8(dbits & 0xFF) + blkb[1] = uint8(dbits >> 8u) + for (i in range(72)) { + blkb[2 + i] = uint8((base * 41 + i * 97 + 17) % 256) + } } elif (fmt == 23l) { // 82B IQ2_S disk block: f16 d, 32 idx + 32 sign bytes, 8 qh, 8 packed scale nibbles - every byte pattern is legal let dbits = f32_to_f16(0.002 + 0.0001 * float(base % 13)) @@ -595,6 +603,8 @@ def repack_kq_grp_fmt(fmt : int64; var kq : uint8?; var ks : uint8?; n, d, mr : repack_k2_grp(kq, ks, n, d, mr) } elif (fmt == 23l) { repack_iq2s_grp(kq, ks, n, d, mr) + } elif (fmt == 24l) { + repack_iq2xs_grp(kq, ks, n, d, mr) } else { repack_k6_grp(kq, ks, n, d, mr) } @@ -607,7 +617,7 @@ def build_kq_fixture(fmt : int64; n, d, ntok : int64) : KqFixture { // nolint: fx.kq |> resize(d * fx.nsb * qsb) fx.ks |> resize(d * fx.nsb * ssb) var blkb : array - blkb |> resize(fmt == 2l ? 84l : (fmt == 23l ? 82l : (fmt == 4l || fmt == 40l || fmt == 45l ? 144l : (fmt == 5l ? 176l : (fmt == 44l ? 136l : (fmt == 3l || fmt == 33l ? 110l : (fmt == 34l ? 98l : 210l))))))) + blkb |> resize(fmt == 2l ? 84l : (fmt == 23l ? 82l : (fmt == 24l ? 74l : (fmt == 4l || fmt == 40l || fmt == 45l ? 144l : (fmt == 5l ? 176l : (fmt == 44l ? 136l : (fmt == 3l || fmt == 33l ? 110l : (fmt == 34l ? 98l : 210l)))))))) for (r in range64(d)) { for (s in range64(fx.nsb)) { let base = int(r * 31l + s * 7l) @@ -633,6 +643,8 @@ def build_kq_fixture(fmt : int64; n, d, ntok : int64) : KqFixture { // nolint: transcode_q2k_superblock(blkb, 0l, fx.kq, sb * qsb, fx.ks, sb * ssb) } elif (fmt == 23l) { transcode_iq2s_superblock(blkb, 0l, fx.kq, sb * qsb, fx.ks, sb * ssb) + } elif (fmt == 24l) { + transcode_iq2xs_superblock(blkb, 0l, fx.kq, sb * qsb, fx.ks, sb * ssb) } else { transcode_q6k_superblock(blkb, 0l, fx.kq, sb * qsb, fx.ks, sb * ssb) } @@ -682,14 +694,17 @@ def build_kq_fixture(fmt : int64; n, d, ntok : int64) : KqFixture { // nolint: : (fmt == 45l ? iq4nl_grp_row_dot(kqg, ksg, r, 4l, addr(fx.xq[0]) + tk * n, addr(fx.xs[0]) + tk * (n / 256l), addr(fx.xbs[0]) + tk * (n / 16l), n) - : (fmt == 23l + : (fmt == 24l + ? iq2xs_grp_row_dot(kqg, ksg, r, 4l, + addr(fx.xq[0]) + tk * n, addr(fx.xs[0]) + tk * (n / 256l), addr(fx.xbs[0]) + tk * (n / 16l), n) + : (fmt == 23l ? iq2s_grp_row_dot(kqg, ksg, r, 4l, addr(fx.xq[0]) + tk * n, addr(fx.xs[0]) + tk * (n / 256l), addr(fx.xbs[0]) + tk * (n / 16l), n) : (fmt == 2l ? k2_grp_row_dot(kqg, ksg, r, 4l, addr(fx.xq[0]) + tk * n, addr(fx.xs[0]) + tk * (n / 256l), addr(fx.xbs[0]) + tk * (n / 16l), n) : kq_grp_row_dot(fmt, kqg, ksg, r, 4l, - addr(fx.xq[0]) + tk * n, addr(fx.xs[0]) + tk * (n / 256l), addr(fx.xbs[0]) + tk * (n / 16l), n))))))))) + addr(fx.xq[0]) + tk * n, addr(fx.xs[0]) + tk * (n / 256l), addr(fx.xbs[0]) + tk * (n / 16l), n)))))))))) } } } @@ -732,6 +747,9 @@ def kq_tile_variants(fmt : int64) : array> { if (fmt == 23l) { return <- iq2sq8_tile_gen_variants() } + if (fmt == 24l) { + return <- iq2xsq8_tile_gen_variants() + } return <- k6q8_tile_gen_variants() } @@ -785,6 +803,12 @@ def kq_gemv_variants_by_suffix(fmt : int64) : table { // nol tab[v._0] = v._1 } delete gvs + } elif (fmt == 24l) { + var gvs <- iq2xsq8_gemv_gen_variants() + for (v in gvs) { + tab[v._0] = v._1 + } + delete gvs } elif (fmt == 23l) { var gvs <- iq2sq8_gemv_gen_variants() for (v in gvs) { @@ -859,6 +883,12 @@ def kq_layout_mrs(fmt : int64) : table { // nolint:STYLE037 — t mrs[v._0] = invoke(v._1) } delete lvs + } elif (fmt == 24l) { + var lvs <- iq2xsq8_layout_gen_variants() + for (v in lvs) { + mrs[v._0] = invoke(v._1) + } + delete lvs } elif (fmt == 23l) { var lvs <- iq2sq8_layout_gen_variants() for (v in lvs) { @@ -888,7 +918,7 @@ def kq_layout_mrs(fmt : int64) : table { // nolint:STYLE037 — t def run_kq_tile(tile; fx : KqFixture; kq : array; ks : array; mr : int64; var y : array) { let ng = fx.d / mr verify(fx.ntok % 4l == 0l) - let packed = fx.fmt == 4l || fx.fmt == 40l || fx.fmt == 44l || fx.fmt == 3l || fx.fmt == 45l || fx.fmt == 2l // packed planes: the tile reads them directly (iq3s/iq3xxs/iq2s unpack their panels per group) + let packed = fx.fmt == 4l || fx.fmt == 40l || fx.fmt == 44l || fx.fmt == 3l || fx.fmt == 45l || fx.fmt == 2l || fx.fmt == 24l // packed planes: the tile reads them directly (iq3s/iq3xxs/iq2s unpack their panels per group) var panel : array if (!packed) { panel |> resize(int(mr * fx.nsb * 256l)) @@ -1402,7 +1432,7 @@ def test_mode : bool { // nolint:STYLE037,STYLE038 — flat per-variant correctn if (v._0 == "reference") { verify(!live) } - if (int(mr) != GEMM_REFERENCE_MR && !live) { + if (int(mr) != GEMM_REFERENCE_MR() && !live) { print(" WITNESS DESYNC {v._0}: layout mr={mr} but witness says not live\n") allok = false } @@ -1970,7 +2000,7 @@ def tune_mode_run : bool { // nolint:STYLE037,STYLE038 — flat family-by-family print("aborting mid-run - no sidecar written\n") return false } - for (fmt in fixed_array(4l, 5l, 6l, 40l, 44l, 3l, 33l, 34l, 45l, 2l, 23l)) { + for (fmt in fixed_array(4l, 5l, 6l, 40l, 44l, 3l, 33l, 34l, 45l, 2l, 23l, 24l)) { interrupt_gate("k{fmt}q8_tile_gen") let kq_t0 = ref_time_ticks() let entry = kq_tile_entry(fmt) diff --git a/modules/dasLLAMA/tests/test_kqformat.das b/modules/dasLLAMA/tests/test_kqformat.das index fa6edec061..d094678d19 100644 --- a/modules/dasLLAMA/tests/test_kqformat.das +++ b/modules/dasLLAMA/tests/test_kqformat.das @@ -27,6 +27,7 @@ def test_kqfmt_enum_values(t : T?) { t |> equal(int(KqFmt.iq4nl), 10) t |> equal(int(KqFmt.k2), 11) t |> equal(int(KqFmt.iq2s), 12) + t |> equal(int(KqFmt.iq2xs), 13) } } @@ -53,6 +54,8 @@ def test_kqfmt_predicates(t : T?) { t |> success(kq_sb(int(KqFmt.k2)), "the int-id predicate agrees for k2") t |> success(kq_sb(KqFmt.iq2s), "iq2s is superblock") t |> success(kq_sb(int(KqFmt.iq2s)), "the int-id predicate agrees for iq2s") + t |> success(kq_sb(KqFmt.iq2xs), "iq2xs is superblock") + t |> success(kq_sb(int(KqFmt.iq2xs)), "the int-id predicate agrees for iq2xs") } } @@ -83,6 +86,8 @@ def test_kqfmt_strides(t : T?) { t |> equal(kq_ssb(KqFmt.k2), 20l) // [f16 d][f16 dmin][16 sc/min bytes] t |> equal(kq_qsb(KqFmt.iq2s), 72l) // [32 idx][32 signs][8 qh], verbatim t |> equal(kq_ssb(KqFmt.iq2s), 20l) // f16 d/8 + 16 x (1 + 2s) strips + t |> equal(kq_qsb(KqFmt.iq2xs), 64l) // 32 u16 [9-bit grid idx][7-bit ksigns idx] words, verbatim + t |> equal(kq_ssb(KqFmt.iq2xs), 20l) // the iq2s row exactly - f16 d/8 + 16 x (1 + 2s) } t |> run("named constants agree with the literals") @(t : T?) { t |> equal(K4_QSB, 128l) @@ -103,6 +108,8 @@ def test_kqfmt_strides(t : T?) { t |> equal(K2_SSB, 20l) t |> equal(IQ2S_QSB, 72l) t |> equal(IQ2S_SSB, 20l) + t |> equal(IQ2XS_QSB, 64l) + t |> equal(IQ2XS_SSB, 20l) t |> equal(IQ3S_QSB, 104l) t |> equal(IQ3S_SSB, 20l) } @@ -142,9 +149,10 @@ def test_kqfmt_id_bridge(t : T?) { t |> equal(kq_schema_id(KqFmt.iq4nl), 45) t |> equal(kq_schema_id(KqFmt.k2), 2) t |> equal(kq_schema_id(KqFmt.iq2s), 23) + t |> equal(kq_schema_id(KqFmt.iq2xs), 24) } t |> run("gemm_schema's int-id forms delegate to the same truth") @(t : T?) { - for (f in [KqFmt.k4, KqFmt.k5, KqFmt.k6, KqFmt.q40, KqFmt.iq4xs, KqFmt.k3, KqFmt.iq3s, KqFmt.iq3xxs, KqFmt.iq4nl, KqFmt.k2, KqFmt.iq2s]) { + for (f in [KqFmt.k4, KqFmt.k5, KqFmt.k6, KqFmt.q40, KqFmt.iq4xs, KqFmt.k3, KqFmt.iq3s, KqFmt.iq3xxs, KqFmt.iq4nl, KqFmt.k2, KqFmt.iq2s, KqFmt.iq2xs]) { t |> equal(kq_qsb(kq_schema_id(f)), kq_qsb(f)) t |> equal(kq_ssb(kq_schema_id(f)), kq_ssb(f)) } @@ -165,6 +173,7 @@ def test_kqfmt_elems_and_stream(t : T?) { t |> equal(kq_elems(KqFmt.iq4nl), 256l) t |> equal(kq_elems(KqFmt.k2), 256l) t |> equal(kq_elems(KqFmt.iq2s), 256l) + t |> equal(kq_elems(KqFmt.iq2xs), 256l) t |> equal(kq_elems(KqFmt.q8), 32l) t |> equal(kq_elems(KqFmt.q51), 32l) } @@ -182,6 +191,7 @@ def test_kqfmt_elems_and_stream(t : T?) { t |> equal(kq_stream_code(KqFmt.iq4nl), 45) t |> equal(kq_stream_code(KqFmt.k2), 20) // kernel id 2 = q51's stream tag t |> equal(kq_stream_code(KqFmt.iq2s), 23) + t |> equal(kq_stream_code(KqFmt.iq2xs), 24) } } diff --git a/modules/dasLLAMA/tests/test_kquant.das b/modules/dasLLAMA/tests/test_kquant.das index 29588e5730..bc9c99c89d 100644 --- a/modules/dasLLAMA/tests/test_kquant.das +++ b/modules/dasLLAMA/tests/test_kquant.das @@ -1,8 +1,8 @@ options gen2 options stack = 524288 // every dasLLAMA program root takes this budget (options stack does not unify up from libs) options persistent_heap // the layout arm loads a multi-GB model; explicit delete below -options _cyclomatic_complexity = 64 // every gate here is the same flat fmt == 4/5/6/40/44/3 ladder repeated per stage -options _function_length = 260 // and each gate carries its whole fixture build + oracle inline +options _cyclomatic_complexity = 70 // every gate here is the same flat fmt == 4/5/6/40/44/3 ladder repeated per stage +options _function_length = 280 // and each gate carries its whole fixture build + oracle inline options _dasllama_internal = true require dastest/testing_boost public @@ -142,7 +142,7 @@ def private build_iq4nl_block() : array { return <- blkb } -def private kq_tag(fmt : int) : string => fmt == 40 ? "q40" : (fmt == 44 ? "iq4xs" : (fmt == 33 ? "iq3s" : (fmt == 34 ? "iq3xxs" : (fmt == 45 ? "iq4nl" : (fmt == 23 ? "iq2s" : "k{fmt}"))))) +def private kq_tag(fmt : int) : string => fmt == 40 ? "q40" : (fmt == 44 ? "iq4xs" : (fmt == 33 ? "iq3s" : (fmt == 34 ? "iq3xxs" : (fmt == 45 ? "iq4nl" : (fmt == 23 ? "iq2s" : (fmt == 24 ? "iq2xs" : "k{fmt}")))))) // iq4xs synthetic superblock: 136B disk block — f16 d = SYNTH_D, the 8 six-bit sub-scales are // SYNTH_SC (packed low nibbles into scales_l, top two bits into scales_h), the shared nibble @@ -298,6 +298,30 @@ def private build_iq2s_block() : array { return <- blkb } +// iq2xs synthetic superblock: 74B IQ2_XS disk block - per-16 scale nibbles SYNTH_SC & 15, +// f16 d, u16 qs words carrying a 9-bit grid index + 7-bit ksigns index pattern (written in the +// PACK direction); the gate recomposes each weight through IQ2XS_GRID + KSIGNS_IQ2XS +def private iq2xs_ix_pat(i : int) : int => (i * 29 + 7) % 512 +def private iq2xs_sg7_pat(i : int) : int => (i * 5 + 3) % 128 +def private build_iq2xs_block() : array { + var blkb : array + blkb |> resize(74) + let dbits = f32_to_f16(SYNTH_D) + blkb[0] = uint8(dbits & 0xFF) + blkb[1] = uint8(dbits >> 8u) + for (w in range(32)) { + let v = iq2xs_ix_pat(w) | (iq2xs_sg7_pat(w) << 9) + blkb[2 + w * 2] = uint8(v & 0xFF) + blkb[3 + w * 2] = uint8(v >> 8) + } + for (g in range(16)) { // scale nibble g: low nibble of byte g/2 (g even), high (g odd) + let v = SYNTH_SC[g % 8] & 15 + let nib = g % 2 == 0 ? v : v << 4 + blkb[66 + g / 2] = uint8(int(blkb[66 + g / 2]) | nib) + } + return <- blkb +} + // q40 synthetic superblock: 8 x 18B q4_0 disk blocks, per-block f16-exact d = (blk+1)/16 and // the shared nibble pattern — weight k = d(k/32) * (q4_pat(k) - 8) def private q40_d(blk : int) : float => float(blk + 1) * 0.0625 @@ -527,6 +551,32 @@ def test_kq_transcode_planes(t : T?) { t |> success(dst[k] == expected, "k2 plane element must match the ggml dequant exactly") } } + t |> run("iq2xs planes eighth the d, decode the strips, and gather the 512-entry grid + ksigns exactly") @(t : T?) { + let blkb <- build_iq2xs_block() + var kq : array + var ks : array + kq |> resize(64) + ks |> resize(20) + transcode_iq2xs_superblock(blkb, 0l, kq, 0l, ks, 0l) + for (g in range(16)) { + t |> equal(int(ks[4 + g]), 1 + 2 * (SYNTH_SC[g % 8] & 15), "decoded strip is 1 + 2s") + } + var dst : array + dst |> resize(256) + dequant_iq2xs_plane_superblock(kq, 0l, ks, 0l, dst, 0l) + let d8 = SYNTH_D * 0.125 + for (k in range(256)) { + let b = k / 32 + let l = (k % 32) / 8 + let j = k % 8 + let idx = iq2xs_ix_pat(b * 4 + l) + let gw = j < 4 ? IQ2XS_GRID[idx * 2] : IQ2XS_GRID[idx * 2 + 1] + let g = int((gw >> uint(8 * (j % 4))) & 0xFFu) + let sgn = ((int(KSIGNS_IQ2XS[iq2xs_sg7_pat(b * 4 + l)]) >> j) & 1) != 0 ? -1.0 : 1.0 + let expected = (d8 * float(1 + 2 * (SYNTH_SC[(2 * b + l / 2) % 8] & 15))) * float(g) * sgn + t |> success(dst[k] == expected, "iq2xs plane element must match the ggml dequant exactly") + } + } t |> run("iq2s planes eighth the d, decode the strips, and gather the u64 grid exactly") @(t : T?) { let blkb <- build_iq2s_block() var kq : array @@ -799,7 +849,7 @@ def test_q51_kernels(t : T?) { // elementwise dot over the plane dequant (the kernels' declared oracle). def private kq_dot_gate(t : T?; fmt : int; n : int64 = 512l) { let nsb = n / 256l - let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : (fmt == 45 ? build_iq4nl_block() : (fmt == 2 ? build_q2k_block() : (fmt == 23 ? build_iq2s_block() : build_q6k_block()))))))))) + let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : (fmt == 45 ? build_iq4nl_block() : (fmt == 2 ? build_q2k_block() : (fmt == 23 ? build_iq2s_block() : (fmt == 24 ? build_iq2xs_block() : build_q6k_block())))))))))) let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) var kq : array @@ -825,6 +875,8 @@ def private kq_dot_gate(t : T?; fmt : int; n : int64 = 512l) { transcode_iq4nl_superblock(blk1, 0l, kq, s * qsb, ks, s * ssb) } elif (fmt == 2) { transcode_q2k_superblock(blk1, 0l, kq, s * qsb, ks, s * ssb) + } elif (fmt == 24) { + transcode_iq2xs_superblock(blk1, 0l, kq, s * qsb, ks, s * ssb) } elif (fmt == 23) { transcode_iq2s_superblock(blk1, 0l, kq, s * qsb, ks, s * ssb) } else { @@ -858,6 +910,8 @@ def private kq_dot_gate(t : T?; fmt : int; n : int64 = 512l) { dequant_iq4nl_plane_superblock(kq, s * qsb, ks, s * ssb, w, s * 256l) } elif (fmt == 2) { dequant_k2_plane_superblock(kq, s * qsb, ks, s * ssb, w, s * 256l) + } elif (fmt == 24) { + dequant_iq2xs_plane_superblock(kq, s * qsb, ks, s * ssb, w, s * 256l) } elif (fmt == 23) { dequant_iq2s_plane_superblock(kq, s * qsb, ks, s * ssb, w, s * 256l) } else { @@ -903,6 +957,8 @@ def private kq_dot_gate(t : T?; fmt : int; n : int64 = 512l) { got = dot_iq4nl_q8(addr(kq[0]), addr(ks[0]), addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } elif (fmt == 2) { got = dot_k2q8(addr(kq[0]), addr(ks[0]), addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + } elif (fmt == 24) { + got = dot_iq2xs_q8(addr(kq[0]), addr(ks[0]), addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } elif (fmt == 23) { got = dot_iq2s_q8(addr(kq[0]), addr(ks[0]), addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } else { @@ -938,7 +994,7 @@ def test_kq_dots(t : T?) { } // 122B-class row widths: n=3072 (12 superblocks — qwen35moe-122B gate/up/cls) and n=1024 // (its k5 down_exps). Every earlier kq model kept n at 512..2560. - for (fmt in [4, 5, 6, 40, 44, 3, 33, 34, 45, 2, 23]) { + for (fmt in [4, 5, 6, 40, 44, 3, 33, 34, 45, 2, 23, 24]) { for (nn in [1024l, 3072l]) { t |> run("dot_{kq_tag(fmt)}q8 matches the fp64 reference at n={int(nn)}") @(t : T?) { kq_dot_gate(t, fmt, nn) @@ -957,7 +1013,7 @@ def private kq_gemv_rows_gate(t : T?; fmt : int; n : int64 = 512l) { let nsb = n / 256l let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) - let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : (fmt == 45 ? build_iq4nl_block() : (fmt == 2 ? build_q2k_block() : (fmt == 23 ? build_iq2s_block() : build_q6k_block()))))))))) + let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : (fmt == 45 ? build_iq4nl_block() : (fmt == 2 ? build_q2k_block() : (fmt == 23 ? build_iq2s_block() : (fmt == 24 ? build_iq2xs_block() : build_q6k_block())))))))))) var kq : array var ks : array kq |> resize(d * nsb * qsb) @@ -982,6 +1038,8 @@ def private kq_gemv_rows_gate(t : T?; fmt : int; n : int64 = 512l) { transcode_iq4nl_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } elif (fmt == 2) { transcode_q2k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 24) { + transcode_iq2xs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } elif (fmt == 23) { transcode_iq2s_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } else { @@ -1034,6 +1092,8 @@ def private kq_gemv_rows_gate(t : T?; fmt : int; n : int64 = 512l) { want = dot_iq4nl_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } elif (fmt == 2) { want = dot_k2q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + } elif (fmt == 24) { + want = dot_iq2xs_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } elif (fmt == 23) { want = dot_iq2s_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } else { @@ -1077,7 +1137,7 @@ def test_kq_gemv_rows(t : T?) { t |> run("portable iq3s GEMV rows bit-match per-row disk dots") @(t : T?) { kq_gemv_rows_gate(t, 33) } - for (fmt in [4, 5, 6, 40, 44, 3, 33, 34, 45, 2, 23]) { + for (fmt in [4, 5, 6, 40, 44, 3, 33, 34, 45, 2, 23, 24]) { for (nn in [1024l, 3072l]) { t |> run("portable {kq_tag(fmt)} GEMV rows bit-match per-row disk dots at n={int(nn)}") @(t : T?) { kq_gemv_rows_gate(t, fmt, nn) @@ -1097,7 +1157,7 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { let nsb = n / 256l let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) - let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : (fmt == 45 ? build_iq4nl_block() : (fmt == 2 ? build_q2k_block() : (fmt == 23 ? build_iq2s_block() : build_q6k_block()))))))))) + let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : (fmt == 45 ? build_iq4nl_block() : (fmt == 2 ? build_q2k_block() : (fmt == 23 ? build_iq2s_block() : (fmt == 24 ? build_iq2xs_block() : build_q6k_block())))))))))) var kq : array var ks : array kq |> resize(d * nsb * qsb) @@ -1122,6 +1182,8 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { transcode_iq4nl_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } elif (fmt == 2) { transcode_q2k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 24) { + transcode_iq2xs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } elif (fmt == 23) { transcode_iq2s_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } else { @@ -1173,6 +1235,8 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { want[r] = dot_iq4nl_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } elif (fmt == 2) { want[r] = dot_k2q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + } elif (fmt == 24) { + want[r] = dot_iq2xs_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } elif (fmt == 23) { want[r] = dot_iq2s_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } else { @@ -1197,6 +1261,8 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { dequant_iq4nl_plane_superblock(kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb, wrow, r * n + s * 256l) } elif (fmt == 2) { dequant_k2_plane_superblock(kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb, wrow, r * n + s * 256l) + } elif (fmt == 24) { + dequant_iq2xs_plane_superblock(kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb, wrow, r * n + s * 256l) } elif (fmt == 23) { dequant_iq2s_plane_superblock(kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb, wrow, r * n + s * 256l) } else { @@ -1222,6 +1288,8 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { repack_q40_grp(addr(kq[0]), addr(ks[0]), n, d, mr) } elif (fmt == 2) { repack_k2_grp(addr(kq[0]), addr(ks[0]), n, d, mr) + } elif (fmt == 24) { + repack_iq2xs_grp(addr(kq[0]), addr(ks[0]), n, d, mr) } elif (fmt == 23) { repack_iq2s_grp(addr(kq[0]), addr(ks[0]), n, d, mr) } else { @@ -1241,8 +1309,8 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { ? iq4xs_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) : (fmt == 3 ? k3_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) - : (fmt == 33 || fmt == 34 || fmt == 45 || fmt == 2 || fmt == 23 - ? (fmt == 33 ? iq3s_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) : (fmt == 34 ? iq3xxs_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) : (fmt == 45 ? iq4nl_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) : (fmt == 2 ? k2_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) : iq2s_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n))))) + : (fmt == 33 || fmt == 34 || fmt == 45 || fmt == 2 || fmt == 23 || fmt == 24 + ? (fmt == 33 ? iq3s_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) : (fmt == 34 ? iq3xxs_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) : (fmt == 45 ? iq4nl_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) : (fmt == 2 ? k2_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) : (fmt == 23 ? iq2s_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) : iq2xs_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n)))))) : kq_grp_row_dot(int64(fmt), kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n))))) if (got != want[r]) { dotbad++ @@ -1270,7 +1338,7 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { [test] def test_kq_repack_grp(t : T?) { - for (fmt in [4, 5, 6, 40, 44, 3, 33, 34, 45, 2, 23]) { + for (fmt in [4, 5, 6, 40, 44, 3, 33, 34, 45, 2, 23, 24]) { for (mr in [4l, 8l, 16l]) { t |> run("repack_{kq_tag(fmt)}_grp mr={mr} preserves dots and row dequants") @(t : T?) { kq_repack_gate(t, fmt, mr) @@ -1295,7 +1363,7 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { // nolint:STYL let mr = kq_layout_of(fmt) // the format's OWN layout companion (per-format tune families) let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) - let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : (fmt == 45 ? build_iq4nl_block() : (fmt == 2 ? build_q2k_block() : (fmt == 23 ? build_iq2s_block() : build_q6k_block()))))))))) + let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : (fmt == 45 ? build_iq4nl_block() : (fmt == 2 ? build_q2k_block() : (fmt == 23 ? build_iq2s_block() : (fmt == 24 ? build_iq2xs_block() : build_q6k_block())))))))))) var kq : array var ks : array kq |> resize(d * nsb * qsb) @@ -1320,6 +1388,8 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { // nolint:STYL transcode_iq4nl_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } elif (fmt == 2) { transcode_q2k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 24) { + transcode_iq2xs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } elif (fmt == 23) { transcode_iq2s_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } else { @@ -1351,6 +1421,8 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { // nolint:STYL repack_q40_grp(addr(kq[0]), addr(ks[0]), n, d, mr) } elif (fmt == 2) { repack_k2_grp(addr(kq[0]), addr(ks[0]), n, d, mr) + } elif (fmt == 24) { + repack_iq2xs_grp(addr(kq[0]), addr(ks[0]), n, d, mr) } elif (fmt == 23) { repack_iq2s_grp(addr(kq[0]), addr(ks[0]), n, d, mr) } else { @@ -1381,7 +1453,7 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { // nolint:STYL // k5/k6 tiles read the byte-expanded panel the batch cell unpacks per (group, token-block) // — the gate covers unpack_kq_panel_grp too (tile-over-panel vs gemv-over-packed planes); // k4/q40 tiles read the packed planes directly - let packed = fmt == 4 || fmt == 40 || fmt == 44 || fmt == 3 || fmt == 45 || fmt == 2 // iq3s/iq3xxs/iq2s tiles read the byte-expanded panel (the batch cell's shape) + let packed = fmt == 4 || fmt == 40 || fmt == 44 || fmt == 3 || fmt == 45 || fmt == 2 || fmt == 24 // iq3s/iq3xxs/iq2s tiles read the byte-expanded panel (the batch cell's shape) var panel : array if (!packed) { panel |> resize(mr * nsb * 256l) @@ -1414,6 +1486,8 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { // nolint:STYL iq4nlq8_tile_gen(addr(ytile[0]), kqg, ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) } elif (fmt == 2) { k2q8_tile_gen(addr(ytile[0]), kqg, ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) + } elif (fmt == 24) { + iq2xsq8_tile_gen(addr(ytile[0]), kqg, ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) } elif (fmt == 23) { iq2sq8_tile_gen(addr(ytile[0]), addr(panel[0]), ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) } else { @@ -1440,6 +1514,8 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { // nolint:STYL iq4nlq8_gemv_gen(addr(ytile[tk * d]), addr(kq[0]), addr(ks[0]), addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, g * mr, (g + 1l) * mr) } elif (fmt == 2) { k2q8_gemv_gen(addr(ytile[tk * d]), addr(kq[0]), addr(ks[0]), addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, g * mr, (g + 1l) * mr) + } elif (fmt == 24) { + iq2xsq8_gemv_gen(addr(ytile[tk * d]), addr(kq[0]), addr(ks[0]), addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, g * mr, (g + 1l) * mr) } elif (fmt == 23) { iq2sq8_gemv_gen(addr(ytile[tk * d]), addr(kq[0]), addr(ks[0]), addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, g * mr, (g + 1l) * mr) } else { @@ -1467,6 +1543,8 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { // nolint:STYL iq4nlq8_gemv_gen(addr(ygemv[p * d]), addr(kq[0]), addr(ks[0]), addr(xq[p * n]), addr(xs[p * (n / 256l)]), addr(xbs[p * (n / 16l)]), n, 0l, d) } elif (fmt == 2) { k2q8_gemv_gen(addr(ygemv[p * d]), addr(kq[0]), addr(ks[0]), addr(xq[p * n]), addr(xs[p * (n / 256l)]), addr(xbs[p * (n / 16l)]), n, 0l, d) + } elif (fmt == 24) { + iq2xsq8_gemv_gen(addr(ygemv[p * d]), addr(kq[0]), addr(ks[0]), addr(xq[p * n]), addr(xs[p * (n / 256l)]), addr(xbs[p * (n / 16l)]), n, 0l, d) } elif (fmt == 23) { iq2sq8_gemv_gen(addr(ygemv[p * d]), addr(kq[0]), addr(ks[0]), addr(xq[p * n]), addr(xs[p * (n / 256l)]), addr(xbs[p * (n / 16l)]), n, 0l, d) } else { @@ -1494,7 +1572,7 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { // nolint:STYL [test] def test_kq_tile(t : T?) { - for (fmt in [4, 5, 6, 40, 44, 3, 33, 34, 45, 2, 23]) { + for (fmt in [4, 5, 6, 40, 44, 3, 33, 34, 45, 2, 23, 24]) { t |> run("{kq_tag(fmt)} 4-token tile bit-matches per-token GEMVs") @(t : T?) { kq_tile_gate(t, fmt) } @@ -1515,7 +1593,7 @@ def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; let nsb = n / 256l let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) - let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : (fmt == 45 ? build_iq4nl_block() : (fmt == 2 ? build_q2k_block() : (fmt == 23 ? build_iq2s_block() : build_q6k_block()))))))))) + let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : (fmt == 45 ? build_iq4nl_block() : (fmt == 2 ? build_q2k_block() : (fmt == 23 ? build_iq2s_block() : (fmt == 24 ? build_iq2xs_block() : build_q6k_block())))))))))) var kq : array var ks : array kq |> resize(nreg * d * nsb * qsb) @@ -1540,6 +1618,8 @@ def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; transcode_iq4nl_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } elif (fmt == 2) { transcode_q2k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 24) { + transcode_iq2xs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } elif (fmt == 23) { transcode_iq2s_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } else { @@ -1617,6 +1697,9 @@ def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; } elif (fmt == 2) { want_sh[r * d + row] = dot_k2q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) want_pr[r * d + row] = dot_k2q8(kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n) + } elif (fmt == 24) { + want_sh[r * d + row] = dot_iq2xs_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + want_pr[r * d + row] = dot_iq2xs_q8(kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n) } elif (fmt == 23) { want_sh[r * d + row] = dot_iq2s_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) want_pr[r * d + row] = dot_iq2s_q8(kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n) @@ -1674,6 +1757,8 @@ def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; repack_q40_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) } elif (fmt == 2) { repack_k2_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) + } elif (fmt == 24) { + repack_iq2xs_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) } elif (fmt == 23) { repack_iq2s_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) } else { @@ -1710,6 +1795,9 @@ def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; } elif (fmt == 2) { k2q8_gemv_gen(addr(want_sh[r * d]), kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, 0l, d) k2q8_gemv_gen(addr(want_pr[r * d]), kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n, 0l, d) + } elif (fmt == 24) { + iq2xsq8_gemv_gen(addr(want_sh[r * d]), kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, 0l, d) + iq2xsq8_gemv_gen(addr(want_pr[r * d]), kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n, 0l, d) } elif (fmt == 23) { iq2sq8_gemv_gen(addr(want_sh[r * d]), kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, 0l, d) iq2sq8_gemv_gen(addr(want_pr[r * d]), kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n, 0l, d) @@ -1748,7 +1836,7 @@ def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; [test] def test_kq_groupn(t : T?) { - for (fmt in [4, 5, 6, 40, 44, 3, 33, 34, 45, 2, 23]) { + for (fmt in [4, 5, 6, 40, 44, 3, 33, 34, 45, 2, 23, 24]) { t |> run("{kq_tag(fmt)} region-list GEMV bit-matches per-row dots (disk + grp slices)") @(t : T?) { kq_groupn_gate(t, fmt) } @@ -1792,7 +1880,7 @@ def private kq_batch_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 for (c in cnts) { nk += c } - let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : (fmt == 45 ? build_iq4nl_block() : (fmt == 2 ? build_q2k_block() : (fmt == 23 ? build_iq2s_block() : build_q6k_block()))))))))) + let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : (fmt == 45 ? build_iq4nl_block() : (fmt == 2 ? build_q2k_block() : (fmt == 23 ? build_iq2s_block() : (fmt == 24 ? build_iq2xs_block() : build_q6k_block())))))))))) var kq : array var ks : array kq |> resize(nreg * d * nsb * qsb) @@ -1817,6 +1905,8 @@ def private kq_batch_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 transcode_iq4nl_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } elif (fmt == 2) { transcode_q2k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 24) { + transcode_iq2xs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } elif (fmt == 23) { transcode_iq2s_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } else { @@ -1849,6 +1939,8 @@ def private kq_batch_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 repack_q40_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) } elif (fmt == 2) { repack_k2_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) + } elif (fmt == 24) { + repack_iq2xs_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) } elif (fmt == 23) { repack_iq2s_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) } else { @@ -1907,6 +1999,8 @@ def private kq_batch_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 iq4nlq8_gemv_gen(addr(want[tk * d]), kqr, ksr, addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, 0l, d) } elif (fmt == 2) { k2q8_gemv_gen(addr(want[tk * d]), kqr, ksr, addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, 0l, d) + } elif (fmt == 24) { + iq2xsq8_gemv_gen(addr(want[tk * d]), kqr, ksr, addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, 0l, d) } elif (fmt == 23) { iq2sq8_gemv_gen(addr(want[tk * d]), kqr, ksr, addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, 0l, d) } else { @@ -1940,7 +2034,7 @@ def private kq_batch_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 [test] def test_kq_batch_groupn(t : T?) { - for (fmt in [4, 5, 6, 40, 44, 3, 33, 34, 45, 2, 23]) { + for (fmt in [4, 5, 6, 40, 44, 3, 33, 34, 45, 2, 23, 24]) { t |> run("{kq_tag(fmt)} batch groupn bit-matches per-token rows-core calls") @(t : T?) { kq_batch_groupn_gate(t, fmt) } From 141944c648add814abe66fdb64ea66fc592b1de6 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Mon, 31 Aug 2026 10:49:03 -0700 Subject: [PATCH 40/61] dasLLAMA: IQ2_XS Phase B - the JIT emitter rides the fmt-23 panel route iq2xs joins the panel lattice as kernel 24, sharing IQ2_S's strip math wholesale: the kq_grp_row_dot_b 23 arms extend `|| fmt == 24l` (signed panel bytes, per-16 UNSIGNED strips, d/8 fold), the tile drops off the packed lists and reads the byte-expanded panel via unpack_iq2xs_panel_grp (each u16 word = a 9-bit iq2xs_grid[512] index doubled into the u64 low/high word pair + a 7-bit ksigns byte through the shared smask negate), and the gemv gathers per superblock via emit_iq2xs_gather off iq2xs_emit_globals' [1024 x i32] private grid plus the SHARED iq3xxs ksigns and iq3s smask globals. Real generators land for gemv + tile; the kfxs24 TEST rows return to the probe (QUIRK 16 closed for the format). Probe: 11/11 k24 perms ok (maddubs 5.2e-7); the tuner crowns dot_maddubs_width256_mr8. QUIRK 15 reran exactly as written: run.tune.json still pinned iq2xsq8_tile_gen "reference" from Phase A, cleared with run.das -- --tune. Gates: test_kquant -jit 246 pass / 4 skips, interp 232 / 18 skips, probe GEN TUNE TEST OK, lint 0. E2e stamped: parity ids 64/64 vs llama.cpp - Phase A's first-ever full-match greedy stream HOLDS under the stamped folds; gen 23 -> 54 t/s. The zen2 bench row and the HOW_TO Phase B note follow with the in-flight measurement. Co-Authored-By: Claude Fable 5 --- .../dasLLAMA/dasllama/dasllama_gemm_gen.das | 139 ++++++++++++++++-- .../dasLLAMA/dasllama/dasllama_math_gen.das | 14 +- modules/dasLLAMA/dasllama/dasllama_repack.das | 44 ++++++ modules/dasLLAMA/harness/gen_tune_probe.das | 5 +- modules/dasLLAMA/tests/test_kquant.das | 4 +- 5 files changed, 185 insertions(+), 21 deletions(-) diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das index ce28b7858b..9f5c044839 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das @@ -1573,6 +1573,121 @@ def private emit_iq2s_gather(var te : TileEmit; var sbi : LLVMOpaqueValue?) { LLVMPositionBuilderAtEnd(b, cont) } +// The iq2xs gemv gather globals: the u64 grid as a [1024 x i32] private constant (512 entries +// as low/high word pairs), the shared ksigns table, and the shared smask. +def private iq2xs_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { + var grid = LLVMGetNamedGlobal(mod, "dasllama.iq2xs.grid") + if (grid == null) { + let g = iq2xs_grid2() + var elems : array + elems |> reserve(1024) + for (i in range(1024)) { + elems |> push(LLVMConstInt(te.types.t_int32, uint64(g[i]), 0)) + } + grid = LLVMAddGlobal(mod, LLVMArrayType(te.types.t_int32, 1024u), "dasllama.iq2xs.grid") + LLVMSetInitializer(grid, LLVMConstArray(te.types.t_int32, array_data_ptr(elems), 1024u)) + LLVMSetGlobalConstant(grid, 1) + LLVMSetLinkage(grid, LLVMLinkage.LLVMPrivateLinkage) + LLVMSetAlignment(grid, 64u) + } + te.iq3s_grid_g = grid + var ksg = LLVMGetNamedGlobal(mod, "dasllama.iq3xxs.ksigns") + if (ksg == null) { + let kt = ksigns_iq2xs() + var elems3 : array + elems3 |> reserve(128) + for (i in range(128)) { + elems3 |> push(LLVMConstInt(te.types.t_int32, uint64(kt[i]), 0)) + } + ksg = LLVMAddGlobal(mod, LLVMArrayType(te.types.t_int32, 128u), "dasllama.iq3xxs.ksigns") + LLVMSetInitializer(ksg, LLVMConstArray(te.types.t_int32, array_data_ptr(elems3), 128u)) + LLVMSetGlobalConstant(ksg, 1) + LLVMSetLinkage(ksg, LLVMLinkage.LLVMPrivateLinkage) + LLVMSetAlignment(ksg, 16u) + } + te.iq3xxs_ksigns_g = ksg + var smask = LLVMGetNamedGlobal(mod, "dasllama.iq3s.smask") + if (smask == null) { + var elems2 : array + elems2 |> reserve(16) + for (m in range(16)) { + var v = 0ul + for (t in range(4)) { + if ((m & (1 << t)) != 0) { + v |= 0xFFul << uint64(8 * t) + } + } + elems2 |> push(LLVMConstInt(te.types.t_int32, v, 0)) + } + smask = LLVMAddGlobal(mod, LLVMArrayType(te.types.t_int32, 16u), "dasllama.iq3s.smask") + LLVMSetInitializer(smask, LLVMConstArray(te.types.t_int32, array_data_ptr(elems2), 16u)) + LLVMSetGlobalConstant(smask, 1) + LLVMSetLinkage(smask, LLVMLinkage.LLVMPrivateLinkage) + LLVMSetAlignment(smask, 16u) + } + te.iq3s_smask_g = smask +} + +// iq2xs gemv: gather superblock `sbi` off the PACKED grp planes into the alloca panel - the +// iq2s gather's two-word u64 form with the u16 qs word's 9-bit index and its high-7-bit +// KSIGNS lookup in place of explicit sign bytes (the iq3xxs sign path). +def private emit_iq2xs_gather(var te : TileEmit; var sbi : LLVMOpaqueValue?) { + let b = te.builder + let mr = te.interleave + var wbP = LLVMBuildMul(b, sbi, te.types->ConstI64(uint64(mr * 64)), "iq2x.wb") + var pre = LLVMGetInsertBlock(b) + var head = LLVMAppendBasicBlockInContext(te.gctx, te.gfn, "iq2x.gath") + var cont = LLVMAppendBasicBlockInContext(te.gctx, te.gfn, "iq2x.cont") + LLVMBuildBr(b, head) + LLVMPositionBuilderAtEnd(b, head) + var rPhi = LLVMBuildPhi(b, te.types.t_int64, "iq2x.r") + var preVals <- [te.types->ConstI64(0ul)] + var preBlocks <- [pre] + LLVMAddIncoming(rPhi, preVals, preBlocks) + var rT4 = LLVMBuildMul(b, rPhi, te.types->ConstI64(4ul), "iq2x.r4") + var base = LLVMBuildAdd(b, wbP, rT4, "iq2x.base") + let c511 = LLVMConstInt(te.types.t_int32, 511ul, 0) + let c15 = LLVMConstInt(te.types.t_int32, 15ul, 0) + let cneg = LLVMConstInt(te.types.t_int32, 0x01010101ul, 0) + for (blk in range(8)) { + for (l in range(4)) { + let lp = (blk * 4 + l) * 2 + var qlp = LLVMBuildGEP2(b, te.types.t_int8, te.wg, LLVMBuildAdd(b, base, te.types->ConstI64(uint64((lp / 4) * mr * 4 + lp % 4)), ""), "") + var qlo = LLVMBuildZExt(b, LLVMBuildLoad2Aligned(b, te.types.t_int8, qlp, 1u, ""), te.types.t_int32, "") + var qhp = LLVMBuildGEP2(b, te.types.t_int8, te.wg, LLVMBuildAdd(b, base, te.types->ConstI64(uint64((lp / 4) * mr * 4 + lp % 4 + 1)), ""), "") + var qhi = LLVMBuildZExt(b, LLVMBuildLoad2Aligned(b, te.types.t_int8, qhp, 1u, ""), te.types.t_int32, "") + var w16 = LLVMBuildOr(b, qlo, LLVMBuildShl(b, qhi, LLVMConstInt(te.types.t_int32, 8ul, 0), ""), "iq2x.w16{blk}_{l}") + var idx = LLVMBuildAnd(b, w16, c511, "iq2x.i{blk}_{l}") + var idx2 = LLVMBuildShl(b, idx, LLVMConstInt(te.types.t_int32, 1ul, 0), "") + var sidx = LLVMBuildLShr(b, w16, LLVMConstInt(te.types.t_int32, 9ul, 0), "") + var ksp2 = LLVMBuildGEP2(b, te.types.t_int32, te.iq3xxs_ksigns_g, sidx, "") + var sgn = LLVMBuildLoad2Aligned(b, te.types.t_int32, ksp2, 4u, "iq2x.sg{blk}_{l}") + var mv : LLVMOpaqueValue? [2] + mv[0] = LLVMBuildAnd(b, sgn, c15, "") + mv[1] = LLVMBuildAnd(b, LLVMBuildLShr(b, sgn, LLVMConstInt(te.types.t_int32, 4ul, 0), ""), c15, "") + for (h in range(2)) { + var gi = h == 0 ? idx2 : LLVMBuildAdd(b, idx2, LLVMConstInt(te.types.t_int32, 1ul, 0), "") + var gp = LLVMBuildGEP2(b, te.types.t_int32, te.iq3s_grid_g, gi, "") + var gw = LLVMBuildLoad2Aligned(b, te.types.t_int32, gp, 4u, "iq2x.g{blk}_{l}_{h}") + var mp = LLVMBuildGEP2(b, te.types.t_int32, te.iq3s_smask_g, mv[h], "") + var mw = LLVMBuildLoad2Aligned(b, te.types.t_int32, mp, 4u, "") + var w = LLVMBuildAdd(b, LLVMBuildXor(b, gw, mw, ""), LLVMBuildAnd(b, mw, cneg, ""), "iq2x.w{blk}_{l}_{h}") + let e = l * 8 + h * 4 + let dd = (e < 16 ? 0 : 128 * mr) + (blk * 4 + (e % 16) / 4) * mr * 4 + var dp = LLVMBuildGEP2(b, te.types.t_int8, te.iq3s_panel, LLVMBuildAdd(b, rT4, te.types->ConstI64(uint64(dd)), ""), "") + LLVMBuildStore(b, w, dp) + } + } + } + var rNext = LLVMBuildAdd(b, rPhi, te.types->ConstI64(1ul), "iq2x.rn") + var more = LLVMBuildICmp(b, LLVMIntPredicate.LLVMIntSLT, rNext, te.types->ConstI64(uint64(mr)), "iq2x.more") + LLVMBuildCondBr(b, more, head, cont) + var loopVals <- [rNext] + var loopBlocks <- [head] + LLVMAddIncoming(rPhi, loopVals, loopBlocks) + LLVMPositionBuilderAtEnd(b, cont) +} + // One 256-weight SUPERBLOCK, IQ4_XS/iq3s grp form (te.kq = 44/33): mx4's LUT decode (44) // or the byte-expanded signed panel (33; the gemv gathers it per superblock, the tile gets the // runtime unpack) — dot_lane's sign-trick lattice, k4's fold with no min term. @@ -1581,8 +1696,8 @@ def private emit_block_iq4xs(var te : TileEmit; var sbi : LLVMOpaqueValue?; var let rq = te.rq let mr = te.interleave let w8 = te.width / 8 - let panel = te.kq == 33 || te.kq == 34 || te.kq == 23 // iq3s/iq3xxs/iq2s: the byte-expanded signed panel replaces the nibble+LUT decode - let iq2 = te.kq == 23 // iq2s: per-16 UNSIGNED strips - lo/hi halves keep split accumulators + let panel = te.kq == 33 || te.kq == 34 || te.kq == 23 || te.kq == 24 // iq3s/iq3xxs/iq2s/iq2xs: the byte-expanded signed panel replaces the nibble+LUT decode + let iq2 = te.kq == 23 || te.kq == 24 // iq2s/iq2xs: per-16 UNSIGNED strips - lo/hi halves keep split accumulators let nl = te.kq == 45 // iq4nl: the 44 LUT decode with q40's PER-BLOCK f16 d fold (no sub-scales, no min term) let gather = panel && te.iq3s_panel != null // gemv mode: gather this superblock off the packed planes first var wbase = te.wg @@ -1592,6 +1707,8 @@ def private emit_block_iq4xs(var te : TileEmit; var sbi : LLVMOpaqueValue?; var emit_iq3s_gather(te, sbi) } elif (te.kq == 23) { emit_iq2s_gather(te, sbi) + } elif (te.kq == 24) { + emit_iq2xs_gather(te, sbi) } else { emit_iq3xxs_gather(te, sbi) } @@ -1720,7 +1837,7 @@ def private emit_block_iq4xs(var te : TileEmit; var sbi : LLVMOpaqueValue?; var // token PAIR — single-token slices ride the sdot lattice instead. kq blocks are SUPERBLOCKS // (the kq gemv/tile drive emit_slice with nb = n/256). def private emit_one_block(var te : TileEmit; var bi : LLVMOpaqueValue?; var f : LLVMOpaqueValue? [8]; tokBase, tokCount : int) { - if (te.kq == 44 || te.kq == 33 || te.kq == 34 || te.kq == 45 || te.kq == 23) { + if (te.kq == 44 || te.kq == 33 || te.kq == 34 || te.kq == 45 || te.kq == 23 || te.kq == 24) { emit_block_iq4xs(te, bi, f, tokBase, tokCount) } elif (te.kq != 0) { emit_block_kqv2(te, bi, f, tokBase, tokCount) @@ -2827,14 +2944,16 @@ def private kq_gemv_gen_impl(var gc : LlvmCodeCtx; fmt : int) : bool { // entry: group range off the row range; strides are disk footprints (quants 128/160/192B, scales 20/20/18B) LLVMPositionBuilderAtEnd(b, entry) - if (fmt == 33 || fmt == 34 || fmt == 23) { - // iq3s/iq3xxs/iq2s: the gemv gathers each superblock into an alloca panel (the tile gets the runtime unpack) + if (fmt == 33 || fmt == 34 || fmt == 23 || fmt == 24) { + // iq3s/iq3xxs/iq2s/iq2xs: the gemv gathers each superblock into an alloca panel (the tile gets the runtime unpack) te.gfn = gc.impl te.gctx = gc.jit.ctx if (fmt == 33) { iq3s_emit_globals(te, LLVMGetGlobalParent(gc.impl)) } elif (fmt == 23) { iq2s_emit_globals(te, LLVMGetGlobalParent(gc.impl)) + } elif (fmt == 24) { + iq2xs_emit_globals(te, LLVMGetGlobalParent(gc.impl)) } else { iq3xxs_emit_globals(te, LLVMGetGlobalParent(gc.impl)) } @@ -3038,12 +3157,10 @@ def private k2_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 2) def private iq2s_gemv_gen(var gc : LlvmCodeCtx) : bool => kq_gemv_gen_impl(gc, 23) def private iq2s_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 23) -// iq2xs: the grid-gather emitter arm lands with its Phase B; until then both stamps decline -// (reference bodies serve) - a declined generator is the framework's own fallback path. -[unused_argument(gc)] -def private iq2xs_gemv_gen(var gc : LlvmCodeCtx) : bool => false -[unused_argument(gc)] -def private iq2xs_tile_gen(var gc : LlvmCodeCtx) : bool => false +// iq2xs rides the iq4xs walk through the panel route: emit_iq2xs_gather feeds the shared +// byte-expanded panel; the iq2 flag's split accumulators and per-16 strips are iq2s's. +def private iq2xs_gemv_gen(var gc : LlvmCodeCtx) : bool => kq_gemv_gen_impl(gc, 24) +def private iq2xs_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 24) def private k4_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 4) def private k5_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 5) diff --git a/modules/dasLLAMA/dasllama/dasllama_math_gen.das b/modules/dasLLAMA/dasllama/dasllama_math_gen.das index a89eca3962..969dcde657 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_gen.das @@ -1042,7 +1042,7 @@ def iq2xsq8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xq unsafe { for (t in range64(4l)) { for (r in range64(mr)) { - yp[(t0 + t) * d + g * mr + r] = iq2xs_grp_row_dot(kqg, ksg, r, mr, xqp + (t0 + t) * n, xsp + (t0 + t) * (n / 256l), xbsp + (t0 + t) * (n / 16l), n) + yp[(t0 + t) * d + g * mr + r] = kq_grp_row_dot_b(24l, kqg, ksg, r, mr, xqp + (t0 + t) * n, xsp + (t0 + t) * (n / 256l), xbsp + (t0 + t) * (n / 16l), n) } } } @@ -1323,8 +1323,8 @@ def kq_grp_row_dot_b(fmt : int64; kqg : uint8 const?; ksg : uint8 const?; r, mr var ihi = 0 for (j in range64(4l)) { for (t in range64(4l)) { - let qlo = fmt == 33l || fmt == 34l || fmt == 23l ? int(int8(kqg[qb + ((blk * 4l + j) * mr + r) * 4l + t])) : int(kqg[qb + ((blk * 4l + j) * mr + r) * 4l + t]) - let qhi = fmt == 33l || fmt == 34l || fmt == 23l ? int(int8(kqg[qb + 128l * mr + ((blk * 4l + j) * mr + r) * 4l + t])) : int(kqg[qb + 128l * mr + ((blk * 4l + j) * mr + r) * 4l + t]) + let qlo = fmt == 33l || fmt == 34l || fmt == 23l || fmt == 24l ? int(int8(kqg[qb + ((blk * 4l + j) * mr + r) * 4l + t])) : int(kqg[qb + ((blk * 4l + j) * mr + r) * 4l + t]) + let qhi = fmt == 33l || fmt == 34l || fmt == 23l || fmt == 24l ? int(int8(kqg[qb + 128l * mr + ((blk * 4l + j) * mr + r) * 4l + t])) : int(kqg[qb + 128l * mr + ((blk * 4l + j) * mr + r) * 4l + t]) ilo += qlo * int(xqp[b * 32l + j * 4l + t]) ihi += qhi * int(xqp[b * 32l + 16l + j * 4l + t]) } @@ -1332,7 +1332,7 @@ def kq_grp_row_dot_b(fmt : int64; kqg : uint8 const?; ksg : uint8 const?; r, mr if (fmt == 33l || fmt == 34l) { let sc = int(int8(ksg[sb + 4l * mr + blk * mr + r])) isum += sc * (ilo + ihi) - } elif (fmt == 23l) { + } elif (fmt == 23l || fmt == 24l) { let s0 = int(ksg[sb + 4l * mr + 2l * blk * mr + r]) let s1 = int(ksg[sb + 4l * mr + (2l * blk + 1l) * mr + r]) isum += s0 * ilo + s1 * ihi @@ -1348,7 +1348,7 @@ def kq_grp_row_dot_b(fmt : int64; kqg : uint8 const?; ksg : uint8 const?; r, mr bsum += mn * (xbsp[b * 2l] + xbsp[b * 2l + 1l]) } } - if (fmt == 33l || fmt == 34l || fmt == 23l) { + if (fmt == 33l || fmt == 34l || fmt == 23l || fmt == 24l) { let di = f16_to_f32(uint(ksg[sb + 2l * r]) | (uint(ksg[sb + 2l * r + 1l]) << 8u)) acc += float(isum) * (di * xsp[sbi]) } elif (fmt == 6l) { @@ -1988,7 +1988,7 @@ def private kq_batch_cell_gen(fmt : int; var myp : float?; kqp : uint8 const?; k let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) let nb16 = n / 16l - let packed = fmt == 4 || fmt == 40 || fmt == 44 || fmt == 3 || fmt == 45 || fmt == 2 || fmt == 24 // packed planes: the tile reads them directly (iq3s/iq3xxs/iq2s expand their grid gathers into the panel) + let packed = fmt == 4 || fmt == 40 || fmt == 44 || fmt == 3 || fmt == 45 || fmt == 2 // packed planes: the tile reads them directly (iq3s/iq3xxs/iq2s expand their grid gathers into the panel) var scratch : array if (!packed) { scratch |> resize(mr * nsb * 256l) @@ -2029,7 +2029,7 @@ def private kq_batch_cell_gen(fmt : int; var myp : float?; kqp : uint8 const?; k } elif (fmt == 23) { iq2sq8_tile_gen(myp, scp, ksg, xqp, xsp, xbsp, n, d, g, tk) } elif (fmt == 24) { - iq2xsq8_tile_gen(myp, kqg, ksg, xqp, xsp, xbsp, n, d, g, tk) + iq2xsq8_tile_gen(myp, scp, ksg, xqp, xsp, xbsp, n, d, g, tk) } else { k6q8_tile_gen(myp, scp, ksg, xqp, xsp, xbsp, n, d, g, tk) } diff --git a/modules/dasLLAMA/dasllama/dasllama_repack.das b/modules/dasLLAMA/dasllama/dasllama_repack.das index beeda51210..621abde579 100644 --- a/modules/dasLLAMA/dasllama/dasllama_repack.das +++ b/modules/dasLLAMA/dasllama/dasllama_repack.das @@ -825,6 +825,46 @@ def unpack_iq2s_panel_grp(kqg : uint8 const?; var dst : uint8?; mr, nsb : int64) } } +//! iq2xs panel unpack: the u16 qs word's 9-bit index doubles into the u64 grid's word pair, +//! its high 7 bits pick the KSIGNS sign byte - packed grp planes -> the shared signed panel. +def unpack_iq2xs_panel_grp(kqg : uint8 const?; var dst : uint8?; mr, nsb : int64) { + let grid = iq2xs_grid2() + let ksgn = ksigns_iq2xs() + let smask = fixed_array(0x00000000u, 0x000000FFu, 0x0000FF00u, 0x0000FFFFu, + 0x00FF0000u, 0x00FF00FFu, 0x00FFFF00u, 0x00FFFFFFu, + 0xFF000000u, 0xFF0000FFu, 0xFF00FF00u, 0xFF00FFFFu, + 0xFFFF0000u, 0xFFFF00FFu, 0xFFFFFF00u, 0xFFFFFFFFu) + unsafe { + for (sbi in range64(nsb)) { + let qb = sbi * 64l * mr + let ob = sbi * 256l * mr + for (r in range64(mr)) { + for (blk in range64(8l)) { + for (l in range64(4l)) { + let lp = (blk * 4l + l) * 2l + let w16 = uint(kqg[qb + ((lp / 4l) * mr + r) * 4l + lp % 4l]) | (uint(kqg[qb + ((lp / 4l) * mr + r) * 4l + lp % 4l + 1l]) << 8u) + let sgn = int(ksgn[int(w16 >> 9u)]) + let ix = int(w16 & 511u) * 2 + let m1 = smask[sgn & 15] + let m2 = smask[(sgn >> 4) & 15] + let w1 = (grid[ix] ^ m1) + (m1 & 0x01010101u) + let w2 = (grid[ix + 1] ^ m2) + (m2 & 0x01010101u) + // element e = l*8 + half*4 + t: e < 16 lands in the lo panel half, else hi + let e1 = l * 8l + let e2 = l * 8l + 4l + let d1 = (e1 < 16l ? 0l : 128l * mr) + ((blk * 4l + (e1 % 16l) / 4l) * mr + r) * 4l + let d2 = (e2 < 16l ? 0l : 128l * mr) + ((blk * 4l + (e2 % 16l) / 4l) * mr + r) * 4l + var wp1 = reinterpret(dst + (ob + d1)) + var wp2 = reinterpret(dst + (ob + d2)) + wp1[0] = w1 + wp2[0] = w2 + } + } + } + } + } +} + def unpack_kq_panel_grp(fmt : int64; kqg : uint8 const?; var dst : uint8?; mr, nsb : int64) { if (fmt == 33l) { unpack_iq3s_panel_grp(kqg, dst, mr, nsb) @@ -838,6 +878,10 @@ def unpack_kq_panel_grp(fmt : int64; kqg : uint8 const?; var dst : uint8?; mr, n unpack_iq2s_panel_grp(kqg, dst, mr, nsb) return } + if (fmt == 24l) { + unpack_iq2xs_panel_grp(kqg, dst, mr, nsb) + return + } let LO4 = 0x0F0F0F0F0F0F0F0Ful let SEL = 0x0804020108040201ul // LE lane t (0..3, twice) selects high-bit t let C7F = 0x7F7F7F7F7F7F7F7Ful diff --git a/modules/dasLLAMA/harness/gen_tune_probe.das b/modules/dasLLAMA/harness/gen_tune_probe.das index ae3e53f1f7..588d99ad10 100644 --- a/modules/dasLLAMA/harness/gen_tune_probe.das +++ b/modules/dasLLAMA/harness/gen_tune_probe.das @@ -918,7 +918,7 @@ def kq_layout_mrs(fmt : int64) : table { // nolint:STYLE037 — t def run_kq_tile(tile; fx : KqFixture; kq : array; ks : array; mr : int64; var y : array) { let ng = fx.d / mr verify(fx.ntok % 4l == 0l) - let packed = fx.fmt == 4l || fx.fmt == 40l || fx.fmt == 44l || fx.fmt == 3l || fx.fmt == 45l || fx.fmt == 2l || fx.fmt == 24l // packed planes: the tile reads them directly (iq3s/iq3xxs/iq2s unpack their panels per group) + let packed = fx.fmt == 4l || fx.fmt == 40l || fx.fmt == 44l || fx.fmt == 3l || fx.fmt == 45l || fx.fmt == 2l // packed planes: the tile reads them directly (iq3s/iq3xxs/iq2s unpack their panels per group) var panel : array if (!packed) { panel |> resize(int(mr * fx.nsb * 256l)) @@ -1405,6 +1405,7 @@ def test_mode : bool { // nolint:STYLE037,STYLE038 — flat per-variant correctn var kfxs45 <- [ <- build_kq_fixture(45l, 256l, 32l, 8l), <- build_kq_fixture(45l, 768l, 32l, 8l), <- build_kq_fixture(45l, 2048l, 32l, 8l)] var kfxs2 <- [ <- build_kq_fixture(2l, 256l, 32l, 8l), <- build_kq_fixture(2l, 768l, 32l, 8l), <- build_kq_fixture(2l, 2048l, 32l, 8l)] var kfxs23 <- [ <- build_kq_fixture(23l, 256l, 32l, 8l), <- build_kq_fixture(23l, 768l, 32l, 8l), <- build_kq_fixture(23l, 2048l, 32l, 8l)] + var kfxs24 <- [ <- build_kq_fixture(24l, 256l, 32l, 8l), <- build_kq_fixture(24l, 768l, 32l, 8l), <- build_kq_fixture(24l, 2048l, 32l, 8l)] var vs <- q8q8_tile_gen_variants() var mrs <- variant_mrs() var wbs <- variant_wbias() @@ -1536,6 +1537,7 @@ def test_mode : bool { // nolint:STYLE037,STYLE038 — flat per-variant correctn allok = kq_test_family(45l, kfxs45) && allok allok = kq_test_family(2l, kfxs2) && allok allok = kq_test_family(23l, kfxs23) && allok + allok = kq_test_family(24l, kfxs24) && allok delete wtab delete mtab delete gtab @@ -1558,6 +1560,7 @@ def test_mode : bool { // nolint:STYLE037,STYLE038 — flat per-variant correctn delete kfxs45 delete kfxs2 delete kfxs23 + delete kfxs24 return allok } diff --git a/modules/dasLLAMA/tests/test_kquant.das b/modules/dasLLAMA/tests/test_kquant.das index bc9c99c89d..57f6b035c9 100644 --- a/modules/dasLLAMA/tests/test_kquant.das +++ b/modules/dasLLAMA/tests/test_kquant.das @@ -1453,7 +1453,7 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { // nolint:STYL // k5/k6 tiles read the byte-expanded panel the batch cell unpacks per (group, token-block) // — the gate covers unpack_kq_panel_grp too (tile-over-panel vs gemv-over-packed planes); // k4/q40 tiles read the packed planes directly - let packed = fmt == 4 || fmt == 40 || fmt == 44 || fmt == 3 || fmt == 45 || fmt == 2 || fmt == 24 // iq3s/iq3xxs/iq2s tiles read the byte-expanded panel (the batch cell's shape) + let packed = fmt == 4 || fmt == 40 || fmt == 44 || fmt == 3 || fmt == 45 || fmt == 2 // iq3s/iq3xxs/iq2s/iq2xs tiles read the byte-expanded panel (the batch cell's shape) var panel : array if (!packed) { panel |> resize(mr * nsb * 256l) @@ -1487,7 +1487,7 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { // nolint:STYL } elif (fmt == 2) { k2q8_tile_gen(addr(ytile[0]), kqg, ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) } elif (fmt == 24) { - iq2xsq8_tile_gen(addr(ytile[0]), kqg, ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) + iq2xsq8_tile_gen(addr(ytile[0]), addr(panel[0]), ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) } elif (fmt == 23) { iq2sq8_tile_gen(addr(ytile[0]), addr(panel[0]), ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) } else { From 9356439c4f94247e198eaf42a2115e1798a83a26 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Mon, 31 Aug 2026 11:12:40 -0700 Subject: [PATCH 41/61] llvm_tune: --tune-only - the one-family re-mint The iquant arc doubled the tuner's family list, and the whole-scope walk crossed 20 minutes per app sidecar - every new format's landing paid it once per app (run, parity, lcpp_bench each root their own). The filter cuts a re-mint to one family: ~30 s of racing, ~3 min end to end. Framework: `--tune-only` on the app's command line (comma-separated tokens, substring match, implies --tune) rides the tuner process chain as DAS_TUNE_ONLY (declared in the llvm_env registry), is stripped from the re-exec, and surfaces as tune_only_filter() / tune_family_selected(name) for harnesses. dasLLAMA halves: the gen half races only matching families (q8q8 brings its e2e confirm; its inline block became q8q8_tune_family(), matching its kq/q51 siblings), the kernels half prunes the bench list and self-skips outright on no match. A skipped family writes no entry, so its sidecar seat survives the upsert; a filtered mint also passes a null runtime (hand-edited TB/L2 knobs survive) and sits out the Metal tensor race. Proof on the IQ2_XS vehicle: --tune-only iq2xsq8_tile_gen races 1 of 14 generator families, skips the loop-hint half, re-crowns dot_maddubs_width256_mr8; all 48 sidecar entries and the runtime section byte-identical after (only the iq2xs race table refreshed). 183 s end to end warm vs the 20-minute walk. Gates: llvm_tune_scope 1/1, llvm_tune_manifest 2/2, llvm_env_registry 12/12, lint 0 on the five files. Docs: skills/tune.md, ENVIRONMENT.md (regenerated), the HOW_TO QUIRK 15 recipe now spells re-mints --tune-only; the IQ2_XS Phase B note rides along (bench: pp512 2.78x, tg128 0.70x vs clean-cpu, parity 64/64 held stamped). Co-Authored-By: Claude Fable 5 --- modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md | 24 ++- modules/dasLLAMA/harness/dasllama_tuner.das | 3 + modules/dasLLAMA/harness/gen_tune_probe.das | 169 ++++++++++++-------- modules/dasLLAMA/harness/tune_kernels.das | 71 +++++--- modules/dasLLVM/ENVIRONMENT.md | 1 + modules/dasLLVM/daslib/llvm_env.das | 4 + modules/dasLLVM/daslib/llvm_tune.das | 48 +++++- skills/tune.md | 10 ++ 8 files changed, 237 insertions(+), 93 deletions(-) diff --git a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md index 16a5ffeae9..40ff3a5831 100644 --- a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md +++ b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md @@ -339,8 +339,12 @@ where, why it is so today, what unquirked looks like. An empty ledger is a legit the original body. Sidecar staleness keys on the binary's mtime, and the emitter is `.das` (JIT-compiled), so landing it invalidates nothing: the next run logs the same `27 tune-stamped`, serves the reference body, and its text and t/s match the pre-emitter run - exactly - a "the emitter changed nothing" reading that is false. Re-mint with `-- --tune` - on the app (a whole-scope re-tune) or delete the sidecar. Unquirked: fold the family's + exactly - a "the emitter changed nothing" reading that is false. Re-mint with + `-- --tune-only q8_tile_gen` on the app (one family races, every other seat and the + runtime knobs survive the upsert; ~3 min end to end vs the 20-minute whole-scope walk the + doubled family list costs now - and each app roots its OWN sidecar, so run/parity/ + lcpp_bench each re-mint on first touch of a new family; bare `--tune` stays the + whole-scope form, skills/tune.md has the flag) or delete the sidecar. Unquirked: fold the family's generator hash (the JIT DLL cache key already carries it) into the sidecar identity, so a generator change reads as stale. 12. **A fresh worktree has no JIT until `lib/LLVM.dll` is staged - on every platform.** The @@ -481,6 +485,22 @@ test_kqformat 18/18, test_kquant 250 (232 pass, 18 env skips), probe GEN TUNE TE lint 0. E2e: ids 64/64 vs llama.cpp - the arc's FIRST full-match greedy stream, no fork anywhere in the window - gen 23 t/s reference bodies. JIT emitter, Vulkan, Metal: pending. +Phase B (JIT emitter, 2026-08-31): iq2xs rides the fmt-23 panel route wholesale - the +kq_grp_row_dot_b 23 arms extend `|| fmt == 24l` (signed panel bytes, per-16 UNSIGNED +strips, d/8 fold), the tile drops off the packed lists and reads the byte-expanded panel +via `unpack_iq2xs_panel_grp` (each u16 = a 9-bit iq2xs_grid[512] index doubled into the +u64 low/high word pair + a 7-bit ksigns byte through the shared smask negate), and the +gemv gathers per superblock via `emit_iq2xs_gather` off `iq2xs_emit_globals`' [1024 x i32] +private grid plus the SHARED iq3xxs ksigns / iq3s smask globals. kfxs24 TEST rows return +(QUIRK 16 closed). Probe: 11/11 k24 perms ok (maddubs 5.2e-7); the tuner crowns +dot_maddubs_width256_mr8. QUIRK 15 reran as written, cleared with a whole-scope +`run.das -- --tune` - the 20-minute walk that motivated `--tune-only` (the QUIRK 15 +recipe above carries the new spelling). Gates: test_kquant -jit 246 pass / 4 skips, interp +232/18, GEN TUNE TEST OK, lint 0. E2e stamped: parity ids 64/64 vs llama.cpp - Phase A's +first-ever full-match stream HOLDS under the stamped folds; gen 23 -> 54 t/s. zen2 16t vs +clean-cpu: pp512 486.4 vs 174.8 (2.78x), tg128 59.4 vs 84.6 (0.70x - the ledgered +grid-format CPU-decode tail, the iq2s class). Vulkan, Metal: pending. + ### IQ2_S Phase A (CPU, 2026-08-31) - the u64-grid tier Shape: 256-superblock grid format, the first with a u64 grid - a 10-bit index (qs byte | diff --git a/modules/dasLLAMA/harness/dasllama_tuner.das b/modules/dasLLAMA/harness/dasllama_tuner.das index 59cf4452f7..84aae07f4a 100644 --- a/modules/dasLLAMA/harness/dasllama_tuner.das +++ b/modules/dasLLAMA/harness/dasllama_tuner.das @@ -178,6 +178,9 @@ def main : int { if (tune_noise_override()) { print("dasllama_tuner: OVERRIDE DAS_TUNE_NOISE_OVERRIDE - a busy-box refusal mints anyway (sidecar stamped 'noise: overridden')\n") } + if (!empty(tune_only_filter())) { + print("dasllama_tuner: FILTER --tune-only '{tune_only_filter()}' - only matching kernel families race; every other sidecar entry survives the upsert\n") + } // pre-mint snapshot: COPY, not rename — the halves UPSERT, and a rename would drop other // scopes' keys from the shared file. The post-mint diff reads it back. let sidecar = g_env_core.tune_manifest diff --git a/modules/dasLLAMA/harness/gen_tune_probe.das b/modules/dasLLAMA/harness/gen_tune_probe.das index 588d99ad10..2ee026de85 100644 --- a/modules/dasLLAMA/harness/gen_tune_probe.das +++ b/modules/dasLLAMA/harness/gen_tune_probe.das @@ -1788,18 +1788,11 @@ def private confirm_winner(winner : string) : string { return fb } -def tune_mode_run : bool { // nolint:STYLE037,STYLE038 — flat family-by-family race rail; gates + deferred flush in call order - let tune_total_t0 = ref_time_ticks() - // refuse before burning minutes measuring noise; every winner below queues into `pending` - // and nothing reaches the sidecar until the end-of-run gate passes - if (!noise_gate("start")) { - return false - } - var pending : table - // q8q8 + four kq formats + q51 + the end-to-end confirm - tune_progress_plan("dasllama_generators", 7) - interrupt_gate("q8q8_tile_gen") - tune_progress_kernel_begin("q8q8_tile_gen", ROUNDS) +// The q8q8 family race, verbatim the old inline tune_mode_run body: the fat-batch tile bench +// picks the tile knobs (and with them the layout), the hot decode bench picks gkstep among the +// rows sharing them, incumbent stickiness last. Returns the merged winner's suffix; "" means a +// correctness gate failed and the tune must abort (the mismatch is printed inside). +def private q8q8_tune_family : string { // nolint:STYLE037,STYLE038 - the q8q8 race rail, one flat pass // batch shape: fat STREAMING ffn like the kq fixture — the old 2048x512x64 kv-projection was // L2-hot; divergent crowns still pass the e2e confirm gate before shipping var fx <- build_fixture(2048l, 8192l, 256l) @@ -1913,13 +1906,13 @@ def tune_mode_run : bool { // nolint:STYLE037,STYLE038 — flat family-by-family var maxdiff = 0.0 if (!close_to_ref(fx, y, maxdiff)) { print("{vs[vi]._0}: tile MISMATCH (maxdiff {maxdiff}) - aborting tune\n") - return false + return "" } run_gemv(gtab[vs[vi]._0], gfx, gbufs[vi].q, gbufs[vi].s, vmr[vi], yg) var gmaxdiff = 0.0 if (!close_to_ref(gfx, yg, gmaxdiff)) { print("{vs[vi]._0}: gemv MISMATCH (maxdiff {gmaxdiff}) - aborting tune\n") - return false + return "" } } // the hot shape runs HOT_REPS back-to-back per timing sample: 1MB of weights stays @@ -1992,18 +1985,78 @@ def tune_mode_run : bool { // nolint:STYLE037,STYLE038 — flat family-by-family merged = gen_sticky("q8q8_tile_gen", vnames, incSameTile ? hbest : best, merged) delete vnames delete keep + let w = vs[merged]._0 + delete y + delete yg + delete yh + delete bufs + delete gbufs + delete hbufs + delete vmr + delete vts + delete vgs + delete best + delete gbest + delete hbest + delete gtab + delete ctab + delete tss + delete kgs + delete wbs + delete mrs + delete vs + delete fx + delete gfx + delete hfx + return w +} + +def tune_mode_run : bool { // nolint:STYLE037,STYLE038 — flat family-by-family race rail; gates + deferred flush in call order + let tune_total_t0 = ref_time_ticks() + // the --tune-only re-mint filter: only matching families race (q8q8 brings its e2e + // confirm along); a skipped family writes no entry, so its previous seat survives the + // upsert - a one-family re-mint after landing a format is seconds, not the full walk + var kqSel <- [for (fmt in fixed_array(4l, 5l, 6l, 40l, 44l, 3l, 33l, 34l, 45l, 2l, 23l, 24l)); fmt; where tune_family_selected(kq_tile_entry(fmt))] + let wantQ8 = tune_family_selected("q8q8_tile_gen") + let wantQ51 = tune_family_selected("q51q8_tile_gen") + let planTotal = (wantQ8 ? 2 : 0) + length(kqSel) + (wantQ51 ? 1 : 0) + if (planTotal == 0) { + print("tune-only '{tune_only_filter()}': no generator family matches - half skipped, sidecar untouched\n") + delete kqSel + return true + } + if (!empty(tune_only_filter())) { + tune_detail("tune-only '{tune_only_filter()}': racing {planTotal - (wantQ8 ? 1 : 0)} of 14 generator families\n") + } + // refuse before burning minutes measuring noise; every winner below queues into `pending` + // and nothing reaches the sidecar until the end-of-run gate passes + if (!noise_gate("start")) { + return false + } + var pending : table + tune_progress_plan("dasllama_generators", planTotal) + var q8winner = "" + if (wantQ8) { + interrupt_gate("q8q8_tile_gen") + let q8_t0 = ref_time_ticks() + tune_progress_kernel_begin("q8q8_tile_gen", ROUNDS) + q8winner = q8q8_tune_family() + tune_detail("TUNE_GEN_TIME q8q8_family {get_time_usec(q8_t0) / 1000} ms\n") + tune_progress_kernel_end("q8q8_tile_gen", q8winner, "beats") + if (empty(q8winner)) { + return false // a tile/gemv correctness gate failed - the mismatch is named above + } + } + if (!noise_gate("mid1")) { + print("aborting mid-run - no sidecar written\n") + return false + } // the kq families tune separately — one manifest entry per format, tile-best wins (the // gemv is nrsplit-independent, so same-mr rows share its plane and kernel shape). No // child-process confirm gate here: the kq stamp only moves the kq planes' interleave, // and the e2e exposure (a tile-optimal mr shaving decode kernel rate) is bounded by the // decode path staying DRAM-bound — validated end-to-end when the entries first landed. - tune_detail("TUNE_GEN_TIME q8q8_family {get_time_usec(tune_total_t0) / 1000} ms\n") - tune_progress_kernel_end("q8q8_tile_gen", vs[merged]._0, "beats") - if (!noise_gate("mid1")) { - print("aborting mid-run - no sidecar written\n") - return false - } - for (fmt in fixed_array(4l, 5l, 6l, 40l, 44l, 3l, 33l, 34l, 45l, 2l, 23l, 24l)) { + for (fmt in kqSel) { interrupt_gate("k{fmt}q8_tile_gen") let kq_t0 = ref_time_ticks() let entry = kq_tile_entry(fmt) @@ -2022,30 +2075,35 @@ def tune_mode_run : bool { // nolint:STYLE037,STYLE038 — flat family-by-family print("aborting mid-run - no sidecar written\n") return false } - interrupt_gate("q51q8_tile_gen") - let q51_t0 = ref_time_ticks() - // the manifest key must match the [tune] fn (the TILE) or the stamp misses its own crown; - // the race still benches the gemv shape - tune_progress_kernel_begin("q51q8_tile_gen", ROUNDS) - var qrows : table - let qw = q51_tune_family(qrows) - tune_detail("TUNE_GEN_TIME q51_family {get_time_usec(q51_t0) / 1000} ms\n") - tune_progress_kernel_end("q51q8_tile_gen", qw, empty(qw) ? "rejected" : "beats") - // same gate-failure fallback as the kq loop: complete the scope on the reference body - pending["q51q8_tile_gen"] = empty(qw) ? "reference" : qw - g_rows["q51q8_tile_gen"] = JV(qrows) - print("q51q8_tile_gen winner: {empty(qw) ? "reference (gate-failure fallback)" : qw}\n") - interrupt_gate("confirm_e2e_prefill") - let confirm_t0 = ref_time_ticks() - tune_progress_kernel_begin("confirm_e2e_prefill", 0) - let cw = confirm_winner(vs[merged]._0) - var ok = !empty(cw) - if (ok) { - pending["q8q8_tile_gen"] = cw - print("q8q8_tile_gen winner: {cw}\n") + if (wantQ51) { + interrupt_gate("q51q8_tile_gen") + let q51_t0 = ref_time_ticks() + // the manifest key must match the [tune] fn (the TILE) or the stamp misses its own crown; + // the race still benches the gemv shape + tune_progress_kernel_begin("q51q8_tile_gen", ROUNDS) + var qrows : table + let qw = q51_tune_family(qrows) + tune_detail("TUNE_GEN_TIME q51_family {get_time_usec(q51_t0) / 1000} ms\n") + tune_progress_kernel_end("q51q8_tile_gen", qw, empty(qw) ? "rejected" : "beats") + // same gate-failure fallback as the kq loop: complete the scope on the reference body + pending["q51q8_tile_gen"] = empty(qw) ? "reference" : qw + g_rows["q51q8_tile_gen"] = JV(qrows) + print("q51q8_tile_gen winner: {empty(qw) ? "reference (gate-failure fallback)" : qw}\n") + } + var ok = true + if (wantQ8) { + interrupt_gate("confirm_e2e_prefill") + let confirm_t0 = ref_time_ticks() + tune_progress_kernel_begin("confirm_e2e_prefill", 0) + let cw = confirm_winner(q8winner) + ok = !empty(cw) + if (ok) { + pending["q8q8_tile_gen"] = cw + print("q8q8_tile_gen winner: {cw}\n") + } + tune_detail("TUNE_GEN_TIME confirm {get_time_usec(confirm_t0) / 1000} ms\n") + tune_progress_kernel_end("confirm_e2e_prefill", "", ok ? "holds" : "rejected") } - tune_detail("TUNE_GEN_TIME confirm {get_time_usec(confirm_t0) / 1000} ms\n") - tune_progress_kernel_end("confirm_e2e_prefill", "", ok ? "holds" : "rejected") tune_progress_finish() // the only write, behind the last gate: a noisy end-probe means NO entry is written — the // winners above were raced in conditions the probe just proved untrustworthy @@ -2071,28 +2129,7 @@ def tune_mode_run : bool { // nolint:STYLE037,STYLE038 — flat family-by-family } delete pending tune_detail("TUNE_GEN_TIME total {get_time_usec(tune_total_t0) / 1000} ms\n") - delete y - delete yg - delete yh - delete bufs - delete gbufs - delete hbufs - delete vmr - delete vts - delete vgs - delete best - delete gbest - delete hbest - delete gtab - delete ctab - delete tss - delete kgs - delete wbs - delete mrs - delete vs - delete fx - delete gfx - delete hfx + delete kqSel return ok } diff --git a/modules/dasLLAMA/harness/tune_kernels.das b/modules/dasLLAMA/harness/tune_kernels.das index 67f19c63fa..b91d8b776f 100644 --- a/modules/dasLLAMA/harness/tune_kernels.das +++ b/modules/dasLLAMA/harness/tune_kernels.das @@ -2343,6 +2343,39 @@ def main : int { // nolint:STYLE037,STYLE038 — flat one-bench-per-kernel rail; if (tuner_paranoid_requested()) { tune_detail("kernel tuner: --tune-paranoid is retired - one protocol, margin-decided; running it\n") } + // every bench by name, declared up front so the --tune-only filter can prune it before + // the job queue even spins up; the sweep runs the list in order, validation re-runs the + // changed ones + var benches <- [ + ("dot", @@bench_dot), ("axpy", @@bench_axpy), ("dot_f16", @@bench_dot_f16), + ("axpy_f16", @@bench_axpy_f16), ("cvt_f32_to_f16", @@bench_cvt_f32_to_f16), + ("cvt_f16_to_f32", @@bench_cvt_f16_to_f32), ("add_inplace", @@bench_add_inplace), + ("mul_inplace", @@bench_mul_inplace), ("scale_inplace", @@bench_scale), + ("copy_floats", @@bench_copy_floats), ("softmax", @@bench_softmax), ("rmsnorm", @@bench_rmsnorm), + ("dot_q4", @@bench_dot_q4), ("dot_q8q8", @@bench_dot_q8q8), ("dot_q8kv", @@bench_dot_q8kv), + ("dot_q8q8kv", @@bench_dot_q8q8kv), ("axpy_q8kv", @@bench_axpy_q8kv), + ("cvt_q8kv_to_f32", @@bench_cvt_q8kv_to_f32), ("quantize_q8kv_row", @@bench_quantize_q8kv_row), + ("dot_mx4q8", @@bench_dot_mx4q8), ("quantize_q8_0_into_ptr", @@bench_quantize), + ("quantize_q8_0_bs_into_ptr", @@bench_quantize_bs), ("rope_scaled_neox_tab", @@bench_rope_tab), + ("gemm_f32_uk_4x16", @@bench_gemm_tile), + ("dot_q8q8_laneq4x4", @@bench_laneq4x4) // last: it pins the repack backend (sticks until process exit) + ] + let filtered = !empty(tune_only_filter()) + if (filtered) { + let nb = length(benches) + for (k in range(nb)) { + let i = nb - 1 - k + if (!tune_family_selected(benches[i]._0)) { + benches |> erase(i) + } + } + if (empty(benches)) { + print("tune-only '{tune_only_filter()}': no loop-hint kernel matches - half skipped, sidecar untouched\n") + delete benches + return 0 + } + tune_detail("tune-only '{tune_only_filter()}': racing {length(benches)} loop-hint kernel(s); the Metal tensor race and the runtime knob snapshot sit out (previous sidecar values survive)\n") + } let affinity_inherited = is_some(g_env_core.jobque_affinity) if (!affinity_inherited) { set_jobque_affinity(2) // JobQue pins its creator (this measurement thread) to physical CPU 0 @@ -2357,24 +2390,9 @@ def main : int { // nolint:STYLE037,STYLE038 — flat one-bench-per-kernel rail; if (!noise_gate("start")) { return 1 } - tune_progress_plan("dasllama_kernels", TUNED_KERNEL_COUNT + VALIDATE_PLAN_ALLOWANCE) + tune_progress_plan("dasllama_kernels", length(benches) + VALIDATE_PLAN_ALLOWANCE) load_incumbents() - // every bench by name: the sweep runs the list in order, validation re-runs the changed ones - var benches <- [ - ("dot", @@bench_dot), ("axpy", @@bench_axpy), ("dot_f16", @@bench_dot_f16), - ("axpy_f16", @@bench_axpy_f16), ("cvt_f32_to_f16", @@bench_cvt_f32_to_f16), - ("cvt_f16_to_f32", @@bench_cvt_f16_to_f32), ("add_inplace", @@bench_add_inplace), - ("mul_inplace", @@bench_mul_inplace), ("scale_inplace", @@bench_scale), - ("copy_floats", @@bench_copy_floats), ("softmax", @@bench_softmax), ("rmsnorm", @@bench_rmsnorm), - ("dot_q4", @@bench_dot_q4), ("dot_q8q8", @@bench_dot_q8q8), ("dot_q8kv", @@bench_dot_q8kv), - ("dot_q8q8kv", @@bench_dot_q8q8kv), ("axpy_q8kv", @@bench_axpy_q8kv), - ("cvt_q8kv_to_f32", @@bench_cvt_q8kv_to_f32), ("quantize_q8kv_row", @@bench_quantize_q8kv_row), - ("dot_mx4q8", @@bench_dot_mx4q8), ("quantize_q8_0_into_ptr", @@bench_quantize), - ("quantize_q8_0_bs_into_ptr", @@bench_quantize_bs), ("rope_scaled_neox_tab", @@bench_rope_tab), - ("gemm_f32_uk_4x16", @@bench_gemm_tile), - ("dot_q8q8_laneq4x4", @@bench_laneq4x4) // last: it pins the repack backend (sticks until process exit) - ] for (b in benches) { let kernel_t0 = kernel_begin(b._0) invoke(b._1) @@ -2424,7 +2442,7 @@ def main : int { // nolint:STYLE037,STYLE038 — flat one-bench-per-kernel rail; ? "\nevery kernel keeps its shipped fallback - the tuner found nothing new on this box.\n" : "\n{changed} kernel(s) beat their shipped fallback on this box.\n") // the one count still written by hand; say so rather than let the bar quietly misreport - if (length(g_results) != TUNED_KERNEL_COUNT) { + if (!filtered && length(g_results) != TUNED_KERNEL_COUNT) { print("note: TUNED_KERNEL_COUNT is {TUNED_KERNEL_COUNT} but {length(g_results)} kernels ran - the progress bar was scaled wrong; update the constant\n") } @@ -2435,8 +2453,12 @@ def main : int { // nolint:STYLE037,STYLE038 — flat one-bench-per-kernel rail; static_if (typeinfo builtin_module_exists(das_metal)) { var mt_wins : array var mt_ran = false - var mt_all <- metal_tensor_race() - var mt_dec <- metal_tensor_race_decode() + var mt_all : array + var mt_dec : array + if (!filtered) { // a filtered mint leaves "runtime" untouched - no crowns to record + mt_all <- metal_tensor_race() + mt_dec <- metal_tensor_race_decode() + } mt_all |> reserve(length(mt_all) + length(mt_dec)) for (r in mt_dec) { mt_all |> emplace(r) @@ -2450,7 +2472,7 @@ def main : int { // nolint:STYLE037,STYLE038 — flat one-bench-per-kernel rail; mt_wins |> push(r.family) } } - if (!mt_ran) { + if (!mt_ran && !filtered) { tune_detail("METAL_TWIN: no Metal device - tensor race skipped\n") } metal_crowns = join(mt_wins, ",") @@ -2480,6 +2502,12 @@ def main : int { // nolint:STYLE037,STYLE038 — flat one-bench-per-kernel rail; threads = get_total_hw_jobs() + 1, metal_tensor = metal_crowns, metal_tall_floor = metal_tall_floor(), metal_devw_small_panel_mb = metal_devw_small_panel_mb(), metal_cvt_min_rows = metal_cvt_min_rows())) // total lanes (workers + computing main), the DAS_JOBQUE_THREADS unit + if (filtered) { + // a filtered mint must not clobber the runtime knobs (hand-edited TB/L2 refinements + // live there): a null runtime leaves the sidecar section untouched + delete_json(runtime) + runtime = null + } // winners come from the results table (validation may have demoted one); the [tuned] // kernels this harness doesn't sweep yet keep their SHIPPED fallback as an explicit entry - // the scope completeness check demands every non-pinned [tuned] kernel @@ -2489,6 +2517,9 @@ def main : int { // nolint:STYLE037,STYLE038 — flat one-bench-per-kernel rail; "dot_q8tq4kv" => "vec16", "quantize_tq4kv_row" => "plain", "dot_q8q8_f16s" => "vec16", "dot_q51e" => "vec16" } + if (filtered) { // a filtered mint seeds nothing - only the raced kernels write + winners |> clear() + } for (r in g_results) { winners[r.kernel] = r.winner if (r.kernel == "softmax") { diff --git a/modules/dasLLVM/ENVIRONMENT.md b/modules/dasLLVM/ENVIRONMENT.md index c50bbaa048..08d83958c1 100644 --- a/modules/dasLLVM/ENVIRONMENT.md +++ b/modules/dasLLVM/ENVIRONMENT.md @@ -37,6 +37,7 @@ Loaded once at context init into `g_env_tune`; tuner children inherit the enviro | `DAS_TUNE_VERBOSITY` | text | normal | What a tune shows: silent, normal, or verbose. Anything unrecognized reads as normal, so a typo never silences a tune. | | `DAS_TUNE_POLICY` | text | declared by [tune_policy] | Override the missing-scope policy: fallback, warn, error, auto, or restart. The announce line says when the environment shaped the policy. | | `DAS_TUNE_CONTROL` | path | unset | A supervisor's stop channel: while the named file exists, tune_interrupt_requested() is true and tuners abort at the next kernel-family boundary without minting. The watchdog sets it and owns the file's lifetime; the measurement in flight always completes. | +| `DAS_TUNE_ONLY` | text | unset (every family races) | Comma-separated re-mint filter: a tuner races only kernel families whose name contains one of these tokens, and every skipped family's sidecar entry survives the upsert. Set by --tune-only on the application; the tuner children inherit it. | ## Ambient variables dasLLVM reads but does not own diff --git a/modules/dasLLVM/daslib/llvm_env.das b/modules/dasLLVM/daslib/llvm_env.das index 46f610d7b2..1b981e4501 100644 --- a/modules/dasLLVM/daslib/llvm_env.das +++ b/modules/dasLLVM/daslib/llvm_env.das @@ -73,6 +73,10 @@ struct public TuneEnv { @clarg_path @clarg_doc = "A supervisor's stop channel: while the named file exists, tune_interrupt_requested() is true and tuners abort at the next kernel-family boundary without minting. The watchdog sets it and owns the file's lifetime; the measurement in flight always completes." tune_control : string = "" + + @clarg_default_doc = "unset (every family races)" + @clarg_doc = "Comma-separated re-mint filter: a tuner races only kernel families whose name contains one of these tokens, and every skipped family's sidecar entry survives the upsert. Set by --tune-only on the application; the tuner children inherit it." + tune_only : string = "" } let public g_env_tune = env_config(type) diff --git a/modules/dasLLVM/daslib/llvm_tune.das b/modules/dasLLVM/daslib/llvm_tune.das index 9bae3b91f8..bcd3d48738 100644 --- a/modules/dasLLVM/daslib/llvm_tune.das +++ b/modules/dasLLVM/daslib/llvm_tune.das @@ -1601,8 +1601,8 @@ class private TuneScopeAnnotation : AstStructureAnnotation { } // The tune flags, parsed off the APP's command line — we share an argv we do not own, so this -// is the two-arg parse_args form (clargs skips unknown flag-shaped tokens). Keep every field a -// bool and never read positionals: an app's own `--model foo.gguf` leaves `foo.gguf` in there. +// is the two-arg parse_args form (clargs skips unknown flag-shaped tokens). Never read +// positionals - an app's own `--model foo.gguf` leaves `foo.gguf` in there. [CommandLineArgs] struct private TuneCliArgs { @clarg_doc = "Re-tune this box's kernels before the app runs" @@ -1615,6 +1615,10 @@ struct private TuneCliArgs { @clarg_name = "tune-verbose" @clarg_doc = "Tune with the full per-variant output instead of a progress display" tune_verbose : bool + + @clarg_name = "tune-only" + @clarg_doc = "Comma-separated name filter: re-tune only kernel families whose name contains one of these tokens (implies --tune); every other sidecar entry survives" + tune_only : string } // Adopts --tune-quiet/--tune-verbose and answers whether --tune was passed. Runs at MACRO @@ -1631,7 +1635,10 @@ def private apply_cli_tune_flags() : bool { } elif (cfg.tune_verbose) { tune_set_verbosity("verbose") } - return cfg.tune + if (!empty(cfg.tune_only)) { + set_env_variable("DAS_TUNE_ONLY", cfg.tune_only) // the tuner children inherit it + } + return cfg.tune || !empty(cfg.tune_only) } // the exact command a user (or the auto path) runs to tune one scope; the env prefix leads @@ -1988,10 +1995,18 @@ def tune_auto_reexec() : bool { var argv <- get_command_line_arguments() var args : array args |> reserve(length(argv)) + var skipValue = false for (a in argv) { - if (a != "--tune") { - args |> push("{a}") + if (skipValue) { // the value token of a `--tune-only ` pair + skipValue = false + continue + } + continue if (a == "--tune" || starts_with(a, "--tune-only=")) + if (a == "--tune-only") { + skipValue = true + continue } + args |> push("{a}") } print("llvm_tune: re-launching to apply the fresh manifests\n") var rc : int @@ -2049,6 +2064,29 @@ def tune_interrupt_requested() : bool { return stat(p, fs) } + //! The `--tune-only` re-mint filter (`DAS_TUNE_ONLY`), verbatim: comma-separated name + //! tokens, empty = race everything. +def tune_only_filter() : string { + return g_env_tune.tune_only +} + + //! True when family `name` should race under the `--tune-only` filter - no filter set, or + //! some comma-separated token is a substring of the name. A harness consults this at each + //! family boundary; a skipped family writes no entry, so its sidecar seat survives. +def tune_family_selected(name : string) : bool { + let f = g_env_tune.tune_only + if (empty(f)) { + return true + } + for (tok in split(f, ",")) { + let t = strip(tok) + if (!empty(t) && find(name, t) >= 0) { + return true + } + } + return false +} + //! Runtime half of `[tune_policy(missing = "restart")]`, injected as a guard at the top of //! the app's `main`: consults the scope resolver first, else runs the tuner; true = main //! returns immediately (winners are compile-time stamps), false = scope complete, carry on. diff --git a/skills/tune.md b/skills/tune.md index 4f67390f57..37e8a6b676 100644 --- a/skills/tune.md +++ b/skills/tune.md @@ -205,6 +205,16 @@ even when the sidecar is complete (a re-tune; the flag is stripped from the re-exec so the child converges). `DAS_TUNE_POLICY` overrides the declared value - `DAS_TUNE_POLICY=fallback` is the CI kill switch. +`--tune-only ` (comma-separated; implies `--tune`) re-tunes only the +kernel families whose name contains one of the tokens - after landing one +family's kernels, `--tune-only iq2xs` re-mints that family in seconds instead +of walking every family the app owns. A skipped family races nothing and +writes nothing, so its sidecar entry survives the upsert. The filter rides +the tuner process chain as `DAS_TUNE_ONLY`, and a harness consults +`tune_family_selected(name)` at each family boundary. On a box with no +complete sidecar a filtered mint leaves the scope incomplete (the startup +warning names the missing kernels) - the first full mint still has to happen. + Two further escapes exist for a run that must not mint. `--jit-opt-level=0` flips the *injected default* to `fallback`, because winners raced under O3 codegen mean nothing at O0; a declared `[tune_policy]` is left alone, and From f2cf3a2c3ebc930d8f2c505740956bad0d52c95f Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Mon, 31 Aug 2026 11:32:45 -0700 Subject: [PATCH 42/61] dasLLAMA: adopt the fast JIT dev loop; ledger the cache-invalidation defects -module-cache + --jit-split-modules=-1 for every correctness-loop run: warm reruns 48 s -> 5.2 s, edit loops 196 s -> 59 s (zen2, lcpp_bench, 99 modules / 7215 fns). Bench rows stay monolith until a split A/B. plans/jit_compile_time.md carries the measurements and the four invalidation items: deser-vs-parse hash divergence (one wasted cold codegen per cache write), chained-prefix obj keys (a repack edit re-emits 72/99 partitions), the all-or-nothing AST cache, and the macro-emitter key hole (QUIRK 21). Co-Authored-By: Claude Fable 5 --- modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md | 9 ++++ plans/jit_compile_time.md | 55 +++++++++++++++++++++++++ 2 files changed, 64 insertions(+) create mode 100644 plans/jit_compile_time.md diff --git a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md index 40ff3a5831..83984c83f7 100644 --- a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md +++ b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md @@ -251,6 +251,15 @@ the MoE GEMV/mul_mm trio for the format - ledgered for the M5 kernel pass. ## 8. End to end +**Dev-loop invocation (adopted 2026-08-31):** every correctness-loop run (probe tests, +parity, run.das spot checks) goes through the fast stack - +`daslang.exe -jit -module-cache .jitted_scripts/module_cache/.dascache .das -- +--jit-split-modules=-1 ...` - warm no-edit reruns drop 48 s -> 5 s, an edit loop 196 s -> +~60 s (zen2, lcpp_bench scale). Bench t/s rows keep the stock monolith invocation (split +loses cross-module inlining); the first run after a cache write pays one cold codegen (the +deser re-key); QUIRK 21 still applies to emitter edits. Numbers, caveats and the +invalidation ledger: `plans/jit_compile_time.md`. + A real file whose every tensor type is now loadable (the header census script in the session scratchpad, or `harness/gguf_dump.das`), through `examples/dasLLAMA/run.das` against `simple_ids.exe` from the llama.cpp reference build for the same prompt; then `test_model_image` diff --git a/plans/jit_compile_time.md b/plans/jit_compile_time.md new file mode 100644 index 0000000000..2da927a036 --- /dev/null +++ b/plans/jit_compile_time.md @@ -0,0 +1,55 @@ +# JIT compile-time program - the fast dev loop and its invalidation ledger + +The iquant arc's session cost is dominated by recompilation, not by racing or by the model: +one zen2 session logged ~20 of 32 hours in front-end + LLVM codegen + full-walk mints. The +mints are fixed (`--tune-only`, skills/tune.md); this file carries the compile-time half: +the measured levers, the adopted dev-loop, and the invalidation defects that cap it. + +## Measurements (2026-08-31, zen2, lcpp_bench: 99 das-modules, 7215 fns, 16 threads) + +| lever | cold | warm | after one mid-chain kernel edit | +|---|---|---|---| +| default (monolith JIT, no AST cache) | 196 s | 48 s | 196 s | +| `--jit-split-modules=-1` (after `--`) | 65 s (codegen 19.2 s vs 155.8 s - 8.1x) | 48 s (codegen 1.1 s) | 65 s (72/99 partitions re-emit) | +| `-module-cache ` (front end 39.7 s -> 3.1 s) | +write | 5.2 s combined | no help - full re-parse + re-write | +| front end alone (parse + infer + interp macros) | 39.7 s | - | 39.7 s | + +The combined warm loop (AST cache + split obj cache) is 5.2 s vs 48 s; the edit loop is +59 s vs 196 s. Most tuner/parity/bench invocations recompile UNEDITED programs (mints, +re-execs, A/B reruns), so the warm 9x is where the session hours went. + +## Adopted dev loop (the kernel workflow) + +``` +daslang.exe -jit -module-cache .jitted_scripts/module_cache/.dascache .das \ + -- --jit-split-modules=-1 +``` + +One cache file per app, under `.jitted_scripts/` (already gitignored). Caveats that stay: + +- **Bench t/s rows run the stock monolith invocation** until an A/B proves split-neutral - + split loses cross-module inlining across partitions, so its artifact is not the shipped one. +- The first cache-backed run re-keys the JIT cache once (ledger item 1) - expect one cold + codegen right after a cache write. +- QUIRK 21 (HOW_TO_ADD_A_FORMAT.md) still applies: macro-side emitter edits are invisible + to every key - `rm -rf .jitted_scripts` after `dasllama_gemm_gen.das` changes (which also + drops the module caches stored under it - correct, the AST changed). + +## Invalidation ledger + +1. **Deser-vs-parse AOT-hash divergence.** A `-module-cache` deserialized AST produces + different per-function AOT hashes than the freshly parsed one, so the first run after a + cache write re-keys the whole JIT DLL/obj cache (one wasted cold codegen); hashes are + stable from then on. Done = the same program hashes identically parsed or deserialized, + witnessed by a cache write followed by a JIT cache HIT. +2. **Split obj keys are a chained prefix fold - one edit re-emits the suffix.** Editing + `dasllama_repack` re-emitted 72 of 99 partitions (18.3 s, nearly full cold): every module + AFTER the edited one in program order re-keys, though their IR is unchanged. Done = + per-module keys derived from the module's own (transitive-interface) hash, so a leaf edit + re-emits ~itself; measured target: an early-chain kernel edit under ~5 s of codegen. +3. **The AST module cache is all-or-nothing under an edit.** The same repack edit paid the + full 39.7 s front end and rewrote the whole 148 MB cache - no per-module reuse of the + unedited prefix. Done = an edit re-parses the edited module + dependents only. +4. **Macro-emitter changes are invisible to every cache key** (QUIRK 21; QUIRK 15's unquirk + note wants the generator hash folded into the sidecar identity too). Done = an emitter + edit invalidates exactly the families it generates. From c8d912b44a29ad848d0ba819e48c2f6fec8f61e0 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Mon, 31 Aug 2026 12:07:02 -0700 Subject: [PATCH 43/61] dasLLAMA: IQ2_XS Phase C - the u64 grid crosses to Vulkan on ksigns-by-parity The iq2s walk with NO extra planes: each u16 qs word carries a 9-bit grid index (TWO staged words out of a 4 KB @workgroup uint[1024]) and a 7-bit KSIGNS_IQ2XS index whose sign byte ksign7 recomputes - the iq3xxs parity trick, so no sign plane and no qh anywhere on the device path; the per-16 UNSIGNED strips and eighth-ed d ride the iq2s spellings verbatim. KqGemvIq2xs, KqBatchIq2xs : KqBatchK6 (stage_w gathers off wq[wsb * 16u + k] - the u32 word index IS k), Iq2xsCm2T over VkIq2xsBlk (the 32 u16 words as 16-bit lanes) on a new gated IQ2XSGRID axis. vk_kq_schema_id 13 -> 24; twelve-format family cells at stride 16 (uint4-ALIGNED, unlike its grid siblings); a parity-fold float witness plus an iq2xsf16 cm2 oracle. Gates: test_vulkan_kernels 84/84 (the three iq2xs cm2 tiles 0-off at 89600 cells each), lint 0. E2e resident: parity ids 64/64 vs llama.cpp - the full-match stream holds on the THIRD tier. 5060 Ti vs llama.cpp b10660 Vulkan -ngl 99: pp512 14023.3 vs 18320.0 (0.77x, above the 0.70x tier class), tg128 188.7 vs 349.9 (0.54x - the grid-format gemv workgroup re-stage amplified on a 1B; pp healthy isolates it to the gemv; ledgered as followup_vulkan `#35` with the lever list). Co-Authored-By: Claude Fable 5 --- modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md | 15 + .../dasllama/dasllama_vulkan_classes.das | 353 ++++++++++++++++++ .../dasllama/dasllama_vulkan_common.das | 1 + .../dasllama/dasllama_vulkan_prefill.das | 17 + modules/dasLLAMA/followup_vulkan.md | 11 + modules/dasLLAMA/tests/_vkd_oracles.das | 84 +++++ .../dasLLAMA/tests/test_vulkan_kernels.das | 150 +++++++- 7 files changed, 623 insertions(+), 8 deletions(-) diff --git a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md index 83984c83f7..284984e590 100644 --- a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md +++ b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md @@ -510,6 +510,21 @@ first-ever full-match stream HOLDS under the stamped folds; gen 23 -> 54 t/s. ze clean-cpu: pp512 486.4 vs 174.8 (2.78x), tg128 59.4 vs 84.6 (0.70x - the ledgered grid-format CPU-decode tail, the iq2s class). Vulkan, Metal: pending. + +Phase C (Vulkan, 2026-08-31): the iq2s walk with ksigns-by-parity and NO extra planes - each +u16 qs word carries its 9-bit grid index (TWO staged words out of a 4 KB `@workgroup +uint[1024]`) and a 7-bit KSIGNS_IQ2XS index whose sign byte `ksign7` recomputes (the iq3xxs +parity trick; no sign plane, no qh), the strips/d epilogue verbatim iq2s. `KqGemvIq2xs`, +`KqBatchIq2xs : KqBatchK6` (stage_w gathers off `wq[wsb * 16u + k]` - the u32 word index IS +k), `Iq2xsCm2T` over `VkIq2xsBlk` (the 32 u16 words as 16-bit lanes) on a new gated +`IQ2XSGRID` axis; schema arm 13 -> 24; twelve-format family cells at stride 16 (uint4-ALIGNED, +unlike iq3s/iq3xxs/iq2s); a parity-fold float witness + an iq2xsf16 cm2 oracle. Gates: +test_vulkan_kernels 84/84 (the three iq2xs cm2 tiles 0-off at 89600 cells each), lint 0. +E2e resident: parity ids 64/64 vs llama.cpp - the full-match stream holds on the THIRD tier. +5060 Ti vs llama.cpp b10660 Vulkan -ngl 99: pp512 14023.3 vs 18320.0 (0.77x - above the +0.70x tier class), tg128 188.7 vs 349.9 (0.54x - the grid-format gemv workgroup re-stage +amplified on a 1B, followup_vulkan #35; pp healthy isolates it to the gemv). Metal: pending. + ### IQ2_S Phase A (CPU, 2026-08-31) - the u64-grid tier Shape: 256-superblock grid format, the first with a u64 grid - a 10-bit index (qs byte | diff --git a/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das b/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das index 2ff271c0d1..6eb51f19bd 100644 --- a/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das +++ b/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das @@ -2540,6 +2540,141 @@ def private iq3s_grid_word(i : int) : uint { return tbl[i] } +// ggml's iq2xs_grid as low/high word pairs - 512 u64 entries, one uint word per call (word +// 2i = magnitudes 0..3 of entry i, word 2i+1 = 4..7); staged into workgroup memory per kernel. +def private iq2xs_grid_word(i : int) : uint { // nolint:STYLE038 - a 1024-word data table, not splittable + let tbl = fixed_array( + 0x08080808, 0x08080808, 0x0808082b, 0x08080808, 0x08081919, 0x08080808, 0x08082b08, 0x08080808, + 0x08082b2b, 0x08080808, 0x08190819, 0x08080808, 0x08191908, 0x08080808, 0x0819192b, 0x08080808, + 0x08192b19, 0x08080808, 0x082b0808, 0x08080808, 0x082b082b, 0x08080808, 0x082b1919, 0x08080808, + 0x082b2b08, 0x08080808, 0x19080819, 0x08080808, 0x19081908, 0x08080808, 0x1908192b, 0x08080808, + 0x19082b19, 0x08080808, 0x19190808, 0x08080808, 0x1919082b, 0x08080808, 0x19191919, 0x08080808, + 0x19192b08, 0x08080808, 0x192b0819, 0x08080808, 0x192b1908, 0x08080808, 0x2b080808, 0x08080808, + 0x2b08082b, 0x08080808, 0x2b081919, 0x08080808, 0x2b082b08, 0x08080808, 0x2b190819, 0x08080808, + 0x2b191908, 0x08080808, 0x2b192b19, 0x08080808, 0x2b2b0808, 0x08080808, 0x08080819, 0x08080819, + 0x08081908, 0x08080819, 0x0808192b, 0x08080819, 0x08082b19, 0x08080819, 0x08190808, 0x08080819, + 0x0819082b, 0x08080819, 0x08191919, 0x08080819, 0x08192b08, 0x08080819, 0x08192b2b, 0x08080819, + 0x082b0819, 0x08080819, 0x082b1908, 0x08080819, 0x19080808, 0x08080819, 0x1908082b, 0x08080819, + 0x19081919, 0x08080819, 0x19082b08, 0x08080819, 0x19190819, 0x08080819, 0x19191908, 0x08080819, + 0x192b0808, 0x08080819, 0x192b2b08, 0x08080819, 0x2b080819, 0x08080819, 0x2b081908, 0x08080819, + 0x2b190808, 0x08080819, 0x08080808, 0x0808082b, 0x0808082b, 0x0808082b, 0x08081919, 0x0808082b, + 0x08082b08, 0x0808082b, 0x08190819, 0x0808082b, 0x08191908, 0x0808082b, 0x082b0808, 0x0808082b, + 0x19080819, 0x0808082b, 0x19081908, 0x0808082b, 0x19190808, 0x0808082b, 0x19191919, 0x0808082b, + 0x2b080808, 0x0808082b, 0x2b082b2b, 0x0808082b, 0x08080819, 0x08081908, 0x08081908, 0x08081908, + 0x0808192b, 0x08081908, 0x08082b19, 0x08081908, 0x08190808, 0x08081908, 0x0819082b, 0x08081908, + 0x08191919, 0x08081908, 0x08192b08, 0x08081908, 0x082b0819, 0x08081908, 0x082b1908, 0x08081908, + 0x19080808, 0x08081908, 0x1908082b, 0x08081908, 0x19081919, 0x08081908, 0x19082b08, 0x08081908, + 0x19190819, 0x08081908, 0x19191908, 0x08081908, 0x1919192b, 0x08081908, 0x192b0808, 0x08081908, + 0x2b080819, 0x08081908, 0x2b081908, 0x08081908, 0x2b190808, 0x08081908, 0x08080808, 0x08081919, + 0x0808082b, 0x08081919, 0x08081919, 0x08081919, 0x08082b08, 0x08081919, 0x08190819, 0x08081919, + 0x08191908, 0x08081919, 0x082b0808, 0x08081919, 0x19080819, 0x08081919, 0x19081908, 0x08081919, + 0x19190808, 0x08081919, 0x192b0819, 0x08081919, 0x2b080808, 0x08081919, 0x08080819, 0x0808192b, + 0x08081908, 0x0808192b, 0x08190808, 0x0808192b, 0x082b192b, 0x0808192b, 0x19080808, 0x0808192b, + 0x1908082b, 0x0808192b, 0x2b081908, 0x0808192b, 0x08080808, 0x08082b08, 0x0808082b, 0x08082b08, + 0x08081919, 0x08082b08, 0x08082b08, 0x08082b08, 0x08082b2b, 0x08082b08, 0x08190819, 0x08082b08, + 0x08191908, 0x08082b08, 0x082b0808, 0x08082b08, 0x082b1919, 0x08082b08, 0x19080819, 0x08082b08, + 0x19081908, 0x08082b08, 0x19190808, 0x08082b08, 0x19192b08, 0x08082b08, 0x2b080808, 0x08082b08, + 0x2b2b0808, 0x08082b08, 0x2b2b2b2b, 0x08082b08, 0x08080819, 0x08082b19, 0x08081908, 0x08082b19, + 0x08190808, 0x08082b19, 0x19080808, 0x08082b19, 0x2b080819, 0x08082b19, 0x2b082b19, 0x08082b19, + 0x08080808, 0x08082b2b, 0x082b0808, 0x08082b2b, 0x082b2b08, 0x08082b2b, 0x2b19192b, 0x08082b2b, + 0x2b2b0808, 0x08082b2b, 0x08080819, 0x08190808, 0x08081908, 0x08190808, 0x0808192b, 0x08190808, + 0x08082b19, 0x08190808, 0x08190808, 0x08190808, 0x0819082b, 0x08190808, 0x08191919, 0x08190808, + 0x08192b08, 0x08190808, 0x082b0819, 0x08190808, 0x082b1908, 0x08190808, 0x19080808, 0x08190808, + 0x1908082b, 0x08190808, 0x19081919, 0x08190808, 0x19082b08, 0x08190808, 0x19190819, 0x08190808, + 0x19191908, 0x08190808, 0x192b0808, 0x08190808, 0x192b2b2b, 0x08190808, 0x2b080819, 0x08190808, + 0x2b081908, 0x08190808, 0x2b190808, 0x08190808, 0x08080808, 0x08190819, 0x0808082b, 0x08190819, + 0x08081919, 0x08190819, 0x08082b08, 0x08190819, 0x08190819, 0x08190819, 0x08191908, 0x08190819, + 0x082b0808, 0x08190819, 0x19080819, 0x08190819, 0x19081908, 0x08190819, 0x19190808, 0x08190819, + 0x2b080808, 0x08190819, 0x2b191908, 0x08190819, 0x2b19192b, 0x08190819, 0x08080819, 0x0819082b, + 0x08081908, 0x0819082b, 0x0808192b, 0x0819082b, 0x08190808, 0x0819082b, 0x19080808, 0x0819082b, + 0x192b0808, 0x0819082b, 0x08080808, 0x08191908, 0x0808082b, 0x08191908, 0x08081919, 0x08191908, + 0x08082b08, 0x08191908, 0x08190819, 0x08191908, 0x08191908, 0x08191908, 0x082b0808, 0x08191908, + 0x19080819, 0x08191908, 0x19081908, 0x08191908, 0x19082b19, 0x08191908, 0x19190808, 0x08191908, + 0x192b1908, 0x08191908, 0x2b080808, 0x08191908, 0x08080819, 0x08191919, 0x08081908, 0x08191919, + 0x08190808, 0x08191919, 0x19080808, 0x08191919, 0x08080808, 0x0819192b, 0x08191908, 0x0819192b, + 0x19082b19, 0x0819192b, 0x08080819, 0x08192b08, 0x08081908, 0x08192b08, 0x08190808, 0x08192b08, + 0x0819082b, 0x08192b08, 0x19080808, 0x08192b08, 0x19191908, 0x08192b08, 0x2b08192b, 0x08192b08, + 0x08080808, 0x08192b19, 0x08081919, 0x08192b19, 0x192b192b, 0x08192b19, 0x19190819, 0x08192b2b, + 0x2b2b2b19, 0x08192b2b, 0x08080808, 0x082b0808, 0x0808082b, 0x082b0808, 0x08081919, 0x082b0808, + 0x08082b08, 0x082b0808, 0x08082b2b, 0x082b0808, 0x08190819, 0x082b0808, 0x08191908, 0x082b0808, + 0x082b0808, 0x082b0808, 0x19080819, 0x082b0808, 0x19081908, 0x082b0808, 0x19190808, 0x082b0808, + 0x2b080808, 0x082b0808, 0x2b2b0808, 0x082b0808, 0x08080819, 0x082b0819, 0x08081908, 0x082b0819, + 0x08190808, 0x082b0819, 0x19080808, 0x082b0819, 0x19082b08, 0x082b0819, 0x192b1919, 0x082b0819, + 0x08080808, 0x082b082b, 0x082b082b, 0x082b082b, 0x2b080808, 0x082b082b, 0x2b2b2b08, 0x082b082b, + 0x08080819, 0x082b1908, 0x08081908, 0x082b1908, 0x08190808, 0x082b1908, 0x082b2b19, 0x082b1908, + 0x19080808, 0x082b1908, 0x08080808, 0x082b1919, 0x19080819, 0x082b1919, 0x1919082b, 0x082b1919, + 0x2b192b19, 0x082b1919, 0x08080819, 0x082b192b, 0x08192b2b, 0x082b192b, 0x2b2b192b, 0x082b192b, + 0x08080808, 0x082b2b08, 0x08082b08, 0x082b2b08, 0x08082b2b, 0x082b2b08, 0x082b0808, 0x082b2b08, + 0x19191919, 0x082b2b08, 0x2b082b08, 0x082b2b08, 0x2b2b082b, 0x082b2b08, 0x192b2b08, 0x082b2b19, + 0x2b190808, 0x082b2b19, 0x08082b08, 0x082b2b2b, 0x082b0808, 0x082b2b2b, 0x2b08082b, 0x082b2b2b, + 0x2b082b08, 0x082b2b2b, 0x2b082b2b, 0x082b2b2b, 0x08080819, 0x19080808, 0x08081908, 0x19080808, + 0x0808192b, 0x19080808, 0x08082b19, 0x19080808, 0x08190808, 0x19080808, 0x0819082b, 0x19080808, + 0x08191919, 0x19080808, 0x08192b08, 0x19080808, 0x082b0819, 0x19080808, 0x082b1908, 0x19080808, + 0x19080808, 0x19080808, 0x1908082b, 0x19080808, 0x19081919, 0x19080808, 0x19082b08, 0x19080808, + 0x19082b2b, 0x19080808, 0x19190819, 0x19080808, 0x19191908, 0x19080808, 0x192b0808, 0x19080808, + 0x192b1919, 0x19080808, 0x2b080819, 0x19080808, 0x2b081908, 0x19080808, 0x2b190808, 0x19080808, + 0x08080808, 0x19080819, 0x0808082b, 0x19080819, 0x08081919, 0x19080819, 0x08082b08, 0x19080819, + 0x08190819, 0x19080819, 0x08191908, 0x19080819, 0x082b0808, 0x19080819, 0x19080819, 0x19080819, + 0x19081908, 0x19080819, 0x19190808, 0x19080819, 0x2b080808, 0x19080819, 0x2b081919, 0x19080819, + 0x2b2b082b, 0x19080819, 0x08080819, 0x1908082b, 0x08081908, 0x1908082b, 0x08190808, 0x1908082b, + 0x0819082b, 0x1908082b, 0x082b2b19, 0x1908082b, 0x19080808, 0x1908082b, 0x08080808, 0x19081908, + 0x0808082b, 0x19081908, 0x08081919, 0x19081908, 0x08082b08, 0x19081908, 0x08190819, 0x19081908, + 0x08191908, 0x19081908, 0x08192b19, 0x19081908, 0x082b0808, 0x19081908, 0x19080819, 0x19081908, + 0x19081908, 0x19081908, 0x19190808, 0x19081908, 0x2b080808, 0x19081908, 0x2b191908, 0x19081908, + 0x08080819, 0x19081919, 0x08081908, 0x19081919, 0x08190808, 0x19081919, 0x082b1908, 0x19081919, + 0x19080808, 0x19081919, 0x2b192b2b, 0x19081919, 0x08080808, 0x1908192b, 0x08082b2b, 0x1908192b, + 0x19081908, 0x1908192b, 0x19190808, 0x1908192b, 0x08080819, 0x19082b08, 0x08081908, 0x19082b08, + 0x08190808, 0x19082b08, 0x19080808, 0x19082b08, 0x19081919, 0x19082b08, 0x19191908, 0x19082b08, + 0x192b082b, 0x19082b08, 0x08080808, 0x19082b19, 0x08190819, 0x19082b19, 0x19081908, 0x19082b19, + 0x19190808, 0x19082b19, 0x192b2b19, 0x19082b19, 0x08081908, 0x19082b2b, 0x08080808, 0x19190808, + 0x0808082b, 0x19190808, 0x08081919, 0x19190808, 0x08082b08, 0x19190808, 0x08190819, 0x19190808, + 0x08191908, 0x19190808, 0x082b0808, 0x19190808, 0x082b2b08, 0x19190808, 0x19080819, 0x19190808, + 0x19081908, 0x19190808, 0x19190808, 0x19190808, 0x2b080808, 0x19190808, 0x08080819, 0x19190819, + 0x08081908, 0x19190819, 0x08190808, 0x19190819, 0x08191919, 0x19190819, 0x19080808, 0x19190819, + 0x1908082b, 0x19190819, 0x08080808, 0x1919082b, 0x19081908, 0x1919082b, 0x2b2b2b2b, 0x1919082b, + 0x08080819, 0x19191908, 0x08081908, 0x19191908, 0x08190808, 0x19191908, 0x082b0819, 0x19191908, + 0x19080808, 0x19191908, 0x192b0808, 0x19191908, 0x2b080819, 0x19191908, 0x2b2b0819, 0x19191908, + 0x08080808, 0x19191919, 0x08082b08, 0x19191919, 0x2b080808, 0x19191919, 0x2b082b08, 0x19191919, + 0x082b0819, 0x1919192b, 0x192b2b08, 0x1919192b, 0x2b2b0819, 0x1919192b, 0x08080808, 0x19192b08, + 0x08191908, 0x19192b08, 0x19080819, 0x19192b08, 0x19190808, 0x19192b08, 0x2b192b19, 0x19192b08, + 0x08192b2b, 0x19192b19, 0x19080808, 0x19192b19, 0x1908082b, 0x19192b19, 0x2b081919, 0x19192b2b, + 0x08080819, 0x192b0808, 0x08081908, 0x192b0808, 0x08190808, 0x192b0808, 0x19080808, 0x192b0808, + 0x19191908, 0x192b0808, 0x192b082b, 0x192b0808, 0x2b08192b, 0x192b0808, 0x2b2b2b19, 0x192b0808, + 0x08080808, 0x192b0819, 0x082b1908, 0x192b082b, 0x19082b2b, 0x192b082b, 0x2b19082b, 0x192b082b, + 0x08080808, 0x192b1908, 0x0819192b, 0x192b1908, 0x08190808, 0x192b1919, 0x19080808, 0x192b1919, + 0x19081919, 0x192b1919, 0x2b2b1908, 0x192b1919, 0x08080819, 0x192b2b08, 0x192b2b2b, 0x192b2b08, + 0x082b1919, 0x192b2b19, 0x0808192b, 0x192b2b2b, 0x19191908, 0x192b2b2b, 0x192b082b, 0x192b2b2b, + 0x08080808, 0x2b080808, 0x0808082b, 0x2b080808, 0x08081919, 0x2b080808, 0x08082b08, 0x2b080808, + 0x08190819, 0x2b080808, 0x08191908, 0x2b080808, 0x082b0808, 0x2b080808, 0x082b2b2b, 0x2b080808, + 0x19080819, 0x2b080808, 0x19081908, 0x2b080808, 0x19190808, 0x2b080808, 0x2b080808, 0x2b080808, + 0x2b08082b, 0x2b080808, 0x2b2b2b08, 0x2b080808, 0x2b2b2b2b, 0x2b080808, 0x08080819, 0x2b080819, + 0x08081908, 0x2b080819, 0x0808192b, 0x2b080819, 0x08190808, 0x2b080819, 0x19080808, 0x2b080819, + 0x19190819, 0x2b080819, 0x19192b19, 0x2b080819, 0x08080808, 0x2b08082b, 0x082b0808, 0x2b08082b, + 0x2b080808, 0x2b08082b, 0x2b08082b, 0x2b08082b, 0x2b2b0808, 0x2b08082b, 0x2b2b2b08, 0x2b08082b, + 0x08080819, 0x2b081908, 0x08081908, 0x2b081908, 0x08190808, 0x2b081908, 0x0819082b, 0x2b081908, + 0x08191919, 0x2b081908, 0x19080808, 0x2b081908, 0x192b0808, 0x2b081908, 0x2b082b19, 0x2b081908, + 0x08080808, 0x2b081919, 0x19081908, 0x2b081919, 0x2b2b1919, 0x2b081919, 0x08192b08, 0x2b08192b, + 0x192b2b2b, 0x2b08192b, 0x08080808, 0x2b082b08, 0x08082b08, 0x2b082b08, 0x082b1919, 0x2b082b08, + 0x19192b2b, 0x2b082b08, 0x2b080808, 0x2b082b08, 0x2b08082b, 0x2b082b08, 0x2b2b2b08, 0x2b082b08, + 0x0808192b, 0x2b082b19, 0x082b082b, 0x2b082b2b, 0x2b080808, 0x2b082b2b, 0x2b082b08, 0x2b082b2b, + 0x2b19192b, 0x2b082b2b, 0x2b2b2b08, 0x2b082b2b, 0x08080819, 0x2b190808, 0x08081908, 0x2b190808, + 0x08190808, 0x2b190808, 0x19080808, 0x2b190808, 0x1919192b, 0x2b190808, 0x2b081908, 0x2b190808, + 0x08080808, 0x2b190819, 0x082b082b, 0x2b190819, 0x192b1908, 0x2b190819, 0x1919192b, 0x2b19082b, + 0x2b082b19, 0x2b19082b, 0x08080808, 0x2b191908, 0x08081919, 0x2b191908, 0x19081908, 0x2b191908, + 0x19190808, 0x2b191908, 0x19192b08, 0x2b191908, 0x082b2b19, 0x2b191919, 0x2b190808, 0x2b191919, + 0x2b19082b, 0x2b191919, 0x19080819, 0x2b19192b, 0x19190819, 0x2b192b08, 0x2b2b192b, 0x2b192b08, + 0x19082b19, 0x2b192b19, 0x08191919, 0x2b192b2b, 0x192b0808, 0x2b192b2b, 0x08080808, 0x2b2b0808, + 0x0808082b, 0x2b2b0808, 0x08082b08, 0x2b2b0808, 0x08082b2b, 0x2b2b0808, 0x082b0808, 0x2b2b0808, + 0x082b2b2b, 0x2b2b0808, 0x2b2b0808, 0x2b2b0808, 0x19190819, 0x2b2b0819, 0x19192b19, 0x2b2b0819, + 0x2b2b192b, 0x2b2b0819, 0x08080808, 0x2b2b082b, 0x0808082b, 0x2b2b082b, 0x08082b08, 0x2b2b082b, + 0x082b2b2b, 0x2b2b082b, 0x2b080808, 0x2b2b082b, 0x2b2b0808, 0x2b2b082b, 0x19080808, 0x2b2b1908, + 0x2b191919, 0x2b2b1908, 0x192b1919, 0x2b2b192b, 0x2b192b08, 0x2b2b192b, 0x08082b2b, 0x2b2b2b08, + 0x082b0808, 0x2b2b2b08, 0x082b082b, 0x2b2b2b08, 0x082b2b08, 0x2b2b2b08, 0x2b2b0808, 0x2b2b2b08, + 0x2b2b2b08, 0x2b2b2b08, 0x08081908, 0x2b2b2b19, 0x2b081908, 0x2b2b2b19, 0x2b08192b, 0x2b2b2b19, + 0x082b2b08, 0x2b2b2b2b, 0x082b2b2b, 0x2b2b2b2b, 0x2b190819, 0x2b2b2b2b, 0x2b2b2b2b, 0x2b2b2b2b) + return tbl[i] +} + // ggml's iq2s_grid as low/high word pairs - 1024 u64 entries, one uint word per call (word 2i = // magnitudes 0..3 of entry i, word 2i+1 = 4..7); the staging loops copy it into workgroup // memory once per kernel, so the local-array storage class never sits on a hot path. @@ -3618,6 +3753,96 @@ class KqBatchIq2s : KqBatchK6 { } } +// IQ2_XS tile: the iq2s gather with ksigns-by-parity - each u16 qs word carries a 9-bit grid +// index (two staged words) and a 7-bit KSIGNS_IQ2XS index whose sign byte ksign7 recomputes +// (no sign plane, no qh); strips and fma ride the iq2s spellings. The 4 KB grid stages once. +[vk_dispatch(name = "kq_batch_iq2xs_cls", kernel = "run", family = "kq_batch_cls", grid = "wgs", params = "wgs : int64")] +class KqBatchIq2xs : KqBatchK6 { + @workgroup gridw : uint[1024] + + def iq3s_signed(g, nib : uint) : uint { + let m1 = (nib * 0x00204081u) & 0x01010101u + let mask = m1 * 255u + return (g ^ mask) + m1 + } + + def override stage_w(wsb0, wcol, nsb, s, word, grp : uint) { + for (ki in range(2)) { + let k = word + uint(ki) * 8u + var wv = uint4(0u) + if (wcol < pa.d) { + let wsb = wsb0 + wcol * nsb + s + let qw = wq[wsb * 16u + k] + let wa = qw & 0xFFFFu + let wb = qw >> 16u + let sg0 = ksign7(wa >> 9u) + let sg1 = ksign7(wb >> 9u) + let i0 = (wa & 511u) * 2u + let i1 = (wb & 511u) * 2u + wv = uint4(iq3s_signed(gridw[i0], sg0 & 15u), iq3s_signed(gridw[i0 + 1u], sg0 >> 4u), + iq3s_signed(gridw[i1], sg1 & 15u), iq3s_signed(gridw[i1 + 1u], sg1 >> 4u)) + } + ww4[grp * 17u + k] = wv + } + } + + def override stage_ws(wsb0, wt, nsb, s, tid : uint) { + let slane = tid % 32u + let sblk = tid / 32u + var sa = 0.0 + var sbv = 0.0 + if (wt * 32u + slane < pa.d) { + let wsb = wsb0 + (wt * 32u + slane) * nsb + s + let w = wsu[wsb * 5u + 1u + sblk / 2u] + let s0 = (w >> ((sblk % 2u) * 16u)) & 255u + let s1 = (w >> ((sblk % 2u) * 16u + 8u)) & 255u + let dd = unpackHalf2x16(wsu[wsb * 5u]).x + sa = dd * float(s0) + sbv = dd * float(s1) + } + wsw[sblk * 32u + slane] = sa + wsw2[sblk * 32u + slane] = sbv + } + + // the split-half fma with UNSIGNED strips: xscl * (d*s0*l + d*s1*h) - no offset, no min + def override blk_fma(bu, grp, word : uint; xscl : float) : float4 { + let kL = bu * 2u + let kH = bu * 2u + 1u + let xwL = xw4[grp * 17u + kL] + let xwH = xw4[grp * 17u + kH] + let wL0 = ww4[word * 17u + kL] + let wL1 = ww4[(word + 8u) * 17u + kL] + let wL2 = ww4[(word + 16u) * 17u + kL] + let wL3 = ww4[(word + 24u) * 17u + kL] + let wH0 = ww4[word * 17u + kH] + let wH1 = ww4[(word + 8u) * 17u + kH] + let wH2 = ww4[(word + 16u) * 17u + kH] + let wH3 = ww4[(word + 24u) * 17u + kH] + let l0 = sdot4(wL0.x, xwL.x) + sdot4(wL0.y, xwL.y) + sdot4(wL0.z, xwL.z) + sdot4(wL0.w, xwL.w) + let l1 = sdot4(wL1.x, xwL.x) + sdot4(wL1.y, xwL.y) + sdot4(wL1.z, xwL.z) + sdot4(wL1.w, xwL.w) + let l2 = sdot4(wL2.x, xwL.x) + sdot4(wL2.y, xwL.y) + sdot4(wL2.z, xwL.z) + sdot4(wL2.w, xwL.w) + let l3 = sdot4(wL3.x, xwL.x) + sdot4(wL3.y, xwL.y) + sdot4(wL3.z, xwL.z) + sdot4(wL3.w, xwL.w) + let h0 = sdot4(wH0.x, xwH.x) + sdot4(wH0.y, xwH.y) + sdot4(wH0.z, xwH.z) + sdot4(wH0.w, xwH.w) + let h1 = sdot4(wH1.x, xwH.x) + sdot4(wH1.y, xwH.y) + sdot4(wH1.z, xwH.z) + sdot4(wH1.w, xwH.w) + let h2 = sdot4(wH2.x, xwH.x) + sdot4(wH2.y, xwH.y) + sdot4(wH2.z, xwH.z) + sdot4(wH2.w, xwH.w) + let h3 = sdot4(wH3.x, xwH.x) + sdot4(wH3.y, xwH.y) + sdot4(wH3.z, xwH.z) + sdot4(wH3.w, xwH.w) + return float4(xscl * (wsw[bu * 32u + word] * float(l0) + wsw2[bu * 32u + word] * float(h0)), + xscl * (wsw[bu * 32u + word + 8u] * float(l1) + wsw2[bu * 32u + word + 8u] * float(h1)), + xscl * (wsw[bu * 32u + word + 16u] * float(l2) + wsw2[bu * 32u + word + 16u] * float(h2)), + xscl * (wsw[bu * 32u + word + 24u] * float(l3) + wsw2[bu * 32u + word + 24u] * float(h3))) + } + + [spirv_kernel(local_size_x = 256, name = "kq_batch_iq2xs_cls_spv")] + def override run { + let tid = gl_LocalInvocationID.x + for (k in range(4)) { + gridw[tid + uint(k) * 256u] = iq2xs_grid_word(int(tid + uint(k) * 256u)) + } + barrier() // the iq2xs grid visible before the first stage_w + tile_shell() + } +} + // ===== the prefill batch GEMMs — cooperative-matrix variants (DASLLAMA_COOPMAT=f16|int8|mm) ===== let private CML_SST = 20u // mm shared row stride in uints (16 f16-pair data + 4 pad — bank spread) @@ -4444,6 +4669,10 @@ struct VkIq2sBlk { qh : int16[4] // ... and its 8 qh bytes (two index high bits per (block, l) window) } +struct VkIq2xsBlk { + qs : int16[32] // one IQ2_XS superblock's 32 u16 qs words as 16-bit lanes (9-bit grid index + 7-bit ksigns index each); scales (f16 d8 + 16 unsigned (1 + 2s) strips) live in ws +} + // The cm2 tile: one class template over the two stamp axes. The token column BN - l 256, m 128, // s 32 (the expert-bucket geometry: STILE's fast path loads a partial column UNCLAMPED and clamps // only the store, so every plane it reads carries 32 rows of slack) - and the weight format: BLK @@ -4469,6 +4698,8 @@ class template KqCm2BatchT : MoeCmBase { @workgroup @template_gate = IQ3XGRID iq3x_gridc : uint[256] @template_constant IQ2SGRID : bool = false // stage the 8 KB iq2s u64 grid into workgroup memory @workgroup @template_gate = IQ2SGRID iq2s_gridc : uint[2048] + @template_constant IQ2XSGRID : bool = false // stage the 4 KB iq2xs u64 grid into workgroup memory + @workgroup @template_gate = IQ2XSGRID iq2xs_gridc : uint[1024] [spirv_kernel(local_size_x = 256), arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-tile-pick-and-default")] def run { // nolint:STYLE037,STYLE038 — the fast/edge path pair, barrier- and register-coupled @@ -4520,6 +4751,12 @@ class template KqCm2BatchT : MoeCmBase { iq2s_gridc[g2 + uint(k) * 256u] = iq2s_grid_word(int(g2 + uint(k) * 256u)) } } + static_if (IQ2XSGRID) { + let g2x = gl_LocalInvocationID.x + for (k in range(4)) { + iq2xs_gridc[g2x + uint(k) * 256u] = iq2xs_grid_word(int(g2x + uint(k) * 256u)) + } + } barrier() // wg_blk0 (and the gated codebook) visible before the first decode load var a : coopmatWgA_f16_128x64 var b : BT @@ -4917,6 +5154,32 @@ class template Iq2sCm2T : KqCm2BatchT { } } +class template Iq2xsCm2T : KqCm2BatchT { + typedef BLK = VkIq2xsBlk + typedef ST = uint + override IQ2XSGRID = true + + [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] + def decode(blk : VkIq2xsBlk; bc, cib : uint2) : float16 { + let e = cib.y + let wi = e >> 3u + let j = e & 7u + let w16 = uint(int(blk.qs[int(wi)])) & 0xFFFFu + let sidx = w16 >> 9u + var tt = sidx ^ (sidx >> 4u) + tt = tt ^ (tt >> 2u) + tt = tt ^ (tt >> 1u) + let sgb = sidx | ((tt & 1u) << 7u) + let gw = iq2xs_gridc[(w16 & 511u) * 2u + (j >> 2u)] + let gb = float((gw >> ((j & 3u) * 8u)) & 0xFFu) + let srow = (wg_blk0 + bc.x * (pa.n >> 8u) + bc.y) * 5u + let dv = unpackHalf2x16(ws[srow]).x + let si = e >> 4u + let sc = (ws[srow + 1u + (si >> 2u)] >> ((si & 3u) * 8u)) & 0xFFu + return float16(dv * float(sc) * (((sgb >> j) & 1u) != 0u ? -gb : gb)) + } +} + // the stamps: the three width columns of each format (the dispatcher keys on (fmt, tc)) [vk_dispatch(name = "q8_batch_cm2l_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] class Q8Cm2LBatch : Q8Cm2T { @@ -5242,6 +5505,33 @@ class Iq2sCm2SBatch : Iq2sCm2T { typedef FLO = tensorLayout2DPad } +[vk_dispatch(name = "kq_batch_iq2xs_cm2l_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Iq2xsCm2LBatch : Iq2xsCm2T { + typedef BT = coopmatWgB_f16_64x256 + typedef ACC = coopmatWgAcc_f16_128x256 + typedef ACCW = coopmatWgAcc_f32_128x256 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_iq2xs_cm2m_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Iq2xsCm2MBatch : Iq2xsCm2T { + override BN = 128u + typedef BT = coopmatWgB_f16_64x128 + typedef ACC = coopmatWgAcc_f16_128x128 + typedef ACCW = coopmatWgAcc_f32_128x128 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_iq2xs_cm2s_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Iq2xsCm2SBatch : Iq2xsCm2T { + override BN = 32u + override STILE = true + typedef BT = coopmatWgB_f16_64x32 + typedef ACC = coopmatWgAcc_f16_128x32 + typedef ACCW = coopmatWgAcc_f32_128x32 + typedef FLO = tensorLayout2DPad +} + // ===== the split-k reduce (sums the cm2 partial planes into y) ===== struct SkRedArgs { @@ -5821,6 +6111,8 @@ def kq_batch_cls_ensure(fmt : int; q40cm : bool) : bool { return ensure_kq_batch_k2_cls() } elif (fmt == int(KqFmt.iq2s)) { return ensure_kq_batch_iq2s_cls() + } elif (fmt == int(KqFmt.iq2xs)) { + return ensure_kq_batch_iq2xs_cls() } panic("dasLLAMA vulkan tier: no kq batch class for format {fmt}") return false @@ -5861,6 +6153,8 @@ def kq_batch_cls_enc_for(fmt : int; q40cm : bool; raw : VkCommandBuffer; var h : enc_kq_batch_k2_cls(raw, h, set_, pc, wgs) } elif (fmt == int(KqFmt.iq2s)) { enc_kq_batch_iq2s_cls(raw, h, set_, pc, wgs) + } elif (fmt == int(KqFmt.iq2xs)) { + enc_kq_batch_iq2xs_cls(raw, h, set_, pc, wgs) } else { panic("dasLLAMA vulkan tier: no kq batch class for format {fmt}") } @@ -5895,6 +6189,8 @@ def gemv_cls_ensure(fmt : int) : bool { return ensure_kq_gemv_k2_cls() } elif (fmt == int(KqFmt.iq2s)) { return ensure_kq_gemv_iq2s_cls() + } elif (fmt == int(KqFmt.iq2xs)) { + return ensure_kq_gemv_iq2xs_cls() } panic("dasLLAMA vulkan tier: no GEMV class for format {fmt}") return false @@ -5935,6 +6231,8 @@ def gemv_cls_enc(fmt : int; raw : VkCommandBuffer; var h : VkHaz; var set_ : VkD enc_kq_gemv_k2_cls(raw, h, set_, pc, wgs) } elif (fmt == int(KqFmt.iq2s)) { enc_kq_gemv_iq2s_cls(raw, h, set_, pc, wgs) + } elif (fmt == int(KqFmt.iq2xs)) { + enc_kq_gemv_iq2xs_cls(raw, h, set_, pc, wgs) } else { panic("dasLLAMA vulkan tier: no GEMV class for format {fmt}") } @@ -6342,3 +6640,58 @@ class KqGemvIq2s : KqGemvBase { gemv_shell() } } + +// IQ2_XS: the iq2s u64-grid walk with ksigns-by-parity - each u16 qs word carries a 9-bit +// grid index (TWO staged words) and a 7-bit KSIGNS_IQ2XS index whose sign byte ksign7 +// recomputes (no sign plane, no qh); per-16 UNSIGNED strips under the eighth-ed d. +[vk_dispatch(name = "kq_gemv_iq2xs_cls", kernel = "run", family = "kq_gemv_cls", grid = "wgs", params = "wgs : int64")] +class KqGemvIq2xs : KqGemvBase { + @workgroup gridw : uint[1024] + + def iq3s_signed(g, nib : uint) : uint { + let m1 = (nib * 0x00204081u) & 0x01010101u + let mask = m1 * 255u + return (g ^ mask) + m1 + } + + def override blk_contrib(wsb, blk, xsb : uint) : float { + let xb4 = xsb * 16u + blk * 2u + let xl = xq4[xb4] + let xh = xq4[xb4 + 1u] + let sw = wsu[wsb * 5u + 1u + blk / 2u] + let s0 = int((sw >> ((blk % 2u) * 16u)) & 255u) + let s1 = int((sw >> ((blk % 2u) * 16u + 8u)) & 255u) + var isum = 0 + for (h in range(2)) { + let qw = wq[wsb * 16u + blk * 2u + uint(h)] + let wa = qw & 0xFFFFu + let wb = qw >> 16u + let sg0 = ksign7(wa >> 9u) + let sg1 = ksign7(wb >> 9u) + let i0 = (wa & 511u) * 2u + let i1 = (wb & 511u) * 2u + let w0 = iq3s_signed(gridw[i0], sg0 & 15u) + let w1 = iq3s_signed(gridw[i0 + 1u], sg0 >> 4u) + let w2 = iq3s_signed(gridw[i1], sg1 & 15u) + let w3 = iq3s_signed(gridw[i1 + 1u], sg1 >> 4u) + if (h == 0) { + isum += s0 * (sdot4(w0, xl.x) + sdot4(w1, xl.y) + sdot4(w2, xl.z) + sdot4(w3, xl.w)) + } else { + isum += s1 * (sdot4(w0, xh.x) + sdot4(w1, xh.y) + sdot4(w2, xh.z) + sdot4(w3, xh.w)) + } + } + let dd = unpackHalf2x16(wsu[wsb * 5u]).x + return axs[xsb] * dd * float(isum) + } + + [spirv_kernel(local_size_x = 64, name = "kq_gemv_iq2xs_cls_spv")] + def run { + let lid = gl_LocalInvocationID.x + for (k in range(16)) { + let idx = lid + uint(k) * 64u + gridw[idx] = iq2xs_grid_word(int(idx)) + } + barrier() // the iq2xs grid visible before the shell's block walk + gemv_shell() + } +} diff --git a/modules/dasLLAMA/dasllama/dasllama_vulkan_common.das b/modules/dasLLAMA/dasllama/dasllama_vulkan_common.das index edaba85d16..6b1b855a03 100644 --- a/modules/dasLLAMA/dasllama/dasllama_vulkan_common.das +++ b/modules/dasLLAMA/dasllama/dasllama_vulkan_common.das @@ -1753,6 +1753,7 @@ def vk_kq_schema_id(fmt : int) : int { if (fmt == 10) return 45 if (fmt == 11) return 2 if (fmt == 12) return 23 + if (fmt == 13) return 24 panic("vk_kq_schema_id: not a kq stack fmt") return 0 } diff --git a/modules/dasLLAMA/dasllama/dasllama_vulkan_prefill.das b/modules/dasLLAMA/dasllama/dasllama_vulkan_prefill.das index df065fadac..14d7b1bb19 100644 --- a/modules/dasLLAMA/dasllama/dasllama_vulkan_prefill.das +++ b/modules/dasLLAMA/dasllama/dasllama_vulkan_prefill.das @@ -91,6 +91,9 @@ def private cm2_cls_ensure(fmt : int; tc : int64) : bool { if (fmt == int(KqFmt.iq2s)) { return tc == 256l ? ensure_kq_batch_iq2s_cm2l_cls() : (tc == 128l ? ensure_kq_batch_iq2s_cm2m_cls() : ensure_kq_batch_iq2s_cm2s_cls()) } + if (fmt == int(KqFmt.iq2xs)) { + return tc == 256l ? ensure_kq_batch_iq2xs_cm2l_cls() : (tc == 128l ? ensure_kq_batch_iq2xs_cm2m_cls() : ensure_kq_batch_iq2xs_cm2s_cls()) + } verify(fmt == int(KqFmt.q8), "vk prefill: the cm2 ensure fell through for a kq format - a format the feed admits must own a tile arm") return tc == 256l ? ensure_q8_batch_cm2l_cls() : (tc == 128l ? ensure_q8_batch_cm2m_cls() : ensure_q8_batch_cm2s_cls()) } @@ -163,6 +166,12 @@ def private cm2_cls_set(fmt : int; tc : int64; bufs : uint64 const[5]; sizes : i } return tc == 128l ? set_kq_batch_iq2s_cm2m_cls(bufs, sizes, gbits) : set_kq_batch_iq2s_cm2s_cls(bufs, sizes, gbits) } + if (fmt == int(KqFmt.iq2xs)) { + if (tc == 256l) { + return set_kq_batch_iq2xs_cm2l_cls(bufs, sizes, gbits) + } + return tc == 128l ? set_kq_batch_iq2xs_cm2m_cls(bufs, sizes, gbits) : set_kq_batch_iq2xs_cm2s_cls(bufs, sizes, gbits) + } verify(fmt == int(KqFmt.q8), "vk prefill: the cm2 set fell through for a kq format - a format the feed admits must own a tile arm") if (tc == 256l) { return set_q8_batch_cm2l_cls(bufs, sizes, gbits) @@ -260,6 +269,14 @@ def private cm2_cls_enc(fmt : int; tc : int64; raw : VkCommandBuffer; var h : Vk } else { enc_kq_batch_iq2s_cm2s_cls(raw, h, s, pc, groups) } + } elif (fmt == int(KqFmt.iq2xs)) { + if (tc == 256l) { + enc_kq_batch_iq2xs_cm2l_cls(raw, h, s, pc, groups) + } elif (tc == 128l) { + enc_kq_batch_iq2xs_cm2m_cls(raw, h, s, pc, groups) + } else { + enc_kq_batch_iq2xs_cm2s_cls(raw, h, s, pc, groups) + } } elif (tc == 256l) { verify(fmt == int(KqFmt.q8), "vk prefill: the cm2 enc fell through for a kq format - a format the feed admits must own a tile arm") enc_q8_batch_cm2l_cls(raw, h, s, pc, groups) diff --git a/modules/dasLLAMA/followup_vulkan.md b/modules/dasLLAMA/followup_vulkan.md index b90a3563f5..a185a036fd 100644 --- a/modules/dasLLAMA/followup_vulkan.md +++ b/modules/dasLLAMA/followup_vulkan.md @@ -544,3 +544,14 @@ module) is independent and can land any time - it is pure structure. schedule a dedicated pass at the END of the iquant-formats arc (after the last format lands), not per-format. Start from the followup 29-32 streamed-layer levers and a kernel-level probe of the cm2 tile vs llama.cpp's mul_mm_cm2 at matched shapes. + +35. **The grid-format GEMV workgroup re-stage is a fixed per-workgroup cost - amplified on + small models.** Every u64-grid gemv (iq2s 8 KB, iq2xs 4 KB) stages the codebook into + workgroup memory per 2-row workgroup, so tg pays a fixed latency the row length must + amortize. On the 3B i1 vehicle iq2s tg landed 0.81x llama.cpp (~350 GB/s effective); + the 1B IQ2_XS vehicle lands 0.54x (188.7 vs 349.9 t/s = ~84 GB/s effective - latency- + bound, while its cm2 pp512 sits at a healthy 0.77x). Levers, in likely order: persist + the staged grid across the row loop (one stage per SM residency, not per workgroup), + widen rows-per-workgroup for grid formats, or fold the grid into a device-buffer read + the L2 serves. Done = 1B-class grid-format tg within the k-format band on the same + vehicle. diff --git a/modules/dasLLAMA/tests/_vkd_oracles.das b/modules/dasLLAMA/tests/_vkd_oracles.das index 2e4737f90e..5f8556fb1b 100644 --- a/modules/dasLLAMA/tests/_vkd_oracles.das +++ b/modules/dasLLAMA/tests/_vkd_oracles.das @@ -195,6 +195,12 @@ def kq_cls_ref(fmt : int; wq, ws, xq : array; xs : array) : KqGemvB k2s.gridw[i] = IQ2S_GRID[i] } k = k2s + } elif (fmt == int(KqFmt.iq2xs)) { + var k2x = new KqGemvIq2xs() + for (i in range(1024)) { // same replay rule, the 512-entry grid's word pairs + k2x.gridw[i] = IQ2XS_GRID[i] + } + k = k2x } else { k = new KqGemvK6() } @@ -420,6 +426,46 @@ def iq2s_gemv_float_oracle(wq, ws, xq : array; xs : array; return <- y } +// IQ2_XS GEMV float oracle, INDEPENDENT of the class bit-math: w = d8 * (1 + 2s) * grid_byte +// * ksign straight off the plane words (32 u16 qs words per superblock, no sign/qh planes - +// the 7-bit ksigns index rides bits 9..15; the k4-shaped scale row with UNSIGNED strips). +def iq2xs_gemv_float_oracle(wq, ws, xq : array; xs : array; + regions : array; n, d, nreg : int) : array { + var y : array + y |> resize(nreg * d) + let nsb = n / 256 + for (r in range(nreg)) { + for (row in range(d)) { + let w0 = int(regions[r * 2]) + row * nsb + let x0 = int(regions[r * 2 + 1]) + var acc = 0.0 + for (s in range(nsb)) { + let sb = w0 + s + let xsb = x0 + s + let dd = half_at(ws, sb * 10) + for (wi in range(32)) { + let w16 = int((wq[sb * 16 + wi / 2] >> uint((wi % 2) * 16)) & 0xFFFFu) + let sidx = w16 >> 9 + var tt = sidx ^ (sidx >> 4) + tt = tt ^ (tt >> 2) + tt = tt ^ (tt >> 1) + let sgb = sidx | ((tt & 1) << 7) + let sc = float(byte_u8(ws, sb * 20 + 4 + wi / 2)) + var bacc = 0.0 + for (j in range(8)) { + let gb = float(int(IQ2XS_GRID[(w16 & 511) * 2 + j / 4] >> uint(8 * (j % 4))) & 0xFF) + let sgn = ((sgb >> j) & 1) != 0 ? -1.0 : 1.0 + bacc += gb * sgn * float(byte_s8(xq, xsb * 256 + wi * 8 + j)) + } + acc += xs[xsb] * dd * sc * bacc + } + } + y[r * d + row] = acc + } + } + return <- y +} + // IQ3_S f16-feed GEMM reference over the device form: grid gather + sign flip per element, // f16-rounded per weight like its siblings ([qs 64][qh 8][signs 32] rows, the k4-shaped // scale row with signed (1 + 2s) sub-scales). @@ -524,6 +570,44 @@ def iq2sf16_gemm_oracle(wq : array; wsu : array; xf : array; } } +// IQ2_XS f16-feed GEMM reference over the device form: u64-grid gather (two words per 9-bit +// index) + ksigns-by-parity off each u16 qs word, f16-rounded per weight like its siblings +// (32 u16 words per superblock, the k4-shaped scale row with UNSIGNED strips, d pre-eighth-ed). +def iq2xsf16_gemm_oracle(wq : array; wsu : array; xf : array; + recs : array; nrec, n, d : int; var y : array) { + let nsb = n / 256 + for (rr in range(nrec)) { + let wsb0 = int(recs[rr * 4]) + let row0 = int(recs[rr * 4 + 1]) + let cnt = int(recs[rr * 4 + 2]) + for (r in range(cnt)) { + for (c in range(d)) { + var acc = 0.0 + for (s in range(nsb)) { + let sb = wsb0 + c * nsb + s + let dv = unpackHalf2x16(wsu[sb * 5]).x + for (e in range(256)) { + let wi = e / 8 + let j = e % 8 + let w16 = int((wq[sb * 16 + wi / 2] >> uint((wi % 2) * 16)) & 0xFFFFu) + let sidx = w16 >> 9 + var tt = sidx ^ (sidx >> 4) + tt = tt ^ (tt >> 2) + tt = tt ^ (tt >> 1) + let sgb = sidx | ((tt & 1) << 7) + let si = e / 16 + let sc = float((wsu[sb * 5 + 1 + si / 4] >> uint((si % 4) * 8)) & 0xFFu) + let gb = float(int(IQ2XS_GRID[(w16 & 511) * 2 + j / 4] >> uint(8 * (j % 4))) & 0xFF) + let w = float(float16(dv * sc * (((sgb >> j) & 1) != 0 ? -gb : gb))) + acc += w * half_at(xf, (row0 + r) * n + s * 256 + e) + } + } + y[(row0 + r) * d + c] = acc + } + } + } +} + // IQ4_NL f16-feed GEMM reference over the device form: q40's walk with the codebook in place // of (q - 8), f16-rounded per weight like its siblings (q40's 32-word rows + 8 f16 d). def iq4nlf16_gemm_oracle(wq : array; wsu : array; xf : array; diff --git a/modules/dasLLAMA/tests/test_vulkan_kernels.das b/modules/dasLLAMA/tests/test_vulkan_kernels.das index b4daf4e4a0..bf40300a23 100644 --- a/modules/dasLLAMA/tests/test_vulkan_kernels.das +++ b/modules/dasLLAMA/tests/test_vulkan_kernels.das @@ -381,7 +381,7 @@ def private attn_row_oracle(q : array; qbase : int; kk : array; kb def test_vkd_kq_gemv_family(t0 : T?) { t0 |> run("kq GEMV class family == the CPU oracle (all 6 formats)") <| @(t : T?) { static_if (typeinfo builtin_module_exists(vulkan)) { - if (!(ensure_kq_gemv_k4_cls() && ensure_kq_gemv_q40_cls() && ensure_kq_gemv_k5_cls() && ensure_kq_gemv_k6_cls() && ensure_kq_gemv_iq4xs_cls() && ensure_kq_gemv_k3_cls() && ensure_kq_gemv_iq3s_cls() && ensure_kq_gemv_iq3xxs_cls() && ensure_kq_gemv_iq4nl_cls() && ensure_kq_gemv_k2_cls() && ensure_kq_gemv_iq2s_cls())) { + if (!(ensure_kq_gemv_k4_cls() && ensure_kq_gemv_q40_cls() && ensure_kq_gemv_k5_cls() && ensure_kq_gemv_k6_cls() && ensure_kq_gemv_iq4xs_cls() && ensure_kq_gemv_k3_cls() && ensure_kq_gemv_iq3s_cls() && ensure_kq_gemv_iq3xxs_cls() && ensure_kq_gemv_iq4nl_cls() && ensure_kq_gemv_k2_cls() && ensure_kq_gemv_iq2s_cls() && ensure_kq_gemv_iq2xs_cls())) { feint("no Vulkan device - skipping\n") return } @@ -389,9 +389,9 @@ def test_vkd_kq_gemv_family(t0 : T?) { let d = 3 // rows per region let nreg = 2 let rows = nreg * d - let nfmt = 11 - let fmts = fixed_array(int(KqFmt.k4), int(KqFmt.q40), int(KqFmt.k5), int(KqFmt.k6), int(KqFmt.iq4xs), int(KqFmt.k3), int(KqFmt.iq3s), int(KqFmt.iq3xxs), int(KqFmt.iq4nl), int(KqFmt.k2), int(KqFmt.iq2s)) - let stride4s = fixed_array(32, 32, 40, 48, 32, 24, 26, 24, 32, 16, 18) // uint WORDS per superblock (iq3s/iq3xxs/iq2s are not uint4-aligned) + let nfmt = 12 + let fmts = fixed_array(int(KqFmt.k4), int(KqFmt.q40), int(KqFmt.k5), int(KqFmt.k6), int(KqFmt.iq4xs), int(KqFmt.k3), int(KqFmt.iq3s), int(KqFmt.iq3xxs), int(KqFmt.iq4nl), int(KqFmt.k2), int(KqFmt.iq2s), int(KqFmt.iq2xs)) + let stride4s = fixed_array(32, 32, 40, 48, 32, 24, 26, 24, 32, 16, 18, 16) // uint WORDS per superblock (iq3s/iq3xxs/iq2s are not uint4-aligned) for (fi in range(nfmt)) { let fmt = fmts[fi] let stride4 = stride4s[fi] @@ -475,6 +475,8 @@ def test_vkd_kq_gemv_family(t0 : T?) { enc_kq_gemv_k2_cls(raw2, h2, s_cls, pc, wgs) } elif (fmt == int(KqFmt.iq2s)) { enc_kq_gemv_iq2s_cls(raw2, h2, s_cls, pc, wgs) + } elif (fmt == int(KqFmt.iq2xs)) { + enc_kq_gemv_iq2xs_cls(raw2, h2, s_cls, pc, wgs) } else { enc_kq_gemv_k6_cls(raw2, h2, s_cls, pc, wgs) } @@ -517,6 +519,12 @@ def test_vkd_kq_gemv_family(t0 : T?) { t |> success(badf == 0, "iq2s: the class bit-math matches the float dequant oracle ({badf} of {rows} rows off)") delete y_flt } + if (fmt == int(KqFmt.iq2xs)) { // the u64-grid/ksigns-parity gather has its own float witness + var y_flt <- iq2xs_gemv_float_oracle(wqh, wsh, xqh, xsh, regs, 512, d, nreg) + let badf = mismatch_approx(y_ref, y_flt) + t |> success(badf == 0, "iq2xs: the class bit-math matches the float dequant oracle ({badf} of {rows} rows off)") + delete y_flt + } delete regs delete y_ref delete wqh @@ -1037,7 +1045,7 @@ def test_vkd_q8_batch_tile(t0 : T?) { def test_vkd_kq_batch_family(t0 : T?) { t0 |> run("kq batch tile class family == the CPU oracle (all 6 formats)") <| @(t : T?) { static_if (typeinfo builtin_module_exists(vulkan)) { - if (!(ensure_kq_batch_k4_cls() && ensure_kq_batch_q40_cls() && ensure_kq_batch_k5_cls() && ensure_kq_batch_k6_cls() && ensure_kq_batch_iq4xs_cls() && ensure_kq_batch_k3_cls() && ensure_kq_batch_iq3s_cls() && ensure_kq_batch_iq3xxs_cls() && ensure_kq_batch_iq4nl_cls() && ensure_kq_batch_k2_cls() && ensure_kq_batch_iq2s_cls())) { + if (!(ensure_kq_batch_k4_cls() && ensure_kq_batch_q40_cls() && ensure_kq_batch_k5_cls() && ensure_kq_batch_k6_cls() && ensure_kq_batch_iq4xs_cls() && ensure_kq_batch_k3_cls() && ensure_kq_batch_iq3s_cls() && ensure_kq_batch_iq3xxs_cls() && ensure_kq_batch_iq4nl_cls() && ensure_kq_batch_k2_cls() && ensure_kq_batch_iq2s_cls() && ensure_kq_batch_iq2xs_cls())) { feint("no Vulkan device - skipping\n") return } @@ -1052,9 +1060,9 @@ def test_vkd_kq_batch_family(t0 : T?) { let wgs1 = ((cnt1 + 31) / 32) * wtiles let wgs = wgs0 + wgs1 let totsb = 2 * d * nsb // superblocks across both regions - let nfmt = 11 - let wstrides = fixed_array(32, 32, 40, 48, 32, 24, 26, 24, 32, 16, 18) // uint words per superblock: k4 q40 k5 k6 iq4xs k3 iq3s iq3xxs iq4nl k2 iq2s - let pipes = fixed_array(int(KqFmt.k4), int(KqFmt.q40), int(KqFmt.k5), int(KqFmt.k6), int(KqFmt.iq4xs), int(KqFmt.k3), int(KqFmt.iq3s), int(KqFmt.iq3xxs), int(KqFmt.iq4nl), int(KqFmt.k2), int(KqFmt.iq2s)) + let nfmt = 12 + let wstrides = fixed_array(32, 32, 40, 48, 32, 24, 26, 24, 32, 16, 18, 16) // uint words per superblock: k4 q40 k5 k6 iq4xs k3 iq3s iq3xxs iq4nl k2 iq2s iq2xs + let pipes = fixed_array(int(KqFmt.k4), int(KqFmt.q40), int(KqFmt.k5), int(KqFmt.k6), int(KqFmt.iq4xs), int(KqFmt.k3), int(KqFmt.iq3s), int(KqFmt.iq3xxs), int(KqFmt.iq4nl), int(KqFmt.k2), int(KqFmt.iq2s), int(KqFmt.iq2xs)) for (fi in range(nfmt)) { let fmt = pipes[fi] let wstride = wstrides[fi] @@ -1141,6 +1149,8 @@ def test_vkd_kq_batch_family(t0 : T?) { enc_kq_batch_k2_cls(raw2, h2, sc, pc, int64(wgs)) } elif (fmt == int(KqFmt.iq2s)) { enc_kq_batch_iq2s_cls(raw2, h2, sc, pc, int64(wgs)) + } elif (fmt == int(KqFmt.iq2xs)) { + enc_kq_batch_iq2xs_cls(raw2, h2, sc, pc, int64(wgs)) } else { enc_kq_batch_k6_cls(raw2, h2, sc, pc, int64(wgs)) } @@ -2680,6 +2690,130 @@ def test_vkd_iq2s_cm2_batch(t0 : T?) { } } +[test] +def test_vkd_iq2xs_cm2_batch(t0 : T?) { + t0 |> run("cm2 IQ2_S l/m/s tiles == the CPU oracle (u64 grid + ksigns-by-parity decode-in-load)") <| @(t : T?) { + static_if (typeinfo builtin_module_exists(vulkan)) { + if (!ensure_q8_batch_cls()) { // engage the device without touching coopmat rails + t |> skip("no Vulkan device") + return + } + if (!(g_gpu.coopmat_mode == 4 && g_gpu.has_coopmat2)) { + t |> skip("the cm2 iq2xs tiles serve only in mode 4 on an NV_coopmat2 device") + return + } + verify(ensure_kq_batch_iq2xs_cm2l_cls() && ensure_kq_batch_iq2xs_cm2m_cls() && ensure_kq_batch_iq2xs_cm2s_cls(), "iq2xs cm2 class rails must engage in cm2 mode") + let n = 768 // one 512-unrolled pass + 4 tail steps crossing superblock bounds + let nsb = n / 256 + let d = 160 // one full 128-tile + a 32 edge + let cnt0 = 300 + let cnt1 = 260 + let rows = cnt0 + cnt1 + let totsb = 2 * d * nsb + var wqh : array + var wsuh : array + var xfh : array + wqh |> resize(totsb * 16) + wsuh |> resize(totsb * 5) + xfh |> resize((rows + 32) * n / 2) // + the s tile's 32-row read slack + for (i in range(totsb * 16)) { + wqh[i] = hash_word(uint(i) + 29u) // arbitrary quant bytes + } + for (sb in range(totsb)) { + wsuh[sb * 5] = packHalf2x16(float2(0.0002 * float(1 + sb % 7), 0.0)) // cool d8 + wsuh[sb * 5 + 1] = hash_word(uint(sb * 4) + 613u) // 16 unsigned strip bytes + wsuh[sb * 5 + 2] = hash_word(uint(sb * 4 + 1) + 613u) + wsuh[sb * 5 + 3] = hash_word(uint(sb * 4 + 2) + 613u) + wsuh[sb * 5 + 4] = hash_word(uint(sb * 4 + 3) + 613u) + } + for (i in range((rows + 32) * n / 2)) { + xfh[i] = ws_word(uint(i) * 3u + 5u) // f16 activation pairs + } + let wq_bytes = int64(totsb) * 64l + let ws_bytes = int64(length(wsuh)) * 4l + let xf_bytes = int64((rows + 32) * n) * 2l + let y_bytes = int64(rows * d) * 4l + let wqd = make_device_buf(wq_bytes) + let wsd = make_device_buf(ws_bytes) + let xfd = make_device_buf(xf_bytes) + let yd2 = make_device_buf(y_bytes) + var host = make_host_buf(y_bytes, true, [cached = true]) + var y_cls : array + y_cls |> resize(rows * d) + var y_ref : array + y_ref |> resize(rows * d) + for (ml in range(3)) { + let ttile = ml == 0 ? 256 : (ml == 1 ? 128 : 32) + let wtiles = (d + 127) / 128 + let wgs0 = ((cnt0 + ttile - 1) / ttile) * wtiles + let wgs1 = ((cnt1 + ttile - 1) / ttile) * wtiles + let wgs = wgs0 + wgs1 + var sched : array + sched |> resize(2 * 4 + wgs) + sched[0] = 0u + sched[1] = 0u + sched[2] = uint(cnt0) + sched[3] = 0u + sched[4] = uint(d * nsb) + sched[5] = uint(cnt0) + sched[6] = uint(cnt1) + sched[7] = uint(wgs0) + for (w in range(wgs)) { + sched[8 + w] = w < wgs0 ? 0u : 1u + } + unsafe { + upload_region_at(wqd, 0l, addr(wqh[0]), wq_bytes) + upload_region_at(wsd, 0l, addr(wsuh[0]), ws_bytes) + upload_region_at(xfd, 0l, addr(xfh[0]), xf_bytes) + let sc_bytes = int64(2 * 4 + wgs) * 4l + let scd = make_device_buf(sc_bytes) + upload_region_at(scd, 0l, addr(sched[0]), sc_bytes) + var sc = (ml == 0 + ? set_kq_batch_iq2xs_cm2l_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)) + : (ml == 1 + ? set_kq_batch_iq2xs_cm2m_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)) + : set_kq_batch_iq2xs_cm2s_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)))) + var raw2 = alloc_cmd() + let begin = VkCommandBufferBeginInfo() + vk_check(vkBeginCommandBuffer(raw2, begin), null) + var h2 : VkHaz + var pc = BatchArgs(n = uint(n), d = uint(d), map_off = uint(2 * 4)) + if (ml == 0) { + enc_kq_batch_iq2xs_cm2l_cls(raw2, h2, sc, pc, int64(wgs)) + } elif (ml == 1) { + enc_kq_batch_iq2xs_cm2m_cls(raw2, h2, sc, pc, int64(wgs)) + } else { + enc_kq_batch_iq2xs_cm2s_cls(raw2, h2, sc, pc, int64(wgs)) + } + vhz_dep(raw2, h2, 16u, 0u, true) + cmd_copy_whole(raw2, yd2, host.buf, y_bytes) + vk_check(vkEndCommandBuffer(raw2), null) + submit_wait(raw2) + memcpy(addr(y_cls[0]), host.mapped, y_bytes) + } + iq2xsf16_gemm_oracle(wqh, wsuh, xfh, sched, 2, n, d, y_ref) + let bad = mismatch_bars(y_cls, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) + to_log(LOG_INFO, "cm2 iq2xs {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile batch: {bad} of {rows * d} off the oracle\n") + t |> success(bad == 0, "cm2 iq2xs {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile matches the CPU oracle ({bad} of {rows * d} off)") + delete sched + } + delete y_ref + delete wqh + delete wsuh + delete xfh + delete y_cls + } else { + t |> skip("dasVulkan not present") + } + } +} + [test] def test_vkd_iq3xxs_cm2_batch(t0 : T?) { t0 |> run("cm2 IQ3_XXS l/m/s tiles == the CPU oracle (halved grid + parity signs decode-in-load)") <| @(t : T?) { From 9b49748612387d6f085c05042fc2407097b30b8f Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Mon, 31 Aug 2026 12:43:58 -0700 Subject: [PATCH 44/61] dasLLAMA: IQ2_XS Phase D - Metal closes the format on all four tiers The iq2s Metal walk with ksigns-by-parity end to end: iq2xs_gw (1024 words, the same all-literal program-scope-constant hoisting), MetalKqGemvIq2xs + the MetalKqMvIq2xsT B2/B4 pair + MetalKqMvB8Iq2xs + MetalKqMulMmIq2xs (its own IQ2XS static_if arm nested over IQ2S's) - every kernel derives the sign byte via ksign7m off the u16 qs word, so there is no sign plane and no qh anywhere on the device path. The "iq2xss" blob arm is the iq2ss split verbatim (16 strips + 2B d8 tail, 18B/sb) over the SAME 20B row; quants bind as the 64B plane (16 uint words/sb, uint4-aligned). Gates: test_metal_gemv 2/2 + test_metal_gemm 2/2 on the M1 (first try - the iq2s shells carried it), zen2 kquant -jit 246/4 regression, lint 0. E2e --ngl 99: parity ids 64/64 vs llama.cpp - the FIRST format whose full-match greedy stream holds on ALL FOUR tiers. M1 benches: CPU das 746.6/51.6 vs llama.cpp 144.8/101.1 (5.16x/0.51x - the ledgered grid-format CPU tg tail), Metal das 3223.1/180.1 vs 3462.0/208.3 (0.93x/0.86x - the iq2s pp class). Four-tier table: zen2 2.78x/0.70x, vk 0.77x/0.54x, M1 CPU 5.16x/0.51x, Metal 0.93x/0.86x. The --tune-only filter carried both m1 sidecar top-ups (1 of 14 families raced). Co-Authored-By: Claude Fable 5 --- modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md | 14 + modules/dasLLAMA/dasllama/dasllama_layout.das | 23 +- .../dasllama/dasllama_metal_common.das | 14 + .../dasllama/dasllama_metal_kernels.das | 403 ++++++++++++++++++ .../dasllama/dasllama_metal_prefill.das | 8 +- .../dasllama/dasllama_metal_shapes.das | 4 +- .../dasLLAMA/tests/_metal_kernel_common.das | 9 +- .../tests/test_metal_gemm_kernels.das | 24 +- .../tests/test_metal_gemv_kernels.das | 40 +- 9 files changed, 501 insertions(+), 38 deletions(-) diff --git a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md index 284984e590..8b9580b101 100644 --- a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md +++ b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md @@ -525,6 +525,20 @@ E2e resident: parity ids 64/64 vs llama.cpp - the full-match stream holds on the 0.70x tier class), tg128 188.7 vs 349.9 (0.54x - the grid-format gemv workgroup re-stage amplified on a 1B, followup_vulkan #35; pp healthy isolates it to the gemv). Metal: pending. + +Phase D (Metal, 2026-08-31): the iq2s Metal walk with the parity trick end to end - `iq2xs_gw` +(1024 words, the same all-literal program-scope-constant hoisting), `MetalKqGemvIq2xs` + +`MetalKqMvIq2xsT` B2/B4 + `MetalKqMvB8Iq2xs` + `MetalKqMulMmIq2xs` (its own `IQ2XS` +static_if arm nested over IQ2S's), every kernel deriving the sign byte via `ksign7m` off the +u16 word (no sign plane, no qh); the "iq2xss" blob arm = the iq2ss split verbatim (16 strips + +2B d8 tail, 18B/sb) over the SAME 20B row. Gates: test_metal_gemv 2/2 + test_metal_gemm 2/2 +on the M1 (first try - the iq2s shells carried it), zen2 kquant -jit 246/4 regression, lint 0. +E2e --ngl 99: parity ids 64/64 - the FIRST format whose full-match stream holds on ALL FOUR +tiers. M1 16GB benches: CPU das 746.6/51.6 vs llama.cpp 144.8/101.1 (5.16x/0.51x - the +#60/#61 CPU tg tail, deepest of the grid formats), Metal das 3223.1/180.1 vs 3462.0/208.3 +(0.93x/0.86x - the iq2s pp class). The format is CLOSED on all four tiers; four-tier table: +zen2 2.78x/0.70x, vk 0.77x/0.54x, M1 CPU 5.16x/0.51x, Metal 0.93x/0.86x. + ### IQ2_S Phase A (CPU, 2026-08-31) - the u64-grid tier Shape: 256-superblock grid format, the first with a u64 grid - a 10-bit index (qs byte | diff --git a/modules/dasLLAMA/dasllama/dasllama_layout.das b/modules/dasLLAMA/dasllama/dasllama_layout.das index 2e87580458..efbbc9082b 100644 --- a/modules/dasLLAMA/dasllama/dasllama_layout.das +++ b/modules/dasLLAMA/dasllama/dasllama_layout.das @@ -29,7 +29,7 @@ require math def private metal_blob_off_ok(off : int64; fmt : KqFmt) : bool { if (off < 0l) return true if (fmt == KqFmt.q8) return off % 256l == 0l - if (fmt == KqFmt.k6 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s || fmt == KqFmt.iq3xxs || fmt == KqFmt.iq2s) return off % 512l == 0l // the split scale form: the (off/256)*2 d-plane bind stays 4B-aligned + if (fmt == KqFmt.k6 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s || fmt == KqFmt.iq3xxs || fmt == KqFmt.iq2s || fmt == KqFmt.iq2xs) return off % 512l == 0l // the split scale form: the (off/256)*2 d-plane bind stays 4B-aligned if (fmt == KqFmt.q51) return off % 128l == 0l // (off/32)*20 and (off/32)*4 both 16B-aligned return true } @@ -245,6 +245,25 @@ def metal_blob_scale_plane(var t : Model; name : string) : bool { } return true } + if (name == "iq2xss") { + // iq2xs: the identical 20B row over t.iq2xss - same split, d8 tail + if (!empty(t.iq2xss)) { + var @exact_size i2xc : array + let nsb = long_length(t.iq2xss) / IQ2XS_SSB + i2xc |> reserve_resize(nsb * 18l) + unsafe { + var dp = addr(i2xc[0]) + let sp = addr < uint8 const? >(t.iq2xss[0]) + for (sb in range64(nsb)) { + memcpy(reinterpret(dp + sb * 16l), reinterpret(sp + sb * IQ2XS_SSB + 4l), 16) + dp[nsb * 16l + sb * 2l] = sp[sb * IQ2XS_SSB] + dp[nsb * 16l + sb * 2l + 1l] = sp[sb * IQ2XS_SSB + 1l] + } + } + t.iq2xss <- i2xc + } + return true + } if (name == "k2s") { // k2: the 20B row splits like k6 - the 16 pair-byte strips first, the 4B d+dmin headers as the tail if (!empty(t.k2s)) { @@ -313,7 +332,7 @@ def metal_blob_commit(var t : Model) : bool { if (!metal_blob_eligible(t)) { return false } - for (nm in ["k4s", "k5s", "k6s", "iq4xss", "k3s", "iq3ss", "iq3xxss", "k2s", "iq2ss"]) { + for (nm in ["k4s", "k5s", "k6s", "iq4xss", "k3s", "iq3ss", "iq3xxss", "k2s", "iq2ss", "iq2xss"]) { metal_blob_scale_plane(t, nm) } t.metal_blob = true diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_common.das b/modules/dasLLAMA/dasllama/dasllama_metal_common.das index df9003ba1e..3c04f7dc76 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_common.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_common.das @@ -282,6 +282,11 @@ var g_pso_kq_mvb2_iq2s : MetalComputePipeline? var g_pso_kq_mvb4_iq2s : MetalComputePipeline? var g_pso_kq_mvb8_iq2s : MetalComputePipeline? var g_pso_kq_mm_b23 : MetalComputePipeline? +var g_pso_kq_iq2xs : MetalComputePipeline? +var g_pso_kq_mvb2_iq2xs : MetalComputePipeline? +var g_pso_kq_mvb4_iq2xs : MetalComputePipeline? +var g_pso_kq_mvb8_iq2xs : MetalComputePipeline? +var g_pso_kq_mm_b24 : MetalComputePipeline? var g_pso_kq_mvb2_iq3xxs : MetalComputePipeline? var g_pso_kq_mvb4_iq3xxs : MetalComputePipeline? var g_pso_kq_mvb8_iq3xxs : MetalComputePipeline? @@ -897,6 +902,10 @@ def kq_quants_of(dev : MetalDevice?; t : Model; fmt : KqFmt; off : int64) : tupl return (buf = plane_buffer(dev, addr < void? >(t.iq2sq[0]), uint64(long_length(t.iq2sq)), t.image_map != null), qoff = uint64(sb0 * 72l)) } + if (fmt == KqFmt.iq2xs) { + return (buf = plane_buffer(dev, addr < void? >(t.iq2xsq[0]), uint64(long_length(t.iq2xsq)), t.image_map != null), + qoff = uint64(sb0 * 64l)) + } return (buf = plane_buffer(dev, addr < void? >(t.k6q[0]), uint64(long_length(t.k6q)), t.image_map != null), qoff = uint64(sb0 * 192l)) } @@ -952,6 +961,11 @@ def kq_scales_of(dev : MetalDevice?; t : Model; fmt : KqFmt; off : int64) : tupl return (buf = plane_buffer(dev, addr < void? >(t.iq2ss[0]), uint64(long_length(t.iq2ss)), t.image_map != null), soff = uint64(sb0 * 16l), doff = uint64(plane_sb * 16l + sb0 * 2l)) } + if (fmt == KqFmt.iq2xs) { // the same split 18B/sb plane over t.iq2xss + let plane_sb = long_length(t.iq2xss) / 18l + return (buf = plane_buffer(dev, addr < void? >(t.iq2xss[0]), uint64(long_length(t.iq2xss)), t.image_map != null), + soff = uint64(sb0 * 16l), doff = uint64(plane_sb * 16l + sb0 * 2l)) + } return (buf = plane_buffer(dev, addr < void? >(t.k5s[0]), uint64(long_length(t.k5s)), t.image_map != null), soff = uint64(sb0 * 16l), doff = 0ul) } diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das index 03569bac69..f1f332ee54 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das @@ -3665,6 +3665,11 @@ def metal_decode_init : bool { // nolint:STYLE038 — a flat compile_pso per k g_pso_kq_mvb4_iq2s = compile_pso(MetalKqMvB4Iq2s_metal_kq_mv_iq2s_msl, MetalKqMvB4Iq2s_metal_kq_mv_iq2s_msl_entry, MetalKqMvB4Iq2s_metal_kq_mv_iq2s_msl_fastmath, ok) g_pso_kq_mvb8_iq2s = compile_pso(metal_kq_mvb8_iq2s_msl, metal_kq_mvb8_iq2s_msl_entry, metal_kq_mvb8_iq2s_msl_fastmath, ok) g_pso_kq_mm_b23 = compile_pso(MetalKqMulMmIq2s_metal_kq_mulmm_msl, MetalKqMulMmIq2s_metal_kq_mulmm_msl_entry, MetalKqMulMmIq2s_metal_kq_mulmm_msl_fastmath, ok) + g_pso_kq_iq2xs = compile_pso(metal_kq_gemv_iq2xs_msl, metal_kq_gemv_iq2xs_msl_entry, metal_kq_gemv_iq2xs_msl_fastmath, ok) + g_pso_kq_mvb2_iq2xs = compile_pso(MetalKqMvB2Iq2xs_metal_kq_mv_iq2xs_msl, MetalKqMvB2Iq2xs_metal_kq_mv_iq2xs_msl_entry, MetalKqMvB2Iq2xs_metal_kq_mv_iq2xs_msl_fastmath, ok) + g_pso_kq_mvb4_iq2xs = compile_pso(MetalKqMvB4Iq2xs_metal_kq_mv_iq2xs_msl, MetalKqMvB4Iq2xs_metal_kq_mv_iq2xs_msl_entry, MetalKqMvB4Iq2xs_metal_kq_mv_iq2xs_msl_fastmath, ok) + g_pso_kq_mvb8_iq2xs = compile_pso(metal_kq_mvb8_iq2xs_msl, metal_kq_mvb8_iq2xs_msl_entry, metal_kq_mvb8_iq2xs_msl_fastmath, ok) + g_pso_kq_mm_b24 = compile_pso(MetalKqMulMmIq2xs_metal_kq_mulmm_msl, MetalKqMulMmIq2xs_metal_kq_mulmm_msl_entry, MetalKqMulMmIq2xs_metal_kq_mulmm_msl_fastmath, ok) g_pso_kq_mm_b45 = compile_pso(MetalKqMulMmIq4nl_metal_kq_mulmm_msl, MetalKqMulMmIq4nl_metal_kq_mulmm_msl_entry, MetalKqMulMmIq4nl_metal_kq_mulmm_msl_fastmath, ok) g_pso_kq_k2 = compile_pso(metal_kq_gemv_k2_msl, metal_kq_gemv_k2_msl_entry, metal_kq_gemv_k2_msl_fastmath, ok) g_pso_kq_mvb2_k2 = compile_pso(MetalKqMvB2K2_metal_kq_mv_k2_msl, MetalKqMvB2K2_metal_kq_mv_k2_msl_entry, MetalKqMvB2K2_metal_kq_mv_k2_msl_fastmath, ok) @@ -3751,6 +3756,8 @@ def enc_kq_gemv(enc : MetalComputeEncoder?; fmt : KqFmt; bq : MetalBuffer?; qoff enc_kq_k2_c(enc, bs, doff, bs, soff, bq, qoff, bx, xoff, by, yoff, bn, bd, rows, n) } elif (fmt == KqFmt.iq2s) { enc_kq_iq2s_c(enc, bs, doff, bs, soff, bq, qoff, bx, xoff, by, yoff, bn, bd, rows, n) + } elif (fmt == KqFmt.iq2xs) { + enc_kq_iq2xs_c(enc, bs, doff, bs, soff, bq, qoff, bx, xoff, by, yoff, bn, bd, rows, n) } elif (fmt == KqFmt.k6) { enc_kq_k6_c(enc, bs, doff, bs, soff, bq, qoff, bx, xoff, by, yoff, bn, bd, rows, n) } elif (fmt == KqFmt.k5 && (n < 3072l || rows <= 1024l)) { @@ -3843,6 +3850,14 @@ def enc_kq_mvb(enc : MetalComputeEncoder?; fmt : KqFmt; bq : MetalBuffer?; qoff } else { enc_kq_mvb4_iq2s_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows, gcols) } + } elif (fmt == KqFmt.iq2xs) { + if (b8) { + enc_kq_mvb8_iq2xs_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows) + } elif (two) { + enc_kq_mvb2_iq2xs_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows) + } else { + enc_kq_mvb4_iq2xs_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows, gcols) + } } elif (fmt == KqFmt.k6) { if (b8) { enc_kq_mvb8_k6_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows) @@ -3889,6 +3904,8 @@ def enc_kq_gemm_mm_b(enc : MetalComputeEncoder?; fmt : KqFmt; bq : MetalBuffer?; enc_kq_mm_k2_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, bk, bn, mp, rows) } elif (fmt == KqFmt.iq2s) { enc_kq_mm_iq2s_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, bk, bn, mp, rows) + } elif (fmt == KqFmt.iq2xs) { + enc_kq_mm_iq2xs_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, bk, bn, mp, rows) } elif (fmt == KqFmt.k6) { enc_kq_mm_k6_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, bk, bn, mp, rows) } elif (fmt == KqFmt.k4) { @@ -4341,6 +4358,11 @@ def metal_kernels_release { // nolint:STYLE037,STYLE038 — a flat release_pso release_pso(g_pso_kq_mvb4_iq2s) release_pso(g_pso_kq_mvb8_iq2s) release_pso(g_pso_kq_mm_b23) + release_pso(g_pso_kq_iq2xs) + release_pso(g_pso_kq_mvb2_iq2xs) + release_pso(g_pso_kq_mvb4_iq2xs) + release_pso(g_pso_kq_mvb8_iq2xs) + release_pso(g_pso_kq_mm_b24) release_pso(g_pso_kq_mvb2_iq4nl) release_pso(g_pso_kq_mvb4_iq4nl) release_pso(g_pso_kq_mvb8_iq4nl) @@ -5582,6 +5604,7 @@ class template MetalKqMulMmK45T : MetalMmTileBase { @template_constant IQ4NL : bool = false // iq4nl: the IQ4XS nibble walk with the per-32-block f16 d plane (no sub-scale; s0off = soff) @template_constant K2 : bool = false // q2_K: per-16 sc/min nibble pairs (one uint4 strip), the split 4B d+dmin tail at s0off @template_constant IQ2S : bool = false // iq2s: the u64-grid gather DIRECT off the hoisted constant table, per-16 UNSIGNED strips + @template_constant IQ2XS : bool = false // iq2xs: iq2s's walk with ksigns-by-parity off each u16 qs word (no sign plane, no qh) @workgroup @template_gate = IQ4XS iq4lut : float[16] // kvalues_iq4nl as floats, staged once per threadgroup @workgroup @template_gate = IQ3S gridw3 : uint[512] // the 2 KB iq3s grid, staged once per threadgroup @workgroup @template_gate = IQ3XXS gridw3x : uint[256] // the 1 KB halved iq3xxs grid, staged once per threadgroup @@ -5702,6 +5725,25 @@ class template MetalKqMulMmK45T : MetalMmTileBase { va[k * 4 + 3] = float16(dsc3 * float(qv & 4278190080u) - dmn) } } else { + static_if (IQ2XS) { + // iq2xs: the iq2s strip/scale reads; each u16 qs word = 9-bit grid + // index + 7-bit ksigns index whose sign byte ksign7m recomputes + let sv = ks4[blk] + let si2 = js * 2u + il0 + let scw = si2 < 8u ? (si2 < 4u ? sv.x : sv.y) : (si2 < 12u ? sv.z : sv.w) + let dsc = float(ksh[blk]) * float((scw >> ((si2 & 3u) * 8u)) & 255u) + let qw2 = kqu[blk * 16u + js * 2u + il0] + for [unroll_full] (k in range(4)) { + let w16 = (qw2 >> (16u * (uint(k) / 2u))) & 0xFFFFu + let idx2 = (w16 & 511u) * 2u + uint(k) % 2u + let sgb2 = ksign7m(w16 >> 9u) + let nib = uint(k) % 2u == 0u ? sgb2 & 15u : sgb2 >> 4u + let sw = iq3s_sw(iq2xs_gw(int(idx2)), nib) + for [unroll_full] (c in range(4)) { + va[k * 4 + c] = float16(dsc * iq3s_sx(sw, uint(c))) + } + } + } else { static_if (IQ2S) { // iq2s: strip byte js*2+il0 is the UNSIGNED per-16 sub-scale; idx byte l of // block js doubles into the u64 grid's word pair, signs off the block's bytes @@ -5846,6 +5888,7 @@ class template MetalKqMulMmK45T : MetalMmTileBase { } } } + } } } barrier() @@ -5921,6 +5964,12 @@ class MetalKqMulMmIq2s : MetalKqMulMmK45T { override IQ2S = true } +// iq2xs rides its own IQ2XS arm; the same split scale plane binds (d8 plane at s0off = doff) +[metal_dispatch(name = "enc_kq_mm_iq2xs_c", pso = "g_pso_kq_mm_b24", tgmem = "MetalKqMulMmIq2xs_metal_kq_mulmm_msl_tgmem", tg = 128, grid = "mp/32, rows/64", params = "mp : int64, rows : int64")] +class MetalKqMulMmIq2xs : MetalKqMulMmK45T { + override IQ2XS = true +} + // iq4nl rides the IQ4XS arm with IQ4NL flipping the scale read to q40's per-32-block d plane [metal_dispatch(name = "enc_kq_mm_iq4nl_c", pso = "g_pso_kq_mm_b45", tgmem = "MetalKqMulMmIq4nl_metal_kq_mulmm_msl_tgmem", tg = 128, grid = "mp/32, rows/64", params = "mp : int64, rows : int64")] class MetalKqMulMmIq4nl : MetalKqMulMmK45T { @@ -6341,6 +6390,141 @@ def private ksign7m(v : uint) : uint { return v | ((t & 1u) << 7u) } +// ggml's iq2xs_grid as low/high word pairs (word 2i = magnitudes 0..3 of u64 entry i, word +// 2i+1 = 4..7); the same all-literal program-scope-constant form as iq2s_gw below. +def private iq2xs_gw(i : int) : uint { // nolint:STYLE038 - a 1024-word data table, not splittable + let tbl = fixed_array( + 0x08080808, 0x08080808, 0x0808082b, 0x08080808, 0x08081919, 0x08080808, 0x08082b08, 0x08080808, + 0x08082b2b, 0x08080808, 0x08190819, 0x08080808, 0x08191908, 0x08080808, 0x0819192b, 0x08080808, + 0x08192b19, 0x08080808, 0x082b0808, 0x08080808, 0x082b082b, 0x08080808, 0x082b1919, 0x08080808, + 0x082b2b08, 0x08080808, 0x19080819, 0x08080808, 0x19081908, 0x08080808, 0x1908192b, 0x08080808, + 0x19082b19, 0x08080808, 0x19190808, 0x08080808, 0x1919082b, 0x08080808, 0x19191919, 0x08080808, + 0x19192b08, 0x08080808, 0x192b0819, 0x08080808, 0x192b1908, 0x08080808, 0x2b080808, 0x08080808, + 0x2b08082b, 0x08080808, 0x2b081919, 0x08080808, 0x2b082b08, 0x08080808, 0x2b190819, 0x08080808, + 0x2b191908, 0x08080808, 0x2b192b19, 0x08080808, 0x2b2b0808, 0x08080808, 0x08080819, 0x08080819, + 0x08081908, 0x08080819, 0x0808192b, 0x08080819, 0x08082b19, 0x08080819, 0x08190808, 0x08080819, + 0x0819082b, 0x08080819, 0x08191919, 0x08080819, 0x08192b08, 0x08080819, 0x08192b2b, 0x08080819, + 0x082b0819, 0x08080819, 0x082b1908, 0x08080819, 0x19080808, 0x08080819, 0x1908082b, 0x08080819, + 0x19081919, 0x08080819, 0x19082b08, 0x08080819, 0x19190819, 0x08080819, 0x19191908, 0x08080819, + 0x192b0808, 0x08080819, 0x192b2b08, 0x08080819, 0x2b080819, 0x08080819, 0x2b081908, 0x08080819, + 0x2b190808, 0x08080819, 0x08080808, 0x0808082b, 0x0808082b, 0x0808082b, 0x08081919, 0x0808082b, + 0x08082b08, 0x0808082b, 0x08190819, 0x0808082b, 0x08191908, 0x0808082b, 0x082b0808, 0x0808082b, + 0x19080819, 0x0808082b, 0x19081908, 0x0808082b, 0x19190808, 0x0808082b, 0x19191919, 0x0808082b, + 0x2b080808, 0x0808082b, 0x2b082b2b, 0x0808082b, 0x08080819, 0x08081908, 0x08081908, 0x08081908, + 0x0808192b, 0x08081908, 0x08082b19, 0x08081908, 0x08190808, 0x08081908, 0x0819082b, 0x08081908, + 0x08191919, 0x08081908, 0x08192b08, 0x08081908, 0x082b0819, 0x08081908, 0x082b1908, 0x08081908, + 0x19080808, 0x08081908, 0x1908082b, 0x08081908, 0x19081919, 0x08081908, 0x19082b08, 0x08081908, + 0x19190819, 0x08081908, 0x19191908, 0x08081908, 0x1919192b, 0x08081908, 0x192b0808, 0x08081908, + 0x2b080819, 0x08081908, 0x2b081908, 0x08081908, 0x2b190808, 0x08081908, 0x08080808, 0x08081919, + 0x0808082b, 0x08081919, 0x08081919, 0x08081919, 0x08082b08, 0x08081919, 0x08190819, 0x08081919, + 0x08191908, 0x08081919, 0x082b0808, 0x08081919, 0x19080819, 0x08081919, 0x19081908, 0x08081919, + 0x19190808, 0x08081919, 0x192b0819, 0x08081919, 0x2b080808, 0x08081919, 0x08080819, 0x0808192b, + 0x08081908, 0x0808192b, 0x08190808, 0x0808192b, 0x082b192b, 0x0808192b, 0x19080808, 0x0808192b, + 0x1908082b, 0x0808192b, 0x2b081908, 0x0808192b, 0x08080808, 0x08082b08, 0x0808082b, 0x08082b08, + 0x08081919, 0x08082b08, 0x08082b08, 0x08082b08, 0x08082b2b, 0x08082b08, 0x08190819, 0x08082b08, + 0x08191908, 0x08082b08, 0x082b0808, 0x08082b08, 0x082b1919, 0x08082b08, 0x19080819, 0x08082b08, + 0x19081908, 0x08082b08, 0x19190808, 0x08082b08, 0x19192b08, 0x08082b08, 0x2b080808, 0x08082b08, + 0x2b2b0808, 0x08082b08, 0x2b2b2b2b, 0x08082b08, 0x08080819, 0x08082b19, 0x08081908, 0x08082b19, + 0x08190808, 0x08082b19, 0x19080808, 0x08082b19, 0x2b080819, 0x08082b19, 0x2b082b19, 0x08082b19, + 0x08080808, 0x08082b2b, 0x082b0808, 0x08082b2b, 0x082b2b08, 0x08082b2b, 0x2b19192b, 0x08082b2b, + 0x2b2b0808, 0x08082b2b, 0x08080819, 0x08190808, 0x08081908, 0x08190808, 0x0808192b, 0x08190808, + 0x08082b19, 0x08190808, 0x08190808, 0x08190808, 0x0819082b, 0x08190808, 0x08191919, 0x08190808, + 0x08192b08, 0x08190808, 0x082b0819, 0x08190808, 0x082b1908, 0x08190808, 0x19080808, 0x08190808, + 0x1908082b, 0x08190808, 0x19081919, 0x08190808, 0x19082b08, 0x08190808, 0x19190819, 0x08190808, + 0x19191908, 0x08190808, 0x192b0808, 0x08190808, 0x192b2b2b, 0x08190808, 0x2b080819, 0x08190808, + 0x2b081908, 0x08190808, 0x2b190808, 0x08190808, 0x08080808, 0x08190819, 0x0808082b, 0x08190819, + 0x08081919, 0x08190819, 0x08082b08, 0x08190819, 0x08190819, 0x08190819, 0x08191908, 0x08190819, + 0x082b0808, 0x08190819, 0x19080819, 0x08190819, 0x19081908, 0x08190819, 0x19190808, 0x08190819, + 0x2b080808, 0x08190819, 0x2b191908, 0x08190819, 0x2b19192b, 0x08190819, 0x08080819, 0x0819082b, + 0x08081908, 0x0819082b, 0x0808192b, 0x0819082b, 0x08190808, 0x0819082b, 0x19080808, 0x0819082b, + 0x192b0808, 0x0819082b, 0x08080808, 0x08191908, 0x0808082b, 0x08191908, 0x08081919, 0x08191908, + 0x08082b08, 0x08191908, 0x08190819, 0x08191908, 0x08191908, 0x08191908, 0x082b0808, 0x08191908, + 0x19080819, 0x08191908, 0x19081908, 0x08191908, 0x19082b19, 0x08191908, 0x19190808, 0x08191908, + 0x192b1908, 0x08191908, 0x2b080808, 0x08191908, 0x08080819, 0x08191919, 0x08081908, 0x08191919, + 0x08190808, 0x08191919, 0x19080808, 0x08191919, 0x08080808, 0x0819192b, 0x08191908, 0x0819192b, + 0x19082b19, 0x0819192b, 0x08080819, 0x08192b08, 0x08081908, 0x08192b08, 0x08190808, 0x08192b08, + 0x0819082b, 0x08192b08, 0x19080808, 0x08192b08, 0x19191908, 0x08192b08, 0x2b08192b, 0x08192b08, + 0x08080808, 0x08192b19, 0x08081919, 0x08192b19, 0x192b192b, 0x08192b19, 0x19190819, 0x08192b2b, + 0x2b2b2b19, 0x08192b2b, 0x08080808, 0x082b0808, 0x0808082b, 0x082b0808, 0x08081919, 0x082b0808, + 0x08082b08, 0x082b0808, 0x08082b2b, 0x082b0808, 0x08190819, 0x082b0808, 0x08191908, 0x082b0808, + 0x082b0808, 0x082b0808, 0x19080819, 0x082b0808, 0x19081908, 0x082b0808, 0x19190808, 0x082b0808, + 0x2b080808, 0x082b0808, 0x2b2b0808, 0x082b0808, 0x08080819, 0x082b0819, 0x08081908, 0x082b0819, + 0x08190808, 0x082b0819, 0x19080808, 0x082b0819, 0x19082b08, 0x082b0819, 0x192b1919, 0x082b0819, + 0x08080808, 0x082b082b, 0x082b082b, 0x082b082b, 0x2b080808, 0x082b082b, 0x2b2b2b08, 0x082b082b, + 0x08080819, 0x082b1908, 0x08081908, 0x082b1908, 0x08190808, 0x082b1908, 0x082b2b19, 0x082b1908, + 0x19080808, 0x082b1908, 0x08080808, 0x082b1919, 0x19080819, 0x082b1919, 0x1919082b, 0x082b1919, + 0x2b192b19, 0x082b1919, 0x08080819, 0x082b192b, 0x08192b2b, 0x082b192b, 0x2b2b192b, 0x082b192b, + 0x08080808, 0x082b2b08, 0x08082b08, 0x082b2b08, 0x08082b2b, 0x082b2b08, 0x082b0808, 0x082b2b08, + 0x19191919, 0x082b2b08, 0x2b082b08, 0x082b2b08, 0x2b2b082b, 0x082b2b08, 0x192b2b08, 0x082b2b19, + 0x2b190808, 0x082b2b19, 0x08082b08, 0x082b2b2b, 0x082b0808, 0x082b2b2b, 0x2b08082b, 0x082b2b2b, + 0x2b082b08, 0x082b2b2b, 0x2b082b2b, 0x082b2b2b, 0x08080819, 0x19080808, 0x08081908, 0x19080808, + 0x0808192b, 0x19080808, 0x08082b19, 0x19080808, 0x08190808, 0x19080808, 0x0819082b, 0x19080808, + 0x08191919, 0x19080808, 0x08192b08, 0x19080808, 0x082b0819, 0x19080808, 0x082b1908, 0x19080808, + 0x19080808, 0x19080808, 0x1908082b, 0x19080808, 0x19081919, 0x19080808, 0x19082b08, 0x19080808, + 0x19082b2b, 0x19080808, 0x19190819, 0x19080808, 0x19191908, 0x19080808, 0x192b0808, 0x19080808, + 0x192b1919, 0x19080808, 0x2b080819, 0x19080808, 0x2b081908, 0x19080808, 0x2b190808, 0x19080808, + 0x08080808, 0x19080819, 0x0808082b, 0x19080819, 0x08081919, 0x19080819, 0x08082b08, 0x19080819, + 0x08190819, 0x19080819, 0x08191908, 0x19080819, 0x082b0808, 0x19080819, 0x19080819, 0x19080819, + 0x19081908, 0x19080819, 0x19190808, 0x19080819, 0x2b080808, 0x19080819, 0x2b081919, 0x19080819, + 0x2b2b082b, 0x19080819, 0x08080819, 0x1908082b, 0x08081908, 0x1908082b, 0x08190808, 0x1908082b, + 0x0819082b, 0x1908082b, 0x082b2b19, 0x1908082b, 0x19080808, 0x1908082b, 0x08080808, 0x19081908, + 0x0808082b, 0x19081908, 0x08081919, 0x19081908, 0x08082b08, 0x19081908, 0x08190819, 0x19081908, + 0x08191908, 0x19081908, 0x08192b19, 0x19081908, 0x082b0808, 0x19081908, 0x19080819, 0x19081908, + 0x19081908, 0x19081908, 0x19190808, 0x19081908, 0x2b080808, 0x19081908, 0x2b191908, 0x19081908, + 0x08080819, 0x19081919, 0x08081908, 0x19081919, 0x08190808, 0x19081919, 0x082b1908, 0x19081919, + 0x19080808, 0x19081919, 0x2b192b2b, 0x19081919, 0x08080808, 0x1908192b, 0x08082b2b, 0x1908192b, + 0x19081908, 0x1908192b, 0x19190808, 0x1908192b, 0x08080819, 0x19082b08, 0x08081908, 0x19082b08, + 0x08190808, 0x19082b08, 0x19080808, 0x19082b08, 0x19081919, 0x19082b08, 0x19191908, 0x19082b08, + 0x192b082b, 0x19082b08, 0x08080808, 0x19082b19, 0x08190819, 0x19082b19, 0x19081908, 0x19082b19, + 0x19190808, 0x19082b19, 0x192b2b19, 0x19082b19, 0x08081908, 0x19082b2b, 0x08080808, 0x19190808, + 0x0808082b, 0x19190808, 0x08081919, 0x19190808, 0x08082b08, 0x19190808, 0x08190819, 0x19190808, + 0x08191908, 0x19190808, 0x082b0808, 0x19190808, 0x082b2b08, 0x19190808, 0x19080819, 0x19190808, + 0x19081908, 0x19190808, 0x19190808, 0x19190808, 0x2b080808, 0x19190808, 0x08080819, 0x19190819, + 0x08081908, 0x19190819, 0x08190808, 0x19190819, 0x08191919, 0x19190819, 0x19080808, 0x19190819, + 0x1908082b, 0x19190819, 0x08080808, 0x1919082b, 0x19081908, 0x1919082b, 0x2b2b2b2b, 0x1919082b, + 0x08080819, 0x19191908, 0x08081908, 0x19191908, 0x08190808, 0x19191908, 0x082b0819, 0x19191908, + 0x19080808, 0x19191908, 0x192b0808, 0x19191908, 0x2b080819, 0x19191908, 0x2b2b0819, 0x19191908, + 0x08080808, 0x19191919, 0x08082b08, 0x19191919, 0x2b080808, 0x19191919, 0x2b082b08, 0x19191919, + 0x082b0819, 0x1919192b, 0x192b2b08, 0x1919192b, 0x2b2b0819, 0x1919192b, 0x08080808, 0x19192b08, + 0x08191908, 0x19192b08, 0x19080819, 0x19192b08, 0x19190808, 0x19192b08, 0x2b192b19, 0x19192b08, + 0x08192b2b, 0x19192b19, 0x19080808, 0x19192b19, 0x1908082b, 0x19192b19, 0x2b081919, 0x19192b2b, + 0x08080819, 0x192b0808, 0x08081908, 0x192b0808, 0x08190808, 0x192b0808, 0x19080808, 0x192b0808, + 0x19191908, 0x192b0808, 0x192b082b, 0x192b0808, 0x2b08192b, 0x192b0808, 0x2b2b2b19, 0x192b0808, + 0x08080808, 0x192b0819, 0x082b1908, 0x192b082b, 0x19082b2b, 0x192b082b, 0x2b19082b, 0x192b082b, + 0x08080808, 0x192b1908, 0x0819192b, 0x192b1908, 0x08190808, 0x192b1919, 0x19080808, 0x192b1919, + 0x19081919, 0x192b1919, 0x2b2b1908, 0x192b1919, 0x08080819, 0x192b2b08, 0x192b2b2b, 0x192b2b08, + 0x082b1919, 0x192b2b19, 0x0808192b, 0x192b2b2b, 0x19191908, 0x192b2b2b, 0x192b082b, 0x192b2b2b, + 0x08080808, 0x2b080808, 0x0808082b, 0x2b080808, 0x08081919, 0x2b080808, 0x08082b08, 0x2b080808, + 0x08190819, 0x2b080808, 0x08191908, 0x2b080808, 0x082b0808, 0x2b080808, 0x082b2b2b, 0x2b080808, + 0x19080819, 0x2b080808, 0x19081908, 0x2b080808, 0x19190808, 0x2b080808, 0x2b080808, 0x2b080808, + 0x2b08082b, 0x2b080808, 0x2b2b2b08, 0x2b080808, 0x2b2b2b2b, 0x2b080808, 0x08080819, 0x2b080819, + 0x08081908, 0x2b080819, 0x0808192b, 0x2b080819, 0x08190808, 0x2b080819, 0x19080808, 0x2b080819, + 0x19190819, 0x2b080819, 0x19192b19, 0x2b080819, 0x08080808, 0x2b08082b, 0x082b0808, 0x2b08082b, + 0x2b080808, 0x2b08082b, 0x2b08082b, 0x2b08082b, 0x2b2b0808, 0x2b08082b, 0x2b2b2b08, 0x2b08082b, + 0x08080819, 0x2b081908, 0x08081908, 0x2b081908, 0x08190808, 0x2b081908, 0x0819082b, 0x2b081908, + 0x08191919, 0x2b081908, 0x19080808, 0x2b081908, 0x192b0808, 0x2b081908, 0x2b082b19, 0x2b081908, + 0x08080808, 0x2b081919, 0x19081908, 0x2b081919, 0x2b2b1919, 0x2b081919, 0x08192b08, 0x2b08192b, + 0x192b2b2b, 0x2b08192b, 0x08080808, 0x2b082b08, 0x08082b08, 0x2b082b08, 0x082b1919, 0x2b082b08, + 0x19192b2b, 0x2b082b08, 0x2b080808, 0x2b082b08, 0x2b08082b, 0x2b082b08, 0x2b2b2b08, 0x2b082b08, + 0x0808192b, 0x2b082b19, 0x082b082b, 0x2b082b2b, 0x2b080808, 0x2b082b2b, 0x2b082b08, 0x2b082b2b, + 0x2b19192b, 0x2b082b2b, 0x2b2b2b08, 0x2b082b2b, 0x08080819, 0x2b190808, 0x08081908, 0x2b190808, + 0x08190808, 0x2b190808, 0x19080808, 0x2b190808, 0x1919192b, 0x2b190808, 0x2b081908, 0x2b190808, + 0x08080808, 0x2b190819, 0x082b082b, 0x2b190819, 0x192b1908, 0x2b190819, 0x1919192b, 0x2b19082b, + 0x2b082b19, 0x2b19082b, 0x08080808, 0x2b191908, 0x08081919, 0x2b191908, 0x19081908, 0x2b191908, + 0x19190808, 0x2b191908, 0x19192b08, 0x2b191908, 0x082b2b19, 0x2b191919, 0x2b190808, 0x2b191919, + 0x2b19082b, 0x2b191919, 0x19080819, 0x2b19192b, 0x19190819, 0x2b192b08, 0x2b2b192b, 0x2b192b08, + 0x19082b19, 0x2b192b19, 0x08191919, 0x2b192b2b, 0x192b0808, 0x2b192b2b, 0x08080808, 0x2b2b0808, + 0x0808082b, 0x2b2b0808, 0x08082b08, 0x2b2b0808, 0x08082b2b, 0x2b2b0808, 0x082b0808, 0x2b2b0808, + 0x082b2b2b, 0x2b2b0808, 0x2b2b0808, 0x2b2b0808, 0x19190819, 0x2b2b0819, 0x19192b19, 0x2b2b0819, + 0x2b2b192b, 0x2b2b0819, 0x08080808, 0x2b2b082b, 0x0808082b, 0x2b2b082b, 0x08082b08, 0x2b2b082b, + 0x082b2b2b, 0x2b2b082b, 0x2b080808, 0x2b2b082b, 0x2b2b0808, 0x2b2b082b, 0x19080808, 0x2b2b1908, + 0x2b191919, 0x2b2b1908, 0x192b1919, 0x2b2b192b, 0x2b192b08, 0x2b2b192b, 0x08082b2b, 0x2b2b2b08, + 0x082b0808, 0x2b2b2b08, 0x082b082b, 0x2b2b2b08, 0x082b2b08, 0x2b2b2b08, 0x2b2b0808, 0x2b2b2b08, + 0x2b2b2b08, 0x2b2b2b08, 0x08081908, 0x2b2b2b19, 0x2b081908, 0x2b2b2b19, 0x2b08192b, 0x2b2b2b19, + 0x082b2b08, 0x2b2b2b2b, 0x082b2b2b, 0x2b2b2b2b, 0x2b190819, 0x2b2b2b2b, 0x2b2b2b2b, 0x2b2b2b2b) + return tbl[i] +} + // ggml's iq2s_grid as low/high word pairs (word 2i = magnitudes 0..3 of u64 entry i, word 2i+1 // = 4..7); an all-literal fixed_array local, so it lowers to a program-scope constant table - // every kernel reads it DIRECT (an f4 slab of 1024 entries would be 32 KB, past the tg budget). @@ -8348,6 +8532,65 @@ class MetalKqGemvIq2s { } } +// IQ2_XS: the iq2s GEMV geometry over the parity-signed u16 words - per (block, half) one u32 +// carries two u16 qs words; 9-bit index doubles into the grid pair, ksign7m recomputes the +// sign byte (no sign plane, no qh); per-16 UNSIGNED strips, d pre-eighth-ed. +[metal_dispatch(name = "enc_kq_iq2xs_c", pso = "g_pso_kq_iq2xs", tg = 64, grid = "rows/8", params = "rows : int64, n : int64")] +class MetalKqGemvIq2xs { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d8 plane - the scale buffer bound at byte nsb*16 (s0off = the caller's doff) + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // 16B strips: 16 UNSIGNED per-16 sub-scales, byte view (same buffer at 0) + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array // iq2xs quant plane, uint view (16 words per superblock: 32 u16 qs words) + @ssbo @binding = 3 @role = "read" @off = "xoff" @span = "n*4" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" @span = "rows*4" y : array + @uniform @binding = 5 ndim : uint + @uniform @binding = 6 ddim : uint + + [metal_kernel(name="metal_kq_gemv_iq2xs_msl")] + def metal_kq_gemv_iq2xs { + let lane = gl_SubgroupInvocationID + let ix = lane / 16u + let it = lane % 16u + let bu = it / 2u + let il = it % 2u + let nb = ndim / 256u + let first_row = (gl_WorkGroupID.x * 2u + gl_SubgroupID) * 4u + var sumf : float[4] + var ib = ix + while (ib < nb) { + let yb4 = ib * 64u + bu * 8u + il * 4u + let y0 = x[yb4] + let y1 = x[yb4 + 1u] + let y2 = x[yb4 + 2u] + let y3 = x[yb4 + 3u] + for [unroll_full] (r in range(4)) { + let blk = (first_row + uint(r)) * nb + ib + let qw = kqu[blk * 16u + bu * 2u + il] + var accv = float4(0.0) + for [unroll_full] (wi in range(2)) { + let w16 = (qw >> (16u * uint(wi))) & 0xFFFFu + let idx2 = (w16 & 511u) * 2u + let sgn = ksign7m(w16 >> 9u) + let w0 = iq3s_sw(iq2xs_gw(int(idx2)), sgn & 15u) + let w1 = iq3s_sw(iq2xs_gw(int(idx2 + 1u)), sgn >> 4u) + let ya = wi == 0 ? y0 : y2 + let yb = wi == 0 ? y1 : y3 + accv += ya * float4(iq3s_sx(w0, 0u), iq3s_sx(w0, 1u), iq3s_sx(w0, 2u), iq3s_sx(w0, 3u)) + accv += yb * float4(iq3s_sx(w1, 0u), iq3s_sx(w1, 1u), iq3s_sx(w1, 2u), iq3s_sx(w1, 3u)) + } + let sc = float(kscb[blk * 16u + bu * 2u + il]) + sumf[r] += float(kdh[blk]) * sc * (accv.x + accv.y + accv.z + accv.w) + } + ib += 2u + } + for [unroll_full] (r in range(4)) { + let s = simd_sum(sumf[r]) + if (lane == 0u && first_row + uint(r) < ddim) { + y[first_row + uint(r)] = s + } + } + } +} + // IQ3_XXS: the iq3s GEMV shape verbatim over the 24-word rows - the 1 KB halved grid as a // float4 magnitude slab, signs from the block's aux32 through the parity helper (no table), // the same (2ls+1)-strip x halved-d fold. 4 rows per simdgroup, dispatch rows/8. @@ -8909,6 +9152,166 @@ class MetalKqMvB8Iq2s { } } +// The iq2xs B2/B4 pair: the iq2s shell with the parity-signed u16 walk - lane tx owns block +// tx; two u32 qs words per block, ksign7m per u16; per-16 UNSIGNED strips as dl0/dl1. +[ |> template_struct_instance] +class template MetalKqMvIq2xsT { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d8 plane - the scale buffer bound at byte nsb*16 + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // 16B strips: 16 UNSIGNED per-16 sub-scales (same buffer at 0) + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array // iq2xs quant plane, uint view + @ssbo @binding = 3 @role = "read" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" y : array + @uniform @binding = 5 ka : KqMvArgs + @template_constant NR : int = 2 + @template_constant NRU : uint = 2u // NR's uint spelling (colbase math) - override BOTH together + @template_constant TILED : bool = false + + [metal_kernel] + def metal_kq_mv_iq2xs { + let lane = gl_SubgroupInvocationID + let tx = lane % 8u + let ty = lane / 8u + let row = gl_WorkGroupID.x * 8u + gl_SubgroupID * 4u + ty + let colbase = gl_WorkGroupID.y * NRU // nolint:LINT002 dead in the TILED=false stamp - measured free + let nb = ka.ndim / 256u + let nb4 = ka.ndim / 4u + var sumf : float[NR] + var sb = 0u + while (sb < nb) { + let blk = row * nb + sb + let dl0 = float(kdh[blk]) * float(kscb[blk * 16u + tx * 2u]) + let dl1 = float(kdh[blk]) * float(kscb[blk * 16u + tx * 2u + 1u]) + let qw0 = kqu[blk * 16u + tx * 2u] + let qw1 = kqu[blk * 16u + tx * 2u + 1u] + var w : float4[8] + for [unroll_full] (l in range(4)) { + let lu = uint(l) + let w16 = ((l < 2 ? qw0 : qw1) >> (16u * (lu & 1u))) & 0xFFFFu + let idx2 = (w16 & 511u) * 2u + let sgn = ksign7m(w16 >> 9u) + let dl = l < 2 ? dl0 : dl1 + let sw0 = iq3s_sw(iq2xs_gw(int(idx2)), sgn & 15u) + let sw1 = iq3s_sw(iq2xs_gw(int(idx2 + 1u)), sgn >> 4u) + w[l * 2] = float4(iq3s_sx(sw0, 0u), iq3s_sx(sw0, 1u), iq3s_sx(sw0, 2u), iq3s_sx(sw0, 3u)) * dl + w[l * 2 + 1] = float4(iq3s_sx(sw1, 0u), iq3s_sx(sw1, 1u), iq3s_sx(sw1, 2u), iq3s_sx(sw1, 3u)) * dl + } + let cx4 = sb * 64u + tx * 8u + static_if (TILED) { + for [unroll_full] (b in range(NR)) { + let xb4 = (colbase + uint(b)) * nb4 + cx4 + var acc = 0.0 + for [unroll_full] (u in range(8)) { + let v = x[xb4 + uint(u)] + acc += v.x * w[u].x + v.y * w[u].y + v.z * w[u].z + v.w * w[u].w + } + sumf[b] += acc + } + } else { + for [unroll_full] (b in range(NR)) { + let xb4 = uint(b) * nb4 + cx4 + var acc = 0.0 + for [unroll_full] (u in range(8)) { + let v = x[xb4 + uint(u)] + acc += v.x * w[u].x + v.y * w[u].y + v.z * w[u].z + v.w * w[u].w + } + sumf[b] += acc + } + } + sb++ + } + for [unroll_full] (b in range(NR)) { + var s = sumf[b] + s += simd_shuffle_down(s, 4u) + s += simd_shuffle_down(s, 2u) + s += simd_shuffle_down(s, 1u) + if (tx == 0u && row < ka.ddim && uint(b) < ka.nr) { + y[uint(b) * ka.ys + row] = s + } + } + } +} + +[metal_dispatch(name = "enc_kq_mvb2_iq2xs_c", pso = "g_pso_kq_mvb2_iq2xs", tg = 64, grid = "rows/8", params = "rows : int64")] +class MetalKqMvB2Iq2xs : MetalKqMvIq2xsT { + override NR = 2 + override NRU = 2u + override TILED = false +} + +[metal_dispatch(name = "enc_kq_mvb4_iq2xs_c", pso = "g_pso_kq_mvb4_iq2xs", tg = 64, grid = "rows/8, gcols", params = "rows : int64, gcols : int64")] +class MetalKqMvB4Iq2xs : MetalKqMvIq2xsT { + override NR = 4 + override NRU = 4u + override TILED = true +} + +// the iq2xs B8 twin: the iq2s B8 shell (one superblock's X panel staged per threadgroup) with +// the parity-signed u16 compose; the grid read direct. +[metal_dispatch(name = "enc_kq_mvb8_iq2xs_c", pso = "g_pso_kq_mvb8_iq2xs", tgmem = "metal_kq_mvb8_iq2xs_msl_tgmem", tg = 64, grid = "rows/8", params = "rows : int64")] +class MetalKqMvB8Iq2xs { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d8 plane - the scale buffer bound at byte nsb*16 + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // 16B strips: 16 UNSIGNED per-16 sub-scales (same buffer at 0) + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array // iq2xs quant plane, uint view + @ssbo @binding = 3 @role = "read" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" y : array + @uniform @binding = 5 ka : KqMvArgs + @workgroup txp : float4[512] // [8 streams x 64 float4] one superblock's X panel + + [metal_kernel(name="metal_kq_mvb8_iq2xs_msl")] + def metal_kq_mvb8_iq2xs { + let lid = gl_LocalInvocationID.x + let lane = gl_SubgroupInvocationID + let tx = lane % 8u + let ty = lane / 8u + let row = gl_WorkGroupID.x * 8u + gl_SubgroupID * 4u + ty + let nb = ka.ndim / 256u + let nb4 = ka.ndim / 4u + var acc : float4[8] + var sb = 0u + while (sb < nb) { + barrier() + for [unroll_full] (j in range(8)) { + let slot = uint(j) * 64u + lid + txp[slot] = x[(slot / 64u) * nb4 + sb * 64u + (slot % 64u)] + } + barrier() + let blk = row * nb + sb + let dl0 = float(kdh[blk]) * float(kscb[blk * 16u + tx * 2u]) + let dl1 = float(kdh[blk]) * float(kscb[blk * 16u + tx * 2u + 1u]) + let qw0 = kqu[blk * 16u + tx * 2u] + let qw1 = kqu[blk * 16u + tx * 2u + 1u] + var w : float4[8] + for [unroll_full] (l in range(4)) { + let lu = uint(l) + let w16 = ((l < 2 ? qw0 : qw1) >> (16u * (lu & 1u))) & 0xFFFFu + let idx2 = (w16 & 511u) * 2u + let sgn = ksign7m(w16 >> 9u) + let dl = l < 2 ? dl0 : dl1 + let sw0 = iq3s_sw(iq2xs_gw(int(idx2)), sgn & 15u) + let sw1 = iq3s_sw(iq2xs_gw(int(idx2 + 1u)), sgn >> 4u) + w[l * 2] = float4(iq3s_sx(sw0, 0u), iq3s_sx(sw0, 1u), iq3s_sx(sw0, 2u), iq3s_sx(sw0, 3u)) * dl + w[l * 2 + 1] = float4(iq3s_sx(sw1, 0u), iq3s_sx(sw1, 1u), iq3s_sx(sw1, 2u), iq3s_sx(sw1, 3u)) * dl + } + let cxl = tx * 8u + for [unroll_full] (b in range(8)) { + for [unroll_full] (u in range(8)) { + acc[b] += txp[uint(b) * 64u + cxl + uint(u)] * w[u] + } + } + sb++ + } + for [unroll_full] (b in range(8)) { + var s = acc[b].x + acc[b].y + acc[b].z + acc[b].w + s += simd_shuffle_down(s, 4u) + s += simd_shuffle_down(s, 2u) + s += simd_shuffle_down(s, 1u) + if (tx == 0u && row < ka.ddim && uint(b) < ka.nr) { + y[uint(b) * ka.ys + row] = s + } + } + } +} + // The iq4xs B2/B4 pair: the K4T shell (row/colbase/TILED twins) over the per-32-block lane map of // the GEMV above — lane tx owns block tx of each superblock, 16 lo + 16 hi weights as float4s. [ |> template_struct_instance] diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das b/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das index dad20b25dd..dd925ff332 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das @@ -4411,7 +4411,7 @@ def private pf_devw_panel(enc : MetalComputeEncoder?; bwblob : MetalBuffer?; wbo //! quant superblocks, plus the k6 2B d tail def private pf_devw_panel_kq(enc : MetalComputeEncoder?; t : Model; fmt : KqFmt; woff : int64; bxh, by, bk, bn : MetalBuffer?; rows, d, kdim : int64; yoff : uint64) : bool { - if (fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s || fmt == KqFmt.iq3xxs || fmt == KqFmt.iq4nl || fmt == KqFmt.k2 || fmt == KqFmt.iq2s) { // no dev-W dequant kernel for these formats yet + if (fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s || fmt == KqFmt.iq3xxs || fmt == KqFmt.iq4nl || fmt == KqFmt.k2 || fmt == KqFmt.iq2s || fmt == KqFmt.iq2xs) { // no dev-W dequant kernel for these formats yet return false } let dq = fmt == KqFmt.k6 ? g_pf_pso_dq_k6 : (fmt == KqFmt.k4 ? g_pf_pso_dq_k4 : g_pf_pso_dq_k5) @@ -4604,6 +4604,10 @@ def private pf_enc_kq_site_mm(enc : MetalComputeEncoder?; t : Model; fmt : KqFmt enc_kq_mm_iq2s_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, by, yoff, bk, bn, mp, rows) return } + if (fmt == KqFmt.iq2xs) { // the base mul_mm only - no tensor / tall / dev-W twins yet + enc_kq_mm_iq2xs_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, by, yoff, bk, bn, mp, rows) + return + } //! deep-class arm: a panel dev-W would split past 8 tiles is DRAM-resident - the tall //! in-kernel-dequant stamp reads the quant plane once per 128-row tile instead of //! materializing and re-streaming a 2B/element f16 panel @@ -4717,6 +4721,8 @@ def private pf_enc_kq_gemv(enc : MetalComputeEncoder?; t : Model; fmt : KqFmt; w enc_kq_iq3xxs_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, xoff, by, 0ul, bn, bd, rows, n) } elif (fmt == KqFmt.iq2s) { enc_kq_iq2s_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, xoff, by, 0ul, bn, bd, rows, n) + } elif (fmt == KqFmt.iq2xs) { + enc_kq_iq2xs_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, xoff, by, 0ul, bn, bd, rows, n) } elif (fmt == KqFmt.iq4nl) { enc_kq_iq4nl_c(enc, bs.buf, bs.soff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, xoff, by, 0ul, bn, bd, rows, n) } elif (fmt == KqFmt.k2) { diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_shapes.das b/modules/dasLLAMA/dasllama/dasllama_metal_shapes.das index 569a74b44e..2168f0fd78 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_shapes.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_shapes.das @@ -180,7 +180,7 @@ def record_needs(var tab : table; missing : MetalNeed) { // formats) must DECLINE kquant_native instead of falling into a wrong-layout kernel branch // (the dispatchers below treat "not k4/k6" as k5) def kq_fmt_gpu_supported(f : KqFmt) : bool { - return f == KqFmt.q8 || f == KqFmt.k4 || f == KqFmt.k5 || f == KqFmt.k6 || f == KqFmt.iq4xs || f == KqFmt.k3 || f == KqFmt.iq3s || f == KqFmt.iq3xxs || f == KqFmt.iq4nl || f == KqFmt.k2 || f == KqFmt.iq2s + return f == KqFmt.q8 || f == KqFmt.k4 || f == KqFmt.k5 || f == KqFmt.k6 || f == KqFmt.iq4xs || f == KqFmt.k3 || f == KqFmt.iq3s || f == KqFmt.iq3xxs || f == KqFmt.iq4nl || f == KqFmt.k2 || f == KqFmt.iq2s || f == KqFmt.iq2xs } def private kq_fmts_gpu_supported(a : array) : bool { @@ -243,7 +243,7 @@ def moe_site_ok(fmt : KqFmt; off, ege : int64) : bool { if (fmt == KqFmt.k4 || fmt == KqFmt.k5) { return off >= 0l && (off % 256l) == 0l && (ege % 256l) == 0l } - if (fmt == KqFmt.k6 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s || fmt == KqFmt.iq3xxs || fmt == KqFmt.iq2s) { // the split scale form: off % 512 keeps the d-plane bind 4B-aligned + if (fmt == KqFmt.k6 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s || fmt == KqFmt.iq3xxs || fmt == KqFmt.iq2s || fmt == KqFmt.iq2xs) { // the split scale form: off % 512 keeps the d-plane bind 4B-aligned return off >= 0l && (off % 512l) == 0l && (ege % 256l) == 0l } if (fmt == KqFmt.q51) { // per-32 planes: off % 128 keeps the 20B/4B binds 16B-aligned diff --git a/modules/dasLLAMA/tests/_metal_kernel_common.das b/modules/dasLLAMA/tests/_metal_kernel_common.das index 6ebb73dcc9..6423b163d2 100644 --- a/modules/dasLLAMA/tests/_metal_kernel_common.das +++ b/modules/dasLLAMA/tests/_metal_kernel_common.das @@ -174,7 +174,7 @@ let KQ_SAFE_SCALES = fixed_array(0.25, 0.5, 1.0, 0.125, 2.0, 0.75, 1.5, 0.375) // [12 packed 6-bit bytes] per superblock, K6 = [16B sub-scales] x nsb + f16 d tail, IQ4_XS (44) = // the same split form with [8 signed sub-scales in -31..31][8 zero] strips, Q3_K (3) = the k6 form over 96B quants def kq_fill_planes(fmt, nsb : int; var kq : array; var ks : array) { // nolint:STYLE037 - the flat per-format fill ladder is the honest shape - let qsb = fmt == 2 ? 64 : (fmt == 23 ? 72 : (fmt == 4 || fmt == 44 || fmt == 45 ? 128 : (fmt == 5 ? 160 : (fmt == 3 ? 96 : (fmt == 33 ? 104 : (fmt == 34 ? 96 : 192)))))) + let qsb = fmt == 2 || fmt == 24 ? 64 : (fmt == 23 ? 72 : (fmt == 4 || fmt == 44 || fmt == 45 ? 128 : (fmt == 5 ? 160 : (fmt == 3 ? 96 : (fmt == 33 ? 104 : (fmt == 34 ? 96 : 192)))))) kq |> resize(nsb * qsb) for (i in range(nsb * qsb)) { kq[i] = kq_hash_byte(i * 3 + fmt) @@ -201,10 +201,10 @@ def kq_fill_planes(fmt, nsb : int; var kq : array; var ks : array) ks[b * 2] = uint8(dbits & 0xFFu) ks[b * 2 + 1] = uint8(dbits >> 8u) } - } elif (fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 23) { + } elif (fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 23 || fmt == 24) { ks |> resize(nsb * 18) for (i in range(nsb * 16)) { - ks[i] = fmt == 6 || fmt == 3 || fmt == 23 ? kq_hash_byte(i * 7 + 13) : uint8(i % 16 < 8 ? int(kq_hash_byte(i * 7 + 13)) % 63 - 31 : 0) // 44/33: signed strip bytes + zero pad; 23: 16 unsigned strips + ks[i] = fmt == 6 || fmt == 3 || fmt == 23 || fmt == 24 ? kq_hash_byte(i * 7 + 13) : uint8(i % 16 < 8 ? int(kq_hash_byte(i * 7 + 13)) % 63 - 31 : 0) // 44/33: signed strip bytes + zero pad; 23: 16 unsigned strips } for (b in range(nsb)) { let dbits = f32_to_f16(KQ_SAFE_SCALES[b % 8]) @@ -287,6 +287,9 @@ def kq_row_ref(fmt, sb_base, nb, nsb_total : int; kq, ks : array; var wro } elif (fmt == 23) { dequant_iq2s_plane_superblock_at(kq, int64(sb * 72), ks, int64(sb * 16), int64(nsb_total * 16 + sb * 2), wrow, int64(ib * 256)) + } elif (fmt == 24) { + dequant_iq2xs_plane_superblock_at(kq, int64(sb * 64), ks, int64(sb * 16), + int64(nsb_total * 16 + sb * 2), wrow, int64(ib * 256)) } else { dequant_k6_plane_superblock_at(kq, int64(sb * 192), ks, int64(sb * 16), int64(nsb_total * 16 + sb * 2), wrow, int64(ib * 256)) diff --git a/modules/dasLLAMA/tests/test_metal_gemm_kernels.das b/modules/dasLLAMA/tests/test_metal_gemm_kernels.das index 2db7885687..6fdb010432 100644 --- a/modules/dasLLAMA/tests/test_metal_gemm_kernels.das +++ b/modules/dasLLAMA/tests/test_metal_gemm_kernels.das @@ -36,28 +36,28 @@ def private kq_mulmm_gate(t : T?; dev, queue; fmt : int; tensor : bool; m, kdim, : halfx ? (fmt == 4 ? MetalKqMulMmK4TH_metal_kq_mulmm_t_msl : (fmt == 5 ? MetalKqMulMmK5TH_metal_kq_mulmm_t_msl : MetalKqMulMmK6TH_metal_kq_mulmm_k6_t_msl)) : (fmt == 4 ? MetalKqMulMmK4T_metal_kq_mulmm_t_msl : (fmt == 5 ? MetalKqMulMmK5T_metal_kq_mulmm_t_msl : MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl))) - : (fmt == 23 ? MetalKqMulMmIq2s_metal_kq_mulmm_msl : (fmt == 45 ? MetalKqMulMmIq4nl_metal_kq_mulmm_msl : (fmt == 2 ? MetalKqMulMmK2_metal_kq_mulmm_msl : (fmt == 34 ? MetalKqMulMmIq3xxs_metal_kq_mulmm_msl : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl : MetalKqMulMmK6_metal_kq_mulmm_msl)))))))))) + : (fmt == 24 ? MetalKqMulMmIq2xs_metal_kq_mulmm_msl : (fmt == 23 ? MetalKqMulMmIq2s_metal_kq_mulmm_msl : (fmt == 45 ? MetalKqMulMmIq4nl_metal_kq_mulmm_msl : (fmt == 2 ? MetalKqMulMmK2_metal_kq_mulmm_msl : (fmt == 34 ? MetalKqMulMmIq3xxs_metal_kq_mulmm_msl : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl : MetalKqMulMmK6_metal_kq_mulmm_msl))))))))))) let entry = (tensor ? (tall ? (fmt == 4 ? MetalKqMulMmK4TH128_metal_kq_mulmm_t_msl_entry : (fmt == 5 ? MetalKqMulMmK5TH128_metal_kq_mulmm_t_msl_entry : MetalKqMulMmK6TH128_metal_kq_mulmm_k6_t_msl_entry)) : halfx ? (fmt == 4 ? MetalKqMulMmK4TH_metal_kq_mulmm_t_msl_entry : (fmt == 5 ? MetalKqMulMmK5TH_metal_kq_mulmm_t_msl_entry : MetalKqMulMmK6TH_metal_kq_mulmm_k6_t_msl_entry)) : (fmt == 4 ? MetalKqMulMmK4T_metal_kq_mulmm_t_msl_entry : (fmt == 5 ? MetalKqMulMmK5T_metal_kq_mulmm_t_msl_entry : MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl_entry))) - : (fmt == 23 ? MetalKqMulMmIq2s_metal_kq_mulmm_msl_entry : (fmt == 45 ? MetalKqMulMmIq4nl_metal_kq_mulmm_msl_entry : (fmt == 2 ? MetalKqMulMmK2_metal_kq_mulmm_msl_entry : (fmt == 34 ? MetalKqMulMmIq3xxs_metal_kq_mulmm_msl_entry : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl_entry : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl_entry : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl_entry : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_entry : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_entry : MetalKqMulMmK6_metal_kq_mulmm_msl_entry)))))))))) + : (fmt == 24 ? MetalKqMulMmIq2xs_metal_kq_mulmm_msl_entry : (fmt == 23 ? MetalKqMulMmIq2s_metal_kq_mulmm_msl_entry : (fmt == 45 ? MetalKqMulMmIq4nl_metal_kq_mulmm_msl_entry : (fmt == 2 ? MetalKqMulMmK2_metal_kq_mulmm_msl_entry : (fmt == 34 ? MetalKqMulMmIq3xxs_metal_kq_mulmm_msl_entry : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl_entry : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl_entry : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl_entry : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_entry : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_entry : MetalKqMulMmK6_metal_kq_mulmm_msl_entry))))))))))) let fm = (tensor ? (tall ? (fmt == 4 ? MetalKqMulMmK4TH128_metal_kq_mulmm_t_msl_fastmath : (fmt == 5 ? MetalKqMulMmK5TH128_metal_kq_mulmm_t_msl_fastmath : MetalKqMulMmK6TH128_metal_kq_mulmm_k6_t_msl_fastmath)) : halfx ? (fmt == 4 ? MetalKqMulMmK4TH_metal_kq_mulmm_t_msl_fastmath : (fmt == 5 ? MetalKqMulMmK5TH_metal_kq_mulmm_t_msl_fastmath : MetalKqMulMmK6TH_metal_kq_mulmm_k6_t_msl_fastmath)) : (fmt == 4 ? MetalKqMulMmK4T_metal_kq_mulmm_t_msl_fastmath : (fmt == 5 ? MetalKqMulMmK5T_metal_kq_mulmm_t_msl_fastmath : MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl_fastmath))) - : (fmt == 23 ? MetalKqMulMmIq2s_metal_kq_mulmm_msl_fastmath : (fmt == 45 ? MetalKqMulMmIq4nl_metal_kq_mulmm_msl_fastmath : (fmt == 2 ? MetalKqMulMmK2_metal_kq_mulmm_msl_fastmath : (fmt == 34 ? MetalKqMulMmIq3xxs_metal_kq_mulmm_msl_fastmath : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl_fastmath : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl_fastmath : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl_fastmath : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_fastmath : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_fastmath : MetalKqMulMmK6_metal_kq_mulmm_msl_fastmath)))))))))) + : (fmt == 24 ? MetalKqMulMmIq2xs_metal_kq_mulmm_msl_fastmath : (fmt == 23 ? MetalKqMulMmIq2s_metal_kq_mulmm_msl_fastmath : (fmt == 45 ? MetalKqMulMmIq4nl_metal_kq_mulmm_msl_fastmath : (fmt == 2 ? MetalKqMulMmK2_metal_kq_mulmm_msl_fastmath : (fmt == 34 ? MetalKqMulMmIq3xxs_metal_kq_mulmm_msl_fastmath : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl_fastmath : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl_fastmath : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl_fastmath : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_fastmath : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_fastmath : MetalKqMulMmK6_metal_kq_mulmm_msl_fastmath))))))))))) let tgm = (tensor ? (tall ? (fmt == 4 ? MetalKqMulMmK4TH128_metal_kq_mulmm_t_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5TH128_metal_kq_mulmm_t_msl_tgmem : MetalKqMulMmK6TH128_metal_kq_mulmm_k6_t_msl_tgmem)) : halfx ? (fmt == 4 ? MetalKqMulMmK4TH_metal_kq_mulmm_t_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5TH_metal_kq_mulmm_t_msl_tgmem : MetalKqMulMmK6TH_metal_kq_mulmm_k6_t_msl_tgmem)) : (fmt == 4 ? MetalKqMulMmK4T_metal_kq_mulmm_t_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5T_metal_kq_mulmm_t_msl_tgmem : MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl_tgmem))) - : (fmt == 23 ? MetalKqMulMmIq2s_metal_kq_mulmm_msl_tgmem : (fmt == 45 ? MetalKqMulMmIq4nl_metal_kq_mulmm_msl_tgmem : (fmt == 2 ? MetalKqMulMmK2_metal_kq_mulmm_msl_tgmem : (fmt == 34 ? MetalKqMulMmIq3xxs_metal_kq_mulmm_msl_tgmem : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl_tgmem : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl_tgmem : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl_tgmem : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_tgmem : MetalKqMulMmK6_metal_kq_mulmm_msl_tgmem)))))))))) + : (fmt == 24 ? MetalKqMulMmIq2xs_metal_kq_mulmm_msl_tgmem : (fmt == 23 ? MetalKqMulMmIq2s_metal_kq_mulmm_msl_tgmem : (fmt == 45 ? MetalKqMulMmIq4nl_metal_kq_mulmm_msl_tgmem : (fmt == 2 ? MetalKqMulMmK2_metal_kq_mulmm_msl_tgmem : (fmt == 34 ? MetalKqMulMmIq3xxs_metal_kq_mulmm_msl_tgmem : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl_tgmem : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl_tgmem : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl_tgmem : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_tgmem : MetalKqMulMmK6_metal_kq_mulmm_msl_tgmem))))))))))) var pso = pipeline_from_source(dev, src, entry, fm, err) if (tensor && pso == null && !metal4_tensor_available(dev)) { to_log(LOG_INFO, "{tag}: no Metal-4 tensor toolchain on this box - arm skipped\n") @@ -123,7 +123,7 @@ def private kq_mulmm_gate(t : T?; dev, queue; fmt : int; tensor : bool; m, kdim, let ran = with_compute_encoder(queue, err) $(enc : MetalComputeEncoder?) { metal_set_pipeline(enc, pso) metal_set_threadgroup_memory_length(enc, tgm, 0) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs: the f16 d tail + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 || fmt == 24 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs: the f16 d tail metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bx, 0ul, 3) @@ -134,7 +134,7 @@ def private kq_mulmm_gate(t : T?; dev, queue; fmt : int; tensor : bool; m, kdim, if (rem_pso != null) { metal_set_pipeline(enc, rem_pso) metal_set_threadgroup_memory_length(enc, fmt == 4 ? MetalKqMulMmK4TH_metal_kq_mulmm_t_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5TH_metal_kq_mulmm_t_msl_tgmem : MetalKqMulMmK6TH_metal_kq_mulmm_k6_t_msl_tgmem), 0) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 ? uint64(nsb * 16) : 0ul, 0) + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 || fmt == 24 ? uint64(nsb * 16) : 0ul, 0) metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bx, uint64(r0 * kdim * 2), 3) @@ -484,7 +484,7 @@ def private kq_devw_gate(t : T?; dev, queue; fmt : int; m, kdim, ndim : int) { var dka = CvtArgs(total = uint(nblk)) let ran = with_compute_encoder(queue, err) $(enc : MetalComputeEncoder?) { metal_set_pipeline(enc, dq_pso) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs: the f16 d tail + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 || fmt == 24 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs: the f16 d tail metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bwh, 0ul, 3) @@ -1461,7 +1461,7 @@ def private moe_mulmm_kq_gate(t : T?; dev, queue; fmt : int; kdim, ndim : int; c let ran = with_compute_encoder(queue, err) $(enc : MetalComputeEncoder?) { metal_set_pipeline(enc, pso) metal_set_threadgroup_memory_length(enc, tgm, 0) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs: the f16 d tail + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 || fmt == 24 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs: the f16 d tail metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bx, 0ul, 3) @@ -1498,7 +1498,7 @@ def private moe_mulmm_kq_gate(t : T?; dev, queue; fmt : int; kdim, ndim : int; c metal_set_threadgroup_memory_length(enc, fmt == 4 ? MetalMoeMulMmK4TH_metal_moe_mulmm_k45_t_msl_tgmem : (fmt == 5 ? MetalMoeMulMmK5TH_metal_moe_mulmm_k45_t_msl_tgmem : MetalMoeMulMmK6TH_metal_moe_mulmm_k6_t_msl_tgmem), 0) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 ? uint64(nsb * 16) : 0ul, 0) + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 || fmt == 24 ? uint64(nsb * 16) : 0ul, 0) metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bxh, 0ul, 3) @@ -1538,7 +1538,7 @@ def private moe_mulmm_kq_gate(t : T?; dev, queue; fmt : int; kdim, ndim : int; c metal_set_threadgroup_memory_length(enc, fmt == 4 ? MetalMoeMulMmK4TH128_metal_moe_mulmm_k45_t_msl_tgmem : (fmt == 5 ? MetalMoeMulMmK5TH128_metal_moe_mulmm_k45_t_msl_tgmem : MetalMoeMulMmK6TH128_metal_moe_mulmm_k6_t_msl_tgmem), 0) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 ? uint64(nsb * 16) : 0ul, 0) + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 || fmt == 24 ? uint64(nsb * 16) : 0ul, 0) metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bxh, 0ul, 3) @@ -1551,7 +1551,7 @@ def private moe_mulmm_kq_gate(t : T?; dev, queue; fmt : int; kdim, ndim : int; c metal_set_threadgroup_memory_length(enc, fmt == 4 ? MetalMoeMulMmK4THR_metal_moe_mulmm_k45_t_msl_tgmem : (fmt == 5 ? MetalMoeMulMmK5THR_metal_moe_mulmm_k45_t_msl_tgmem : MetalMoeMulMmK6THR_metal_moe_mulmm_k6_t_msl_tgmem), 0) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 ? uint64(nsb * 16) : 0ul, 0) + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 || fmt == 24 ? uint64(nsb * 16) : 0ul, 0) metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bxh, 0ul, 3) @@ -2061,6 +2061,8 @@ def test_metal_gemm_kernels(t : T?) { kq_mulmm_gate(t, dev, queue, 2, false, 32, 256, 128) kq_mulmm_gate(t, dev, queue, 23, false, 64, 512, 64) // iq2s: the base form only (no tensor / tall twins) kq_mulmm_gate(t, dev, queue, 23, false, 32, 256, 128) + kq_mulmm_gate(t, dev, queue, 24, false, 64, 512, 64) // iq2xs: the base form only (no tensor / tall twins) + kq_mulmm_gate(t, dev, queue, 24, false, 32, 256, 128) q8_mulmm_t_gate(t, dev, queue, 64, 256, 64) q8_mulmm_t_gate(t, dev, queue, 64, 256, 64, halfx = true) q8_mulmm_t_gate(t, dev, queue, 64, 160, 64) // kdim % 64 == 32: the bk=64 helper's 32-tail chunk diff --git a/modules/dasLLAMA/tests/test_metal_gemv_kernels.das b/modules/dasLLAMA/tests/test_metal_gemv_kernels.das index 699c8a9f2a..c0431afb07 100644 --- a/modules/dasLLAMA/tests/test_metal_gemv_kernels.das +++ b/modules/dasLLAMA/tests/test_metal_gemv_kernels.das @@ -23,12 +23,12 @@ require math // ===== single-stream kq GEMV (MetalKqGemvK4 / K5 / K5C / K6) ===== def private kq_gemv_gate(t : T?; dev, queue; vform : string; n, d : int) { - let fmt = vform == "k4" ? 4 : (vform == "k6" ? 6 : (vform == "iq4xs" ? 44 : (vform == "k3" ? 3 : (vform == "iq3s" ? 33 : (vform == "iq3xxs" ? 34 : (vform == "iq4nl" ? 45 : (vform == "iq2s" ? 23 : (vform == "k2" ? 2 : 5)))))))) + let fmt = vform == "k4" ? 4 : (vform == "k6" ? 6 : (vform == "iq4xs" ? 44 : (vform == "k3" ? 3 : (vform == "iq3s" ? 33 : (vform == "iq3xxs" ? 34 : (vform == "iq4nl" ? 45 : (vform == "iq2s" ? 23 : (vform == "iq2xs" ? 24 : (vform == "k2" ? 2 : 5))))))))) let tag = "kq_gemv_{vform} n={n} d={d}" var err : string - let src = vform == "iq2s" ? metal_kq_gemv_iq2s_msl : (vform == "k2" ? metal_kq_gemv_k2_msl : (vform == "iq4nl" ? metal_kq_gemv_iq4nl_msl : (vform == "iq3xxs" ? metal_kq_gemv_iq3xxs_msl : (vform == "iq3s" ? metal_kq_gemv_iq3s_msl : (vform == "k3" ? metal_kq_gemv_k3_msl : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl : (vform == "k4" ? metal_kq_gemv_k4_msl : (vform == "k5" ? metal_kq_gemv_k5_msl : (vform == "k5c" ? metal_kq_gemv_k5c_msl : metal_kq_gemv_k6_msl))))))))) - let entry = vform == "iq2s" ? metal_kq_gemv_iq2s_msl_entry : (vform == "k2" ? metal_kq_gemv_k2_msl_entry : (vform == "iq4nl" ? metal_kq_gemv_iq4nl_msl_entry : (vform == "iq3xxs" ? metal_kq_gemv_iq3xxs_msl_entry : (vform == "iq3s" ? metal_kq_gemv_iq3s_msl_entry : (vform == "k3" ? metal_kq_gemv_k3_msl_entry : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl_entry : (vform == "k4" ? metal_kq_gemv_k4_msl_entry : (vform == "k5" ? metal_kq_gemv_k5_msl_entry : (vform == "k5c" ? metal_kq_gemv_k5c_msl_entry : metal_kq_gemv_k6_msl_entry))))))))) - let fm = vform == "iq2s" ? metal_kq_gemv_iq2s_msl_fastmath : (vform == "k2" ? metal_kq_gemv_k2_msl_fastmath : (vform == "iq4nl" ? metal_kq_gemv_iq4nl_msl_fastmath : (vform == "iq3xxs" ? metal_kq_gemv_iq3xxs_msl_fastmath : (vform == "iq3s" ? metal_kq_gemv_iq3s_msl_fastmath : (vform == "k3" ? metal_kq_gemv_k3_msl_fastmath : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl_fastmath : (vform == "k4" ? metal_kq_gemv_k4_msl_fastmath : (vform == "k5" ? metal_kq_gemv_k5_msl_fastmath : (vform == "k5c" ? metal_kq_gemv_k5c_msl_fastmath : metal_kq_gemv_k6_msl_fastmath))))))))) + let src = vform == "iq2xs" ? metal_kq_gemv_iq2xs_msl : (vform == "iq2s" ? metal_kq_gemv_iq2s_msl : (vform == "k2" ? metal_kq_gemv_k2_msl : (vform == "iq4nl" ? metal_kq_gemv_iq4nl_msl : (vform == "iq3xxs" ? metal_kq_gemv_iq3xxs_msl : (vform == "iq3s" ? metal_kq_gemv_iq3s_msl : (vform == "k3" ? metal_kq_gemv_k3_msl : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl : (vform == "k4" ? metal_kq_gemv_k4_msl : (vform == "k5" ? metal_kq_gemv_k5_msl : (vform == "k5c" ? metal_kq_gemv_k5c_msl : metal_kq_gemv_k6_msl)))))))))) + let entry = vform == "iq2xs" ? metal_kq_gemv_iq2xs_msl_entry : (vform == "iq2s" ? metal_kq_gemv_iq2s_msl_entry : (vform == "k2" ? metal_kq_gemv_k2_msl_entry : (vform == "iq4nl" ? metal_kq_gemv_iq4nl_msl_entry : (vform == "iq3xxs" ? metal_kq_gemv_iq3xxs_msl_entry : (vform == "iq3s" ? metal_kq_gemv_iq3s_msl_entry : (vform == "k3" ? metal_kq_gemv_k3_msl_entry : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl_entry : (vform == "k4" ? metal_kq_gemv_k4_msl_entry : (vform == "k5" ? metal_kq_gemv_k5_msl_entry : (vform == "k5c" ? metal_kq_gemv_k5c_msl_entry : metal_kq_gemv_k6_msl_entry)))))))))) + let fm = vform == "iq2xs" ? metal_kq_gemv_iq2xs_msl_fastmath : (vform == "iq2s" ? metal_kq_gemv_iq2s_msl_fastmath : (vform == "k2" ? metal_kq_gemv_k2_msl_fastmath : (vform == "iq4nl" ? metal_kq_gemv_iq4nl_msl_fastmath : (vform == "iq3xxs" ? metal_kq_gemv_iq3xxs_msl_fastmath : (vform == "iq3s" ? metal_kq_gemv_iq3s_msl_fastmath : (vform == "k3" ? metal_kq_gemv_k3_msl_fastmath : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl_fastmath : (vform == "k4" ? metal_kq_gemv_k4_msl_fastmath : (vform == "k5" ? metal_kq_gemv_k5_msl_fastmath : (vform == "k5c" ? metal_kq_gemv_k5c_msl_fastmath : metal_kq_gemv_k6_msl_fastmath)))))))))) var pso = pipeline_from_source(dev, src, entry, fm, err) t |> success(pso != null, "{tag}: pipeline: {err}") return if (pso == null) @@ -66,7 +66,7 @@ def private kq_gemv_gate(t : T?; dev, queue; vform : string; n, d : int) { var by = buf_fill(dev, d, -1000.0) var bn = buf_u32(dev, uint(n)) var bd = buf_u32(dev, uint(d)) - let groups = vform == "k5c" ? (d + 1) / 2 : (vform == "iq3s" || vform == "iq3xxs" || vform == "iq2s" ? (d + 7) / 8 : (d + 3) / 4) + let groups = vform == "k5c" ? (d + 1) / 2 : (vform == "iq3s" || vform == "iq3xxs" || vform == "iq2s" || vform == "iq2xs" ? (d + 7) / 8 : (d + 3) / 4) let ran = with_compute_encoder(queue, err) $(enc : MetalComputeEncoder?) { metal_set_pipeline(enc, pso) if (vform == "iq4xs") { @@ -81,7 +81,7 @@ def private kq_gemv_gate(t : T?; dev, queue; vform : string; n, d : int) { if (vform == "iq4nl") { metal_set_threadgroup_memory_length(enc, metal_kq_gemv_iq4nl_msl_tgmem, 0) // the codebook slab } - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs/k3: the f16 d tail + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 || fmt == 24 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs/k3: the f16 d tail metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bx, 0ul, 3) @@ -114,15 +114,15 @@ def private kq_gemv_gate(t : T?; dev, queue; vform : string; n, d : int) { def private kq_mvb_gate(t : T?; dev, queue; fmt, bwidth, n, d, nr, ys : int) { let tag = "kq_mvb{bwidth}_k{fmt} n={n} d={d} nr={nr} ys={ys}" var err : string - let src = (bwidth == 2 ? (fmt == 23 ? MetalKqMvB2Iq2s_metal_kq_mv_iq2s_msl : (fmt == 45 ? MetalKqMvB2Iq4nl_metal_kq_mv_iq4nl_msl : (fmt == 2 ? MetalKqMvB2K2_metal_kq_mv_k2_msl : (fmt == 34 ? MetalKqMvB2Iq3xxs_metal_kq_mv_iq3xxs_msl : (fmt == 33 ? MetalKqMvB2Iq3s_metal_kq_mv_iq3s_msl : (fmt == 3 ? MetalKqMvB2K3_metal_kq_mv_k3_msl : (fmt == 44 ? MetalKqMvB2Iq4xs_metal_kq_mv_iq4xs_msl : (fmt == 4 ? MetalKqMvB2K4_metal_kq_mv_k4_msl : (fmt == 5 ? MetalKqMvB2K5_metal_kq_mv_k5_msl : MetalKqMvB2K6_metal_kq_mv_k6_msl))))))))) - : (bwidth == 4 ? (fmt == 23 ? MetalKqMvB4Iq2s_metal_kq_mv_iq2s_msl : (fmt == 45 ? MetalKqMvB4Iq4nl_metal_kq_mv_iq4nl_msl : (fmt == 2 ? MetalKqMvB4K2_metal_kq_mv_k2_msl : (fmt == 34 ? MetalKqMvB4Iq3xxs_metal_kq_mv_iq3xxs_msl : (fmt == 33 ? MetalKqMvB4Iq3s_metal_kq_mv_iq3s_msl : (fmt == 3 ? MetalKqMvB4K3_metal_kq_mv_k3_msl : (fmt == 44 ? MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl : (fmt == 4 ? MetalKqMvB4K4_metal_kq_mv_k4_msl : (fmt == 5 ? MetalKqMvB4K5_metal_kq_mv_k5_msl : MetalKqMvB4K6_metal_kq_mv_k6_msl))))))))) - : (fmt == 23 ? metal_kq_mvb8_iq2s_msl : (fmt == 45 ? metal_kq_mvb8_iq4nl_msl : (fmt == 2 ? metal_kq_mvb8_k2_msl : (fmt == 34 ? metal_kq_mvb8_iq3xxs_msl : (fmt == 33 ? metal_kq_mvb8_iq3s_msl : (fmt == 3 ? metal_kq_mvb8_k3_msl : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl : metal_kq_mvb8_k6_msl))))))))))) - let entry = (bwidth == 2 ? (fmt == 23 ? MetalKqMvB2Iq2s_metal_kq_mv_iq2s_msl_entry : (fmt == 45 ? MetalKqMvB2Iq4nl_metal_kq_mv_iq4nl_msl_entry : (fmt == 2 ? MetalKqMvB2K2_metal_kq_mv_k2_msl_entry : (fmt == 34 ? MetalKqMvB2Iq3xxs_metal_kq_mv_iq3xxs_msl_entry : (fmt == 33 ? MetalKqMvB2Iq3s_metal_kq_mv_iq3s_msl_entry : (fmt == 3 ? MetalKqMvB2K3_metal_kq_mv_k3_msl_entry : (fmt == 44 ? MetalKqMvB2Iq4xs_metal_kq_mv_iq4xs_msl_entry : (fmt == 4 ? MetalKqMvB2K4_metal_kq_mv_k4_msl_entry : (fmt == 5 ? MetalKqMvB2K5_metal_kq_mv_k5_msl_entry : MetalKqMvB2K6_metal_kq_mv_k6_msl_entry))))))))) - : (bwidth == 4 ? (fmt == 23 ? MetalKqMvB4Iq2s_metal_kq_mv_iq2s_msl_entry : (fmt == 45 ? MetalKqMvB4Iq4nl_metal_kq_mv_iq4nl_msl_entry : (fmt == 2 ? MetalKqMvB4K2_metal_kq_mv_k2_msl_entry : (fmt == 34 ? MetalKqMvB4Iq3xxs_metal_kq_mv_iq3xxs_msl_entry : (fmt == 33 ? MetalKqMvB4Iq3s_metal_kq_mv_iq3s_msl_entry : (fmt == 3 ? MetalKqMvB4K3_metal_kq_mv_k3_msl_entry : (fmt == 44 ? MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl_entry : (fmt == 4 ? MetalKqMvB4K4_metal_kq_mv_k4_msl_entry : (fmt == 5 ? MetalKqMvB4K5_metal_kq_mv_k5_msl_entry : MetalKqMvB4K6_metal_kq_mv_k6_msl_entry))))))))) - : (fmt == 23 ? metal_kq_mvb8_iq2s_msl_entry : (fmt == 45 ? metal_kq_mvb8_iq4nl_msl_entry : (fmt == 2 ? metal_kq_mvb8_k2_msl_entry : (fmt == 34 ? metal_kq_mvb8_iq3xxs_msl_entry : (fmt == 33 ? metal_kq_mvb8_iq3s_msl_entry : (fmt == 3 ? metal_kq_mvb8_k3_msl_entry : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl_entry : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_entry : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_entry : metal_kq_mvb8_k6_msl_entry))))))))))) - let fm = (bwidth == 2 ? (fmt == 23 ? MetalKqMvB2Iq2s_metal_kq_mv_iq2s_msl_fastmath : (fmt == 45 ? MetalKqMvB2Iq4nl_metal_kq_mv_iq4nl_msl_fastmath : (fmt == 2 ? MetalKqMvB2K2_metal_kq_mv_k2_msl_fastmath : (fmt == 34 ? MetalKqMvB2Iq3xxs_metal_kq_mv_iq3xxs_msl_fastmath : (fmt == 33 ? MetalKqMvB2Iq3s_metal_kq_mv_iq3s_msl_fastmath : (fmt == 3 ? MetalKqMvB2K3_metal_kq_mv_k3_msl_fastmath : (fmt == 44 ? MetalKqMvB2Iq4xs_metal_kq_mv_iq4xs_msl_fastmath : (fmt == 4 ? MetalKqMvB2K4_metal_kq_mv_k4_msl_fastmath : (fmt == 5 ? MetalKqMvB2K5_metal_kq_mv_k5_msl_fastmath : MetalKqMvB2K6_metal_kq_mv_k6_msl_fastmath))))))))) - : (bwidth == 4 ? (fmt == 23 ? MetalKqMvB4Iq2s_metal_kq_mv_iq2s_msl_fastmath : (fmt == 45 ? MetalKqMvB4Iq4nl_metal_kq_mv_iq4nl_msl_fastmath : (fmt == 2 ? MetalKqMvB4K2_metal_kq_mv_k2_msl_fastmath : (fmt == 34 ? MetalKqMvB4Iq3xxs_metal_kq_mv_iq3xxs_msl_fastmath : (fmt == 33 ? MetalKqMvB4Iq3s_metal_kq_mv_iq3s_msl_fastmath : (fmt == 3 ? MetalKqMvB4K3_metal_kq_mv_k3_msl_fastmath : (fmt == 44 ? MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl_fastmath : (fmt == 4 ? MetalKqMvB4K4_metal_kq_mv_k4_msl_fastmath : (fmt == 5 ? MetalKqMvB4K5_metal_kq_mv_k5_msl_fastmath : MetalKqMvB4K6_metal_kq_mv_k6_msl_fastmath))))))))) - : (fmt == 23 ? metal_kq_mvb8_iq2s_msl_fastmath : (fmt == 45 ? metal_kq_mvb8_iq4nl_msl_fastmath : (fmt == 2 ? metal_kq_mvb8_k2_msl_fastmath : (fmt == 34 ? metal_kq_mvb8_iq3xxs_msl_fastmath : (fmt == 33 ? metal_kq_mvb8_iq3s_msl_fastmath : (fmt == 3 ? metal_kq_mvb8_k3_msl_fastmath : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl_fastmath : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_fastmath : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_fastmath : metal_kq_mvb8_k6_msl_fastmath))))))))))) + let src = (bwidth == 2 ? (fmt == 24 ? MetalKqMvB2Iq2xs_metal_kq_mv_iq2xs_msl : (fmt == 23 ? MetalKqMvB2Iq2s_metal_kq_mv_iq2s_msl : (fmt == 45 ? MetalKqMvB2Iq4nl_metal_kq_mv_iq4nl_msl : (fmt == 2 ? MetalKqMvB2K2_metal_kq_mv_k2_msl : (fmt == 34 ? MetalKqMvB2Iq3xxs_metal_kq_mv_iq3xxs_msl : (fmt == 33 ? MetalKqMvB2Iq3s_metal_kq_mv_iq3s_msl : (fmt == 3 ? MetalKqMvB2K3_metal_kq_mv_k3_msl : (fmt == 44 ? MetalKqMvB2Iq4xs_metal_kq_mv_iq4xs_msl : (fmt == 4 ? MetalKqMvB2K4_metal_kq_mv_k4_msl : (fmt == 5 ? MetalKqMvB2K5_metal_kq_mv_k5_msl : MetalKqMvB2K6_metal_kq_mv_k6_msl)))))))))) + : (bwidth == 4 ? (fmt == 24 ? MetalKqMvB4Iq2xs_metal_kq_mv_iq2xs_msl : (fmt == 23 ? MetalKqMvB4Iq2s_metal_kq_mv_iq2s_msl : (fmt == 45 ? MetalKqMvB4Iq4nl_metal_kq_mv_iq4nl_msl : (fmt == 2 ? MetalKqMvB4K2_metal_kq_mv_k2_msl : (fmt == 34 ? MetalKqMvB4Iq3xxs_metal_kq_mv_iq3xxs_msl : (fmt == 33 ? MetalKqMvB4Iq3s_metal_kq_mv_iq3s_msl : (fmt == 3 ? MetalKqMvB4K3_metal_kq_mv_k3_msl : (fmt == 44 ? MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl : (fmt == 4 ? MetalKqMvB4K4_metal_kq_mv_k4_msl : (fmt == 5 ? MetalKqMvB4K5_metal_kq_mv_k5_msl : MetalKqMvB4K6_metal_kq_mv_k6_msl)))))))))) + : (fmt == 24 ? metal_kq_mvb8_iq2xs_msl : (fmt == 23 ? metal_kq_mvb8_iq2s_msl : (fmt == 45 ? metal_kq_mvb8_iq4nl_msl : (fmt == 2 ? metal_kq_mvb8_k2_msl : (fmt == 34 ? metal_kq_mvb8_iq3xxs_msl : (fmt == 33 ? metal_kq_mvb8_iq3s_msl : (fmt == 3 ? metal_kq_mvb8_k3_msl : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl : metal_kq_mvb8_k6_msl)))))))))))) + let entry = (bwidth == 2 ? (fmt == 24 ? MetalKqMvB2Iq2xs_metal_kq_mv_iq2xs_msl_entry : (fmt == 23 ? MetalKqMvB2Iq2s_metal_kq_mv_iq2s_msl_entry : (fmt == 45 ? MetalKqMvB2Iq4nl_metal_kq_mv_iq4nl_msl_entry : (fmt == 2 ? MetalKqMvB2K2_metal_kq_mv_k2_msl_entry : (fmt == 34 ? MetalKqMvB2Iq3xxs_metal_kq_mv_iq3xxs_msl_entry : (fmt == 33 ? MetalKqMvB2Iq3s_metal_kq_mv_iq3s_msl_entry : (fmt == 3 ? MetalKqMvB2K3_metal_kq_mv_k3_msl_entry : (fmt == 44 ? MetalKqMvB2Iq4xs_metal_kq_mv_iq4xs_msl_entry : (fmt == 4 ? MetalKqMvB2K4_metal_kq_mv_k4_msl_entry : (fmt == 5 ? MetalKqMvB2K5_metal_kq_mv_k5_msl_entry : MetalKqMvB2K6_metal_kq_mv_k6_msl_entry)))))))))) + : (bwidth == 4 ? (fmt == 24 ? MetalKqMvB4Iq2xs_metal_kq_mv_iq2xs_msl_entry : (fmt == 23 ? MetalKqMvB4Iq2s_metal_kq_mv_iq2s_msl_entry : (fmt == 45 ? MetalKqMvB4Iq4nl_metal_kq_mv_iq4nl_msl_entry : (fmt == 2 ? MetalKqMvB4K2_metal_kq_mv_k2_msl_entry : (fmt == 34 ? MetalKqMvB4Iq3xxs_metal_kq_mv_iq3xxs_msl_entry : (fmt == 33 ? MetalKqMvB4Iq3s_metal_kq_mv_iq3s_msl_entry : (fmt == 3 ? MetalKqMvB4K3_metal_kq_mv_k3_msl_entry : (fmt == 44 ? MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl_entry : (fmt == 4 ? MetalKqMvB4K4_metal_kq_mv_k4_msl_entry : (fmt == 5 ? MetalKqMvB4K5_metal_kq_mv_k5_msl_entry : MetalKqMvB4K6_metal_kq_mv_k6_msl_entry)))))))))) + : (fmt == 24 ? metal_kq_mvb8_iq2xs_msl_entry : (fmt == 23 ? metal_kq_mvb8_iq2s_msl_entry : (fmt == 45 ? metal_kq_mvb8_iq4nl_msl_entry : (fmt == 2 ? metal_kq_mvb8_k2_msl_entry : (fmt == 34 ? metal_kq_mvb8_iq3xxs_msl_entry : (fmt == 33 ? metal_kq_mvb8_iq3s_msl_entry : (fmt == 3 ? metal_kq_mvb8_k3_msl_entry : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl_entry : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_entry : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_entry : metal_kq_mvb8_k6_msl_entry)))))))))))) + let fm = (bwidth == 2 ? (fmt == 24 ? MetalKqMvB2Iq2xs_metal_kq_mv_iq2xs_msl_fastmath : (fmt == 23 ? MetalKqMvB2Iq2s_metal_kq_mv_iq2s_msl_fastmath : (fmt == 45 ? MetalKqMvB2Iq4nl_metal_kq_mv_iq4nl_msl_fastmath : (fmt == 2 ? MetalKqMvB2K2_metal_kq_mv_k2_msl_fastmath : (fmt == 34 ? MetalKqMvB2Iq3xxs_metal_kq_mv_iq3xxs_msl_fastmath : (fmt == 33 ? MetalKqMvB2Iq3s_metal_kq_mv_iq3s_msl_fastmath : (fmt == 3 ? MetalKqMvB2K3_metal_kq_mv_k3_msl_fastmath : (fmt == 44 ? MetalKqMvB2Iq4xs_metal_kq_mv_iq4xs_msl_fastmath : (fmt == 4 ? MetalKqMvB2K4_metal_kq_mv_k4_msl_fastmath : (fmt == 5 ? MetalKqMvB2K5_metal_kq_mv_k5_msl_fastmath : MetalKqMvB2K6_metal_kq_mv_k6_msl_fastmath)))))))))) + : (bwidth == 4 ? (fmt == 24 ? MetalKqMvB4Iq2xs_metal_kq_mv_iq2xs_msl_fastmath : (fmt == 23 ? MetalKqMvB4Iq2s_metal_kq_mv_iq2s_msl_fastmath : (fmt == 45 ? MetalKqMvB4Iq4nl_metal_kq_mv_iq4nl_msl_fastmath : (fmt == 2 ? MetalKqMvB4K2_metal_kq_mv_k2_msl_fastmath : (fmt == 34 ? MetalKqMvB4Iq3xxs_metal_kq_mv_iq3xxs_msl_fastmath : (fmt == 33 ? MetalKqMvB4Iq3s_metal_kq_mv_iq3s_msl_fastmath : (fmt == 3 ? MetalKqMvB4K3_metal_kq_mv_k3_msl_fastmath : (fmt == 44 ? MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl_fastmath : (fmt == 4 ? MetalKqMvB4K4_metal_kq_mv_k4_msl_fastmath : (fmt == 5 ? MetalKqMvB4K5_metal_kq_mv_k5_msl_fastmath : MetalKqMvB4K6_metal_kq_mv_k6_msl_fastmath)))))))))) + : (fmt == 24 ? metal_kq_mvb8_iq2xs_msl_fastmath : (fmt == 23 ? metal_kq_mvb8_iq2s_msl_fastmath : (fmt == 45 ? metal_kq_mvb8_iq4nl_msl_fastmath : (fmt == 2 ? metal_kq_mvb8_k2_msl_fastmath : (fmt == 34 ? metal_kq_mvb8_iq3xxs_msl_fastmath : (fmt == 33 ? metal_kq_mvb8_iq3s_msl_fastmath : (fmt == 3 ? metal_kq_mvb8_k3_msl_fastmath : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl_fastmath : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_fastmath : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_fastmath : metal_kq_mvb8_k6_msl_fastmath)))))))))))) var pso = pipeline_from_source(dev, src, entry, fm, err) t |> success(pso != null, "{tag}: pipeline: {err}") return if (pso == null) @@ -172,9 +172,9 @@ def private kq_mvb_gate(t : T?; dev, queue; fmt, bwidth, n, d, nr, ys : int) { metal_set_pipeline(enc, pso) if (bwidth == 8) { metal_set_threadgroup_memory_length(enc, - fmt == 23 ? metal_kq_mvb8_iq2s_msl_tgmem : (fmt == 2 ? metal_kq_mvb8_k2_msl_tgmem : (fmt == 45 ? metal_kq_mvb8_iq4nl_msl_tgmem : (fmt == 34 ? metal_kq_mvb8_iq3xxs_msl_tgmem : (fmt == 33 ? metal_kq_mvb8_iq3s_msl_tgmem : (fmt == 3 ? metal_kq_mvb8_k3_msl_tgmem : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl_tgmem : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_tgmem : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_tgmem : metal_kq_mvb8_k6_msl_tgmem)))))))), 0) + fmt == 24 ? metal_kq_mvb8_iq2xs_msl_tgmem : (fmt == 23 ? metal_kq_mvb8_iq2s_msl_tgmem : (fmt == 2 ? metal_kq_mvb8_k2_msl_tgmem : (fmt == 45 ? metal_kq_mvb8_iq4nl_msl_tgmem : (fmt == 34 ? metal_kq_mvb8_iq3xxs_msl_tgmem : (fmt == 33 ? metal_kq_mvb8_iq3s_msl_tgmem : (fmt == 3 ? metal_kq_mvb8_k3_msl_tgmem : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl_tgmem : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_tgmem : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_tgmem : metal_kq_mvb8_k6_msl_tgmem))))))))), 0) } - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs/k3: the f16 d tail + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 || fmt == 24 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs/k3: the f16 d tail metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bx, 0ul, 3) @@ -263,7 +263,7 @@ def private moe_kq_gemv_gate(t : T?; dev, queue; fmt, n, d, net, k, nst : int; p bxs = uint((perslot ? k : 1) * n), bys = uint(k * d), hasb = 0u) let ran = with_compute_encoder(queue, err) $(enc : MetalComputeEncoder?) { metal_set_pipeline(enc, pso) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs/k3: the f16 d tail + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 || fmt == 24 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs/k3: the f16 d tail metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bx, 0ul, 3) @@ -747,8 +747,10 @@ def test_metal_kq_gemv_kernels(t : T?) { kq_gemv_gate(t, dev, queue, "k2", 512, 30) kq_gemv_gate(t, dev, queue, "iq2s", 1280, 11) kq_gemv_gate(t, dev, queue, "iq2s", 512, 30) + kq_gemv_gate(t, dev, queue, "iq2xs", 1280, 11) + kq_gemv_gate(t, dev, queue, "iq2xs", 512, 30) // the small-batch twins: nr = 2 / 3 (b4 col guard) / 6 (b8 pad), ys > d - for (fmt in [4, 5, 6, 44, 3, 33, 34, 45, 2, 23]) { + for (fmt in [4, 5, 6, 44, 3, 33, 34, 45, 2, 23, 24]) { kq_mvb_gate(t, dev, queue, fmt, 2, 768, 17, 2, 22) kq_mvb_gate(t, dev, queue, fmt, 4, 768, 17, 3, 22) kq_mvb_gate(t, dev, queue, fmt, 8, 768, 17, 6, 22) From 810c539fc6bb0377f5c48dbfdb2178fce423c11f Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Mon, 31 Aug 2026 13:55:19 -0700 Subject: [PATCH 45/61] dasLLAMA: IQ2_XXS joins the kq lattice (CPU slice) - the LAST format iq3xxs's aux32 machinery over the iq2xs-style u64 grid: enum iq2xxs (14), kernel/stream 25, 66B disk = [f16 d][8 x [4 byte indices into iq2xxs_grid[256]][aux32: four 7-bit ksigns + per-32 ls nibble]]. The 64B qs region rides VERBATIM as the quant plane (aux in-plane, 16 grp columns); the scale row is the iq3xxs shape (d EIGHTH-ed, 8 per-32 (1+2s) strips). Full CPU walk: transcode/dequant/dot/rows/grp dot/ repack, every dispatch ladder, declined emitter stubs, the tune-list row, and test arms across every gate plus a synthetic plane gate recomposing through grid + the shared ksigns. Most of the walk was GENERATED - a twinning script replayed the IQ2_XS Phase A commit's pure-insert hunks under iq2xs->iq2xxs / 24->25 maps (~500 of ~900 lines); hand work was the aux32 codec bodies, the 512-word grid, and the chain tails. The gates caught the three real traps (HOW_TO Phase A note): the PACKED fmt-25 arms in kq_batch_cell_gen and the test tile ladder (the fmt-24 template is a panel arm; the miss segfaults JIT-only), the grid fn's direct fixed_array return (a let-local round trip broke the JIT cmres), and gen_tune_probe registration (the --tune-only re-mint's "no family matches" doubled as the registration gate). Gates: test_kqformat 18/18, test_kquant interp 248 pass / -jit 263 pass, 0 failed, probe GEN TUNE TEST OK, lint 0 (20 files). QUIRK 15 pinned "reference" via --tune-only iq2xxsq8_tile_gen (1 of 15 families, seconds - the filter's first Phase A). E2e: parity ids 64/64 vs llama.cpp - the THIRD consecutive full-match greedy stream. JIT emitter, Vulkan, Metal: pending. Co-Authored-By: Claude Fable 5 --- modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md | 26 ++++ modules/dasLLAMA/dasllama/dasllama_blocks.das | 2 +- modules/dasLLAMA/dasllama/dasllama_common.das | 39 ++++- modules/dasLLAMA/dasllama/dasllama_config.das | 3 +- .../dasLLAMA/dasllama/dasllama_convert.das | 49 +++++++ .../dasLLAMA/dasllama/dasllama_gemm_gen.das | 10 ++ .../dasllama/dasllama_gemm_schema.das | 2 + modules/dasLLAMA/dasllama/dasllama_gguf.das | 43 ++++++ .../dasllama/dasllama_gpu_resident.das | 9 +- modules/dasLLAMA/dasllama/dasllama_image.das | 7 +- .../dasLLAMA/dasllama/dasllama_kqformat.das | 93 +++++++++++- modules/dasLLAMA/dasllama/dasllama_layout.das | 22 ++- modules/dasLLAMA/dasllama/dasllama_load.das | 41 +++++- modules/dasLLAMA/dasllama/dasllama_math.das | 9 +- .../dasllama/dasllama_math_default.das | 87 ++++++++++++ .../dasLLAMA/dasllama/dasllama_math_gen.das | 106 +++++++++++++- modules/dasLLAMA/dasllama/dasllama_ple.das | 4 + modules/dasLLAMA/dasllama/dasllama_repack.das | 44 ++++++ modules/dasLLAMA/harness/gen_tune_probe.das | 48 +++++-- modules/dasLLAMA/tests/test_kqformat.das | 12 +- modules/dasLLAMA/tests/test_kquant.das | 133 +++++++++++++++--- 21 files changed, 732 insertions(+), 57 deletions(-) diff --git a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md index 8b9580b101..51945a816d 100644 --- a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md +++ b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md @@ -472,6 +472,32 @@ where, why it is so today, what unquirked looks like. An empty ledger is a legit ## Per-format notes +### IQ2_XXS Phase A (CPU, 2026-08-31) - the LAST format: iq3xxs's aux32 over a 256-entry u64 grid + +Shape: 66B disk = [f16 d][8 blocks x 8B], each block = aux32[0] (4 BYTE indices into +iq2xxs_grid[256], u64 entries, magnitudes {8, 25, 43}) + aux32[1] (four 7-bit KSIGNS_IQ2XS +indices + the per-32 scale nibble in the top 4 bits) - iq3xxs's sign/scale machinery over the +iq2xs-style two-word u64 grid. Planes: the 64B qs region VERBATIM (aux stays in-plane; +IQ2XXS_QSB 64, 16 uniform grp columns - column 2b grid bytes, 2b+1 aux32); scale row = the +iq3xxs shape (d EIGHTH-ed, 8 per-32 (1+2s) strips + 8 pad, IQ2XXS_SSB 20). Ids: enum +iq2xxs = 14, kernel/stream 25, GGML_TYPE_IQ2_XXS = 16, IMAGE_VERSION 26. Vehicle: local +requant again - census iq2_xxs x94 + q4_K x16 + q2_K x2 + q5_K embd, the exact IQ2_XS mix. + +MOST of the walk was GENERATED: a twinning script replayed the IQ2_XS Phase A commit's +pure-insert hunks with iq2xs->iq2xxs / 24->25 substitutions (~500 of ~900 inserted lines); +the hand work was the codec bodies (transcode/dequant/dot/grp-dot: the aux32 walk), the +512-word grid table, and the chain-tail extensions the twin cannot express. TRAPS the gates +caught: kq_batch_cell_gen and the tests' tile ladder needed their PACKED fmt-25 arms BY HAND +(the fmt-24 template is a panel arm - a missing cell arm falls to k6's panel path and +segfaults JIT-ONLY, since the groupn gate skips off-JIT); the grid fn must `return +fixed_array(...)` DIRECTLY (a let-local round trip crashed the JIT's cmres return); +the generated per-format sections in gen_tune_probe silently kept iq2xs sizes until the +filtered mint reported "no generator family matches" - the --tune-only re-mint doubles as a +REGISTRATION gate. Gates: kqformat 18/18, kquant interp 248/0 + -jit 263/0, GEN TUNE TEST +OK, lint 0 (20 files). QUIRK 15 pinned via `--tune-only iq2xxsq8_tile_gen` (1 of 15 families, +seconds). E2e: parity ids 64/64 vs llama.cpp - the THIRD consecutive full-match stream. +JIT emitter, Vulkan, Metal: pending. + ### IQ2_XS Phase A (CPU, 2026-08-31) - the ksigns u64 tier Shape: 256-superblock grid format - each of the 32 u16 qs words carries a 9-bit index into diff --git a/modules/dasLLAMA/dasllama/dasllama_blocks.das b/modules/dasLLAMA/dasllama/dasllama_blocks.das index c0a9d860c7..06bf167997 100644 --- a/modules/dasLLAMA/dasllama/dasllama_blocks.das +++ b/modules/dasLLAMA/dasllama/dasllama_blocks.das @@ -1421,7 +1421,7 @@ def private kq_bytes_per_weight(f : KqFmt) : float { if (f == KqFmt.iq2s) { return 92.0 / 256.0 // 72B quants + the 20B scale row per 256 } - if (f == KqFmt.iq2xs) { + if (f == KqFmt.iq2xs || f == KqFmt.iq2xxs) { return 84.0 / 256.0 // 64B quants + the 20B scale row per 256 } return 34.0 / 32.0 // q8: 32 quants + an f16 scale per block diff --git a/modules/dasLLAMA/dasllama/dasllama_common.das b/modules/dasLLAMA/dasllama/dasllama_common.das index d52b01f196..4655d97f14 100644 --- a/modules/dasLLAMA/dasllama/dasllama_common.das +++ b/modules/dasLLAMA/dasllama/dasllama_common.das @@ -859,6 +859,9 @@ struct Model { // the iq2xs tier's plane pair: 32 u16 qs words per superblock + the 20B row (f16 d/8, 16 x (1 + 2s)) iq2xsq : array iq2xss : array + // the iq2xxs tier's plane pair: 32 u16 qs words per superblock + the 20B row (f16 d/8, 16 x (1 + 2s)) + iq2xxsq : array + iq2xxss : array kquant_native : bool = false // kq stage 4: the kq planes were repacked at load into the active backend's grp layout. // From then on every kq matmul MUST run the backend slots — disk-order portable kernels would @@ -877,6 +880,7 @@ struct Model { kq_repack_mr2 : int64 = 4l kq_repack_mr23 : int64 = 4l kq_repack_mr24 : int64 = 4l + kq_repack_mr25 : int64 = 4l // per-layer weight format tags (KqFmt; empty = all q8). Filled by detect_kq_formats before // layout_offsets so the layout walks each tensor into its format's cursor. wq_fmt : array @@ -1102,7 +1106,7 @@ def model_weights_bytes(t : Model) : int64 { + long_length(t.iq4xsq) + long_length(t.iq4xss) + long_length(t.k3q) + long_length(t.k3s) + long_length(t.iq3sq) + long_length(t.iq3ss) + long_length(t.iq3xxsq) + long_length(t.iq3xxss) - + long_length(t.iq4nlq) + long_length(t.iq4nls) + long_length(t.k2q) + long_length(t.k2s) + long_length(t.iq2sq) + long_length(t.iq2ss) + long_length(t.iq2xsq) + long_length(t.iq2xss) + + long_length(t.iq4nlq) + long_length(t.iq4nls) + long_length(t.k2q) + long_length(t.k2s) + long_length(t.iq2sq) + long_length(t.iq2ss) + long_length(t.iq2xsq) + long_length(t.iq2xss) + long_length(t.iq2xxsq) + long_length(t.iq2xxss) + long_length(t.q51q) + long_length(t.q51s) + long_length(t.mblob) + long_length(t.vkblob) + long_length(t.embq) + long_length(t.embs)) @@ -1499,6 +1503,7 @@ def private dlim_cpu_source_impl(var c : DlimCpuConfig) { c.kq_mr2 = active_kq_layout_mr(2) c.kq_mr23 = active_kq_layout_mr(23) c.kq_mr24 = active_kq_layout_mr(24) + c.kq_mr25 = active_kq_layout_mr(25) c.q51_mr = active_q51_layout_mr() // OUTCOME, not request: a backend without s16 twins keeps f32 scale planes (wscale_convert_f16) c.wscale_f16 = g_wscale_f16 && kernel_backend_has_wscale16() @@ -1591,7 +1596,7 @@ def kq_active_mr(t : Model; f : KqFmt) : int64 { if (!t.kq_repacked) { return 1l } - return f == KqFmt.k4 ? t.kq_repack_mr4 : (f == KqFmt.k5 ? t.kq_repack_mr5 : (f == KqFmt.k6 ? t.kq_repack_mr6 : (f == KqFmt.q40 ? t.kq_repack_mr40 : (f == KqFmt.iq4xs ? t.kq_repack_mr44 : (f == KqFmt.k3 ? t.kq_repack_mr3 : (f == KqFmt.iq3s ? t.kq_repack_mr33 : (f == KqFmt.iq3xxs ? t.kq_repack_mr34 : (f == KqFmt.iq4nl ? t.kq_repack_mr45 : (f == KqFmt.k2 ? t.kq_repack_mr2 : (f == KqFmt.iq2s ? t.kq_repack_mr23 : t.kq_repack_mr24)))))))))) + return f == KqFmt.k4 ? t.kq_repack_mr4 : (f == KqFmt.k5 ? t.kq_repack_mr5 : (f == KqFmt.k6 ? t.kq_repack_mr6 : (f == KqFmt.q40 ? t.kq_repack_mr40 : (f == KqFmt.iq4xs ? t.kq_repack_mr44 : (f == KqFmt.k3 ? t.kq_repack_mr3 : (f == KqFmt.iq3s ? t.kq_repack_mr33 : (f == KqFmt.iq3xxs ? t.kq_repack_mr34 : (f == KqFmt.iq4nl ? t.kq_repack_mr45 : (f == KqFmt.k2 ? t.kq_repack_mr2 : (f == KqFmt.iq2s ? t.kq_repack_mr23 : (f == KqFmt.iq2xs ? t.kq_repack_mr24 : t.kq_repack_mr25))))))))))) } //! Load a llama-architecture GGUF into the split layout at the requested precision — straight into @@ -1654,8 +1659,8 @@ def footprint(t : Model) : MemFootprint { let q4s = long_length(t.q4scales) * 4l let mx4 = long_length(t.mxq) // uint8 (two nibbles) -> 1 byte each let mx4s = long_length(t.mxs) // raw E8M0 -> 1 byte per 32-block - let kq = long_length(t.k4q) + long_length(t.k5q) + long_length(t.k6q) + long_length(t.q40q) + long_length(t.q51q) + long_length(t.iq4xsq) + long_length(t.k3q) + long_length(t.iq3sq) + long_length(t.iq3xxsq) + long_length(t.iq4nlq) + long_length(t.k2q) + long_length(t.iq2sq) + long_length(t.iq2xsq) - let kqs = long_length(t.k4s) + long_length(t.k5s) + long_length(t.k6s) + long_length(t.q40s) + long_length(t.q51s) + long_length(t.iq4xss) + long_length(t.k3s) + long_length(t.iq3ss) + long_length(t.iq3xxss) + long_length(t.iq4nls) + long_length(t.k2s) + long_length(t.iq2ss) + long_length(t.iq2xss) + let kq = long_length(t.k4q) + long_length(t.k5q) + long_length(t.k6q) + long_length(t.q40q) + long_length(t.q51q) + long_length(t.iq4xsq) + long_length(t.k3q) + long_length(t.iq3sq) + long_length(t.iq3xxsq) + long_length(t.iq4nlq) + long_length(t.k2q) + long_length(t.iq2sq) + long_length(t.iq2xsq) + long_length(t.iq2xxsq) + let kqs = long_length(t.k4s) + long_length(t.k5s) + long_length(t.k6s) + long_length(t.q40s) + long_length(t.q51s) + long_length(t.iq4xss) + long_length(t.k3s) + long_length(t.iq3ss) + long_length(t.iq3xxss) + long_length(t.iq4nls) + long_length(t.k2s) + long_length(t.iq2ss) + long_length(t.iq2xss) + long_length(t.iq2xxss) return MemFootprint(aux_fp32_bytes = aux, weight_fp32_bytes = wf, q8_bytes = q8, q8_scale_bytes = q8s, q4_bytes = q4, q4_scale_bytes = q4s, mx4_bytes = mx4, mx4_scale_bytes = mx4s, kq_bytes = kq, kq_scale_bytes = kqs, @@ -3329,7 +3334,7 @@ def mm_b_q51_groupn(var y : array; t : Model; offs : array; nregio // KqFmt -> the kernel-layer format id (matmul_kq*/kq_rows_fn/kq_qsb take the int form). q8 maps // to 0, a poison id no kq kernel accepts — kq_rows_for calls this for EVERY tag, so it must stay // total even though q8 branches never read the result. -def private kq_fi(fmt : KqFmt) : int => fmt == KqFmt.k4 ? 4 : (fmt == KqFmt.k5 ? 5 : (fmt == KqFmt.k6 ? 6 : (fmt == KqFmt.q40 ? 40 : (fmt == KqFmt.iq4xs ? 44 : (fmt == KqFmt.k3 ? 3 : (fmt == KqFmt.iq3s ? 33 : (fmt == KqFmt.iq3xxs ? 34 : (fmt == KqFmt.iq4nl ? 45 : (fmt == KqFmt.k2 ? 2 : (fmt == KqFmt.iq2s ? 23 : (fmt == KqFmt.iq2xs ? 24 : 0))))))))))) +def private kq_fi(fmt : KqFmt) : int => fmt == KqFmt.k4 ? 4 : (fmt == KqFmt.k5 ? 5 : (fmt == KqFmt.k6 ? 6 : (fmt == KqFmt.q40 ? 40 : (fmt == KqFmt.iq4xs ? 44 : (fmt == KqFmt.k3 ? 3 : (fmt == KqFmt.iq3s ? 33 : (fmt == KqFmt.iq3xxs ? 34 : (fmt == KqFmt.iq4nl ? 45 : (fmt == KqFmt.k2 ? 2 : (fmt == KqFmt.iq2s ? 23 : (fmt == KqFmt.iq2xs ? 24 : (fmt == KqFmt.iq2xxs ? 25 : 0)))))))))))) // K-quant GEMV against a PRE-quantized activation (xq/xs/xbs already filled by the bs quantizer). // kq_repacked loads run the backend's stamped cores; disk-order loads run the portable kernels. @@ -3359,6 +3364,8 @@ def mm_at_kq_pre(var y : array; t : Model; fmt : KqFmt; woff : int64; xq matmul_kq_active(23, y, t.iq2sq, t.iq2ss, woff, xq, xs, xbs, n, d, yoff) } elif (fmt == KqFmt.iq2xs) { matmul_kq_active(24, y, t.iq2xsq, t.iq2xss, woff, xq, xs, xbs, n, d, yoff) + } elif (fmt == KqFmt.iq2xxs) { + matmul_kq_active(25, y, t.iq2xxsq, t.iq2xxss, woff, xq, xs, xbs, n, d, yoff) } else { panic("mm_at_kq_pre: '{fmt}' has no kq plane pair") // a new KqFmt must claim its arm, never ride q40's } @@ -3386,6 +3393,8 @@ def mm_at_kq_pre(var y : array; t : Model; fmt : KqFmt; woff : int64; xq matmul_kq(23, y, t.iq2sq, t.iq2ss, woff, xq, xs, xbs, n, d, yoff) } elif (fmt == KqFmt.iq2xs) { matmul_kq(24, y, t.iq2xsq, t.iq2xss, woff, xq, xs, xbs, n, d, yoff) + } elif (fmt == KqFmt.iq2xxs) { + matmul_kq(25, y, t.iq2xxsq, t.iq2xxss, woff, xq, xs, xbs, n, d, yoff) } else { panic("mm_at_kq_pre: '{fmt}' has no kq plane pair") } @@ -3433,6 +3442,8 @@ def mm_b_kq(var s : Session; var y : array; t : Model; fmt : KqFmt; woff matmul_kq_batch(23, y, t.iq2sq, t.iq2ss, woff, s.xqb, s.xsb, s.xbsb, n, d, npos) } elif (fmt == KqFmt.iq2xs) { matmul_kq_batch(24, y, t.iq2xsq, t.iq2xss, woff, s.xqb, s.xsb, s.xbsb, n, d, npos) + } elif (fmt == KqFmt.iq2xxs) { + matmul_kq_batch(25, y, t.iq2xxsq, t.iq2xxss, woff, s.xqb, s.xsb, s.xbsb, n, d, npos) } else { panic("mm_b_kq: '{fmt}' has no kq plane pair") } @@ -3476,6 +3487,8 @@ def mm_at_kq_groupn(var y : array; t : Model; fmt : KqFmt; offs : array; t : Model; fmt : KqFmt; offs : array; t : Model; fmt : KqFmt; offs : array; t : Model; fmt : KqFmt; woff : int64; xq : matmul_kq_batch(23, y, t.iq2sq, t.iq2ss, woff, xq, xs, xbs, n, d, ntok) } elif (fmt == KqFmt.iq2xs) { matmul_kq_batch(24, y, t.iq2xsq, t.iq2xss, woff, xq, xs, xbs, n, d, ntok) + } elif (fmt == KqFmt.iq2xxs) { + matmul_kq_batch(25, y, t.iq2xxsq, t.iq2xxss, woff, xq, xs, xbs, n, d, ntok) } else { panic("mm_b_kq_pre: '{fmt}' has no kq plane pair") } @@ -3589,6 +3608,7 @@ def kq_plane_q(t : Model; fmt : KqFmt; woff : int64) : uint8 const? { if (fmt == KqFmt.k2) return unsafe(addr(t.k2q[sb * K2_QSB])) if (fmt == KqFmt.iq2s) return unsafe(addr(t.iq2sq[sb * IQ2S_QSB])) if (fmt == KqFmt.iq2xs) return unsafe(addr(t.iq2xsq[sb * IQ2XS_QSB])) + if (fmt == KqFmt.iq2xxs) return unsafe(addr(t.iq2xxsq[sb * IQ2XXS_QSB])) return null } @@ -3606,6 +3626,7 @@ def kq_plane_s(t : Model; fmt : KqFmt; woff : int64) : uint8 const? { if (fmt == KqFmt.k2) return unsafe(addr(t.k2s[sb * K2_SSB])) if (fmt == KqFmt.iq2s) return unsafe(addr(t.iq2ss[sb * IQ2S_SSB])) if (fmt == KqFmt.iq2xs) return unsafe(addr(t.iq2xss[sb * IQ2XS_SSB])) + if (fmt == KqFmt.iq2xxs) return unsafe(addr(t.iq2xxss[sb * IQ2XXS_SSB])) return null } @@ -3982,6 +4003,8 @@ def embed_row(t : Model; token : int64; var dst : float?) { // nolint:STYLE037 dequant_kq_row_grp(fmt, addr(t.embq[sbg * IQ2S_QSB]), addr(t.embs[sbg * IQ2S_SSB]), token % mr, mr, dim, dst) } elif (t.emb_fmt == KqFmt.iq2xs) { dequant_kq_row_grp(fmt, addr(t.embq[sbg * IQ2XS_QSB]), addr(t.embs[sbg * IQ2XS_SSB]), token % mr, mr, dim, dst) + } elif (t.emb_fmt == KqFmt.iq2xxs) { + dequant_kq_row_grp(fmt, addr(t.embq[sbg * IQ2XXS_QSB]), addr(t.embs[sbg * IQ2XXS_SSB]), token % mr, mr, dim, dst) } else { panic("embed_row: '{t.emb_fmt}' has no kq plane pair") } @@ -4009,6 +4032,8 @@ def embed_row(t : Model; token : int64; var dst : float?) { // nolint:STYLE037 dequant_kq_row_grp(fmt, addr(t.iq2sq[sbg * IQ2S_QSB]), addr(t.iq2ss[sbg * IQ2S_SSB]), token % mr, mr, dim, dst) } elif (t.emb_fmt == KqFmt.iq2xs) { dequant_kq_row_grp(fmt, addr(t.iq2xsq[sbg * IQ2XS_QSB]), addr(t.iq2xss[sbg * IQ2XS_SSB]), token % mr, mr, dim, dst) + } elif (t.emb_fmt == KqFmt.iq2xxs) { + dequant_kq_row_grp(fmt, addr(t.iq2xxsq[sbg * IQ2XXS_QSB]), addr(t.iq2xxss[sbg * IQ2XXS_SSB]), token % mr, mr, dim, dst) } else { panic("embed_row: '{t.emb_fmt}' has no kq plane pair") } @@ -4047,6 +4072,8 @@ def embed_row(t : Model; token : int64; var dst : float?) { // nolint:STYLE037 dequant_iq2s_plane_superblock(t.embq, (sb0 + s) * IQ2S_QSB, t.embs, (sb0 + s) * IQ2S_SSB, row, s * 256l) } elif (t.emb_fmt == KqFmt.iq2xs) { dequant_iq2xs_plane_superblock(t.embq, (sb0 + s) * IQ2XS_QSB, t.embs, (sb0 + s) * IQ2XS_SSB, row, s * 256l) + } elif (t.emb_fmt == KqFmt.iq2xxs) { + dequant_iq2xxs_plane_superblock(t.embq, (sb0 + s) * IQ2XXS_QSB, t.embs, (sb0 + s) * IQ2XXS_SSB, row, s * 256l) } else { panic("embed_row: '{t.emb_fmt}' has no kq plane pair") } @@ -4078,6 +4105,8 @@ def embed_row(t : Model; token : int64; var dst : float?) { // nolint:STYLE037 dequant_iq2s_plane_superblock(t.iq2sq, (sb0 + s) * IQ2S_QSB, t.iq2ss, (sb0 + s) * IQ2S_SSB, row, s * 256l) } elif (t.emb_fmt == KqFmt.iq2xs) { dequant_iq2xs_plane_superblock(t.iq2xsq, (sb0 + s) * IQ2XS_QSB, t.iq2xss, (sb0 + s) * IQ2XS_SSB, row, s * 256l) + } elif (t.emb_fmt == KqFmt.iq2xxs) { + dequant_iq2xxs_plane_superblock(t.iq2xxsq, (sb0 + s) * IQ2XXS_QSB, t.iq2xxss, (sb0 + s) * IQ2XXS_SSB, row, s * 256l) } else { panic("embed_row: '{t.emb_fmt}' has no kq plane pair") } diff --git a/modules/dasLLAMA/dasllama/dasllama_config.das b/modules/dasLLAMA/dasllama/dasllama_config.das index b141cdf027..82c7507b1d 100644 --- a/modules/dasLLAMA/dasllama/dasllama_config.das +++ b/modules/dasLLAMA/dasllama/dasllama_config.das @@ -35,6 +35,7 @@ struct public DlimCpuConfig { kq_mr2 : int64 kq_mr23 : int64 kq_mr24 : int64 + kq_mr25 : int64 q51_mr : int64 wscale_f16 : bool kquant_native : bool @@ -147,7 +148,7 @@ def public dlim_config_current(quant : string = "q8") : DlimConfiguration { def public dlim_identity(c : DlimConfiguration; image_version : int; tag : string = "") : string { return ("v{image_version}|{c.quant}|{c.cpu.backend}|{c.cpu.wscale_f16 ? "s16" : "s32"}" + "|q8 mr{c.cpu.q8_mr} b{c.cpu.q8_wbias} g{c.cpu.q8_kgroup}" - + "|kq {c.cpu.kq_mr4}/{c.cpu.kq_mr5}/{c.cpu.kq_mr6}/{c.cpu.kq_mr40}/{c.cpu.kq_mr44}/{c.cpu.kq_mr3}/{c.cpu.kq_mr33}/{c.cpu.kq_mr34}/{c.cpu.kq_mr45}/{c.cpu.kq_mr2}/{c.cpu.kq_mr23}/{c.cpu.kq_mr24}" + + "|kq {c.cpu.kq_mr4}/{c.cpu.kq_mr5}/{c.cpu.kq_mr6}/{c.cpu.kq_mr40}/{c.cpu.kq_mr44}/{c.cpu.kq_mr3}/{c.cpu.kq_mr33}/{c.cpu.kq_mr34}/{c.cpu.kq_mr45}/{c.cpu.kq_mr2}/{c.cpu.kq_mr23}/{c.cpu.kq_mr24}/{c.cpu.kq_mr25}" + "|q51 mr{c.cpu.q51_mr}" + "|nat {c.cpu.kquant_native ? 1 : 0}{c.cpu.kq_q40_native ? 1 : 0}{c.cpu.kq_q50_native ? 1 : 0}{c.cpu.kq_q51_native ? 1 : 0}" + (tag != "" ? "|{tag}" : "")) diff --git a/modules/dasLLAMA/dasllama/dasllama_convert.das b/modules/dasLLAMA/dasllama/dasllama_convert.das index 0f8ca8efb6..f9ba080f58 100644 --- a/modules/dasLLAMA/dasllama/dasllama_convert.das +++ b/modules/dasLLAMA/dasllama/dasllama_convert.das @@ -959,6 +959,55 @@ def dequant_iq2xs_plane_superblock_at(kq : array | #; kqo : int64; ks : a } } + +//! Transcode one IQ2_XXS superblock (66 bytes at `bo`: f16 d, 8 x [4 grid bytes][aux32]) into +//! the iq2xxs planes: the 64B qs region verbatim to kq[kqo..+64), d EIGHTH-ed (ggml's +//! (0.5+ls)*0.25 fold becomes (2ls+1) x d/8) + the 8 per-32 strips (1 + 2s) to ks[kso..+20). +def transcode_iq2xxs_superblock(bytes : array | #; bo : int64; var kq : array; kqo : int64; var ks : array; kso : int64) { + for (i in range64(64l)) { + kq[kqo + i] = bytes[bo + 2l + i] + } + let d8 = f16_half_bits(f16_half_bits(f16_half_bits(uint(bytes[bo]) | (uint(bytes[bo + 1l]) << 8u)))) + ks[kso] = uint8(d8 & 0xFFu) + ks[kso + 1l] = uint8(d8 >> 8u) + ks[kso + 2l] = uint8(0) + ks[kso + 3l] = uint8(0) + for (b in range64(8l)) { + ks[kso + 4l + b] = uint8(1 + 2 * (int(bytes[bo + 2l + b * 8l + 7l]) >> 4)) + } + for (i in range64(12l, 20l)) { + ks[kso + i] = uint8(0) + } +} + +//! Reference dequant of one iq2xxs-plane superblock: w = (d8 * (1 + 2s)) * grid_byte * sign - +//! block b's aux32[0] bytes pick u64 grid entries, its aux32[1] carries four 7-bit ksigns +//! indices (bit j flips element j). ggml's dequantize_row_iq2_xxs order. +def dequant_iq2xxs_plane_superblock(kq : array | #; kqo : int64; ks : array | #; kso : int64; var dst : array | #; doff : int64) { + dequant_iq2xxs_plane_superblock_at(kq, kqo, ks, kso + 4l, kso, dst, doff) +} + +//! The split-layout twin: `sso` names the 8-strip base, `dso` the f16 d8 halfword. +def dequant_iq2xxs_plane_superblock_at(kq : array | #; kqo : int64; ks : array | #; sso, dso : int64; var dst : array | #; doff : int64) { + let d8 = f16_to_f32(rd_u16(ks, dso)) + let grid = iq2xxs_grid2() + let signs = ksigns_iq2xs() + for (b in range64(8l)) { + let db = d8 * float(int(ks[sso + b])) + let aux = uint(kq[kqo + b * 8l + 4l]) | (uint(kq[kqo + b * 8l + 5l]) << 8u) | (uint(kq[kqo + b * 8l + 6l]) << 16u) | (uint(kq[kqo + b * 8l + 7l]) << 24u) + for (l in range64(4l)) { + let sgn = int(signs[int((aux >> uint(7l * l)) & 127u)]) + let idx = int(kq[kqo + b * 8l + l]) + let g1 = grid[idx * 2] + let g2 = grid[idx * 2 + 1] + for (j in range64(4l)) { + dst[doff + b * 32l + l * 8l + j] = db * float((g1 >> uint(8l * j)) & 0xFFu) * (((sgn >> int(j)) & 1) != 0 ? -1.0 : 1.0) + dst[doff + b * 32l + l * 8l + 4l + j] = db * float((g2 >> uint(8l * j)) & 0xFFu) * (((sgn >> int(4l + j)) & 1) != 0 ? -1.0 : 1.0) + } + } + } +} + //! Transcode one Q5_1 disk block (24 bytes at `bo`: d f16, m f16, qh u32, 16 nibble bytes) into //! the q51 planes — verbatim splits, exact. The per-block array form the tests drive; the bulk //! loader (gguf_transcode_q51) runs the same split pointerized and threaded. diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das index 9f5c044839..910ae1b5ee 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das @@ -3216,6 +3216,14 @@ def private q51_tile_gen(var gc : LlvmCodeCtx) : bool { // Called from llvm_user_modules::register_user_llvm_code_generators — the JIT invokes that in the // context that reads the registry, so the direct call lands in the right table copy. + +// iq2xxs: the grid-gather emitter arm lands with its Phase B; until then both stamps decline +// (reference bodies serve) - a declined generator is the framework's own fallback path. +[unused_argument(gc)] +def private iq2xxs_gemv_gen(var gc : LlvmCodeCtx) : bool => false +[unused_argument(gc)] +def private iq2xxs_tile_gen(var gc : LlvmCodeCtx) : bool => false + [macro_function] def public register_dasllama_gemm_generators { register_llvm_code_generator("dasllama_gemm_gen::q8q8_tile", @@tile_gen) @@ -3255,5 +3263,7 @@ def public register_dasllama_gemm_generators { register_llvm_code_generator("dasllama_gemm_gen::iq2s_tile", @@iq2s_tile_gen) register_llvm_code_generator("dasllama_gemm_gen::iq2xs_gemv", @@iq2xs_gemv_gen) register_llvm_code_generator("dasllama_gemm_gen::iq2xs_tile", @@iq2xs_tile_gen) + register_llvm_code_generator("dasllama_gemm_gen::iq2xxs_gemv", @@iq2xxs_gemv_gen) + register_llvm_code_generator("dasllama_gemm_gen::iq2xxs_tile", @@iq2xxs_tile_gen) register_llvm_code_generator("dasllama_gemm_gen::q8q8_witness", @@witness_gen) } diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_schema.das b/modules/dasLLAMA/dasllama/dasllama_gemm_schema.das index 02fb985946..b9059f9f57 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_schema.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_schema.das @@ -73,6 +73,7 @@ def kq_qsb(fmt : int) : int64 { if (fmt == 2) return kq_qsb(KqFmt.k2) if (fmt == 23) return kq_qsb(KqFmt.iq2s) if (fmt == 24) return kq_qsb(KqFmt.iq2xs) + if (fmt == 25) return kq_qsb(KqFmt.iq2xxs) panic("kq_qsb: unknown kq format id {fmt}") return 0l } @@ -93,6 +94,7 @@ def kq_ssb(fmt : int) : int64 { if (fmt == 2) return kq_ssb(KqFmt.k2) if (fmt == 23) return kq_ssb(KqFmt.iq2s) if (fmt == 24) return kq_ssb(KqFmt.iq2xs) + if (fmt == 25) return kq_ssb(KqFmt.iq2xxs) panic("kq_ssb: unknown kq format id {fmt}") return 0l } diff --git a/modules/dasLLAMA/dasllama/dasllama_gguf.das b/modules/dasLLAMA/dasllama/dasllama_gguf.das index bc72f907c4..821bdd6a0b 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gguf.das +++ b/modules/dasLLAMA/dasllama/dasllama_gguf.das @@ -52,6 +52,7 @@ let GGML_TYPE_Q2_K = 10 let GGML_TYPE_IQ3_S = 21 let GGML_TYPE_IQ2_S = 22 let GGML_TYPE_IQ2_XS = 17 +let GGML_TYPE_IQ2_XXS = 16 let GGML_TYPE_IQ3_XXS = 18 let GGML_TYPE_IQ4_NL = 20 let GGML_TYPE_Q4_K = 12 @@ -1020,6 +1021,48 @@ def gguf_transcode_iq2xs(m : GGUFMeta; srcbytes : array | #; name : strin } } + +//! Transcode an IQ2_XXS tensor into the iq2xxs planes (strides 64/20, exact): the 66B disk +//! superblock's qs region goes verbatim, d eighth-ed, the 8 per-32 strips decoded to +//! (1 + 2s) - transcode_iq2xxs_superblock. +def gguf_transcode_iq2xxs(m : GGUFMeta; srcbytes : array | #; name : string; var kq : array; var ks : array; eloff, expect_n : int64; src_off : int64 = 0l) { + let ti = kq_transcode_check(m, name, GGML_TYPE_IQ2_XXS, "IQ2_XXS", src_off, expect_n) + let nb = expect_n / 256l + if (nb <= 0l) { + return + } + guard_dst(name, "iq2xxs quant plane", (eloff / 256l) * IQ2XXS_QSB, nb * IQ2XXS_QSB, long_length(kq)) + guard_dst(name, "iq2xxs scale plane", (eloff / 256l) * IQ2XXS_SSB, nb * IQ2XXS_SSB, long_length(ks)) + with_tensor_view(m, srcbytes, ti) $(bytes, tbase) { + let bo = tbase + (src_off / 256l) * 66l + unsafe { + let srcp = addr(bytes[bo]) + var kqp = addr(kq[(eloff / 256l) * IQ2XXS_QSB]) + var ksp = addr(ks[(eloff / 256l) * IQ2XXS_SSB]) + maybe_parallel_for(0, int(nb), transcode_jobs(nb, 66l)) $(rb, re) { + unsafe { + for (sb in range64(int64(rb), int64(re))) { + let src = srcp + sb * 66l + var row = ksp + sb * IQ2XXS_SSB + let d8 = f16_half_bits(f16_half_bits(f16_half_bits(uint(src[0l]) | (uint(src[1l]) << 8u)))) + row[0l] = uint8(d8 & 0xFFu) + row[1l] = uint8(d8 >> 8u) + row[2l] = uint8(0) + row[3l] = uint8(0) + for (b in range64(8l)) { + row[4l + b] = uint8(1 + 2 * (int(src[2l + b * 8l + 7l]) >> 4)) + } + for (i in range64(12l, 20l)) { + row[i] = uint8(0) + } + bcopy(kqp + sb * IQ2XXS_QSB, src + 2l, 64l) + } + } + } + } + } +} + //! Transcode a Q5_1 tensor into the q51 planes (per 32-BLOCK strides 20/4 — see Q51_QB): each //! 24B disk block splits into 2B d + 2B m (scale plane) and 16B nibbles + 4B qh (quant plane), //! all verbatim. Exact; `eloff`/`src_off`/`expect_n` are element offsets, % 32. diff --git a/modules/dasLLAMA/dasllama/dasllama_gpu_resident.das b/modules/dasLLAMA/dasllama/dasllama_gpu_resident.das index a114e91329..76ae3ca0ca 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gpu_resident.das +++ b/modules/dasLLAMA/dasllama/dasllama_gpu_resident.das @@ -227,7 +227,7 @@ def private moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, row //! P3 trim (generate-then-free): pack the emb/cls region into embq/embs, mark the model //! trimmed, and FREE the big CPU weight families — the save then writes them as empty planes. //! Arena-only plans qualify (pure resident-driver dense); false = declined, nothing changed. -def trim_model_planes(var t : Model) : bool { // nolint:STYLE038 — the flat per-format plane-trim ladder (one arm per KqFmt) is the honest shape +def trim_model_planes(var t : Model) : bool { // nolint:STYLE037,STYLE038 — the flat per-format plane-trim ladder (one arm per KqFmt) is the honest shape if (empty(t.vkplan) || t.config.n_layer_nextn > 0l) { return false } @@ -310,6 +310,11 @@ def trim_model_planes(var t : Model) : bool { // nolint:STYLE038 — the flat t.embs |> reserve_resize(nsb * IQ2XS_SSB) memcpy(addr(t.embq[0]), addr(t.iq2xsq[sb0 * IQ2XS_QSB]), nsb * IQ2XS_QSB) memcpy(addr(t.embs[0]), addr(t.iq2xss[sb0 * IQ2XS_SSB]), nsb * IQ2XS_SSB) + } elif (t.emb_fmt == KqFmt.iq2xxs) { + t.embq |> reserve_resize(nsb * IQ2XXS_QSB) + t.embs |> reserve_resize(nsb * IQ2XXS_SSB) + memcpy(addr(t.embq[0]), addr(t.iq2xxsq[sb0 * IQ2XXS_QSB]), nsb * IQ2XXS_QSB) + memcpy(addr(t.embs[0]), addr(t.iq2xxss[sb0 * IQ2XXS_SSB]), nsb * IQ2XXS_SSB) } elif (t.emb_fmt == KqFmt.iq3xxs) { t.embq |> resize(nsb * IQ3XXS_QSB) t.embs |> resize(nsb * IQ3XXS_SSB) @@ -354,6 +359,8 @@ def trim_model_planes(var t : Model) : bool { // nolint:STYLE038 — the flat delete t.iq2ss delete t.iq2xsq delete t.iq2xss + delete t.iq2xxsq + delete t.iq2xxss delete t.q40q delete t.q40s delete t.q51q diff --git a/modules/dasLLAMA/dasllama/dasllama_image.das b/modules/dasLLAMA/dasllama/dasllama_image.das index c54ed4a90e..a51fea7c90 100644 --- a/modules/dasLLAMA/dasllama/dasllama_image.das +++ b/modules/dasLLAMA/dasllama/dasllama_image.das @@ -38,7 +38,7 @@ require dasllama/dasllama_load // WhisperModel.enc) contribute their planes under dotted names ("enc.fblob"); string-array // fields ride the meta blob via serialize_strings — raw string pointers can't be planes. -let IMAGE_VERSION = 25 // 25: the iq2xs plane pair (IQ2_XS native tier); 24: iq2s; 23: k2 +let IMAGE_VERSION = 26 // 26: the iq2xxs plane pair (IQ2_XXS native tier); 25: iq2xs; 24: iq2s //! The metal (blob-only) flavor's identity tag: q8 planes ride the 34B block_q8_0 blob and the //! kq scale planes their GPU forms (convert_model_to_metal_blob) — flavors are per-config and @@ -642,6 +642,7 @@ def serialize_image_meta(var arch : Archive; var t : Model) { arch |> serialize_raw(t.kq_repack_mr2) arch |> serialize_raw(t.kq_repack_mr23) arch |> serialize_raw(t.kq_repack_mr24) + arch |> serialize_raw(t.kq_repack_mr25) arch |> serialize_raw(t.wcls_fmt) arch |> serialize_raw(t.emb_fmt) arch |> serialize_raw(t.ple_emb_fmt) @@ -681,7 +682,7 @@ def serialize_image_meta(var arch : Archive; var t : Model) { } // 67 serialized fields + 3 deliberate skips (blocks, image_map, image_bytes) -let IMAGE_META_FIELDS = 74 + 3 +let IMAGE_META_FIELDS = 75 + 3 //! Count of meta-carried fields of any struct: non-arrays plus string arrays (those cannot //! be raw planes — see serialize_strings). Pair with a per-type field-count constant to @@ -965,7 +966,7 @@ def save_model_image(var t : Model; path : string; tag : string = ""; quant : st // streamed planes size as empty, so their bytes plus a page each are added here; over-reserve is free (dwrite_close truncates) def private stream_extra_bytes(t : Model; jobs : array) : uint64 { var extra = 0ul - for (fname in ["qblob", "mblob", "k4q", "k5q", "k6q", "q40q", "q51q", "iq4xsq", "k3q", "iq3sq", "iq3xxsq", "iq4nlq", "k2q", "iq2sq", "iq2xsq"]) { + for (fname in ["qblob", "mblob", "k4q", "k5q", "k6q", "q40q", "q51q", "iq4xsq", "k3q", "iq3sq", "iq3xxsq", "iq4nlq", "k2q", "iq2sq", "iq2xsq", "iq2xxsq"]) { let sb = stream_plane_bytes(t, jobs, fname) if (sb > 0l) { extra += uint64(sb) + uint64(IMAGE_PAGE) diff --git a/modules/dasLLAMA/dasllama/dasllama_kqformat.das b/modules/dasLLAMA/dasllama/dasllama_kqformat.das index 826901d2bc..5173eb39c1 100644 --- a/modules/dasLLAMA/dasllama/dasllama_kqformat.das +++ b/modules/dasLLAMA/dasllama/dasllama_kqformat.das @@ -13,7 +13,7 @@ module dasllama_kqformat shared public require dasllama/dasllama_lint public //! Per-weight storage format under a q8-mode load with native planes: q8 = qblob/qscales; -//! k4/k5/k6/q40/iq4xs/k3/iq3s/iq3xxs/iq4nl/k2/iq2s/iq2xs = superblock plane pairs (mixed PER TENSOR); q51 = native Q5_1 per-32 +//! k4/k5/k6/q40/iq4xs/k3/iq3s/iq3xxs/iq4nl/k2/iq2s/iq2xs/iq2xxs = superblock plane pairs (mixed PER TENSOR); q51 = native Q5_1 per-32 //! planes. New members append — the int value is the device stack tag and the image plane id. enum KqFmt : uint8 { q8 @@ -30,15 +30,16 @@ enum KqFmt : uint8 { k2 iq2s iq2xs + iq2xxs } //! The superblock-lattice formats (Q8_K-form activations, % 256 rows, repack + stamped kq //! kernels). q51 deliberately fails this: it rides per-32 planes with Q8_0-form activations — //! a `fmt != KqFmt.q8` test does not imply the kq lattice; branch on kq_sb where it does. -def kq_sb(f : KqFmt) : bool => f == KqFmt.k4 || f == KqFmt.k5 || f == KqFmt.k6 || f == KqFmt.q40 || f == KqFmt.iq4xs || f == KqFmt.k3 || f == KqFmt.iq3s || f == KqFmt.iq3xxs || f == KqFmt.iq4nl || f == KqFmt.k2 || f == KqFmt.iq2s || f == KqFmt.iq2xs +def kq_sb(f : KqFmt) : bool => f == KqFmt.k4 || f == KqFmt.k5 || f == KqFmt.k6 || f == KqFmt.q40 || f == KqFmt.iq4xs || f == KqFmt.k3 || f == KqFmt.iq3s || f == KqFmt.iq3xxs || f == KqFmt.iq4nl || f == KqFmt.k2 || f == KqFmt.iq2s || f == KqFmt.iq2xs || f == KqFmt.iq2xxs //! The same predicate over the int id space the GPU drivers carry per stack/plane. -def kq_sb(fmt : int) : bool => fmt == int(KqFmt.k4) || fmt == int(KqFmt.k5) || fmt == int(KqFmt.k6) || fmt == int(KqFmt.q40) || fmt == int(KqFmt.iq4xs) || fmt == int(KqFmt.k3) || fmt == int(KqFmt.iq3s) || fmt == int(KqFmt.iq3xxs) || fmt == int(KqFmt.iq4nl) || fmt == int(KqFmt.k2) || fmt == int(KqFmt.iq2s) || fmt == int(KqFmt.iq2xs) +def kq_sb(fmt : int) : bool => fmt == int(KqFmt.k4) || fmt == int(KqFmt.k5) || fmt == int(KqFmt.k6) || fmt == int(KqFmt.q40) || fmt == int(KqFmt.iq4xs) || fmt == int(KqFmt.k3) || fmt == int(KqFmt.iq3s) || fmt == int(KqFmt.iq3xxs) || fmt == int(KqFmt.iq4nl) || fmt == int(KqFmt.k2) || fmt == int(KqFmt.iq2s) || fmt == int(KqFmt.iq2xs) || fmt == int(KqFmt.iq2xxs) //! KqFmt -> the kernel/IR format id space — the ONE bridge between the enum and the int ids the //! generated kernels take as runtime params. Mnemonic ids: a K-quant is its bit width (3/4/5/6), @@ -80,6 +81,9 @@ def kq_schema_id(f : KqFmt) : int { if (f == KqFmt.iq2xs) { return 24 } + if (f == KqFmt.iq2xxs) { + return 25 + } panic("kq_schema_id: not a kq superblock format") return 0 } @@ -110,6 +114,8 @@ let IQ2S_QSB = 72l // IQ2_S quant plane: [32 grid-index low bytes][32 sign by let IQ2S_SSB = 20l // IQ2_S scale plane: f16 d EIGHTH-ed (ggml's (0.5+ls)*0.25 = (2ls+1)/8), 2 pad, 16 x uint8 (2ls+1) - per-16 strips in the 20B row let IQ2XS_QSB = 64l // IQ2_XS quant plane: 32 u16 words ([9-bit grid index][7-bit ksigns index]), verbatim disk order let IQ2XS_SSB = 20l // IQ2_XS scale plane: the iq2s row exactly - f16 d EIGHTH-ed, 2 pad, 16 x uint8 (2ls+1) per-16 strips +let IQ2XXS_QSB = 64l // IQ2_XXS quant plane: 8 blocks x [4 iq2xxs_grid byte indices][aux32: 4 x 7-bit ksigns + 4-bit ls], verbatim disk order +let IQ2XXS_SSB = 20l // IQ2_XXS scale plane: the iq3xxs row shape - f16 d EIGHTH-ed, 2 pad, 8 x uint8 (2ls+1) per-32 strips, 8 pad // IQ4_NL reuses Q40_QSB/Q40_SSB outright: the disk bytes are q40's (f16 d + 16 nibble bytes // per 32-block, the k/k+16 pairing) — only the nibble SEMANTICS differ (IQ4NL_LUT, no -8). @@ -677,6 +683,82 @@ def iq2xs_grid2() : uint[1024] { // nolint:STYLE038 - a 1024-word data table, //! a module global as zeros; kernels take `iq2xs_grid2()` instead. let IQ2XS_GRID = iq2xs_grid2() + +//! The IQ2_XXS grid (ggml's iq2xxs_grid, 256 u64 entries as low/high uint pairs): a block's +//! aux32[0] BYTES each select EIGHT magnitudes ({8, 25, 43}), flipped through the shared +//! ksigns_iq2xs table. Per-call local (2 KB), worker-safe; IQ2XXS_GRID = main-context copy. +def iq2xxs_grid2() : uint[512] { + return fixed_array( + 0x08080808, 0x08080808, 0x0808082b, 0x08080808, 0x08081919, 0x08080808, 0x08082b08, 0x08080808, + 0x08082b2b, 0x08080808, 0x08190819, 0x08080808, 0x08191908, 0x08080808, 0x082b0808, 0x08080808, + 0x082b082b, 0x08080808, 0x082b2b08, 0x08080808, 0x082b2b2b, 0x08080808, 0x19080819, 0x08080808, + 0x19081908, 0x08080808, 0x19190808, 0x08080808, 0x19192b08, 0x08080808, 0x192b0819, 0x08080808, + 0x192b1908, 0x08080808, 0x2b080808, 0x08080808, 0x2b08082b, 0x08080808, 0x2b082b2b, 0x08080808, + 0x2b2b082b, 0x08080808, 0x08080819, 0x08080819, 0x08081908, 0x08080819, 0x08190808, 0x08080819, + 0x08191919, 0x08080819, 0x19080808, 0x08080819, 0x2b081908, 0x08080819, 0x2b192b08, 0x08080819, + 0x08080808, 0x0808082b, 0x0808082b, 0x0808082b, 0x082b082b, 0x0808082b, 0x2b08082b, 0x0808082b, + 0x08080819, 0x08081908, 0x08081908, 0x08081908, 0x08190808, 0x08081908, 0x082b0819, 0x08081908, + 0x082b1908, 0x08081908, 0x19080808, 0x08081908, 0x1908082b, 0x08081908, 0x19082b08, 0x08081908, + 0x192b0808, 0x08081908, 0x2b080819, 0x08081908, 0x2b081908, 0x08081908, 0x2b190808, 0x08081908, + 0x2b2b1908, 0x08081908, 0x08080808, 0x08081919, 0x0808082b, 0x08081919, 0x08082b08, 0x08081919, + 0x082b0808, 0x08081919, 0x1908192b, 0x08081919, 0x192b2b19, 0x08081919, 0x2b080808, 0x08081919, + 0x2b190819, 0x08081919, 0x08082b19, 0x0808192b, 0x08190808, 0x0808192b, 0x19080808, 0x0808192b, + 0x2b081908, 0x0808192b, 0x2b2b1908, 0x0808192b, 0x08080808, 0x08082b08, 0x08081919, 0x08082b08, + 0x08082b08, 0x08082b08, 0x08191908, 0x08082b08, 0x082b2b08, 0x08082b08, 0x19080819, 0x08082b08, + 0x19081908, 0x08082b08, 0x19190808, 0x08082b08, 0x1919082b, 0x08082b08, 0x2b082b08, 0x08082b08, + 0x08081908, 0x08082b19, 0x19080808, 0x08082b19, 0x0808082b, 0x08082b2b, 0x08191908, 0x08082b2b, + 0x08080819, 0x08190808, 0x08081908, 0x08190808, 0x08190808, 0x08190808, 0x082b0819, 0x08190808, + 0x19080808, 0x08190808, 0x192b0808, 0x08190808, 0x2b081908, 0x08190808, 0x2b190808, 0x08190808, + 0x2b191919, 0x08190808, 0x08080808, 0x08190819, 0x08082b08, 0x08190819, 0x082b0808, 0x08190819, + 0x19190808, 0x08190819, 0x19192b2b, 0x08190819, 0x2b080808, 0x08190819, 0x082b1908, 0x0819082b, + 0x19081919, 0x0819082b, 0x08080808, 0x08191908, 0x08082b08, 0x08191908, 0x082b0808, 0x08191908, + 0x082b1919, 0x08191908, 0x19082b19, 0x08191908, 0x2b080808, 0x08191908, 0x08192b08, 0x08191919, + 0x192b082b, 0x08191919, 0x08080808, 0x0819192b, 0x0819192b, 0x0819192b, 0x08080819, 0x08192b08, + 0x08081908, 0x08192b08, 0x08190808, 0x08192b08, 0x19080808, 0x08192b08, 0x2b080819, 0x08192b08, + 0x08080808, 0x08192b19, 0x08081919, 0x08192b19, 0x2b2b0808, 0x08192b19, 0x19190819, 0x08192b2b, + 0x08080808, 0x082b0808, 0x0808082b, 0x082b0808, 0x08082b2b, 0x082b0808, 0x19081908, 0x082b0808, + 0x192b0819, 0x082b0808, 0x2b080808, 0x082b0808, 0x2b08082b, 0x082b0808, 0x082b2b19, 0x082b0819, + 0x19082b08, 0x082b0819, 0x08080808, 0x082b082b, 0x0808082b, 0x082b082b, 0x08080819, 0x082b1908, + 0x08081908, 0x082b1908, 0x08190808, 0x082b1908, 0x19080808, 0x082b1908, 0x1919192b, 0x082b1908, + 0x08080808, 0x082b1919, 0x19080819, 0x082b1919, 0x192b1908, 0x082b1919, 0x2b190808, 0x082b192b, + 0x08082b08, 0x082b2b08, 0x082b0808, 0x082b2b08, 0x2b191908, 0x082b2b08, 0x19081908, 0x082b2b2b, + 0x08080819, 0x19080808, 0x08081908, 0x19080808, 0x08190808, 0x19080808, 0x08192b08, 0x19080808, + 0x082b0819, 0x19080808, 0x082b1908, 0x19080808, 0x19080808, 0x19080808, 0x19082b08, 0x19080808, + 0x1919192b, 0x19080808, 0x192b0808, 0x19080808, 0x2b080819, 0x19080808, 0x2b081908, 0x19080808, + 0x2b190808, 0x19080808, 0x08080808, 0x19080819, 0x082b0808, 0x19080819, 0x192b0819, 0x19080819, + 0x2b080808, 0x19080819, 0x2b081919, 0x19080819, 0x08080819, 0x1908082b, 0x08190808, 0x1908082b, + 0x19082b08, 0x1908082b, 0x1919192b, 0x1908082b, 0x192b2b08, 0x1908082b, 0x08080808, 0x19081908, + 0x08082b08, 0x19081908, 0x082b0808, 0x19081908, 0x2b080808, 0x19081908, 0x2b192b19, 0x19081908, + 0x0819082b, 0x19081919, 0x082b1908, 0x19081919, 0x08080808, 0x1908192b, 0x08080819, 0x19082b08, + 0x08081908, 0x19082b08, 0x08190808, 0x19082b08, 0x19080808, 0x19082b08, 0x19081919, 0x19082b08, + 0x08080808, 0x19082b19, 0x19192b08, 0x19082b19, 0x192b0819, 0x19082b19, 0x2b08082b, 0x19082b19, + 0x19081919, 0x19082b2b, 0x2b190808, 0x19082b2b, 0x08080808, 0x19190808, 0x08082b08, 0x19190808, + 0x08190819, 0x19190808, 0x08192b19, 0x19190808, 0x082b0808, 0x19190808, 0x2b080808, 0x19190808, + 0x2b082b08, 0x19190808, 0x08081908, 0x19190819, 0x1908082b, 0x19190819, 0x2b2b1908, 0x19190819, + 0x2b190819, 0x1919082b, 0x2b190808, 0x19191908, 0x2b19082b, 0x19191908, 0x08082b2b, 0x19191919, + 0x08080819, 0x1919192b, 0x19191908, 0x1919192b, 0x08080808, 0x19192b08, 0x08190819, 0x19192b08, + 0x08192b19, 0x19192b08, 0x192b1908, 0x19192b08, 0x19080808, 0x19192b19, 0x08082b08, 0x19192b2b, + 0x08081908, 0x192b0808, 0x08190808, 0x192b0808, 0x19080808, 0x192b0808, 0x192b2b08, 0x192b0808, + 0x08080808, 0x192b0819, 0x19191919, 0x192b0819, 0x08192b08, 0x192b082b, 0x192b0808, 0x192b082b, + 0x08080808, 0x192b1908, 0x08081919, 0x192b1908, 0x08190808, 0x192b1919, 0x0819082b, 0x192b1919, + 0x2b081908, 0x192b1919, 0x1908082b, 0x192b2b08, 0x08080808, 0x2b080808, 0x0808082b, 0x2b080808, + 0x08082b2b, 0x2b080808, 0x19080819, 0x2b080808, 0x2b08082b, 0x2b080808, 0x08081908, 0x2b080819, + 0x08192b08, 0x2b080819, 0x19080808, 0x2b080819, 0x08190819, 0x2b08082b, 0x08080819, 0x2b081908, + 0x08081908, 0x2b081908, 0x08190808, 0x2b081908, 0x08191919, 0x2b081908, 0x19080808, 0x2b081908, + 0x192b0808, 0x2b081908, 0x08080808, 0x2b081919, 0x1908192b, 0x2b081919, 0x2b191908, 0x2b081919, + 0x08082b19, 0x2b08192b, 0x19080808, 0x2b08192b, 0x192b0808, 0x2b08192b, 0x0808082b, 0x2b082b08, + 0x08081908, 0x2b082b19, 0x08190819, 0x2b082b2b, 0x08081908, 0x2b190808, 0x08190808, 0x2b190808, + 0x082b1908, 0x2b190808, 0x19080808, 0x2b190808, 0x2b2b0819, 0x2b190808, 0x0819192b, 0x2b190819, + 0x2b080808, 0x2b190819, 0x19081919, 0x2b19082b, 0x08080808, 0x2b191908, 0x082b082b, 0x2b191908, + 0x19081908, 0x2b191908, 0x19190819, 0x2b191919, 0x2b080819, 0x2b192b08, 0x082b0808, 0x2b192b19, + 0x0808082b, 0x2b2b0808, 0x19190808, 0x2b2b0808, 0x2b081919, 0x2b2b0808, 0x08082b19, 0x2b2b0819, + 0x08080808, 0x2b2b082b, 0x08192b08, 0x2b2b1908, 0x19190808, 0x2b2b2b08, 0x08081908, 0x2b2b2b19) +} + +//! Main-context use only (tests, oracles, the emitter's constant bake) - a forked worker reads +//! a module global as zeros; kernels take `iq2xxs_grid2()` instead. +let IQ2XXS_GRID = iq2xxs_grid2() + //! Quant-plane bytes per stride unit — the typed stride surface. The stride unit is one //! 256-weight superblock for the kq lattice and one 32-weight block for q51 (see kq_elems). //! q8 panics (its planes are byte-per-elem, not stride-walked here). @@ -711,13 +793,16 @@ def kq_qsb(f : KqFmt) : int64 { if (f == KqFmt.iq2xs) { return IQ2XS_QSB } + if (f == KqFmt.iq2xxs) { + return IQ2XXS_QSB + } panic("kq_qsb: no plane stride for this format") return 0l } //! Scale-plane bytes per stride unit (see kq_qsb for the unit). def kq_ssb(f : KqFmt) : int64 { - if (f == KqFmt.k4 || f == KqFmt.k5 || f == KqFmt.iq4xs || f == KqFmt.iq3s || f == KqFmt.iq3xxs || f == KqFmt.k2 || f == KqFmt.iq2s || f == KqFmt.iq2xs) { + if (f == KqFmt.k4 || f == KqFmt.k5 || f == KqFmt.iq4xs || f == KqFmt.iq3s || f == KqFmt.iq3xxs || f == KqFmt.k2 || f == KqFmt.iq2s || f == KqFmt.iq2xs || f == KqFmt.iq2xxs) { return K4_SSB } if (f == KqFmt.k6 || f == KqFmt.k3) { diff --git a/modules/dasLLAMA/dasllama/dasllama_layout.das b/modules/dasLLAMA/dasllama/dasllama_layout.das index efbbc9082b..0378721501 100644 --- a/modules/dasLLAMA/dasllama/dasllama_layout.das +++ b/modules/dasLLAMA/dasllama/dasllama_layout.das @@ -458,6 +458,10 @@ def private repack_regions(var t : Model; regs : array) { var iq2xssp : uint8? = null if (!empty(t.iq2xsq)) { iq2xsqp = addr(t.iq2xsq[0]) } if (!empty(t.iq2xss)) { iq2xssp = addr(t.iq2xss[0]) } + var iq2xxsqp : uint8? = null + var iq2xxssp : uint8? = null + if (!empty(t.iq2xxsq)) { iq2xxsqp = addr(t.iq2xxsq[0]) } + if (!empty(t.iq2xxss)) { iq2xxssp = addr(t.iq2xxss[0]) } if (!empty(t.qblob)) { qbp = addr(t.qblob[0]) } if (!empty(t.qscales)) { qsp = addr(t.qscales[0]) } if (!empty(t.mxq)) { mxqp = addr(t.mxq[0]) } @@ -490,8 +494,8 @@ def private repack_regions(var t : Model; regs : array) { let fk = f == 20 ? 2 : f let qsb = kq_qsb(fk) let ssb = kq_ssb(fk) - var kqp = f == 4 ? k4qp : (f == 5 ? k5qp : (f == 6 ? k6qp : (f == 40 ? q40qp : (f == 44 ? iq4xsqp : (f == 3 ? k3qp : (f == 33 ? iq3sqp : (f == 34 ? iq3xxsqp : (f == 45 ? iq4nlqp : (fk == 2 ? k2qp : (f == 23 ? iq2sqp : iq2xsqp)))))))))) - var ksp = f == 4 ? k4sp : (f == 5 ? k5sp : (f == 6 ? k6sp : (f == 40 ? q40sp : (f == 44 ? iq4xssp : (f == 3 ? k3sp : (f == 33 ? iq3ssp : (f == 34 ? iq3xxssp : (f == 45 ? iq4nlsp : (fk == 2 ? k2sp : (f == 23 ? iq2ssp : iq2xssp)))))))))) + var kqp = f == 4 ? k4qp : (f == 5 ? k5qp : (f == 6 ? k6qp : (f == 40 ? q40qp : (f == 44 ? iq4xsqp : (f == 3 ? k3qp : (f == 33 ? iq3sqp : (f == 34 ? iq3xxsqp : (f == 45 ? iq4nlqp : (fk == 2 ? k2qp : (f == 23 ? iq2sqp : (f == 24 ? iq2xsqp : iq2xxsqp))))))))))) + var ksp = f == 4 ? k4sp : (f == 5 ? k5sp : (f == 6 ? k6sp : (f == 40 ? q40sp : (f == 44 ? iq4xssp : (f == 3 ? k3sp : (f == 33 ? iq3ssp : (f == 34 ? iq3xxssp : (f == 45 ? iq4nlsp : (fk == 2 ? k2sp : (f == 23 ? iq2ssp : (f == 24 ? iq2xssp : iq2xxssp))))))))))) invoke(rkq, fk, kqp + sb * qsb, ksp + sb * ssb, rp[i].n, rp[i].d) } } @@ -633,6 +637,8 @@ def private push_repack_kq(var regs : array; fmt : KqFmt; woff, n, d push_repack(regs, 23, woff, n, d) } elif (fmt == KqFmt.iq2xs) { push_repack(regs, 24, woff, n, d) + } elif (fmt == KqFmt.iq2xxs) { + push_repack(regs, 25, woff, n, d) } } @@ -910,13 +916,13 @@ def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice : (fmt == KqFmt.k6 ? addr(t.k6q[sb0 * kq_qsb(6)]) : (fmt == KqFmt.q40 ? addr(t.q40q[sb0 * kq_qsb(40)]) : (fmt == KqFmt.iq4xs ? addr(t.iq4xsq[sb0 * kq_qsb(44)]) - : (fmt == KqFmt.k3 ? addr(t.k3q[sb0 * kq_qsb(3)]) : (fmt == KqFmt.iq3s ? addr(t.iq3sq[sb0 * kq_qsb(33)]) : (fmt == KqFmt.iq3xxs ? addr(t.iq3xxsq[sb0 * kq_qsb(34)]) : (fmt == KqFmt.iq4nl ? addr(t.iq4nlq[sb0 * kq_qsb(45)]) : (fmt == KqFmt.k2 ? addr(t.k2q[sb0 * kq_qsb(2)]) : (fmt == KqFmt.iq2s ? addr(t.iq2sq[sb0 * kq_qsb(23)]) : addr(t.iq2xsq[sb0 * kq_qsb(24)])))))))))))) + : (fmt == KqFmt.k3 ? addr(t.k3q[sb0 * kq_qsb(3)]) : (fmt == KqFmt.iq3s ? addr(t.iq3sq[sb0 * kq_qsb(33)]) : (fmt == KqFmt.iq3xxs ? addr(t.iq3xxsq[sb0 * kq_qsb(34)]) : (fmt == KqFmt.iq4nl ? addr(t.iq4nlq[sb0 * kq_qsb(45)]) : (fmt == KqFmt.k2 ? addr(t.k2q[sb0 * kq_qsb(2)]) : (fmt == KqFmt.iq2s ? addr(t.iq2sq[sb0 * kq_qsb(23)]) : (fmt == KqFmt.iq2xs ? addr(t.iq2xsq[sb0 * kq_qsb(24)]) : addr(t.iq2xxsq[sb0 * kq_qsb(25)]))))))))))))) let sp = (fmt == KqFmt.k4 ? addr(t.k4s[sb0 * kq_ssb(4)]) : (fmt == KqFmt.k5 ? addr(t.k5s[sb0 * kq_ssb(5)]) : (fmt == KqFmt.k6 ? addr(t.k6s[sb0 * kq_ssb(6)]) : (fmt == KqFmt.q40 ? addr(t.q40s[sb0 * kq_ssb(40)]) : (fmt == KqFmt.iq4xs ? addr(t.iq4xss[sb0 * kq_ssb(44)]) - : (fmt == KqFmt.k3 ? addr(t.k3s[sb0 * kq_ssb(3)]) : (fmt == KqFmt.iq3s ? addr(t.iq3ss[sb0 * kq_ssb(33)]) : (fmt == KqFmt.iq3xxs ? addr(t.iq3xxss[sb0 * kq_ssb(34)]) : (fmt == KqFmt.iq4nl ? addr(t.iq4nls[sb0 * kq_ssb(45)]) : (fmt == KqFmt.k2 ? addr(t.k2s[sb0 * kq_ssb(2)]) : (fmt == KqFmt.iq2s ? addr(t.iq2ss[sb0 * kq_ssb(23)]) : addr(t.iq2xss[sb0 * kq_ssb(24)])))))))))))) + : (fmt == KqFmt.k3 ? addr(t.k3s[sb0 * kq_ssb(3)]) : (fmt == KqFmt.iq3s ? addr(t.iq3ss[sb0 * kq_ssb(33)]) : (fmt == KqFmt.iq3xxs ? addr(t.iq3xxss[sb0 * kq_ssb(34)]) : (fmt == KqFmt.iq4nl ? addr(t.iq4nls[sb0 * kq_ssb(45)]) : (fmt == KqFmt.k2 ? addr(t.k2s[sb0 * kq_ssb(2)]) : (fmt == KqFmt.iq2s ? addr(t.iq2ss[sb0 * kq_ssb(23)]) : (fmt == KqFmt.iq2xs ? addr(t.iq2xss[sb0 * kq_ssb(24)]) : addr(t.iq2xxss[sb0 * kq_ssb(25)]))))))))))))) var wqp = addr(wq[0]) var wsp = addr(ws[0]) let njobs = is_job_que_available() ? min(nslices, 4 * (get_total_hw_jobs() + 1)) : 1 @@ -963,6 +969,10 @@ def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice for (m in range64(64l)) { dq[m] = gq[((m / 4l) * mr + r) * 4l + m % 4l] } + } elif (fmt == KqFmt.iq2xxs) { // 16 uniform 4-byte columns over the u16 qs words + for (m in range64(64l)) { + dq[m] = gq[((m / 4l) * mr + r) * 4l + m % 4l] + } } else { for (m in range64(128l)) { // nibble region: verbatim per (blk, j, t) dq[m] = gq[((m / 4l) * mr + r) * 4l + m % 4l] @@ -1014,7 +1024,7 @@ def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice } else { let rq = qp + sliceQ + ri * qrow + sbi * qsb let rs = sp + sliceS + ri * srow + sbi * dssb - if (fmt == KqFmt.q40 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s || fmt == KqFmt.iq3xxs || fmt == KqFmt.iq4nl || fmt == KqFmt.k2 || fmt == KqFmt.iq2s || fmt == KqFmt.iq2xs) { + if (fmt == KqFmt.q40 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s || fmt == KqFmt.iq3xxs || fmt == KqFmt.iq4nl || fmt == KqFmt.k2 || fmt == KqFmt.iq2s || fmt == KqFmt.iq2xs || fmt == KqFmt.iq2xxs) { for (m in range64(qsb)) { // disk bytes already in the device form (k/k+16 pairs; k3's lanes) dq[m] = rq[m] } @@ -1041,7 +1051,7 @@ def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice dq[128l + p] = rq[128l + p] } } - if (fmt == KqFmt.k6 || fmt == KqFmt.q40 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s || fmt == KqFmt.iq3xxs || fmt == KqFmt.iq4nl || fmt == KqFmt.k2 || fmt == KqFmt.iq2s || fmt == KqFmt.iq2xs) { // rows already in the decoded device form + if (fmt == KqFmt.k6 || fmt == KqFmt.q40 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s || fmt == KqFmt.iq3xxs || fmt == KqFmt.iq4nl || fmt == KqFmt.k2 || fmt == KqFmt.iq2s || fmt == KqFmt.iq2xs || fmt == KqFmt.iq2xxs) { // rows already in the decoded device form for (idx in range64(dssb)) { dsc[idx] = rs[idx] } diff --git a/modules/dasLLAMA/dasllama/dasllama_load.das b/modules/dasLLAMA/dasllama/dasllama_load.das index 91379c37fd..6fcd08ae21 100644 --- a/modules/dasLLAMA/dasllama/dasllama_load.das +++ b/modules/dasLLAMA/dasllama/dasllama_load.das @@ -49,6 +49,7 @@ struct private LayoutSizes { k2_n : int64 iq2s_n : int64 iq2xs_n : int64 + iq2xxs_n : int64 pleq8_n : int64 // the dedicated q8 plane holding ONLY the PLE table (fp32/q4 serving) } @@ -69,6 +70,7 @@ struct private KqCursors { k2 : int64 iq2s : int64 iq2xs : int64 + iq2xxs : int64 } def private kq_take(var cur : KqCursors; f : KqFmt; n : int64) : int64 { @@ -137,6 +139,11 @@ def private kq_take(var cur : KqCursors; f : KqFmt; n : int64) : int64 { cur.iq2xs += n return o } + if (f == KqFmt.iq2xxs) { + let o = cur.iq2xxs + cur.iq2xxs += n + return o + } let o = cur.wo cur.wo += n return o @@ -436,7 +443,7 @@ def private layout_offsets(var t : Model) : LayoutSizes { // nolint:STYLE037,S fo += dim } return LayoutSizes(fblob_n = fo, wblob_n = cur.wo, mx_n = mx, bf16_n = bf16, - k4_n = cur.k4, k5_n = cur.k5, k6_n = cur.k6, q40_n = cur.q40, q51_n = cur.q51, iq4xs_n = cur.iq4xs, k3_n = cur.k3, iq3s_n = cur.iq3s, iq3xxs_n = cur.iq3xxs, iq4nl_n = cur.iq4nl, k2_n = cur.k2, iq2s_n = cur.iq2s, iq2xs_n = cur.iq2xs, pleq8_n = pleq8) + k4_n = cur.k4, k5_n = cur.k5, k6_n = cur.k6, q40_n = cur.q40, q51_n = cur.q51, iq4xs_n = cur.iq4xs, k3_n = cur.k3, iq3s_n = cur.iq3s, iq3xxs_n = cur.iq3xxs, iq4nl_n = cur.iq4nl, k2_n = cur.k2, iq2s_n = cur.iq2s, iq2xs_n = cur.iq2xs, iq2xxs_n = cur.iq2xxs, pleq8_n = pleq8) } @@ -711,6 +718,8 @@ def stream_field_of(t : Model; fmt : KqFmt) : string { return "iq2sq" } elif (fmt == KqFmt.iq2xs) { return "iq2xsq" + } elif (fmt == KqFmt.iq2xxs) { + return "iq2xxsq" } elif (t.quant == QuantMode.q8) { // the blob flavor carries q8 as gguf-native 34B blocks in ONE plane, never the split pair return t.metal_blob ? "mblob" : "qblob" @@ -843,6 +852,8 @@ def fill_stream_plane(m : GGUFMeta; bytes : array | #; var t : Model; fie memcpy(addr(t.iq2ss[(j.woff / 256l) * IQ2S_SSB]), addr(temp_s[0]), sh.sb) } elif (j.fmt == KqFmt.iq2xs) { memcpy(addr(t.iq2xss[(j.woff / 256l) * IQ2XS_SSB]), addr(temp_s[0]), sh.sb) + } elif (j.fmt == KqFmt.iq2xxs) { + memcpy(addr(t.iq2xxss[(j.woff / 256l) * IQ2XXS_SSB]), addr(temp_s[0]), sh.sb) } elif (j.fmt == KqFmt.q51) { memcpy(addr(t.q51s[(j.woff / 32l) * Q51_SB]), addr(temp_s[0]), sh.sb) } elif (sh.sb == 0l && j.n > 0l && !t.metal_blob) { @@ -898,6 +909,9 @@ def fill_stream_plane(m : GGUFMeta; bytes : array | #; var t : Model; fie } elif (r.fmt == 24) { invoke(g_stream_repack, 24, addr(temp_q[((r.off - j.woff) / 256l) * IQ2XS_QSB]), addr(t.iq2xss[(r.off / 256l) * IQ2XS_SSB]), r.n, r.d) + } elif (r.fmt == 25) { + invoke(g_stream_repack, 25, addr(temp_q[((r.off - j.woff) / 256l) * IQ2XXS_QSB]), + addr(t.iq2xxss[(r.off / 256l) * IQ2XXS_SSB]), r.n, r.d) } } } @@ -1134,6 +1148,9 @@ def private transcode_kq_tensor(m : GGUFMeta; bytes : array | #; name : s } elif (fmt == KqFmt.iq2xs) { gguf_transcode_iq2xs(m, bytes, name, kq, ks, eloff, n, src_off) return "iq2xs transcode (IQ2_XS)" + } elif (fmt == KqFmt.iq2xxs) { + gguf_transcode_iq2xxs(m, bytes, name, kq, ks, eloff, n, src_off) + return "iq2xxs transcode (IQ2_XXS)" } gguf_transcode_q51(m, bytes, name, kq, ks, eloff, n, src_off) return "q51 transcode (Q5_1)" @@ -1188,6 +1205,8 @@ def private load_big(m : GGUFMeta; bytes : array | #; name : string; var kind = transcode_kq_tensor(m, bytes, name, fmt, t.iq2sq, t.iq2ss, woff, n, src_off, scratch) } elif (fmt == KqFmt.iq2xs) { kind = transcode_kq_tensor(m, bytes, name, fmt, t.iq2xsq, t.iq2xss, woff, n, src_off, scratch) + } elif (fmt == KqFmt.iq2xxs) { + kind = transcode_kq_tensor(m, bytes, name, fmt, t.iq2xxsq, t.iq2xxss, woff, n, src_off, scratch) } elif (fmt == KqFmt.q51) { kind = transcode_kq_tensor(m, bytes, name, fmt, t.q51q, t.q51s, woff, n, src_off, scratch) } elif (t.quant == QuantMode.q8) { @@ -1287,8 +1306,8 @@ def private log_load_report(t : Model) { to_log(LOG_INFO, "dasLLAMA noisy: backend '{active_kernel_backend()}' | kquant_native {get_kquant_native()} q40_native {get_kq_q40_native()} q50_native {get_kq_q50_native()}\n") // the tier arms at [init], before a --noisy CLI flag can land — restate it here so the flag alone suffices to_log(LOG_INFO, "dasLLAMA noisy: GPU tier installed {moe_gpu_tier_installed()} | want auto {gpu_want_auto()}, moe layers {gpu_want_moe_layers()}, stream {gpu_want_moe_stream()}\n") - to_log(LOG_INFO, "dasLLAMA noisy: format tags - q8 {tally_fmt(t, KqFmt.q8)}, k4 {tally_fmt(t, KqFmt.k4)}, k5 {tally_fmt(t, KqFmt.k5)}, k6 {tally_fmt(t, KqFmt.k6)}, q40 {tally_fmt(t, KqFmt.q40)}, q51 {tally_fmt(t, KqFmt.q51)}, iq4xs {tally_fmt(t, KqFmt.iq4xs)}, k3 {tally_fmt(t, KqFmt.k3)}, iq3s {tally_fmt(t, KqFmt.iq3s)}, iq3xxs {tally_fmt(t, KqFmt.iq3xxs)}, iq4nl {tally_fmt(t, KqFmt.iq4nl)}, k2 {tally_fmt(t, KqFmt.k2)}, iq2s {tally_fmt(t, KqFmt.iq2s)}, iq2xs {tally_fmt(t, KqFmt.iq2xs)}\n") - to_log(LOG_INFO, "dasLLAMA noisy: plane MB - q8 {float(long_length(t.qblob)) / mb} (+scales {float(long_length(t.qscales) * 4l) / mb}), k4 {float(long_length(t.k4q) + long_length(t.k4s)) / mb}, k5 {float(long_length(t.k5q) + long_length(t.k5s)) / mb}, k6 {float(long_length(t.k6q) + long_length(t.k6s)) / mb}, q40 {float(long_length(t.q40q) + long_length(t.q40s)) / mb}, q51 {float(long_length(t.q51q) + long_length(t.q51s)) / mb}, iq4xs {float(long_length(t.iq4xsq) + long_length(t.iq4xss)) / mb}, k3 {float(long_length(t.k3q) + long_length(t.k3s)) / mb}, iq3s {float(long_length(t.iq3sq) + long_length(t.iq3ss)) / mb}, iq3xxs {float(long_length(t.iq3xxsq) + long_length(t.iq3xxss)) / mb}, iq4nl {float(long_length(t.iq4nlq) + long_length(t.iq4nls)) / mb}, k2 {float(long_length(t.k2q) + long_length(t.k2s)) / mb}, iq2s {float(long_length(t.iq2sq) + long_length(t.iq2ss)) / mb}, iq2xs {float(long_length(t.iq2xsq) + long_length(t.iq2xss)) / mb}, f32 {float(long_length(t.wblob) * 4l) / mb}\n") + to_log(LOG_INFO, "dasLLAMA noisy: format tags - q8 {tally_fmt(t, KqFmt.q8)}, k4 {tally_fmt(t, KqFmt.k4)}, k5 {tally_fmt(t, KqFmt.k5)}, k6 {tally_fmt(t, KqFmt.k6)}, q40 {tally_fmt(t, KqFmt.q40)}, q51 {tally_fmt(t, KqFmt.q51)}, iq4xs {tally_fmt(t, KqFmt.iq4xs)}, k3 {tally_fmt(t, KqFmt.k3)}, iq3s {tally_fmt(t, KqFmt.iq3s)}, iq3xxs {tally_fmt(t, KqFmt.iq3xxs)}, iq4nl {tally_fmt(t, KqFmt.iq4nl)}, k2 {tally_fmt(t, KqFmt.k2)}, iq2s {tally_fmt(t, KqFmt.iq2s)}, iq2xs {tally_fmt(t, KqFmt.iq2xs)}, iq2xxs {tally_fmt(t, KqFmt.iq2xxs)}\n") + to_log(LOG_INFO, "dasLLAMA noisy: plane MB - q8 {float(long_length(t.qblob)) / mb} (+scales {float(long_length(t.qscales) * 4l) / mb}), k4 {float(long_length(t.k4q) + long_length(t.k4s)) / mb}, k5 {float(long_length(t.k5q) + long_length(t.k5s)) / mb}, k6 {float(long_length(t.k6q) + long_length(t.k6s)) / mb}, q40 {float(long_length(t.q40q) + long_length(t.q40s)) / mb}, q51 {float(long_length(t.q51q) + long_length(t.q51s)) / mb}, iq4xs {float(long_length(t.iq4xsq) + long_length(t.iq4xss)) / mb}, k3 {float(long_length(t.k3q) + long_length(t.k3s)) / mb}, iq3s {float(long_length(t.iq3sq) + long_length(t.iq3ss)) / mb}, iq3xxs {float(long_length(t.iq3xxsq) + long_length(t.iq3xxss)) / mb}, iq4nl {float(long_length(t.iq4nlq) + long_length(t.iq4nls)) / mb}, k2 {float(long_length(t.k2q) + long_length(t.k2s)) / mb}, iq2s {float(long_length(t.iq2sq) + long_length(t.iq2ss)) / mb}, iq2xs {float(long_length(t.iq2xsq) + long_length(t.iq2xss)) / mb}, iq2xxs {float(long_length(t.iq2xxsq) + long_length(t.iq2xxss)) / mb}, f32 {float(long_length(t.wblob) * 4l) / mb}\n") } // GGML disk type -> plane format tag (non-K-quant types ride the classic q8 path) @@ -1326,6 +1345,9 @@ def private kq_fmt_of(gt : int) : KqFmt { if (gt == GGML_TYPE_IQ2_XS) { return KqFmt.iq2xs } + if (gt == GGML_TYPE_IQ2_XXS) { + return KqFmt.iq2xxs + } if (gt == GGML_TYPE_Q4_0 && get_kq_q40_native()) { return KqFmt.q40 } @@ -2167,6 +2189,7 @@ def private load_gguf_parsed(var t : Model; m : GGUFMeta; bytes : array | g_stream_plane_total["k2q"] = (sz.k2_n / 256l) * K2_QSB g_stream_plane_total["iq2sq"] = (sz.iq2s_n / 256l) * IQ2S_QSB g_stream_plane_total["iq2xsq"] = (sz.iq2xs_n / 256l) * IQ2XS_QSB + g_stream_plane_total["iq2xxsq"] = (sz.iq2xxs_n / 256l) * IQ2XXS_QSB let wb = mode == QuantMode.q8 ? sz.wblob_n : 0l g_stream_plane_total[t.metal_blob ? "mblob" : "qblob"] = (t.metal_blob ? (wb / Q8_BLOCK_ELEMS) * (Q8_QPB + Q8_SPB) : wb) @@ -2275,6 +2298,14 @@ def private load_gguf_parsed(var t : Model; m : GGUFMeta; bytes : array | t.iq2xss |> reserve((sz.iq2xs_n / 256l) * IQ2XS_SSB) t.iq2xss |> resize((sz.iq2xs_n / 256l) * IQ2XS_SSB) } + if (sz.iq2xxs_n > 0l) { + if (!stream_q) { + t.iq2xxsq |> reserve((sz.iq2xxs_n / 256l) * IQ2XXS_QSB) + t.iq2xxsq |> resize((sz.iq2xxs_n / 256l) * IQ2XXS_QSB) + } + t.iq2xxss |> reserve((sz.iq2xxs_n / 256l) * IQ2XXS_SSB) + t.iq2xxss |> resize((sz.iq2xxs_n / 256l) * IQ2XXS_SSB) + } if (mode == QuantMode.q8) { if (!stream_q) { t.qblob |> reserve(sz.wblob_n) @@ -2589,6 +2620,7 @@ def private load_gguf_parsed(var t : Model; m : GGUFMeta; bytes : array | t.kq_repack_mr2 = active_kq_layout_mr(2) t.kq_repack_mr23 = active_kq_layout_mr(23) t.kq_repack_mr24 = active_kq_layout_mr(24) + t.kq_repack_mr25 = active_kq_layout_mr(25) } } else { layout_repack_q8(t) @@ -2613,7 +2645,8 @@ def private load_gguf_parsed(var t : Model; m : GGUFMeta; bytes : array | t.kq_repack_mr2 = active_kq_layout_mr(2) t.kq_repack_mr23 = active_kq_layout_mr(23) t.kq_repack_mr24 = active_kq_layout_mr(24) - to_log(LOG_INFO, "dasLLAMA: K-quant planes repacked to the '{active_kernel_backend()}' layouts (k4 grp{t.kq_repack_mr4}, k5 grp{t.kq_repack_mr5}, k6 grp{t.kq_repack_mr6}, q40 grp{t.kq_repack_mr40}, iq4xs grp{t.kq_repack_mr44}, k3 grp{t.kq_repack_mr3}, iq3s grp{t.kq_repack_mr33}, iq3xxs grp{t.kq_repack_mr34}, iq4nl grp{t.kq_repack_mr45}, k2 grp{t.kq_repack_mr2}, iq2s grp{t.kq_repack_mr23}, iq2xs grp{t.kq_repack_mr24})\n") + t.kq_repack_mr25 = active_kq_layout_mr(25) + to_log(LOG_INFO, "dasLLAMA: K-quant planes repacked to the '{active_kernel_backend()}' layouts (k4 grp{t.kq_repack_mr4}, k5 grp{t.kq_repack_mr5}, k6 grp{t.kq_repack_mr6}, q40 grp{t.kq_repack_mr40}, iq4xs grp{t.kq_repack_mr44}, k3 grp{t.kq_repack_mr3}, iq3s grp{t.kq_repack_mr33}, iq3xxs grp{t.kq_repack_mr34}, iq4nl grp{t.kq_repack_mr45}, k2 grp{t.kq_repack_mr2}, iq2s grp{t.kq_repack_mr23}, iq2xs grp{t.kq_repack_mr24}, iq2xxs grp{t.kq_repack_mr25})\n") } } } diff --git a/modules/dasLLAMA/dasllama/dasllama_math.das b/modules/dasLLAMA/dasllama/dasllama_math.das index 39216c4e08..868c319512 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math.das +++ b/modules/dasLLAMA/dasllama/dasllama_math.das @@ -1092,6 +1092,7 @@ struct KernelBackend { kq_rows_k2 : MatmulKqRowsFn = @@kq_unset_rows kq_rows_iq2s : MatmulKqRowsFn = @@kq_unset_rows kq_rows_iq2xs : MatmulKqRowsFn = @@kq_unset_rows + kq_rows_iq2xxs : MatmulKqRowsFn = @@kq_unset_rows kq_batch : MatmulKqBatchFn = @@kq_unset_batch kq_groupn : MatmulKqGroupNFn = @@kq_unset_groupn kq_batch_groupn : MatmulKqBatchGroupNFn = @@kq_unset_batch_groupn @@ -1162,6 +1163,7 @@ var g_kq_rows_iq4nl = @@kq_unset_rows var g_kq_rows_k2 = @@kq_unset_rows var g_kq_rows_iq2s = @@kq_unset_rows var g_kq_rows_iq2xs = @@kq_unset_rows +var g_kq_rows_iq2xxs = @@kq_unset_rows var g_mm_kq_batch = @@kq_unset_batch var g_mm_kq_groupn = @@kq_unset_groupn var g_mm_kq_batch_groupn = @@kq_unset_batch_groupn @@ -1489,7 +1491,7 @@ def active_q8_layout_mr() : int64 => g_bake_cpu_override ? g_bake_cpu.q8_mr : g_ //! family split; the loader freezes these onto the Model at repack time. def active_kq_layout_mr(fmt : int) : int64 { if (g_bake_cpu_override) { - return fmt == 4 ? g_bake_cpu.kq_mr4 : (fmt == 5 ? g_bake_cpu.kq_mr5 : (fmt == 40 ? g_bake_cpu.kq_mr40 : (fmt == 44 ? g_bake_cpu.kq_mr44 : (fmt == 3 ? g_bake_cpu.kq_mr3 : (fmt == 33 ? g_bake_cpu.kq_mr33 : (fmt == 34 ? g_bake_cpu.kq_mr34 : (fmt == 45 ? g_bake_cpu.kq_mr45 : (fmt == 2 ? g_bake_cpu.kq_mr2 : (fmt == 23 ? g_bake_cpu.kq_mr23 : (fmt == 24 ? g_bake_cpu.kq_mr24 : g_bake_cpu.kq_mr6)))))))))) + return fmt == 4 ? g_bake_cpu.kq_mr4 : (fmt == 5 ? g_bake_cpu.kq_mr5 : (fmt == 40 ? g_bake_cpu.kq_mr40 : (fmt == 44 ? g_bake_cpu.kq_mr44 : (fmt == 3 ? g_bake_cpu.kq_mr3 : (fmt == 33 ? g_bake_cpu.kq_mr33 : (fmt == 34 ? g_bake_cpu.kq_mr34 : (fmt == 45 ? g_bake_cpu.kq_mr45 : (fmt == 2 ? g_bake_cpu.kq_mr2 : (fmt == 23 ? g_bake_cpu.kq_mr23 : (fmt == 24 ? g_bake_cpu.kq_mr24 : (fmt == 25 ? g_bake_cpu.kq_mr25 : g_bake_cpu.kq_mr6))))))))))) } return int64(invoke(g_active_kq_layout, fmt)) } @@ -1673,6 +1675,7 @@ def private activate(be : KernelBackend) { g_kq_rows_k2 = be.kq_rows_k2 g_kq_rows_iq2s = be.kq_rows_iq2s g_kq_rows_iq2xs = be.kq_rows_iq2xs + g_kq_rows_iq2xxs = be.kq_rows_iq2xxs g_mm_kq_batch = be.kq_batch g_mm_kq_groupn = be.kq_groupn g_mm_kq_batch_groupn = be.kq_batch_groupn @@ -1883,6 +1886,7 @@ def kq_rows_fn(fmt : int) : MatmulKqRowsFn { if (fmt == 2) return g_kq_rows_k2 if (fmt == 23) return g_kq_rows_iq2s if (fmt == 24) return g_kq_rows_iq2xs + if (fmt == 25) return g_kq_rows_iq2xxs return @@kq_unset_rows } @@ -3267,6 +3271,9 @@ def private restore_math_fn_defaults { // nolint:STYLE037,STYLE038 — flat seam if (g_kq_rows_iq2xs == null) { g_kq_rows_iq2xs = @@kq_unset_rows } + if (g_kq_rows_iq2xxs == null) { + g_kq_rows_iq2xxs = @@kq_unset_rows + } if (g_mm_kq_batch == null) { g_mm_kq_batch = @@kq_unset_batch } diff --git a/modules/dasLLAMA/dasllama/dasllama_math_default.das b/modules/dasLLAMA/dasllama/dasllama_math_default.das index 1e0e1c35ca..126d1f932b 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_default.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_default.das @@ -727,6 +727,44 @@ def dot_iq2xs_q8(kqrow : uint8 const?; ksrow : uint8 const?; xqp : int8 const?; return acc } +//! IQ2_XXS row dot off the DISK-ORDER planes: per block the aux32[0] bytes pick u64 grid +//! entries (two words each), the aux32[1]'s 7-bit ksigns indices flip signs, its top nibble +//! is the per-32 strip (1 + 2s) on the eighth-ed d - decoded to the strip byte at transcode. +[unused_argument(xbsp)] +def dot_iq2xxs_q8(kqrow : uint8 const?; ksrow : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n : int64) : float { + var acc = 0.0 + let grid = iq2xxs_grid2() + let signs = ksigns_iq2xs() + unsafe { + for (sb in range64(n / 256l)) { + let kqo = sb * 64l + let kso = sb * 20l + let d8 = f16_to_f32(uint(ksrow[kso]) | (uint(ksrow[kso + 1l]) << 8u)) + var isum = 0 + for (blk in range64(8l)) { + let aux = uint(kqrow[kqo + blk * 8l + 4l]) | (uint(kqrow[kqo + blk * 8l + 5l]) << 8u) | (uint(kqrow[kqo + blk * 8l + 6l]) << 16u) | (uint(kqrow[kqo + blk * 8l + 7l]) << 24u) + let ab = (sb * 8l + blk) * 32l + var ib = 0 + for (l in range64(4l)) { + let sgn = int(signs[int((aux >> uint(7l * l)) & 127u)]) + let idx = int(kqrow[kqo + blk * 8l + l]) + let g1 = grid[idx * 2] + let g2 = grid[idx * 2 + 1] + for (j in range64(4l)) { + let w1 = int((g1 >> uint(8l * j)) & 0xFFu) + let w2 = int((g2 >> uint(8l * j)) & 0xFFu) + ib += (((sgn >> int(j)) & 1) != 0 ? -w1 : w1) * int(xqp[ab + l * 8l + j]) + ib += (((sgn >> int(4l + j)) & 1) != 0 ? -w2 : w2) * int(xqp[ab + l * 8l + 4l + j]) + } + } + isum += int(ksrow[kso + 4l + blk]) * ib + } + acc += float(isum) * (d8 * xsp[sb]) + } + } + return acc +} + // Row-range cores + full GEMVs per format. Not `private`: invoked through hoisted function // pointers from lifted worker lambdas (the fused chains) and the dispatch wrappers in common. def k4_rows_kernel(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, rb, re : int64) { @@ -931,6 +969,15 @@ def iq2xs_rows_kernel(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; x } } +def iq2xxs_rows_kernel(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, rb, re : int64) { + let nsb = n / 256l + unsafe { + for (i in range64(rb, re)) { + yp[i] = dot_iq2xxs_q8(kqp + i * nsb * 64l, ksp + i * nsb * 20l, xqp, xsp, xbsp, n) + } + } +} + def private kq_gemv_kernel(fmt : int; var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d : int64) { var myp = yp maybe_parallel_for(0, int(d), matmul_chunks_gemv(int(d), 1, n * d)) $(rb, re) { @@ -956,6 +1003,8 @@ def private kq_gemv_kernel(fmt : int; var yp : float?; kqp : uint8 const?; ksp : iq2s_rows_kernel(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb), int64(re)) } elif (fmt == 24) { iq2xs_rows_kernel(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb), int64(re)) + } elif (fmt == 25) { + iq2xxs_rows_kernel(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb), int64(re)) } else { k6_rows_kernel(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb), int64(re)) } @@ -1209,6 +1258,38 @@ def private dequant_iq2xs_row_grp(kqg : uint8 const?; ksg : uint8 const?; r, mr, } } +//! iq2xxs grp row dequant: 16 uniform 4-byte columns per superblock row - column 2b carries +//! block b's four grid bytes, column 2b+1 its aux32 (signs + the per-32 scale nibble, whose +//! strip byte the scale row already carries). +def private dequant_iq2xxs_row_grp(kqg : uint8 const?; ksg : uint8 const?; r, mr, nsb : int64; var dst : float?) { + let grid = iq2xxs_grid2() + let ksgn = ksigns_iq2xs() + unsafe { + for (sbi in range64(nsb)) { + let qb = sbi * 64l * mr + let sb = sbi * 20l * mr + let d8 = f16_to_f32(uint(ksg[sb + 2l * r]) | (uint(ksg[sb + 2l * r + 1l]) << 8u)) + for (b in range64(8l)) { + let db = d8 * float(int(ksg[sb + 4l * mr + b * mr + r])) + let ac = (2l * b + 1l) * mr + r + let aux = uint(kqg[qb + ac * 4l]) | (uint(kqg[qb + ac * 4l + 1l]) << 8u) | (uint(kqg[qb + ac * 4l + 2l]) << 16u) | (uint(kqg[qb + ac * 4l + 3l]) << 24u) + let gc = (2l * b) * mr + r + for (l in range64(4l)) { + let sgn = int(ksgn[int((aux >> uint(7l * l)) & 127u)]) + let idx = int(kqg[qb + gc * 4l + l]) + let g1 = grid[idx * 2] + let g2 = grid[idx * 2 + 1] + let kb = sbi * 256l + b * 32l + l * 8l + for (j in range64(4l)) { + dst[kb + j] = db * float((g1 >> uint(8l * j)) & 0xFFu) * (((sgn >> int(j)) & 1) != 0 ? -1.0 : 1.0) + dst[kb + 4l + j] = db * float((g2 >> uint(8l * j)) & 0xFFu) * (((sgn >> int(4l + j)) & 1) != 0 ? -1.0 : 1.0) + } + } + } + } + } +} + //! Dequant one row off the grp-REPACKED K-quant planes (repack_k4/k5/k6_grp layout). //! kqg/ksg = row's GROUP plane base (group g = row/mr); r = row % mr. //! embed_row's kq_repacked path — per-token, so scalar is fine. @@ -1252,6 +1333,10 @@ def dequant_kq_row_grp(fmt : int64; kqg : uint8 const?; ksg : uint8 const?; r, m dequant_iq2xs_row_grp(kqg, ksg, r, mr, nsb, dst) return } + if (fmt == 25l) { + dequant_iq2xxs_row_grp(kqg, ksg, r, mr, nsb, dst) + return + } unsafe { for (sbi in range64(nsb)) { let qb = sbi * qsb * mr @@ -1362,6 +1447,8 @@ def matmul_kq_groupn(fmt : int; var y : array | #; kq : array | #; myp[ii] = dot_iq2s_q8(kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) } elif (fmt == 24) { myp[ii] = dot_iq2xs_q8(kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) + } elif (fmt == 25) { + myp[ii] = dot_iq2xxs_q8(kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) } else { myp[ii] = dot_k6q8(kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) } diff --git a/modules/dasLLAMA/dasllama/dasllama_math_gen.das b/modules/dasLLAMA/dasllama/dasllama_math_gen.das index 969dcde657..d3a18370de 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_gen.das @@ -275,6 +275,10 @@ def iq2xsq8_layout_gen() : int { return q8q8_repack_type(GEMM_REFERENCE_MR()).interleave } +def iq2xxsq8_layout_gen() : int { + return q8q8_repack_type(GEMM_REFERENCE_MR()).interleave +} + // the KernelBackend.kq_layout slot form of kq_layout_of (registered on both gen backends) def private kq_layout_fmt_gen(fmt : int) : int => int(kq_layout_of(fmt)) @@ -314,6 +318,9 @@ def kq_layout_of(fmt : int) : int64 { if (fmt == 24) { return int64(iq2xsq8_layout_gen()) } + if (fmt == 25) { + return int64(iq2xxsq8_layout_gen()) + } return int64(k6q8_layout_gen()) } @@ -1048,6 +1055,79 @@ def iq2xsq8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xq } } + +//! One row's dot off the grp iq2xxs planes, scalar - the iq2xxs stubs' reference body and +//! the repack oracle: column 2b carries block b's grid bytes, column 2b+1 its aux32 (ksigns + +//! per-32 scale, mirrored into the strip row). Same fold order as dot_iq2xxs_q8. +[unused_argument(xbsp)] +def iq2xxs_grp_row_dot(kqg : uint8 const?; ksg : uint8 const?; r, mr : int64; xqp : int8 const?; xsp : float const?; xbsp : int const?; n : int64) : float { + var acc = 0.0 + let nsb = n / 256l + let grid = iq2xxs_grid2() + let ksgn = ksigns_iq2xs() + unsafe { + for (sbi in range64(nsb)) { + let qb = sbi * 64l * mr + let sb = sbi * 20l * mr + var isum = 0 + for (b in range64(8l)) { + let ac = (2l * b + 1l) * mr + r + let aux = uint(kqg[qb + ac * 4l]) | (uint(kqg[qb + ac * 4l + 1l]) << 8u) | (uint(kqg[qb + ac * 4l + 2l]) << 16u) | (uint(kqg[qb + ac * 4l + 3l]) << 24u) + let gc = (2l * b) * mr + r + var ibb = 0 + for (l in range64(4l)) { + let sgn = int(ksgn[int((aux >> uint(7l * l)) & 127u)]) + let idx = int(kqg[qb + gc * 4l + l]) + let g1 = grid[idx * 2] + let g2 = grid[idx * 2 + 1] + for (j in range64(4l)) { + let w1 = int((g1 >> uint(8l * j)) & 0xFFu) + let w2 = int((g2 >> uint(8l * j)) & 0xFFu) + ibb += (((sgn >> int(j)) & 1) != 0 ? -w1 : w1) * int(xqp[(sbi * 8l + b) * 32l + l * 8l + j]) + ibb += (((sgn >> int(4l + j)) & 1) != 0 ? -w2 : w2) * int(xqp[(sbi * 8l + b) * 32l + l * 8l + 4l + j]) + } + } + isum += int(ksg[sb + 4l * mr + b * mr + r]) * ibb + } + let dk = f16_to_f32(uint(ksg[sb + 2l * r]) | (uint(ksg[sb + 2l * r + 1l]) << 8u)) + acc += float(isum) * (dk * xsp[sbi]) + } + } + return acc +} + +[hint(unsafe_range_check, noalias = kqp, noalias = ksp, noalias = xqp, noalias = xsp, noalias = xbsp)] +def iq2xxsq8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, rb, re : int64) : void { + let mr = int64(iq2xxsq8_layout_gen()) + let nsb = n / 256l + unsafe { + for (i in range64(rb, re)) { + let g = i / mr + yp[i] = iq2xxs_grp_row_dot(kqp + g * mr * nsb * 64l, ksp + g * mr * nsb * 20l, i % mr, mr, xqp, xsp, xbsp, n) + } + } +} + +[tune_perm(mr = 4), tune_perm(mr = 4, nrsplit = 2), tune_perm(mr = 8), tune_perm(mr = 8, nrsplit = 2), + tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "maddubs", width = 256, mr = 8, nrsplit = 2, requires = "avx2"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, nrsplit = 2, requires = "avxvnni|avx512vnni"), + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, nrsplit = 2, requires = "avx512vnni,avx512bw"), + tune_companion(fn = "iq2xxsq8_gemv_gen", gen = "dasllama_gemm_gen::iq2xxs_gemv"), + tune_companion(fn = "iq2xxsq8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), + tune(gen = "dasllama_gemm_gen::iq2xxs_tile", + fallback = "dot_vpdpbusd_width512_mr16;dot_vpdpbusd_width256_mr8;dot_maddubs_width256_mr8;mr4"), + hint(unsafe_range_check, noalias = kqg, noalias = ksg, noalias = xqp, noalias = xsp, noalias = xbsp)] +def iq2xxsq8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d, g, t0 : int64) : void { + let mr = int64(iq2xxsq8_layout_gen()) + unsafe { + for (t in range64(4l)) { + for (r in range64(mr)) { + yp[(t0 + t) * d + g * mr + r] = iq2xxs_grp_row_dot(kqg, ksg, r, mr, xqp + (t0 + t) * n, xsp + (t0 + t) * (n / 256l), xbsp + (t0 + t) * (n / 16l), n) + } + } + } +} + //! The f16-scale GEMV companion (wscale_f16 rail): q8q8_gemv_gen's twin over the raw binary16 //! group-scale plane. Same rows-range contract; the generated body differs only in the scale //! fold's widen (fcvtl/vcvtph2ps). Not `private`: it IS the mm_rows_s16 slot. @@ -1391,6 +1471,8 @@ def private repack_kq_gen(fmt : int; var kq : uint8?; var ks : uint8?; n, d : in repack_iq2s_grp(kq, ks, n, d, kq_layout_of(23)) } elif (fmt == 24) { repack_iq2xs_grp(kq, ks, n, d, kq_layout_of(24)) + } elif (fmt == 25) { + repack_iq2xxs_grp(kq, ks, n, d, kq_layout_of(25)) } else { repack_k6_grp(kq, ks, n, d, kq_layout_of(6)) } @@ -1419,6 +1501,8 @@ def private repack_kq_bake(fmt : int; var kq : uint8?; var ks : uint8?; n, d : i repack_iq2s_grp(kq, ks, n, d, active_kq_layout_mr(23)) } elif (fmt == 24) { repack_iq2xs_grp(kq, ks, n, d, active_kq_layout_mr(24)) + } elif (fmt == 25) { + repack_iq2xxs_grp(kq, ks, n, d, active_kq_layout_mr(25)) } else { repack_k6_grp(kq, ks, n, d, active_kq_layout_mr(6)) } @@ -1945,6 +2029,8 @@ def private kq_kernel_gen(fmt : int; var yp : float?; kqp : uint8 const?; ksp : iq2sq8_gemv_gen(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb) * mr, int64(re) * mr) } elif (fmt == 24) { iq2xsq8_gemv_gen(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb) * mr, int64(re) * mr) + } elif (fmt == 25) { + iq2xxsq8_gemv_gen(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb) * mr, int64(re) * mr) } else { k6q8_gemv_gen(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb) * mr, int64(re) * mr) } @@ -1973,6 +2059,8 @@ def private kq_kernel_gen(fmt : int; var yp : float?; kqp : uint8 const?; ksp : myp[i] = dot_iq2s_q8(kqp + i * nsb * 72l, ksp + i * nsb * 20l, xqp, xsp, xbsp, n) } elif (fmt == 24) { myp[i] = dot_iq2xs_q8(kqp + i * nsb * 64l, ksp + i * nsb * 20l, xqp, xsp, xbsp, n) + } elif (fmt == 25) { + myp[i] = dot_iq2xxs_q8(kqp + i * nsb * 64l, ksp + i * nsb * 20l, xqp, xsp, xbsp, n) } else { myp[i] = dot_k6q8(kqp + i * nsb * 192l, ksp + i * nsb * 18l, xqp, xsp, xbsp, n) } @@ -1988,7 +2076,7 @@ def private kq_batch_cell_gen(fmt : int; var myp : float?; kqp : uint8 const?; k let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) let nb16 = n / 16l - let packed = fmt == 4 || fmt == 40 || fmt == 44 || fmt == 3 || fmt == 45 || fmt == 2 // packed planes: the tile reads them directly (iq3s/iq3xxs/iq2s expand their grid gathers into the panel) + let packed = fmt == 4 || fmt == 40 || fmt == 44 || fmt == 3 || fmt == 45 || fmt == 2 || fmt == 25 // packed planes: the tile reads them directly (iq3s/iq3xxs/iq2s expand their grid gathers into the panel) var scratch : array if (!packed) { scratch |> resize(mr * nsb * 256l) @@ -2026,6 +2114,8 @@ def private kq_batch_cell_gen(fmt : int; var myp : float?; kqp : uint8 const?; k iq4nlq8_tile_gen(myp, kqg, ksg, xqp, xsp, xbsp, n, d, g, tk) } elif (fmt == 2) { k2q8_tile_gen(myp, kqg, ksg, xqp, xsp, xbsp, n, d, g, tk) + } elif (fmt == 25) { + iq2xxsq8_tile_gen(myp, kqg, ksg, xqp, xsp, xbsp, n, d, g, tk) } elif (fmt == 23) { iq2sq8_tile_gen(myp, scp, ksg, xqp, xsp, xbsp, n, d, g, tk) } elif (fmt == 24) { @@ -2058,6 +2148,8 @@ def private kq_batch_cell_gen(fmt : int; var myp : float?; kqp : uint8 const?; k iq2sq8_gemv_gen(myp + tk * d, kqp, ksp, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n, g * mr, (g + 1l) * mr) } elif (fmt == 24) { iq2xsq8_gemv_gen(myp + tk * d, kqp, ksp, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n, g * mr, (g + 1l) * mr) + } elif (fmt == 25) { + iq2xxsq8_gemv_gen(myp + tk * d, kqp, ksp, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n, g * mr, (g + 1l) * mr) } else { k6q8_gemv_gen(myp + tk * d, kqp, ksp, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n, g * mr, (g + 1l) * mr) } @@ -2107,6 +2199,8 @@ def private kq_batch_kernel_gen(fmt : int; var yp : float?; kqp : uint8 const?; myp[tk * d + i] = dot_iq2s_q8(kqp + i * nsb * 72l, ksp + i * nsb * 20l, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * (n / 16l), n) } elif (fmt == 24) { myp[tk * d + i] = dot_iq2xs_q8(kqp + i * nsb * 64l, ksp + i * nsb * 20l, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * (n / 16l), n) + } elif (fmt == 25) { + myp[tk * d + i] = dot_iq2xxs_q8(kqp + i * nsb * 64l, ksp + i * nsb * 20l, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * (n / 16l), n) } else { myp[tk * d + i] = dot_k6q8(kqp + i * nsb * 192l, ksp + i * nsb * 18l, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * (n / 16l), n) } @@ -2171,6 +2265,8 @@ def private kq_batch_groupn_gen(fmt : int; var yp : float?; kqp : uint8 const?; myp[tk * d + i] = dot_iq2s_q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n) } elif (fmt == 24) { myp[tk * d + i] = dot_iq2xs_q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n) + } elif (fmt == 25) { + myp[tk * d + i] = dot_iq2xxs_q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n) } else { myp[tk * d + i] = dot_k6q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n) } @@ -2344,6 +2440,8 @@ def kq_groupn_gen(fmt : int; var yp : float?; kqp : uint8 const?; ksp : uint8 co iq2sq8_gemv_gen(myp + r * d, kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n, gLocal * mr, gEnd * mr) } elif (fmt == 24) { iq2xsq8_gemv_gen(myp + r * d, kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n, gLocal * mr, gEnd * mr) + } elif (fmt == 25) { + iq2xxsq8_gemv_gen(myp + r * d, kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n, gLocal * mr, gEnd * mr) } else { k6q8_gemv_gen(myp + r * d, kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n, gLocal * mr, gEnd * mr) } @@ -2378,6 +2476,8 @@ def kq_groupn_gen(fmt : int; var yp : float?; kqp : uint8 const?; ksp : uint8 co myp[r * d + i] = dot_iq2s_q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) } elif (fmt == 24) { myp[r * d + i] = dot_iq2xs_q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) + } elif (fmt == 25) { + myp[r * d + i] = dot_iq2xxs_q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) } else { myp[r * d + i] = dot_k6q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) } @@ -2671,7 +2771,7 @@ def dasllama_math_gen_register() { // the no-witness stance above covers kq too mm_kq = @@kq_kernel_gen, repack_kq = @@repack_kq_gen, kq_rows_k4 = @@k4q8_gemv_gen, kq_rows_k5 = @@k5q8_gemv_gen, - kq_rows_k6 = @@k6q8_gemv_gen, kq_rows_q40 = @@q40q8_gemv_gen, kq_rows_iq4xs = @@iq4xsq8_gemv_gen, kq_rows_k3 = @@k3q8_gemv_gen, kq_rows_iq3s = @@iq3sq8_gemv_gen, kq_rows_iq3xxs = @@iq3xxsq8_gemv_gen, kq_rows_iq4nl = @@iq4nlq8_gemv_gen, kq_rows_k2 = @@k2q8_gemv_gen, kq_rows_iq2s = @@iq2sq8_gemv_gen, kq_rows_iq2xs = @@iq2xsq8_gemv_gen, + kq_rows_k6 = @@k6q8_gemv_gen, kq_rows_q40 = @@q40q8_gemv_gen, kq_rows_iq4xs = @@iq4xsq8_gemv_gen, kq_rows_k3 = @@k3q8_gemv_gen, kq_rows_iq3s = @@iq3sq8_gemv_gen, kq_rows_iq3xxs = @@iq3xxsq8_gemv_gen, kq_rows_iq4nl = @@iq4nlq8_gemv_gen, kq_rows_k2 = @@k2q8_gemv_gen, kq_rows_iq2s = @@iq2sq8_gemv_gen, kq_rows_iq2xs = @@iq2xsq8_gemv_gen, kq_rows_iq2xxs = @@iq2xxsq8_gemv_gen, kq_batch = @@kq_batch_kernel_gen, kq_groupn = @@kq_groupn_gen, kq_batch_groupn = @@kq_batch_groupn_gen, // q51 family: tile + GEMV off the grp planes (the tile family's companions stamp @@ -2700,7 +2800,7 @@ def dasllama_math_gen_register() { mm_rows_s16 = @@q8q8_gemv_s16_gen, mm_kq = @@kq_kernel_gen, repack_kq = @@repack_kq_gen, kq_rows_k4 = @@k4q8_gemv_gen, kq_rows_k5 = @@k5q8_gemv_gen, - kq_rows_k6 = @@k6q8_gemv_gen, kq_rows_q40 = @@q40q8_gemv_gen, kq_rows_iq4xs = @@iq4xsq8_gemv_gen, kq_rows_k3 = @@k3q8_gemv_gen, kq_rows_iq3s = @@iq3sq8_gemv_gen, kq_rows_iq3xxs = @@iq3xxsq8_gemv_gen, kq_rows_iq4nl = @@iq4nlq8_gemv_gen, kq_rows_k2 = @@k2q8_gemv_gen, kq_rows_iq2s = @@iq2sq8_gemv_gen, kq_rows_iq2xs = @@iq2xsq8_gemv_gen, + kq_rows_k6 = @@k6q8_gemv_gen, kq_rows_q40 = @@q40q8_gemv_gen, kq_rows_iq4xs = @@iq4xsq8_gemv_gen, kq_rows_k3 = @@k3q8_gemv_gen, kq_rows_iq3s = @@iq3sq8_gemv_gen, kq_rows_iq3xxs = @@iq3xxsq8_gemv_gen, kq_rows_iq4nl = @@iq4nlq8_gemv_gen, kq_rows_k2 = @@k2q8_gemv_gen, kq_rows_iq2s = @@iq2sq8_gemv_gen, kq_rows_iq2xs = @@iq2xsq8_gemv_gen, kq_rows_iq2xxs = @@iq2xxsq8_gemv_gen, kq_batch = @@kq_batch_kernel_gen, kq_groupn = @@kq_groupn_gen, kq_batch_groupn = @@kq_batch_groupn_gen, groupn_q51 = @@q51q8_groupn_gen, repack_q51 = @@repack_q51_gen, diff --git a/modules/dasLLAMA/dasllama/dasllama_ple.das b/modules/dasLLAMA/dasllama/dasllama_ple.das index 36a44acccb..d3967ad802 100644 --- a/modules/dasLLAMA/dasllama/dasllama_ple.das +++ b/modules/dasLLAMA/dasllama/dasllama_ple.das @@ -78,6 +78,8 @@ def ple_check_table(t : Model; origin : string) { have = (long_length(t.iq2sq) / IQ2S_QSB) * 256l } elif (t.ple_emb_fmt == KqFmt.iq2xs) { have = (long_length(t.iq2xsq) / IQ2XS_QSB) * 256l + } elif (t.ple_emb_fmt == KqFmt.iq2xxs) { + have = (long_length(t.iq2xxsq) / IQ2XXS_QSB) * 256l } if (have < need) { panic("dasLLAMA: the PLE token table is not resident ({origin}): fmt {t.ple_emb_fmt} plane holds {have} of {need} elements - regenerate this carrier (it predates the pinned-table rail)") @@ -127,6 +129,8 @@ def private ple_gather_row(t : Model; row : int64; var dst : array; doff dequant_iq2s_plane_superblock(t.iq2sq, sb * IQ2S_QSB, t.iq2ss, sb * IQ2S_SSB, dst, doff + blk * 256l) } elif (t.ple_emb_fmt == KqFmt.iq2xs) { dequant_iq2xs_plane_superblock(t.iq2xsq, sb * IQ2XS_QSB, t.iq2xss, sb * IQ2XS_SSB, dst, doff + blk * 256l) + } elif (t.ple_emb_fmt == KqFmt.iq2xxs) { + dequant_iq2xxs_plane_superblock(t.iq2xxsq, sb * IQ2XXS_QSB, t.iq2xxss, sb * IQ2XXS_SSB, dst, doff + blk * 256l) } else { panic("ple_gather_row: '{t.ple_emb_fmt}' has no kq plane pair") } diff --git a/modules/dasLLAMA/dasllama/dasllama_repack.das b/modules/dasLLAMA/dasllama/dasllama_repack.das index 621abde579..3628e12d57 100644 --- a/modules/dasLLAMA/dasllama/dasllama_repack.das +++ b/modules/dasLLAMA/dasllama/dasllama_repack.das @@ -643,6 +643,50 @@ def repack_iq2xs_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { delete ts } +//! iq2xxs grp layout: the whole 64B row as 16 four-byte columns x mr rows - the u16 qs words +//! stay in place per column (each word's two bytes share a column); scales interleave as k4's +//! row with 16 strips ([d8 f16 x mr][pad x mr][16 strips x mr]). +def repack_iq2xxs_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { + let nsb = n / 256l + let qrow = nsb * 64l + let srow = nsb * 20l + let ng = d / mr + var tq : array + var ts : array + tq |> resize(d * qrow) + ts |> resize(d * srow) + unsafe { + var tqp = addr(tq[0]) + var tsp = addr(ts[0]) + memcpy(tqp, kq, d * qrow) + memcpy(tsp, ks, d * srow) + for (g in range64(ng)) { + for (sbi in range64(nsb)) { + let dq = g * mr * qrow + sbi * 64l * mr + let ds = g * mr * srow + sbi * 20l * mr + for (r in range64(mr)) { + let sq = (g * mr + r) * qrow + sbi * 64l + let ss = (g * mr + r) * srow + sbi * 20l + for (c in range64(16l)) { + for (t in range64(4l)) { + kq[dq + (c * mr + r) * 4l + t] = tqp[sq + c * 4l + t] + } + } + ks[ds + 2l * r] = tsp[ss] + ks[ds + 2l * r + 1l] = tsp[ss + 1l] + ks[ds + 2l * mr + 2l * r] = tsp[ss + 2l] + ks[ds + 2l * mr + 2l * r + 1l] = tsp[ss + 3l] + for (g16 in range64(16l)) { + ks[ds + 4l * mr + g16 * mr + r] = tsp[ss + 4l + g16] + } + } + } + } + } + delete tq + delete ts +} + // ===== grp panel unpack (the byte-expanded tile form the kq stamps load) ===== //! Unpack ONE group's packed grp k5/k6 quant panel into the BYTE-EXPANDED tile form (wlo/whi diff --git a/modules/dasLLAMA/harness/gen_tune_probe.das b/modules/dasLLAMA/harness/gen_tune_probe.das index 2ee026de85..a2a00972f2 100644 --- a/modules/dasLLAMA/harness/gen_tune_probe.das +++ b/modules/dasLLAMA/harness/gen_tune_probe.das @@ -441,7 +441,7 @@ def kq_qsb(fmt : int64) : int64 => dasllama_gemm_schema::kq_qsb(int(fmt)) def kq_ssb(fmt : int64) : int64 => dasllama_gemm_schema::kq_ssb(int(fmt)) // the family's sidecar entry = its [tune] function name (the K-quants by bit width - k3 included - q40/iq4xs by tag) -def kq_tile_entry(fmt : int64) : string => fmt == 40l ? "q40q8_tile_gen" : (fmt == 44l ? "iq4xsq8_tile_gen" : (fmt == 33l ? "iq3sq8_tile_gen" : (fmt == 34l ? "iq3xxsq8_tile_gen" : (fmt == 45l ? "iq4nlq8_tile_gen" : (fmt == 23l ? "iq2sq8_tile_gen" : (fmt == 24l ? "iq2xsq8_tile_gen" : "k{fmt}q8_tile_gen")))))) +def kq_tile_entry(fmt : int64) : string => fmt == 40l ? "q40q8_tile_gen" : (fmt == 44l ? "iq4xsq8_tile_gen" : (fmt == 33l ? "iq3sq8_tile_gen" : (fmt == 34l ? "iq3xxsq8_tile_gen" : (fmt == 45l ? "iq4nlq8_tile_gen" : (fmt == 23l ? "iq2sq8_tile_gen" : (fmt == 24l ? "iq2xsq8_tile_gen" : (fmt == 25l ? "iq2xxsq8_tile_gen" : "k{fmt}q8_tile_gen"))))))) // synthetic disk superblocks, varied per (row, superblock) so no two blocks repeat: any byte // is a legal quant, and the small d/dmin keep the f16 (s, o) pairs finite and row sums tame @@ -545,6 +545,14 @@ def pack_kq_disk_block(fmt : int64; var blkb : array; base : int) { // for (i in range(72)) { blkb[2 + i] = uint8((base * 41 + i * 97 + 17) % 256) } + } elif (fmt == 25l) { + // 66B IQ2_XXS disk block: f16 d, then 8 blocks x [4 grid bytes][aux32 signs + ls] - every byte pattern is legal + let dbits = f32_to_f16(0.002 + 0.0001 * float(base % 13)) + blkb[0] = uint8(dbits & 0xFF) + blkb[1] = uint8(dbits >> 8u) + for (i in range(64)) { + blkb[2 + i] = uint8((base * 41 + i * 97 + 17) % 256) + } } elif (fmt == 23l) { // 82B IQ2_S disk block: f16 d, 32 idx + 32 sign bytes, 8 qh, 8 packed scale nibbles - every byte pattern is legal let dbits = f32_to_f16(0.002 + 0.0001 * float(base % 13)) @@ -605,6 +613,8 @@ def repack_kq_grp_fmt(fmt : int64; var kq : uint8?; var ks : uint8?; n, d, mr : repack_iq2s_grp(kq, ks, n, d, mr) } elif (fmt == 24l) { repack_iq2xs_grp(kq, ks, n, d, mr) + } elif (fmt == 25l) { + repack_iq2xxs_grp(kq, ks, n, d, mr) } else { repack_k6_grp(kq, ks, n, d, mr) } @@ -617,7 +627,7 @@ def build_kq_fixture(fmt : int64; n, d, ntok : int64) : KqFixture { // nolint: fx.kq |> resize(d * fx.nsb * qsb) fx.ks |> resize(d * fx.nsb * ssb) var blkb : array - blkb |> resize(fmt == 2l ? 84l : (fmt == 23l ? 82l : (fmt == 24l ? 74l : (fmt == 4l || fmt == 40l || fmt == 45l ? 144l : (fmt == 5l ? 176l : (fmt == 44l ? 136l : (fmt == 3l || fmt == 33l ? 110l : (fmt == 34l ? 98l : 210l)))))))) + blkb |> resize(fmt == 2l ? 84l : (fmt == 23l ? 82l : (fmt == 24l ? 74l : (fmt == 25l ? 66l : (fmt == 4l || fmt == 40l || fmt == 45l ? 144l : (fmt == 5l ? 176l : (fmt == 44l ? 136l : (fmt == 3l || fmt == 33l ? 110l : (fmt == 34l ? 98l : 210l))))))))) for (r in range64(d)) { for (s in range64(fx.nsb)) { let base = int(r * 31l + s * 7l) @@ -645,6 +655,8 @@ def build_kq_fixture(fmt : int64; n, d, ntok : int64) : KqFixture { // nolint: transcode_iq2s_superblock(blkb, 0l, fx.kq, sb * qsb, fx.ks, sb * ssb) } elif (fmt == 24l) { transcode_iq2xs_superblock(blkb, 0l, fx.kq, sb * qsb, fx.ks, sb * ssb) + } elif (fmt == 25l) { + transcode_iq2xxs_superblock(blkb, 0l, fx.kq, sb * qsb, fx.ks, sb * ssb) } else { transcode_q6k_superblock(blkb, 0l, fx.kq, sb * qsb, fx.ks, sb * ssb) } @@ -694,7 +706,10 @@ def build_kq_fixture(fmt : int64; n, d, ntok : int64) : KqFixture { // nolint: : (fmt == 45l ? iq4nl_grp_row_dot(kqg, ksg, r, 4l, addr(fx.xq[0]) + tk * n, addr(fx.xs[0]) + tk * (n / 256l), addr(fx.xbs[0]) + tk * (n / 16l), n) - : (fmt == 24l + : (fmt == 25l + ? iq2xxs_grp_row_dot(kqg, ksg, r, 4l, + addr(fx.xq[0]) + tk * n, addr(fx.xs[0]) + tk * (n / 256l), addr(fx.xbs[0]) + tk * (n / 16l), n) + : (fmt == 24l ? iq2xs_grp_row_dot(kqg, ksg, r, 4l, addr(fx.xq[0]) + tk * n, addr(fx.xs[0]) + tk * (n / 256l), addr(fx.xbs[0]) + tk * (n / 16l), n) : (fmt == 23l @@ -704,7 +719,7 @@ def build_kq_fixture(fmt : int64; n, d, ntok : int64) : KqFixture { // nolint: ? k2_grp_row_dot(kqg, ksg, r, 4l, addr(fx.xq[0]) + tk * n, addr(fx.xs[0]) + tk * (n / 256l), addr(fx.xbs[0]) + tk * (n / 16l), n) : kq_grp_row_dot(fmt, kqg, ksg, r, 4l, - addr(fx.xq[0]) + tk * n, addr(fx.xs[0]) + tk * (n / 256l), addr(fx.xbs[0]) + tk * (n / 16l), n)))))))))) + addr(fx.xq[0]) + tk * n, addr(fx.xs[0]) + tk * (n / 256l), addr(fx.xbs[0]) + tk * (n / 16l), n))))))))))) } } } @@ -750,10 +765,13 @@ def kq_tile_variants(fmt : int64) : array> { if (fmt == 24l) { return <- iq2xsq8_tile_gen_variants() } + if (fmt == 25l) { + return <- iq2xxsq8_tile_gen_variants() + } return <- k6q8_tile_gen_variants() } -def kq_gemv_variants_by_suffix(fmt : int64) : table { // nolint:STYLE037 — the flat per-format ladder (one arm per kq format) is the honest shape +def kq_gemv_variants_by_suffix(fmt : int64) : table { // nolint:STYLE037,STYLE038 — the flat per-format ladder (one arm per kq format) is the honest shape var tab : table if (fmt == 4l) { var gvs <- k4q8_gemv_gen_variants() @@ -809,6 +827,12 @@ def kq_gemv_variants_by_suffix(fmt : int64) : table { // nol tab[v._0] = v._1 } delete gvs + } elif (fmt == 25l) { + var gvs <- iq2xxsq8_gemv_gen_variants() + for (v in gvs) { + tab[v._0] = v._1 + } + delete gvs } elif (fmt == 23l) { var gvs <- iq2sq8_gemv_gen_variants() for (v in gvs) { @@ -833,7 +857,7 @@ def kq_gemv_variants_by_suffix(fmt : int64) : table { // nol // suffix -> plane interleave off the family's OWN layout companion registry (per-format // layouts since the family split — declined rows report the reference grp4) -def kq_layout_mrs(fmt : int64) : table { // nolint:STYLE037 — the flat per-format ladder (one arm per kq format) is the honest shape +def kq_layout_mrs(fmt : int64) : table { // nolint:STYLE037,STYLE038 — the flat per-format ladder (one arm per kq format) is the honest shape var mrs : table if (fmt == 4l) { var lvs <- k4q8_layout_gen_variants() @@ -889,6 +913,12 @@ def kq_layout_mrs(fmt : int64) : table { // nolint:STYLE037 — t mrs[v._0] = invoke(v._1) } delete lvs + } elif (fmt == 25l) { + var lvs <- iq2xxsq8_layout_gen_variants() + for (v in lvs) { + mrs[v._0] = invoke(v._1) + } + delete lvs } elif (fmt == 23l) { var lvs <- iq2sq8_layout_gen_variants() for (v in lvs) { @@ -918,7 +948,7 @@ def kq_layout_mrs(fmt : int64) : table { // nolint:STYLE037 — t def run_kq_tile(tile; fx : KqFixture; kq : array; ks : array; mr : int64; var y : array) { let ng = fx.d / mr verify(fx.ntok % 4l == 0l) - let packed = fx.fmt == 4l || fx.fmt == 40l || fx.fmt == 44l || fx.fmt == 3l || fx.fmt == 45l || fx.fmt == 2l // packed planes: the tile reads them directly (iq3s/iq3xxs/iq2s unpack their panels per group) + let packed = fx.fmt == 4l || fx.fmt == 40l || fx.fmt == 44l || fx.fmt == 3l || fx.fmt == 45l || fx.fmt == 2l || fx.fmt == 25l // packed planes: the tile reads them directly (iq3s/iq3xxs/iq2s unpack their panels per group) var panel : array if (!packed) { panel |> resize(int(mr * fx.nsb * 256l)) @@ -2016,7 +2046,7 @@ def tune_mode_run : bool { // nolint:STYLE037,STYLE038 — flat family-by-family // the --tune-only re-mint filter: only matching families race (q8q8 brings its e2e // confirm along); a skipped family writes no entry, so its previous seat survives the // upsert - a one-family re-mint after landing a format is seconds, not the full walk - var kqSel <- [for (fmt in fixed_array(4l, 5l, 6l, 40l, 44l, 3l, 33l, 34l, 45l, 2l, 23l, 24l)); fmt; where tune_family_selected(kq_tile_entry(fmt))] + var kqSel <- [for (fmt in fixed_array(4l, 5l, 6l, 40l, 44l, 3l, 33l, 34l, 45l, 2l, 23l, 24l, 25l)); fmt; where tune_family_selected(kq_tile_entry(fmt))] let wantQ8 = tune_family_selected("q8q8_tile_gen") let wantQ51 = tune_family_selected("q51q8_tile_gen") let planTotal = (wantQ8 ? 2 : 0) + length(kqSel) + (wantQ51 ? 1 : 0) @@ -2026,7 +2056,7 @@ def tune_mode_run : bool { // nolint:STYLE037,STYLE038 — flat family-by-family return true } if (!empty(tune_only_filter())) { - tune_detail("tune-only '{tune_only_filter()}': racing {planTotal - (wantQ8 ? 1 : 0)} of 14 generator families\n") + tune_detail("tune-only '{tune_only_filter()}': racing {planTotal - (wantQ8 ? 1 : 0)} of 15 generator families\n") } // refuse before burning minutes measuring noise; every winner below queues into `pending` // and nothing reaches the sidecar until the end-of-run gate passes diff --git a/modules/dasLLAMA/tests/test_kqformat.das b/modules/dasLLAMA/tests/test_kqformat.das index d094678d19..4c654c49f7 100644 --- a/modules/dasLLAMA/tests/test_kqformat.das +++ b/modules/dasLLAMA/tests/test_kqformat.das @@ -28,6 +28,7 @@ def test_kqfmt_enum_values(t : T?) { t |> equal(int(KqFmt.k2), 11) t |> equal(int(KqFmt.iq2s), 12) t |> equal(int(KqFmt.iq2xs), 13) + t |> equal(int(KqFmt.iq2xxs), 14) } } @@ -56,6 +57,8 @@ def test_kqfmt_predicates(t : T?) { t |> success(kq_sb(int(KqFmt.iq2s)), "the int-id predicate agrees for iq2s") t |> success(kq_sb(KqFmt.iq2xs), "iq2xs is superblock") t |> success(kq_sb(int(KqFmt.iq2xs)), "the int-id predicate agrees for iq2xs") + t |> success(kq_sb(KqFmt.iq2xxs), "iq2xxs is superblock") + t |> success(kq_sb(int(KqFmt.iq2xxs)), "the int-id predicate agrees for iq2xxs") } } @@ -88,6 +91,8 @@ def test_kqfmt_strides(t : T?) { t |> equal(kq_ssb(KqFmt.iq2s), 20l) // f16 d/8 + 16 x (1 + 2s) strips t |> equal(kq_qsb(KqFmt.iq2xs), 64l) // 32 u16 [9-bit grid idx][7-bit ksigns idx] words, verbatim t |> equal(kq_ssb(KqFmt.iq2xs), 20l) // the iq2s row exactly - f16 d/8 + 16 x (1 + 2s) + t |> equal(kq_qsb(KqFmt.iq2xxs), 64l) // 8 blocks x [4 grid bytes][aux32 signs + ls], verbatim + t |> equal(kq_ssb(KqFmt.iq2xxs), 20l) // the iq2s row exactly - f16 d/8 + 16 x (1 + 2s) } t |> run("named constants agree with the literals") @(t : T?) { t |> equal(K4_QSB, 128l) @@ -110,6 +115,8 @@ def test_kqfmt_strides(t : T?) { t |> equal(IQ2S_SSB, 20l) t |> equal(IQ2XS_QSB, 64l) t |> equal(IQ2XS_SSB, 20l) + t |> equal(IQ2XXS_QSB, 64l) + t |> equal(IQ2XXS_SSB, 20l) t |> equal(IQ3S_QSB, 104l) t |> equal(IQ3S_SSB, 20l) } @@ -150,9 +157,10 @@ def test_kqfmt_id_bridge(t : T?) { t |> equal(kq_schema_id(KqFmt.k2), 2) t |> equal(kq_schema_id(KqFmt.iq2s), 23) t |> equal(kq_schema_id(KqFmt.iq2xs), 24) + t |> equal(kq_schema_id(KqFmt.iq2xxs), 25) } t |> run("gemm_schema's int-id forms delegate to the same truth") @(t : T?) { - for (f in [KqFmt.k4, KqFmt.k5, KqFmt.k6, KqFmt.q40, KqFmt.iq4xs, KqFmt.k3, KqFmt.iq3s, KqFmt.iq3xxs, KqFmt.iq4nl, KqFmt.k2, KqFmt.iq2s, KqFmt.iq2xs]) { + for (f in [KqFmt.k4, KqFmt.k5, KqFmt.k6, KqFmt.q40, KqFmt.iq4xs, KqFmt.k3, KqFmt.iq3s, KqFmt.iq3xxs, KqFmt.iq4nl, KqFmt.k2, KqFmt.iq2s, KqFmt.iq2xs, KqFmt.iq2xxs]) { t |> equal(kq_qsb(kq_schema_id(f)), kq_qsb(f)) t |> equal(kq_ssb(kq_schema_id(f)), kq_ssb(f)) } @@ -174,6 +182,7 @@ def test_kqfmt_elems_and_stream(t : T?) { t |> equal(kq_elems(KqFmt.k2), 256l) t |> equal(kq_elems(KqFmt.iq2s), 256l) t |> equal(kq_elems(KqFmt.iq2xs), 256l) + t |> equal(kq_elems(KqFmt.iq2xxs), 256l) t |> equal(kq_elems(KqFmt.q8), 32l) t |> equal(kq_elems(KqFmt.q51), 32l) } @@ -192,6 +201,7 @@ def test_kqfmt_elems_and_stream(t : T?) { t |> equal(kq_stream_code(KqFmt.k2), 20) // kernel id 2 = q51's stream tag t |> equal(kq_stream_code(KqFmt.iq2s), 23) t |> equal(kq_stream_code(KqFmt.iq2xs), 24) + t |> equal(kq_stream_code(KqFmt.iq2xxs), 25) } } diff --git a/modules/dasLLAMA/tests/test_kquant.das b/modules/dasLLAMA/tests/test_kquant.das index 57f6b035c9..99e4e4933d 100644 --- a/modules/dasLLAMA/tests/test_kquant.das +++ b/modules/dasLLAMA/tests/test_kquant.das @@ -1,8 +1,8 @@ options gen2 options stack = 524288 // every dasLLAMA program root takes this budget (options stack does not unify up from libs) options persistent_heap // the layout arm loads a multi-GB model; explicit delete below -options _cyclomatic_complexity = 70 // every gate here is the same flat fmt == 4/5/6/40/44/3 ladder repeated per stage -options _function_length = 280 // and each gate carries its whole fixture build + oracle inline +options _cyclomatic_complexity = 76 // every gate here is the same flat fmt == 4/5/6/40/44/3 ladder repeated per stage +options _function_length = 300 // and each gate carries its whole fixture build + oracle inline options _dasllama_internal = true require dastest/testing_boost public @@ -142,7 +142,7 @@ def private build_iq4nl_block() : array { return <- blkb } -def private kq_tag(fmt : int) : string => fmt == 40 ? "q40" : (fmt == 44 ? "iq4xs" : (fmt == 33 ? "iq3s" : (fmt == 34 ? "iq3xxs" : (fmt == 45 ? "iq4nl" : (fmt == 23 ? "iq2s" : (fmt == 24 ? "iq2xs" : "k{fmt}")))))) +def private kq_tag(fmt : int) : string => fmt == 40 ? "q40" : (fmt == 44 ? "iq4xs" : (fmt == 33 ? "iq3s" : (fmt == 34 ? "iq3xxs" : (fmt == 45 ? "iq4nl" : (fmt == 23 ? "iq2s" : (fmt == 24 ? "iq2xs" : (fmt == 25 ? "iq2xxs" : "k{fmt}"))))))) // iq4xs synthetic superblock: 136B disk block — f16 d = SYNTH_D, the 8 six-bit sub-scales are // SYNTH_SC (packed low nibbles into scales_l, top two bits into scales_h), the shared nibble @@ -322,6 +322,33 @@ def private build_iq2xs_block() : array { return <- blkb } +// iq2xxs synthetic superblock: 66B IQ2_XXS disk block - f16 d, per block 4 grid byte indices +// + an aux32 packing four 7-bit ksigns indices and the per-32 scale nibble (written in the +// PACK direction); the gate recomposes each weight through IQ2XXS_GRID + the shared KSIGNS_IQ2XS +def private iq2xxs_ix_pat(i : int) : int => (i * 29 + 7) % 256 +def private iq2xxs_sg7_pat(i : int) : int => (i * 5 + 3) % 128 +def private build_iq2xxs_block() : array { + var blkb : array + blkb |> resize(66) + let dbits = f32_to_f16(SYNTH_D) + blkb[0] = uint8(dbits & 0xFF) + blkb[1] = uint8(dbits >> 8u) + for (b in range(8)) { + for (l in range(4)) { + blkb[2 + b * 8 + l] = uint8(iq2xxs_ix_pat(b * 4 + l)) + } + var aux = 0 + for (l in range(4)) { + aux = aux | (iq2xxs_sg7_pat(b * 4 + l) << (7 * l)) + } + aux = aux | ((SYNTH_SC[b % 8] & 15) << 28) + for (k in range(4)) { + blkb[2 + b * 8 + 4 + k] = uint8((aux >> (8 * k)) & 0xFF) + } + } + return <- blkb +} + // q40 synthetic superblock: 8 x 18B q4_0 disk blocks, per-block f16-exact d = (blk+1)/16 and // the shared nibble pattern — weight k = d(k/32) * (q4_pat(k) - 8) def private q40_d(blk : int) : float => float(blk + 1) * 0.0625 @@ -577,6 +604,32 @@ def test_kq_transcode_planes(t : T?) { t |> success(dst[k] == expected, "iq2xs plane element must match the ggml dequant exactly") } } + t |> run("iq2xxs planes eighth the d, decode the per-32 strips, and gather the 256-entry grid + ksigns exactly") @(t : T?) { + let blkb <- build_iq2xxs_block() + var kq : array + var ks : array + kq |> resize(64) + ks |> resize(20) + transcode_iq2xxs_superblock(blkb, 0l, kq, 0l, ks, 0l) + for (b in range(8)) { + t |> equal(int(ks[4 + b]), 1 + 2 * (SYNTH_SC[b % 8] & 15), "decoded per-32 strip is 1 + 2s") + } + var dst : array + dst |> resize(256) + dequant_iq2xxs_plane_superblock(kq, 0l, ks, 0l, dst, 0l) + let d8 = SYNTH_D * 0.125 + for (k in range(256)) { + let b = k / 32 + let l = (k % 32) / 8 + let j = k % 8 + let idx = iq2xxs_ix_pat(b * 4 + l) + let gw = j < 4 ? IQ2XXS_GRID[idx * 2] : IQ2XXS_GRID[idx * 2 + 1] + let g = int((gw >> uint(8 * (j % 4))) & 0xFFu) + let sgn = ((int(KSIGNS_IQ2XS[iq2xxs_sg7_pat(b * 4 + l)]) >> j) & 1) != 0 ? -1.0 : 1.0 + let expected = (d8 * float(1 + 2 * (SYNTH_SC[b % 8] & 15))) * float(g) * sgn + t |> success(dst[k] == expected, "iq2xxs plane element must match the ggml dequant exactly") + } + } t |> run("iq2s planes eighth the d, decode the strips, and gather the u64 grid exactly") @(t : T?) { let blkb <- build_iq2s_block() var kq : array @@ -849,7 +902,7 @@ def test_q51_kernels(t : T?) { // elementwise dot over the plane dequant (the kernels' declared oracle). def private kq_dot_gate(t : T?; fmt : int; n : int64 = 512l) { let nsb = n / 256l - let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : (fmt == 45 ? build_iq4nl_block() : (fmt == 2 ? build_q2k_block() : (fmt == 23 ? build_iq2s_block() : (fmt == 24 ? build_iq2xs_block() : build_q6k_block())))))))))) + let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : (fmt == 45 ? build_iq4nl_block() : (fmt == 2 ? build_q2k_block() : (fmt == 23 ? build_iq2s_block() : (fmt == 24 ? build_iq2xs_block() : (fmt == 25 ? build_iq2xxs_block() : build_q6k_block()))))))))))) let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) var kq : array @@ -877,6 +930,8 @@ def private kq_dot_gate(t : T?; fmt : int; n : int64 = 512l) { transcode_q2k_superblock(blk1, 0l, kq, s * qsb, ks, s * ssb) } elif (fmt == 24) { transcode_iq2xs_superblock(blk1, 0l, kq, s * qsb, ks, s * ssb) + } elif (fmt == 25) { + transcode_iq2xxs_superblock(blk1, 0l, kq, s * qsb, ks, s * ssb) } elif (fmt == 23) { transcode_iq2s_superblock(blk1, 0l, kq, s * qsb, ks, s * ssb) } else { @@ -912,6 +967,8 @@ def private kq_dot_gate(t : T?; fmt : int; n : int64 = 512l) { dequant_k2_plane_superblock(kq, s * qsb, ks, s * ssb, w, s * 256l) } elif (fmt == 24) { dequant_iq2xs_plane_superblock(kq, s * qsb, ks, s * ssb, w, s * 256l) + } elif (fmt == 25) { + dequant_iq2xxs_plane_superblock(kq, s * qsb, ks, s * ssb, w, s * 256l) } elif (fmt == 23) { dequant_iq2s_plane_superblock(kq, s * qsb, ks, s * ssb, w, s * 256l) } else { @@ -959,6 +1016,8 @@ def private kq_dot_gate(t : T?; fmt : int; n : int64 = 512l) { got = dot_k2q8(addr(kq[0]), addr(ks[0]), addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } elif (fmt == 24) { got = dot_iq2xs_q8(addr(kq[0]), addr(ks[0]), addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + } elif (fmt == 25) { + got = dot_iq2xxs_q8(addr(kq[0]), addr(ks[0]), addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } elif (fmt == 23) { got = dot_iq2s_q8(addr(kq[0]), addr(ks[0]), addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } else { @@ -994,7 +1053,7 @@ def test_kq_dots(t : T?) { } // 122B-class row widths: n=3072 (12 superblocks — qwen35moe-122B gate/up/cls) and n=1024 // (its k5 down_exps). Every earlier kq model kept n at 512..2560. - for (fmt in [4, 5, 6, 40, 44, 3, 33, 34, 45, 2, 23, 24]) { + for (fmt in [4, 5, 6, 40, 44, 3, 33, 34, 45, 2, 23, 24, 25]) { for (nn in [1024l, 3072l]) { t |> run("dot_{kq_tag(fmt)}q8 matches the fp64 reference at n={int(nn)}") @(t : T?) { kq_dot_gate(t, fmt, nn) @@ -1013,7 +1072,7 @@ def private kq_gemv_rows_gate(t : T?; fmt : int; n : int64 = 512l) { let nsb = n / 256l let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) - let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : (fmt == 45 ? build_iq4nl_block() : (fmt == 2 ? build_q2k_block() : (fmt == 23 ? build_iq2s_block() : (fmt == 24 ? build_iq2xs_block() : build_q6k_block())))))))))) + let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : (fmt == 45 ? build_iq4nl_block() : (fmt == 2 ? build_q2k_block() : (fmt == 23 ? build_iq2s_block() : (fmt == 24 ? build_iq2xs_block() : (fmt == 25 ? build_iq2xxs_block() : build_q6k_block()))))))))))) var kq : array var ks : array kq |> resize(d * nsb * qsb) @@ -1040,6 +1099,8 @@ def private kq_gemv_rows_gate(t : T?; fmt : int; n : int64 = 512l) { transcode_q2k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } elif (fmt == 24) { transcode_iq2xs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 25) { + transcode_iq2xxs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } elif (fmt == 23) { transcode_iq2s_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } else { @@ -1094,6 +1155,8 @@ def private kq_gemv_rows_gate(t : T?; fmt : int; n : int64 = 512l) { want = dot_k2q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } elif (fmt == 24) { want = dot_iq2xs_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + } elif (fmt == 25) { + want = dot_iq2xxs_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } elif (fmt == 23) { want = dot_iq2s_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } else { @@ -1137,7 +1200,7 @@ def test_kq_gemv_rows(t : T?) { t |> run("portable iq3s GEMV rows bit-match per-row disk dots") @(t : T?) { kq_gemv_rows_gate(t, 33) } - for (fmt in [4, 5, 6, 40, 44, 3, 33, 34, 45, 2, 23, 24]) { + for (fmt in [4, 5, 6, 40, 44, 3, 33, 34, 45, 2, 23, 24, 25]) { for (nn in [1024l, 3072l]) { t |> run("portable {kq_tag(fmt)} GEMV rows bit-match per-row disk dots at n={int(nn)}") @(t : T?) { kq_gemv_rows_gate(t, fmt, nn) @@ -1157,7 +1220,7 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { let nsb = n / 256l let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) - let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : (fmt == 45 ? build_iq4nl_block() : (fmt == 2 ? build_q2k_block() : (fmt == 23 ? build_iq2s_block() : (fmt == 24 ? build_iq2xs_block() : build_q6k_block())))))))))) + let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : (fmt == 45 ? build_iq4nl_block() : (fmt == 2 ? build_q2k_block() : (fmt == 23 ? build_iq2s_block() : (fmt == 24 ? build_iq2xs_block() : (fmt == 25 ? build_iq2xxs_block() : build_q6k_block()))))))))))) var kq : array var ks : array kq |> resize(d * nsb * qsb) @@ -1184,6 +1247,8 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { transcode_q2k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } elif (fmt == 24) { transcode_iq2xs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 25) { + transcode_iq2xxs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } elif (fmt == 23) { transcode_iq2s_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } else { @@ -1237,6 +1302,8 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { want[r] = dot_k2q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } elif (fmt == 24) { want[r] = dot_iq2xs_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + } elif (fmt == 25) { + want[r] = dot_iq2xxs_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } elif (fmt == 23) { want[r] = dot_iq2s_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } else { @@ -1263,6 +1330,8 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { dequant_k2_plane_superblock(kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb, wrow, r * n + s * 256l) } elif (fmt == 24) { dequant_iq2xs_plane_superblock(kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb, wrow, r * n + s * 256l) + } elif (fmt == 25) { + dequant_iq2xxs_plane_superblock(kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb, wrow, r * n + s * 256l) } elif (fmt == 23) { dequant_iq2s_plane_superblock(kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb, wrow, r * n + s * 256l) } else { @@ -1290,6 +1359,8 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { repack_k2_grp(addr(kq[0]), addr(ks[0]), n, d, mr) } elif (fmt == 24) { repack_iq2xs_grp(addr(kq[0]), addr(ks[0]), n, d, mr) + } elif (fmt == 25) { + repack_iq2xxs_grp(addr(kq[0]), addr(ks[0]), n, d, mr) } elif (fmt == 23) { repack_iq2s_grp(addr(kq[0]), addr(ks[0]), n, d, mr) } else { @@ -1309,8 +1380,8 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { ? iq4xs_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) : (fmt == 3 ? k3_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) - : (fmt == 33 || fmt == 34 || fmt == 45 || fmt == 2 || fmt == 23 || fmt == 24 - ? (fmt == 33 ? iq3s_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) : (fmt == 34 ? iq3xxs_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) : (fmt == 45 ? iq4nl_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) : (fmt == 2 ? k2_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) : (fmt == 23 ? iq2s_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) : iq2xs_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n)))))) + : (fmt == 33 || fmt == 34 || fmt == 45 || fmt == 2 || fmt == 23 || fmt == 24 || fmt == 25 + ? (fmt == 33 ? iq3s_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) : (fmt == 34 ? iq3xxs_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) : (fmt == 45 ? iq4nl_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) : (fmt == 2 ? k2_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) : (fmt == 23 ? iq2s_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) : (fmt == 24 ? iq2xs_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) : iq2xxs_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n))))))) : kq_grp_row_dot(int64(fmt), kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n))))) if (got != want[r]) { dotbad++ @@ -1338,7 +1409,7 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { [test] def test_kq_repack_grp(t : T?) { - for (fmt in [4, 5, 6, 40, 44, 3, 33, 34, 45, 2, 23, 24]) { + for (fmt in [4, 5, 6, 40, 44, 3, 33, 34, 45, 2, 23, 24, 25]) { for (mr in [4l, 8l, 16l]) { t |> run("repack_{kq_tag(fmt)}_grp mr={mr} preserves dots and row dequants") @(t : T?) { kq_repack_gate(t, fmt, mr) @@ -1363,7 +1434,7 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { // nolint:STYL let mr = kq_layout_of(fmt) // the format's OWN layout companion (per-format tune families) let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) - let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : (fmt == 45 ? build_iq4nl_block() : (fmt == 2 ? build_q2k_block() : (fmt == 23 ? build_iq2s_block() : (fmt == 24 ? build_iq2xs_block() : build_q6k_block())))))))))) + let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : (fmt == 45 ? build_iq4nl_block() : (fmt == 2 ? build_q2k_block() : (fmt == 23 ? build_iq2s_block() : (fmt == 24 ? build_iq2xs_block() : (fmt == 25 ? build_iq2xxs_block() : build_q6k_block()))))))))))) var kq : array var ks : array kq |> resize(d * nsb * qsb) @@ -1390,6 +1461,8 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { // nolint:STYL transcode_q2k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } elif (fmt == 24) { transcode_iq2xs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 25) { + transcode_iq2xxs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } elif (fmt == 23) { transcode_iq2s_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } else { @@ -1423,6 +1496,8 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { // nolint:STYL repack_k2_grp(addr(kq[0]), addr(ks[0]), n, d, mr) } elif (fmt == 24) { repack_iq2xs_grp(addr(kq[0]), addr(ks[0]), n, d, mr) + } elif (fmt == 25) { + repack_iq2xxs_grp(addr(kq[0]), addr(ks[0]), n, d, mr) } elif (fmt == 23) { repack_iq2s_grp(addr(kq[0]), addr(ks[0]), n, d, mr) } else { @@ -1453,7 +1528,7 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { // nolint:STYL // k5/k6 tiles read the byte-expanded panel the batch cell unpacks per (group, token-block) // — the gate covers unpack_kq_panel_grp too (tile-over-panel vs gemv-over-packed planes); // k4/q40 tiles read the packed planes directly - let packed = fmt == 4 || fmt == 40 || fmt == 44 || fmt == 3 || fmt == 45 || fmt == 2 // iq3s/iq3xxs/iq2s/iq2xs tiles read the byte-expanded panel (the batch cell's shape) + let packed = fmt == 4 || fmt == 40 || fmt == 44 || fmt == 3 || fmt == 45 || fmt == 2 || fmt == 25 // iq3s/iq3xxs/iq2s/iq2xs tiles read the byte-expanded panel (the batch cell's shape) var panel : array if (!packed) { panel |> resize(mr * nsb * 256l) @@ -1486,6 +1561,8 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { // nolint:STYL iq4nlq8_tile_gen(addr(ytile[0]), kqg, ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) } elif (fmt == 2) { k2q8_tile_gen(addr(ytile[0]), kqg, ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) + } elif (fmt == 25) { + iq2xxsq8_tile_gen(addr(ytile[0]), kqg, ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) } elif (fmt == 24) { iq2xsq8_tile_gen(addr(ytile[0]), addr(panel[0]), ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) } elif (fmt == 23) { @@ -1516,6 +1593,8 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { // nolint:STYL k2q8_gemv_gen(addr(ytile[tk * d]), addr(kq[0]), addr(ks[0]), addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, g * mr, (g + 1l) * mr) } elif (fmt == 24) { iq2xsq8_gemv_gen(addr(ytile[tk * d]), addr(kq[0]), addr(ks[0]), addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, g * mr, (g + 1l) * mr) + } elif (fmt == 25) { + iq2xxsq8_gemv_gen(addr(ytile[tk * d]), addr(kq[0]), addr(ks[0]), addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, g * mr, (g + 1l) * mr) } elif (fmt == 23) { iq2sq8_gemv_gen(addr(ytile[tk * d]), addr(kq[0]), addr(ks[0]), addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, g * mr, (g + 1l) * mr) } else { @@ -1545,6 +1624,8 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { // nolint:STYL k2q8_gemv_gen(addr(ygemv[p * d]), addr(kq[0]), addr(ks[0]), addr(xq[p * n]), addr(xs[p * (n / 256l)]), addr(xbs[p * (n / 16l)]), n, 0l, d) } elif (fmt == 24) { iq2xsq8_gemv_gen(addr(ygemv[p * d]), addr(kq[0]), addr(ks[0]), addr(xq[p * n]), addr(xs[p * (n / 256l)]), addr(xbs[p * (n / 16l)]), n, 0l, d) + } elif (fmt == 25) { + iq2xxsq8_gemv_gen(addr(ygemv[p * d]), addr(kq[0]), addr(ks[0]), addr(xq[p * n]), addr(xs[p * (n / 256l)]), addr(xbs[p * (n / 16l)]), n, 0l, d) } elif (fmt == 23) { iq2sq8_gemv_gen(addr(ygemv[p * d]), addr(kq[0]), addr(ks[0]), addr(xq[p * n]), addr(xs[p * (n / 256l)]), addr(xbs[p * (n / 16l)]), n, 0l, d) } else { @@ -1572,7 +1653,7 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { // nolint:STYL [test] def test_kq_tile(t : T?) { - for (fmt in [4, 5, 6, 40, 44, 3, 33, 34, 45, 2, 23, 24]) { + for (fmt in [4, 5, 6, 40, 44, 3, 33, 34, 45, 2, 23, 24, 25]) { t |> run("{kq_tag(fmt)} 4-token tile bit-matches per-token GEMVs") @(t : T?) { kq_tile_gate(t, fmt) } @@ -1593,7 +1674,7 @@ def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; let nsb = n / 256l let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) - let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : (fmt == 45 ? build_iq4nl_block() : (fmt == 2 ? build_q2k_block() : (fmt == 23 ? build_iq2s_block() : (fmt == 24 ? build_iq2xs_block() : build_q6k_block())))))))))) + let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : (fmt == 45 ? build_iq4nl_block() : (fmt == 2 ? build_q2k_block() : (fmt == 23 ? build_iq2s_block() : (fmt == 24 ? build_iq2xs_block() : (fmt == 25 ? build_iq2xxs_block() : build_q6k_block()))))))))))) var kq : array var ks : array kq |> resize(nreg * d * nsb * qsb) @@ -1620,6 +1701,8 @@ def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; transcode_q2k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } elif (fmt == 24) { transcode_iq2xs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 25) { + transcode_iq2xxs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } elif (fmt == 23) { transcode_iq2s_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } else { @@ -1700,6 +1783,9 @@ def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; } elif (fmt == 24) { want_sh[r * d + row] = dot_iq2xs_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) want_pr[r * d + row] = dot_iq2xs_q8(kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n) + } elif (fmt == 25) { + want_sh[r * d + row] = dot_iq2xxs_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + want_pr[r * d + row] = dot_iq2xxs_q8(kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n) } elif (fmt == 23) { want_sh[r * d + row] = dot_iq2s_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) want_pr[r * d + row] = dot_iq2s_q8(kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n) @@ -1759,6 +1845,8 @@ def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; repack_k2_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) } elif (fmt == 24) { repack_iq2xs_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) + } elif (fmt == 25) { + repack_iq2xxs_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) } elif (fmt == 23) { repack_iq2s_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) } else { @@ -1798,6 +1886,9 @@ def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; } elif (fmt == 24) { iq2xsq8_gemv_gen(addr(want_sh[r * d]), kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, 0l, d) iq2xsq8_gemv_gen(addr(want_pr[r * d]), kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n, 0l, d) + } elif (fmt == 25) { + iq2xxsq8_gemv_gen(addr(want_sh[r * d]), kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, 0l, d) + iq2xxsq8_gemv_gen(addr(want_pr[r * d]), kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n, 0l, d) } elif (fmt == 23) { iq2sq8_gemv_gen(addr(want_sh[r * d]), kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, 0l, d) iq2sq8_gemv_gen(addr(want_pr[r * d]), kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n, 0l, d) @@ -1836,7 +1927,7 @@ def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; [test] def test_kq_groupn(t : T?) { - for (fmt in [4, 5, 6, 40, 44, 3, 33, 34, 45, 2, 23, 24]) { + for (fmt in [4, 5, 6, 40, 44, 3, 33, 34, 45, 2, 23, 24, 25]) { t |> run("{kq_tag(fmt)} region-list GEMV bit-matches per-row dots (disk + grp slices)") @(t : T?) { kq_groupn_gate(t, fmt) } @@ -1880,7 +1971,7 @@ def private kq_batch_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 for (c in cnts) { nk += c } - let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : (fmt == 45 ? build_iq4nl_block() : (fmt == 2 ? build_q2k_block() : (fmt == 23 ? build_iq2s_block() : (fmt == 24 ? build_iq2xs_block() : build_q6k_block())))))))))) + let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : (fmt == 45 ? build_iq4nl_block() : (fmt == 2 ? build_q2k_block() : (fmt == 23 ? build_iq2s_block() : (fmt == 24 ? build_iq2xs_block() : (fmt == 25 ? build_iq2xxs_block() : build_q6k_block()))))))))))) var kq : array var ks : array kq |> resize(nreg * d * nsb * qsb) @@ -1907,6 +1998,8 @@ def private kq_batch_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 transcode_q2k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } elif (fmt == 24) { transcode_iq2xs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 25) { + transcode_iq2xxs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } elif (fmt == 23) { transcode_iq2s_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } else { @@ -1941,6 +2034,8 @@ def private kq_batch_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 repack_k2_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) } elif (fmt == 24) { repack_iq2xs_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) + } elif (fmt == 25) { + repack_iq2xxs_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) } elif (fmt == 23) { repack_iq2s_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) } else { @@ -2001,6 +2096,8 @@ def private kq_batch_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 k2q8_gemv_gen(addr(want[tk * d]), kqr, ksr, addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, 0l, d) } elif (fmt == 24) { iq2xsq8_gemv_gen(addr(want[tk * d]), kqr, ksr, addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, 0l, d) + } elif (fmt == 25) { + iq2xxsq8_gemv_gen(addr(want[tk * d]), kqr, ksr, addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, 0l, d) } elif (fmt == 23) { iq2sq8_gemv_gen(addr(want[tk * d]), kqr, ksr, addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, 0l, d) } else { @@ -2034,7 +2131,7 @@ def private kq_batch_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 [test] def test_kq_batch_groupn(t : T?) { - for (fmt in [4, 5, 6, 40, 44, 3, 33, 34, 45, 2, 23, 24]) { + for (fmt in [4, 5, 6, 40, 44, 3, 33, 34, 45, 2, 23, 24, 25]) { t |> run("{kq_tag(fmt)} batch groupn bit-matches per-token rows-core calls") @(t : T?) { kq_batch_groupn_gate(t, fmt) } From e4190da85cb1c39b09a23a99db1b40399be381f7 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Mon, 31 Aug 2026 14:32:51 -0700 Subject: [PATCH 46/61] dasLLAMA: IQ2_XXS Phase B - the panel route absorbs the aux32 form with one arm kq_grp_row_dot_b grows the fmt-25 strip arm (per-32 UNSIGNED, the iq3xxs shape) and the emitter needs NO new fold at all: (2ls+1) strip bytes stay under 128, so the shared per-32 SExt load reads them exactly - iq2xxs sets `panel` but not the `iq2` split-accumulator flag and rides the iq3s/iq3xxs scale path verbatim. emit_iq2xxs_gather composes each block's aux32 once, then per l a grid-BYTE index doubles into a [512 x i32] private grid; signs ride the shared iq3xxs ksigns global and the shared smask negate. unpack_iq2xxs_panel_grp mirrors it for the tile; the packed lists drop 25; kfxs25 TEST rows return (QUIRK 16 closed for the format). Probe: 11/11 k25 perms ok (maddubs 9.5e-7); the tuner crowns dot_maddubs_width256_mr8 via --tune-only iq2xxsq8_tile_gen (1 of 15 families, per app). Gates: test_kquant -jit 263 pass / interp 248 pass, 0 failed, lint 0. TRAP for the ledger: run.das takes the model as the LAST .gguf argv token - flags after it silently select DEFAULT_MODEL. E2e stamped: the reference 64/64 full match moves to a step-3 fork on a 0.0267-logit top-2 near-tie (das picks the oracle's `#2` - the arc's tightest; stamped folds move flip points, not the class). zen2 16t vs clean-cpu: pp512 519.0 vs 181.6 (2.86x), tg128 61.1 vs 87.3 (0.70x - the grid-format decode tail class, IQ2_XS's numbers exactly). Co-Authored-By: Claude Fable 5 --- modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md | 17 +++ .../dasLLAMA/dasllama/dasllama_gemm_gen.das | 139 ++++++++++++++++-- .../dasLLAMA/dasllama/dasllama_math_gen.das | 15 +- modules/dasLLAMA/dasllama/dasllama_repack.das | 45 ++++++ modules/dasLLAMA/harness/gen_tune_probe.das | 5 +- modules/dasLLAMA/tests/test_kquant.das | 4 +- 6 files changed, 207 insertions(+), 18 deletions(-) diff --git a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md index 51945a816d..d7ae05798b 100644 --- a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md +++ b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md @@ -498,6 +498,23 @@ OK, lint 0 (20 files). QUIRK 15 pinned via `--tune-only iq2xxsq8_tile_gen` (1 of seconds). E2e: parity ids 64/64 vs llama.cpp - the THIRD consecutive full-match stream. JIT emitter, Vulkan, Metal: pending. + +Phase B (JIT emitter, 2026-08-31): the panel route absorbs the aux32 form with ONE new arm - +`kq_grp_row_dot_b` grows a fmt-25 strip arm (per-32 UNSIGNED, the iq3xxs shape; the emitter +needs NO new fold at all, because (2ls+1) strip bytes stay under 128 and the shared per-32 +SExt load reads them exactly - iq2xxs sets `panel` but NOT the `iq2` split-accumulator flag). +`emit_iq2xxs_gather` loads each block's aux32 once (four byte loads composed), then per l a +grid-BYTE index doubles into the [512 x i32] private grid; signs ride the shared iq3xxs +ksigns global. `unpack_iq2xxs_panel_grp` mirrors it for the tile. kfxs25 TEST rows returned. +Probe: 11/11 k25 perms ok (maddubs 9.5e-7). Gates: kquant -jit 263 / interp 248, 0 failed, +lint 0. QUIRK 15 cleared per-app with `--tune-only iq2xxsq8_tile_gen` (crowned +dot_maddubs_width256_mr8); TRAP: run.das takes the model as the LAST `.gguf` argv token - +flags placed after it silently fall back to DEFAULT_MODEL (put the model last). E2e stamped: +the reference stream's 64/64 full match moves to a fork at step 3 that is a 0.0267-logit +top-2 near-tie (das picks the oracle's #2 - the ARC'S TIGHTEST; stamped folds move flip +points, not the class). zen2 16t vs clean-cpu: pp512 519.0 vs 181.6 (2.86x), tg128 61.1 vs +87.3 (0.70x - the grid-format decode tail class). Vulkan, Metal: pending. + ### IQ2_XS Phase A (CPU, 2026-08-31) - the ksigns u64 tier Shape: 256-superblock grid format - each of the 32 u16 qs words carries a 9-bit index into diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das index 910ae1b5ee..e41068eb7f 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das @@ -1628,6 +1628,61 @@ def private iq2xs_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { te.iq3s_smask_g = smask } +// The iq2xxs gemv gather globals: the u64 grid as a [512 x i32] private constant (256 entries +// as low/high word pairs), the shared ksigns table, and the shared smask. +def private iq2xxs_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { + var grid = LLVMGetNamedGlobal(mod, "dasllama.iq2xxs.grid") + if (grid == null) { + let g = iq2xxs_grid2() + var elems : array + elems |> reserve(512) + for (i in range(512)) { + elems |> push(LLVMConstInt(te.types.t_int32, uint64(g[i]), 0)) + } + grid = LLVMAddGlobal(mod, LLVMArrayType(te.types.t_int32, 512u), "dasllama.iq2xxs.grid") + LLVMSetInitializer(grid, LLVMConstArray(te.types.t_int32, array_data_ptr(elems), 512u)) + LLVMSetGlobalConstant(grid, 1) + LLVMSetLinkage(grid, LLVMLinkage.LLVMPrivateLinkage) + LLVMSetAlignment(grid, 64u) + } + te.iq3s_grid_g = grid + var ksg = LLVMGetNamedGlobal(mod, "dasllama.iq3xxs.ksigns") + if (ksg == null) { + let kt = ksigns_iq2xs() + var elems3 : array + elems3 |> reserve(128) + for (i in range(128)) { + elems3 |> push(LLVMConstInt(te.types.t_int32, uint64(kt[i]), 0)) + } + ksg = LLVMAddGlobal(mod, LLVMArrayType(te.types.t_int32, 128u), "dasllama.iq3xxs.ksigns") + LLVMSetInitializer(ksg, LLVMConstArray(te.types.t_int32, array_data_ptr(elems3), 128u)) + LLVMSetGlobalConstant(ksg, 1) + LLVMSetLinkage(ksg, LLVMLinkage.LLVMPrivateLinkage) + LLVMSetAlignment(ksg, 16u) + } + te.iq3xxs_ksigns_g = ksg + var smask = LLVMGetNamedGlobal(mod, "dasllama.iq3s.smask") + if (smask == null) { + var elems2 : array + elems2 |> reserve(16) + for (m in range(16)) { + var v = 0ul + for (t in range(4)) { + if ((m & (1 << t)) != 0) { + v |= 0xFFul << uint64(8 * t) + } + } + elems2 |> push(LLVMConstInt(te.types.t_int32, v, 0)) + } + smask = LLVMAddGlobal(mod, LLVMArrayType(te.types.t_int32, 16u), "dasllama.iq3s.smask") + LLVMSetInitializer(smask, LLVMConstArray(te.types.t_int32, array_data_ptr(elems2), 16u)) + LLVMSetGlobalConstant(smask, 1) + LLVMSetLinkage(smask, LLVMLinkage.LLVMPrivateLinkage) + LLVMSetAlignment(smask, 16u) + } + te.iq3s_smask_g = smask +} + // iq2xs gemv: gather superblock `sbi` off the PACKED grp planes into the alloca panel - the // iq2s gather's two-word u64 form with the u16 qs word's 9-bit index and its high-7-bit // KSIGNS lookup in place of explicit sign bytes (the iq3xxs sign path). @@ -1688,6 +1743,69 @@ def private emit_iq2xs_gather(var te : TileEmit; var sbi : LLVMOpaqueValue?) { LLVMPositionBuilderAtEnd(b, cont) } +// iq2xxs gemv: gather superblock `sbi` off the PACKED grp planes into the alloca panel - the +// aux32 form: block b's grid BYTES (column 2b) each double into the u64 grid's word pair, its +// aux32 (column 2b+1) carries the four 7-bit KSIGNS indices (the per-32 scale nibble rides the +// scale row, not the gather). +def private emit_iq2xxs_gather(var te : TileEmit; var sbi : LLVMOpaqueValue?) { + let b = te.builder + let mr = te.interleave + var wbP = LLVMBuildMul(b, sbi, te.types->ConstI64(uint64(mr * 64)), "iq2xx.wb") + var pre = LLVMGetInsertBlock(b) + var head = LLVMAppendBasicBlockInContext(te.gctx, te.gfn, "iq2xx.gath") + var cont = LLVMAppendBasicBlockInContext(te.gctx, te.gfn, "iq2xx.cont") + LLVMBuildBr(b, head) + LLVMPositionBuilderAtEnd(b, head) + var rPhi = LLVMBuildPhi(b, te.types.t_int64, "iq2xx.r") + var preVals <- [te.types->ConstI64(0ul)] + var preBlocks <- [pre] + LLVMAddIncoming(rPhi, preVals, preBlocks) + var rT4 = LLVMBuildMul(b, rPhi, te.types->ConstI64(4ul), "iq2xx.r4") + var base = LLVMBuildAdd(b, wbP, rT4, "iq2xx.base") + let c15 = LLVMConstInt(te.types.t_int32, 15ul, 0) + let c127 = LLVMConstInt(te.types.t_int32, 127ul, 0) + let cneg = LLVMConstInt(te.types.t_int32, 0x01010101ul, 0) + for (blk in range(8)) { + var a0p = LLVMBuildGEP2(b, te.types.t_int8, te.wg, LLVMBuildAdd(b, base, te.types->ConstI64(uint64((2 * blk + 1) * mr * 4)), ""), "") + var aux = LLVMBuildZExt(b, LLVMBuildLoad2Aligned(b, te.types.t_int8, a0p, 1u, ""), te.types.t_int32, "") + for (k in range(1, 4)) { + var akp = LLVMBuildGEP2(b, te.types.t_int8, te.wg, LLVMBuildAdd(b, base, te.types->ConstI64(uint64((2 * blk + 1) * mr * 4 + k)), ""), "") + var ab = LLVMBuildZExt(b, LLVMBuildLoad2Aligned(b, te.types.t_int8, akp, 1u, ""), te.types.t_int32, "") + aux = LLVMBuildOr(b, aux, LLVMBuildShl(b, ab, LLVMConstInt(te.types.t_int32, uint64(8 * k), 0), ""), "iq2xx.aux{blk}_{k}") + } + for (l in range(4)) { + var ip = LLVMBuildGEP2(b, te.types.t_int8, te.wg, LLVMBuildAdd(b, base, te.types->ConstI64(uint64(2 * blk * mr * 4 + l)), ""), "") + var idx = LLVMBuildZExt(b, LLVMBuildLoad2Aligned(b, te.types.t_int8, ip, 1u, ""), te.types.t_int32, "iq2xx.i{blk}_{l}") + var idx2 = LLVMBuildShl(b, idx, LLVMConstInt(te.types.t_int32, 1ul, 0), "") + var sidx = LLVMBuildAnd(b, LLVMBuildLShr(b, aux, LLVMConstInt(te.types.t_int32, uint64(7 * l), 0), ""), c127, "") + var ksp2 = LLVMBuildGEP2(b, te.types.t_int32, te.iq3xxs_ksigns_g, sidx, "") + var sgn = LLVMBuildLoad2Aligned(b, te.types.t_int32, ksp2, 4u, "iq2xx.sg{blk}_{l}") + var mv : LLVMOpaqueValue? [2] + mv[0] = LLVMBuildAnd(b, sgn, c15, "") + mv[1] = LLVMBuildAnd(b, LLVMBuildLShr(b, sgn, LLVMConstInt(te.types.t_int32, 4ul, 0), ""), c15, "") + for (h in range(2)) { + var gi = h == 0 ? idx2 : LLVMBuildAdd(b, idx2, LLVMConstInt(te.types.t_int32, 1ul, 0), "") + var gp = LLVMBuildGEP2(b, te.types.t_int32, te.iq3s_grid_g, gi, "") + var gw = LLVMBuildLoad2Aligned(b, te.types.t_int32, gp, 4u, "iq2xx.g{blk}_{l}_{h}") + var mp = LLVMBuildGEP2(b, te.types.t_int32, te.iq3s_smask_g, mv[h], "") + var mw = LLVMBuildLoad2Aligned(b, te.types.t_int32, mp, 4u, "") + var w = LLVMBuildAdd(b, LLVMBuildXor(b, gw, mw, ""), LLVMBuildAnd(b, mw, cneg, ""), "iq2xx.w{blk}_{l}_{h}") + let e = l * 8 + h * 4 + let dd = (e < 16 ? 0 : 128 * mr) + (blk * 4 + (e % 16) / 4) * mr * 4 + var dp = LLVMBuildGEP2(b, te.types.t_int8, te.iq3s_panel, LLVMBuildAdd(b, rT4, te.types->ConstI64(uint64(dd)), ""), "") + LLVMBuildStore(b, w, dp) + } + } + } + var rNext = LLVMBuildAdd(b, rPhi, te.types->ConstI64(1ul), "iq2xx.rn") + var more = LLVMBuildICmp(b, LLVMIntPredicate.LLVMIntSLT, rNext, te.types->ConstI64(uint64(mr)), "iq2xx.more") + LLVMBuildCondBr(b, more, head, cont) + var loopVals <- [rNext] + var loopBlocks <- [head] + LLVMAddIncoming(rPhi, loopVals, loopBlocks) + LLVMPositionBuilderAtEnd(b, cont) +} + // One 256-weight SUPERBLOCK, IQ4_XS/iq3s grp form (te.kq = 44/33): mx4's LUT decode (44) // or the byte-expanded signed panel (33; the gemv gathers it per superblock, the tile gets the // runtime unpack) — dot_lane's sign-trick lattice, k4's fold with no min term. @@ -1696,7 +1814,7 @@ def private emit_block_iq4xs(var te : TileEmit; var sbi : LLVMOpaqueValue?; var let rq = te.rq let mr = te.interleave let w8 = te.width / 8 - let panel = te.kq == 33 || te.kq == 34 || te.kq == 23 || te.kq == 24 // iq3s/iq3xxs/iq2s/iq2xs: the byte-expanded signed panel replaces the nibble+LUT decode + let panel = te.kq == 33 || te.kq == 34 || te.kq == 23 || te.kq == 24 || te.kq == 25 // iq3s/iq3xxs/iq2s/iq2xs/iq2xxs: the byte-expanded signed panel replaces the nibble+LUT decode let iq2 = te.kq == 23 || te.kq == 24 // iq2s/iq2xs: per-16 UNSIGNED strips - lo/hi halves keep split accumulators let nl = te.kq == 45 // iq4nl: the 44 LUT decode with q40's PER-BLOCK f16 d fold (no sub-scales, no min term) let gather = panel && te.iq3s_panel != null // gemv mode: gather this superblock off the packed planes first @@ -1709,6 +1827,8 @@ def private emit_block_iq4xs(var te : TileEmit; var sbi : LLVMOpaqueValue?; var emit_iq2s_gather(te, sbi) } elif (te.kq == 24) { emit_iq2xs_gather(te, sbi) + } elif (te.kq == 25) { + emit_iq2xxs_gather(te, sbi) } else { emit_iq3xxs_gather(te, sbi) } @@ -1837,7 +1957,7 @@ def private emit_block_iq4xs(var te : TileEmit; var sbi : LLVMOpaqueValue?; var // token PAIR — single-token slices ride the sdot lattice instead. kq blocks are SUPERBLOCKS // (the kq gemv/tile drive emit_slice with nb = n/256). def private emit_one_block(var te : TileEmit; var bi : LLVMOpaqueValue?; var f : LLVMOpaqueValue? [8]; tokBase, tokCount : int) { - if (te.kq == 44 || te.kq == 33 || te.kq == 34 || te.kq == 45 || te.kq == 23 || te.kq == 24) { + if (te.kq == 44 || te.kq == 33 || te.kq == 34 || te.kq == 45 || te.kq == 23 || te.kq == 24 || te.kq == 25) { emit_block_iq4xs(te, bi, f, tokBase, tokCount) } elif (te.kq != 0) { emit_block_kqv2(te, bi, f, tokBase, tokCount) @@ -2944,7 +3064,7 @@ def private kq_gemv_gen_impl(var gc : LlvmCodeCtx; fmt : int) : bool { // entry: group range off the row range; strides are disk footprints (quants 128/160/192B, scales 20/20/18B) LLVMPositionBuilderAtEnd(b, entry) - if (fmt == 33 || fmt == 34 || fmt == 23 || fmt == 24) { + if (fmt == 33 || fmt == 34 || fmt == 23 || fmt == 24 || fmt == 25) { // iq3s/iq3xxs/iq2s/iq2xs: the gemv gathers each superblock into an alloca panel (the tile gets the runtime unpack) te.gfn = gc.impl te.gctx = gc.jit.ctx @@ -2954,6 +3074,8 @@ def private kq_gemv_gen_impl(var gc : LlvmCodeCtx; fmt : int) : bool { iq2s_emit_globals(te, LLVMGetGlobalParent(gc.impl)) } elif (fmt == 24) { iq2xs_emit_globals(te, LLVMGetGlobalParent(gc.impl)) + } elif (fmt == 25) { + iq2xxs_emit_globals(te, LLVMGetGlobalParent(gc.impl)) } else { iq3xxs_emit_globals(te, LLVMGetGlobalParent(gc.impl)) } @@ -3217,12 +3339,11 @@ def private q51_tile_gen(var gc : LlvmCodeCtx) : bool { // Called from llvm_user_modules::register_user_llvm_code_generators — the JIT invokes that in the // context that reads the registry, so the direct call lands in the right table copy. -// iq2xxs: the grid-gather emitter arm lands with its Phase B; until then both stamps decline -// (reference bodies serve) - a declined generator is the framework's own fallback path. -[unused_argument(gc)] -def private iq2xxs_gemv_gen(var gc : LlvmCodeCtx) : bool => false -[unused_argument(gc)] -def private iq2xxs_tile_gen(var gc : LlvmCodeCtx) : bool => false +// iq2xxs rides the iq4xs walk through the panel route: emit_iq2xxs_gather feeds the shared +// byte-expanded panel; per-32 UNSIGNED strips ride the iq3s/iq3xxs scale fold (the strip +// bytes stay under 128, so the shared SExt load reads them exactly). +def private iq2xxs_gemv_gen(var gc : LlvmCodeCtx) : bool => kq_gemv_gen_impl(gc, 25) +def private iq2xxs_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 25) [macro_function] def public register_dasllama_gemm_generators { diff --git a/modules/dasLLAMA/dasllama/dasllama_math_gen.das b/modules/dasLLAMA/dasllama/dasllama_math_gen.das index d3a18370de..f02886cfaa 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_gen.das @@ -1122,7 +1122,7 @@ def iq2xxsq8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; x unsafe { for (t in range64(4l)) { for (r in range64(mr)) { - yp[(t0 + t) * d + g * mr + r] = iq2xxs_grp_row_dot(kqg, ksg, r, mr, xqp + (t0 + t) * n, xsp + (t0 + t) * (n / 256l), xbsp + (t0 + t) * (n / 16l), n) + yp[(t0 + t) * d + g * mr + r] = kq_grp_row_dot_b(25l, kqg, ksg, r, mr, xqp + (t0 + t) * n, xsp + (t0 + t) * (n / 256l), xbsp + (t0 + t) * (n / 16l), n) } } } @@ -1403,8 +1403,8 @@ def kq_grp_row_dot_b(fmt : int64; kqg : uint8 const?; ksg : uint8 const?; r, mr var ihi = 0 for (j in range64(4l)) { for (t in range64(4l)) { - let qlo = fmt == 33l || fmt == 34l || fmt == 23l || fmt == 24l ? int(int8(kqg[qb + ((blk * 4l + j) * mr + r) * 4l + t])) : int(kqg[qb + ((blk * 4l + j) * mr + r) * 4l + t]) - let qhi = fmt == 33l || fmt == 34l || fmt == 23l || fmt == 24l ? int(int8(kqg[qb + 128l * mr + ((blk * 4l + j) * mr + r) * 4l + t])) : int(kqg[qb + 128l * mr + ((blk * 4l + j) * mr + r) * 4l + t]) + let qlo = fmt == 33l || fmt == 34l || fmt == 23l || fmt == 24l || fmt == 25l ? int(int8(kqg[qb + ((blk * 4l + j) * mr + r) * 4l + t])) : int(kqg[qb + ((blk * 4l + j) * mr + r) * 4l + t]) + let qhi = fmt == 33l || fmt == 34l || fmt == 23l || fmt == 24l || fmt == 25l ? int(int8(kqg[qb + 128l * mr + ((blk * 4l + j) * mr + r) * 4l + t])) : int(kqg[qb + 128l * mr + ((blk * 4l + j) * mr + r) * 4l + t]) ilo += qlo * int(xqp[b * 32l + j * 4l + t]) ihi += qhi * int(xqp[b * 32l + 16l + j * 4l + t]) } @@ -1416,6 +1416,9 @@ def kq_grp_row_dot_b(fmt : int64; kqg : uint8 const?; ksg : uint8 const?; r, mr let s0 = int(ksg[sb + 4l * mr + 2l * blk * mr + r]) let s1 = int(ksg[sb + 4l * mr + (2l * blk + 1l) * mr + r]) isum += s0 * ilo + s1 * ihi + } elif (fmt == 25l) { + let sc = int(ksg[sb + 4l * mr + blk * mr + r]) // per-32 UNSIGNED strip + isum += sc * (ilo + ihi) } elif (fmt == 6l) { let s0 = int(int8(ksg[sb + 2l * blk * mr + r])) let s1 = int(int8(ksg[sb + (2l * blk + 1l) * mr + r])) @@ -1428,7 +1431,7 @@ def kq_grp_row_dot_b(fmt : int64; kqg : uint8 const?; ksg : uint8 const?; r, mr bsum += mn * (xbsp[b * 2l] + xbsp[b * 2l + 1l]) } } - if (fmt == 33l || fmt == 34l || fmt == 23l || fmt == 24l) { + if (fmt == 33l || fmt == 34l || fmt == 23l || fmt == 24l || fmt == 25l) { let di = f16_to_f32(uint(ksg[sb + 2l * r]) | (uint(ksg[sb + 2l * r + 1l]) << 8u)) acc += float(isum) * (di * xsp[sbi]) } elif (fmt == 6l) { @@ -2076,7 +2079,7 @@ def private kq_batch_cell_gen(fmt : int; var myp : float?; kqp : uint8 const?; k let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) let nb16 = n / 16l - let packed = fmt == 4 || fmt == 40 || fmt == 44 || fmt == 3 || fmt == 45 || fmt == 2 || fmt == 25 // packed planes: the tile reads them directly (iq3s/iq3xxs/iq2s expand their grid gathers into the panel) + let packed = fmt == 4 || fmt == 40 || fmt == 44 || fmt == 3 || fmt == 45 || fmt == 2 // packed planes: the tile reads them directly (iq3s/iq3xxs/iq2s expand their grid gathers into the panel) var scratch : array if (!packed) { scratch |> resize(mr * nsb * 256l) @@ -2115,7 +2118,7 @@ def private kq_batch_cell_gen(fmt : int; var myp : float?; kqp : uint8 const?; k } elif (fmt == 2) { k2q8_tile_gen(myp, kqg, ksg, xqp, xsp, xbsp, n, d, g, tk) } elif (fmt == 25) { - iq2xxsq8_tile_gen(myp, kqg, ksg, xqp, xsp, xbsp, n, d, g, tk) + iq2xxsq8_tile_gen(myp, scp, ksg, xqp, xsp, xbsp, n, d, g, tk) } elif (fmt == 23) { iq2sq8_tile_gen(myp, scp, ksg, xqp, xsp, xbsp, n, d, g, tk) } elif (fmt == 24) { diff --git a/modules/dasLLAMA/dasllama/dasllama_repack.das b/modules/dasLLAMA/dasllama/dasllama_repack.das index 3628e12d57..b44d1470dd 100644 --- a/modules/dasLLAMA/dasllama/dasllama_repack.das +++ b/modules/dasLLAMA/dasllama/dasllama_repack.das @@ -909,6 +909,47 @@ def unpack_iq2xs_panel_grp(kqg : uint8 const?; var dst : uint8?; mr, nsb : int64 } } +//! iq2xxs panel unpack: block b's grid BYTES (column 2b) each double into the u64 grid's word +//! pair, its aux32 (column 2b+1) picks the KSIGNS sign bytes - packed grp planes -> the +//! shared signed panel (the per-32 scale rides the strip row). +def unpack_iq2xxs_panel_grp(kqg : uint8 const?; var dst : uint8?; mr, nsb : int64) { + let grid = iq2xxs_grid2() + let ksgn = ksigns_iq2xs() + let smask = fixed_array(0x00000000u, 0x000000FFu, 0x0000FF00u, 0x0000FFFFu, + 0x00FF0000u, 0x00FF00FFu, 0x00FFFF00u, 0x00FFFFFFu, + 0xFF000000u, 0xFF0000FFu, 0xFF00FF00u, 0xFF00FFFFu, + 0xFFFF0000u, 0xFFFF00FFu, 0xFFFFFF00u, 0xFFFFFFFFu) + unsafe { + for (sbi in range64(nsb)) { + let qb = sbi * 64l * mr + let ob = sbi * 256l * mr + for (r in range64(mr)) { + for (blk in range64(8l)) { + let ac = (2l * blk + 1l) * mr + r + let aux = uint(kqg[qb + ac * 4l]) | (uint(kqg[qb + ac * 4l + 1l]) << 8u) | (uint(kqg[qb + ac * 4l + 2l]) << 16u) | (uint(kqg[qb + ac * 4l + 3l]) << 24u) + let gc = (2l * blk) * mr + r + for (l in range64(4l)) { + let sgn = int(ksgn[int((aux >> uint(7l * l)) & 127u)]) + let ix = int(kqg[qb + gc * 4l + l]) * 2 + let m1 = smask[sgn & 15] + let m2 = smask[(sgn >> 4) & 15] + let w1 = (grid[ix] ^ m1) + (m1 & 0x01010101u) + let w2 = (grid[ix + 1] ^ m2) + (m2 & 0x01010101u) + let e1 = l * 8l + let e2 = l * 8l + 4l + let d1 = (e1 < 16l ? 0l : 128l * mr) + ((blk * 4l + (e1 % 16l) / 4l) * mr + r) * 4l + let d2 = (e2 < 16l ? 0l : 128l * mr) + ((blk * 4l + (e2 % 16l) / 4l) * mr + r) * 4l + var wp1 = reinterpret(dst + (ob + d1)) + var wp2 = reinterpret(dst + (ob + d2)) + wp1[0] = w1 + wp2[0] = w2 + } + } + } + } + } +} + def unpack_kq_panel_grp(fmt : int64; kqg : uint8 const?; var dst : uint8?; mr, nsb : int64) { if (fmt == 33l) { unpack_iq3s_panel_grp(kqg, dst, mr, nsb) @@ -922,6 +963,10 @@ def unpack_kq_panel_grp(fmt : int64; kqg : uint8 const?; var dst : uint8?; mr, n unpack_iq2s_panel_grp(kqg, dst, mr, nsb) return } + if (fmt == 25l) { + unpack_iq2xxs_panel_grp(kqg, dst, mr, nsb) + return + } if (fmt == 24l) { unpack_iq2xs_panel_grp(kqg, dst, mr, nsb) return diff --git a/modules/dasLLAMA/harness/gen_tune_probe.das b/modules/dasLLAMA/harness/gen_tune_probe.das index a2a00972f2..5bd7c9b8b7 100644 --- a/modules/dasLLAMA/harness/gen_tune_probe.das +++ b/modules/dasLLAMA/harness/gen_tune_probe.das @@ -948,7 +948,7 @@ def kq_layout_mrs(fmt : int64) : table { // nolint:STYLE037,STYLE def run_kq_tile(tile; fx : KqFixture; kq : array; ks : array; mr : int64; var y : array) { let ng = fx.d / mr verify(fx.ntok % 4l == 0l) - let packed = fx.fmt == 4l || fx.fmt == 40l || fx.fmt == 44l || fx.fmt == 3l || fx.fmt == 45l || fx.fmt == 2l || fx.fmt == 25l // packed planes: the tile reads them directly (iq3s/iq3xxs/iq2s unpack their panels per group) + let packed = fx.fmt == 4l || fx.fmt == 40l || fx.fmt == 44l || fx.fmt == 3l || fx.fmt == 45l || fx.fmt == 2l // packed planes: the tile reads them directly (iq3s/iq3xxs/iq2s unpack their panels per group) var panel : array if (!packed) { panel |> resize(int(mr * fx.nsb * 256l)) @@ -1436,6 +1436,7 @@ def test_mode : bool { // nolint:STYLE037,STYLE038 — flat per-variant correctn var kfxs2 <- [ <- build_kq_fixture(2l, 256l, 32l, 8l), <- build_kq_fixture(2l, 768l, 32l, 8l), <- build_kq_fixture(2l, 2048l, 32l, 8l)] var kfxs23 <- [ <- build_kq_fixture(23l, 256l, 32l, 8l), <- build_kq_fixture(23l, 768l, 32l, 8l), <- build_kq_fixture(23l, 2048l, 32l, 8l)] var kfxs24 <- [ <- build_kq_fixture(24l, 256l, 32l, 8l), <- build_kq_fixture(24l, 768l, 32l, 8l), <- build_kq_fixture(24l, 2048l, 32l, 8l)] + var kfxs25 <- [ <- build_kq_fixture(25l, 256l, 32l, 8l), <- build_kq_fixture(25l, 768l, 32l, 8l), <- build_kq_fixture(25l, 2048l, 32l, 8l)] var vs <- q8q8_tile_gen_variants() var mrs <- variant_mrs() var wbs <- variant_wbias() @@ -1568,6 +1569,7 @@ def test_mode : bool { // nolint:STYLE037,STYLE038 — flat per-variant correctn allok = kq_test_family(2l, kfxs2) && allok allok = kq_test_family(23l, kfxs23) && allok allok = kq_test_family(24l, kfxs24) && allok + allok = kq_test_family(25l, kfxs25) && allok delete wtab delete mtab delete gtab @@ -1591,6 +1593,7 @@ def test_mode : bool { // nolint:STYLE037,STYLE038 — flat per-variant correctn delete kfxs2 delete kfxs23 delete kfxs24 + delete kfxs25 return allok } diff --git a/modules/dasLLAMA/tests/test_kquant.das b/modules/dasLLAMA/tests/test_kquant.das index 99e4e4933d..dee5d07672 100644 --- a/modules/dasLLAMA/tests/test_kquant.das +++ b/modules/dasLLAMA/tests/test_kquant.das @@ -1528,7 +1528,7 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { // nolint:STYL // k5/k6 tiles read the byte-expanded panel the batch cell unpacks per (group, token-block) // — the gate covers unpack_kq_panel_grp too (tile-over-panel vs gemv-over-packed planes); // k4/q40 tiles read the packed planes directly - let packed = fmt == 4 || fmt == 40 || fmt == 44 || fmt == 3 || fmt == 45 || fmt == 2 || fmt == 25 // iq3s/iq3xxs/iq2s/iq2xs tiles read the byte-expanded panel (the batch cell's shape) + let packed = fmt == 4 || fmt == 40 || fmt == 44 || fmt == 3 || fmt == 45 || fmt == 2 // iq3s/iq3xxs/iq2s/iq2xs tiles read the byte-expanded panel (the batch cell's shape) var panel : array if (!packed) { panel |> resize(mr * nsb * 256l) @@ -1562,7 +1562,7 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { // nolint:STYL } elif (fmt == 2) { k2q8_tile_gen(addr(ytile[0]), kqg, ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) } elif (fmt == 25) { - iq2xxsq8_tile_gen(addr(ytile[0]), kqg, ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) + iq2xxsq8_tile_gen(addr(ytile[0]), addr(panel[0]), ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) } elif (fmt == 24) { iq2xsq8_tile_gen(addr(ytile[0]), addr(panel[0]), ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) } elif (fmt == 23) { From 5615454decfcd33ea318bc0373c60f4c20c6d882 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Mon, 31 Aug 2026 14:33:53 -0700 Subject: [PATCH 47/61] dasLLAMA: trim the iq2xxs gather comment to the 3-line cap (STYLE014) Co-Authored-By: Claude Fable 5 --- modules/dasLLAMA/dasllama/dasllama_gemm_gen.das | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das index e41068eb7f..5e5ef3b408 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das @@ -1743,10 +1743,9 @@ def private emit_iq2xs_gather(var te : TileEmit; var sbi : LLVMOpaqueValue?) { LLVMPositionBuilderAtEnd(b, cont) } -// iq2xxs gemv: gather superblock `sbi` off the PACKED grp planes into the alloca panel - the -// aux32 form: block b's grid BYTES (column 2b) each double into the u64 grid's word pair, its -// aux32 (column 2b+1) carries the four 7-bit KSIGNS indices (the per-32 scale nibble rides the -// scale row, not the gather). +// iq2xxs gemv: gather superblock `sbi` off the PACKED grp planes into the alloca panel - block +// b's grid BYTES (column 2b) double into the u64 grid's word pair, its aux32 (column 2b+1) +// carries the 7-bit KSIGNS indices (the per-32 scale nibble rides the scale row, not the gather). def private emit_iq2xxs_gather(var te : TileEmit; var sbi : LLVMOpaqueValue?) { let b = te.builder let mr = te.interleave From 80070e28f8f809a490bb4f6d9d69053b7f5a7538 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Mon, 31 Aug 2026 15:34:42 -0700 Subject: [PATCH 48/61] dasLLAMA: IQ2_XXS Phase C - the iq3xxs Vulkan shell over the two-word grid KqGemvIq2xxs (2 KB workgroup grid, byte indices + ksign7 parity signs), KqBatchIq2xxs : KqBatchIq4xs with the per-32 strip plane and fma fully INHERITED - (2ls+1) strips stay under 128, so the base's signed extract reads them exactly - and Iq2xxsCm2T / VkIq2xxsBlk on a gated IQ2XXSGRID axis; schema 14 -> 25; thirteen-format family cells at stride 16; a parity-fold float witness + an iq2xxsf16 cm2 oracle. SIGN TRAP for the HOW_TO: the family's random-byte tests demand SIGNED per-32 strip reads everywhere - unsigned reads agree on real strips but fork the hash-fill fixtures one tier at a time. Gates: test_vulkan_kernels 86/86, lint 0. E2e resident: the Vulkan stream is 64/64 IDENTICAL to the stamped CPU stream (tier-bit consistency; both carry only the step-3 0.0267 top-2 flip vs llama.cpp). 5060 Ti vs llama.cpp b10660 Vulkan -ngl 99: pp512 14423.2 vs 18572.5 (0.78x, above the 0.70x class), tg128 297.0 vs 421.4 (0.70x - the 2 KB grid re-stage, milder than iq2xs's 4 KB; followup `#35`). Co-Authored-By: Claude Fable 5 --- modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md | 17 ++ .../dasllama/dasllama_vulkan_classes.das | 247 ++++++++++++++++++ .../dasllama/dasllama_vulkan_common.das | 1 + .../dasllama/dasllama_vulkan_prefill.das | 17 ++ modules/dasLLAMA/tests/_vkd_oracles.das | 92 +++++++ .../dasLLAMA/tests/test_vulkan_kernels.das | 152 ++++++++++- 6 files changed, 517 insertions(+), 9 deletions(-) diff --git a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md index d7ae05798b..b53e72dfb8 100644 --- a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md +++ b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md @@ -515,6 +515,23 @@ top-2 near-tie (das picks the oracle's #2 - the ARC'S TIGHTEST; stamped folds mo points, not the class). zen2 16t vs clean-cpu: pp512 519.0 vs 181.6 (2.86x), tg128 61.1 vs 87.3 (0.70x - the grid-format decode tail class). Vulkan, Metal: pending. + +Phase C (Vulkan, 2026-08-31): the iq3xxs shell over the two-word u64 grid - `KqGemvIq2xxs` +(2 KB `@workgroup uint[512]`, 8 words per thread; block b's grid word carries four BYTE +indices, its aux32 the parity-derived ksigns via `ksign7`), `KqBatchIq2xxs : KqBatchIq4xs` +(the per-32 strip plane and fma INHERIT - (2ls+1) strips stay under 128, so the base's +signed extract reads them exactly), `Iq2xxsCm2T` / `VkIq2xxsBlk` (the 64B qs as 16-bit +lanes) on a gated `IQ2XXSGRID` axis; schema 14 -> 25; thirteen-format family cells at +stride 16. SIGN CONVENTION TRAP: the family's random-byte tests demand SIGNED per-32 strip +reads EVERYWHERE (class, cm2 decode, float + f16 oracles alike) - an unsigned `& 0xFF` read +agrees on real (2ls+1 < 128) strips but forks the hash-fill fixtures, and the mismatch +surfaces one tier at a time (batch first, then cm2) as each side is aligned. Gates: +test_vulkan_kernels 86/86, lint 0. E2e resident (DASLLAMA_GPU=1): the Vulkan stream is +64/64 IDENTICAL to the stamped CPU stream - tier-bit-consistency; both carry only the +step-3 0.0267 top-2 flip vs llama.cpp. 5060 Ti vs llama.cpp b10660 -ngl 99: pp512 14423.2 +vs 18572.5 (0.78x - above the 0.70x class), tg128 297.0 vs 421.4 (0.70x - the grid-gemv +re-stage at 2 KB, milder than iq2xs's 4 KB; followup_vulkan #35). Metal: pending. + ### IQ2_XS Phase A (CPU, 2026-08-31) - the ksigns u64 tier Shape: 256-superblock grid format - each of the 32 u16 qs words carries a 9-bit index into diff --git a/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das b/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das index 6eb51f19bd..7c601e8e35 100644 --- a/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das +++ b/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das @@ -2540,6 +2540,77 @@ def private iq3s_grid_word(i : int) : uint { return tbl[i] } +// ggml's iq2xxs_grid as low/high word pairs - 256 u64 entries, one uint word per call (word +// 2i = magnitudes 0..3 of entry i, word 2i+1 = 4..7); staged into workgroup memory per kernel. +def private iq2xxs_grid_word(i : int) : uint { + let tbl = fixed_array( + 0x08080808, 0x08080808, 0x0808082b, 0x08080808, 0x08081919, 0x08080808, 0x08082b08, 0x08080808, + 0x08082b2b, 0x08080808, 0x08190819, 0x08080808, 0x08191908, 0x08080808, 0x082b0808, 0x08080808, + 0x082b082b, 0x08080808, 0x082b2b08, 0x08080808, 0x082b2b2b, 0x08080808, 0x19080819, 0x08080808, + 0x19081908, 0x08080808, 0x19190808, 0x08080808, 0x19192b08, 0x08080808, 0x192b0819, 0x08080808, + 0x192b1908, 0x08080808, 0x2b080808, 0x08080808, 0x2b08082b, 0x08080808, 0x2b082b2b, 0x08080808, + 0x2b2b082b, 0x08080808, 0x08080819, 0x08080819, 0x08081908, 0x08080819, 0x08190808, 0x08080819, + 0x08191919, 0x08080819, 0x19080808, 0x08080819, 0x2b081908, 0x08080819, 0x2b192b08, 0x08080819, + 0x08080808, 0x0808082b, 0x0808082b, 0x0808082b, 0x082b082b, 0x0808082b, 0x2b08082b, 0x0808082b, + 0x08080819, 0x08081908, 0x08081908, 0x08081908, 0x08190808, 0x08081908, 0x082b0819, 0x08081908, + 0x082b1908, 0x08081908, 0x19080808, 0x08081908, 0x1908082b, 0x08081908, 0x19082b08, 0x08081908, + 0x192b0808, 0x08081908, 0x2b080819, 0x08081908, 0x2b081908, 0x08081908, 0x2b190808, 0x08081908, + 0x2b2b1908, 0x08081908, 0x08080808, 0x08081919, 0x0808082b, 0x08081919, 0x08082b08, 0x08081919, + 0x082b0808, 0x08081919, 0x1908192b, 0x08081919, 0x192b2b19, 0x08081919, 0x2b080808, 0x08081919, + 0x2b190819, 0x08081919, 0x08082b19, 0x0808192b, 0x08190808, 0x0808192b, 0x19080808, 0x0808192b, + 0x2b081908, 0x0808192b, 0x2b2b1908, 0x0808192b, 0x08080808, 0x08082b08, 0x08081919, 0x08082b08, + 0x08082b08, 0x08082b08, 0x08191908, 0x08082b08, 0x082b2b08, 0x08082b08, 0x19080819, 0x08082b08, + 0x19081908, 0x08082b08, 0x19190808, 0x08082b08, 0x1919082b, 0x08082b08, 0x2b082b08, 0x08082b08, + 0x08081908, 0x08082b19, 0x19080808, 0x08082b19, 0x0808082b, 0x08082b2b, 0x08191908, 0x08082b2b, + 0x08080819, 0x08190808, 0x08081908, 0x08190808, 0x08190808, 0x08190808, 0x082b0819, 0x08190808, + 0x19080808, 0x08190808, 0x192b0808, 0x08190808, 0x2b081908, 0x08190808, 0x2b190808, 0x08190808, + 0x2b191919, 0x08190808, 0x08080808, 0x08190819, 0x08082b08, 0x08190819, 0x082b0808, 0x08190819, + 0x19190808, 0x08190819, 0x19192b2b, 0x08190819, 0x2b080808, 0x08190819, 0x082b1908, 0x0819082b, + 0x19081919, 0x0819082b, 0x08080808, 0x08191908, 0x08082b08, 0x08191908, 0x082b0808, 0x08191908, + 0x082b1919, 0x08191908, 0x19082b19, 0x08191908, 0x2b080808, 0x08191908, 0x08192b08, 0x08191919, + 0x192b082b, 0x08191919, 0x08080808, 0x0819192b, 0x0819192b, 0x0819192b, 0x08080819, 0x08192b08, + 0x08081908, 0x08192b08, 0x08190808, 0x08192b08, 0x19080808, 0x08192b08, 0x2b080819, 0x08192b08, + 0x08080808, 0x08192b19, 0x08081919, 0x08192b19, 0x2b2b0808, 0x08192b19, 0x19190819, 0x08192b2b, + 0x08080808, 0x082b0808, 0x0808082b, 0x082b0808, 0x08082b2b, 0x082b0808, 0x19081908, 0x082b0808, + 0x192b0819, 0x082b0808, 0x2b080808, 0x082b0808, 0x2b08082b, 0x082b0808, 0x082b2b19, 0x082b0819, + 0x19082b08, 0x082b0819, 0x08080808, 0x082b082b, 0x0808082b, 0x082b082b, 0x08080819, 0x082b1908, + 0x08081908, 0x082b1908, 0x08190808, 0x082b1908, 0x19080808, 0x082b1908, 0x1919192b, 0x082b1908, + 0x08080808, 0x082b1919, 0x19080819, 0x082b1919, 0x192b1908, 0x082b1919, 0x2b190808, 0x082b192b, + 0x08082b08, 0x082b2b08, 0x082b0808, 0x082b2b08, 0x2b191908, 0x082b2b08, 0x19081908, 0x082b2b2b, + 0x08080819, 0x19080808, 0x08081908, 0x19080808, 0x08190808, 0x19080808, 0x08192b08, 0x19080808, + 0x082b0819, 0x19080808, 0x082b1908, 0x19080808, 0x19080808, 0x19080808, 0x19082b08, 0x19080808, + 0x1919192b, 0x19080808, 0x192b0808, 0x19080808, 0x2b080819, 0x19080808, 0x2b081908, 0x19080808, + 0x2b190808, 0x19080808, 0x08080808, 0x19080819, 0x082b0808, 0x19080819, 0x192b0819, 0x19080819, + 0x2b080808, 0x19080819, 0x2b081919, 0x19080819, 0x08080819, 0x1908082b, 0x08190808, 0x1908082b, + 0x19082b08, 0x1908082b, 0x1919192b, 0x1908082b, 0x192b2b08, 0x1908082b, 0x08080808, 0x19081908, + 0x08082b08, 0x19081908, 0x082b0808, 0x19081908, 0x2b080808, 0x19081908, 0x2b192b19, 0x19081908, + 0x0819082b, 0x19081919, 0x082b1908, 0x19081919, 0x08080808, 0x1908192b, 0x08080819, 0x19082b08, + 0x08081908, 0x19082b08, 0x08190808, 0x19082b08, 0x19080808, 0x19082b08, 0x19081919, 0x19082b08, + 0x08080808, 0x19082b19, 0x19192b08, 0x19082b19, 0x192b0819, 0x19082b19, 0x2b08082b, 0x19082b19, + 0x19081919, 0x19082b2b, 0x2b190808, 0x19082b2b, 0x08080808, 0x19190808, 0x08082b08, 0x19190808, + 0x08190819, 0x19190808, 0x08192b19, 0x19190808, 0x082b0808, 0x19190808, 0x2b080808, 0x19190808, + 0x2b082b08, 0x19190808, 0x08081908, 0x19190819, 0x1908082b, 0x19190819, 0x2b2b1908, 0x19190819, + 0x2b190819, 0x1919082b, 0x2b190808, 0x19191908, 0x2b19082b, 0x19191908, 0x08082b2b, 0x19191919, + 0x08080819, 0x1919192b, 0x19191908, 0x1919192b, 0x08080808, 0x19192b08, 0x08190819, 0x19192b08, + 0x08192b19, 0x19192b08, 0x192b1908, 0x19192b08, 0x19080808, 0x19192b19, 0x08082b08, 0x19192b2b, + 0x08081908, 0x192b0808, 0x08190808, 0x192b0808, 0x19080808, 0x192b0808, 0x192b2b08, 0x192b0808, + 0x08080808, 0x192b0819, 0x19191919, 0x192b0819, 0x08192b08, 0x192b082b, 0x192b0808, 0x192b082b, + 0x08080808, 0x192b1908, 0x08081919, 0x192b1908, 0x08190808, 0x192b1919, 0x0819082b, 0x192b1919, + 0x2b081908, 0x192b1919, 0x1908082b, 0x192b2b08, 0x08080808, 0x2b080808, 0x0808082b, 0x2b080808, + 0x08082b2b, 0x2b080808, 0x19080819, 0x2b080808, 0x2b08082b, 0x2b080808, 0x08081908, 0x2b080819, + 0x08192b08, 0x2b080819, 0x19080808, 0x2b080819, 0x08190819, 0x2b08082b, 0x08080819, 0x2b081908, + 0x08081908, 0x2b081908, 0x08190808, 0x2b081908, 0x08191919, 0x2b081908, 0x19080808, 0x2b081908, + 0x192b0808, 0x2b081908, 0x08080808, 0x2b081919, 0x1908192b, 0x2b081919, 0x2b191908, 0x2b081919, + 0x08082b19, 0x2b08192b, 0x19080808, 0x2b08192b, 0x192b0808, 0x2b08192b, 0x0808082b, 0x2b082b08, + 0x08081908, 0x2b082b19, 0x08190819, 0x2b082b2b, 0x08081908, 0x2b190808, 0x08190808, 0x2b190808, + 0x082b1908, 0x2b190808, 0x19080808, 0x2b190808, 0x2b2b0819, 0x2b190808, 0x0819192b, 0x2b190819, + 0x2b080808, 0x2b190819, 0x19081919, 0x2b19082b, 0x08080808, 0x2b191908, 0x082b082b, 0x2b191908, + 0x19081908, 0x2b191908, 0x19190819, 0x2b191919, 0x2b080819, 0x2b192b08, 0x082b0808, 0x2b192b19, + 0x0808082b, 0x2b2b0808, 0x19190808, 0x2b2b0808, 0x2b081919, 0x2b2b0808, 0x08082b19, 0x2b2b0819, + 0x08080808, 0x2b2b082b, 0x08192b08, 0x2b2b1908, 0x19190808, 0x2b2b2b08, 0x08081908, 0x2b2b2b19) + return tbl[i] +} + // ggml's iq2xs_grid as low/high word pairs - 512 u64 entries, one uint word per call (word // 2i = magnitudes 0..3 of entry i, word 2i+1 = 4..7); staged into workgroup memory per kernel. def private iq2xs_grid_word(i : int) : uint { // nolint:STYLE038 - a 1024-word data table, not splittable @@ -3361,6 +3432,51 @@ class KqBatchIq3xxs : KqBatchIq4xs { } } +// IQ2_XXS tile: the iq3xxs shell over the two-word u64 grid - block b's grid word (column 2b) +// carries four BYTE indices, its aux32 (column 2b+1) the parity-derived ksigns; the per-32 +// strip plane and fma inherit from KqBatchIq4xs (the (2ls+1) strips read exactly under the +// signed extract, staying below 128). +[vk_dispatch(name = "kq_batch_iq2xxs_cls", kernel = "run", family = "kq_batch_cls", grid = "wgs", params = "wgs : int64")] +class KqBatchIq2xxs : KqBatchIq4xs { + @workgroup gridw : uint[512] + + def iq3s_signed(g, nib : uint) : uint { + let m1 = (nib * 0x00204081u) & 0x01010101u + let mask = m1 * 255u + return (g ^ mask) + m1 + } + + def override stage_w(wsb0, wcol, nsb, s, word, grp : uint) { + for (ki in range(2)) { + let k = word + uint(ki) * 8u + let bu = k / 2u + let hh = k % 2u + var wv = uint4(0u) + if (wcol < pa.d) { + let wsb = wsb0 + wcol * nsb + s + let gw = wq[wsb * 16u + bu * 2u] + let aux = wq[wsb * 16u + bu * 2u + 1u] + let b0 = (gw >> (16u * hh)) & 255u + let b1 = (gw >> (16u * hh + 8u)) & 255u + let sg0 = ksign7((aux >> (14u * hh)) & 127u) + let sg1 = ksign7((aux >> (14u * hh + 7u)) & 127u) + wv = uint4(iq3s_signed(gridw[b0 * 2u], sg0 & 15u), iq3s_signed(gridw[b0 * 2u + 1u], sg0 >> 4u), + iq3s_signed(gridw[b1 * 2u], sg1 & 15u), iq3s_signed(gridw[b1 * 2u + 1u], sg1 >> 4u)) + } + ww4[grp * 17u + k] = wv + } + } + + [spirv_kernel(local_size_x = 256, name = "kq_batch_iq2xxs_cls_spv")] + def override run { + let tid = gl_LocalInvocationID.x + gridw[tid] = iq2xxs_grid_word(int(tid)) + gridw[tid + 256u] = iq2xxs_grid_word(int(tid + 256u)) + barrier() // the iq2xxs grid visible before the first stage_w + tile_shell() + } +} + // IQ4_NL tile: iq4xs's LUT staging (stage_w and the no-min blk_fma inherit) with q40's // per-block f16 d staged as the scale - no sub-scales, no bsum term anywhere. [vk_dispatch(name = "kq_batch_iq4nl_cls", kernel = "run", family = "kq_batch_cls", grid = "wgs", params = "wgs : int64")] @@ -4673,6 +4789,10 @@ struct VkIq2xsBlk { qs : int16[32] // one IQ2_XS superblock's 32 u16 qs words as 16-bit lanes (9-bit grid index + 7-bit ksigns index each); scales (f16 d8 + 16 unsigned (1 + 2s) strips) live in ws } +struct VkIq2xxsBlk { + qs : int16[32] // one IQ2_XXS superblock's 64B qs as 16-bit lanes: per block 4 grid byte indices + its aux32 (4 x 7-bit ksigns + per-32 ls nibble); scales (f16 d8 + 8 unsigned (1 + 2s) strips) live in ws +} + // The cm2 tile: one class template over the two stamp axes. The token column BN - l 256, m 128, // s 32 (the expert-bucket geometry: STILE's fast path loads a partial column UNCLAMPED and clamps // only the store, so every plane it reads carries 32 rows of slack) - and the weight format: BLK @@ -4700,6 +4820,8 @@ class template KqCm2BatchT : MoeCmBase { @workgroup @template_gate = IQ2SGRID iq2s_gridc : uint[2048] @template_constant IQ2XSGRID : bool = false // stage the 4 KB iq2xs u64 grid into workgroup memory @workgroup @template_gate = IQ2XSGRID iq2xs_gridc : uint[1024] + @template_constant IQ2XXSGRID : bool = false // stage the 2 KB iq2xxs u64 grid into workgroup memory + @workgroup @template_gate = IQ2XXSGRID iq2xxs_gridc : uint[512] [spirv_kernel(local_size_x = 256), arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-tile-pick-and-default")] def run { // nolint:STYLE037,STYLE038 — the fast/edge path pair, barrier- and register-coupled @@ -4757,6 +4879,12 @@ class template KqCm2BatchT : MoeCmBase { iq2xs_gridc[g2x + uint(k) * 256u] = iq2xs_grid_word(int(g2x + uint(k) * 256u)) } } + static_if (IQ2XXSGRID) { + let g2xx = gl_LocalInvocationID.x + for (k in range(2)) { + iq2xxs_gridc[g2xx + uint(k) * 256u] = iq2xxs_grid_word(int(g2xx + uint(k) * 256u)) + } + } barrier() // wg_blk0 (and the gated codebook) visible before the first decode load var a : coopmatWgA_f16_128x64 var b : BT @@ -5180,6 +5308,38 @@ class template Iq2xsCm2T : KqCm2BatchT { } } +class template Iq2xxsCm2T : KqCm2BatchT { + typedef BLK = VkIq2xxsBlk + typedef ST = uint + override IQ2XXSGRID = true + + [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] + def decode(blk : VkIq2xxsBlk; bc, cib : uint2) : float16 { + let e = cib.y + let g = e >> 5u + let r = e & 31u + let l = r >> 3u + let j = r & 7u + let qi = g * 8u + l + let qb = uint(int(unpack8(blk.qs[int(qi >> 1u)])[int(qi & 1u)])) & 0xFFu + let bl = 7u * l + let a0i = g * 8u + 4u + (bl >> 3u) + let b0 = uint(int(unpack8(blk.qs[int(a0i >> 1u)])[int(a0i & 1u)])) & 0xFFu + let b1 = uint(int(unpack8(blk.qs[int((a0i + 1u) >> 1u)])[int((a0i + 1u) & 1u)])) & 0xFFu + let sidx = ((b0 | (b1 << 8u)) >> (bl & 7u)) & 127u + var tt = sidx ^ (sidx >> 4u) + tt = tt ^ (tt >> 2u) + tt = tt ^ (tt >> 1u) + let sgb = sidx | ((tt & 1u) << 7u) + let gw = iq2xxs_gridc[qb * 2u + (j >> 2u)] + let gb = float((gw >> ((j & 3u) * 8u)) & 0xFFu) + let srow = (wg_blk0 + bc.x * (pa.n >> 8u) + bc.y) * 5u + let dv = unpackHalf2x16(ws[srow]).x + let sc = int(ws[srow + 1u + (g >> 2u)] << ((3u - (g & 3u)) * 8u)) >> 24 + return float16(dv * float(sc) * (((sgb >> j) & 1u) != 0u ? -gb : gb)) + } +} + // the stamps: the three width columns of each format (the dispatcher keys on (fmt, tc)) [vk_dispatch(name = "q8_batch_cm2l_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] class Q8Cm2LBatch : Q8Cm2T { @@ -5532,6 +5692,33 @@ class Iq2xsCm2SBatch : Iq2xsCm2T { typedef FLO = tensorLayout2DPad } +[vk_dispatch(name = "kq_batch_iq2xxs_cm2l_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Iq2xxsCm2LBatch : Iq2xxsCm2T { + typedef BT = coopmatWgB_f16_64x256 + typedef ACC = coopmatWgAcc_f16_128x256 + typedef ACCW = coopmatWgAcc_f32_128x256 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_iq2xxs_cm2m_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Iq2xxsCm2MBatch : Iq2xxsCm2T { + override BN = 128u + typedef BT = coopmatWgB_f16_64x128 + typedef ACC = coopmatWgAcc_f16_128x128 + typedef ACCW = coopmatWgAcc_f32_128x128 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_iq2xxs_cm2s_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Iq2xxsCm2SBatch : Iq2xxsCm2T { + override BN = 32u + override STILE = true + typedef BT = coopmatWgB_f16_64x32 + typedef ACC = coopmatWgAcc_f16_128x32 + typedef ACCW = coopmatWgAcc_f32_128x32 + typedef FLO = tensorLayout2DPad +} + // ===== the split-k reduce (sums the cm2 partial planes into y) ===== struct SkRedArgs { @@ -6113,6 +6300,8 @@ def kq_batch_cls_ensure(fmt : int; q40cm : bool) : bool { return ensure_kq_batch_iq2s_cls() } elif (fmt == int(KqFmt.iq2xs)) { return ensure_kq_batch_iq2xs_cls() + } elif (fmt == int(KqFmt.iq2xxs)) { + return ensure_kq_batch_iq2xxs_cls() } panic("dasLLAMA vulkan tier: no kq batch class for format {fmt}") return false @@ -6155,6 +6344,8 @@ def kq_batch_cls_enc_for(fmt : int; q40cm : bool; raw : VkCommandBuffer; var h : enc_kq_batch_iq2s_cls(raw, h, set_, pc, wgs) } elif (fmt == int(KqFmt.iq2xs)) { enc_kq_batch_iq2xs_cls(raw, h, set_, pc, wgs) + } elif (fmt == int(KqFmt.iq2xxs)) { + enc_kq_batch_iq2xxs_cls(raw, h, set_, pc, wgs) } else { panic("dasLLAMA vulkan tier: no kq batch class for format {fmt}") } @@ -6191,6 +6382,8 @@ def gemv_cls_ensure(fmt : int) : bool { return ensure_kq_gemv_iq2s_cls() } elif (fmt == int(KqFmt.iq2xs)) { return ensure_kq_gemv_iq2xs_cls() + } elif (fmt == int(KqFmt.iq2xxs)) { + return ensure_kq_gemv_iq2xxs_cls() } panic("dasLLAMA vulkan tier: no GEMV class for format {fmt}") return false @@ -6233,6 +6426,8 @@ def gemv_cls_enc(fmt : int; raw : VkCommandBuffer; var h : VkHaz; var set_ : VkD enc_kq_gemv_iq2s_cls(raw, h, set_, pc, wgs) } elif (fmt == int(KqFmt.iq2xs)) { enc_kq_gemv_iq2xs_cls(raw, h, set_, pc, wgs) + } elif (fmt == int(KqFmt.iq2xxs)) { + enc_kq_gemv_iq2xxs_cls(raw, h, set_, pc, wgs) } else { panic("dasLLAMA vulkan tier: no GEMV class for format {fmt}") } @@ -6695,3 +6890,55 @@ class KqGemvIq2xs : KqGemvBase { gemv_shell() } } + +// IQ2_XXS: the iq3xxs GEMV shell over the two-word u64 grid - block b's grid word carries +// four BYTE indices (two staged words each), its aux32 the parity-derived ksigns; the per-32 +// (1 + 2s) strip under the eighth-ed d reads exactly through the shared signed extract. +[vk_dispatch(name = "kq_gemv_iq2xxs_cls", kernel = "run", family = "kq_gemv_cls", grid = "wgs", params = "wgs : int64")] +class KqGemvIq2xxs : KqGemvBase { + @workgroup gridw : uint[512] + + def iq3s_signed(g, nib : uint) : uint { + let m1 = (nib * 0x00204081u) & 0x01010101u + let mask = m1 * 255u + return (g ^ mask) + m1 + } + + def override blk_contrib(wsb, blk, xsb : uint) : float { + let gw = wq[wsb * 16u + blk * 2u] + let aux = wq[wsb * 16u + blk * 2u + 1u] + let xb4 = xsb * 16u + blk * 2u + let xl = xq4[xb4] + let xh = xq4[xb4 + 1u] + var isum = 0 + for (h in range(2)) { + let hu = uint(h) + let b0 = (gw >> (16u * hu)) & 255u + let b1 = (gw >> (16u * hu + 8u)) & 255u + let sg0 = ksign7((aux >> (14u * hu)) & 127u) + let sg1 = ksign7((aux >> (14u * hu + 7u)) & 127u) + let w0 = iq3s_signed(gridw[b0 * 2u], sg0 & 15u) + let w1 = iq3s_signed(gridw[b0 * 2u + 1u], sg0 >> 4u) + let w2 = iq3s_signed(gridw[b1 * 2u], sg1 & 15u) + let w3 = iq3s_signed(gridw[b1 * 2u + 1u], sg1 >> 4u) + if (h == 0) { + isum += sdot4(w0, xl.x) + sdot4(w1, xl.y) + sdot4(w2, xl.z) + sdot4(w3, xl.w) + } else { + isum += sdot4(w0, xh.x) + sdot4(w1, xh.y) + sdot4(w2, xh.z) + sdot4(w3, xh.w) + } + } + let dd = unpackHalf2x16(wsu[wsb * 5u]).x + return axs[xsb] * dd * float(iq4_sc(wsu[wsb * 5u + 1u + blk / 4u], blk) * isum) + } + + [spirv_kernel(local_size_x = 64, name = "kq_gemv_iq2xxs_cls_spv")] + def run { + let lid = gl_LocalInvocationID.x + for (k in range(8)) { + let idx = lid + uint(k) * 64u + gridw[idx] = iq2xxs_grid_word(int(idx)) + } + barrier() // the iq2xxs grid visible before the shell's block walk + gemv_shell() + } +} diff --git a/modules/dasLLAMA/dasllama/dasllama_vulkan_common.das b/modules/dasLLAMA/dasllama/dasllama_vulkan_common.das index 6b1b855a03..bfb23db506 100644 --- a/modules/dasLLAMA/dasllama/dasllama_vulkan_common.das +++ b/modules/dasLLAMA/dasllama/dasllama_vulkan_common.das @@ -1754,6 +1754,7 @@ def vk_kq_schema_id(fmt : int) : int { if (fmt == 11) return 2 if (fmt == 12) return 23 if (fmt == 13) return 24 + if (fmt == 14) return 25 panic("vk_kq_schema_id: not a kq stack fmt") return 0 } diff --git a/modules/dasLLAMA/dasllama/dasllama_vulkan_prefill.das b/modules/dasLLAMA/dasllama/dasllama_vulkan_prefill.das index 14d7b1bb19..3b4cdc501a 100644 --- a/modules/dasLLAMA/dasllama/dasllama_vulkan_prefill.das +++ b/modules/dasLLAMA/dasllama/dasllama_vulkan_prefill.das @@ -94,6 +94,9 @@ def private cm2_cls_ensure(fmt : int; tc : int64) : bool { if (fmt == int(KqFmt.iq2xs)) { return tc == 256l ? ensure_kq_batch_iq2xs_cm2l_cls() : (tc == 128l ? ensure_kq_batch_iq2xs_cm2m_cls() : ensure_kq_batch_iq2xs_cm2s_cls()) } + if (fmt == int(KqFmt.iq2xxs)) { + return tc == 256l ? ensure_kq_batch_iq2xxs_cm2l_cls() : (tc == 128l ? ensure_kq_batch_iq2xxs_cm2m_cls() : ensure_kq_batch_iq2xxs_cm2s_cls()) + } verify(fmt == int(KqFmt.q8), "vk prefill: the cm2 ensure fell through for a kq format - a format the feed admits must own a tile arm") return tc == 256l ? ensure_q8_batch_cm2l_cls() : (tc == 128l ? ensure_q8_batch_cm2m_cls() : ensure_q8_batch_cm2s_cls()) } @@ -172,6 +175,12 @@ def private cm2_cls_set(fmt : int; tc : int64; bufs : uint64 const[5]; sizes : i } return tc == 128l ? set_kq_batch_iq2xs_cm2m_cls(bufs, sizes, gbits) : set_kq_batch_iq2xs_cm2s_cls(bufs, sizes, gbits) } + if (fmt == int(KqFmt.iq2xxs)) { + if (tc == 256l) { + return set_kq_batch_iq2xxs_cm2l_cls(bufs, sizes, gbits) + } + return tc == 128l ? set_kq_batch_iq2xxs_cm2m_cls(bufs, sizes, gbits) : set_kq_batch_iq2xxs_cm2s_cls(bufs, sizes, gbits) + } verify(fmt == int(KqFmt.q8), "vk prefill: the cm2 set fell through for a kq format - a format the feed admits must own a tile arm") if (tc == 256l) { return set_q8_batch_cm2l_cls(bufs, sizes, gbits) @@ -277,6 +286,14 @@ def private cm2_cls_enc(fmt : int; tc : int64; raw : VkCommandBuffer; var h : Vk } else { enc_kq_batch_iq2xs_cm2s_cls(raw, h, s, pc, groups) } + } elif (fmt == int(KqFmt.iq2xxs)) { + if (tc == 256l) { + enc_kq_batch_iq2xxs_cm2l_cls(raw, h, s, pc, groups) + } elif (tc == 128l) { + enc_kq_batch_iq2xxs_cm2m_cls(raw, h, s, pc, groups) + } else { + enc_kq_batch_iq2xxs_cm2s_cls(raw, h, s, pc, groups) + } } elif (tc == 256l) { verify(fmt == int(KqFmt.q8), "vk prefill: the cm2 enc fell through for a kq format - a format the feed admits must own a tile arm") enc_q8_batch_cm2l_cls(raw, h, s, pc, groups) diff --git a/modules/dasLLAMA/tests/_vkd_oracles.das b/modules/dasLLAMA/tests/_vkd_oracles.das index 5f8556fb1b..3af9753c64 100644 --- a/modules/dasLLAMA/tests/_vkd_oracles.das +++ b/modules/dasLLAMA/tests/_vkd_oracles.das @@ -201,6 +201,12 @@ def kq_cls_ref(fmt : int; wq, ws, xq : array; xs : array) : KqGemvB k2x.gridw[i] = IQ2XS_GRID[i] } k = k2x + } elif (fmt == int(KqFmt.iq2xxs)) { + var k2xx = new KqGemvIq2xxs() + for (i in range(512)) { // same replay rule, the 256-entry grid's word pairs + k2xx.gridw[i] = IQ2XXS_GRID[i] + } + k = k2xx } else { k = new KqGemvK6() } @@ -466,6 +472,50 @@ def iq2xs_gemv_float_oracle(wq, ws, xq : array; xs : array; return <- y } +// IQ2_XXS GEMV float oracle, INDEPENDENT of the class bit-math: w = d8 * (1 + 2s) * grid_byte +// * ksign straight off the plane words (per block a grid word of 4 byte indices + its aux32; +// the k4-shaped scale row with 8 UNSIGNED per-32 strips and the pre-eighth-ed d). +def iq2xxs_gemv_float_oracle(wq, ws, xq : array; xs : array; + regions : array; n, d, nreg : int) : array { + var y : array + y |> resize(nreg * d) + let nsb = n / 256 + for (r in range(nreg)) { + for (row in range(d)) { + let w0 = int(regions[r * 2]) + row * nsb + let x0 = int(regions[r * 2 + 1]) + var acc = 0.0 + for (s in range(nsb)) { + let sb = w0 + s + let xsb = x0 + s + let dd = half_at(ws, sb * 10) + for (b in range(8)) { + let gw = wq[sb * 16 + b * 2] + let aux = int(wq[sb * 16 + b * 2 + 1]) + let sc = float(byte_s8(ws, sb * 20 + 4 + b)) // SIGNED like the class/batch stage - real (2ls+1) strips stay positive + var bacc = 0.0 + for (l in range(4)) { + let idx = int((gw >> uint(8 * l)) & 0xFFu) + let sidx = (aux >> (7 * l)) & 127 + var tt = sidx ^ (sidx >> 4) + tt = tt ^ (tt >> 2) + tt = tt ^ (tt >> 1) + let sgb = sidx | ((tt & 1) << 7) + for (j in range(8)) { + let gb = float(int(IQ2XXS_GRID[idx * 2 + j / 4] >> uint(8 * (j % 4))) & 0xFF) + let sgn = ((sgb >> j) & 1) != 0 ? -1.0 : 1.0 + bacc += gb * sgn * float(byte_s8(xq, xsb * 256 + b * 32 + l * 8 + j)) + } + } + acc += xs[xsb] * dd * sc * bacc + } + } + y[r * d + row] = acc + } + } + return <- y +} + // IQ3_S f16-feed GEMM reference over the device form: grid gather + sign flip per element, // f16-rounded per weight like its siblings ([qs 64][qh 8][signs 32] rows, the k4-shaped // scale row with signed (1 + 2s) sub-scales). @@ -608,6 +658,48 @@ def iq2xsf16_gemm_oracle(wq : array; wsu : array; xf : array; } } +// IQ2_XXS f16-feed GEMM reference over the device form: per block a grid word of 4 byte +// indices (two u64 words each) + its aux32 (ksigns by parity), f16-rounded per weight like +// its siblings (16 uint words per superblock, the k4-shaped scale row with 8 UNSIGNED per-32 +// strips, d pre-eighth-ed). +def iq2xxsf16_gemm_oracle(wq : array; wsu : array; xf : array; + recs : array; nrec, n, d : int; var y : array) { + let nsb = n / 256 + for (rr in range(nrec)) { + let wsb0 = int(recs[rr * 4]) + let row0 = int(recs[rr * 4 + 1]) + let cnt = int(recs[rr * 4 + 2]) + for (r in range(cnt)) { + for (c in range(d)) { + var acc = 0.0 + for (s in range(nsb)) { + let sb = wsb0 + c * nsb + s + let dv = unpackHalf2x16(wsu[sb * 5]).x + for (e in range(256)) { + let g = e / 32 + let rr2 = e % 32 + let l = rr2 / 8 + let j = rr2 % 8 + let gw = wq[sb * 16 + g * 2] + let aux = int(wq[sb * 16 + g * 2 + 1]) + let idx = int((gw >> uint(8 * l)) & 0xFFu) + let sidx = (aux >> (7 * l)) & 127 + var tt = sidx ^ (sidx >> 4) + tt = tt ^ (tt >> 2) + tt = tt ^ (tt >> 1) + let sgb = sidx | ((tt & 1) << 7) + let sc = float(int(wsu[sb * 5 + 1 + g / 4] << uint((3 - (g % 4)) * 8)) >> 24) + let gb = float(int(IQ2XXS_GRID[idx * 2 + j / 4] >> uint(8 * (j % 4))) & 0xFF) + let w = float(float16(dv * sc * (((sgb >> j) & 1) != 0 ? -gb : gb))) + acc += w * half_at(xf, (row0 + r) * n + s * 256 + e) + } + } + y[(row0 + r) * d + c] = acc + } + } + } +} + // IQ4_NL f16-feed GEMM reference over the device form: q40's walk with the codebook in place // of (q - 8), f16-rounded per weight like its siblings (q40's 32-word rows + 8 f16 d). def iq4nlf16_gemm_oracle(wq : array; wsu : array; xf : array; diff --git a/modules/dasLLAMA/tests/test_vulkan_kernels.das b/modules/dasLLAMA/tests/test_vulkan_kernels.das index bf40300a23..f6a9302a73 100644 --- a/modules/dasLLAMA/tests/test_vulkan_kernels.das +++ b/modules/dasLLAMA/tests/test_vulkan_kernels.das @@ -381,7 +381,7 @@ def private attn_row_oracle(q : array; qbase : int; kk : array; kb def test_vkd_kq_gemv_family(t0 : T?) { t0 |> run("kq GEMV class family == the CPU oracle (all 6 formats)") <| @(t : T?) { static_if (typeinfo builtin_module_exists(vulkan)) { - if (!(ensure_kq_gemv_k4_cls() && ensure_kq_gemv_q40_cls() && ensure_kq_gemv_k5_cls() && ensure_kq_gemv_k6_cls() && ensure_kq_gemv_iq4xs_cls() && ensure_kq_gemv_k3_cls() && ensure_kq_gemv_iq3s_cls() && ensure_kq_gemv_iq3xxs_cls() && ensure_kq_gemv_iq4nl_cls() && ensure_kq_gemv_k2_cls() && ensure_kq_gemv_iq2s_cls() && ensure_kq_gemv_iq2xs_cls())) { + if (!(ensure_kq_gemv_k4_cls() && ensure_kq_gemv_q40_cls() && ensure_kq_gemv_k5_cls() && ensure_kq_gemv_k6_cls() && ensure_kq_gemv_iq4xs_cls() && ensure_kq_gemv_k3_cls() && ensure_kq_gemv_iq3s_cls() && ensure_kq_gemv_iq3xxs_cls() && ensure_kq_gemv_iq4nl_cls() && ensure_kq_gemv_k2_cls() && ensure_kq_gemv_iq2s_cls() && ensure_kq_gemv_iq2xs_cls() && ensure_kq_gemv_iq2xxs_cls())) { feint("no Vulkan device - skipping\n") return } @@ -389,9 +389,9 @@ def test_vkd_kq_gemv_family(t0 : T?) { let d = 3 // rows per region let nreg = 2 let rows = nreg * d - let nfmt = 12 - let fmts = fixed_array(int(KqFmt.k4), int(KqFmt.q40), int(KqFmt.k5), int(KqFmt.k6), int(KqFmt.iq4xs), int(KqFmt.k3), int(KqFmt.iq3s), int(KqFmt.iq3xxs), int(KqFmt.iq4nl), int(KqFmt.k2), int(KqFmt.iq2s), int(KqFmt.iq2xs)) - let stride4s = fixed_array(32, 32, 40, 48, 32, 24, 26, 24, 32, 16, 18, 16) // uint WORDS per superblock (iq3s/iq3xxs/iq2s are not uint4-aligned) + let nfmt = 13 + let fmts = fixed_array(int(KqFmt.k4), int(KqFmt.q40), int(KqFmt.k5), int(KqFmt.k6), int(KqFmt.iq4xs), int(KqFmt.k3), int(KqFmt.iq3s), int(KqFmt.iq3xxs), int(KqFmt.iq4nl), int(KqFmt.k2), int(KqFmt.iq2s), int(KqFmt.iq2xs), int(KqFmt.iq2xxs)) + let stride4s = fixed_array(32, 32, 40, 48, 32, 24, 26, 24, 32, 16, 18, 16, 16) // uint WORDS per superblock (iq3s/iq3xxs/iq2s are not uint4-aligned) for (fi in range(nfmt)) { let fmt = fmts[fi] let stride4 = stride4s[fi] @@ -477,6 +477,8 @@ def test_vkd_kq_gemv_family(t0 : T?) { enc_kq_gemv_iq2s_cls(raw2, h2, s_cls, pc, wgs) } elif (fmt == int(KqFmt.iq2xs)) { enc_kq_gemv_iq2xs_cls(raw2, h2, s_cls, pc, wgs) + } elif (fmt == int(KqFmt.iq2xxs)) { + enc_kq_gemv_iq2xxs_cls(raw2, h2, s_cls, pc, wgs) } else { enc_kq_gemv_k6_cls(raw2, h2, s_cls, pc, wgs) } @@ -525,6 +527,12 @@ def test_vkd_kq_gemv_family(t0 : T?) { t |> success(badf == 0, "iq2xs: the class bit-math matches the float dequant oracle ({badf} of {rows} rows off)") delete y_flt } + if (fmt == int(KqFmt.iq2xxs)) { // the byte-grid/aux32 gather has its own float witness + var y_flt <- iq2xxs_gemv_float_oracle(wqh, wsh, xqh, xsh, regs, 512, d, nreg) + let badf = mismatch_approx(y_ref, y_flt) + t |> success(badf == 0, "iq2xxs: the class bit-math matches the float dequant oracle ({badf} of {rows} rows off)") + delete y_flt + } delete regs delete y_ref delete wqh @@ -1045,7 +1053,7 @@ def test_vkd_q8_batch_tile(t0 : T?) { def test_vkd_kq_batch_family(t0 : T?) { t0 |> run("kq batch tile class family == the CPU oracle (all 6 formats)") <| @(t : T?) { static_if (typeinfo builtin_module_exists(vulkan)) { - if (!(ensure_kq_batch_k4_cls() && ensure_kq_batch_q40_cls() && ensure_kq_batch_k5_cls() && ensure_kq_batch_k6_cls() && ensure_kq_batch_iq4xs_cls() && ensure_kq_batch_k3_cls() && ensure_kq_batch_iq3s_cls() && ensure_kq_batch_iq3xxs_cls() && ensure_kq_batch_iq4nl_cls() && ensure_kq_batch_k2_cls() && ensure_kq_batch_iq2s_cls() && ensure_kq_batch_iq2xs_cls())) { + if (!(ensure_kq_batch_k4_cls() && ensure_kq_batch_q40_cls() && ensure_kq_batch_k5_cls() && ensure_kq_batch_k6_cls() && ensure_kq_batch_iq4xs_cls() && ensure_kq_batch_k3_cls() && ensure_kq_batch_iq3s_cls() && ensure_kq_batch_iq3xxs_cls() && ensure_kq_batch_iq4nl_cls() && ensure_kq_batch_k2_cls() && ensure_kq_batch_iq2s_cls() && ensure_kq_batch_iq2xs_cls() && ensure_kq_batch_iq2xxs_cls())) { feint("no Vulkan device - skipping\n") return } @@ -1060,9 +1068,9 @@ def test_vkd_kq_batch_family(t0 : T?) { let wgs1 = ((cnt1 + 31) / 32) * wtiles let wgs = wgs0 + wgs1 let totsb = 2 * d * nsb // superblocks across both regions - let nfmt = 12 - let wstrides = fixed_array(32, 32, 40, 48, 32, 24, 26, 24, 32, 16, 18, 16) // uint words per superblock: k4 q40 k5 k6 iq4xs k3 iq3s iq3xxs iq4nl k2 iq2s iq2xs - let pipes = fixed_array(int(KqFmt.k4), int(KqFmt.q40), int(KqFmt.k5), int(KqFmt.k6), int(KqFmt.iq4xs), int(KqFmt.k3), int(KqFmt.iq3s), int(KqFmt.iq3xxs), int(KqFmt.iq4nl), int(KqFmt.k2), int(KqFmt.iq2s), int(KqFmt.iq2xs)) + let nfmt = 13 + let wstrides = fixed_array(32, 32, 40, 48, 32, 24, 26, 24, 32, 16, 18, 16, 16) // uint words per superblock: k4 q40 k5 k6 iq4xs k3 iq3s iq3xxs iq4nl k2 iq2s iq2xs iq2xxs + let pipes = fixed_array(int(KqFmt.k4), int(KqFmt.q40), int(KqFmt.k5), int(KqFmt.k6), int(KqFmt.iq4xs), int(KqFmt.k3), int(KqFmt.iq3s), int(KqFmt.iq3xxs), int(KqFmt.iq4nl), int(KqFmt.k2), int(KqFmt.iq2s), int(KqFmt.iq2xs), int(KqFmt.iq2xxs)) for (fi in range(nfmt)) { let fmt = pipes[fi] let wstride = wstrides[fi] @@ -1151,6 +1159,8 @@ def test_vkd_kq_batch_family(t0 : T?) { enc_kq_batch_iq2s_cls(raw2, h2, sc, pc, int64(wgs)) } elif (fmt == int(KqFmt.iq2xs)) { enc_kq_batch_iq2xs_cls(raw2, h2, sc, pc, int64(wgs)) + } elif (fmt == int(KqFmt.iq2xxs)) { + enc_kq_batch_iq2xxs_cls(raw2, h2, sc, pc, int64(wgs)) } else { enc_kq_batch_k6_cls(raw2, h2, sc, pc, int64(wgs)) } @@ -2692,7 +2702,7 @@ def test_vkd_iq2s_cm2_batch(t0 : T?) { [test] def test_vkd_iq2xs_cm2_batch(t0 : T?) { - t0 |> run("cm2 IQ2_S l/m/s tiles == the CPU oracle (u64 grid + ksigns-by-parity decode-in-load)") <| @(t : T?) { + t0 |> run("cm2 IQ2_XS l/m/s tiles == the CPU oracle (u64 grid + ksigns-by-parity decode-in-load)") <| @(t : T?) { static_if (typeinfo builtin_module_exists(vulkan)) { if (!ensure_q8_batch_cls()) { // engage the device without touching coopmat rails t |> skip("no Vulkan device") @@ -2814,6 +2824,130 @@ def test_vkd_iq2xs_cm2_batch(t0 : T?) { } } +[test] +def test_vkd_iq2xxs_cm2_batch(t0 : T?) { + t0 |> run("cm2 IQ2_XXS l/m/s tiles == the CPU oracle (byte-grid + aux32 ksigns decode-in-load)") <| @(t : T?) { + static_if (typeinfo builtin_module_exists(vulkan)) { + if (!ensure_q8_batch_cls()) { // engage the device without touching coopmat rails + t |> skip("no Vulkan device") + return + } + if (!(g_gpu.coopmat_mode == 4 && g_gpu.has_coopmat2)) { + t |> skip("the cm2 iq2xxs tiles serve only in mode 4 on an NV_coopmat2 device") + return + } + verify(ensure_kq_batch_iq2xxs_cm2l_cls() && ensure_kq_batch_iq2xxs_cm2m_cls() && ensure_kq_batch_iq2xxs_cm2s_cls(), "iq2xxs cm2 class rails must engage in cm2 mode") + let n = 768 // one 512-unrolled pass + 4 tail steps crossing superblock bounds + let nsb = n / 256 + let d = 160 // one full 128-tile + a 32 edge + let cnt0 = 300 + let cnt1 = 260 + let rows = cnt0 + cnt1 + let totsb = 2 * d * nsb + var wqh : array + var wsuh : array + var xfh : array + wqh |> resize(totsb * 16) + wsuh |> resize(totsb * 5) + xfh |> resize((rows + 32) * n / 2) // + the s tile's 32-row read slack + for (i in range(totsb * 16)) { + wqh[i] = hash_word(uint(i) + 29u) // arbitrary quant bytes + } + for (sb in range(totsb)) { + wsuh[sb * 5] = packHalf2x16(float2(0.0002 * float(1 + sb % 7), 0.0)) // cool d8 + wsuh[sb * 5 + 1] = hash_word(uint(sb * 4) + 613u) // 16 unsigned strip bytes + wsuh[sb * 5 + 2] = hash_word(uint(sb * 4 + 1) + 613u) + wsuh[sb * 5 + 3] = hash_word(uint(sb * 4 + 2) + 613u) + wsuh[sb * 5 + 4] = hash_word(uint(sb * 4 + 3) + 613u) + } + for (i in range((rows + 32) * n / 2)) { + xfh[i] = ws_word(uint(i) * 3u + 5u) // f16 activation pairs + } + let wq_bytes = int64(totsb) * 64l + let ws_bytes = int64(length(wsuh)) * 4l + let xf_bytes = int64((rows + 32) * n) * 2l + let y_bytes = int64(rows * d) * 4l + let wqd = make_device_buf(wq_bytes) + let wsd = make_device_buf(ws_bytes) + let xfd = make_device_buf(xf_bytes) + let yd2 = make_device_buf(y_bytes) + var host = make_host_buf(y_bytes, true, [cached = true]) + var y_cls : array + y_cls |> resize(rows * d) + var y_ref : array + y_ref |> resize(rows * d) + for (ml in range(3)) { + let ttile = ml == 0 ? 256 : (ml == 1 ? 128 : 32) + let wtiles = (d + 127) / 128 + let wgs0 = ((cnt0 + ttile - 1) / ttile) * wtiles + let wgs1 = ((cnt1 + ttile - 1) / ttile) * wtiles + let wgs = wgs0 + wgs1 + var sched : array + sched |> resize(2 * 4 + wgs) + sched[0] = 0u + sched[1] = 0u + sched[2] = uint(cnt0) + sched[3] = 0u + sched[4] = uint(d * nsb) + sched[5] = uint(cnt0) + sched[6] = uint(cnt1) + sched[7] = uint(wgs0) + for (w in range(wgs)) { + sched[8 + w] = w < wgs0 ? 0u : 1u + } + unsafe { + upload_region_at(wqd, 0l, addr(wqh[0]), wq_bytes) + upload_region_at(wsd, 0l, addr(wsuh[0]), ws_bytes) + upload_region_at(xfd, 0l, addr(xfh[0]), xf_bytes) + let sc_bytes = int64(2 * 4 + wgs) * 4l + let scd = make_device_buf(sc_bytes) + upload_region_at(scd, 0l, addr(sched[0]), sc_bytes) + var sc = (ml == 0 + ? set_kq_batch_iq2xxs_cm2l_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)) + : (ml == 1 + ? set_kq_batch_iq2xxs_cm2m_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)) + : set_kq_batch_iq2xxs_cm2s_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)))) + var raw2 = alloc_cmd() + let begin = VkCommandBufferBeginInfo() + vk_check(vkBeginCommandBuffer(raw2, begin), null) + var h2 : VkHaz + var pc = BatchArgs(n = uint(n), d = uint(d), map_off = uint(2 * 4)) + if (ml == 0) { + enc_kq_batch_iq2xxs_cm2l_cls(raw2, h2, sc, pc, int64(wgs)) + } elif (ml == 1) { + enc_kq_batch_iq2xxs_cm2m_cls(raw2, h2, sc, pc, int64(wgs)) + } else { + enc_kq_batch_iq2xxs_cm2s_cls(raw2, h2, sc, pc, int64(wgs)) + } + vhz_dep(raw2, h2, 16u, 0u, true) + cmd_copy_whole(raw2, yd2, host.buf, y_bytes) + vk_check(vkEndCommandBuffer(raw2), null) + submit_wait(raw2) + memcpy(addr(y_cls[0]), host.mapped, y_bytes) + } + iq2xxsf16_gemm_oracle(wqh, wsuh, xfh, sched, 2, n, d, y_ref) + let bad = mismatch_bars(y_cls, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) + to_log(LOG_INFO, "cm2 iq2xxs {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile batch: {bad} of {rows * d} off the oracle\n") + t |> success(bad == 0, "cm2 iq2xxs {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile matches the CPU oracle ({bad} of {rows * d} off)") + delete sched + } + delete y_ref + delete wqh + delete wsuh + delete xfh + delete y_cls + } else { + t |> skip("dasVulkan not present") + } + } +} + [test] def test_vkd_iq3xxs_cm2_batch(t0 : T?) { t0 |> run("cm2 IQ3_XXS l/m/s tiles == the CPU oracle (halved grid + parity signs decode-in-load)") <| @(t : T?) { From 66b088d3a342fd7f737f464586f25f9837e7be34 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Mon, 31 Aug 2026 15:56:24 -0700 Subject: [PATCH 49/61] dasLLAMA: IQ2_XXS Phase D - Metal closes the format, and the format ladder iq2xxs_gw (512 words, program-scope constant), MetalKqGemvIq2xxs + MetalKqMvIq2xxsT B2/B4 + MetalKqMvB8Iq2xxs + MetalKqMulMmIq2xxs (its own IQ2XXS static_if arm) - signs via ksign7m off the block's aux32, ONE per-32 UNSIGNED strip per block (the 8 pad bytes of the 16B strip region ride along unread); the "iq2xxss" blob arm is the iq2xss split verbatim. Gates: metal gemv 2/2 + gemm 2/2 FIRST TRY, zen2 kquant -jit 263/0 regression, lint 0. E2e --ngl 99: the Metal stream is 64/64 IDENTICAL to the stamped CPU stream - ALL FOUR TIERS of IQ2_XXS are bit-consistent with each other, diverging from llama.cpp only at the step-3 0.0267-logit top-2 flip. M1: CPU das 897.4/56.3 vs llama.cpp 139.9/98.4 (6.41x/0.57x), Metal das 3227.6/180.7 vs 3473.3/231.5 (0.93x/0.78x). Four-tier table: zen2 2.86x/0.70x, vk 0.78x/0.70x, M1 CPU 6.41x/0.57x, Metal 0.93x/0.78x. The ELEVENTH kq format lands its fourth tier: the iquant format ladder is COMPLETE. Co-Authored-By: Claude Fable 5 --- modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md | 14 + modules/dasLLAMA/dasllama/dasllama_layout.das | 23 +- .../dasllama/dasllama_metal_common.das | 14 + .../dasllama/dasllama_metal_kernels.das | 334 ++++++++++++++++++ .../dasllama/dasllama_metal_prefill.das | 8 +- .../dasllama/dasllama_metal_shapes.das | 4 +- .../dasLLAMA/tests/_metal_kernel_common.das | 9 +- .../tests/test_metal_gemm_kernels.das | 24 +- .../tests/test_metal_gemv_kernels.das | 40 ++- 9 files changed, 432 insertions(+), 38 deletions(-) diff --git a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md index b53e72dfb8..0f389d952f 100644 --- a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md +++ b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md @@ -532,6 +532,20 @@ step-3 0.0267 top-2 flip vs llama.cpp. 5060 Ti vs llama.cpp b10660 -ngl 99: pp51 vs 18572.5 (0.78x - above the 0.70x class), tg128 297.0 vs 421.4 (0.70x - the grid-gemv re-stage at 2 KB, milder than iq2xs's 4 KB; followup_vulkan #35). Metal: pending. + +Phase D (Metal, 2026-08-31): the iq2xs Metal walk over the aux32 form - `iq2xxs_gw` (512 +words, program-scope constant), `MetalKqGemvIq2xxs` + `MetalKqMvIq2xxsT` B2/B4 + +`MetalKqMvB8Iq2xxs` + `MetalKqMulMmIq2xxs` (its own `IQ2XXS` static_if arm), every kernel +deriving signs via `ksign7m` off the block's aux32 and reading the ONE per-32 UNSIGNED strip +(byte `bu` of the 16B strip region - the 8 pad bytes ride along unread); the "iq2xxss" blob +arm is the iq2xss split verbatim. Gates: metal gemv 2/2 + gemm 2/2 FIRST TRY, zen2 kquant +-jit 263/0 regression, lint 0. E2e --ngl 99: the Metal stream is 64/64 IDENTICAL to the +stamped CPU stream - ALL FOUR TIERS of IQ2_XXS are bit-consistent with each other, carrying +only the step-3 0.0267 top-2 flip vs llama.cpp. M1 benches: CPU das 897.4/56.3 vs llama.cpp +139.9/98.4 (6.41x/0.57x), Metal das 3227.6/180.7 vs 3473.3/231.5 (0.93x/0.78x). The format +is CLOSED - and with it THE FORMAT LADDER: four-tier table zen2 2.86x/0.70x, vk 0.78x/0.70x, +M1 CPU 6.41x/0.57x, Metal 0.93x/0.78x. + ### IQ2_XS Phase A (CPU, 2026-08-31) - the ksigns u64 tier Shape: 256-superblock grid format - each of the 32 u16 qs words carries a 9-bit index into diff --git a/modules/dasLLAMA/dasllama/dasllama_layout.das b/modules/dasLLAMA/dasllama/dasllama_layout.das index 0378721501..2f0b17b442 100644 --- a/modules/dasLLAMA/dasllama/dasllama_layout.das +++ b/modules/dasLLAMA/dasllama/dasllama_layout.das @@ -29,7 +29,7 @@ require math def private metal_blob_off_ok(off : int64; fmt : KqFmt) : bool { if (off < 0l) return true if (fmt == KqFmt.q8) return off % 256l == 0l - if (fmt == KqFmt.k6 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s || fmt == KqFmt.iq3xxs || fmt == KqFmt.iq2s || fmt == KqFmt.iq2xs) return off % 512l == 0l // the split scale form: the (off/256)*2 d-plane bind stays 4B-aligned + if (fmt == KqFmt.k6 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s || fmt == KqFmt.iq3xxs || fmt == KqFmt.iq2s || fmt == KqFmt.iq2xs || fmt == KqFmt.iq2xxs) return off % 512l == 0l // the split scale form: the (off/256)*2 d-plane bind stays 4B-aligned if (fmt == KqFmt.q51) return off % 128l == 0l // (off/32)*20 and (off/32)*4 both 16B-aligned return true } @@ -264,6 +264,25 @@ def metal_blob_scale_plane(var t : Model; name : string) : bool { } return true } + if (name == "iq2xxss") { + // iq2xxs: the identical 20B row over t.iq2xxss (8 strips + 8 pad ride the 16B region) + if (!empty(t.iq2xxss)) { + var @exact_size i2xxc : array + let nsb = long_length(t.iq2xxss) / IQ2XXS_SSB + i2xxc |> reserve_resize(nsb * 18l) + unsafe { + var dp = addr(i2xxc[0]) + let sp = addr < uint8 const? >(t.iq2xxss[0]) + for (sb in range64(nsb)) { + memcpy(reinterpret(dp + sb * 16l), reinterpret(sp + sb * IQ2XXS_SSB + 4l), 16) + dp[nsb * 16l + sb * 2l] = sp[sb * IQ2XXS_SSB] + dp[nsb * 16l + sb * 2l + 1l] = sp[sb * IQ2XXS_SSB + 1l] + } + } + t.iq2xxss <- i2xxc + } + return true + } if (name == "k2s") { // k2: the 20B row splits like k6 - the 16 pair-byte strips first, the 4B d+dmin headers as the tail if (!empty(t.k2s)) { @@ -332,7 +351,7 @@ def metal_blob_commit(var t : Model) : bool { if (!metal_blob_eligible(t)) { return false } - for (nm in ["k4s", "k5s", "k6s", "iq4xss", "k3s", "iq3ss", "iq3xxss", "k2s", "iq2ss", "iq2xss"]) { + for (nm in ["k4s", "k5s", "k6s", "iq4xss", "k3s", "iq3ss", "iq3xxss", "k2s", "iq2ss", "iq2xss", "iq2xxss"]) { metal_blob_scale_plane(t, nm) } t.metal_blob = true diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_common.das b/modules/dasLLAMA/dasllama/dasllama_metal_common.das index 3c04f7dc76..9cea1b0e47 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_common.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_common.das @@ -287,6 +287,11 @@ var g_pso_kq_mvb2_iq2xs : MetalComputePipeline? var g_pso_kq_mvb4_iq2xs : MetalComputePipeline? var g_pso_kq_mvb8_iq2xs : MetalComputePipeline? var g_pso_kq_mm_b24 : MetalComputePipeline? +var g_pso_kq_iq2xxs : MetalComputePipeline? +var g_pso_kq_mvb2_iq2xxs : MetalComputePipeline? +var g_pso_kq_mvb4_iq2xxs : MetalComputePipeline? +var g_pso_kq_mvb8_iq2xxs : MetalComputePipeline? +var g_pso_kq_mm_b25 : MetalComputePipeline? var g_pso_kq_mvb2_iq3xxs : MetalComputePipeline? var g_pso_kq_mvb4_iq3xxs : MetalComputePipeline? var g_pso_kq_mvb8_iq3xxs : MetalComputePipeline? @@ -906,6 +911,10 @@ def kq_quants_of(dev : MetalDevice?; t : Model; fmt : KqFmt; off : int64) : tupl return (buf = plane_buffer(dev, addr < void? >(t.iq2xsq[0]), uint64(long_length(t.iq2xsq)), t.image_map != null), qoff = uint64(sb0 * 64l)) } + if (fmt == KqFmt.iq2xxs) { + return (buf = plane_buffer(dev, addr < void? >(t.iq2xxsq[0]), uint64(long_length(t.iq2xxsq)), t.image_map != null), + qoff = uint64(sb0 * 64l)) + } return (buf = plane_buffer(dev, addr < void? >(t.k6q[0]), uint64(long_length(t.k6q)), t.image_map != null), qoff = uint64(sb0 * 192l)) } @@ -966,6 +975,11 @@ def kq_scales_of(dev : MetalDevice?; t : Model; fmt : KqFmt; off : int64) : tupl return (buf = plane_buffer(dev, addr < void? >(t.iq2xss[0]), uint64(long_length(t.iq2xss)), t.image_map != null), soff = uint64(sb0 * 16l), doff = uint64(plane_sb * 16l + sb0 * 2l)) } + if (fmt == KqFmt.iq2xxs) { // the same split 18B/sb plane over t.iq2xxss + let plane_sb = long_length(t.iq2xxss) / 18l + return (buf = plane_buffer(dev, addr < void? >(t.iq2xxss[0]), uint64(long_length(t.iq2xxss)), t.image_map != null), + soff = uint64(sb0 * 16l), doff = uint64(plane_sb * 16l + sb0 * 2l)) + } return (buf = plane_buffer(dev, addr < void? >(t.k5s[0]), uint64(long_length(t.k5s)), t.image_map != null), soff = uint64(sb0 * 16l), doff = 0ul) } diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das index f1f332ee54..db94e0ed55 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das @@ -3670,6 +3670,11 @@ def metal_decode_init : bool { // nolint:STYLE038 — a flat compile_pso per k g_pso_kq_mvb4_iq2xs = compile_pso(MetalKqMvB4Iq2xs_metal_kq_mv_iq2xs_msl, MetalKqMvB4Iq2xs_metal_kq_mv_iq2xs_msl_entry, MetalKqMvB4Iq2xs_metal_kq_mv_iq2xs_msl_fastmath, ok) g_pso_kq_mvb8_iq2xs = compile_pso(metal_kq_mvb8_iq2xs_msl, metal_kq_mvb8_iq2xs_msl_entry, metal_kq_mvb8_iq2xs_msl_fastmath, ok) g_pso_kq_mm_b24 = compile_pso(MetalKqMulMmIq2xs_metal_kq_mulmm_msl, MetalKqMulMmIq2xs_metal_kq_mulmm_msl_entry, MetalKqMulMmIq2xs_metal_kq_mulmm_msl_fastmath, ok) + g_pso_kq_iq2xxs = compile_pso(metal_kq_gemv_iq2xxs_msl, metal_kq_gemv_iq2xxs_msl_entry, metal_kq_gemv_iq2xxs_msl_fastmath, ok) + g_pso_kq_mvb2_iq2xxs = compile_pso(MetalKqMvB2Iq2xxs_metal_kq_mv_iq2xxs_msl, MetalKqMvB2Iq2xxs_metal_kq_mv_iq2xxs_msl_entry, MetalKqMvB2Iq2xxs_metal_kq_mv_iq2xxs_msl_fastmath, ok) + g_pso_kq_mvb4_iq2xxs = compile_pso(MetalKqMvB4Iq2xxs_metal_kq_mv_iq2xxs_msl, MetalKqMvB4Iq2xxs_metal_kq_mv_iq2xxs_msl_entry, MetalKqMvB4Iq2xxs_metal_kq_mv_iq2xxs_msl_fastmath, ok) + g_pso_kq_mvb8_iq2xxs = compile_pso(metal_kq_mvb8_iq2xxs_msl, metal_kq_mvb8_iq2xxs_msl_entry, metal_kq_mvb8_iq2xxs_msl_fastmath, ok) + g_pso_kq_mm_b25 = compile_pso(MetalKqMulMmIq2xxs_metal_kq_mulmm_msl, MetalKqMulMmIq2xxs_metal_kq_mulmm_msl_entry, MetalKqMulMmIq2xxs_metal_kq_mulmm_msl_fastmath, ok) g_pso_kq_mm_b45 = compile_pso(MetalKqMulMmIq4nl_metal_kq_mulmm_msl, MetalKqMulMmIq4nl_metal_kq_mulmm_msl_entry, MetalKqMulMmIq4nl_metal_kq_mulmm_msl_fastmath, ok) g_pso_kq_k2 = compile_pso(metal_kq_gemv_k2_msl, metal_kq_gemv_k2_msl_entry, metal_kq_gemv_k2_msl_fastmath, ok) g_pso_kq_mvb2_k2 = compile_pso(MetalKqMvB2K2_metal_kq_mv_k2_msl, MetalKqMvB2K2_metal_kq_mv_k2_msl_entry, MetalKqMvB2K2_metal_kq_mv_k2_msl_fastmath, ok) @@ -3758,6 +3763,8 @@ def enc_kq_gemv(enc : MetalComputeEncoder?; fmt : KqFmt; bq : MetalBuffer?; qoff enc_kq_iq2s_c(enc, bs, doff, bs, soff, bq, qoff, bx, xoff, by, yoff, bn, bd, rows, n) } elif (fmt == KqFmt.iq2xs) { enc_kq_iq2xs_c(enc, bs, doff, bs, soff, bq, qoff, bx, xoff, by, yoff, bn, bd, rows, n) + } elif (fmt == KqFmt.iq2xxs) { + enc_kq_iq2xxs_c(enc, bs, doff, bs, soff, bq, qoff, bx, xoff, by, yoff, bn, bd, rows, n) } elif (fmt == KqFmt.k6) { enc_kq_k6_c(enc, bs, doff, bs, soff, bq, qoff, bx, xoff, by, yoff, bn, bd, rows, n) } elif (fmt == KqFmt.k5 && (n < 3072l || rows <= 1024l)) { @@ -3858,6 +3865,14 @@ def enc_kq_mvb(enc : MetalComputeEncoder?; fmt : KqFmt; bq : MetalBuffer?; qoff } else { enc_kq_mvb4_iq2xs_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows, gcols) } + } elif (fmt == KqFmt.iq2xxs) { + if (b8) { + enc_kq_mvb8_iq2xxs_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows) + } elif (two) { + enc_kq_mvb2_iq2xxs_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows) + } else { + enc_kq_mvb4_iq2xxs_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows, gcols) + } } elif (fmt == KqFmt.k6) { if (b8) { enc_kq_mvb8_k6_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows) @@ -3906,6 +3921,8 @@ def enc_kq_gemm_mm_b(enc : MetalComputeEncoder?; fmt : KqFmt; bq : MetalBuffer?; enc_kq_mm_iq2s_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, bk, bn, mp, rows) } elif (fmt == KqFmt.iq2xs) { enc_kq_mm_iq2xs_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, bk, bn, mp, rows) + } elif (fmt == KqFmt.iq2xxs) { + enc_kq_mm_iq2xxs_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, bk, bn, mp, rows) } elif (fmt == KqFmt.k6) { enc_kq_mm_k6_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, bk, bn, mp, rows) } elif (fmt == KqFmt.k4) { @@ -4363,6 +4380,11 @@ def metal_kernels_release { // nolint:STYLE037,STYLE038 — a flat release_pso release_pso(g_pso_kq_mvb4_iq2xs) release_pso(g_pso_kq_mvb8_iq2xs) release_pso(g_pso_kq_mm_b24) + release_pso(g_pso_kq_iq2xxs) + release_pso(g_pso_kq_mvb2_iq2xxs) + release_pso(g_pso_kq_mvb4_iq2xxs) + release_pso(g_pso_kq_mvb8_iq2xxs) + release_pso(g_pso_kq_mm_b25) release_pso(g_pso_kq_mvb2_iq4nl) release_pso(g_pso_kq_mvb4_iq4nl) release_pso(g_pso_kq_mvb8_iq4nl) @@ -5605,6 +5627,7 @@ class template MetalKqMulMmK45T : MetalMmTileBase { @template_constant K2 : bool = false // q2_K: per-16 sc/min nibble pairs (one uint4 strip), the split 4B d+dmin tail at s0off @template_constant IQ2S : bool = false // iq2s: the u64-grid gather DIRECT off the hoisted constant table, per-16 UNSIGNED strips @template_constant IQ2XS : bool = false // iq2xs: iq2s's walk with ksigns-by-parity off each u16 qs word (no sign plane, no qh) + @template_constant IQ2XXS : bool = false // iq2xxs: byte grid indices + aux32 parity ksigns, per-32 UNSIGNED strips @workgroup @template_gate = IQ4XS iq4lut : float[16] // kvalues_iq4nl as floats, staged once per threadgroup @workgroup @template_gate = IQ3S gridw3 : uint[512] // the 2 KB iq3s grid, staged once per threadgroup @workgroup @template_gate = IQ3XXS gridw3x : uint[256] // the 1 KB halved iq3xxs grid, staged once per threadgroup @@ -5725,6 +5748,25 @@ class template MetalKqMulMmK45T : MetalMmTileBase { va[k * 4 + 3] = float16(dsc3 * float(qv & 4278190080u) - dmn) } } else { + static_if (IQ2XXS) { + // iq2xxs: per-32 UNSIGNED strip byte js; block js's grid word carries + // four byte indices, its aux32 the parity-derived ksigns + let sv = ks4[blk] + let scw = js < 4u ? sv.x : sv.y + let dsc = float(ksh[blk]) * float((scw >> ((js & 3u) * 8u)) & 255u) + let gw2 = kqu[blk * 16u + js * 2u] + let aux2 = kqu[blk * 16u + js * 2u + 1u] + for [unroll_full] (k in range(4)) { + let q3 = il0 * 2u + uint(k) / 2u + let bidx = (gw2 >> (8u * q3)) & 255u + let sgb2 = ksign7m((aux2 >> (7u * q3)) & 127u) + let nib = uint(k) % 2u == 0u ? sgb2 & 15u : sgb2 >> 4u + let sw = iq3s_sw(iq2xxs_gw(int(bidx * 2u + uint(k) % 2u)), nib) + for [unroll_full] (c in range(4)) { + va[k * 4 + c] = float16(dsc * iq3s_sx(sw, uint(c))) + } + } + } else { static_if (IQ2XS) { // iq2xs: the iq2s strip/scale reads; each u16 qs word = 9-bit grid // index + 7-bit ksigns index whose sign byte ksign7m recomputes @@ -5889,6 +5931,7 @@ class template MetalKqMulMmK45T : MetalMmTileBase { } } } + } } } barrier() @@ -5970,6 +6013,12 @@ class MetalKqMulMmIq2xs : MetalKqMulMmK45T { override IQ2XS = true } +// iq2xxs rides its own IQ2XXS arm; the same split scale plane binds (d8 plane at s0off = doff) +[metal_dispatch(name = "enc_kq_mm_iq2xxs_c", pso = "g_pso_kq_mm_b25", tgmem = "MetalKqMulMmIq2xxs_metal_kq_mulmm_msl_tgmem", tg = 128, grid = "mp/32, rows/64", params = "mp : int64, rows : int64")] +class MetalKqMulMmIq2xxs : MetalKqMulMmK45T { + override IQ2XXS = true +} + // iq4nl rides the IQ4XS arm with IQ4NL flipping the scale read to q40's per-32-block d plane [metal_dispatch(name = "enc_kq_mm_iq4nl_c", pso = "g_pso_kq_mm_b45", tgmem = "MetalKqMulMmIq4nl_metal_kq_mulmm_msl_tgmem", tg = 128, grid = "mp/32, rows/64", params = "mp : int64, rows : int64")] class MetalKqMulMmIq4nl : MetalKqMulMmK45T { @@ -6390,6 +6439,77 @@ def private ksign7m(v : uint) : uint { return v | ((t & 1u) << 7u) } +// ggml's iq2xxs_grid as low/high word pairs (word 2i = magnitudes 0..3 of u64 entry i, word +// 2i+1 = 4..7); the same all-literal program-scope-constant form as its grid siblings. +def private iq2xxs_gw(i : int) : uint { + let tbl = fixed_array( + 0x08080808, 0x08080808, 0x0808082b, 0x08080808, 0x08081919, 0x08080808, 0x08082b08, 0x08080808, + 0x08082b2b, 0x08080808, 0x08190819, 0x08080808, 0x08191908, 0x08080808, 0x082b0808, 0x08080808, + 0x082b082b, 0x08080808, 0x082b2b08, 0x08080808, 0x082b2b2b, 0x08080808, 0x19080819, 0x08080808, + 0x19081908, 0x08080808, 0x19190808, 0x08080808, 0x19192b08, 0x08080808, 0x192b0819, 0x08080808, + 0x192b1908, 0x08080808, 0x2b080808, 0x08080808, 0x2b08082b, 0x08080808, 0x2b082b2b, 0x08080808, + 0x2b2b082b, 0x08080808, 0x08080819, 0x08080819, 0x08081908, 0x08080819, 0x08190808, 0x08080819, + 0x08191919, 0x08080819, 0x19080808, 0x08080819, 0x2b081908, 0x08080819, 0x2b192b08, 0x08080819, + 0x08080808, 0x0808082b, 0x0808082b, 0x0808082b, 0x082b082b, 0x0808082b, 0x2b08082b, 0x0808082b, + 0x08080819, 0x08081908, 0x08081908, 0x08081908, 0x08190808, 0x08081908, 0x082b0819, 0x08081908, + 0x082b1908, 0x08081908, 0x19080808, 0x08081908, 0x1908082b, 0x08081908, 0x19082b08, 0x08081908, + 0x192b0808, 0x08081908, 0x2b080819, 0x08081908, 0x2b081908, 0x08081908, 0x2b190808, 0x08081908, + 0x2b2b1908, 0x08081908, 0x08080808, 0x08081919, 0x0808082b, 0x08081919, 0x08082b08, 0x08081919, + 0x082b0808, 0x08081919, 0x1908192b, 0x08081919, 0x192b2b19, 0x08081919, 0x2b080808, 0x08081919, + 0x2b190819, 0x08081919, 0x08082b19, 0x0808192b, 0x08190808, 0x0808192b, 0x19080808, 0x0808192b, + 0x2b081908, 0x0808192b, 0x2b2b1908, 0x0808192b, 0x08080808, 0x08082b08, 0x08081919, 0x08082b08, + 0x08082b08, 0x08082b08, 0x08191908, 0x08082b08, 0x082b2b08, 0x08082b08, 0x19080819, 0x08082b08, + 0x19081908, 0x08082b08, 0x19190808, 0x08082b08, 0x1919082b, 0x08082b08, 0x2b082b08, 0x08082b08, + 0x08081908, 0x08082b19, 0x19080808, 0x08082b19, 0x0808082b, 0x08082b2b, 0x08191908, 0x08082b2b, + 0x08080819, 0x08190808, 0x08081908, 0x08190808, 0x08190808, 0x08190808, 0x082b0819, 0x08190808, + 0x19080808, 0x08190808, 0x192b0808, 0x08190808, 0x2b081908, 0x08190808, 0x2b190808, 0x08190808, + 0x2b191919, 0x08190808, 0x08080808, 0x08190819, 0x08082b08, 0x08190819, 0x082b0808, 0x08190819, + 0x19190808, 0x08190819, 0x19192b2b, 0x08190819, 0x2b080808, 0x08190819, 0x082b1908, 0x0819082b, + 0x19081919, 0x0819082b, 0x08080808, 0x08191908, 0x08082b08, 0x08191908, 0x082b0808, 0x08191908, + 0x082b1919, 0x08191908, 0x19082b19, 0x08191908, 0x2b080808, 0x08191908, 0x08192b08, 0x08191919, + 0x192b082b, 0x08191919, 0x08080808, 0x0819192b, 0x0819192b, 0x0819192b, 0x08080819, 0x08192b08, + 0x08081908, 0x08192b08, 0x08190808, 0x08192b08, 0x19080808, 0x08192b08, 0x2b080819, 0x08192b08, + 0x08080808, 0x08192b19, 0x08081919, 0x08192b19, 0x2b2b0808, 0x08192b19, 0x19190819, 0x08192b2b, + 0x08080808, 0x082b0808, 0x0808082b, 0x082b0808, 0x08082b2b, 0x082b0808, 0x19081908, 0x082b0808, + 0x192b0819, 0x082b0808, 0x2b080808, 0x082b0808, 0x2b08082b, 0x082b0808, 0x082b2b19, 0x082b0819, + 0x19082b08, 0x082b0819, 0x08080808, 0x082b082b, 0x0808082b, 0x082b082b, 0x08080819, 0x082b1908, + 0x08081908, 0x082b1908, 0x08190808, 0x082b1908, 0x19080808, 0x082b1908, 0x1919192b, 0x082b1908, + 0x08080808, 0x082b1919, 0x19080819, 0x082b1919, 0x192b1908, 0x082b1919, 0x2b190808, 0x082b192b, + 0x08082b08, 0x082b2b08, 0x082b0808, 0x082b2b08, 0x2b191908, 0x082b2b08, 0x19081908, 0x082b2b2b, + 0x08080819, 0x19080808, 0x08081908, 0x19080808, 0x08190808, 0x19080808, 0x08192b08, 0x19080808, + 0x082b0819, 0x19080808, 0x082b1908, 0x19080808, 0x19080808, 0x19080808, 0x19082b08, 0x19080808, + 0x1919192b, 0x19080808, 0x192b0808, 0x19080808, 0x2b080819, 0x19080808, 0x2b081908, 0x19080808, + 0x2b190808, 0x19080808, 0x08080808, 0x19080819, 0x082b0808, 0x19080819, 0x192b0819, 0x19080819, + 0x2b080808, 0x19080819, 0x2b081919, 0x19080819, 0x08080819, 0x1908082b, 0x08190808, 0x1908082b, + 0x19082b08, 0x1908082b, 0x1919192b, 0x1908082b, 0x192b2b08, 0x1908082b, 0x08080808, 0x19081908, + 0x08082b08, 0x19081908, 0x082b0808, 0x19081908, 0x2b080808, 0x19081908, 0x2b192b19, 0x19081908, + 0x0819082b, 0x19081919, 0x082b1908, 0x19081919, 0x08080808, 0x1908192b, 0x08080819, 0x19082b08, + 0x08081908, 0x19082b08, 0x08190808, 0x19082b08, 0x19080808, 0x19082b08, 0x19081919, 0x19082b08, + 0x08080808, 0x19082b19, 0x19192b08, 0x19082b19, 0x192b0819, 0x19082b19, 0x2b08082b, 0x19082b19, + 0x19081919, 0x19082b2b, 0x2b190808, 0x19082b2b, 0x08080808, 0x19190808, 0x08082b08, 0x19190808, + 0x08190819, 0x19190808, 0x08192b19, 0x19190808, 0x082b0808, 0x19190808, 0x2b080808, 0x19190808, + 0x2b082b08, 0x19190808, 0x08081908, 0x19190819, 0x1908082b, 0x19190819, 0x2b2b1908, 0x19190819, + 0x2b190819, 0x1919082b, 0x2b190808, 0x19191908, 0x2b19082b, 0x19191908, 0x08082b2b, 0x19191919, + 0x08080819, 0x1919192b, 0x19191908, 0x1919192b, 0x08080808, 0x19192b08, 0x08190819, 0x19192b08, + 0x08192b19, 0x19192b08, 0x192b1908, 0x19192b08, 0x19080808, 0x19192b19, 0x08082b08, 0x19192b2b, + 0x08081908, 0x192b0808, 0x08190808, 0x192b0808, 0x19080808, 0x192b0808, 0x192b2b08, 0x192b0808, + 0x08080808, 0x192b0819, 0x19191919, 0x192b0819, 0x08192b08, 0x192b082b, 0x192b0808, 0x192b082b, + 0x08080808, 0x192b1908, 0x08081919, 0x192b1908, 0x08190808, 0x192b1919, 0x0819082b, 0x192b1919, + 0x2b081908, 0x192b1919, 0x1908082b, 0x192b2b08, 0x08080808, 0x2b080808, 0x0808082b, 0x2b080808, + 0x08082b2b, 0x2b080808, 0x19080819, 0x2b080808, 0x2b08082b, 0x2b080808, 0x08081908, 0x2b080819, + 0x08192b08, 0x2b080819, 0x19080808, 0x2b080819, 0x08190819, 0x2b08082b, 0x08080819, 0x2b081908, + 0x08081908, 0x2b081908, 0x08190808, 0x2b081908, 0x08191919, 0x2b081908, 0x19080808, 0x2b081908, + 0x192b0808, 0x2b081908, 0x08080808, 0x2b081919, 0x1908192b, 0x2b081919, 0x2b191908, 0x2b081919, + 0x08082b19, 0x2b08192b, 0x19080808, 0x2b08192b, 0x192b0808, 0x2b08192b, 0x0808082b, 0x2b082b08, + 0x08081908, 0x2b082b19, 0x08190819, 0x2b082b2b, 0x08081908, 0x2b190808, 0x08190808, 0x2b190808, + 0x082b1908, 0x2b190808, 0x19080808, 0x2b190808, 0x2b2b0819, 0x2b190808, 0x0819192b, 0x2b190819, + 0x2b080808, 0x2b190819, 0x19081919, 0x2b19082b, 0x08080808, 0x2b191908, 0x082b082b, 0x2b191908, + 0x19081908, 0x2b191908, 0x19190819, 0x2b191919, 0x2b080819, 0x2b192b08, 0x082b0808, 0x2b192b19, + 0x0808082b, 0x2b2b0808, 0x19190808, 0x2b2b0808, 0x2b081919, 0x2b2b0808, 0x08082b19, 0x2b2b0819, + 0x08080808, 0x2b2b082b, 0x08192b08, 0x2b2b1908, 0x19190808, 0x2b2b2b08, 0x08081908, 0x2b2b2b19) + return tbl[i] +} + // ggml's iq2xs_grid as low/high word pairs (word 2i = magnitudes 0..3 of u64 entry i, word // 2i+1 = 4..7); the same all-literal program-scope-constant form as iq2s_gw below. def private iq2xs_gw(i : int) : uint { // nolint:STYLE038 - a 1024-word data table, not splittable @@ -8591,6 +8711,66 @@ class MetalKqGemvIq2xs { } } +// IQ2_XXS: the iq2xs GEMV geometry over the aux32 form - per (block, half) the block's grid +// word supplies two BYTE indices (each doubling into the u64 grid pair), its aux32 the +// parity ksigns; ONE per-32 UNSIGNED strip scales the whole block on the eighth-ed d. +[metal_dispatch(name = "enc_kq_iq2xxs_c", pso = "g_pso_kq_iq2xxs", tg = 64, grid = "rows/8", params = "rows : int64, n : int64")] +class MetalKqGemvIq2xxs { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d8 plane - the scale buffer bound at byte nsb*16 (s0off = the caller's doff) + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // strip region: 8 UNSIGNED per-32 sub-scales + 8 pad, byte view (same buffer at 0) + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array // iq2xxs quant plane, uint view (16 words per superblock: [grid bytes][aux32] per block) + @ssbo @binding = 3 @role = "read" @off = "xoff" @span = "n*4" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" @span = "rows*4" y : array + @uniform @binding = 5 ndim : uint + @uniform @binding = 6 ddim : uint + + [metal_kernel(name="metal_kq_gemv_iq2xxs_msl")] + def metal_kq_gemv_iq2xxs { + let lane = gl_SubgroupInvocationID + let ix = lane / 16u + let it = lane % 16u + let bu = it / 2u + let il = it % 2u + let nb = ndim / 256u + let first_row = (gl_WorkGroupID.x * 2u + gl_SubgroupID) * 4u + var sumf : float[4] + var ib = ix + while (ib < nb) { + let yb4 = ib * 64u + bu * 8u + il * 4u + let y0 = x[yb4] + let y1 = x[yb4 + 1u] + let y2 = x[yb4 + 2u] + let y3 = x[yb4 + 3u] + for [unroll_full] (r in range(4)) { + let blk = (first_row + uint(r)) * nb + ib + let gw = kqu[blk * 16u + bu * 2u] + let aux = kqu[blk * 16u + bu * 2u + 1u] + var accv = float4(0.0) + for [unroll_full] (wi in range(2)) { + let q3 = il * 2u + uint(wi) + let bidx = (gw >> (8u * q3)) & 255u + let sgn = ksign7m((aux >> (7u * q3)) & 127u) + let w0 = iq3s_sw(iq2xxs_gw(int(bidx * 2u)), sgn & 15u) + let w1 = iq3s_sw(iq2xxs_gw(int(bidx * 2u + 1u)), sgn >> 4u) + let ya = wi == 0 ? y0 : y2 + let yb = wi == 0 ? y1 : y3 + accv += ya * float4(iq3s_sx(w0, 0u), iq3s_sx(w0, 1u), iq3s_sx(w0, 2u), iq3s_sx(w0, 3u)) + accv += yb * float4(iq3s_sx(w1, 0u), iq3s_sx(w1, 1u), iq3s_sx(w1, 2u), iq3s_sx(w1, 3u)) + } + let sc = float(kscb[blk * 16u + bu]) + sumf[r] += float(kdh[blk]) * sc * (accv.x + accv.y + accv.z + accv.w) + } + ib += 2u + } + for [unroll_full] (r in range(4)) { + let s = simd_sum(sumf[r]) + if (lane == 0u && first_row + uint(r) < ddim) { + y[first_row + uint(r)] = s + } + } + } +} + // IQ3_XXS: the iq3s GEMV shape verbatim over the 24-word rows - the 1 KB halved grid as a // float4 magnitude slab, signs from the block's aux32 through the parity helper (no table), // the same (2ls+1)-strip x halved-d fold. 4 rows per simdgroup, dispatch rows/8. @@ -9312,6 +9492,160 @@ class MetalKqMvB8Iq2xs { } } +// The iq2xxs B2/B4 pair: the iq2xs shell over the aux32 walk - lane tx owns block tx; the +// block's grid word supplies four byte indices, ksign7m per index; ONE per-32 strip as dl. +[ |> template_struct_instance] +class template MetalKqMvIq2xxsT { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d8 plane - the scale buffer bound at byte nsb*16 + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // strip region: 8 UNSIGNED per-32 sub-scales + 8 pad (same buffer at 0) + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array // iq2xxs quant plane, uint view + @ssbo @binding = 3 @role = "read" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" y : array + @uniform @binding = 5 ka : KqMvArgs + @template_constant NR : int = 2 + @template_constant NRU : uint = 2u // NR's uint spelling (colbase math) - override BOTH together + @template_constant TILED : bool = false + + [metal_kernel] + def metal_kq_mv_iq2xxs { + let lane = gl_SubgroupInvocationID + let tx = lane % 8u + let ty = lane / 8u + let row = gl_WorkGroupID.x * 8u + gl_SubgroupID * 4u + ty + let colbase = gl_WorkGroupID.y * NRU // nolint:LINT002 dead in the TILED=false stamp - measured free + let nb = ka.ndim / 256u + let nb4 = ka.ndim / 4u + var sumf : float[NR] + var sb = 0u + while (sb < nb) { + let blk = row * nb + sb + let dl = float(kdh[blk]) * float(kscb[blk * 16u + tx]) + let gw = kqu[blk * 16u + tx * 2u] + let aux = kqu[blk * 16u + tx * 2u + 1u] + var w : float4[8] + for [unroll_full] (l in range(4)) { + let lu = uint(l) + let bidx = (gw >> (8u * lu)) & 255u + let sgn = ksign7m((aux >> (7u * lu)) & 127u) + let sw0 = iq3s_sw(iq2xxs_gw(int(bidx * 2u)), sgn & 15u) + let sw1 = iq3s_sw(iq2xxs_gw(int(bidx * 2u + 1u)), sgn >> 4u) + w[l * 2] = float4(iq3s_sx(sw0, 0u), iq3s_sx(sw0, 1u), iq3s_sx(sw0, 2u), iq3s_sx(sw0, 3u)) * dl + w[l * 2 + 1] = float4(iq3s_sx(sw1, 0u), iq3s_sx(sw1, 1u), iq3s_sx(sw1, 2u), iq3s_sx(sw1, 3u)) * dl + } + let cx4 = sb * 64u + tx * 8u + static_if (TILED) { + for [unroll_full] (b in range(NR)) { + let xb4 = (colbase + uint(b)) * nb4 + cx4 + var acc = 0.0 + for [unroll_full] (u in range(8)) { + let v = x[xb4 + uint(u)] + acc += v.x * w[u].x + v.y * w[u].y + v.z * w[u].z + v.w * w[u].w + } + sumf[b] += acc + } + } else { + for [unroll_full] (b in range(NR)) { + let xb4 = uint(b) * nb4 + cx4 + var acc = 0.0 + for [unroll_full] (u in range(8)) { + let v = x[xb4 + uint(u)] + acc += v.x * w[u].x + v.y * w[u].y + v.z * w[u].z + v.w * w[u].w + } + sumf[b] += acc + } + } + sb++ + } + for [unroll_full] (b in range(NR)) { + var s = sumf[b] + s += simd_shuffle_down(s, 4u) + s += simd_shuffle_down(s, 2u) + s += simd_shuffle_down(s, 1u) + if (tx == 0u && row < ka.ddim && uint(b) < ka.nr) { + y[uint(b) * ka.ys + row] = s + } + } + } +} + +[metal_dispatch(name = "enc_kq_mvb2_iq2xxs_c", pso = "g_pso_kq_mvb2_iq2xxs", tg = 64, grid = "rows/8", params = "rows : int64")] +class MetalKqMvB2Iq2xxs : MetalKqMvIq2xxsT { + override NR = 2 + override NRU = 2u + override TILED = false +} + +[metal_dispatch(name = "enc_kq_mvb4_iq2xxs_c", pso = "g_pso_kq_mvb4_iq2xxs", tg = 64, grid = "rows/8, gcols", params = "rows : int64, gcols : int64")] +class MetalKqMvB4Iq2xxs : MetalKqMvIq2xxsT { + override NR = 4 + override NRU = 4u + override TILED = true +} + +// the iq2xxs B8 twin: the iq2xs B8 shell (one superblock's X panel staged per threadgroup) +// with the aux32 compose; the grid read direct. +[metal_dispatch(name = "enc_kq_mvb8_iq2xxs_c", pso = "g_pso_kq_mvb8_iq2xxs", tgmem = "metal_kq_mvb8_iq2xxs_msl_tgmem", tg = 64, grid = "rows/8", params = "rows : int64")] +class MetalKqMvB8Iq2xxs { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d8 plane - the scale buffer bound at byte nsb*16 + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // strip region: 8 UNSIGNED per-32 sub-scales + 8 pad (same buffer at 0) + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array // iq2xxs quant plane, uint view + @ssbo @binding = 3 @role = "read" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" y : array + @uniform @binding = 5 ka : KqMvArgs + @workgroup txp : float4[512] // [8 streams x 64 float4] one superblock's X panel + + [metal_kernel(name="metal_kq_mvb8_iq2xxs_msl")] + def metal_kq_mvb8_iq2xxs { + let lid = gl_LocalInvocationID.x + let lane = gl_SubgroupInvocationID + let tx = lane % 8u + let ty = lane / 8u + let row = gl_WorkGroupID.x * 8u + gl_SubgroupID * 4u + ty + let nb = ka.ndim / 256u + let nb4 = ka.ndim / 4u + var acc : float4[8] + var sb = 0u + while (sb < nb) { + barrier() + for [unroll_full] (j in range(8)) { + let slot = uint(j) * 64u + lid + txp[slot] = x[(slot / 64u) * nb4 + sb * 64u + (slot % 64u)] + } + barrier() + let blk = row * nb + sb + let dl = float(kdh[blk]) * float(kscb[blk * 16u + tx]) + let gw = kqu[blk * 16u + tx * 2u] + let aux = kqu[blk * 16u + tx * 2u + 1u] + var w : float4[8] + for [unroll_full] (l in range(4)) { + let lu = uint(l) + let bidx = (gw >> (8u * lu)) & 255u + let sgn = ksign7m((aux >> (7u * lu)) & 127u) + let sw0 = iq3s_sw(iq2xxs_gw(int(bidx * 2u)), sgn & 15u) + let sw1 = iq3s_sw(iq2xxs_gw(int(bidx * 2u + 1u)), sgn >> 4u) + w[l * 2] = float4(iq3s_sx(sw0, 0u), iq3s_sx(sw0, 1u), iq3s_sx(sw0, 2u), iq3s_sx(sw0, 3u)) * dl + w[l * 2 + 1] = float4(iq3s_sx(sw1, 0u), iq3s_sx(sw1, 1u), iq3s_sx(sw1, 2u), iq3s_sx(sw1, 3u)) * dl + } + let cxl = tx * 8u + for [unroll_full] (b in range(8)) { + for [unroll_full] (u in range(8)) { + acc[b] += txp[uint(b) * 64u + cxl + uint(u)] * w[u] + } + } + sb++ + } + for [unroll_full] (b in range(8)) { + var s = acc[b].x + acc[b].y + acc[b].z + acc[b].w + s += simd_shuffle_down(s, 4u) + s += simd_shuffle_down(s, 2u) + s += simd_shuffle_down(s, 1u) + if (tx == 0u && row < ka.ddim && uint(b) < ka.nr) { + y[uint(b) * ka.ys + row] = s + } + } + } +} + // The iq4xs B2/B4 pair: the K4T shell (row/colbase/TILED twins) over the per-32-block lane map of // the GEMV above — lane tx owns block tx of each superblock, 16 lo + 16 hi weights as float4s. [ |> template_struct_instance] diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das b/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das index dd925ff332..7a0b3adba9 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das @@ -4411,7 +4411,7 @@ def private pf_devw_panel(enc : MetalComputeEncoder?; bwblob : MetalBuffer?; wbo //! quant superblocks, plus the k6 2B d tail def private pf_devw_panel_kq(enc : MetalComputeEncoder?; t : Model; fmt : KqFmt; woff : int64; bxh, by, bk, bn : MetalBuffer?; rows, d, kdim : int64; yoff : uint64) : bool { - if (fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s || fmt == KqFmt.iq3xxs || fmt == KqFmt.iq4nl || fmt == KqFmt.k2 || fmt == KqFmt.iq2s || fmt == KqFmt.iq2xs) { // no dev-W dequant kernel for these formats yet + if (fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s || fmt == KqFmt.iq3xxs || fmt == KqFmt.iq4nl || fmt == KqFmt.k2 || fmt == KqFmt.iq2s || fmt == KqFmt.iq2xs || fmt == KqFmt.iq2xxs) { // no dev-W dequant kernel for these formats yet return false } let dq = fmt == KqFmt.k6 ? g_pf_pso_dq_k6 : (fmt == KqFmt.k4 ? g_pf_pso_dq_k4 : g_pf_pso_dq_k5) @@ -4608,6 +4608,10 @@ def private pf_enc_kq_site_mm(enc : MetalComputeEncoder?; t : Model; fmt : KqFmt enc_kq_mm_iq2xs_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, by, yoff, bk, bn, mp, rows) return } + if (fmt == KqFmt.iq2xxs) { // the base mul_mm only - no tensor / tall / dev-W twins yet + enc_kq_mm_iq2xxs_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, by, yoff, bk, bn, mp, rows) + return + } //! deep-class arm: a panel dev-W would split past 8 tiles is DRAM-resident - the tall //! in-kernel-dequant stamp reads the quant plane once per 128-row tile instead of //! materializing and re-streaming a 2B/element f16 panel @@ -4723,6 +4727,8 @@ def private pf_enc_kq_gemv(enc : MetalComputeEncoder?; t : Model; fmt : KqFmt; w enc_kq_iq2s_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, xoff, by, 0ul, bn, bd, rows, n) } elif (fmt == KqFmt.iq2xs) { enc_kq_iq2xs_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, xoff, by, 0ul, bn, bd, rows, n) + } elif (fmt == KqFmt.iq2xxs) { + enc_kq_iq2xxs_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, xoff, by, 0ul, bn, bd, rows, n) } elif (fmt == KqFmt.iq4nl) { enc_kq_iq4nl_c(enc, bs.buf, bs.soff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, xoff, by, 0ul, bn, bd, rows, n) } elif (fmt == KqFmt.k2) { diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_shapes.das b/modules/dasLLAMA/dasllama/dasllama_metal_shapes.das index 2168f0fd78..41435e5fe7 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_shapes.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_shapes.das @@ -180,7 +180,7 @@ def record_needs(var tab : table; missing : MetalNeed) { // formats) must DECLINE kquant_native instead of falling into a wrong-layout kernel branch // (the dispatchers below treat "not k4/k6" as k5) def kq_fmt_gpu_supported(f : KqFmt) : bool { - return f == KqFmt.q8 || f == KqFmt.k4 || f == KqFmt.k5 || f == KqFmt.k6 || f == KqFmt.iq4xs || f == KqFmt.k3 || f == KqFmt.iq3s || f == KqFmt.iq3xxs || f == KqFmt.iq4nl || f == KqFmt.k2 || f == KqFmt.iq2s || f == KqFmt.iq2xs + return f == KqFmt.q8 || f == KqFmt.k4 || f == KqFmt.k5 || f == KqFmt.k6 || f == KqFmt.iq4xs || f == KqFmt.k3 || f == KqFmt.iq3s || f == KqFmt.iq3xxs || f == KqFmt.iq4nl || f == KqFmt.k2 || f == KqFmt.iq2s || f == KqFmt.iq2xs || f == KqFmt.iq2xxs } def private kq_fmts_gpu_supported(a : array) : bool { @@ -243,7 +243,7 @@ def moe_site_ok(fmt : KqFmt; off, ege : int64) : bool { if (fmt == KqFmt.k4 || fmt == KqFmt.k5) { return off >= 0l && (off % 256l) == 0l && (ege % 256l) == 0l } - if (fmt == KqFmt.k6 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s || fmt == KqFmt.iq3xxs || fmt == KqFmt.iq2s || fmt == KqFmt.iq2xs) { // the split scale form: off % 512 keeps the d-plane bind 4B-aligned + if (fmt == KqFmt.k6 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s || fmt == KqFmt.iq3xxs || fmt == KqFmt.iq2s || fmt == KqFmt.iq2xs || fmt == KqFmt.iq2xxs) { // the split scale form: off % 512 keeps the d-plane bind 4B-aligned return off >= 0l && (off % 512l) == 0l && (ege % 256l) == 0l } if (fmt == KqFmt.q51) { // per-32 planes: off % 128 keeps the 20B/4B binds 16B-aligned diff --git a/modules/dasLLAMA/tests/_metal_kernel_common.das b/modules/dasLLAMA/tests/_metal_kernel_common.das index 6423b163d2..b058f6de6a 100644 --- a/modules/dasLLAMA/tests/_metal_kernel_common.das +++ b/modules/dasLLAMA/tests/_metal_kernel_common.das @@ -174,7 +174,7 @@ let KQ_SAFE_SCALES = fixed_array(0.25, 0.5, 1.0, 0.125, 2.0, 0.75, 1.5, 0.375) // [12 packed 6-bit bytes] per superblock, K6 = [16B sub-scales] x nsb + f16 d tail, IQ4_XS (44) = // the same split form with [8 signed sub-scales in -31..31][8 zero] strips, Q3_K (3) = the k6 form over 96B quants def kq_fill_planes(fmt, nsb : int; var kq : array; var ks : array) { // nolint:STYLE037 - the flat per-format fill ladder is the honest shape - let qsb = fmt == 2 || fmt == 24 ? 64 : (fmt == 23 ? 72 : (fmt == 4 || fmt == 44 || fmt == 45 ? 128 : (fmt == 5 ? 160 : (fmt == 3 ? 96 : (fmt == 33 ? 104 : (fmt == 34 ? 96 : 192)))))) + let qsb = fmt == 2 || fmt == 24 || fmt == 25 ? 64 : (fmt == 23 ? 72 : (fmt == 4 || fmt == 44 || fmt == 45 ? 128 : (fmt == 5 ? 160 : (fmt == 3 ? 96 : (fmt == 33 ? 104 : (fmt == 34 ? 96 : 192)))))) kq |> resize(nsb * qsb) for (i in range(nsb * qsb)) { kq[i] = kq_hash_byte(i * 3 + fmt) @@ -201,10 +201,10 @@ def kq_fill_planes(fmt, nsb : int; var kq : array; var ks : array) ks[b * 2] = uint8(dbits & 0xFFu) ks[b * 2 + 1] = uint8(dbits >> 8u) } - } elif (fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 23 || fmt == 24) { + } elif (fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 23 || fmt == 24 || fmt == 25) { ks |> resize(nsb * 18) for (i in range(nsb * 16)) { - ks[i] = fmt == 6 || fmt == 3 || fmt == 23 || fmt == 24 ? kq_hash_byte(i * 7 + 13) : uint8(i % 16 < 8 ? int(kq_hash_byte(i * 7 + 13)) % 63 - 31 : 0) // 44/33: signed strip bytes + zero pad; 23: 16 unsigned strips + ks[i] = fmt == 6 || fmt == 3 || fmt == 23 || fmt == 24 || fmt == 25 ? kq_hash_byte(i * 7 + 13) : uint8(i % 16 < 8 ? int(kq_hash_byte(i * 7 + 13)) % 63 - 31 : 0) // 44/33: signed strip bytes + zero pad; 23: 16 unsigned strips } for (b in range(nsb)) { let dbits = f32_to_f16(KQ_SAFE_SCALES[b % 8]) @@ -290,6 +290,9 @@ def kq_row_ref(fmt, sb_base, nb, nsb_total : int; kq, ks : array; var wro } elif (fmt == 24) { dequant_iq2xs_plane_superblock_at(kq, int64(sb * 64), ks, int64(sb * 16), int64(nsb_total * 16 + sb * 2), wrow, int64(ib * 256)) + } elif (fmt == 25) { + dequant_iq2xxs_plane_superblock_at(kq, int64(sb * 64), ks, int64(sb * 16), + int64(nsb_total * 16 + sb * 2), wrow, int64(ib * 256)) } else { dequant_k6_plane_superblock_at(kq, int64(sb * 192), ks, int64(sb * 16), int64(nsb_total * 16 + sb * 2), wrow, int64(ib * 256)) diff --git a/modules/dasLLAMA/tests/test_metal_gemm_kernels.das b/modules/dasLLAMA/tests/test_metal_gemm_kernels.das index 6fdb010432..d9d3a486ec 100644 --- a/modules/dasLLAMA/tests/test_metal_gemm_kernels.das +++ b/modules/dasLLAMA/tests/test_metal_gemm_kernels.das @@ -36,28 +36,28 @@ def private kq_mulmm_gate(t : T?; dev, queue; fmt : int; tensor : bool; m, kdim, : halfx ? (fmt == 4 ? MetalKqMulMmK4TH_metal_kq_mulmm_t_msl : (fmt == 5 ? MetalKqMulMmK5TH_metal_kq_mulmm_t_msl : MetalKqMulMmK6TH_metal_kq_mulmm_k6_t_msl)) : (fmt == 4 ? MetalKqMulMmK4T_metal_kq_mulmm_t_msl : (fmt == 5 ? MetalKqMulMmK5T_metal_kq_mulmm_t_msl : MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl))) - : (fmt == 24 ? MetalKqMulMmIq2xs_metal_kq_mulmm_msl : (fmt == 23 ? MetalKqMulMmIq2s_metal_kq_mulmm_msl : (fmt == 45 ? MetalKqMulMmIq4nl_metal_kq_mulmm_msl : (fmt == 2 ? MetalKqMulMmK2_metal_kq_mulmm_msl : (fmt == 34 ? MetalKqMulMmIq3xxs_metal_kq_mulmm_msl : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl : MetalKqMulMmK6_metal_kq_mulmm_msl))))))))))) + : (fmt == 25 ? MetalKqMulMmIq2xxs_metal_kq_mulmm_msl : (fmt == 24 ? MetalKqMulMmIq2xs_metal_kq_mulmm_msl : (fmt == 23 ? MetalKqMulMmIq2s_metal_kq_mulmm_msl : (fmt == 45 ? MetalKqMulMmIq4nl_metal_kq_mulmm_msl : (fmt == 2 ? MetalKqMulMmK2_metal_kq_mulmm_msl : (fmt == 34 ? MetalKqMulMmIq3xxs_metal_kq_mulmm_msl : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl : MetalKqMulMmK6_metal_kq_mulmm_msl)))))))))))) let entry = (tensor ? (tall ? (fmt == 4 ? MetalKqMulMmK4TH128_metal_kq_mulmm_t_msl_entry : (fmt == 5 ? MetalKqMulMmK5TH128_metal_kq_mulmm_t_msl_entry : MetalKqMulMmK6TH128_metal_kq_mulmm_k6_t_msl_entry)) : halfx ? (fmt == 4 ? MetalKqMulMmK4TH_metal_kq_mulmm_t_msl_entry : (fmt == 5 ? MetalKqMulMmK5TH_metal_kq_mulmm_t_msl_entry : MetalKqMulMmK6TH_metal_kq_mulmm_k6_t_msl_entry)) : (fmt == 4 ? MetalKqMulMmK4T_metal_kq_mulmm_t_msl_entry : (fmt == 5 ? MetalKqMulMmK5T_metal_kq_mulmm_t_msl_entry : MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl_entry))) - : (fmt == 24 ? MetalKqMulMmIq2xs_metal_kq_mulmm_msl_entry : (fmt == 23 ? MetalKqMulMmIq2s_metal_kq_mulmm_msl_entry : (fmt == 45 ? MetalKqMulMmIq4nl_metal_kq_mulmm_msl_entry : (fmt == 2 ? MetalKqMulMmK2_metal_kq_mulmm_msl_entry : (fmt == 34 ? MetalKqMulMmIq3xxs_metal_kq_mulmm_msl_entry : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl_entry : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl_entry : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl_entry : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_entry : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_entry : MetalKqMulMmK6_metal_kq_mulmm_msl_entry))))))))))) + : (fmt == 25 ? MetalKqMulMmIq2xxs_metal_kq_mulmm_msl_entry : (fmt == 24 ? MetalKqMulMmIq2xs_metal_kq_mulmm_msl_entry : (fmt == 23 ? MetalKqMulMmIq2s_metal_kq_mulmm_msl_entry : (fmt == 45 ? MetalKqMulMmIq4nl_metal_kq_mulmm_msl_entry : (fmt == 2 ? MetalKqMulMmK2_metal_kq_mulmm_msl_entry : (fmt == 34 ? MetalKqMulMmIq3xxs_metal_kq_mulmm_msl_entry : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl_entry : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl_entry : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl_entry : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_entry : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_entry : MetalKqMulMmK6_metal_kq_mulmm_msl_entry)))))))))))) let fm = (tensor ? (tall ? (fmt == 4 ? MetalKqMulMmK4TH128_metal_kq_mulmm_t_msl_fastmath : (fmt == 5 ? MetalKqMulMmK5TH128_metal_kq_mulmm_t_msl_fastmath : MetalKqMulMmK6TH128_metal_kq_mulmm_k6_t_msl_fastmath)) : halfx ? (fmt == 4 ? MetalKqMulMmK4TH_metal_kq_mulmm_t_msl_fastmath : (fmt == 5 ? MetalKqMulMmK5TH_metal_kq_mulmm_t_msl_fastmath : MetalKqMulMmK6TH_metal_kq_mulmm_k6_t_msl_fastmath)) : (fmt == 4 ? MetalKqMulMmK4T_metal_kq_mulmm_t_msl_fastmath : (fmt == 5 ? MetalKqMulMmK5T_metal_kq_mulmm_t_msl_fastmath : MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl_fastmath))) - : (fmt == 24 ? MetalKqMulMmIq2xs_metal_kq_mulmm_msl_fastmath : (fmt == 23 ? MetalKqMulMmIq2s_metal_kq_mulmm_msl_fastmath : (fmt == 45 ? MetalKqMulMmIq4nl_metal_kq_mulmm_msl_fastmath : (fmt == 2 ? MetalKqMulMmK2_metal_kq_mulmm_msl_fastmath : (fmt == 34 ? MetalKqMulMmIq3xxs_metal_kq_mulmm_msl_fastmath : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl_fastmath : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl_fastmath : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl_fastmath : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_fastmath : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_fastmath : MetalKqMulMmK6_metal_kq_mulmm_msl_fastmath))))))))))) + : (fmt == 25 ? MetalKqMulMmIq2xxs_metal_kq_mulmm_msl_fastmath : (fmt == 24 ? MetalKqMulMmIq2xs_metal_kq_mulmm_msl_fastmath : (fmt == 23 ? MetalKqMulMmIq2s_metal_kq_mulmm_msl_fastmath : (fmt == 45 ? MetalKqMulMmIq4nl_metal_kq_mulmm_msl_fastmath : (fmt == 2 ? MetalKqMulMmK2_metal_kq_mulmm_msl_fastmath : (fmt == 34 ? MetalKqMulMmIq3xxs_metal_kq_mulmm_msl_fastmath : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl_fastmath : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl_fastmath : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl_fastmath : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_fastmath : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_fastmath : MetalKqMulMmK6_metal_kq_mulmm_msl_fastmath)))))))))))) let tgm = (tensor ? (tall ? (fmt == 4 ? MetalKqMulMmK4TH128_metal_kq_mulmm_t_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5TH128_metal_kq_mulmm_t_msl_tgmem : MetalKqMulMmK6TH128_metal_kq_mulmm_k6_t_msl_tgmem)) : halfx ? (fmt == 4 ? MetalKqMulMmK4TH_metal_kq_mulmm_t_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5TH_metal_kq_mulmm_t_msl_tgmem : MetalKqMulMmK6TH_metal_kq_mulmm_k6_t_msl_tgmem)) : (fmt == 4 ? MetalKqMulMmK4T_metal_kq_mulmm_t_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5T_metal_kq_mulmm_t_msl_tgmem : MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl_tgmem))) - : (fmt == 24 ? MetalKqMulMmIq2xs_metal_kq_mulmm_msl_tgmem : (fmt == 23 ? MetalKqMulMmIq2s_metal_kq_mulmm_msl_tgmem : (fmt == 45 ? MetalKqMulMmIq4nl_metal_kq_mulmm_msl_tgmem : (fmt == 2 ? MetalKqMulMmK2_metal_kq_mulmm_msl_tgmem : (fmt == 34 ? MetalKqMulMmIq3xxs_metal_kq_mulmm_msl_tgmem : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl_tgmem : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl_tgmem : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl_tgmem : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_tgmem : MetalKqMulMmK6_metal_kq_mulmm_msl_tgmem))))))))))) + : (fmt == 25 ? MetalKqMulMmIq2xxs_metal_kq_mulmm_msl_tgmem : (fmt == 24 ? MetalKqMulMmIq2xs_metal_kq_mulmm_msl_tgmem : (fmt == 23 ? MetalKqMulMmIq2s_metal_kq_mulmm_msl_tgmem : (fmt == 45 ? MetalKqMulMmIq4nl_metal_kq_mulmm_msl_tgmem : (fmt == 2 ? MetalKqMulMmK2_metal_kq_mulmm_msl_tgmem : (fmt == 34 ? MetalKqMulMmIq3xxs_metal_kq_mulmm_msl_tgmem : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl_tgmem : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl_tgmem : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl_tgmem : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_tgmem : MetalKqMulMmK6_metal_kq_mulmm_msl_tgmem)))))))))))) var pso = pipeline_from_source(dev, src, entry, fm, err) if (tensor && pso == null && !metal4_tensor_available(dev)) { to_log(LOG_INFO, "{tag}: no Metal-4 tensor toolchain on this box - arm skipped\n") @@ -123,7 +123,7 @@ def private kq_mulmm_gate(t : T?; dev, queue; fmt : int; tensor : bool; m, kdim, let ran = with_compute_encoder(queue, err) $(enc : MetalComputeEncoder?) { metal_set_pipeline(enc, pso) metal_set_threadgroup_memory_length(enc, tgm, 0) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 || fmt == 24 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs: the f16 d tail + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 || fmt == 24 || fmt == 25 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs: the f16 d tail metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bx, 0ul, 3) @@ -134,7 +134,7 @@ def private kq_mulmm_gate(t : T?; dev, queue; fmt : int; tensor : bool; m, kdim, if (rem_pso != null) { metal_set_pipeline(enc, rem_pso) metal_set_threadgroup_memory_length(enc, fmt == 4 ? MetalKqMulMmK4TH_metal_kq_mulmm_t_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5TH_metal_kq_mulmm_t_msl_tgmem : MetalKqMulMmK6TH_metal_kq_mulmm_k6_t_msl_tgmem), 0) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 || fmt == 24 ? uint64(nsb * 16) : 0ul, 0) + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 || fmt == 24 || fmt == 25 ? uint64(nsb * 16) : 0ul, 0) metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bx, uint64(r0 * kdim * 2), 3) @@ -484,7 +484,7 @@ def private kq_devw_gate(t : T?; dev, queue; fmt : int; m, kdim, ndim : int) { var dka = CvtArgs(total = uint(nblk)) let ran = with_compute_encoder(queue, err) $(enc : MetalComputeEncoder?) { metal_set_pipeline(enc, dq_pso) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 || fmt == 24 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs: the f16 d tail + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 || fmt == 24 || fmt == 25 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs: the f16 d tail metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bwh, 0ul, 3) @@ -1461,7 +1461,7 @@ def private moe_mulmm_kq_gate(t : T?; dev, queue; fmt : int; kdim, ndim : int; c let ran = with_compute_encoder(queue, err) $(enc : MetalComputeEncoder?) { metal_set_pipeline(enc, pso) metal_set_threadgroup_memory_length(enc, tgm, 0) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 || fmt == 24 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs: the f16 d tail + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 || fmt == 24 || fmt == 25 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs: the f16 d tail metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bx, 0ul, 3) @@ -1498,7 +1498,7 @@ def private moe_mulmm_kq_gate(t : T?; dev, queue; fmt : int; kdim, ndim : int; c metal_set_threadgroup_memory_length(enc, fmt == 4 ? MetalMoeMulMmK4TH_metal_moe_mulmm_k45_t_msl_tgmem : (fmt == 5 ? MetalMoeMulMmK5TH_metal_moe_mulmm_k45_t_msl_tgmem : MetalMoeMulMmK6TH_metal_moe_mulmm_k6_t_msl_tgmem), 0) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 || fmt == 24 ? uint64(nsb * 16) : 0ul, 0) + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 || fmt == 24 || fmt == 25 ? uint64(nsb * 16) : 0ul, 0) metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bxh, 0ul, 3) @@ -1538,7 +1538,7 @@ def private moe_mulmm_kq_gate(t : T?; dev, queue; fmt : int; kdim, ndim : int; c metal_set_threadgroup_memory_length(enc, fmt == 4 ? MetalMoeMulMmK4TH128_metal_moe_mulmm_k45_t_msl_tgmem : (fmt == 5 ? MetalMoeMulMmK5TH128_metal_moe_mulmm_k45_t_msl_tgmem : MetalMoeMulMmK6TH128_metal_moe_mulmm_k6_t_msl_tgmem), 0) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 || fmt == 24 ? uint64(nsb * 16) : 0ul, 0) + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 || fmt == 24 || fmt == 25 ? uint64(nsb * 16) : 0ul, 0) metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bxh, 0ul, 3) @@ -1551,7 +1551,7 @@ def private moe_mulmm_kq_gate(t : T?; dev, queue; fmt : int; kdim, ndim : int; c metal_set_threadgroup_memory_length(enc, fmt == 4 ? MetalMoeMulMmK4THR_metal_moe_mulmm_k45_t_msl_tgmem : (fmt == 5 ? MetalMoeMulMmK5THR_metal_moe_mulmm_k45_t_msl_tgmem : MetalMoeMulMmK6THR_metal_moe_mulmm_k6_t_msl_tgmem), 0) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 || fmt == 24 ? uint64(nsb * 16) : 0ul, 0) + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 || fmt == 24 || fmt == 25 ? uint64(nsb * 16) : 0ul, 0) metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bxh, 0ul, 3) @@ -2063,6 +2063,8 @@ def test_metal_gemm_kernels(t : T?) { kq_mulmm_gate(t, dev, queue, 23, false, 32, 256, 128) kq_mulmm_gate(t, dev, queue, 24, false, 64, 512, 64) // iq2xs: the base form only (no tensor / tall twins) kq_mulmm_gate(t, dev, queue, 24, false, 32, 256, 128) + kq_mulmm_gate(t, dev, queue, 25, false, 64, 512, 64) // iq2xxs: the base form only (no tensor / tall twins) + kq_mulmm_gate(t, dev, queue, 25, false, 32, 256, 128) q8_mulmm_t_gate(t, dev, queue, 64, 256, 64) q8_mulmm_t_gate(t, dev, queue, 64, 256, 64, halfx = true) q8_mulmm_t_gate(t, dev, queue, 64, 160, 64) // kdim % 64 == 32: the bk=64 helper's 32-tail chunk diff --git a/modules/dasLLAMA/tests/test_metal_gemv_kernels.das b/modules/dasLLAMA/tests/test_metal_gemv_kernels.das index c0431afb07..f1441069bd 100644 --- a/modules/dasLLAMA/tests/test_metal_gemv_kernels.das +++ b/modules/dasLLAMA/tests/test_metal_gemv_kernels.das @@ -23,12 +23,12 @@ require math // ===== single-stream kq GEMV (MetalKqGemvK4 / K5 / K5C / K6) ===== def private kq_gemv_gate(t : T?; dev, queue; vform : string; n, d : int) { - let fmt = vform == "k4" ? 4 : (vform == "k6" ? 6 : (vform == "iq4xs" ? 44 : (vform == "k3" ? 3 : (vform == "iq3s" ? 33 : (vform == "iq3xxs" ? 34 : (vform == "iq4nl" ? 45 : (vform == "iq2s" ? 23 : (vform == "iq2xs" ? 24 : (vform == "k2" ? 2 : 5))))))))) + let fmt = vform == "k4" ? 4 : (vform == "k6" ? 6 : (vform == "iq4xs" ? 44 : (vform == "k3" ? 3 : (vform == "iq3s" ? 33 : (vform == "iq3xxs" ? 34 : (vform == "iq4nl" ? 45 : (vform == "iq2s" ? 23 : (vform == "iq2xs" ? 24 : (vform == "iq2xxs" ? 25 : (vform == "k2" ? 2 : 5)))))))))) let tag = "kq_gemv_{vform} n={n} d={d}" var err : string - let src = vform == "iq2xs" ? metal_kq_gemv_iq2xs_msl : (vform == "iq2s" ? metal_kq_gemv_iq2s_msl : (vform == "k2" ? metal_kq_gemv_k2_msl : (vform == "iq4nl" ? metal_kq_gemv_iq4nl_msl : (vform == "iq3xxs" ? metal_kq_gemv_iq3xxs_msl : (vform == "iq3s" ? metal_kq_gemv_iq3s_msl : (vform == "k3" ? metal_kq_gemv_k3_msl : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl : (vform == "k4" ? metal_kq_gemv_k4_msl : (vform == "k5" ? metal_kq_gemv_k5_msl : (vform == "k5c" ? metal_kq_gemv_k5c_msl : metal_kq_gemv_k6_msl)))))))))) - let entry = vform == "iq2xs" ? metal_kq_gemv_iq2xs_msl_entry : (vform == "iq2s" ? metal_kq_gemv_iq2s_msl_entry : (vform == "k2" ? metal_kq_gemv_k2_msl_entry : (vform == "iq4nl" ? metal_kq_gemv_iq4nl_msl_entry : (vform == "iq3xxs" ? metal_kq_gemv_iq3xxs_msl_entry : (vform == "iq3s" ? metal_kq_gemv_iq3s_msl_entry : (vform == "k3" ? metal_kq_gemv_k3_msl_entry : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl_entry : (vform == "k4" ? metal_kq_gemv_k4_msl_entry : (vform == "k5" ? metal_kq_gemv_k5_msl_entry : (vform == "k5c" ? metal_kq_gemv_k5c_msl_entry : metal_kq_gemv_k6_msl_entry)))))))))) - let fm = vform == "iq2xs" ? metal_kq_gemv_iq2xs_msl_fastmath : (vform == "iq2s" ? metal_kq_gemv_iq2s_msl_fastmath : (vform == "k2" ? metal_kq_gemv_k2_msl_fastmath : (vform == "iq4nl" ? metal_kq_gemv_iq4nl_msl_fastmath : (vform == "iq3xxs" ? metal_kq_gemv_iq3xxs_msl_fastmath : (vform == "iq3s" ? metal_kq_gemv_iq3s_msl_fastmath : (vform == "k3" ? metal_kq_gemv_k3_msl_fastmath : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl_fastmath : (vform == "k4" ? metal_kq_gemv_k4_msl_fastmath : (vform == "k5" ? metal_kq_gemv_k5_msl_fastmath : (vform == "k5c" ? metal_kq_gemv_k5c_msl_fastmath : metal_kq_gemv_k6_msl_fastmath)))))))))) + let src = vform == "iq2xxs" ? metal_kq_gemv_iq2xxs_msl : (vform == "iq2xs" ? metal_kq_gemv_iq2xs_msl : (vform == "iq2s" ? metal_kq_gemv_iq2s_msl : (vform == "k2" ? metal_kq_gemv_k2_msl : (vform == "iq4nl" ? metal_kq_gemv_iq4nl_msl : (vform == "iq3xxs" ? metal_kq_gemv_iq3xxs_msl : (vform == "iq3s" ? metal_kq_gemv_iq3s_msl : (vform == "k3" ? metal_kq_gemv_k3_msl : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl : (vform == "k4" ? metal_kq_gemv_k4_msl : (vform == "k5" ? metal_kq_gemv_k5_msl : (vform == "k5c" ? metal_kq_gemv_k5c_msl : metal_kq_gemv_k6_msl))))))))))) + let entry = vform == "iq2xxs" ? metal_kq_gemv_iq2xxs_msl_entry : (vform == "iq2xs" ? metal_kq_gemv_iq2xs_msl_entry : (vform == "iq2s" ? metal_kq_gemv_iq2s_msl_entry : (vform == "k2" ? metal_kq_gemv_k2_msl_entry : (vform == "iq4nl" ? metal_kq_gemv_iq4nl_msl_entry : (vform == "iq3xxs" ? metal_kq_gemv_iq3xxs_msl_entry : (vform == "iq3s" ? metal_kq_gemv_iq3s_msl_entry : (vform == "k3" ? metal_kq_gemv_k3_msl_entry : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl_entry : (vform == "k4" ? metal_kq_gemv_k4_msl_entry : (vform == "k5" ? metal_kq_gemv_k5_msl_entry : (vform == "k5c" ? metal_kq_gemv_k5c_msl_entry : metal_kq_gemv_k6_msl_entry))))))))))) + let fm = vform == "iq2xxs" ? metal_kq_gemv_iq2xxs_msl_fastmath : (vform == "iq2xs" ? metal_kq_gemv_iq2xs_msl_fastmath : (vform == "iq2s" ? metal_kq_gemv_iq2s_msl_fastmath : (vform == "k2" ? metal_kq_gemv_k2_msl_fastmath : (vform == "iq4nl" ? metal_kq_gemv_iq4nl_msl_fastmath : (vform == "iq3xxs" ? metal_kq_gemv_iq3xxs_msl_fastmath : (vform == "iq3s" ? metal_kq_gemv_iq3s_msl_fastmath : (vform == "k3" ? metal_kq_gemv_k3_msl_fastmath : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl_fastmath : (vform == "k4" ? metal_kq_gemv_k4_msl_fastmath : (vform == "k5" ? metal_kq_gemv_k5_msl_fastmath : (vform == "k5c" ? metal_kq_gemv_k5c_msl_fastmath : metal_kq_gemv_k6_msl_fastmath))))))))))) var pso = pipeline_from_source(dev, src, entry, fm, err) t |> success(pso != null, "{tag}: pipeline: {err}") return if (pso == null) @@ -66,7 +66,7 @@ def private kq_gemv_gate(t : T?; dev, queue; vform : string; n, d : int) { var by = buf_fill(dev, d, -1000.0) var bn = buf_u32(dev, uint(n)) var bd = buf_u32(dev, uint(d)) - let groups = vform == "k5c" ? (d + 1) / 2 : (vform == "iq3s" || vform == "iq3xxs" || vform == "iq2s" || vform == "iq2xs" ? (d + 7) / 8 : (d + 3) / 4) + let groups = vform == "k5c" ? (d + 1) / 2 : (vform == "iq3s" || vform == "iq3xxs" || vform == "iq2s" || vform == "iq2xs" || vform == "iq2xxs" ? (d + 7) / 8 : (d + 3) / 4) let ran = with_compute_encoder(queue, err) $(enc : MetalComputeEncoder?) { metal_set_pipeline(enc, pso) if (vform == "iq4xs") { @@ -81,7 +81,7 @@ def private kq_gemv_gate(t : T?; dev, queue; vform : string; n, d : int) { if (vform == "iq4nl") { metal_set_threadgroup_memory_length(enc, metal_kq_gemv_iq4nl_msl_tgmem, 0) // the codebook slab } - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 || fmt == 24 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs/k3: the f16 d tail + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 || fmt == 24 || fmt == 25 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs/k3: the f16 d tail metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bx, 0ul, 3) @@ -114,15 +114,15 @@ def private kq_gemv_gate(t : T?; dev, queue; vform : string; n, d : int) { def private kq_mvb_gate(t : T?; dev, queue; fmt, bwidth, n, d, nr, ys : int) { let tag = "kq_mvb{bwidth}_k{fmt} n={n} d={d} nr={nr} ys={ys}" var err : string - let src = (bwidth == 2 ? (fmt == 24 ? MetalKqMvB2Iq2xs_metal_kq_mv_iq2xs_msl : (fmt == 23 ? MetalKqMvB2Iq2s_metal_kq_mv_iq2s_msl : (fmt == 45 ? MetalKqMvB2Iq4nl_metal_kq_mv_iq4nl_msl : (fmt == 2 ? MetalKqMvB2K2_metal_kq_mv_k2_msl : (fmt == 34 ? MetalKqMvB2Iq3xxs_metal_kq_mv_iq3xxs_msl : (fmt == 33 ? MetalKqMvB2Iq3s_metal_kq_mv_iq3s_msl : (fmt == 3 ? MetalKqMvB2K3_metal_kq_mv_k3_msl : (fmt == 44 ? MetalKqMvB2Iq4xs_metal_kq_mv_iq4xs_msl : (fmt == 4 ? MetalKqMvB2K4_metal_kq_mv_k4_msl : (fmt == 5 ? MetalKqMvB2K5_metal_kq_mv_k5_msl : MetalKqMvB2K6_metal_kq_mv_k6_msl)))))))))) - : (bwidth == 4 ? (fmt == 24 ? MetalKqMvB4Iq2xs_metal_kq_mv_iq2xs_msl : (fmt == 23 ? MetalKqMvB4Iq2s_metal_kq_mv_iq2s_msl : (fmt == 45 ? MetalKqMvB4Iq4nl_metal_kq_mv_iq4nl_msl : (fmt == 2 ? MetalKqMvB4K2_metal_kq_mv_k2_msl : (fmt == 34 ? MetalKqMvB4Iq3xxs_metal_kq_mv_iq3xxs_msl : (fmt == 33 ? MetalKqMvB4Iq3s_metal_kq_mv_iq3s_msl : (fmt == 3 ? MetalKqMvB4K3_metal_kq_mv_k3_msl : (fmt == 44 ? MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl : (fmt == 4 ? MetalKqMvB4K4_metal_kq_mv_k4_msl : (fmt == 5 ? MetalKqMvB4K5_metal_kq_mv_k5_msl : MetalKqMvB4K6_metal_kq_mv_k6_msl)))))))))) - : (fmt == 24 ? metal_kq_mvb8_iq2xs_msl : (fmt == 23 ? metal_kq_mvb8_iq2s_msl : (fmt == 45 ? metal_kq_mvb8_iq4nl_msl : (fmt == 2 ? metal_kq_mvb8_k2_msl : (fmt == 34 ? metal_kq_mvb8_iq3xxs_msl : (fmt == 33 ? metal_kq_mvb8_iq3s_msl : (fmt == 3 ? metal_kq_mvb8_k3_msl : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl : metal_kq_mvb8_k6_msl)))))))))))) - let entry = (bwidth == 2 ? (fmt == 24 ? MetalKqMvB2Iq2xs_metal_kq_mv_iq2xs_msl_entry : (fmt == 23 ? MetalKqMvB2Iq2s_metal_kq_mv_iq2s_msl_entry : (fmt == 45 ? MetalKqMvB2Iq4nl_metal_kq_mv_iq4nl_msl_entry : (fmt == 2 ? MetalKqMvB2K2_metal_kq_mv_k2_msl_entry : (fmt == 34 ? MetalKqMvB2Iq3xxs_metal_kq_mv_iq3xxs_msl_entry : (fmt == 33 ? MetalKqMvB2Iq3s_metal_kq_mv_iq3s_msl_entry : (fmt == 3 ? MetalKqMvB2K3_metal_kq_mv_k3_msl_entry : (fmt == 44 ? MetalKqMvB2Iq4xs_metal_kq_mv_iq4xs_msl_entry : (fmt == 4 ? MetalKqMvB2K4_metal_kq_mv_k4_msl_entry : (fmt == 5 ? MetalKqMvB2K5_metal_kq_mv_k5_msl_entry : MetalKqMvB2K6_metal_kq_mv_k6_msl_entry)))))))))) - : (bwidth == 4 ? (fmt == 24 ? MetalKqMvB4Iq2xs_metal_kq_mv_iq2xs_msl_entry : (fmt == 23 ? MetalKqMvB4Iq2s_metal_kq_mv_iq2s_msl_entry : (fmt == 45 ? MetalKqMvB4Iq4nl_metal_kq_mv_iq4nl_msl_entry : (fmt == 2 ? MetalKqMvB4K2_metal_kq_mv_k2_msl_entry : (fmt == 34 ? MetalKqMvB4Iq3xxs_metal_kq_mv_iq3xxs_msl_entry : (fmt == 33 ? MetalKqMvB4Iq3s_metal_kq_mv_iq3s_msl_entry : (fmt == 3 ? MetalKqMvB4K3_metal_kq_mv_k3_msl_entry : (fmt == 44 ? MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl_entry : (fmt == 4 ? MetalKqMvB4K4_metal_kq_mv_k4_msl_entry : (fmt == 5 ? MetalKqMvB4K5_metal_kq_mv_k5_msl_entry : MetalKqMvB4K6_metal_kq_mv_k6_msl_entry)))))))))) - : (fmt == 24 ? metal_kq_mvb8_iq2xs_msl_entry : (fmt == 23 ? metal_kq_mvb8_iq2s_msl_entry : (fmt == 45 ? metal_kq_mvb8_iq4nl_msl_entry : (fmt == 2 ? metal_kq_mvb8_k2_msl_entry : (fmt == 34 ? metal_kq_mvb8_iq3xxs_msl_entry : (fmt == 33 ? metal_kq_mvb8_iq3s_msl_entry : (fmt == 3 ? metal_kq_mvb8_k3_msl_entry : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl_entry : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_entry : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_entry : metal_kq_mvb8_k6_msl_entry)))))))))))) - let fm = (bwidth == 2 ? (fmt == 24 ? MetalKqMvB2Iq2xs_metal_kq_mv_iq2xs_msl_fastmath : (fmt == 23 ? MetalKqMvB2Iq2s_metal_kq_mv_iq2s_msl_fastmath : (fmt == 45 ? MetalKqMvB2Iq4nl_metal_kq_mv_iq4nl_msl_fastmath : (fmt == 2 ? MetalKqMvB2K2_metal_kq_mv_k2_msl_fastmath : (fmt == 34 ? MetalKqMvB2Iq3xxs_metal_kq_mv_iq3xxs_msl_fastmath : (fmt == 33 ? MetalKqMvB2Iq3s_metal_kq_mv_iq3s_msl_fastmath : (fmt == 3 ? MetalKqMvB2K3_metal_kq_mv_k3_msl_fastmath : (fmt == 44 ? MetalKqMvB2Iq4xs_metal_kq_mv_iq4xs_msl_fastmath : (fmt == 4 ? MetalKqMvB2K4_metal_kq_mv_k4_msl_fastmath : (fmt == 5 ? MetalKqMvB2K5_metal_kq_mv_k5_msl_fastmath : MetalKqMvB2K6_metal_kq_mv_k6_msl_fastmath)))))))))) - : (bwidth == 4 ? (fmt == 24 ? MetalKqMvB4Iq2xs_metal_kq_mv_iq2xs_msl_fastmath : (fmt == 23 ? MetalKqMvB4Iq2s_metal_kq_mv_iq2s_msl_fastmath : (fmt == 45 ? MetalKqMvB4Iq4nl_metal_kq_mv_iq4nl_msl_fastmath : (fmt == 2 ? MetalKqMvB4K2_metal_kq_mv_k2_msl_fastmath : (fmt == 34 ? MetalKqMvB4Iq3xxs_metal_kq_mv_iq3xxs_msl_fastmath : (fmt == 33 ? MetalKqMvB4Iq3s_metal_kq_mv_iq3s_msl_fastmath : (fmt == 3 ? MetalKqMvB4K3_metal_kq_mv_k3_msl_fastmath : (fmt == 44 ? MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl_fastmath : (fmt == 4 ? MetalKqMvB4K4_metal_kq_mv_k4_msl_fastmath : (fmt == 5 ? MetalKqMvB4K5_metal_kq_mv_k5_msl_fastmath : MetalKqMvB4K6_metal_kq_mv_k6_msl_fastmath)))))))))) - : (fmt == 24 ? metal_kq_mvb8_iq2xs_msl_fastmath : (fmt == 23 ? metal_kq_mvb8_iq2s_msl_fastmath : (fmt == 45 ? metal_kq_mvb8_iq4nl_msl_fastmath : (fmt == 2 ? metal_kq_mvb8_k2_msl_fastmath : (fmt == 34 ? metal_kq_mvb8_iq3xxs_msl_fastmath : (fmt == 33 ? metal_kq_mvb8_iq3s_msl_fastmath : (fmt == 3 ? metal_kq_mvb8_k3_msl_fastmath : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl_fastmath : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_fastmath : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_fastmath : metal_kq_mvb8_k6_msl_fastmath)))))))))))) + let src = (bwidth == 2 ? (fmt == 25 ? MetalKqMvB2Iq2xxs_metal_kq_mv_iq2xxs_msl : (fmt == 24 ? MetalKqMvB2Iq2xs_metal_kq_mv_iq2xs_msl : (fmt == 23 ? MetalKqMvB2Iq2s_metal_kq_mv_iq2s_msl : (fmt == 45 ? MetalKqMvB2Iq4nl_metal_kq_mv_iq4nl_msl : (fmt == 2 ? MetalKqMvB2K2_metal_kq_mv_k2_msl : (fmt == 34 ? MetalKqMvB2Iq3xxs_metal_kq_mv_iq3xxs_msl : (fmt == 33 ? MetalKqMvB2Iq3s_metal_kq_mv_iq3s_msl : (fmt == 3 ? MetalKqMvB2K3_metal_kq_mv_k3_msl : (fmt == 44 ? MetalKqMvB2Iq4xs_metal_kq_mv_iq4xs_msl : (fmt == 4 ? MetalKqMvB2K4_metal_kq_mv_k4_msl : (fmt == 5 ? MetalKqMvB2K5_metal_kq_mv_k5_msl : MetalKqMvB2K6_metal_kq_mv_k6_msl))))))))))) + : (bwidth == 4 ? (fmt == 25 ? MetalKqMvB4Iq2xxs_metal_kq_mv_iq2xxs_msl : (fmt == 24 ? MetalKqMvB4Iq2xs_metal_kq_mv_iq2xs_msl : (fmt == 23 ? MetalKqMvB4Iq2s_metal_kq_mv_iq2s_msl : (fmt == 45 ? MetalKqMvB4Iq4nl_metal_kq_mv_iq4nl_msl : (fmt == 2 ? MetalKqMvB4K2_metal_kq_mv_k2_msl : (fmt == 34 ? MetalKqMvB4Iq3xxs_metal_kq_mv_iq3xxs_msl : (fmt == 33 ? MetalKqMvB4Iq3s_metal_kq_mv_iq3s_msl : (fmt == 3 ? MetalKqMvB4K3_metal_kq_mv_k3_msl : (fmt == 44 ? MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl : (fmt == 4 ? MetalKqMvB4K4_metal_kq_mv_k4_msl : (fmt == 5 ? MetalKqMvB4K5_metal_kq_mv_k5_msl : MetalKqMvB4K6_metal_kq_mv_k6_msl))))))))))) + : (fmt == 25 ? metal_kq_mvb8_iq2xxs_msl : (fmt == 24 ? metal_kq_mvb8_iq2xs_msl : (fmt == 23 ? metal_kq_mvb8_iq2s_msl : (fmt == 45 ? metal_kq_mvb8_iq4nl_msl : (fmt == 2 ? metal_kq_mvb8_k2_msl : (fmt == 34 ? metal_kq_mvb8_iq3xxs_msl : (fmt == 33 ? metal_kq_mvb8_iq3s_msl : (fmt == 3 ? metal_kq_mvb8_k3_msl : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl : metal_kq_mvb8_k6_msl))))))))))))) + let entry = (bwidth == 2 ? (fmt == 25 ? MetalKqMvB2Iq2xxs_metal_kq_mv_iq2xxs_msl_entry : (fmt == 24 ? MetalKqMvB2Iq2xs_metal_kq_mv_iq2xs_msl_entry : (fmt == 23 ? MetalKqMvB2Iq2s_metal_kq_mv_iq2s_msl_entry : (fmt == 45 ? MetalKqMvB2Iq4nl_metal_kq_mv_iq4nl_msl_entry : (fmt == 2 ? MetalKqMvB2K2_metal_kq_mv_k2_msl_entry : (fmt == 34 ? MetalKqMvB2Iq3xxs_metal_kq_mv_iq3xxs_msl_entry : (fmt == 33 ? MetalKqMvB2Iq3s_metal_kq_mv_iq3s_msl_entry : (fmt == 3 ? MetalKqMvB2K3_metal_kq_mv_k3_msl_entry : (fmt == 44 ? MetalKqMvB2Iq4xs_metal_kq_mv_iq4xs_msl_entry : (fmt == 4 ? MetalKqMvB2K4_metal_kq_mv_k4_msl_entry : (fmt == 5 ? MetalKqMvB2K5_metal_kq_mv_k5_msl_entry : MetalKqMvB2K6_metal_kq_mv_k6_msl_entry))))))))))) + : (bwidth == 4 ? (fmt == 25 ? MetalKqMvB4Iq2xxs_metal_kq_mv_iq2xxs_msl_entry : (fmt == 24 ? MetalKqMvB4Iq2xs_metal_kq_mv_iq2xs_msl_entry : (fmt == 23 ? MetalKqMvB4Iq2s_metal_kq_mv_iq2s_msl_entry : (fmt == 45 ? MetalKqMvB4Iq4nl_metal_kq_mv_iq4nl_msl_entry : (fmt == 2 ? MetalKqMvB4K2_metal_kq_mv_k2_msl_entry : (fmt == 34 ? MetalKqMvB4Iq3xxs_metal_kq_mv_iq3xxs_msl_entry : (fmt == 33 ? MetalKqMvB4Iq3s_metal_kq_mv_iq3s_msl_entry : (fmt == 3 ? MetalKqMvB4K3_metal_kq_mv_k3_msl_entry : (fmt == 44 ? MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl_entry : (fmt == 4 ? MetalKqMvB4K4_metal_kq_mv_k4_msl_entry : (fmt == 5 ? MetalKqMvB4K5_metal_kq_mv_k5_msl_entry : MetalKqMvB4K6_metal_kq_mv_k6_msl_entry))))))))))) + : (fmt == 25 ? metal_kq_mvb8_iq2xxs_msl_entry : (fmt == 24 ? metal_kq_mvb8_iq2xs_msl_entry : (fmt == 23 ? metal_kq_mvb8_iq2s_msl_entry : (fmt == 45 ? metal_kq_mvb8_iq4nl_msl_entry : (fmt == 2 ? metal_kq_mvb8_k2_msl_entry : (fmt == 34 ? metal_kq_mvb8_iq3xxs_msl_entry : (fmt == 33 ? metal_kq_mvb8_iq3s_msl_entry : (fmt == 3 ? metal_kq_mvb8_k3_msl_entry : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl_entry : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_entry : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_entry : metal_kq_mvb8_k6_msl_entry))))))))))))) + let fm = (bwidth == 2 ? (fmt == 25 ? MetalKqMvB2Iq2xxs_metal_kq_mv_iq2xxs_msl_fastmath : (fmt == 24 ? MetalKqMvB2Iq2xs_metal_kq_mv_iq2xs_msl_fastmath : (fmt == 23 ? MetalKqMvB2Iq2s_metal_kq_mv_iq2s_msl_fastmath : (fmt == 45 ? MetalKqMvB2Iq4nl_metal_kq_mv_iq4nl_msl_fastmath : (fmt == 2 ? MetalKqMvB2K2_metal_kq_mv_k2_msl_fastmath : (fmt == 34 ? MetalKqMvB2Iq3xxs_metal_kq_mv_iq3xxs_msl_fastmath : (fmt == 33 ? MetalKqMvB2Iq3s_metal_kq_mv_iq3s_msl_fastmath : (fmt == 3 ? MetalKqMvB2K3_metal_kq_mv_k3_msl_fastmath : (fmt == 44 ? MetalKqMvB2Iq4xs_metal_kq_mv_iq4xs_msl_fastmath : (fmt == 4 ? MetalKqMvB2K4_metal_kq_mv_k4_msl_fastmath : (fmt == 5 ? MetalKqMvB2K5_metal_kq_mv_k5_msl_fastmath : MetalKqMvB2K6_metal_kq_mv_k6_msl_fastmath))))))))))) + : (bwidth == 4 ? (fmt == 25 ? MetalKqMvB4Iq2xxs_metal_kq_mv_iq2xxs_msl_fastmath : (fmt == 24 ? MetalKqMvB4Iq2xs_metal_kq_mv_iq2xs_msl_fastmath : (fmt == 23 ? MetalKqMvB4Iq2s_metal_kq_mv_iq2s_msl_fastmath : (fmt == 45 ? MetalKqMvB4Iq4nl_metal_kq_mv_iq4nl_msl_fastmath : (fmt == 2 ? MetalKqMvB4K2_metal_kq_mv_k2_msl_fastmath : (fmt == 34 ? MetalKqMvB4Iq3xxs_metal_kq_mv_iq3xxs_msl_fastmath : (fmt == 33 ? MetalKqMvB4Iq3s_metal_kq_mv_iq3s_msl_fastmath : (fmt == 3 ? MetalKqMvB4K3_metal_kq_mv_k3_msl_fastmath : (fmt == 44 ? MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl_fastmath : (fmt == 4 ? MetalKqMvB4K4_metal_kq_mv_k4_msl_fastmath : (fmt == 5 ? MetalKqMvB4K5_metal_kq_mv_k5_msl_fastmath : MetalKqMvB4K6_metal_kq_mv_k6_msl_fastmath))))))))))) + : (fmt == 25 ? metal_kq_mvb8_iq2xxs_msl_fastmath : (fmt == 24 ? metal_kq_mvb8_iq2xs_msl_fastmath : (fmt == 23 ? metal_kq_mvb8_iq2s_msl_fastmath : (fmt == 45 ? metal_kq_mvb8_iq4nl_msl_fastmath : (fmt == 2 ? metal_kq_mvb8_k2_msl_fastmath : (fmt == 34 ? metal_kq_mvb8_iq3xxs_msl_fastmath : (fmt == 33 ? metal_kq_mvb8_iq3s_msl_fastmath : (fmt == 3 ? metal_kq_mvb8_k3_msl_fastmath : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl_fastmath : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_fastmath : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_fastmath : metal_kq_mvb8_k6_msl_fastmath))))))))))))) var pso = pipeline_from_source(dev, src, entry, fm, err) t |> success(pso != null, "{tag}: pipeline: {err}") return if (pso == null) @@ -172,9 +172,9 @@ def private kq_mvb_gate(t : T?; dev, queue; fmt, bwidth, n, d, nr, ys : int) { metal_set_pipeline(enc, pso) if (bwidth == 8) { metal_set_threadgroup_memory_length(enc, - fmt == 24 ? metal_kq_mvb8_iq2xs_msl_tgmem : (fmt == 23 ? metal_kq_mvb8_iq2s_msl_tgmem : (fmt == 2 ? metal_kq_mvb8_k2_msl_tgmem : (fmt == 45 ? metal_kq_mvb8_iq4nl_msl_tgmem : (fmt == 34 ? metal_kq_mvb8_iq3xxs_msl_tgmem : (fmt == 33 ? metal_kq_mvb8_iq3s_msl_tgmem : (fmt == 3 ? metal_kq_mvb8_k3_msl_tgmem : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl_tgmem : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_tgmem : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_tgmem : metal_kq_mvb8_k6_msl_tgmem))))))))), 0) + fmt == 25 ? metal_kq_mvb8_iq2xxs_msl_tgmem : (fmt == 24 ? metal_kq_mvb8_iq2xs_msl_tgmem : (fmt == 23 ? metal_kq_mvb8_iq2s_msl_tgmem : (fmt == 2 ? metal_kq_mvb8_k2_msl_tgmem : (fmt == 45 ? metal_kq_mvb8_iq4nl_msl_tgmem : (fmt == 34 ? metal_kq_mvb8_iq3xxs_msl_tgmem : (fmt == 33 ? metal_kq_mvb8_iq3s_msl_tgmem : (fmt == 3 ? metal_kq_mvb8_k3_msl_tgmem : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl_tgmem : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_tgmem : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_tgmem : metal_kq_mvb8_k6_msl_tgmem)))))))))), 0) } - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 || fmt == 24 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs/k3: the f16 d tail + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 || fmt == 24 || fmt == 25 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs/k3: the f16 d tail metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bx, 0ul, 3) @@ -263,7 +263,7 @@ def private moe_kq_gemv_gate(t : T?; dev, queue; fmt, n, d, net, k, nst : int; p bxs = uint((perslot ? k : 1) * n), bys = uint(k * d), hasb = 0u) let ran = with_compute_encoder(queue, err) $(enc : MetalComputeEncoder?) { metal_set_pipeline(enc, pso) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 || fmt == 24 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs/k3: the f16 d tail + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 || fmt == 24 || fmt == 25 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs/k3: the f16 d tail metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bx, 0ul, 3) @@ -749,8 +749,10 @@ def test_metal_kq_gemv_kernels(t : T?) { kq_gemv_gate(t, dev, queue, "iq2s", 512, 30) kq_gemv_gate(t, dev, queue, "iq2xs", 1280, 11) kq_gemv_gate(t, dev, queue, "iq2xs", 512, 30) + kq_gemv_gate(t, dev, queue, "iq2xxs", 1280, 11) + kq_gemv_gate(t, dev, queue, "iq2xxs", 512, 30) // the small-batch twins: nr = 2 / 3 (b4 col guard) / 6 (b8 pad), ys > d - for (fmt in [4, 5, 6, 44, 3, 33, 34, 45, 2, 23, 24]) { + for (fmt in [4, 5, 6, 44, 3, 33, 34, 45, 2, 23, 24, 25]) { kq_mvb_gate(t, dev, queue, fmt, 2, 768, 17, 2, 22) kq_mvb_gate(t, dev, queue, fmt, 4, 768, 17, 3, 22) kq_mvb_gate(t, dev, queue, fmt, 8, 768, 17, 6, 22) From ce7d8d80e0052a4530342c6f43d96082e3df13a0 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Mon, 31 Aug 2026 17:28:41 -0700 Subject: [PATCH 50/61] dasLLAMA: the tune race keeps one seat per ISA tier (unquirk pass, B1) The 2026-08-31 sidecar audit across zen2/m1/m4/m5: the kq tile races crowned dot_maddubs_width256_mr8 (mr8 on ARM) on every format and every box, and the 20-seat [tuned] utility walk never spread winner-vs-runner-up past 1.4%. The clones only cost mint time, so the grids now carry one seat per ISA tier: - kq tiles 11 -> 4 seats (mr8 / maddubs avx2 / vpdpbusd vnni256 / vnni512); fallback chains end at mr8 (the mr4 crown loses double-digit % at model scale - the probe header's own finding). - q8q8 ~34 -> 23: per-ISA winners + mr8_budget (the declining reference seat, the M5 crown) + the untouched smmla/amx/vpdpbssd legs, every x86 leg now carrying its requires= gate so race-on-unlock can key on it. - [tuned] grid 20 -> 7: plain/u2/vec8/vec8_u2/vec16/vec16_u2 + the vec4_u4 hand row dot_q4 pins (NEON nibble widening). Gates: gen probe TEST mode green over every surviving perm x 16 families; tune mode green, same crowns (maddubs mr8; q8q8 kstep2 confirms as the per-ISA fallback). Pruned probe tile walk: 372 s vs the ~20-minute pre-prune app mints. plans/unquirk_pass.md carries the full end-of-arc plan. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01MyYCDutugdc9xL5pd2qcvT --- .../dasLLAMA/dasllama/dasllama_math_gen.das | 169 +++++++++--------- modules/dasLLAMA/dasllama/dasllama_tune.das | 9 +- modules/dasLLAMA/harness/gen_tune_probe.das | 4 +- plans/unquirk_pass.md | 161 +++++++++++++++++ 4 files changed, 248 insertions(+), 95 deletions(-) create mode 100644 plans/unquirk_pass.md diff --git a/modules/dasLLAMA/dasllama/dasllama_math_gen.das b/modules/dasLLAMA/dasllama/dasllama_math_gen.das index f02886cfaa..282745de14 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_gen.das @@ -522,21 +522,21 @@ def q40q8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp } } -// The kq tune grids (one [tune] family per format): kstep pinned 1, grid is (mr, nrsplit) per ISA -// tier. nrsplit token-splits the 4-token tile; nrsplit-4 (single unpack) wins wherever q-reg -// budget admits it. bias128/smmla/amx rows are absent — kq planes are natively unsigned. +// The kq tune grids (one [tune] family per format): kstep pinned 1, ONE seat per ISA tier — +// mr4/nrsplit clones pruned (no box ever crowned one outside noise; the race exists only where +// an ISA unlock changes the kernel). bias128/smmla/amx rows absent — kq planes are natively unsigned. //! The K-quant TILE kernels (kq batch family): 4 tokens x mr rows per call — kqg is the group's //! QUANT panel (k4 the packed grp plane; k5/k6 a BYTE-EXPANDED scratch unpacked once per //! (group, token-block)). One call is bit-exact vs 4 per-token GEMVs over the same weights. -[tune_perm(mr = 4), tune_perm(mr = 4, nrsplit = 2), tune_perm(mr = 8), tune_perm(mr = 8, nrsplit = 2), - tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "maddubs", width = 256, mr = 8, nrsplit = 2, requires = "avx2"), - tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, nrsplit = 2, requires = "avxvnni|avx512vnni"), - tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, nrsplit = 2, requires = "avx512vnni,avx512bw"), +[tune_perm(mr = 8), + tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), tune_companion(fn = "k4q8_gemv_gen", gen = "dasllama_gemm_gen::k4_gemv"), tune_companion(fn = "k4q8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::k4_tile", - fallback = "dot_vpdpbusd_width512_mr16;dot_vpdpbusd_width256_mr8;dot_maddubs_width256_mr8;mr4"), + fallback = "dot_vpdpbusd_width512_mr16;dot_vpdpbusd_width256_mr8;dot_maddubs_width256_mr8;mr8"), hint(unsafe_range_check, noalias = kqg, noalias = ksg, noalias = xqp, noalias = xsp, noalias = xbsp)] def k4q8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d, g, t0 : int64) : void { let mr = int64(k4q8_layout_gen()) @@ -549,14 +549,14 @@ def k4q8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp : } } -[tune_perm(mr = 4), tune_perm(mr = 4, nrsplit = 2), tune_perm(mr = 8), tune_perm(mr = 8, nrsplit = 2), - tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "maddubs", width = 256, mr = 8, nrsplit = 2, requires = "avx2"), - tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, nrsplit = 2, requires = "avxvnni|avx512vnni"), - tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, nrsplit = 2, requires = "avx512vnni,avx512bw"), +[tune_perm(mr = 8), + tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), tune_companion(fn = "k5q8_gemv_gen", gen = "dasllama_gemm_gen::k5_gemv"), tune_companion(fn = "k5q8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::k5_tile", - fallback = "dot_vpdpbusd_width512_mr16;dot_vpdpbusd_width256_mr8;dot_maddubs_width256_mr8;mr4"), + fallback = "dot_vpdpbusd_width512_mr16;dot_vpdpbusd_width256_mr8;dot_maddubs_width256_mr8;mr8"), hint(unsafe_range_check, noalias = kqg, noalias = ksg, noalias = xqp, noalias = xsp, noalias = xbsp)] def k5q8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d, g, t0 : int64) : void { let mr = int64(k5q8_layout_gen()) @@ -569,14 +569,14 @@ def k5q8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp : } } -[tune_perm(mr = 4), tune_perm(mr = 4, nrsplit = 2), tune_perm(mr = 8), tune_perm(mr = 8, nrsplit = 2), - tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "maddubs", width = 256, mr = 8, nrsplit = 2, requires = "avx2"), - tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, nrsplit = 2, requires = "avxvnni|avx512vnni"), - tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, nrsplit = 2, requires = "avx512vnni,avx512bw"), +[tune_perm(mr = 8), + tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), tune_companion(fn = "k6q8_gemv_gen", gen = "dasllama_gemm_gen::k6_gemv"), tune_companion(fn = "k6q8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::k6_tile", - fallback = "dot_vpdpbusd_width512_mr16;dot_vpdpbusd_width256_mr8;dot_maddubs_width256_mr8;mr4"), + fallback = "dot_vpdpbusd_width512_mr16;dot_vpdpbusd_width256_mr8;dot_maddubs_width256_mr8;mr8"), hint(unsafe_range_check, noalias = kqg, noalias = ksg, noalias = xqp, noalias = xsp, noalias = xbsp)] def k6q8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d, g, t0 : int64) : void { let mr = int64(k6q8_layout_gen()) @@ -589,14 +589,14 @@ def k6q8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp : } } -[tune_perm(mr = 4), tune_perm(mr = 4, nrsplit = 2), tune_perm(mr = 8), tune_perm(mr = 8, nrsplit = 2), - tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "maddubs", width = 256, mr = 8, nrsplit = 2, requires = "avx2"), - tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, nrsplit = 2, requires = "avxvnni|avx512vnni"), - tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, nrsplit = 2, requires = "avx512vnni,avx512bw"), +[tune_perm(mr = 8), + tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), tune_companion(fn = "q40q8_gemv_gen", gen = "dasllama_gemm_gen::q40_gemv"), tune_companion(fn = "q40q8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::q40_tile", - fallback = "dot_vpdpbusd_width512_mr16;dot_vpdpbusd_width256_mr8;dot_maddubs_width256_mr8;mr4"), + fallback = "dot_vpdpbusd_width512_mr16;dot_vpdpbusd_width256_mr8;dot_maddubs_width256_mr8;mr8"), hint(unsafe_range_check, noalias = kqg, noalias = ksg, noalias = xqp, noalias = xsp, noalias = xbsp)] def q40q8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d, g, t0 : int64) : void { let mr = int64(q40q8_layout_gen()) @@ -621,14 +621,14 @@ def iq4xsq8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xq } } -[tune_perm(mr = 4), tune_perm(mr = 4, nrsplit = 2), tune_perm(mr = 8), tune_perm(mr = 8, nrsplit = 2), - tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "maddubs", width = 256, mr = 8, nrsplit = 2, requires = "avx2"), - tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, nrsplit = 2, requires = "avxvnni|avx512vnni"), - tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, nrsplit = 2, requires = "avx512vnni,avx512bw"), +[tune_perm(mr = 8), + tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), tune_companion(fn = "iq4xsq8_gemv_gen", gen = "dasllama_gemm_gen::iq4xs_gemv"), tune_companion(fn = "iq4xsq8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::iq4xs_tile", - fallback = "dot_vpdpbusd_width512_mr16;dot_vpdpbusd_width256_mr8;dot_maddubs_width256_mr8;mr4"), + fallback = "dot_vpdpbusd_width512_mr16;dot_vpdpbusd_width256_mr8;dot_maddubs_width256_mr8;mr8"), hint(unsafe_range_check, noalias = kqg, noalias = ksg, noalias = xqp, noalias = xsp, noalias = xbsp)] def iq4xsq8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d, g, t0 : int64) : void { let mr = int64(iq4xsq8_layout_gen()) @@ -653,14 +653,14 @@ def k3q8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : } } -[tune_perm(mr = 4), tune_perm(mr = 4, nrsplit = 2), tune_perm(mr = 8), tune_perm(mr = 8, nrsplit = 2), - tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "maddubs", width = 256, mr = 8, nrsplit = 2, requires = "avx2"), - tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, nrsplit = 2, requires = "avxvnni|avx512vnni"), - tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, nrsplit = 2, requires = "avx512vnni,avx512bw"), +[tune_perm(mr = 8), + tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), tune_companion(fn = "k3q8_gemv_gen", gen = "dasllama_gemm_gen::k3_gemv"), tune_companion(fn = "k3q8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::k3_tile", - fallback = "dot_vpdpbusd_width512_mr16;dot_vpdpbusd_width256_mr8;dot_maddubs_width256_mr8;mr4"), + fallback = "dot_vpdpbusd_width512_mr16;dot_vpdpbusd_width256_mr8;dot_maddubs_width256_mr8;mr8"), hint(unsafe_range_check, noalias = kqg, noalias = ksg, noalias = xqp, noalias = xsp, noalias = xbsp)] def k3q8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d, g, t0 : int64) : void { let mr = int64(k3q8_layout_gen()) @@ -685,14 +685,14 @@ def iq3sq8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp } } -[tune_perm(mr = 4), tune_perm(mr = 4, nrsplit = 2), tune_perm(mr = 8), tune_perm(mr = 8, nrsplit = 2), - tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "maddubs", width = 256, mr = 8, nrsplit = 2, requires = "avx2"), - tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, nrsplit = 2, requires = "avxvnni|avx512vnni"), - tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, nrsplit = 2, requires = "avx512vnni,avx512bw"), +[tune_perm(mr = 8), + tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), tune_companion(fn = "iq3sq8_gemv_gen", gen = "dasllama_gemm_gen::iq3s_gemv"), tune_companion(fn = "iq3sq8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::iq3s_tile", - fallback = "dot_vpdpbusd_width512_mr16;dot_vpdpbusd_width256_mr8;dot_maddubs_width256_mr8;mr4"), + fallback = "dot_vpdpbusd_width512_mr16;dot_vpdpbusd_width256_mr8;dot_maddubs_width256_mr8;mr8"), hint(unsafe_range_check, noalias = kqg, noalias = ksg, noalias = xqp, noalias = xsp, noalias = xbsp)] def iq3sq8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d, g, t0 : int64) : void { let mr = int64(iq3sq8_layout_gen()) @@ -760,14 +760,14 @@ def iq3xxsq8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; x } } -[tune_perm(mr = 4), tune_perm(mr = 4, nrsplit = 2), tune_perm(mr = 8), tune_perm(mr = 8, nrsplit = 2), - tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "maddubs", width = 256, mr = 8, nrsplit = 2, requires = "avx2"), - tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, nrsplit = 2, requires = "avxvnni|avx512vnni"), - tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, nrsplit = 2, requires = "avx512vnni,avx512bw"), +[tune_perm(mr = 8), + tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), tune_companion(fn = "iq3xxsq8_gemv_gen", gen = "dasllama_gemm_gen::iq3xxs_gemv"), tune_companion(fn = "iq3xxsq8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::iq3xxs_tile", - fallback = "dot_vpdpbusd_width512_mr16;dot_vpdpbusd_width256_mr8;dot_maddubs_width256_mr8;mr4"), + fallback = "dot_vpdpbusd_width512_mr16;dot_vpdpbusd_width256_mr8;dot_maddubs_width256_mr8;mr8"), hint(unsafe_range_check, noalias = kqg, noalias = ksg, noalias = xqp, noalias = xsp, noalias = xbsp)] def iq3xxsq8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d, g, t0 : int64) : void { let mr = int64(iq3xxsq8_layout_gen()) @@ -823,14 +823,14 @@ def iq4nlq8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xq } } -[tune_perm(mr = 4), tune_perm(mr = 4, nrsplit = 2), tune_perm(mr = 8), tune_perm(mr = 8, nrsplit = 2), - tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "maddubs", width = 256, mr = 8, nrsplit = 2, requires = "avx2"), - tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, nrsplit = 2, requires = "avxvnni|avx512vnni"), - tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, nrsplit = 2, requires = "avx512vnni,avx512bw"), +[tune_perm(mr = 8), + tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), tune_companion(fn = "iq4nlq8_gemv_gen", gen = "dasllama_gemm_gen::iq4nl_gemv"), tune_companion(fn = "iq4nlq8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::iq4nl_tile", - fallback = "dot_vpdpbusd_width512_mr16;dot_vpdpbusd_width256_mr8;dot_maddubs_width256_mr8;mr4"), + fallback = "dot_vpdpbusd_width512_mr16;dot_vpdpbusd_width256_mr8;dot_maddubs_width256_mr8;mr8"), hint(unsafe_range_check, noalias = kqg, noalias = ksg, noalias = xqp, noalias = xsp, noalias = xbsp)] def iq4nlq8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d, g, t0 : int64) : void { let mr = int64(iq4nlq8_layout_gen()) @@ -892,14 +892,14 @@ def k2q8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : } } -[tune_perm(mr = 4), tune_perm(mr = 4, nrsplit = 2), tune_perm(mr = 8), tune_perm(mr = 8, nrsplit = 2), - tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "maddubs", width = 256, mr = 8, nrsplit = 2, requires = "avx2"), - tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, nrsplit = 2, requires = "avxvnni|avx512vnni"), - tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, nrsplit = 2, requires = "avx512vnni,avx512bw"), +[tune_perm(mr = 8), + tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), tune_companion(fn = "k2q8_gemv_gen", gen = "dasllama_gemm_gen::k2_gemv"), tune_companion(fn = "k2q8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::k2_tile", - fallback = "dot_vpdpbusd_width512_mr16;dot_vpdpbusd_width256_mr8;dot_maddubs_width256_mr8;mr4"), + fallback = "dot_vpdpbusd_width512_mr16;dot_vpdpbusd_width256_mr8;dot_maddubs_width256_mr8;mr8"), hint(unsafe_range_check, noalias = kqg, noalias = ksg, noalias = xqp, noalias = xsp, noalias = xbsp)] def k2q8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d, g, t0 : int64) : void { let mr = int64(k2q8_layout_gen()) @@ -964,14 +964,14 @@ def iq2sq8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp } } -[tune_perm(mr = 4), tune_perm(mr = 4, nrsplit = 2), tune_perm(mr = 8), tune_perm(mr = 8, nrsplit = 2), - tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "maddubs", width = 256, mr = 8, nrsplit = 2, requires = "avx2"), - tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, nrsplit = 2, requires = "avxvnni|avx512vnni"), - tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, nrsplit = 2, requires = "avx512vnni,avx512bw"), +[tune_perm(mr = 8), + tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), tune_companion(fn = "iq2sq8_gemv_gen", gen = "dasllama_gemm_gen::iq2s_gemv"), tune_companion(fn = "iq2sq8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::iq2s_tile", - fallback = "dot_vpdpbusd_width512_mr16;dot_vpdpbusd_width256_mr8;dot_maddubs_width256_mr8;mr4"), + fallback = "dot_vpdpbusd_width512_mr16;dot_vpdpbusd_width256_mr8;dot_maddubs_width256_mr8;mr8"), hint(unsafe_range_check, noalias = kqg, noalias = ksg, noalias = xqp, noalias = xsp, noalias = xbsp)] def iq2sq8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d, g, t0 : int64) : void { let mr = int64(iq2sq8_layout_gen()) @@ -1035,14 +1035,14 @@ def iq2xsq8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xq } } -[tune_perm(mr = 4), tune_perm(mr = 4, nrsplit = 2), tune_perm(mr = 8), tune_perm(mr = 8, nrsplit = 2), - tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "maddubs", width = 256, mr = 8, nrsplit = 2, requires = "avx2"), - tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, nrsplit = 2, requires = "avxvnni|avx512vnni"), - tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, nrsplit = 2, requires = "avx512vnni,avx512bw"), +[tune_perm(mr = 8), + tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), tune_companion(fn = "iq2xsq8_gemv_gen", gen = "dasllama_gemm_gen::iq2xs_gemv"), tune_companion(fn = "iq2xsq8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::iq2xs_tile", - fallback = "dot_vpdpbusd_width512_mr16;dot_vpdpbusd_width256_mr8;dot_maddubs_width256_mr8;mr4"), + fallback = "dot_vpdpbusd_width512_mr16;dot_vpdpbusd_width256_mr8;dot_maddubs_width256_mr8;mr8"), hint(unsafe_range_check, noalias = kqg, noalias = ksg, noalias = xqp, noalias = xsp, noalias = xbsp)] def iq2xsq8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d, g, t0 : int64) : void { let mr = int64(iq2xsq8_layout_gen()) @@ -1108,14 +1108,14 @@ def iq2xxsq8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; x } } -[tune_perm(mr = 4), tune_perm(mr = 4, nrsplit = 2), tune_perm(mr = 8), tune_perm(mr = 8, nrsplit = 2), - tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "maddubs", width = 256, mr = 8, nrsplit = 2, requires = "avx2"), - tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, nrsplit = 2, requires = "avxvnni|avx512vnni"), - tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, nrsplit = 2, requires = "avx512vnni,avx512bw"), +[tune_perm(mr = 8), + tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), tune_companion(fn = "iq2xxsq8_gemv_gen", gen = "dasllama_gemm_gen::iq2xxs_gemv"), tune_companion(fn = "iq2xxsq8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::iq2xxs_tile", - fallback = "dot_vpdpbusd_width512_mr16;dot_vpdpbusd_width256_mr8;dot_maddubs_width256_mr8;mr4"), + fallback = "dot_vpdpbusd_width512_mr16;dot_vpdpbusd_width256_mr8;dot_maddubs_width256_mr8;mr8"), hint(unsafe_range_check, noalias = kqg, noalias = ksg, noalias = xqp, noalias = xsp, noalias = xbsp)] def iq2xxsq8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d, g, t0 : int64) : void { let mr = int64(iq2xxsq8_layout_gen()) @@ -1166,21 +1166,13 @@ def q8q8_tile_s16_gen(var yp : float?; wg : int8 const?; sg : uint16 const?; xqp //! The generated tile: mr interleaved rows x 4 tokens (the q8q8_repack_type(mr) layout). Reference //! body = 4 single-token laneq4 dots, runs on interp/AOT/non-arm64 JIT/generator-declined; ignores //! xbsp unless a biased stamp reads it. dot=/width= rows are the x64 legs (slice F), one grid per ISA. -[tune_perm(kstep = 1), tune_perm(kstep = 2), tune_perm(kstep = 4), tune_perm(kstep = 2, nrsplit = 2), - tune_perm(kstep = 1, nrsplit = 2), tune_perm(kstep = 4, nrsplit = 2), - tune_perm(kstep = 1, nrsplit = 2, mr = 8), tune_perm(kstep = 2, nrsplit = 2, mr = 8), tune_perm(kstep = 4, nrsplit = 2, mr = 8), - tune_perm(kstep = 2, gkstep = 2), tune_perm(kstep = 2, gkstep = 4), - tune_perm(kstep = 4, nrsplit = 2, mr = 8, gkstep = 2), tune_perm(kstep = 4, nrsplit = 2, mr = 8, gkstep = 4), - tune_perm(dot = "maddubs", width = 256, mr = 8, kstep = 2, requires = "avx2"), tune_perm(dot = "maddubs", width = 256, mr = 8, kstep = 4, nrsplit = 2), - tune_perm(dot = "vpdpbusd", width = 256, mr = 8, kstep = 2), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, kstep = 4, nrsplit = 2), - tune_perm(dot = "vpdpbusd", width = 256, mr = 16, kstep = 2, nrsplit = 2), tune_perm(dot = "vpdpbssd", width = 256, mr = 8, kstep = 2), - tune_perm(dot = "maddubs", width = 512, mr = 16, kstep = 2), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, kstep = 2), - tune_perm(dot = "vpdpbusd", width = 512, mr = 16, kstep = 4, nrsplit = 2), tune_perm(dot = "vpdpbusd", width = 512, mr = 32, kstep = 2, nrsplit = 2), - tune_perm(dot = "vpdpbusd", width = 256, mr = 8, kstep = 2, gkstep = 2), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, kstep = 2, gkstep = 2), - tune_perm(dot = "vpdpbusd", width = 256, mr = 8, kstep = 2, bias = 128), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, kstep = 2, gkstep = 2, bias = 128, requires = "avxvnni|avx512vnni"), - tune_perm(dot = "vpdpbusd", width = 512, mr = 16, kstep = 2, bias = 128), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, kstep = 2, gkstep = 2, bias = 128, requires = "avx512vnni,avx512bw"), - tune_perm(dot = "vpdpbusd", width = 512, mr = 16, kstep = 4, nrsplit = 2, bias = 128), - tune_perm(dot = "maddubs", width = 256, mr = 8, kstep = 2, bias = 128), +[tune_perm(kstep = 2), tune_perm(suffix = "mr8_budget", mr = 8), + tune_perm(dot = "maddubs", width = 256, mr = 8, kstep = 2, requires = "avx2"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 8, kstep = 2, requires = "avxvnni|avx512vnni"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 8, kstep = 2, gkstep = 2, bias = 128, requires = "avxvnni|avx512vnni"), + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, kstep = 2, requires = "avx512vnni,avx512bw"), + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, kstep = 2, gkstep = 2, bias = 128, requires = "avx512vnni,avx512bw"), + tune_perm(dot = "vpdpbssd", width = 256, mr = 8, kstep = 2, requires = "avxvnniint8"), tune_perm(dot = "smmla", mr = 4, kstep = 2, requires = "i8mm"), tune_perm(dot = "smmla", mr = 4, kstep = 2, nrsplit = 2, requires = "i8mm"), tune_perm(dot = "smmla", mr = 8, kstep = 2, nrsplit = 2, requires = "i8mm"), tune_perm(dot = "smmla", mr = 8, kstep = 4, nrsplit = 2, requires = "i8mm"), tune_perm(dot = "smmla", mr = 8, kstep = 2, nrsplit = 2, gkstep = 2, requires = "i8mm"), @@ -1194,7 +1186,6 @@ def q8q8_tile_s16_gen(var yp : float?; wg : int8 const?; sg : uint16 const?; xqp tune_perm(dot = "amx_int8", width = 512, mr = 16, kstep = 1, nrsplit = 2, pipe = 1, bias = 128, requires = "amx-int8"), tune_perm(dot = "amx_int8", width = 512, mr = 16, kstep = 1, nrsplit = 2, latch = 1, bias = 128, requires = "amx-int8"), tune_perm(dot = "amx_int8", width = 512, mr = 16, kstep = 1, nrsplit = 2, pipe = 1, latch = 1, bias = 128, requires = "amx-int8"), - tune_perm(suffix = "mr8_budget", mr = 8), tune_perm(suffix = "mr2_lanes", mr = 2), tune_perm(dot = "vpdpbusd"), tune_companion(fn = "q8q8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune_companion(fn = "q8q8_wbias_gen", gen = "dasllama_gemm_gen::q8q8_wbias"), tune_companion(fn = "q8q8_kgroup_gen", gen = "dasllama_gemm_gen::q8q8_kgroup"), @@ -1313,14 +1304,14 @@ def q51q8_gemv_gen(var yp : float?; qgp : uint8 const?; sgp : uint8 const?; xqp //! The q51 TILE kernel (the family anchor): 4 tokens x mr rows per call off the group's packed //! grp plane pair — the kq tile contract with the 32-weight block unit (Q8_0-form //! activations: per-32 xs and per-32 asum planes). Bit-exact vs 4 per-token GEMVs. -[tune_perm(mr = 4), tune_perm(mr = 4, nrsplit = 2), tune_perm(mr = 8), tune_perm(mr = 8, nrsplit = 2), - tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "maddubs", width = 256, mr = 8, nrsplit = 2, requires = "avx2"), - tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, nrsplit = 2, requires = "avxvnni|avx512vnni"), - tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, nrsplit = 2, requires = "avx512vnni,avx512bw"), +[tune_perm(mr = 8), + tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), tune_companion(fn = "q51q8_gemv_gen", gen = "dasllama_gemm_gen::q51_gemv"), tune_companion(fn = "q51q8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::q51_tile", - fallback = "dot_vpdpbusd_width512_mr16;dot_vpdpbusd_width256_mr8;dot_maddubs_width256_mr8;mr4"), + fallback = "dot_vpdpbusd_width512_mr16;dot_vpdpbusd_width256_mr8;dot_maddubs_width256_mr8;mr8"), hint(unsafe_range_check, noalias = qgp, noalias = sgp, noalias = xqp, noalias = xsp, noalias = xbsp)] def q51q8_tile_gen(var yp : float?; qgp : uint8 const?; sgp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d, g, t0 : int64) : void { let mr = int64(q51q8_layout_gen()) diff --git a/modules/dasLLAMA/dasllama/dasllama_tune.das b/modules/dasLLAMA/dasllama/dasllama_tune.das index 5ecd6a3a86..bb1bbb6909 100644 --- a/modules/dasLLAMA/dasllama/dasllama_tune.das +++ b/modules/dasLLAMA/dasllama/dasllama_tune.das @@ -30,11 +30,11 @@ struct TunePerm { } def private build_grid() : array { - // unroll=1 OMITS unroll_count so each row stays byte-identical to the original hand hints (DEFAULT_PERM vec8_u2 included); width 4/32 rarely win on M1 but x64 AVX-512 picks them. - let WIDTHS = fixed_array(4, 8, 16, 32) - let UNROLLS = fixed_array(2, 4, 8) // unroll-only rows; u1 == plain + // pruned to seats that win beyond noise (the 20-seat walk never spread >1.4% on any box); unroll=1 omits unroll_count (rows byte-identical to the hand hints); vec4_u4 = dot_q4's pinned NEON nibble-widening winner + let WIDTHS = fixed_array(8, 16) + let UNROLLS = fixed_array(2) // unroll-only rows; u1 == plain var grid : array - grid |> reserve(1 + length(UNROLLS) + length(WIDTHS) * (1 + length(UNROLLS))) + grid |> reserve(2 + length(UNROLLS) + length(WIDTHS) * (1 + length(UNROLLS))) grid |> emplace(TunePerm(suffix = "plain")) for (u in UNROLLS) { grid |> emplace(TunePerm(suffix = "u{u}", hints <- [TuneHint(tune = 1, unroll = u)])) @@ -46,6 +46,7 @@ def private build_grid() : array { hints <- [TuneHint(tune = 1, vectorize = true, width = w, unroll = u)])) } } + grid |> emplace(TunePerm(suffix = "vec4_u4", hints <- [TuneHint(tune = 1, vectorize = true, width = 4, unroll = 4)])) return <- grid } diff --git a/modules/dasLLAMA/harness/gen_tune_probe.das b/modules/dasLLAMA/harness/gen_tune_probe.das index 5bd7c9b8b7..b73a920b20 100644 --- a/modules/dasLLAMA/harness/gen_tune_probe.das +++ b/modules/dasLLAMA/harness/gen_tune_probe.das @@ -21,8 +21,8 @@ options _dasllama_internal = true // equality: variants run under full fast-math, so the backend may legally regroup a // fold per loop shape (~1e-6 relative; the reference holds no cross-kernel bit-parity // either) — real generator bugs (wrong stride/lane/scale) are orders louder and -// still die here. Declining perms (mr8_budget / mr2_lanes / dot_vpdpbusd) run the -// reference bodies. +// still die here. Declining perms (mr8_budget, and every ISA-gated row the box +// fails) run the reference bodies. // tune: iso-bench every variant (interleaved best-of) on its own repacked buffers — the // tile on a batch shape, the gemv core on a single-token decode shape — print both // tables, and tune_manifest_set the merged winner: the row whose TILE knobs win the diff --git a/plans/unquirk_pass.md b/plans/unquirk_pass.md new file mode 100644 index 0000000000..e9c720640e --- /dev/null +++ b/plans/unquirk_pass.md @@ -0,0 +1,161 @@ +# Unquirk pass — end of the iquant format ladder arc + +The ladder is complete (9 formats x 4 tiers, ea1f41d2c). This plan covers everything between +that and the arc's PR: the tuner rework, the KqPlanes unquirk, the cache/one-off fixes, the +HOW_TO document split, and two PRs. Order: B + C1 + D1 -> **PR-1** (the ladder ships, the +parallel M5/M1 session forks) -> A + D2 + E -> **PR-2**; every phase ends green and +committed. Kernel catch-up runs on zen2 in parallel with the mac session after PR-1. + +Evidence base (2026-08-31 sidecar audit): a full mint races 39 families / 637 seats. The 16 +tile-gen families crown `dot_maddubs_width256_mr8` on every format on zen2 and mr8-shapes on +ARM; the 22 `[tuned]` vector-utility families have winner-vs-runner-up gaps of 0.0-1.4% on +zen2 AND on M5 (25/25 under 2%). Cross-sidecar winner agreement 34/39; all 5 diffs are noise +ties or a skipped-seat artifact. The race rediscovers ~2 facts for ~20 minutes per app per box. +M5 Max carries FEAT_I8MM + FEAT_SME2/2p1 + BF16 that our ARM seat space does not offer a +single seat for — its mint provably lands on M1's answers today. + +## Phase B — tuner rework (kills QUIRK 11, 15, 17, 18) + +Target: mint on a known box = zero races; full race on new silicon (zen4, M5) = a couple of +minutes. If that target holds, the dasllama.io tune service is removed entirely (Boris's call, +outside this PR). + +**B1. Prune the seat space.** +- Tile gens (`dasllama_math_gen.das`, 16 families x 11 seats): keep `mr8` (generic/ARM + baseline), `maddubs w256 mr8` (requires avx2 — the x86 baseline), `vpdpbusd w256 mr8` + (requires vnni), `vpdpbusd w512 mr16` (requires avx512vnni+bw). Drop mr4 and every nrsplit2 + twin (never won beyond noise on any box). 11 -> 4 seats, only 2 raceable per box class. +- q8q8 (15 seats): keep the observed winners' axes only — `kstep2` (zen2 crown), + `mr8_budget` (M5 crown), plus the maddubs/vpdpbusd kstep2 twins under their gates. ~15 -> 4. +- `[tuned]` utilities (`dasllama_tune.das::build_grid()`, one place): grid 20 -> the union of + every winner ever recorded across zen2/m1/m4/m5 sidecars (~6: plain, u2, vec8_u2, vec16, + vec16_u2, vec4_u2). With B2 they stop racing on known classes anyway; the pruned grid is + what a new-silicon full race walks. + +**B2. Arch default profiles + the CPU/GPU sidecar split.** +- The sidecar splits into a CPU tune (kernel winners + CPU runtime knobs: jobque, par + thresholds, chunk sizes, threads, q8_l2_budget) and a GPU tune (device knobs: + metal_devw_*, metal_tensor, metal_cvt_min_rows, metal_tall_floor, batch_grid_2d, lane + caps). CPU tunes key by CPU feature-class and are portable; GPU tunes key by device name + and never travel. Old flat sidecars read back as merged (compat shim in the loader). +- Checked-in CPU profiles: `modules/dasLLAMA/performance/defaults/.tune.json`, + classes `x86-avx2` (minted from zen2), `arm-neon` (minted from m1/m5 — provably identical + today). `x86-vnni512` gets minted by the zen4 trial, `arm-i8mm`/`arm-sme` exist only once + those kernels do. +- Resolution order becomes: `perm=` pin > sidecar > **arch profile** > `fallback=` chain > + DEFAULT_PERM. Framework side (`llvm_tune.das`): a profile hit satisfies the completeness + rule — no race demanded. +- Race-on-unlock: a mint races ONLY seats whose `requires=` the host passes AND whose feature + is not already the profile class's own (zen2: nothing; zen4: the 2 vpdpbusd seats x tile + families ~= 32 races; M1: nothing; M5: nothing until i8mm/SME seats exist). Bare `--tune` + keeps meaning "race what this box unlocks"; the honest full walk moves behind + `--tune-full` (new-silicon minting). `--tune-only` unchanged. + +**B3. `DAS_TUNE_POLICY=reference`** — a policy value that serves the original bodies outright +(QUIRK 18's missing spelling). Small, framework-side. + +**B4. Validation + the measurement Boris's service decision needs.** +- dasLLVM tune tests (test_tuned/test_grid/llvm_tune_*) extended for profile resolution + + the split sidecar; gen probe test+tune modes; parity 40/40; one bench row vs current + numbers (winners are unchanged by construction — prove it, don't assume it). +- zen2: delete sidecar, cold start -> 0 races, numbers match. Wall time recorded. +- M5 (ssh): fresh mint under arm-neon profile -> 0 races. Wall time recorded. +- zen4 (ssh, off-hours — it hosts dasweb): `--tune-full` new-silicon race, wall time + recorded (target <= ~2-3 min), its x86-vnni512 profile checked in. First box where + vpdpbusd can actually win — report the crowns. +- skills/tune.md + tune_for_this_box.md + HOW_TO QUIRK 11/15/17/18 text updated in-phase. + +## Phase A — the KqPlanes table (kills QUIRK 1, 2, 3, 5, 7, 8, 13, 19, 25, 26) + +One per-format record on `Model` — quant plane, scale plane, mr, qsb/ssb strides, stream +code, schema id, device-form flag — indexed by `KqFmt`; every `fmt ==` ladder becomes a +lookup. Staged so each commit is green; the fast dev loop (`-module-cache` + +`--jit-split-modules=-1`) keeps the cycle ~1 min; after EVERY stage that touches layout or +pack, delete `*.dlim` and e2e the MIXED vehicle (the iq2xs requant carrying q2_K — QUIRK +26's lesson), not a pure one. + +- **A1.** The table + accessors (`kq_plane_q/s`, `kq_mr`, ...) + the id-derivation table + (schema id + stream code from `KqFmt` — QUIRK 5/25 become one row each) + `test_kqformat` + pins table<->enum. No consumers move yet. +- **A2.** Collapse the load/common/layout ladders (12 + 13 + 3 arms-per-format sites). +- **A3.** Collapse math_gen/math_default/math (8 + 3 + 4); the repack-mr freeze ladder + (QUIRK 8) becomes a table write; `serialize_image_meta` serializes the table and + `dlim_identity` formats from it (QUIRK 3/7 — the hand list and the count tripwire go). + Image meta layout changes -> images re-bake once; C1 lands before this so stale images + die loudly. +- **A4.** `tests/_kq_fixtures.das`: one per-format dispatch (`kq_transcode_sb`, + `kq_dequant_sb`, `kq_dot`, `kq_repack`, `kq_gemv_gen`, `kq_tile_gen`) shared by + `test_kquant` and `gen_tune_probe`; ONE family array walks both probe modes (QUIRK 2; the + stub-family rule of QUIRK 16 gets a `stub` flag in that array instead of a hand list). +- **A5.** Metal test records (QUIRK 13): per-format `{src, entry, fastmath, tgmem}` table + replaces the four parallel ternary chains. m1 sync + metal gemv/gemm gates. +- **A6.** The device gather's two branches key on the table's device-form flag (QUIRK 19). + +## Phase C — cache invalidation + +- **C1** (lands FIRST, before A3): a `PACK_VERSION` folded into the `.dlim` filename hash + (QUIRK 24) — a layout/pack edit re-bakes instead of serving stale planes. Gate: bump -> + next run logs "baked", not "mapped". +- QUIRK 21 (generator bodies into the JIT DLL hash) STAYS on `plans/jit_compile_time.md` — + cache-keying work, not this pass. + +## Phase D — one-offs + +- **D1**: posix arm for `stage_jit_backend` in `utils/mcp/setup.das` (QUIRK 12). +- **D2**: `static_if ... elif` in the MSL emitter (QUIRK 14) — timeboxed to ~2h, dropped to + the metal followup doc if the emitter fights. +- Ledgered, not done here: QUIRK 6 resolves as an INTRINSIC builtin (Boris's ruling: daslang + has no u8/u16 operator math and will not start — a `byte_or(...)`-shaped builtin, never + `|=` on uint8), SPIR-V constant-storage lowering for literal fixed_arrays (QUIRK 20, + emitter). Both proposed as lint rules too: + module-global read reachable from a worker/team-lane kernel (QUIRK 9/27, bitten twice), + dynamically-indexed fixed_array local in a kernel class (QUIRK 20's cheap half). + +## Phase E — the document split (the detour) + +`HOW_TO_ADD_A_FORMAT.md` (today: recipe + 27 quirks + 9 phase diaries) becomes: + +1. **`HOW_TO_ADD_A_FORMAT.md`** — an actual how-to: steps 0-8 rewritten post-unquirk (the + table shrinks the step count), the E-cluster rules folded in as steps/warnings (stubs + first Q4; worker-table rule Q9/27; stale-binder Q10; stub-out-of-test-list Q16; + kernel-probe-not-e2e Q22; census-the-vehicle Q23), and the quirk ledger cut to the + survivors (whatever A/B/C/D did not kill). +2. **`performance/PERF_LEDGER.md`** — the nine per-format four-tier tables, the three gap + classes, each format's vehicle GGUF, and the box inventory: which model files sit on + zen2 for testing, the m1/m5 paths, the ref llama.cpp builds per box. +3. **`followup_metal.md`** — twin of followup_vulkan.md: the Metal tg items (0.78-0.93x), + the Q22 dispatch-loop probe method, D2 if dropped. +4. The per-format phase diaries -> `/history` (doc_archiving skill), linked from the ledger. + +LAWS.md (modules/dasLLAMA) gets the rulings from this session: prune the families, saved +zen2/M1 defaults, race only on feature unlock, CPU/GPU tune split, the doc split. + +## Phase F — TWO PRs (the mid-plan cut that starts the M5 session) + +- **PR-1** (after B): the 9-format ladder the branch already carries + the tuner rework + + C1 + D1. Full make_pr chain. On merge the parallel M5/M1 session forks from master — + free mints (arm-neon profile), fast dev loop, pack-versioned images. +- **PR-2** (fresh branch off master): A (KqPlanes) + D2 + E (the doc split — it depends on + A's survivor-quirk ledger anyway). Full make_pr chain again. +- Parallel-window rule: the mac session owns kernel BODIES and additive arms (i8mm/SME + seats, Metal levers); the zen2 session's A owns dispatch/tables. A5 (the Metal test + records) runs EARLY in A's sequence to shrink the shared-file window. + +## Open questions (Boris) + +1. The new-silicon trial runs on AWS (us-west-2, CLI configured on this box), not the dasweb + zen4. No UMA + built-in-GPU x86 box exists on AWS; per goal the candidates are + c7a.2xlarge (zen4, avx512-vnni — the vnni512 mint), c7i.2xlarge (Sapphire Rapids, + vnni + AMX), c8g.2xlarge (Graviton4: i8mm + SVE2 — the arm-i8mm class once those + kernels exist), g4ad.xlarge (zen2 + Radeon V520 dGPU — Vulkan-on-AMD-driver, not UMA). + True UMA + Vulkan is Strix Halo territory, outside any cloud. Boris picks the box(es). +2. RESOLVED: profile classes key by the feature set that gates seats (winners follow + instruction sets, not box identity). Host resolves to the highest class it satisfies + that has a shipped profile, falling back down the chain (M5 today -> arm-neon). +3. RESOLVED yes: `performance/PERF_LEDGER.md` + `followup_metal.md`. +4. RESOLVED yes: D2 timeboxed, dropped to followup_metal.md if the emitter fights. + +## Rough wall-clock + +B ~1 day incl. the m5/zen4 trials; A ~1-2 days (the wide mechanical stages, each gated); +C/D hours; E ~half day; F the usual chain. Everything on `bbatkin/iquant-formats`. From 1e4067991bbfe4fefc869503e47e8208acf316e0 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Mon, 31 Aug 2026 17:33:20 -0700 Subject: [PATCH 51/61] dasLLAMA: followup_metal.md opens with the dark smmla leg The mac-session ledger the unquirk plan's Phase E fills out. First entry: the five gated smmla seats q8q8 ships have never raced anywhere - macOS returns an empty LLVMGetHostCPUFeatures so i8mm never detects on Apple Silicon - and the order of operations once B2's darwin feature map lands (prove the emitter arms under forced i8mm, race on M5, then the kq-tile smmla arm as the first real mac kernel work; SME after, research first). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01MyYCDutugdc9xL5pd2qcvT --- modules/dasLLAMA/followup_metal.md | 31 ++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 modules/dasLLAMA/followup_metal.md diff --git a/modules/dasLLAMA/followup_metal.md b/modules/dasLLAMA/followup_metal.md new file mode 100644 index 0000000000..4c44a31824 --- /dev/null +++ b/modules/dasLLAMA/followup_metal.md @@ -0,0 +1,31 @@ +# Metal / mac-session follow-up - the M5 catch-up ledger + +**Status: OPEN (2026-08-31, unquirk pass).** The ledger for the parallel mac session that +forks after PR-1 of `plans/unquirk_pass.md`. Metal-tier perf items move here from the +per-format notes during the Phase E doc split; the CPU items below are mac-session work too - +the M-series CPU tiers are minted and raced from that box. + +## 1. The dark smmla (i8mm) leg - free race, never run + +`q8q8_tile_gen` ships five `dot = "smmla"` seats gated `requires = "i8mm"` (mr4/mr8 x +kstep/nrsplit/gkstep), and NO box has ever raced them: `LLVMGetHostCPUFeatures()` returns an +EMPTY string on macOS (llvm_jit_common.das documents it beside `g_target_arm64_i8mm`), so +i8mm never detects on Apple Silicon and the seats gate-skip. M1 lacks i8mm; M2+ has it; the +M5 Max additionally has FEAT_SME2p1 + BF16/EBF16. The unquirk pass's B2 adds a darwin +host-CPU-name feature map, after which an M5 mint races smmla-vs-NEON with zero new kernel +work. Mac-session order: + +1. After B2 lands: `--tune-full` on the M5, confirm the smmla seats emit (they have never + been exercised - treat the emitter arms as unproven, gate with the gen probe TEST mode + under `DAS_JIT_ARM64_FORCE_FEATURES=i8mm` first) and report the crowns. +2. If smmla wins q8q8: the kq tile families have NO ARM ISA seats at all (mr8 NEON is the + whole grid) - an smmla kq tile emitter arm is the highest-leverage CPU kernel work on + the mac, and it transfers to Graviton3+ (c8g) verbatim. +3. SME/SME2 is the tier after: no seats, no emitter, new kernel design (streaming mode + + ZA tiles) - research first per the standing research-before-kernel-work rule. + +## 2. Metal tier perf items + +Placeholder - the per-format Metal notes (tg 0.78-0.93x tails, the IQ4_XS lane-map gap of +followup_general #58, the Q22 dispatch-loop probe method) consolidate here in Phase E of +`plans/unquirk_pass.md`; until then they live in `HOW_TO_ADD_A_FORMAT.md`'s per-format notes. From d736dfc08778eae1dbdf6e87069b2f5e22f7bcd6 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Mon, 31 Aug 2026 18:15:26 -0700 Subject: [PATCH 52/61] llvm_tune: shipped defaults profiles + DAS_TUNE_POLICY=reference (unquirk B2/B3) Kernel winners follow instruction sets, not boxes, so a scope can now ship them: [tune_scope(defaults = dir)] names a directory of checked-in .tune-defaults.json profiles (kernels-only - runtime knobs mix device and box-shape values and never travel). An untuned auto/restart start walks tune_class_chain() down from tune_cpu_class(), adopts the first version-matching profile into the app sidecar, and races only the residue: families the profile predates plus ISA-gated families this box unlocks - decided against the features fingerprint every sidecar save now stamps - armed through the existing DAS_TUNE_ONLY machinery. --tune skips adoption outright and stays the full new-silicon walk; the scope resolver seam is consulted after profiles (local answers outrank remote). DAS_TUNE_POLICY=reference is the true kill switch QUIRK 18 asked for: original bodies for [tune] families and [tuned] loop-hint kernels alike - fallback can serve the very stamp under test when the chain's first viable perm is the crown. Also unlocks the dark smmla leg on Apple Silicon: LLVMGetHostCPUFeatures is empty on macOS, so g_target_arm64_i8mm now also consults cpu_supports (sysctl truth) and the target machine appends +i8mm when the host has it - the gated q8q8 smmla seats become raceable on M2+ with no force env. dasLLAMA: the dasllama scope declares defaults=, x86-avx2 profile checked in (minted fresh on zen2 under the pruned grids: full mint 488 s, noise cv <= 0.33%), harness/export_tune_profile.das mints a profile from a sidecar. Gates: new llvm_tune_profiles.das end-to-end (adopt / race-on-unlock with the armed filter / --tune outranks / no-profile baseline) green under -jit; llvm_tune_scope + manifest + requires + modes green; reference policy verified on the scope client (STATUS source=reference); parity cold-start adopts x86-avx2 with ZERO races and reproduces the recorded stamped stream on the IQ2_XXS vehicle; lint 0 across the changed set. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01MyYCDutugdc9xL5pd2qcvT --- modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md | 18 +- .../dasLLAMA/dasllama/dasllama_math_gen.das | 3 +- modules/dasLLAMA/dasllama/dasllama_tune.das | 4 + .../dasLLAMA/harness/export_tune_profile.das | 39 ++ .../defaults/x86-avx2.tune-defaults.json | 70 ++++ modules/dasLLVM/daslib/llvm_jit_common.das | 11 +- modules/dasLLVM/daslib/llvm_tune.das | 358 +++++++++++++++++- modules/dasLLVM/tests/llvm_tune_profiles.das | 123 ++++++ .../tests/llvm_tune_profiles_client.das | 18 + .../dasLLVM/tests/llvm_tune_profiles_lib.das | 21 + .../tests/llvm_tune_profiles_tuner.das | 16 + plans/unquirk_pass.md | 39 +- skills/tune.md | 54 ++- 13 files changed, 730 insertions(+), 44 deletions(-) create mode 100644 modules/dasLLAMA/harness/export_tune_profile.das create mode 100644 modules/dasLLAMA/performance/defaults/x86-avx2.tune-defaults.json create mode 100644 modules/dasLLVM/tests/llvm_tune_profiles.das create mode 100644 modules/dasLLVM/tests/llvm_tune_profiles_client.das create mode 100644 modules/dasLLVM/tests/llvm_tune_profiles_lib.das create mode 100644 modules/dasLLVM/tests/llvm_tune_profiles_tuner.das diff --git a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md index 0f389d952f..94a30e7609 100644 --- a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md +++ b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md @@ -373,7 +373,11 @@ where, why it is so today, what unquirked looks like. An empty ledger is a legit 15. **First run on a fresh box tunes before it serves.** `run.das` on the M1 spent its first minutes minting `run.tune.json` (every family, the confirm pass included) before loading the model - the auto policy, working as designed, but a Mac session's first end-to-end - "hang" is that mint. Watch `@tune begin/end` lines in the log. + "hang" is that mint. Watch `@tune begin/end` lines in the log. UNQUIRKED for known CPU + classes: the scope ships defaults profiles (`performance/defaults/.tune-defaults.json`), + and an untuned start ADOPTS its class's profile instead of racing - the mint survives only + on new silicon (`--tune`, or the race-on-unlock residue). `skills/tune.md`, the profiles + section. 16. **A stub family fails the probe's test mode on every perm whose layout companion generates.** `q8q8_layout` is shared and generates for any perm `perm_declines` admits, so under `=> false` stubs the k3 family's `dot_maddubs_width256_mr8` perm repacked its planes @@ -386,14 +390,16 @@ where, why it is so today, what unquirked looks like. An empty ledger is a legit entry for a demanded kernel re-tunes the whole scope (the framework's completeness rule), so `run.das` on the box spent minutes minting before the Q3_K_L load - and every other sidecar on the box (`chat.das`, the harness probes) does the same on its next run. Budget - for it; do not read the first end-to-end wall time as a load-time regression. + for it; do not read the first end-to-end wall time as a load-time regression. SOFTENED by + the defaults profiles: profile-covered families adopt, and only the families the profile + predates race (the adopt path arms `--tune-only` with exactly those) - so land the new + family in the checked-in profiles (re-export on the reference boxes) in the same arc. 18. **`DAS_TUNE_POLICY=fallback` is not "the reference body".** It walks the family's `fallback=` chain, and on a box where the chain's first viable perm is the crowned one the kill switch serves the very stamp under test - the k3 stamped-vs-"reference" rows diff came - back bit-identical for that reason. The original body is reached only by a `"reference"` - sidecar entry for the family (copy the app's sidecar next to the probe as - `.tune.json`, edit the one entry). Unquirked: a policy value that names the reference - tier outright. + back bit-identical for that reason. UNQUIRKED: `DAS_TUNE_POLICY=reference` now serves the + original bodies outright - `[tune]` families and `[tuned]` loop-hint kernels alike - so an + A/B against the true reference tier is one env var, no sidecar editing. 19. **The device gather's tail-row branch is a second ladder.** `moe_gpu_gather_stack_kq` walks grouped rows (the grp interleave) and ungrouped rows (`d % mr` tails, or an unrepacked load) through two different per-format ladders; the ungrouped one re-pairs k4 diff --git a/modules/dasLLAMA/dasllama/dasllama_math_gen.das b/modules/dasLLAMA/dasllama/dasllama_math_gen.das index 282745de14..a019539b23 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_gen.das @@ -56,7 +56,8 @@ require math // entries for (the completeness set — drift shows up as a loud "missing:" list at startup). [tune_scope(name = "dasllama", tuner = "../harness/dasllama_tuner.das", covers = "dasllama_math;dasllama_math_default;dasllama_math_aarch64_neon;dasllama_kv_codec;dasllama_convert", - version_of = "dasllama_version/DASLLAMA_RELEASE", version_key = "dasllama_version")] + version_of = "dasllama_version/DASLLAMA_RELEASE", version_key = "dasllama_version", + defaults = "../performance/defaults")] struct private DasllamaTuneScope {} //! The layout companion (six-function stamp): the repack interleave (mr) the stamped tile reads. diff --git a/modules/dasLLAMA/dasllama/dasllama_tune.das b/modules/dasLLAMA/dasllama/dasllama_tune.das index bb1bbb6909..aa7e7ef99e 100644 --- a/modules/dasLLAMA/dasllama/dasllama_tune.das +++ b/modules/dasLLAMA/dasllama/dasllama_tune.das @@ -324,6 +324,10 @@ def private resolve_perm(func : FunctionPtr; args : AnnotationArgumentList) : st let fname = "{func.name}" let fb = fallback_perm(args) g_fallbacks[fname] = fb + if (tune_policy_env() == "reference") { + // the true kill switch, [tune]'s twin: the template's own loops, no hints stamped + return report_perm(fname, "plain", "reference policy") + } let av = find_arg(args, "perm") if (av is tString) return av as tString let cfg = config_perm_for(fname) diff --git a/modules/dasLLAMA/harness/export_tune_profile.das b/modules/dasLLAMA/harness/export_tune_profile.das new file mode 100644 index 0000000000..abee9f224f --- /dev/null +++ b/modules/dasLLAMA/harness/export_tune_profile.das @@ -0,0 +1,39 @@ +options gen2 + +require llvm/daslib/llvm_tune +require daslib/clargs +require daslib/fio +require daslib/module_path + +[CommandLineArgs] +struct ExportArgs { + @clarg_doc = "Sidecar to export from (default: the DAS_TUNE_MANIFEST / app-beside resolution)" + sidecar : string + + @clarg_doc = "Output directory (default: modules/dasLLAMA/performance/defaults beside this script)" + out_dir : string + + @clarg_doc = "CPU class name to export as (default: this box's tune_cpu_class())" + klass : string +} + +//! Export a minted sidecar's kernel winners as the shipped defaults profile for this box's +//! CPU class — run on a reference box after a full `--tune` mint, then check the file in. +//! `daslang harness/export_tune_profile.das -- --sidecar benchmarks/lcpp_bench.tune.json` +[export] +def main : int { + var r <- parse_args(type) + if (r |> is_err) { + print("{r |> unwrap_err}\n") + return 1 + } + let cfg <- r |> move_unwrap + if (!empty(cfg.sidecar)) { + set_tune_manifest_runtime_path(cfg.sidecar) + } + let klass = empty(cfg.klass) ? tune_cpu_class() : cfg.klass + let outDir = empty(cfg.out_dir) ? path_join(path_join(get_this_module_dir(), ".."), "performance/defaults") : cfg.out_dir + mkdir(outDir) + let outPath = path_join(outDir, "{klass}.tune-defaults.json") + return tune_profile_export(outPath, klass) ? 0 : 1 +} diff --git a/modules/dasLLAMA/performance/defaults/x86-avx2.tune-defaults.json b/modules/dasLLAMA/performance/defaults/x86-avx2.tune-defaults.json new file mode 100644 index 0000000000..936d2e9b4a --- /dev/null +++ b/modules/dasLLAMA/performance/defaults/x86-avx2.tune-defaults.json @@ -0,0 +1,70 @@ +{ + "kernels" : { + "add_inplace" : "vec8_u2", + "cvt_f32_to_f16" : "vec8_u2", + "rope_scaled_neox_tab" : "plain", + "q51q8_tile_gen" : "dot_maddubs_width256_mr8", + "softmax" : "vec8_u2", + "mul_inplace" : "vec8_u2", + "quantize_q8_0_bs_into_ptr" : "vec16_u2", + "iq2xsq8_tile_gen" : "dot_maddubs_width256_mr8", + "dot_q8kv" : "vec4_u4", + "dot_q8q8" : "vec16", + "iq4xsq8_tile_gen" : "dot_maddubs_width256_mr8", + "iq4nlq8_tile_gen" : "dot_maddubs_width256_mr8", + "quantize_q8kv_row" : "plain", + "axpy_f16" : "vec8_u2", + "q40q8_tile_gen" : "dot_maddubs_width256_mr8", + "axpy_tq4kv" : "vec8_u2", + "cvt_tq4kv_to_f32" : "vec8_u2", + "axpy" : "vec8_u2", + "dot_q8q8kv" : "vec16", + "dot_mx4q8" : "u2", + "softmax_sink" : "vec8_u2", + "iq2xxsq8_tile_gen" : "dot_maddubs_width256_mr8", + "dot_q8q8_laneq4x4" : "", + "dot_bf16" : "vec8_u2", + "iq2sq8_tile_gen" : "dot_maddubs_width256_mr8", + "add_scale_inplace" : "vec8_u2", + "cvt_q8kv_to_f32" : "vec8_u2", + "axpy_q8kv" : "vec8_u2", + "dot_q8q8_f16s" : "vec16", + "q8q8_tile_gen" : "dot_maddubs_width256_mr8_kstep2", + "quantize_q8_0_into_ptr" : "plain", + "k4q8_tile_gen" : "dot_maddubs_width256_mr8", + "k5q8_tile_gen" : "dot_maddubs_width256_mr8", + "k6q8_tile_gen" : "dot_maddubs_width256_mr8", + "k3q8_tile_gen" : "dot_maddubs_width256_mr8", + "k2q8_tile_gen" : "dot_maddubs_width256_mr8", + "gemm_f32_uk_4x16" : "plain", + "dot_q51e" : "vec16", + "dot_f16" : "vec8_u2", + "cvt_f16_to_f32" : "vec8_u2", + "dot" : "vec8_u2", + "iq3xxsq8_tile_gen" : "dot_maddubs_width256_mr8", + "iq3sq8_tile_gen" : "dot_maddubs_width256_mr8", + "dot_q8tq4kv" : "vec16", + "scale_inplace" : "vec8_u2", + "dot_q4" : "vec8_u2", + "quantize_tq4kv_row" : "plain", + "copy_floats" : "vec8_u2", + "rmsnorm" : "vec16_u2" + }, + "provenance" : { + "validation" : "ok", + "noise_probes" : "start cv 0.28%; mid1 cv 0.33%; mid2 cv 0.15%; end cv 0.31%", + "platform" : "windows", + "noise_floor_cv_pct" : "0.33", + "features" : "avx2;f16c;fma;sse4.2", + "engine_sha" : "cb659766d", + "class" : "x86-avx2", + "written" : "2026-09-01T01:06:53.218Z", + "validation_demoted" : "0", + "mode" : "normal", + "dasllama_version" : "15", + "origin" : "profile", + "noise" : "ok", + "arch" : "x86_64", + "validation_max_drift_pct" : "19.46" + } +} \ No newline at end of file diff --git a/modules/dasLLVM/daslib/llvm_jit_common.das b/modules/dasLLVM/daslib/llvm_jit_common.das index dd392a1960..a2a8e7b2f0 100644 --- a/modules/dasLLVM/daslib/llvm_jit_common.das +++ b/modules/dasLLVM/daslib/llvm_jit_common.das @@ -633,9 +633,10 @@ def public init_jit_target_flags(target_triple : string = ""; host_features : bo ? get_platform_name() == "linux" : find(target_triple, "linux") >= 0) // aarch64 i8mm tier (see the g_target_arm64_i8mm declaration comment): host targets read - // LLVM's host-features string OR the arm force env; cross aarch64 triples the env only. + // LLVM's host string OR cpu_supports — macOS hands back an EMPTY host string, which left + // i8mm dark on every Apple M2+ — OR the arm force env; cross aarch64 triples the env only. g_target_arm64_i8mm = g_target_is_aarch64 && (arm64_forced_feature("i8mm") - || (empty(target_triple) && host_features && host_llvm_feature("i8mm"))) + || (empty(target_triple) && host_features && (host_llvm_feature("i8mm") || cpu_supports("i8mm")))) // aarch64 fullfp16 (see declaration comment): darwin-arm64 == Apple Silicon == always. // Generic-exe builds stay off — the promote/narrow path is ARMv8.0-legal, fullfp16 is not. g_target_arm64_fullfp16 = g_target_is_aarch64 && (arm64_forced_feature("fullfp16") @@ -921,6 +922,12 @@ def public create_default_target_machine(opt_level : uint; use_host_cpu : bool) var targetMachine : LLVMTargetMachineRef if (use_host_cpu && g_target_is_aarch64) { var feats = empty(features_msg) ? "+dotprod" : "{features_msg},+dotprod" + // same protection for i8mm: an Apple part this LLVM cannot name maps to "generic" where + // SMMLA has no instruction to select. cpu_supports reads sysctl truth, so the append is + // execution-safe on this host (unlike the force env, which may name absent silicon). + if (cpu_supports("i8mm")) { + feats = "{feats},+i8mm" + } // DAS_JIT_ARM64_FORCE_FEATURES (e.g. "i8mm"): the x64 rail's twin — a pre-i8mm host EMITS // AND LINKS an artifact for newer silicon, which the same-env run on the target box then // cache-hits without a linker. Executing forced instructions here SIGILLs; gate execution. diff --git a/modules/dasLLVM/daslib/llvm_tune.das b/modules/dasLLVM/daslib/llvm_tune.das index bcd3d48738..67dbea3e9f 100644 --- a/modules/dasLLVM/daslib/llvm_tune.das +++ b/modules/dasLLVM/daslib/llvm_tune.das @@ -421,12 +421,13 @@ def private load_sidecar_doc(path : string) : JsonValue? { // version stamp, and two pinned scopes then invalidate each other (the auto policy loops) def private save_sidecar_doc(path : string; var doc : JsonValue?) : bool { var prov = JV((binary = running_binary(), platform = get_platform_name(), - arch = get_architecture_name(), box = tune_box_identity(), written = iso8601_now())) + arch = get_architecture_name(), box = tune_box_identity(), written = iso8601_now(), + features = tune_feature_fingerprint())) let old = doc?["provenance"] if (old != null && old is _object) { for (k, v in keys(old as _object), values(old as _object)) { continue if (k == "binary" || k == "platform" || k == "arch" || k == "box" || k == "written" - || g_prov_notes |> key_exists(k)) + || k == "features" || g_prov_notes |> key_exists(k)) if (v != null && v is _string) { update(prov, k, JV(v as _string)) } elif (v != null && v is _number) { @@ -539,6 +540,40 @@ def tune_sidecar_section_merge(section : string; var entries : table JV(v as _string); where v != null && v is _string} + var prov : table + let old = doc?["provenance"] + if (old != null && old.value is _object) { + for (k, v in keys(old as _object), values(old as _object)) { + continue if (k == "binary" || k == "box" || k == "written" || k == "origin" || k == "class") + if (v != null && v is _string) { + prov[k] = JV(v as _string) + } + } + } + prov["origin"] = JV("profile") + prov["class"] = JV(klass) + prov["written"] = JV(iso8601_now()) + var out = JV({ "kernels" => JV(kernels), "provenance" => JV(prov) }) + let ok = fwrite(outPath, write_json(out)) + print(ok ? "llvm_tune: profile '{klass}' exported -> {outPath} ({length(ks as _object)} kernel entries)\n" + : "llvm_tune: profile export FAILED writing {outPath}\n") + delete_json(out) + delete_json(doc) + return ok +} + // ===== the tune history (~/.tune-history) ===== var private g_history_dir = "" @@ -789,6 +824,7 @@ struct private TuneScope { tunerPath : string // resolved against the declaring file; "" = no tuner declared versionKey : string // provenance key the scope pins ("" = unpinned) versionExpected : string + defaultsPath : string // resolved against the declaring file; "" = no shipped profiles } // `version_of = "module/CONST"` on a [tune_scope]: winners are valid only while the sidecar's @@ -880,6 +916,46 @@ def private scope_fnames_joined(scope : TuneScope) : string { return joined } +// the ';'-joined `fname=exprA+exprB` map of a scope's ISA-gated [tune] families (every +// requires= its [tune_perm] rows carry), read off the AST at guard-emission time — what lets +// the runtime guard ask which families have seats the adopted profile's box could not race +def private scope_gated_joined(scope : TuneScope) : string { + var wanted : table + wanted |> insert(scope.modName) + for (m in split(scope.covers, ";")) { + if (!empty(m)) { + wanted |> insert(m) + } + } + var rows : array + program_for_each_module(compiling_program()) $(mod) { + if (wanted |> key_exists("{mod.name}")) { + for_each_function(mod, "") $(fn) { + var isTune = false + var exprs : array + for (ann in fn.annotations) { + if (ann.annotation.name == "tune") { + isTune = true + } elif (ann.annotation.name == "tune_perm") { + let rv = find_arg(ann.arguments, "requires") + if (rv is tString && !empty(rv as tString)) { + exprs |> push(rv as tString) + } + } + } + if (isTune && !empty(exprs)) { + rows |> push("{fn.name}={join(exprs, "+")}") + } + delete exprs + } + } + } + let joined = join(rows, ";") + delete wanted + delete rows + return joined +} + //! One `[tune_scope]`'s identity + completeness, as reported by `tune_scopes_status()` — //! the macro-time surface build orchestrators (the `-exe` release-deps JSON, daspkg) read //! to decide whether to run tuners before a rebuild. @@ -1157,6 +1233,87 @@ def tune_pick_fallback(chain : string) : string { return "" } +// every feature name a requires= expression may query, in one place — the fingerprint and the +// class ladder both walk it, so a new gate name added here is immediately profile-aware +let private TUNE_KNOWN_FEATURES = "avx2;f16c;fma;sse4.2;avxvnni;avxvnniint8;avx512f;avx512bw;avx512vl;avx512vnni;amx-tile;amx-int8;neon;dotprod;i8mm;bf16;fullfp16;sve;lse" + + //! The ';'-joined subset of the known `requires=` feature names this box passes — stamped + //! into every sidecar's provenance as `features`, which is what lets a shipped profile say + //! exactly which gated seats its minting box could and could not race. +def tune_feature_fingerprint() : string { + var have <- [for (f in split(TUNE_KNOWN_FEATURES, ";")); f; where fallback_feature_ok(f)] + let joined = join(have, ";") + delete have + return joined +} + + //! This box's CPU feature class — the name a shipped defaults profile is keyed by. Classes + //! follow the seats `requires=` can gate, not box models: two boxes of one class stamp the + //! same winners (measured spread inside a class is noise), so one profile serves them all. +def tune_cpu_class() : string { + let arch = get_architecture_name() + if (arch == "x86_64") { + if (fallback_feature_ok("avx512vnni") && fallback_feature_ok("avx512bw")) { + return "x86-vnni512" + } + if (fallback_feature_ok("avxvnni")) { + return "x86-vnni256" + } + return fallback_feature_ok("avx2") ? "x86-avx2" : "x86-base" + } + if (arch == "arm64" || arch == "aarch64") { + return fallback_feature_ok("i8mm") ? "arm-i8mm" : "arm-neon" + } + return "{get_platform_name()}-{arch}" // an honest key; no profile will match, and that is correct +} + + //! Profile candidates for this box, most specific first — a box above the best shipped + //! profile adopts the nearest lower class and races only what that class could not. +def tune_class_chain() : array { + let ladder = (get_architecture_name() == "x86_64" + ? "x86-vnni512;x86-vnni256;x86-avx2;x86-base" + : "arm-i8mm;arm-neon") + let own = tune_cpu_class() + var chain : array + var seen = false + for (c in split(ladder, ";")) { + seen ||= c == own + if (seen) { + chain |> push(c) + } + } + if (!seen) { + chain |> push(own) + } + return <- chain +} + +// evaluate a requires= expression against a ';'-joined feature set instead of this box — +// "did the PROFILE's minting box pass this gate" is what separates covered from unlocked seats +def private expr_in_features(expr : string; featCsv : string) : bool { + return true if (empty(expr)) + var have : table + for (f in split(featCsv, ";")) { + if (!empty(f)) { + have |> insert(f) + } + } + var ok = true + for (conj in split(expr, ",")) { + continue if (empty(conj)) + var anyOk = false + for (alt in split(conj, "|")) { + anyOk ||= !empty(alt) && have |> key_exists(alt) + } + if (!anyOk) { + ok = false + break + } + } + delete have + return ok +} + def private perm_requires_ok(p : TunePerm) : bool { return tune_requires_ok(p.requires) } @@ -1456,6 +1613,20 @@ class private TuneAnnotation : AstFunctionAnnotation { delete comps return false } + // DAS_TUNE_POLICY=reference is the true kill switch: original bodies, no stamps at all — + // unlike `fallback`, whose chain's first viable perm can be the very stamp under test + if (g_env_tune.tune_policy == "reference") { + let noPerms : array + var rok = emit_registry(compiling_module(), func, noPerms, errors) + for (c in comps) { + if (rok) { + rok = emit_registry(compiling_module(), c.fn, noPerms, errors) + } + } + delete perms + delete comps + return rok + } // no entry falls back rather than dropping to reference, so a sidecar written before a kernel family landed cannot untune it let fname = "{func.name}" var manifestFound = false @@ -1591,11 +1762,17 @@ class private TuneScopeAnnotation : AstStructureAnnotation { return false } } + var defaults = "" + let dv = find_arg(args, "defaults") + if (dv is tString) { + defaults = resolve_decl_path(string(st.at.fileInfo.name), dv as tString) + } // overwrite, not error: shared-module macro state persists across compiles in one process g_scopes["{compiling_module().name}"] = TuneScope(scopeName = sname, modName = "{compiling_module().name}", covers = covers, manifestPath = tune_manifest_path(), tunerPath = tuner, - versionKey = versionKey, versionExpected = versionExpected) + versionKey = versionKey, versionExpected = versionExpected, + defaultsPath = defaults) return true } } @@ -1956,15 +2133,131 @@ def public tune_suppress_mint(knob : string) { g_mint_suppressed = knob } - //! Runtime half of `[tune_policy(missing = "auto")]` / `--tune`, injected at the top of the - //! app's `main` and followed by `tune_auto_reexec()`. Tuning happens at RUNTIME because the - //! winners are compile-time stamps. An empty or corrupt sidecar reads as untuned, so it self-heals. -def tune_auto_prepare(scopeName : string; manifestPath : string; tunerPath : string; force : bool; fnames : string) { +// ===== shipped defaults profiles ([tune_scope(defaults = "dir")]) ===== + +// satisfy an untuned scope from a shipped class profile before any race: "" = nothing adopted, +// "*" = fully covered, else the ','-joined families still owed a race (profile-missing plus +// ISA-unlocked), which the caller arms as the DAS_TUNE_ONLY filter for the tuner it spawns +def private profile_try_adopt(defaultsDir, scopeName, manifestPath, fnames, gated : string) : string { // nolint:STYLE037,STYLE038 — one straight adopt pipeline: locate, validate, merge, diff + // a redirected runtime manifest path must never adopt into the wrong file + return "" if (empty(defaultsDir) || tune_manifest_path() != manifestPath) + // the version pin from the demand: the profile must record the same value or it predates this build + var verKey = "" + var verVal = "" + for (tok in split(fnames, ";")) { + if (tok |> starts_with(TUNE_VERSION_TOKEN)) { + let kv = slice(tok, length(TUNE_VERSION_TOKEN)) + let eq = find(kv, "=") + if (eq >= 0) { + verKey = slice(kv, 0, eq) + verVal = slice(kv, eq + 1) + } + } + } + var chain <- tune_class_chain() + var klass = "" + var doc : JsonValue? + for (c in chain) { + let p = path_join(defaultsDir, "{c}.tune-defaults.json") + let text = fread(p) + continue if (empty(text)) + var err = "" + var d = read_json(text, err) + if (d == null || !(d.value is _object)) { + print("llvm_tune: defaults profile {p} is not valid JSON ({err}) - skipped\n") + delete_json(d) + continue + } + if (!empty(verKey) && "{d?["provenance"]?[verKey] ?? ""}" != verVal) { + print("llvm_tune: defaults profile '{c}' predates this build ({verKey} != {verVal}) - skipped\n") + delete_json(d) + continue + } + klass = c + doc = d + break + } + delete chain + return "" if (doc == null) + var kernels : table + let ks = doc?["kernels"] + if (ks != null && ks.value is _object) { + for (k, v in keys(ks as _object), values(ks as _object)) { + if (v != null && v is _string) { + kernels[k] = v as _string + } + } + } + if (empty(kernels)) { + delete_json(doc) + return "" + } + // owed a race: demanded-but-absent from the profile, plus gated families this box unlocks + var race : array + for (tok in split(fnames, ";")) { + continue if (empty(tok) || tok |> starts_with(TUNE_VERSION_TOKEN)) + if (!(kernels |> key_exists(tok))) { + race |> push(tok) + } + } + let pfeat = "{doc?["provenance"]?["features"] ?? ""}" + for (row in split(gated, ";")) { + continue if (empty(row)) + let eq = find(row, "=") + continue if (eq < 0) + let fam = slice(row, 0, eq) + continue if (race |> find_index(fam) >= 0) + for (expr in split(slice(row, eq + 1), "+")) { + continue if (empty(expr)) + if (tune_requires_ok(expr) && !expr_in_features(expr, pfeat)) { + race |> push(fam) + break + } + } + } + if (!empty(verKey)) { + tune_provenance_note(verKey, verVal) + } + tune_provenance_note("adopted_profile", klass) + var noRuntime : JsonValue? + let merged = tune_sidecar_merge(kernels, noRuntime) + if (!merged) { + print("llvm_tune: scope '{scopeName}' could not write adopted profile '{klass}' into {manifestPath}\n") + delete kernels + delete race + delete_json(doc) + return "" + } + let tail = empty(race) ? "" : " - racing {length(race)} unlocked/new families" + print("llvm_tune: scope '{scopeName}' adopted the '{klass}' defaults profile ({length(kernels)} kernel entries){tail}\n") + let res = empty(race) ? "*" : join(race, ",") + delete kernels + delete race + delete_json(doc) + return res +} + + //! Runtime half of `[tune_policy(missing = "auto")]` / `--tune`, injected at the top of `main` + //! and followed by `tune_auto_reexec()`. Adopts a declared defaults profile first, then tunes + //! the residue at RUNTIME (winners are compile-time stamps). A corrupt sidecar self-heals. +def tune_auto_prepare(scopeName : string; manifestPath : string; tunerPath : string; force : bool; fnames : string; + defaultsDir : string = ""; gated : string = "") { let v = tune_sidecar_verdict(manifestPath, fnames) if (!force && v.complete) { return } - if (!force && scope_resolved_externally(scopeName, manifestPath, fnames)) { + var onlyFilter = "" + if (!force) { + let adopt = profile_try_adopt(defaultsDir, scopeName, manifestPath, fnames, gated) + if (adopt == "*") { + g_tuned_scopes++ // fully covered by the shipped profile; the re-exec stamps it + return + } + if (!empty(adopt)) { + onlyFilter = adopt // adopted a base; the tuner below races only what it lacked + } + } + if (!force && empty(onlyFilter) && scope_resolved_externally(scopeName, manifestPath, fnames)) { g_tuned_scopes++ // the re-exec picks the resolved sidecar up like a fresh tune return } @@ -1974,8 +2267,14 @@ def tune_auto_prepare(scopeName : string; manifestPath : string; tunerPath : str } return } + if (!empty(onlyFilter)) { + set_env_variable("DAS_TUNE_ONLY", onlyFilter) // the spawned tuner inherits the filter + } announce_tuning(scopeName, force, true, v.detail) run_scope_tuner(TuneScope(scopeName = scopeName, manifestPath = manifestPath, tunerPath = tunerPath)) + if (!empty(onlyFilter)) { + set_env_variable("DAS_TUNE_ONLY", "") // the re-exec child must race unfiltered if it ever tunes + } let after = tune_sidecar_verdict(manifestPath, fnames) if (after.complete) { g_tuned_scopes++ @@ -2064,6 +2363,12 @@ def tune_interrupt_requested() : bool { return stat(p, fs) } + //! The `DAS_TUNE_POLICY` override, verbatim ("" = none) — how a sibling stamping macro + //! (e.g. a library's `[tuned]` loop-hint rail) honors `reference` and `fallback` too. +def tune_policy_env() : string { + return g_env_tune.tune_policy +} + //! The `--tune-only` re-mint filter (`DAS_TUNE_ONLY`), verbatim: comma-separated name //! tokens, empty = race everything. def tune_only_filter() : string { @@ -2090,12 +2395,24 @@ def tune_family_selected(name : string) : bool { //! Runtime half of `[tune_policy(missing = "restart")]`, injected as a guard at the top of //! the app's `main`: consults the scope resolver first, else runs the tuner; true = main //! returns immediately (winners are compile-time stamps), false = scope complete, carry on. -def tune_restart_needed(scopeName : string; manifestPath : string; tunerPath : string; fnames : string) : bool { +def tune_restart_needed(scopeName : string; manifestPath : string; tunerPath : string; fnames : string; + defaultsDir : string = ""; gated : string = "") : bool { let v = tune_sidecar_verdict(manifestPath, fnames) if (v.complete) { return false } - if (scope_resolved_externally(scopeName, manifestPath, fnames)) { + var onlyFilter = "" + { + let adopt = profile_try_adopt(defaultsDir, scopeName, manifestPath, fnames, gated) + if (adopt == "*") { + print("llvm_tune: scope '{scopeName}' adopted -> {manifestPath}\nllvm_tune: restart to apply the winners\n") + return true + } + if (!empty(adopt)) { + onlyFilter = adopt + } + } + if (empty(onlyFilter) && scope_resolved_externally(scopeName, manifestPath, fnames)) { return true // resolved without a tune; the resolver printed its own restart marker } if (!empty(g_mint_suppressed)) { @@ -2111,7 +2428,13 @@ def tune_restart_needed(scopeName : string; manifestPath : string; tunerPath : s return true } announce_tuning(scopeName, false, false, v.detail) + if (!empty(onlyFilter)) { + set_env_variable("DAS_TUNE_ONLY", onlyFilter) // the spawned tuner inherits the filter + } run_scope_tuner(TuneScope(scopeName = scopeName, manifestPath = manifestPath, tunerPath = tunerPath)) + if (!empty(onlyFilter)) { + set_env_variable("DAS_TUNE_ONLY", "") + } let after = tune_sidecar_verdict(manifestPath, fnames) if (after.complete) { print("llvm_tune: scope '{scopeName}' tuned -> {manifestPath}\nllvm_tune: restart to apply the winners\n") @@ -2232,8 +2555,8 @@ def private apply_tune_policy(var func : FunctionPtr; declared : string; var err envNote = "llvm_tune: policy '{policy}' set by DAS_TUNE_POLICY (declared/default '{declared}')\n" } } - if (policy != "fallback" && policy != "warn" && policy != "error" && policy != "auto" && policy != "restart") { - errors = "[tune_policy] on {func.name}: missing=\"{policy}\" - use fallback | warn | error | auto | restart" + if (policy != "fallback" && policy != "reference" && policy != "warn" && policy != "error" && policy != "auto" && policy != "restart") { + errors = "[tune_policy] on {func.name}: missing=\"{policy}\" - use fallback | reference | warn | error | auto | restart" return false } let force = apply_cli_tune_flags() @@ -2276,15 +2599,18 @@ def private apply_tune_policy(var func : FunctionPtr; declared : string; var err errors = "[tune_policy]: restart - {func.name} must return void or int" } else { var guard : ExpressionPtr + let scopeGated = scope_gated_joined(scope) if (func.result.baseType == Type.tInt) { guard = qmacro_expr() { - if (tune_restart_needed($v(scope.scopeName), $v(scope.manifestPath), $v(scope.tunerPath), $v(scopeFnames))) { + if (tune_restart_needed($v(scope.scopeName), $v(scope.manifestPath), $v(scope.tunerPath), $v(scopeFnames), + $v(scope.defaultsPath), $v(scopeGated))) { return 3 } } } else { guard = qmacro_expr() { - if (tune_restart_needed($v(scope.scopeName), $v(scope.manifestPath), $v(scope.tunerPath), $v(scopeFnames))) { + if (tune_restart_needed($v(scope.scopeName), $v(scope.manifestPath), $v(scope.tunerPath), $v(scopeFnames), + $v(scope.defaultsPath), $v(scopeGated))) { return } } @@ -2310,8 +2636,10 @@ def private apply_tune_policy(var func : FunctionPtr; declared : string; var err guards |> reserve(length(autoScopes) + 1) for (scope in autoScopes) { let fnames = scope_fnames_joined(scope) + let gated = scope_gated_joined(scope) guards |> push <| qmacro_expr() { - tune_auto_prepare($v(scope.scopeName), $v(scope.manifestPath), $v(scope.tunerPath), $v(force), $v(fnames)) + tune_auto_prepare($v(scope.scopeName), $v(scope.manifestPath), $v(scope.tunerPath), $v(force), $v(fnames), + $v(scope.defaultsPath), $v(gated)) } } if (func.result.baseType == Type.tInt) { diff --git a/modules/dasLLVM/tests/llvm_tune_profiles.das b/modules/dasLLVM/tests/llvm_tune_profiles.das new file mode 100644 index 0000000000..7f46857eee --- /dev/null +++ b/modules/dasLLVM/tests/llvm_tune_profiles.das @@ -0,0 +1,123 @@ +options gen2 + +require dastest/testing_boost +require daslib/module_path +require daslib/strings_boost +require daslib/fio +require strings +require llvm/daslib/llvm_tune + +// The shipped-defaults-profile rail ([tune_scope(defaults = ...)]) through child daslang +// processes: an untuned start adopts the class profile instead of racing (source=manifest, +// no tuner spawn); a profile whose minting box lacked a feature this box has races ONLY the +// unlocked family (DAS_TUNE_ONLY armed for the tuner, cleared after); no profile file = +// the old full race. The profile files are written HERE — their name carries this box's +// tune_cpu_class(), and their features field is this box's fingerprint (or empty, to fake +// a less-capable minting box). + +def private tune_env() : string { + return (get_platform_name() == "windows" ? + "set DAS_TUNE_MODE=&& set DAS_TUNE_MANIFEST=&& set DAS_TUNE_VERBOSITY=verbose&& set DAS_TUNE_POLICY=&& set DAS_TUNE_ONLY=&& " : + "DAS_TUNE_MODE= DAS_TUNE_MANIFEST= DAS_TUNE_VERBOSITY=verbose DAS_TUNE_POLICY= DAS_TUNE_ONLY= ") +} + +def private spawn_child(cmd : string; var lines : array) : int { + var rc : int + unsafe { + rc = popen_timeout("{cmd} 2>&1", 300.0) $(f) { + if (f == null) { + return + } + while (!feof(f)) { + let ln = strip(fgets(f)) + if (!empty(ln)) { + lines |> push("{ln}") + } + } + } + } + return rc +} + +def private has(lines : array; needle : string) : bool { + for (ln in lines) { + if (find(ln, needle) >= 0) { + return true + } + } + return false +} + +def private write_profile(dir, klass, winner, features : string) { + mkdir(dir) + let body = "\{ \"kernels\" : \{ \"prof_add\" : \"{winner}\" \}, \"provenance\" : \{ \"features\" : \"{features}\" \} \}" + fwrite(path_join(dir, "{klass}.tune-defaults.json"), body) +} + +[test] +def test_llvm_tune_profiles(t : T?) { + if (!jit_enabled()) { + // the stamped-tier fingerprints need the generator tier; children run -jit + t |> success(true) + return + } + let args <- get_command_line_arguments() // locked view — do not delete + let bin = args[0] + let client = "{get_this_module_dir()}/llvm_tune_profiles_client.das" + let sidecar = "{get_this_module_dir()}/llvm_tune_profiles_client.tune.json" + let defdir = "{get_this_module_dir()}/llvm_tune_profiles_defaults" + let klass = tune_cpu_class() + let profile = path_join(defdir, "{klass}.tune-defaults.json") + + // 1. no profile file: the plain auto flow — full race (empty filter), the tuner's k2 runs + remove(sidecar) + remove(profile) + var l1 : array + t |> equal(spawn_child("{tune_env()}\"{bin}\" -jit \"{client}\"", l1), 0) + t |> success(l1 |> has("FAKE TUNER")) + t |> success(l1 |> has("ONLY=''")) + t |> success(!(l1 |> has("defaults profile"))) + t |> success(l1 |> has("RESULT 7")) + delete l1 + + // 2. full-cover profile (this box's own fingerprint): adopted, no tuner, its winner stamps + remove(sidecar) + write_profile(defdir, klass, "k0", tune_feature_fingerprint()) + var l2 : array + t |> equal(spawn_child("{tune_env()}\"{bin}\" -jit \"{client}\"", l2), 0) + t |> success(l2 |> has("adopted the '{klass}' defaults profile")) + t |> success(!(l2 |> has("FAKE TUNER"))) + t |> success(l2 |> has("RESULT 5")) + t |> success(l2 |> has("STATUS prof_add|llvm_tune_profiles_probe|k0|manifest")) + let side2 = fread(sidecar) + t |> success(find(side2, "\"adopted_profile\"") >= 0) + delete l2 + + // 3. a profile minted on a less-capable box (empty features): the k3 requires= row is an + // unlocked seat here, so the guard adopts AND races just this family — the tuner sees + // the filter and its k2 outranks the adopted k0 + remove(sidecar) + write_profile(defdir, klass, "k0", "") + var l3 : array + t |> equal(spawn_child("{tune_env()}\"{bin}\" -jit \"{client}\"", l3), 0) + t |> success(l3 |> has("racing 1 unlocked/new families")) + t |> success(l3 |> has("ONLY='prof_add'")) + t |> success(l3 |> has("RESULT 7")) + t |> success(l3 |> has("STATUS prof_add|llvm_tune_profiles_probe|k2|manifest")) + delete l3 + + // 4. --tune outranks the profile: full re-race, no adoption + var l4 : array + t |> equal(spawn_child("{tune_env()}\"{bin}\" -jit \"{client}\" -- --tune", l4), 0) + t |> success(!(l4 |> has("defaults profile"))) + t |> success(l4 |> has("FAKE TUNER")) + delete l4 + + remove(sidecar) + remove(profile) +} + +[export] +def main { + return +} diff --git a/modules/dasLLVM/tests/llvm_tune_profiles_client.das b/modules/dasLLVM/tests/llvm_tune_profiles_client.das new file mode 100644 index 0000000000..1af5b55381 --- /dev/null +++ b/modules/dasLLVM/tests/llvm_tune_profiles_client.das @@ -0,0 +1,18 @@ +options gen2 + +require llvm_tune_profiles_lib + +// App-face fixture for llvm_tune_profiles.das (no [test] on purpose — the suite just +// compiles it): default policy (auto), so an untuned start walks adopt -> filtered race -> +// re-exec. RESULT is the k-fingerprint of the stamped tier (jit: 2+3+k; interp reference: 5). +[export] +def main : int { + var n = 2 // nolint:LINT003 — var defeats const-fold so the call survives to codegen + print("RESULT {prof_add(n, 3)}\n") + var st <- tune_status() + for (s in st) { + print("STATUS {s.fname}|{s.scope}|{s.suffix}|{s.source}\n") + } + delete st + return 0 +} diff --git a/modules/dasLLVM/tests/llvm_tune_profiles_lib.das b/modules/dasLLVM/tests/llvm_tune_profiles_lib.das new file mode 100644 index 0000000000..102da66cd2 --- /dev/null +++ b/modules/dasLLVM/tests/llvm_tune_profiles_lib.das @@ -0,0 +1,21 @@ +options gen2 + +module llvm_tune_profiles_lib public + +require llvm/daslib/llvm_tune public + +// Library-side fixture for llvm_tune_profiles.das (no [test] on purpose — the suite just +// compiles it): a scope that also ships defaults profiles. The defaults dir is created and +// filled BY THE TEST (the profile file name carries this box's tune_cpu_class(), which the +// test computes in-process). The k3 row's requires= passes on every test box (avx2 on x64, +// neon on arm) — the "unlocked seat" whose absence from a profile's features forces a race. +[tune_scope(name = "llvm_tune_profiles_probe", tuner = "./llvm_tune_profiles_tuner.das", + defaults = "./llvm_tune_profiles_defaults")] +struct private ScopeDecl {} + +[tune_perm(k = 0), tune_perm(k = 1), tune_perm(k = 2), + tune_perm(k = 3, requires = "avx2|neon"), + tune(gen = "llvm_code_selftest::add_plus_k", fallback = "k1")] +def prof_add(a, b : int) : int { + return a + b +} diff --git a/modules/dasLLVM/tests/llvm_tune_profiles_tuner.das b/modules/dasLLVM/tests/llvm_tune_profiles_tuner.das new file mode 100644 index 0000000000..64c3d2cb33 --- /dev/null +++ b/modules/dasLLVM/tests/llvm_tune_profiles_tuner.das @@ -0,0 +1,16 @@ +options gen2 + +require llvm/daslib/llvm_tune +require daslib/fio + +// Instant fake tuner for llvm_tune_profiles.das (no [test] on purpose): prints the +// DAS_TUNE_ONLY filter the guard armed (the race-on-unlock proof) and records k2 — a winner +// distinct from both the fallback (k1) and the profiles the test writes, so RESULT +// fingerprints which path stamped. +[export] +def main { + print("FAKE TUNER -> {tune_manifest_path()} ONLY='{get_env_variable("DAS_TUNE_ONLY")}'\n") + if (tune_family_selected("prof_add")) { + tune_manifest_set("prof_add", "k2") + } +} diff --git a/plans/unquirk_pass.md b/plans/unquirk_pass.md index e9c720640e..0bc3e1b262 100644 --- a/plans/unquirk_pass.md +++ b/plans/unquirk_pass.md @@ -32,24 +32,27 @@ outside this PR). vec16_u2, vec4_u2). With B2 they stop racing on known classes anyway; the pruned grid is what a new-silicon full race walks. -**B2. Arch default profiles + the CPU/GPU sidecar split.** -- The sidecar splits into a CPU tune (kernel winners + CPU runtime knobs: jobque, par - thresholds, chunk sizes, threads, q8_l2_budget) and a GPU tune (device knobs: - metal_devw_*, metal_tensor, metal_cvt_min_rows, metal_tall_floor, batch_grid_2d, lane - caps). CPU tunes key by CPU feature-class and are portable; GPU tunes key by device name - and never travel. Old flat sidecars read back as merged (compat shim in the loader). -- Checked-in CPU profiles: `modules/dasLLAMA/performance/defaults/.tune.json`, - classes `x86-avx2` (minted from zen2), `arm-neon` (minted from m1/m5 — provably identical - today). `x86-vnni512` gets minted by the zen4 trial, `arm-i8mm`/`arm-sme` exist only once - those kernels do. -- Resolution order becomes: `perm=` pin > sidecar > **arch profile** > `fallback=` chain > - DEFAULT_PERM. Framework side (`llvm_tune.das`): a profile hit satisfies the completeness - rule — no race demanded. -- Race-on-unlock: a mint races ONLY seats whose `requires=` the host passes AND whose feature - is not already the profile class's own (zen2: nothing; zen4: the 2 vpdpbusd seats x tile - families ~= 32 races; M1: nothing; M5: nothing until i8mm/SME seats exist). Bare `--tune` - keeps meaning "race what this box unlocks"; the honest full walk moves behind - `--tune-full` (new-silicon minting). `--tune-only` unchanged. +**B2. Arch default profiles (kernels-only — the sharper form of the CPU/GPU split).** +- Profiles carry KERNEL WINNERS ONLY. The runtime-knob section never travels: its knobs mix + device knobs (metal_*, batch_grid_2d, lane caps) with box-shape knobs (core counts, cache + budgets, thread receipts) — neither is class-portable, so nothing GPU- or box-side can + mis-inherit by construction. Knobs stay code-defaults on a fresh box until a deliberate + `--tune`; existing minted sidecars keep theirs. +- Checked-in profiles: `modules/dasLLAMA/performance/defaults/.tune-defaults.json` + (the extension dodges the `*.tune.json` gitignore), `x86-avx2` minted from zen2, + `arm-neon` from m1. `x86-vnni512` comes from the c7a trial; `arm-i8mm`/`arm-sme` exist + only once those kernels do. `harness/export_tune_profile.das` mints one from a sidecar; + the framework fn is `tune_profile_export`. +- Adoption is FRAMEWORK-side, declared on the scope — `[tune_scope(defaults = "dir")]` — + and runs in the auto/restart guards ahead of the scope resolver: walk `tune_class_chain()` + down from `tune_cpu_class()`, adopt the first matching profile (version-pin checked) into + the app sidecar as a normal local write. A covered scope races NOTHING. +- Race-on-unlock: an adopting start races ONLY families with a seat whose `requires=` the + host passes but the profile's minting box did not (recorded as the `features` fingerprint + every sidecar save now stamps) — plus families the profile predates. The residue rides the + existing `--tune-only` machinery (the guard arms `DAS_TUNE_ONLY` for the tuner it spawns). + No new flag: bare `--tune` already skips adoption entirely — it IS the full new-silicon + walk. `--tune-only` unchanged. **B3. `DAS_TUNE_POLICY=reference`** — a policy value that serves the original bodies outright (QUIRK 18's missing spelling). Small, framework-side. diff --git a/skills/tune.md b/skills/tune.md index 37e8a6b676..458d717d28 100644 --- a/skills/tune.md +++ b/skills/tune.md @@ -136,7 +136,10 @@ warning **names the missing kernels**. `version_of=` (optional, `"module/CONST"`) pins the scope to a library version: the sidecar's provenance must record that int constant's current value (under the lowercased constant name; `version_key=` overrides), so bumping the constant -on kernel work invalidates every box's winners. The annotation names that +on kernel work invalidates every box's winners. `defaults=` (optional, +resolved against the declaring file) names the directory of **shipped +defaults profiles** — see the section below; an untuned box adopts its CPU +class's profile instead of racing. The annotation names that module by string only, so the declaring module must `require` it as well (suppress the unused-require lint), and the scope's tuner must stamp the value with `tune_provenance_note` - an unstamped pin re-tunes on every @@ -158,6 +161,42 @@ public surface (a blanket `public` on a module that also re-exports `jobque_boost` floods requirers with name ambiguities). ``` +## Shipped defaults profiles - `[tune_scope(defaults = "dir")]` + +Kernel winners follow **instruction sets, not boxes**: within one CPU feature +class the measured spread between seats is noise, so one minted answer serves +every box of the class. A library ships those answers as checked-in profile +files - `/.tune-defaults.json`, each the `"kernels"` section of a +reference box's full mint plus a provenance recording the minting box's +`features` fingerprint (and any `version_of=` pin value). Runtime knobs, race +tables and box identity never travel - knobs are per-box, devices are +per-device. + +The class names follow the features `requires=` can gate: `x86-vnni512` / +`x86-vnni256` / `x86-avx2` / `x86-base`, `arm-i8mm` / `arm-neon` +(`tune_cpu_class()` computes this box's, `tune_class_chain()` its adoption +ladder). An untuned auto/restart start walks the ladder from the box's own +class downward, adopts the first profile found into the app sidecar (a normal +local write - staleness, box identity and the JIT DLL cache re-key all behave +as for a mint), and then races only what the profile could not answer: + +* families the profile has no entry for (it predates them), and +* families with an ISA-gated seat this box unlocks that the profile's minting + box could not race - decided by comparing each `requires=` against the + profile's recorded `features`. + +That residue races through the ordinary tuner spawn with the `--tune-only` +filter armed, so a zen4 box adopting the `x86-avx2` profile races only the +vpdpbusd seats, and a box whose class profile is exact races **nothing**. +Adoption is skipped entirely under `--tune` (a forced re-race stays a full +local mint), when the profile's pinned version mismatches, or when no profile +file matches any class in the ladder. + +A maintainer produces a profile on a reference box after a full `--tune` mint +with `tune_profile_export(path, klass)` - it refuses an empty or stale +sidecar. Every sidecar save also stamps the box's `features` fingerprint into +provenance, which is what makes a future export race-on-unlock-aware. + ## Application policy - `[tune_policy]` and `--tune` **Untuned does not start.** Any application whose program root has a `main` @@ -184,6 +223,11 @@ def main { * - `fallback` - stamp `fallback=` silently (also what `DAS_TUNE_POLICY=fallback` - the CI kill switch - forces everywhere) +* - `reference` + - serve the ORIGINAL bodies: no stamps at all, for `[tune]` families and + loop-hint `[tuned]` kernels alike. The A/B truth tier - `fallback` is + not it, because a chain's first viable perm can be the very stamp under + test. Usually via `DAS_TUNE_POLICY=reference`. * - `warn` - loud compile-time banner with the exact tuner command * - `error` @@ -320,7 +364,8 @@ Two seams let a supervisor or a network service participate: - `tune_set_scope_resolver(fn)` - registered from an `[init]` (which must run before the guard at the top of `main`), consulted by - the auto/restart policy guards before spawning a scope's tuner. A resolver + the auto/restart policy guards before spawning a scope's tuner (and after + the shipped-defaults profiles above - local answers outrank remote ones). A resolver that can satisfy the scope another way (dasLLAMA's exchange client downloads a matching per-box sidecar from dasllama.io) returns true; completeness is re-checked, never trusted, and `--tune` never consults it. @@ -513,10 +558,15 @@ tables behind each multi-variant winner under `"race"`, and `"provenance"` { "kernels" : { "gemm" : "kstep4", "gemm_gemv" : "reference" }, "provenance" : { "binary" : "...", "platform" : "windows", "arch" : "x86_64", + "features" : "avx2;f16c;fma;sse4.2", "noise" : "ok", "noise_probes" : "start cv 0.40%; mid1 cv 0.14%; end cv 0.28%" } } ``` +`features` is the box's fingerprint over the known `requires=` feature names, +stamped on every save - it is what a shipped defaults profile exported from +this sidecar uses to tell covered seats from unlocked ones. + It is a per-app, per-box artifact - gitignored (`*.tune.json`), and any change re-keys the JIT DLL cache automatically (the winning permutation's args fold into the DLL basename). From 4d1c85bd633f67f9689b751ff7d75aa9b90092f3 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Mon, 31 Aug 2026 18:23:57 -0700 Subject: [PATCH 53/61] dasLLAMA: the SDK bundle carries the shipped defaults profiles The scope resolves defaults= against the installed dasllama_math_gen.das, so performance/defaults/*.tune-defaults.json must install beside it or an SDK box never adopts and falls back to the tuner it does not ship. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01MyYCDutugdc9xL5pd2qcvT --- modules/dasLLAMA/CMakeLists.txt | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/modules/dasLLAMA/CMakeLists.txt b/modules/dasLLAMA/CMakeLists.txt index 9138e85621..bb23897198 100644 --- a/modules/dasLLAMA/CMakeLists.txt +++ b/modules/dasLLAMA/CMakeLists.txt @@ -38,6 +38,14 @@ IF(NOT DAS_LLAMA_INCLUDED) install(FILES ${PROJECT_SOURCE_DIR}/modules/dasLLAMA/performance/model_specs.das DESTINATION ${DAS_INSTALL_MODULESDIR}/dasLLAMA/performance ) + # shipped defaults profiles ([tune_scope(defaults = ...)]): an untuned SDK box adopts its + # CPU class's winners instead of racing — the scope resolves this dir against the installed + # dasllama_math_gen.das, so the bundle must carry it + install(DIRECTORY ${PROJECT_SOURCE_DIR}/modules/dasLLAMA/performance/defaults + DESTINATION ${DAS_INSTALL_MODULESDIR}/dasLLAMA/performance + FILES_MATCHING + PATTERN "*.tune-defaults.json" + ) # third-party notices: the ported reference implementations (MIT) + the weights terms install(FILES ${PROJECT_SOURCE_DIR}/modules/dasLLAMA/LICENSE.PARAKEET DESTINATION ${DAS_INSTALL_DOCDIR} RENAME PARAKEET.LICENSE) install(FILES ${PROJECT_SOURCE_DIR}/modules/dasLLAMA/LICENSE.SILERO DESTINATION ${DAS_INSTALL_DOCDIR} RENAME SILERO.LICENSE) From 28b883f3c29bc7a003f6830f4cb1f80d1b97cc7a Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Mon, 31 Aug 2026 18:38:14 -0700 Subject: [PATCH 54/61] dasLLAMA: arm-neon + x86-vnni512 profiles, the tune_kernels Metal-arm compile fix (unquirk B4) The two remaining reference profiles, minted fresh under the pruned grids: - arm-neon from the M1 worktree (gen half 734 s, [tuned] half 70 s; mr8 on every tile, the Metal tensor twins lose to simdgroup on M1); - x86-vnni512 from a c7a.4xlarge (EPYC 9R14 zen4): full new-silicon mint 316 s, noise cv <= 0.21% - dot_vpdpbusd_width512_mr16 crowns EVERY tile family and q8q8 takes the bias128/gkstep2 vnni512 stamp. The instance is terminated; the recipe is in the arc memory. tune_kernels.das failed to COMPILE on macOS: the two Metal race halves reach MetalTensorRaceResult through different require paths and their arrays no longer unify into one typed local. A generic absorb helper takes each returned row where it lands - no named element type, same log lines, same crown list. Windows never compiled that static_if arm, so the rot sat unnoticed under the --tune-only mints. followup_metal.md: the smmla note records the actual fix (cpu_supports at the emitter tier) and the post-PR-1 M5 order. plans/unquirk_pass.md: B4 numbers. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01MyYCDutugdc9xL5pd2qcvT --- modules/dasLLAMA/followup_metal.md | 16 +++-- modules/dasLLAMA/harness/tune_kernels.das | 34 ++++----- .../defaults/arm-neon.tune-defaults.json | 70 +++++++++++++++++++ .../defaults/x86-vnni512.tune-defaults.json | 70 +++++++++++++++++++ plans/unquirk_pass.md | 10 ++- 5 files changed, 175 insertions(+), 25 deletions(-) create mode 100644 modules/dasLLAMA/performance/defaults/arm-neon.tune-defaults.json create mode 100644 modules/dasLLAMA/performance/defaults/x86-vnni512.tune-defaults.json diff --git a/modules/dasLLAMA/followup_metal.md b/modules/dasLLAMA/followup_metal.md index 4c44a31824..74e76d3174 100644 --- a/modules/dasLLAMA/followup_metal.md +++ b/modules/dasLLAMA/followup_metal.md @@ -10,14 +10,16 @@ the M-series CPU tiers are minted and raced from that box. `q8q8_tile_gen` ships five `dot = "smmla"` seats gated `requires = "i8mm"` (mr4/mr8 x kstep/nrsplit/gkstep), and NO box has ever raced them: `LLVMGetHostCPUFeatures()` returns an EMPTY string on macOS (llvm_jit_common.das documents it beside `g_target_arm64_i8mm`), so -i8mm never detects on Apple Silicon and the seats gate-skip. M1 lacks i8mm; M2+ has it; the -M5 Max additionally has FEAT_SME2p1 + BF16/EBF16. The unquirk pass's B2 adds a darwin -host-CPU-name feature map, after which an M5 mint races smmla-vs-NEON with zero new kernel -work. Mac-session order: +i8mm never detected on Apple Silicon at the EMITTER tier (`g_target_arm64_i8mm`), so the +generators declined and the seats never raced - even though `cpu_supports("i8mm")` answered +correctly via sysctl. M1 lacks i8mm; M2+ has it; the M5 Max additionally has FEAT_SME2p1 + +BF16/EBF16. FIXED in the unquirk pass (278b3c765): `g_target_arm64_i8mm` also consults +`cpu_supports`, and the target machine appends `+i8mm` when the host has it - an M5 `--tune` +now races smmla-vs-NEON with zero new kernel work. Mac-session order: -1. After B2 lands: `--tune-full` on the M5, confirm the smmla seats emit (they have never - been exercised - treat the emitter arms as unproven, gate with the gen probe TEST mode - under `DAS_JIT_ARM64_FORCE_FEATURES=i8mm` first) and report the crowns. +1. On the M5 (post PR-1): `--tune`, confirm the smmla seats EMIT (they have never been + exercised - treat the emitter arms as unproven, gate with the gen probe TEST mode first) + and report the crowns; export the `arm-i8mm` profile if they win. 2. If smmla wins q8q8: the kq tile families have NO ARM ISA seats at all (mr8 NEON is the whole grid) - an smmla kq tile emitter arm is the highest-leverage CPU kernel work on the mac, and it transfers to Graviton3+ (c8g) verbatim. diff --git a/modules/dasLLAMA/harness/tune_kernels.das b/modules/dasLLAMA/harness/tune_kernels.das index b91d8b776f..bdd6a945c9 100644 --- a/modules/dasLLAMA/harness/tune_kernels.das +++ b/modules/dasLLAMA/harness/tune_kernels.das @@ -2338,6 +2338,16 @@ def bench_laneq4x4() : string { // nolint:STYLE038 — one bench: fixture, backe // fallback: the tuner rail drives this half explicitly (DAS_TUNE_MODE=tune) — the auto // policy must not fire a nested tuner off this program's own missing per-app sidecar +// one METAL_TWIN race row: log it and bank the family when the tensor twin won +def private metal_twin_absorb(var wins : array; var ran : bool&; r) { + ran = true + let mt_note = empty(r.note) ? "" : " ({r.note})" + tune_detail("METAL_TWIN {r.family}: base={r.base_ms}ms tensor={r.twin_ms}ms -> {r.winner}{mt_note}\n") + if (r.winner == "tensor") { + wins |> push(r.family) + } +} + [export, tune_policy(missing = "fallback")] def main : int { // nolint:STYLE037,STYLE038 — flat one-bench-per-kernel rail; the noise probes are its checkpoints if (tuner_paranoid_requested()) { @@ -2453,23 +2463,14 @@ def main : int { // nolint:STYLE037,STYLE038 — flat one-bench-per-kernel rail; static_if (typeinfo builtin_module_exists(das_metal)) { var mt_wins : array var mt_ran = false - var mt_all : array - var mt_dec : array if (!filtered) { // a filtered mint leaves "runtime" untouched - no crowns to record - mt_all <- metal_tensor_race() - mt_dec <- metal_tensor_race_decode() - } - mt_all |> reserve(length(mt_all) + length(mt_dec)) - for (r in mt_dec) { - mt_all |> emplace(r) - } - delete mt_dec - for (r in mt_all) { - mt_ran = true - let mt_note = empty(r.note) ? "" : " ({r.note})" - tune_detail("METAL_TWIN {r.family}: base={r.base_ms}ms tensor={r.twin_ms}ms -> {r.winner}{mt_note}\n") - if (r.winner == "tensor") { - mt_wins |> push(r.family) + // the two race halves come through different require paths, so their result element + // types do not unify into one typed local - absorb each returned array where it lands + for (r in metal_tensor_race()) { + metal_twin_absorb(mt_wins, mt_ran, r) + } + for (r in metal_tensor_race_decode()) { + metal_twin_absorb(mt_wins, mt_ran, r) } } if (!mt_ran && !filtered) { @@ -2477,7 +2478,6 @@ def main : int { // nolint:STYLE037,STYLE038 — flat one-bench-per-kernel rail; } metal_crowns = join(mt_wins, ",") delete mt_wins - delete mt_all } // softmax_sink shares softmax's loop shape exactly (one extra scalar joins the reduction), so it diff --git a/modules/dasLLAMA/performance/defaults/arm-neon.tune-defaults.json b/modules/dasLLAMA/performance/defaults/arm-neon.tune-defaults.json new file mode 100644 index 0000000000..f038e3319c --- /dev/null +++ b/modules/dasLLAMA/performance/defaults/arm-neon.tune-defaults.json @@ -0,0 +1,70 @@ +{ + "kernels" : { + "add_inplace" : "vec8_u2", + "cvt_f32_to_f16" : "vec8_u2", + "rope_scaled_neox_tab" : "vec16", + "q51q8_tile_gen" : "mr8", + "softmax" : "vec8_u2", + "mul_inplace" : "vec8_u2", + "quantize_q8_0_bs_into_ptr" : "plain", + "iq2xsq8_tile_gen" : "mr8", + "dot_q8kv" : "vec4_u4", + "dot_q8q8" : "vec16", + "iq4xsq8_tile_gen" : "mr8", + "iq4nlq8_tile_gen" : "mr8", + "quantize_q8kv_row" : "plain", + "axpy_f16" : "vec4_u4", + "q40q8_tile_gen" : "mr8", + "axpy_tq4kv" : "vec8_u2", + "cvt_tq4kv_to_f32" : "vec8_u2", + "axpy" : "vec8_u2", + "dot_q8q8kv" : "vec16", + "dot_mx4q8" : "vec4_u4", + "softmax_sink" : "vec8_u2", + "iq2xxsq8_tile_gen" : "mr8", + "dot_q8q8_laneq4x4" : "u2", + "dot_bf16" : "vec8_u2", + "iq2sq8_tile_gen" : "mr8", + "add_scale_inplace" : "vec8_u2", + "cvt_q8kv_to_f32" : "vec8_u2", + "axpy_q8kv" : "u2", + "dot_q8q8_f16s" : "vec16", + "q8q8_tile_gen" : "mr8_budget", + "quantize_q8_0_into_ptr" : "plain", + "k4q8_tile_gen" : "mr8", + "k5q8_tile_gen" : "mr8", + "k6q8_tile_gen" : "mr8", + "k3q8_tile_gen" : "mr8", + "k2q8_tile_gen" : "mr8", + "gemm_f32_uk_4x16" : "u2", + "dot_q51e" : "vec16", + "dot_f16" : "vec8_u2", + "cvt_f16_to_f32" : "vec16", + "dot" : "vec8_u2", + "iq3xxsq8_tile_gen" : "mr8", + "iq3sq8_tile_gen" : "mr8", + "dot_q8tq4kv" : "vec16", + "scale_inplace" : "vec8_u2", + "dot_q4" : "vec4_u4", + "quantize_tq4kv_row" : "plain", + "copy_floats" : "vec8_u2", + "rmsnorm" : "vec8" + }, + "provenance" : { + "validation" : "ok", + "noise_probes" : "start cv 0.22%; mid1 cv 0.11%; mid2 cv 0.10%; end cv 0.09%", + "platform" : "darwin", + "noise_floor_cv_pct" : "0.22", + "features" : "neon;dotprod;fullfp16;lse", + "engine_sha" : "278b3c765", + "class" : "arm-neon", + "written" : "2026-09-01T01:37:39.701Z", + "validation_demoted" : "0", + "mode" : "normal", + "dasllama_version" : "15", + "origin" : "profile", + "noise" : "ok", + "arch" : "arm64", + "validation_max_drift_pct" : "0.13" + } +} \ No newline at end of file diff --git a/modules/dasLLAMA/performance/defaults/x86-vnni512.tune-defaults.json b/modules/dasLLAMA/performance/defaults/x86-vnni512.tune-defaults.json new file mode 100644 index 0000000000..53e07aee10 --- /dev/null +++ b/modules/dasLLAMA/performance/defaults/x86-vnni512.tune-defaults.json @@ -0,0 +1,70 @@ +{ + "kernels" : { + "add_inplace" : "vec8_u2", + "cvt_f32_to_f16" : "plain", + "rope_scaled_neox_tab" : "vec8_u2", + "q51q8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "softmax" : "vec8_u2", + "mul_inplace" : "vec8_u2", + "quantize_q8_0_bs_into_ptr" : "u2", + "iq2xsq8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "dot_q8kv" : "vec8_u2", + "dot_q8q8" : "u2", + "iq4xsq8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "iq4nlq8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "quantize_q8kv_row" : "plain", + "axpy_f16" : "vec8_u2", + "q40q8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "axpy_tq4kv" : "vec8_u2", + "cvt_tq4kv_to_f32" : "vec8_u2", + "axpy" : "vec8_u2", + "dot_q8q8kv" : "u2", + "dot_mx4q8" : "u2", + "softmax_sink" : "vec8_u2", + "iq2xxsq8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "dot_q8q8_laneq4x4" : "", + "dot_bf16" : "vec8_u2", + "iq2sq8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "add_scale_inplace" : "vec8_u2", + "cvt_q8kv_to_f32" : "vec8_u2", + "axpy_q8kv" : "plain", + "dot_q8q8_f16s" : "vec16", + "q8q8_tile_gen" : "dot_vpdpbusd_width512_mr16_kstep2_gkstep2_bias128", + "quantize_q8_0_into_ptr" : "plain", + "k4q8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "k5q8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "k6q8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "k3q8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "k2q8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "gemm_f32_uk_4x16" : "u2", + "dot_q51e" : "vec16", + "dot_f16" : "vec16", + "cvt_f16_to_f32" : "vec8_u2", + "dot" : "vec16", + "iq3xxsq8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "iq3sq8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "dot_q8tq4kv" : "vec16", + "scale_inplace" : "vec8_u2", + "dot_q4" : "vec8_u2", + "quantize_tq4kv_row" : "plain", + "copy_floats" : "vec8_u2", + "rmsnorm" : "vec8" + }, + "provenance" : { + "validation" : "ok", + "noise_probes" : "start cv 0.09%; mid1 cv 0.04%; mid2 cv 0.21%; end cv 0.06%", + "platform" : "linux", + "noise_floor_cv_pct" : "0.21", + "features" : "avx2;f16c;fma;sse4.2;avx512f;avx512bw;avx512vl;avx512vnni", + "engine_sha" : "278b3c765", + "class" : "x86-vnni512", + "written" : "2026-09-01T01:30:04.021Z", + "validation_demoted" : "3", + "mode" : "normal", + "dasllama_version" : "15", + "origin" : "profile", + "noise" : "ok", + "arch" : "x86_64", + "validation_max_drift_pct" : "21.77" + } +} \ No newline at end of file diff --git a/plans/unquirk_pass.md b/plans/unquirk_pass.md index 0bc3e1b262..f7d7d75fc4 100644 --- a/plans/unquirk_pass.md +++ b/plans/unquirk_pass.md @@ -57,7 +57,15 @@ outside this PR). **B3. `DAS_TUNE_POLICY=reference`** — a policy value that serves the original bodies outright (QUIRK 18's missing spelling). Small, framework-side. -**B4. Validation + the measurement Boris's service decision needs.** +**B4. Validation + the measurement Boris's service decision needs.** MEASURED 2026-08-31: +zen2 fresh full mint under the pruned grids 488 s (gen probe 414 + tuned/knobs 66; noise cv +<= 0.33%); zen2 parity cold start against the shipped x86-avx2 profile = 0 races (the 285 s +wall is two monolith JIT compiles, pre-adopt + re-exec), stream identical to the recorded +stamped one; c7a.4xlarge (EPYC 9R14 zen4, 16 vCPU) full new-silicon mint **316 s** (gen probe +253 + tuned 58, noise cv <= 0.21%) - `dot_vpdpbusd_width512_mr16` crowns EVERY tile family +and q8q8 takes the bias128/gkstep2 vnni512 stamp; x86-vnni512 profile exported + checked in. +m1 gen half 734 s (M1, 8 threads; mr8 everywhere) - its `tune_kernels` Metal arm had a +macOS-only compile error (two require paths for MetalTensorRaceResult), fixed in-pass. - dasLLVM tune tests (test_tuned/test_grid/llvm_tune_*) extended for profile resolution + the split sidecar; gen probe test+tune modes; parity 40/40; one bench row vs current numbers (winners are unchanged by construction — prove it, don't assume it). From 8b241bec37e61fe4684281c0b5607f9ba49f3b57 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Mon, 31 Aug 2026 18:47:28 -0700 Subject: [PATCH 55/61] dasLLAMA: the .dlim identity carries the pack-code version; JIT staging on every platform (C1/D1) QUIRK 24: a layout/pack edit left cached images serving stale planes because the image filename hash folded model + schema and nothing about the packing code. PACK_VERSION now lives beside the pack walkers in dasllama_layout.das and folds into every identity through image_identity_of - the ONE spelling the loader, the peek verdicts, the converter's want/have compare and the tests all format through, so schema and pack versions can never be folded two ways. Bump it with the edit; the image re-bakes instead of being mapped. QUIRK 12: stage_jit_backend staged LLVM.dll on Windows only. It now stages lib/LLVM.dll + its .version stamp everywhere and adds bin/lld-link.exe on Windows, so a fresh worktree on macOS/Linux gets -jit without the hand copy. Gates: lint 0 over the six changed files; test_model_image 48 passed / 19 model-gated skips / 0 failed under -jit; dasllama-convert and setup run. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01MyYCDutugdc9xL5pd2qcvT --- modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md | 10 ++++++---- modules/dasLLAMA/dasllama/dasllama_config.das | 8 +++++--- modules/dasLLAMA/dasllama/dasllama_image.das | 11 +++++++++-- modules/dasLLAMA/dasllama/dasllama_layout.das | 5 +++++ modules/dasLLAMA/tests/test_model_image.das | 4 ++-- utils/dasllama-convert/main.das | 6 +++--- utils/mcp/setup.das | 10 ++++++---- 7 files changed, 36 insertions(+), 18 deletions(-) diff --git a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md index 94a30e7609..f67f33c8a2 100644 --- a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md +++ b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md @@ -358,8 +358,8 @@ where, why it is so today, what unquirked looks like. An empty ledger is a legit generator change reads as stale. 12. **A fresh worktree has no JIT until `lib/LLVM.dll` is staged - on every platform.** The M1 worktree's first `-jit` run died on `can't load library LLVM.dll`; `utils/mcp/setup.das` - stages the codegen backend on Windows only. Copy `
/lib/LLVM.dll` (+ `.version`) into - the worktree's `lib/` by hand on macOS/Linux. Unquirked: the posix arm of `stage_jit_backend`. + staged the codegen backend on Windows only. UNQUIRKED: `stage_jit_backend` now stages + `lib/LLVM.dll` + `.version` on every platform (Windows additionally `bin/lld-link.exe`). 13. **The Metal test ladders are nested ternaries with an `else` = k6.** `kq_gemv_gate`, `kq_mvb_gate`, `kq_mulmm_gate` and the fixtures pick MSL sources / entries / fastmath / tgmem names per format in four parallel ternary chains each; a format missing from any one @@ -444,8 +444,10 @@ where, why it is so today, what unquirked looks like. An empty ledger is a legit copy fix, the rerun "mapped" the stale image and kept serving the broken device planes - garbage text and all-zero decode logits survived a correct fix. Delete the model's `*.dlim` beside the GGUF after ANY `dasllama_layout.das`/pack edit and confirm the next - log line says "baked", not "mapped". Unquirked: fold a pack-code version into the image - hash, the way QUIRK 21's fix would version the JIT DLL cache. + log line says "baked", not "mapped". UNQUIRKED: `PACK_VERSION` (dasllama_layout.das) + folds into every image identity through `image_identity_of` - the ONE identity spelling + all consumers format through - so bumping it with a pack edit re-bakes; the discipline is + now "bump the constant beside the code you edited", not "remember to delete files". 25. **The stream-code space is NOT the kernel-id space: q51 squats on 2.** Stream region tags are q8=0, mx4=1, q51=2, then the kq kernel ids - Q2_K's mnemonic id 2 collided and diff --git a/modules/dasLLAMA/dasllama/dasllama_config.das b/modules/dasLLAMA/dasllama/dasllama_config.das index 82c7507b1d..3704b36156 100644 --- a/modules/dasLLAMA/dasllama/dasllama_config.das +++ b/modules/dasLLAMA/dasllama/dasllama_config.das @@ -144,9 +144,11 @@ def public dlim_config_current(quant : string = "q8") : DlimConfiguration { // ===== pure formatters (identity, tags, JSON) ===== //! The identity string an image file/header is keyed by — a pure formatter of the struct. -//! `image_version` comes from the caller (dasllama_image owns the version constant). -def public dlim_identity(c : DlimConfiguration; image_version : int; tag : string = "") : string { - return ("v{image_version}|{c.quant}|{c.cpu.backend}|{c.cpu.wscale_f16 ? "s16" : "s32"}" +//! `image_version` / `pack_version` come from the caller (dasllama_image owns the schema +//! version, dasllama_layout the pack-code version; 0 = the pre-versioned pack spelling). +def public dlim_identity(c : DlimConfiguration; image_version : int; tag : string = ""; pack_version : int = 0) : string { + let pk = pack_version > 0 ? "p{pack_version}" : "" + return ("v{image_version}{pk}|{c.quant}|{c.cpu.backend}|{c.cpu.wscale_f16 ? "s16" : "s32"}" + "|q8 mr{c.cpu.q8_mr} b{c.cpu.q8_wbias} g{c.cpu.q8_kgroup}" + "|kq {c.cpu.kq_mr4}/{c.cpu.kq_mr5}/{c.cpu.kq_mr6}/{c.cpu.kq_mr40}/{c.cpu.kq_mr44}/{c.cpu.kq_mr3}/{c.cpu.kq_mr33}/{c.cpu.kq_mr34}/{c.cpu.kq_mr45}/{c.cpu.kq_mr2}/{c.cpu.kq_mr23}/{c.cpu.kq_mr24}/{c.cpu.kq_mr25}" + "|q51 mr{c.cpu.q51_mr}" diff --git a/modules/dasLLAMA/dasllama/dasllama_image.das b/modules/dasLLAMA/dasllama/dasllama_image.das index a51fea7c90..fdbba2fbb4 100644 --- a/modules/dasLLAMA/dasllama/dasllama_image.das +++ b/modules/dasLLAMA/dasllama/dasllama_image.das @@ -57,7 +57,14 @@ struct ImgSection { [arch(at = "../ARCHITECTURE_IMAGE.md#image-identity-backend-order")] def image_identity(tag : string = ""; quant : string = "q8") : string { - return dlim_identity(dlim_config_current(quant), IMAGE_VERSION, tag) + return image_identity_of(dlim_config_current(quant), tag) +} + +//! The identity of an explicit configuration — the ONE spelling every identity consumer (the +//! loader, the peek verdicts, the converter's want/have compare, the tests) formats through, so +//! the schema version and the pack-code version can never be folded in two different ways. +def image_identity_of(c : DlimConfiguration; tag : string = "") : string { + return dlim_identity(c, IMAGE_VERSION, tag, PACK_VERSION) } // a 64-bit identity hash per file: a collision would leave two identities re-saving over one path forever @@ -775,7 +782,7 @@ def private build_image(var t; dst : string; in_mem : bool; extra : uint64; plane : block<(name : string; var w : ImgWriter; var sections : array) : bool>) : ImgWriter { // one config snapshot serves the meta embed AND the header hash — they must agree let dcfg = dlim_config_current(quant) - let ident = dlim_identity(dcfg, IMAGE_VERSION, tag) + let ident = image_identity_of(dcfg, tag) var fend = 0ul var meta_off = 0ul var meta_bytes = 0ul diff --git a/modules/dasLLAMA/dasllama/dasllama_layout.das b/modules/dasLLAMA/dasllama/dasllama_layout.das index 2f0b17b442..5d439db2b3 100644 --- a/modules/dasLLAMA/dasllama/dasllama_layout.das +++ b/modules/dasLLAMA/dasllama/dasllama_layout.das @@ -21,6 +21,11 @@ require math // walkers, and the GPU tier gathers. Requires dasllama_common back for Model/KqFmt; the loader // reaches the entry points through the hooks registered at [init] (register_model_layout). +//! The pack-code version folded into every .dlim identity: bump it with ANY edit to a pack or +//! repack walker in this file, so a cached image baked by the old code re-bakes instead of +//! being mapped and served (an image's filename hash carries model + schema + this). +let PACK_VERSION = 1 + // ===== metal-flavor blob transform (the blob-only .dlim redesign) ===== // A q8 region's blob bind offset (off/32)*34 lands 16B-aligned iff off % 256 == 0; a k6 diff --git a/modules/dasLLAMA/tests/test_model_image.das b/modules/dasLLAMA/tests/test_model_image.das index 9ed26ae205..e10a682c12 100644 --- a/modules/dasLLAMA/tests/test_model_image.das +++ b/modules/dasLLAMA/tests/test_model_image.das @@ -275,7 +275,7 @@ def test_image_mechanics(t : T?) { t |> equal(pk.identity, image_identity("img-probe"), "peek: baked identity string") var pcfg = DlimConfiguration() t |> success(dlim_config_from_json(pk.config_json, pcfg), "peek: embedded config JSON parses") - t |> equal(dlim_identity(pcfg, IMAGE_VERSION, "img-probe"), pk.identity, "peek: config re-derives the identity") + t |> equal(image_identity_of(pcfg, "img-probe"), pk.identity, "peek: config re-derives the identity") var m3 = ImgProbe() t |> success(!load_image(path, m3, "wrong-tag"), "identity mismatch declines") @@ -378,7 +378,7 @@ def test_image_mechanics(t : T?) { t |> success(!empty(js), "config serializes") var back = DlimConfiguration() t |> success(dlim_config_from_json(js, back), "JSON parses back") - t |> equal(dlim_identity(back, IMAGE_VERSION), dlim_identity(dc, IMAGE_VERSION), "identity survives the round trip") + t |> equal(image_identity_of(back), image_identity_of(dc), "identity survives the round trip") var junk = DlimConfiguration() t |> success(!dlim_config_from_json("not a config", junk), "garbage declines") } diff --git a/utils/dasllama-convert/main.das b/utils/dasllama-convert/main.das index fceba938a1..8ddb399f8c 100644 --- a/utils/dasllama-convert/main.das +++ b/utils/dasllama-convert/main.das @@ -358,7 +358,7 @@ def main() : int { } else { print("{js}\n") } - to_log(LOG_INFO, "dasllama-convert: identity {dlim_identity(dc, IMAGE_VERSION)}\n") + to_log(LOG_INFO, "dasllama-convert: identity {image_identity_of(dc)}\n") return rc0 } if (cfg.model == "") { @@ -416,8 +416,8 @@ def main() : int { to_log(LOG_ERROR, "dasllama-convert: --config targets '{want.cpu.backend}' and this host has no repack family to bake it with\n") return 2 } - let want_ident = dlim_identity(want, IMAGE_VERSION) - let have_ident = dlim_identity(dlim_config_current(want.quant), IMAGE_VERSION) + let want_ident = image_identity_of(want) + let have_ident = image_identity_of(dlim_config_current(want.quant)) if (have_ident != want_ident) { to_log(LOG_ERROR, "dasllama-convert: --config could not be fully applied on this host:\n want {want_ident}\n have {have_ident}\n") return 2 diff --git a/utils/mcp/setup.das b/utils/mcp/setup.das index 3c97812d77..123c55c984 100644 --- a/utils/mcp/setup.das +++ b/utils/mcp/setup.das @@ -193,9 +193,6 @@ def root_from_daslang_binary(binary : string) : string { } def stage_jit_backend(root : string) { - if (get_platform_name() != "windows") { - return - } let src_root = get_das_root() var fallback_root = "" var fallback_source = "DASLANG is unset" @@ -208,7 +205,12 @@ def stage_jit_backend(root : string) { fallback_source = "DASLANG does not name an existing binary" } } - let jit_files = ["lib/LLVM.dll", "bin/lld-link.exe"] + // the codegen backend is LLVM.dll on every platform (+ its .version stamp, which CMake's + // re-download check reads); only Windows also needs the lld-link.exe linker beside the binary + var jit_files <- ["lib/LLVM.dll", "lib/LLVM.dll.version"] + if (get_platform_name() == "windows") { + jit_files |> push("bin/lld-link.exe") + } for (rel in jit_files) { let dst = path_join(root, rel) if (fexist(dst)) { From b1fb8c10739d6567bcd690564c1636e585fe1fec Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Mon, 31 Aug 2026 19:17:48 -0700 Subject: [PATCH 56/61] PR-1 comment harvest: rules, facts and citations land; the tri-state adopt return gets a shape Per-module harvest of the comments this branch adds (make_pr row 0a0). Landings: - dasLLVM: REVIEW.md gains the TUNE_KNOWN_FEATURES duty and the arm-then-clear DAS_TUNE_ONLY duty; ARCHITECTURE.md sec.4 "Host CPU feature truth on aarch64" (cited from init_jit_target_flags and create_default_target_machine); profile_try_adopt returns bool + a race csv instead of a ""/"*"/csv string; expr_in_features -> requires_ok_in_fingerprint, skipValue -> skipTuneOnlyValue; three docs the diff had left stale corrected; tests/README covers _lib fixtures. - dasLLAMA: REVIEW.md gains the def-not-let team-lane constant rule and the dot_q8q8_laneq4x4-stays-last bench rule; ARCHITECTURE_GPU_VULKAN.md sec.2.2l carries the cm2 fast-path / no-split-arm facts; ARCHITECTURE_GPU.md sec.2.2y "The Metal kq split scale plane" (four citers); followup_general 58 extended to all nine formats' missing mul_mm twins, 59 closed by --tune-only + profiles; smi/sni -> row_half/col_half in the cm2 tile; attribution stripped from 21 comments; bare measured figures name their harness. - utils: the setup.das staging comment compressed; a Windows-only claim D1 falsified deleted. Gates: lint 0 over the 21 touched .das files; probe TEST OK; Vulkan 86/86; dasLLVM profiles + scope tests green. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01MyYCDutugdc9xL5pd2qcvT --- modules/dasLLAMA/ARCHITECTURE.md | 2 +- modules/dasLLAMA/ARCHITECTURE_GPU.md | 16 + modules/dasLLAMA/ARCHITECTURE_GPU_VULKAN.md | 15 + modules/dasLLAMA/REVIEW.md | 8 + .../dasllama/dasllama_gemm_schema.das | 2 +- modules/dasLLAMA/dasllama/dasllama_image.das | 4 +- .../dasLLAMA/dasllama/dasllama_kqformat.das | 4 +- modules/dasLLAMA/dasllama/dasllama_layout.das | 12 +- .../dasllama/dasllama_metal_common.das | 15 +- .../dasllama/dasllama_metal_kernels.das | 31 +- .../dasllama/dasllama_metal_prefill.das | 20 +- .../dasllama/dasllama_metal_shapes.das | 3 +- modules/dasLLAMA/dasllama/dasllama_tune.das | 2 +- .../dasllama/dasllama_vulkan_classes.das | 312 +++++++++--------- .../dasllama/dasllama_vulkan_common.das | 6 +- modules/dasLLAMA/followup_general.md | 15 +- modules/dasLLAMA/harness/gen_tune_probe.das | 2 +- modules/dasLLAMA/harness/tune_kernels.das | 6 +- modules/dasLLAMA/tests/test_kqformat.das | 2 +- modules/dasLLAMA/tests/test_kquant.das | 2 +- .../tests/test_metal_gemm_kernels.das | 24 +- .../tests/test_metal_gemv_kernels.das | 6 +- modules/dasLLVM/ARCHITECTURE.md | 17 + modules/dasLLVM/REVIEW.md | 9 + modules/dasLLVM/daslib/llvm_jit_common.das | 12 +- modules/dasLLVM/daslib/llvm_tune.das | 87 ++--- modules/dasLLVM/tests/README.md | 2 +- modules/dasLLVM/tests/llvm_tune_profiles.das | 14 +- skills/LAWS.md | 10 + utils/mcp/setup.das | 4 +- 30 files changed, 348 insertions(+), 316 deletions(-) diff --git a/modules/dasLLAMA/ARCHITECTURE.md b/modules/dasLLAMA/ARCHITECTURE.md index e0da7ceef5..dc4ea42a2f 100644 --- a/modules/dasLLAMA/ARCHITECTURE.md +++ b/modules/dasLLAMA/ARCHITECTURE.md @@ -38,7 +38,7 @@ re-transcoding `$LCPP/src/unicode-data.cpp`). - `ARCHITECTURE_IMAGE.md` - sec.2.1-2.1i: the prepared-image rail, the baked dev-W f16 plane, and the baked tower twin-W plane. -- `ARCHITECTURE_GPU.md` - sec.2.2b, 2.2w-2.2x: the tensor-GEMM and fused-attention shapes that +- `ARCHITECTURE_GPU.md` - sec.2.2b, 2.2w-2.2y: the tensor-GEMM and fused-attention shapes that measured out, the tower attention routes, and the tower driver's encode chains. - `ARCHITECTURE_GPU_PREFILL.md` - sec.2.2c-2.2i, 2.2u-2.2v: the Metal prefill driver's GEMM form ladder, dev-W knee map, attention slab, MoE bucket rail, chunked submission, the f16 twin diff --git a/modules/dasLLAMA/ARCHITECTURE_GPU.md b/modules/dasLLAMA/ARCHITECTURE_GPU.md index 8921e33661..bd9eac0cbe 100644 --- a/modules/dasLLAMA/ARCHITECTURE_GPU.md +++ b/modules/dasLLAMA/ARCHITECTURE_GPU.md @@ -256,3 +256,19 @@ specification and the CPU-vs-GPU transcript cells are its parity instrument. Eve best-effort: it answers false (or -1) on any shape, knob, quant-mode or device decline, and the CPU chain serves that encode. Engage is read from counter deltas (`metal_tower_stats`, `metal_tower_f16_encodes`), never from "the model ran". + +### 2.2y The Metal kq split scale plane {#metal-kq-split-scale-plane} + +Every superblock format but k4, k5, q40 and iq4nl stores its Metal-blob scale row SPLIT into two +regions of one buffer: the 16-byte sub-scale strips of every superblock first, then the packed +per-superblock d tail. A kernel binds that one buffer twice - the strips at `soff = sb0 * 16` and +the tail at `doff = nsb * 16 + sb0 * 2` - so the two reads stride independently and the strip +read stays 16-byte aligned. k2 is the one shape variation: its tail is 4 bytes per superblock +(`nsb * 16 + sb0 * 4`), because it carries d and dmin. `kq_scales_of` builds the pair; +`metal_blob_scale_plane` mints it at bake time, folding each format's 20-byte decoded row into +`[16B strips][2B d]` (k3's row is 18 bytes and is already in that shape). The 2-byte tail is why +a region's bind offset must be a multiple of 512 elements - the `(off/256)*2` d-plane bind is +4-byte aligned only then - which is what `metal_blob_off_ok` and `moe_site_ok` check. iq4nl is +the exception: it reuses q40's 16-byte plane of eight f16 d per superblock, binds once, and +ignores `doff`. The Vulkan tier does not use this form - it binds the decoded 20-byte row as five +uints per superblock. diff --git a/modules/dasLLAMA/ARCHITECTURE_GPU_VULKAN.md b/modules/dasLLAMA/ARCHITECTURE_GPU_VULKAN.md index 64138433b9..264a1d2544 100644 --- a/modules/dasLLAMA/ARCHITECTURE_GPU_VULKAN.md +++ b/modules/dasLLAMA/ARCHITECTURE_GPU_VULKAN.md @@ -114,6 +114,21 @@ NV_cooperative_matrix2, else mm where it has KHR_cooperative_matrix, else sdot4; the extension lands on mm. The same resolver stamps the mode into the `.dlim` flavor configuration, so the recorded mode and the running mode cannot drift. +**The tile's fast path is what makes the loads unclamped.** It runs when the weight tile is +whole (`m0 + 128 <= d`), the token column is whole or stamped s, and K is a whole number of BK +steps; the layouts are then created clamp-Undefined and the B and output strides are masked to +a multiple of 8 f16 (`stride &= ~7`). The mask is an identity on today's shapes - `n` and `d` +are 32-multiples - and it exists to make the alignment PROVABLE to the driver's address +analysis, which is what keeps the loads on the wide path. The s column gates only the weight +tile: its partial token column loads unclamped and its store clamps. Everything else takes the +edge path with clamped layouts. + +**The no-split arm keeps literal loop bounds and a literal store base.** Where `ksplit` is zero +the k loop runs the literal `0 .. n` with the store at the row base rather than the general +`k0`/`k1`/`ybase` form, although those values are exactly `0`, `n` and `0` on that path: the +general spelling cost 27% of prefill throughput (`benchmarks/lcpp_bench.das` pp512, 5060 Ti). +The split arm keeps the general form. + ### 2.2m Class-pipeline creation is the Vulkan tier's one shader A/B seat {#vk-class-pipeline-build} `vkd_class_pipe` is the single place a class kernel's SPIR-V becomes a pipeline, so both shader diff --git a/modules/dasLLAMA/REVIEW.md b/modules/dasLLAMA/REVIEW.md index 46392225f4..293be6a38c 100644 --- a/modules/dasLLAMA/REVIEW.md +++ b/modules/dasLLAMA/REVIEW.md @@ -294,3 +294,11 @@ family name on a shared path, is a defect - it carries declarative registration changes none of its arithmetic - deleting or rewriting the CPU form in the same change is a defect.** The hook returns a decline value (`false`, or `-1` for the chunk hooks), and the CPU form serves every box with no driver. + +**A constant that a team-mode job lane reads is declared as a `def` returning the value, never +as a `let` global.** A team lane never runs global initializers, so a `let` reads zero there +while every single-threaded run reads the right value. + +**A diff that adds a row to `harness/tune_kernels.das`'s bench list puts it ahead of +`dot_q8q8_laneq4x4`, which stays last.** That bench pins the repack backend for the rest of the +process, so a row after it races against the pinned backend instead of the one it selects. diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_schema.das b/modules/dasLLAMA/dasllama/dasllama_gemm_schema.das index b9059f9f57..4990b5b630 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_schema.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_schema.das @@ -40,7 +40,7 @@ def q8q8_repack_type(mr : int; wbias : int = 0; kgroup : int = 4) : Q8RepackType //! The mr of the REFERENCE tier's layout (q8q8_repack_type(4), the old hand arm64-laneq layout //! the reference stub bodies read). Non-reference mr runs the generated grp kernel/layout pair, //! stamped from one manifest entry so both decline in lockstep (M4). -def GEMM_REFERENCE_MR : int => 4 // a FUNCTION, not a let global: team-mode job lanes never run global init, so a let here reads 0 on the lanes the unstamped reference layout/repack run on +def GEMM_REFERENCE_MR : int => 4 //! GPU-tier device-plane shape of the Q8 rail: 32-weight blocks, 32B quants + one f16 scale //! halfword per block. Every q8 plane sizing (gathers, arena/stack math) derives from these. diff --git a/modules/dasLLAMA/dasllama/dasllama_image.das b/modules/dasLLAMA/dasllama/dasllama_image.das index fdbba2fbb4..61e438c2c5 100644 --- a/modules/dasLLAMA/dasllama/dasllama_image.das +++ b/modules/dasLLAMA/dasllama/dasllama_image.das @@ -38,7 +38,7 @@ require dasllama/dasllama_load // WhisperModel.enc) contribute their planes under dotted names ("enc.fblob"); string-array // fields ride the meta blob via serialize_strings — raw string pointers can't be planes. -let IMAGE_VERSION = 26 // 26: the iq2xxs plane pair (IQ2_XXS native tier); 25: iq2xs; 24: iq2s +let IMAGE_VERSION = 26 // 26: the iq2xxs plane pair (IQ2_XXS native tier) //! The metal (blob-only) flavor's identity tag: q8 planes ride the 34B block_q8_0 blob and the //! kq scale planes their GPU forms (convert_model_to_metal_blob) — flavors are per-config and @@ -688,7 +688,7 @@ def serialize_image_meta(var arch : Archive; var t : Model) { arch |> serialize_raw(t.mtp_headnorm_off) } -// 67 serialized fields + 3 deliberate skips (blocks, image_map, image_bytes) +// the 3 deliberate skips: blocks, image_map, image_bytes let IMAGE_META_FIELDS = 75 + 3 //! Count of meta-carried fields of any struct: non-arrays plus string arrays (those cannot diff --git a/modules/dasLLAMA/dasllama/dasllama_kqformat.das b/modules/dasLLAMA/dasllama/dasllama_kqformat.das index 5173eb39c1..9c69f8d538 100644 --- a/modules/dasLLAMA/dasllama/dasllama_kqformat.das +++ b/modules/dasLLAMA/dasllama/dasllama_kqformat.das @@ -107,11 +107,11 @@ let K3_SSB = 18l // Q3_K scale plane: 16 x int8 (6-bit scale - 32, decoded at let IQ3S_QSB = 104l // IQ3_S quant plane: [64 grid-index qs][8 qh ninth bits][32 sign bytes], verbatim disk order let IQ3S_SSB = 20l // IQ3_S scale plane: f16 d, 2 pad, 8 x int8 (1 + 2s), 8 pad — the iq4xs/k4 row shape let IQ3XXS_QSB = 96l // IQ3_XXS quant plane: [64 grid-index qs][32 aux bytes: per block 4x7-bit sign indices + 4-bit scale], verbatim disk order -let IQ3XXS_SSB = 20l // IQ3_XXS scale plane: f16 d HALVED, 2 pad, 8 x uint8 (2*ls + 1), 8 pad — the iq3s row shape and fold (ggml's global 0.25 = the halved d x the halved grid) +let IQ3XXS_SSB = 20l // IQ3_XXS scale plane: f16 d HALVED, 2 pad, 8 x uint8 (2*ls + 1), 8 pad — the iq3s row shape and fold (the format's global 0.25 = the halved d x the halved grid) let K2_QSB = 64l // Q2_K quant plane: 64 qs bytes (four 2-bit lanes each), verbatim disk order let K2_SSB = 20l // Q2_K scale plane: [f16 d][f16 dmin][16 sc/min nibble-pair bytes] - the disk scale block, header first let IQ2S_QSB = 72l // IQ2_S quant plane: [32 grid-index low bytes][32 sign bytes][8 qh], verbatim disk order -let IQ2S_SSB = 20l // IQ2_S scale plane: f16 d EIGHTH-ed (ggml's (0.5+ls)*0.25 = (2ls+1)/8), 2 pad, 16 x uint8 (2ls+1) - per-16 strips in the 20B row +let IQ2S_SSB = 20l // IQ2_S scale plane: f16 d EIGHTH-ed (the disk (0.5+ls)*0.25 = (2ls+1)/8), 2 pad, 16 x uint8 (2ls+1) - per-16 strips in the 20B row let IQ2XS_QSB = 64l // IQ2_XS quant plane: 32 u16 words ([9-bit grid index][7-bit ksigns index]), verbatim disk order let IQ2XS_SSB = 20l // IQ2_XS scale plane: the iq2s row exactly - f16 d EIGHTH-ed, 2 pad, 16 x uint8 (2ls+1) per-16 strips let IQ2XXS_QSB = 64l // IQ2_XXS quant plane: 8 blocks x [4 iq2xxs_grid byte indices][aux32: 4 x 7-bit ksigns + 4-bit ls], verbatim disk order diff --git a/modules/dasLLAMA/dasllama/dasllama_layout.das b/modules/dasLLAMA/dasllama/dasllama_layout.das index 5d439db2b3..f29cc20d07 100644 --- a/modules/dasLLAMA/dasllama/dasllama_layout.das +++ b/modules/dasLLAMA/dasllama/dasllama_layout.das @@ -31,10 +31,11 @@ let PACK_VERSION = 1 // A q8 region's blob bind offset (off/32)*34 lands 16B-aligned iff off % 256 == 0; a k6 // region's d-plane bind (NSB*16 + (off/256)*2) needs 4B alignment iff off % 512 == 0. // k4/k5/q40 quant + scale binds are 16B-multiples by construction. +[arch(at = "../ARCHITECTURE_GPU.md#metal-kq-split-scale-plane")] def private metal_blob_off_ok(off : int64; fmt : KqFmt) : bool { if (off < 0l) return true if (fmt == KqFmt.q8) return off % 256l == 0l - if (fmt == KqFmt.k6 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s || fmt == KqFmt.iq3xxs || fmt == KqFmt.iq2s || fmt == KqFmt.iq2xs || fmt == KqFmt.iq2xxs) return off % 512l == 0l // the split scale form: the (off/256)*2 d-plane bind stays 4B-aligned + if (fmt == KqFmt.k6 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s || fmt == KqFmt.iq3xxs || fmt == KqFmt.iq2s || fmt == KqFmt.iq2xs || fmt == KqFmt.iq2xxs) return off % 512l == 0l if (fmt == KqFmt.q51) return off % 128l == 0l // (off/32)*20 and (off/32)*4 both 16B-aligned return true } @@ -159,6 +160,7 @@ def metal_blob_fill_blocks(t : Model; b0, nblk : int64; var dst : array) //! Put ONE kq scale plane into its metal device form, named by its Model field; false = not a kq //! scale plane. Per-plane because the streamed image walk applies each as its own turn comes, //! which is the only point where that plane's fills are done and its bytes are not yet written. +[arch(at = "../ARCHITECTURE_GPU.md#metal-kq-split-scale-plane")] def metal_blob_scale_plane(var t : Model; name : string) : bool { // k4/k5: the 16B verbatim disk scale block per superblock — the 4B pad is CPU-repack // scratch (see the plane-layout comment in dasllama_gguf) and a blob model never repacks @@ -175,7 +177,6 @@ def metal_blob_scale_plane(var t : Model; name : string) : bool { return true } if (name == "iq4xss") { - // iq4xs: the k6 split form — strips = bytes 4..19 of the 20B row ([8 signed sc][8 pad]), d = bytes 0..1 if (!empty(t.iq4xss)) { var @exact_size xc : array let nsb = long_length(t.iq4xss) / IQ4XS_SSB @@ -194,7 +195,6 @@ def metal_blob_scale_plane(var t : Model; name : string) : bool { return true } if (name == "iq3ss") { - // iq3s: the row IS the iq4xs 20B shape — the same split (strips = bytes 4..19, d = bytes 0..1), over t.iq3ss if (!empty(t.iq3ss)) { var @exact_size i3c : array let nsb = long_length(t.iq3ss) / IQ3S_SSB @@ -213,7 +213,6 @@ def metal_blob_scale_plane(var t : Model; name : string) : bool { return true } if (name == "iq3xxss") { - // iq3xxs: the row IS the iq4xs 20B shape — the same split (strips = bytes 4..19, d = bytes 0..1), over t.iq3xxss if (!empty(t.iq3xxss)) { var @exact_size i3xc : array let nsb = long_length(t.iq3xxss) / IQ3XXS_SSB @@ -232,7 +231,6 @@ def metal_blob_scale_plane(var t : Model; name : string) : bool { return true } if (name == "iq2ss") { - // iq2s: the row IS the iq4xs 20B shape - the same split (strips = bytes 4..19, d8 = bytes 0..1), over t.iq2ss if (!empty(t.iq2ss)) { var @exact_size i2c : array let nsb = long_length(t.iq2ss) / IQ2S_SSB @@ -251,7 +249,6 @@ def metal_blob_scale_plane(var t : Model; name : string) : bool { return true } if (name == "iq2xss") { - // iq2xs: the identical 20B row over t.iq2xss - same split, d8 tail if (!empty(t.iq2xss)) { var @exact_size i2xc : array let nsb = long_length(t.iq2xss) / IQ2XS_SSB @@ -270,7 +267,6 @@ def metal_blob_scale_plane(var t : Model; name : string) : bool { return true } if (name == "iq2xxss") { - // iq2xxs: the identical 20B row over t.iq2xxss (8 strips + 8 pad ride the 16B region) if (!empty(t.iq2xxss)) { var @exact_size i2xxc : array let nsb = long_length(t.iq2xxss) / IQ2XXS_SSB @@ -289,7 +285,6 @@ def metal_blob_scale_plane(var t : Model; name : string) : bool { return true } if (name == "k2s") { - // k2: the 20B row splits like k6 - the 16 pair-byte strips first, the 4B d+dmin headers as the tail if (!empty(t.k2s)) { var @exact_size k2c : array let nsb = long_length(t.k2s) / K2_SSB @@ -309,7 +304,6 @@ def metal_blob_scale_plane(var t : Model; name : string) : bool { return true } if (name == "k3s") { - // k3: the CPU row IS the k6 form ([16 int8][f16 d]) — the same split, over t.k3s if (!empty(t.k3s)) { var @exact_size k3c : array let nsb = long_length(t.k3s) / K3_SSB diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_common.das b/modules/dasLLAMA/dasllama/dasllama_metal_common.das index 9cea1b0e47..f43a3dc1f9 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_common.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_common.das @@ -300,7 +300,7 @@ var g_pso_kq_mm_b4 : MetalComputePipeline? // batch B>=9: the prefill-owned kq var g_pso_kq_mm_b5 : MetalComputePipeline? var g_pso_kq_mm_b6 : MetalComputePipeline? var g_pso_kq_mm_b44 : MetalComputePipeline? -var g_pso_kq_mm_b3 : MetalComputePipeline? // fused W1|W3+swiglu (s16 path) +var g_pso_kq_mm_b3 : MetalComputePipeline? // the batched-step set (P4): fixed-B GEMVs + the M-pad-32 GEMM twin (f32 X) + row-table kernels var g_pso_gemv_b2 : MetalComputePipeline? var g_pso_gemv_b4 : MetalComputePipeline? @@ -923,6 +923,7 @@ def kq_quants_of(dev : MetalDevice?; t : Model; fmt : KqFmt; off : int64) : tupl //! One kq tensor's GPU-form scale bind (the gemv lab's layouts, pre-baked by the blob //! transform): k4/k5 = 16B compact blocks at soff; k6 = the split plane — sub-scale strips at //! soff, the packed f16 d plane at doff (the kernels bind the buffer twice, once per offset). +[arch(at = "../ARCHITECTURE_GPU.md#metal-kq-split-scale-plane")] def kq_scales_of(dev : MetalDevice?; t : Model; fmt : KqFmt; off : int64) : tuple { assert(t.metal_blob) let sb0 = off / 256l @@ -936,22 +937,22 @@ def kq_scales_of(dev : MetalDevice?; t : Model; fmt : KqFmt; off : int64) : tupl return (buf = plane_buffer(dev, addr < void? >(t.k4s[0]), uint64(long_length(t.k4s)), t.image_map != null), soff = uint64(sb0 * 16l), doff = 0ul) } - if (fmt == KqFmt.k3) { // the k6 split form over the 18B/sb device plane + if (fmt == KqFmt.k3) { let plane_sb = long_length(t.k3s) / 18l return (buf = plane_buffer(dev, addr < void? >(t.k3s[0]), uint64(long_length(t.k3s)), t.image_map != null), soff = uint64(sb0 * 16l), doff = uint64(plane_sb * 16l + sb0 * 2l)) } - if (fmt == KqFmt.iq3s) { // the k6 split form over the 18B/sb device plane + if (fmt == KqFmt.iq3s) { let plane_sb = long_length(t.iq3ss) / 18l return (buf = plane_buffer(dev, addr < void? >(t.iq3ss[0]), uint64(long_length(t.iq3ss)), t.image_map != null), soff = uint64(sb0 * 16l), doff = uint64(plane_sb * 16l + sb0 * 2l)) } - if (fmt == KqFmt.iq3xxs) { // the k6 split form over the 18B/sb device plane + if (fmt == KqFmt.iq3xxs) { let plane_sb = long_length(t.iq3xxss) / 18l return (buf = plane_buffer(dev, addr < void? >(t.iq3xxss[0]), uint64(long_length(t.iq3xxss)), t.image_map != null), soff = uint64(sb0 * 16l), doff = uint64(plane_sb * 16l + sb0 * 2l)) } - if (fmt == KqFmt.iq4xs) { // the k6 split form over the 18B/sb device plane + if (fmt == KqFmt.iq4xs) { let plane_sb = long_length(t.iq4xss) / 18l return (buf = plane_buffer(dev, addr < void? >(t.iq4xss[0]), uint64(long_length(t.iq4xss)), t.image_map != null), soff = uint64(sb0 * 16l), doff = uint64(plane_sb * 16l + sb0 * 2l)) @@ -970,12 +971,12 @@ def kq_scales_of(dev : MetalDevice?; t : Model; fmt : KqFmt; off : int64) : tupl return (buf = plane_buffer(dev, addr < void? >(t.iq2ss[0]), uint64(long_length(t.iq2ss)), t.image_map != null), soff = uint64(sb0 * 16l), doff = uint64(plane_sb * 16l + sb0 * 2l)) } - if (fmt == KqFmt.iq2xs) { // the same split 18B/sb plane over t.iq2xss + if (fmt == KqFmt.iq2xs) { let plane_sb = long_length(t.iq2xss) / 18l return (buf = plane_buffer(dev, addr < void? >(t.iq2xss[0]), uint64(long_length(t.iq2xss)), t.image_map != null), soff = uint64(sb0 * 16l), doff = uint64(plane_sb * 16l + sb0 * 2l)) } - if (fmt == KqFmt.iq2xxs) { // the same split 18B/sb plane over t.iq2xxss + if (fmt == KqFmt.iq2xxs) { let plane_sb = long_length(t.iq2xxss) / 18l return (buf = plane_buffer(dev, addr < void? >(t.iq2xxss[0]), uint64(long_length(t.iq2xxss)), t.image_map != null), soff = uint64(sb0 * 16l), doff = uint64(plane_sb * 16l + sb0 * 2l)) diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das index db94e0ed55..ea16bc6f65 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das @@ -5972,19 +5972,16 @@ class MetalKqMulMmK6 : MetalKqMulMmK45T { override SIXBIT = true } -// iq4xs rides the IQ4XS arm; the split scale plane binds twice like k6 (d plane at s0off = doff) [metal_dispatch(name = "enc_kq_mm_iq4xs_c", pso = "g_pso_kq_mm_b44", tgmem = "MetalKqMulMmIq4xs_metal_kq_mulmm_msl_tgmem", tg = 128, grid = "mp/32, rows/64", params = "mp : int64, rows : int64")] class MetalKqMulMmIq4xs : MetalKqMulMmK45T { override IQ4XS = true } -// q3_K rides the K3 arm; the split scale plane binds twice like k6 (d plane at s0off = doff) [metal_dispatch(name = "enc_kq_mm_k3_c", pso = "g_pso_kq_mm_b3", tgmem = "MetalKqMulMmK3_metal_kq_mulmm_msl_tgmem", tg = 128, grid = "mp/32, rows/64", params = "mp : int64, rows : int64")] class MetalKqMulMmK3 : MetalKqMulMmK45T { override K3 = true } -// iq3s rides the IQ3S arm; the split scale plane binds twice like k6 (d plane at s0off = doff) [metal_dispatch(name = "enc_kq_mm_iq3s_c", pso = "g_pso_kq_mm_b33", tgmem = "MetalKqMulMmIq3s_metal_kq_mulmm_msl_tgmem", tg = 128, grid = "mp/32, rows/64", params = "mp : int64, rows : int64")] class MetalKqMulMmIq3s : MetalKqMulMmK45T { override IQ3S = true @@ -5995,31 +5992,27 @@ class MetalKqMulMmIq3xxs : MetalKqMulMmK45T { override IQ3XXS = true } -// q2_K rides its own K2 arm; the split scale plane binds twice like k6 (the 4B tail at s0off = doff) [metal_dispatch(name = "enc_kq_mm_k2_c", pso = "g_pso_kq_mm_b2", tgmem = "MetalKqMulMmK2_metal_kq_mulmm_msl_tgmem", tg = 128, grid = "mp/32, rows/64", params = "mp : int64, rows : int64")] class MetalKqMulMmK2 : MetalKqMulMmK45T { override K2 = true } -// iq2s rides its own IQ2S arm; the split scale plane binds twice like k6 (d8 plane at s0off = doff) [metal_dispatch(name = "enc_kq_mm_iq2s_c", pso = "g_pso_kq_mm_b23", tgmem = "MetalKqMulMmIq2s_metal_kq_mulmm_msl_tgmem", tg = 128, grid = "mp/32, rows/64", params = "mp : int64, rows : int64")] class MetalKqMulMmIq2s : MetalKqMulMmK45T { override IQ2S = true } -// iq2xs rides its own IQ2XS arm; the same split scale plane binds (d8 plane at s0off = doff) [metal_dispatch(name = "enc_kq_mm_iq2xs_c", pso = "g_pso_kq_mm_b24", tgmem = "MetalKqMulMmIq2xs_metal_kq_mulmm_msl_tgmem", tg = 128, grid = "mp/32, rows/64", params = "mp : int64, rows : int64")] class MetalKqMulMmIq2xs : MetalKqMulMmK45T { override IQ2XS = true } -// iq2xxs rides its own IQ2XXS arm; the same split scale plane binds (d8 plane at s0off = doff) [metal_dispatch(name = "enc_kq_mm_iq2xxs_c", pso = "g_pso_kq_mm_b25", tgmem = "MetalKqMulMmIq2xxs_metal_kq_mulmm_msl_tgmem", tg = 128, grid = "mp/32, rows/64", params = "mp : int64, rows : int64")] class MetalKqMulMmIq2xxs : MetalKqMulMmK45T { override IQ2XXS = true } -// iq4nl rides the IQ4XS arm with IQ4NL flipping the scale read to q40's per-32-block d plane +// IQ4NL flips the scale read to q40's per-32-block d plane [metal_dispatch(name = "enc_kq_mm_iq4nl_c", pso = "g_pso_kq_mm_b45", tgmem = "MetalKqMulMmIq4nl_metal_kq_mulmm_msl_tgmem", tg = 128, grid = "mp/32, rows/64", params = "mp : int64, rows : int64")] class MetalKqMulMmIq4nl : MetalKqMulMmK45T { override IQ4XS = true @@ -6303,15 +6296,15 @@ def private kmask_scales(sv : uint4; iq : uint) : uint4 { ((hw4 >> 4u) & 0x0F0F) | ((hw2 & 0xC0C0) >> 2u)) } -// IQ4_XS codebook (ggml's kvalues_iq4nl) as four packed words, selected + byte-extracted — +// IQ4_XS codebook (kvalues_iq4nl) as four packed words, selected + byte-extracted — // no table memory; the k6 sign trick widens the byte def private iq4_lut(q : uint) : float { let w = q < 8u ? (q < 4u ? 3215825025u : 4142587343u) : (q < 12u ? 639175937u : 1901675829u) return float((int((w >> ((q & 3u) * 8u)) & 255u) ^ 128) - 128) } -// ggml's iq3s_grid, one word per call - the GEMV/mul_mm stage it into threadgroup memory once -// per group; the mv twins read it directly (unmeasured batch shapes - followup #58's stance). +// iq3s_grid, one word per call - the GEMV/mul_mm stage it into threadgroup memory once +// per group; the mv twins read it directly. def private iq3s_gw(i : int) : uint { let tbl = fixed_array( 0x01010101u, 0x01010103u, 0x01010105u, 0x0101010bu, 0x0101010fu, 0x01010301u, 0x01010303u, 0x01010305u, @@ -6392,7 +6385,7 @@ def private iq3s_sx(sw, t : uint) : float { return float((int((sw >> (t * 8u)) & 255u) ^ 128) - 128) } -// the HALVED iq3xxs grid (ggml's iq3xxs_grid, every byte / 2 - the plane convention), one +// the HALVED iq3xxs grid (iq3xxs_grid, every byte / 2 - the plane convention), one // word per call; the GEMV and mul_mm stage it, the mv twins read it directly. def private iq3xxs_gw(i : int) : uint { let tbl = fixed_array( @@ -6439,7 +6432,7 @@ def private ksign7m(v : uint) : uint { return v | ((t & 1u) << 7u) } -// ggml's iq2xxs_grid as low/high word pairs (word 2i = magnitudes 0..3 of u64 entry i, word +// iq2xxs_grid as low/high word pairs (word 2i = magnitudes 0..3 of u64 entry i, word // 2i+1 = 4..7); the same all-literal program-scope-constant form as its grid siblings. def private iq2xxs_gw(i : int) : uint { let tbl = fixed_array( @@ -6510,7 +6503,7 @@ def private iq2xxs_gw(i : int) : uint { return tbl[i] } -// ggml's iq2xs_grid as low/high word pairs (word 2i = magnitudes 0..3 of u64 entry i, word +// iq2xs_grid as low/high word pairs (word 2i = magnitudes 0..3 of u64 entry i, word // 2i+1 = 4..7); the same all-literal program-scope-constant form as iq2s_gw below. def private iq2xs_gw(i : int) : uint { // nolint:STYLE038 - a 1024-word data table, not splittable let tbl = fixed_array( @@ -6645,7 +6638,7 @@ def private iq2xs_gw(i : int) : uint { // nolint:STYLE038 - a 1024-word data t return tbl[i] } -// ggml's iq2s_grid as low/high word pairs (word 2i = magnitudes 0..3 of u64 entry i, word 2i+1 +// iq2s_grid as low/high word pairs (word 2i = magnitudes 0..3 of u64 entry i, word 2i+1 // = 4..7); an all-literal fixed_array local, so it lowers to a program-scope constant table - // every kernel reads it DIRECT (an f4 slab of 1024 entries would be 32 KB, past the tg budget). def private iq2s_gw(i : int) : uint { // nolint:STYLE038 - a 2048-word data table, not splittable @@ -8524,7 +8517,7 @@ class MetalKqGemvIq4nl { } } -// IQ3_S: w = d*sc*(+-grid byte); the 2 KB grid in threadgroup memory (llama.cpp's shape), a +// IQ3_S: w = d*sc*(+-grid byte); the 2 KB grid in threadgroup memory, a // 16-lane half-simdgroup per superblock, lane (bu, il) = block bu's half il: one qs word = 4 // grid words = 16 elems = 4 CONSECUTIVE float4 x loads. Scale plane = the iq4xs/k6 split form. [metal_dispatch(name = "enc_kq_iq3s_c", pso = "g_pso_kq_iq3s", tgmem = "metal_kq_gemv_iq3s_msl_tgmem", tg = 64, grid = "rows/8", params = "rows : int64, n : int64")] @@ -8536,7 +8529,7 @@ class MetalKqGemvIq3s { @ssbo @binding = 4 @role = "write" @off = "yoff" @span = "rows*4" y : array @uniform @binding = 5 ndim : uint @uniform @binding = 6 ddim : uint - @workgroup gridf : float4[512] // magnitudes pre-expanded at stage time: the inner loop reads one float4 and flips signs, no byte extracts (the best of the eight raced forms - llama.cpp's own geometry included) + @workgroup gridf : float4[512] // magnitudes pre-expanded at stage time: the inner loop reads one float4 and flips signs, no byte extracts (the best of the eight raced forms, the reference exe's geometry included) [metal_kernel(name="metal_kq_gemv_iq3s_msl")] def metal_kq_gemv_iq3s { @@ -8553,7 +8546,7 @@ class MetalKqGemvIq3s { let bu = it / 2u let il = it % 2u let nb = ndim / 256u - let first_row = (gl_WorkGroupID.x * 2u + gl_SubgroupID) * 4u // 4 rows per simdgroup - y loads and grid gathers amortize x4 (llama.cpp N_R0_IQ3_S) + let first_row = (gl_WorkGroupID.x * 2u + gl_SubgroupID) * 4u // 4 rows per simdgroup - y loads and grid gathers amortize x4 var sumf : float[4] var ib = ix while (ib < nb) { @@ -8839,7 +8832,7 @@ class MetalKqGemvIq3xxs { } // The iq3xxs B2/B4 pair: the iq3s shells over the 24-word rows - direct halved-grid reads, -// parity signs (unmeasured batch shapes - followup #58's stance). +// parity signs. [ |> template_struct_instance] class template MetalKqMvIq3xxsT { @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d plane — the scale buffer bound at byte nsb*16 diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das b/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das index 7a0b3adba9..b6687d0a7e 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das @@ -4411,7 +4411,7 @@ def private pf_devw_panel(enc : MetalComputeEncoder?; bwblob : MetalBuffer?; wbo //! quant superblocks, plus the k6 2B d tail def private pf_devw_panel_kq(enc : MetalComputeEncoder?; t : Model; fmt : KqFmt; woff : int64; bxh, by, bk, bn : MetalBuffer?; rows, d, kdim : int64; yoff : uint64) : bool { - if (fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s || fmt == KqFmt.iq3xxs || fmt == KqFmt.iq4nl || fmt == KqFmt.k2 || fmt == KqFmt.iq2s || fmt == KqFmt.iq2xs || fmt == KqFmt.iq2xxs) { // no dev-W dequant kernel for these formats yet + if (fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s || fmt == KqFmt.iq3xxs || fmt == KqFmt.iq4nl || fmt == KqFmt.k2 || fmt == KqFmt.iq2s || fmt == KqFmt.iq2xs || fmt == KqFmt.iq2xxs) { //! no dev-W dequant kernel for these formats return false } let dq = fmt == KqFmt.k6 ? g_pf_pso_dq_k6 : (fmt == KqFmt.k4 ? g_pf_pso_dq_k4 : g_pf_pso_dq_k5) @@ -4576,39 +4576,39 @@ def private pf_enc_kq_site_mm(enc : MetalComputeEncoder?; t : Model; fmt : KqFmt let fmt_tensor = fmt == KqFmt.k6 ? g_pf_kq_mm6_tensor : (fmt == KqFmt.k4 ? g_pf_kq_mm4_tensor : g_pf_kq_mm5_tensor) let bq = kq_quants_of(g_dev, t, fmt, woff) let bs = kq_scales_of(g_dev, t, fmt, woff) - if (fmt == KqFmt.iq4xs) { // the base mul_mm only - no tensor / tall / dev-W twins yet + if (fmt == KqFmt.iq4xs) { enc_kq_mm_iq4xs_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, by, yoff, bk, bn, mp, rows) return } - if (fmt == KqFmt.k3) { // the base mul_mm only - no tensor / tall / dev-W twins yet + if (fmt == KqFmt.k3) { enc_kq_mm_k3_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, by, yoff, bk, bn, mp, rows) return } - if (fmt == KqFmt.iq3s) { // the base mul_mm only - no tensor / tall / dev-W twins yet + if (fmt == KqFmt.iq3s) { enc_kq_mm_iq3s_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, by, yoff, bk, bn, mp, rows) return } - if (fmt == KqFmt.iq3xxs) { // the base mul_mm only - no tensor / tall / dev-W twins yet + if (fmt == KqFmt.iq3xxs) { enc_kq_mm_iq3xxs_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, by, yoff, bk, bn, mp, rows) return } - if (fmt == KqFmt.iq4nl) { // the base mul_mm only - no tensor / tall / dev-W twins yet + if (fmt == KqFmt.iq4nl) { enc_kq_mm_iq4nl_c(enc, bs.buf, bs.soff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, by, yoff, bk, bn, mp, rows) return } - if (fmt == KqFmt.k2) { // the base mul_mm only - no tensor / tall / dev-W twins yet + if (fmt == KqFmt.k2) { enc_kq_mm_k2_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, by, yoff, bk, bn, mp, rows) return } - if (fmt == KqFmt.iq2s) { // the base mul_mm only - no tensor / tall / dev-W twins yet + if (fmt == KqFmt.iq2s) { enc_kq_mm_iq2s_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, by, yoff, bk, bn, mp, rows) return } - if (fmt == KqFmt.iq2xs) { // the base mul_mm only - no tensor / tall / dev-W twins yet + if (fmt == KqFmt.iq2xs) { enc_kq_mm_iq2xs_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, by, yoff, bk, bn, mp, rows) return } - if (fmt == KqFmt.iq2xxs) { // the base mul_mm only - no tensor / tall / dev-W twins yet + if (fmt == KqFmt.iq2xxs) { enc_kq_mm_iq2xxs_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, by, yoff, bk, bn, mp, rows) return } diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_shapes.das b/modules/dasLLAMA/dasllama/dasllama_metal_shapes.das index 41435e5fe7..08003d490e 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_shapes.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_shapes.das @@ -236,6 +236,7 @@ def dn_metal_ok(t : Model) : bool { } // (q8 32-blocks, kq 256-superblocks — the expert shift is index math, no base multiple) +[arch(at = "../ARCHITECTURE_GPU.md#metal-kq-split-scale-plane")] def moe_site_ok(fmt : KqFmt; off, ege : int64) : bool { if (fmt == KqFmt.q8) { return off >= 0l && (off % 256l) == 0l && (ege % 32l) == 0l @@ -243,7 +244,7 @@ def moe_site_ok(fmt : KqFmt; off, ege : int64) : bool { if (fmt == KqFmt.k4 || fmt == KqFmt.k5) { return off >= 0l && (off % 256l) == 0l && (ege % 256l) == 0l } - if (fmt == KqFmt.k6 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s || fmt == KqFmt.iq3xxs || fmt == KqFmt.iq2s || fmt == KqFmt.iq2xs || fmt == KqFmt.iq2xxs) { // the split scale form: off % 512 keeps the d-plane bind 4B-aligned + if (fmt == KqFmt.k6 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s || fmt == KqFmt.iq3xxs || fmt == KqFmt.iq2s || fmt == KqFmt.iq2xs || fmt == KqFmt.iq2xxs) { return off >= 0l && (off % 512l) == 0l && (ege % 256l) == 0l } if (fmt == KqFmt.q51) { // per-32 planes: off % 128 keeps the 20B/4B binds 16B-aligned diff --git a/modules/dasLLAMA/dasllama/dasllama_tune.das b/modules/dasLLAMA/dasllama/dasllama_tune.das index aa7e7ef99e..90fadbbbbd 100644 --- a/modules/dasLLAMA/dasllama/dasllama_tune.das +++ b/modules/dasLLAMA/dasllama/dasllama_tune.das @@ -30,7 +30,7 @@ struct TunePerm { } def private build_grid() : array { - // pruned to seats that win beyond noise (the 20-seat walk never spread >1.4% on any box); unroll=1 omits unroll_count (rows byte-identical to the hand hints); vec4_u4 = dot_q4's pinned NEON nibble-widening winner + // pruned to seats that win beyond noise (the 20-seat walk never spread >1.4% winner-vs-runner-up in any box's sidecar race tables); unroll=1 omits unroll_count (rows byte-identical to the hand hints); vec4_u4 = dot_q4's pinned NEON nibble-widening winner let WIDTHS = fixed_array(8, 16) let UNROLLS = fixed_array(2) // unroll-only rows; u1 == plain var grid : array diff --git a/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das b/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das index 7c601e8e35..b39c43d303 100644 --- a/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das +++ b/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das @@ -2465,10 +2465,10 @@ def private k5_dep(hb : uint) : uint { } // IQ4_XS codebook decode of one word of 4 nibble lanes (bits 7..4 of each byte clear) into 4 -// SIGNED int8 lanes (ggml's kvalues_iq4nl) — the sdot4 operand form; batch and gemv share it. +// SIGNED int8 lanes (kvalues_iq4nl) — the sdot4 operand form; batch and gemv share it. // The codebook is four packed words picked by a dynamic vector index and byte-extracted: pure -// ALU (a fixed_array local was a Function-storage table, 3.4x slower on the decode GEMV) -// ggml's iq3s_grid, one word per call - the staging loops copy it into workgroup memory once +// ALU (a fixed_array local was a Function-storage table, 3.4x slower on the decode GEMV - `benchmarks/lcpp_bench.das` tg128, 5060 Ti) +// iq3s_grid, one word per call - the staging loops copy it into workgroup memory once // per kernel (a few indexed reads per thread), so the local-array storage class (QUIRK 20's // slow path) never sits on a hot path; a shader function cannot return a fixed array. def private iq3s_grid_word(i : int) : uint { @@ -2540,7 +2540,7 @@ def private iq3s_grid_word(i : int) : uint { return tbl[i] } -// ggml's iq2xxs_grid as low/high word pairs - 256 u64 entries, one uint word per call (word +// iq2xxs_grid as low/high word pairs - 256 u64 entries, one uint word per call (word // 2i = magnitudes 0..3 of entry i, word 2i+1 = 4..7); staged into workgroup memory per kernel. def private iq2xxs_grid_word(i : int) : uint { let tbl = fixed_array( @@ -2611,142 +2611,142 @@ def private iq2xxs_grid_word(i : int) : uint { return tbl[i] } -// ggml's iq2xs_grid as low/high word pairs - 512 u64 entries, one uint word per call (word +// iq2xs_grid as low/high word pairs - 512 u64 entries, one uint word per call (word // 2i = magnitudes 0..3 of entry i, word 2i+1 = 4..7); staged into workgroup memory per kernel. def private iq2xs_grid_word(i : int) : uint { // nolint:STYLE038 - a 1024-word data table, not splittable let tbl = fixed_array( - 0x08080808, 0x08080808, 0x0808082b, 0x08080808, 0x08081919, 0x08080808, 0x08082b08, 0x08080808, - 0x08082b2b, 0x08080808, 0x08190819, 0x08080808, 0x08191908, 0x08080808, 0x0819192b, 0x08080808, - 0x08192b19, 0x08080808, 0x082b0808, 0x08080808, 0x082b082b, 0x08080808, 0x082b1919, 0x08080808, - 0x082b2b08, 0x08080808, 0x19080819, 0x08080808, 0x19081908, 0x08080808, 0x1908192b, 0x08080808, - 0x19082b19, 0x08080808, 0x19190808, 0x08080808, 0x1919082b, 0x08080808, 0x19191919, 0x08080808, - 0x19192b08, 0x08080808, 0x192b0819, 0x08080808, 0x192b1908, 0x08080808, 0x2b080808, 0x08080808, - 0x2b08082b, 0x08080808, 0x2b081919, 0x08080808, 0x2b082b08, 0x08080808, 0x2b190819, 0x08080808, - 0x2b191908, 0x08080808, 0x2b192b19, 0x08080808, 0x2b2b0808, 0x08080808, 0x08080819, 0x08080819, - 0x08081908, 0x08080819, 0x0808192b, 0x08080819, 0x08082b19, 0x08080819, 0x08190808, 0x08080819, - 0x0819082b, 0x08080819, 0x08191919, 0x08080819, 0x08192b08, 0x08080819, 0x08192b2b, 0x08080819, - 0x082b0819, 0x08080819, 0x082b1908, 0x08080819, 0x19080808, 0x08080819, 0x1908082b, 0x08080819, - 0x19081919, 0x08080819, 0x19082b08, 0x08080819, 0x19190819, 0x08080819, 0x19191908, 0x08080819, - 0x192b0808, 0x08080819, 0x192b2b08, 0x08080819, 0x2b080819, 0x08080819, 0x2b081908, 0x08080819, - 0x2b190808, 0x08080819, 0x08080808, 0x0808082b, 0x0808082b, 0x0808082b, 0x08081919, 0x0808082b, - 0x08082b08, 0x0808082b, 0x08190819, 0x0808082b, 0x08191908, 0x0808082b, 0x082b0808, 0x0808082b, - 0x19080819, 0x0808082b, 0x19081908, 0x0808082b, 0x19190808, 0x0808082b, 0x19191919, 0x0808082b, - 0x2b080808, 0x0808082b, 0x2b082b2b, 0x0808082b, 0x08080819, 0x08081908, 0x08081908, 0x08081908, - 0x0808192b, 0x08081908, 0x08082b19, 0x08081908, 0x08190808, 0x08081908, 0x0819082b, 0x08081908, - 0x08191919, 0x08081908, 0x08192b08, 0x08081908, 0x082b0819, 0x08081908, 0x082b1908, 0x08081908, - 0x19080808, 0x08081908, 0x1908082b, 0x08081908, 0x19081919, 0x08081908, 0x19082b08, 0x08081908, - 0x19190819, 0x08081908, 0x19191908, 0x08081908, 0x1919192b, 0x08081908, 0x192b0808, 0x08081908, - 0x2b080819, 0x08081908, 0x2b081908, 0x08081908, 0x2b190808, 0x08081908, 0x08080808, 0x08081919, - 0x0808082b, 0x08081919, 0x08081919, 0x08081919, 0x08082b08, 0x08081919, 0x08190819, 0x08081919, - 0x08191908, 0x08081919, 0x082b0808, 0x08081919, 0x19080819, 0x08081919, 0x19081908, 0x08081919, - 0x19190808, 0x08081919, 0x192b0819, 0x08081919, 0x2b080808, 0x08081919, 0x08080819, 0x0808192b, - 0x08081908, 0x0808192b, 0x08190808, 0x0808192b, 0x082b192b, 0x0808192b, 0x19080808, 0x0808192b, - 0x1908082b, 0x0808192b, 0x2b081908, 0x0808192b, 0x08080808, 0x08082b08, 0x0808082b, 0x08082b08, - 0x08081919, 0x08082b08, 0x08082b08, 0x08082b08, 0x08082b2b, 0x08082b08, 0x08190819, 0x08082b08, - 0x08191908, 0x08082b08, 0x082b0808, 0x08082b08, 0x082b1919, 0x08082b08, 0x19080819, 0x08082b08, - 0x19081908, 0x08082b08, 0x19190808, 0x08082b08, 0x19192b08, 0x08082b08, 0x2b080808, 0x08082b08, - 0x2b2b0808, 0x08082b08, 0x2b2b2b2b, 0x08082b08, 0x08080819, 0x08082b19, 0x08081908, 0x08082b19, - 0x08190808, 0x08082b19, 0x19080808, 0x08082b19, 0x2b080819, 0x08082b19, 0x2b082b19, 0x08082b19, - 0x08080808, 0x08082b2b, 0x082b0808, 0x08082b2b, 0x082b2b08, 0x08082b2b, 0x2b19192b, 0x08082b2b, - 0x2b2b0808, 0x08082b2b, 0x08080819, 0x08190808, 0x08081908, 0x08190808, 0x0808192b, 0x08190808, - 0x08082b19, 0x08190808, 0x08190808, 0x08190808, 0x0819082b, 0x08190808, 0x08191919, 0x08190808, - 0x08192b08, 0x08190808, 0x082b0819, 0x08190808, 0x082b1908, 0x08190808, 0x19080808, 0x08190808, - 0x1908082b, 0x08190808, 0x19081919, 0x08190808, 0x19082b08, 0x08190808, 0x19190819, 0x08190808, - 0x19191908, 0x08190808, 0x192b0808, 0x08190808, 0x192b2b2b, 0x08190808, 0x2b080819, 0x08190808, - 0x2b081908, 0x08190808, 0x2b190808, 0x08190808, 0x08080808, 0x08190819, 0x0808082b, 0x08190819, - 0x08081919, 0x08190819, 0x08082b08, 0x08190819, 0x08190819, 0x08190819, 0x08191908, 0x08190819, - 0x082b0808, 0x08190819, 0x19080819, 0x08190819, 0x19081908, 0x08190819, 0x19190808, 0x08190819, - 0x2b080808, 0x08190819, 0x2b191908, 0x08190819, 0x2b19192b, 0x08190819, 0x08080819, 0x0819082b, - 0x08081908, 0x0819082b, 0x0808192b, 0x0819082b, 0x08190808, 0x0819082b, 0x19080808, 0x0819082b, - 0x192b0808, 0x0819082b, 0x08080808, 0x08191908, 0x0808082b, 0x08191908, 0x08081919, 0x08191908, - 0x08082b08, 0x08191908, 0x08190819, 0x08191908, 0x08191908, 0x08191908, 0x082b0808, 0x08191908, - 0x19080819, 0x08191908, 0x19081908, 0x08191908, 0x19082b19, 0x08191908, 0x19190808, 0x08191908, - 0x192b1908, 0x08191908, 0x2b080808, 0x08191908, 0x08080819, 0x08191919, 0x08081908, 0x08191919, - 0x08190808, 0x08191919, 0x19080808, 0x08191919, 0x08080808, 0x0819192b, 0x08191908, 0x0819192b, - 0x19082b19, 0x0819192b, 0x08080819, 0x08192b08, 0x08081908, 0x08192b08, 0x08190808, 0x08192b08, - 0x0819082b, 0x08192b08, 0x19080808, 0x08192b08, 0x19191908, 0x08192b08, 0x2b08192b, 0x08192b08, - 0x08080808, 0x08192b19, 0x08081919, 0x08192b19, 0x192b192b, 0x08192b19, 0x19190819, 0x08192b2b, - 0x2b2b2b19, 0x08192b2b, 0x08080808, 0x082b0808, 0x0808082b, 0x082b0808, 0x08081919, 0x082b0808, - 0x08082b08, 0x082b0808, 0x08082b2b, 0x082b0808, 0x08190819, 0x082b0808, 0x08191908, 0x082b0808, - 0x082b0808, 0x082b0808, 0x19080819, 0x082b0808, 0x19081908, 0x082b0808, 0x19190808, 0x082b0808, - 0x2b080808, 0x082b0808, 0x2b2b0808, 0x082b0808, 0x08080819, 0x082b0819, 0x08081908, 0x082b0819, - 0x08190808, 0x082b0819, 0x19080808, 0x082b0819, 0x19082b08, 0x082b0819, 0x192b1919, 0x082b0819, - 0x08080808, 0x082b082b, 0x082b082b, 0x082b082b, 0x2b080808, 0x082b082b, 0x2b2b2b08, 0x082b082b, - 0x08080819, 0x082b1908, 0x08081908, 0x082b1908, 0x08190808, 0x082b1908, 0x082b2b19, 0x082b1908, - 0x19080808, 0x082b1908, 0x08080808, 0x082b1919, 0x19080819, 0x082b1919, 0x1919082b, 0x082b1919, - 0x2b192b19, 0x082b1919, 0x08080819, 0x082b192b, 0x08192b2b, 0x082b192b, 0x2b2b192b, 0x082b192b, - 0x08080808, 0x082b2b08, 0x08082b08, 0x082b2b08, 0x08082b2b, 0x082b2b08, 0x082b0808, 0x082b2b08, - 0x19191919, 0x082b2b08, 0x2b082b08, 0x082b2b08, 0x2b2b082b, 0x082b2b08, 0x192b2b08, 0x082b2b19, - 0x2b190808, 0x082b2b19, 0x08082b08, 0x082b2b2b, 0x082b0808, 0x082b2b2b, 0x2b08082b, 0x082b2b2b, - 0x2b082b08, 0x082b2b2b, 0x2b082b2b, 0x082b2b2b, 0x08080819, 0x19080808, 0x08081908, 0x19080808, - 0x0808192b, 0x19080808, 0x08082b19, 0x19080808, 0x08190808, 0x19080808, 0x0819082b, 0x19080808, - 0x08191919, 0x19080808, 0x08192b08, 0x19080808, 0x082b0819, 0x19080808, 0x082b1908, 0x19080808, - 0x19080808, 0x19080808, 0x1908082b, 0x19080808, 0x19081919, 0x19080808, 0x19082b08, 0x19080808, - 0x19082b2b, 0x19080808, 0x19190819, 0x19080808, 0x19191908, 0x19080808, 0x192b0808, 0x19080808, - 0x192b1919, 0x19080808, 0x2b080819, 0x19080808, 0x2b081908, 0x19080808, 0x2b190808, 0x19080808, - 0x08080808, 0x19080819, 0x0808082b, 0x19080819, 0x08081919, 0x19080819, 0x08082b08, 0x19080819, - 0x08190819, 0x19080819, 0x08191908, 0x19080819, 0x082b0808, 0x19080819, 0x19080819, 0x19080819, - 0x19081908, 0x19080819, 0x19190808, 0x19080819, 0x2b080808, 0x19080819, 0x2b081919, 0x19080819, - 0x2b2b082b, 0x19080819, 0x08080819, 0x1908082b, 0x08081908, 0x1908082b, 0x08190808, 0x1908082b, - 0x0819082b, 0x1908082b, 0x082b2b19, 0x1908082b, 0x19080808, 0x1908082b, 0x08080808, 0x19081908, - 0x0808082b, 0x19081908, 0x08081919, 0x19081908, 0x08082b08, 0x19081908, 0x08190819, 0x19081908, - 0x08191908, 0x19081908, 0x08192b19, 0x19081908, 0x082b0808, 0x19081908, 0x19080819, 0x19081908, - 0x19081908, 0x19081908, 0x19190808, 0x19081908, 0x2b080808, 0x19081908, 0x2b191908, 0x19081908, - 0x08080819, 0x19081919, 0x08081908, 0x19081919, 0x08190808, 0x19081919, 0x082b1908, 0x19081919, - 0x19080808, 0x19081919, 0x2b192b2b, 0x19081919, 0x08080808, 0x1908192b, 0x08082b2b, 0x1908192b, - 0x19081908, 0x1908192b, 0x19190808, 0x1908192b, 0x08080819, 0x19082b08, 0x08081908, 0x19082b08, - 0x08190808, 0x19082b08, 0x19080808, 0x19082b08, 0x19081919, 0x19082b08, 0x19191908, 0x19082b08, - 0x192b082b, 0x19082b08, 0x08080808, 0x19082b19, 0x08190819, 0x19082b19, 0x19081908, 0x19082b19, - 0x19190808, 0x19082b19, 0x192b2b19, 0x19082b19, 0x08081908, 0x19082b2b, 0x08080808, 0x19190808, - 0x0808082b, 0x19190808, 0x08081919, 0x19190808, 0x08082b08, 0x19190808, 0x08190819, 0x19190808, - 0x08191908, 0x19190808, 0x082b0808, 0x19190808, 0x082b2b08, 0x19190808, 0x19080819, 0x19190808, - 0x19081908, 0x19190808, 0x19190808, 0x19190808, 0x2b080808, 0x19190808, 0x08080819, 0x19190819, - 0x08081908, 0x19190819, 0x08190808, 0x19190819, 0x08191919, 0x19190819, 0x19080808, 0x19190819, - 0x1908082b, 0x19190819, 0x08080808, 0x1919082b, 0x19081908, 0x1919082b, 0x2b2b2b2b, 0x1919082b, - 0x08080819, 0x19191908, 0x08081908, 0x19191908, 0x08190808, 0x19191908, 0x082b0819, 0x19191908, - 0x19080808, 0x19191908, 0x192b0808, 0x19191908, 0x2b080819, 0x19191908, 0x2b2b0819, 0x19191908, - 0x08080808, 0x19191919, 0x08082b08, 0x19191919, 0x2b080808, 0x19191919, 0x2b082b08, 0x19191919, - 0x082b0819, 0x1919192b, 0x192b2b08, 0x1919192b, 0x2b2b0819, 0x1919192b, 0x08080808, 0x19192b08, - 0x08191908, 0x19192b08, 0x19080819, 0x19192b08, 0x19190808, 0x19192b08, 0x2b192b19, 0x19192b08, - 0x08192b2b, 0x19192b19, 0x19080808, 0x19192b19, 0x1908082b, 0x19192b19, 0x2b081919, 0x19192b2b, - 0x08080819, 0x192b0808, 0x08081908, 0x192b0808, 0x08190808, 0x192b0808, 0x19080808, 0x192b0808, - 0x19191908, 0x192b0808, 0x192b082b, 0x192b0808, 0x2b08192b, 0x192b0808, 0x2b2b2b19, 0x192b0808, - 0x08080808, 0x192b0819, 0x082b1908, 0x192b082b, 0x19082b2b, 0x192b082b, 0x2b19082b, 0x192b082b, - 0x08080808, 0x192b1908, 0x0819192b, 0x192b1908, 0x08190808, 0x192b1919, 0x19080808, 0x192b1919, - 0x19081919, 0x192b1919, 0x2b2b1908, 0x192b1919, 0x08080819, 0x192b2b08, 0x192b2b2b, 0x192b2b08, - 0x082b1919, 0x192b2b19, 0x0808192b, 0x192b2b2b, 0x19191908, 0x192b2b2b, 0x192b082b, 0x192b2b2b, - 0x08080808, 0x2b080808, 0x0808082b, 0x2b080808, 0x08081919, 0x2b080808, 0x08082b08, 0x2b080808, - 0x08190819, 0x2b080808, 0x08191908, 0x2b080808, 0x082b0808, 0x2b080808, 0x082b2b2b, 0x2b080808, - 0x19080819, 0x2b080808, 0x19081908, 0x2b080808, 0x19190808, 0x2b080808, 0x2b080808, 0x2b080808, - 0x2b08082b, 0x2b080808, 0x2b2b2b08, 0x2b080808, 0x2b2b2b2b, 0x2b080808, 0x08080819, 0x2b080819, - 0x08081908, 0x2b080819, 0x0808192b, 0x2b080819, 0x08190808, 0x2b080819, 0x19080808, 0x2b080819, - 0x19190819, 0x2b080819, 0x19192b19, 0x2b080819, 0x08080808, 0x2b08082b, 0x082b0808, 0x2b08082b, - 0x2b080808, 0x2b08082b, 0x2b08082b, 0x2b08082b, 0x2b2b0808, 0x2b08082b, 0x2b2b2b08, 0x2b08082b, - 0x08080819, 0x2b081908, 0x08081908, 0x2b081908, 0x08190808, 0x2b081908, 0x0819082b, 0x2b081908, - 0x08191919, 0x2b081908, 0x19080808, 0x2b081908, 0x192b0808, 0x2b081908, 0x2b082b19, 0x2b081908, - 0x08080808, 0x2b081919, 0x19081908, 0x2b081919, 0x2b2b1919, 0x2b081919, 0x08192b08, 0x2b08192b, - 0x192b2b2b, 0x2b08192b, 0x08080808, 0x2b082b08, 0x08082b08, 0x2b082b08, 0x082b1919, 0x2b082b08, - 0x19192b2b, 0x2b082b08, 0x2b080808, 0x2b082b08, 0x2b08082b, 0x2b082b08, 0x2b2b2b08, 0x2b082b08, - 0x0808192b, 0x2b082b19, 0x082b082b, 0x2b082b2b, 0x2b080808, 0x2b082b2b, 0x2b082b08, 0x2b082b2b, - 0x2b19192b, 0x2b082b2b, 0x2b2b2b08, 0x2b082b2b, 0x08080819, 0x2b190808, 0x08081908, 0x2b190808, - 0x08190808, 0x2b190808, 0x19080808, 0x2b190808, 0x1919192b, 0x2b190808, 0x2b081908, 0x2b190808, - 0x08080808, 0x2b190819, 0x082b082b, 0x2b190819, 0x192b1908, 0x2b190819, 0x1919192b, 0x2b19082b, - 0x2b082b19, 0x2b19082b, 0x08080808, 0x2b191908, 0x08081919, 0x2b191908, 0x19081908, 0x2b191908, - 0x19190808, 0x2b191908, 0x19192b08, 0x2b191908, 0x082b2b19, 0x2b191919, 0x2b190808, 0x2b191919, - 0x2b19082b, 0x2b191919, 0x19080819, 0x2b19192b, 0x19190819, 0x2b192b08, 0x2b2b192b, 0x2b192b08, - 0x19082b19, 0x2b192b19, 0x08191919, 0x2b192b2b, 0x192b0808, 0x2b192b2b, 0x08080808, 0x2b2b0808, - 0x0808082b, 0x2b2b0808, 0x08082b08, 0x2b2b0808, 0x08082b2b, 0x2b2b0808, 0x082b0808, 0x2b2b0808, - 0x082b2b2b, 0x2b2b0808, 0x2b2b0808, 0x2b2b0808, 0x19190819, 0x2b2b0819, 0x19192b19, 0x2b2b0819, - 0x2b2b192b, 0x2b2b0819, 0x08080808, 0x2b2b082b, 0x0808082b, 0x2b2b082b, 0x08082b08, 0x2b2b082b, - 0x082b2b2b, 0x2b2b082b, 0x2b080808, 0x2b2b082b, 0x2b2b0808, 0x2b2b082b, 0x19080808, 0x2b2b1908, - 0x2b191919, 0x2b2b1908, 0x192b1919, 0x2b2b192b, 0x2b192b08, 0x2b2b192b, 0x08082b2b, 0x2b2b2b08, - 0x082b0808, 0x2b2b2b08, 0x082b082b, 0x2b2b2b08, 0x082b2b08, 0x2b2b2b08, 0x2b2b0808, 0x2b2b2b08, - 0x2b2b2b08, 0x2b2b2b08, 0x08081908, 0x2b2b2b19, 0x2b081908, 0x2b2b2b19, 0x2b08192b, 0x2b2b2b19, + 0x08080808, 0x08080808, 0x0808082b, 0x08080808, 0x08081919, 0x08080808, 0x08082b08, 0x08080808, + 0x08082b2b, 0x08080808, 0x08190819, 0x08080808, 0x08191908, 0x08080808, 0x0819192b, 0x08080808, + 0x08192b19, 0x08080808, 0x082b0808, 0x08080808, 0x082b082b, 0x08080808, 0x082b1919, 0x08080808, + 0x082b2b08, 0x08080808, 0x19080819, 0x08080808, 0x19081908, 0x08080808, 0x1908192b, 0x08080808, + 0x19082b19, 0x08080808, 0x19190808, 0x08080808, 0x1919082b, 0x08080808, 0x19191919, 0x08080808, + 0x19192b08, 0x08080808, 0x192b0819, 0x08080808, 0x192b1908, 0x08080808, 0x2b080808, 0x08080808, + 0x2b08082b, 0x08080808, 0x2b081919, 0x08080808, 0x2b082b08, 0x08080808, 0x2b190819, 0x08080808, + 0x2b191908, 0x08080808, 0x2b192b19, 0x08080808, 0x2b2b0808, 0x08080808, 0x08080819, 0x08080819, + 0x08081908, 0x08080819, 0x0808192b, 0x08080819, 0x08082b19, 0x08080819, 0x08190808, 0x08080819, + 0x0819082b, 0x08080819, 0x08191919, 0x08080819, 0x08192b08, 0x08080819, 0x08192b2b, 0x08080819, + 0x082b0819, 0x08080819, 0x082b1908, 0x08080819, 0x19080808, 0x08080819, 0x1908082b, 0x08080819, + 0x19081919, 0x08080819, 0x19082b08, 0x08080819, 0x19190819, 0x08080819, 0x19191908, 0x08080819, + 0x192b0808, 0x08080819, 0x192b2b08, 0x08080819, 0x2b080819, 0x08080819, 0x2b081908, 0x08080819, + 0x2b190808, 0x08080819, 0x08080808, 0x0808082b, 0x0808082b, 0x0808082b, 0x08081919, 0x0808082b, + 0x08082b08, 0x0808082b, 0x08190819, 0x0808082b, 0x08191908, 0x0808082b, 0x082b0808, 0x0808082b, + 0x19080819, 0x0808082b, 0x19081908, 0x0808082b, 0x19190808, 0x0808082b, 0x19191919, 0x0808082b, + 0x2b080808, 0x0808082b, 0x2b082b2b, 0x0808082b, 0x08080819, 0x08081908, 0x08081908, 0x08081908, + 0x0808192b, 0x08081908, 0x08082b19, 0x08081908, 0x08190808, 0x08081908, 0x0819082b, 0x08081908, + 0x08191919, 0x08081908, 0x08192b08, 0x08081908, 0x082b0819, 0x08081908, 0x082b1908, 0x08081908, + 0x19080808, 0x08081908, 0x1908082b, 0x08081908, 0x19081919, 0x08081908, 0x19082b08, 0x08081908, + 0x19190819, 0x08081908, 0x19191908, 0x08081908, 0x1919192b, 0x08081908, 0x192b0808, 0x08081908, + 0x2b080819, 0x08081908, 0x2b081908, 0x08081908, 0x2b190808, 0x08081908, 0x08080808, 0x08081919, + 0x0808082b, 0x08081919, 0x08081919, 0x08081919, 0x08082b08, 0x08081919, 0x08190819, 0x08081919, + 0x08191908, 0x08081919, 0x082b0808, 0x08081919, 0x19080819, 0x08081919, 0x19081908, 0x08081919, + 0x19190808, 0x08081919, 0x192b0819, 0x08081919, 0x2b080808, 0x08081919, 0x08080819, 0x0808192b, + 0x08081908, 0x0808192b, 0x08190808, 0x0808192b, 0x082b192b, 0x0808192b, 0x19080808, 0x0808192b, + 0x1908082b, 0x0808192b, 0x2b081908, 0x0808192b, 0x08080808, 0x08082b08, 0x0808082b, 0x08082b08, + 0x08081919, 0x08082b08, 0x08082b08, 0x08082b08, 0x08082b2b, 0x08082b08, 0x08190819, 0x08082b08, + 0x08191908, 0x08082b08, 0x082b0808, 0x08082b08, 0x082b1919, 0x08082b08, 0x19080819, 0x08082b08, + 0x19081908, 0x08082b08, 0x19190808, 0x08082b08, 0x19192b08, 0x08082b08, 0x2b080808, 0x08082b08, + 0x2b2b0808, 0x08082b08, 0x2b2b2b2b, 0x08082b08, 0x08080819, 0x08082b19, 0x08081908, 0x08082b19, + 0x08190808, 0x08082b19, 0x19080808, 0x08082b19, 0x2b080819, 0x08082b19, 0x2b082b19, 0x08082b19, + 0x08080808, 0x08082b2b, 0x082b0808, 0x08082b2b, 0x082b2b08, 0x08082b2b, 0x2b19192b, 0x08082b2b, + 0x2b2b0808, 0x08082b2b, 0x08080819, 0x08190808, 0x08081908, 0x08190808, 0x0808192b, 0x08190808, + 0x08082b19, 0x08190808, 0x08190808, 0x08190808, 0x0819082b, 0x08190808, 0x08191919, 0x08190808, + 0x08192b08, 0x08190808, 0x082b0819, 0x08190808, 0x082b1908, 0x08190808, 0x19080808, 0x08190808, + 0x1908082b, 0x08190808, 0x19081919, 0x08190808, 0x19082b08, 0x08190808, 0x19190819, 0x08190808, + 0x19191908, 0x08190808, 0x192b0808, 0x08190808, 0x192b2b2b, 0x08190808, 0x2b080819, 0x08190808, + 0x2b081908, 0x08190808, 0x2b190808, 0x08190808, 0x08080808, 0x08190819, 0x0808082b, 0x08190819, + 0x08081919, 0x08190819, 0x08082b08, 0x08190819, 0x08190819, 0x08190819, 0x08191908, 0x08190819, + 0x082b0808, 0x08190819, 0x19080819, 0x08190819, 0x19081908, 0x08190819, 0x19190808, 0x08190819, + 0x2b080808, 0x08190819, 0x2b191908, 0x08190819, 0x2b19192b, 0x08190819, 0x08080819, 0x0819082b, + 0x08081908, 0x0819082b, 0x0808192b, 0x0819082b, 0x08190808, 0x0819082b, 0x19080808, 0x0819082b, + 0x192b0808, 0x0819082b, 0x08080808, 0x08191908, 0x0808082b, 0x08191908, 0x08081919, 0x08191908, + 0x08082b08, 0x08191908, 0x08190819, 0x08191908, 0x08191908, 0x08191908, 0x082b0808, 0x08191908, + 0x19080819, 0x08191908, 0x19081908, 0x08191908, 0x19082b19, 0x08191908, 0x19190808, 0x08191908, + 0x192b1908, 0x08191908, 0x2b080808, 0x08191908, 0x08080819, 0x08191919, 0x08081908, 0x08191919, + 0x08190808, 0x08191919, 0x19080808, 0x08191919, 0x08080808, 0x0819192b, 0x08191908, 0x0819192b, + 0x19082b19, 0x0819192b, 0x08080819, 0x08192b08, 0x08081908, 0x08192b08, 0x08190808, 0x08192b08, + 0x0819082b, 0x08192b08, 0x19080808, 0x08192b08, 0x19191908, 0x08192b08, 0x2b08192b, 0x08192b08, + 0x08080808, 0x08192b19, 0x08081919, 0x08192b19, 0x192b192b, 0x08192b19, 0x19190819, 0x08192b2b, + 0x2b2b2b19, 0x08192b2b, 0x08080808, 0x082b0808, 0x0808082b, 0x082b0808, 0x08081919, 0x082b0808, + 0x08082b08, 0x082b0808, 0x08082b2b, 0x082b0808, 0x08190819, 0x082b0808, 0x08191908, 0x082b0808, + 0x082b0808, 0x082b0808, 0x19080819, 0x082b0808, 0x19081908, 0x082b0808, 0x19190808, 0x082b0808, + 0x2b080808, 0x082b0808, 0x2b2b0808, 0x082b0808, 0x08080819, 0x082b0819, 0x08081908, 0x082b0819, + 0x08190808, 0x082b0819, 0x19080808, 0x082b0819, 0x19082b08, 0x082b0819, 0x192b1919, 0x082b0819, + 0x08080808, 0x082b082b, 0x082b082b, 0x082b082b, 0x2b080808, 0x082b082b, 0x2b2b2b08, 0x082b082b, + 0x08080819, 0x082b1908, 0x08081908, 0x082b1908, 0x08190808, 0x082b1908, 0x082b2b19, 0x082b1908, + 0x19080808, 0x082b1908, 0x08080808, 0x082b1919, 0x19080819, 0x082b1919, 0x1919082b, 0x082b1919, + 0x2b192b19, 0x082b1919, 0x08080819, 0x082b192b, 0x08192b2b, 0x082b192b, 0x2b2b192b, 0x082b192b, + 0x08080808, 0x082b2b08, 0x08082b08, 0x082b2b08, 0x08082b2b, 0x082b2b08, 0x082b0808, 0x082b2b08, + 0x19191919, 0x082b2b08, 0x2b082b08, 0x082b2b08, 0x2b2b082b, 0x082b2b08, 0x192b2b08, 0x082b2b19, + 0x2b190808, 0x082b2b19, 0x08082b08, 0x082b2b2b, 0x082b0808, 0x082b2b2b, 0x2b08082b, 0x082b2b2b, + 0x2b082b08, 0x082b2b2b, 0x2b082b2b, 0x082b2b2b, 0x08080819, 0x19080808, 0x08081908, 0x19080808, + 0x0808192b, 0x19080808, 0x08082b19, 0x19080808, 0x08190808, 0x19080808, 0x0819082b, 0x19080808, + 0x08191919, 0x19080808, 0x08192b08, 0x19080808, 0x082b0819, 0x19080808, 0x082b1908, 0x19080808, + 0x19080808, 0x19080808, 0x1908082b, 0x19080808, 0x19081919, 0x19080808, 0x19082b08, 0x19080808, + 0x19082b2b, 0x19080808, 0x19190819, 0x19080808, 0x19191908, 0x19080808, 0x192b0808, 0x19080808, + 0x192b1919, 0x19080808, 0x2b080819, 0x19080808, 0x2b081908, 0x19080808, 0x2b190808, 0x19080808, + 0x08080808, 0x19080819, 0x0808082b, 0x19080819, 0x08081919, 0x19080819, 0x08082b08, 0x19080819, + 0x08190819, 0x19080819, 0x08191908, 0x19080819, 0x082b0808, 0x19080819, 0x19080819, 0x19080819, + 0x19081908, 0x19080819, 0x19190808, 0x19080819, 0x2b080808, 0x19080819, 0x2b081919, 0x19080819, + 0x2b2b082b, 0x19080819, 0x08080819, 0x1908082b, 0x08081908, 0x1908082b, 0x08190808, 0x1908082b, + 0x0819082b, 0x1908082b, 0x082b2b19, 0x1908082b, 0x19080808, 0x1908082b, 0x08080808, 0x19081908, + 0x0808082b, 0x19081908, 0x08081919, 0x19081908, 0x08082b08, 0x19081908, 0x08190819, 0x19081908, + 0x08191908, 0x19081908, 0x08192b19, 0x19081908, 0x082b0808, 0x19081908, 0x19080819, 0x19081908, + 0x19081908, 0x19081908, 0x19190808, 0x19081908, 0x2b080808, 0x19081908, 0x2b191908, 0x19081908, + 0x08080819, 0x19081919, 0x08081908, 0x19081919, 0x08190808, 0x19081919, 0x082b1908, 0x19081919, + 0x19080808, 0x19081919, 0x2b192b2b, 0x19081919, 0x08080808, 0x1908192b, 0x08082b2b, 0x1908192b, + 0x19081908, 0x1908192b, 0x19190808, 0x1908192b, 0x08080819, 0x19082b08, 0x08081908, 0x19082b08, + 0x08190808, 0x19082b08, 0x19080808, 0x19082b08, 0x19081919, 0x19082b08, 0x19191908, 0x19082b08, + 0x192b082b, 0x19082b08, 0x08080808, 0x19082b19, 0x08190819, 0x19082b19, 0x19081908, 0x19082b19, + 0x19190808, 0x19082b19, 0x192b2b19, 0x19082b19, 0x08081908, 0x19082b2b, 0x08080808, 0x19190808, + 0x0808082b, 0x19190808, 0x08081919, 0x19190808, 0x08082b08, 0x19190808, 0x08190819, 0x19190808, + 0x08191908, 0x19190808, 0x082b0808, 0x19190808, 0x082b2b08, 0x19190808, 0x19080819, 0x19190808, + 0x19081908, 0x19190808, 0x19190808, 0x19190808, 0x2b080808, 0x19190808, 0x08080819, 0x19190819, + 0x08081908, 0x19190819, 0x08190808, 0x19190819, 0x08191919, 0x19190819, 0x19080808, 0x19190819, + 0x1908082b, 0x19190819, 0x08080808, 0x1919082b, 0x19081908, 0x1919082b, 0x2b2b2b2b, 0x1919082b, + 0x08080819, 0x19191908, 0x08081908, 0x19191908, 0x08190808, 0x19191908, 0x082b0819, 0x19191908, + 0x19080808, 0x19191908, 0x192b0808, 0x19191908, 0x2b080819, 0x19191908, 0x2b2b0819, 0x19191908, + 0x08080808, 0x19191919, 0x08082b08, 0x19191919, 0x2b080808, 0x19191919, 0x2b082b08, 0x19191919, + 0x082b0819, 0x1919192b, 0x192b2b08, 0x1919192b, 0x2b2b0819, 0x1919192b, 0x08080808, 0x19192b08, + 0x08191908, 0x19192b08, 0x19080819, 0x19192b08, 0x19190808, 0x19192b08, 0x2b192b19, 0x19192b08, + 0x08192b2b, 0x19192b19, 0x19080808, 0x19192b19, 0x1908082b, 0x19192b19, 0x2b081919, 0x19192b2b, + 0x08080819, 0x192b0808, 0x08081908, 0x192b0808, 0x08190808, 0x192b0808, 0x19080808, 0x192b0808, + 0x19191908, 0x192b0808, 0x192b082b, 0x192b0808, 0x2b08192b, 0x192b0808, 0x2b2b2b19, 0x192b0808, + 0x08080808, 0x192b0819, 0x082b1908, 0x192b082b, 0x19082b2b, 0x192b082b, 0x2b19082b, 0x192b082b, + 0x08080808, 0x192b1908, 0x0819192b, 0x192b1908, 0x08190808, 0x192b1919, 0x19080808, 0x192b1919, + 0x19081919, 0x192b1919, 0x2b2b1908, 0x192b1919, 0x08080819, 0x192b2b08, 0x192b2b2b, 0x192b2b08, + 0x082b1919, 0x192b2b19, 0x0808192b, 0x192b2b2b, 0x19191908, 0x192b2b2b, 0x192b082b, 0x192b2b2b, + 0x08080808, 0x2b080808, 0x0808082b, 0x2b080808, 0x08081919, 0x2b080808, 0x08082b08, 0x2b080808, + 0x08190819, 0x2b080808, 0x08191908, 0x2b080808, 0x082b0808, 0x2b080808, 0x082b2b2b, 0x2b080808, + 0x19080819, 0x2b080808, 0x19081908, 0x2b080808, 0x19190808, 0x2b080808, 0x2b080808, 0x2b080808, + 0x2b08082b, 0x2b080808, 0x2b2b2b08, 0x2b080808, 0x2b2b2b2b, 0x2b080808, 0x08080819, 0x2b080819, + 0x08081908, 0x2b080819, 0x0808192b, 0x2b080819, 0x08190808, 0x2b080819, 0x19080808, 0x2b080819, + 0x19190819, 0x2b080819, 0x19192b19, 0x2b080819, 0x08080808, 0x2b08082b, 0x082b0808, 0x2b08082b, + 0x2b080808, 0x2b08082b, 0x2b08082b, 0x2b08082b, 0x2b2b0808, 0x2b08082b, 0x2b2b2b08, 0x2b08082b, + 0x08080819, 0x2b081908, 0x08081908, 0x2b081908, 0x08190808, 0x2b081908, 0x0819082b, 0x2b081908, + 0x08191919, 0x2b081908, 0x19080808, 0x2b081908, 0x192b0808, 0x2b081908, 0x2b082b19, 0x2b081908, + 0x08080808, 0x2b081919, 0x19081908, 0x2b081919, 0x2b2b1919, 0x2b081919, 0x08192b08, 0x2b08192b, + 0x192b2b2b, 0x2b08192b, 0x08080808, 0x2b082b08, 0x08082b08, 0x2b082b08, 0x082b1919, 0x2b082b08, + 0x19192b2b, 0x2b082b08, 0x2b080808, 0x2b082b08, 0x2b08082b, 0x2b082b08, 0x2b2b2b08, 0x2b082b08, + 0x0808192b, 0x2b082b19, 0x082b082b, 0x2b082b2b, 0x2b080808, 0x2b082b2b, 0x2b082b08, 0x2b082b2b, + 0x2b19192b, 0x2b082b2b, 0x2b2b2b08, 0x2b082b2b, 0x08080819, 0x2b190808, 0x08081908, 0x2b190808, + 0x08190808, 0x2b190808, 0x19080808, 0x2b190808, 0x1919192b, 0x2b190808, 0x2b081908, 0x2b190808, + 0x08080808, 0x2b190819, 0x082b082b, 0x2b190819, 0x192b1908, 0x2b190819, 0x1919192b, 0x2b19082b, + 0x2b082b19, 0x2b19082b, 0x08080808, 0x2b191908, 0x08081919, 0x2b191908, 0x19081908, 0x2b191908, + 0x19190808, 0x2b191908, 0x19192b08, 0x2b191908, 0x082b2b19, 0x2b191919, 0x2b190808, 0x2b191919, + 0x2b19082b, 0x2b191919, 0x19080819, 0x2b19192b, 0x19190819, 0x2b192b08, 0x2b2b192b, 0x2b192b08, + 0x19082b19, 0x2b192b19, 0x08191919, 0x2b192b2b, 0x192b0808, 0x2b192b2b, 0x08080808, 0x2b2b0808, + 0x0808082b, 0x2b2b0808, 0x08082b08, 0x2b2b0808, 0x08082b2b, 0x2b2b0808, 0x082b0808, 0x2b2b0808, + 0x082b2b2b, 0x2b2b0808, 0x2b2b0808, 0x2b2b0808, 0x19190819, 0x2b2b0819, 0x19192b19, 0x2b2b0819, + 0x2b2b192b, 0x2b2b0819, 0x08080808, 0x2b2b082b, 0x0808082b, 0x2b2b082b, 0x08082b08, 0x2b2b082b, + 0x082b2b2b, 0x2b2b082b, 0x2b080808, 0x2b2b082b, 0x2b2b0808, 0x2b2b082b, 0x19080808, 0x2b2b1908, + 0x2b191919, 0x2b2b1908, 0x192b1919, 0x2b2b192b, 0x2b192b08, 0x2b2b192b, 0x08082b2b, 0x2b2b2b08, + 0x082b0808, 0x2b2b2b08, 0x082b082b, 0x2b2b2b08, 0x082b2b08, 0x2b2b2b08, 0x2b2b0808, 0x2b2b2b08, + 0x2b2b2b08, 0x2b2b2b08, 0x08081908, 0x2b2b2b19, 0x2b081908, 0x2b2b2b19, 0x2b08192b, 0x2b2b2b19, 0x082b2b08, 0x2b2b2b2b, 0x082b2b2b, 0x2b2b2b2b, 0x2b190819, 0x2b2b2b2b, 0x2b2b2b2b, 0x2b2b2b2b) return tbl[i] } -// ggml's iq2s_grid as low/high word pairs - 1024 u64 entries, one uint word per call (word 2i = +// iq2s_grid as low/high word pairs - 1024 u64 entries, one uint word per call (word 2i = // magnitudes 0..3 of entry i, word 2i+1 = 4..7); the staging loops copy it into workgroup // memory once per kernel, so the local-array storage class never sits on a hot path. def private iq2s_grid_word(i : int) : uint { // nolint:STYLE038 - a 2048-word data table, not splittable @@ -3010,7 +3010,7 @@ def private iq2s_grid_word(i : int) : uint { // nolint:STYLE038 - a 2048-word return tbl[i] } -// the HALVED iq3xxs grid (ggml's iq3xxs_grid with every byte / 2 - the plane convention that +// the HALVED iq3xxs grid (iq3xxs_grid with every byte / 2 - the plane convention that // absorbs the format's 0.25 into the stored d), one word per call; staging loops copy it into // workgroup memory (QUIRK 20's slow path stays off the hot loop). def private iq3xxs_grid_word(i : int) : uint { @@ -4003,8 +4003,8 @@ class CmF16Batch : MoeCmBase { let nsteps = (nbb + 7u) / 8u let tid = gl_LocalInvocationID.x let sg = gl_SubgroupID // 0..3 (subgroupSize 32) - let smi = sg / 2u // which 16-row half of the tile - let sni = sg % 2u // which 16-col half of the tile + let row_half = sg / 2u // which 16-row half of the tile + let col_half = sg % 2u // which 16-col half of the tile var acc : coopmatAcc_f32_16x16 var s = 0u while (s < nsteps) { @@ -4056,8 +4056,8 @@ class CmF16Batch : MoeCmBase { for (ks in range(2)) { var a : coopmatA_f16_16x16 var b : coopmatB_f16_16x16 - coopmatLoad(a, cm_as, int(smi * 16u * 256u + uint(bb) * 32u + uint(ks) * 16u), 256, 0) - coopmatLoad(b, cm_bs, int((uint(bb) * 32u + uint(ks) * 16u) * 32u + sni * 16u), 32, 0) + coopmatLoad(a, cm_as, int(row_half * 16u * 256u + uint(bb) * 32u + uint(ks) * 16u), 256, 0) + coopmatLoad(b, cm_bs, int((uint(bb) * 32u + uint(ks) * 16u) * 32u + col_half * 16u), 32, 0) acc = coopmatMulAdd(a, b, acc) } } @@ -4071,8 +4071,8 @@ class CmF16Batch : MoeCmBase { let idx = lane * 8u + uint(e) let m = idx / 16u let nn = idx % 16u - let orow = xt * 32u + smi * 16u + m - let ocol = wt * 32u + sni * 16u + nn + let orow = xt * 32u + row_half * 16u + m + let ocol = wt * 32u + col_half * 16u + nn if (orow < cnt && ocol < pa.d) { y[(row0 + orow) * pa.d + ocol] = cm_out[sg * 256u + idx] } @@ -4105,10 +4105,10 @@ class CmI8Batch : MoeCmBase { let nbb = pa.n / 32u let sg = gl_SubgroupID let lane = gl_SubgroupInvocationID - let smi = sg / 2u - let sni = sg % 2u - let arow = xt * 32u + smi * 16u // this subgroup's local token-row base - let bcol = wt * 32u + sni * 16u // this subgroup's output-col base + let row_half = sg / 2u + let col_half = sg % 2u + let arow = xt * 32u + row_half * 16u // this subgroup's local token-row base + let bcol = wt * 32u + col_half * 16u // this subgroup's output-col base for (e in range(8)) { cmi_acc[sg * 256u + lane * 8u + uint(e)] = 0.0 } @@ -4656,8 +4656,8 @@ class KqQ40CmF16 : MoeCmBase { let nsb = pa.n / 256u let tid = gl_LocalInvocationID.x let sg = gl_SubgroupID - let smi = sg / 2u - let sni = sg % 2u + let row_half = sg / 2u + let col_half = sg % 2u var acc : coopmatAcc_f32_16x16 var s = 0u while (s < nsb) { @@ -4715,8 +4715,8 @@ class KqQ40CmF16 : MoeCmBase { for (ks in range(2)) { var a : coopmatA_f16_16x16 var b : coopmatB_f16_16x16 - coopmatLoad(a, cm_as, int(smi * 16u * 256u + uint(bb) * 32u + uint(ks) * 16u), 256, 0) - coopmatLoad(b, cm_bs, int((uint(bb) * 32u + uint(ks) * 16u) * 32u + sni * 16u), 32, 0) + coopmatLoad(a, cm_as, int(row_half * 16u * 256u + uint(bb) * 32u + uint(ks) * 16u), 256, 0) + coopmatLoad(b, cm_bs, int((uint(bb) * 32u + uint(ks) * 16u) * 32u + col_half * 16u), 32, 0) acc = coopmatMulAdd(a, b, acc) } } @@ -4730,8 +4730,8 @@ class KqQ40CmF16 : MoeCmBase { let idx = lane * 8u + uint(e) let m = idx / 16u let nn = idx % 16u - let orow = xt * 32u + smi * 16u + m - let ocol = wt * 32u + sni * 16u + nn + let orow = xt * 32u + row_half * 16u + m + let ocol = wt * 32u + col_half * 16u + nn if (orow < cnt && ocol < pa.d) { y[(row0 + orow) * pa.d + ocol] = cm_out[sg * 256u + idx] } @@ -4794,8 +4794,7 @@ struct VkIq2xxsBlk { } // The cm2 tile: one class template over the two stamp axes. The token column BN - l 256, m 128, -// s 32 (the expert-bucket geometry: STILE's fast path loads a partial column UNCLAMPED and clamps -// only the store, so every plane it reads carries 32 rows of slack) - and the weight format: BLK +// s 32 (the expert-bucket geometry) - and the weight format: BLK // the 16-bit-lane block struct, ST the scale plane's element, BLKW the block's element width, and // the format template's decode. 256 threads, one 128-weight x BN-token tile per wg, BK=64 per // step, an 8x-unrolled k loop; the decode METHOD reads the separate scale plane by block index - @@ -4811,7 +4810,7 @@ class template KqCm2BatchT : MoeCmBase { @template_constant STILE : bool = false // the 32-row column's partial-column fast path @template_constant BLKW : uint = 256u // elements per weight block @template_constant IQLUT : bool = false // stage the iq4nl codebook into workgroup memory - @workgroup @template_gate = IQLUT iq4lut : float16[16] // kvalues_iq4nl as f16 (llama.cpp's init_iq_shmem form) + @workgroup @template_gate = IQLUT iq4lut : float16[16] // kvalues_iq4nl as f16 @template_constant IQ3GRID : bool = false // stage the 2 KB iq3s grid into workgroup memory @workgroup @template_gate = IQ3GRID iq3s_gridc : uint[512] @template_constant IQ3XGRID : bool = false // stage the 1 KB halved iq3xxs grid into workgroup memory @@ -4891,11 +4890,6 @@ class template KqCm2BatchT : MoeCmBase { var acc : ACC let t0 = row0 + xt * BN let m0 = wt * 128u - // FAST PATH — every access of this tile provably in bounds and K a whole number of - // BK steps: unclamped (clamp-Undefined) layouts, and the B/out strides masked to a - // multiple of 8 f16 (identity here — pa.n and pa.d are 32-multiples — but the mask makes - // the alignment PROVABLE to the driver's address analysis via stride &= ~7). The s column - // gates only the weight tile: its partial token column loads unclamped, its store clamps if (m0 + 128u <= pa.d && (STILE || xt * BN + BN <= cnt) && (pa.n & 63u) == 0u) { var fla : tensorLayout2D tensorLayoutCreate(fla) @@ -4910,8 +4904,6 @@ class template KqCm2BatchT : MoeCmBase { tensorLayoutCreate(flo) tensorLayoutSetDimension(flo, row0 + cnt, pa.d) tensorLayoutSetStride(flo, pa.d & ~7u, 1u) - // the no-split arm keeps LITERAL loop bounds and store base: the k0/k1/ybase form - // in this hot loop measured -27% pp on the 5060 Ti even when the values were 0/pa.n/0 if (pa.ksplit == 0u) { var k = 0u for (_i in range(int(pa.n / 512u))) { diff --git a/modules/dasLLAMA/dasllama/dasllama_vulkan_common.das b/modules/dasLLAMA/dasllama/dasllama_vulkan_common.das index bfb23db506..47f7a0dc88 100644 --- a/modules/dasLLAMA/dasllama/dasllama_vulkan_common.das +++ b/modules/dasLLAMA/dasllama/dasllama_vulkan_common.das @@ -1709,7 +1709,7 @@ struct ArenaSlab { } struct ArenaFmt { - fmt : int // int(KqFmt): 0 = q8, 1/2/3 = k4/k5/k6, 4 = q40, 6 = iq4xs, 7 = k3, 8 = iq3s, 9 = iq3xxs, 10 = iq4nl, 11 = k2 + fmt : int // int(KqFmt) slabs : array cur : int // slab the bump cursor is in slab_cap : int64 // max blocks one slab may carry (msr over both plane strides) @@ -1740,7 +1740,7 @@ def arena_local_blk(blk : int64) : int64 => blk & ((1l << 32l) - 1l) // per-block plane strides — the block is 32 weights for q8 and a 256-weight superblock for kq. // Mirrors stack_plane_bytes: it computes the same totals as count * these strides. -// the tier's stack fmt space is int(KqFmt) — bridge to the gemm_schema kq id space (4/5/6/40/44/3/33/34/45/2) +// the tier's stack fmt space is int(KqFmt) — bridge to the gemm_schema kq id space def vk_kq_schema_id(fmt : int) : int { if (fmt == 1) return 4 if (fmt == 2) return 5 @@ -1759,7 +1759,7 @@ def vk_kq_schema_id(fmt : int) : int { return 0 } -// fmt space here = int(KqFmt): 0 q8 (32-elem blocks), 1-4 and 6..11 the kq lattice (+ q40, iq4xs, k3, iq3s, iq3xxs, iq4nl, k2) +// fmt space here = int(KqFmt): 0 = q8 (32-elem blocks), every other value the kq lattice def vk_fmt_b32(fmt : int) : bool => fmt == 0 // meta/activation block unit of a stack format — q8 counts 32-weight blocks, the kq diff --git a/modules/dasLLAMA/followup_general.md b/modules/dasLLAMA/followup_general.md index c42fa72f6c..4a9b3a8a0e 100644 --- a/modules/dasLLAMA/followup_general.md +++ b/modules/dasLLAMA/followup_general.md @@ -730,10 +730,17 @@ dequant twins, and no MoE GEMV / mul_mm trio for the format. Done = the twins stamped on the existing templates, the LUT cost measured against a `constant` table and against a byte-pair decode on the M5, and `bench_metal_gemv_kernels` / `bench_metal_kq_mm_lab` rows - for the format beside k4's. - -59. **Partial mint (Boris, 2026-08-30, for after the formats arc): "takes existing mint and - mints new kernels only. at least in debug-only mode."** Today a sidecar missing ANY demanded + for the format beside k4's. The same gap now covers every format this arc added: `k3`, + `iq3s`, `iq3xxs`, `iq4nl`, `k2`, `iq2s`, `iq2xs` and `iq2xxs` take the base `mul_mm` at the + prefill site - no tensor (`_t`), tall (`_th128`), double-buffered (`_thdb`) or dev-W dequant + twins - and `pf_devw_panel_kq` declines all nine formats outright. Done = the twins stamped + on the existing templates for whichever formats measure worth it, and that decline list + shortened to match. + +59. **DONE (2026-08-31, the unquirk pass): `--tune-only ` re-mints one family into the + existing sidecar, and the shipped defaults profiles adopt-then-race only the residue.** + Original ask - partial mint (Boris, 2026-08-30, for after the formats arc): "takes existing mint and + mints new kernels only. at least in debug-only mode." Today a sidecar missing ANY demanded kernel re-tunes the whole scope (the completeness rule), so every new `[tune]` family - one per format in this arc - re-mints every application sidecar on the box on its next start, minutes each (HOW_TO_ADD_A_FORMAT.md QUIRK 17), and the same rule keeps a stub-era diff --git a/modules/dasLLAMA/harness/gen_tune_probe.das b/modules/dasLLAMA/harness/gen_tune_probe.das index b73a920b20..89e7879674 100644 --- a/modules/dasLLAMA/harness/gen_tune_probe.das +++ b/modules/dasLLAMA/harness/gen_tune_probe.das @@ -1821,7 +1821,7 @@ def private confirm_winner(winner : string) : string { return fb } -// The q8q8 family race, verbatim the old inline tune_mode_run body: the fat-batch tile bench +// The q8q8 family race: the fat-batch tile bench // picks the tile knobs (and with them the layout), the hot decode bench picks gkstep among the // rows sharing them, incumbent stickiness last. Returns the merged winner's suffix; "" means a // correctness gate failed and the tune must abort (the mismatch is printed inside). diff --git a/modules/dasLLAMA/harness/tune_kernels.das b/modules/dasLLAMA/harness/tune_kernels.das index bdd6a945c9..da2bd800c9 100644 --- a/modules/dasLLAMA/harness/tune_kernels.das +++ b/modules/dasLLAMA/harness/tune_kernels.das @@ -2336,8 +2336,6 @@ def bench_laneq4x4() : string { // nolint:STYLE038 — one bench: fixture, backe return report("dot_q8q8_laneq4x4", names, best, ok) } -// fallback: the tuner rail drives this half explicitly (DAS_TUNE_MODE=tune) — the auto -// policy must not fire a nested tuner off this program's own missing per-app sidecar // one METAL_TWIN race row: log it and bank the family when the tensor twin won def private metal_twin_absorb(var wins : array; var ran : bool&; r) { ran = true @@ -2348,6 +2346,8 @@ def private metal_twin_absorb(var wins : array; var ran : bool&; r) { } } +// fallback: the tuner rail drives this half explicitly (DAS_TUNE_MODE=tune) — the auto +// policy must not fire a nested tuner off this program's own missing per-app sidecar [export, tune_policy(missing = "fallback")] def main : int { // nolint:STYLE037,STYLE038 — flat one-bench-per-kernel rail; the noise probes are its checkpoints if (tuner_paranoid_requested()) { @@ -2368,7 +2368,7 @@ def main : int { // nolint:STYLE037,STYLE038 — flat one-bench-per-kernel rail; ("dot_mx4q8", @@bench_dot_mx4q8), ("quantize_q8_0_into_ptr", @@bench_quantize), ("quantize_q8_0_bs_into_ptr", @@bench_quantize_bs), ("rope_scaled_neox_tab", @@bench_rope_tab), ("gemm_f32_uk_4x16", @@bench_gemm_tile), - ("dot_q8q8_laneq4x4", @@bench_laneq4x4) // last: it pins the repack backend (sticks until process exit) + ("dot_q8q8_laneq4x4", @@bench_laneq4x4) ] let filtered = !empty(tune_only_filter()) if (filtered) { diff --git a/modules/dasLLAMA/tests/test_kqformat.das b/modules/dasLLAMA/tests/test_kqformat.das index 4c654c49f7..c2aec80cc3 100644 --- a/modules/dasLLAMA/tests/test_kqformat.das +++ b/modules/dasLLAMA/tests/test_kqformat.das @@ -213,7 +213,7 @@ def test_iq3xxs_tables(t : T?) { t |> equal(IQ3XXS_GRID[i], lut[i]) for (b in range(4)) { let v = int((lut[i] >> uint(8 * b)) & 0xFFu) - // ggml levels 4..62 halved: even 2..30 plus 31 (the 62 top level) + // the disk levels 4..62 halved: even 2..30 plus 31 (the 62 top level) t |> success(v == 31 || (v >= 2 && v <= 30 && v % 2 == 0), "halved grid byte in range") } } diff --git a/modules/dasLLAMA/tests/test_kquant.das b/modules/dasLLAMA/tests/test_kquant.das index dee5d07672..af7308aed6 100644 --- a/modules/dasLLAMA/tests/test_kquant.das +++ b/modules/dasLLAMA/tests/test_kquant.das @@ -171,7 +171,7 @@ def private build_iq4xs_block() : array { } // k3 synthetic superblock: 110B Q3_K disk block — 16 six-bit sub-scales SYNTH_SC[is % 8] packed in -// ggml's kmask form (written in the PACK direction), f16 d = SYNTH_D, the 2-bit lanes + hmask bits +// the disk kmask form (written in the PACK direction), f16 d = SYNTH_D, the 2-bit lanes + hmask bits // of q3_pat — weight k = (d * (SYNTH_SC[(k/16) % 8] - 32)) * (q3_pat(k) - 4) def private build_q3k_block() : array { var blkb : array diff --git a/modules/dasLLAMA/tests/test_metal_gemm_kernels.das b/modules/dasLLAMA/tests/test_metal_gemm_kernels.das index d9d3a486ec..a5d3d0d29c 100644 --- a/modules/dasLLAMA/tests/test_metal_gemm_kernels.das +++ b/modules/dasLLAMA/tests/test_metal_gemm_kernels.das @@ -123,7 +123,7 @@ def private kq_mulmm_gate(t : T?; dev, queue; fmt : int; tensor : bool; m, kdim, let ran = with_compute_encoder(queue, err) $(enc : MetalComputeEncoder?) { metal_set_pipeline(enc, pso) metal_set_threadgroup_memory_length(enc, tgm, 0) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 || fmt == 24 || fmt == 25 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs: the f16 d tail + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 || fmt == 24 || fmt == 25 ? uint64(nsb * 16) : 0ul, 0) // the split-scale formats: the f16 d tail at nsb*16 metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bx, 0ul, 3) @@ -484,7 +484,7 @@ def private kq_devw_gate(t : T?; dev, queue; fmt : int; m, kdim, ndim : int) { var dka = CvtArgs(total = uint(nblk)) let ran = with_compute_encoder(queue, err) $(enc : MetalComputeEncoder?) { metal_set_pipeline(enc, dq_pso) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 || fmt == 24 || fmt == 25 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs: the f16 d tail + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 || fmt == 24 || fmt == 25 ? uint64(nsb * 16) : 0ul, 0) // the split-scale formats: the f16 d tail at nsb*16 metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bwh, 0ul, 3) @@ -1461,7 +1461,7 @@ def private moe_mulmm_kq_gate(t : T?; dev, queue; fmt : int; kdim, ndim : int; c let ran = with_compute_encoder(queue, err) $(enc : MetalComputeEncoder?) { metal_set_pipeline(enc, pso) metal_set_threadgroup_memory_length(enc, tgm, 0) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 || fmt == 24 || fmt == 25 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs: the f16 d tail + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 || fmt == 24 || fmt == 25 ? uint64(nsb * 16) : 0ul, 0) // the split-scale formats: the f16 d tail at nsb*16 metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bx, 0ul, 3) @@ -2047,23 +2047,23 @@ def test_metal_gemm_kernels(t : T?) { kq_mulmm_gate(t, dev, queue, fmt, true, 320, 512, 64, halfx = true, tall = true) // off the 128 floor: the remainder pair at nonzero x/y offsets } kq_mulmm_gate(t, dev, queue, 4, false, 32, 256, 128) // second shape: 2 col tiles - kq_mulmm_gate(t, dev, queue, 44, false, 64, 512, 64) // iq4xs: the base form only (no tensor / tall twins) + kq_mulmm_gate(t, dev, queue, 44, false, 64, 512, 64) kq_mulmm_gate(t, dev, queue, 44, false, 32, 256, 128) - kq_mulmm_gate(t, dev, queue, 3, false, 64, 512, 64) // k3: the base form only (no tensor / tall twins) + kq_mulmm_gate(t, dev, queue, 3, false, 64, 512, 64) kq_mulmm_gate(t, dev, queue, 3, false, 32, 256, 128) - kq_mulmm_gate(t, dev, queue, 33, false, 64, 512, 64) // iq3s: the base form only (no tensor / tall twins) + kq_mulmm_gate(t, dev, queue, 33, false, 64, 512, 64) kq_mulmm_gate(t, dev, queue, 33, false, 32, 256, 128) - kq_mulmm_gate(t, dev, queue, 34, false, 64, 512, 64) // iq3xxs: the base form only (no tensor / tall twins) + kq_mulmm_gate(t, dev, queue, 34, false, 64, 512, 64) kq_mulmm_gate(t, dev, queue, 34, false, 32, 256, 128) - kq_mulmm_gate(t, dev, queue, 45, false, 64, 512, 64) // iq4nl: the base form only (no tensor / tall twins) + kq_mulmm_gate(t, dev, queue, 45, false, 64, 512, 64) kq_mulmm_gate(t, dev, queue, 45, false, 32, 256, 128) - kq_mulmm_gate(t, dev, queue, 2, false, 64, 512, 64) // k2: the base form only (no tensor / tall twins) + kq_mulmm_gate(t, dev, queue, 2, false, 64, 512, 64) kq_mulmm_gate(t, dev, queue, 2, false, 32, 256, 128) - kq_mulmm_gate(t, dev, queue, 23, false, 64, 512, 64) // iq2s: the base form only (no tensor / tall twins) + kq_mulmm_gate(t, dev, queue, 23, false, 64, 512, 64) kq_mulmm_gate(t, dev, queue, 23, false, 32, 256, 128) - kq_mulmm_gate(t, dev, queue, 24, false, 64, 512, 64) // iq2xs: the base form only (no tensor / tall twins) + kq_mulmm_gate(t, dev, queue, 24, false, 64, 512, 64) kq_mulmm_gate(t, dev, queue, 24, false, 32, 256, 128) - kq_mulmm_gate(t, dev, queue, 25, false, 64, 512, 64) // iq2xxs: the base form only (no tensor / tall twins) + kq_mulmm_gate(t, dev, queue, 25, false, 64, 512, 64) kq_mulmm_gate(t, dev, queue, 25, false, 32, 256, 128) q8_mulmm_t_gate(t, dev, queue, 64, 256, 64) q8_mulmm_t_gate(t, dev, queue, 64, 256, 64, halfx = true) diff --git a/modules/dasLLAMA/tests/test_metal_gemv_kernels.das b/modules/dasLLAMA/tests/test_metal_gemv_kernels.das index f1441069bd..c97620e11c 100644 --- a/modules/dasLLAMA/tests/test_metal_gemv_kernels.das +++ b/modules/dasLLAMA/tests/test_metal_gemv_kernels.das @@ -81,7 +81,7 @@ def private kq_gemv_gate(t : T?; dev, queue; vform : string; n, d : int) { if (vform == "iq4nl") { metal_set_threadgroup_memory_length(enc, metal_kq_gemv_iq4nl_msl_tgmem, 0) // the codebook slab } - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 || fmt == 24 || fmt == 25 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs/k3: the f16 d tail + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 || fmt == 24 || fmt == 25 ? uint64(nsb * 16) : 0ul, 0) // the split-scale formats: the f16 d tail at nsb*16 metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bx, 0ul, 3) @@ -174,7 +174,7 @@ def private kq_mvb_gate(t : T?; dev, queue; fmt, bwidth, n, d, nr, ys : int) { metal_set_threadgroup_memory_length(enc, fmt == 25 ? metal_kq_mvb8_iq2xxs_msl_tgmem : (fmt == 24 ? metal_kq_mvb8_iq2xs_msl_tgmem : (fmt == 23 ? metal_kq_mvb8_iq2s_msl_tgmem : (fmt == 2 ? metal_kq_mvb8_k2_msl_tgmem : (fmt == 45 ? metal_kq_mvb8_iq4nl_msl_tgmem : (fmt == 34 ? metal_kq_mvb8_iq3xxs_msl_tgmem : (fmt == 33 ? metal_kq_mvb8_iq3s_msl_tgmem : (fmt == 3 ? metal_kq_mvb8_k3_msl_tgmem : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl_tgmem : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_tgmem : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_tgmem : metal_kq_mvb8_k6_msl_tgmem)))))))))), 0) } - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 || fmt == 24 || fmt == 25 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs/k3: the f16 d tail + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 || fmt == 24 || fmt == 25 ? uint64(nsb * 16) : 0ul, 0) // the split-scale formats: the f16 d tail at nsb*16 metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bx, 0ul, 3) @@ -263,7 +263,7 @@ def private moe_kq_gemv_gate(t : T?; dev, queue; fmt, n, d, net, k, nst : int; p bxs = uint((perslot ? k : 1) * n), bys = uint(k * d), hasb = 0u) let ran = with_compute_encoder(queue, err) $(enc : MetalComputeEncoder?) { metal_set_pipeline(enc, pso) - metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 || fmt == 24 || fmt == 25 ? uint64(nsb * 16) : 0ul, 0) // k6/iq4xs/k3: the f16 d tail + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 || fmt == 24 || fmt == 25 ? uint64(nsb * 16) : 0ul, 0) // the split-scale formats: the f16 d tail at nsb*16 metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bx, 0ul, 3) diff --git a/modules/dasLLVM/ARCHITECTURE.md b/modules/dasLLVM/ARCHITECTURE.md index 6789e3eac5..3e1548490c 100644 --- a/modules/dasLLVM/ARCHITECTURE.md +++ b/modules/dasLLVM/ARCHITECTURE.md @@ -125,3 +125,20 @@ Environment knobs load ONCE, at context init, into the `[EnvConfig]` structs `g_ `g_env_tune` (`llvm_env.das`) - a mid-process `setenv` changes nothing the backend reads. In-process overrides therefore go through the tune setters (`tune_set_verbosity`, `tune_set_noise_cv`, ...), which also arm spawned children by exporting the matching variable. + +## 4. Host CPU feature truth on aarch64 {#aarch64-feature-truth} + +An aarch64 host target reads its CPU features from two sources, because neither answers alone. +`LLVMGetHostCPUFeatures` returns an EMPTY string on macOS - there the CPU name is meant to imply +the features - and a part this LLVM cannot name maps to the generic CPU, where SDOT and SMMLA +have no instruction to select and codegen aborts. `cpu_supports` reads the operating system +instead (sysctl / `AT_HWCAP` / `IsProcessorFeaturePresent`), so it answers for silicon LLVM has +never heard of. Both the tier gates (`init_jit_target_flags`) and the target machine's feature +string (`create_default_target_machine`) therefore take the union of the two: an LLVM host-string +hit OR a `cpu_supports` hit. A cross-compile triple takes neither - only the force env. + +The two ways a feature reaches the target machine's string license different things. A +detection-derived append - `+dotprod` always, `+i8mm` when `cpu_supports` confirms it - is +EXECUTION-safe: the silicon running this process really has the instruction. A +`DAS_JIT_ARM64_FORCE_FEATURES` append is EMISSION-only: it may name silicon this box does not +have, so the artifact is for another machine and executing it here traps. diff --git a/modules/dasLLVM/REVIEW.md b/modules/dasLLVM/REVIEW.md index 1f94d4b62e..4dd29c0620 100644 --- a/modules/dasLLVM/REVIEW.md +++ b/modules/dasLLVM/REVIEW.md @@ -59,3 +59,12 @@ `has_env_variable(expr)` outside `daslib/llvm_env.das`. Spell the name as a literal through the declared forms, or declare the knob, instead** (the literal-name forms are scanner-enforced by `tests/llvm_env_registry.das`; weakening that test is a defect). + +- **A diff that lets a `requires=` expression name a new CPU feature adds that name to + `TUNE_KNOWN_FEATURES` (`daslib/llvm_tune.das`) in the same change.** Every sidecar save + stamps this box's pass/fail over that list as the `features` fingerprint, so a name + missing from it makes a shipped defaults profile claim a seat its minting box never raced. + +- **A diff that arms `DAS_TUNE_ONLY` for a single tuner spawn clears it immediately after + that spawn.** The auto/restart policy guard re-execs the process, and a filter left armed + makes the re-exec's own mint partial. diff --git a/modules/dasLLVM/daslib/llvm_jit_common.das b/modules/dasLLVM/daslib/llvm_jit_common.das index a2a8e7b2f0..f28fd5724e 100644 --- a/modules/dasLLVM/daslib/llvm_jit_common.das +++ b/modules/dasLLVM/daslib/llvm_jit_common.das @@ -444,8 +444,7 @@ var public g_target_x64_amx = false var public g_target_os_linux = false // ARMv8.6 i8mm (SMMLA — 2×2 s8 matrix-multiply-accumulate; Apple M2/A15+, Graviton3+; M1 has -// DotProd only). Host truth is LLVMGetHostCPUFeatures "+i8mm" — populated on Linux aarch64, EMPTY -// on macOS where the CPU name implies features — so there the force env is the rail on both boxes. +// DotProd only). Host truth is LLVMGetHostCPUFeatures "+i8mm" OR cpu_supports (ARCHITECTURE.md sec.4). var public g_target_arm64_i8mm = false // ARMv8.2 fullfp16 (native half arithmetic — fadd.4h/8h etc.). Detected as i8mm, except @@ -603,6 +602,7 @@ def public init_jit_clopts() { // Pure target truth, no LLVM state touched: safe and required BEFORE init_jit, whose DisableJitVisitor // consults these gates. init_jit re-runs it — pass the SAME host_features or pass and emission diverge. // host_features=false targets a GENERIC CPU: cpuid gates stay off, since leaked F16C emits libcalls. +[arch(at = "../ARCHITECTURE.md#aarch64-feature-truth")] def public init_jit_target_flags(target_triple : string = ""; host_features : bool = true) { g_target_host_features = host_features g_target_is_wasm = target_triple |> starts_with("wasm") @@ -632,9 +632,6 @@ def public init_jit_target_flags(target_triple : string = ""; host_features : bo g_target_os_linux = (empty(target_triple) ? get_platform_name() == "linux" : find(target_triple, "linux") >= 0) - // aarch64 i8mm tier (see the g_target_arm64_i8mm declaration comment): host targets read - // LLVM's host string OR cpu_supports — macOS hands back an EMPTY host string, which left - // i8mm dark on every Apple M2+ — OR the arm force env; cross aarch64 triples the env only. g_target_arm64_i8mm = g_target_is_aarch64 && (arm64_forced_feature("i8mm") || (empty(target_triple) && host_features && (host_llvm_feature("i8mm") || cpu_supports("i8mm")))) // aarch64 fullfp16 (see declaration comment): darwin-arm64 == Apple Silicon == always. @@ -894,7 +891,7 @@ def public with_target_machine(triple, cpu, features : string; opt_level : uint; LLVMDisposeTargetMachine(targetMachine) } -[macro_function] +[macro_function, arch(at = "../ARCHITECTURE.md#aarch64-feature-truth")] def public create_default_target_machine(opt_level : uint; use_host_cpu : bool) : LLVMTargetMachineRef { LLVMInitializeAllTargetInfos() LLVMInitializeAllTargets() @@ -922,9 +919,6 @@ def public create_default_target_machine(opt_level : uint; use_host_cpu : bool) var targetMachine : LLVMTargetMachineRef if (use_host_cpu && g_target_is_aarch64) { var feats = empty(features_msg) ? "+dotprod" : "{features_msg},+dotprod" - // same protection for i8mm: an Apple part this LLVM cannot name maps to "generic" where - // SMMLA has no instruction to select. cpu_supports reads sysctl truth, so the append is - // execution-safe on this host (unlike the force env, which may name absent silicon). if (cpu_supports("i8mm")) { feats = "{feats},+i8mm" } diff --git a/modules/dasLLVM/daslib/llvm_tune.das b/modules/dasLLVM/daslib/llvm_tune.das index 67dbea3e9f..310052f7ba 100644 --- a/modules/dasLLVM/daslib/llvm_tune.das +++ b/modules/dasLLVM/daslib/llvm_tune.das @@ -916,9 +916,7 @@ def private scope_fnames_joined(scope : TuneScope) : string { return joined } -// the ';'-joined `fname=exprA+exprB` map of a scope's ISA-gated [tune] families (every -// requires= its [tune_perm] rows carry), read off the AST at guard-emission time — what lets -// the runtime guard ask which families have seats the adopted profile's box could not race +//! the ';'-joined `fname=exprA+exprB` map of a scope's ISA-gated [tune] families - every `requires=` its [tune_perm] rows carry, read off the AST for the runtime guard def private scope_gated_joined(scope : TuneScope) : string { var wanted : table wanted |> insert(scope.modName) @@ -1233,8 +1231,6 @@ def tune_pick_fallback(chain : string) : string { return "" } -// every feature name a requires= expression may query, in one place — the fingerprint and the -// class ladder both walk it, so a new gate name added here is immediately profile-aware let private TUNE_KNOWN_FEATURES = "avx2;f16c;fma;sse4.2;avxvnni;avxvnniint8;avx512f;avx512bw;avx512vl;avx512vnni;amx-tile;amx-int8;neon;dotprod;i8mm;bf16;fullfp16;sve;lse" //! The ';'-joined subset of the known `requires=` feature names this box passes — stamped @@ -1264,7 +1260,7 @@ def tune_cpu_class() : string { if (arch == "arm64" || arch == "aarch64") { return fallback_feature_ok("i8mm") ? "arm-i8mm" : "arm-neon" } - return "{get_platform_name()}-{arch}" // an honest key; no profile will match, and that is correct + return "{get_platform_name()}-{arch}" //! an arch with no ladder: a key no shipped profile can match, deliberately } //! Profile candidates for this box, most specific first — a box above the best shipped @@ -1290,16 +1286,16 @@ def tune_class_chain() : array { // evaluate a requires= expression against a ';'-joined feature set instead of this box — // "did the PROFILE's minting box pass this gate" is what separates covered from unlocked seats -def private expr_in_features(expr : string; featCsv : string) : bool { - return true if (empty(expr)) +def private requires_ok_in_fingerprint(requires : string; fingerprint : string) : bool { + return true if (empty(requires)) var have : table - for (f in split(featCsv, ";")) { + for (f in split(fingerprint, ";")) { if (!empty(f)) { have |> insert(f) } } var ok = true - for (conj in split(expr, ",")) { + for (conj in split(requires, ",")) { continue if (empty(conj)) var anyOk = false for (alt in split(conj, "|")) { @@ -1613,8 +1609,6 @@ class private TuneAnnotation : AstFunctionAnnotation { delete comps return false } - // DAS_TUNE_POLICY=reference is the true kill switch: original bodies, no stamps at all — - // unlike `fallback`, whose chain's first viable perm can be the very stamp under test if (g_env_tune.tune_policy == "reference") { let noPerms : array var rok = emit_registry(compiling_module(), func, noPerms, errors) @@ -1813,7 +1807,7 @@ def private apply_cli_tune_flags() : bool { tune_set_verbosity("verbose") } if (!empty(cfg.tune_only)) { - set_env_variable("DAS_TUNE_ONLY", cfg.tune_only) // the tuner children inherit it + set_env_variable("DAS_TUNE_ONLY", cfg.tune_only) } return cfg.tune || !empty(cfg.tune_only) } @@ -2135,13 +2129,11 @@ def public tune_suppress_mint(knob : string) { // ===== shipped defaults profiles ([tune_scope(defaults = "dir")]) ===== -// satisfy an untuned scope from a shipped class profile before any race: "" = nothing adopted, -// "*" = fully covered, else the ','-joined families still owed a race (profile-missing plus -// ISA-unlocked), which the caller arms as the DAS_TUNE_ONLY filter for the tuner it spawns -def private profile_try_adopt(defaultsDir, scopeName, manifestPath, fnames, gated : string) : string { // nolint:STYLE037,STYLE038 — one straight adopt pipeline: locate, validate, merge, diff - // a redirected runtime manifest path must never adopt into the wrong file - return "" if (empty(defaultsDir) || tune_manifest_path() != manifestPath) - // the version pin from the demand: the profile must record the same value or it predates this build +//! Adopt a shipped class profile into an untuned scope before any race. True = adopted; `raceCsv` is then the ','-joined families the caller still owes a race ("" = fully covered). +def private profile_try_adopt(defaultsDir, scopeName, manifestPath, fnames, gated : string; var raceCsv : string&) : bool { // nolint:STYLE037,STYLE038 — one straight adopt pipeline: locate, validate, merge, diff + raceCsv = "" + //! a runtime-redirected manifest path declines - adoption never lands in a file this guard was not compiled for + return false if (empty(defaultsDir) || tune_manifest_path() != manifestPath) var verKey = "" var verVal = "" for (tok in split(fnames, ";")) { @@ -2178,7 +2170,7 @@ def private profile_try_adopt(defaultsDir, scopeName, manifestPath, fnames, gate break } delete chain - return "" if (doc == null) + return false if (doc == null) var kernels : table let ks = doc?["kernels"] if (ks != null && ks.value is _object) { @@ -2190,9 +2182,8 @@ def private profile_try_adopt(defaultsDir, scopeName, manifestPath, fnames, gate } if (empty(kernels)) { delete_json(doc) - return "" + return false } - // owed a race: demanded-but-absent from the profile, plus gated families this box unlocks var race : array for (tok in split(fnames, ";")) { continue if (empty(tok) || tok |> starts_with(TUNE_VERSION_TOKEN)) @@ -2209,7 +2200,7 @@ def private profile_try_adopt(defaultsDir, scopeName, manifestPath, fnames, gate continue if (race |> find_index(fam) >= 0) for (expr in split(slice(row, eq + 1), "+")) { continue if (empty(expr)) - if (tune_requires_ok(expr) && !expr_in_features(expr, pfeat)) { + if (tune_requires_ok(expr) && !requires_ok_in_fingerprint(expr, pfeat)) { race |> push(fam) break } @@ -2226,15 +2217,15 @@ def private profile_try_adopt(defaultsDir, scopeName, manifestPath, fnames, gate delete kernels delete race delete_json(doc) - return "" + return false } let tail = empty(race) ? "" : " - racing {length(race)} unlocked/new families" print("llvm_tune: scope '{scopeName}' adopted the '{klass}' defaults profile ({length(kernels)} kernel entries){tail}\n") - let res = empty(race) ? "*" : join(race, ",") + raceCsv = join(race, ",") delete kernels delete race delete_json(doc) - return res + return true } //! Runtime half of `[tune_policy(missing = "auto")]` / `--tune`, injected at the top of `main` @@ -2247,15 +2238,9 @@ def tune_auto_prepare(scopeName : string; manifestPath : string; tunerPath : str return } var onlyFilter = "" - if (!force) { - let adopt = profile_try_adopt(defaultsDir, scopeName, manifestPath, fnames, gated) - if (adopt == "*") { - g_tuned_scopes++ // fully covered by the shipped profile; the re-exec stamps it - return - } - if (!empty(adopt)) { - onlyFilter = adopt // adopted a base; the tuner below races only what it lacked - } + if (!force && profile_try_adopt(defaultsDir, scopeName, manifestPath, fnames, gated, onlyFilter) && empty(onlyFilter)) { + g_tuned_scopes++ //! nothing raced, yet counted: the re-exec is what stamps the adopted winners + return } if (!force && empty(onlyFilter) && scope_resolved_externally(scopeName, manifestPath, fnames)) { g_tuned_scopes++ // the re-exec picks the resolved sidecar up like a fresh tune @@ -2268,12 +2253,12 @@ def tune_auto_prepare(scopeName : string; manifestPath : string; tunerPath : str return } if (!empty(onlyFilter)) { - set_env_variable("DAS_TUNE_ONLY", onlyFilter) // the spawned tuner inherits the filter + set_env_variable("DAS_TUNE_ONLY", onlyFilter) } announce_tuning(scopeName, force, true, v.detail) run_scope_tuner(TuneScope(scopeName = scopeName, manifestPath = manifestPath, tunerPath = tunerPath)) if (!empty(onlyFilter)) { - set_env_variable("DAS_TUNE_ONLY", "") // the re-exec child must race unfiltered if it ever tunes + set_env_variable("DAS_TUNE_ONLY", "") } let after = tune_sidecar_verdict(manifestPath, fnames) if (after.complete) { @@ -2284,7 +2269,7 @@ def tune_auto_prepare(scopeName : string; manifestPath : string; tunerPath : str } //! Relaunch the process so a fresh compile stamps the just-written manifests — they cannot be - //! adopted mid-run. `--tune` is stripped from the relaunch so the child converges. True means + //! adopted mid-run. `--tune` / `--tune-only` are stripped from the relaunch so the child converges. True means //! the child ran and main must return `tune_reexec_code()` immediately. def tune_auto_reexec() : bool { if (g_tuned_scopes == 0) { @@ -2294,15 +2279,15 @@ def tune_auto_reexec() : bool { var argv <- get_command_line_arguments() var args : array args |> reserve(length(argv)) - var skipValue = false + var skipTuneOnlyValue = false for (a in argv) { - if (skipValue) { // the value token of a `--tune-only ` pair - skipValue = false + if (skipTuneOnlyValue) { // the value token of a `--tune-only ` pair + skipTuneOnlyValue = false continue } continue if (a == "--tune" || starts_with(a, "--tune-only=")) if (a == "--tune-only") { - skipValue = true + skipTuneOnlyValue = true continue } args |> push("{a}") @@ -2393,7 +2378,7 @@ def tune_family_selected(name : string) : bool { } //! Runtime half of `[tune_policy(missing = "restart")]`, injected as a guard at the top of - //! the app's `main`: consults the scope resolver first, else runs the tuner; true = main + //! the app's `main`: adopts a declared defaults profile, then consults the scope resolver, else runs the tuner; true = main //! returns immediately (winners are compile-time stamps), false = scope complete, carry on. def tune_restart_needed(scopeName : string; manifestPath : string; tunerPath : string; fnames : string; defaultsDir : string = ""; gated : string = "") : bool { @@ -2402,15 +2387,9 @@ def tune_restart_needed(scopeName : string; manifestPath : string; tunerPath : s return false } var onlyFilter = "" - { - let adopt = profile_try_adopt(defaultsDir, scopeName, manifestPath, fnames, gated) - if (adopt == "*") { - print("llvm_tune: scope '{scopeName}' adopted -> {manifestPath}\nllvm_tune: restart to apply the winners\n") - return true - } - if (!empty(adopt)) { - onlyFilter = adopt - } + if (profile_try_adopt(defaultsDir, scopeName, manifestPath, fnames, gated, onlyFilter) && empty(onlyFilter)) { + print("llvm_tune: scope '{scopeName}' adopted -> {manifestPath}\nllvm_tune: restart to apply the winners\n") + return true } if (empty(onlyFilter) && scope_resolved_externally(scopeName, manifestPath, fnames)) { return true // resolved without a tune; the resolver printed its own restart marker @@ -2429,7 +2408,7 @@ def tune_restart_needed(scopeName : string; manifestPath : string; tunerPath : s } announce_tuning(scopeName, false, false, v.detail) if (!empty(onlyFilter)) { - set_env_variable("DAS_TUNE_ONLY", onlyFilter) // the spawned tuner inherits the filter + set_env_variable("DAS_TUNE_ONLY", onlyFilter) } run_scope_tuner(TuneScope(scopeName = scopeName, manifestPath = manifestPath, tunerPath = tunerPath)) if (!empty(onlyFilter)) { diff --git a/modules/dasLLVM/tests/README.md b/modules/dasLLVM/tests/README.md index f811939c0f..a84240b9eb 100644 --- a/modules/dasLLVM/tests/README.md +++ b/modules/dasLLVM/tests/README.md @@ -11,5 +11,5 @@ Run the suite explicitly from the repository root, on an LLVM-enabled build bin/Release/daslang.exe dastest/dastest.das -jit -- --timing-outliers 10 --test modules/dasLLVM/tests ``` -Files ending in `_client`, `_client_lib`, or `_tuner` are child-process fixtures rather than +Files ending in `_client`, `_client_lib`, `_lib`, or `_tuner` are child-process fixtures rather than standalone test cases. diff --git a/modules/dasLLVM/tests/llvm_tune_profiles.das b/modules/dasLLVM/tests/llvm_tune_profiles.das index 7f46857eee..215190b0cb 100644 --- a/modules/dasLLVM/tests/llvm_tune_profiles.das +++ b/modules/dasLLVM/tests/llvm_tune_profiles.das @@ -57,11 +57,11 @@ def private write_profile(dir, klass, winner, features : string) { [test] def test_llvm_tune_profiles(t : T?) { if (!jit_enabled()) { - // the stamped-tier fingerprints need the generator tier; children run -jit + //! no generator tier without the jit, so no stamped-tier fingerprint to read t |> success(true) return } - let args <- get_command_line_arguments() // locked view — do not delete + let args <- get_command_line_arguments() //! locked process-global view - never delete it let bin = args[0] let client = "{get_this_module_dir()}/llvm_tune_profiles_client.das" let sidecar = "{get_this_module_dir()}/llvm_tune_profiles_client.tune.json" @@ -69,7 +69,7 @@ def test_llvm_tune_profiles(t : T?) { let klass = tune_cpu_class() let profile = path_join(defdir, "{klass}.tune-defaults.json") - // 1. no profile file: the plain auto flow — full race (empty filter), the tuner's k2 runs + //! 1. no profile file: the plain auto flow - full race (empty filter), the tuner's k2 runs remove(sidecar) remove(profile) var l1 : array @@ -80,7 +80,7 @@ def test_llvm_tune_profiles(t : T?) { t |> success(l1 |> has("RESULT 7")) delete l1 - // 2. full-cover profile (this box's own fingerprint): adopted, no tuner, its winner stamps + //! 2. full-cover profile (this box's own fingerprint): adopted, no tuner, its winner stamps remove(sidecar) write_profile(defdir, klass, "k0", tune_feature_fingerprint()) var l2 : array @@ -93,9 +93,7 @@ def test_llvm_tune_profiles(t : T?) { t |> success(find(side2, "\"adopted_profile\"") >= 0) delete l2 - // 3. a profile minted on a less-capable box (empty features): the k3 requires= row is an - // unlocked seat here, so the guard adopts AND races just this family — the tuner sees - // the filter and its k2 outranks the adopted k0 + //! 3. a profile minted with no features leaves the k3 requires= seat unlocked here: adopt AND race just that family - the tuner's k2 outranks the adopted k0 remove(sidecar) write_profile(defdir, klass, "k0", "") var l3 : array @@ -106,7 +104,7 @@ def test_llvm_tune_profiles(t : T?) { t |> success(l3 |> has("STATUS prof_add|llvm_tune_profiles_probe|k2|manifest")) delete l3 - // 4. --tune outranks the profile: full re-race, no adoption + //! 4. --tune outranks the profile: full re-race, no adoption var l4 : array t |> equal(spawn_child("{tune_env()}\"{bin}\" -jit \"{client}\" -- --tune", l4), 0) t |> success(!(l4 |> has("defaults profile"))) diff --git a/skills/LAWS.md b/skills/LAWS.md index eabca6ccb9..2d5713a478 100644 --- a/skills/LAWS.md +++ b/skills/LAWS.md @@ -13,3 +13,13 @@ compacted, or cited as rules. - 2026-08-31 - `review_md.md`: review-round flashlight item 4. Boris ruled ("yes") to bless a charter-carrying `README.md` as an architecture doc rather than splitting one out - `site/README.md` is the precedent; both playground checklists keep their pointers. + +## 2026-08-31 - skills/tune.md (shipped defaults profiles, DAS_TUNE_POLICY=reference) + +Boris, on the tune mint being the workflow blocker (20+ minutes per app per box): "we prune the +families. we save M1/zen2 defaults. and if winners are available (vnni), some magic M5 stuff ... +then there is a race. if nothing available - no race." Profiles are keyed by CPU feature class +("profile classes key by the feature set that gates seats" - he asked to elaborate and accepted +it), and separate CPU from GPU: "do u want to separate CPU and GPU tunes?" - resolved as +kernels-only profiles, runtime knobs never travel. He will remove the dasllama.io tune service +if the new-silicon race drops to a couple of minutes. diff --git a/utils/mcp/setup.das b/utils/mcp/setup.das index 123c55c984..151638a641 100644 --- a/utils/mcp/setup.das +++ b/utils/mcp/setup.das @@ -182,7 +182,6 @@ def ensure_sgconfig(root : string) { // Copy them from the source tree of the daslang running this setup (get_das_root()), which is a // JIT-capable build. When setup itself runs under the target worktree's binary, use DASLANG as a // fallback SDK binary: a worktree-local root cannot supply a backend file that it is missing. -// Windows-only for now (the codegen dll name / linker differ on posix — staging there is TBD). def root_from_daslang_binary(binary : string) : string { let executable_dir = dir_name(get_full_file_name(binary)) let leaf = base_name(executable_dir) @@ -205,8 +204,7 @@ def stage_jit_backend(root : string) { fallback_source = "DASLANG does not name an existing binary" } } - // the codegen backend is LLVM.dll on every platform (+ its .version stamp, which CMake's - // re-download check reads); only Windows also needs the lld-link.exe linker beside the binary + //! lib/LLVM.dll is the codegen backend's name on every platform; its .version stamp stages with it or a later cmake refetches LLVM. var jit_files <- ["lib/LLVM.dll", "lib/LLVM.dll.version"] if (get_platform_name() == "windows") { jit_files |> push("bin/lld-link.exe") From c0f8f3fbcc465785db8db36012f69a01f440e12d Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Mon, 31 Aug 2026 20:38:55 -0700 Subject: [PATCH 57/61] PR-1 audit round: the Metal-blob split-plane reads, the f32 fallback, the row demotion, and the checklists Three real defects the external round surfaced, each fixed red-first: - embed_row and the PLE gather read a Metal-blob model's SPLIT scale plane with the CPU 20B row stride for every split format but k6. One dispatcher now owns the plane-form question - dequant_kq_plane_sb(fmt, kq, ks, sb, metal_blob, ...) in dasllama_convert.das, kq_planes_of(t, fmt) in common - and test_kquant proves the split form reads to the row form's values for the six 20B-row formats (random f16 scale bits can be NaN on both sides, so the compares are NaN-aware). - gguf_read_tensor_f32 panicked on every new superblock disk type, so the non-native transcode fallback could not load them: it now transcodes one superblock into scratch planes and runs the plane dequant - the native load's own two steps - and test_gguf_quant pins reader == native for all nine types. - kq_fmt_row_ok demoted q40 and k5 on non-256 rows but not iq4nl, a 32-block disk format riding the same lattice; test_kqformat pins the demotion. The audits' own landings: profiles carry no engine_sha (export strips it, performance/REVIEW.das sweeps defaults/); LLVM_JIT_CODEGEN_VERSION 0x59 and aarch64 i8mm folded into jit_env_salt (the +i8mm append is an emission input); DAS_TUNE_POLICY=reference announces on every compile and is rejected as a declared value; run_scope_tuner owns the DAS_TUNE_ONLY arm/clear; a --tune-only filter matching nothing exits 3 per half and fails the mint only when both halves say so; CENSUS_NEVER_DISPATCHED names the 45 new Metal classes; the eleven cm2 cells get poison controls and the family guards skip instead of vanishing; test_kquant joins the model-free suite; the msl const-table rename arm gets its census kind, fixture and a tests/metal oracle; the hoist asymmetry is ledgered in dasMetal's ARCHITECTURE.md; the profiles test covers --tune-only convergence, the class chain and the reference policy. dasLLAMA/REVIEW.md splits its placement rules into REVIEW_PLACEMENT.md (LINT027); the dragon round's wording repairs land across eleven checklists and the two tune skills; followup_metal.md carries the per-format vehicle table (which GGUF, where on zen2 and the M1). Gates: lint 0 over the changed set; Markdown ASCII gate clean; REVIEW.das x4 green; test_kquant 265, test_gguf_quant 9, test_kqformat 20, test_model_image 50, Vulkan 86/86, msl 101/101, coverage, probe TEST, dasLLVM suite incl. the seven-case profiles test - all green under -jit. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01MyYCDutugdc9xL5pd2qcvT --- modules/dasLLAMA/ARCHITECTURE_ENGINE.md | 5 +- modules/dasLLAMA/ARCHITECTURE_GPU_VULKAN.md | 2 +- modules/dasLLAMA/ARCHITECTURE_MEASUREMENT.md | 9 ++ modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md | 4 +- modules/dasLLAMA/README.md | 7 +- modules/dasLLAMA/REVIEW.md | 127 +++++----------- modules/dasLLAMA/REVIEW_MEASUREMENT.md | 12 +- modules/dasLLAMA/REVIEW_PLACEMENT.md | 75 ++++++++++ modules/dasLLAMA/benchmarks/REVIEW.md | 8 + modules/dasLLAMA/dasllama/dasllama_common.das | 92 +++++++----- .../dasLLAMA/dasllama/dasllama_convert.das | 64 ++++++++ .../dasLLAMA/dasllama/dasllama_gemm_gen.das | 8 +- .../dasllama/dasllama_gemm_schema.das | 8 +- modules/dasLLAMA/dasllama/dasllama_gguf.das | 95 ++++++++++-- modules/dasLLAMA/dasllama/dasllama_image.das | 2 +- .../dasLLAMA/dasllama/dasllama_kqformat.das | 24 +-- modules/dasLLAMA/dasllama/dasllama_layout.das | 4 +- .../dasLLAMA/dasllama/dasllama_math_gen.das | 2 +- .../dasllama/dasllama_metal_common.das | 20 +-- modules/dasLLAMA/dasllama/dasllama_ple.das | 40 +---- modules/dasLLAMA/dasllama/dasllama_tune.das | 4 +- .../dasllama/dasllama_vulkan_classes.das | 4 +- .../dasllama/dasllama_vulkan_common.das | 26 ++-- modules/dasLLAMA/followup_general.md | 2 +- modules/dasLLAMA/followup_metal.md | 29 +++- modules/dasLLAMA/harness/dasllama_tuner.das | 12 ++ modules/dasLLAMA/harness/gen_tune_probe.das | 8 +- modules/dasLLAMA/harness/tune_kernels.das | 20 ++- modules/dasLLAMA/performance/REVIEW.das | 42 ++++++ modules/dasLLAMA/performance/REVIEW.md | 36 +++-- .../defaults/arm-neon.tune-defaults.json | 137 +++++++++--------- .../defaults/x86-avx2.tune-defaults.json | 137 +++++++++--------- .../defaults/x86-vnni512.tune-defaults.json | 137 +++++++++--------- modules/dasLLAMA/tests/REVIEW.md | 36 ++--- modules/dasLLAMA/tests/run.das | 1 + modules/dasLLAMA/tests/test_gguf_quant.das | 61 ++++++++ .../dasLLAMA/tests/test_kernel_coverage.das | 16 +- modules/dasLLAMA/tests/test_kqformat.das | 19 ++- modules/dasLLAMA/tests/test_kquant.das | 89 ++++++++++-- modules/dasLLAMA/tests/test_model_image.das | 13 ++ .../dasLLAMA/tests/test_vulkan_kernels.das | 52 ++++++- modules/dasLLVM/ARCHITECTURE.md | 7 +- modules/dasLLVM/ENVIRONMENT.md | 2 +- modules/dasLLVM/REVIEW.md | 19 +-- modules/dasLLVM/daslib/llvm_env.das | 2 +- modules/dasLLVM/daslib/llvm_jit_common.das | 11 +- modules/dasLLVM/daslib/llvm_jit_run.das | 6 +- modules/dasLLVM/daslib/llvm_tune.das | 97 ++++++------- modules/dasLLVM/tests/llvm_tune_profiles.das | 130 ++++++++++++----- .../tests/llvm_tune_profiles_tuner.das | 3 +- modules/dasLLVM/tests/test_grid.das | 4 +- modules/dasLLVM/tests/test_tuned.das | 6 +- modules/dasMetal/ARCHITECTURE.md | 7 + modules/dasMetal/REVIEW.md | 57 ++++---- modules/dasMetal/metal/msl_emit.das | 1 + plans/unquirk_pass.md | 72 ++++----- skills/internal/llvm_tune_internals.md | 29 +++- skills/tune.md | 73 ++++++---- tests/metal/test_metal_consttab.das | 107 ++++++++++++++ tests/msl/REVIEW.md | 7 +- tests/msl/_msl_common.das | 9 +- tests/msl/test_msl_functions.das | 2 + utils/REVIEW.md | 25 +++- utils/mcp/REVIEW.md | 10 +- 64 files changed, 1443 insertions(+), 732 deletions(-) create mode 100644 modules/dasLLAMA/REVIEW_PLACEMENT.md create mode 100644 tests/metal/test_metal_consttab.das diff --git a/modules/dasLLAMA/ARCHITECTURE_ENGINE.md b/modules/dasLLAMA/ARCHITECTURE_ENGINE.md index a4f1a67e05..8e8eb5a060 100644 --- a/modules/dasLLAMA/ARCHITECTURE_ENGINE.md +++ b/modules/dasLLAMA/ARCHITECTURE_ENGINE.md @@ -78,7 +78,10 @@ stay the reviewer's. A mis-numbered arm dispatches, reads the wrong buffer, and image plus its element count; the image owns the bytes, a carrier owns nothing but its backing. Requires nothing in dasllama - the image rail binds planes, every carrier holds them. - **`dasllama_kqformat.das`** - format IDENTITY: the `KqFmt` enum, the per-format descriptor table - (plane strides, block geometry, stream codes), format predicates. It requires nothing else in + (plane strides, block geometry, stream codes), format predicates, and the shared decode + tables the grid and codebook formats key off - each as a builder function (`iq3s_grid()`, + `iq4nl_lut()`) for kernels that may run on a team lane, plus a global twin for tests, + oracles and the emitter's constant bake. It requires nothing else in dasllama, because it is the taxonomy everything keys off. ONE id space - the enum; integer ids exist only at the IR/kernel-param boundary. `kq_sb` is the superblock-lattice predicate: a `fmt != q8` test does not imply the lattice, so branch on the predicate. diff --git a/modules/dasLLAMA/ARCHITECTURE_GPU_VULKAN.md b/modules/dasLLAMA/ARCHITECTURE_GPU_VULKAN.md index 264a1d2544..d45152be9b 100644 --- a/modules/dasLLAMA/ARCHITECTURE_GPU_VULKAN.md +++ b/modules/dasLLAMA/ARCHITECTURE_GPU_VULKAN.md @@ -79,7 +79,7 @@ indexed by a runtime value (a byte4 dynamic select) drops the whole kernel off t path, to about a third of the rate. Every cm2 decode - q8 and the six kq superblock formats - is spelled the 16-bit way, which is why the block structs are `int16` arrays over the same bytes. The IQ4_XS codebook is the one runtime-indexed read a decode makes: it is staged into a -16-entry `@workgroup` f16 table ahead of the tile loop (llama.cpp's `init_iq_shmem` form), +16-entry `@workgroup` f16 table ahead of the tile loop (the reference exe's shared-memory table-staging form), never selected out of a register vector per element. ### 2.2l The cm2 tile pick and the coopmat default ladder {#cm2-tile-pick-and-default} diff --git a/modules/dasLLAMA/ARCHITECTURE_MEASUREMENT.md b/modules/dasLLAMA/ARCHITECTURE_MEASUREMENT.md index 9b6ec93045..c5a2d7897b 100644 --- a/modules/dasLLAMA/ARCHITECTURE_MEASUREMENT.md +++ b/modules/dasLLAMA/ARCHITECTURE_MEASUREMENT.md @@ -108,3 +108,12 @@ same math differ in float terms - while one that changes only WHEN work happens a CLI flag is never an override (it is the run's own command line, visible where the run is launched). + +### Re-stamping inside the content-addressed archive + +A sidecar archived as `records/.tune..json` is content-addressed: its filename +carries the hash of its bytes. Re-stamping such a file's `provenance.engine_sha` to a reachable +commit (the remedy `performance/REVIEW.md` allows when the measured `modules/dasLLAMA/` tree is +byte-identical) therefore re-hashes and renames the file, and every `records/.json` row +whose `tune_sha` named the old file is repointed in the same change - a row left on the old +name points at a file that no longer exists. diff --git a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md index f67f33c8a2..5e2703b677 100644 --- a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md +++ b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md @@ -128,7 +128,7 @@ add the arm next to `q40`'s. The compiler does not find these - a missing arm fa repack at mr 4/8/16 (dots and row dequants bit-exact), 4-token tile vs per-token GEMVs, groupn (disk + grp slices), batch groupn. -Gate: `test_kqformat` + `test_kquant` on the interpreter binary (the stubs' reference bodies). +Gate: `test_kqformat` + `test_kquant` under `-jit` (the stubs decline, so their reference bodies run either way; every dasLLAMA test run is a `-jit` run). ## 5. The JIT emitter - `dasllama_gemm_gen.das` @@ -339,7 +339,7 @@ where, why it is so today, what unquirked looks like. An empty ledger is a legit 10. **The worktree session sees a stale binder.** LSP/MCP diagnostics in a fresh worktree come from the main tree's binary and report the new enum member as "not inferred" and every dasllama file as broken (`get_total_perf_cores` missing). Trust only the worktree binary: - `bin/Release/daslang.exe dastest/dastest.das -- --test `. Run the session inside the + `bin/Release/daslang.exe -jit dastest/dastest.das -- --test `. Run the session inside the worktree once it is bootstrapped. 11. **A sidecar minted while the stubs declined pins the family to `"reference"`.** The app's auto-policy tune ran during the first end-to-end (QUIRK 4's stubs in place), every perm of diff --git a/modules/dasLLAMA/README.md b/modules/dasLLAMA/README.md index ec4076d652..3244ddea86 100644 --- a/modules/dasLLAMA/README.md +++ b/modules/dasLLAMA/README.md @@ -6,7 +6,8 @@ per-box auto-tuner picks the kernel forms for *your* machine, so the same source code on an M1 and a Threadripper alike. Backends stack behind one registry: the portable / NEON / AVX CPU tiers, an Accelerate "+AMX" float tier on Apple silicon, and **Metal** or **Vulkan** compute for GPU-resident serving. It loads stock GGUF files (single-file or -multi-shard splits, F32/F16/Q8_0/Q4_0/Q5_0/MXFP4 and native Q4_K/Q5_K/Q6_K +multi-shard splits, F32/F16/Q8_0/Q4_0/Q5_0/MXFP4 and native Q2_K/Q3_K/Q4_K/Q5_K/Q6_K plus the +i-quants IQ4_XS/IQ4_NL/IQ3_S/IQ3_XXS/IQ2_S/IQ2_XS/IQ2_XXS planes), runs text LLMs, audio-input "omni" chat models, speech-to-text, and voice-activity detection, and it is fast: consistently ahead of each model's reference C++ engine on prompt processing (up to ~1.7×), trading blows on generation, and up to ~6× ahead on audio/omni workloads — live @@ -322,7 +323,7 @@ What a model needs to "just work" today: | Feature | Supported | |---|---| -| GGUF weight types (read directly) | **F32, F16, Q8_0, Q4_0, MXFP4, and the K-quant planes Q4_K / Q5_K / Q6_K** — K-quant tensors keep their native format and run on dedicated kq kernels (a Q4_K_M file mixes formats per tensor); MXFP4 expert stacks stay native too (mx4·q8 kernels); bf16 audio-tower mmprojs read exactly | +| GGUF weight types (read directly) | **F32, F16, Q8_0, Q4_0, MXFP4, the K-quant planes Q2_K / Q3_K / Q4_K / Q5_K / Q6_K, and the i-quants IQ4_XS / IQ4_NL / IQ3_S / IQ3_XXS / IQ2_S / IQ2_XS / IQ2_XXS** — K-quant and i-quant tensors keep their native format and run on dedicated kq kernels (a Q4_K_M file mixes formats per tensor); MXFP4 expert stacks stay native too (mx4·q8 kernels); bf16 audio-tower mmprojs read exactly | | On-the-fly self-quantization | Q8, Q4 (from an F16/F32 model) | | Architecture | `llama`, `mistral3`, `qwen2`, `qwen2vl`, `qwen3`, `qwen3vl`, `phi3`, `gemma2`, `gemma3`, `gemma4`, `qwen2moe`, `qwen3moe`, `qwen3vlmoe`, `qwen35`, `qwen35moe`, `qwen3next`, `glm4moe`, `gpt-oss` — a self-registering arch registry (`dasllama_arch_*.das`, `[init]`); the loader dispatches on GGUF `general.architecture`, splits Phi3's fused attn_qkv / gate_up at load, and panics with the registered list on an unknown arch | | Attention | MHA **and** GQA (grouped-query); sliding-window with a per-layer pattern (Gemma-2 alternating, Gemma-3 5 local : 1 global, Gemma-4 explicit per-layer bool array); heterogeneous per-layer geometry (Gemma-4: sliding vs global layers differ in head size AND kv-head count, incl. V-from-K layers with no attn_v tensor); **cross-layer KV sharing** (Gemma-4 E-series: later layers carry no K/V of their own — Q only, attending against an earlier layer's cached K/V); configurable attention-score scale (Gemma-4: 1.0); per-head attention-sink logits in the softmax denominator (gpt-oss, all three prefill cores + decode); attention + final-logit soft-capping; suppressed-token logit bias | @@ -389,7 +390,7 @@ So there's no ambiguity about what will fail: `gemma4v`, gemma-3 `gemma3v`, the whole qwen family — Qwen3-Omni / dense Qwen3-VL `qwen3vl_merger` incl. deepstack, Qwen2.5-Omni/VL `qwen2.5o`) — vision on the remaining multimodal families (pixtral, minicpm-v, …) is what stays out of scope. -- **GGUF weight types beyond F32 / F16 / Q8_0 / Q4_0 / MXFP4 / Q4_K / Q5_K / Q6_K** — no IQ +- **GGUF weight types beyond F32 / F16 / Q8_0 / Q4_0 / MXFP4 / Q2_K-Q6_K / IQ4_XS / IQ4_NL / IQ3_S / IQ3_XXS / IQ2_S / IQ2_XS / IQ2_XXS** — no IQ1 or quants; BF16 is read only for the audio-tower mmprojs, not as an LLM weight format. - `encode(..., parse_special)` is reserved and currently a no-op — the chat renderer injects special-token *ids* directly (so templates work); parsing special tokens out of free text diff --git a/modules/dasLLAMA/REVIEW.md b/modules/dasLLAMA/REVIEW.md index 293be6a38c..bb187c85c0 100644 --- a/modules/dasLLAMA/REVIEW.md +++ b/modules/dasLLAMA/REVIEW.md @@ -1,11 +1,10 @@ # dasLLAMA Code Review Checklist **Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist.** Architecture -docs: `ARCHITECTURE.md` and the `ARCHITECTURE_*.md` companions it indexes - the rules below -cite `ARCHITECTURE_ENGINE.md`, `ARCHITECTURE_GPU.md`, `ARCHITECTURE_MEDIA.md`, and -`ARCHITECTURE_MEASUREMENT.md`. Planned work: -`followup_general.md`, `followup_vulkan.md`, `PERF_LEDGER.md` (performance goes to the perf -ledger, everything else to the followup ledgers). +docs: `ARCHITECTURE.md` and the `ARCHITECTURE_*.md` companions it indexes. Planned work: +`followup_general.md`, `followup_vulkan.md`, `followup_metal.md` (the Metal tier, and CPU work +measured on macOS), `PERF_LEDGER.md` (performance goes to the perf ledger, everything else to +the followup ledgers). **A dasLLAMA `[test]` file, wherever the diff puts it, answers to this module's `tests/REVIEW.md`.** @@ -28,7 +27,9 @@ winners back. **A change to the sidecar-exchange client (`dasllama/dasllama_exchange.das`), or to a tune-boot path that reaches it, applies `REVIEW_EXCHANGE.md`.** -**Every `dasllama/` change applies this folder's `tests/REVIEW.md`.** +**Every `dasllama/` change applies this folder's `tests/REVIEW.md`.** The folder walk surfaces +that checklist only for diffs under `tests/`; its census, kargs and new-value duties trigger +on `dasllama/`. **A GPU kernel, driver, dispatch-class, or K/V-mirror change applies `REVIEW_GPU.md`.** @@ -54,6 +55,10 @@ schedules such a stream, applies `REVIEW_VISION.md`.** `REVIEW_AUDIO.md` and `REVIEW_VISION.md`;** a family file that only CALLS a shared rail does not thereby pick up the other modality's checklist. +**A diff that adds a file under `dasllama/`, moves code between files, or lands a kernel, +codec, transform, tokenizer, tool-wire, media-IO or registration concern in a new place +applies `REVIEW_PLACEMENT.md`** - the what-lands-where rules. + **A routed file applies BOTH the checklist it routes to and this one; every other file under `modules/dasLLAMA/` applies this one.** @@ -99,12 +104,6 @@ planes) are out of scope; a site that must stay f32 for another reason is ledger file's sec.1 charter line in `ARCHITECTURE_ENGINE.md`, `ARCHITECTURE_GPU.md`, or `ARCHITECTURE_MEDIA.md`, not commented into compliance. -**Platform-specific code in an engine file (`dasllama/`) lands only in that platform's backend -file.** - -**A diff that adds a new engine concern that is not `Model`/`Session`/`Config` state to -`dasllama/dasllama_common.das` is a defect - give the concern its own file.** - **A boot-path prompt (code that runs at startup, before the first request) that reads stdin without first proving both stdin and stdout are terminals is a defect - emit the question as a `@sidecar` event instead.** A supervised or piped boot must never block on input. @@ -190,18 +189,19 @@ the file it checks - in `ARCHITECTURE_ENGINE.md`, `ARCHITECTURE_MEDIA.md`, or `ARCHITECTURE_GPU.md`.** The line names the check and the names it licenses. A licensed name is one that check does not flag. When the check licenses no names, the line says so. -**An upstream mechanism is described in our own terms, not attributed** - no -"lifted/ported verbatim from" and no upstream symbol, header, or constant names in any -checked-in prose this module owns (docs and comments, any language); state what the code -does and why its shape wins. A symbol the file's own code calls or carries as a value may be -named - naming your own callee or data key is not attribution. Provenance is -not attribution: a path naming where checked-in data is regenerated FROM, an env-knob row -in `ENVIRONMENT.md` whose value locates the reference binary, and a command line or flag -list in `METHODOLOGY.md`, `PROFILE.md`, or `BRINGUP.md`, and a follow-up ledger's board row -naming the build it compares against, all name the binary outright; -every other `.md` line and `.das` comment writes "the reference exe" or "upstream". Legal -attribution lives in `THIRD_PARTY_NOTICES.md` and the `LICENSE.*` files, so prose never -carries it. +**Checked-in prose this module owns - docs and comments, any language - describes an upstream +mechanism in our own terms: no "lifted/ported verbatim from", and no upstream symbol, header, +or constant name; write "the reference exe" or "upstream" instead.** A symbol this file's own +code calls or carries as a value is its own name, not attribution. + +**A line whose job is to locate or reproduce work against the reference build names the +binary outright** - a path naming where checked-in data is regenerated FROM, an env-knob row +whose value locates the reference binary, a command line or flag list in a methodology, +profile, bring-up, or how-to document, and a follow-up ledger's board row naming the build it +compares against. + +**Legal attribution never appears in prose - it lives in `THIRD_PARTY_NOTICES.md` and the +`LICENSE.*` files.** **A def of `dasllama/dasllama.das` - and a new OVERLOAD of one - is TAUGHT: demonstrated in runnable code in a `tutorials/dasLLAMA/*.das` source and narrated on a @@ -229,76 +229,19 @@ companions, lands `ARCHITECTURE.md`'s index line and section range and repoints LINT026-gated; the prose ones are not, and a prose citation of a section that left its file sends the reader to nothing. -**A per-file inventory restated in this checklist is a defect of the checklist.** The sec.1 -charters - `ARCHITECTURE_ENGINE.md`, `ARCHITECTURE_GPU.md`, `ARCHITECTURE_MEDIA.md` - own the -per-file list. A rule naming what KIND of code lands in which file is the checklist's own. - -**A tensor format conversion lands in `dasllama/dasllama_convert.das`.** - -**A disk-order -> compute-order transform lands per scope: kernel-layout in -`dasllama/dasllama_repack.das`, load-scope in `dasllama/dasllama_layout.das`.** - -**A CPU KV-cache store, read, score dot, or V-accumulate lands in `dasllama/dasllama_kv_codec.das`, -its format family kept whole.** GPU twins land in their backend kernel file. - -**A pre-tokenizer split lands in `dasllama/dasllama_pretok.das`; a merge algorithm in its backend file -(`dasllama/dasllama_spm.das` / `dasllama/dasllama_bpe.das`).** - -**A kernel body lands in its owner's backend file.** A GPU kernel body lands in the file where -its PSO is compiled and released. A CPU-tier kernel body lands in that tier's -`dasllama/dasllama_math_.das`. A kernel body never lands in -`dasllama/dasllama_math.das` or in a lens/dispatch macro file. - -**A family quirk lands in the family file; a piece two families need moves UP into the -concern's shared file (its own file when none exists)** - never sideways into a sibling. - -**A family gaining an arm for a media kind adds that kind's span markers to that family's chat -template, never to a second renderer.** Span markers are the template text that opens and -closes the media rows. A family whose template or vocab lacks them has no arm for that media -kind - `create_chat_` panics at create, not at render. - -**No signature in `dasllama/dasllama_tower.das` takes a type that -`dasllama/dasllama_audio.das`, `dasllama/dasllama_vision.das`, or a family file declares.** -`dasllama/dasllama_tower.das` is the shared encoder-tower home, and it requires none of those -files. A doc comment naming the family a helper was built for is fine. - -**A `dasllama/dasllama_tower.das` helper with one calling family lands in that family's -file** - a single-caller helper sanctioned as tower-worthy is ledgered on -`ARCHITECTURE_MEDIA.md` sec.1.7's tower charter line, not argued in review. - -**A harness whose run can end with zero result rows exits non-zero when it does - wrong -flags, failed load, a device that declines.** - -**Tool wire text (the text of a model's tool/function call, built or parsed) is produced only -in `dasllama/dasllama_tools.das`.** - -**No engine file (`dasllama/`) other than `dasllama/dasllama_audio_io.das` requires `audio` (the -miniaudio decode module).** - -**No engine file (`dasllama/`) other than `dasllama/dasllama_vision_io.das` requires `stbimage`.** -Benchmarks, harnesses, and tests decode their own fixtures. - -**Engine, HTTP, or writer logic never lands in `dasllama/dasllama_scheduler.das`** - engine logic in -engine files, HTTP in the server, writer logic in the writer's own file. - -**An `[init]`-only side-effect require in an engine file (`dasllama/`) lives in -`dasllama/dasllama_transformer.das`** - arch registrations, GPU tiers, every module requiring -the engine back; it sits in `dasllama/dasllama_common.das` only if engine code needs it. A -program root (test, harness, benchmark, tool) requires the registration module it needs -directly. - -**An architecture file (`dasllama/dasllama_arch_*.das`) that changes a forward loop, or tests a -family name on a shared path, is a defect - it carries declarative registration only.** - **A diff that moves a family encode stage onto a GPU hook leaves the CPU form in place and changes none of its arithmetic - deleting or rewriting the CPU form in the same change is a defect.** The hook returns a decline value (`false`, or `-1` for the chunk hooks), and the CPU form serves every box with no driver. -**A constant that a team-mode job lane reads is declared as a `def` returning the value, never -as a `let` global.** A team lane never runs global initializers, so a `let` reads zero there -while every single-threaded run reads the right value. - -**A diff that adds a row to `harness/tune_kernels.das`'s bench list puts it ahead of -`dot_q8q8_laneq4x4`, which stays last.** That bench pins the repack backend for the rest of the -process, so a row after it races against the pinned backend instead of the one it selects. +**A diff that writes a CPU feature name in a `requires=` argument that `TUNE_KNOWN_FEATURES` +(`modules/dasLLVM/daslib/llvm_tune.das`, repo root) does not list adds it there in the same +change.** The `features` fingerprint saved with every sidecar is this box's pass/fail over +that list, so a name outside it is never recorded and a box adopting a shipped profile re-runs +the tuning the profile was meant to save. + +**A value that a team-lane kernel reads - anything reachable from a `team_parallel_for` / +`team_parallel_for_indexed` / `team_parallel_stages` body (`dasllama/dasllama_par.das`) - is a `def` +returning it, never a module global with a declaration initializer (`let` or `var`).** A team +lane never runs global initializers, so the global reads zero there while every +single-threaded run reads the right value. diff --git a/modules/dasLLAMA/REVIEW_MEASUREMENT.md b/modules/dasLLAMA/REVIEW_MEASUREMENT.md index 7bcf185d71..64d77f451b 100644 --- a/modules/dasLLAMA/REVIEW_MEASUREMENT.md +++ b/modules/dasLLAMA/REVIEW_MEASUREMENT.md @@ -3,10 +3,9 @@ **Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist.** Architecture doc: `ARCHITECTURE_MEASUREMENT.md`. Planned work: `PERF_LEDGER.md`. -The two figure rules below bind the surfaces this module owns - its ledgers, docs, code -comments, and the PR body of a diff under it. A number on a served page answers to the site -checklists (`site/REVIEW.md`, `site-dasllama/REVIEW.md`), which carry their own provenance -duty. +Figure rules here bind the surfaces this module owns - its ledgers, docs, code comments, and +the PR body of a diff under it. A number on a served page answers to `site/REVIEW.md` or +`site-dasllama/REVIEW.md` (repo root) instead. **A self-measured served-turn time entering `PERF_LEDGER.md` comes from the released `lcpp_bench` exe, never from the `-jit` script.** A served-turn time is a tok/s figure or a @@ -43,7 +42,10 @@ whole-turn model or engine comparison - is a defect wherever this module writes no cell behind it: a checked-in doc, a ledger, a code comment, or a PR description.** The cell states its quant mode and stamps box and engine provenance, so a number can never silently describe a format nobody serves or a kernel set nobody ships. A figure labeled as a -prediction is not a reading - the prediction log is mandated and needs no cell. +prediction is not a reading - the prediction log is mandated and needs no cell. A bring-up +log in a how-to or ledger that names harness, flags, box, and the exe or script that ran it, +at section level, is a stage figure, not a served-turn measurement; the cell it owes is its +format's board row. **A figure measuring one engine stage inside a served turn, or any other measured margin - a lab margin, a kernel-form delta, a gate knee - names the harness and flags that diff --git a/modules/dasLLAMA/REVIEW_PLACEMENT.md b/modules/dasLLAMA/REVIEW_PLACEMENT.md new file mode 100644 index 0000000000..6ad853c663 --- /dev/null +++ b/modules/dasLLAMA/REVIEW_PLACEMENT.md @@ -0,0 +1,75 @@ +# dasLLAMA Code Review Checklist - placement + +**Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist.** The per-file +charters are `ARCHITECTURE_ENGINE.md`, `ARCHITECTURE_GPU.md` and `ARCHITECTURE_MEDIA.md` sec.1. + +**Routed from `REVIEW.md`: a diff that adds a file under `dasllama/`, moves code between files, +or lands a kernel, codec, transform, tokenizer, tool-wire, media-IO or registration concern +in a new place applies this list together with `REVIEW.md`.** + +**A per-file inventory restated in this checklist is a defect of the checklist.** The sec.1 +charters - `ARCHITECTURE_ENGINE.md`, `ARCHITECTURE_GPU.md`, `ARCHITECTURE_MEDIA.md` - own the +per-file list. A rule naming what KIND of code lands in which file is the checklist's own. + +**A tensor format conversion lands in `dasllama/dasllama_convert.das`.** + +**A disk-order -> compute-order transform lands per scope: kernel-layout in +`dasllama/dasllama_repack.das`, load-scope in `dasllama/dasllama_layout.das`.** + +**A CPU KV-cache store, read, score dot, or V-accumulate lands in `dasllama/dasllama_kv_codec.das`, +its format family kept whole.** GPU twins land in their backend kernel file. + +**A pre-tokenizer split lands in `dasllama/dasllama_pretok.das`; a merge algorithm in its backend file +(`dasllama/dasllama_spm.das` / `dasllama/dasllama_bpe.das`).** + +**A kernel body lands in its owner's backend file.** A GPU kernel body lands in the file where +its PSO is compiled and released. A CPU-tier kernel body lands in that tier's +`dasllama/dasllama_math_.das`. A kernel body never lands in +`dasllama/dasllama_math.das` or in a lens/dispatch macro file. + +**A family quirk lands in the family file; a piece two families need moves UP into the +concern's shared file (its own file when none exists)** - never sideways into a sibling. + +**A family gaining an arm for a media kind adds that kind's span markers to that family's chat +template, never to a second renderer.** Span markers are the template text that opens and +closes the media rows. A family whose template or vocab lacks them has no arm for that media +kind - `create_chat_` panics at create, not at render. + +**No signature in `dasllama/dasllama_tower.das` takes a type that +`dasllama/dasllama_audio.das`, `dasllama/dasllama_vision.das`, or a family file declares.** +`dasllama/dasllama_tower.das` is the shared encoder-tower home. A doc comment naming the +family a helper was built for is fine. + +**`dasllama/dasllama_tower.das` requires none of `dasllama/dasllama_audio.das`, +`dasllama/dasllama_vision.das`, or a family file - a diff adding such a require is a defect.** + +**A `dasllama/dasllama_tower.das` helper with one calling family lands in that family's +file** - a single-caller helper sanctioned as tower-worthy is ledgered on +`ARCHITECTURE_MEDIA.md` sec.1.7's tower charter line, not argued in review. + +**Tool wire text (the text of a model's tool/function call, built or parsed) is produced only +in `dasllama/dasllama_tools.das`.** + +**No engine file (`dasllama/`) other than `dasllama/dasllama_audio_io.das` requires `audio` (the +miniaudio decode module).** + +**No engine file (`dasllama/`) other than `dasllama/dasllama_vision_io.das` requires `stbimage`.** +Benchmarks, harnesses, and tests decode their own fixtures. + +**Engine, HTTP, or writer logic never lands in `dasllama/dasllama_scheduler.das`** - engine logic in +engine files, HTTP in the server, writer logic in the writer's own file. + +**An `[init]`-only side-effect require in an engine file (`dasllama/`) lives in +`dasllama/dasllama_transformer.das`** - arch registrations, GPU tiers, every module requiring +the engine back; it sits in `dasllama/dasllama_common.das` only if engine code needs it. A +program root (test, harness, benchmark, tool) requires the registration module it needs +directly. + +**An architecture file (`dasllama/dasllama_arch_*.das`) that changes a forward loop, or tests a +family name on a shared path, is a defect - it carries declarative registration only.** + +**Platform-specific code in an engine file (`dasllama/`) lands only in that platform's backend +file.** + +**A diff that adds a new engine concern that is not `Model`/`Session`/`Config` state to +`dasllama/dasllama_common.das` is a defect - give the concern its own file.** diff --git a/modules/dasLLAMA/benchmarks/REVIEW.md b/modules/dasLLAMA/benchmarks/REVIEW.md index af17be89f6..7d26cf1830 100644 --- a/modules/dasLLAMA/benchmarks/REVIEW.md +++ b/modules/dasLLAMA/benchmarks/REVIEW.md @@ -62,3 +62,11 @@ body.** A board cell is a timed cell of the published results board: one `../PROFILE.md` section. What a cell times changes when a change to its code, to its input corpus, or to the pinned reference build - `DEFAULT_REF_SHA` in `setup_lcpp_ref.das`, the targets it builds, or a patch it applies - moves the measured quantity. + +**A diff that adds or moves a row in `../harness/tune_kernels.das`'s bench list keeps +`dot_q8q8_laneq4x4` last.** That bench pins the repack backend for the rest of the process, +so a row after it races against the pinned backend instead of the one it selects. + +**A harness whose run can end with zero result rows exits non-zero when it does - wrong +flags, failed load, a device that declines.** A run that matched nothing and reported success +leaves a sidecar or a record untouched and a caller none the wiser. diff --git a/modules/dasLLAMA/dasllama/dasllama_common.das b/modules/dasLLAMA/dasllama/dasllama_common.das index 4655d97f14..3146f37c0a 100644 --- a/modules/dasLLAMA/dasllama/dasllama_common.das +++ b/modules/dasLLAMA/dasllama/dasllama_common.das @@ -1527,11 +1527,59 @@ def dlim_config_sources_register() { set_dlim_metal_source(@@dlim_metal_source_impl) } -// Dense-position demotion: q40/k5-from-Q5_0 need % 256 rows (superblock walkers); q51 always -// demotes — dense positions have no q51 dispatch (v1 is expert stacks — kq_fmt_expert_ok). +//! The Model's plane pair for a kq format - the dispatcher every plane-form consumer keys on. +def kq_planes_of(t : Model; fmt : KqFmt; var kq : array const? &; var ks : array const? &) { // nolint:STYLE037 — the format ladder + unsafe { + if (fmt == KqFmt.k4) { + kq = addr(t.k4q) + ks = addr(t.k4s) + } elif (fmt == KqFmt.k5) { + kq = addr(t.k5q) + ks = addr(t.k5s) + } elif (fmt == KqFmt.k6) { + kq = addr(t.k6q) + ks = addr(t.k6s) + } elif (fmt == KqFmt.q40) { + kq = addr(t.q40q) + ks = addr(t.q40s) + } elif (fmt == KqFmt.iq4xs) { + kq = addr(t.iq4xsq) + ks = addr(t.iq4xss) + } elif (fmt == KqFmt.k3) { + kq = addr(t.k3q) + ks = addr(t.k3s) + } elif (fmt == KqFmt.iq3s) { + kq = addr(t.iq3sq) + ks = addr(t.iq3ss) + } elif (fmt == KqFmt.iq3xxs) { + kq = addr(t.iq3xxsq) + ks = addr(t.iq3xxss) + } elif (fmt == KqFmt.iq4nl) { + kq = addr(t.iq4nlq) + ks = addr(t.iq4nls) + } elif (fmt == KqFmt.k2) { + kq = addr(t.k2q) + ks = addr(t.k2s) + } elif (fmt == KqFmt.iq2s) { + kq = addr(t.iq2sq) + ks = addr(t.iq2ss) + } elif (fmt == KqFmt.iq2xs) { + kq = addr(t.iq2xsq) + ks = addr(t.iq2xss) + } elif (fmt == KqFmt.iq2xxs) { + kq = addr(t.iq2xxsq) + ks = addr(t.iq2xxss) + } else { + panic("kq_planes_of: '{fmt}' has no kq plane pair") + } + } +} + +// Dense-position demotion: q40/iq4nl/k5-from-Q5_0 need % 256 rows (superblock walkers over +// 32-block disk formats); q51 always demotes — dense positions have no q51 dispatch (v1 is expert stacks — kq_fmt_expert_ok). def kq_fmt_row_ok(f : KqFmt; n : int64) : KqFmt { if (f == KqFmt.q51) return KqFmt.q8 - return (f == KqFmt.q40 || f == KqFmt.k5) && n % 256l != 0l ? KqFmt.q8 : f + return (f == KqFmt.q40 || f == KqFmt.iq4nl || f == KqFmt.k5) && n % 256l != 0l ? KqFmt.q8 : f } // Expert-stack tag rule: q51 serves natively off per-32 planes (its whole reason to exist — the @@ -4043,9 +4091,6 @@ def embed_row(t : Model; token : int64; var dst : float?) { // nolint:STYLE037 var row = temp_array(dst, dim, type) // trimmed: embq/embs hold the region's plane pair verbatim, rebased to 0 let sb0 = t.planes_trimmed ? (token * dim) / 256l : (t.wcls_off + token * dim) / 256l - // metal-blob planes: k4/k5 strips are 16B, k6 is the split GPU form (d in the tail) - let ssb45 = t.metal_blob ? 16l : K4_SSB - let k6d0 = t.metal_blob ? (long_length(t.k6s) / K6_SSB) * 16l : 0l for (s in range64(dim / 256l)) { if (t.planes_trimmed) { if (t.emb_fmt == KqFmt.k4) { @@ -4077,38 +4122,11 @@ def embed_row(t : Model; token : int64; var dst : float?) { // nolint:STYLE037 } else { panic("embed_row: '{t.emb_fmt}' has no kq plane pair") } - } elif (t.emb_fmt == KqFmt.k4) { - dequant_k4_plane_superblock(t.k4q, (sb0 + s) * K4_QSB, t.k4s, (sb0 + s) * ssb45, row, s * 256l) - } elif (t.emb_fmt == KqFmt.k5) { - dequant_k5_plane_superblock(t.k5q, (sb0 + s) * K5_QSB, t.k5s, (sb0 + s) * ssb45, row, s * 256l) - } elif (t.emb_fmt == KqFmt.k6) { - if (t.metal_blob) { - dequant_k6_plane_superblock_at(t.k6q, (sb0 + s) * K6_QSB, t.k6s, (sb0 + s) * 16l, k6d0 + (sb0 + s) * 2l, row, s * 256l) - } else { - dequant_k6_plane_superblock(t.k6q, (sb0 + s) * K6_QSB, t.k6s, (sb0 + s) * K6_SSB, row, s * 256l) - } - } elif (t.emb_fmt == KqFmt.q40) { - dequant_q40_plane_superblock(t.q40q, (sb0 + s) * Q40_QSB, t.q40s, (sb0 + s) * Q40_SSB, row, s * 256l) - } elif (t.emb_fmt == KqFmt.iq4xs) { - dequant_iq4xs_plane_superblock(t.iq4xsq, (sb0 + s) * IQ4XS_QSB, t.iq4xss, (sb0 + s) * IQ4XS_SSB, row, s * 256l) - } elif (t.emb_fmt == KqFmt.k3) { - dequant_k3_plane_superblock(t.k3q, (sb0 + s) * K3_QSB, t.k3s, (sb0 + s) * K3_SSB, row, s * 256l) - } elif (t.emb_fmt == KqFmt.iq3s) { - dequant_iq3s_plane_superblock(t.iq3sq, (sb0 + s) * IQ3S_QSB, t.iq3ss, (sb0 + s) * IQ3S_SSB, row, s * 256l) - } elif (t.emb_fmt == KqFmt.iq3xxs) { - dequant_iq3xxs_plane_superblock(t.iq3xxsq, (sb0 + s) * IQ3XXS_QSB, t.iq3xxss, (sb0 + s) * IQ3XXS_SSB, row, s * 256l) - } elif (t.emb_fmt == KqFmt.iq4nl) { - dequant_iq4nl_plane_superblock(t.iq4nlq, (sb0 + s) * Q40_QSB, t.iq4nls, (sb0 + s) * Q40_SSB, row, s * 256l) - } elif (t.emb_fmt == KqFmt.k2) { - dequant_k2_plane_superblock(t.k2q, (sb0 + s) * K2_QSB, t.k2s, (sb0 + s) * K2_SSB, row, s * 256l) - } elif (t.emb_fmt == KqFmt.iq2s) { - dequant_iq2s_plane_superblock(t.iq2sq, (sb0 + s) * IQ2S_QSB, t.iq2ss, (sb0 + s) * IQ2S_SSB, row, s * 256l) - } elif (t.emb_fmt == KqFmt.iq2xs) { - dequant_iq2xs_plane_superblock(t.iq2xsq, (sb0 + s) * IQ2XS_QSB, t.iq2xss, (sb0 + s) * IQ2XS_SSB, row, s * 256l) - } elif (t.emb_fmt == KqFmt.iq2xxs) { - dequant_iq2xxs_plane_superblock(t.iq2xxsq, (sb0 + s) * IQ2XXS_QSB, t.iq2xxss, (sb0 + s) * IQ2XXS_SSB, row, s * 256l) } else { - panic("embed_row: '{t.emb_fmt}' has no kq plane pair") + var kqp : array const? + var ksp : array const? + kq_planes_of(t, t.emb_fmt, kqp, ksp) + dequant_kq_plane_sb(t.emb_fmt, *kqp, *ksp, sb0 + s, t.metal_blob, row, s * 256l) } } } diff --git a/modules/dasLLAMA/dasllama/dasllama_convert.das b/modules/dasLLAMA/dasllama/dasllama_convert.das index f9ba080f58..36d4b4692d 100644 --- a/modules/dasLLAMA/dasllama/dasllama_convert.das +++ b/modules/dasLLAMA/dasllama/dasllama_convert.das @@ -561,6 +561,70 @@ def dequant_k6_plane_superblock(kq : array | #; kqo : int64; ks : array | #; ks : array | #; sb : int64; metal_blob : bool; var dst : array | #; doff : int64) { // nolint:STYLE037,STYLE038 — one arm per format, two carrier forms + if (fmt == KqFmt.k4) { + dequant_k4_plane_superblock(kq, sb * K4_QSB, ks, sb * (metal_blob ? 16l : K4_SSB), dst, doff) + } elif (fmt == KqFmt.k5) { + dequant_k5_plane_superblock(kq, sb * K5_QSB, ks, sb * (metal_blob ? 16l : K5_SSB), dst, doff) + } elif (fmt == KqFmt.q40) { + dequant_q40_plane_superblock(kq, sb * Q40_QSB, ks, sb * Q40_SSB, dst, doff) + } elif (fmt == KqFmt.iq4nl) { + dequant_iq4nl_plane_superblock(kq, sb * Q40_QSB, ks, sb * Q40_SSB, dst, doff) + } elif (!metal_blob) { + if (fmt == KqFmt.k6) { + dequant_k6_plane_superblock(kq, sb * K6_QSB, ks, sb * K6_SSB, dst, doff) + } elif (fmt == KqFmt.iq4xs) { + dequant_iq4xs_plane_superblock(kq, sb * IQ4XS_QSB, ks, sb * IQ4XS_SSB, dst, doff) + } elif (fmt == KqFmt.k3) { + dequant_k3_plane_superblock(kq, sb * K3_QSB, ks, sb * K3_SSB, dst, doff) + } elif (fmt == KqFmt.iq3s) { + dequant_iq3s_plane_superblock(kq, sb * IQ3S_QSB, ks, sb * IQ3S_SSB, dst, doff) + } elif (fmt == KqFmt.iq3xxs) { + dequant_iq3xxs_plane_superblock(kq, sb * IQ3XXS_QSB, ks, sb * IQ3XXS_SSB, dst, doff) + } elif (fmt == KqFmt.k2) { + dequant_k2_plane_superblock(kq, sb * K2_QSB, ks, sb * K2_SSB, dst, doff) + } elif (fmt == KqFmt.iq2s) { + dequant_iq2s_plane_superblock(kq, sb * IQ2S_QSB, ks, sb * IQ2S_SSB, dst, doff) + } elif (fmt == KqFmt.iq2xs) { + dequant_iq2xs_plane_superblock(kq, sb * IQ2XS_QSB, ks, sb * IQ2XS_SSB, dst, doff) + } elif (fmt == KqFmt.iq2xxs) { + dequant_iq2xxs_plane_superblock(kq, sb * IQ2XXS_QSB, ks, sb * IQ2XXS_SSB, dst, doff) + } else { + panic("dequant_kq_plane_sb: '{fmt}' has no kq plane pair") + } + } else { + // the split GPU form: every strip region is 16B/sb, the tail 2B/sb (k2: 4B - d and dmin) + let tail = fmt == KqFmt.k2 ? 4l : 2l + let d0 = (long_length(ks) / (16l + tail)) * 16l + if (fmt == KqFmt.k6) { + dequant_k6_plane_superblock_at(kq, sb * K6_QSB, ks, sb * 16l, d0 + sb * 2l, dst, doff) + } elif (fmt == KqFmt.iq4xs) { + dequant_iq4xs_plane_superblock_at(kq, sb * IQ4XS_QSB, ks, sb * 16l, d0 + sb * 2l, dst, doff) + } elif (fmt == KqFmt.k3) { + dequant_k3_plane_superblock_at(kq, sb * K3_QSB, ks, sb * 16l, d0 + sb * 2l, dst, doff) + } elif (fmt == KqFmt.iq3s) { + dequant_iq3s_plane_superblock_at(kq, sb * IQ3S_QSB, ks, sb * 16l, d0 + sb * 2l, dst, doff) + } elif (fmt == KqFmt.iq3xxs) { + dequant_iq3xxs_plane_superblock_at(kq, sb * IQ3XXS_QSB, ks, sb * 16l, d0 + sb * 2l, dst, doff) + } elif (fmt == KqFmt.k2) { + dequant_k2_plane_superblock_at(kq, sb * K2_QSB, ks, sb * 16l, d0 + sb * 4l, dst, doff) + } elif (fmt == KqFmt.iq2s) { + dequant_iq2s_plane_superblock_at(kq, sb * IQ2S_QSB, ks, sb * 16l, d0 + sb * 2l, dst, doff) + } elif (fmt == KqFmt.iq2xs) { + dequant_iq2xs_plane_superblock_at(kq, sb * IQ2XS_QSB, ks, sb * 16l, d0 + sb * 2l, dst, doff) + } elif (fmt == KqFmt.iq2xxs) { + dequant_iq2xxs_plane_superblock_at(kq, sb * IQ2XXS_QSB, ks, sb * 16l, d0 + sb * 2l, dst, doff) + } else { + panic("dequant_kq_plane_sb: '{fmt}' has no kq plane pair") + } + } +} + + //! The split-layout twin: `dso` names the f16 d halfword's byte offset within ks independently //! of the 16B sub-scale strip at kso — metal-blob models keep k6s in the GPU split form //! ([nsb x 16B sub-scales][nsb x f16 d]) where d lives in the plane's tail. diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das index 5e5ef3b408..2d13a33379 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das @@ -275,7 +275,7 @@ struct private TileEmit { mx4 : bool // block emitter: mx4 LUT-dequant instead of Q8 loads q51 : bool // block emitter: q51 nibble+qh unpack (32-weight blocks, // per-block f16 d/m scale pairs + xbs min-term) - kq : int // 3/4/5/6/40 = K-quant superblock emitter, 44 = IQ4_XS (0 = not kq): the + kq : int // the gemm_schema kq id (0 = not kq): the // block unit is a 256-weight superblock over the grp // kq planes; dots run unsigned-q (kq_dot_lane) kqBytes : bool // kq weights are a BYTE-EXPANDED panel (the tile's @@ -932,7 +932,7 @@ def private or_bit_x10(te : TileEmit; var w, bytes, maskv : LLVMOpaqueValue?; na return LLVMBuildOr(b, w, sel, name) } -// One 256-weight SUPERBLOCK, K-quant grp form (te.kq = 3/4/5/6/40): weight vectors unpacked +// One 256-weight SUPERBLOCK, K-quant grp form (te.kq set): weight vectors unpacked // ONCE per (sub-block, dword-group) and dotted against every token — weight-stationary, // bit-exact vs per-token GEMVs (tokCount=1). kq v2 folds integer sub-scales/bsums against Q8_K-form activations. def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f : LLVMOpaqueValue? [8]; tokBase, tokCount : int) { // nolint:STYLE037,STYLE038 — one register-coupled superblock loop nest; the k-quant decode arms share live values @@ -1840,7 +1840,7 @@ def private emit_block_iq4xs(var te : TileEmit; var sbi : LLVMOpaqueValue?; var var xb = LLVMBuildMul(b, sbi, te.types->ConstI64(0x100ul), "xb") var vri8 = LLVMVectorType(te.types.t_int8, uint(te.rv)) var dv : LLVMOpaqueValue? [2] - if (!nl) { // 44/33/34: one superblock d per row lane; iq4nl loads its per-block d inside the walk + if (!nl) { // one superblock d per row lane; iq4nl loads its per-block d inside the walk for (qd in range(rq)) { dv[qd] = load_f16_vec_at(te, te.sg, LLVMBuildAdd(b, sb, te.types->ConstI64(uint64(2 * (qd * te.rv))), ""), "d{qd}") } @@ -3203,7 +3203,7 @@ def private kq_tile_gen_impl(var gc : LlvmCodeCtx; fmt : int) : bool { if (perm_declines(gc, p0)) return false let p = companion_perm(p0) - var te = TileEmit(kq = fmt, kqBytes = fmt == 5 || fmt == 6) // k4/q40/iq4xs/k3/iq4nl tiles read the packed planes + var te = TileEmit(kq = fmt, kqBytes = fmt == 5 || fmt == 6) // only the k5/k6 tiles get the byte-expanded panel if (!setup_tile_emit(te, gc, p, fmt == 44 || fmt == 45)) return false let b = gc.jit.builder diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_schema.das b/modules/dasLLAMA/dasllama/dasllama_gemm_schema.das index 4990b5b630..f476357100 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_schema.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_schema.das @@ -58,7 +58,7 @@ let Q51_QPB = 20l let Q51_SPB = 4l //! Quant-plane bytes per 256-weight superblock per row for a kq format id (4/5/6 = Q4_K/Q5_K/ -//! Q6_K, 40 = Q4_0, 44 = IQ4_XS, 3 = Q3_K, 33 = IQ3_S, 34 = IQ3_XXS, 45 = IQ4_NL). The ONE stride source for every fmt-branched kq walker/ +//! Q6_K, 40 = Q4_0, 44 = IQ4_XS, 3 = Q3_K, 33 = IQ3_S, 34 = IQ3_XXS, 45 = IQ4_NL, 2 = Q2_K, 23 = IQ2_S, 24 = IQ2_XS, 25 = IQ2_XXS). The ONE stride source for every fmt-branched kq walker/ //! kernel/repack — an unknown id panics instead of silently walking another format's stride. def kq_qsb(fmt : int) : int64 { if (fmt == 4) return kq_qsb(KqFmt.k4) @@ -98,3 +98,9 @@ def kq_ssb(fmt : int) : int64 { panic("kq_ssb: unknown kq format id {fmt}") return 0l } + +//! True for the kq ids whose tile reads the packed quant plane directly; the others (k5/k6 and +//! the grid formats) read a byte-expanded panel unpacked once per (group, token-block). +def kq_reads_packed_planes(fmt : int64) : bool { + return fmt == 4l || fmt == 40l || fmt == 44l || fmt == 3l || fmt == 45l || fmt == 2l +} diff --git a/modules/dasLLAMA/dasllama/dasllama_gguf.das b/modules/dasLLAMA/dasllama/dasllama_gguf.das index 821bdd6a0b..80cbae4abd 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gguf.das +++ b/modules/dasLLAMA/dasllama/dasllama_gguf.das @@ -40,7 +40,7 @@ let private GT_U64 = 10 let private GT_I64 = 11 let private GT_F64 = 12 -// ggml tensor data types (subset we read) +// upstream tensor data types (subset we read) let GGML_TYPE_F32 = 0 let GGML_TYPE_F16 = 1 let GGML_TYPE_Q4_0 = 2 @@ -403,7 +403,7 @@ def gguf_str(m : GGUFMeta; bytes : array | #; key : string) : string { return rd_str(bytes, o) } -//! Read a string array (e.g. tokenizer.ggml.tokens). +//! Read a string array (e.g. tokenizer.upstream.tokens). def gguf_str_array(m : GGUFMeta; bytes : array | #; key : string) : array { if (!key_exists(m.kv, key)) { panic("gguf: missing key '{key}'") @@ -421,7 +421,7 @@ def gguf_str_array(m : GGUFMeta; bytes : array | #; key : string) : array return <- out } -//! Read a float32 array (e.g. tokenizer.ggml.scores). +//! Read a float32 array (e.g. tokenizer.upstream.scores). def gguf_f32_array(m : GGUFMeta; bytes : array | #; key : string) : array { if (!key_exists(m.kv, key)) { panic("gguf: missing key '{key}'") @@ -440,7 +440,7 @@ def gguf_f32_array(m : GGUFMeta; bytes : array | #; key : string) : array return <- out } -//! Read an int32 array (e.g. tokenizer.ggml.token_type), widened to int64. +//! Read an int32 array (e.g. tokenizer.upstream.token_type), widened to int64. def gguf_i32_array(m : GGUFMeta; bytes : array | #; key : string) : array { if (!key_exists(m.kv, key)) { panic("gguf: missing key '{key}'") @@ -870,7 +870,7 @@ def gguf_transcode_iq3s(m : GGUFMeta; srcbytes : array | #; name : string //! Transcode an IQ3_XXS tensor into the iq3xxs planes (strides 96/20, exact): qs+aux copy //! verbatim off the 98B disk block, the iq3s-shaped scale row carries d HALVED (f16_half_bits) -//! and strips (2*ls + 1) — with the halved grid this absorbs ggml's 0.25 exactly. +//! and strips (2*ls + 1) — with the halved grid this absorbs upstream's 0.25 exactly. def gguf_transcode_iq3xxs(m : GGUFMeta; srcbytes : array | #; name : string; var kq : array; var ks : array; eloff, expect_n : int64; src_off : int64 = 0l) { let ti = kq_transcode_check(m, name, GGML_TYPE_IQ3_XXS, "IQ3_XXS", src_off, expect_n) let nb = expect_n / 256l @@ -1097,10 +1097,63 @@ def gguf_transcode_q51(m : GGUFMeta; srcbytes : array | #; name : string; } } -//! Decode tensor `name` (F32 or F16 on disk) into dst[dst_off ..] as fp32. Panics if the -//! tensor is missing, has a different element count than expected, or uses a type we don't -//! yet read (quantized types are wired in a later step). -def gguf_read_tensor_f32(m : GGUFMeta; srcbytes : array | #; name : string; var dst : array; dst_off, expect_n : int64; src_off : int64 = 0l) { // nolint:STYLE038 — one arm per on-disk ggml type; each arm is a self-contained decode kernel +//! The kq format tag of a superblock i-quant / Q2_K / Q3_K GGML type (q8 = not one of them). +def gguf_kq_fmt_of_type(gtype : int) : KqFmt { + if (gtype == GGML_TYPE_IQ4_XS) return KqFmt.iq4xs + if (gtype == GGML_TYPE_Q3_K) return KqFmt.k3 + if (gtype == GGML_TYPE_IQ3_S) return KqFmt.iq3s + if (gtype == GGML_TYPE_IQ3_XXS) return KqFmt.iq3xxs + if (gtype == GGML_TYPE_IQ4_NL) return KqFmt.iq4nl + if (gtype == GGML_TYPE_Q2_K) return KqFmt.k2 + if (gtype == GGML_TYPE_IQ2_S) return KqFmt.iq2s + if (gtype == GGML_TYPE_IQ2_XS) return KqFmt.iq2xs + if (gtype == GGML_TYPE_IQ2_XXS) return KqFmt.iq2xxs + return KqFmt.q8 +} + +//! Disk bytes per 256-superblock of the i-quant / Q2_K / Q3_K GGML types the native tier reads +//! (0 = not one of them). IQ4_NL is eight 18B blocks of 32. +def gguf_kq_disk_sbb(gtype : int) : int64 { + if (gtype == GGML_TYPE_IQ4_XS) return 136l + if (gtype == GGML_TYPE_Q3_K || gtype == GGML_TYPE_IQ3_S) return 110l + if (gtype == GGML_TYPE_IQ3_XXS) return 98l + if (gtype == GGML_TYPE_IQ4_NL) return 144l + if (gtype == GGML_TYPE_Q2_K) return 84l + if (gtype == GGML_TYPE_IQ2_S) return 82l + if (gtype == GGML_TYPE_IQ2_XS) return 74l + if (gtype == GGML_TYPE_IQ2_XXS) return 66l + return 0l +} + +//! One superblock's disk -> plane transcode, keyed by format - the array form the tests and the +//! f32 fallback drive (the bulk loaders are the per-format `gguf_transcode_*`). +def kq_transcode_superblock(fmt : KqFmt; bytes : array | #; bo : int64; var kq : array; kqo : int64; var ks : array; kso : int64) { // nolint:STYLE037 — the format ladder + if (fmt == KqFmt.iq4xs) { + transcode_iq4xs_superblock(bytes, bo, kq, kqo, ks, kso) + } elif (fmt == KqFmt.k3) { + transcode_q3k_superblock(bytes, bo, kq, kqo, ks, kso) + } elif (fmt == KqFmt.iq3s) { + transcode_iq3s_superblock(bytes, bo, kq, kqo, ks, kso) + } elif (fmt == KqFmt.iq3xxs) { + transcode_iq3xxs_superblock(bytes, bo, kq, kqo, ks, kso) + } elif (fmt == KqFmt.iq4nl) { + transcode_iq4nl_superblock(bytes, bo, kq, kqo, ks, kso) + } elif (fmt == KqFmt.k2) { + transcode_q2k_superblock(bytes, bo, kq, kqo, ks, kso) + } elif (fmt == KqFmt.iq2s) { + transcode_iq2s_superblock(bytes, bo, kq, kqo, ks, kso) + } elif (fmt == KqFmt.iq2xs) { + transcode_iq2xs_superblock(bytes, bo, kq, kqo, ks, kso) + } elif (fmt == KqFmt.iq2xxs) { + transcode_iq2xxs_superblock(bytes, bo, kq, kqo, ks, kso) + } else { + panic("kq_transcode_superblock: '{fmt}' has no superblock transcode") + } +} + +//! Decode tensor `name` into dst[dst_off ..] as fp32 from any on-disk type the native tier reads. +//! Panics if the tensor is missing, its element count differs, or its type has no decoder here. +def gguf_read_tensor_f32(m : GGUFMeta; srcbytes : array | #; name : string; var dst : array; dst_off, expect_n : int64; src_off : int64 = 0l) { // nolint:STYLE038 — one arm per on-disk upstream type; each arm is a self-contained decode kernel let ti = gguf_find_tensor(m, name) if (ti < 0) { panic("gguf: tensor '{name}' not found") @@ -1114,7 +1167,7 @@ def gguf_read_tensor_f32(m : GGUFMeta; srcbytes : array | #; name : strin } let gtype = m.tensors[ti].ggml_type guard_dst(name, "f32 dst", dst_off, expect_n, long_length(dst)) - with_tensor_view(m, srcbytes, ti) $(bytes, base) { // nolint:STYLE037 — the ggml-type ladder; the arms share only the borrowed view + with_tensor_view(m, srcbytes, ti) $(bytes, base) { // nolint:STYLE037 — the upstream-type ladder; the arms share only the borrowed view if (gtype == GGML_TYPE_F32) { unsafe { // raw copy, threaded in u64 words (+ one odd-float tail) let sp = addr(bytes[base + src_off * 4l]) @@ -1253,6 +1306,26 @@ def gguf_read_tensor_f32(m : GGUFMeta; srcbytes : array | #; name : strin } } } + } elif (gguf_kq_disk_sbb(gtype) > 0l) { + // transcode one superblock into scratch planes + its plane dequant: the native load's own two steps, value for value + if (src_off % 256l != 0l || expect_n % 256l != 0l) { + panic("gguf: tensor '{name}' superblock slice [{src_off}, +{expect_n}) is not superblock-aligned") + } + let sbb = gguf_kq_disk_sbb(gtype) + let fmt = gguf_kq_fmt_of_type(gtype) + var kq : array + var ks : array + kq |> resize(int(kq_qsb(fmt))) + ks |> resize(int(kq_ssb(fmt))) + let nb = expect_n / 256l + for (blk in range64(nb)) { + let bo = base + (src_off / 256l ++ blk) * sbb + kq_transcode_superblock(fmt, bytes, bo, kq, 0l, ks, 0l) + dequant_kq_plane_sb(fmt, kq, ks, 0l, false, dst, dst_off + blk * 256l) + } + delete kq + delete ks } elif (gtype == GGML_TYPE_MXFP4) { // block of 32: 1 E8M0 scale byte + 16 nibble bytes (low nibbles = elems 0..15, high = 16..31); // x = e2m1[nibble] * scale — both sides carry the ×2/÷2 on-disk convention (see MXFP4_KVALUES) @@ -1297,7 +1370,7 @@ def gguf_transcode_bf16(m : GGUFMeta; srcbytes : array | #; name : string } } -//! The on-disk ggml type of tensor `name` (GGML_TYPE_*), or -1 if the tensor is absent. Lets a +//! The on-disk upstream type of tensor `name` (GGML_TYPE_*), or -1 if the tensor is absent. Lets a //! caller pick the direct-transcode path when the disk type already matches the target precision. def gguf_tensor_type(m : GGUFMeta; name : string) : int { let ti = gguf_find_tensor(m, name) diff --git a/modules/dasLLAMA/dasllama/dasllama_image.das b/modules/dasLLAMA/dasllama/dasllama_image.das index 61e438c2c5..88d36e48b9 100644 --- a/modules/dasLLAMA/dasllama/dasllama_image.das +++ b/modules/dasLLAMA/dasllama/dasllama_image.das @@ -38,7 +38,7 @@ require dasllama/dasllama_load // WhisperModel.enc) contribute their planes under dotted names ("enc.fblob"); string-array // fields ride the meta blob via serialize_strings — raw string pointers can't be planes. -let IMAGE_VERSION = 26 // 26: the iq2xxs plane pair (IQ2_XXS native tier) +let IMAGE_VERSION = 26 // the iq2xxs plane pair (IQ2_XXS native tier) //! The metal (blob-only) flavor's identity tag: q8 planes ride the 34B block_q8_0 blob and the //! kq scale planes their GPU forms (convert_model_to_metal_blob) — flavors are per-config and diff --git a/modules/dasLLAMA/dasllama/dasllama_kqformat.das b/modules/dasLLAMA/dasllama/dasllama_kqformat.das index 9c69f8d538..8312aed05d 100644 --- a/modules/dasLLAMA/dasllama/dasllama_kqformat.das +++ b/modules/dasLLAMA/dasllama/dasllama_kqformat.das @@ -119,7 +119,7 @@ let IQ2XXS_SSB = 20l // IQ2_XXS scale plane: the iq3xxs row shape - f16 d EIGH // IQ4_NL reuses Q40_QSB/Q40_SSB outright: the disk bytes are q40's (f16 d + 16 nibble bytes // per 32-block, the k/k+16 pairing) — only the nibble SEMANTICS differ (IQ4NL_LUT, no -8). -//! The IQ4_NL / IQ4_XS nibble codebook: weight = scale x IQ4NL_LUT[nibble] (ggml's kvalues_iq4nl). +//! The IQ4_NL / IQ4_XS nibble codebook: weight = scale x IQ4NL_LUT[nibble] (upstream's kvalues_iq4nl). //! Main-context use only (tests, oracles, the emitter's constant bake): a kernel that can run on a //! forked worker reads a module global as zeros - kernels take `iq4nl_lut()` as a local instead. let IQ4NL_LUT = fixed_array(-127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113) @@ -128,7 +128,7 @@ let IQ4NL_LUT = fixed_array(-127, -104, -83, -65, -49, -35, -22, -10, 1, 1 //! worker-run kernel reads. def iq4nl_lut() : int8[16] => fixed_array(-127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113) -//! The IQ3_S grid (ggml's iq3s_grid): entry qs | (qh bit << 8) packs FOUR weight magnitudes as +//! The IQ3_S grid (upstream's iq3s_grid): entry qs | (qh bit << 8) packs FOUR weight magnitudes as //! bytes; the per-element sign rides the block's own sign bytes. Per-call local (2 KB on the //! caller's frame) - the form every worker-run kernel reads; IQ3S_GRID is the main-context copy. def iq3s_grid() : uint[512] { @@ -203,8 +203,8 @@ def iq3s_grid() : uint[512] { //! a module global as zeros; kernels take `iq3s_grid()` instead. let IQ3S_GRID = iq3s_grid() -//! The IQ3_XXS grid, every byte HALVED (ggml levels are even, 4..62 -> 2..31): one qs byte = -//! FOUR magnitudes; the halved grid + halved stored d absorb ggml's 0.25 exactly, so the fold +//! The IQ3_XXS grid, every byte HALVED (upstream levels are even, 4..62 -> 2..31): one qs byte = +//! FOUR magnitudes; the halved grid + halved stored d absorb upstream's 0.25 exactly, so the fold //! is iq3s's d x strip x byte. Per-call local (1 KB) - the worker-safe form kernels read. def iq3xxs_grid() : uint[256] { return fixed_array( @@ -245,7 +245,7 @@ def iq3xxs_grid() : uint[256] { //! Main-context copy of the halved IQ3_XXS grid (tests, oracles, emitter bakes). let IQ3XXS_GRID = iq3xxs_grid() -//! ggml's ksigns_iq2xs: 7-bit sign index -> 8 sign bits (bit 7 = even parity). Shared by the +//! upstream's ksigns_iq2xs: 7-bit sign index -> 8 sign bits (bit 7 = even parity). Shared by the //! IQ3_XXS and IQ2 families. Per-call local - the worker-safe form. def ksigns_iq2xs() : uint8[128] { return fixed_array( @@ -263,7 +263,7 @@ def ksigns_iq2xs() : uint8[128] { let KSIGNS_IQ2XS = ksigns_iq2xs() //! Halve an f16 bit pattern exactly (exponent decrement; subnormal results truncate-halve). -//! The IQ3_XXS transcode folds ggml's 0.25 into the stored d (one half) and the grid (the other). +//! The IQ3_XXS transcode folds upstream's 0.25 into the stored d (one half) and the grid (the other). def f16_half_bits(u : uint) : uint { let e = (u >> 10u) & 0x1Fu if (e >= 2u) { @@ -276,7 +276,7 @@ def f16_half_bits(u : uint) : uint { } -//! The IQ2_S grid (ggml's iq2s_grid, 1024 u64 entries as low/high uint pairs): a 10-bit index +//! The IQ2_S grid (upstream's iq2s_grid, 1024 u64 entries as low/high uint pairs): a 10-bit index //! (qs byte | qh 2 bits) selects EIGHT magnitudes ({8, 25, 43}); signs ride the block's own //! sign bytes. Per-call local (8 KB) - the worker-safe form; IQ2S_GRID is the main-context copy. def iq2s_grid2() : uint[2048] { // nolint:STYLE038 - a 2048-word data table, not splittable @@ -544,7 +544,7 @@ def iq2s_grid2() : uint[2048] { // nolint:STYLE038 - a 2048-word data table, n let IQ2S_GRID = iq2s_grid2() -//! The IQ2_XS grid (ggml's iq2xs_grid, 512 u64 entries as low/high uint pairs): the u16 word's +//! The IQ2_XS grid (upstream's iq2xs_grid, 512 u64 entries as low/high uint pairs): the u16 word's //! low 9 bits select EIGHT magnitudes ({8, 25, 43}), its high 7 bits the KSIGNS_IQ2XS sign //! byte (bit j flips element j). Per-call local (4 KB), worker-safe; IQ2XS_GRID = main-context copy. def iq2xs_grid2() : uint[1024] { // nolint:STYLE038 - a 1024-word data table, not splittable @@ -684,7 +684,7 @@ def iq2xs_grid2() : uint[1024] { // nolint:STYLE038 - a 1024-word data table, let IQ2XS_GRID = iq2xs_grid2() -//! The IQ2_XXS grid (ggml's iq2xxs_grid, 256 u64 entries as low/high uint pairs): a block's +//! The IQ2_XXS grid (upstream's iq2xxs_grid, 256 u64 entries as low/high uint pairs): a block's //! aux32[0] BYTES each select EIGHT magnitudes ({8, 25, 43}), flipped through the shared //! ksigns_iq2xs table. Per-call local (2 KB), worker-safe; IQ2XXS_GRID = main-context copy. def iq2xxs_grid2() : uint[512] { @@ -823,6 +823,10 @@ def kq_ssb(f : KqFmt) : int64 { def kq_elems(f : KqFmt) : int64 => kq_sb(f) ? 256l : 32l //! The loader's stream/repack region tag for a format (stream_repack_one / push_repack codes). +//! k2's stream-region code: kernel id 2 IS q51's stream tag, so k2 streams under 20 and the +//! repack dispatchers translate it back to kernel id 2. +let K2_STREAM_TAG = 20 + def kq_stream_code(f : KqFmt) : int { if (f == KqFmt.q8) { return 0 @@ -831,7 +835,7 @@ def kq_stream_code(f : KqFmt) : int { return 2 } if (f == KqFmt.k2) { - return 20 // kernel id 2 IS q51's stream tag - k2 streams under 20, translated back at the repack dispatchers + return K2_STREAM_TAG } return kq_schema_id(f) } diff --git a/modules/dasLLAMA/dasllama/dasllama_layout.das b/modules/dasLLAMA/dasllama/dasllama_layout.das index f29cc20d07..920696275b 100644 --- a/modules/dasLLAMA/dasllama/dasllama_layout.das +++ b/modules/dasLLAMA/dasllama/dasllama_layout.das @@ -509,7 +509,7 @@ def private repack_regions(var t : Model; regs : array) { invoke(rq51, q51qp + (rp[i].off / 32l) * Q51_QPB, q51sp + (rp[i].off / 32l) * Q51_SPB, rp[i].n, rp[i].d) } else { let sb = rp[i].off / 256l - let fk = f == 20 ? 2 : f + let fk = f == K2_STREAM_TAG ? 2 : f let qsb = kq_qsb(fk) let ssb = kq_ssb(fk) var kqp = f == 4 ? k4qp : (f == 5 ? k5qp : (f == 6 ? k6qp : (f == 40 ? q40qp : (f == 44 ? iq4xsqp : (f == 3 ? k3qp : (f == 33 ? iq3sqp : (f == 34 ? iq3xxsqp : (f == 45 ? iq4nlqp : (fk == 2 ? k2qp : (f == 23 ? iq2sqp : (f == 24 ? iq2xsqp : iq2xxsqp))))))))))) @@ -792,7 +792,7 @@ def private stream_repack_one(fmt : int; qp, sp : void?; n, d : int64) { invoke(active_repack_q8q8(), reinterpret(qp), reinterpret(sp), n, d) } elif (fmt == 2) { invoke(active_repack_q51(), reinterpret(qp), reinterpret(sp), n, d) - } elif (fmt == 20) { // k2's stream tag - the kq repack runs under its kernel id 2 + } elif (fmt == K2_STREAM_TAG) { invoke(active_repack_kq(), 2, reinterpret(qp), reinterpret(sp), n, d) } else { invoke(active_repack_kq(), fmt, reinterpret(qp), reinterpret(sp), n, d) diff --git a/modules/dasLLAMA/dasllama/dasllama_math_gen.das b/modules/dasLLAMA/dasllama/dasllama_math_gen.das index a019539b23..27f4e2932d 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_gen.das @@ -2071,7 +2071,7 @@ def private kq_batch_cell_gen(fmt : int; var myp : float?; kqp : uint8 const?; k let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) let nb16 = n / 16l - let packed = fmt == 4 || fmt == 40 || fmt == 44 || fmt == 3 || fmt == 45 || fmt == 2 // packed planes: the tile reads them directly (iq3s/iq3xxs/iq2s expand their grid gathers into the panel) + let packed = kq_reads_packed_planes(int64(fmt)) var scratch : array if (!packed) { scratch |> resize(mr * nsb * 256l) diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_common.das b/modules/dasLLAMA/dasllama/dasllama_metal_common.das index f43a3dc1f9..8df3f0d33e 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_common.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_common.das @@ -873,7 +873,7 @@ def kq_quants_of(dev : MetalDevice?; t : Model; fmt : KqFmt; off : int64) : tupl unsafe { if (fmt == KqFmt.k4) { return (buf = plane_buffer(dev, addr < void? >(t.k4q[0]), uint64(long_length(t.k4q)), t.image_map != null), - qoff = uint64(sb0 * 128l)) + qoff = uint64(sb0 * kq_qsb(fmt))) } if (fmt == KqFmt.k5) { return (buf = plane_buffer(dev, addr < void? >(t.k5q[0]), uint64(long_length(t.k5q)), t.image_map != null), @@ -881,39 +881,39 @@ def kq_quants_of(dev : MetalDevice?; t : Model; fmt : KqFmt; off : int64) : tupl } if (fmt == KqFmt.iq4xs) { return (buf = plane_buffer(dev, addr < void? >(t.iq4xsq[0]), uint64(long_length(t.iq4xsq)), t.image_map != null), - qoff = uint64(sb0 * 128l)) + qoff = uint64(sb0 * kq_qsb(fmt))) } if (fmt == KqFmt.k3) { return (buf = plane_buffer(dev, addr < void? >(t.k3q[0]), uint64(long_length(t.k3q)), t.image_map != null), - qoff = uint64(sb0 * 96l)) + qoff = uint64(sb0 * kq_qsb(fmt))) } if (fmt == KqFmt.iq3s) { return (buf = plane_buffer(dev, addr < void? >(t.iq3sq[0]), uint64(long_length(t.iq3sq)), t.image_map != null), - qoff = uint64(sb0 * 104l)) + qoff = uint64(sb0 * kq_qsb(fmt))) } if (fmt == KqFmt.iq3xxs) { return (buf = plane_buffer(dev, addr < void? >(t.iq3xxsq[0]), uint64(long_length(t.iq3xxsq)), t.image_map != null), - qoff = uint64(sb0 * 96l)) + qoff = uint64(sb0 * kq_qsb(fmt))) } if (fmt == KqFmt.iq4nl) { return (buf = plane_buffer(dev, addr < void? >(t.iq4nlq[0]), uint64(long_length(t.iq4nlq)), t.image_map != null), - qoff = uint64(sb0 * 128l)) + qoff = uint64(sb0 * kq_qsb(fmt))) } if (fmt == KqFmt.k2) { return (buf = plane_buffer(dev, addr < void? >(t.k2q[0]), uint64(long_length(t.k2q)), t.image_map != null), - qoff = uint64(sb0 * 64l)) + qoff = uint64(sb0 * kq_qsb(fmt))) } if (fmt == KqFmt.iq2s) { return (buf = plane_buffer(dev, addr < void? >(t.iq2sq[0]), uint64(long_length(t.iq2sq)), t.image_map != null), - qoff = uint64(sb0 * 72l)) + qoff = uint64(sb0 * kq_qsb(fmt))) } if (fmt == KqFmt.iq2xs) { return (buf = plane_buffer(dev, addr < void? >(t.iq2xsq[0]), uint64(long_length(t.iq2xsq)), t.image_map != null), - qoff = uint64(sb0 * 64l)) + qoff = uint64(sb0 * kq_qsb(fmt))) } if (fmt == KqFmt.iq2xxs) { return (buf = plane_buffer(dev, addr < void? >(t.iq2xxsq[0]), uint64(long_length(t.iq2xxsq)), t.image_map != null), - qoff = uint64(sb0 * 64l)) + qoff = uint64(sb0 * kq_qsb(fmt))) } return (buf = plane_buffer(dev, addr < void? >(t.k6q[0]), uint64(long_length(t.k6q)), t.image_map != null), qoff = uint64(sb0 * 192l)) diff --git a/modules/dasLLAMA/dasllama/dasllama_ple.das b/modules/dasLLAMA/dasllama/dasllama_ple.das index d3967ad802..2754fea480 100644 --- a/modules/dasLLAMA/dasllama/dasllama_ple.das +++ b/modules/dasLLAMA/dasllama/dasllama_ple.das @@ -96,44 +96,12 @@ def private ple_gather_row(t : Model; row : int64; var dst : array; doff dequant_q8_row(t, eloff, all, unsafe(addr(dst[doff]))) return } - // metal-blob carriers compact k4/k5 scale strips to 16B and split k6 (embed_row's strides) - let ssb45 = t.metal_blob ? 16l : K4_SSB - let k6d0 = t.metal_blob ? (long_length(t.k6s) / K6_SSB) * 16l : 0l + var kqp : array const? + var ksp : array const? + kq_planes_of(t, t.ple_emb_fmt, kqp, ksp) for (blk in range64(all / 256l)) { let sb = (eloff + blk * 256l) / 256l - if (t.ple_emb_fmt == KqFmt.k4) { - dequant_k4_plane_superblock(t.k4q, sb * K4_QSB, t.k4s, sb * ssb45, dst, doff + blk * 256l) - } elif (t.ple_emb_fmt == KqFmt.k5) { - dequant_k5_plane_superblock(t.k5q, sb * K5_QSB, t.k5s, sb * ssb45, dst, doff + blk * 256l) - } elif (t.ple_emb_fmt == KqFmt.k6) { - if (t.metal_blob) { - dequant_k6_plane_superblock_at(t.k6q, sb * K6_QSB, t.k6s, sb * 16l, k6d0 + sb * 2l, dst, doff + blk * 256l) - } else { - dequant_k6_plane_superblock(t.k6q, sb * K6_QSB, t.k6s, sb * K6_SSB, dst, doff + blk * 256l) - } - } elif (t.ple_emb_fmt == KqFmt.q40) { - dequant_q40_plane_superblock(t.q40q, sb * Q40_QSB, t.q40s, sb * Q40_SSB, dst, doff + blk * 256l) - } elif (t.ple_emb_fmt == KqFmt.iq4xs) { - dequant_iq4xs_plane_superblock(t.iq4xsq, sb * IQ4XS_QSB, t.iq4xss, sb * IQ4XS_SSB, dst, doff + blk * 256l) - } elif (t.ple_emb_fmt == KqFmt.k3) { - dequant_k3_plane_superblock(t.k3q, sb * K3_QSB, t.k3s, sb * K3_SSB, dst, doff + blk * 256l) - } elif (t.ple_emb_fmt == KqFmt.iq3s) { - dequant_iq3s_plane_superblock(t.iq3sq, sb * IQ3S_QSB, t.iq3ss, sb * IQ3S_SSB, dst, doff + blk * 256l) - } elif (t.ple_emb_fmt == KqFmt.iq3xxs) { - dequant_iq3xxs_plane_superblock(t.iq3xxsq, sb * IQ3XXS_QSB, t.iq3xxss, sb * IQ3XXS_SSB, dst, doff + blk * 256l) - } elif (t.ple_emb_fmt == KqFmt.iq4nl) { - dequant_iq4nl_plane_superblock(t.iq4nlq, sb * Q40_QSB, t.iq4nls, sb * Q40_SSB, dst, doff + blk * 256l) - } elif (t.ple_emb_fmt == KqFmt.k2) { - dequant_k2_plane_superblock(t.k2q, sb * K2_QSB, t.k2s, sb * K2_SSB, dst, doff + blk * 256l) - } elif (t.ple_emb_fmt == KqFmt.iq2s) { - dequant_iq2s_plane_superblock(t.iq2sq, sb * IQ2S_QSB, t.iq2ss, sb * IQ2S_SSB, dst, doff + blk * 256l) - } elif (t.ple_emb_fmt == KqFmt.iq2xs) { - dequant_iq2xs_plane_superblock(t.iq2xsq, sb * IQ2XS_QSB, t.iq2xss, sb * IQ2XS_SSB, dst, doff + blk * 256l) - } elif (t.ple_emb_fmt == KqFmt.iq2xxs) { - dequant_iq2xxs_plane_superblock(t.iq2xxsq, sb * IQ2XXS_QSB, t.iq2xxss, sb * IQ2XXS_SSB, dst, doff + blk * 256l) - } else { - panic("ple_gather_row: '{t.ple_emb_fmt}' has no kq plane pair") - } + dequant_kq_plane_sb(t.ple_emb_fmt, *kqp, *ksp, sb, t.metal_blob, dst, doff + blk * 256l) } } diff --git a/modules/dasLLAMA/dasllama/dasllama_tune.das b/modules/dasLLAMA/dasllama/dasllama_tune.das index 90fadbbbbd..ba83e64231 100644 --- a/modules/dasLLAMA/dasllama/dasllama_tune.das +++ b/modules/dasLLAMA/dasllama/dasllama_tune.das @@ -30,7 +30,7 @@ struct TunePerm { } def private build_grid() : array { - // pruned to seats that win beyond noise (the 20-seat walk never spread >1.4% winner-vs-runner-up in any box's sidecar race tables); unroll=1 omits unroll_count (rows byte-identical to the hand hints); vec4_u4 = dot_q4's pinned NEON nibble-widening winner + // pruned to the seats that win beyond noise (the full 20-seat walk never spread >1.4% winner-vs-runner-up in any box's sidecar race tables - zen2 lcpp_bench/parity, m1, m4, m5); unroll=1 omits unroll_count so rows stay byte-identical to the hand hints let WIDTHS = fixed_array(8, 16) let UNROLLS = fixed_array(2) // unroll-only rows; u1 == plain var grid : array @@ -46,7 +46,7 @@ def private build_grid() : array { hints <- [TuneHint(tune = 1, vectorize = true, width = w, unroll = u)])) } } - grid |> emplace(TunePerm(suffix = "vec4_u4", hints <- [TuneHint(tune = 1, vectorize = true, width = 4, unroll = 4)])) + grid |> emplace(TunePerm(suffix = "vec4_u4", hints <- [TuneHint(tune = 1, vectorize = true, width = 4, unroll = 4)])) // dot_q4's pinned NEON nibble-widening winner return <- grid } diff --git a/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das b/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das index b39c43d303..fd964a8dfe 100644 --- a/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das +++ b/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das @@ -6426,8 +6426,8 @@ def gemv_cls_enc(fmt : int; raw : VkCommandBuffer; var h : VkHaz; var set_ : VkD } // Q4_K: w = d*sc*q - dmin*mn*1, q in [0,15] — per 32-block axs*(d*sc*idot - dmin*mn*bsum). -// The ten formats share family=kq_gemv_cls: ONE set layout + ONE set_kq_gemv_cls serves all -// ten pipelines (identical inherited members), each leaf keeping its own ensure_/enc_ names. +// Every kq format shares family=kq_gemv_cls: ONE set layout + ONE set_kq_gemv_cls serves every +// pipeline (identical inherited members), each leaf keeping its own ensure_/enc_ names. [vk_dispatch(name = "kq_gemv_k4_cls", kernel = "run", family = "kq_gemv_cls", grid = "wgs", params = "wgs : int64")] class KqGemvK4 : KqGemvBase { def override blk_contrib(wsb, blk, xsb : uint) : float { diff --git a/modules/dasLLAMA/dasllama/dasllama_vulkan_common.das b/modules/dasLLAMA/dasllama/dasllama_vulkan_common.das index 47f7a0dc88..f41b13e29d 100644 --- a/modules/dasLLAMA/dasllama/dasllama_vulkan_common.das +++ b/modules/dasLLAMA/dasllama/dasllama_vulkan_common.das @@ -1742,19 +1742,19 @@ def arena_local_blk(blk : int64) : int64 => blk & ((1l << 32l) - 1l) // Mirrors stack_plane_bytes: it computes the same totals as count * these strides. // the tier's stack fmt space is int(KqFmt) — bridge to the gemm_schema kq id space def vk_kq_schema_id(fmt : int) : int { - if (fmt == 1) return 4 - if (fmt == 2) return 5 - if (fmt == 3) return 6 - if (fmt == 4) return 40 - if (fmt == 6) return 44 - if (fmt == 7) return 3 - if (fmt == 8) return 33 - if (fmt == 9) return 34 - if (fmt == 10) return 45 - if (fmt == 11) return 2 - if (fmt == 12) return 23 - if (fmt == 13) return 24 - if (fmt == 14) return 25 + if (fmt == int(KqFmt.k4)) return 4 + if (fmt == int(KqFmt.k5)) return 5 + if (fmt == int(KqFmt.k6)) return 6 + if (fmt == int(KqFmt.q40)) return 40 + if (fmt == int(KqFmt.iq4xs)) return 44 + if (fmt == int(KqFmt.k3)) return 3 + if (fmt == int(KqFmt.iq3s)) return 33 + if (fmt == int(KqFmt.iq3xxs)) return 34 + if (fmt == int(KqFmt.iq4nl)) return 45 + if (fmt == int(KqFmt.k2)) return 2 + if (fmt == int(KqFmt.iq2s)) return 23 + if (fmt == int(KqFmt.iq2xs)) return 24 + if (fmt == int(KqFmt.iq2xxs)) return 25 panic("vk_kq_schema_id: not a kq stack fmt") return 0 } diff --git a/modules/dasLLAMA/followup_general.md b/modules/dasLLAMA/followup_general.md index 4a9b3a8a0e..97abd94787 100644 --- a/modules/dasLLAMA/followup_general.md +++ b/modules/dasLLAMA/followup_general.md @@ -756,7 +756,7 @@ ones - lets ledger towards the end of this arc").** The IQ3_S CPU arm is the first written against llama.cpp's arch kernel (signs-on-activation, sllv index compose, scalar grid loads beating hardware gather); IQ4_XS and Q3_K were written from the disk format alone. - Toward the end of the arc: read `ggml_vec_dot_iq4_xs_q8_K` and `ggml_vec_dot_q3_K_q8_K` + Toward the end of the arc: read the reference exe's vec-dot and the reference exe's vec-dot (x86 + arm), list every technique ours does not measure, land the missing ones as `[tune_perm]` spellings and let the probe judge. Done = a per-format note naming what was raced and what won, beside the existing bench rows. diff --git a/modules/dasLLAMA/followup_metal.md b/modules/dasLLAMA/followup_metal.md index 74e76d3174..b6819e92dc 100644 --- a/modules/dasLLAMA/followup_metal.md +++ b/modules/dasLLAMA/followup_metal.md @@ -13,7 +13,7 @@ EMPTY string on macOS (llvm_jit_common.das documents it beside `g_target_arm64_i i8mm never detected on Apple Silicon at the EMITTER tier (`g_target_arm64_i8mm`), so the generators declined and the seats never raced - even though `cpu_supports("i8mm")` answered correctly via sysctl. M1 lacks i8mm; M2+ has it; the M5 Max additionally has FEAT_SME2p1 + -BF16/EBF16. FIXED in the unquirk pass (278b3c765): `g_target_arm64_i8mm` also consults +BF16/EBF16. FIXED in the unquirk pass (PR-1): `g_target_arm64_i8mm` also consults `cpu_supports`, and the target machine appends `+i8mm` when the host has it - an M5 `--tune` now races smmla-vs-NEON with zero new kernel work. Mac-session order: @@ -26,7 +26,32 @@ now races smmla-vs-NEON with zero new kernel work. Mac-session order: 3. SME/SME2 is the tier after: no seats, no emitter, new kernel design (streaming mode + ZA tiles) - research first per the standing research-before-kernel-work rule. -## 2. Metal tier perf items +## 2. The fixtures - which GGUF per format, and where to get it + +Every vehicle is a Llama-3.2-1B-Instruct requant unless named otherwise. zen2 holds them at +`D:\Work\llama.cpp\models\`; the M1 worktree box holds the same set at +`~/Work/llama.cpp/models/` together with the baked `.dlim` images beside each; the M5 starts +empty - `scp m1:~/Work/llama.cpp/models/ ~/Work/llama.cpp/models/` (or from zen2) per row. +The `-local` files are our own `llama-quantize --allow-requantize` products (the IQ ones with +`--imatrix`), the rest are bartowski / mradermacher downloads. + +| format | vehicle GGUF | notes | +|---|---|---| +| IQ4_XS | `Llama-3.2-1B-Instruct-IQ4_XS.gguf` | the pilot (bartowski) | +| Q3_K | `Llama-3.2-1B-Instruct-Q3_K_L.gguf` | a third of the bytes are k5/k6 | +| IQ3_S | `Qwen2.5-1.5B-Instruct-IQ3_XS.gguf` (kernels) + `Llama-3.2-1B-Instruct-IQ3_M.gguf` (e2e/bench) | the Qwen file proves kernels, the local IQ3_M carries iq3_s on the bench rows | +| IQ3_XXS | `Llama-3.2-1B-Instruct-IQ3_XXS-local.gguf` | `Llama-3.2-1B-Instruct.i1-IQ3_XXS.gguf` (mradermacher) mixes in IQ2_S attn - the IQ2_S vehicle below | +| IQ4_NL | `Llama-3.2-1B-Instruct-IQ4_NL-local.gguf` | no imatrix needed | +| Q2_K | `Llama-3.2-1B-Instruct-Q2_K-local.gguf` | | +| IQ2_S | `Llama-3.2-1B-Instruct.i1-IQ3_XXS.gguf` | IQ2_S attn x32 + IQ3_XXS/IQ3_S/Q4_K/Q5_K - a MIXED vehicle by construction | +| IQ2_XS | `Llama-3.2-1B-Instruct-IQ2_XS-local.gguf` | carries q2_K on blk.0/1 ffn_down - the mixed-vehicle e2e of QUIRK 26 | +| IQ2_XXS | `Llama-3.2-1B-Instruct-IQ2_XXS-local.gguf` | prompt ids `128000,12805,5304,264,892`; the step-3 0.0267 top-2 tie | + +The reference build for the ratios is llama.cpp b10660 (`~/Work/llama.cpp/build/bin/llama-bench` +/ `llama-simple` on the M1; the same tree serves the `simple_ids` oracle). Parity prompt for +every format: `harness/parity.das -- -m -n 40 --ids 128000,12805,5304,264,892`. + +## 3. Metal tier perf items Placeholder - the per-format Metal notes (tg 0.78-0.93x tails, the IQ4_XS lane-map gap of followup_general #58, the Q22 dispatch-loop probe method) consolidate here in Phase E of diff --git a/modules/dasLLAMA/harness/dasllama_tuner.das b/modules/dasLLAMA/harness/dasllama_tuner.das index 84aae07f4a..623ea2932b 100644 --- a/modules/dasLLAMA/harness/dasllama_tuner.das +++ b/modules/dasLLAMA/harness/dasllama_tuner.das @@ -151,6 +151,10 @@ def private run_half(name : string; paranoid : bool) : bool { } let elapsed_ms = get_time_usec(t0) / 1000 print("dasllama_tuner: {name} finished in {elapsed_ms} ms (rc={rc})\n") + if (rc == TUNE_ONLY_NO_MATCH) { + g_no_match_halves++ + return true // the filter named nothing here; main fails the mint only when BOTH halves say so + } if (rc != 0) { // the refusal detail scrolled by minutes ago mid-stream — the closing line carries the // WHY and the escape hatches so a nonzero exit is self-diagnosing where you look. @@ -165,6 +169,10 @@ def private run_half(name : string; paranoid : bool) : bool { return rc == 0 } +// exit code of a half whose --tune-only filter matched nothing (tune_kernels / gen_tune_probe) +let TUNE_ONLY_NO_MATCH = 3 +var private g_no_match_halves = 0 + [export] def main : int { let t0 = ref_time_ticks() @@ -193,6 +201,10 @@ def main : int { } let ok1 = run_half("gen_tune_probe.das", paranoid) let ok2 = run_half("tune_kernels.das", paranoid) + if (g_no_match_halves == 2) { + print("dasllama_tuner: --tune-only '{tune_only_filter()}' matched no family in either half - a typo'd token; sidecar untouched\n") + return 1 + } if (!empty(sidecar)) { if (ok1 && ok2) { if (!empty(oldText)) { diff --git a/modules/dasLLAMA/harness/gen_tune_probe.das b/modules/dasLLAMA/harness/gen_tune_probe.das index 89e7879674..3eaa4ea0b1 100644 --- a/modules/dasLLAMA/harness/gen_tune_probe.das +++ b/modules/dasLLAMA/harness/gen_tune_probe.das @@ -93,6 +93,9 @@ def private gen_round_step(r : int; keep : array) { // the previous sidecar's seat for `entry`, read raw (a stale sidecar still names the seats) — // a re-mint's crown only moves when it beats the incumbent by gen_margin() var private g_incumbent : table +// the --tune-only filter matched no generator family here: exit 3, so the wrapper can tell a +// half-that-sat-out from a typo'd token (both halves reporting it fails the mint) +var private g_tune_only_no_match = false var private g_incumbent_loaded = false def private gen_incumbent(entry : string) : string { @@ -948,7 +951,7 @@ def kq_layout_mrs(fmt : int64) : table { // nolint:STYLE037,STYLE def run_kq_tile(tile; fx : KqFixture; kq : array; ks : array; mr : int64; var y : array) { let ng = fx.d / mr verify(fx.ntok % 4l == 0l) - let packed = fx.fmt == 4l || fx.fmt == 40l || fx.fmt == 44l || fx.fmt == 3l || fx.fmt == 45l || fx.fmt == 2l // packed planes: the tile reads them directly (iq3s/iq3xxs/iq2s unpack their panels per group) + let packed = kq_reads_packed_planes(fx.fmt) var panel : array if (!packed) { panel |> resize(int(mr * fx.nsb * 256l)) @@ -2056,6 +2059,7 @@ def tune_mode_run : bool { // nolint:STYLE037,STYLE038 — flat family-by-family if (planTotal == 0) { print("tune-only '{tune_only_filter()}': no generator family matches - half skipped, sidecar untouched\n") delete kqSel + g_tune_only_no_match = true return true } if (!empty(tune_only_filter())) { @@ -2205,7 +2209,7 @@ def main : int { if (mode == "tune") { let rok = tune_mode_run() print(rok ? "GEN TUNE RUN OK\n" : "GEN TUNE RUN FAIL\n") - return rok ? 0 : 1 + return rok ? (g_tune_only_no_match ? 3 : 0) : 1 } print("set DAS_TUNE_MODE=test or =tune (compile-time env: it stamps the variant grid)\n") return 1 diff --git a/modules/dasLLAMA/harness/tune_kernels.das b/modules/dasLLAMA/harness/tune_kernels.das index da2bd800c9..f8660d817d 100644 --- a/modules/dasLLAMA/harness/tune_kernels.das +++ b/modules/dasLLAMA/harness/tune_kernels.das @@ -2348,6 +2348,10 @@ def private metal_twin_absorb(var wins : array; var ran : bool&; r) { // fallback: the tuner rail drives this half explicitly (DAS_TUNE_MODE=tune) — the auto // policy must not fire a nested tuner off this program's own missing per-app sidecar +//! Exit code of a `--tune-only` half whose filter matched nothing here: the wrapper fails the mint +//! only when BOTH halves report it (a filter naming one half's family is not a typo). +let TUNE_ONLY_NO_MATCH = 3 + [export, tune_policy(missing = "fallback")] def main : int { // nolint:STYLE037,STYLE038 — flat one-bench-per-kernel rail; the noise probes are its checkpoints if (tuner_paranoid_requested()) { @@ -2370,8 +2374,8 @@ def main : int { // nolint:STYLE037,STYLE038 — flat one-bench-per-kernel rail; ("gemm_f32_uk_4x16", @@bench_gemm_tile), ("dot_q8q8_laneq4x4", @@bench_laneq4x4) ] - let filtered = !empty(tune_only_filter()) - if (filtered) { + let partialMint = !empty(tune_only_filter()) + if (partialMint) { let nb = length(benches) for (k in range(nb)) { let i = nb - 1 - k @@ -2382,7 +2386,7 @@ def main : int { // nolint:STYLE037,STYLE038 — flat one-bench-per-kernel rail; if (empty(benches)) { print("tune-only '{tune_only_filter()}': no loop-hint kernel matches - half skipped, sidecar untouched\n") delete benches - return 0 + return TUNE_ONLY_NO_MATCH } tune_detail("tune-only '{tune_only_filter()}': racing {length(benches)} loop-hint kernel(s); the Metal tensor race and the runtime knob snapshot sit out (previous sidecar values survive)\n") } @@ -2452,7 +2456,7 @@ def main : int { // nolint:STYLE037,STYLE038 — flat one-bench-per-kernel rail; ? "\nevery kernel keeps its shipped fallback - the tuner found nothing new on this box.\n" : "\n{changed} kernel(s) beat their shipped fallback on this box.\n") // the one count still written by hand; say so rather than let the bar quietly misreport - if (!filtered && length(g_results) != TUNED_KERNEL_COUNT) { + if (!partialMint && length(g_results) != TUNED_KERNEL_COUNT) { print("note: TUNED_KERNEL_COUNT is {TUNED_KERNEL_COUNT} but {length(g_results)} kernels ran - the progress bar was scaled wrong; update the constant\n") } @@ -2463,7 +2467,7 @@ def main : int { // nolint:STYLE037,STYLE038 — flat one-bench-per-kernel rail; static_if (typeinfo builtin_module_exists(das_metal)) { var mt_wins : array var mt_ran = false - if (!filtered) { // a filtered mint leaves "runtime" untouched - no crowns to record + if (!partialMint) { // a filtered mint leaves "runtime" untouched - no crowns to record // the two race halves come through different require paths, so their result element // types do not unify into one typed local - absorb each returned array where it lands for (r in metal_tensor_race()) { @@ -2473,7 +2477,7 @@ def main : int { // nolint:STYLE037,STYLE038 — flat one-bench-per-kernel rail; metal_twin_absorb(mt_wins, mt_ran, r) } } - if (!mt_ran && !filtered) { + if (!mt_ran && !partialMint) { tune_detail("METAL_TWIN: no Metal device - tensor race skipped\n") } metal_crowns = join(mt_wins, ",") @@ -2502,7 +2506,7 @@ def main : int { // nolint:STYLE037,STYLE038 — flat one-bench-per-kernel rail; threads = get_total_hw_jobs() + 1, metal_tensor = metal_crowns, metal_tall_floor = metal_tall_floor(), metal_devw_small_panel_mb = metal_devw_small_panel_mb(), metal_cvt_min_rows = metal_cvt_min_rows())) // total lanes (workers + computing main), the DAS_JOBQUE_THREADS unit - if (filtered) { + if (partialMint) { // a filtered mint must not clobber the runtime knobs (hand-edited TB/L2 refinements // live there): a null runtime leaves the sidecar section untouched delete_json(runtime) @@ -2517,7 +2521,7 @@ def main : int { // nolint:STYLE037,STYLE038 — flat one-bench-per-kernel rail; "dot_q8tq4kv" => "vec16", "quantize_tq4kv_row" => "plain", "dot_q8q8_f16s" => "vec16", "dot_q51e" => "vec16" } - if (filtered) { // a filtered mint seeds nothing - only the raced kernels write + if (partialMint) { // a filtered mint seeds nothing - only the raced kernels write winners |> clear() } for (r in g_results) { diff --git a/modules/dasLLAMA/performance/REVIEW.das b/modules/dasLLAMA/performance/REVIEW.das index 2ecfa66fbc..ef6847311b 100644 --- a/modules/dasLLAMA/performance/REVIEW.das +++ b/modules/dasLLAMA/performance/REVIEW.das @@ -105,6 +105,47 @@ def private check_last_known_good { // historical rows legitimately carry older pins and shas - so they stay out of this sweep; // reachability runs in make-pr's chain, where the orphan hazard appears.) let private RECORDS_DIR = "modules/dasLLAMA/performance/records" +let private DEFAULTS_DIR = "modules/dasLLAMA/performance/defaults" + +// the shipped defaults profiles: a quiet mint, a version pin, and NO commit stamp (a +// profile's validity is its pin + feature class; an engine_sha would be an orphan after +// every rebase and says nothing a reader can act on) +def private check_defaults_profiles { + dir(DEFAULTS_DIR) $(name) { + return if (!(name |> ends_with(".tune-defaults.json"))) + let full = "{DEFAULTS_DIR}/{name}" + let text = fread(full) + if (empty(text)) { + gate_finding(full, "missing or unreadable") + return + } + var jerr = "" + var doc = read_json(text, jerr) + defer() { + unsafe { + delete doc + } + } + if (doc == null) { + gate_finding(full, "does not parse as JSON: {jerr}") + return + } + let noise = "{doc?["provenance"]?["noise"] ?? ""}" + if (noise != "ok") { + gate_finding(full, "provenance.noise is '{noise}' - a shipped profile carries a quiet mint (noise ok)") + } + if (empty("{doc?["provenance"]?["dasllama_version"] ?? ""}")) { + gate_finding(full, "no provenance.dasllama_version - a profile with no version pin adopts on every build") + } + if (!empty("{doc?["provenance"]?["engine_sha"] ?? ""}")) { + gate_finding(full, "carries provenance.engine_sha - a profile ships no commit stamp (tune_profile_export strips it)") + } + let kernels = doc?["kernels"] + if (kernels == null || !(kernels.value is _object) || empty(kernels.value as _object)) { + gate_finding(full, "no kernels section - not a profile") + } + } +} def private check_records_corpus { dir(RECORDS_DIR) $(name) { @@ -156,6 +197,7 @@ def main() : int { return 2 } check_last_known_good() + check_defaults_profiles() check_schema_engine_free() check_single_exchange_client() check_records_corpus() diff --git a/modules/dasLLAMA/performance/REVIEW.md b/modules/dasLLAMA/performance/REVIEW.md index 8cbcd39137..12827343f5 100644 --- a/modules/dasLLAMA/performance/REVIEW.md +++ b/modules/dasLLAMA/performance/REVIEW.md @@ -9,28 +9,36 @@ validate through `../dasllama/dasllama_exchange_schema.das` instead.** The engin that gate is a defect. **Weakening `REVIEW.das`'s record-hygiene checks - `hardware.remote_desktop` `off` on every -stored run, `provenance.noise` `ok` on every sidecar - is a defect; the fix for a red is a -re-mint on a quiet, session-free box, never an edit.** +stored run, `provenance.noise` `ok` on every checked-in JSON under this folder that carries a +`provenance` block, no `engine_sha` in a `defaults/` profile - is a defect; the fix for a red +is a re-mint on a quiet, session-free box, never an edit.** -**A diff that writes a commit stamp anywhere under this folder - `records/`, -`last_known_good_sidecar.json`, any provenance manifest - naming a commit the branch under -review cannot reach is a defect - re-mint, or re-stamp to a reachable commit whose +**A diff that writes a commit stamp anywhere under this folder naming a commit the branch +under review cannot reach is a defect - re-mint, or re-stamp to a reachable commit whose `modules/dasLLAMA/` tree is byte-identical to the tree that was measured, with the PR body -naming the re-stamp.** The commit stamps are a `das` row's `sha` and a sidecar's -`provenance.engine_sha`. A stamp that resolves to no commit at all counts as unreachable. -Re-stamping inside a content-addressed archive (`records/.tune..json`) re-hashes -and renames the file and repoints every row's `tune_sha` in the same change. +naming the re-stamp.** The commit stamps are a `das` row's `sha` and a `provenance.engine_sha` +in any checked-in JSON under this folder. A stamp that resolves to no commit at all counts as +unreachable. + +**A diff that re-stamps an archived sidecar (`records/.tune..json`) re-hashes and +renames the file and repoints every `records/.json` row whose `tune_sha` named the old +name, in the same change.** The archive is content-addressed; a row left on the old name +points at a file that no longer exists. **A diff that writes a reference-engine row to `records/` whose `sha` names anything but the standing ref pin (`DEFAULT_REF_SHA`, `../benchmarks/setup_lcpp_ref.das`) is a defect - re-mint.** A reference row that carries no `sha` (the cli and python reference tools) is pinned by its builder instead - the cli tools by that same ref pin's checkout, the python -legs by `../benchmarks/asr/requirements-*.txt` - and a diff that moves one of those pins -re-mints every row the moved pin's tool minted, or withdraws them. +legs by `../benchmarks/asr/requirements-*.txt`. + +**A diff that moves a reference-tool pin - the ref pin's checkout, or a python leg's +`requirements-*.txt` - re-mints every row the moved pin's tool minted, or withdraws them.** -**A diff that writes a sidecar to `records/` whose `provenance.dasllama_version` differs from -`DASLLAMA_RELEASE` (`../dasllama/dasllama_version.das`) is a defect - re-mint.** Read -`DASLLAMA_RELEASE` at the commit the sidecar's `provenance.engine_sha` names. +**A diff that writes a provenance manifest under this folder whose `provenance.dasllama_version` +differs from the `DASLLAMA_RELEASE` string (`../dasllama/dasllama_version.das`; the JSON key +stays `dasllama_version` across constant renames) is a defect - re-mint.** For a sidecar +with an `engine_sha`, read the value at that commit; a `defaults/` profile compares against +the branch under review. **A diff that writes a row to `records/.json` mints that row from a board cell.** A board cell is one `gen_bench_records.das` spawns, or a manual `../benchmarks/lcpp_bench.das` cell diff --git a/modules/dasLLAMA/performance/defaults/arm-neon.tune-defaults.json b/modules/dasLLAMA/performance/defaults/arm-neon.tune-defaults.json index f038e3319c..7bef73466d 100644 --- a/modules/dasLLAMA/performance/defaults/arm-neon.tune-defaults.json +++ b/modules/dasLLAMA/performance/defaults/arm-neon.tune-defaults.json @@ -1,70 +1,69 @@ { - "kernels" : { - "add_inplace" : "vec8_u2", - "cvt_f32_to_f16" : "vec8_u2", - "rope_scaled_neox_tab" : "vec16", - "q51q8_tile_gen" : "mr8", - "softmax" : "vec8_u2", - "mul_inplace" : "vec8_u2", - "quantize_q8_0_bs_into_ptr" : "plain", - "iq2xsq8_tile_gen" : "mr8", - "dot_q8kv" : "vec4_u4", - "dot_q8q8" : "vec16", - "iq4xsq8_tile_gen" : "mr8", - "iq4nlq8_tile_gen" : "mr8", - "quantize_q8kv_row" : "plain", - "axpy_f16" : "vec4_u4", - "q40q8_tile_gen" : "mr8", - "axpy_tq4kv" : "vec8_u2", - "cvt_tq4kv_to_f32" : "vec8_u2", - "axpy" : "vec8_u2", - "dot_q8q8kv" : "vec16", - "dot_mx4q8" : "vec4_u4", - "softmax_sink" : "vec8_u2", - "iq2xxsq8_tile_gen" : "mr8", - "dot_q8q8_laneq4x4" : "u2", - "dot_bf16" : "vec8_u2", - "iq2sq8_tile_gen" : "mr8", - "add_scale_inplace" : "vec8_u2", - "cvt_q8kv_to_f32" : "vec8_u2", - "axpy_q8kv" : "u2", - "dot_q8q8_f16s" : "vec16", - "q8q8_tile_gen" : "mr8_budget", - "quantize_q8_0_into_ptr" : "plain", - "k4q8_tile_gen" : "mr8", - "k5q8_tile_gen" : "mr8", - "k6q8_tile_gen" : "mr8", - "k3q8_tile_gen" : "mr8", - "k2q8_tile_gen" : "mr8", - "gemm_f32_uk_4x16" : "u2", - "dot_q51e" : "vec16", - "dot_f16" : "vec8_u2", - "cvt_f16_to_f32" : "vec16", - "dot" : "vec8_u2", - "iq3xxsq8_tile_gen" : "mr8", - "iq3sq8_tile_gen" : "mr8", - "dot_q8tq4kv" : "vec16", - "scale_inplace" : "vec8_u2", - "dot_q4" : "vec4_u4", - "quantize_tq4kv_row" : "plain", - "copy_floats" : "vec8_u2", - "rmsnorm" : "vec8" - }, - "provenance" : { - "validation" : "ok", - "noise_probes" : "start cv 0.22%; mid1 cv 0.11%; mid2 cv 0.10%; end cv 0.09%", - "platform" : "darwin", - "noise_floor_cv_pct" : "0.22", - "features" : "neon;dotprod;fullfp16;lse", - "engine_sha" : "278b3c765", - "class" : "arm-neon", - "written" : "2026-09-01T01:37:39.701Z", - "validation_demoted" : "0", - "mode" : "normal", - "dasllama_version" : "15", - "origin" : "profile", - "noise" : "ok", - "arch" : "arm64", - "validation_max_drift_pct" : "0.13" - } -} \ No newline at end of file + "kernels": { + "add_inplace": "vec8_u2", + "add_scale_inplace": "vec8_u2", + "axpy": "vec8_u2", + "axpy_f16": "vec4_u4", + "axpy_q8kv": "u2", + "axpy_tq4kv": "vec8_u2", + "copy_floats": "vec8_u2", + "cvt_f16_to_f32": "vec16", + "cvt_f32_to_f16": "vec8_u2", + "cvt_q8kv_to_f32": "vec8_u2", + "cvt_tq4kv_to_f32": "vec8_u2", + "dot": "vec8_u2", + "dot_bf16": "vec8_u2", + "dot_f16": "vec8_u2", + "dot_mx4q8": "vec4_u4", + "dot_q4": "vec4_u4", + "dot_q51e": "vec16", + "dot_q8kv": "vec4_u4", + "dot_q8q8": "vec16", + "dot_q8q8_f16s": "vec16", + "dot_q8q8_laneq4x4": "u2", + "dot_q8q8kv": "vec16", + "dot_q8tq4kv": "vec16", + "gemm_f32_uk_4x16": "u2", + "iq2sq8_tile_gen": "mr8", + "iq2xsq8_tile_gen": "mr8", + "iq2xxsq8_tile_gen": "mr8", + "iq3sq8_tile_gen": "mr8", + "iq3xxsq8_tile_gen": "mr8", + "iq4nlq8_tile_gen": "mr8", + "iq4xsq8_tile_gen": "mr8", + "k2q8_tile_gen": "mr8", + "k3q8_tile_gen": "mr8", + "k4q8_tile_gen": "mr8", + "k5q8_tile_gen": "mr8", + "k6q8_tile_gen": "mr8", + "mul_inplace": "vec8_u2", + "q40q8_tile_gen": "mr8", + "q51q8_tile_gen": "mr8", + "q8q8_tile_gen": "mr8_budget", + "quantize_q8_0_bs_into_ptr": "plain", + "quantize_q8_0_into_ptr": "plain", + "quantize_q8kv_row": "plain", + "quantize_tq4kv_row": "plain", + "rmsnorm": "vec8", + "rope_scaled_neox_tab": "vec16", + "scale_inplace": "vec8_u2", + "softmax": "vec8_u2", + "softmax_sink": "vec8_u2" + }, + "provenance": { + "arch": "arm64", + "class": "arm-neon", + "dasllama_version": "15", + "features": "neon;dotprod;fullfp16;lse", + "mode": "normal", + "noise": "ok", + "noise_floor_cv_pct": "0.22", + "noise_probes": "start cv 0.22%; mid1 cv 0.11%; mid2 cv 0.10%; end cv 0.09%", + "origin": "profile", + "platform": "darwin", + "validation": "ok", + "validation_demoted": "0", + "validation_max_drift_pct": "0.13", + "written": "2026-09-01T01:37:39.701Z" + } +} diff --git a/modules/dasLLAMA/performance/defaults/x86-avx2.tune-defaults.json b/modules/dasLLAMA/performance/defaults/x86-avx2.tune-defaults.json index 936d2e9b4a..3392983b1a 100644 --- a/modules/dasLLAMA/performance/defaults/x86-avx2.tune-defaults.json +++ b/modules/dasLLAMA/performance/defaults/x86-avx2.tune-defaults.json @@ -1,70 +1,69 @@ { - "kernels" : { - "add_inplace" : "vec8_u2", - "cvt_f32_to_f16" : "vec8_u2", - "rope_scaled_neox_tab" : "plain", - "q51q8_tile_gen" : "dot_maddubs_width256_mr8", - "softmax" : "vec8_u2", - "mul_inplace" : "vec8_u2", - "quantize_q8_0_bs_into_ptr" : "vec16_u2", - "iq2xsq8_tile_gen" : "dot_maddubs_width256_mr8", - "dot_q8kv" : "vec4_u4", - "dot_q8q8" : "vec16", - "iq4xsq8_tile_gen" : "dot_maddubs_width256_mr8", - "iq4nlq8_tile_gen" : "dot_maddubs_width256_mr8", - "quantize_q8kv_row" : "plain", - "axpy_f16" : "vec8_u2", - "q40q8_tile_gen" : "dot_maddubs_width256_mr8", - "axpy_tq4kv" : "vec8_u2", - "cvt_tq4kv_to_f32" : "vec8_u2", - "axpy" : "vec8_u2", - "dot_q8q8kv" : "vec16", - "dot_mx4q8" : "u2", - "softmax_sink" : "vec8_u2", - "iq2xxsq8_tile_gen" : "dot_maddubs_width256_mr8", - "dot_q8q8_laneq4x4" : "", - "dot_bf16" : "vec8_u2", - "iq2sq8_tile_gen" : "dot_maddubs_width256_mr8", - "add_scale_inplace" : "vec8_u2", - "cvt_q8kv_to_f32" : "vec8_u2", - "axpy_q8kv" : "vec8_u2", - "dot_q8q8_f16s" : "vec16", - "q8q8_tile_gen" : "dot_maddubs_width256_mr8_kstep2", - "quantize_q8_0_into_ptr" : "plain", - "k4q8_tile_gen" : "dot_maddubs_width256_mr8", - "k5q8_tile_gen" : "dot_maddubs_width256_mr8", - "k6q8_tile_gen" : "dot_maddubs_width256_mr8", - "k3q8_tile_gen" : "dot_maddubs_width256_mr8", - "k2q8_tile_gen" : "dot_maddubs_width256_mr8", - "gemm_f32_uk_4x16" : "plain", - "dot_q51e" : "vec16", - "dot_f16" : "vec8_u2", - "cvt_f16_to_f32" : "vec8_u2", - "dot" : "vec8_u2", - "iq3xxsq8_tile_gen" : "dot_maddubs_width256_mr8", - "iq3sq8_tile_gen" : "dot_maddubs_width256_mr8", - "dot_q8tq4kv" : "vec16", - "scale_inplace" : "vec8_u2", - "dot_q4" : "vec8_u2", - "quantize_tq4kv_row" : "plain", - "copy_floats" : "vec8_u2", - "rmsnorm" : "vec16_u2" - }, - "provenance" : { - "validation" : "ok", - "noise_probes" : "start cv 0.28%; mid1 cv 0.33%; mid2 cv 0.15%; end cv 0.31%", - "platform" : "windows", - "noise_floor_cv_pct" : "0.33", - "features" : "avx2;f16c;fma;sse4.2", - "engine_sha" : "cb659766d", - "class" : "x86-avx2", - "written" : "2026-09-01T01:06:53.218Z", - "validation_demoted" : "0", - "mode" : "normal", - "dasllama_version" : "15", - "origin" : "profile", - "noise" : "ok", - "arch" : "x86_64", - "validation_max_drift_pct" : "19.46" - } -} \ No newline at end of file + "kernels": { + "add_inplace": "vec8_u2", + "add_scale_inplace": "vec8_u2", + "axpy": "vec8_u2", + "axpy_f16": "vec8_u2", + "axpy_q8kv": "vec8_u2", + "axpy_tq4kv": "vec8_u2", + "copy_floats": "vec8_u2", + "cvt_f16_to_f32": "vec8_u2", + "cvt_f32_to_f16": "vec8_u2", + "cvt_q8kv_to_f32": "vec8_u2", + "cvt_tq4kv_to_f32": "vec8_u2", + "dot": "vec8_u2", + "dot_bf16": "vec8_u2", + "dot_f16": "vec8_u2", + "dot_mx4q8": "u2", + "dot_q4": "vec8_u2", + "dot_q51e": "vec16", + "dot_q8kv": "vec4_u4", + "dot_q8q8": "vec16", + "dot_q8q8_f16s": "vec16", + "dot_q8q8_laneq4x4": "", + "dot_q8q8kv": "vec16", + "dot_q8tq4kv": "vec16", + "gemm_f32_uk_4x16": "plain", + "iq2sq8_tile_gen": "dot_maddubs_width256_mr8", + "iq2xsq8_tile_gen": "dot_maddubs_width256_mr8", + "iq2xxsq8_tile_gen": "dot_maddubs_width256_mr8", + "iq3sq8_tile_gen": "dot_maddubs_width256_mr8", + "iq3xxsq8_tile_gen": "dot_maddubs_width256_mr8", + "iq4nlq8_tile_gen": "dot_maddubs_width256_mr8", + "iq4xsq8_tile_gen": "dot_maddubs_width256_mr8", + "k2q8_tile_gen": "dot_maddubs_width256_mr8", + "k3q8_tile_gen": "dot_maddubs_width256_mr8", + "k4q8_tile_gen": "dot_maddubs_width256_mr8", + "k5q8_tile_gen": "dot_maddubs_width256_mr8", + "k6q8_tile_gen": "dot_maddubs_width256_mr8", + "mul_inplace": "vec8_u2", + "q40q8_tile_gen": "dot_maddubs_width256_mr8", + "q51q8_tile_gen": "dot_maddubs_width256_mr8", + "q8q8_tile_gen": "dot_maddubs_width256_mr8_kstep2", + "quantize_q8_0_bs_into_ptr": "vec16_u2", + "quantize_q8_0_into_ptr": "plain", + "quantize_q8kv_row": "plain", + "quantize_tq4kv_row": "plain", + "rmsnorm": "vec16_u2", + "rope_scaled_neox_tab": "plain", + "scale_inplace": "vec8_u2", + "softmax": "vec8_u2", + "softmax_sink": "vec8_u2" + }, + "provenance": { + "arch": "x86_64", + "class": "x86-avx2", + "dasllama_version": "15", + "features": "avx2;f16c;fma;sse4.2", + "mode": "normal", + "noise": "ok", + "noise_floor_cv_pct": "0.33", + "noise_probes": "start cv 0.28%; mid1 cv 0.33%; mid2 cv 0.15%; end cv 0.31%", + "origin": "profile", + "platform": "windows", + "validation": "ok", + "validation_demoted": "0", + "validation_max_drift_pct": "19.46", + "written": "2026-09-01T01:06:53.218Z" + } +} diff --git a/modules/dasLLAMA/performance/defaults/x86-vnni512.tune-defaults.json b/modules/dasLLAMA/performance/defaults/x86-vnni512.tune-defaults.json index 53e07aee10..ebedf6abd3 100644 --- a/modules/dasLLAMA/performance/defaults/x86-vnni512.tune-defaults.json +++ b/modules/dasLLAMA/performance/defaults/x86-vnni512.tune-defaults.json @@ -1,70 +1,69 @@ { - "kernels" : { - "add_inplace" : "vec8_u2", - "cvt_f32_to_f16" : "plain", - "rope_scaled_neox_tab" : "vec8_u2", - "q51q8_tile_gen" : "dot_vpdpbusd_width512_mr16", - "softmax" : "vec8_u2", - "mul_inplace" : "vec8_u2", - "quantize_q8_0_bs_into_ptr" : "u2", - "iq2xsq8_tile_gen" : "dot_vpdpbusd_width512_mr16", - "dot_q8kv" : "vec8_u2", - "dot_q8q8" : "u2", - "iq4xsq8_tile_gen" : "dot_vpdpbusd_width512_mr16", - "iq4nlq8_tile_gen" : "dot_vpdpbusd_width512_mr16", - "quantize_q8kv_row" : "plain", - "axpy_f16" : "vec8_u2", - "q40q8_tile_gen" : "dot_vpdpbusd_width512_mr16", - "axpy_tq4kv" : "vec8_u2", - "cvt_tq4kv_to_f32" : "vec8_u2", - "axpy" : "vec8_u2", - "dot_q8q8kv" : "u2", - "dot_mx4q8" : "u2", - "softmax_sink" : "vec8_u2", - "iq2xxsq8_tile_gen" : "dot_vpdpbusd_width512_mr16", - "dot_q8q8_laneq4x4" : "", - "dot_bf16" : "vec8_u2", - "iq2sq8_tile_gen" : "dot_vpdpbusd_width512_mr16", - "add_scale_inplace" : "vec8_u2", - "cvt_q8kv_to_f32" : "vec8_u2", - "axpy_q8kv" : "plain", - "dot_q8q8_f16s" : "vec16", - "q8q8_tile_gen" : "dot_vpdpbusd_width512_mr16_kstep2_gkstep2_bias128", - "quantize_q8_0_into_ptr" : "plain", - "k4q8_tile_gen" : "dot_vpdpbusd_width512_mr16", - "k5q8_tile_gen" : "dot_vpdpbusd_width512_mr16", - "k6q8_tile_gen" : "dot_vpdpbusd_width512_mr16", - "k3q8_tile_gen" : "dot_vpdpbusd_width512_mr16", - "k2q8_tile_gen" : "dot_vpdpbusd_width512_mr16", - "gemm_f32_uk_4x16" : "u2", - "dot_q51e" : "vec16", - "dot_f16" : "vec16", - "cvt_f16_to_f32" : "vec8_u2", - "dot" : "vec16", - "iq3xxsq8_tile_gen" : "dot_vpdpbusd_width512_mr16", - "iq3sq8_tile_gen" : "dot_vpdpbusd_width512_mr16", - "dot_q8tq4kv" : "vec16", - "scale_inplace" : "vec8_u2", - "dot_q4" : "vec8_u2", - "quantize_tq4kv_row" : "plain", - "copy_floats" : "vec8_u2", - "rmsnorm" : "vec8" - }, - "provenance" : { - "validation" : "ok", - "noise_probes" : "start cv 0.09%; mid1 cv 0.04%; mid2 cv 0.21%; end cv 0.06%", - "platform" : "linux", - "noise_floor_cv_pct" : "0.21", - "features" : "avx2;f16c;fma;sse4.2;avx512f;avx512bw;avx512vl;avx512vnni", - "engine_sha" : "278b3c765", - "class" : "x86-vnni512", - "written" : "2026-09-01T01:30:04.021Z", - "validation_demoted" : "3", - "mode" : "normal", - "dasllama_version" : "15", - "origin" : "profile", - "noise" : "ok", - "arch" : "x86_64", - "validation_max_drift_pct" : "21.77" - } -} \ No newline at end of file + "kernels": { + "add_inplace": "vec8_u2", + "add_scale_inplace": "vec8_u2", + "axpy": "vec8_u2", + "axpy_f16": "vec8_u2", + "axpy_q8kv": "plain", + "axpy_tq4kv": "vec8_u2", + "copy_floats": "vec8_u2", + "cvt_f16_to_f32": "vec8_u2", + "cvt_f32_to_f16": "plain", + "cvt_q8kv_to_f32": "vec8_u2", + "cvt_tq4kv_to_f32": "vec8_u2", + "dot": "vec16", + "dot_bf16": "vec8_u2", + "dot_f16": "vec16", + "dot_mx4q8": "u2", + "dot_q4": "vec8_u2", + "dot_q51e": "vec16", + "dot_q8kv": "vec8_u2", + "dot_q8q8": "u2", + "dot_q8q8_f16s": "vec16", + "dot_q8q8_laneq4x4": "", + "dot_q8q8kv": "u2", + "dot_q8tq4kv": "vec16", + "gemm_f32_uk_4x16": "u2", + "iq2sq8_tile_gen": "dot_vpdpbusd_width512_mr16", + "iq2xsq8_tile_gen": "dot_vpdpbusd_width512_mr16", + "iq2xxsq8_tile_gen": "dot_vpdpbusd_width512_mr16", + "iq3sq8_tile_gen": "dot_vpdpbusd_width512_mr16", + "iq3xxsq8_tile_gen": "dot_vpdpbusd_width512_mr16", + "iq4nlq8_tile_gen": "dot_vpdpbusd_width512_mr16", + "iq4xsq8_tile_gen": "dot_vpdpbusd_width512_mr16", + "k2q8_tile_gen": "dot_vpdpbusd_width512_mr16", + "k3q8_tile_gen": "dot_vpdpbusd_width512_mr16", + "k4q8_tile_gen": "dot_vpdpbusd_width512_mr16", + "k5q8_tile_gen": "dot_vpdpbusd_width512_mr16", + "k6q8_tile_gen": "dot_vpdpbusd_width512_mr16", + "mul_inplace": "vec8_u2", + "q40q8_tile_gen": "dot_vpdpbusd_width512_mr16", + "q51q8_tile_gen": "dot_vpdpbusd_width512_mr16", + "q8q8_tile_gen": "dot_vpdpbusd_width512_mr16_kstep2_gkstep2_bias128", + "quantize_q8_0_bs_into_ptr": "u2", + "quantize_q8_0_into_ptr": "plain", + "quantize_q8kv_row": "plain", + "quantize_tq4kv_row": "plain", + "rmsnorm": "vec8", + "rope_scaled_neox_tab": "vec8_u2", + "scale_inplace": "vec8_u2", + "softmax": "vec8_u2", + "softmax_sink": "vec8_u2" + }, + "provenance": { + "arch": "x86_64", + "class": "x86-vnni512", + "dasllama_version": "15", + "features": "avx2;f16c;fma;sse4.2;avx512f;avx512bw;avx512vl;avx512vnni", + "mode": "normal", + "noise": "ok", + "noise_floor_cv_pct": "0.21", + "noise_probes": "start cv 0.09%; mid1 cv 0.04%; mid2 cv 0.21%; end cv 0.06%", + "origin": "profile", + "platform": "linux", + "validation": "ok", + "validation_demoted": "3", + "validation_max_drift_pct": "21.77", + "written": "2026-09-01T01:30:04.021Z" + } +} diff --git a/modules/dasLLAMA/tests/REVIEW.md b/modules/dasLLAMA/tests/REVIEW.md index 255e718346..9adc6a8098 100644 --- a/modules/dasLLAMA/tests/REVIEW.md +++ b/modules/dasLLAMA/tests/REVIEW.md @@ -8,11 +8,10 @@ the whole directory.** A change reaches a test when it alters anything the test' depends on - the test file, a shared helper, engine code it exercises, an in-tree fixture or corpus it reads, or a name it asserts on; a comment-only edit reaches none. -**Leaving a test file that no `run.das` model suite lists out of the `model-free` suite is a -defect, when its cells still assert what they claim under `DASLLAMA_CPU_PREFILL=1`.** The model -suites are every suite but `model-free`, and that variable is what the runner arms for -`model-free`. The listing lands in the same change that adds the file, and the file skips -honestly when its models are absent. +**Leaving a test file out of every `run.das` suite is a defect, unless the file's header +states why its cells cannot hold under `DASLLAMA_CPU_PREFILL=1`.** `DASLLAMA_CPU_PREFILL=1` is +what the runner arms for `model-free`. The listing lands in the same change that adds the +file, and the file skips honestly when its models are absent. **Listing a test file that `DASLLAMA_CPU_PREFILL=1` disarms in any `run.das` suite is a defect, and so is leaving that fact out of the file's header.** `DASLLAMA_CPU_PREFILL=1` is what the @@ -146,8 +145,9 @@ class a census row could dispatch is a defect. output - missing a compare against a CPU oracle that can witness the cell's property is a defect.** (A census row dispatches without asserting on output, so it is not one.) -**A kernel-unit cell that bit-compares a buffer the GPU writes prefills its output buffers with -a sentinel.** An unprefilled output can pass a bit compare by staying stale. +**A kernel-unit cell fills a GPU output buffer with a sentinel before every dispatch whose +output it then reads.** An unprefilled output can pass by staying stale - the previous +dispatch's values, or garbage that happens to sit inside the tolerance bar. **A cross-dispatch bit-identity compare - comparing the outputs of two dispatches - runs GPU against GPU.** No CPU oracle can witness that property. @@ -194,8 +194,7 @@ through a `.dlim`-baking loader (`load__tower` / `load__encoder` beside the model, and the next direct-image load in another suite panics on the wrong identity. The one residue: a cell whose SUBJECT is a facade lane knob (`load_asr_model` under `set_asr_tower_fp32`) keeps the facade loader - the image identity folds the pin, so -minting around it would unmake the claim; the purge exposure on a cold box is followup 61's -owed in-process image-off seam. +minting around it would unmake the claim. **A CPU-vs-GPU arm that does not run a PLANAR model for its CPU stages, and that model's `blob_twin(t, path, seq_cap)` for override-selected stages, is a defect.** One session spans @@ -207,8 +206,8 @@ model. tag is the token passed to `family_on(t, name)` (`_model_tier.das`). An untagged block silently joins every family's gate. -**No CPU-control batch parity runs against the 70B.** Its batch coverage is ENGAGE-only in the -support matrix. The batched code paths get their parity on small models, through pins. +**No CPU-control batch parity runs against the 70B.** The batched code paths get their parity +on small models, through pins. **Setting a knob a cell can reach only through the environment after the process that reads it starts is a defect - set it before that process starts.** That process is a child the cell @@ -245,10 +244,12 @@ exact fixtures. **An embedding-parity cell that does not name its fixture, or does not log the measured maxdiff on green as well as red, is a defect.** -**A new gate, or a new or loosened tolerance bar, ships a control that reds it - a poison, a -knockout, a disconnected mechanism, or a cross-lane witness - in the same change.** A bar -nothing has ever exceeded is not known to discriminate, and a gate that reads state the same -code path wrote can be a tautology - only the control proves either can fail. +**A new gate, or a new or loosened tolerance bar, ships a control that reds it in the same +change.** A control is a run of the same gate that must RED - a poisoned input, a poisoned +expectation, a disconnected mechanism, or a second independent lane; a gate's own reference is +never its control. A bar nothing has ever exceeded is not known to discriminate, and a gate +that reads state the same code path wrote can be a tautology - only the control proves either +can fail. **A family that gains a live thinking or tool format ships its recognition tests in the same change** - the wire-shape pins, the render pins, and a live server leg gated on the family's @@ -274,7 +275,8 @@ plane alone is a valid control there, while a route reading both planes needs bo poison the served route never reads passes on a broken kernel. **An ASR cell comparing transcripts across two serving lanes asserts TOKEN equality; the one -exception is the crowned-lane twin, whose ruled grade is WORD equality (the tensor twins' -rounding legitimately flips tokens - followup 60's resolution).** A cell that cannot hold +cell comparing a crowned lane - the raced kernel form a tune sidecar arms as the serving one - +against its tensor twin asserts WORD equality, because the twins' rounding legitimately flips +tokens.** A cell that cannot hold its grade converts to the forced-feed logits-tolerance form - never to a looser text compare. diff --git a/modules/dasLLAMA/tests/run.das b/modules/dasLLAMA/tests/run.das index fccaa00427..f9e5f06cf3 100644 --- a/modules/dasLLAMA/tests/run.das +++ b/modules/dasLLAMA/tests/run.das @@ -106,6 +106,7 @@ def private suite_files(name : string) : array { // nolint:STYLE038 - "modules/dasLLAMA/tests/test_kernel_backend.das", "modules/dasLLAMA/tests/test_kgroup_repack.das", "modules/dasLLAMA/tests/test_kqformat.das", + "modules/dasLLAMA/tests/test_kquant.das", "modules/dasLLAMA/tests/test_kv_prefix.das", "modules/dasLLAMA/tests/test_matmul.das", "modules/dasLLAMA/tests/test_matmul_batch.das", diff --git a/modules/dasLLAMA/tests/test_gguf_quant.das b/modules/dasLLAMA/tests/test_gguf_quant.das index f2a1445332..0f5dbafbe4 100644 --- a/modules/dasLLAMA/tests/test_gguf_quant.das +++ b/modules/dasLLAMA/tests/test_gguf_quant.das @@ -4,6 +4,7 @@ options _dasllama_internal = true require dastest/testing_boost public require dasllama/dasllama_convert // the codec layer under test — direct require pins the new module API require dasllama/dasllama_gguf +require dasllama/dasllama_common // dequant_kq_plane_sb - the plane dequant the f32 fallback must match require math // Byte-level GGUF quant-format gates on a synthetic in-memory file: the dequant arm must match an @@ -210,3 +211,63 @@ def test_q5k_encoder(t : T?) { delete ks } } + +// the superblock disk types the native tier reads: the f32 fallback decodes every one of them, +// value for value the way the native load does (transcode, then plane dequant). Random bytes +// are valid input for all nine (grid indices, sign bits and scale nibbles have no invalid values). +def private synth_sb_bytes(sbb : int64; seed : int) : array { + var b : array + b |> reserve(int(sbb * 2l)) + var x = uint(seed) * 2654435761u + 12345u + for (i in range(int(sbb * 2l))) { + x = x * 1664525u + 1013904223u + b |> push(uint8((x >> 24u) & 0xFFu)) + } + return <- b +} + +[test] +def test_gguf_f32_reader_superblock_types(t : T?) { + t |> run("gguf_read_tensor_f32 decodes every superblock i-quant / K-quant the native tier reads") <| @(t : T?) { + let types <- [GGML_TYPE_IQ4_XS, GGML_TYPE_Q3_K, GGML_TYPE_IQ3_S, GGML_TYPE_IQ3_XXS, GGML_TYPE_IQ4_NL, + GGML_TYPE_Q2_K, GGML_TYPE_IQ2_S, GGML_TYPE_IQ2_XS, GGML_TYPE_IQ2_XXS] + var ti = 0 + for (gtype in types) { + let sbb = gguf_kq_disk_sbb(gtype) + t |> success(sbb > 0l, "type {gtype} has a disk stride") + let fmt = gguf_kq_fmt_of_type(gtype) + let data <- synth_sb_bytes(sbb, 7 + ti) + let gg <- synth_gguf(gtype, 512l, data) + var m <- parse_gguf_meta(gg) + var deq : array + deq |> resize(512) + gguf_read_tensor_f32(m, gg, "t", deq, 0l, 512l) + // the oracle: the native two-step over the same bytes + var kq : array + var ks : array + kq |> resize(int(kq_qsb(fmt))) + ks |> resize(int(kq_ssb(fmt))) + var want : array + want |> resize(512) + let base = long_length(gg) - sbb * 2l + for (blk in range64(2l)) { + kq_transcode_superblock(fmt, gg, base + blk * sbb, kq, 0l, ks, 0l) + dequant_kq_plane_sb(fmt, kq, ks, 0l, false, want, blk * 256l) + } + var bad = 0 + for (i in range(512)) { + // random f16 scale bits can be NaN on both sides - a NaN pair is agreement + if (deq[i] != want[i] && !(deq[i] != deq[i] && want[i] != want[i])) { + bad++ + } + } + t |> equal(bad, 0, "type {gtype}: reader == transcode+dequant ({bad} of 512 differ)") + delete m + delete deq + delete want + delete kq + delete ks + ti++ + } + } +} diff --git a/modules/dasLLAMA/tests/test_kernel_coverage.das b/modules/dasLLAMA/tests/test_kernel_coverage.das index 88eabaa228..55d1a10cf2 100644 --- a/modules/dasLLAMA/tests/test_kernel_coverage.das +++ b/modules/dasLLAMA/tests/test_kernel_coverage.das @@ -69,7 +69,21 @@ let private CENSUS_NEVER_DISPATCHED <- [ // the dense tall in-kernel-dequant kq stamps: they engage only when a site's f16 panel // would reach TALLKQ_MIN_PANEL (96 MiB) - the zoo's small carriers top out far below it. // Their coverage is the tall kernel-unit leg in test_metal_gemm_kernels.das (kq_mulmm_gate) - "MetalKqMulMmK4TH128", "MetalKqMulMmK5TH128", "MetalKqMulMmK6TH128" + "MetalKqMulMmK4TH128", "MetalKqMulMmK5TH128", "MetalKqMulMmK6TH128", + // the nine formats the iquant arc added (Q2_K, Q3_K, IQ4_XS, IQ4_NL, IQ3_S, IQ3_XXS, IQ2_S, + // IQ2_XS, IQ2_XXS): no stocked census carrier holds a plane in any of them - the zoo quants + // are Q8_0 / Q5_K_M / Q4_K_M / mxfp4. Their coverage is the kernel-unit gates in + // test_metal_gemv_kernels.das / test_metal_gemm_kernels.das and the per-format e2e parity + // rows recorded in HOW_TO_ADD_A_FORMAT.md + "MetalKqGemvK2", "MetalKqMvB2K2", "MetalKqMvB4K2", "MetalKqMvB8K2", "MetalKqMulMmK2", + "MetalKqGemvK3", "MetalKqMvB2K3", "MetalKqMvB4K3", "MetalKqMvB8K3", "MetalKqMulMmK3", + "MetalKqGemvIq4xs", "MetalKqMvB2Iq4xs", "MetalKqMvB4Iq4xs", "MetalKqMvB8Iq4xs", "MetalKqMulMmIq4xs", + "MetalKqGemvIq4nl", "MetalKqMvB2Iq4nl", "MetalKqMvB4Iq4nl", "MetalKqMvB8Iq4nl", "MetalKqMulMmIq4nl", + "MetalKqGemvIq3s", "MetalKqMvB2Iq3s", "MetalKqMvB4Iq3s", "MetalKqMvB8Iq3s", "MetalKqMulMmIq3s", + "MetalKqGemvIq3xxs", "MetalKqMvB2Iq3xxs", "MetalKqMvB4Iq3xxs", "MetalKqMvB8Iq3xxs", "MetalKqMulMmIq3xxs", + "MetalKqGemvIq2s", "MetalKqMvB2Iq2s", "MetalKqMvB4Iq2s", "MetalKqMvB8Iq2s", "MetalKqMulMmIq2s", + "MetalKqGemvIq2xs", "MetalKqMvB2Iq2xs", "MetalKqMvB4Iq2xs", "MetalKqMvB8Iq2xs", "MetalKqMulMmIq2xs", + "MetalKqGemvIq2xxs", "MetalKqMvB2Iq2xxs", "MetalKqMvB4Iq2xxs", "MetalKqMvB8Iq2xxs", "MetalKqMulMmIq2xxs" ] def private counting_prompt(start, last : int) : string { diff --git a/modules/dasLLAMA/tests/test_kqformat.das b/modules/dasLLAMA/tests/test_kqformat.das index c2aec80cc3..6037f39254 100644 --- a/modules/dasLLAMA/tests/test_kqformat.das +++ b/modules/dasLLAMA/tests/test_kqformat.das @@ -9,6 +9,7 @@ options _dasllama_internal = true require dastest/testing_boost public require dasllama/dasllama_kqformat +require dasllama/dasllama_common // kq_fmt_row_ok - the dense-row demotion require dasllama/dasllama_gemm_schema // the int-id delegation under test [test] @@ -120,7 +121,7 @@ def test_kqfmt_strides(t : T?) { t |> equal(IQ3S_QSB, 104l) t |> equal(IQ3S_SSB, 20l) } - t |> run("the IQ4_NL codebook is ggml's kvalues_iq4nl") @(t : T?) { + t |> run("the IQ4_NL codebook is upstream's kvalues_iq4nl") @(t : T?) { t |> equal(int(IQ4NL_LUT[0]), -127) t |> equal(int(IQ4NL_LUT[7]), -10) t |> equal(int(IQ4NL_LUT[8]), 1) @@ -133,7 +134,7 @@ def test_kqfmt_strides(t : T?) { [test] def test_kqfmt_grid(t : T?) { - t |> run("the IQ3_S grid is ggml's iq3s_grid, and the worker-safe local agrees") @(t : T?) { + t |> run("the IQ3_S grid is upstream's iq3s_grid, and the worker-safe local agrees") @(t : T?) { t |> equal(IQ3S_GRID[0], 0x01010101u) t |> equal(IQ3S_GRID[511], 0x0f0f0101u) let lut = iq3s_grid() @@ -233,3 +234,17 @@ def test_iq3xxs_tables(t : T?) { t |> equal(f16_half_bits(0x0400u), 0x0200u) } } + +[test] +def test_kq_fmt_row_demotion(t : T?) { + t |> run("the 32-block disk formats demote to q8 on rows the superblock walkers cannot tile") <| @(t : T?) { + // 704 = 22 blocks of 32, not a 256-multiple: q40, iq4nl and Q5_0-born k5 all fall to q8 + t |> equal(kq_fmt_row_ok(KqFmt.q40, 704l), KqFmt.q8) + t |> equal(kq_fmt_row_ok(KqFmt.iq4nl, 704l), KqFmt.q8) + t |> equal(kq_fmt_row_ok(KqFmt.k5, 704l), KqFmt.q8) + t |> equal(kq_fmt_row_ok(KqFmt.iq4nl, 768l), KqFmt.iq4nl) + t |> equal(kq_fmt_row_ok(KqFmt.q40, 768l), KqFmt.q40) + // the 256-superblock disk formats never see a non-multiple row (the file cannot hold one) + t |> equal(kq_fmt_row_ok(KqFmt.iq2xxs, 768l), KqFmt.iq2xxs) + } +} diff --git a/modules/dasLLAMA/tests/test_kquant.das b/modules/dasLLAMA/tests/test_kquant.das index af7308aed6..4a691ed78f 100644 --- a/modules/dasLLAMA/tests/test_kquant.das +++ b/modules/dasLLAMA/tests/test_kquant.das @@ -1,13 +1,14 @@ options gen2 options stack = 524288 // every dasLLAMA program root takes this budget (options stack does not unify up from libs) options persistent_heap // the layout arm loads a multi-GB model; explicit delete below -options _cyclomatic_complexity = 76 // every gate here is the same flat fmt == 4/5/6/40/44/3 ladder repeated per stage +options _cyclomatic_complexity = 76 // every gate here is the same flat per-format ladder (thirteen formats) repeated per stage options _function_length = 300 // and each gate carries its whole fixture build + oracle inline options _dasllama_internal = true require dastest/testing_boost public require _model_tier require dasllama/dasllama_gguf +require dasllama/dasllama_common // dequant_kq_plane_sb - the carrier-form dispatcher require dasllama/dasllama_convert require dasllama/dasllama_math // pin_kernel_backend — the layout arm needs disk-order planes require dasllama/dasllama_math_default // dot_k4q8/k5/k6/q40q8 — the kernel arms gate them vs the plane dequant @@ -517,7 +518,7 @@ def test_kq_transcode_planes(t : T?) { dequant_iq4xs_plane_superblock(kq, 0l, ks, 0l, dst, 0l) for (k in range(256)) { let expected = (SYNTH_D * float(SYNTH_SC[k / 32] - 32)) * float(IQ4NL_LUT[q4_pat(k)]) - t |> success(dst[k] == expected, "iq4xs plane element must match the ggml dequant exactly") + t |> success(dst[k] == expected, "iq4xs plane element must match the upstream dequant exactly") } } t |> run("k3 planes decode the packed 6-bit sub-scales and compose the 3-bit quants exactly") @(t : T?) { @@ -535,7 +536,7 @@ def test_kq_transcode_planes(t : T?) { dequant_k3_plane_superblock(kq, 0l, ks, 0l, dst, 0l) for (k in range(256)) { let expected = (SYNTH_D * float(SYNTH_SC[(k / 16) % 8] - 32)) * float(q3_pat(k) - 4) - t |> success(dst[k] == expected, "k3 plane element must match the ggml dequant exactly") + t |> success(dst[k] == expected, "k3 plane element must match the upstream dequant exactly") } } t |> run("iq3s planes decode the 4-bit sub-scales and gather the grid exactly") @(t : T?) { @@ -559,7 +560,7 @@ def test_kq_transcode_planes(t : T?) { let g = int((IQ3S_GRID[qi] >> uint(8 * (hj % 4))) & 0xFFu) let sgn = ((iq3s_sg_pat(b * 4 + l) >> hj) & 1) != 0 ? -1.0 : 1.0 let expected = (SYNTH_D * float(1 + 2 * (SYNTH_SC[b] & 15))) * float(g) * sgn - t |> success(dst[k] == expected, "iq3s plane element must match the ggml dequant exactly") + t |> success(dst[k] == expected, "iq3s plane element must match the upstream dequant exactly") } } t |> run("k2 planes reorder the header and compose the 2-bit quants exactly") @(t : T?) { @@ -575,7 +576,7 @@ def test_kq_transcode_planes(t : T?) { for (k in range(256)) { let g = k / 16 let expected = SYNTH_D * float(SYNTH_SC[g % 8] & 15) * float(q2_pat(k)) - SYNTH_DMIN * float(SYNTH_MN[g % 8] & 15) - t |> success(dst[k] == expected, "k2 plane element must match the ggml dequant exactly") + t |> success(dst[k] == expected, "k2 plane element must match the upstream dequant exactly") } } t |> run("iq2xs planes eighth the d, decode the strips, and gather the 512-entry grid + ksigns exactly") @(t : T?) { @@ -601,7 +602,7 @@ def test_kq_transcode_planes(t : T?) { let g = int((gw >> uint(8 * (j % 4))) & 0xFFu) let sgn = ((int(KSIGNS_IQ2XS[iq2xs_sg7_pat(b * 4 + l)]) >> j) & 1) != 0 ? -1.0 : 1.0 let expected = (d8 * float(1 + 2 * (SYNTH_SC[(2 * b + l / 2) % 8] & 15))) * float(g) * sgn - t |> success(dst[k] == expected, "iq2xs plane element must match the ggml dequant exactly") + t |> success(dst[k] == expected, "iq2xs plane element must match the upstream dequant exactly") } } t |> run("iq2xxs planes eighth the d, decode the per-32 strips, and gather the 256-entry grid + ksigns exactly") @(t : T?) { @@ -627,7 +628,7 @@ def test_kq_transcode_planes(t : T?) { let g = int((gw >> uint(8 * (j % 4))) & 0xFFu) let sgn = ((int(KSIGNS_IQ2XS[iq2xxs_sg7_pat(b * 4 + l)]) >> j) & 1) != 0 ? -1.0 : 1.0 let expected = (d8 * float(1 + 2 * (SYNTH_SC[b % 8] & 15))) * float(g) * sgn - t |> success(dst[k] == expected, "iq2xxs plane element must match the ggml dequant exactly") + t |> success(dst[k] == expected, "iq2xxs plane element must match the upstream dequant exactly") } } t |> run("iq2s planes eighth the d, decode the strips, and gather the u64 grid exactly") @(t : T?) { @@ -653,7 +654,7 @@ def test_kq_transcode_planes(t : T?) { let g = int((gw >> uint(8 * (j % 4))) & 0xFFu) let sgn = ((iq2s_sg_pat(b * 4 + l) >> j) & 1) != 0 ? -1.0 : 1.0 let expected = (d8 * float(1 + 2 * (SYNTH_SC[(2 * b + l / 2) % 8] & 15))) * float(g) * sgn - t |> success(dst[k] == expected, "iq2s plane element must match the ggml dequant exactly") + t |> success(dst[k] == expected, "iq2s plane element must match the upstream dequant exactly") } } t |> run("q51 planes unpack the hand-packed Q5_1 blocks exactly") @(t : T?) { @@ -1528,7 +1529,7 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { // nolint:STYL // k5/k6 tiles read the byte-expanded panel the batch cell unpacks per (group, token-block) // — the gate covers unpack_kq_panel_grp too (tile-over-panel vs gemv-over-packed planes); // k4/q40 tiles read the packed planes directly - let packed = fmt == 4 || fmt == 40 || fmt == 44 || fmt == 3 || fmt == 45 || fmt == 2 // iq3s/iq3xxs/iq2s/iq2xs tiles read the byte-expanded panel (the batch cell's shape) + let packed = kq_reads_packed_planes(int64(fmt)) var panel : array if (!packed) { panel |> resize(mr * nsb * 256l) @@ -2493,3 +2494,73 @@ def test_iq4nl_planes(t : T?) { delete dst } } + +// the Metal-blob carrier splits a format's scale plane into [16B strips of every superblock] +// [the d tails] (ARCHITECTURE_GPU.md sec.2.2y); this rebuilds that form from the CPU row form +// the way metal_blob_scale_plane does for the 20B rows: [f16 d][2 pad][16B strips] +def private split_scale_plane(ks : array; ssb : int64) : array { + let nsb = long_length(ks) / ssb + var out : array + out |> resize(int(nsb * 18l)) + for (sb in range64(nsb)) { + for (i in range64(16l)) { + out[sb * 16l + i] = ks[sb * ssb + 4l + i] + } + out[nsb * 16l + sb * 2l] = ks[sb * ssb] + out[nsb * 16l + sb * 2l + 1l] = ks[sb * ssb + 1l] + } + return <- out +} + +[test] +def test_kq_plane_sb_metal_blob_form(t : T?) { + t |> run("dequant_kq_plane_sb reads the Metal-blob split scale plane to the same values as the row form") <| @(t : T?) { + // one superblock per format from random disk bytes: transcode, dequant the row form, then + // dequant the split form through the metal_blob arm - the values must agree exactly + let gtypes <- [GGML_TYPE_IQ2_XXS, GGML_TYPE_IQ2_XS, GGML_TYPE_IQ2_S, GGML_TYPE_IQ3_XXS, GGML_TYPE_IQ3_S, GGML_TYPE_IQ4_XS] + var seed = 11 + for (gtype in gtypes) { + let fmt = gguf_kq_fmt_of_type(gtype) + let sbb = gguf_kq_disk_sbb(gtype) + var bytes : array + bytes |> reserve(int(sbb * 3l)) + var x = uint(seed) * 2654435761u + for (i in range(int(sbb * 3l))) { + x = x * 1664525u + 1013904223u + bytes |> push(uint8((x >> 24u) & 0xFFu)) + } + let qsb = kq_qsb(fmt) + let ssb = kq_ssb(fmt) + var kq : array + var ks : array + kq |> resize(int(qsb * 3l)) + ks |> resize(int(ssb * 3l)) + for (sb in range64(3l)) { + kq_transcode_superblock(fmt, bytes, sb * sbb, kq, sb * qsb, ks, sb * ssb) + } + var rowform : array + var blobform : array + rowform |> resize(768) + blobform |> resize(768) + let split <- split_scale_plane(ks, ssb) + for (sb in range64(3l)) { + dequant_kq_plane_sb(fmt, kq, ks, sb, false, rowform, sb * 256l) + dequant_kq_plane_sb(fmt, kq, split, sb, true, blobform, sb * 256l) + } + var bad = 0 + for (i in range(768)) { + // random f16 scale bits can be NaN on both sides - a NaN pair is agreement + if (rowform[i] != blobform[i] && !(rowform[i] != rowform[i] && blobform[i] != blobform[i])) { + bad++ + } + } + t |> equal(bad, 0, "{fmt}: split form == row form ({bad} of 768 differ)") + delete bytes + delete kq + delete ks + delete rowform + delete blobform + seed++ + } + } +} diff --git a/modules/dasLLAMA/tests/test_model_image.das b/modules/dasLLAMA/tests/test_model_image.das index e10a682c12..b10d90b371 100644 --- a/modules/dasLLAMA/tests/test_model_image.das +++ b/modules/dasLLAMA/tests/test_model_image.das @@ -2379,3 +2379,16 @@ def test_layout_fingerprint(t : T?) { remove(path) } } + +[test] +def test_dlim_identity_pack_version(t : T?) { + t |> run("the pack-code version folds into the identity and nothing else moves") <| @(t : T?) { + var c = DlimConfiguration() + let base = dlim_identity(c, IMAGE_VERSION, "", 0) + let packed = dlim_identity(c, IMAGE_VERSION, "", 7) + t |> success(base != packed, "pack_version 7 changes the identity") + t |> success(packed |> starts_with("v{IMAGE_VERSION}p7|"), "the pack version rides the schema version: {packed}") + t |> success(base |> starts_with("v{IMAGE_VERSION}|"), "pack_version 0 is the pre-versioned spelling: {base}") + t |> equal(image_identity_of(c), dlim_identity(c, IMAGE_VERSION, "", PACK_VERSION), "image_identity_of folds the layout's PACK_VERSION") + } +} diff --git a/modules/dasLLAMA/tests/test_vulkan_kernels.das b/modules/dasLLAMA/tests/test_vulkan_kernels.das index f6a9302a73..cb7a23720e 100644 --- a/modules/dasLLAMA/tests/test_vulkan_kernels.das +++ b/modules/dasLLAMA/tests/test_vulkan_kernels.das @@ -379,10 +379,10 @@ def private attn_row_oracle(q : array; qbase : int; kk : array; kb [test] def test_vkd_kq_gemv_family(t0 : T?) { - t0 |> run("kq GEMV class family == the CPU oracle (all 6 formats)") <| @(t : T?) { + t0 |> run("kq GEMV class family == the CPU oracle (all 13 formats)") <| @(t : T?) { static_if (typeinfo builtin_module_exists(vulkan)) { if (!(ensure_kq_gemv_k4_cls() && ensure_kq_gemv_q40_cls() && ensure_kq_gemv_k5_cls() && ensure_kq_gemv_k6_cls() && ensure_kq_gemv_iq4xs_cls() && ensure_kq_gemv_k3_cls() && ensure_kq_gemv_iq3s_cls() && ensure_kq_gemv_iq3xxs_cls() && ensure_kq_gemv_iq4nl_cls() && ensure_kq_gemv_k2_cls() && ensure_kq_gemv_iq2s_cls() && ensure_kq_gemv_iq2xs_cls() && ensure_kq_gemv_iq2xxs_cls())) { - feint("no Vulkan device - skipping\n") + t |> skip("no Vulkan device, or a kq GEMV class pipeline failed to build") return } let nsb = 2 // superblocks per row reduction (n = 512) @@ -1051,10 +1051,10 @@ def test_vkd_q8_batch_tile(t0 : T?) { [test] def test_vkd_kq_batch_family(t0 : T?) { - t0 |> run("kq batch tile class family == the CPU oracle (all 6 formats)") <| @(t : T?) { + t0 |> run("kq batch tile class family == the CPU oracle (all 13 formats)") <| @(t : T?) { static_if (typeinfo builtin_module_exists(vulkan)) { if (!(ensure_kq_batch_k4_cls() && ensure_kq_batch_q40_cls() && ensure_kq_batch_k5_cls() && ensure_kq_batch_k6_cls() && ensure_kq_batch_iq4xs_cls() && ensure_kq_batch_k3_cls() && ensure_kq_batch_iq3s_cls() && ensure_kq_batch_iq3xxs_cls() && ensure_kq_batch_iq4nl_cls() && ensure_kq_batch_k2_cls() && ensure_kq_batch_iq2s_cls() && ensure_kq_batch_iq2xs_cls() && ensure_kq_batch_iq2xxs_cls())) { - feint("no Vulkan device - skipping\n") + t |> skip("no Vulkan device, or a kq batch class pipeline failed to build") return } let n = 512 // nsb = 2 superblocks @@ -1821,6 +1821,10 @@ def test_vkd_k5_cm2_batch(t0 : T?) { let bad = mismatch_bars(y_cls, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) to_log(LOG_INFO, "cm2 k5 {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile batch: {bad} of {rows * d} off the oracle\n") t |> success(bad == 0, "cm2 k5 {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile matches the CPU oracle ({bad} of {rows * d} off)") + var y_poison := y_cls // the bar's control: one element pushed past both bars must red + y_poison[0] += 1.0 + 2.0 * max_abs(y_ref) + t |> success(mismatch_bars(y_poison, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) > 0, "the k5 bar reds a poisoned element") + delete y_poison delete sched } delete y_ref @@ -1945,6 +1949,10 @@ def test_vkd_q40_cm2_batch(t0 : T?) { let bad = mismatch_bars(y_cls, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) to_log(LOG_INFO, "cm2 q40 {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile batch: {bad} of {rows * d} off the oracle\n") t |> success(bad == 0, "cm2 q40 {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile matches the CPU oracle ({bad} of {rows * d} off)") + var y_poison := y_cls // the bar's control: one element pushed past both bars must red + y_poison[0] += 1.0 + 2.0 * max_abs(y_ref) + t |> success(mismatch_bars(y_poison, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) > 0, "the q40 bar reds a poisoned element") + delete y_poison delete sched } delete y_ref @@ -2068,6 +2076,10 @@ def test_vkd_k2_cm2_batch(t0 : T?) { let bad = mismatch_bars(y_cls, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) to_log(LOG_INFO, "cm2 k2 {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile batch: {bad} of {rows * d} off the oracle\n") t |> success(bad == 0, "cm2 k2 {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile matches the CPU oracle ({bad} of {rows * d} off)") + var y_poison := y_cls // the bar's control: one element pushed past both bars must red + y_poison[0] += 1.0 + 2.0 * max_abs(y_ref) + t |> success(mismatch_bars(y_poison, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) > 0, "the k2 bar reds a poisoned element") + delete y_poison delete sched } delete y_ref @@ -2192,6 +2204,10 @@ def test_vkd_iq4nl_cm2_batch(t0 : T?) { let bad = mismatch_bars(y_cls, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) to_log(LOG_INFO, "cm2 iq4nl {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile batch: {bad} of {rows * d} off the oracle\n") t |> success(bad == 0, "cm2 iq4nl {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile matches the CPU oracle ({bad} of {rows * d} off)") + var y_poison := y_cls // the bar's control: one element pushed past both bars must red + y_poison[0] += 1.0 + 2.0 * max_abs(y_ref) + t |> success(mismatch_bars(y_poison, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) > 0, "the iq4nl bar reds a poisoned element") + delete y_poison delete sched } delete y_ref @@ -2316,6 +2332,10 @@ def test_vkd_iq4xs_cm2_batch(t0 : T?) { let bad = mismatch_bars(y_cls, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) to_log(LOG_INFO, "cm2 iq4xs {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile batch: {bad} of {rows * d} off the oracle\n") t |> success(bad == 0, "cm2 iq4xs {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile matches the CPU oracle ({bad} of {rows * d} off)") + var y_poison := y_cls // the bar's control: one element pushed past both bars must red + y_poison[0] += 1.0 + 2.0 * max_abs(y_ref) + t |> success(mismatch_bars(y_poison, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) > 0, "the iq4xs bar reds a poisoned element") + delete y_poison delete sched } delete y_ref @@ -2439,6 +2459,10 @@ def test_vkd_k3_cm2_batch(t0 : T?) { let bad = mismatch_bars(y_cls, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) to_log(LOG_INFO, "cm2 k3 {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile batch: {bad} of {rows * d} off the oracle\n") t |> success(bad == 0, "cm2 k3 {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile matches the CPU oracle ({bad} of {rows * d} off)") + var y_poison := y_cls // the bar's control: one element pushed past both bars must red + y_poison[0] += 1.0 + 2.0 * max_abs(y_ref) + t |> success(mismatch_bars(y_poison, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) > 0, "the k3 bar reds a poisoned element") + delete y_poison delete sched } delete y_ref @@ -2563,6 +2587,10 @@ def test_vkd_iq3s_cm2_batch(t0 : T?) { let bad = mismatch_bars(y_cls, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) to_log(LOG_INFO, "cm2 iq3s {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile batch: {bad} of {rows * d} off the oracle\n") t |> success(bad == 0, "cm2 iq3s {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile matches the CPU oracle ({bad} of {rows * d} off)") + var y_poison := y_cls // the bar's control: one element pushed past both bars must red + y_poison[0] += 1.0 + 2.0 * max_abs(y_ref) + t |> success(mismatch_bars(y_poison, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) > 0, "the iq3s bar reds a poisoned element") + delete y_poison delete sched } delete y_ref @@ -2687,6 +2715,10 @@ def test_vkd_iq2s_cm2_batch(t0 : T?) { let bad = mismatch_bars(y_cls, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) to_log(LOG_INFO, "cm2 iq2s {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile batch: {bad} of {rows * d} off the oracle\n") t |> success(bad == 0, "cm2 iq2s {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile matches the CPU oracle ({bad} of {rows * d} off)") + var y_poison := y_cls // the bar's control: one element pushed past both bars must red + y_poison[0] += 1.0 + 2.0 * max_abs(y_ref) + t |> success(mismatch_bars(y_poison, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) > 0, "the iq2s bar reds a poisoned element") + delete y_poison delete sched } delete y_ref @@ -2811,6 +2843,10 @@ def test_vkd_iq2xs_cm2_batch(t0 : T?) { let bad = mismatch_bars(y_cls, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) to_log(LOG_INFO, "cm2 iq2xs {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile batch: {bad} of {rows * d} off the oracle\n") t |> success(bad == 0, "cm2 iq2xs {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile matches the CPU oracle ({bad} of {rows * d} off)") + var y_poison := y_cls // the bar's control: one element pushed past both bars must red + y_poison[0] += 1.0 + 2.0 * max_abs(y_ref) + t |> success(mismatch_bars(y_poison, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) > 0, "the iq2xs bar reds a poisoned element") + delete y_poison delete sched } delete y_ref @@ -2935,6 +2971,10 @@ def test_vkd_iq2xxs_cm2_batch(t0 : T?) { let bad = mismatch_bars(y_cls, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) to_log(LOG_INFO, "cm2 iq2xxs {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile batch: {bad} of {rows * d} off the oracle\n") t |> success(bad == 0, "cm2 iq2xxs {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile matches the CPU oracle ({bad} of {rows * d} off)") + var y_poison := y_cls // the bar's control: one element pushed past both bars must red + y_poison[0] += 1.0 + 2.0 * max_abs(y_ref) + t |> success(mismatch_bars(y_poison, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) > 0, "the iq2xxs bar reds a poisoned element") + delete y_poison delete sched } delete y_ref @@ -3059,6 +3099,10 @@ def test_vkd_iq3xxs_cm2_batch(t0 : T?) { let bad = mismatch_bars(y_cls, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) to_log(LOG_INFO, "cm2 iq3xxs {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile batch: {bad} of {rows * d} off the oracle\n") t |> success(bad == 0, "cm2 iq3xxs {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile matches the CPU oracle ({bad} of {rows * d} off)") + var y_poison := y_cls // the bar's control: one element pushed past both bars must red + y_poison[0] += 1.0 + 2.0 * max_abs(y_ref) + t |> success(mismatch_bars(y_poison, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) > 0, "the iq3xxs bar reds a poisoned element") + delete y_poison delete sched } delete y_ref diff --git a/modules/dasLLVM/ARCHITECTURE.md b/modules/dasLLVM/ARCHITECTURE.md index 3e1548490c..1894290013 100644 --- a/modules/dasLLVM/ARCHITECTURE.md +++ b/modules/dasLLVM/ARCHITECTURE.md @@ -106,7 +106,9 @@ The backend's override knobs - the escapes that change what a run compiles, tune beyond its defaults - are: `DAS_TUNE_POLICY` (replaces the declared/injected tune policy), `DAS_TUNE_MODE` (grid/tuner compile modes), `DAS_TUNE_MANIFEST` (pins the sidecar), `DAS_TUNE_NOISE_CV` (recalibrates the tuner noise gate), `DAS_TUNE_NOISE_OVERRIDE` (mints -through a failing gate), `--tune` (forced re-mint), `--jit-obj-cache=0` (forces every split +through a failing gate), `--tune` (forced re-mint), `--tune-only` / `DAS_TUNE_ONLY` (re-mints +only the named families; the policy guard arms it itself for a profile's residue), +`DAS_TUNE_CONTROL` (a supervisor's stop request - tuners abort between families), `--jit-obj-cache=0` (forces every split partition to re-emit, bypassing the obj cache), `DAS_JIT_PROBE_LTO` (split partitions emit bitcode and the link runs lld LTO - a dev probe artifact), `DAS_JIT_X64_FORCE_FEATURES` / `DAS_JIT_ARM64_FORCE_FEATURES` (force CPU features past detection - emission, the cache keys, @@ -135,7 +137,8 @@ have no instruction to select and codegen aborts. `cpu_supports` reads the opera instead (sysctl / `AT_HWCAP` / `IsProcessorFeaturePresent`), so it answers for silicon LLVM has never heard of. Both the tier gates (`init_jit_target_flags`) and the target machine's feature string (`create_default_target_machine`) therefore take the union of the two: an LLVM host-string -hit OR a `cpu_supports` hit. A cross-compile triple takes neither - only the force env. +hit OR a `cpu_supports` hit (fullfp16 additionally reads darwin-arm64 as always-on - every +Apple Silicon part has it). A cross-compile triple takes neither - only the force env. The two ways a feature reaches the target machine's string license different things. A detection-derived append - `+dotprod` always, `+i8mm` when `cpu_supports` confirms it - is diff --git a/modules/dasLLVM/ENVIRONMENT.md b/modules/dasLLVM/ENVIRONMENT.md index 08d83958c1..ceed549549 100644 --- a/modules/dasLLVM/ENVIRONMENT.md +++ b/modules/dasLLVM/ENVIRONMENT.md @@ -35,7 +35,7 @@ Loaded once at context init into `g_env_tune`; tuner children inherit the enviro | `DAS_TUNE_NOISE_CV` | number | 2 | The noise probe's note threshold in percent (a louder probe stamps the mint noisy; refusal sits at max(10, this)) - a calibration lever. Garbage or <= 0 reads as unset. | | `DAS_TUNE_NOISE_OVERRIDE` | flag | off | Mint through the busy-box refusal (a probe cv past the hard ceiling). The sidecar is stamped noise overridden, so the escape always leaves a mark. Validation never refuses - a winner that does not hold is demoted per kernel. | | `DAS_TUNE_VERBOSITY` | text | normal | What a tune shows: silent, normal, or verbose. Anything unrecognized reads as normal, so a typo never silences a tune. | -| `DAS_TUNE_POLICY` | text | declared by [tune_policy] | Override the missing-scope policy: fallback, warn, error, auto, or restart. The announce line says when the environment shaped the policy. | +| `DAS_TUNE_POLICY` | text | declared by [tune_policy] | Override the missing-scope policy: fallback, reference, warn, error, auto, or restart. The announce line says when the environment shaped the policy. | | `DAS_TUNE_CONTROL` | path | unset | A supervisor's stop channel: while the named file exists, tune_interrupt_requested() is true and tuners abort at the next kernel-family boundary without minting. The watchdog sets it and owns the file's lifetime; the measurement in flight always completes. | | `DAS_TUNE_ONLY` | text | unset (every family races) | Comma-separated re-mint filter: a tuner races only kernel families whose name contains one of these tokens, and every skipped family's sidecar entry survives the upsert. Set by --tune-only on the application; the tuner children inherit it. | diff --git a/modules/dasLLVM/REVIEW.md b/modules/dasLLVM/REVIEW.md index 4dd29c0620..57a44b17ab 100644 --- a/modules/dasLLVM/REVIEW.md +++ b/modules/dasLLVM/REVIEW.md @@ -7,12 +7,14 @@ `tests/README.md` here). The suite is outside the core `tests/` sweep, so no other lane covers it. -- **A change gated on the host platform - a `get_platform_name()` test - runs the - module-owned suite on that platform.** +- **A change whose behavior depends on the host - a branch on `get_platform_name()`, + `get_architecture_name()`, or `cpu_supports()` - runs the module-owned suite on a machine + matching that condition.** -- **A diff whose new behavior sits under a target-triple branch - one it adds or one already - there - states in its PR body the cross-compile (`write_exe`) for that target that exercised - the new behavior.** The suite runs on the host; a target-triple branch is checked only by +- **A diff whose new behavior sits under a code path taken only when a target triple is set - + a cross-compile, not the host - whether the diff adds the path or finds it there, states + in its PR body the cross-compile (`write_exe`) for that target that exercised the new + behavior.** The suite runs on the host; a target-triple branch is checked only by the artifact built for that target. - **A diff that adds work to, or moves work within, what `run_jit` @@ -60,11 +62,4 @@ the declared forms, or declare the knob, instead** (the literal-name forms are scanner-enforced by `tests/llvm_env_registry.das`; weakening that test is a defect). -- **A diff that lets a `requires=` expression name a new CPU feature adds that name to - `TUNE_KNOWN_FEATURES` (`daslib/llvm_tune.das`) in the same change.** Every sidecar save - stamps this box's pass/fail over that list as the `features` fingerprint, so a name - missing from it makes a shipped defaults profile claim a seat its minting box never raced. -- **A diff that arms `DAS_TUNE_ONLY` for a single tuner spawn clears it immediately after - that spawn.** The auto/restart policy guard re-execs the process, and a filter left armed - makes the re-exec's own mint partial. diff --git a/modules/dasLLVM/daslib/llvm_env.das b/modules/dasLLVM/daslib/llvm_env.das index 1b981e4501..96c2fea95d 100644 --- a/modules/dasLLVM/daslib/llvm_env.das +++ b/modules/dasLLVM/daslib/llvm_env.das @@ -67,7 +67,7 @@ struct public TuneEnv { tune_verbosity : string = "" @clarg_default_doc = "declared by [tune_policy]" - @clarg_doc = "Override the missing-scope policy: fallback, warn, error, auto, or restart. The announce line says when the environment shaped the policy." + @clarg_doc = "Override the missing-scope policy: fallback, reference, warn, error, auto, or restart. The announce line says when the environment shaped the policy." tune_policy : string = "" @clarg_path diff --git a/modules/dasLLVM/daslib/llvm_jit_common.das b/modules/dasLLVM/daslib/llvm_jit_common.das index f28fd5724e..5faf21bebc 100644 --- a/modules/dasLLVM/daslib/llvm_jit_common.das +++ b/modules/dasLLVM/daslib/llvm_jit_common.das @@ -444,7 +444,7 @@ var public g_target_x64_amx = false var public g_target_os_linux = false // ARMv8.6 i8mm (SMMLA — 2×2 s8 matrix-multiply-accumulate; Apple M2/A15+, Graviton3+; M1 has -// DotProd only). Host truth is LLVMGetHostCPUFeatures "+i8mm" OR cpu_supports (ARCHITECTURE.md sec.4). +// DotProd only). Host truth is LLVMGetHostCPUFeatures "+i8mm" OR cpu_supports (ARCHITECTURE.md#aarch64-feature-truth). var public g_target_arm64_i8mm = false // ARMv8.2 fullfp16 (native half arithmetic — fadd.4h/8h etc.). Detected as i8mm, except @@ -637,7 +637,7 @@ def public init_jit_target_flags(target_triple : string = ""; host_features : bo // aarch64 fullfp16 (see declaration comment): darwin-arm64 == Apple Silicon == always. // Generic-exe builds stay off — the promote/narrow path is ARMv8.0-legal, fullfp16 is not. g_target_arm64_fullfp16 = g_target_is_aarch64 && (arm64_forced_feature("fullfp16") - || (empty(target_triple) && host_features && (host_llvm_feature("fullfp16") || get_platform_name() == "darwin"))) + || (empty(target_triple) && host_features && (host_llvm_feature("fullfp16") || cpu_supports("fullfp16") || get_platform_name() == "darwin"))) // fp16 operators lower as promote-compute-narrow IR wherever half converts are hardware // (routing truth for isExprOp2/Op1_Func — lives in llvm_boost). aarch64 needs no feature // gate: f16<->f32 converts are ARMv8.0 baseline, so even generic-exe aarch64 stays native. @@ -913,18 +913,13 @@ def public create_default_target_machine(opt_level : uint; use_host_cpu : bool) let cpu_msg = use_host_cpu ? LLVMGetHostCPUName() : "" let features_msg = use_host_cpu ? LLVMGetHostCPUFeatures() : "" - // JIT host artifact on aarch64: force +dotprod in. LLVMGetHostCPUFeatures is empty on macOS - // (the CPU name implies features), so a part an older LLVM cannot name maps to "generic", - // where SDOT has no instruction to select and codegen aborts. Every aarch64 host has dotprod. var targetMachine : LLVMTargetMachineRef if (use_host_cpu && g_target_is_aarch64) { var feats = empty(features_msg) ? "+dotprod" : "{features_msg},+dotprod" if (cpu_supports("i8mm")) { feats = "{feats},+i8mm" } - // DAS_JIT_ARM64_FORCE_FEATURES (e.g. "i8mm"): the x64 rail's twin — a pre-i8mm host EMITS - // AND LINKS an artifact for newer silicon, which the same-env run on the target box then - // cache-hits without a linker. Executing forced instructions here SIGILLs; gate execution. + // a forced feature's artifact cache-hits on the target box without a linker there let forced_arm <- [for (part in split(g_env_jit.jit_arm64_force_features, ",")); "+{part}"; where !empty(part)] if (!empty(forced_arm)) { feats = "{feats},{join(forced_arm, ",")}" diff --git a/modules/dasLLVM/daslib/llvm_jit_run.das b/modules/dasLLVM/daslib/llvm_jit_run.das index 27832e5e6b..413f84d152 100644 --- a/modules/dasLLVM/daslib/llvm_jit_run.das +++ b/modules/dasLLVM/daslib/llvm_jit_run.das @@ -36,7 +36,7 @@ var LINK_WHOLE_LIB = false // when true, standalone exe links against the whole // invalidates cached DLLs (e.g. edits to llvm_jit.das, llvm_macro.das, llvm_jit_common.das, // runtime helper ABI, default target triple). Cache filenames fold this in, so a bump // makes every previously written DLL miss the cache on the next run and get GC'd. -let LLVM_JIT_CODEGEN_VERSION : uint64 = 0x58ul // [hint(unsafe_division_check)] drops the sdiv/srem guards (0x57: INT_MIN / -1 and % -1 guards on sdiv/srem) +let LLVM_JIT_CODEGEN_VERSION : uint64 = 0x59ul // aarch64 hosts append +i8mm when cpu_supports says so (0x58: [hint(unsafe_division_check)] drops the sdiv/srem guards) // Read by tests-cpp/small/test_jit_emitter_pin.cpp: FNV-1a64 of the emitter sources // (normalized to LF; file list in the test) @@ -176,7 +176,7 @@ def private has_generated_kernel(prog : Program?; jit_all_functions : bool) : bo // The environment/config salt shared by the whole-DLL key AND the per-partition obj cache key. // ONE helper on purpose: a salt component folded into the DLL key but not the partition key // would let a config change reuse stale cached partition objects. -def jit_env_salt(opt_level : int; size_level : int; emit_prologue : bool; debug_info : bool; +def jit_env_salt(opt_level : int; size_level : int; emit_prologue : bool; debug_info : bool; // nolint:STYLE037 - one flat fold per emission input; a split would hide which inputs key the cache target_triple : string; split : bool; use_host_cpu : bool) : uint64 { var h = LLVM_JIT_CODEGEN_VERSION // host/binary identity: object format follows the triple (COFF vs ELF on one checkout), and the emitting binary bakes its Context offsets into every global access - neither is in any AST hash @@ -243,6 +243,8 @@ def jit_env_salt(opt_level : int; size_level : int; emit_prologue : bool; debug_ if (!empty(forced_arm)) { h = (h ^ hash(forced_arm)) * JIT_FNV_PRIME } + // aarch64 i8mm is an emission input (the +i8mm append): a warm pre-i8mm cache must not serve an i8mm host + h = (h ^ (cpu_supports("i8mm") ? 64ul : 0ul)) * JIT_FNV_PRIME // Cross-compile artifacts key on their triple — a --jit-target DLL must not cache-hit a host run (foreign-arch dlopen) if (!empty(target_triple)) { h = (h ^ hash(target_triple)) * JIT_FNV_PRIME diff --git a/modules/dasLLVM/daslib/llvm_tune.das b/modules/dasLLVM/daslib/llvm_tune.das index 310052f7ba..cd3aef2f45 100644 --- a/modules/dasLLVM/daslib/llvm_tune.das +++ b/modules/dasLLVM/daslib/llvm_tune.das @@ -556,7 +556,8 @@ def tune_profile_export(outPath : string; klass : string) : bool { let old = doc?["provenance"] if (old != null && old.value is _object) { for (k, v in keys(old as _object), values(old as _object)) { - continue if (k == "binary" || k == "box" || k == "written" || k == "origin" || k == "class") + // a profile's validity is its version pin + feature class, never a commit: engine_sha stays behind with the box identity + continue if (k == "binary" || k == "box" || k == "written" || k == "origin" || k == "class" || k == "engine_sha") if (v != null && v is _string) { prov[k] = JV(v as _string) } @@ -916,8 +917,8 @@ def private scope_fnames_joined(scope : TuneScope) : string { return joined } -//! the ';'-joined `fname=exprA+exprB` map of a scope's ISA-gated [tune] families - every `requires=` its [tune_perm] rows carry, read off the AST for the runtime guard -def private scope_gated_joined(scope : TuneScope) : string { +// every `requires=` a scope's [tune_perm] rows carry, read off the AST for the runtime guard +def private scope_gated_families_joined(scope : TuneScope) : string { var wanted : table wanted |> insert(scope.modName) for (m in split(scope.covers, ";")) { @@ -1245,7 +1246,7 @@ def tune_feature_fingerprint() : string { //! This box's CPU feature class — the name a shipped defaults profile is keyed by. Classes //! follow the seats `requires=` can gate, not box models: two boxes of one class stamp the - //! same winners (measured spread inside a class is noise), so one profile serves them all. + //! same winners, so one profile serves them all; an arch with no ladder gets a key no profile matches. def tune_cpu_class() : string { let arch = get_architecture_name() if (arch == "x86_64") { @@ -1260,7 +1261,7 @@ def tune_cpu_class() : string { if (arch == "arm64" || arch == "aarch64") { return fallback_feature_ok("i8mm") ? "arm-i8mm" : "arm-neon" } - return "{get_platform_name()}-{arch}" //! an arch with no ladder: a key no shipped profile can match, deliberately + return "{get_platform_name()}-{arch}" } //! Profile candidates for this box, most specific first — a box above the best shipped @@ -1284,7 +1285,6 @@ def tune_class_chain() : array { return <- chain } -// evaluate a requires= expression against a ';'-joined feature set instead of this box — // "did the PROFILE's minting box pass this gate" is what separates covered from unlocked seats def private requires_ok_in_fingerprint(requires : string; fingerprint : string) : bool { return true if (empty(requires)) @@ -1613,9 +1613,8 @@ class private TuneAnnotation : AstFunctionAnnotation { let noPerms : array var rok = emit_registry(compiling_module(), func, noPerms, errors) for (c in comps) { - if (rok) { - rok = emit_registry(compiling_module(), c.fn, noPerms, errors) - } + break if (!rok) + rok = emit_registry(compiling_module(), c.fn, noPerms, errors) } delete perms delete comps @@ -1792,9 +1791,9 @@ struct private TuneCliArgs { tune_only : string } -// Adopts --tune-quiet/--tune-verbose and answers whether --tune was passed. Runs at MACRO -// time, so the verbosity has to travel by env: the runtime guard is a different context and -// would not see this context's global. get_user_args also gets the standalone-exe slice right. +// Adopts --tune-quiet/--tune-verbose/--tune-only and answers whether a forced tune was asked for +// (--tune, or --tune-only alone). Runs at MACRO time, so the verbosity travels by env - the runtime +// guard is another context; get_user_args also gets the standalone-exe slice right. def private apply_cli_tune_flags() : bool { var r <- parse_args(type, get_user_args()) if (r |> is_err) { @@ -2085,10 +2084,10 @@ def private announce_tuning(scopeName : string; force, selfRestarts : bool; deta print("llvm_tune: scope '{scopeName}' {why} - tuning now. This takes a while; {tail}.\n") } -// spawn the scope's tuner, relaying its output LIVE — RUNTIME only (the auto and restart -// guards); never from macro context. Streaming is load-bearing: a tune is minutes of work, and -// buffering it to a file left us and any supervisor with no signal until the child exited. -def private run_scope_tuner(scope : TuneScope) : bool { +// spawn the scope's tuner, relaying its output LIVE — RUNTIME only (the auto and restart guards). +// Streaming is load-bearing: buffering minutes of tune output leaves every supervisor blind. +// `onlyFilter` arms DAS_TUNE_ONLY for this one spawn and clears it after (the re-exec child must not inherit it). +def private run_scope_tuner(scope : TuneScope; onlyFilter : string = "") : bool { let cmd = "{tuner_cmd(scope)} 2>&1" if (tune_verbosity() != "silent") { print("llvm_tune: tuning scope '{scope.scopeName}' -> {scope.manifestPath}\n") @@ -2096,6 +2095,9 @@ def private run_scope_tuner(scope : TuneScope) : bool { if (tune_verbosity() == "verbose") { print("llvm_tune: {cmd}\n") } + if (!empty(onlyFilter)) { + set_env_variable("DAS_TUNE_ONLY", onlyFilter) + } var rc : int unsafe { rc = popen(cmd) $(f) { @@ -2106,6 +2108,9 @@ def private run_scope_tuner(scope : TuneScope) : bool { } } } + if (!empty(onlyFilter)) { + set_env_variable("DAS_TUNE_ONLY", "") + } finish_progress() if (rc != 0 && tune_verbosity() != "silent") { print("llvm_tune: tuner for scope '{scope.scopeName}' exited with {rc}\n") @@ -2129,10 +2134,11 @@ def public tune_suppress_mint(knob : string) { // ===== shipped defaults profiles ([tune_scope(defaults = "dir")]) ===== -//! Adopt a shipped class profile into an untuned scope before any race. True = adopted; `raceCsv` is then the ','-joined families the caller still owes a race ("" = fully covered). -def private profile_try_adopt(defaultsDir, scopeName, manifestPath, fnames, gated : string; var raceCsv : string&) : bool { // nolint:STYLE037,STYLE038 — one straight adopt pipeline: locate, validate, merge, diff - raceCsv = "" - //! a runtime-redirected manifest path declines - adoption never lands in a file this guard was not compiled for +// adopt a shipped class profile into an untuned scope before any race: true = adopted, and +// `onlyFilter` is then the ','-joined families the caller still owes a race ("" = fully covered) +def private profile_try_adopt(defaultsDir, scopeName, manifestPath, fnames, gatedFamilies : string; var onlyFilter : string&) : bool { // nolint:STYLE037,STYLE038 — one straight adopt pipeline: locate, validate, merge, diff + onlyFilter = "" + // a runtime-redirected manifest path declines - adoption never lands in a file this guard was not compiled for return false if (empty(defaultsDir) || tune_manifest_path() != manifestPath) var verKey = "" var verVal = "" @@ -2181,6 +2187,7 @@ def private profile_try_adopt(defaultsDir, scopeName, manifestPath, fnames, gate } } if (empty(kernels)) { + print("llvm_tune: defaults profile '{klass}' has no kernel entries - skipped\n") delete_json(doc) return false } @@ -2192,7 +2199,7 @@ def private profile_try_adopt(defaultsDir, scopeName, manifestPath, fnames, gate } } let pfeat = "{doc?["provenance"]?["features"] ?? ""}" - for (row in split(gated, ";")) { + for (row in split(gatedFamilies, ";")) { continue if (empty(row)) let eq = find(row, "=") continue if (eq < 0) @@ -2221,7 +2228,7 @@ def private profile_try_adopt(defaultsDir, scopeName, manifestPath, fnames, gate } let tail = empty(race) ? "" : " - racing {length(race)} unlocked/new families" print("llvm_tune: scope '{scopeName}' adopted the '{klass}' defaults profile ({length(kernels)} kernel entries){tail}\n") - raceCsv = join(race, ",") + onlyFilter = join(race, ",") delete kernels delete race delete_json(doc) @@ -2232,14 +2239,14 @@ def private profile_try_adopt(defaultsDir, scopeName, manifestPath, fnames, gate //! and followed by `tune_auto_reexec()`. Adopts a declared defaults profile first, then tunes //! the residue at RUNTIME (winners are compile-time stamps). A corrupt sidecar self-heals. def tune_auto_prepare(scopeName : string; manifestPath : string; tunerPath : string; force : bool; fnames : string; - defaultsDir : string = ""; gated : string = "") { + defaultsDir : string = ""; gatedFamilies : string = "") { let v = tune_sidecar_verdict(manifestPath, fnames) if (!force && v.complete) { return } var onlyFilter = "" - if (!force && profile_try_adopt(defaultsDir, scopeName, manifestPath, fnames, gated, onlyFilter) && empty(onlyFilter)) { - g_tuned_scopes++ //! nothing raced, yet counted: the re-exec is what stamps the adopted winners + if (!force && profile_try_adopt(defaultsDir, scopeName, manifestPath, fnames, gatedFamilies, onlyFilter) && empty(onlyFilter)) { + g_tuned_scopes++ // nothing raced, yet counted: the re-exec is what stamps the adopted winners return } if (!force && empty(onlyFilter) && scope_resolved_externally(scopeName, manifestPath, fnames)) { @@ -2252,14 +2259,8 @@ def tune_auto_prepare(scopeName : string; manifestPath : string; tunerPath : str } return } - if (!empty(onlyFilter)) { - set_env_variable("DAS_TUNE_ONLY", onlyFilter) - } announce_tuning(scopeName, force, true, v.detail) - run_scope_tuner(TuneScope(scopeName = scopeName, manifestPath = manifestPath, tunerPath = tunerPath)) - if (!empty(onlyFilter)) { - set_env_variable("DAS_TUNE_ONLY", "") - } + run_scope_tuner(TuneScope(scopeName = scopeName, manifestPath = manifestPath, tunerPath = tunerPath), onlyFilter) let after = tune_sidecar_verdict(manifestPath, fnames) if (after.complete) { g_tuned_scopes++ @@ -2281,7 +2282,7 @@ def tune_auto_reexec() : bool { args |> reserve(length(argv)) var skipTuneOnlyValue = false for (a in argv) { - if (skipTuneOnlyValue) { // the value token of a `--tune-only ` pair + if (skipTuneOnlyValue) { skipTuneOnlyValue = false continue } @@ -2381,13 +2382,13 @@ def tune_family_selected(name : string) : bool { //! the app's `main`: adopts a declared defaults profile, then consults the scope resolver, else runs the tuner; true = main //! returns immediately (winners are compile-time stamps), false = scope complete, carry on. def tune_restart_needed(scopeName : string; manifestPath : string; tunerPath : string; fnames : string; - defaultsDir : string = ""; gated : string = "") : bool { + defaultsDir : string = ""; gatedFamilies : string = "") : bool { let v = tune_sidecar_verdict(manifestPath, fnames) if (v.complete) { return false } var onlyFilter = "" - if (profile_try_adopt(defaultsDir, scopeName, manifestPath, fnames, gated, onlyFilter) && empty(onlyFilter)) { + if (profile_try_adopt(defaultsDir, scopeName, manifestPath, fnames, gatedFamilies, onlyFilter) && empty(onlyFilter)) { print("llvm_tune: scope '{scopeName}' adopted -> {manifestPath}\nllvm_tune: restart to apply the winners\n") return true } @@ -2407,13 +2408,7 @@ def tune_restart_needed(scopeName : string; manifestPath : string; tunerPath : s return true } announce_tuning(scopeName, false, false, v.detail) - if (!empty(onlyFilter)) { - set_env_variable("DAS_TUNE_ONLY", onlyFilter) - } - run_scope_tuner(TuneScope(scopeName = scopeName, manifestPath = manifestPath, tunerPath = tunerPath)) - if (!empty(onlyFilter)) { - set_env_variable("DAS_TUNE_ONLY", "") - } + run_scope_tuner(TuneScope(scopeName = scopeName, manifestPath = manifestPath, tunerPath = tunerPath), onlyFilter) let after = tune_sidecar_verdict(manifestPath, fnames) if (after.complete) { print("llvm_tune: scope '{scopeName}' tuned -> {manifestPath}\nllvm_tune: restart to apply the winners\n") @@ -2526,18 +2521,22 @@ def private apply_tune_policy(var func : FunctionPtr; declared : string; var err var policy = declared { let penv = g_env_tune.tune_policy - // announce deferred to the first INCOMPLETE scope - a fully-tuned start stays silent - var envNote = "" if (!empty(penv)) { policy = penv + // the announce fires where the override changes the outcome - `reference` alters every stamp even on a fully tuned box if (tune_verbosity() != "silent") { - envNote = "llvm_tune: policy '{policy}' set by DAS_TUNE_POLICY (declared/default '{declared}')\n" + print("llvm_tune: policy '{policy}' set by DAS_TUNE_POLICY (declared/default '{declared}')\n") } } if (policy != "fallback" && policy != "reference" && policy != "warn" && policy != "error" && policy != "auto" && policy != "restart") { errors = "[tune_policy] on {func.name}: missing=\"{policy}\" - use fallback | reference | warn | error | auto | restart" return false } + if (declared == "reference" && penv != "reference") { + // the [tune] stamps in required libraries compile before this root exists and read the env only + errors = "[tune_policy] on {func.name}: missing=\"reference\" is an environment override (DAS_TUNE_POLICY=reference), not a declarable value" + return false + } let force = apply_cli_tune_flags() if (force && empty(g_scopes)) { print("llvm_tune: --tune requested but no [tune_scope] is declared by any required library\n") @@ -2553,10 +2552,6 @@ def private apply_tune_policy(var func : FunctionPtr; declared : string; var err let scopeFnames = scope_fnames_joined(scope) let v = tune_sidecar_verdict(scope.manifestPath, scopeFnames) continue if (!force && v.complete) - if (!empty(envNote)) { - print(envNote) - envNote = "" - } if (force || policy == "auto") { // a runtime guard tunes and RE-EXECS: restamping a required module's compiled function is a cross-module mutation re-infer cannot survive if (empty(scope.tunerPath)) { @@ -2578,7 +2573,7 @@ def private apply_tune_policy(var func : FunctionPtr; declared : string; var err errors = "[tune_policy]: restart - {func.name} must return void or int" } else { var guard : ExpressionPtr - let scopeGated = scope_gated_joined(scope) + let scopeGated = scope_gated_families_joined(scope) if (func.result.baseType == Type.tInt) { guard = qmacro_expr() { if (tune_restart_needed($v(scope.scopeName), $v(scope.manifestPath), $v(scope.tunerPath), $v(scopeFnames), @@ -2615,7 +2610,7 @@ def private apply_tune_policy(var func : FunctionPtr; declared : string; var err guards |> reserve(length(autoScopes) + 1) for (scope in autoScopes) { let fnames = scope_fnames_joined(scope) - let gated = scope_gated_joined(scope) + let gated = scope_gated_families_joined(scope) guards |> push <| qmacro_expr() { tune_auto_prepare($v(scope.scopeName), $v(scope.manifestPath), $v(scope.tunerPath), $v(force), $v(fnames), $v(scope.defaultsPath), $v(gated)) diff --git a/modules/dasLLVM/tests/llvm_tune_profiles.das b/modules/dasLLVM/tests/llvm_tune_profiles.das index 215190b0cb..c528c2d0f2 100644 --- a/modules/dasLLVM/tests/llvm_tune_profiles.das +++ b/modules/dasLLVM/tests/llvm_tune_profiles.das @@ -15,7 +15,7 @@ require llvm/daslib/llvm_tune // tune_cpu_class(), and their features field is this box's fingerprint (or empty, to fake // a less-capable minting box). -def private tune_env() : string { +def private cleared_tune_env_prefix() : string { return (get_platform_name() == "windows" ? "set DAS_TUNE_MODE=&& set DAS_TUNE_MANIFEST=&& set DAS_TUNE_VERBOSITY=verbose&& set DAS_TUNE_POLICY=&& set DAS_TUNE_ONLY=&& " : "DAS_TUNE_MODE= DAS_TUNE_MANIFEST= DAS_TUNE_VERBOSITY=verbose DAS_TUNE_POLICY= DAS_TUNE_ONLY= ") @@ -39,6 +39,16 @@ def private spawn_child(cmd : string; var lines : array) : int { return rc } +def private count_hits(lines : array; needle : string) : int { + var n = 0 + for (ln in lines) { + if (find(ln, needle) >= 0) { + n++ + } + } + return n +} + def private has(lines : array; needle : string) : bool { for (ln in lines) { if (find(ln, needle) >= 0) { @@ -55,13 +65,13 @@ def private write_profile(dir, klass, winner, features : string) { } [test] -def test_llvm_tune_profiles(t : T?) { +def test_llvm_tune_profiles(t : T?) { // nolint:STYLE038 - one flat spawn-and-assert rail per scenario; a split would hide the sequence the scenarios build on if (!jit_enabled()) { - //! no generator tier without the jit, so no stamped-tier fingerprint to read + // no generator tier without the jit, so no stamped-tier fingerprint to read t |> success(true) return } - let args <- get_command_line_arguments() //! locked process-global view - never delete it + let args <- get_command_line_arguments() // aliases a process-global Array header - do not delete let bin = args[0] let client = "{get_this_module_dir()}/llvm_tune_profiles_client.das" let sidecar = "{get_this_module_dir()}/llvm_tune_profiles_client.tune.json" @@ -69,47 +79,89 @@ def test_llvm_tune_profiles(t : T?) { let klass = tune_cpu_class() let profile = path_join(defdir, "{klass}.tune-defaults.json") - //! 1. no profile file: the plain auto flow - full race (empty filter), the tuner's k2 runs + // 1. no profile file: the plain auto flow - full race (empty filter), the tuner's k2 runs remove(sidecar) remove(profile) - var l1 : array - t |> equal(spawn_child("{tune_env()}\"{bin}\" -jit \"{client}\"", l1), 0) - t |> success(l1 |> has("FAKE TUNER")) - t |> success(l1 |> has("ONLY=''")) - t |> success(!(l1 |> has("defaults profile"))) - t |> success(l1 |> has("RESULT 7")) - delete l1 - - //! 2. full-cover profile (this box's own fingerprint): adopted, no tuner, its winner stamps + var noProfile : array + t |> equal(spawn_child("{cleared_tune_env_prefix()}\"{bin}\" -jit \"{client}\"", noProfile), 0) + t |> success(noProfile |> has("FAKE TUNER")) + t |> success(noProfile |> has("ONLY=''")) + t |> success(!(noProfile |> has("defaults profile"))) + t |> success(noProfile |> has("RESULT 7")) + delete noProfile + + // 2. full-cover profile (this box's own fingerprint): adopted, no tuner, its winner stamps remove(sidecar) write_profile(defdir, klass, "k0", tune_feature_fingerprint()) - var l2 : array - t |> equal(spawn_child("{tune_env()}\"{bin}\" -jit \"{client}\"", l2), 0) - t |> success(l2 |> has("adopted the '{klass}' defaults profile")) - t |> success(!(l2 |> has("FAKE TUNER"))) - t |> success(l2 |> has("RESULT 5")) - t |> success(l2 |> has("STATUS prof_add|llvm_tune_profiles_probe|k0|manifest")) - let side2 = fread(sidecar) - t |> success(find(side2, "\"adopted_profile\"") >= 0) - delete l2 - - //! 3. a profile minted with no features leaves the k3 requires= seat unlocked here: adopt AND race just that family - the tuner's k2 outranks the adopted k0 + var fullCover : array + t |> equal(spawn_child("{cleared_tune_env_prefix()}\"{bin}\" -jit \"{client}\"", fullCover), 0) + t |> success(fullCover |> has("adopted the '{klass}' defaults profile")) + t |> success(!(fullCover |> has("FAKE TUNER"))) + t |> success(fullCover |> has("RESULT 5")) + t |> success(fullCover |> has("STATUS prof_add|llvm_tune_profiles_probe|k0|manifest")) + let adoptedSidecar = fread(sidecar) + t |> success(find(adoptedSidecar, "\"adopted_profile\"") >= 0) + delete fullCover + + // 3. a profile minted with no features leaves the k3 requires= seat unlocked here: adopt AND race just that family - the tuner's k2 outranks the adopted k0 remove(sidecar) write_profile(defdir, klass, "k0", "") - var l3 : array - t |> equal(spawn_child("{tune_env()}\"{bin}\" -jit \"{client}\"", l3), 0) - t |> success(l3 |> has("racing 1 unlocked/new families")) - t |> success(l3 |> has("ONLY='prof_add'")) - t |> success(l3 |> has("RESULT 7")) - t |> success(l3 |> has("STATUS prof_add|llvm_tune_profiles_probe|k2|manifest")) - delete l3 - - //! 4. --tune outranks the profile: full re-race, no adoption - var l4 : array - t |> equal(spawn_child("{tune_env()}\"{bin}\" -jit \"{client}\" -- --tune", l4), 0) - t |> success(!(l4 |> has("defaults profile"))) - t |> success(l4 |> has("FAKE TUNER")) - delete l4 + var unlockedSeat : array + t |> equal(spawn_child("{cleared_tune_env_prefix()}\"{bin}\" -jit \"{client}\"", unlockedSeat), 0) + t |> success(unlockedSeat |> has("racing 1 unlocked/new families")) + t |> success(unlockedSeat |> has("ONLY='prof_add'")) + t |> success(unlockedSeat |> has("RESULT 7")) + t |> success(unlockedSeat |> has("STATUS prof_add|llvm_tune_profiles_probe|k2|manifest")) + delete unlockedSeat + + // 4. --tune outranks the profile: full re-race, no adoption + var forcedTune : array + t |> equal(spawn_child("{cleared_tune_env_prefix()}\"{bin}\" -jit \"{client}\" -- --tune", forcedTune), 0) + t |> success(!(forcedTune |> has("defaults profile"))) + t |> success(forcedTune |> has("FAKE TUNER")) + delete forcedTune + + // 5. --tune-only converges: the relaunch strips both spellings, so exactly one tuner runs and + // the child does not re-tune forever (the strip arms are what keep it from looping) + var onlyOne : array + t |> equal(spawn_child("{cleared_tune_env_prefix()}\"{bin}\" -jit \"{client}\" -- --tune-only prof_add", onlyOne), 0) + t |> equal(count_hits(onlyOne, "FAKE TUNER"), 1) + t |> success(onlyOne |> has("ONLY='prof_add'")) + t |> success(onlyOne |> has("RESULT 7")) + delete onlyOne + var onlyEq : array + t |> equal(spawn_child("{cleared_tune_env_prefix()}\"{bin}\" -jit \"{client}\" -- --tune-only=prof_add", onlyEq), 0) + t |> equal(count_hits(onlyEq, "FAKE TUNER"), 1) + delete onlyEq + + // 6. the class chain: only a LOWER class's profile on disk, and the box adopts it (the + // residue race then covers whatever its own class unlocks) + var chain <- tune_class_chain() + if (length(chain) > 1) { + let lower = chain[length(chain) - 1] + remove(sidecar) + remove(profile) + write_profile(defdir, lower, "k0", tune_feature_fingerprint()) + var lowerAdopt : array + t |> equal(spawn_child("{cleared_tune_env_prefix()}\"{bin}\" -jit \"{client}\"", lowerAdopt), 0) + t |> success(lowerAdopt |> has("adopted the '{lower}' defaults profile")) + t |> success(!(lowerAdopt |> has("FAKE TUNER"))) + delete lowerAdopt + remove(path_join(defdir, "{lower}.tune-defaults.json")) + } + delete chain + + // 7. DAS_TUNE_POLICY=reference serves the original bodies and announces itself even on a + // fully tuned box (the sidecar from case 5 is complete) + var refRun : array + let refEnv = (get_platform_name() == "windows" ? + "set DAS_TUNE_MODE=&& set DAS_TUNE_MANIFEST=&& set DAS_TUNE_VERBOSITY=verbose&& set DAS_TUNE_POLICY=reference&& set DAS_TUNE_ONLY=&& " : + "DAS_TUNE_MODE= DAS_TUNE_MANIFEST= DAS_TUNE_VERBOSITY=verbose DAS_TUNE_POLICY=reference DAS_TUNE_ONLY= ") + t |> equal(spawn_child("{refEnv}\"{bin}\" -jit \"{client}\"", refRun), 0) + t |> success(refRun |> has("policy 'reference' set by DAS_TUNE_POLICY")) + t |> success(refRun |> has("STATUS prof_add|llvm_tune_profiles_probe||reference")) + t |> success(refRun |> has("RESULT 5")) + delete refRun remove(sidecar) remove(profile) diff --git a/modules/dasLLVM/tests/llvm_tune_profiles_tuner.das b/modules/dasLLVM/tests/llvm_tune_profiles_tuner.das index 64c3d2cb33..77f9a1085a 100644 --- a/modules/dasLLVM/tests/llvm_tune_profiles_tuner.das +++ b/modules/dasLLVM/tests/llvm_tune_profiles_tuner.das @@ -1,7 +1,6 @@ options gen2 require llvm/daslib/llvm_tune -require daslib/fio // Instant fake tuner for llvm_tune_profiles.das (no [test] on purpose): prints the // DAS_TUNE_ONLY filter the guard armed (the race-on-unlock proof) and records k2 — a winner @@ -9,7 +8,7 @@ require daslib/fio // fingerprints which path stamped. [export] def main { - print("FAKE TUNER -> {tune_manifest_path()} ONLY='{get_env_variable("DAS_TUNE_ONLY")}'\n") + print("FAKE TUNER -> {tune_manifest_path()} ONLY='{tune_only_filter()}'\n") if (tune_family_selected("prof_add")) { tune_manifest_set("prof_add", "k2") } diff --git a/modules/dasLLVM/tests/test_grid.das b/modules/dasLLVM/tests/test_grid.das index 041fdef558..5ba22564bd 100644 --- a/modules/dasLLVM/tests/test_grid.das +++ b/modules/dasLLVM/tests/test_grid.das @@ -34,7 +34,7 @@ def test_grid_registry(t : T?) { let want = reference(a, b) let vs <- grid_dot_template_variants() - t |> equal(length(vs), 20, "grid emits all 20 permutations") + t |> equal(length(vs), 7, "grid emits all 7 permutations (plain, u2, vec8, vec8_u2, vec16, vec16_u2, vec4_u4)") // Every emitted variant is callable and numerically correct (vectorize reassociates FP, // so compare within tolerance rather than bit-exact). @@ -65,7 +65,7 @@ struct GridScale {} [test] def test_grid_void_registry(t : T?) { let vs <- grid_scale_template_variants() - t |> equal(length(vs), 20, "void-template grid emits all 20 permutations") + t |> equal(length(vs), 7, "void-template grid emits all 7 permutations") var d : array d |> resize(8) diff --git a/modules/dasLLVM/tests/test_tuned.das b/modules/dasLLVM/tests/test_tuned.das index 899da4b1d6..30fd2952a0 100644 --- a/modules/dasLLVM/tests/test_tuned.das +++ b/modules/dasLLVM/tests/test_tuned.das @@ -23,8 +23,8 @@ def sample_dot(a, b : array) : float {} // an unknown fallback name fails the compile with "tuned: unknown perm" (negative-probed), so this // compiling proves the fallback names a valid grid perm — the one consulted whenever no profile // entry / perm pin applies (and no profile carries this test-local kernel's key). -[tuned(fallback = "u4", src = "sample_dot_template")] -def sample_dot_u4(a, b : array) : float {} +[tuned(fallback = "u2", src = "sample_dot_template")] +def sample_dot_u2(a, b : array) : float {} def private reference(a, b : array) : float { var s = 0f @@ -39,5 +39,5 @@ def test_tuned_reconstitutes(t : T?) { let a = [for (i in range(64)); float(i)] let b = [for (_i in range(64)); 1.0f] t |> equal(sample_dot(a, b), reference(a, b), "tuned sample_dot == reference") - t |> equal(sample_dot_u4(a, b), reference(a, b), "tuned default=u4 sample_dot_u4 == reference") + t |> equal(sample_dot_u2(a, b), reference(a, b), "tuned default=u2 sample_dot_u2 == reference") } diff --git a/modules/dasMetal/ARCHITECTURE.md b/modules/dasMetal/ARCHITECTURE.md index cc7a7ef959..aa7ca93c0b 100644 --- a/modules/dasMetal/ARCHITECTURE.md +++ b/modules/dasMetal/ARCHITECTURE.md @@ -170,6 +170,13 @@ than a second GPU. Cross-GPU parity of one source is secondary. Current entries: SPV_NV_cooperative_matrix2 tensor loads. Metal-4 tensors have no decode-callback analogue, so the Metal quant GEMMs stage dequant through threadgroup memory instead. Deliberate, target-specific - not a pending port. +- **Literal fixed-array hoisting is Metal-only; pending, not deliberate.** `msl_emit` lowers a + `let` fixed-array local whose elements are all literals to a program-scope `constant` table + (renamed on a same-name/different-content collision); `spirv_emit` keeps such a local in + Function storage, where a dynamically indexed read spills to private memory - the codebook + and grid quant kernels on Vulkan therefore pack their tables into `uint4` words by hand. + Done = the SPIR-V twin (a constant-storage array for the same shape) or a lint on a + dynamically indexed fixed-array local inside a kernel class. ## 7. `@uniform` structs - the kargs form diff --git a/modules/dasMetal/REVIEW.md b/modules/dasMetal/REVIEW.md index 682e5f1dc0..387306721e 100644 --- a/modules/dasMetal/REVIEW.md +++ b/modules/dasMetal/REVIEW.md @@ -8,11 +8,12 @@ **A CPU-oracle or host-side test file answers to `tests/metal/REVIEW.md` (repo root), wherever the diff puts it.** An emitted-text fixture answers to `tests/msl/REVIEW.md` (repo root). -- **A new emitter capability ships a text fixture under `tests/msl/` (repo root) and its own - census kind, in the same change.** A new emitter capability is a new emit site or a newly - accepted construct, including the path a `[metal_kernel]` takes when it has no `name=` +- **A new emitter capability ships a text fixture under `tests/msl/` (repo root) and a census + kind per emit shape, in the same change.** A new emitter capability is a new emit site or a + newly accepted construct, including the path a `[metal_kernel]` takes when it has no `name=` argument. A census kind is the construct label the emitter records at emit time, declared in - `declared_msl_census` (`tests/msl/_msl_common.das`, repo root). Two emit shapes sharing one + `declared_msl_census` (`tests/msl/_msl_common.das`, repo root); an emit shape is one distinct + text the site can produce (a renamed twin is a second shape). Two emit shapes sharing one kind lets either one go untested. - **A new or changed `[metal_kernel]` annotation argument ships a `tests/msl/` (repo root) @@ -20,17 +21,17 @@ the diff puts it.** An emitted-text fixture answers to `tests/msl/REVIEW.md` (re macro declares for it - a module-level global holding the kernel's MSL text or a compile option - or the difference it makes to the emitted text. -- **A diff that makes the MSL emitter reject at compile time a construct it accepted before - - a new rejection, or a widened condition on an existing one - ships a - `tests/msl/_fail_closed/` (repo root) fixture for that construct in the same change.** An - error needle is the substring of the compile error that names the rejected construct; the - fixture-and-assert pairing itself is `REVIEW.das`'s to hold (`check_fail_closed_sync`, - both directions, `tests/msl/test_msl_fail_closed.das` (repo root)). +- **A diff that adds an MSL emitter rejection, widens an existing one's condition, or changes a + rejection's message text ships a `tests/msl/_fail_closed/` (repo root) fixture asserting that + rejection's needle, in the same change.** An error needle is the substring of the compile + error that names the rejected construct; `REVIEW.das` checks the fixture-and-assert pairing + both directions (`check_fail_closed_sync`, `tests/msl/test_msl_fail_closed.das` (repo root)). - **A kernel behavioral change ships a CPU-oracle test under `tests/metal/` (repo root).** A - kernel behavioral change is a change to what a kernel computes - its emitted arithmetic, - indexing, or synchronization; a CPU-oracle test compares the GPU result against a - CPU-computed expectation. + kernel behavioral change is a change to what an existing kernel computes, or a diff that + makes the emitter produce arithmetic, indexing, or synchronization it did not produce before + - a newly accepted construct that lowers to a read, an index, or a barrier; a CPU-oracle + test compares the GPU result against a CPU-computed expectation. - **A change visible only in the emitted text ships a `tests/msl/` (repo root) fixture.** The fixture asserts the emitted text that the change alters. @@ -40,18 +41,19 @@ the diff puts it.** An emitted-text fixture answers to `tests/msl/REVIEW.md` (re - **A new or changed host extern under `modules/dasMetal/src/` ships a host-side test under `tests/metal/` (repo root) in the same change.** A changed public function in - `metal/das_metal_boost.das` fires this rule too. The test `feint`s when no Metal device is - present. A regression in dasMetal alone must fail here, not in a consumer module's tests. + `metal/das_metal_boost.das` fires this rule too. The test reports a `feint` - dastest's skip + result - when no Metal device is present. A regression in dasMetal alone must fail here, not in a consumer module's tests. - **Weakening `REVIEW.das`'s descriptor check is a defect.** That check requires every `matmul2d_descriptor` written in `metal/msl_emit.das` to set `relaxed_precision = true`. Narrowing the file it scans weakens it. So does making its finding text stop naming what failed. `relaxed_precision = false` keeps the op off the tensor-unit fast path. -- **Weakening `REVIEW.das`'s fail-closed sync check is a defect.** That check holds the - `tests/msl/_fail_closed/` fixture list and `test_msl_fail_closed.das`'s `check_rejects` - asserts equal both directions - a fixture without its assert leaves a rejection unpinned, - and a stranded assert is a rejection the emitter no longer makes. +- **Weakening `REVIEW.das`'s fail-closed sync check is a defect.** That check requires every + `tests/msl/_fail_closed/` fixture to have a `check_rejects` call in + `test_msl_fail_closed.das`, and every such call to have a fixture - a fixture without its + call leaves a rejection unpinned, and a call without its fixture is a rejection the emitter + no longer makes. - **Never zero a cooperative tensor element by element before a `matmul2d` `run` accumulates into it - `get_destination_cooperative_tensor` already hands it back zeroed.** That walk @@ -63,9 +65,14 @@ the diff puts it.** An emitted-text fixture answers to `tests/msl/REVIEW.md` (re das body replays the same tile indexing, so a one-sided change reads the wrong halves with no compile error. -- **A diff that changes the threadgroup row stride `tmm2d_helper_text` writes - its `dstp` index - and the matching tensor extent in `metal/msl_emit.das` - also changes the `wt` size documented - on `tmm2d_q8u_f32` (`metal/metal_builtins.das`) and every `@workgroup` staging array a - `tests/msl/` or `tests/metal/` (repo root) fixture sizes for that stride, in the same change.** - A staged weight tile is the dequantized W the emitter parks in threadgroup memory; nothing - checks the caller's allocation against the stride, so a one-sided change overruns it silently. +- **A diff that changes the threadgroup row stride `tmm2d_helper_text` writes (its `dstp` + index and the matching tensor extent in `metal/msl_emit.das`) changes the `wt` size + documented on `tmm2d_q8u_f32` (`metal/metal_builtins.das`) in the same change.** `wt` is the + staged weight tile - the dequantized W the emitter stores in threadgroup memory - and the + documented size is the caller's only statement of the stride. + +- **A diff that changes the threadgroup row stride `tmm2d_helper_text` writes + (`metal/msl_emit.das`) also resizes every `@workgroup` staging array a `tests/msl/` or + `tests/metal/` (repo root) fixture sizes for that stride, in the same change.** + Nothing checks a fixture's allocation against the stride, so a one-sided change overruns + it silently. diff --git a/modules/dasMetal/metal/msl_emit.das b/modules/dasMetal/metal/msl_emit.das index 63f0a415db..ba843db1f5 100644 --- a/modules/dasMetal/metal/msl_emit.das +++ b/modules/dasMetal/metal/msl_emit.das @@ -3027,6 +3027,7 @@ def private hoist_const_table(var ctx : MslCtx; v : VariablePtr) { an = "{an}_c{ctx.splice_seq}" // a same-named table with different content in another helper ctx.splice_seq++ txt = "constant {ename} {an}[{n}] = \{\n{vtxt}\};" + note(ctx, "stmt.let.const_table.renamed") } ctx.renames[aname] = an ctx.const_tables[an] = txt diff --git a/plans/unquirk_pass.md b/plans/unquirk_pass.md index f7d7d75fc4..6d5283f09f 100644 --- a/plans/unquirk_pass.md +++ b/plans/unquirk_pass.md @@ -1,4 +1,4 @@ -# Unquirk pass — end of the iquant format ladder arc +# Unquirk pass - end of the iquant format ladder arc The ladder is complete (9 formats x 4 tiers, ea1f41d2c). This plan covers everything between that and the arc's PR: the tuner rework, the KqPlanes unquirk, the cache/one-off fixes, the @@ -12,9 +12,9 @@ ARM; the 22 `[tuned]` vector-utility families have winner-vs-runner-up gaps of 0 zen2 AND on M5 (25/25 under 2%). Cross-sidecar winner agreement 34/39; all 5 diffs are noise ties or a skipped-seat artifact. The race rediscovers ~2 facts for ~20 minutes per app per box. M5 Max carries FEAT_I8MM + FEAT_SME2/2p1 + BF16 that our ARM seat space does not offer a -single seat for — its mint provably lands on M1's answers today. +single seat for - its mint provably lands on M1's answers today. -## Phase B — tuner rework (kills QUIRK 11, 15, 17, 18) +## Phase B - tuner rework (kills QUIRK 11, 15, 17, 18) Target: mint on a known box = zero races; full race on new silicon (zen4, M5) = a couple of minutes. If that target holds, the dasllama.io tune service is removed entirely (Boris's call, @@ -22,20 +22,20 @@ outside this PR). **B1. Prune the seat space.** - Tile gens (`dasllama_math_gen.das`, 16 families x 11 seats): keep `mr8` (generic/ARM - baseline), `maddubs w256 mr8` (requires avx2 — the x86 baseline), `vpdpbusd w256 mr8` + baseline), `maddubs w256 mr8` (requires avx2 - the x86 baseline), `vpdpbusd w256 mr8` (requires vnni), `vpdpbusd w512 mr16` (requires avx512vnni+bw). Drop mr4 and every nrsplit2 twin (never won beyond noise on any box). 11 -> 4 seats, only 2 raceable per box class. -- q8q8 (15 seats): keep the observed winners' axes only — `kstep2` (zen2 crown), +- q8q8 (15 seats): keep the observed winners' axes only - `kstep2` (zen2 crown), `mr8_budget` (M5 crown), plus the maddubs/vpdpbusd kstep2 twins under their gates. ~15 -> 4. - `[tuned]` utilities (`dasllama_tune.das::build_grid()`, one place): grid 20 -> the union of every winner ever recorded across zen2/m1/m4/m5 sidecars (~6: plain, u2, vec8_u2, vec16, vec16_u2, vec4_u2). With B2 they stop racing on known classes anyway; the pruned grid is what a new-silicon full race walks. -**B2. Arch default profiles (kernels-only — the sharper form of the CPU/GPU split).** +**B2. Arch default profiles (kernels-only - the sharper form of the CPU/GPU split).** - Profiles carry KERNEL WINNERS ONLY. The runtime-knob section never travels: its knobs mix device knobs (metal_*, batch_grid_2d, lane caps) with box-shape knobs (core counts, cache - budgets, thread receipts) — neither is class-portable, so nothing GPU- or box-side can + budgets, thread receipts) - neither is class-portable, so nothing GPU- or box-side can mis-inherit by construction. Knobs stay code-defaults on a fresh box until a deliberate `--tune`; existing minted sidecars keep theirs. - Checked-in profiles: `modules/dasLLAMA/performance/defaults/.tune-defaults.json` @@ -43,18 +43,18 @@ outside this PR). `arm-neon` from m1. `x86-vnni512` comes from the c7a trial; `arm-i8mm`/`arm-sme` exist only once those kernels do. `harness/export_tune_profile.das` mints one from a sidecar; the framework fn is `tune_profile_export`. -- Adoption is FRAMEWORK-side, declared on the scope — `[tune_scope(defaults = "dir")]` — +- Adoption is FRAMEWORK-side, declared on the scope - `[tune_scope(defaults = "dir")]` - and runs in the auto/restart guards ahead of the scope resolver: walk `tune_class_chain()` down from `tune_cpu_class()`, adopt the first matching profile (version-pin checked) into the app sidecar as a normal local write. A covered scope races NOTHING. - Race-on-unlock: an adopting start races ONLY families with a seat whose `requires=` the host passes but the profile's minting box did not (recorded as the `features` fingerprint - every sidecar save now stamps) — plus families the profile predates. The residue rides the + every sidecar save now stamps) - plus families the profile predates. The residue rides the existing `--tune-only` machinery (the guard arms `DAS_TUNE_ONLY` for the tuner it spawns). - No new flag: bare `--tune` already skips adoption entirely — it IS the full new-silicon + No new flag: bare `--tune` already skips adoption entirely - it IS the full new-silicon walk. `--tune-only` unchanged. -**B3. `DAS_TUNE_POLICY=reference`** — a policy value that serves the original bodies outright +**B3. `DAS_TUNE_POLICY=reference`** - a policy value that serves the original bodies outright (QUIRK 18's missing spelling). Small, framework-side. **B4. Validation + the measurement Boris's service decision needs.** MEASURED 2026-08-31: @@ -68,30 +68,30 @@ m1 gen half 734 s (M1, 8 threads; mr8 everywhere) - its `tune_kernels` Metal arm macOS-only compile error (two require paths for MetalTensorRaceResult), fixed in-pass. - dasLLVM tune tests (test_tuned/test_grid/llvm_tune_*) extended for profile resolution + the split sidecar; gen probe test+tune modes; parity 40/40; one bench row vs current - numbers (winners are unchanged by construction — prove it, don't assume it). + numbers (winners are unchanged by construction - prove it, don't assume it). - zen2: delete sidecar, cold start -> 0 races, numbers match. Wall time recorded. - M5 (ssh): fresh mint under arm-neon profile -> 0 races. Wall time recorded. -- zen4 (ssh, off-hours — it hosts dasweb): `--tune-full` new-silicon race, wall time +- zen4 (ssh, off-hours - it hosts dasweb): `--tune-full` new-silicon race, wall time recorded (target <= ~2-3 min), its x86-vnni512 profile checked in. First box where - vpdpbusd can actually win — report the crowns. + vpdpbusd can actually win - report the crowns. - skills/tune.md + tune_for_this_box.md + HOW_TO QUIRK 11/15/17/18 text updated in-phase. -## Phase A — the KqPlanes table (kills QUIRK 1, 2, 3, 5, 7, 8, 13, 19, 25, 26) +## Phase A - the KqPlanes table (kills QUIRK 1, 2, 3, 5, 7, 8, 13, 19, 25, 26) -One per-format record on `Model` — quant plane, scale plane, mr, qsb/ssb strides, stream -code, schema id, device-form flag — indexed by `KqFmt`; every `fmt ==` ladder becomes a +One per-format record on `Model` - quant plane, scale plane, mr, qsb/ssb strides, stream +code, schema id, device-form flag - indexed by `KqFmt`; every `fmt ==` ladder becomes a lookup. Staged so each commit is green; the fast dev loop (`-module-cache` + `--jit-split-modules=-1`) keeps the cycle ~1 min; after EVERY stage that touches layout or -pack, delete `*.dlim` and e2e the MIXED vehicle (the iq2xs requant carrying q2_K — QUIRK +pack, delete `*.dlim` and e2e the MIXED vehicle (the iq2xs requant carrying q2_K - QUIRK 26's lesson), not a pure one. - **A1.** The table + accessors (`kq_plane_q/s`, `kq_mr`, ...) + the id-derivation table - (schema id + stream code from `KqFmt` — QUIRK 5/25 become one row each) + `test_kqformat` + (schema id + stream code from `KqFmt` - QUIRK 5/25 become one row each) + `test_kqformat` pins table<->enum. No consumers move yet. - **A2.** Collapse the load/common/layout ladders (12 + 13 + 3 arms-per-format sites). - **A3.** Collapse math_gen/math_default/math (8 + 3 + 4); the repack-mr freeze ladder (QUIRK 8) becomes a table write; `serialize_image_meta` serializes the table and - `dlim_identity` formats from it (QUIRK 3/7 — the hand list and the count tripwire go). + `dlim_identity` formats from it (QUIRK 3/7 - the hand list and the count tripwire go). Image meta layout changes -> images re-bake once; C1 lands before this so stale images die loudly. - **A4.** `tests/_kq_fixtures.das`: one per-format dispatch (`kq_transcode_sb`, @@ -102,51 +102,51 @@ pack, delete `*.dlim` and e2e the MIXED vehicle (the iq2xs requant carrying q2_K replaces the four parallel ternary chains. m1 sync + metal gemv/gemm gates. - **A6.** The device gather's two branches key on the table's device-form flag (QUIRK 19). -## Phase C — cache invalidation +## Phase C - cache invalidation - **C1** (lands FIRST, before A3): a `PACK_VERSION` folded into the `.dlim` filename hash - (QUIRK 24) — a layout/pack edit re-bakes instead of serving stale planes. Gate: bump -> + (QUIRK 24) - a layout/pack edit re-bakes instead of serving stale planes. Gate: bump -> next run logs "baked", not "mapped". -- QUIRK 21 (generator bodies into the JIT DLL hash) STAYS on `plans/jit_compile_time.md` — +- QUIRK 21 (generator bodies into the JIT DLL hash) STAYS on `plans/jit_compile_time.md` - cache-keying work, not this pass. -## Phase D — one-offs +## Phase D - one-offs - **D1**: posix arm for `stage_jit_backend` in `utils/mcp/setup.das` (QUIRK 12). -- **D2**: `static_if ... elif` in the MSL emitter (QUIRK 14) — timeboxed to ~2h, dropped to +- **D2**: `static_if ... elif` in the MSL emitter (QUIRK 14) - timeboxed to ~2h, dropped to the metal followup doc if the emitter fights. - Ledgered, not done here: QUIRK 6 resolves as an INTRINSIC builtin (Boris's ruling: daslang - has no u8/u16 operator math and will not start — a `byte_or(...)`-shaped builtin, never + has no u8/u16 operator math and will not start - a `byte_or(...)`-shaped builtin, never `|=` on uint8), SPIR-V constant-storage lowering for literal fixed_arrays (QUIRK 20, emitter). Both proposed as lint rules too: module-global read reachable from a worker/team-lane kernel (QUIRK 9/27, bitten twice), dynamically-indexed fixed_array local in a kernel class (QUIRK 20's cheap half). -## Phase E — the document split (the detour) +## Phase E - the document split (the detour) `HOW_TO_ADD_A_FORMAT.md` (today: recipe + 27 quirks + 9 phase diaries) becomes: -1. **`HOW_TO_ADD_A_FORMAT.md`** — an actual how-to: steps 0-8 rewritten post-unquirk (the +1. **`HOW_TO_ADD_A_FORMAT.md`** - an actual how-to: steps 0-8 rewritten post-unquirk (the table shrinks the step count), the E-cluster rules folded in as steps/warnings (stubs first Q4; worker-table rule Q9/27; stale-binder Q10; stub-out-of-test-list Q16; kernel-probe-not-e2e Q22; census-the-vehicle Q23), and the quirk ledger cut to the survivors (whatever A/B/C/D did not kill). -2. **`performance/PERF_LEDGER.md`** — the nine per-format four-tier tables, the three gap +2. **`performance/PERF_LEDGER.md`** - the nine per-format four-tier tables, the three gap classes, each format's vehicle GGUF, and the box inventory: which model files sit on zen2 for testing, the m1/m5 paths, the ref llama.cpp builds per box. -3. **`followup_metal.md`** — twin of followup_vulkan.md: the Metal tg items (0.78-0.93x), +3. **`followup_metal.md`** - twin of followup_vulkan.md: the Metal tg items (0.78-0.93x), the Q22 dispatch-loop probe method, D2 if dropped. 4. The per-format phase diaries -> `/history` (doc_archiving skill), linked from the ledger. LAWS.md (modules/dasLLAMA) gets the rulings from this session: prune the families, saved zen2/M1 defaults, race only on feature unlock, CPU/GPU tune split, the doc split. -## Phase F — TWO PRs (the mid-plan cut that starts the M5 session) +## Phase F - TWO PRs (the mid-plan cut that starts the M5 session) - **PR-1** (after B): the 9-format ladder the branch already carries + the tuner rework + - C1 + D1. Full make_pr chain. On merge the parallel M5/M1 session forks from master — + C1 + D1. Full make_pr chain. On merge the parallel M5/M1 session forks from master - free mints (arm-neon profile), fast dev loop, pack-versioned images. -- **PR-2** (fresh branch off master): A (KqPlanes) + D2 + E (the doc split — it depends on +- **PR-2** (fresh branch off master): A (KqPlanes) + D2 + E (the doc split - it depends on A's survivor-quirk ledger anyway). Full make_pr chain again. - Parallel-window rule: the mac session owns kernel BODIES and additive arms (i8mm/SME seats, Metal levers); the zen2 session's A owns dispatch/tables. A5 (the Metal test @@ -156,9 +156,9 @@ zen2/M1 defaults, race only on feature unlock, CPU/GPU tune split, the doc split 1. The new-silicon trial runs on AWS (us-west-2, CLI configured on this box), not the dasweb zen4. No UMA + built-in-GPU x86 box exists on AWS; per goal the candidates are - c7a.2xlarge (zen4, avx512-vnni — the vnni512 mint), c7i.2xlarge (Sapphire Rapids, - vnni + AMX), c8g.2xlarge (Graviton4: i8mm + SVE2 — the arm-i8mm class once those - kernels exist), g4ad.xlarge (zen2 + Radeon V520 dGPU — Vulkan-on-AMD-driver, not UMA). + c7a.2xlarge (zen4, avx512-vnni - the vnni512 mint), c7i.2xlarge (Sapphire Rapids, + vnni + AMX), c8g.2xlarge (Graviton4: i8mm + SVE2 - the arm-i8mm class once those + kernels exist), g4ad.xlarge (zen2 + Radeon V520 dGPU - Vulkan-on-AMD-driver, not UMA). True UMA + Vulkan is Strix Halo territory, outside any cloud. Boris picks the box(es). 2. RESOLVED: profile classes key by the feature set that gates seats (winners follow instruction sets, not box identity). Host resolves to the highest class it satisfies diff --git a/skills/internal/llvm_tune_internals.md b/skills/internal/llvm_tune_internals.md index 9dd9a30d0d..3fe0d1f0b3 100644 --- a/skills/internal/llvm_tune_internals.md +++ b/skills/internal/llvm_tune_internals.md @@ -45,6 +45,9 @@ file is only what maintaining the implementation needs on top of it. - `cpu_supports` must answer on arm64 as well as x86 (sysctl / `AT_HWCAP` / `IsProcessorFeaturePresent`, LLVM target-feature spellings). An x86-only probe that fails closed leaves every arm `requires=` gate silently never firing. +- `DAS_TUNE_POLICY=reference` is env-only by construction: the `[tune]` stamps in required + libraries compile before the root that would declare it, so `missing = "reference"` is + rejected at the annotation. ## The gates @@ -85,4 +88,28 @@ tuner is a seconds-fast fake that upserts through `tune_manifest_set`. Two scope sidecar prove the upsert preserves the other's keys, and that tuned-ness is per-key completeness. `llvm_tune_manifest.das` covers the write->stamp round trip and the staleness rail (back-dating with `set_mtime` + `mktime`). Both run under `-jit` and short-circuit to -pass when `!jit_enabled()`. +pass when `!jit_enabled()`. The defaults-profile rail is `llvm_tune_profiles.das` (+ its +`_client` / `_lib` / `_tuner` fixtures): it writes its profile files at runtime under the +untracked `llvm_tune_profiles_defaults/`, named by this box's own `tune_cpu_class()`, and fakes +a less-capable minting box with an empty `features` field to force the unlocked-seat race. + +## Shipped defaults profiles - where the pieces live + +- `defaults=` resolves against the declaring file at `[tune_scope]` apply and rides + `TuneScope.defaultsPath`; the guard emission bakes it, together with the scope's + `scope_gated_families_joined` string - `family=requires+requires` rows, `;`-joined, only for + families with a gated row, read off the AST - into the `tune_auto_prepare` / + `tune_restart_needed` call, because the runtime has no AST. +- Adoption is runtime-side (`profile_try_adopt`): walk `tune_class_chain()` from + `tune_cpu_class()`, skip a profile whose version pin differs (the walk continues down), + `tune_sidecar_merge` the kernels into the app sidecar as a normal local write, then diff the + gated string against the profile's `features` fingerprint. It returns adopted-or-not and + hands the residue back in its `onlyFilter` out-param; the two signals read together - true + with an empty filter means fully covered, true with a filter means adopted-and-owing, and + an owing adoption skips the scope resolver and rides `run_scope_tuner(scope, onlyFilter)`, + which arms `DAS_TUNE_ONLY` for that one spawn. It declines outright when + `tune_manifest_path()` no longer equals the `manifestPath` the guard was compiled for. +- `tune_profile_export` carries `kernels` plus the string provenance that survives, dropping + `binary` / `box` / `engine_sha` and re-stamping `origin` = `profile`, `class` = the export's + class argument, and `written`: a profile's identity is class + version pin + features, never + a commit. diff --git a/skills/tune.md b/skills/tune.md index 458d717d28..cb5a7b0fcf 100644 --- a/skills/tune.md +++ b/skills/tune.md @@ -136,10 +136,7 @@ warning **names the missing kernels**. `version_of=` (optional, `"module/CONST"`) pins the scope to a library version: the sidecar's provenance must record that int constant's current value (under the lowercased constant name; `version_key=` overrides), so bumping the constant -on kernel work invalidates every box's winners. `defaults=` (optional, -resolved against the declaring file) names the directory of **shipped -defaults profiles** — see the section below; an untuned box adopts its CPU -class's profile instead of racing. The annotation names that +on kernel work invalidates every box's winners. The annotation names that module by string only, so the declaring module must `require` it as well (suppress the unused-require lint), and the scope's tuner must stamp the value with `tune_provenance_note` - an unstamped pin re-tunes on every @@ -149,7 +146,10 @@ completeness check everywhere it runs - the policy rail, `daspkg release per-app file - every library's tuner **upserts its own keys** and preserves everyone else's (that upsert is the isolation contract; "is this scope tuned" is per-key completeness, not file existence). Reading winners needs no scope -at all - every `[tune]` resolves against the app sidecar. +at all - every `[tune]` resolves against the app sidecar. `defaults=` (optional, +resolved against the declaring file) names the directory of shipped defaults +profiles (the *Shipped defaults profiles* section): an untuned box adopts its +CPU class's profile instead of racing. ```{warning} @@ -164,18 +164,21 @@ public surface (a blanket `public` on a module that also re-exports ## Shipped defaults profiles - `[tune_scope(defaults = "dir")]` Kernel winners follow **instruction sets, not boxes**: within one CPU feature -class the measured spread between seats is noise, so one minted answer serves -every box of the class. A library ships those answers as checked-in profile -files - `/.tune-defaults.json`, each the `"kernels"` section of a -reference box's full mint plus a provenance recording the minting box's -`features` fingerprint (and any `version_of=` pin value). Runtime knobs, race -tables and box identity never travel - knobs are per-box, devices are -per-device. +class the same permutation wins on every box - what differs from box to box +inside a class sits in the noise band - so one minted answer serves them all. +(A seat is the permutation slot one kernel family's winner occupies.) A +library ships those answers as checked-in profile files - +`/.tune-defaults.json`, each the `"kernels"` section of the +minting box's full mint plus a provenance recording that box's `features` +fingerprint (and any `version_of=` pin value). Runtime knobs, race tables and +box identity never travel - all three are properties of the box that measured +them, not of its class. The class names follow the features `requires=` can gate: `x86-vnni512` / -`x86-vnni256` / `x86-avx2` / `x86-base`, `arm-i8mm` / `arm-neon` -(`tune_cpu_class()` computes this box's, `tune_class_chain()` its adoption -ladder). An untuned auto/restart start walks the ladder from the box's own +`x86-vnni256` / `x86-avx2` / `x86-base`, `arm-i8mm` / `arm-neon`; any other +architecture gets `-`, a key no shipped profile matches, so +such a box always races (`tune_cpu_class()` computes this box's, +`tune_class_chain()` its adoption ladder). An untuned auto/restart start walks the ladder from the box's own class downward, adopts the first profile found into the app sidecar (a normal local write - staleness, box identity and the JIT DLL cache re-key all behave as for a mint), and then races only what the profile could not answer: @@ -186,16 +189,19 @@ as for a mint), and then races only what the profile could not answer: profile's recorded `features`. That residue races through the ordinary tuner spawn with the `--tune-only` -filter armed, so a zen4 box adopting the `x86-avx2` profile races only the -vpdpbusd seats, and a box whose class profile is exact races **nothing**. -Adoption is skipped entirely under `--tune` (a forced re-race stays a full -local mint), when the profile's pinned version mismatches, or when no profile -file matches any class in the ladder. - -A maintainer produces a profile on a reference box after a full `--tune` mint +filter armed, so a box whose own class is `x86-vnni512`, adopting an +`x86-avx2` profile, races only the families whose `requires=` names an +AVX-512 VNNI feature - and a box whose class profile is exact races +**nothing**. Adoption is skipped entirely under `--tune` - a forced re-race +stays a full local mint. A profile whose pinned version differs is skipped and +the walk continues to the next class down; when nothing in the ladder both +exists and matches, the box races the full grid. + +A maintainer produces a profile on the minting box after a full `--tune` mint with `tune_profile_export(path, klass)` - it refuses an empty or stale sidecar. Every sidecar save also stamps the box's `features` fingerprint into -provenance, which is what makes a future export race-on-unlock-aware. +provenance - that is what later lets an adopting box tell the seats the +profile's minting box already raced from the ones its own ISA unlocks. ## Application policy - `[tune_policy]` and `--tune` @@ -223,11 +229,6 @@ def main { * - `fallback` - stamp `fallback=` silently (also what `DAS_TUNE_POLICY=fallback` - the CI kill switch - forces everywhere) -* - `reference` - - serve the ORIGINAL bodies: no stamps at all, for `[tune]` families and - loop-hint `[tuned]` kernels alike. The A/B truth tier - `fallback` is - not it, because a chain's first viable perm can be the very stamp under - test. Usually via `DAS_TUNE_POLICY=reference`. * - `warn` - loud compile-time banner with the exact tuner command * - `error` @@ -244,6 +245,14 @@ def main { Programs whose root has no `main` never get the default - dastest-driven test files run `[test]` functions, so the test suite never tunes-on-start. +`DAS_TUNE_POLICY=reference` is the A/B truth tier: it serves the ORIGINAL +bodies - no stamps at all, for `[tune]` families and loop-hint `[tuned]` +kernels alike - on a fully tuned box too, and announces itself once per +compile. `fallback` is not it, because a chain's first viable perm can be the +very stamp under test. It is environment-only: `[tune_policy(missing = +"reference")]` is a compile error, because the `[tune]` stamps in required +libraries compile before the root that would declare it. + `--tune` after `--` on the application's command line forces the tune path even when the sidecar is complete (a re-tune; the flag is stripped from the re-exec so the child converges). `DAS_TUNE_POLICY` overrides the declared @@ -251,7 +260,7 @@ value - `DAS_TUNE_POLICY=fallback` is the CI kill switch. `--tune-only ` (comma-separated; implies `--tune`) re-tunes only the kernel families whose name contains one of the tokens - after landing one -family's kernels, `--tune-only iq2xs` re-mints that family in seconds instead +family's kernels, `--tune-only gemm` re-mints that family in seconds instead of walking every family the app owns. A skipped family races nothing and writes nothing, so its sidecar entry survives the upsert. The filter rides the tuner process chain as `DAS_TUNE_ONLY`, and a harness consults @@ -364,8 +373,10 @@ Two seams let a supervisor or a network service participate: - `tune_set_scope_resolver(fn)` - registered from an `[init]` (which must run before the guard at the top of `main`), consulted by - the auto/restart policy guards before spawning a scope's tuner (and after - the shipped-defaults profiles above - local answers outrank remote ones). A resolver + the auto/restart policy guards before spawning a scope's tuner - and after + the *Shipped defaults profiles* section's adoption, which outranks it: a + partial adoption, one that still owes a race, skips the resolver entirely + and races the residue instead. A resolver that can satisfy the scope another way (dasLLAMA's exchange client downloads a matching per-box sidecar from dasllama.io) returns true; completeness is re-checked, never trusted, and `--tune` never consults it. diff --git a/tests/metal/test_metal_consttab.das b/tests/metal/test_metal_consttab.das new file mode 100644 index 0000000000..2df2e9a674 --- /dev/null +++ b/tests/metal/test_metal_consttab.das @@ -0,0 +1,107 @@ +options gen2 +options indenting = 4 + +// Real-GPU behavioral gate for the hoisted constant tables: a `let` fixed-array local of literals +// lowers to a program-scope `constant` array (renamed when a helper reuses the name with different +// content), and the kernel's reads through it must return the staged values. The CPU run of the +// same body is the oracle; the GPU half is static_if-gated on das_metal and skips without a device. + +require dastest/testing_boost public +require metal/msl_shader +require _metal_common // nolint:STYLE030 — used inside the das_metal static_if arm only + +let N = 256 +let TPG = 64u + +def private ct_pick(i : uint) : uint { + let tbl = fixed_array(0x11u, 0x22u, 0x33u, 0x44u, 0x55u, 0x66u, 0x77u, 0x88u) + return tbl[i & 7u] +} + +// the same local name with different content: the emitter renames the hoisted twin +def private ct_pick2(i : uint) : uint { + let tbl = fixed_array(0x1u, 0x2u, 0x4u, 0x8u, 0x10u, 0x20u, 0x40u, 0x80u) + return tbl[i & 7u] +} + +class ConstTabGpu { + @ssbo @binding = 0 src : array + @ssbo @binding = 1 dst : array + @ssbo @binding = 2 fdst : array + + [metal_kernel(name="consttab_gpu_msl"), marker(no_coverage)] + def consttab { + let i = gl_GlobalInvocationID.x + let sgn = fixed_array(1.0, 0.5) + dst[i] = src[i] + ct_pick(i) + ct_pick2(i) + fdst[i] = sgn[i & 1u] * 2.0 + } +} + +[test] +def test_consttab_gpu_vs_cpu(t : T?) { + t |> run("hoisted constant tables: GPU == CPU-reference") <| @(t : T?) { + var oracle = new ConstTabGpu() + oracle.src |> resize(N) + oracle.dst |> resize(N) + oracle.fdst |> resize(N) + for (k in range(N)) { + oracle.src[k] = uint(k * 3) + } + for (k in range(N)) { + gl_GlobalInvocationID = uint3(uint(k), 0u, 0u) + oracle->consttab() + } + t |> equal(oracle.dst[1], 3u + 0x22u + 0x2u) // the oracle itself: src 3, tbl[1], tbl_c0[1] + t |> equal(oracle.fdst[1], 1.0) // sgn[1] * 2 + static_if (typeinfo builtin_module_exists(das_metal)) { + var gpu_bad = -1 + with_metal_device() $(dev : MetalDevice?) { + if (dev == null) { + feint("no Metal device on this box; GPU compare skipped\n") + return + } + var perr : string + var pso = pipeline_from_source(dev, consttab_gpu_msl, consttab_gpu_msl_entry, consttab_gpu_msl_fastmath, perr) + t |> success(pso != null, "pipeline_from_source: {perr}") + if (pso == null) { + return + } + var queue = metal_new_command_queue(dev) + var bsrc = buf_upload(dev, oracle.src) + var bdst = buf_fill(dev, N, 0u) + var bfdst = buf_fill(dev, N, 0.0) + var bufs <- [bsrc, bdst, bfdst] + var rerr : string + let ran = run_compute_1d(queue, pso, bufs, uint(N), TPG, rerr) + t |> success(ran, "run_compute_1d: {rerr}") + if (ran) { + gpu_bad = buf_mismatch_exact(bdst, oracle.dst) + buf_mismatch_exact(bfdst, oracle.fdst) + } + bufs |> clear() // non-owning handles — clear() skips pointee finalize + unsafe { + delete bufs + } + metal_release(bsrc) + metal_release(bdst) + metal_release(bfdst) + metal_release(pso) + metal_release(queue) + } + if (gpu_bad >= 0) { + t |> equal(gpu_bad, 0) + } + t |> equal(metal_live_object_count(), 0l) + } else { + feint("das_metal is not built on this platform; CPU-reference half only\n") + } + unsafe { + delete oracle + } + } +} + +[export] +def main { + return +} diff --git a/tests/msl/REVIEW.md b/tests/msl/REVIEW.md index cf5cf5f07f..e576b04203 100644 --- a/tests/msl/REVIEW.md +++ b/tests/msl/REVIEW.md @@ -11,8 +11,11 @@ from `declared_msl_census` or a fixture's census from `all_msl_censuses` (both i `_msl_common.das`). Emitted text cannot be read back into constructs, so the census is the only measure of coverage. +**A diff that adds a fixture outside `_fail_closed/` adds its census to `all_msl_censuses` +(`_msl_common.das`) in the same change.** A fixture left out of that list emits kinds the +check never sees, so both directions pass with the fixture uncounted. + **Weakening `test_msl_fail_closed.das`'s `check_rejects` is a defect.** It asserts each `_fail_closed/` fixture fails to compile and that the compile error contains the fixture's needle - the substring naming the rejected construct. The fixture-has-its-assert pairing is -machine-checked both directions (`modules/dasMetal/REVIEW.das`'s `check_fail_closed_sync`; -its weakening rule lives with that checklist). +machine-checked both directions (`modules/dasMetal/REVIEW.das`'s `check_fail_closed_sync`). diff --git a/tests/msl/_msl_common.das b/tests/msl/_msl_common.das index ab2024d2e5..c09c142bb3 100644 --- a/tests/msl/_msl_common.das +++ b/tests/msl/_msl_common.das @@ -701,6 +701,12 @@ def private ctb_pick(i : uint) : uint { return tbl[i & 7u] } +// the same local name with different content: the hoist renames the second table (tbl_c) +def private ctb_pick2(i : uint) : uint { + let tbl = fixed_array(0x1u, 0x2u, 0x4u, 0x8u, 0x10u, 0x20u, 0x40u, 0x80u) + return tbl[i & 7u] +} + class ConstTab { @ssbo @binding = 0 src : array @ssbo @binding = 1 dst : array @@ -710,7 +716,7 @@ class ConstTab { def consttab { let i = gl_GlobalInvocationID.x let sgn = fixed_array(1.0, 0.5) - dst[i] = src[i] + ctb_pick(i) + dst[i] = src[i] + ctb_pick(i) + ctb_pick2(i) fdst[i] = sgn[i & 1u] * 2.0 } } @@ -1317,6 +1323,7 @@ def public declared_msl_census : table { // nolint:STYLE038 — one fl // statements "stmt.let", "stmt.let.const_table", + "stmt.let.const_table.renamed", "stmt.var", "stmt.var.zero", "stmt.assign", diff --git a/tests/msl/test_msl_functions.das b/tests/msl/test_msl_functions.das index cba0ff1d75..aa8681a3f7 100644 --- a/tests/msl/test_msl_functions.das +++ b/tests/msl/test_msl_functions.das @@ -147,7 +147,9 @@ def private hasq(t : T?; needle : string) { def test_msl_const_tables(t : T?) { t |> run("literal fixed-array locals hoist to program-scope constant tables") <| @(t : T?) { hasq(t, "constant uint tbl[8] = \{") // hoisted from the helper's body + hasq(t, "constant uint tbl_c0[8] = \{") // the same-named different-content table renamed, not merged hasq(t, "constant float sgn[2] = \{") // hoisted from the kernel body hasq(t, "tbl[") // reads index the hoisted table by name + hasq(t, "tbl_c0[") // the renamed table's reads follow the rename } } diff --git a/utils/REVIEW.md b/utils/REVIEW.md index 96962616dc..30d67c0e55 100644 --- a/utils/REVIEW.md +++ b/utils/REVIEW.md @@ -14,6 +14,11 @@ checklist - not with the checklist of the directory it sits in.** **A file in a `utils/` library directory (`common/`) is reviewed with this checklist and with the checklist of every tool that requires it.** +**A diff under `utils/` that changes how a `.dlim` is built from a gguf, how one is loaded, or +what identifies one - the fields that decide whether two `.dlim`s are the same image - answers +to `modules/dasLLAMA/REVIEW.md` (repo root) too.** A `utils/` diff never opens that checklist +on its own. + **A tool's file outside `utils/` answers to the checklist of the folder that contains it as well as to this one.** @@ -31,21 +36,25 @@ removed entry. **A diff that deletes a tool outright records the decision beside `DAS_UTILS_SHIPPED_EXES` in `CMakeLists.txt` (beside this file), in the same change.** -**A test the diff adds or changes alongside a change under `utils/`, whose load-bearing -assertions a CI lane runs against the change - the assertions that prove it, not a skip-path +**A test the diff adds or changes that covers a change under `utils/`, whose load-bearing +assertions a CI lane can run against the change - the assertions that prove it, not a skip-path assertion - ships with a CI row that executes those assertions, wherever the diff puts the test, added in the same change if no row already covers it.** A row that only compile-checks the test (`dastest --compile-only`) does not execute them. A test whose assertions no row executes never runs again. -**A test whose only executing row runs against an already-deployed artifact (a nightly lane -driving the live site) takes the same obligation as a compile-only row: the PR description +**A test the diff adds or changes that covers a change under `utils/`, whose only executing +row runs against an already-deployed artifact (a nightly lane driving the live site), takes +the same obligation as a compile-only row: the PR description records an executed local run of those assertions against the change.** A lane that tests production after merge proves nothing about the diff under review. -**A test the diff adds or changes alongside a change under `utils/`, whose load-bearing +**A test the diff adds or changes that covers a change under `utils/`, whose load-bearing assertions no CI lane can run, ships with a CI row that compile-checks it.** -**A test the diff adds or changes alongside a change under `utils/` that gets a compile-only row records its executed -run in the PR description**: the machine the assertions ran on, what that machine had that CI -lacks, and the pass count. +**A test the diff adds or changes that covers a change under `utils/`, whose only row +compile-checks it or whose only executing row runs against an already-deployed artifact - a +nightly lane driving the live site - records its executed run in the PR description**: the +machine the assertions ran on, what that machine had that CI lacks, and the pass count. A lane +that tests production after merge proves nothing about the diff under review. + diff --git a/utils/mcp/REVIEW.md b/utils/mcp/REVIEW.md index 4e67bfac41..8ac4a30221 100644 --- a/utils/mcp/REVIEW.md +++ b/utils/mcp/REVIEW.md @@ -8,7 +8,9 @@ interpreted through `.mcp.json` instead.** Development runs the server through the python keep-alive supervisor, so an exe form would never be used in development before it ships. -**A diff that adds a top-level `.das` under `utils/mcp/` that `main.das` reaches also adds it to -the `install(FILES ...)` block in `CMakeLists.txt` (repo root), in the same change.** `tools/` -and `subtools/` are globbed; a top-level file left out of the list dies in the shipped SDK on -`error[20605] missing prerequisite` while the in-tree server keeps working. +**A diff that adds a top-level `.das` under `utils/mcp/` that the shipped SDK runs or loads - +`main.das` reaches it, or it has its own `main` that something in the shipped SDK runs - also +adds it to the `install(FILES ...)` block that lists `utils/mcp/main.das` in `CMakeLists.txt` +(repo root), in the same change.** `tools/` and `subtools/` are globbed; a top-level file left +out of the list dies in the shipped SDK on `error[20605] missing prerequisite` while the +in-tree server keeps working. From e86c13907a2cc80f1e9fb482d58e73ef587db628 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Mon, 31 Aug 2026 20:42:15 -0700 Subject: [PATCH 58/61] the fresh dragon's wording repairs across the six checklists The tests-route enumeration, the attribution pair's boundary, the team-lane pointer, the record-hygiene census, the suite-listing duplicate, the media-carrier and ASR carve-outs folded into their triggers, the timing-instrument word, the utils duplicate rule, and the over-long wraps. setup.das carries the earlier comment caps. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- modules/dasLLAMA/REVIEW.md | 30 +++++++++++++------------- modules/dasLLAMA/REVIEW_PLACEMENT.md | 21 +++++++++--------- modules/dasLLAMA/benchmarks/REVIEW.md | 7 +++--- modules/dasLLAMA/performance/REVIEW.md | 17 ++++++--------- modules/dasLLAMA/tests/REVIEW.md | 26 +++++++++++----------- modules/dasLLVM/REVIEW.md | 2 -- utils/REVIEW.md | 6 ------ utils/mcp/setup.das | 5 +++-- 8 files changed, 51 insertions(+), 63 deletions(-) diff --git a/modules/dasLLAMA/REVIEW.md b/modules/dasLLAMA/REVIEW.md index bb187c85c0..22699c3a26 100644 --- a/modules/dasLLAMA/REVIEW.md +++ b/modules/dasLLAMA/REVIEW.md @@ -28,8 +28,7 @@ winners back. tune-boot path that reaches it, applies `REVIEW_EXCHANGE.md`.** **Every `dasllama/` change applies this folder's `tests/REVIEW.md`.** The folder walk surfaces -that checklist only for diffs under `tests/`; its census, kargs and new-value duties trigger -on `dasllama/`. +that checklist only for diffs under `tests/`. **A GPU kernel, driver, dispatch-class, or K/V-mirror change applies `REVIEW_GPU.md`.** @@ -189,16 +188,17 @@ the file it checks - in `ARCHITECTURE_ENGINE.md`, `ARCHITECTURE_MEDIA.md`, or `ARCHITECTURE_GPU.md`.** The line names the check and the names it licenses. A licensed name is one that check does not flag. When the check licenses no names, the line says so. -**Checked-in prose this module owns - docs and comments, any language - describes an upstream -mechanism in our own terms: no "lifted/ported verbatim from", and no upstream symbol, header, -or constant name; write "the reference exe" or "upstream" instead.** A symbol this file's own -code calls or carries as a value is its own name, not attribution. +**Checked-in prose this module owns - docs and comments, any language - that is not locating +or reproducing work against the reference build describes an upstream mechanism in our own +terms: no "lifted/ported verbatim from", and no upstream symbol, header, constant, or binary +name - write "the reference exe" or "upstream" instead.** A symbol the file carrying that +prose calls or holds as a value is its own name, not attribution. -**A line whose job is to locate or reproduce work against the reference build names the -binary outright** - a path naming where checked-in data is regenerated FROM, an env-knob row -whose value locates the reference binary, a command line or flag list in a methodology, -profile, bring-up, or how-to document, and a follow-up ledger's board row naming the build it -compares against. +**A line whose job is to locate or reproduce work against the reference build names that +build's binary outright** - a path naming where checked-in data is regenerated FROM, an +env-knob row whose value locates the reference binary, a command line or flag list in a +methodology, profile, bring-up, or how-to document, and a follow-up ledger's board row naming +the build it compares against. **Legal attribution never appears in prose - it lives in `THIRD_PARTY_NOTICES.md` and the `LICENSE.*` files.** @@ -241,7 +241,7 @@ that list, so a name outside it is never recorded and a box adopting a shipped p the tuning the profile was meant to save. **A value that a team-lane kernel reads - anything reachable from a `team_parallel_for` / -`team_parallel_for_indexed` / `team_parallel_stages` body (`dasllama/dasllama_par.das`) - is a `def` -returning it, never a module global with a declaration initializer (`let` or `var`).** A team -lane never runs global initializers, so the global reads zero there while every -single-threaded run reads the right value. +`team_parallel_for_indexed` / `team_parallel_stages` body (`daslib/jobque_boost.das`, repo +root) - is a `def` returning it, never a module global with a declaration initializer (`let` +or `var`).** A team lane never runs global initializers, so the global reads zero there while +every single-threaded run reads the right value. diff --git a/modules/dasLLAMA/REVIEW_PLACEMENT.md b/modules/dasLLAMA/REVIEW_PLACEMENT.md index 6ad853c663..52ec83e497 100644 --- a/modules/dasLLAMA/REVIEW_PLACEMENT.md +++ b/modules/dasLLAMA/REVIEW_PLACEMENT.md @@ -16,11 +16,12 @@ per-file list. A rule naming what KIND of code lands in which file is the checkl **A disk-order -> compute-order transform lands per scope: kernel-layout in `dasllama/dasllama_repack.das`, load-scope in `dasllama/dasllama_layout.das`.** -**A CPU KV-cache store, read, score dot, or V-accumulate lands in `dasllama/dasllama_kv_codec.das`, -its format family kept whole.** GPU twins land in their backend kernel file. +**A CPU KV-cache store, read, score dot, or V-accumulate lands in +`dasllama/dasllama_kv_codec.das`, its format family kept whole.** GPU twins land in their +backend kernel file. -**A pre-tokenizer split lands in `dasllama/dasllama_pretok.das`; a merge algorithm in its backend file -(`dasllama/dasllama_spm.das` / `dasllama/dasllama_bpe.das`).** +**A pre-tokenizer split lands in `dasllama/dasllama_pretok.das`; a merge algorithm in its +backend file (`dasllama/dasllama_spm.das` / `dasllama/dasllama_bpe.das`).** **A kernel body lands in its owner's backend file.** A GPU kernel body lands in the file where its PSO is compiled and released. A CPU-tier kernel body lands in that tier's @@ -50,14 +51,14 @@ file** - a single-caller helper sanctioned as tower-worthy is ledgered on **Tool wire text (the text of a model's tool/function call, built or parsed) is produced only in `dasllama/dasllama_tools.das`.** -**No engine file (`dasllama/`) other than `dasllama/dasllama_audio_io.das` requires `audio` (the -miniaudio decode module).** +**No engine file (`dasllama/`) other than `dasllama/dasllama_audio_io.das` requires `audio` +(the miniaudio decode module).** -**No engine file (`dasllama/`) other than `dasllama/dasllama_vision_io.das` requires `stbimage`.** -Benchmarks, harnesses, and tests decode their own fixtures. +**No engine file (`dasllama/`) other than `dasllama/dasllama_vision_io.das` requires +`stbimage`.** Benchmarks, harnesses, and tests decode their own fixtures. -**Engine, HTTP, or writer logic never lands in `dasllama/dasllama_scheduler.das`** - engine logic in -engine files, HTTP in the server, writer logic in the writer's own file. +**Engine, HTTP, or writer logic never lands in `dasllama/dasllama_scheduler.das`** - engine +logic in engine files, HTTP in the server, writer logic in the writer's own file. **An `[init]`-only side-effect require in an engine file (`dasllama/`) lives in `dasllama/dasllama_transformer.das`** - arch registrations, GPU tiers, every module requiring diff --git a/modules/dasLLAMA/benchmarks/REVIEW.md b/modules/dasLLAMA/benchmarks/REVIEW.md index 7d26cf1830..202e264bae 100644 --- a/modules/dasLLAMA/benchmarks/REVIEW.md +++ b/modules/dasLLAMA/benchmarks/REVIEW.md @@ -67,6 +67,7 @@ targets it builds, or a patch it applies - moves the measured quantity. `dot_q8q8_laneq4x4` last.** That bench pins the repack backend for the rest of the process, so a row after it races against the pinned backend instead of the one it selects. -**A harness whose run can end with zero result rows exits non-zero when it does - wrong -flags, failed load, a device that declines.** A run that matched nothing and reported success -leaves a sidecar or a record untouched and a caller none the wiser. +**A timing instrument this checklist governs whose run can end with zero result rows exits +non-zero when it does - wrong flags, failed load, a device that declines.** A run that matched +nothing and reported success leaves a sidecar or a record untouched and a caller none the +wiser. diff --git a/modules/dasLLAMA/performance/REVIEW.md b/modules/dasLLAMA/performance/REVIEW.md index 12827343f5..ab1cde9915 100644 --- a/modules/dasLLAMA/performance/REVIEW.md +++ b/modules/dasLLAMA/performance/REVIEW.md @@ -8,10 +8,9 @@ validate through `../dasllama/dasllama_exchange_schema.das` instead.** The engin `dasllama/` require beyond the lint macro module) is `REVIEW.das`'s to enforce; weakening that gate is a defect. -**Weakening `REVIEW.das`'s record-hygiene checks - `hardware.remote_desktop` `off` on every -stored run, `provenance.noise` `ok` on every checked-in JSON under this folder that carries a -`provenance` block, no `engine_sha` in a `defaults/` profile - is a defect; the fix for a red -is a re-mint on a quiet, session-free box, never an edit.** +**Weakening `REVIEW.das`'s record-hygiene checks is a defect; the fix for a red is a re-mint +on a quiet, session-free box, never an edit.** What each check enforces is read from the gate +itself; each check's finding text states its own rule. **A diff that writes a commit stamp anywhere under this folder naming a commit the branch under review cannot reach is a defect - re-mint, or re-stamp to a reachable commit whose @@ -35,10 +34,9 @@ legs by `../benchmarks/asr/requirements-*.txt`. `requirements-*.txt` - re-mints every row the moved pin's tool minted, or withdraws them.** **A diff that writes a provenance manifest under this folder whose `provenance.dasllama_version` -differs from the `DASLLAMA_RELEASE` string (`../dasllama/dasllama_version.das`; the JSON key -stays `dasllama_version` across constant renames) is a defect - re-mint.** For a sidecar -with an `engine_sha`, read the value at that commit; a `defaults/` profile compares against -the branch under review. +differs from the `DASLLAMA_RELEASE` string (`../dasllama/dasllama_version.das`) is a defect - +re-mint.** For a sidecar with an `engine_sha`, read the value at that commit; a `defaults/` +profile compares against the branch under review. **A diff that writes a row to `records/.json` mints that row from a board cell.** A board cell is one `gen_bench_records.das` spawns, or a manual `../benchmarks/lcpp_bench.das` cell @@ -99,9 +97,6 @@ the row's canonical values** - the verify never reads the url of a file already **A diff that refreshes `last_known_good_sidecar.json` replaces it with one complete mint from the box its provenance names, at the current `DASLLAMA_RELEASE` - never a hand-edited copy.** -`REVIEW.das` (beside this file) machine-checks the age-independent half (`noise` `ok`, -`validation` `ok`, every `race` winner equal to its `kernels` value); weakening that gate is a -defect. **A diff that changes the exchange consent notice wording updates every checked-in copy in the same change, and a diff that adds a copy names it here in the same change: diff --git a/modules/dasLLAMA/tests/REVIEW.md b/modules/dasLLAMA/tests/REVIEW.md index 9adc6a8098..137356e550 100644 --- a/modules/dasLLAMA/tests/REVIEW.md +++ b/modules/dasLLAMA/tests/REVIEW.md @@ -14,8 +14,7 @@ what the runner arms for `model-free`. The listing lands in the same change that file, and the file skips honestly when its models are absent. **Listing a test file that `DASLLAMA_CPU_PREFILL=1` disarms in any `run.das` suite is a defect, -and so is leaving that fact out of the file's header.** `DASLLAMA_CPU_PREFILL=1` is what the -runner arms for `model-free`. +and so is leaving that fact out of the file's header.** **Invoking dastest directly on a test file in a `run.das` model suite (every suite but `model-free`) is a defect; such a file runs only through `run.das`. A `model-free` file runs @@ -188,13 +187,13 @@ it to the documented default. A runtime decline standing in for a pin measures w lane the box's policy picked. **A cell that loads a media carrier under a lane pin - a `set__q8`-class knob or a -tensor-crown pin - mints in memory through the family's `stage_*` + `mint_*` pair, never -through a `.dlim`-baking loader (`load__tower` / `load__encoder` / -`load_model*`).** A disk bake under a pinned lane GC-purges the serving lane's `.dlim` -beside the model, and the next direct-image load in another suite panics on the wrong -identity. The one residue: a cell whose SUBJECT is a facade lane knob (`load_asr_model` -under `set_asr_tower_fp32`) keeps the facade loader - the image identity folds the pin, so -minting around it would unmake the claim. +tensor-crown pin - and whose subject is not that lane knob itself mints in memory through the +family's `stage_*` + `mint_*` pair, never through a `.dlim`-baking loader +(`load__tower` / `load__encoder` / `load_model*`).** A disk bake under a +pinned lane GC-purges the serving lane's `.dlim` beside the model, and the next direct-image +load in another suite panics on the wrong identity. A cell whose subject is the lane knob +(`load_asr_model` under `set_asr_tower_fp32`) keeps the facade loader: the image identity +folds the pin, so minting around it would unmake the claim. **A CPU-vs-GPU arm that does not run a PLANAR model for its CPU stages, and that model's `blob_twin(t, path, seq_cap)` for override-selected stages, is a defect.** One session spans @@ -274,9 +273,8 @@ route's to say: a twin-W route reads the baked halfword twin (`wblob`), so poiso plane alone is a valid control there, while a route reading both planes needs both zeroed. A poison the served route never reads passes on a broken kernel. -**An ASR cell comparing transcripts across two serving lanes asserts TOKEN equality; the one -cell comparing a crowned lane - the raced kernel form a tune sidecar arms as the serving one - +**An ASR cell comparing transcripts across two serving lanes asserts TOKEN equality; a cell +comparing a crowned lane - the raced kernel form a tune sidecar arms as the serving one - against its tensor twin asserts WORD equality, because the twins' rounding legitimately flips -tokens.** A cell that cannot hold -its grade converts to the forced-feed logits-tolerance form - never to a looser text -compare. +tokens.** A cell that cannot hold its grade converts to the forced-feed logits-tolerance form - +never to a looser text compare. diff --git a/modules/dasLLVM/REVIEW.md b/modules/dasLLVM/REVIEW.md index 57a44b17ab..8f8bb7e33f 100644 --- a/modules/dasLLVM/REVIEW.md +++ b/modules/dasLLVM/REVIEW.md @@ -61,5 +61,3 @@ `has_env_variable(expr)` outside `daslib/llvm_env.das`. Spell the name as a literal through the declared forms, or declare the knob, instead** (the literal-name forms are scanner-enforced by `tests/llvm_env_registry.das`; weakening that test is a defect). - - diff --git a/utils/REVIEW.md b/utils/REVIEW.md index 30d67c0e55..9f0d2cbdab 100644 --- a/utils/REVIEW.md +++ b/utils/REVIEW.md @@ -43,12 +43,6 @@ test, added in the same change if no row already covers it.** A row that only co the test (`dastest --compile-only`) does not execute them. A test whose assertions no row executes never runs again. -**A test the diff adds or changes that covers a change under `utils/`, whose only executing -row runs against an already-deployed artifact (a nightly lane driving the live site), takes -the same obligation as a compile-only row: the PR description -records an executed local run of those assertions against the change.** A lane that tests -production after merge proves nothing about the diff under review. - **A test the diff adds or changes that covers a change under `utils/`, whose load-bearing assertions no CI lane can run, ships with a CI row that compile-checks it.** diff --git a/utils/mcp/setup.das b/utils/mcp/setup.das index 151638a641..567c51d4f5 100644 --- a/utils/mcp/setup.das +++ b/utils/mcp/setup.das @@ -176,7 +176,7 @@ def ensure_sgconfig(root : string) { // Stage the JIT backend files that a plain `--target daslang` build does NOT produce: // /lib/LLVM.dll (the LLVM codegen backend, loaded by name from dasLLVM's bindings) and -// /bin/lld-link.exe (links each JIT-emitted .dll). Only the dasLLVM shared-lib CMake step +// on Windows /bin/lld-link.exe (links each JIT-emitted .dll). Only the dasLLVM shared-lib CMake step // stages these, and building just the `daslang` target skips it — so without this a fresh // worktree binary silently falls back to the interpreter (`-jit` fails: "can't load LLVM.dll"). // Copy them from the source tree of the daslang running this setup (get_das_root()), which is a @@ -204,7 +204,8 @@ def stage_jit_backend(root : string) { fallback_source = "DASLANG does not name an existing binary" } } - //! lib/LLVM.dll is the codegen backend's name on every platform; its .version stamp stages with it or a later cmake refetches LLVM. + // lib/LLVM.dll is the codegen backend's name on every platform + // its .version stamp stages with it, or a later cmake configure refetches LLVM var jit_files <- ["lib/LLVM.dll", "lib/LLVM.dll.version"] if (get_platform_name() == "windows") { jit_files |> push("bin/lld-link.exe") From d9a3b7ed09ed1537e77fb9b49a8408c6f40fc806 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Mon, 31 Aug 2026 21:50:44 -0700 Subject: [PATCH 59/61] PR-1 gate round: the emitter's renamed-table probe, the IQ3_XXS trim reserve, the emitter pin, and the lint/format residue The external round's two P2s, each red-first: - the msl emitter renamed a same-named constant table to tbl_c without checking whether a helper's own table already held that name, so the rename overwrote it and that helper read the wrong constants. The generated name now probes past every held name; the msl fixture gains a helper owning tbl_c0, the test asserts three surviving tables and the tbl_c1 reads, and the tests/metal oracle proves GPU == CPU with the held name in play. - the IQ3_XXS trim arm bare-resized the @exact_size embedding planes where every sibling arm reserves first; past max_unreserved_size that panics before the copy. PERF032 missed it: its reserve check is function-scoped, so an earlier arm's reserve satisfies every later arm of the ladder. The chain's residue: LLVM_JIT_EMITTER_HASH re-pinned for the i8mm emitter change; two files formatted to the gate's spelling; LINT018/LINT020/LINT002/ LINT003 in the new tests; the dragon's round-4 wording repairs (the crown pin named by its setters, the reference-tool pin named by DEFAULT_REF_SHA, the tensor-twin gloss, PSO expanded, the tests-route and locate-rule wording). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- modules/dasLLAMA/REVIEW.md | 23 ++++++------ modules/dasLLAMA/REVIEW_PLACEMENT.md | 13 +++---- modules/dasLLAMA/benchmarks/REVIEW.md | 3 ++ modules/dasLLAMA/dasllama/dasllama_common.das | 2 +- .../dasllama/dasllama_gpu_resident.das | 4 +-- .../dasllama/dasllama_metal_kernels.das | 2 +- modules/dasLLAMA/performance/REVIEW.md | 7 ++-- modules/dasLLAMA/tests/REVIEW.md | 35 ++++++++++--------- modules/dasLLAMA/tests/test_gguf_quant.das | 4 +-- modules/dasLLAMA/tests/test_kquant.das | 6 ++-- modules/dasLLAMA/tests/test_model_image.das | 2 +- modules/dasLLVM/REVIEW.md | 10 +++--- modules/dasLLVM/daslib/llvm_jit_run.das | 2 +- modules/dasMetal/metal/msl_emit.das | 7 +++- tests/metal/test_metal_consttab.das | 9 +++-- tests/msl/_msl_common.das | 8 ++++- tests/msl/test_msl_functions.das | 16 +++++++-- utils/REVIEW.md | 1 - 18 files changed, 95 insertions(+), 59 deletions(-) diff --git a/modules/dasLLAMA/REVIEW.md b/modules/dasLLAMA/REVIEW.md index 22699c3a26..0b87b0991b 100644 --- a/modules/dasLLAMA/REVIEW.md +++ b/modules/dasLLAMA/REVIEW.md @@ -27,8 +27,8 @@ winners back. **A change to the sidecar-exchange client (`dasllama/dasllama_exchange.das`), or to a tune-boot path that reaches it, applies `REVIEW_EXCHANGE.md`.** -**Every `dasllama/` change applies this folder's `tests/REVIEW.md`.** The folder walk surfaces -that checklist only for diffs under `tests/`. +**Every `dasllama/` change applies this folder's `tests/REVIEW.md`.** A `dasllama/` diff never +opens that checklist on its own. **A GPU kernel, driver, dispatch-class, or K/V-mirror change applies `REVIEW_GPU.md`.** @@ -133,13 +133,14 @@ the name in the same change). **A change to code or data of `encode`/`bpe_encode` or anything they reach in `dasllama/dasllama_spm.das` / `dasllama/dasllama_bpe.das` / `dasllama/dasllama_pretok.das` -ships before/after `--tok` rows (this folder's `benchmarks/lcpp_bench.das`) for the affected backend** - the -instrument is the scaling ratio across the size ladder, and superlinear is a defect. +ships before/after `--tok` rows (this folder's `benchmarks/lcpp_bench.das`) for the affected +backend** - the instrument is the scaling ratio across the size ladder, and superlinear is a +defect. **A change to code or data in `dasllama/dasllama_tokenizer.das`, `dasllama/dasllama_spm.das`, `dasllama/dasllama_bpe.das`, or `dasllama/dasllama_pretok.das`, or to the special-token or -template strings any of them look up, records a run of this folder's `tests/test_tokenizer.das` with its cases -EXECUTED, not skipped.** +template strings any of them look up, records a run of this folder's +`tests/test_tokenizer.das` with its cases EXECUTED, not skipped.** **A diff that adds an override, or gives one a new effect, without the announce is a defect.** An override is an environment knob, an exported runtime setter, or an on-disk state file that @@ -191,14 +192,16 @@ is one that check does not flag. When the check licenses no names, the line says **Checked-in prose this module owns - docs and comments, any language - that is not locating or reproducing work against the reference build describes an upstream mechanism in our own terms: no "lifted/ported verbatim from", and no upstream symbol, header, constant, or binary -name - write "the reference exe" or "upstream" instead.** A symbol the file carrying that -prose calls or holds as a value is its own name, not attribution. +name - write "the reference exe" or "upstream" instead.** The reference build is the +third-party engine this module measures itself against - the checkout +`benchmarks/setup_lcpp_ref.das` pins. A symbol the file carrying that prose calls or holds as +a value is its own name, not attribution. **A line whose job is to locate or reproduce work against the reference build names that build's binary outright** - a path naming where checked-in data is regenerated FROM, an env-knob row whose value locates the reference binary, a command line or flag list in a -methodology, profile, bring-up, or how-to document, and a follow-up ledger's board row naming -the build it compares against. +methodology, profile, bring-up, or how-to document, and a follow-up ledger's row naming the +build its numbers compare against. **Legal attribution never appears in prose - it lives in `THIRD_PARTY_NOTICES.md` and the `LICENSE.*` files.** diff --git a/modules/dasLLAMA/REVIEW_PLACEMENT.md b/modules/dasLLAMA/REVIEW_PLACEMENT.md index 52ec83e497..88c36f2945 100644 --- a/modules/dasLLAMA/REVIEW_PLACEMENT.md +++ b/modules/dasLLAMA/REVIEW_PLACEMENT.md @@ -8,8 +8,8 @@ or lands a kernel, codec, transform, tokenizer, tool-wire, media-IO or registrat in a new place applies this list together with `REVIEW.md`.** **A per-file inventory restated in this checklist is a defect of the checklist.** The sec.1 -charters - `ARCHITECTURE_ENGINE.md`, `ARCHITECTURE_GPU.md`, `ARCHITECTURE_MEDIA.md` - own the -per-file list. A rule naming what KIND of code lands in which file is the checklist's own. +charters own the per-file list; a rule naming what KIND of code lands in which file is the +checklist's own. **A tensor format conversion lands in `dasllama/dasllama_convert.das`.** @@ -24,9 +24,9 @@ backend kernel file. backend file (`dasllama/dasllama_spm.das` / `dasllama/dasllama_bpe.das`).** **A kernel body lands in its owner's backend file.** A GPU kernel body lands in the file where -its PSO is compiled and released. A CPU-tier kernel body lands in that tier's -`dasllama/dasllama_math_.das`. A kernel body never lands in -`dasllama/dasllama_math.das` or in a lens/dispatch macro file. +its pipeline state object (PSO) is compiled and released. A CPU-tier kernel body lands in that +tier's `dasllama/dasllama_math_.das`. A kernel body never lands in +`dasllama/dasllama_math.das` or in a file whose job is declaring kernels and routing dispatch. **A family quirk lands in the family file; a piece two families need moves UP into the concern's shared file (its own file when none exists)** - never sideways into a sibling. @@ -62,7 +62,8 @@ logic in engine files, HTTP in the server, writer logic in the writer's own file **An `[init]`-only side-effect require in an engine file (`dasllama/`) lives in `dasllama/dasllama_transformer.das`** - arch registrations, GPU tiers, every module requiring -the engine back; it sits in `dasllama/dasllama_common.das` only if engine code needs it. A +the engine back. It lives in `dasllama/dasllama_common.das` instead when code in +`dasllama/dasllama_common.das` itself depends on that module's registration having run. A program root (test, harness, benchmark, tool) requires the registration module it needs directly. diff --git a/modules/dasLLAMA/benchmarks/REVIEW.md b/modules/dasLLAMA/benchmarks/REVIEW.md index 202e264bae..48a6cc4f9e 100644 --- a/modules/dasLLAMA/benchmarks/REVIEW.md +++ b/modules/dasLLAMA/benchmarks/REVIEW.md @@ -5,6 +5,9 @@ doc: `../ARCHITECTURE_MEASUREMENT.md` (the benchmark rig: `#one-benchmark-rig`). `../followup_vulkan.md` for anything about the Vulkan backend, `../PERF_LEDGER.md` for any other performance followup, `../followup_general.md` for everything else. +**Routed from `../REVIEW.md`: a timing rig - a script whose output is a measured wall or rate - +wherever the diff puts it, applies this list together with `../REVIEW.md`.** + **A diff that adds or changes an instrument whose timed body runs a forward pass through a pipeline the dasLLAMA engine selected also calls `tune_gate()` (`../performance/profile_common.das`) before that instrument's first timed rep.** A pipeline diff --git a/modules/dasLLAMA/dasllama/dasllama_common.das b/modules/dasLLAMA/dasllama/dasllama_common.das index 3146f37c0a..be280140ac 100644 --- a/modules/dasLLAMA/dasllama/dasllama_common.das +++ b/modules/dasLLAMA/dasllama/dasllama_common.das @@ -1528,7 +1528,7 @@ def dlim_config_sources_register() { } //! The Model's plane pair for a kq format - the dispatcher every plane-form consumer keys on. -def kq_planes_of(t : Model; fmt : KqFmt; var kq : array const? &; var ks : array const? &) { // nolint:STYLE037 — the format ladder +def kq_planes_of(t : Model; fmt : KqFmt; var kq : array const?&; var ks : array const?&) { // nolint:STYLE037 — the format ladder unsafe { if (fmt == KqFmt.k4) { kq = addr(t.k4q) diff --git a/modules/dasLLAMA/dasllama/dasllama_gpu_resident.das b/modules/dasLLAMA/dasllama/dasllama_gpu_resident.das index 76ae3ca0ca..5b848fe4be 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gpu_resident.das +++ b/modules/dasLLAMA/dasllama/dasllama_gpu_resident.das @@ -316,8 +316,8 @@ def trim_model_planes(var t : Model) : bool { // nolint:STYLE037,STYLE038 — memcpy(addr(t.embq[0]), addr(t.iq2xxsq[sb0 * IQ2XXS_QSB]), nsb * IQ2XXS_QSB) memcpy(addr(t.embs[0]), addr(t.iq2xxss[sb0 * IQ2XXS_SSB]), nsb * IQ2XXS_SSB) } elif (t.emb_fmt == KqFmt.iq3xxs) { - t.embq |> resize(nsb * IQ3XXS_QSB) - t.embs |> resize(nsb * IQ3XXS_SSB) + t.embq |> reserve_resize(nsb * IQ3XXS_QSB) + t.embs |> reserve_resize(nsb * IQ3XXS_SSB) memcpy(addr(t.embq[0]), addr(t.iq3xxsq[sb0 * IQ3XXS_QSB]), nsb * IQ3XXS_QSB) memcpy(addr(t.embs[0]), addr(t.iq3xxss[sb0 * IQ3XXS_SSB]), nsb * IQ3XXS_SSB) } elif (t.emb_fmt == KqFmt.iq3s) { diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das index ea16bc6f65..e9d00cff8d 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das @@ -3796,7 +3796,7 @@ def private kq_b8_on : bool { } def enc_kq_mvb(enc : MetalComputeEncoder?; fmt : KqFmt; bq : MetalBuffer?; qoff : uint64; // nolint:STYLE037 — the flat per-format ladder (one arm per kq format) is the honest shape - bs : MetalBuffer?; soff, doff : uint64; bx, by : MetalBuffer?; yoff : uint64; + bs : MetalBuffer ?; soff, doff : uint64; bx, by : MetalBuffer?; yoff : uint64; var ka : KqMvArgs; rows, nlive : int64) { let two = nlive == 2l let b8 = nlive >= 5l && kq_b8_on() diff --git a/modules/dasLLAMA/performance/REVIEW.md b/modules/dasLLAMA/performance/REVIEW.md index ab1cde9915..fa6a5bd4f3 100644 --- a/modules/dasLLAMA/performance/REVIEW.md +++ b/modules/dasLLAMA/performance/REVIEW.md @@ -30,8 +30,10 @@ re-mint.** A reference row that carries no `sha` (the cli and python reference t pinned by its builder instead - the cli tools by that same ref pin's checkout, the python legs by `../benchmarks/asr/requirements-*.txt`. -**A diff that moves a reference-tool pin - the ref pin's checkout, or a python leg's -`requirements-*.txt` - re-mints every row the moved pin's tool minted, or withdraws them.** +**A diff that changes a reference-tool pin - `DEFAULT_REF_SHA` in +`../benchmarks/setup_lcpp_ref.das`, or a python leg's +`../benchmarks/asr/requirements-*.txt` - re-mints every row that pin's tool minted, or +withdraws them.** **A diff that writes a provenance manifest under this folder whose `provenance.dasllama_version` differs from the `DASLLAMA_RELEASE` string (`../dasllama/dasllama_version.das`) is a defect - @@ -104,4 +106,3 @@ the same change, and a diff that adds a copy names it here in the same change: `CONSENT_TEXT` (`utils/watchdog/watchdog.py`, repo root), and the captured `utils/dasllama-server/tests/fixtures/exchange.json` (repo root).** The console prompt and the control page render the served constant, so they are not copies. - diff --git a/modules/dasLLAMA/tests/REVIEW.md b/modules/dasLLAMA/tests/REVIEW.md index 137356e550..7c2e2303ac 100644 --- a/modules/dasLLAMA/tests/REVIEW.md +++ b/modules/dasLLAMA/tests/REVIEW.md @@ -134,11 +134,11 @@ a defect.** A resize cap is not evidence. logits-tolerance form. Counting cells stay token-exact. **A diff that adds a GPU kernel class under `../dasllama/` - a `[metal_kernel]` def, a -`[vk_dispatch]` declaration, or a new instance of a template carrying one - covers that class in -`test_kernel_coverage.das`, one of two ways.** Either a census row there dispatches the class, or the diff names it in -that file's `CENSUS_NEVER_DISPATCHED`, with the reason no row can reach it. A -`DASLLAMA_PARITY_FULL`-gated census row counts as the dispatching arm. Naming a -class a census row could dispatch is a defect. +`[vk_dispatch]` declaration, or a new instance of a template carrying one - covers that class +in `test_kernel_coverage.das`, one of two ways.** Either a census row there dispatches the +class, or the diff names it in that file's `CENSUS_NEVER_DISPATCHED`, with the reason no row +can reach it. A `DASLLAMA_PARITY_FULL`-gated census row counts as the dispatching arm. Naming +a class a census row could dispatch is a defect. **A kernel-unit cell - a model-less cell that dispatches one kernel class and asserts on its output - missing a compare against a CPU oracle that can witness the cell's property is a @@ -186,14 +186,14 @@ parameter that takes the lane, is a defect.** A family whose knob has no reset t it to the documented default. A runtime decline standing in for a pin measures whichever lane the box's policy picked. -**A cell that loads a media carrier under a lane pin - a `set__q8`-class knob or a -tensor-crown pin - and whose subject is not that lane knob itself mints in memory through the -family's `stage_*` + `mint_*` pair, never through a `.dlim`-baking loader -(`load__tower` / `load__encoder` / `load_model*`).** A disk bake under a -pinned lane GC-purges the serving lane's `.dlim` beside the model, and the next direct-image -load in another suite panics on the wrong identity. A cell whose subject is the lane knob -(`load_asr_model` under `set_asr_tower_fp32`) keeps the facade loader: the image identity -folds the pin, so minting around it would unmake the claim. +**A cell that loads a media carrier under a lane pin - a `set__q8`-class knob, or a +`set_metal_tensor_crowns` / `pin_metal_tensor_crowns` pin - and whose subject is not that lane +knob itself mints in memory through the family's `stage_*` + `mint_*` pair, never through a +`.dlim`-baking loader (`load__tower` / `load__encoder` / `load_model*`).** A +disk bake under a pinned lane GC-purges the serving lane's `.dlim` beside the model, and the +next direct-image load in another suite panics on the wrong identity. A cell whose subject is +the lane knob (`load_asr_model` under `set_asr_tower_fp32`) keeps the facade loader: the image +identity folds the pin, so minting around it would unmake the claim. **A CPU-vs-GPU arm that does not run a PLANAR model for its CPU stages, and that model's `blob_twin(t, path, seq_cap)` for override-selected stages, is a defect.** One session spans @@ -274,7 +274,8 @@ plane alone is a valid control there, while a route reading both planes needs bo poison the served route never reads passes on a broken kernel. **An ASR cell comparing transcripts across two serving lanes asserts TOKEN equality; a cell -comparing a crowned lane - the raced kernel form a tune sidecar arms as the serving one - -against its tensor twin asserts WORD equality, because the twins' rounding legitimately flips -tokens.** A cell that cannot hold its grade converts to the forced-feed logits-tolerance form - -never to a looser text compare. +comparing a crowned lane against its tensor twin asserts WORD equality, because the twins' +rounding legitimately flips tokens.** A crowned lane is the raced kernel form a tune sidecar +arms as the serving one; its tensor twin is the same kernel written on Metal's tensor +primitives. A cell that cannot hold its grade converts to the forced-feed logits-tolerance +form - never to a looser text compare. diff --git a/modules/dasLLAMA/tests/test_gguf_quant.das b/modules/dasLLAMA/tests/test_gguf_quant.das index 0f5dbafbe4..c71dc34289 100644 --- a/modules/dasLLAMA/tests/test_gguf_quant.das +++ b/modules/dasLLAMA/tests/test_gguf_quant.das @@ -217,9 +217,9 @@ def test_q5k_encoder(t : T?) { // are valid input for all nine (grid indices, sign bits and scale nibbles have no invalid values). def private synth_sb_bytes(sbb : int64; seed : int) : array { var b : array - b |> reserve(int(sbb * 2l)) + b |> reserve(sbb * 2l) var x = uint(seed) * 2654435761u + 12345u - for (i in range(int(sbb * 2l))) { + for (_i in range64(sbb * 2l)) { x = x * 1664525u + 1013904223u b |> push(uint8((x >> 24u) & 0xFFu)) } diff --git a/modules/dasLLAMA/tests/test_kquant.das b/modules/dasLLAMA/tests/test_kquant.das index 4a691ed78f..c1bf709a7a 100644 --- a/modules/dasLLAMA/tests/test_kquant.das +++ b/modules/dasLLAMA/tests/test_kquant.das @@ -2501,7 +2501,7 @@ def test_iq4nl_planes(t : T?) { def private split_scale_plane(ks : array; ssb : int64) : array { let nsb = long_length(ks) / ssb var out : array - out |> resize(int(nsb * 18l)) + out |> resize(nsb * 18l) for (sb in range64(nsb)) { for (i in range64(16l)) { out[sb * 16l + i] = ks[sb * ssb + 4l + i] @@ -2523,9 +2523,9 @@ def test_kq_plane_sb_metal_blob_form(t : T?) { let fmt = gguf_kq_fmt_of_type(gtype) let sbb = gguf_kq_disk_sbb(gtype) var bytes : array - bytes |> reserve(int(sbb * 3l)) + bytes |> reserve(sbb * 3l) var x = uint(seed) * 2654435761u - for (i in range(int(sbb * 3l))) { + for (_i in range64(sbb * 3l)) { x = x * 1664525u + 1013904223u bytes |> push(uint8((x >> 24u) & 0xFFu)) } diff --git a/modules/dasLLAMA/tests/test_model_image.das b/modules/dasLLAMA/tests/test_model_image.das index b10d90b371..11c1673a88 100644 --- a/modules/dasLLAMA/tests/test_model_image.das +++ b/modules/dasLLAMA/tests/test_model_image.das @@ -2383,7 +2383,7 @@ def test_layout_fingerprint(t : T?) { [test] def test_dlim_identity_pack_version(t : T?) { t |> run("the pack-code version folds into the identity and nothing else moves") <| @(t : T?) { - var c = DlimConfiguration() + let c = DlimConfiguration() let base = dlim_identity(c, IMAGE_VERSION, "", 0) let packed = dlim_identity(c, IMAGE_VERSION, "", 7) t |> success(base != packed, "pack_version 7 changes the identity") diff --git a/modules/dasLLVM/REVIEW.md b/modules/dasLLVM/REVIEW.md index 8f8bb7e33f..030f00d7f8 100644 --- a/modules/dasLLVM/REVIEW.md +++ b/modules/dasLLVM/REVIEW.md @@ -11,11 +11,11 @@ `get_architecture_name()`, or `cpu_supports()` - runs the module-owned suite on a machine matching that condition.** -- **A diff whose new behavior sits under a code path taken only when a target triple is set - - a cross-compile, not the host - whether the diff adds the path or finds it there, states - in its PR body the cross-compile (`write_exe`) for that target that exercised the new - behavior.** The suite runs on the host; a target-triple branch is checked only by - the artifact built for that target. +- **A diff whose new behavior runs only under a set target triple - a cross-compile, not the + host - names in its PR body the cross-compile (`write_exe`) for that target that exercised + the behavior.** This holds whether the diff adds that code path or finds it already there; + the suite runs on the host, so a target-triple branch is checked only by the artifact built + for that target. - **A diff that adds work to, or moves work within, what `run_jit` (`daslib/llvm_jit_run.das`) executes - its own body or any callee - also prints an diff --git a/modules/dasLLVM/daslib/llvm_jit_run.das b/modules/dasLLVM/daslib/llvm_jit_run.das index 413f84d152..ca7780097e 100644 --- a/modules/dasLLVM/daslib/llvm_jit_run.das +++ b/modules/dasLLVM/daslib/llvm_jit_run.das @@ -40,7 +40,7 @@ let LLVM_JIT_CODEGEN_VERSION : uint64 = 0x59ul // aarch64 hosts append +i8mm w // Read by tests-cpp/small/test_jit_emitter_pin.cpp: FNV-1a64 of the emitter sources // (normalized to LF; file list in the test) -let LLVM_JIT_EMITTER_HASH : uint64 = 0xee1f4cdc69b614f5ul +let LLVM_JIT_EMITTER_HASH : uint64 = 0xfcc23a40234a227ul let JIT_FNV_PRIME : uint64 = 1099511628211ul diff --git a/modules/dasMetal/metal/msl_emit.das b/modules/dasMetal/metal/msl_emit.das index ba843db1f5..082f9b0ea4 100644 --- a/modules/dasMetal/metal/msl_emit.das +++ b/modules/dasMetal/metal/msl_emit.das @@ -3024,8 +3024,13 @@ def private hoist_const_table(var ctx : MslCtx; v : VariablePtr) { var an = local_msl_name(ctx, aname) var txt = "constant {ename} {an}[{n}] = \{\n{vtxt}\};" if (key_exists(ctx.const_tables, an) && ctx.const_tables[an] != txt) { - an = "{an}_c{ctx.splice_seq}" // a same-named table with different content in another helper + var cand = "{an}_c{ctx.splice_seq}" // a same-named table with different content in another helper + while (key_exists(ctx.const_tables, cand)) { // a helper's own table may already hold the generated name + ctx.splice_seq++ + cand = "{an}_c{ctx.splice_seq}" + } ctx.splice_seq++ + an = cand txt = "constant {ename} {an}[{n}] = \{\n{vtxt}\};" note(ctx, "stmt.let.const_table.renamed") } diff --git a/tests/metal/test_metal_consttab.das b/tests/metal/test_metal_consttab.das index 2df2e9a674..355d2dfba4 100644 --- a/tests/metal/test_metal_consttab.das +++ b/tests/metal/test_metal_consttab.das @@ -18,6 +18,11 @@ def private ct_pick(i : uint) : uint { return tbl[i & 7u] } +def private ct_pick0(i : uint) : uint { // its own table carries the name the rename would generate + let tbl_c0 = fixed_array(0x100u, 0x200u, 0x300u, 0x400u, 0x500u, 0x600u, 0x700u, 0x800u) + return tbl_c0[i & 7u] +} + // the same local name with different content: the emitter renames the hoisted twin def private ct_pick2(i : uint) : uint { let tbl = fixed_array(0x1u, 0x2u, 0x4u, 0x8u, 0x10u, 0x20u, 0x40u, 0x80u) @@ -33,7 +38,7 @@ class ConstTabGpu { def consttab { let i = gl_GlobalInvocationID.x let sgn = fixed_array(1.0, 0.5) - dst[i] = src[i] + ct_pick(i) + ct_pick2(i) + dst[i] = src[i] + ct_pick0(i) + ct_pick(i) + ct_pick2(i) fdst[i] = sgn[i & 1u] * 2.0 } } @@ -52,7 +57,7 @@ def test_consttab_gpu_vs_cpu(t : T?) { gl_GlobalInvocationID = uint3(uint(k), 0u, 0u) oracle->consttab() } - t |> equal(oracle.dst[1], 3u + 0x22u + 0x2u) // the oracle itself: src 3, tbl[1], tbl_c0[1] + t |> equal(oracle.dst[1], 3u + 0x200u + 0x22u + 0x2u) // the oracle itself: src 3, the helper's own tbl_c0[1], tbl[1], the renamed tbl_c1[1] t |> equal(oracle.fdst[1], 1.0) // sgn[1] * 2 static_if (typeinfo builtin_module_exists(das_metal)) { var gpu_bad = -1 diff --git a/tests/msl/_msl_common.das b/tests/msl/_msl_common.das index c09c142bb3..fb3541e83e 100644 --- a/tests/msl/_msl_common.das +++ b/tests/msl/_msl_common.das @@ -696,6 +696,12 @@ class UkStride { // A `let` fixed-array local with all-literal elements lowers to a program-scope `constant` // array - the codebook/grid shape quant kernels stage from; reads index it by the local's name. +// a helper's own table already carries a generated-looking name: the rename below must probe past it +def private ctb_pick0(i : uint) : uint { + let tbl_c0 = fixed_array(0x100u, 0x200u, 0x300u, 0x400u, 0x500u, 0x600u, 0x700u, 0x800u) + return tbl_c0[i & 7u] +} + def private ctb_pick(i : uint) : uint { let tbl = fixed_array(0x11u, 0x22u, 0x33u, 0x44u, 0x55u, 0x66u, 0x77u, 0x88u) return tbl[i & 7u] @@ -716,7 +722,7 @@ class ConstTab { def consttab { let i = gl_GlobalInvocationID.x let sgn = fixed_array(1.0, 0.5) - dst[i] = src[i] + ctb_pick(i) + ctb_pick2(i) + dst[i] = src[i] + ctb_pick0(i) + ctb_pick(i) + ctb_pick2(i) fdst[i] = sgn[i & 1u] * 2.0 } } diff --git a/tests/msl/test_msl_functions.das b/tests/msl/test_msl_functions.das index aa8681a3f7..216807d8d1 100644 --- a/tests/msl/test_msl_functions.das +++ b/tests/msl/test_msl_functions.das @@ -143,13 +143,25 @@ def private hasq(t : T?; needle : string) { } } +def private count_in_consttab(needle : string) : int { + var n = 0 + var at = find(consttab_msl, needle) + while (at >= 0) { + n++ + at = find(consttab_msl, needle, at + 1) + } + return n +} + [test] def test_msl_const_tables(t : T?) { t |> run("literal fixed-array locals hoist to program-scope constant tables") <| @(t : T?) { hasq(t, "constant uint tbl[8] = \{") // hoisted from the helper's body - hasq(t, "constant uint tbl_c0[8] = \{") // the same-named different-content table renamed, not merged + hasq(t, "constant uint tbl_c0[8] = \{") // a helper's OWN table under the generated-looking name + hasq(t, "constant uint tbl_c1[8] = \{") // the same-named different-content table renamed past it, not merged hasq(t, "constant float sgn[2] = \{") // hoisted from the kernel body hasq(t, "tbl[") // reads index the hoisted table by name - hasq(t, "tbl_c0[") // the renamed table's reads follow the rename + hasq(t, "tbl_c1[") // the renamed table's reads follow the rename + t |> equal(count_in_consttab("constant uint tbl"), 3, "three distinct tables, none overwritten by a rename") } } diff --git a/utils/REVIEW.md b/utils/REVIEW.md index 9f0d2cbdab..6d91ccff7e 100644 --- a/utils/REVIEW.md +++ b/utils/REVIEW.md @@ -51,4 +51,3 @@ compile-checks it or whose only executing row runs against an already-deployed a nightly lane driving the live site - records its executed run in the PR description**: the machine the assertions ran on, what that machine had that CI lacks, and the pass count. A lane that tests production after merge proves nothing about the diff under review. - From 7dbbf2ec11b2f049011719d2f9d96edccd7f5612 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Mon, 31 Aug 2026 22:24:08 -0700 Subject: [PATCH 60/61] the benchmarks checklist's last dragon pass: the zero-rows rule keyed on the diff, the bench-list WHY without the race verb, no routed-from restatement Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- modules/dasLLAMA/benchmarks/REVIEW.md | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/modules/dasLLAMA/benchmarks/REVIEW.md b/modules/dasLLAMA/benchmarks/REVIEW.md index 48a6cc4f9e..c5e5c6b742 100644 --- a/modules/dasLLAMA/benchmarks/REVIEW.md +++ b/modules/dasLLAMA/benchmarks/REVIEW.md @@ -5,9 +5,6 @@ doc: `../ARCHITECTURE_MEASUREMENT.md` (the benchmark rig: `#one-benchmark-rig`). `../followup_vulkan.md` for anything about the Vulkan backend, `../PERF_LEDGER.md` for any other performance followup, `../followup_general.md` for everything else. -**Routed from `../REVIEW.md`: a timing rig - a script whose output is a measured wall or rate - -wherever the diff puts it, applies this list together with `../REVIEW.md`.** - **A diff that adds or changes an instrument whose timed body runs a forward pass through a pipeline the dasLLAMA engine selected also calls `tune_gate()` (`../performance/profile_common.das`) before that instrument's first timed rep.** A pipeline @@ -67,10 +64,10 @@ corpus, or to the pinned reference build - `DEFAULT_REF_SHA` in `setup_lcpp_ref. targets it builds, or a patch it applies - moves the measured quantity. **A diff that adds or moves a row in `../harness/tune_kernels.das`'s bench list keeps -`dot_q8q8_laneq4x4` last.** That bench pins the repack backend for the rest of the process, -so a row after it races against the pinned backend instead of the one it selects. +`dot_q8q8_laneq4x4` last.** That bench pins the repack backend for the rest of the process, so +a row after it is timed against the pinned backend instead of the backend it would select. -**A timing instrument this checklist governs whose run can end with zero result rows exits -non-zero when it does - wrong flags, failed load, a device that declines.** A run that matched -nothing and reported success leaves a sidecar or a record untouched and a caller none the -wiser. +**A diff that adds or changes a timing instrument makes it exit non-zero on a run that ends +with zero result rows - wrong flags, failed load, a device that declines.** A run that matched +nothing and reported success leaves a sidecar or a record untouched, and its caller cannot +tell. From 0c29ca0af470cc1185d50f1b1cb1e59e301588dd Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Mon, 31 Aug 2026 22:39:35 -0700 Subject: [PATCH 61/61] the constant-table hoist names what it accepts: a fixed_array of exactly N elements, integer or float scalar literals Both refusals said 'every element must be a literal', which a bool, half or vector literal contradicts; the fail-closed fixture pins the new text. The accepted set itself is unchanged - no kernel declares such a table, and the hoist fails closed on shapes it does not emit. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- modules/dasMetal/metal/msl_emit.das | 4 ++-- tests/msl/test_msl_fail_closed.das | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/dasMetal/metal/msl_emit.das b/modules/dasMetal/metal/msl_emit.das index 082f9b0ea4..e85e9ea34b 100644 --- a/modules/dasMetal/metal/msl_emit.das +++ b/modules/dasMetal/metal/msl_emit.das @@ -2990,7 +2990,7 @@ def private hoist_const_table(var ctx : MslCtx; v : VariablePtr) { } } if (mk == null || length(mk.values) != n) { - err(ctx, v.at, "fixed-array local `{v.name}` initializer hoists to a program-scope constant table - every element must be a literal") + err(ctx, v.at, "fixed-array local `{v.name}` hoists to a program-scope constant table - its initializer must be a `fixed_array(...)` of exactly {n} elements") return } var vals : array @@ -2999,7 +2999,7 @@ def private hoist_const_table(var ctx : MslCtx; v : VariablePtr) { if (!(el is ExprConstInt || el is ExprConstUInt || el is ExprConstInt64 || el is ExprConstUInt64 || el is ExprConstFloat || el is ExprConstDouble || el is ExprConstInt8 || el is ExprConstUInt8 || el is ExprConstInt16 || el is ExprConstUInt16)) { - err(ctx, v.at, "fixed-array local `{v.name}` initializer hoists to a program-scope constant table - every element must be a literal") + err(ctx, v.at, "fixed-array local `{v.name}` hoists to a program-scope constant table - every element must be an integer or float scalar literal (bool, half and vector literals do not hoist)") return } vals |> push(emit_value(ctx, el)) diff --git a/tests/msl/test_msl_fail_closed.das b/tests/msl/test_msl_fail_closed.das index 8929cfc222..5ee33b1518 100644 --- a/tests/msl/test_msl_fail_closed.das +++ b/tests/msl/test_msl_fail_closed.das @@ -59,7 +59,7 @@ def test_fail_closed_rejections(t : T?) { check_rejects(t, "_fc_uniform_written", "@uniform member `n` is written") check_rejects(t, "_fc_nan_literal", "non-finite float literal has no MSL form") check_rejects(t, "_fc_uniform_array", "array members are @ssbo buffers") - check_rejects(t, "_fc_array_init_nonconst", "every element must be a literal") + check_rejects(t, "_fc_array_init_nonconst", "must be an integer or float scalar literal") check_rejects(t, "_fc_return_value", "kernels return void") check_rejects(t, "_fc_recursion", "is recursive - MSL forbids recursion") check_rejects(t, "_fc_ufn_param", "type has no MSL form")