diff --git a/CLAUDE.md b/CLAUDE.md index 30e9cdb205..d099e6c7ec 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -162,6 +162,7 @@ Task-specific instructions are split into skill files under `skills/`. You MUST | `skills/daslang/references/queries.md` | Filter/map/sort/group/aggregate transforms - comprehension -> linq_boost -> plain `for`; avoid `daslib/functional` for new code | | `skills/decs.md` | Programming with `daslib/decs` / `decs_boost` - entities, components, queries, `[decs_template]`, stages | | `skills/internal/aot_hash_desync_debugging.md` | `error[50101]: AOT link failed` - semantic-hash desync diagnostics | +| `modules/dasLLAMA/CLAUDE.md` | Any work under `modules/dasLLAMA/` - the module's HOW_TO series (`HOW_TO_ADD_A_FORMAT.md` for a new weight format) and its architecture/review set | Multiple skill files may apply to one task: creating a new daslib module needs `skills/das_formatting.md`, `skills/daslib_modules.md`, and possibly `skills/internal/documentation_rst.md`. diff --git a/modules/dasLLAMA/ARCHITECTURE.md b/modules/dasLLAMA/ARCHITECTURE.md index e0da7ceef5..dc4ea42a2f 100644 --- a/modules/dasLLAMA/ARCHITECTURE.md +++ b/modules/dasLLAMA/ARCHITECTURE.md @@ -38,7 +38,7 @@ re-transcoding `$LCPP/src/unicode-data.cpp`). - `ARCHITECTURE_IMAGE.md` - sec.2.1-2.1i: the prepared-image rail, the baked dev-W f16 plane, and the baked tower twin-W plane. -- `ARCHITECTURE_GPU.md` - sec.2.2b, 2.2w-2.2x: the tensor-GEMM and fused-attention shapes that +- `ARCHITECTURE_GPU.md` - sec.2.2b, 2.2w-2.2y: the tensor-GEMM and fused-attention shapes that measured out, the tower attention routes, and the tower driver's encode chains. - `ARCHITECTURE_GPU_PREFILL.md` - sec.2.2c-2.2i, 2.2u-2.2v: the Metal prefill driver's GEMM form ladder, dev-W knee map, attention slab, MoE bucket rail, chunked submission, the f16 twin diff --git a/modules/dasLLAMA/ARCHITECTURE_ENGINE.md b/modules/dasLLAMA/ARCHITECTURE_ENGINE.md index a4f1a67e05..8e8eb5a060 100644 --- a/modules/dasLLAMA/ARCHITECTURE_ENGINE.md +++ b/modules/dasLLAMA/ARCHITECTURE_ENGINE.md @@ -78,7 +78,10 @@ stay the reviewer's. A mis-numbered arm dispatches, reads the wrong buffer, and image plus its element count; the image owns the bytes, a carrier owns nothing but its backing. Requires nothing in dasllama - the image rail binds planes, every carrier holds them. - **`dasllama_kqformat.das`** - format IDENTITY: the `KqFmt` enum, the per-format descriptor table - (plane strides, block geometry, stream codes), format predicates. It requires nothing else in + (plane strides, block geometry, stream codes), format predicates, and the shared decode + tables the grid and codebook formats key off - each as a builder function (`iq3s_grid()`, + `iq4nl_lut()`) for kernels that may run on a team lane, plus a global twin for tests, + oracles and the emitter's constant bake. It requires nothing else in dasllama, because it is the taxonomy everything keys off. ONE id space - the enum; integer ids exist only at the IR/kernel-param boundary. `kq_sb` is the superblock-lattice predicate: a `fmt != q8` test does not imply the lattice, so branch on the predicate. diff --git a/modules/dasLLAMA/ARCHITECTURE_GPU.md b/modules/dasLLAMA/ARCHITECTURE_GPU.md index 8921e33661..bd9eac0cbe 100644 --- a/modules/dasLLAMA/ARCHITECTURE_GPU.md +++ b/modules/dasLLAMA/ARCHITECTURE_GPU.md @@ -256,3 +256,19 @@ specification and the CPU-vs-GPU transcript cells are its parity instrument. Eve best-effort: it answers false (or -1) on any shape, knob, quant-mode or device decline, and the CPU chain serves that encode. Engage is read from counter deltas (`metal_tower_stats`, `metal_tower_f16_encodes`), never from "the model ran". + +### 2.2y The Metal kq split scale plane {#metal-kq-split-scale-plane} + +Every superblock format but k4, k5, q40 and iq4nl stores its Metal-blob scale row SPLIT into two +regions of one buffer: the 16-byte sub-scale strips of every superblock first, then the packed +per-superblock d tail. A kernel binds that one buffer twice - the strips at `soff = sb0 * 16` and +the tail at `doff = nsb * 16 + sb0 * 2` - so the two reads stride independently and the strip +read stays 16-byte aligned. k2 is the one shape variation: its tail is 4 bytes per superblock +(`nsb * 16 + sb0 * 4`), because it carries d and dmin. `kq_scales_of` builds the pair; +`metal_blob_scale_plane` mints it at bake time, folding each format's 20-byte decoded row into +`[16B strips][2B d]` (k3's row is 18 bytes and is already in that shape). The 2-byte tail is why +a region's bind offset must be a multiple of 512 elements - the `(off/256)*2` d-plane bind is +4-byte aligned only then - which is what `metal_blob_off_ok` and `moe_site_ok` check. iq4nl is +the exception: it reuses q40's 16-byte plane of eight f16 d per superblock, binds once, and +ignores `doff`. The Vulkan tier does not use this form - it binds the decoded 20-byte row as five +uints per superblock. diff --git a/modules/dasLLAMA/ARCHITECTURE_GPU_VULKAN.md b/modules/dasLLAMA/ARCHITECTURE_GPU_VULKAN.md index 3c6f42c045..d45152be9b 100644 --- a/modules/dasLLAMA/ARCHITECTURE_GPU_VULKAN.md +++ b/modules/dasLLAMA/ARCHITECTURE_GPU_VULKAN.md @@ -76,8 +76,11 @@ compiler pattern-matches only one spelling into that path: a 16-bit load (`int16 members) followed by `unpack8(w)[i & 1u]` - a byte2 lane select - with sub-fields pulled out by shift and mask. A 32-bit word with a variable shift runs slower; an `unpack8` of a 32-bit word indexed by a runtime value (a byte4 dynamic select) drops the whole kernel off the block-load -path, to about a third of the rate. Every cm2 decode - q8, Q4_K, Q6_K - is spelled the 16-bit -way, which is why the block structs are `int16` arrays over the same bytes. +path, to about a third of the rate. Every cm2 decode - q8 and the six kq superblock formats - +is spelled the 16-bit way, which is why the block structs are `int16` arrays over the same +bytes. The IQ4_XS codebook is the one runtime-indexed read a decode makes: it is staged into a +16-entry `@workgroup` f16 table ahead of the tile loop (the reference exe's shared-memory table-staging form), +never selected out of a register vector per element. ### 2.2l The cm2 tile pick and the coopmat default ladder {#cm2-tile-pick-and-default} @@ -98,8 +101,8 @@ and clamps only the store, so every f16 plane the chain feeds it - the gathered image and the hidden plane - is sized with 32 rows of slack past its last region (`ffn_cm2_chunk_rows`). -**The f16 feed admits exactly three weight formats - q8, Q4_K and Q6_K** - the same set the cm2 -decode callbacks cover (sec.2.2k) - and each (format, tile) pair has ONE generated class. The +**The f16 feed admits q8 and every kq superblock format** (`kq_sb`) - the set the cm2 decode +callbacks cover (sec.2.2k) - and each (format, tile) pair has ONE stamped class. The prefill driver reaches them through one dispatcher per stage (`cm2_cls_ensure`, `cm2_cls_set`, `cm2_cls_enc`), all three keyed on the same `(fmt, ml)` pair, so the pipeline a role ensures, the set it binds and the kernel it encodes can never be three different classes. The decode @@ -111,6 +114,21 @@ NV_cooperative_matrix2, else mm where it has KHR_cooperative_matrix, else sdot4; the extension lands on mm. The same resolver stamps the mode into the `.dlim` flavor configuration, so the recorded mode and the running mode cannot drift. +**The tile's fast path is what makes the loads unclamped.** It runs when the weight tile is +whole (`m0 + 128 <= d`), the token column is whole or stamped s, and K is a whole number of BK +steps; the layouts are then created clamp-Undefined and the B and output strides are masked to +a multiple of 8 f16 (`stride &= ~7`). The mask is an identity on today's shapes - `n` and `d` +are 32-multiples - and it exists to make the alignment PROVABLE to the driver's address +analysis, which is what keeps the loads on the wide path. The s column gates only the weight +tile: its partial token column loads unclamped and its store clamps. Everything else takes the +edge path with clamped layouts. + +**The no-split arm keeps literal loop bounds and a literal store base.** Where `ksplit` is zero +the k loop runs the literal `0 .. n` with the store at the row base rather than the general +`k0`/`k1`/`ybase` form, although those values are exactly `0`, `n` and `0` on that path: the +general spelling cost 27% of prefill throughput (`benchmarks/lcpp_bench.das` pp512, 5060 Ti). +The split arm keeps the general form. + ### 2.2m Class-pipeline creation is the Vulkan tier's one shader A/B seat {#vk-class-pipeline-build} `vkd_class_pipe` is the single place a class kernel's SPIR-V becomes a pipeline, so both shader diff --git a/modules/dasLLAMA/ARCHITECTURE_MEASUREMENT.md b/modules/dasLLAMA/ARCHITECTURE_MEASUREMENT.md index 9b6ec93045..c5a2d7897b 100644 --- a/modules/dasLLAMA/ARCHITECTURE_MEASUREMENT.md +++ b/modules/dasLLAMA/ARCHITECTURE_MEASUREMENT.md @@ -108,3 +108,12 @@ same math differ in float terms - while one that changes only WHEN work happens a CLI flag is never an override (it is the run's own command line, visible where the run is launched). + +### Re-stamping inside the content-addressed archive + +A sidecar archived as `records/.tune..json` is content-addressed: its filename +carries the hash of its bytes. Re-stamping such a file's `provenance.engine_sha` to a reachable +commit (the remedy `performance/REVIEW.md` allows when the measured `modules/dasLLAMA/` tree is +byte-identical) therefore re-hashes and renames the file, and every `records/.json` row +whose `tune_sha` named the old file is repointed in the same change - a row left on the old +name points at a file that no longer exists. diff --git a/modules/dasLLAMA/CLAUDE.md b/modules/dasLLAMA/CLAUDE.md new file mode 100644 index 0000000000..bd5bc7fc44 --- /dev/null +++ b/modules/dasLLAMA/CLAUDE.md @@ -0,0 +1,36 @@ +# dasLLAMA module instructions + +dasLLAMA is the daslang LLM / ASR / vision engine, in-tree at `modules/dasLLAMA/`. **How it is +built and why is the `ARCHITECTURE*.md` set beside this file** (`ARCHITECTURE.md` routes to the +engine, image, GPU, Vulkan, Metal, measurement and media companions) - read the section you +are about to work in before writing code here. The rules binding a diff are `REVIEW*.md`; +`ENVIRONMENT.md` lists every knob; `followup_general.md` / `followup_vulkan.md` are the ledgers; +`PERF_LEDGER.md` is the measured record; `tests/CLAUDE.md` is the test discipline (run suites +ONLY through `tests/run.das`). + +Follow the daslang **gen2** conventions - the root `CLAUDE.md` rules apply to every `.das` file +here. + +## HOW_TO documents (REQUIRED for the task they name) + +A HOW_TO is a procedure: imperative, ordered, each step citing the architecture section that +owns it, validated by execution, with a QUIRKS ledger of every place the pattern broke so a +follow-up arc can unquirk it. Read the one that matches your task before the first edit, and +fix it in the same session when a step turns out wrong. + +| Document | Read BEFORE... | +|---|---| +| `HOW_TO_ADD_A_FORMAT.md` | Adding a weight format (a new `KqFmt`): GGUF type -> planes -> CPU kernels -> tune family -> Vulkan -> Metal -> tests | +| `BRINGUP.md` | Bringing a profiling box up from zero (the records rig; `METHODOLOGY.md` is the published method) | + +Planned entries in the series: adding a model family, a vision tower, an audio tower, TTS. + +## Skill files (REQUIRED) + +| Skill file | Read BEFORE... | +|---|---| +| `skills/tune.md` | Touching any `[tune]` / `[tune_perm]` kernel family or the sidecar | +| `skills/internal/llvm_tune_internals.md` | Editing the tune framework itself | +| `skills/perf_lint.md` / `skills/style_lint.md` | Suppressing any lint finding here | +| `skills/internal/tests_in_repo.md` | Adding a test (the deep-engine rules: `options stack`, `T?`-free helpers) | +| `skills/writing_benchmarks.md` / `skills/internal/benchmarks_in_repo.md` | Anything under `benchmarks/` or `performance/` | diff --git a/modules/dasLLAMA/CMakeLists.txt b/modules/dasLLAMA/CMakeLists.txt index 9138e85621..bb23897198 100644 --- a/modules/dasLLAMA/CMakeLists.txt +++ b/modules/dasLLAMA/CMakeLists.txt @@ -38,6 +38,14 @@ IF(NOT DAS_LLAMA_INCLUDED) install(FILES ${PROJECT_SOURCE_DIR}/modules/dasLLAMA/performance/model_specs.das DESTINATION ${DAS_INSTALL_MODULESDIR}/dasLLAMA/performance ) + # shipped defaults profiles ([tune_scope(defaults = ...)]): an untuned SDK box adopts its + # CPU class's winners instead of racing — the scope resolves this dir against the installed + # dasllama_math_gen.das, so the bundle must carry it + install(DIRECTORY ${PROJECT_SOURCE_DIR}/modules/dasLLAMA/performance/defaults + DESTINATION ${DAS_INSTALL_MODULESDIR}/dasLLAMA/performance + FILES_MATCHING + PATTERN "*.tune-defaults.json" + ) # third-party notices: the ported reference implementations (MIT) + the weights terms install(FILES ${PROJECT_SOURCE_DIR}/modules/dasLLAMA/LICENSE.PARAKEET DESTINATION ${DAS_INSTALL_DOCDIR} RENAME PARAKEET.LICENSE) install(FILES ${PROJECT_SOURCE_DIR}/modules/dasLLAMA/LICENSE.SILERO DESTINATION ${DAS_INSTALL_DOCDIR} RENAME SILERO.LICENSE) diff --git a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md new file mode 100644 index 0000000000..5e2703b677 --- /dev/null +++ b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md @@ -0,0 +1,1138 @@ +# How to add a weight format to dasLLAMA + +A HOW_TO is the fourth document mood: `REVIEW.md` binds a diff, `ARCHITECTURE.md` says how the +system is, `/history` says what happened - this says **do this, then this**. It walks the code in +the order a new format lands, cites the architecture section that owns each step instead of +restating it, and keeps a QUIRKS ledger of every step where the pattern broke. It is validated by +execution: every step below was run in the session that wrote it, on the format named in the +per-format notes, or is marked untested. A step that turns out wrong during the next format's +bring-up is fixed here in that session. + +The worked example is IQ4_XS (the pilot). The per-format notes at the end say what each later +format changed in the walk. + +## 0. Decide the format's shape before touching code + +Three questions decide which existing family the format rides; answer them from +`ggml-common.h` (the block struct) and `ggml-quants.c` (`dequantize_row_`): + +1. **Block geometry.** 256-weight superblock (every K-quant, every i-quant) or 32-weight block + (`Q4_0`, `Q5_1`, `IQ4_NL`)? A superblock format joins the `kq_sb` lattice (Q8_K activations, + `% 256` rows, the grp repack, the stamped kq kernels); a 32-block format rides per-32 + planes like `q51`. `ARCHITECTURE_ENGINE.md` sec.1.2 owns the lattice split. +2. **Weight reconstruction.** Shift/mask nibbles (`q4_0`, K-quants), a 16-entry codebook on the + nibble (`IQ4_XS`, `IQ4_NL`), or a grid gather (`IQ2_*`, `IQ3_*`)? Shift/mask and codebook + formats keep the k4 nibble tiling and add an unpack step; grid formats decode to bytes first + (the byte-expanded panel k5/k6 already use). +3. **Scale structure and its fold.** Per-block f16 (`q40`), superblock d/dmin + 6-bit sub-scale + and min (`k4`/`k5`), superblock d + signed int8 sub-scale (`k6`, `IQ4_XS`). The fold decides + whether the dot needs the activation block sums (`xbsp`): a min or an unsigned-nibble offset + needs them, a signed reconstruction (codebook, k6's -32) does not. + +Write the answers down; they are the first lines of the per-format note. + +## 1. Identity - `dasllama/dasllama_kqformat.das` + +The taxonomy every other file keys off. One edit here, then the compiler finds the ladders. + +- Append the member to `KqFmt` - **append, never reorder**: the int value is the device stack + tag (`vk_kq_schema_id`) and the image plane id. +- `kq_sb` (both overloads) if it is a superblock format. +- `kq_schema_id`: the kernel/IR id. The ids are mnemonics - K-quants by bit width (4/5/6), + Q4_0 = 40, i-quants = bit width x 10 + a variant digit (IQ4_XS = 44). Three id spaces exist + (QUIRK 5); this is the only bridge between two of them. +- The stride constants `_QSB` / `_SSB` (bytes per superblock row of the quant and + scale planes) and the `kq_qsb` / `kq_ssb` arms. A codebook goes here too (`IQ4NL_LUT`). +- `dasllama_gemm_schema.das`: the int-id twins `kq_qsb(int)` / `kq_ssb(int)`. +- `tests/test_kqformat.das`: pin the enum value, the predicate, the strides, the id, the stream + code, and the codebook's edge values. + +## 2. Codec - `dasllama/dasllama_convert.das`, `dasllama/dasllama_gguf.das` + +- `GGML_TYPE_` constant (`dasllama_gguf.das`). +- `transcode__superblock(bytes, bo, kq, kqo, ks, kso)` - the per-superblock disk -> plane + split, array form (what the tests drive). +- `dequant__plane_superblock` - the reference dequant off the planes, in ggml's own float + order (`dequantize_row_` decides the order; match it operation for operation so the + plane dequant is bit-exact against the file dequant). +- `gguf_transcode_` (`dasllama_gguf.das`) - the bulk, threaded, pointerized twin of the + superblock transcode (`kq_transcode_check` + `guard_dst` + `with_tensor_view` + + `maybe_parallel_for` over superblocks). +- `tests/test_kquant.das`: a hand-packed synthetic superblock (`build__block`, written in + the PACK direction so a misread cannot cancel) and an arm asserting the plane dequant equals + the hand-computed weights exactly. + +**Plane layout rule.** Quant plane = the disk nibbles verbatim when the disk pairing is already +what the kernels want (Q4_0 and IQ4_XS pair k/k+16 inside a 32-block - the k4 tiling; Q4_K +pairs k/k+32 and re-pairs at repack). Scale plane = a fixed-stride row per superblock. The kq +v2 rule keeps the DISK scale block verbatim and decodes at repack (k4/k5: 16B block + 4B pad); +a format whose decoded form FITS the row may decode at transcode instead (IQ4_XS: f16 d, 2 pad, +8 x int8 sub-scale, 8 pad = the k4 decoded-row shape, so k4's repack addressing serves). + +## 3. Planes and the loader - `dasllama_common.das`, `dasllama_load.das`, `dasllama_layout.das` + +This is the ladder walk (QUIRK 1). Every site is a flat `if (fmt == KqFmt.k4) ... elif` chain; +add the arm next to `q40`'s. The compiler does not find these - a missing arm falls to the +`else` panic (good) or silently to k6 (bad, the `kq_kernel_gen` shape); grep `KqFmt.q40` and +`== 40` and visit every hit. + +- `Model`: the plane pair `q` / `s` and `kq_repack_mr`. +- `dasllama_load.das`: `LayoutSizes._n`, `KqCursors.`, `kq_take`, the `LayoutSizes` + constructor, `stream_field_of` (the streamed image plane name), the scale-half landing + `memcpy`, the stream repack `invoke(g_stream_repack, , ...)`, `transcode_kq_tensor`, + `load_big`, `kq_fmt_of` (GGML type -> tag), the two `noisy` log lines, + `g_stream_plane_total["q"]`, the plane `reserve`/`resize` block. +- `dasllama_common.das`: `kq_active_mr`, `kq_fi`, `mm_at_kq_pre` (two arms), `mm_b_kq`, + `mm_at_kq_groupn` (two arms), `mm_b_kq_groupn`, `mm_b_kq_pre`, `kq_plane_q`, `kq_plane_s`, + `embed_row` (four arms: grp and plane form, trimmed and untrimmed), the bake config fill + (`c.kq_mr = active_kq_layout_mr()`). +- `dasllama_layout.das`: the plane base pointers + the `rkq` ternary, `push_repack_kq`, the + `moe_gpu_gather_stack_kq` plane ternaries. +- `dasllama_gpu_resident.das` (embed trim), `dasllama_ple.das` (two arms), + `dasllama_blocks.das` (`kq_bytes_per_weight`). +- `dasllama_config.das`: `DlimCpuConfig.kq_mr` AND the identity string in + `dlim_identity` (QUIRK 3); `dasllama_image.das`: `IMAGE_VERSION` bump + the streamed plane + name list in `stream_extra_bytes`. + +## 4. CPU kernels - `dasllama_math_default.das`, `dasllama_math_gen.das`, `dasllama_math.das`, `dasllama_repack.das` + +- **Read llama.cpp's CPU kernel for the format FIRST** - `ggml-cpu/arch/x86/quants.c` + `ggml_vec_dot__q8_K` (and the arm twin) in the local clone. Map each technique it uses + onto the lattice as its own `[tune_perm]` spelling where it fits; the probe IS the side-by-side + (perms race each other and the reference, per box). Our CPU kernels typically win - keep it + that way by never leaving one of their tricks unmeasured. +- `dot__q8` - the portable disk-order row dot (exact integer inner sums, one float fold per + superblock), `_rows_kernel`, the `kq_gemv_kernel` arm, the `matmul_kq_groupn` arm. +- `dequant__row_grp` - the grp row dequant (own helper; `dequant_kq_row_grp` dispatches). +- `repack__grp` (`dasllama_repack.das`) - disk-order -> grp planes; tail rows (d % mr) + stay disk-order untouched. +- `dasllama_math_gen.das`: `q8_layout_gen` (the layout companion), `_grp_row_dot` + (the scalar grp reference = the stubs' body and the repack oracle), `q8_gemv_gen` + + `q8_tile_gen` with the `[tune_perm]` grid copied from q40's and + `tune(gen = "dasllama_gemm_gen::_tile", ...)`, `kq_layout_of`, `repack_kq_gen` / + `repack_kq_bake`, `kq_kernel_gen` (two ladders), `kq_batch_cell_gen` (`packed` + tile + + tail ladders), `kq_batch_kernel_gen` tail, `kq_batch_groupn_gen` tail, `kq_groupn_gen` (two + ladders), both `register_kernel_backend` rows (`kq_rows_ = @@q8_gemv_gen`). +- `dasllama_math.das`: the `KernelBackend.kq_rows_` slot, its `g_kq_rows_` global, + the activation copy, `kq_rows_fn`, the null-guard, the bake arm in `active_kq_layout_mr`. +- `dasllama_gemm_gen.das`: `register_llvm_code_generator` for `_gemv` and `_tile`. + Until the emitter arm exists, register generators that return `false` - a declined + generator IS the framework's fallback (QUIRK 4); the reference bodies serve. +- `harness/gen_tune_probe.das`: `pack_kq_disk_block`, `repack_kq_grp_fmt`, `build_kq_fixture` + (block size, transcode, the yref oracle), `kq_tile_variants`, `kq_gemv_variants_by_suffix`, + `kq_layout_mrs`, `run_kq_tile`'s `packed`, `kq_tile_entry`, the family list. Without this + the scope's completeness check demands a sidecar entry the tuner never writes, and every + start re-tunes. +- `tests/test_kquant.das`: add the format to every gate's ladder and to the `[4, 5, 6, 40]` + lists (QUIRK 2). The gates: dot vs the fp64 plane-dequant oracle, portable GEMV rows, + repack at mr 4/8/16 (dots and row dequants bit-exact), 4-token tile vs per-token GEMVs, + groupn (disk + grp slices), batch groupn. + +Gate: `test_kqformat` + `test_kquant` under `-jit` (the stubs decline, so their reference bodies run either way; every dasLLAMA test run is a `-jit` run). + +## 5. The JIT emitter - `dasllama_gemm_gen.das` + +`kq_tile_gen_impl(gc, fmt)` / `kq_gemv_gen_impl(gc, fmt)` are one emitter specialized on `fmt` +at generation time: the group walk, the slice/loop machinery (`emit_slice`) and the store +epilogue are format-agnostic; only the block body and the primitives `setup_tile_emit` wires +differ. IQ4_XS took: + +1. A block body, `emit_block_iq4xs`: `emit_block_mx4`'s nibble unpack + `lut_lookup` (`tbl1` + / `pshufb`, the codebook baked as a constant vector) + `dot_lane` with `abs_w` - the SIGNED + sign-trick lattice, because LUT weights are int8; `emit_block_kqv2`'s `kq_dot_lane` / + `kq_dot_mem` run the unsigned-nibble lattice and do not apply - then k4's fold with the min + term dropped: per block `iacc += sext(sc) * (idot_lo + idot_hi)`, per superblock + `f += iacc * (d * d8)`. Plane addressing is the repack's: nibbles at + `wb + ((blk*16 + j*4)*mr + qd*w8)`, `sc` at `sb + 4*mr + blk*mr + r`, `d` at `sb + 2*r`. +2. `emit_one_block`: `te.kq == 44` routes to the new body ahead of the `te.kq != 0` arm. +3. `setup_tile_emit(te, gc, p, needMx4 = fmt == 44)`: the mx4 primitive wiring is reused as + is; the LUT bake picks `iq4nl_lut()` over the e2m1 table on `te.kq == 44`. +4. The two stubs become `=> kq_gemv_gen_impl(gc, 44)` / `kq_tile_gen_impl(gc, 44)`. +5. No `perm_declines` change: the tbl1 rail already sits on every sdot perm (it is the mx4 + companion's), and pshufb is implied by the x64 tiers. + +Gates, in order: `DAS_TUNE_MODE=test bin/Release/daslang.exe -jit harness/gen_tune_probe.das` +(the family must be in BOTH of the probe's lists - QUIRK 2) gates every perm of the family's +grid against the scalar oracle - a stamped perm reports its layout companion's `mr` (8 on this +box) and a fast-math-sized maxdiff (1.5e-5), a declined one `mr=4` and maxdiff 0 (that is the +reference body answering, not the emitter); then `test_kquant` under `-jit` (the tile-vs-GEMV +gate is bit-exact by construction); then the end-to-end run - and read QUIRK 11 before +trusting its numbers. + +## 6. Vulkan - `dasllama_vulkan_classes.das`, `dasllama_vulkan_common.das` + +The tier reads the CPU planes verbatim (`stack_plane_bytes` -> `arena_block_bytes` -> +`kq_qsb(vk_kq_schema_id(fmt))` x `KQ_DEV_SSB`), so a format whose plane pair already has the +20 B decoded scale row needs no upload work - only the id bridge and the kernels. IQ4_XS took: + +1. `vk_kq_schema_id` (`dasllama_vulkan_common.das`): the `int(KqFmt)` -> kernel-id arm + (`6 -> 44`). This is the third id space of QUIRK 5 at its Vulkan seam; without the arm the + arena plan panics on the first iq4xs stack. +2. `KqGemvIq4xs : KqGemvBase` - `def override blk_contrib`: the q40 nibble tiling + (`wq4[wsb * 8 + blk]`), each nibble word decoded through `iq4_word` (a `fixed_array` LUT + local - the SPIR-V emitter lowers a `let` fixed array to a Function-storage variable and + indexes it) into SIGNED lanes for `sdot4` (OpSDot, signed x signed - the block-sum trick of + q40/k4 does not apply and is not needed), scale `d * sc` with `sc` the signed byte off word + 1..2 of the 5-word row (`unpack8` sign-extends, the k6 spelling). +3. `KqBatchIq4xs : KqBatchBase` - `stage_w` decodes the staged words through `iq4_word` + (k4's staging otherwise), `stage_ws` fills ONE plane with `d * sc`, `blk_fma` is + `xscl * ws * idot` (q40's without the `- 8 * bsum`). +4. Ladders: `kq_batch_cls_ensure` / `kq_batch_cls_enc_for` / `gemv_cls_ensure` / + `gemv_cls_enc` gain an arm; `gemv_cls_set`'s four-way `||` became `kq_sb(fmt)`. +5. Tests: `tests/_vkd_oracles.das` `kq_cls_ref` arm (the class-on-CPU oracle), the two family + cells in `tests/test_vulkan_kernels.das` go to five formats, and - because the codebook pack + is new bit-math that a class-vs-device compare cannot see (both sides run the same + `iq4_word`) - `iq4xs_gemv_float_oracle`, a float dequant straight off the plane bytes that + the class oracle must match. + +Not done, by ruling: a cm2 decode-in-load tile (`[spirv_decode] def decode_iq4xs`). The f16 feed +admits q8/k4/k6 only (`pf_f16_feed`), q40 and k5 have no cm2 tile either, and +`followup_vulkan.md` item 24 rules that new formats land on the one class template, not as +three more hand-stamped bodies. IQ4_XS prefill rides the kq batch tile like q40 does. + +### 6b. The cm2 prefill tile - a decode method on the template + +On an NV_coopmat2 device the f16 feed serves every kq format through ONE tile template +(`KqCm2BatchT`): a new format is a format template authoring `[spirv_decode] def decode` over +the DEVICE forms (quants as the gather lays them out - k4/k5 re-paired k/k+16, q40/iq4xs/k3 +verbatim; scales the 20 B `KQ_DEV_SSB` row) plus three eight-line width stamps, arms in the +`cm2_cls_ensure/set/enc` ladders, and `pf_f16_feed` admits it via `kq_sb` automatically. A +codebook format raises the `IQLUT` axis - a gated `@workgroup` f16 table staged ahead of the +tile loop (llama.cpp's `init_iq_shmem` form); never select codes out of a register vector per +element inside a decode callback. Gate: a device-form CPU oracle (`f16_gemm_oracle`) and +an l/m/s cell in `tests/test_vulkan_kernels.das`. Payoff on the 1B: iq4xs pp512 5161 -> 15334, +k3 5174 -> 14031 (0.90x / 0.80x llama.cpp's Vulkan, from 0.30x). + +## 7. Metal - `dasllama_metal_kernels.das`, `_common`, `_prefill`, `_shapes`, `dasllama_layout.das` + +Done for IQ4_XS over ssh on the M1 (the tier compiles MSL and runs only there). The tier is +NOT a plane-verbatim consumer like Vulkan: a Metal-served model is the `metal_blob` flavor, +whose kq scale planes are rebaked into device forms (k4/k5: the 16 B disk block, k6: the split +form) and whose CPU never reads a plane again. So a format on Metal starts with a device-form +decision, and every ladder below keys on `KqFmt` with an `else` that means "k5" or "k6" - +without its own arm a new format runs a wrong-layout kernel silently, which is why the tier's +gate (`kq_fmt_gpu_supported`) is closed by default. IQ4_XS took: + +1. **The device scale form: the k6 split form.** `metal_blob_scale_plane("iq4xss")` + (`dasllama_layout.das`) turns the 20 B CPU row into `[nsb x 16 B strips][nsb x f16 d]` - + the strip is the row's bytes 4..19 (`[8 signed sc][8 pad]`), d is bytes 0..1; 18 B/sb, the + k6 numbers exactly. That choice makes `kq_scales_of` the k6 arm over `t.iq4xss` + (`soff = sb0*16`, `doff = plane_sb*16 + sb0*2`), `metal_blob_off_ok` the k6 rule + (`off % 512`), and every kernel bind the k6 pair (d plane at `s0off = doff`, strips at + `soff`). `metal_blob_commit` lists the plane by name. Quants: `kq_quants_of` binds + `t.iq4xsq` verbatim (128 B/sb). +2. **The codebook in MSL, `iq4_lut`:** four packed words selected by nibble range, byte + extracted, widened with the k6 sign trick - no table memory, no fixed-array literal (the MSL + emitter takes fixed arrays as `var` locals with per-index stores, not as initializers). +3. **Kernels** (`dasllama_metal_kernels.das`), each a copy of the k4/k6 shape with the per-32 + pairing (lane = one 32-block of every 4th superblock: 4 uints, 16 lo + 16 hi weights) and + `d * sc * LUT[q]` with no min term: `MetalKqGemvIq4xs` (decode GEMV), `MetalKqMvIq4xsT` + B2/B4 + `MetalKqMvB8Iq4xs` (small batch), and an `IQ4XS` arm in `MetalKqMulMmK45T` + (prefill mul_mm; `MetalKqMulMmIq4xs` stamps it). PSO globals + `compile_pso` + + `release_pso` lines in the three ladders. +4. **Ladders:** `enc_kq_gemv`, `enc_kq_mvb`, `enc_kq_gemm_mm_b` (kernels), `pf_enc_kq_site_mm` + (the base mul_mm only - no tensor / tall / dev-W twins: those are the M5 kernel pass), + `pf_devw_panel_kq` (returns false - its `dq` pick would otherwise be k5's), `pf_enc_kq_gemv` + (prefill), `moe_site_ok` + the `sb1/2/3` predicates (shapes), and last the gate. +5. **Tests:** `dequant_iq4xs_plane_superblock_at` (`dasllama_convert.das`, the split-layout + twin the CPU row now calls), fixtures at fmt 44 in `tests/_metal_kernel_common.das`, the + ladders + calls in `test_metal_gemv_kernels.das` (GEMV, B2/B4/B8) and + `test_metal_gemm_kernels.das` (mul_mm, base form). Gate proof: a one-byte codebook mutation + turns the iq4xs cells red. + +Result: `test_metal_gemv_kernels` 2/2 and `test_metal_gemm_kernels` 2/2 on the M1 Max; the +tuner's first mint on that box crowned `iq4xsq8_tile_gen: mr8` (the NEON `tbl1` path of the +JIT emitter, `verdict=beats`), and `run.das` on the CPU/NEON tier reproduces the text at gen 131 t/s; +on the Metal tier (`set_metal_mode(MetalMode.required)` BEFORE `load_model_`, then +`convert_model_to_metal_blob` - `run.das` itself declares CPU intent, and `DASLLAMA_GPU=1` on a +Mac build with dasVulkan arms Vulkan-on-MoltenVK, 96 t/s, not Metal) the 1B IQ4_XS decodes the +same text at gen 153 t/s with `metal decode/prefill: resident path live on Apple M1 Max`. Not done: the tensor / tall / dev-W mul_mm twins and +the MoE GEMV/mul_mm trio for the format - ledgered for the M5 kernel pass. + +## 8. End to end + +**Dev-loop invocation (adopted 2026-08-31):** every correctness-loop run (probe tests, +parity, run.das spot checks) goes through the fast stack - +`daslang.exe -jit -module-cache .jitted_scripts/module_cache/.dascache .das -- +--jit-split-modules=-1 ...` - warm no-edit reruns drop 48 s -> 5 s, an edit loop 196 s -> +~60 s (zen2, lcpp_bench scale). Bench t/s rows keep the stock monolith invocation (split +loses cross-module inlining); the first run after a cache write pays one cold codegen (the +deser re-key); QUIRK 21 still applies to emitter edits. Numbers, caveats and the +invalidation ledger: `plans/jit_compile_time.md`. + +A real file whose every tensor type is now loadable (the header census script in the session +scratchpad, or `harness/gguf_dump.das`), through `examples/dasLLAMA/run.das` against +`simple_ids.exe` from the llama.cpp reference build for the same prompt; then `test_model_image` +for the image round trip, then the records rig. + +## QUIRKS + +Every step where adding a format was not "the template plus one condition". Each entry: what, +where, why it is so today, what unquirked looks like. An empty ledger is a legitimate outcome. + +1. **Per-format plane fields and ~40 hand ladders.** `Model` holds `k4q/k4s, k5q/k5s, k6q/k6s, + q40q/q40s, q51q/q51s, iq4xsq/iq4xss`, and every consumer selects the pair with an + `if (fmt == KqFmt.k4) ... elif` chain: 12 ladders in `dasllama_load.das`, 13 in + `dasllama_common.das`, 3 in `dasllama_layout.das`, 8 in `dasllama_math_gen.das`, 3 in + `dasllama_math_default.das`, 4 in `dasllama_math.das`, one each in `ple`, `gpu_resident`, + `blocks`, `config`, `image`. Why: the planes grew one format at a time and each arm carries + a different literal stride. Unquirked: one `KqPlanes` (quant, scale, mr) indexed by `KqFmt` + on `Model`, and `kq_plane_q/s` as the only accessors - every ladder above collapses to one + table lookup, and a new format is the enum member plus its strides. +2. **The test file ladders per format too.** `tests/test_kquant.das` builds fixtures, transcodes, + dequants, dots, repacks and calls the stubs through the same `fmt == 4/5/6/40` chains in + five gates (28 arms for one format), and raises `_cyclomatic_complexity` / + `_function_length` per format added. `harness/gen_tune_probe.das` repeats the shape (9 + arms), and its test mode gates a hand list of families (the `kq_test_family(4l/5l/6l/40l)` + calls plus their fixture arrays) SEPARATE from the tune-mode family array - IQ4_XS sat in + the tune list and not the test list, so `GEN TUNE TEST OK` said nothing about it until the + call was added. Unquirked: per-format dispatch helpers in one `_kq_fixtures.das` shared by + the test and the harness (`kq_transcode_sb`, `kq_dequant_sb`, `kq_dot`, `kq_repack`, + `kq_gemv_gen`, `kq_tile_gen`), each a single ladder, and ONE family array both probe modes + walk. +3. **The bake identity is hand-formatted.** `DlimCpuConfig` gains `kq_mr`, and + `dlim_identity` must ALSO append it to the identity string by hand - a field added without + the string leaves two images with different repack interleaves keyed identically. Why: the + identity is a formatter, not a reflection. Unquirked: format the `kq_mr*` fields from a table + the `KqFmt` enum drives. +4. **The `[tune]` family must exist before the emitter does.** `iq4xsq8_tile_gen` carries the + full `[tune_perm]` grid and a `tune(gen = ...)` key from day one because the loader's kq + dispatch, the layout companion and the tuner's completeness check all resolve the family by + name; the generator behind the key is a two-line `=> false` until the emitter arm lands. + Why: a declined generator is the framework's own fallback path, so the stubs are the + cheapest way to have a real family. Not a defect - record it so nobody "fixes" the stubs. +5. **Three int id spaces.** `int(KqFmt)` (device stack tags, image plane ids, `vk_kq_schema_id`'s + input), the kernel/IR id (`kq_schema_id`: 4/5/6/40/44 - runtime params of generated + kernels, `kq_qsb(int)`, `kq_rows_fn`), and the stream/repack region code (`kq_stream_code`: + 0/2 for q8/q51, else the kernel id). A new format touches all three; `test_kqformat` pins + them. Unquirked: one id, or at least one table that derives the other two. +6. **`|=` is not defined on `uint8`** in daslang (`blkb[i] |= uint8(x)` is a compile error). + Build the byte in an `int` and store it once. +7. **The image meta list is hand-maintained behind a count tripwire.** `kq_repack_mr` is a + non-array `Model` field, so `serialize_image_meta` (`dasllama_image.das`) must + `serialize_raw` it AND `IMAGE_META_FIELDS` must grow by one, or the first model load + dies on `verify(count_meta_fields(t) == IMAGE_META_FIELDS)` - after the tokenizer build, + not at compile time. The tripwire is deliberate (it catches a forgotten field); the hand + list is the quirk. Unquirked: the per-format `mr` moves into the `KqPlanes` table of + QUIRK 1 and the image serializes the table. +8. **The repack interleave is frozen onto the Model in a ladder of its own.** `layout_repack_kq` + repacks every format with the layout companion's `mr`, and a SEPARATE ladder in + `dasllama_load.das` (two copies: the streamed-save arm and the eager arm) records + `t.kq_repack_mr = active_kq_layout_mr()`. A format missing from that ladder keeps + the field's default (4) while its planes sit at the companion's `mr` (8 on this box): the + GEMV rail still reads the companion directly and is right, every `kq_active_mr` consumer - + the fused decode chains, `embed_row`, the grp row dequant - reads the wrong interleave, and + the model emits "time time time" with no diagnostic. Caught only by an end-to-end run; the + unit gates build their planes in-process and never see the freeze. Unquirked: the repack + records its own `mr` (one field per format in the `KqPlanes` table of QUIRK 1), and the + freeze ladder disappears. +9. **A kernel that runs on a forked worker reads a module-level table as zeros.** The fused + chains invoke the rows cores inside `team_parallel_stages` / `maybe_parallel_for` job + contexts; a `let` global (`IQ4NL_LUT`) read there returns 0, so a codebook format's rows + came out zero on every worker-run row and right on the main-context rows - the GEMV probes + (no job queue) all passed, the forward was garbage. The k4/k5/k6/q40 cores never had a + table; the mx4 path already hands its workers a `lutp` pointer for exactly this reason. + Rule: a kernel that can run on a worker takes its table as a per-call local + (`iq4nl_lut()`) or a pointer argument, never a module global; keep the global for tests, + oracles and the emitter's constant bake. Probe that catches it: any kernel gate run under + `with_job_que() { setup_dasllama_jobque() ... }` with enough rows to fork. +10. **The worktree session sees a stale binder.** LSP/MCP diagnostics in a fresh worktree come + from the main tree's binary and report the new enum member as "not inferred" and every + dasllama file as broken (`get_total_perf_cores` missing). Trust only the worktree binary: + `bin/Release/daslang.exe -jit dastest/dastest.das -- --test `. Run the session inside the + worktree once it is bootstrapped. +11. **A sidecar minted while the stubs declined pins the family to `"reference"`.** The app's + auto-policy tune ran during the first end-to-end (QUIRK 4's stubs in place), every perm of + `iq4xsq8_tile_gen` declined, and `examples/dasLLAMA/run.tune.json` recorded + `"iq4xsq8_tile_gen" : "reference"` - the framework's explicit-reference form, which forces + the original body. Sidecar staleness keys on the binary's mtime, and the emitter is `.das` + (JIT-compiled), so landing it invalidates nothing: the next run logs the same + `27 tune-stamped`, serves the reference body, and its text and t/s match the pre-emitter run + exactly - a "the emitter changed nothing" reading that is false. Re-mint with + `-- --tune-only q8_tile_gen` on the app (one family races, every other seat and the + runtime knobs survive the upsert; ~3 min end to end vs the 20-minute whole-scope walk the + doubled family list costs now - and each app roots its OWN sidecar, so run/parity/ + lcpp_bench each re-mint on first touch of a new family; bare `--tune` stays the + whole-scope form, skills/tune.md has the flag) or delete the sidecar. Unquirked: fold the family's + generator hash (the JIT DLL cache key already carries it) into the sidecar identity, so a + generator change reads as stale. +12. **A fresh worktree has no JIT until `lib/LLVM.dll` is staged - on every platform.** The + M1 worktree's first `-jit` run died on `can't load library LLVM.dll`; `utils/mcp/setup.das` + staged the codegen backend on Windows only. UNQUIRKED: `stage_jit_backend` now stages + `lib/LLVM.dll` + `.version` on every platform (Windows additionally `bin/lld-link.exe`). +13. **The Metal test ladders are nested ternaries with an `else` = k6.** `kq_gemv_gate`, + `kq_mvb_gate`, `kq_mulmm_gate` and the fixtures pick MSL sources / entries / fastmath / + tgmem names per format in four parallel ternary chains each; a format missing from any one + of them silently tests k6's kernel under the new format's tag. Same shape as the tier's + own dispatch ladders (QUIRK 1's cousin on Metal). Unquirked: one per-format record + (`src, entry, fastmath, tgmem`) per kernel family, indexed by format. +14. **The mul_mm template has no `static_if ... elif`.** Adding a format arm to + `MetalKqMulMmK45T` beside `SIXBIT` meant re-nesting the k4/k5 arm one level deeper + (`static_if (IQ4XS) {...} else { }`); the diff is mostly indentation. + Unquirked: an `elif` on `static_if` in the MSL emitter. +15. **First run on a fresh box tunes before it serves.** `run.das` on the M1 spent its first + minutes minting `run.tune.json` (every family, the confirm pass included) before loading + the model - the auto policy, working as designed, but a Mac session's first end-to-end + "hang" is that mint. Watch `@tune begin/end` lines in the log. UNQUIRKED for known CPU + classes: the scope ships defaults profiles (`performance/defaults/.tune-defaults.json`), + and an untuned start ADOPTS its class's profile instead of racing - the mint survives only + on new silicon (`--tune`, or the race-on-unlock residue). `skills/tune.md`, the profiles + section. +16. **A stub family fails the probe's test mode on every perm whose layout companion + generates.** `q8q8_layout` is shared and generates for any perm `perm_declines` admits, so + under `=> false` stubs the k3 family's `dot_maddubs_width256_mr8` perm repacked its planes + at mr 8 while the declined tile ran the reference body at `k3q8_layout_gen()` = 4 - + `maxdiff nan`. The lockstep between layout and tile declines IS `perm_declines`, which a + stub does not share. Keep a stub family out of the probe's TEST list (the tune-mode list is + fine: the race gates each perm and mints "reference"); it joins the test list with its + emitter arm. This is why iq4xs entered that list only in Phase B. +17. **A new `[tune]` family re-tunes every application on its first start.** A sidecar with no + entry for a demanded kernel re-tunes the whole scope (the framework's completeness rule), so + `run.das` on the box spent minutes minting before the Q3_K_L load - and every other + sidecar on the box (`chat.das`, the harness probes) does the same on its next run. Budget + for it; do not read the first end-to-end wall time as a load-time regression. SOFTENED by + the defaults profiles: profile-covered families adopt, and only the families the profile + predates race (the adopt path arms `--tune-only` with exactly those) - so land the new + family in the checked-in profiles (re-export on the reference boxes) in the same arc. +18. **`DAS_TUNE_POLICY=fallback` is not "the reference body".** It walks the family's + `fallback=` chain, and on a box where the chain's first viable perm is the crowned one the + kill switch serves the very stamp under test - the k3 stamped-vs-"reference" rows diff came + back bit-identical for that reason. UNQUIRKED: `DAS_TUNE_POLICY=reference` now serves the + original bodies outright - `[tune]` families and `[tuned]` loop-hint kernels alike - so an + A/B against the true reference tier is one env var, no sidecar editing. +19. **The device gather's tail-row branch is a second ladder.** `moe_gpu_gather_stack_kq` + walks grouped rows (the grp interleave) and ungrouped rows (`d % mr` tails, or an + unrepacked load) through two different per-format ladders; the ungrouped one re-pairs k4 + nibbles and re-decodes the 6-bit scale packing in its `else`, so a format that is already in + the device form (q40, iq4xs, k3) needs its verbatim arm in BOTH. iq4xs had only the grouped + one - latent, because every row of the dense 1B is grouped - and the k3 walk found it. + Unquirked: one per-format "device form" predicate both branches key on. +20. **A `fixed_array` local in a SPIR-V kernel is Function-storage memory.** `iq4_word`'s + first form held `kvalues_iq4nl` as a 16-entry `fixed_array` local and indexed it per + nibble; the emitter lowers a dynamically indexed local array to Function storage - private + memory the driver spills - and the IQ4_XS Vulkan GEMV decoded at 105.7 t/s against k3's + 372.8 on fewer bytes per weight. The form that runs at speed packs the codebook into four + `uint4` words (four int8 codes per word) and selects with a dynamic vector index plus a byte + shift, `(tbl[q >> 2] >> ((q & 3) * 8)) & 0xFF`: 338.4 t/s, 0.99x llama.cpp, kernel suite + 64/64. SPIR-V-only: the Metal twin `iq4_lut` rewritten the same way measured flat (tg128 + 144.3 +- 6.2 against 138.7 +- 5.4, pp512 2802 against 2953), so MSL's constant-address + array stays - the Metal IQ4_XS decode gap (0.56x llama.cpp) is the lane map, not the + lookup, and belongs to followup_general #58. Unquirked: the SPIR-V emitter lowers a `let` + fixed_array of literals to a constant-storage array, or a lint on a dynamically indexed + fixed_array local inside a kernel class. +21. **An emitter-arm edit does NOT invalidate the JIT DLL cache.** The registered code + generators (`dasllama_gemm_gen.das`) run at codegen time and their bodies do not fold into + the cached DLL's hash - after an emitter change, every `-jit` run that hits the cache + executes the OLD stamps with no signal (the iq3s gemv arm "failed" three fix rounds in a + row on byte-identical numbers before the cache-hit line gave it away; the hash was the same + 0xa3a02e12... across every edit). After ANY emitter change: `rm -rf .jitted_scripts` (or + bump `LLVM_JIT_CODEGEN_VERSION`) before trusting a probe or bench. Unquirked: the cache + hash folds the generator bodies (plans/interpreter ledger has the sibling + `jit_dll_semantic_hash` item). + +22. **A Metal kernel lever is judged by a kernel-level probe, never by tg128 e2e.** A tg128 + row on the M1 carries +-8 t/s of noise; the iq3s f4-slab GEMV form (+9% at the kernel) + measured FLAT on e2e twice and was nearly discarded. The dispatch-loop probe (50 dispatches + per encoder, best-of-3 encoders, GB/s off plane bytes) resolves 3% in seconds. +23. **Census the vehicle GGUF before committing to it.** An ftype is a MIX: the natural + "IQ3_XXS" download (mradermacher i1) quantizes attn_k/q to IQ2_S - unsupported until the + iq2 tier - so the file cannot load end to end. `gguf_census.py` (scratch) reads the tensor + type table in seconds; pick a vehicle whose mix is {supported} + the new format only + (bartowski's Qwen2.5-1.5B IQ3_XS carries iq3_xxs with iq3_s/q4_K/q6_K siblings). +24. **A layout/pack edit does NOT invalidate cached `.dlim` images.** The prepared-image + filename hash folds the model and schema, not the packing code: after the grouped-scale + copy fix, the rerun "mapped" the stale image and kept serving the broken device planes - + garbage text and all-zero decode logits survived a correct fix. Delete the model's + `*.dlim` beside the GGUF after ANY `dasllama_layout.das`/pack edit and confirm the next + log line says "baked", not "mapped". UNQUIRKED: `PACK_VERSION` (dasllama_layout.das) + folds into every image identity through `image_identity_of` - the ONE identity spelling + all consumers format through - so bumping it with a pack edit re-bakes; the discipline is + now "bump the constant beside the code you edited", not "remember to delete files". + +25. **The stream-code space is NOT the kernel-id space: q51 squats on 2.** Stream region + tags are q8=0, mx4=1, q51=2, then the kq kernel ids - Q2_K's mnemonic id 2 collided and + the first load dispatched k2 regions down the q51 repack arm (index out of range on the + empty q51s plane). k2 streams under code 20 (kq_stream_code), translated back to kernel + id 2 at the two dispatch boundaries (stream_repack_one's arm, repack_regions' fk). A new + format's kernel id must dodge 0/1/2 in the stream space or claim a distinct code the + same way. + +26. **Region `f` is the STREAM code at the repack-pointer ternaries too - an `f == 2` arm + there is DEAD (q51 intercepts 2 upstream), and k2 arrives as f = 20.** QUIRK 25's third + boundary: the big-load repack pointer pick (layout.das, the kqp/ksp ternaries beside the + `fk` translation) originally ended `... : k2qp`, so k2's 20 landed on the ELSE by + position. The iq2s walk turned that else into a keyed chain (`f == 2 ? k2qp : iq2sqp`) - + from that day every k2 region fell through to the NEWEST format's planes, latent until + the first vehicle mixing Q2_K with an i-quant (the IQ2_XS requant: llama.cpp's ftype + puts q2_K on blk.0/1 ffn_down). repack_k2_grp(mr 8) ground over the iq2xs planes: + k2-shaped scale headers, scattered garbage/NaN f16 scale reads, logits all NaN, argmax + token 0 forever. Every serial gate passes (fixtures repack through the fn directly) - + the corruption exists only in the LOADED model, and the first rows can look clean. The + arms key `fk` now. When a walk touches any region/stream ladder, check stream-code vs + kernel-id keying at EVERY boundary, and e2e a MIXED vehicle, not just a pure one. + +27. **Team-mode job lanes never run global init - a `let` module global reads ZERO there.** + (Boris's standing rule, stated during the same hunt.) Nothing reachable from a + team-lane kernel may read a das module global: tables ride per-call builders + (`iq2s_grid2()`), constants ride functions - `GEMM_REFERENCE_MR` is a function for this + reason. Audit a new format's whole rows/tile/dot call graph for global reads; a global + that happens to const-fold today is one refactor away from a lane read of zero. + +## Per-format notes + +### IQ2_XXS Phase A (CPU, 2026-08-31) - the LAST format: iq3xxs's aux32 over a 256-entry u64 grid + +Shape: 66B disk = [f16 d][8 blocks x 8B], each block = aux32[0] (4 BYTE indices into +iq2xxs_grid[256], u64 entries, magnitudes {8, 25, 43}) + aux32[1] (four 7-bit KSIGNS_IQ2XS +indices + the per-32 scale nibble in the top 4 bits) - iq3xxs's sign/scale machinery over the +iq2xs-style two-word u64 grid. Planes: the 64B qs region VERBATIM (aux stays in-plane; +IQ2XXS_QSB 64, 16 uniform grp columns - column 2b grid bytes, 2b+1 aux32); scale row = the +iq3xxs shape (d EIGHTH-ed, 8 per-32 (1+2s) strips + 8 pad, IQ2XXS_SSB 20). Ids: enum +iq2xxs = 14, kernel/stream 25, GGML_TYPE_IQ2_XXS = 16, IMAGE_VERSION 26. Vehicle: local +requant again - census iq2_xxs x94 + q4_K x16 + q2_K x2 + q5_K embd, the exact IQ2_XS mix. + +MOST of the walk was GENERATED: a twinning script replayed the IQ2_XS Phase A commit's +pure-insert hunks with iq2xs->iq2xxs / 24->25 substitutions (~500 of ~900 inserted lines); +the hand work was the codec bodies (transcode/dequant/dot/grp-dot: the aux32 walk), the +512-word grid table, and the chain-tail extensions the twin cannot express. TRAPS the gates +caught: kq_batch_cell_gen and the tests' tile ladder needed their PACKED fmt-25 arms BY HAND +(the fmt-24 template is a panel arm - a missing cell arm falls to k6's panel path and +segfaults JIT-ONLY, since the groupn gate skips off-JIT); the grid fn must `return +fixed_array(...)` DIRECTLY (a let-local round trip crashed the JIT's cmres return); +the generated per-format sections in gen_tune_probe silently kept iq2xs sizes until the +filtered mint reported "no generator family matches" - the --tune-only re-mint doubles as a +REGISTRATION gate. Gates: kqformat 18/18, kquant interp 248/0 + -jit 263/0, GEN TUNE TEST +OK, lint 0 (20 files). QUIRK 15 pinned via `--tune-only iq2xxsq8_tile_gen` (1 of 15 families, +seconds). E2e: parity ids 64/64 vs llama.cpp - the THIRD consecutive full-match stream. +JIT emitter, Vulkan, Metal: pending. + + +Phase B (JIT emitter, 2026-08-31): the panel route absorbs the aux32 form with ONE new arm - +`kq_grp_row_dot_b` grows a fmt-25 strip arm (per-32 UNSIGNED, the iq3xxs shape; the emitter +needs NO new fold at all, because (2ls+1) strip bytes stay under 128 and the shared per-32 +SExt load reads them exactly - iq2xxs sets `panel` but NOT the `iq2` split-accumulator flag). +`emit_iq2xxs_gather` loads each block's aux32 once (four byte loads composed), then per l a +grid-BYTE index doubles into the [512 x i32] private grid; signs ride the shared iq3xxs +ksigns global. `unpack_iq2xxs_panel_grp` mirrors it for the tile. kfxs25 TEST rows returned. +Probe: 11/11 k25 perms ok (maddubs 9.5e-7). Gates: kquant -jit 263 / interp 248, 0 failed, +lint 0. QUIRK 15 cleared per-app with `--tune-only iq2xxsq8_tile_gen` (crowned +dot_maddubs_width256_mr8); TRAP: run.das takes the model as the LAST `.gguf` argv token - +flags placed after it silently fall back to DEFAULT_MODEL (put the model last). E2e stamped: +the reference stream's 64/64 full match moves to a fork at step 3 that is a 0.0267-logit +top-2 near-tie (das picks the oracle's #2 - the ARC'S TIGHTEST; stamped folds move flip +points, not the class). zen2 16t vs clean-cpu: pp512 519.0 vs 181.6 (2.86x), tg128 61.1 vs +87.3 (0.70x - the grid-format decode tail class). Vulkan, Metal: pending. + + +Phase C (Vulkan, 2026-08-31): the iq3xxs shell over the two-word u64 grid - `KqGemvIq2xxs` +(2 KB `@workgroup uint[512]`, 8 words per thread; block b's grid word carries four BYTE +indices, its aux32 the parity-derived ksigns via `ksign7`), `KqBatchIq2xxs : KqBatchIq4xs` +(the per-32 strip plane and fma INHERIT - (2ls+1) strips stay under 128, so the base's +signed extract reads them exactly), `Iq2xxsCm2T` / `VkIq2xxsBlk` (the 64B qs as 16-bit +lanes) on a gated `IQ2XXSGRID` axis; schema 14 -> 25; thirteen-format family cells at +stride 16. SIGN CONVENTION TRAP: the family's random-byte tests demand SIGNED per-32 strip +reads EVERYWHERE (class, cm2 decode, float + f16 oracles alike) - an unsigned `& 0xFF` read +agrees on real (2ls+1 < 128) strips but forks the hash-fill fixtures, and the mismatch +surfaces one tier at a time (batch first, then cm2) as each side is aligned. Gates: +test_vulkan_kernels 86/86, lint 0. E2e resident (DASLLAMA_GPU=1): the Vulkan stream is +64/64 IDENTICAL to the stamped CPU stream - tier-bit-consistency; both carry only the +step-3 0.0267 top-2 flip vs llama.cpp. 5060 Ti vs llama.cpp b10660 -ngl 99: pp512 14423.2 +vs 18572.5 (0.78x - above the 0.70x class), tg128 297.0 vs 421.4 (0.70x - the grid-gemv +re-stage at 2 KB, milder than iq2xs's 4 KB; followup_vulkan #35). Metal: pending. + + +Phase D (Metal, 2026-08-31): the iq2xs Metal walk over the aux32 form - `iq2xxs_gw` (512 +words, program-scope constant), `MetalKqGemvIq2xxs` + `MetalKqMvIq2xxsT` B2/B4 + +`MetalKqMvB8Iq2xxs` + `MetalKqMulMmIq2xxs` (its own `IQ2XXS` static_if arm), every kernel +deriving signs via `ksign7m` off the block's aux32 and reading the ONE per-32 UNSIGNED strip +(byte `bu` of the 16B strip region - the 8 pad bytes ride along unread); the "iq2xxss" blob +arm is the iq2xss split verbatim. Gates: metal gemv 2/2 + gemm 2/2 FIRST TRY, zen2 kquant +-jit 263/0 regression, lint 0. E2e --ngl 99: the Metal stream is 64/64 IDENTICAL to the +stamped CPU stream - ALL FOUR TIERS of IQ2_XXS are bit-consistent with each other, carrying +only the step-3 0.0267 top-2 flip vs llama.cpp. M1 benches: CPU das 897.4/56.3 vs llama.cpp +139.9/98.4 (6.41x/0.57x), Metal das 3227.6/180.7 vs 3473.3/231.5 (0.93x/0.78x). The format +is CLOSED - and with it THE FORMAT LADDER: four-tier table zen2 2.86x/0.70x, vk 0.78x/0.70x, +M1 CPU 6.41x/0.57x, Metal 0.93x/0.78x. + +### IQ2_XS Phase A (CPU, 2026-08-31) - the ksigns u64 tier + +Shape: 256-superblock grid format - each of the 32 u16 qs words carries a 9-bit index into +iq2xs_grid[512] (u64 entries, magnitudes {8, 25, 43}) and a 7-bit KSIGNS_IQ2XS index (bit j +flips element j) - the iq3xxs sign machinery over the iq2s-style u64 grid. Scales are the +IQ2_S row EXACTLY: per-16 nibbles folded (0.5+ls)*0.25 = (2ls+1) x d/8. Disk 74B: [f16 d] +[32 u16 qs][8 scale nibble bytes]. Planes: the 64B qs region verbatim (IQ2XS_QSB 64 - 16 +uniform grp columns; a u16's two bytes always share a column), the 20B eighth-ed-d strip +row (IQ2XS_SSB 20). Ids: KqFmt.iq2xs = 13, kernel id 24, stream code 24. Grid: +iq2xs_grid2() (4 KB per-call local, 1024 words as low/high pairs) + IQ2XS_GRID. Vehicle: a +local requant (llama-quantize --allow-requantize --imatrix, Q8_0 -> IQ2_XS) - census CLEAN: +iq2_xs x94 + q4_K x16 (attn_v) + q2_K x2 (blk.0/1 ffn_down) + q5_K embd, every sibling +already supported. + +The walk surfaced TWO buried defects, now QUIRKs 26/27: the DEAD `f == 2` repack-pointer +arm (k2 regions fell through to the newest format's planes since the iq2s walk - this +vehicle's q2_K ffn_down ground repack_k2_grp over the iq2xs planes; the arms key `fk` now), +and Boris's team-lane global-init rule (GEMM_REFERENCE_MR is a function now). Gates: +test_kqformat 18/18, test_kquant 250 (232 pass, 18 env skips), probe GEN TUNE TEST OK, +lint 0. E2e: ids 64/64 vs llama.cpp - the arc's FIRST full-match greedy stream, no fork +anywhere in the window - gen 23 t/s reference bodies. JIT emitter, Vulkan, Metal: pending. + +Phase B (JIT emitter, 2026-08-31): iq2xs rides the fmt-23 panel route wholesale - the +kq_grp_row_dot_b 23 arms extend `|| fmt == 24l` (signed panel bytes, per-16 UNSIGNED +strips, d/8 fold), the tile drops off the packed lists and reads the byte-expanded panel +via `unpack_iq2xs_panel_grp` (each u16 = a 9-bit iq2xs_grid[512] index doubled into the +u64 low/high word pair + a 7-bit ksigns byte through the shared smask negate), and the +gemv gathers per superblock via `emit_iq2xs_gather` off `iq2xs_emit_globals`' [1024 x i32] +private grid plus the SHARED iq3xxs ksigns / iq3s smask globals. kfxs24 TEST rows return +(QUIRK 16 closed). Probe: 11/11 k24 perms ok (maddubs 5.2e-7); the tuner crowns +dot_maddubs_width256_mr8. QUIRK 15 reran as written, cleared with a whole-scope +`run.das -- --tune` - the 20-minute walk that motivated `--tune-only` (the QUIRK 15 +recipe above carries the new spelling). Gates: test_kquant -jit 246 pass / 4 skips, interp +232/18, GEN TUNE TEST OK, lint 0. E2e stamped: parity ids 64/64 vs llama.cpp - Phase A's +first-ever full-match stream HOLDS under the stamped folds; gen 23 -> 54 t/s. zen2 16t vs +clean-cpu: pp512 486.4 vs 174.8 (2.78x), tg128 59.4 vs 84.6 (0.70x - the ledgered +grid-format CPU-decode tail, the iq2s class). Vulkan, Metal: pending. + + +Phase C (Vulkan, 2026-08-31): the iq2s walk with ksigns-by-parity and NO extra planes - each +u16 qs word carries its 9-bit grid index (TWO staged words out of a 4 KB `@workgroup +uint[1024]`) and a 7-bit KSIGNS_IQ2XS index whose sign byte `ksign7` recomputes (the iq3xxs +parity trick; no sign plane, no qh), the strips/d epilogue verbatim iq2s. `KqGemvIq2xs`, +`KqBatchIq2xs : KqBatchK6` (stage_w gathers off `wq[wsb * 16u + k]` - the u32 word index IS +k), `Iq2xsCm2T` over `VkIq2xsBlk` (the 32 u16 words as 16-bit lanes) on a new gated +`IQ2XSGRID` axis; schema arm 13 -> 24; twelve-format family cells at stride 16 (uint4-ALIGNED, +unlike iq3s/iq3xxs/iq2s); a parity-fold float witness + an iq2xsf16 cm2 oracle. Gates: +test_vulkan_kernels 84/84 (the three iq2xs cm2 tiles 0-off at 89600 cells each), lint 0. +E2e resident: parity ids 64/64 vs llama.cpp - the full-match stream holds on the THIRD tier. +5060 Ti vs llama.cpp b10660 Vulkan -ngl 99: pp512 14023.3 vs 18320.0 (0.77x - above the +0.70x tier class), tg128 188.7 vs 349.9 (0.54x - the grid-format gemv workgroup re-stage +amplified on a 1B, followup_vulkan #35; pp healthy isolates it to the gemv). Metal: pending. + + +Phase D (Metal, 2026-08-31): the iq2s Metal walk with the parity trick end to end - `iq2xs_gw` +(1024 words, the same all-literal program-scope-constant hoisting), `MetalKqGemvIq2xs` + +`MetalKqMvIq2xsT` B2/B4 + `MetalKqMvB8Iq2xs` + `MetalKqMulMmIq2xs` (its own `IQ2XS` +static_if arm nested over IQ2S's), every kernel deriving the sign byte via `ksign7m` off the +u16 word (no sign plane, no qh); the "iq2xss" blob arm = the iq2ss split verbatim (16 strips + +2B d8 tail, 18B/sb) over the SAME 20B row. Gates: test_metal_gemv 2/2 + test_metal_gemm 2/2 +on the M1 (first try - the iq2s shells carried it), zen2 kquant -jit 246/4 regression, lint 0. +E2e --ngl 99: parity ids 64/64 - the FIRST format whose full-match stream holds on ALL FOUR +tiers. M1 16GB benches: CPU das 746.6/51.6 vs llama.cpp 144.8/101.1 (5.16x/0.51x - the +#60/#61 CPU tg tail, deepest of the grid formats), Metal das 3223.1/180.1 vs 3462.0/208.3 +(0.93x/0.86x - the iq2s pp class). The format is CLOSED on all four tiers; four-tier table: +zen2 2.78x/0.70x, vk 0.77x/0.54x, M1 CPU 5.16x/0.51x, Metal 0.93x/0.86x. + +### IQ2_S Phase A (CPU, 2026-08-31) - the u64-grid tier + +Shape: 256-superblock grid format, the first with a u64 grid - a 10-bit index (qs byte | +qh 2 bits) selects EIGHT magnitudes ({8, 25, 43}) from iq2s_grid[1024]; the block's own +sign bytes flip them (bit j - no ksigns table), scales are PER-16 nibbles folded as +(0.5+ls)*0.25 = (2ls+1) x d/8. Disk 82B: [f16 d][32 idx][32 signs][8 qh][8 scale-nibble +bytes]. Planes: [idx][signs][qh] verbatim (IQ2S_QSB 72), scale row the k4 20B STRIDE with +d EIGHTH-ed at transcode (three f16_half_bits) + 16 strips (1+2s) - the row fills exactly, +no pad. Ids: KqFmt.iq2s = 12, kernel id 23, stream code 23 (clears QUIRK 25's 0/1/2). The +grid ships as iq2s_grid2() - the per-call local at 8 KB (2048 uint words, low/high pairs) - +plus the main-context IQ2S_GRID. Repack: 18 uniform 4-byte columns; the k4 scale interleave +with 16 strips. Gates: test_kqformat 18/18, test_kquant 233 (216 pass, 17 env skips), +lint 0. E2e: the mradermacher i1-IQ3_XXS vehicle (on disk since the iq3xxs phase, blocked +on its IQ2_S attn x32) now loads and decodes FIVE formats in one graph - ids 10/64 vs +llama.cpp with the fork a 0.040-logit near-tie (the arc's tightest; top2 IS our token), +gen 41 t/s reference bodies. + +Phase B (JIT emitter, 2026-08-31): iq2s joins the fmt-33/34 PANEL route end to end - the +tile drops off the packed lists (probe, batch cell, test gate) and reads the byte-expanded +panel via a new `unpack_iq2s_panel_grp` (the 10-bit qs|qh index - `(qh << (8-2l)) & 0x300` - +doubled into the u64 grid's low/high word pair, sign bytes through the same smask +expansion), with `kq_grp_row_dot_b` growing a 23 arm: SIGNED panel bytes but per-16 +UNSIGNED strips (s0*ilo + s1*ihi) under the 33/34 d fold (d pre-eighth-ed at transcode). +The gemv gathers per superblock via `emit_iq2s_gather` off `iq2s_emit_globals`' [2048 x +i32] private grid + the shared smask; in `emit_block_iq4xs` the iq2 flag SPLITS the lo/hi +accumulators (a/a1) and loads two ZExt strip vectors per block, since per-16 strips cannot +share the fused 33/34 dot. Probe: 11/11 k23 perms ok (maddubs 8.3e-7); the tuner crowns +`dot_maddubs_width256_mr8` verdict=beats (the zen2 usual). QUIRK 15 reran exactly as +written: run.tune.json still pinned `iq2sq8_tile_gen : "reference"` from Phase A, cleared +with `run.das -- --tune`. Gates: test_kquant -jit 229 pass, interp 216/17 skips, +kqformat 18/18, M1 probe + kquant -jit 232/1, lint 0. E2e stamped: gen 41 -> 47 t/s; the +fork vs llama.cpp moves from step 10 to step 5 (0.211-logit top-2 near-tie - stamped folds +move the flip point, not the class; the iq3xxs precedent). zen2 16t vs clean-cpu on the i1 +vehicle: pp512 501.5 vs 138.5 (3.62x), tg128 55.8 vs 73.5 (0.76x - the ledgered #60/#61 +CPU-decode tail). + +Phase C (Vulkan, 2026-08-31): the iq3s walk widened to the u64 grid. `iq2s_grid_word` ships +the 8 KB table as low/high word pairs (word 2i / 2i+1); `KqGemvIq2s` = the iq3s shell with +`@workgroup uint[2048]` (32 words staged per thread at local_size 64), TWO grid words per +10-bit qs|qh index (`(qh << (8-2l)) & 0x300`), the block's own sign bytes through the same +smask negate, and the per-16 UNSIGNED strips folded k2-style off wsu words 1..4 (split +lo/hi dots, d pre-eighth-ed). `KqBatchIq2s : KqBatchK6` - the iq3s gather in stage_w, the +strip pair on the wsw/wsw2 planes, and a plain split fma (no offset, no min term). +`Iq2sCm2T` = `VkIq2sBlk` ([qs 16][sg 16][qh 4] int16 lanes) on a new `IQ2SGRID` gated axis +(8 words per thread at 256); the device scale row is the CPU 20B row VERBATIM (d + 2B hole ++ 16 strips - the grouped gather's k4/k5 else arm already emits exactly that, no layout +edit). vk_kq_schema_id 12 -> 23; eleven-format family cells at stride 18 words; a dedicated +float witness (`iq2s_gemv_float_oracle`) plus `iq2sf16_gemm_oracle` for the cm2 tiles. +Gates: the suite 82/82 (the three iq2s cm2 tiles 0-off at 89600 cells each); lint 0. The +resident driver arms on the i1 vehicle: gen 233 t/s, the fork vs llama.cpp at step 5 = the +SAME 0.211-logit top-2 near-tie as the CPU-JIT tier. Rows (5060 Ti vs llama.cpp b10660 +build-vulkan -ngl 99): pp512 12099.3 / 17377.5 (0.70x - the tier class), tg128 292.4 / +362.5 (0.81x; the gemv re-stages the 8 KB grid per 2-row workgroup - the obvious tg lever, +same ledger class as the pp tier). + +Phase D (Metal, 2026-08-31): the constant-table commit pays off - `iq2s_gw` ships the 8 KB +grid as low/high word pairs through the 363654b0d hoisting (an all-literal fixed_array +local lowers to a program-scope constant table), so EVERY kernel reads it DIRECT: no +threadgroup slab anywhere (an f4-expanded slab of 1024 entries would be 32 KB, past the tg +budget - the iq3s gridf form does not scale to u64 grids). `MetalKqGemvIq2s` = the iq3s +GEMV geometry (4 rows/simdgroup, dispatch rows/8) with per-word `iq3s_sw`/`iq3s_sx` +compose and per-16 UNSIGNED strips (plain byte reads, no ^128 sign trick); the B2/B4/B8 +twins split dl into dl0/dl1 per half; the mul_mm rides a new IQ2S arm (strip byte js*2+il0 +ZExt, two va words per idx byte). The "iq2ss" blob arm is iq3ss's verbatim twin ([16 +strips][2B d8 tail], 18B/sb); ladders: kq_fmt_gpu_supported, moe_site_ok / blob_off_ok % +512, the 72B quant bind, the k6-split scale bind. Gates on the M1 Max: +test_metal_gemv_kernels 2/2, test_metal_gemm_kernels 2/2 (iq2s gemv/mv/mul_mm rows + the +shared fill/rowref arms); lint 0. E2e (parity --ngl 99, resident Metal decode): ids 40/64 - +the fork at step 40 is a 0.0654-logit top-2 near-tie (the arc's DEEPEST match), on the same +Sophia stream as the CPU-JIT tier. TRAP from the walk: a paren-safe ternary-ladder insert +must land INSIDE the bwidth arm, not before it (three b4 lines rebuilt). + +Against llama.cpp b10660 (`lcpp_bench --for-debug-purposes`; zen2 = 16 threads, M1 = 8; +the mradermacher i1 vehicle - IQ2_S attn x32 + IQ3_XXS/IQ3_S/Q4_K/Q5_K): + +| tier | pp512 das / llama.cpp | tg128 das / llama.cpp | +|---|---|---| +| zen2 CPU | 501.5 / 138.5 (3.62x) | 55.8 / 73.5 (0.76x) | +| 5060 Ti Vulkan | 12099.3 / 17377.5 (0.70x - the tier class) | 292.4 / 362.5 (0.81x) | +| M1 CPU | 883.3 / 413.4 (2.14x) | 53.7 / 73.9 (0.73x) | +| M1 Metal | 3170.5 / 3427.2 (0.93x) | 205.8 / 220.7 (0.93x) | + +(The CPU tg tails - zen2 0.76x, M1 0.73x - are the ledgered #60/#61 class; the Vulkan tg +0.81x adds the per-workgroup 8 KB grid re-stage to the same ledger.) + +### Q2_K Phase A (CPU, 2026-08-31) + +Shape: 256-superblock, the k4/k5 scale STRUCTURE at k6's granularity - 16 per-16-element +sc/min nibble-pair bytes folded as (d*sc)*q - (dmin*mn); the 2-bit lanes are unsigned, so +the min term rides the activation 16-sums (xbsp), exactly dot_k4q8's shape. Disk 84B: +[16 sc/min][64 qs][f16 d][f16 dmin]. Planes: qs verbatim (K2_QSB 64 - k3's lanes minus the +hmask), the scale row reordered header-first [d][dmin][16 sc/min] (K2_SSB 20). Ids: +KqFmt.k2 = 11, kernel id 2, stream code 20 (QUIRK 25). Kernels: dot_k2q8 (dot_k4q8's fold +over 16 groups), k2_grp_row_dot, repack_k2_grp (16 four-byte columns x mr; scale +[16 sc x mr][mr x 4B header]); the tile rides the packed planes. Gates: test_kqformat +18/18, test_kquant 216 (200 pass, 16 env-gated skips), lint 0. E2e: the local requant +(Q2_K x64 + Q3_K/Q4_K/Q6_K siblings, all supported - llama-quantize with the imatrix, no +--tensor-type override needed) decodes coherently at gen 30 t/s reference bodies, greedy +ids 19/64 with the fork a 0.153-logit near-tie (top2 IS our token; the lossiest format +diverges earliest). No whole-scope re-mint fired on the first e2e (zero "@tune begin" +lines), unlike QUIRK 17's precedent - resolved at Phase B: a declined-stub family demands +no sidecar entry; the mint fires once real generators exist. + +Phase B (JIT emitter, 2026-08-31): a fourth arm through emit_block_kqv2 - the k2 flag +swaps the quant compose to k3's column walk minus the hmask (2-bit lane at shift +2*(blk%4)), the scale reads to per-16 nibble extracts (ZExt the pair bytes to i32 lanes, +then &15 / >>4 - no i8-width splat needed; scv/mnv carry the two groups' sc, mnlo/mnhi +their mins), keeps the k63 split lo/hi accumulators behind a per16 flag +(vecBsums/fuseAcc/madd16/flush/bs0-bs1), and takes the k4/k5 two-fma epilogue (bacc folds +on dmin). Prerequisite refactor: the grp scale header went FIELD-MAJOR +([16 sc x mr][mr x f16 d][mr x f16 dmin]) so load_f16_vec_at serves the d/dmin vectors - +repack_k2_grp and both grp readers moved together, re-gated bit-exact. Gates: probe test +mode 11/11 k2 perms (maddubs mr8 stamps at 4.8e-7; vpdpbusd declines on zen2 as every +format), test_kquant -jit 212/216. Crowned dot_maddubs_width256_mr8; stamped e2e gen +30 -> 75 t/s, ids 19/64 with the same token-19 near-tie. zen2 16t vs llama.cpp clean-cpu +b10660: pp512 418.6 / 412.0 (1.02x), tg128 77.1 / 81.1 (0.95x). + +Phase C (Vulkan, 2026-08-31): KqGemvK2 = k3's dot minus the hmask (its int4 (ilo, ihi, +blo, bhi) already carries the half sums) folded by the pair-byte nibble scales and the k4 +d|dmin word: dm.x*(sc_lo*ilo + sc_hi*ihi) - dm.y*(mn_lo*blo + mn_hi*bhi). KqBatchK2 : +KqBatchK6 stages the unsigned lanes (k3's staging minus the hmask), FOUR scale planes +(d*sc and dmin*mn per 16-half - two new @workgroup arrays), and a split-half fma folding +the min sides on the half sums. K2Cm2T = VkK2Blk (int16[32]) with K4Cm2T's srow walk over +the pair bytes. The device scale row is the CPU 20B row VERBATIM - both gather paths copy +through (the grouped path re-packs the field-major grp header). vk_kq_schema_id 11 -> 2; +ten-format family cells at stride 16 words. TRAP replayed from k3: a vk class child must +FOLLOW its parent in the file (KqBatchK2 first landed above KqBatchK6 - "parent structure +not found"). Gates: the suite 80/80, the three k2 cm2 tiles 0-off; resident e2e armed +(fresh bake), gen 284 t/s, ids 19/64 with the same token-19 near-tie. Rows (5060 Ti vs +llama.cpp b10660 build-vulkan): pp512 14544.0 / 16752.7 (0.87x - above the 0.69-0.78 tier +class), tg128 442.8 / 424.1 (1.04x). + +Phase D (Metal, 2026-08-31): the "k2s" blob arm splits the 20B row k6-style into [16 +pair-byte strips x nsb][4B d+dmin tails x nsb] (a verbatim 20B bind would need off % 1024; +the split keeps every bind aligned free - no off_ok arm). dequant_k2_plane_superblock grew +its _at twin for the split read. Kernels are the k3 shells minus the hmask: MetalKqGemvK2 +(the k3 lane map; sc scales the dot, mn folds on per-block x sums against dmin off the 4B +tail), MetalKqMvK2T B2/B4 + MetalKqMvB8K2 (w = q*d*sc - dmin*mn per element), and a K2 arm +on MetalKqMulMmK45T (the pair byte from the one uint4 strip; wrapped around the k4/k5 +default - a hand brace-balance close landed one line early and the default ran after the +arm, "redefinition of sv" in the emitted MSL; balance from INSIDE the else). Gates on the +M1 Max: gemv 2/2, gemm 2/2; the metal-blob e2e decodes the same stream at gen 246 t/s, +ids 19/64 with the same token-19 near-tie. + +Where Q2_K landed (vs llama.cpp b10660, the local requant): + +| tier | pp512 (ours / theirs) | tg128 (ours / theirs) | +|---|---|---| +| zen2 CPU | 418.6 / 412.0 (1.02x) | 77.1 / 81.1 (0.95x) | +| M1 CPU | 543.4 / 210.0 (2.59x) | 155.8 / 128.2 (1.21x) | +| 5060 Ti Vulkan | 14544.0 / 16752.7 (0.87x) | 442.8 / 424.1 (1.04x) | +| M1 Metal | 3577.6 / 3568.1 (1.00x) | 213.7 / 237.1 (0.90x, both refs drift +-11 thermally) | + +### IQ4_NL (the near-free one, 2026-08-30) + +Shape: 32-element blocks, 18B each on disk - f16 d + 16 nibble bytes with the k/k+16 +pairing. That is Q4_0's disk shape BYTE FOR BYTE; only the nibble semantics differ +(`kvalues_iq4nl` - already shipped as `IQ4NL_LUT` - instead of q - 8, so no offset and no +bsum term). The whole CPU story follows from that identity: planes = `Q40_QSB`/`Q40_SSB` +verbatim, transcode = q40's byte split with the type check swapped, grp repack = literally +`repack_q40_grp` (routed, not copied), and the tile rides the PACKED lists - no panel, no +gather. Identity 45. The scalar dot is iq4xs's LUT loop with q40's per-block f16-d fold; +the JIT emitter is one `nl` mode on `emit_block_iq4xs`: the per-block d vectors load with +the same `load_f16_vec_at` helper over q40's scale interleave, and the fold moves INTO the +block loop (float per-block fma) replacing the sub-scale integer sum - plus the LUT-bake +condition widening to `te.kq == 44 || 45`. QUIRK 16 replayed on cue: the pre-emitter gate +stamped 10 perms whose maddubs rows mismatched (declined generators at layout mr 8); the +emitter arm turned all 11 k45 rows green (maxdiff ~2e-6) in the same sitting. + +Vehicle: a local requant again - IQ4_NL needs NO imatrix, just `--allow-requantize` +(mix: iq4_nl x94 + q5_K x18 + q6_K embd, all supported; census first, QUIRK 23). Gates: +`test_kqformat` 18/18, `test_kquant` 199 tests 0 failed, the probe 11/11 k45 perms. E2e +stamped: coherent text at gen 60 t/s, 11/64 greedy ids vs `simple_ids.exe` where the fork +is a 0.042-logit near-tie (the window's smallest margin by 100x) whose top-2 IS our token. +zen2 rows vs llama.cpp clean-cpu: pp512 618.3 vs 540.6 (1.14x), tg128 64.2 vs +62.2 (1.03x). + +Vulkan followed as pure composition - not one new decode was written: `KqGemvIq4nl` = +iq4xs's `iq4_dot` codebook compose folded by q40's `q40_d` per-block d (no sub-scale, no +bsum); `KqBatchIq4nl : KqBatchIq4xs` overrides ONLY `stage_ws` with q40's (the LUT stage_w +and the no-min blk_fma inherit); `Iq4nlCm2T` = `VkK4Blk` + the existing `IQLUT` axis with +Q40Cm2T's d pick times `iq4lut[q]`. Schema fmt 10 -> 45, the nine-format family cells at +stride 32 words, the cm2 cell mirrors q40's with the codebook oracle. Gates: the suite +78/78 (the three cm2 tiles 0-off at 89600 cells each); the resident driver arms and runs +gen 245 t/s, greedy ids 11/64 vs `simple_ids.exe` (the near-tie class - the margin +oracle's step-11 tie is 0.042 logits on this stream). Rows (5060 Ti vs llama.cpp b10660 +Vulkan): pp512 15027.4 vs 19177.8 (0.78x), tg128 340.7 vs 363.6 (0.94x). + +Metal closed the format as the promised compose - the iq4xs kernels with the scale fold +swapped. No blob arm, no off-gate row, no split plane: q40's 16B/sb plane of 8 f16 d's IS +the device form, bound once at soff (kq_scales_of hands doff = 0), and every kernel reads +block bu's d at `kdh[blk*8+bu]` where iq4xs read per-sb d x strip byte. `MetalKqGemvIq4nl`, +`MetalKqMvIq4nlT` B2/B4 + `MetalKqMvB8Iq4nl` (the iq4xs shells over the threadgroup +codebook), and an `IQ4NL` constant on `MetalKqMulMmK45T` riding the IQ4XS arm with only the +dsc read flipped. One checklist line this walk misses easily: the `g_pso_*` globals are +DECLARED in dasllama_metal_common.das - the `[metal_dispatch]` pso= attribute only NAMES +them - so five new kernels = five decl lines there, or every reference errors 30838. Gates +on the M1 Max: test_metal_gemv_kernels 2/2, test_metal_gemm_kernels 2/2 (116 s corpus); the +metal-blob e2e decodes the same stream at gen 285 t/s, ids 11/64 with the fork = the +format's 0.042-logit near-tie at step 11 on every tier. + +Where IQ4_NL landed (vs llama.cpp b10660, the local requant): + +| tier | pp512 (ours / theirs) | tg128 (ours / theirs) | +|---|---|---| +| zen2 CPU | 618.3 / 540.6 (1.14x) | 64.2 / 62.2 (1.03x) | +| M1 CPU | 727.1 / 724.1 (1.00x) | 134.4 / 133.7 (1.01x) | +| 5060 Ti Vulkan | 15027.4 / 19177.8 (0.78x) | 340.7 / 363.6 (0.94x) | +| M1 Metal | 3599.5 / 3774.6 (0.95x) | 248.4 / 250.2 (0.99x) | + +The nearly-free prediction held: not one new decode on any tier, no CPU tg tail (no +gather), and the only real bug of the format was the layout copy arm QUIRK 24 caught. + +### IQ3_XXS (2026-08-30) + +Shape: 256-superblock, 98B disk block - f16 d, 64 grid-index bytes (`iq3xxs_grid[256]`, one +byte = FOUR magnitudes), 8 x aux32 per superblock (bits 0..27 = four 7-bit `ksigns_iq2xs` +indices, bits 28..31 = the block scale ls; ggml folds w = 0.25 * d * (2ls+1) * grid * sign). +The plane design removes the 0.25 EXACTLY: every ggml grid level is even (4..62), so our +plane grid stores the bytes HALVED (2..31) and the transcode stores d halved +(`f16_half_bits` - an exponent decrement, exact down to the subnormal edge) - the kernel +fold is then iq3s's `d x strip x grid_byte` verbatim with strip = 2ls+1, and the smask sign +trick still holds (it needs g != 0, not odd). Planes 96/20: quants = [qs][aux] verbatim +(24 uniform grp columns), scale = the iq3s row. `ksigns_iq2xs` (bit 7 = even parity) ships +in kqformat and is shared with the coming IQ2 family. Identity 34. + +The panel form is SHARED with iq3s: `unpack_iq3xxs_panel_grp` emits the same signed-byte +panel, so `kq_grp_row_dot_b`, the tile ladders and the batch cell take `fmt == 33 || 34` +conditions rather than new arms. Vehicle: `Qwen2.5-1.5B-Instruct-IQ3_XS` (bartowski - iq3_xxs +on attn_k/q + parts of ffn, everything else already supported); the obvious +mradermacher i1-IQ3_XXS 1B carries IQ2_S attn tensors and must WAIT for the iq2 tier - +census the candidate GGUF first (QUIRK 23). Gates: `test_kqformat` 18/18 (tables, parity +property, the halver edges), `test_kquant` 181 tests 0 failed with fmt 34 in every gate +loop - and the tile gate's `packed` list wrongly held 33 (the iq3s tile reads the +byte-expanded panel), a pre-existing red on x64 fixed in the same change. End to end +(reference bodies - both generators decline until the emitter arc): coherent text, 22/64 +greedy ids vs llama.cpp's `simple_ids.exe` where the fork is a 0.12-logit near-tie whose +top-2 IS our token (`simple_ids_margin`), gen 19 t/s. + +The JIT emitter followed the same day and is the smallest arm of the arc: the TILE is free +(the panel route widens - `emit_one_block`'s and `emit_block_iq4xs`'s `te.kq == 33` become +`33 || 34`; stride, scale row and fold are already the panel's), and only the gemv gather is +new. `emit_iq3xxs_gather` mirrors the iq3s gather with three changes: the aux32 sign word is +ONE aligned i32 column load per block (the four aux bytes of a block land contiguous in the +grp column layout), sign bytes come from a `dasllama.iq3xxs.ksigns` [128 x i32] global +(then the SAME smask nibble expansion), and the grid index is the bare qs byte into the +[256 x i32] HALVED grid - no 9th bit. Two probe lists must both gain the format or the gate +silently skips it: the test-mode fixture families AND the tune loop's +`fixed_array(4l, 5l, ...)` - the first run stamped 10 perms and gated NONE of them (grep +`k34` in the test log to prove coverage). Gates: 11/11 k34 perms ok vs the oracle (maxdiff +~2e-6); the tuner crowns `dot_maddubs_width256_mr8` verdict=beats (the zen2 board's usual +winner, same as iq3s/k3). E2e stamped: prefill 6 -> 17 t/s, gen 19 -> 41 t/s; ids 17/64 +with the fork again ON the margin oracle's top-2 (0.43 logits at step 17 - stamped float +folds move the flip point, they do not change the class). + +Vulkan came as the iq3s walk compressed: `vk_kq_schema_id` 9 -> 34; the 1 KB HALVED grid +stages into `@workgroup uint[256]` (one word per thread on the tile, four on the gemv), and +the ksigns TABLE DISAPPEARS - bit 7 of the sign byte is the even parity of the 7-bit index, +five ALU ops (`t = v ^ (v >> 4); t = t ^ (t >> 2); t = t ^ (t >> 1)`; the SPIR-V emitter has +no `^=`, spell the fold as plain assignments). `KqGemvIq3xxs : KqGemvBase` (24-word rows, +one aligned aux word per block, iq3s's fold via `iq4_sc` - the (2ls+1) strips read the same); +`KqBatchIq3xxs : KqBatchIq4xs` (the halved-grid gather at stage time); `Iq3xxsCm2T` on a new +`IQ3XGRID` gated axis (the aux 7-bit field spans at most two bytes - two unpack8 picks per +element). The sdot4 arms, cm2 dispatcher arms and the family/witness/cm2 test cells mirror +iq3s's rows exactly. VEHICLE trap beyond QUIRK 23: the Qwen2.5 vehicle proves kernels but +CANNOT arm the resident decode driver (attention QKV bias - a silent decline), so the +Vulkan e2e/bench vehicle is a LOCAL requant: `llama-quantize --allow-requantize --imatrix + --tensor-type attn_k=iq3_xxs --tensor-type attn_q=iq3_xxs out IQ3_XXS` +- llama.cpp's own ftype recipe puts IQ2_S on attn_k/q, the override keeps the mix inside +the supported set. Gates: the suite 76/76 (the three cm2 tiles 0-off at 89600 cells each, +the float witness in the family cell); the resident driver arms and matches llama.cpp's +greedy ids 32/64 with the fork a 0.093-logit near-tie whose top-2 IS our token (the +smallest margin in the window), gen 202 t/s. Rows (5060 Ti vs llama.cpp b10660 Vulkan, +the local requant): pp512 12225.7 vs 17807.7 (0.69x), tg128 372.1 vs 389.9 (0.95x). +Metal closed the format the same day, and the iq3s walk paid for itself twice: the blob +"iq3xxss" arm is the iq4xs 20->18 split verbatim, `kq_quants_of` binds at sb0*96, and the +kernels are the iq3s shapes with three deltas - `iq3xxs_gw` (the 1 KB halved grid through +the constant-table hoisting), `ksign7m` (the parity sign byte - no ksigns table on Metal +either), and 24-word rows with one aux word per block. The GEMV ships directly in the +crowned iq3s form (float4 magnitude slab + sign selects, 4 rows/simdgroup, dispatch +rows/8 - QUIRK 22's race already ran); the B2/B4/B8 twins and the `IQ3XXS` mul_mm arm (a +gated uint[256], the chain re-nested once more) follow their iq3s siblings. One +cross-tier find: the CPU dequant/dot strip reads went SIGNED (int8) to match the GPU +kernels' decode - real strips are 1..31 either way, but the synthetic test fills carry +high-bit bytes and the split-form fill is shared across formats. Gates on the M1 Max: +`test_metal_gemv_kernels` 2/2, `test_metal_gemm_kernels` 2/2; e2e decodes the SAME story +stream as the CPU/Vulkan tiers at gen 218 t/s (ttft 106ms). + +Against llama.cpp b10660 (`lcpp_bench --for-debug-purposes`; zen2 = 16 threads, M1 = 8; +the local --tensor-type requant, iq3_xxs on attn_k/q + all ffn): + +| tier | pp512 das / llama.cpp | tg128 das / llama.cpp | +|---|---|---| +| zen2 CPU | 507.2 / 136.0 (3.73x) | 56.7 / 72.6 (0.78x) | +| 5060 Ti Vulkan | 12225.7 / 17807.7 (0.69x - the tier class) | 372.1 / 389.9 (0.95x) | +| M1 CPU | 906.0 / 410.5 (2.21x) | 53.5 / 74.0 (0.72x) | +| M1 Metal | 3224.0 / 3429.9 (0.94x) | 213.5 / 227.3 (0.94x) | + +(The tg tails - zen2 0.78x, M1 CPU 0.72x - are the arc's ledgered CPU-decode class, +followup_general #60/#61: the per-superblock gather never amortizes at one token.) + +### IQ3_S (the third format - and the first grid format, 2026-08-30) + +Shape: 256-superblock grid format - an 8-bit grid index plus a qh ninth bit selects +`iq3s_grid[512]` (each uint32 = 4 weight magnitudes), the block's own sign bytes flip them (no +ksigns table anywhere - the disk carries explicit signs), scale = f16 d x (1 + 2 x 4-bit +sub-scale); signed reconstruction, so no `xbsp` term. Disk block 110 B: f16 d, 64 qs, 8 qh, +32 signs, 4 packed scale nibbles. Plane pair: quants [qs 64][qh 8][signs 32] VERBATIM (104 B - +the k3 "disk is already the device form" answer), scale row = the iq4xs/k4 20 B shape with +(1 + 2s) decoded at transcode, so every k4-row consumer serves unchanged. Ids: `KqFmt.iq3s` += 8, kernel id 33, stream code 33. The codebook question a grid format adds: the 2 KB table +ships as `iq3s_grid()` - the per-call-local worker-safe form `IQ4NL_LUT` documented, just 512 +entries now - plus the main-context `IQ3S_GRID`; the repack is 26 uniform 4-byte columns (no +per-region split - simpler than k3's, since nothing shifts per lane). Gates on the worktree +binary: `test_kqformat` 16/16, `test_kquant` 163 tests 150 pass / 13 env-gated skips (the +batch-groupn family class every format shares); the emitter stubs decline (QUIRK 4), the +family minted `verdict=rejected` (QUIRK 16's shape) after the whole-scope re-tune (QUIRK 17), +and the reference bodies serve. End to end: `Llama-3.2-1B-Instruct-IQ3_M.gguf` (bartowski: +IQ3_S x78 + Q4_K x34 + Q6_K embd) through `run.das` matches llama.cpp's greedy ids **64 of +64** at gen 23 t/s - the first format to hold the whole comparison window token-for-token. +JIT emitter (section 5, the first side-by-side under the llama.cpp-kernel rule): the TILE +rides the panel route - `unpack_iq3s_panel_grp` gathers grid words + signs into the k5/k6 +byte-expanded panel per group (one i32 store per grid word, sign nibble through a 16-entry +mask table, negate = `(w ^ m) + (m & 0x01010101)` - magnitudes are odd 1..15, no cross-byte +carry) - and the GEMV gathers each superblock into an alloca panel via an emitted per-row +loop (`emit_iq3s_gather`; grid + mask as private module constants); both then ride +`emit_block_iq4xs`'s sign-trick lattice with panel loads in place of the nibble+LUT. Two +finds along the way: `emit_slice` recorded SAVED block handles as phi incomings - a +block-splitting emitter leaves the builder elsewhere, so back-edges were malformed (fixed: +capture `LLVMGetInsertBlock` at the branch points); and QUIRK 21 ate three fix rounds. +Probe test mode: every k33 perm ok (maddubs mr8 stamped, 1.9e-6); the tuner crowned +`dot_maddubs_width256_mr8` at ~4.9x the reference body. Side by side on the zen2 (16t, +llama.cpp b10660 clean-cpu): pp512 516.9 vs 104.9 (**4.93x** - the panel amortizes the +gather across the tile; their per-row kernel re-gathers per token), tg128 52.4 vs 57.0 +(0.92x - nothing amortizes at one token; the no-panel gemv spelling to close it is +followup_general #61). Stamped e2e: 63/64 greedy ids (the flip is the FINAL token, the +stamped-vs-reference near-tie class), gen 44 t/s. + +Vulkan (section 6, the first grid format on the tier): `vk_kq_schema_id` 8 -> 33; the 2 KB +grid stages into `@workgroup uint[512]` once per kernel (llama.cpp's `init_iq_shmem` form) - +the source is `iq3s_grid_word`, a per-index accessor over a fixed_array local (QUIRK 20's +slow path, off the hot loop; a shader function cannot RETURN a fixed array), and the sign +nibble expands arithmetically (`mask = ((nib * 0x00204081) & 0x01010101) * 255`, negate = +`(g ^ mask) + m1`). `KqGemvIq3s : KqGemvBase` (26-word rows read as scalar uints - 104 B is +not uint4-aligned; iq4xs's scale fold via `iq4_sc`); `KqBatchIq3s : KqBatchIq4xs` (the grid +gather AT STAGE TIME per staged uint4; scale row and blk_fma inherited verbatim). TWO traps +this walk found: a test family cell whose enc ladder lacks the new arm silently dispatches +the ELSE format's pipeline over the new planes (fmt 8 ran the k6 tiles - 2200/2200 off with +byte-stable garbage across "fix" rounds; add the enc arm to EVERY per-fmt ladder in the +cell, not just ensure/strides); and `pf_f16_feed` admitting a format via `kq_sb` before its +cm2 tile exists sent iq3s prefill through the cm2 dispatcher whose fall-through served the +q8 tiles - garbage text at full speed. The feed now excludes iq3s until its tile lands and +all three cm2 ladders `verify` on a non-q8 fall-through. The CPU replay of a class whose +kernel stages workgroup state (the grid) must fill that state in `kq_cls_ref` - the replay +never runs the kernel head. Gates: the suite 71/72 with the seven-format family cells and +the iq3s float witness; the resident driver matches llama.cpp's greedy ids **64 of 64** at +gen 262 t/s. The cm2 tile followed in the same phase: an `IQ3GRID` gated axis on +`KqCm2BatchT` stages the 2 KB grid into a `@workgroup uint[512]` before the tile loop, and +`Iq3sCm2T`'s decode gathers per element over the 16-bit-lane block views (`VkIq3sBlk`: +qs/qh/signs as int16 arrays) with iq4xs's scale fold; three width stamps, the dispatcher +arms, and iq3s re-admitted to `pf_f16_feed`. The three tiles gate 0-off (89600 cells each); +the e2e holds 63/64 on the f16 feed (the final-token near-tie; the quant feed ran 64/64). +Rows (5060 Ti vs llama.cpp b10660 Vulkan): tg128 288.1 vs 324.2 (0.89x), pp512 12540 vs +17865 (0.70x - AT the tier's shared 1B-shape class; the k4 control on this box is 0.67x), +up from 6241 on the quant feed. + +Metal came in five surfaces plus one emitter feature. The blob "iq3ss" arm is the iq4xs +20->18 split VERBATIM over `t.iq3ss` (same strip/d addressing, so `kq_scales_of` is a copy of +the iq4xs arm with the plane swapped); quants bind at `sb0*104`. The MSL emitter grew +constant-table hoisting for the grid (its own dasMetal commit: a `let` fixed-array local with +all-literal elements lowers to a program-scope `constant T name[N] = {...}` - before that, +ANY fixed-array initializer was a compile error, and there is no other way to put 2 KB of +data into a kernel). `iq3s_gw()` carries the 512-word literal; the GEMV and mul_mm stage it +into threadgroup memory, the Mv twins read it direct (unmeasured batch shapes). Kernels: +`MetalKqGemvIq3s` (the iq4xs parity shape but 4 rows/simdgroup - llama.cpp's N_R0_IQ3_S - +with the grid staged as a `float4[512]` MAGNITUDE slab and signs applied by select), +`MetalKqMvIq3sT` B2/B4 + `MetalKqMvB8Iq3s` (iq4xs shells, per-32-block lane map, consecutive +float4 x loads), an `IQ3S` arm in `MetalKqMulMmK45T` (threadgroup grid under +`@template_gate`; re-nest the chain - `} else static_if` is a parse error, QUIRK 14's rule +holds in the MSL emitter too). + +The decode-GEMV gap got the full treatment: at n=2048 d=8192 the shipped kernel raced EIGHT +forms (tg-slab uint gather, constant-table gather, duplicated slab, gather deleted, sign +flip deleted, llama.cpp's exact 1-lane-per-block geometry, 2- vs 4-row, f4 magnitude slab) - +all land in 127-141 GB/s while k4 does 204 and k6 287 in the same harness. The compose chain +(9-bit index + per-nibble signs) is format-intrinsic on M1; the f4 slab is the best form +(+9%) and ships. Gates on the M1 Max: `test_metal_gemv_kernels` 2/2, `test_metal_gemm_kernels` +2/2 (~108s corpus), e2e decodes the coherent story at gen 227 t/s. + +Against llama.cpp b10660 (`lcpp_bench --for-debug-purposes`, das = the debug-jit instrument; +zen2 = 16 threads, M1 = 8; IQ3_M, so attn_v/attn_output/ffn_down are Q4_K and the tied +embedding head is Q6_K - three formats share every decode step): + +| tier | pp512 das / llama.cpp | tg128 das / llama.cpp | +|---|---|---| +| zen2 CPU | 516.9 / 104.9 (4.93x) | 52.4 / 57.0 (0.92x) | +| 5060 Ti Vulkan | 12539.6 / 17865 (0.70x) | 288.1 / 324.2 (0.89x) | +| M1 CPU | 886.2 / 433.6 (2.04x) | 57.4 / 66.6 (0.86x) | +| M1 Metal | 3237.6 / 3344.3 (0.97x) | 199.4 / 209.0 (0.95x) | + +(The M1 Metal ref tg drifts 209-230 across back-to-back rounds - thermal; ratios are +same-run. The tg tails on every tier are the one ledgered class: followup_general #61/#62.) + +### Q3_K (the second format, 2026-08-30) + +Shape: 256-superblock, k6's scale structure exactly (16 per-16 signed sub-scales + f16 d, the +18 B row) over a 3-bit quant composed from a 2-bit lane (byte h*32 + l of `qs`, shift 2j, block +b = 4h + j) and an `hmask` bit (bit b of byte l), offset -4 folded like k6's -32. Disk block +110 B: hmask[32], qs[64], 12 packed 6-bit scales, f16 d. Planes: quants `[qs 64][hmask 32]` +verbatim (K3_QSB 96), the scale row DECODED at transcode (`q3k_scale6` = ggml's kmask unpack, +stored as int8 scale - 32; K3_SSB 18). Ids: `KqFmt.k3` = 7, kernel id 3 (bit width), stream +code 3. Kernels are k6's with a different compose: `dot_k3q8`, `k3_grp_row_dot` +(`isum - 4 * bsum`), `repack_k3_grp` (the 2-bit lanes and mask bits stay in place as 4-byte +columns x mr; k6's scale interleave). The tile reads the PACKED planes (`packed` includes 3) - +no byte-expanded panel. Walk cost: one Python patch twinning every `iq4xs` arm of the pilot's +CPU-slice commit (`63da7571e`) for k3 plus the hand-written codec/kernels/repack; the only +misses the census found were two plane-byte accounting sums the pilot itself had skipped. +Gates: `test_kqformat` 14/14, `test_kquant` 132/144 interpreted and 140/144 under `-jit`, +the probe's tune-mode family (test mode waits for the emitter - QUIRK 16). End to end: +`Llama-3.2-1B-Instruct-Q3_K_L.gguf` (bartowski: Q3_K x64 + Q5_K x48 + Q6_K embd) through `run.das` matches llama.cpp's greedy ids (`simple_ids.exe`, compared through `llama-tokenize --ids` on our text) for 52 of 64 tokens on the reference bodies, gen 22 t/s. + +JIT emitter: no new block body - a `k3` flag through `emit_block_kqv2`'s k6 arms (`k63` +shares the scale row, the per-16 fold and the i16 flush cadence; `koff` = 4 / 32), the compose +being qs column `(blk/4)*8 + j` (lo) / `+ 4` (hi) shifted by `2 * (blk % 4)` OR the hmask +column `j` / `j + 4` bit `blk` shifted up two. The stubs became `kq_*_gen_impl(gc, 3)`, k3 +joined the probe's test list. Gates: the probe 10/10 perms on x64 (`dot_maddubs_width256_mr8` +live, maxdiff 4.8e-7) and on the M1 (sdot mr 4/8 stamps bit-exact), `test_kquant -jit` +140/144 on both. After the sidecar re-mint (`k3q8_tile_gen: dot_maddubs_width256_mr8`, +`verdict=beats`) the 1B decodes at gen 60 t/s (22 on the reference body) and its text flips +at token 15 - a near tie: against the reference body on two real k3 tensors (10240 output rows, +scratchpad `k3_rows.das` with a `"reference"` sidecar entry) 4957 rows are bit-identical and +the worst relative difference is 1.7e-4 on a row of magnitude 6e-5 (fp32 fold-order noise). + +Vulkan: `vk_kq_schema_id` 7 -> 3; `KqGemvK3 : KqGemvBase` (k6's blk_contrib with `k3_dot` - +the lane/mask compose over 6 uint4 per superblock, qs 0..3 and hmask 4..5 - and the offset 4) +and `KqBatchK3 : KqBatchK6` (a child of the k6 class: `stage_ws` inherited, `stage_w` composes +the staged words, `blk_fma` is k6's with 4). A child class must follow its parent in the file +(the class rail resolves parents in order). Five ladders, the class oracle arm, family cells at +six formats, `k3_gemv_float_oracle` as the witness. The device gather +(`moe_gpu_gather_stack_kq`) gained k3 arms in both its grouped and tail-row branches - and the +tail-row branch turned out to lack iq4xs arms too (QUIRK 19). `test_vulkan_kernels` 64/64; +the 1B Q3_K_L on the resident driver matches llama.cpp's greedy ids for 51 of 64 tokens at gen +284 t/s. + +Metal: the k3 scale row IS the k6 form, so `metal_blob_scale_plane("k3s")` is the k6 split +verbatim over `t.k3s`, `kq_scales_of` the k6 arm, `metal_blob_off_ok` the k6 rule; quants bind +verbatim (96 B). Kernels are k6's twins with the compose: `MetalKqGemvK3` (k6's lane map - a +lane's 4 elements of each of a half's 4 blocks share the qs bytes at shifts 0/2/4/6 and the +hmask bits 4ip..4ip+3, exactly k6's `sums[0..3]` shape), `MetalKqMvK3T` B2/B4 + `MetalKqMvB8K3` +(k6's slot map over 24 uints per superblock: qs 0..15, hmask 16..23), a `K3` arm in +`MetalKqMulMmK45T` beside `SIXBIT` (the k4/k5/iq4xs arms re-nested once more - QUIRK 14). A +local named `half3` is a reserved vector type name in daslang; the emitter reports it as a +syntax error at the `let`. Ladders as iq4xs's; fixtures at fmt 3 reuse the k6 split-form fill, +`kq_row_ref` via `dequant_k3_plane_superblock_at`. Gates on the M1 Max: +`test_metal_gemv_kernels` 2/2, `test_metal_gemm_kernels` 2/2; the 1B Q3_K_L on the Metal tier +(`k3_metal_probe.das`, the iq4xs probe with the model swapped) decodes a coherent story at gen 223 +t/s, taking the other side of the same token-15 near tie the CPU stamp took. + +Against llama.cpp b10660 (`lcpp_bench`, das = the debug-jit instrument; zen2 = 16 threads, M1 = +8; Q3_K_L, so a third of the weight bytes are k5/k6): + +| tier | pp512 das / llama.cpp | tg128 das / llama.cpp | +|---|---|---| +| zen2 CPU | 543.6 / 310.8 (1.75x) | 66.4 / 65.1 (1.02x) | +| 5060 Ti Vulkan | 14031 / 17509 (0.80x) | 374.5 / 349.1 (1.07x) | +| M1 CPU | 524.6 / 223.5 (2.35x) | 127.3 / 110.6 (1.15x) | +| M1 Metal | 3316 / 3219 (1.03x) | 200.9 / 193.0 (1.04x) | + +Decode is at or above llama.cpp on every tier; the Vulkan prefill gap closed with the cm2 +tile (section 6b, 0.30x -> 0.80x). + +### IQ4_XS (the pilot, 2026-08-30) + +Shape: 256-superblock, codebook nibble (`kvalues_iq4nl`, signed, no offset), scale = f16 d x +6-bit sub-scale - 32. Disk block 136 B: f16 d, u16 scales_h, 4 scales_l bytes, 128 nibbles +paired k/k+16 per 32-block. Plane pair: quants verbatim (the q40 tiling), scale row 20 B decoded +at transcode ([f16 d][2 pad][8 x int8 (ls-32)][8 pad] - the k4 decoded-row shape, so +`repack_k4_grp`'s scale addressing is reused verbatim in `repack_iq4xs_grp`). Dot fold: +`sum_blk sc * sum_k LUT[q] * x`, then `* (d * xs)` per superblock; no `xbsp` term. Ids: +`KqFmt.iq4xs` = 6, kernel id 44, stream code 44. Gates green on the interpreter binary +AND under `-jit`: `test_kqformat` 14/14, `test_kquant` 114/114 (iq4xs arms: transcode+dequant +exact, dot vs fp64 at n=512/1024/3072, GEMV rows, repack mr=4/8/16, tile, groupn). End to end: +`Llama-3.2-1B-Instruct-IQ4_XS.gguf` (bartowski; IQ4_XS x96 + Q5_K + Q6_K) through +`examples/dasLLAMA/run.das` reproduces llama.cpp's greedy continuation of "Once upon a time" +for ~45 tokens before the fast-math divergence (gen 42 t/s, reference bodies). Two bugs the +unit gates could not see, both caught only by the run: QUIRK 8 (the repack-mr freeze) and +QUIRK 9 (the codebook global on workers). Method that found them: a Python oracle that +dequantizes rows straight from the GGUF bytes, a daslang probe calling `mm_at_kq_pre` on the +same rows (run it UNDER the job queue), a layer bisect (`config.n_layers = L`, logits cosine +against a sibling quant of the same model), and a dump of the image's group-0 plane bytes +against the grp layout computed in Python. JIT emitter: `emit_block_iq4xs` (section 5); +family gate 10/10 perms, live stamp on this box `dot_maddubs_width256_mr8` (mr 8, maxdiff +1.5e-5), `test_kquant` 121/125 under `-jit`; after the sidecar re-mint (QUIRK 11) the 1B +decodes at 59-60 t/s against 39 t/s on the reference body, same text. The body rides mx4's +chunk-load + lane-splat dot path; `emit_block_kqv2`'s x64 `vpbroadcastd` / `madd16` chains are +the untried next lever. Vulkan (section 6): `KqGemvIq4xs` + `KqBatchIq4xs`, the kernel suite +64/64 with the five-format family cells and the float witness, and the 1B IQ4_XS model on the +resident driver reproduces the CPU text at gen 102 t/s (prefill 38 t/s on the 5-token prompt); +the codebook lookup then cost 3.5x on the GEMV until QUIRK 20's packed-word form. Metal +(section 7): `MetalKqGemvIq4xs`, the `MetalKqMvIq4xsT` B2/B4 + `MetalKqMvB8Iq4xs` twins, the +`IQ4XS` mul_mm arm; gates 2/2 + 2/2 on the M1 Max. + +Against llama.cpp b10660 (`lcpp_bench`, das = the debug-jit instrument, ~8% under the tuned +exe; zen2 = 16 threads, M1 = 8): + +| tier | pp512 das / llama.cpp | tg128 das / llama.cpp | +|---|---|---| +| zen2 CPU | 475.7 / 256.9 (1.85x) | 65.0 / 59.5 (1.09x) | +| 5060 Ti Vulkan | 15334 / 17060 (0.90x) | 334.7 / 340.7 (0.98x) | +| M1 CPU | 796.0 / 263.0 (3.03x) | 132.3 / 128.2 (1.03x) | +| M1 Metal | 3612 / 3575 (1.01x) | 249.4 / 249.3 (1.01x) | + +Both gaps closed 2026-08-30: the cm2 decode tile (section 6b) took Vulkan prefill past the k4 +control's 13144, and the threadgroup-LUT GEMV/mul_mm rework took Metal to parity on both axes; +what remains anywhere is the tier-wide small-model prefill gap the k4 control shares. diff --git a/modules/dasLLAMA/LAWS.md b/modules/dasLLAMA/LAWS.md index 024423a9eb..53e63a0d93 100644 --- a/modules/dasLLAMA/LAWS.md +++ b/modules/dasLLAMA/LAWS.md @@ -66,3 +66,33 @@ clause - a bump's diff also carries the LAWS.md entry recording the declared release; the declaration itself stays his call, on feel ("the intent was to bump due to major changes ... i go on feel on that one" - "4. agreee"). + +- **2026-08-30** (`HOW_TO_ADD_A_FORMAT.md`, `CLAUDE.md`): Boris opened the HOW_TO document + series with this pilot - "document entire process on how to add new format, as well as all + the quirks which popped up. so that the followup arc can unquirk them (if need be)"; a + separate kernel per card "is fine, as long as its a similar template setup ... just another + cond"; the series continues with adding a model family, vision, audio, TTS. Linked from the + module's new `CLAUDE.md` and the root table. Arc ruling the same day: the community i-quants + are read as-is ("B for sure. we are not big enough for people to bother with our own fork"), + Qwen3.8-27B on a 16 GB card is the Vulkan arc's endgame. + +## 2026-08-30 - HOW_TO_ADD_A_FORMAT.md, followup_general.md (#58) + +Boris, on the IQ4_XS pilot's Metal section and QUIRKS 12-15: "this is exactly the kind of info +we need for that document. once we're done with all formats - we'll unquirk all we can, but +otherwise it'll be like step1-step2-step3 instruction - this is how to add format. like it +should." The QUIRKS ledger stays open and honest through every format of the arc; the unquirk +pass comes AFTER the last format lands, and the surviving document is the numbered walk. Earlier +the same day, on the Metal portion: "we have an option of ssh m1 ... go for it. it feels right +somehow. plus validates the how-to. and we'll ledger M5 pass on new kernels for later" - the M1 +is in the loop for every format from here, and performance work on the new Metal kernels is +the ledgered M5 pass, not this arc. + +## 2026-08-30 - HOW_TO_ADD_A_FORMAT.md (section 4), followup_general.md (#60) + +Boris, on the IQ3_S emitter design after reading llama.cpp's AVX2 iq3_s kernel: "our CPU +kernels typically faster - so if they map closer, perhaps worth looking at and doing side by +side?" and then "lets make sure we do for all new CPU kernels, and if we skipped for previous +ones - lets ledger towards the end of this arc." Every new format's CPU kernel work starts by +reading llama.cpp's arch kernel and racing its techniques as tune perms; the formats that +skipped this (IQ4_XS, Q3_K) get the retroactive audit at the end of the arc (#60). diff --git a/modules/dasLLAMA/README.md b/modules/dasLLAMA/README.md index ec4076d652..3244ddea86 100644 --- a/modules/dasLLAMA/README.md +++ b/modules/dasLLAMA/README.md @@ -6,7 +6,8 @@ per-box auto-tuner picks the kernel forms for *your* machine, so the same source code on an M1 and a Threadripper alike. Backends stack behind one registry: the portable / NEON / AVX CPU tiers, an Accelerate "+AMX" float tier on Apple silicon, and **Metal** or **Vulkan** compute for GPU-resident serving. It loads stock GGUF files (single-file or -multi-shard splits, F32/F16/Q8_0/Q4_0/Q5_0/MXFP4 and native Q4_K/Q5_K/Q6_K +multi-shard splits, F32/F16/Q8_0/Q4_0/Q5_0/MXFP4 and native Q2_K/Q3_K/Q4_K/Q5_K/Q6_K plus the +i-quants IQ4_XS/IQ4_NL/IQ3_S/IQ3_XXS/IQ2_S/IQ2_XS/IQ2_XXS planes), runs text LLMs, audio-input "omni" chat models, speech-to-text, and voice-activity detection, and it is fast: consistently ahead of each model's reference C++ engine on prompt processing (up to ~1.7×), trading blows on generation, and up to ~6× ahead on audio/omni workloads — live @@ -322,7 +323,7 @@ What a model needs to "just work" today: | Feature | Supported | |---|---| -| GGUF weight types (read directly) | **F32, F16, Q8_0, Q4_0, MXFP4, and the K-quant planes Q4_K / Q5_K / Q6_K** — K-quant tensors keep their native format and run on dedicated kq kernels (a Q4_K_M file mixes formats per tensor); MXFP4 expert stacks stay native too (mx4·q8 kernels); bf16 audio-tower mmprojs read exactly | +| GGUF weight types (read directly) | **F32, F16, Q8_0, Q4_0, MXFP4, the K-quant planes Q2_K / Q3_K / Q4_K / Q5_K / Q6_K, and the i-quants IQ4_XS / IQ4_NL / IQ3_S / IQ3_XXS / IQ2_S / IQ2_XS / IQ2_XXS** — K-quant and i-quant tensors keep their native format and run on dedicated kq kernels (a Q4_K_M file mixes formats per tensor); MXFP4 expert stacks stay native too (mx4·q8 kernels); bf16 audio-tower mmprojs read exactly | | On-the-fly self-quantization | Q8, Q4 (from an F16/F32 model) | | Architecture | `llama`, `mistral3`, `qwen2`, `qwen2vl`, `qwen3`, `qwen3vl`, `phi3`, `gemma2`, `gemma3`, `gemma4`, `qwen2moe`, `qwen3moe`, `qwen3vlmoe`, `qwen35`, `qwen35moe`, `qwen3next`, `glm4moe`, `gpt-oss` — a self-registering arch registry (`dasllama_arch_*.das`, `[init]`); the loader dispatches on GGUF `general.architecture`, splits Phi3's fused attn_qkv / gate_up at load, and panics with the registered list on an unknown arch | | Attention | MHA **and** GQA (grouped-query); sliding-window with a per-layer pattern (Gemma-2 alternating, Gemma-3 5 local : 1 global, Gemma-4 explicit per-layer bool array); heterogeneous per-layer geometry (Gemma-4: sliding vs global layers differ in head size AND kv-head count, incl. V-from-K layers with no attn_v tensor); **cross-layer KV sharing** (Gemma-4 E-series: later layers carry no K/V of their own — Q only, attending against an earlier layer's cached K/V); configurable attention-score scale (Gemma-4: 1.0); per-head attention-sink logits in the softmax denominator (gpt-oss, all three prefill cores + decode); attention + final-logit soft-capping; suppressed-token logit bias | @@ -389,7 +390,7 @@ So there's no ambiguity about what will fail: `gemma4v`, gemma-3 `gemma3v`, the whole qwen family — Qwen3-Omni / dense Qwen3-VL `qwen3vl_merger` incl. deepstack, Qwen2.5-Omni/VL `qwen2.5o`) — vision on the remaining multimodal families (pixtral, minicpm-v, …) is what stays out of scope. -- **GGUF weight types beyond F32 / F16 / Q8_0 / Q4_0 / MXFP4 / Q4_K / Q5_K / Q6_K** — no IQ +- **GGUF weight types beyond F32 / F16 / Q8_0 / Q4_0 / MXFP4 / Q2_K-Q6_K / IQ4_XS / IQ4_NL / IQ3_S / IQ3_XXS / IQ2_S / IQ2_XS / IQ2_XXS** — no IQ1 or quants; BF16 is read only for the audio-tower mmprojs, not as an LLM weight format. - `encode(..., parse_special)` is reserved and currently a no-op — the chat renderer injects special-token *ids* directly (so templates work); parsing special tokens out of free text diff --git a/modules/dasLLAMA/REVIEW.md b/modules/dasLLAMA/REVIEW.md index 46392225f4..0b87b0991b 100644 --- a/modules/dasLLAMA/REVIEW.md +++ b/modules/dasLLAMA/REVIEW.md @@ -1,11 +1,10 @@ # dasLLAMA Code Review Checklist **Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist.** Architecture -docs: `ARCHITECTURE.md` and the `ARCHITECTURE_*.md` companions it indexes - the rules below -cite `ARCHITECTURE_ENGINE.md`, `ARCHITECTURE_GPU.md`, `ARCHITECTURE_MEDIA.md`, and -`ARCHITECTURE_MEASUREMENT.md`. Planned work: -`followup_general.md`, `followup_vulkan.md`, `PERF_LEDGER.md` (performance goes to the perf -ledger, everything else to the followup ledgers). +docs: `ARCHITECTURE.md` and the `ARCHITECTURE_*.md` companions it indexes. Planned work: +`followup_general.md`, `followup_vulkan.md`, `followup_metal.md` (the Metal tier, and CPU work +measured on macOS), `PERF_LEDGER.md` (performance goes to the perf ledger, everything else to +the followup ledgers). **A dasLLAMA `[test]` file, wherever the diff puts it, answers to this module's `tests/REVIEW.md`.** @@ -28,7 +27,8 @@ winners back. **A change to the sidecar-exchange client (`dasllama/dasllama_exchange.das`), or to a tune-boot path that reaches it, applies `REVIEW_EXCHANGE.md`.** -**Every `dasllama/` change applies this folder's `tests/REVIEW.md`.** +**Every `dasllama/` change applies this folder's `tests/REVIEW.md`.** A `dasllama/` diff never +opens that checklist on its own. **A GPU kernel, driver, dispatch-class, or K/V-mirror change applies `REVIEW_GPU.md`.** @@ -54,6 +54,10 @@ schedules such a stream, applies `REVIEW_VISION.md`.** `REVIEW_AUDIO.md` and `REVIEW_VISION.md`;** a family file that only CALLS a shared rail does not thereby pick up the other modality's checklist. +**A diff that adds a file under `dasllama/`, moves code between files, or lands a kernel, +codec, transform, tokenizer, tool-wire, media-IO or registration concern in a new place +applies `REVIEW_PLACEMENT.md`** - the what-lands-where rules. + **A routed file applies BOTH the checklist it routes to and this one; every other file under `modules/dasLLAMA/` applies this one.** @@ -99,12 +103,6 @@ planes) are out of scope; a site that must stay f32 for another reason is ledger file's sec.1 charter line in `ARCHITECTURE_ENGINE.md`, `ARCHITECTURE_GPU.md`, or `ARCHITECTURE_MEDIA.md`, not commented into compliance. -**Platform-specific code in an engine file (`dasllama/`) lands only in that platform's backend -file.** - -**A diff that adds a new engine concern that is not `Model`/`Session`/`Config` state to -`dasllama/dasllama_common.das` is a defect - give the concern its own file.** - **A boot-path prompt (code that runs at startup, before the first request) that reads stdin without first proving both stdin and stdout are terminals is a defect - emit the question as a `@sidecar` event instead.** A supervised or piped boot must never block on input. @@ -135,13 +133,14 @@ the name in the same change). **A change to code or data of `encode`/`bpe_encode` or anything they reach in `dasllama/dasllama_spm.das` / `dasllama/dasllama_bpe.das` / `dasllama/dasllama_pretok.das` -ships before/after `--tok` rows (this folder's `benchmarks/lcpp_bench.das`) for the affected backend** - the -instrument is the scaling ratio across the size ladder, and superlinear is a defect. +ships before/after `--tok` rows (this folder's `benchmarks/lcpp_bench.das`) for the affected +backend** - the instrument is the scaling ratio across the size ladder, and superlinear is a +defect. **A change to code or data in `dasllama/dasllama_tokenizer.das`, `dasllama/dasllama_spm.das`, `dasllama/dasllama_bpe.das`, or `dasllama/dasllama_pretok.das`, or to the special-token or -template strings any of them look up, records a run of this folder's `tests/test_tokenizer.das` with its cases -EXECUTED, not skipped.** +template strings any of them look up, records a run of this folder's +`tests/test_tokenizer.das` with its cases EXECUTED, not skipped.** **A diff that adds an override, or gives one a new effect, without the announce is a defect.** An override is an environment knob, an exported runtime setter, or an on-disk state file that @@ -190,18 +189,22 @@ the file it checks - in `ARCHITECTURE_ENGINE.md`, `ARCHITECTURE_MEDIA.md`, or `ARCHITECTURE_GPU.md`.** The line names the check and the names it licenses. A licensed name is one that check does not flag. When the check licenses no names, the line says so. -**An upstream mechanism is described in our own terms, not attributed** - no -"lifted/ported verbatim from" and no upstream symbol, header, or constant names in any -checked-in prose this module owns (docs and comments, any language); state what the code -does and why its shape wins. A symbol the file's own code calls or carries as a value may be -named - naming your own callee or data key is not attribution. Provenance is -not attribution: a path naming where checked-in data is regenerated FROM, an env-knob row -in `ENVIRONMENT.md` whose value locates the reference binary, and a command line or flag -list in `METHODOLOGY.md`, `PROFILE.md`, or `BRINGUP.md`, and a follow-up ledger's board row -naming the build it compares against, all name the binary outright; -every other `.md` line and `.das` comment writes "the reference exe" or "upstream". Legal -attribution lives in `THIRD_PARTY_NOTICES.md` and the `LICENSE.*` files, so prose never -carries it. +**Checked-in prose this module owns - docs and comments, any language - that is not locating +or reproducing work against the reference build describes an upstream mechanism in our own +terms: no "lifted/ported verbatim from", and no upstream symbol, header, constant, or binary +name - write "the reference exe" or "upstream" instead.** The reference build is the +third-party engine this module measures itself against - the checkout +`benchmarks/setup_lcpp_ref.das` pins. A symbol the file carrying that prose calls or holds as +a value is its own name, not attribution. + +**A line whose job is to locate or reproduce work against the reference build names that +build's binary outright** - a path naming where checked-in data is regenerated FROM, an +env-knob row whose value locates the reference binary, a command line or flag list in a +methodology, profile, bring-up, or how-to document, and a follow-up ledger's row naming the +build its numbers compare against. + +**Legal attribution never appears in prose - it lives in `THIRD_PARTY_NOTICES.md` and the +`LICENSE.*` files.** **A def of `dasllama/dasllama.das` - and a new OVERLOAD of one - is TAUGHT: demonstrated in runnable code in a `tutorials/dasLLAMA/*.das` source and narrated on a @@ -229,68 +232,19 @@ companions, lands `ARCHITECTURE.md`'s index line and section range and repoints LINT026-gated; the prose ones are not, and a prose citation of a section that left its file sends the reader to nothing. -**A per-file inventory restated in this checklist is a defect of the checklist.** The sec.1 -charters - `ARCHITECTURE_ENGINE.md`, `ARCHITECTURE_GPU.md`, `ARCHITECTURE_MEDIA.md` - own the -per-file list. A rule naming what KIND of code lands in which file is the checklist's own. - -**A tensor format conversion lands in `dasllama/dasllama_convert.das`.** - -**A disk-order -> compute-order transform lands per scope: kernel-layout in -`dasllama/dasllama_repack.das`, load-scope in `dasllama/dasllama_layout.das`.** - -**A CPU KV-cache store, read, score dot, or V-accumulate lands in `dasllama/dasllama_kv_codec.das`, -its format family kept whole.** GPU twins land in their backend kernel file. - -**A pre-tokenizer split lands in `dasllama/dasllama_pretok.das`; a merge algorithm in its backend file -(`dasllama/dasllama_spm.das` / `dasllama/dasllama_bpe.das`).** - -**A kernel body lands in its owner's backend file.** A GPU kernel body lands in the file where -its PSO is compiled and released. A CPU-tier kernel body lands in that tier's -`dasllama/dasllama_math_.das`. A kernel body never lands in -`dasllama/dasllama_math.das` or in a lens/dispatch macro file. - -**A family quirk lands in the family file; a piece two families need moves UP into the -concern's shared file (its own file when none exists)** - never sideways into a sibling. - -**A family gaining an arm for a media kind adds that kind's span markers to that family's chat -template, never to a second renderer.** Span markers are the template text that opens and -closes the media rows. A family whose template or vocab lacks them has no arm for that media -kind - `create_chat_` panics at create, not at render. - -**No signature in `dasllama/dasllama_tower.das` takes a type that -`dasllama/dasllama_audio.das`, `dasllama/dasllama_vision.das`, or a family file declares.** -`dasllama/dasllama_tower.das` is the shared encoder-tower home, and it requires none of those -files. A doc comment naming the family a helper was built for is fine. - -**A `dasllama/dasllama_tower.das` helper with one calling family lands in that family's -file** - a single-caller helper sanctioned as tower-worthy is ledgered on -`ARCHITECTURE_MEDIA.md` sec.1.7's tower charter line, not argued in review. - -**A harness whose run can end with zero result rows exits non-zero when it does - wrong -flags, failed load, a device that declines.** - -**Tool wire text (the text of a model's tool/function call, built or parsed) is produced only -in `dasllama/dasllama_tools.das`.** - -**No engine file (`dasllama/`) other than `dasllama/dasllama_audio_io.das` requires `audio` (the -miniaudio decode module).** - -**No engine file (`dasllama/`) other than `dasllama/dasllama_vision_io.das` requires `stbimage`.** -Benchmarks, harnesses, and tests decode their own fixtures. - -**Engine, HTTP, or writer logic never lands in `dasllama/dasllama_scheduler.das`** - engine logic in -engine files, HTTP in the server, writer logic in the writer's own file. - -**An `[init]`-only side-effect require in an engine file (`dasllama/`) lives in -`dasllama/dasllama_transformer.das`** - arch registrations, GPU tiers, every module requiring -the engine back; it sits in `dasllama/dasllama_common.das` only if engine code needs it. A -program root (test, harness, benchmark, tool) requires the registration module it needs -directly. - -**An architecture file (`dasllama/dasllama_arch_*.das`) that changes a forward loop, or tests a -family name on a shared path, is a defect - it carries declarative registration only.** - **A diff that moves a family encode stage onto a GPU hook leaves the CPU form in place and changes none of its arithmetic - deleting or rewriting the CPU form in the same change is a defect.** The hook returns a decline value (`false`, or `-1` for the chunk hooks), and the CPU form serves every box with no driver. + +**A diff that writes a CPU feature name in a `requires=` argument that `TUNE_KNOWN_FEATURES` +(`modules/dasLLVM/daslib/llvm_tune.das`, repo root) does not list adds it there in the same +change.** The `features` fingerprint saved with every sidecar is this box's pass/fail over +that list, so a name outside it is never recorded and a box adopting a shipped profile re-runs +the tuning the profile was meant to save. + +**A value that a team-lane kernel reads - anything reachable from a `team_parallel_for` / +`team_parallel_for_indexed` / `team_parallel_stages` body (`daslib/jobque_boost.das`, repo +root) - is a `def` returning it, never a module global with a declaration initializer (`let` +or `var`).** A team lane never runs global initializers, so the global reads zero there while +every single-threaded run reads the right value. diff --git a/modules/dasLLAMA/REVIEW_MEASUREMENT.md b/modules/dasLLAMA/REVIEW_MEASUREMENT.md index 7bcf185d71..64d77f451b 100644 --- a/modules/dasLLAMA/REVIEW_MEASUREMENT.md +++ b/modules/dasLLAMA/REVIEW_MEASUREMENT.md @@ -3,10 +3,9 @@ **Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist.** Architecture doc: `ARCHITECTURE_MEASUREMENT.md`. Planned work: `PERF_LEDGER.md`. -The two figure rules below bind the surfaces this module owns - its ledgers, docs, code -comments, and the PR body of a diff under it. A number on a served page answers to the site -checklists (`site/REVIEW.md`, `site-dasllama/REVIEW.md`), which carry their own provenance -duty. +Figure rules here bind the surfaces this module owns - its ledgers, docs, code comments, and +the PR body of a diff under it. A number on a served page answers to `site/REVIEW.md` or +`site-dasllama/REVIEW.md` (repo root) instead. **A self-measured served-turn time entering `PERF_LEDGER.md` comes from the released `lcpp_bench` exe, never from the `-jit` script.** A served-turn time is a tok/s figure or a @@ -43,7 +42,10 @@ whole-turn model or engine comparison - is a defect wherever this module writes no cell behind it: a checked-in doc, a ledger, a code comment, or a PR description.** The cell states its quant mode and stamps box and engine provenance, so a number can never silently describe a format nobody serves or a kernel set nobody ships. A figure labeled as a -prediction is not a reading - the prediction log is mandated and needs no cell. +prediction is not a reading - the prediction log is mandated and needs no cell. A bring-up +log in a how-to or ledger that names harness, flags, box, and the exe or script that ran it, +at section level, is a stage figure, not a served-turn measurement; the cell it owes is its +format's board row. **A figure measuring one engine stage inside a served turn, or any other measured margin - a lab margin, a kernel-form delta, a gate knee - names the harness and flags that diff --git a/modules/dasLLAMA/REVIEW_PLACEMENT.md b/modules/dasLLAMA/REVIEW_PLACEMENT.md new file mode 100644 index 0000000000..88c36f2945 --- /dev/null +++ b/modules/dasLLAMA/REVIEW_PLACEMENT.md @@ -0,0 +1,77 @@ +# dasLLAMA Code Review Checklist - placement + +**Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist.** The per-file +charters are `ARCHITECTURE_ENGINE.md`, `ARCHITECTURE_GPU.md` and `ARCHITECTURE_MEDIA.md` sec.1. + +**Routed from `REVIEW.md`: a diff that adds a file under `dasllama/`, moves code between files, +or lands a kernel, codec, transform, tokenizer, tool-wire, media-IO or registration concern +in a new place applies this list together with `REVIEW.md`.** + +**A per-file inventory restated in this checklist is a defect of the checklist.** The sec.1 +charters own the per-file list; a rule naming what KIND of code lands in which file is the +checklist's own. + +**A tensor format conversion lands in `dasllama/dasllama_convert.das`.** + +**A disk-order -> compute-order transform lands per scope: kernel-layout in +`dasllama/dasllama_repack.das`, load-scope in `dasllama/dasllama_layout.das`.** + +**A CPU KV-cache store, read, score dot, or V-accumulate lands in +`dasllama/dasllama_kv_codec.das`, its format family kept whole.** GPU twins land in their +backend kernel file. + +**A pre-tokenizer split lands in `dasllama/dasllama_pretok.das`; a merge algorithm in its +backend file (`dasllama/dasllama_spm.das` / `dasllama/dasllama_bpe.das`).** + +**A kernel body lands in its owner's backend file.** A GPU kernel body lands in the file where +its pipeline state object (PSO) is compiled and released. A CPU-tier kernel body lands in that +tier's `dasllama/dasllama_math_.das`. A kernel body never lands in +`dasllama/dasllama_math.das` or in a file whose job is declaring kernels and routing dispatch. + +**A family quirk lands in the family file; a piece two families need moves UP into the +concern's shared file (its own file when none exists)** - never sideways into a sibling. + +**A family gaining an arm for a media kind adds that kind's span markers to that family's chat +template, never to a second renderer.** Span markers are the template text that opens and +closes the media rows. A family whose template or vocab lacks them has no arm for that media +kind - `create_chat_` panics at create, not at render. + +**No signature in `dasllama/dasllama_tower.das` takes a type that +`dasllama/dasllama_audio.das`, `dasllama/dasllama_vision.das`, or a family file declares.** +`dasllama/dasllama_tower.das` is the shared encoder-tower home. A doc comment naming the +family a helper was built for is fine. + +**`dasllama/dasllama_tower.das` requires none of `dasllama/dasllama_audio.das`, +`dasllama/dasllama_vision.das`, or a family file - a diff adding such a require is a defect.** + +**A `dasllama/dasllama_tower.das` helper with one calling family lands in that family's +file** - a single-caller helper sanctioned as tower-worthy is ledgered on +`ARCHITECTURE_MEDIA.md` sec.1.7's tower charter line, not argued in review. + +**Tool wire text (the text of a model's tool/function call, built or parsed) is produced only +in `dasllama/dasllama_tools.das`.** + +**No engine file (`dasllama/`) other than `dasllama/dasllama_audio_io.das` requires `audio` +(the miniaudio decode module).** + +**No engine file (`dasllama/`) other than `dasllama/dasllama_vision_io.das` requires +`stbimage`.** Benchmarks, harnesses, and tests decode their own fixtures. + +**Engine, HTTP, or writer logic never lands in `dasllama/dasllama_scheduler.das`** - engine +logic in engine files, HTTP in the server, writer logic in the writer's own file. + +**An `[init]`-only side-effect require in an engine file (`dasllama/`) lives in +`dasllama/dasllama_transformer.das`** - arch registrations, GPU tiers, every module requiring +the engine back. It lives in `dasllama/dasllama_common.das` instead when code in +`dasllama/dasllama_common.das` itself depends on that module's registration having run. A +program root (test, harness, benchmark, tool) requires the registration module it needs +directly. + +**An architecture file (`dasllama/dasllama_arch_*.das`) that changes a forward loop, or tests a +family name on a shared path, is a defect - it carries declarative registration only.** + +**Platform-specific code in an engine file (`dasllama/`) lands only in that platform's backend +file.** + +**A diff that adds a new engine concern that is not `Model`/`Session`/`Config` state to +`dasllama/dasllama_common.das` is a defect - give the concern its own file.** diff --git a/modules/dasLLAMA/benchmarks/REVIEW.md b/modules/dasLLAMA/benchmarks/REVIEW.md index af17be89f6..c5e5c6b742 100644 --- a/modules/dasLLAMA/benchmarks/REVIEW.md +++ b/modules/dasLLAMA/benchmarks/REVIEW.md @@ -62,3 +62,12 @@ body.** A board cell is a timed cell of the published results board: one `../PROFILE.md` section. What a cell times changes when a change to its code, to its input corpus, or to the pinned reference build - `DEFAULT_REF_SHA` in `setup_lcpp_ref.das`, the targets it builds, or a patch it applies - moves the measured quantity. + +**A diff that adds or moves a row in `../harness/tune_kernels.das`'s bench list keeps +`dot_q8q8_laneq4x4` last.** That bench pins the repack backend for the rest of the process, so +a row after it is timed against the pinned backend instead of the backend it would select. + +**A diff that adds or changes a timing instrument makes it exit non-zero on a run that ends +with zero result rows - wrong flags, failed load, a device that declines.** A run that matched +nothing and reported success leaves a sidecar or a record untouched, and its caller cannot +tell. diff --git a/modules/dasLLAMA/dasllama/dasllama_blocks.das b/modules/dasLLAMA/dasllama/dasllama_blocks.das index cb8a1d4df8..06bf167997 100644 --- a/modules/dasLLAMA/dasllama/dasllama_blocks.das +++ b/modules/dasLLAMA/dasllama/dasllama_blocks.das @@ -1394,7 +1394,7 @@ def private kq_bytes_per_weight(f : KqFmt) : float { if (f == KqFmt.q40) { return 144.0 / 256.0 // 4-bit quants plus an f16 scale per 32 } - if (f == KqFmt.k4) { + if (f == KqFmt.k4 || f == KqFmt.iq4xs) { return 148.0 / 256.0 } if (f == KqFmt.k5) { @@ -1403,6 +1403,27 @@ def private kq_bytes_per_weight(f : KqFmt) : float { if (f == KqFmt.k6) { return 212.0 / 256.0 } + if (f == KqFmt.k3) { + return 114.0 / 256.0 // 96B quants + the 18B scale row per 256 + } + if (f == KqFmt.iq3s) { + return 124.0 / 256.0 // 104B quants + the 20B scale row per 256 + } + if (f == KqFmt.iq3xxs) { + return 116.0 / 256.0 // 96B quants + the 20B scale row per 256 + } + if (f == KqFmt.iq4nl) { + return 144.0 / 256.0 // 128B quants + the 16B scale row per 256 (q40's shapes) + } + if (f == KqFmt.k2) { + return 84.0 / 256.0 // 64B quants + the 20B scale row per 256 + } + if (f == KqFmt.iq2s) { + return 92.0 / 256.0 // 72B quants + the 20B scale row per 256 + } + if (f == KqFmt.iq2xs || f == KqFmt.iq2xxs) { + return 84.0 / 256.0 // 64B quants + the 20B scale row per 256 + } return 34.0 / 32.0 // q8: 32 quants + an f16 scale per block } diff --git a/modules/dasLLAMA/dasllama/dasllama_common.das b/modules/dasLLAMA/dasllama/dasllama_common.das index accbe371e0..be280140ac 100644 --- a/modules/dasLLAMA/dasllama/dasllama_common.das +++ b/modules/dasLLAMA/dasllama/dasllama_common.das @@ -835,6 +835,33 @@ struct Model { // the q51 tier's plane pair, per 32-BLOCK: 20B nibbles+qh / 4B f16 d+m (never repacked) q51q : array q51s : array + // the iq4xs tier's plane pair: 128B nibbles in the q40 tiling + a 20B decoded scale row per superblock + iq4xsq : array + iq4xss : array + // the k3 tier's plane pair: [64 qs][32 hmask] per superblock + the k6-shaped 18B scale row (16 x int8 (sc - 32), f16 d) + k3q : array + k3s : array + // the iq3s tier's plane pair: [64 qs][8 qh][32 signs] per superblock + the k4-shaped 20B scale row (f16 d, 8 x int8 (1 + 2s)) + iq3sq : array + iq3ss : array + // the iq3xxs tier's plane pair: [64 qs][32 aux] per superblock + the iq3s-shaped 20B scale row (f16 d halved, 8 x uint8 (2ls + 1)) + iq3xxsq : array + iq3xxss : array + // the iq4nl tier's plane pair: q40's shapes exactly (128B nibbles + 8 x f16 d per superblock), LUT decode + iq4nlq : array + iq4nls : array + // the k2 tier's plane pair: 64 qs bytes per superblock + the 20B scale row [f16 d][f16 dmin][16 sc/min bytes] + k2q : array + k2s : array + // the iq2s tier's plane pair: [32 idx][32 signs][8 qh] per superblock + the 20B row (f16 d/8, 16 x (1 + 2s)) + iq2sq : array + iq2ss : array + // the iq2xs tier's plane pair: 32 u16 qs words per superblock + the 20B row (f16 d/8, 16 x (1 + 2s)) + iq2xsq : array + iq2xss : array + // the iq2xxs tier's plane pair: 32 u16 qs words per superblock + the 20B row (f16 d/8, 16 x (1 + 2s)) + iq2xxsq : array + iq2xxss : array kquant_native : bool = false // kq stage 4: the kq planes were repacked at load into the active backend's grp layout. // From then on every kq matmul MUST run the backend slots — disk-order portable kernels would @@ -845,6 +872,15 @@ struct Model { kq_repack_mr5 : int64 = 4l kq_repack_mr6 : int64 = 4l kq_repack_mr40 : int64 = 4l + kq_repack_mr44 : int64 = 4l + kq_repack_mr3 : int64 = 4l + kq_repack_mr33 : int64 = 4l + kq_repack_mr34 : int64 = 4l + kq_repack_mr45 : int64 = 4l + kq_repack_mr2 : int64 = 4l + kq_repack_mr23 : int64 = 4l + kq_repack_mr24 : int64 = 4l + kq_repack_mr25 : int64 = 4l // per-layer weight format tags (KqFmt; empty = all q8). Filled by detect_kq_formats before // layout_offsets so the layout walks each tensor into its format's cursor. wq_fmt : array @@ -1067,6 +1103,10 @@ def model_weights_bytes(t : Model) : int64 { + long_length(t.bf16blob) * 2l + long_length(t.k4q) + long_length(t.k4s) + long_length(t.k5q) + long_length(t.k5s) + long_length(t.k6q) + long_length(t.k6s) + long_length(t.q40q) + long_length(t.q40s) + + long_length(t.iq4xsq) + long_length(t.iq4xss) + long_length(t.k3q) + long_length(t.k3s) + + long_length(t.iq3sq) + long_length(t.iq3ss) + + long_length(t.iq3xxsq) + long_length(t.iq3xxss) + + long_length(t.iq4nlq) + long_length(t.iq4nls) + long_length(t.k2q) + long_length(t.k2s) + long_length(t.iq2sq) + long_length(t.iq2ss) + long_length(t.iq2xsq) + long_length(t.iq2xss) + long_length(t.iq2xxsq) + long_length(t.iq2xxss) + long_length(t.q51q) + long_length(t.q51s) + long_length(t.mblob) + long_length(t.vkblob) + long_length(t.embq) + long_length(t.embs)) @@ -1455,6 +1495,15 @@ def private dlim_cpu_source_impl(var c : DlimCpuConfig) { c.kq_mr5 = active_kq_layout_mr(5) c.kq_mr6 = active_kq_layout_mr(6) c.kq_mr40 = active_kq_layout_mr(40) + c.kq_mr44 = active_kq_layout_mr(44) + c.kq_mr3 = active_kq_layout_mr(3) + c.kq_mr33 = active_kq_layout_mr(33) + c.kq_mr34 = active_kq_layout_mr(34) + c.kq_mr45 = active_kq_layout_mr(45) + c.kq_mr2 = active_kq_layout_mr(2) + c.kq_mr23 = active_kq_layout_mr(23) + c.kq_mr24 = active_kq_layout_mr(24) + c.kq_mr25 = active_kq_layout_mr(25) c.q51_mr = active_q51_layout_mr() // OUTCOME, not request: a backend without s16 twins keeps f32 scale planes (wscale_convert_f16) c.wscale_f16 = g_wscale_f16 && kernel_backend_has_wscale16() @@ -1478,11 +1527,59 @@ def dlim_config_sources_register() { set_dlim_metal_source(@@dlim_metal_source_impl) } -// Dense-position demotion: q40/k5-from-Q5_0 need % 256 rows (superblock walkers); q51 always -// demotes — dense positions have no q51 dispatch (v1 is expert stacks — kq_fmt_expert_ok). +//! The Model's plane pair for a kq format - the dispatcher every plane-form consumer keys on. +def kq_planes_of(t : Model; fmt : KqFmt; var kq : array const?&; var ks : array const?&) { // nolint:STYLE037 — the format ladder + unsafe { + if (fmt == KqFmt.k4) { + kq = addr(t.k4q) + ks = addr(t.k4s) + } elif (fmt == KqFmt.k5) { + kq = addr(t.k5q) + ks = addr(t.k5s) + } elif (fmt == KqFmt.k6) { + kq = addr(t.k6q) + ks = addr(t.k6s) + } elif (fmt == KqFmt.q40) { + kq = addr(t.q40q) + ks = addr(t.q40s) + } elif (fmt == KqFmt.iq4xs) { + kq = addr(t.iq4xsq) + ks = addr(t.iq4xss) + } elif (fmt == KqFmt.k3) { + kq = addr(t.k3q) + ks = addr(t.k3s) + } elif (fmt == KqFmt.iq3s) { + kq = addr(t.iq3sq) + ks = addr(t.iq3ss) + } elif (fmt == KqFmt.iq3xxs) { + kq = addr(t.iq3xxsq) + ks = addr(t.iq3xxss) + } elif (fmt == KqFmt.iq4nl) { + kq = addr(t.iq4nlq) + ks = addr(t.iq4nls) + } elif (fmt == KqFmt.k2) { + kq = addr(t.k2q) + ks = addr(t.k2s) + } elif (fmt == KqFmt.iq2s) { + kq = addr(t.iq2sq) + ks = addr(t.iq2ss) + } elif (fmt == KqFmt.iq2xs) { + kq = addr(t.iq2xsq) + ks = addr(t.iq2xss) + } elif (fmt == KqFmt.iq2xxs) { + kq = addr(t.iq2xxsq) + ks = addr(t.iq2xxss) + } else { + panic("kq_planes_of: '{fmt}' has no kq plane pair") + } + } +} + +// Dense-position demotion: q40/iq4nl/k5-from-Q5_0 need % 256 rows (superblock walkers over +// 32-block disk formats); q51 always demotes — dense positions have no q51 dispatch (v1 is expert stacks — kq_fmt_expert_ok). def kq_fmt_row_ok(f : KqFmt; n : int64) : KqFmt { if (f == KqFmt.q51) return KqFmt.q8 - return (f == KqFmt.q40 || f == KqFmt.k5) && n % 256l != 0l ? KqFmt.q8 : f + return (f == KqFmt.q40 || f == KqFmt.iq4nl || f == KqFmt.k5) && n % 256l != 0l ? KqFmt.q8 : f } // Expert-stack tag rule: q51 serves natively off per-32 planes (its whole reason to exist — the @@ -1547,7 +1644,7 @@ def kq_active_mr(t : Model; f : KqFmt) : int64 { if (!t.kq_repacked) { return 1l } - return f == KqFmt.k4 ? t.kq_repack_mr4 : (f == KqFmt.k5 ? t.kq_repack_mr5 : (f == KqFmt.k6 ? t.kq_repack_mr6 : t.kq_repack_mr40)) + return f == KqFmt.k4 ? t.kq_repack_mr4 : (f == KqFmt.k5 ? t.kq_repack_mr5 : (f == KqFmt.k6 ? t.kq_repack_mr6 : (f == KqFmt.q40 ? t.kq_repack_mr40 : (f == KqFmt.iq4xs ? t.kq_repack_mr44 : (f == KqFmt.k3 ? t.kq_repack_mr3 : (f == KqFmt.iq3s ? t.kq_repack_mr33 : (f == KqFmt.iq3xxs ? t.kq_repack_mr34 : (f == KqFmt.iq4nl ? t.kq_repack_mr45 : (f == KqFmt.k2 ? t.kq_repack_mr2 : (f == KqFmt.iq2s ? t.kq_repack_mr23 : (f == KqFmt.iq2xs ? t.kq_repack_mr24 : t.kq_repack_mr25))))))))))) } //! Load a llama-architecture GGUF into the split layout at the requested precision — straight into @@ -1610,8 +1707,8 @@ def footprint(t : Model) : MemFootprint { let q4s = long_length(t.q4scales) * 4l let mx4 = long_length(t.mxq) // uint8 (two nibbles) -> 1 byte each let mx4s = long_length(t.mxs) // raw E8M0 -> 1 byte per 32-block - let kq = long_length(t.k4q) + long_length(t.k5q) + long_length(t.k6q) + long_length(t.q40q) + long_length(t.q51q) - let kqs = long_length(t.k4s) + long_length(t.k5s) + long_length(t.k6s) + long_length(t.q40s) + long_length(t.q51s) + let kq = long_length(t.k4q) + long_length(t.k5q) + long_length(t.k6q) + long_length(t.q40q) + long_length(t.q51q) + long_length(t.iq4xsq) + long_length(t.k3q) + long_length(t.iq3sq) + long_length(t.iq3xxsq) + long_length(t.iq4nlq) + long_length(t.k2q) + long_length(t.iq2sq) + long_length(t.iq2xsq) + long_length(t.iq2xxsq) + let kqs = long_length(t.k4s) + long_length(t.k5s) + long_length(t.k6s) + long_length(t.q40s) + long_length(t.q51s) + long_length(t.iq4xss) + long_length(t.k3s) + long_length(t.iq3ss) + long_length(t.iq3xxss) + long_length(t.iq4nls) + long_length(t.k2s) + long_length(t.iq2ss) + long_length(t.iq2xss) + long_length(t.iq2xxss) return MemFootprint(aux_fp32_bytes = aux, weight_fp32_bytes = wf, q8_bytes = q8, q8_scale_bytes = q8s, q4_bytes = q4, q4_scale_bytes = q4s, mx4_bytes = mx4, mx4_scale_bytes = mx4s, kq_bytes = kq, kq_scale_bytes = kqs, @@ -3285,11 +3382,11 @@ def mm_b_q51_groupn(var y : array; t : Model; offs : array; nregio // KqFmt -> the kernel-layer format id (matmul_kq*/kq_rows_fn/kq_qsb take the int form). q8 maps // to 0, a poison id no kq kernel accepts — kq_rows_for calls this for EVERY tag, so it must stay // total even though q8 branches never read the result. -def private kq_fi(fmt : KqFmt) : int => fmt == KqFmt.k4 ? 4 : (fmt == KqFmt.k5 ? 5 : (fmt == KqFmt.k6 ? 6 : (fmt == KqFmt.q40 ? 40 : 0))) +def private kq_fi(fmt : KqFmt) : int => fmt == KqFmt.k4 ? 4 : (fmt == KqFmt.k5 ? 5 : (fmt == KqFmt.k6 ? 6 : (fmt == KqFmt.q40 ? 40 : (fmt == KqFmt.iq4xs ? 44 : (fmt == KqFmt.k3 ? 3 : (fmt == KqFmt.iq3s ? 33 : (fmt == KqFmt.iq3xxs ? 34 : (fmt == KqFmt.iq4nl ? 45 : (fmt == KqFmt.k2 ? 2 : (fmt == KqFmt.iq2s ? 23 : (fmt == KqFmt.iq2xs ? 24 : (fmt == KqFmt.iq2xxs ? 25 : 0)))))))))))) // K-quant GEMV against a PRE-quantized activation (xq/xs/xbs already filled by the bs quantizer). // kq_repacked loads run the backend's stamped cores; disk-order loads run the portable kernels. -def mm_at_kq_pre(var y : array; t : Model; fmt : KqFmt; woff : int64; xq : array; xs : array; xbs : array; n, d : int64; yoff : int64 = 0l) { +def mm_at_kq_pre(var y : array; t : Model; fmt : KqFmt; woff : int64; xq : array; xs : array; xbs : array; n, d : int64; yoff : int64 = 0l) { // nolint:STYLE037 — the flat per-format ladder (one arm per kq format) is the honest shape if (t.kq_repacked) { if (fmt == KqFmt.k4) { matmul_kq_active(4, y, t.k4q, t.k4s, woff, xq, xs, xbs, n, d, yoff) @@ -3299,6 +3396,24 @@ def mm_at_kq_pre(var y : array; t : Model; fmt : KqFmt; woff : int64; xq matmul_kq_active(6, y, t.k6q, t.k6s, woff, xq, xs, xbs, n, d, yoff) } elif (fmt == KqFmt.q40) { matmul_kq_active(40, y, t.q40q, t.q40s, woff, xq, xs, xbs, n, d, yoff) + } elif (fmt == KqFmt.iq4xs) { + matmul_kq_active(44, y, t.iq4xsq, t.iq4xss, woff, xq, xs, xbs, n, d, yoff) + } elif (fmt == KqFmt.k3) { + matmul_kq_active(3, y, t.k3q, t.k3s, woff, xq, xs, xbs, n, d, yoff) + } elif (fmt == KqFmt.iq3s) { + matmul_kq_active(33, y, t.iq3sq, t.iq3ss, woff, xq, xs, xbs, n, d, yoff) + } elif (fmt == KqFmt.iq3xxs) { + matmul_kq_active(34, y, t.iq3xxsq, t.iq3xxss, woff, xq, xs, xbs, n, d, yoff) + } elif (fmt == KqFmt.iq4nl) { + matmul_kq_active(45, y, t.iq4nlq, t.iq4nls, woff, xq, xs, xbs, n, d, yoff) + } elif (fmt == KqFmt.k2) { + matmul_kq_active(2, y, t.k2q, t.k2s, woff, xq, xs, xbs, n, d, yoff) + } elif (fmt == KqFmt.iq2s) { + matmul_kq_active(23, y, t.iq2sq, t.iq2ss, woff, xq, xs, xbs, n, d, yoff) + } elif (fmt == KqFmt.iq2xs) { + matmul_kq_active(24, y, t.iq2xsq, t.iq2xss, woff, xq, xs, xbs, n, d, yoff) + } elif (fmt == KqFmt.iq2xxs) { + matmul_kq_active(25, y, t.iq2xxsq, t.iq2xxss, woff, xq, xs, xbs, n, d, yoff) } else { panic("mm_at_kq_pre: '{fmt}' has no kq plane pair") // a new KqFmt must claim its arm, never ride q40's } @@ -3310,6 +3425,24 @@ def mm_at_kq_pre(var y : array; t : Model; fmt : KqFmt; woff : int64; xq matmul_kq(6, y, t.k6q, t.k6s, woff, xq, xs, xbs, n, d, yoff) } elif (fmt == KqFmt.q40) { matmul_kq(40, y, t.q40q, t.q40s, woff, xq, xs, xbs, n, d, yoff) + } elif (fmt == KqFmt.iq4xs) { + matmul_kq(44, y, t.iq4xsq, t.iq4xss, woff, xq, xs, xbs, n, d, yoff) + } elif (fmt == KqFmt.k3) { + matmul_kq(3, y, t.k3q, t.k3s, woff, xq, xs, xbs, n, d, yoff) + } elif (fmt == KqFmt.iq3s) { + matmul_kq(33, y, t.iq3sq, t.iq3ss, woff, xq, xs, xbs, n, d, yoff) + } elif (fmt == KqFmt.iq3xxs) { + matmul_kq(34, y, t.iq3xxsq, t.iq3xxss, woff, xq, xs, xbs, n, d, yoff) + } elif (fmt == KqFmt.iq4nl) { + matmul_kq(45, y, t.iq4nlq, t.iq4nls, woff, xq, xs, xbs, n, d, yoff) + } elif (fmt == KqFmt.k2) { + matmul_kq(2, y, t.k2q, t.k2s, woff, xq, xs, xbs, n, d, yoff) + } elif (fmt == KqFmt.iq2s) { + matmul_kq(23, y, t.iq2sq, t.iq2ss, woff, xq, xs, xbs, n, d, yoff) + } elif (fmt == KqFmt.iq2xs) { + matmul_kq(24, y, t.iq2xsq, t.iq2xss, woff, xq, xs, xbs, n, d, yoff) + } elif (fmt == KqFmt.iq2xxs) { + matmul_kq(25, y, t.iq2xxsq, t.iq2xxss, woff, xq, xs, xbs, n, d, yoff) } else { panic("mm_at_kq_pre: '{fmt}' has no kq plane pair") } @@ -3341,6 +3474,24 @@ def mm_b_kq(var s : Session; var y : array; t : Model; fmt : KqFmt; woff matmul_kq_batch(6, y, t.k6q, t.k6s, woff, s.xqb, s.xsb, s.xbsb, n, d, npos) } elif (fmt == KqFmt.q40) { matmul_kq_batch(40, y, t.q40q, t.q40s, woff, s.xqb, s.xsb, s.xbsb, n, d, npos) + } elif (fmt == KqFmt.iq4xs) { + matmul_kq_batch(44, y, t.iq4xsq, t.iq4xss, woff, s.xqb, s.xsb, s.xbsb, n, d, npos) + } elif (fmt == KqFmt.k3) { + matmul_kq_batch(3, y, t.k3q, t.k3s, woff, s.xqb, s.xsb, s.xbsb, n, d, npos) + } elif (fmt == KqFmt.iq3s) { + matmul_kq_batch(33, y, t.iq3sq, t.iq3ss, woff, s.xqb, s.xsb, s.xbsb, n, d, npos) + } elif (fmt == KqFmt.iq3xxs) { + matmul_kq_batch(34, y, t.iq3xxsq, t.iq3xxss, woff, s.xqb, s.xsb, s.xbsb, n, d, npos) + } elif (fmt == KqFmt.iq4nl) { + matmul_kq_batch(45, y, t.iq4nlq, t.iq4nls, woff, s.xqb, s.xsb, s.xbsb, n, d, npos) + } elif (fmt == KqFmt.k2) { + matmul_kq_batch(2, y, t.k2q, t.k2s, woff, s.xqb, s.xsb, s.xbsb, n, d, npos) + } elif (fmt == KqFmt.iq2s) { + matmul_kq_batch(23, y, t.iq2sq, t.iq2ss, woff, s.xqb, s.xsb, s.xbsb, n, d, npos) + } elif (fmt == KqFmt.iq2xs) { + matmul_kq_batch(24, y, t.iq2xsq, t.iq2xss, woff, s.xqb, s.xsb, s.xbsb, n, d, npos) + } elif (fmt == KqFmt.iq2xxs) { + matmul_kq_batch(25, y, t.iq2xxsq, t.iq2xxss, woff, s.xqb, s.xsb, s.xbsb, n, d, npos) } else { panic("mm_b_kq: '{fmt}' has no kq plane pair") } @@ -3358,7 +3509,7 @@ def mm_b_kq(var s : Session; var y : array; t : Model; fmt : KqFmt; woff // Region-list kq GEMV over the model's planes (the MoE expert dispatch's kq twin of // mm_at_q8_groupn; offs = (weight, activation) element-offset pairs within fmt's plane). // Repacked loads run the backend's kq_groupn; disk-order loads run the portable region walk. -def mm_at_kq_groupn(var y : array; t : Model; fmt : KqFmt; offs : array; nregions : int64; xq : array; xs : array; xbs : array; n, d : int64) { +def mm_at_kq_groupn(var y : array; t : Model; fmt : KqFmt; offs : array; nregions : int64; xq : array; xs : array; xbs : array; n, d : int64) { // nolint:STYLE037 — the flat per-format ladder (one arm per kq format) is the honest shape if (t.kq_repacked) { if (fmt == KqFmt.k4) { matmul_kq_groupn_active(4, y, t.k4q, t.k4s, offs, nregions, xq, xs, xbs, n, d) @@ -3368,6 +3519,24 @@ def mm_at_kq_groupn(var y : array; t : Model; fmt : KqFmt; offs : array; t : Model; fmt : KqFmt; offs : array; t : Model; fmt : KqFmt; offs : array; t : Model; fmt : KqFmt; woff : int64; xq : matmul_kq_batch(6, y, t.k6q, t.k6s, woff, xq, xs, xbs, n, d, ntok) } elif (fmt == KqFmt.q40) { matmul_kq_batch(40, y, t.q40q, t.q40s, woff, xq, xs, xbs, n, d, ntok) + } elif (fmt == KqFmt.iq4xs) { + matmul_kq_batch(44, y, t.iq4xsq, t.iq4xss, woff, xq, xs, xbs, n, d, ntok) + } elif (fmt == KqFmt.k3) { + matmul_kq_batch(3, y, t.k3q, t.k3s, woff, xq, xs, xbs, n, d, ntok) + } elif (fmt == KqFmt.iq3s) { + matmul_kq_batch(33, y, t.iq3sq, t.iq3ss, woff, xq, xs, xbs, n, d, ntok) + } elif (fmt == KqFmt.iq3xxs) { + matmul_kq_batch(34, y, t.iq3xxsq, t.iq3xxss, woff, xq, xs, xbs, n, d, ntok) + } elif (fmt == KqFmt.iq4nl) { + matmul_kq_batch(45, y, t.iq4nlq, t.iq4nls, woff, xq, xs, xbs, n, d, ntok) + } elif (fmt == KqFmt.k2) { + matmul_kq_batch(2, y, t.k2q, t.k2s, woff, xq, xs, xbs, n, d, ntok) + } elif (fmt == KqFmt.iq2s) { + matmul_kq_batch(23, y, t.iq2sq, t.iq2ss, woff, xq, xs, xbs, n, d, ntok) + } elif (fmt == KqFmt.iq2xs) { + matmul_kq_batch(24, y, t.iq2xsq, t.iq2xss, woff, xq, xs, xbs, n, d, ntok) + } elif (fmt == KqFmt.iq2xxs) { + matmul_kq_batch(25, y, t.iq2xxsq, t.iq2xxss, woff, xq, xs, xbs, n, d, ntok) } else { panic("mm_b_kq_pre: '{fmt}' has no kq plane pair") } @@ -3425,6 +3648,15 @@ def kq_plane_q(t : Model; fmt : KqFmt; woff : int64) : uint8 const? { if (fmt == KqFmt.k5) return unsafe(addr(t.k5q[sb * K5_QSB])) if (fmt == KqFmt.k6) return unsafe(addr(t.k6q[sb * K6_QSB])) if (fmt == KqFmt.q40) return unsafe(addr(t.q40q[sb * Q40_QSB])) + if (fmt == KqFmt.iq4xs) return unsafe(addr(t.iq4xsq[sb * IQ4XS_QSB])) + if (fmt == KqFmt.k3) return unsafe(addr(t.k3q[sb * K3_QSB])) + if (fmt == KqFmt.iq3s) return unsafe(addr(t.iq3sq[sb * IQ3S_QSB])) + if (fmt == KqFmt.iq3xxs) return unsafe(addr(t.iq3xxsq[sb * IQ3XXS_QSB])) + if (fmt == KqFmt.iq4nl) return unsafe(addr(t.iq4nlq[sb * Q40_QSB])) + if (fmt == KqFmt.k2) return unsafe(addr(t.k2q[sb * K2_QSB])) + if (fmt == KqFmt.iq2s) return unsafe(addr(t.iq2sq[sb * IQ2S_QSB])) + if (fmt == KqFmt.iq2xs) return unsafe(addr(t.iq2xsq[sb * IQ2XS_QSB])) + if (fmt == KqFmt.iq2xxs) return unsafe(addr(t.iq2xxsq[sb * IQ2XXS_QSB])) return null } @@ -3434,6 +3666,15 @@ def kq_plane_s(t : Model; fmt : KqFmt; woff : int64) : uint8 const? { if (fmt == KqFmt.k5) return unsafe(addr(t.k5s[sb * K5_SSB])) if (fmt == KqFmt.k6) return unsafe(addr(t.k6s[sb * K6_SSB])) if (fmt == KqFmt.q40) return unsafe(addr(t.q40s[sb * Q40_SSB])) + if (fmt == KqFmt.iq4xs) return unsafe(addr(t.iq4xss[sb * IQ4XS_SSB])) + if (fmt == KqFmt.k3) return unsafe(addr(t.k3s[sb * K3_SSB])) + if (fmt == KqFmt.iq3s) return unsafe(addr(t.iq3ss[sb * IQ3S_SSB])) + if (fmt == KqFmt.iq3xxs) return unsafe(addr(t.iq3xxss[sb * IQ3XXS_SSB])) + if (fmt == KqFmt.iq4nl) return unsafe(addr(t.iq4nls[sb * Q40_SSB])) + if (fmt == KqFmt.k2) return unsafe(addr(t.k2s[sb * K2_SSB])) + if (fmt == KqFmt.iq2s) return unsafe(addr(t.iq2ss[sb * IQ2S_SSB])) + if (fmt == KqFmt.iq2xs) return unsafe(addr(t.iq2xss[sb * IQ2XS_SSB])) + if (fmt == KqFmt.iq2xxs) return unsafe(addr(t.iq2xxss[sb * IQ2XXS_SSB])) return null } @@ -3794,6 +4035,24 @@ def embed_row(t : Model; token : int64; var dst : float?) { // nolint:STYLE037 dequant_kq_row_grp(fmt, addr(t.embq[sbg * K6_QSB]), addr(t.embs[sbg * K6_SSB]), token % mr, mr, dim, dst) } elif (t.emb_fmt == KqFmt.q40) { dequant_kq_row_grp(fmt, addr(t.embq[sbg * Q40_QSB]), addr(t.embs[sbg * Q40_SSB]), token % mr, mr, dim, dst) + } elif (t.emb_fmt == KqFmt.iq4xs) { + dequant_kq_row_grp(fmt, addr(t.embq[sbg * IQ4XS_QSB]), addr(t.embs[sbg * IQ4XS_SSB]), token % mr, mr, dim, dst) + } elif (t.emb_fmt == KqFmt.k3) { + dequant_kq_row_grp(fmt, addr(t.embq[sbg * K3_QSB]), addr(t.embs[sbg * K3_SSB]), token % mr, mr, dim, dst) + } elif (t.emb_fmt == KqFmt.iq3s) { + dequant_kq_row_grp(fmt, addr(t.embq[sbg * IQ3S_QSB]), addr(t.embs[sbg * IQ3S_SSB]), token % mr, mr, dim, dst) + } elif (t.emb_fmt == KqFmt.iq3xxs) { + dequant_kq_row_grp(fmt, addr(t.embq[sbg * IQ3XXS_QSB]), addr(t.embs[sbg * IQ3XXS_SSB]), token % mr, mr, dim, dst) + } elif (t.emb_fmt == KqFmt.iq4nl) { + dequant_kq_row_grp(fmt, addr(t.embq[sbg * Q40_QSB]), addr(t.embs[sbg * Q40_SSB]), token % mr, mr, dim, dst) + } elif (t.emb_fmt == KqFmt.k2) { + dequant_kq_row_grp(fmt, addr(t.embq[sbg * K2_QSB]), addr(t.embs[sbg * K2_SSB]), token % mr, mr, dim, dst) + } elif (t.emb_fmt == KqFmt.iq2s) { + dequant_kq_row_grp(fmt, addr(t.embq[sbg * IQ2S_QSB]), addr(t.embs[sbg * IQ2S_SSB]), token % mr, mr, dim, dst) + } elif (t.emb_fmt == KqFmt.iq2xs) { + dequant_kq_row_grp(fmt, addr(t.embq[sbg * IQ2XS_QSB]), addr(t.embs[sbg * IQ2XS_SSB]), token % mr, mr, dim, dst) + } elif (t.emb_fmt == KqFmt.iq2xxs) { + dequant_kq_row_grp(fmt, addr(t.embq[sbg * IQ2XXS_QSB]), addr(t.embs[sbg * IQ2XXS_SSB]), token % mr, mr, dim, dst) } else { panic("embed_row: '{t.emb_fmt}' has no kq plane pair") } @@ -3805,6 +4064,24 @@ def embed_row(t : Model; token : int64; var dst : float?) { // nolint:STYLE037 dequant_kq_row_grp(fmt, addr(t.k6q[sbg * K6_QSB]), addr(t.k6s[sbg * K6_SSB]), token % mr, mr, dim, dst) } elif (t.emb_fmt == KqFmt.q40) { dequant_kq_row_grp(fmt, addr(t.q40q[sbg * Q40_QSB]), addr(t.q40s[sbg * Q40_SSB]), token % mr, mr, dim, dst) + } elif (t.emb_fmt == KqFmt.iq4xs) { + dequant_kq_row_grp(fmt, addr(t.iq4xsq[sbg * IQ4XS_QSB]), addr(t.iq4xss[sbg * IQ4XS_SSB]), token % mr, mr, dim, dst) + } elif (t.emb_fmt == KqFmt.k3) { + dequant_kq_row_grp(fmt, addr(t.k3q[sbg * K3_QSB]), addr(t.k3s[sbg * K3_SSB]), token % mr, mr, dim, dst) + } elif (t.emb_fmt == KqFmt.iq3s) { + dequant_kq_row_grp(fmt, addr(t.iq3sq[sbg * IQ3S_QSB]), addr(t.iq3ss[sbg * IQ3S_SSB]), token % mr, mr, dim, dst) + } elif (t.emb_fmt == KqFmt.iq3xxs) { + dequant_kq_row_grp(fmt, addr(t.iq3xxsq[sbg * IQ3XXS_QSB]), addr(t.iq3xxss[sbg * IQ3XXS_SSB]), token % mr, mr, dim, dst) + } elif (t.emb_fmt == KqFmt.iq4nl) { + dequant_kq_row_grp(fmt, addr(t.iq4nlq[sbg * Q40_QSB]), addr(t.iq4nls[sbg * Q40_SSB]), token % mr, mr, dim, dst) + } elif (t.emb_fmt == KqFmt.k2) { + dequant_kq_row_grp(fmt, addr(t.k2q[sbg * K2_QSB]), addr(t.k2s[sbg * K2_SSB]), token % mr, mr, dim, dst) + } elif (t.emb_fmt == KqFmt.iq2s) { + dequant_kq_row_grp(fmt, addr(t.iq2sq[sbg * IQ2S_QSB]), addr(t.iq2ss[sbg * IQ2S_SSB]), token % mr, mr, dim, dst) + } elif (t.emb_fmt == KqFmt.iq2xs) { + dequant_kq_row_grp(fmt, addr(t.iq2xsq[sbg * IQ2XS_QSB]), addr(t.iq2xss[sbg * IQ2XS_SSB]), token % mr, mr, dim, dst) + } elif (t.emb_fmt == KqFmt.iq2xxs) { + dequant_kq_row_grp(fmt, addr(t.iq2xxsq[sbg * IQ2XXS_QSB]), addr(t.iq2xxss[sbg * IQ2XXS_SSB]), token % mr, mr, dim, dst) } else { panic("embed_row: '{t.emb_fmt}' has no kq plane pair") } @@ -3814,9 +4091,6 @@ def embed_row(t : Model; token : int64; var dst : float?) { // nolint:STYLE037 var row = temp_array(dst, dim, type) // trimmed: embq/embs hold the region's plane pair verbatim, rebased to 0 let sb0 = t.planes_trimmed ? (token * dim) / 256l : (t.wcls_off + token * dim) / 256l - // metal-blob planes: k4/k5 strips are 16B, k6 is the split GPU form (d in the tail) - let ssb45 = t.metal_blob ? 16l : K4_SSB - let k6d0 = t.metal_blob ? (long_length(t.k6s) / K6_SSB) * 16l : 0l for (s in range64(dim / 256l)) { if (t.planes_trimmed) { if (t.emb_fmt == KqFmt.k4) { @@ -3827,23 +4101,32 @@ def embed_row(t : Model; token : int64; var dst : float?) { // nolint:STYLE037 dequant_k6_plane_superblock(t.embq, (sb0 + s) * K6_QSB, t.embs, (sb0 + s) * K6_SSB, row, s * 256l) } elif (t.emb_fmt == KqFmt.q40) { dequant_q40_plane_superblock(t.embq, (sb0 + s) * Q40_QSB, t.embs, (sb0 + s) * Q40_SSB, row, s * 256l) + } elif (t.emb_fmt == KqFmt.iq4xs) { + dequant_iq4xs_plane_superblock(t.embq, (sb0 + s) * IQ4XS_QSB, t.embs, (sb0 + s) * IQ4XS_SSB, row, s * 256l) + } elif (t.emb_fmt == KqFmt.k3) { + dequant_k3_plane_superblock(t.embq, (sb0 + s) * K3_QSB, t.embs, (sb0 + s) * K3_SSB, row, s * 256l) + } elif (t.emb_fmt == KqFmt.iq3s) { + dequant_iq3s_plane_superblock(t.embq, (sb0 + s) * IQ3S_QSB, t.embs, (sb0 + s) * IQ3S_SSB, row, s * 256l) + } elif (t.emb_fmt == KqFmt.iq3xxs) { + dequant_iq3xxs_plane_superblock(t.embq, (sb0 + s) * IQ3XXS_QSB, t.embs, (sb0 + s) * IQ3XXS_SSB, row, s * 256l) + } elif (t.emb_fmt == KqFmt.iq4nl) { + dequant_iq4nl_plane_superblock(t.embq, (sb0 + s) * Q40_QSB, t.embs, (sb0 + s) * Q40_SSB, row, s * 256l) + } elif (t.emb_fmt == KqFmt.k2) { + dequant_k2_plane_superblock(t.embq, (sb0 + s) * K2_QSB, t.embs, (sb0 + s) * K2_SSB, row, s * 256l) + } elif (t.emb_fmt == KqFmt.iq2s) { + dequant_iq2s_plane_superblock(t.embq, (sb0 + s) * IQ2S_QSB, t.embs, (sb0 + s) * IQ2S_SSB, row, s * 256l) + } elif (t.emb_fmt == KqFmt.iq2xs) { + dequant_iq2xs_plane_superblock(t.embq, (sb0 + s) * IQ2XS_QSB, t.embs, (sb0 + s) * IQ2XS_SSB, row, s * 256l) + } elif (t.emb_fmt == KqFmt.iq2xxs) { + dequant_iq2xxs_plane_superblock(t.embq, (sb0 + s) * IQ2XXS_QSB, t.embs, (sb0 + s) * IQ2XXS_SSB, row, s * 256l) } else { panic("embed_row: '{t.emb_fmt}' has no kq plane pair") } - } elif (t.emb_fmt == KqFmt.k4) { - dequant_k4_plane_superblock(t.k4q, (sb0 + s) * K4_QSB, t.k4s, (sb0 + s) * ssb45, row, s * 256l) - } elif (t.emb_fmt == KqFmt.k5) { - dequant_k5_plane_superblock(t.k5q, (sb0 + s) * K5_QSB, t.k5s, (sb0 + s) * ssb45, row, s * 256l) - } elif (t.emb_fmt == KqFmt.k6) { - if (t.metal_blob) { - dequant_k6_plane_superblock_at(t.k6q, (sb0 + s) * K6_QSB, t.k6s, (sb0 + s) * 16l, k6d0 + (sb0 + s) * 2l, row, s * 256l) - } else { - dequant_k6_plane_superblock(t.k6q, (sb0 + s) * K6_QSB, t.k6s, (sb0 + s) * K6_SSB, row, s * 256l) - } - } elif (t.emb_fmt == KqFmt.q40) { - dequant_q40_plane_superblock(t.q40q, (sb0 + s) * Q40_QSB, t.q40s, (sb0 + s) * Q40_SSB, row, s * 256l) } else { - panic("embed_row: '{t.emb_fmt}' has no kq plane pair") + var kqp : array const? + var ksp : array const? + kq_planes_of(t, t.emb_fmt, kqp, ksp) + dequant_kq_plane_sb(t.emb_fmt, *kqp, *ksp, sb0 + s, t.metal_blob, row, s * 256l) } } } diff --git a/modules/dasLLAMA/dasllama/dasllama_config.das b/modules/dasLLAMA/dasllama/dasllama_config.das index 88b181b4fe..3704b36156 100644 --- a/modules/dasLLAMA/dasllama/dasllama_config.das +++ b/modules/dasLLAMA/dasllama/dasllama_config.das @@ -27,6 +27,15 @@ struct public DlimCpuConfig { kq_mr5 : int64 kq_mr6 : int64 kq_mr40 : int64 + kq_mr44 : int64 + kq_mr3 : int64 + kq_mr33 : int64 + kq_mr34 : int64 + kq_mr45 : int64 + kq_mr2 : int64 + kq_mr23 : int64 + kq_mr24 : int64 + kq_mr25 : int64 q51_mr : int64 wscale_f16 : bool kquant_native : bool @@ -135,11 +144,13 @@ def public dlim_config_current(quant : string = "q8") : DlimConfiguration { // ===== pure formatters (identity, tags, JSON) ===== //! The identity string an image file/header is keyed by — a pure formatter of the struct. -//! `image_version` comes from the caller (dasllama_image owns the version constant). -def public dlim_identity(c : DlimConfiguration; image_version : int; tag : string = "") : string { - return ("v{image_version}|{c.quant}|{c.cpu.backend}|{c.cpu.wscale_f16 ? "s16" : "s32"}" +//! `image_version` / `pack_version` come from the caller (dasllama_image owns the schema +//! version, dasllama_layout the pack-code version; 0 = the pre-versioned pack spelling). +def public dlim_identity(c : DlimConfiguration; image_version : int; tag : string = ""; pack_version : int = 0) : string { + let pk = pack_version > 0 ? "p{pack_version}" : "" + return ("v{image_version}{pk}|{c.quant}|{c.cpu.backend}|{c.cpu.wscale_f16 ? "s16" : "s32"}" + "|q8 mr{c.cpu.q8_mr} b{c.cpu.q8_wbias} g{c.cpu.q8_kgroup}" - + "|kq {c.cpu.kq_mr4}/{c.cpu.kq_mr5}/{c.cpu.kq_mr6}/{c.cpu.kq_mr40}" + + "|kq {c.cpu.kq_mr4}/{c.cpu.kq_mr5}/{c.cpu.kq_mr6}/{c.cpu.kq_mr40}/{c.cpu.kq_mr44}/{c.cpu.kq_mr3}/{c.cpu.kq_mr33}/{c.cpu.kq_mr34}/{c.cpu.kq_mr45}/{c.cpu.kq_mr2}/{c.cpu.kq_mr23}/{c.cpu.kq_mr24}/{c.cpu.kq_mr25}" + "|q51 mr{c.cpu.q51_mr}" + "|nat {c.cpu.kquant_native ? 1 : 0}{c.cpu.kq_q40_native ? 1 : 0}{c.cpu.kq_q50_native ? 1 : 0}{c.cpu.kq_q51_native ? 1 : 0}" + (tag != "" ? "|{tag}" : "")) diff --git a/modules/dasLLAMA/dasllama/dasllama_convert.das b/modules/dasLLAMA/dasllama/dasllama_convert.das index 0fd918bf2f..36d4b4692d 100644 --- a/modules/dasLLAMA/dasllama/dasllama_convert.das +++ b/modules/dasLLAMA/dasllama/dasllama_convert.das @@ -561,6 +561,70 @@ def dequant_k6_plane_superblock(kq : array | #; kqo : int64; ks : array | #; ks : array | #; sb : int64; metal_blob : bool; var dst : array | #; doff : int64) { // nolint:STYLE037,STYLE038 — one arm per format, two carrier forms + if (fmt == KqFmt.k4) { + dequant_k4_plane_superblock(kq, sb * K4_QSB, ks, sb * (metal_blob ? 16l : K4_SSB), dst, doff) + } elif (fmt == KqFmt.k5) { + dequant_k5_plane_superblock(kq, sb * K5_QSB, ks, sb * (metal_blob ? 16l : K5_SSB), dst, doff) + } elif (fmt == KqFmt.q40) { + dequant_q40_plane_superblock(kq, sb * Q40_QSB, ks, sb * Q40_SSB, dst, doff) + } elif (fmt == KqFmt.iq4nl) { + dequant_iq4nl_plane_superblock(kq, sb * Q40_QSB, ks, sb * Q40_SSB, dst, doff) + } elif (!metal_blob) { + if (fmt == KqFmt.k6) { + dequant_k6_plane_superblock(kq, sb * K6_QSB, ks, sb * K6_SSB, dst, doff) + } elif (fmt == KqFmt.iq4xs) { + dequant_iq4xs_plane_superblock(kq, sb * IQ4XS_QSB, ks, sb * IQ4XS_SSB, dst, doff) + } elif (fmt == KqFmt.k3) { + dequant_k3_plane_superblock(kq, sb * K3_QSB, ks, sb * K3_SSB, dst, doff) + } elif (fmt == KqFmt.iq3s) { + dequant_iq3s_plane_superblock(kq, sb * IQ3S_QSB, ks, sb * IQ3S_SSB, dst, doff) + } elif (fmt == KqFmt.iq3xxs) { + dequant_iq3xxs_plane_superblock(kq, sb * IQ3XXS_QSB, ks, sb * IQ3XXS_SSB, dst, doff) + } elif (fmt == KqFmt.k2) { + dequant_k2_plane_superblock(kq, sb * K2_QSB, ks, sb * K2_SSB, dst, doff) + } elif (fmt == KqFmt.iq2s) { + dequant_iq2s_plane_superblock(kq, sb * IQ2S_QSB, ks, sb * IQ2S_SSB, dst, doff) + } elif (fmt == KqFmt.iq2xs) { + dequant_iq2xs_plane_superblock(kq, sb * IQ2XS_QSB, ks, sb * IQ2XS_SSB, dst, doff) + } elif (fmt == KqFmt.iq2xxs) { + dequant_iq2xxs_plane_superblock(kq, sb * IQ2XXS_QSB, ks, sb * IQ2XXS_SSB, dst, doff) + } else { + panic("dequant_kq_plane_sb: '{fmt}' has no kq plane pair") + } + } else { + // the split GPU form: every strip region is 16B/sb, the tail 2B/sb (k2: 4B - d and dmin) + let tail = fmt == KqFmt.k2 ? 4l : 2l + let d0 = (long_length(ks) / (16l + tail)) * 16l + if (fmt == KqFmt.k6) { + dequant_k6_plane_superblock_at(kq, sb * K6_QSB, ks, sb * 16l, d0 + sb * 2l, dst, doff) + } elif (fmt == KqFmt.iq4xs) { + dequant_iq4xs_plane_superblock_at(kq, sb * IQ4XS_QSB, ks, sb * 16l, d0 + sb * 2l, dst, doff) + } elif (fmt == KqFmt.k3) { + dequant_k3_plane_superblock_at(kq, sb * K3_QSB, ks, sb * 16l, d0 + sb * 2l, dst, doff) + } elif (fmt == KqFmt.iq3s) { + dequant_iq3s_plane_superblock_at(kq, sb * IQ3S_QSB, ks, sb * 16l, d0 + sb * 2l, dst, doff) + } elif (fmt == KqFmt.iq3xxs) { + dequant_iq3xxs_plane_superblock_at(kq, sb * IQ3XXS_QSB, ks, sb * 16l, d0 + sb * 2l, dst, doff) + } elif (fmt == KqFmt.k2) { + dequant_k2_plane_superblock_at(kq, sb * K2_QSB, ks, sb * 16l, d0 + sb * 4l, dst, doff) + } elif (fmt == KqFmt.iq2s) { + dequant_iq2s_plane_superblock_at(kq, sb * IQ2S_QSB, ks, sb * 16l, d0 + sb * 2l, dst, doff) + } elif (fmt == KqFmt.iq2xs) { + dequant_iq2xs_plane_superblock_at(kq, sb * IQ2XS_QSB, ks, sb * 16l, d0 + sb * 2l, dst, doff) + } elif (fmt == KqFmt.iq2xxs) { + dequant_iq2xxs_plane_superblock_at(kq, sb * IQ2XXS_QSB, ks, sb * 16l, d0 + sb * 2l, dst, doff) + } else { + panic("dequant_kq_plane_sb: '{fmt}' has no kq plane pair") + } + } +} + + //! The split-layout twin: `dso` names the f16 d halfword's byte offset within ks independently //! of the 16B sub-scale strip at kso — metal-blob models keep k6s in the GPU split form //! ([nsb x 16B sub-scales][nsb x f16 d]) where d lives in the plane's tail. @@ -615,6 +679,399 @@ def dequant_q40_plane_superblock(kq : array | #; kqo : int64; ks : array< } } +//! Transcode one IQ4_XS superblock (136 bytes at `bo`: f16 d, u16 scales_h, 4 scales_l bytes, +//! 128 nibble bytes) into the iq4xs planes: nibbles verbatim (the disk k/k+16 pairing IS the +//! q40 tiling), the scale row DECODED — f16 d, 2 pad, 8 x int8 (ls - 32), 8 pad (exact). +def transcode_iq4xs_superblock(bytes : array | #; bo : int64; var kq : array; kqo : int64; var ks : array; kso : int64) { + ks[kso] = bytes[bo] + ks[kso + 1l] = bytes[bo + 1l] + ks[kso + 2l] = uint8(0) + ks[kso + 3l] = uint8(0) + let sh = uint(bytes[bo + 2l]) | (uint(bytes[bo + 3l]) << 8u) + for (blk in range64(8l)) { + let lo = (uint(bytes[bo + 4l + blk / 2l]) >> uint(4l * (blk % 2l))) & 15u + let ls = int(lo | (((sh >> uint(2l * blk)) & 3u) << 4u)) + ks[kso + 4l + blk] = uint8(ls - 32) + } + for (i in range64(8l)) { + ks[kso + 12l + i] = uint8(0) + } + for (i in range64(128l)) { + kq[kqo + i] = bytes[bo + 8l + i] + } +} + +//! The signed sub-scale of an iq4xs plane row: byte 4 + blk of the 20B row, read as int8. +def iq4xs_sc(ks : array | #; kso, blk : int64) : int => rd_i8(ks, kso + 4l + blk) + +//! Reference dequant of one iq4xs-plane superblock: w = (d * sc) * IQ4NL_LUT[q] per 32-block +//! (ggml's own float order: dl = d * (ls - 32), then dl * kvalue), q = nibble (low = k, high = +//! k+16 — the disk pairing, kept verbatim by the transcode). +def dequant_iq4xs_plane_superblock(kq : array | #; kqo : int64; ks : array | #; kso : int64; var dst : array | #; doff : int64) { + dequant_iq4xs_plane_superblock_at(kq, kqo, ks, kso + 4l, kso, dst, doff) +} + +//! The split-layout twin: `dso` names the f16 d halfword's byte offset independently of the +//! 8-byte signed sub-scale strip at kso — metal-blob models keep iq4xss in the k6 split form +//! ([nsb x 16B strips][nsb x f16 d]); the CPU row is the same math at kso + 4 / kso. +def dequant_iq4xs_plane_superblock_at(kq : array | #; kqo : int64; ks : array | #; kso, dso : int64; var dst : array | #; doff : int64) { + let d = f16_to_f32(rd_u16(ks, dso)) + let lut = iq4nl_lut() + for (blk in range64(8l)) { + let dl = d * float(rd_i8(ks, kso + blk)) + for (l in range64(16l)) { + let b = int(kq[kqo + blk * 16l + l]) + dst[doff + blk * 32l + l] = dl * float(lut[b & 15]) + dst[doff + blk * 32l + 16l + l] = dl * float(lut[b >> 4]) + } + } +} + +//! Transcode one Q2_K superblock (84 bytes at `bo`: 16 sc/min nibble-pair bytes, 64 qs, f16 d, +//! f16 dmin) into the k2 planes: qs verbatim to kq[kqo..+64), the scale row REORDERED to +//! [f16 d][f16 dmin][16 sc/min bytes] at ks[kso..+20) - all bytes verbatim (exact). +def transcode_q2k_superblock(bytes : array | #; bo : int64; var kq : array; kqo : int64; var ks : array; kso : int64) { + ks[kso] = bytes[bo + 80l] + ks[kso + 1l] = bytes[bo + 81l] + ks[kso + 2l] = bytes[bo + 82l] + ks[kso + 3l] = bytes[bo + 83l] + for (i in range64(16l)) { + ks[kso + 4l + i] = bytes[bo + i] + } + for (i in range64(64l)) { + kq[kqo + i] = bytes[bo + 16l + i] + } +} + +//! Reference dequant of one k2-plane superblock: w = (d * sc) * q - (dmin * mn) per 16-elem group +//! g (sc/mn = the group's nibble pair); q = the 2-bit lane of qs byte (g/8)*32 + (g%2)*16 + l at +//! shift 2*((g%8)/2) - ggml's dequantize_row_q2_K float order (dl*q - ml). +def dequant_k2_plane_superblock(kq : array | #; kqo : int64; ks : array | #; kso : int64; var dst : array | #; doff : int64) { + dequant_k2_plane_superblock_at(kq, kqo, ks, kso + 4l, kso, dst, doff) +} + +//! The split-layout twin: `sso` names the 16 pair-byte strip, `dso` the 4B d+dmin header - +//! independent bases, so the metal-blob split plane reads through the same math. +def dequant_k2_plane_superblock_at(kq : array | #; kqo : int64; ks : array | #; sso, dso : int64; var dst : array | #; doff : int64) { + let d = f16_to_f32(rd_u16(ks, dso)) + let dmin = f16_to_f32(rd_u16(ks, dso + 2l)) + for (g in range64(16l)) { + let scb = int(ks[sso + g]) + let dl = d * float(scb & 15) + let ml = dmin * float(scb >> 4) + let qb = kqo + (g / 8l) * 32l + (g % 2l) * 16l + let sh = int(((g % 8l) / 2l) * 2l) + for (l in range64(16l)) { + dst[doff + g * 16l + l] = dl * float((int(kq[qb + l]) >> sh) & 3) - ml + } + } +} + +//! The 6-bit sub-scale i (0..15) of a Q3_K disk block off its 12 packed scale bytes at `so` — +//! ggml's kmask unpack: low nibbles of bytes 0..7 carry scales 0..7, their high nibbles 8..15, +//! bytes 8..11 carry every scale's top two bits at shift 2 * (i / 4). +def q3k_scale6(bytes : array | #; so, i : int64) : int { + let lo = i < 8l ? int(bytes[so + i]) & 15 : int(bytes[so + i - 8l]) >> 4 + let hi = int(bytes[so + 8l + i % 4l]) >> int(2l * (i / 4l)) + return lo | ((hi & 3) << 4) +} + +//! Transcode one Q3_K superblock (110 bytes at `bo`: 32 hmask, 64 qs, 12 packed 6-bit scales, +//! f16 d) into the k3 planes: [qs 64][hmask 32] verbatim to kq[kqo..+96), the 16 sub-scales +//! DECODED to int8 (scale - 32) + the f16 d to ks[kso..+18) — the k6 row shape (exact). +def transcode_q3k_superblock(bytes : array | #; bo : int64; var kq : array; kqo : int64; var ks : array; kso : int64) { + for (i in range64(64l)) { + kq[kqo + i] = bytes[bo + 32l + i] + } + for (i in range64(32l)) { + kq[kqo + 64l + i] = bytes[bo + i] + } + for (i in range64(16l)) { + ks[kso + i] = uint8(q3k_scale6(bytes, bo + 96l, i) - 32) + } + ks[kso + 16l] = bytes[bo + 108l] + ks[kso + 17l] = bytes[bo + 109l] +} + +//! Reference dequant of one k3-plane superblock: w = (d * sc16) * (q - 4) with q = the 2-bit lane +//! of qs byte h*32 + l at shift 2j | hmask bit (4h + j) of byte l << 2, block 4h + j — ggml's +//! own float order (dl = d * (scale - 32), then dl * (q3 - 4)). +def dequant_k3_plane_superblock(kq : array | #; kqo : int64; ks : array | #; kso : int64; var dst : array | #; doff : int64) { + dequant_k3_plane_superblock_at(kq, kqo, ks, kso, kso + 16l, dst, doff) +} + +//! The split-layout twin: `dso` names the f16 d halfword's byte offset independently of the +//! 16-byte sub-scale strip at kso — the k6 split form a metal-blob model keeps. +def dequant_k3_plane_superblock_at(kq : array | #; kqo : int64; ks : array | #; kso, dso : int64; var dst : array | #; doff : int64) { + let d = f16_to_f32(rd_u16(ks, dso)) + for (blk in range64(8l)) { + let qb = kqo + (blk / 4l) * 32l + let sh = int((blk % 4l) * 2l) + let dl0 = d * float(rd_i8(ks, kso + 2l * blk)) + let dl1 = d * float(rd_i8(ks, kso + 2l * blk + 1l)) + for (l in range64(32l)) { + let q2 = (int(kq[qb + l]) >> sh) & 3 + let hb = (int(kq[kqo + 64l + l]) >> int(blk)) & 1 + dst[doff + blk * 32l + l] = (l < 16l ? dl0 : dl1) * float((q2 | (hb << 2)) - 4) + } + } +} + +//! Transcode one IQ3_S superblock (110 bytes at `bo`: f16 d, 64 qs, 8 qh, 32 signs, 4 packed +//! 4-bit scales) into the iq3s planes: [qs][qh][signs] verbatim to kq[kqo..+104), the 8 +//! sub-scales DECODED to int8 (1 + 2s) + the f16 d to ks[kso..+20) — the k4 row shape (exact). +def transcode_iq4nl_superblock(bytes : array | #; bo : int64; var kq : array; kqo : int64; var ks : array; kso : int64) { + for (blk in range64(8l)) { + ks[kso + blk * 2l] = bytes[bo + blk * 18l] + ks[kso + blk * 2l + 1l] = bytes[bo + blk * 18l + 1l] + for (i in range64(16l)) { + kq[kqo + blk * 16l + i] = bytes[bo + blk * 18l + 2l + i] + } + } +} + +//! Reference dequant of one iq4nl-plane superblock: w = d * IQ4NL_LUT[nibble], the q40 k/k+16 +//! pairing, per-32-block f16 d - ggml's dequantize_row_iq4_nl float order. +def dequant_iq4nl_plane_superblock(kq : array | #; kqo : int64; ks : array | #; kso : int64; var dst : array | #; doff : int64) { + for (blk in range64(8l)) { + let d = f16_to_f32(rd_u16(ks, kso + blk * 2l)) + for (l in range64(16l)) { + let b = int(kq[kqo + blk * 16l + l]) + dst[doff + blk * 32l + l] = d * float(int(IQ4NL_LUT[b & 15])) + dst[doff + blk * 32l + 16l + l] = d * float(int(IQ4NL_LUT[b >> 4])) + } + } +} + +def transcode_iq3xxs_superblock(bytes : array | #; bo : int64; var kq : array; kqo : int64; var ks : array; kso : int64) { + for (i in range64(96l)) { + kq[kqo + i] = bytes[bo + 2l + i] + } + let dh = f16_half_bits(uint(bytes[bo]) | (uint(bytes[bo + 1l]) << 8u)) + ks[kso] = uint8(dh & 0xFFu) + ks[kso + 1l] = uint8(dh >> 8u) + ks[kso + 2l] = uint8(0) + ks[kso + 3l] = uint8(0) + for (b in range64(8l)) { + ks[kso + 4l + b] = uint8(1 + 2 * (int(bytes[bo + 66l + b * 4l + 3l]) >> 4)) + } + for (i in range64(12l, 20l)) { + ks[kso + i] = uint8(0) + } +} + +//! Reference dequant of one iq3xxs-plane superblock: w = (d_halved * (2ls + 1)) * halved_grid +//! byte * sign — ggml's 0.25 * d * (2ls+1) * grid * sign exactly (both halvings are lossless). +//! Grid entry = one qs byte; sign byte = ksigns_iq2xs[(aux32 >> 7l) & 127], bit j (bit 7 = parity). +def dequant_iq3xxs_plane_superblock(kq : array | #; kqo : int64; ks : array | #; kso : int64; var dst : array | #; doff : int64) { + dequant_iq3xxs_plane_superblock_at(kq, kqo, ks, kso + 4l, kso, dst, doff) +} + +//! The split-layout twin: `dso` names the f16 d halfword's byte offset independently of the +//! 8-byte sub-scale strip at kso (the metal split form keeps them apart). +def dequant_iq3xxs_plane_superblock_at(kq : array | #; kqo : int64; ks : array | #; kso, dso : int64; var dst : array | #; doff : int64) { + let d = f16_to_f32(rd_u16(ks, dso)) + let grid = iq3xxs_grid() + let signs = ksigns_iq2xs() + for (b in range64(8l)) { + let db = d * float(int(int8(ks[kso + b]))) + let aux = uint(kq[kqo + 64l + b * 4l]) | (uint(kq[kqo + 64l + b * 4l + 1l]) << 8u) | (uint(kq[kqo + 64l + b * 4l + 2l]) << 16u) | (uint(kq[kqo + 64l + b * 4l + 3l]) << 24u) + for (l in range64(4l)) { + let sgn = int(signs[int((aux >> uint(7l * l)) & 127u)]) + let g1 = grid[int(kq[kqo + b * 8l + 2l * l])] + let g2 = grid[int(kq[kqo + b * 8l + 2l * l + 1l])] + for (j in range64(4l)) { + dst[doff + b * 32l + l * 8l + j] = db * float((g1 >> uint(8l * j)) & 0xFFu) * (((sgn >> int(j)) & 1) != 0 ? -1.0 : 1.0) + dst[doff + b * 32l + l * 8l + 4l + j] = db * float((g2 >> uint(8l * j)) & 0xFFu) * (((sgn >> int(4l + j)) & 1) != 0 ? -1.0 : 1.0) + } + } + } +} + +def transcode_iq3s_superblock(bytes : array | #; bo : int64; var kq : array; kqo : int64; var ks : array; kso : int64) { + for (i in range64(104l)) { + kq[kqo + i] = bytes[bo + 2l + i] + } + ks[kso] = bytes[bo] + ks[kso + 1l] = bytes[bo + 1l] + ks[kso + 2l] = uint8(0) + ks[kso + 3l] = uint8(0) + for (b in range64(8l)) { + let nib = b % 2l == 0l ? int(bytes[bo + 106l + b / 2l]) & 15 : int(bytes[bo + 106l + b / 2l]) >> 4 + ks[kso + 4l + b] = uint8(1 + 2 * nib) + } + for (i in range64(12l, 20l)) { + ks[kso + i] = uint8(0) + } +} + +//! Reference dequant of one iq3s-plane superblock: w = (d * (1 + 2s)) * grid_byte * sign, grid +//! entry qs[b*8 + 2l + half] | (qh[b] bit (2l + half) << 8), sign byte signs[b*4 + l] bit j — +//! ggml's own float order (dequantize_row_iq3_s: db per block, then db * grid * +-1). +def dequant_iq3s_plane_superblock(kq : array | #; kqo : int64; ks : array | #; kso : int64; var dst : array | #; doff : int64) { + dequant_iq3s_plane_superblock_at(kq, kqo, ks, kso + 4l, kso, dst, doff) +} + +//! The split-layout twin: `dso` names the f16 d halfword's byte offset independently of the +//! 8-byte sub-scale strip at kso (the metal split form keeps them apart). +def dequant_iq3s_plane_superblock_at(kq : array | #; kqo : int64; ks : array | #; kso, dso : int64; var dst : array | #; doff : int64) { + let d = f16_to_f32(rd_u16(ks, dso)) + let grid = iq3s_grid() + for (b in range64(8l)) { + let db = d * float(int(int8(ks[kso + b]))) + let qh = int(kq[kqo + 64l + b]) + for (l in range64(4l)) { + let sgn = int(kq[kqo + 72l + b * 4l + l]) + let g1 = grid[int(kq[kqo + b * 8l + 2l * l]) | ((qh << int(8l - 2l * l)) & 256)] + let g2 = grid[int(kq[kqo + b * 8l + 2l * l + 1l]) | ((qh << int(7l - 2l * l)) & 256)] + for (j in range64(4l)) { + dst[doff + b * 32l + l * 8l + j] = db * float((g1 >> uint(8l * j)) & 0xFFu) * (((sgn >> int(j)) & 1) != 0 ? -1.0 : 1.0) + dst[doff + b * 32l + l * 8l + 4l + j] = db * float((g2 >> uint(8l * j)) & 0xFFu) * (((sgn >> int(4l + j)) & 1) != 0 ? -1.0 : 1.0) + } + } + } +} + +//! Transcode one IQ2_S superblock (82 B at `bo`: f16 d, 32 idx + 32 sign bytes, 8 qh, 8 packed +//! scale nibbles): [idx][signs][qh] verbatim to kq[kqo..+72), d EIGHTH-ed (three exact exponent +//! decrements - ggml's (0.5+ls)*0.25 becomes (2ls+1) x d/8) + 16 strips (1+2s) to ks[kso..+20). +def transcode_iq2s_superblock(bytes : array | #; bo : int64; var kq : array; kqo : int64; var ks : array; kso : int64) { + for (i in range64(72l)) { + kq[kqo + i] = bytes[bo + 2l + i] + } + let d8 = f16_half_bits(f16_half_bits(f16_half_bits(uint(bytes[bo]) | (uint(bytes[bo + 1l]) << 8u)))) + ks[kso] = uint8(d8 & 0xFFu) + ks[kso + 1l] = uint8(d8 >> 8u) + ks[kso + 2l] = uint8(0) + ks[kso + 3l] = uint8(0) + for (g in range64(16l)) { + let nib = g % 2l == 0l ? int(bytes[bo + 74l + g / 2l]) & 15 : int(bytes[bo + 74l + g / 2l]) >> 4 + ks[kso + 4l + g] = uint8(1 + 2 * nib) + } +} + +//! Reference dequant of one iq2s-plane superblock: w = (d8 * (1 + 2s)) * grid_byte * sign - +//! grid entry idx[b*4+l] | ((qh[b] << (8-2l)) & 0x300) is TWO uint words (8 magnitudes), sign +//! byte signs[b*4+l] bit j, strip = the 16-chunk 2b + l/2. ggml's dequantize_row_iq2_s order. +def dequant_iq2s_plane_superblock(kq : array | #; kqo : int64; ks : array | #; kso : int64; var dst : array | #; doff : int64) { + dequant_iq2s_plane_superblock_at(kq, kqo, ks, kso + 4l, kso, dst, doff) +} + +//! The split-layout twin: `sso` names the 16-strip base, `dso` the f16 d8 halfword. +def dequant_iq2s_plane_superblock_at(kq : array | #; kqo : int64; ks : array | #; sso, dso : int64; var dst : array | #; doff : int64) { + let d8 = f16_to_f32(rd_u16(ks, dso)) + let grid = iq2s_grid2() + for (b in range64(8l)) { + let qh = int(kq[kqo + 64l + b]) + for (l in range64(4l)) { + let db = d8 * float(int(ks[sso + 2l * b + l / 2l])) + let sgn = int(kq[kqo + 32l + b * 4l + l]) + let idx = int(kq[kqo + b * 4l + l]) | ((qh << int(8l - 2l * l)) & 0x300) + let g1 = grid[idx * 2] + let g2 = grid[idx * 2 + 1] + for (j in range64(4l)) { + dst[doff + b * 32l + l * 8l + j] = db * float((g1 >> uint(8l * j)) & 0xFFu) * (((sgn >> int(j)) & 1) != 0 ? -1.0 : 1.0) + dst[doff + b * 32l + l * 8l + 4l + j] = db * float((g2 >> uint(8l * j)) & 0xFFu) * (((sgn >> int(4l + j)) & 1) != 0 ? -1.0 : 1.0) + } + } + } +} + +//! Transcode one IQ2_XS superblock (74 bytes at `bo`: f16 d, 32 u16 qs words, 8 packed scale +//! nibbles) into the iq2xs planes: the 64B qs region verbatim to kq[kqo..+64), d EIGHTH-ed +//! (ggml's (0.5+ls)*0.25 fold becomes (2ls+1) x d/8) + the 16 strips (1 + 2s) to ks[kso..+20). +def transcode_iq2xs_superblock(bytes : array | #; bo : int64; var kq : array; kqo : int64; var ks : array; kso : int64) { + for (i in range64(64l)) { + kq[kqo + i] = bytes[bo + 2l + i] + } + let d8 = f16_half_bits(f16_half_bits(f16_half_bits(uint(bytes[bo]) | (uint(bytes[bo + 1l]) << 8u)))) + ks[kso] = uint8(d8 & 0xFFu) + ks[kso + 1l] = uint8(d8 >> 8u) + ks[kso + 2l] = uint8(0) + ks[kso + 3l] = uint8(0) + for (g in range64(16l)) { + let nib = g % 2l == 0l ? int(bytes[bo + 66l + g / 2l]) & 15 : int(bytes[bo + 66l + g / 2l]) >> 4 + ks[kso + 4l + g] = uint8(1 + 2 * nib) + } +} + +//! Reference dequant of one iq2xs-plane superblock: w = (d8 * (1 + 2s)) * grid_byte * sign - +//! u16 word (b*4+l)'s low 9 bits pick TWO grid words (8 magnitudes), its high 7 bits pick the +//! ksigns sign byte (bit j), strip = the 16-chunk 2b + l/2. ggml's dequantize_row_iq2_xs order. +def dequant_iq2xs_plane_superblock(kq : array | #; kqo : int64; ks : array | #; kso : int64; var dst : array | #; doff : int64) { + dequant_iq2xs_plane_superblock_at(kq, kqo, ks, kso + 4l, kso, dst, doff) +} + +//! The split-layout twin: `sso` names the 16-strip base, `dso` the f16 d8 halfword. +def dequant_iq2xs_plane_superblock_at(kq : array | #; kqo : int64; ks : array | #; sso, dso : int64; var dst : array | #; doff : int64) { + let d8 = f16_to_f32(rd_u16(ks, dso)) + let grid = iq2xs_grid2() + let ksg = ksigns_iq2xs() + for (b in range64(8l)) { + for (l in range64(4l)) { + let db = d8 * float(int(ks[sso + 2l * b + l / 2l])) + let w16 = uint(kq[kqo + (b * 4l + l) * 2l]) | (uint(kq[kqo + (b * 4l + l) * 2l + 1l]) << 8u) + let sgn = int(ksg[int(w16 >> 9u)]) + let idx = int(w16 & 511u) + let g1 = grid[idx * 2] + let g2 = grid[idx * 2 + 1] + for (j in range64(4l)) { + dst[doff + b * 32l + l * 8l + j] = db * float((g1 >> uint(8l * j)) & 0xFFu) * (((sgn >> int(j)) & 1) != 0 ? -1.0 : 1.0) + dst[doff + b * 32l + l * 8l + 4l + j] = db * float((g2 >> uint(8l * j)) & 0xFFu) * (((sgn >> int(4l + j)) & 1) != 0 ? -1.0 : 1.0) + } + } + } +} + + +//! Transcode one IQ2_XXS superblock (66 bytes at `bo`: f16 d, 8 x [4 grid bytes][aux32]) into +//! the iq2xxs planes: the 64B qs region verbatim to kq[kqo..+64), d EIGHTH-ed (ggml's +//! (0.5+ls)*0.25 fold becomes (2ls+1) x d/8) + the 8 per-32 strips (1 + 2s) to ks[kso..+20). +def transcode_iq2xxs_superblock(bytes : array | #; bo : int64; var kq : array; kqo : int64; var ks : array; kso : int64) { + for (i in range64(64l)) { + kq[kqo + i] = bytes[bo + 2l + i] + } + let d8 = f16_half_bits(f16_half_bits(f16_half_bits(uint(bytes[bo]) | (uint(bytes[bo + 1l]) << 8u)))) + ks[kso] = uint8(d8 & 0xFFu) + ks[kso + 1l] = uint8(d8 >> 8u) + ks[kso + 2l] = uint8(0) + ks[kso + 3l] = uint8(0) + for (b in range64(8l)) { + ks[kso + 4l + b] = uint8(1 + 2 * (int(bytes[bo + 2l + b * 8l + 7l]) >> 4)) + } + for (i in range64(12l, 20l)) { + ks[kso + i] = uint8(0) + } +} + +//! Reference dequant of one iq2xxs-plane superblock: w = (d8 * (1 + 2s)) * grid_byte * sign - +//! block b's aux32[0] bytes pick u64 grid entries, its aux32[1] carries four 7-bit ksigns +//! indices (bit j flips element j). ggml's dequantize_row_iq2_xxs order. +def dequant_iq2xxs_plane_superblock(kq : array | #; kqo : int64; ks : array | #; kso : int64; var dst : array | #; doff : int64) { + dequant_iq2xxs_plane_superblock_at(kq, kqo, ks, kso + 4l, kso, dst, doff) +} + +//! The split-layout twin: `sso` names the 8-strip base, `dso` the f16 d8 halfword. +def dequant_iq2xxs_plane_superblock_at(kq : array | #; kqo : int64; ks : array | #; sso, dso : int64; var dst : array | #; doff : int64) { + let d8 = f16_to_f32(rd_u16(ks, dso)) + let grid = iq2xxs_grid2() + let signs = ksigns_iq2xs() + for (b in range64(8l)) { + let db = d8 * float(int(ks[sso + b])) + let aux = uint(kq[kqo + b * 8l + 4l]) | (uint(kq[kqo + b * 8l + 5l]) << 8u) | (uint(kq[kqo + b * 8l + 6l]) << 16u) | (uint(kq[kqo + b * 8l + 7l]) << 24u) + for (l in range64(4l)) { + let sgn = int(signs[int((aux >> uint(7l * l)) & 127u)]) + let idx = int(kq[kqo + b * 8l + l]) + let g1 = grid[idx * 2] + let g2 = grid[idx * 2 + 1] + for (j in range64(4l)) { + dst[doff + b * 32l + l * 8l + j] = db * float((g1 >> uint(8l * j)) & 0xFFu) * (((sgn >> int(j)) & 1) != 0 ? -1.0 : 1.0) + dst[doff + b * 32l + l * 8l + 4l + j] = db * float((g2 >> uint(8l * j)) & 0xFFu) * (((sgn >> int(4l + j)) & 1) != 0 ? -1.0 : 1.0) + } + } + } +} + //! Transcode one Q5_1 disk block (24 bytes at `bo`: d f16, m f16, qh u32, 16 nibble bytes) into //! the q51 planes — verbatim splits, exact. The per-block array form the tests drive; the bulk //! loader (gguf_transcode_q51) runs the same split pointerized and threaded. diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das index 2cd27152fc..2d13a33379 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das @@ -87,7 +87,7 @@ def private kgroup_of(p : TilePerm) : int => p.dotPrim == "smmla" ? 8 : 4 struct private TilePerm { kstep : int = 2 nrsplit : int = 4 - mr : int = GEMM_REFERENCE_MR + mr : int = GEMM_REFERENCE_MR() gkstep : int = 1 dotPrim : string = "sdot" width : int = 128 @@ -245,6 +245,12 @@ struct private TileEmit { abs_decl : LLVMOpaqueValue? // x64 sign trick: llvm.abs (VPABSB) abs_ty : LLVMOpaqueType? psign_decl : LLVMOpaqueValue? // x64 width-256 sign apply (VPSIGNB) + iq3s_panel : LLVMOpaqueValue? // iq3s/iq3xxs gemv: the per-superblock alloca panel (null = tile mode, the caller unpacked) + iq3s_grid_g : LLVMOpaqueValue? // iq3s gemv: the [512 x i32] grid global (iq3xxs: its [256 x i32] halved grid) + iq3s_smask_g : LLVMOpaqueValue? // iq3s gemv: the [16 x i32] sign-nibble byte-mask global (iq3xxs shares it) + iq3xxs_ksigns_g : LLVMOpaqueValue? // iq3xxs gemv: the [128 x i32] ksigns_iq2xs global (7-bit index -> sign byte) + gfn : LLVMOpaqueValue? // the function under emit (block creation for emitted inner loops) + gctx : LLVMOpaqueContext? psign_ty : LLVMOpaqueType? madd_decl : LLVMOpaqueValue? // x64 maddubs pair (VPMADDUBSW) madd_ty : LLVMOpaqueType? @@ -269,7 +275,7 @@ struct private TileEmit { mx4 : bool // block emitter: mx4 LUT-dequant instead of Q8 loads q51 : bool // block emitter: q51 nibble+qh unpack (32-weight blocks, // per-block f16 d/m scale pairs + xbs min-term) - kq : int // 4/5/6 = K-quant superblock emitter (0 = not kq): the + kq : int // the gemm_schema kq id (0 = not kq): the // block unit is a 256-weight superblock over the grp // kq planes; dots run unsigned-q (kq_dot_lane) kqBytes : bool // kq weights are a BYTE-EXPANDED panel (the tile's @@ -926,7 +932,7 @@ def private or_bit_x10(te : TileEmit; var w, bytes, maskv : LLVMOpaqueValue?; na return LLVMBuildOr(b, w, sel, name) } -// One 256-weight SUPERBLOCK, K-quant grp form (te.kq = 4/5/6): weight vectors unpacked +// One 256-weight SUPERBLOCK, K-quant grp form (te.kq set): weight vectors unpacked // ONCE per (sub-block, dword-group) and dotted against every token — weight-stationary, // bit-exact vs per-token GEMVs (tokCount=1). kq v2 folds integer sub-scales/bsums against Q8_K-form activations. def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f : LLVMOpaqueValue? [8]; tokBase, tokCount : int) { // nolint:STYLE037,STYLE038 — one register-coupled superblock loop nest; the k-quant decode arms share live values @@ -935,10 +941,15 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f let mr = te.interleave let w8 = te.width / 8 let k6 = te.kq == 6 + let k3 = te.kq == 3 + let k2 = te.kq == 2 + let k63 = k6 || k3 // k3 rides k6's scale row and fold; only the quant compose and the offset differ + let per16 = k63 || k2 // per-16 scale granularity: split lo/hi accumulators, unsummed bs0/bs1 + let koff = k3 ? 4 : 32 // the signed-lattice offset the bsum fold removes let q40 = te.kq == 40 - let qsb = (te.kq == 4 || q40) ? 128 : (te.kqBytes ? 256 : (te.kq == 5 ? 160 : 192)) + let qsb = (te.kq == 4 || q40) ? 128 : (k3 ? 96 : (k2 ? 64 : (te.kqBytes ? 256 : (te.kq == 5 ? 160 : 192)))) var wb = LLVMBuildMul(b, sbi, te.types->ConstI64(uint64(mr * qsb)), "wb") - var sb = LLVMBuildMul(b, sbi, te.types->ConstI64(uint64(mr * (k6 ? 18 : (q40 ? 16 : 20)))), "sb") + var sb = LLVMBuildMul(b, sbi, te.types->ConstI64(uint64(mr * (k63 ? 18 : (q40 ? 16 : 20)))), "sb") var xb = LLVMBuildMul(b, sbi, te.types->ConstI64(0x100ul), "xb") var gb = LLVMBuildShl(b, sbi, te.types->ConstI64(3ul), "gb") var maskLoElems <- [for (i in range(w8)); LLVMConstInt(te.types.t_int8, uint64(1 << (i % 4)), 0)] @@ -949,13 +960,16 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f let memBcast = te.dotKind != DOT_SDOT && te.dotKind != DOT_SMMLA let madd16 = memBcast && te.dotKind == DOT_MADDUBS var vri8 = LLVMVectorType(te.types.t_int8, uint(te.rv)) - // weight-side superblock scale rows: k4/k5 f16 d/dmin; k6 f16 d only; q40 has none (per-32 d loads live below) + // weight-side superblock scale rows: k4/k5 f16 d/dmin; k6/k3 f16 d only; q40 has none (per-32 d loads live below) var dv : LLVMOpaqueValue? [2] var mv : LLVMOpaqueValue? [2] if (!q40) { for (qd in range(rq)) { - if (k6) { + if (k63 || k2) { dv[qd] = load_f16_vec_at(te, te.sg, LLVMBuildAdd(b, sb, te.types->ConstI64(uint64(16 * mr + 2 * (qd * te.rv))), ""), "d{qd}") + if (k2) { + mv[qd] = load_f16_vec_at(te, te.sg, LLVMBuildAdd(b, sb, te.types->ConstI64(uint64(18 * mr + 2 * (qd * te.rv))), ""), "dm{qd}") + } } else { dv[qd] = load_f16_vec_at(te, te.sg, LLVMBuildAdd(b, sb, te.types->ConstI64(uint64(2 * (qd * te.rv))), ""), "d{qd}") mv[qd] = load_f16_vec_at(te, te.sg, LLVMBuildAdd(b, sb, te.types->ConstI64(uint64(2 * mr + 2 * (qd * te.rv))), ""), "dm{qd}") @@ -971,9 +985,9 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f facc[i] = LLVMConstNull(te.vnf32) } // vector bsums: sdot leg only (x64's vpbroadcastd makes scalars free; k6 needs bs0/bs1 unsummed) - let vecBsums = !memBcast && !k6 + let vecBsums = !memBcast && !per16 // mr>=8 fuses lo+hi into ONE acc (i32-exact — the fold adds them anyway); mr4 keeps split accs for ILP - let fuseAcc = !memBcast && mr >= 8 && !k6 + let fuseAcc = !memBcast && mr >= 8 && !per16 var a32v8 : LLVMOpaqueValue? [8] // [tok * 2 + blk / 4], lane blk % 4 if (vecBsums) { var xb16 = LLVMBuildShl(b, sbi, te.types->ConstI64(4ul), "xb16") @@ -1018,7 +1032,29 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f var noff = LLVMBuildAdd(b, wb, te.types->ConstI64(uint64((blk * 16 + j * 4) * mr + qd * w8)), "") var wlo : LLVMOpaqueValue? var whi : LLVMOpaqueValue? - if (te.kqBytes) { + if (k3) { + // k3: qs column (blk/4)*8 + j (lo) / + 4 (hi), 2-bit lane at shift 2*(blk%4); hmask column j / j+4, bit blk + let c0 = (blk / 4) * 8 + j + let sh = (blk % 4) * 2 + var q3l = load_vec(te, te.vwi8, te.wg, LLVMBuildAdd(b, wb, te.types->ConstI64(uint64(c0 * 4 * mr + qd * w8)), ""), "q3l{blk}_{j * rq + qd}") + var q3h = load_vec(te, te.vwi8, te.wg, LLVMBuildAdd(b, wb, te.types->ConstI64(uint64((c0 + 4) * 4 * mr + qd * w8)), ""), "q3h{blk}_{j * rq + qd}") + var h3l = load_vec(te, te.vwi8, te.wg, LLVMBuildAdd(b, wb, te.types->ConstI64(uint64((64 + j * 4) * mr + qd * w8)), ""), "h3l{blk}_{j * rq + qd}") + var h3h = load_vec(te, te.vwi8, te.wg, LLVMBuildAdd(b, wb, te.types->ConstI64(uint64((64 + (j + 4) * 4) * mr + qd * w8)), ""), "h3h{blk}_{j * rq + qd}") + var lo2 = LLVMBuildAnd(b, LLVMBuildLShr(b, q3l, splat_i8w(te, sh), ""), splat_i8w(te, 3), "") + var hi2 = LLVMBuildAnd(b, LLVMBuildLShr(b, q3h, splat_i8w(te, sh), ""), splat_i8w(te, 3), "") + var lob = LLVMBuildShl(b, LLVMBuildAnd(b, LLVMBuildLShr(b, h3l, splat_i8w(te, blk), ""), splat_i8w(te, 1), ""), splat_i8w(te, 2), "") + var hib = LLVMBuildShl(b, LLVMBuildAnd(b, LLVMBuildLShr(b, h3h, splat_i8w(te, blk), ""), splat_i8w(te, 1), ""), splat_i8w(te, 2), "") + wlo = LLVMBuildOr(b, lo2, lob, "wlo3{blk}_{j * rq + qd}") + whi = LLVMBuildOr(b, hi2, hib, "whi3{blk}_{j * rq + qd}") + } elif (k2) { + // k2: qs column (blk/4)*8 + j (lo 16) / + 4 (hi 16), 2-bit lane at shift 2*(blk%4) - k3's walk minus the hmask + let c0 = (blk / 4) * 8 + j + let sh = (blk % 4) * 2 + var q2l = load_vec(te, te.vwi8, te.wg, LLVMBuildAdd(b, wb, te.types->ConstI64(uint64(c0 * 4 * mr + qd * w8)), ""), "q2l{blk}_{j * rq + qd}") + var q2h = load_vec(te, te.vwi8, te.wg, LLVMBuildAdd(b, wb, te.types->ConstI64(uint64((c0 + 4) * 4 * mr + qd * w8)), ""), "q2h{blk}_{j * rq + qd}") + wlo = LLVMBuildAnd(b, LLVMBuildLShr(b, q2l, splat_i8w(te, sh), ""), splat_i8w(te, 3), "wlo2{blk}_{j * rq + qd}") + whi = LLVMBuildAnd(b, LLVMBuildLShr(b, q2h, splat_i8w(te, sh), ""), splat_i8w(te, 3), "whi2{blk}_{j * rq + qd}") + } elif (te.kqBytes) { var hoff2 = LLVMBuildAdd(b, noff, te.types->ConstI64(uint64(128 * mr)), "") wlo = load_vec(te, te.vwi8, te.wg, noff, "wlo{blk}_{j * rq + qd}") whi = load_vec(te, te.vwi8, te.wg, hoff2, "whi{blk}_{j * rq + qd}") @@ -1052,7 +1088,7 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f var xhi = bcast_dword(te, te.x[tokBase + i], doff1, "xd{tokBase + i}_{blk}_{j}b") if (madd16) { let k = i * rq + qd - if (k6) { + if (per16) { p16lo[k] = madd16_acc(te, p16lo[k], wlo, xlo) p16hi[k] = madd16_acc(te, p16hi[k], whi, xhi) } else { @@ -1074,12 +1110,12 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f } } } - // i16 chain flushes at the format's overflow bound: k5 after 4 madds, k6 after 2; k4/q40 run the full 8 + // i16 chain flushes at the format's overflow bound: k5 after 4 madds, k6 (and k3, conservatively) after 2; k4/q40 run the full 8 if (madd16 && (j == 1 || j == 3) && te.kq != 4 && !q40) { for (k in range(tokCount * rq)) { a0[k] = madd16_flush(te, a0[k], p16lo[k]) p16lo[k] = null - if (k6) { + if (per16) { a1[k] = madd16_flush(te, a1[k], p16hi[k]) p16hi[k] = null } @@ -1095,15 +1131,27 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f // the integer fold — no float ops per block, except q40 (its per-32 f16 d admits no cross-block integer fold) var scv : LLVMOpaqueValue? [2] var mnv : LLVMOpaqueValue? [2] + var mnlo : LLVMOpaqueValue? [2] // k2: the two groups' min nibbles (scv/mnv carry their sc nibbles) + var mnhi : LLVMOpaqueValue? [2] var dvb : LLVMOpaqueValue? [2] for (qd in range(rq)) { if (q40) { dvb[qd] = load_f16_vec_at(te, te.sg, LLVMBuildAdd(b, sb, te.types->ConstI64(uint64(blk * 2 * mr + 2 * (qd * te.rv))), ""), "d{blk}_{qd}") - } elif (k6) { + } elif (k63) { var sc0p = LLVMBuildGEP2(b, te.types.t_int8, te.sg, LLVMBuildAdd(b, sb, te.types->ConstI64(uint64(2 * blk * mr + qd * te.rv)), ""), "scp{blk}_{qd}a") var sc1p = LLVMBuildGEP2(b, te.types.t_int8, te.sg, LLVMBuildAdd(b, sb, te.types->ConstI64(uint64((2 * blk + 1) * mr + qd * te.rv)), ""), "scp{blk}_{qd}b") scv[qd] = LLVMBuildSExt(b, LLVMBuildLoad2Aligned(b, vri8, sc0p, 1u, ""), te.vni32, "sc{blk}_{qd}a") mnv[qd] = LLVMBuildSExt(b, LLVMBuildLoad2Aligned(b, vri8, sc1p, 1u, ""), te.vni32, "sc{blk}_{qd}b") + } elif (k2) { + // per-16 nibble pairs: byte 2blk = the lo group's sc|mn<<4, byte 2blk+1 = the hi group's + var b0p = LLVMBuildGEP2(b, te.types.t_int8, te.sg, LLVMBuildAdd(b, sb, te.types->ConstI64(uint64(2 * blk * mr + qd * te.rv)), ""), "k2p{blk}_{qd}a") + var b1p = LLVMBuildGEP2(b, te.types.t_int8, te.sg, LLVMBuildAdd(b, sb, te.types->ConstI64(uint64((2 * blk + 1) * mr + qd * te.rv)), ""), "k2p{blk}_{qd}b") + var z0 = LLVMBuildZExt(b, LLVMBuildLoad2Aligned(b, vri8, b0p, 1u, ""), te.vni32, "k2z{blk}_{qd}a") + var z1 = LLVMBuildZExt(b, LLVMBuildLoad2Aligned(b, vri8, b1p, 1u, ""), te.vni32, "k2z{blk}_{qd}b") + scv[qd] = LLVMBuildAnd(b, z0, splat_i32n(te, 15), "sc2{blk}_{qd}a") + mnv[qd] = LLVMBuildAnd(b, z1, splat_i32n(te, 15), "sc2{blk}_{qd}b") + mnlo[qd] = LLVMBuildLShr(b, z0, splat_i32n(te, 4), "mn2{blk}_{qd}a") + mnhi[qd] = LLVMBuildLShr(b, z1, splat_i32n(te, 4), "mn2{blk}_{qd}b") } else { var scp = LLVMBuildGEP2(b, te.types.t_int8, te.sg, LLVMBuildAdd(b, sb, te.types->ConstI64(uint64(4 * mr + blk * mr + qd * te.rv)), ""), "scp{blk}_{qd}") scv[qd] = LLVMBuildZExt(b, LLVMBuildLoad2Aligned(b, vri8, scp, 1u, ""), te.vni32, "sc{blk}_{qd}") @@ -1128,7 +1176,7 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f bs0 = LLVMBuildLoad2Aligned(b, te.types.t_int32, bp0, 4u, "bs{tk}_{blk}a") var bp1 = LLVMBuildGEP2(b, te.types.t_int32, te.xbs[tk], bi2b, "bp{tk}_{blk}b") bs1 = LLVMBuildLoad2Aligned(b, te.types.t_int32, bp1, 4u, "bs{tk}_{blk}b") - if (!k6) { + if (!per16) { a32v = splat_i32(te, LLVMBuildAdd(b, bs0, bs1, "a32{tk}_{blk}"), "a32v{tk}_{blk}") } } @@ -1138,7 +1186,7 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f var isub = LLVMBuildSub(b, acc, LLVMBuildMul(b, a32v, splat_i32n(te, 8), ""), "is{tk}_{blk}_{qd}") facc[i * rq + qd] = fold_fma(te, LLVMBuildSIToFP(b, isub, te.vnf32, ""), dvb[qd], facc[i * rq + qd], "fa{tk}_{blk}_{qd}") } - } elif (k6) { + } elif (k63) { var bs0v = splat_i32(te, bs0, "bs0v{tk}_{blk}") var bs1v = splat_i32(te, bs1, "bs1v{tk}_{blk}") for (qd in range(rq)) { @@ -1147,6 +1195,16 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f iacc[i * rq + qd] = LLVMBuildAdd(b, iacc[i * rq + qd], ia, "ia{tk}_{blk}_{qd}") bacc[i * rq + qd] = LLVMBuildAdd(b, bacc[i * rq + qd], ba, "ba{tk}_{blk}_{qd}") } + } elif (k2) { + // ia = sc_lo*dot_lo + sc_hi*dot_hi; ba = mn_lo*bs0 + mn_hi*bs1 - the k4/k5 epilogue folds bacc on dmin + var bs0v = splat_i32(te, bs0, "bs0v{tk}_{blk}") + var bs1v = splat_i32(te, bs1, "bs1v{tk}_{blk}") + for (qd in range(rq)) { + var ia = LLVMBuildAdd(b, LLVMBuildMul(b, scv[qd], a0[i * rq + qd], ""), LLVMBuildMul(b, mnv[qd], a1[i * rq + qd], ""), "ia2{tk}_{blk}_{qd}") + var ba = LLVMBuildAdd(b, LLVMBuildMul(b, mnlo[qd], bs0v, ""), LLVMBuildMul(b, mnhi[qd], bs1v, ""), "ba2{tk}_{blk}_{qd}") + iacc[i * rq + qd] = LLVMBuildAdd(b, iacc[i * rq + qd], ia, "ia{tk}_{blk}_{qd}k2") + bacc[i * rq + qd] = LLVMBuildAdd(b, bacc[i * rq + qd], ba, "ba{tk}_{blk}_{qd}k2") + } } else { for (qd in range(rq)) { var acc = fuseAcc ? a0[i * rq + qd] : LLVMBuildAdd(b, a0[i * rq + qd], a1[i * rq + qd], "acc{tk}_{blk}_{qd}") @@ -1171,7 +1229,7 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f } } } - // superblock fold: k4/k5 iacc·d8 − bacc·dmin·d8; k6 (iacc−32·bacc)·d·d8 exact int32; q40 facc·d8 + // superblock fold: k4/k5 iacc·d8 − bacc·dmin·d8; k6/k3 (iacc−off·bacc)·d·d8 exact int32 (off 32 / 4); q40 facc·d8 for (i in range(tokCount)) { let tk = tokBase + i var qp = LLVMBuildGEP2(b, te.types.t_float, te.xs[tk], sbi, "qp{tk}") @@ -1183,8 +1241,8 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f continue } var ds = LLVMBuildFMul(b, dv[qd], d8v, "ds{tk}_{qd}") - if (k6) { - var isub = LLVMBuildSub(b, iacc[i * rq + qd], LLVMBuildMul(b, bacc[i * rq + qd], splat_i32n(te, 32), ""), "is{tk}_{qd}") + if (k63) { + var isub = LLVMBuildSub(b, iacc[i * rq + qd], LLVMBuildMul(b, bacc[i * rq + qd], splat_i32n(te, koff), ""), "is{tk}_{qd}") f[i * rq + qd] = fold_fma(te, LLVMBuildSIToFP(b, isub, te.vnf32, ""), ds, f[i * rq + qd], "f{tk}_{qd}s") } else { var ms = LLVMBuildFNeg(b, LLVMBuildFMul(b, mv[qd], d8v, ""), "ms{tk}_{qd}") @@ -1195,11 +1253,712 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f } } +// The iq3s gemv gather globals: the grid as a [512 x i32] private constant, the 4-bit sign +// nibble expanded to a byte mask via a [16 x i32] table (negation is (w ^ m) + (m & 0x01010101); +// grid magnitudes are odd 1..15, so no carry crosses a byte lane). +def private iq3s_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { + var grid = LLVMGetNamedGlobal(mod, "dasllama.iq3s.grid") + if (grid == null) { + let g = iq3s_grid() + var elems : array + elems |> reserve(512) + for (i in range(512)) { + elems |> push(LLVMConstInt(te.types.t_int32, uint64(g[i]), 0)) + } + grid = LLVMAddGlobal(mod, LLVMArrayType(te.types.t_int32, 512u), "dasllama.iq3s.grid") + LLVMSetInitializer(grid, LLVMConstArray(te.types.t_int32, array_data_ptr(elems), 512u)) + LLVMSetGlobalConstant(grid, 1) + LLVMSetLinkage(grid, LLVMLinkage.LLVMPrivateLinkage) + LLVMSetAlignment(grid, 64u) + } + te.iq3s_grid_g = grid + var smask = LLVMGetNamedGlobal(mod, "dasllama.iq3s.smask") + if (smask == null) { + var elems2 : array + elems2 |> reserve(16) + for (m in range(16)) { + var v = 0ul + for (t in range(4)) { + if ((m & (1 << t)) != 0) { + v |= 0xFFul << uint64(8 * t) + } + } + elems2 |> push(LLVMConstInt(te.types.t_int32, v, 0)) + } + smask = LLVMAddGlobal(mod, LLVMArrayType(te.types.t_int32, 16u), "dasllama.iq3s.smask") + LLVMSetInitializer(smask, LLVMConstArray(te.types.t_int32, array_data_ptr(elems2), 16u)) + LLVMSetGlobalConstant(smask, 1) + LLVMSetLinkage(smask, LLVMLinkage.LLVMPrivateLinkage) + LLVMSetAlignment(smask, 16u) + } + te.iq3s_smask_g = smask +} + +// iq3s gemv: gather superblock `sbi`'s grid words + signs off the PACKED grp planes into the +// alloca panel — one emitted loop over the mr rows (32 grid words unrolled per iteration); a +// single straight-line loop block, so the phi's back edge is the block itself. +def private emit_iq3s_gather(var te : TileEmit; var sbi : LLVMOpaqueValue?) { + let b = te.builder + let mr = te.interleave + var wbP = LLVMBuildMul(b, sbi, te.types->ConstI64(uint64(mr * 104)), "iq3s.wb") + var pre = LLVMGetInsertBlock(b) + var head = LLVMAppendBasicBlockInContext(te.gctx, te.gfn, "iq3s.gath") + var cont = LLVMAppendBasicBlockInContext(te.gctx, te.gfn, "iq3s.cont") + LLVMBuildBr(b, head) + LLVMPositionBuilderAtEnd(b, head) + var rPhi = LLVMBuildPhi(b, te.types.t_int64, "iq3s.r") + var preVals <- [te.types->ConstI64(0ul)] + var preBlocks <- [pre] + LLVMAddIncoming(rPhi, preVals, preBlocks) + var rT4 = LLVMBuildMul(b, rPhi, te.types->ConstI64(4ul), "iq3s.r4") + var base = LLVMBuildAdd(b, wbP, rT4, "iq3s.base") + let c256 = LLVMConstInt(te.types.t_int32, 256ul, 0) + let c15 = LLVMConstInt(te.types.t_int32, 15ul, 0) + let cneg = LLVMConstInt(te.types.t_int32, 0x01010101ul, 0) + for (blk in range(8)) { + let hp = 64 + blk + var qhp = LLVMBuildGEP2(b, te.types.t_int8, te.wg, LLVMBuildAdd(b, base, te.types->ConstI64(uint64((hp / 4) * mr * 4 + hp % 4)), ""), "") + var qh = LLVMBuildZExt(b, LLVMBuildLoad2Aligned(b, te.types.t_int8, qhp, 1u, ""), te.types.t_int32, "iq3s.qh{blk}") + for (l in range(4)) { + let sp = 72 + blk * 4 + l + var sgp = LLVMBuildGEP2(b, te.types.t_int8, te.wg, LLVMBuildAdd(b, base, te.types->ConstI64(uint64((sp / 4) * mr * 4 + sp % 4)), ""), "") + var sgn = LLVMBuildZExt(b, LLVMBuildLoad2Aligned(b, te.types.t_int8, sgp, 1u, ""), te.types.t_int32, "iq3s.sg{blk}_{l}") + var iv : LLVMOpaqueValue? [2] + for (h in range(2)) { + let qp1 = blk * 8 + 2 * l + h + var qpp = LLVMBuildGEP2(b, te.types.t_int8, te.wg, LLVMBuildAdd(b, base, te.types->ConstI64(uint64((qp1 / 4) * mr * 4 + qp1 % 4)), ""), "") + var qv = LLVMBuildZExt(b, LLVMBuildLoad2Aligned(b, te.types.t_int8, qpp, 1u, ""), te.types.t_int32, "") + var hb = LLVMBuildAnd(b, LLVMBuildShl(b, qh, LLVMConstInt(te.types.t_int32, uint64(8 - h - 2 * l), 0), ""), c256, "") + iv[h] = LLVMBuildOr(b, qv, hb, "iq3s.i{blk}_{l}_{h}") + } + var mv : LLVMOpaqueValue? [2] + mv[0] = LLVMBuildAnd(b, sgn, c15, "") + mv[1] = LLVMBuildAnd(b, LLVMBuildLShr(b, sgn, LLVMConstInt(te.types.t_int32, 4ul, 0), ""), c15, "") + for (h in range(2)) { + var gp = LLVMBuildGEP2(b, te.types.t_int32, te.iq3s_grid_g, iv[h], "") + var gw = LLVMBuildLoad2Aligned(b, te.types.t_int32, gp, 4u, "iq3s.g{blk}_{l}_{h}") + var mp = LLVMBuildGEP2(b, te.types.t_int32, te.iq3s_smask_g, mv[h], "") + var mw = LLVMBuildLoad2Aligned(b, te.types.t_int32, mp, 4u, "") + var w = LLVMBuildAdd(b, LLVMBuildXor(b, gw, mw, ""), LLVMBuildAnd(b, mw, cneg, ""), "iq3s.w{blk}_{l}_{h}") + let e = l * 8 + h * 4 + let dd = (e < 16 ? 0 : 128 * mr) + (blk * 4 + (e % 16) / 4) * mr * 4 + var dp = LLVMBuildGEP2(b, te.types.t_int8, te.iq3s_panel, LLVMBuildAdd(b, rT4, te.types->ConstI64(uint64(dd)), ""), "") + LLVMBuildStore(b, w, dp) + } + } + } + var rNext = LLVMBuildAdd(b, rPhi, te.types->ConstI64(1ul), "iq3s.rn") + var more = LLVMBuildICmp(b, LLVMIntPredicate.LLVMIntSLT, rNext, te.types->ConstI64(uint64(mr)), "iq3s.more") + LLVMBuildCondBr(b, more, head, cont) + var loopVals <- [rNext] + var loopBlocks <- [head] + LLVMAddIncoming(rPhi, loopVals, loopBlocks) + LLVMPositionBuilderAtEnd(b, cont) +} + +// The iq3xxs gemv gather globals: the HALVED grid as a [256 x i32] private constant, ksigns +// as a [128 x i32] (7-bit index -> sign byte, bit 7 = parity), plus the shared smask table +// (the same nibble -> byte-mask expansion; magnitudes 2..31 are nonzero, so no carry). +def private iq3xxs_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { + var grid = LLVMGetNamedGlobal(mod, "dasllama.iq3xxs.grid") + if (grid == null) { + let g = iq3xxs_grid() + var elems : array + elems |> reserve(256) + for (i in range(256)) { + elems |> push(LLVMConstInt(te.types.t_int32, uint64(g[i]), 0)) + } + grid = LLVMAddGlobal(mod, LLVMArrayType(te.types.t_int32, 256u), "dasllama.iq3xxs.grid") + LLVMSetInitializer(grid, LLVMConstArray(te.types.t_int32, array_data_ptr(elems), 256u)) + LLVMSetGlobalConstant(grid, 1) + LLVMSetLinkage(grid, LLVMLinkage.LLVMPrivateLinkage) + LLVMSetAlignment(grid, 64u) + } + te.iq3s_grid_g = grid + var ksg = LLVMGetNamedGlobal(mod, "dasllama.iq3xxs.ksigns") + if (ksg == null) { + let kt = ksigns_iq2xs() + var elems3 : array + elems3 |> reserve(128) + for (i in range(128)) { + elems3 |> push(LLVMConstInt(te.types.t_int32, uint64(kt[i]), 0)) + } + ksg = LLVMAddGlobal(mod, LLVMArrayType(te.types.t_int32, 128u), "dasllama.iq3xxs.ksigns") + LLVMSetInitializer(ksg, LLVMConstArray(te.types.t_int32, array_data_ptr(elems3), 128u)) + LLVMSetGlobalConstant(ksg, 1) + LLVMSetLinkage(ksg, LLVMLinkage.LLVMPrivateLinkage) + LLVMSetAlignment(ksg, 16u) + } + te.iq3xxs_ksigns_g = ksg + var smask = LLVMGetNamedGlobal(mod, "dasllama.iq3s.smask") + if (smask == null) { + var elems2 : array + elems2 |> reserve(16) + for (m in range(16)) { + var v = 0ul + for (t in range(4)) { + if ((m & (1 << t)) != 0) { + v |= 0xFFul << uint64(8 * t) + } + } + elems2 |> push(LLVMConstInt(te.types.t_int32, v, 0)) + } + smask = LLVMAddGlobal(mod, LLVMArrayType(te.types.t_int32, 16u), "dasllama.iq3s.smask") + LLVMSetInitializer(smask, LLVMConstArray(te.types.t_int32, array_data_ptr(elems2), 16u)) + LLVMSetGlobalConstant(smask, 1) + LLVMSetLinkage(smask, LLVMLinkage.LLVMPrivateLinkage) + LLVMSetAlignment(smask, 16u) + } + te.iq3s_smask_g = smask +} + +// iq3xxs gemv: gather superblock `sbi` off the PACKED grp planes into the alloca panel - the +// iq3s gather's shape with the aux32 sign word (ONE aligned i32 column load per block) and +// ksigns lookups in place of the explicit sign bytes, and no 9th grid-index bit. +def private emit_iq3xxs_gather(var te : TileEmit; var sbi : LLVMOpaqueValue?) { + let b = te.builder + let mr = te.interleave + var wbP = LLVMBuildMul(b, sbi, te.types->ConstI64(uint64(mr * 96)), "iq3x.wb") + var pre = LLVMGetInsertBlock(b) + var head = LLVMAppendBasicBlockInContext(te.gctx, te.gfn, "iq3x.gath") + var cont = LLVMAppendBasicBlockInContext(te.gctx, te.gfn, "iq3x.cont") + LLVMBuildBr(b, head) + LLVMPositionBuilderAtEnd(b, head) + var rPhi = LLVMBuildPhi(b, te.types.t_int64, "iq3x.r") + var preVals <- [te.types->ConstI64(0ul)] + var preBlocks <- [pre] + LLVMAddIncoming(rPhi, preVals, preBlocks) + var rT4 = LLVMBuildMul(b, rPhi, te.types->ConstI64(4ul), "iq3x.r4") + var base = LLVMBuildAdd(b, wbP, rT4, "iq3x.base") + let c15 = LLVMConstInt(te.types.t_int32, 15ul, 0) + let c127 = LLVMConstInt(te.types.t_int32, 127ul, 0) + let cneg = LLVMConstInt(te.types.t_int32, 0x01010101ul, 0) + for (blk in range(8)) { + var axp = LLVMBuildGEP2(b, te.types.t_int8, te.wg, LLVMBuildAdd(b, base, te.types->ConstI64(uint64((16 + blk) * mr * 4)), ""), "") + var aux = LLVMBuildLoad2Aligned(b, te.types.t_int32, axp, 4u, "iq3x.aux{blk}") + for (l in range(4)) { + var sidx = LLVMBuildAnd(b, l == 0 ? aux : LLVMBuildLShr(b, aux, LLVMConstInt(te.types.t_int32, uint64(7 * l), 0), ""), c127, "") + var ksp2 = LLVMBuildGEP2(b, te.types.t_int32, te.iq3xxs_ksigns_g, sidx, "") + var sgn = LLVMBuildLoad2Aligned(b, te.types.t_int32, ksp2, 4u, "iq3x.sg{blk}_{l}") + var iv : LLVMOpaqueValue? [2] + for (h in range(2)) { + let qp1 = blk * 8 + 2 * l + h + var qpp = LLVMBuildGEP2(b, te.types.t_int8, te.wg, LLVMBuildAdd(b, base, te.types->ConstI64(uint64((qp1 / 4) * mr * 4 + qp1 % 4)), ""), "") + iv[h] = LLVMBuildZExt(b, LLVMBuildLoad2Aligned(b, te.types.t_int8, qpp, 1u, ""), te.types.t_int32, "iq3x.i{blk}_{l}_{h}") + } + var mv : LLVMOpaqueValue? [2] + mv[0] = LLVMBuildAnd(b, sgn, c15, "") + mv[1] = LLVMBuildAnd(b, LLVMBuildLShr(b, sgn, LLVMConstInt(te.types.t_int32, 4ul, 0), ""), c15, "") + for (h in range(2)) { + var gp = LLVMBuildGEP2(b, te.types.t_int32, te.iq3s_grid_g, iv[h], "") + var gw = LLVMBuildLoad2Aligned(b, te.types.t_int32, gp, 4u, "iq3x.g{blk}_{l}_{h}") + var mp = LLVMBuildGEP2(b, te.types.t_int32, te.iq3s_smask_g, mv[h], "") + var mw = LLVMBuildLoad2Aligned(b, te.types.t_int32, mp, 4u, "") + var w = LLVMBuildAdd(b, LLVMBuildXor(b, gw, mw, ""), LLVMBuildAnd(b, mw, cneg, ""), "iq3x.w{blk}_{l}_{h}") + let e = l * 8 + h * 4 + let dd = (e < 16 ? 0 : 128 * mr) + (blk * 4 + (e % 16) / 4) * mr * 4 + var dp = LLVMBuildGEP2(b, te.types.t_int8, te.iq3s_panel, LLVMBuildAdd(b, rT4, te.types->ConstI64(uint64(dd)), ""), "") + LLVMBuildStore(b, w, dp) + } + } + } + var rNext = LLVMBuildAdd(b, rPhi, te.types->ConstI64(1ul), "iq3x.rn") + var more = LLVMBuildICmp(b, LLVMIntPredicate.LLVMIntSLT, rNext, te.types->ConstI64(uint64(mr)), "iq3x.more") + LLVMBuildCondBr(b, more, head, cont) + var loopVals <- [rNext] + var loopBlocks <- [head] + LLVMAddIncoming(rPhi, loopVals, loopBlocks) + LLVMPositionBuilderAtEnd(b, cont) +} + +// The iq2s gemv gather globals: the u64 grid as a [2048 x i32] private constant (1024 entries +// as low/high word pairs), plus the shared smask table (magnitudes {8, 25, 43} are nonzero - +// no carry crosses a byte lane on negation). +def private iq2s_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { + var grid = LLVMGetNamedGlobal(mod, "dasllama.iq2s.grid") + if (grid == null) { + let g = iq2s_grid2() + var elems : array + elems |> reserve(2048) + for (i in range(2048)) { + elems |> push(LLVMConstInt(te.types.t_int32, uint64(g[i]), 0)) + } + grid = LLVMAddGlobal(mod, LLVMArrayType(te.types.t_int32, 2048u), "dasllama.iq2s.grid") + LLVMSetInitializer(grid, LLVMConstArray(te.types.t_int32, array_data_ptr(elems), 2048u)) + LLVMSetGlobalConstant(grid, 1) + LLVMSetLinkage(grid, LLVMLinkage.LLVMPrivateLinkage) + LLVMSetAlignment(grid, 64u) + } + te.iq3s_grid_g = grid + var smask = LLVMGetNamedGlobal(mod, "dasllama.iq3s.smask") + if (smask == null) { + var elems2 : array + elems2 |> reserve(16) + for (m in range(16)) { + var v = 0ul + for (t in range(4)) { + if ((m & (1 << t)) != 0) { + v |= 0xFFul << uint64(8 * t) + } + } + elems2 |> push(LLVMConstInt(te.types.t_int32, v, 0)) + } + smask = LLVMAddGlobal(mod, LLVMArrayType(te.types.t_int32, 16u), "dasllama.iq3s.smask") + LLVMSetInitializer(smask, LLVMConstArray(te.types.t_int32, array_data_ptr(elems2), 16u)) + LLVMSetGlobalConstant(smask, 1) + LLVMSetLinkage(smask, LLVMLinkage.LLVMPrivateLinkage) + LLVMSetAlignment(smask, 16u) + } + te.iq3s_smask_g = smask +} + +// iq2s gemv: gather superblock `sbi` off the PACKED grp planes into the alloca panel - the +// iq3s gather's shape with the 10-bit index (qs byte | qh's 0x300 window) doubled into the +// u64 grid's low/high word pair; the sign byte is explicit like iq3s's. +def private emit_iq2s_gather(var te : TileEmit; var sbi : LLVMOpaqueValue?) { + let b = te.builder + let mr = te.interleave + var wbP = LLVMBuildMul(b, sbi, te.types->ConstI64(uint64(mr * 72)), "iq2s.wb") + var pre = LLVMGetInsertBlock(b) + var head = LLVMAppendBasicBlockInContext(te.gctx, te.gfn, "iq2s.gath") + var cont = LLVMAppendBasicBlockInContext(te.gctx, te.gfn, "iq2s.cont") + LLVMBuildBr(b, head) + LLVMPositionBuilderAtEnd(b, head) + var rPhi = LLVMBuildPhi(b, te.types.t_int64, "iq2s.r") + var preVals <- [te.types->ConstI64(0ul)] + var preBlocks <- [pre] + LLVMAddIncoming(rPhi, preVals, preBlocks) + var rT4 = LLVMBuildMul(b, rPhi, te.types->ConstI64(4ul), "iq2s.r4") + var base = LLVMBuildAdd(b, wbP, rT4, "iq2s.base") + let c768 = LLVMConstInt(te.types.t_int32, 0x300ul, 0) + let c15 = LLVMConstInt(te.types.t_int32, 15ul, 0) + let cneg = LLVMConstInt(te.types.t_int32, 0x01010101ul, 0) + for (blk in range(8)) { + let hp = 64 + blk + var qhp = LLVMBuildGEP2(b, te.types.t_int8, te.wg, LLVMBuildAdd(b, base, te.types->ConstI64(uint64((hp / 4) * mr * 4 + hp % 4)), ""), "") + var qh = LLVMBuildZExt(b, LLVMBuildLoad2Aligned(b, te.types.t_int8, qhp, 1u, ""), te.types.t_int32, "iq2s.qh{blk}") + for (l in range(4)) { + let sp = 32 + blk * 4 + l + var sgp = LLVMBuildGEP2(b, te.types.t_int8, te.wg, LLVMBuildAdd(b, base, te.types->ConstI64(uint64((sp / 4) * mr * 4 + sp % 4)), ""), "") + var sgn = LLVMBuildZExt(b, LLVMBuildLoad2Aligned(b, te.types.t_int8, sgp, 1u, ""), te.types.t_int32, "iq2s.sg{blk}_{l}") + let ip = blk * 4 + l + var ipp = LLVMBuildGEP2(b, te.types.t_int8, te.wg, LLVMBuildAdd(b, base, te.types->ConstI64(uint64((ip / 4) * mr * 4 + ip % 4)), ""), "") + var iv0 = LLVMBuildZExt(b, LLVMBuildLoad2Aligned(b, te.types.t_int8, ipp, 1u, ""), te.types.t_int32, "") + var hb = LLVMBuildAnd(b, LLVMBuildShl(b, qh, LLVMConstInt(te.types.t_int32, uint64(8 - 2 * l), 0), ""), c768, "") + var idx = LLVMBuildOr(b, iv0, hb, "iq2s.i{blk}_{l}") + var idx2 = LLVMBuildShl(b, idx, LLVMConstInt(te.types.t_int32, 1ul, 0), "") + var mv : LLVMOpaqueValue? [2] + mv[0] = LLVMBuildAnd(b, sgn, c15, "") + mv[1] = LLVMBuildAnd(b, LLVMBuildLShr(b, sgn, LLVMConstInt(te.types.t_int32, 4ul, 0), ""), c15, "") + for (h in range(2)) { + var gi = h == 0 ? idx2 : LLVMBuildAdd(b, idx2, LLVMConstInt(te.types.t_int32, 1ul, 0), "") + var gp = LLVMBuildGEP2(b, te.types.t_int32, te.iq3s_grid_g, gi, "") + var gw = LLVMBuildLoad2Aligned(b, te.types.t_int32, gp, 4u, "iq2s.g{blk}_{l}_{h}") + var mp = LLVMBuildGEP2(b, te.types.t_int32, te.iq3s_smask_g, mv[h], "") + var mw = LLVMBuildLoad2Aligned(b, te.types.t_int32, mp, 4u, "") + var w = LLVMBuildAdd(b, LLVMBuildXor(b, gw, mw, ""), LLVMBuildAnd(b, mw, cneg, ""), "iq2s.w{blk}_{l}_{h}") + let e = l * 8 + h * 4 + let dd = (e < 16 ? 0 : 128 * mr) + (blk * 4 + (e % 16) / 4) * mr * 4 + var dp = LLVMBuildGEP2(b, te.types.t_int8, te.iq3s_panel, LLVMBuildAdd(b, rT4, te.types->ConstI64(uint64(dd)), ""), "") + LLVMBuildStore(b, w, dp) + } + } + } + var rNext = LLVMBuildAdd(b, rPhi, te.types->ConstI64(1ul), "iq2s.rn") + var more = LLVMBuildICmp(b, LLVMIntPredicate.LLVMIntSLT, rNext, te.types->ConstI64(uint64(mr)), "iq2s.more") + LLVMBuildCondBr(b, more, head, cont) + var loopVals <- [rNext] + var loopBlocks <- [head] + LLVMAddIncoming(rPhi, loopVals, loopBlocks) + LLVMPositionBuilderAtEnd(b, cont) +} + +// The iq2xs gemv gather globals: the u64 grid as a [1024 x i32] private constant (512 entries +// as low/high word pairs), the shared ksigns table, and the shared smask. +def private iq2xs_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { + var grid = LLVMGetNamedGlobal(mod, "dasllama.iq2xs.grid") + if (grid == null) { + let g = iq2xs_grid2() + var elems : array + elems |> reserve(1024) + for (i in range(1024)) { + elems |> push(LLVMConstInt(te.types.t_int32, uint64(g[i]), 0)) + } + grid = LLVMAddGlobal(mod, LLVMArrayType(te.types.t_int32, 1024u), "dasllama.iq2xs.grid") + LLVMSetInitializer(grid, LLVMConstArray(te.types.t_int32, array_data_ptr(elems), 1024u)) + LLVMSetGlobalConstant(grid, 1) + LLVMSetLinkage(grid, LLVMLinkage.LLVMPrivateLinkage) + LLVMSetAlignment(grid, 64u) + } + te.iq3s_grid_g = grid + var ksg = LLVMGetNamedGlobal(mod, "dasllama.iq3xxs.ksigns") + if (ksg == null) { + let kt = ksigns_iq2xs() + var elems3 : array + elems3 |> reserve(128) + for (i in range(128)) { + elems3 |> push(LLVMConstInt(te.types.t_int32, uint64(kt[i]), 0)) + } + ksg = LLVMAddGlobal(mod, LLVMArrayType(te.types.t_int32, 128u), "dasllama.iq3xxs.ksigns") + LLVMSetInitializer(ksg, LLVMConstArray(te.types.t_int32, array_data_ptr(elems3), 128u)) + LLVMSetGlobalConstant(ksg, 1) + LLVMSetLinkage(ksg, LLVMLinkage.LLVMPrivateLinkage) + LLVMSetAlignment(ksg, 16u) + } + te.iq3xxs_ksigns_g = ksg + var smask = LLVMGetNamedGlobal(mod, "dasllama.iq3s.smask") + if (smask == null) { + var elems2 : array + elems2 |> reserve(16) + for (m in range(16)) { + var v = 0ul + for (t in range(4)) { + if ((m & (1 << t)) != 0) { + v |= 0xFFul << uint64(8 * t) + } + } + elems2 |> push(LLVMConstInt(te.types.t_int32, v, 0)) + } + smask = LLVMAddGlobal(mod, LLVMArrayType(te.types.t_int32, 16u), "dasllama.iq3s.smask") + LLVMSetInitializer(smask, LLVMConstArray(te.types.t_int32, array_data_ptr(elems2), 16u)) + LLVMSetGlobalConstant(smask, 1) + LLVMSetLinkage(smask, LLVMLinkage.LLVMPrivateLinkage) + LLVMSetAlignment(smask, 16u) + } + te.iq3s_smask_g = smask +} + +// The iq2xxs gemv gather globals: the u64 grid as a [512 x i32] private constant (256 entries +// as low/high word pairs), the shared ksigns table, and the shared smask. +def private iq2xxs_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { + var grid = LLVMGetNamedGlobal(mod, "dasllama.iq2xxs.grid") + if (grid == null) { + let g = iq2xxs_grid2() + var elems : array + elems |> reserve(512) + for (i in range(512)) { + elems |> push(LLVMConstInt(te.types.t_int32, uint64(g[i]), 0)) + } + grid = LLVMAddGlobal(mod, LLVMArrayType(te.types.t_int32, 512u), "dasllama.iq2xxs.grid") + LLVMSetInitializer(grid, LLVMConstArray(te.types.t_int32, array_data_ptr(elems), 512u)) + LLVMSetGlobalConstant(grid, 1) + LLVMSetLinkage(grid, LLVMLinkage.LLVMPrivateLinkage) + LLVMSetAlignment(grid, 64u) + } + te.iq3s_grid_g = grid + var ksg = LLVMGetNamedGlobal(mod, "dasllama.iq3xxs.ksigns") + if (ksg == null) { + let kt = ksigns_iq2xs() + var elems3 : array + elems3 |> reserve(128) + for (i in range(128)) { + elems3 |> push(LLVMConstInt(te.types.t_int32, uint64(kt[i]), 0)) + } + ksg = LLVMAddGlobal(mod, LLVMArrayType(te.types.t_int32, 128u), "dasllama.iq3xxs.ksigns") + LLVMSetInitializer(ksg, LLVMConstArray(te.types.t_int32, array_data_ptr(elems3), 128u)) + LLVMSetGlobalConstant(ksg, 1) + LLVMSetLinkage(ksg, LLVMLinkage.LLVMPrivateLinkage) + LLVMSetAlignment(ksg, 16u) + } + te.iq3xxs_ksigns_g = ksg + var smask = LLVMGetNamedGlobal(mod, "dasllama.iq3s.smask") + if (smask == null) { + var elems2 : array + elems2 |> reserve(16) + for (m in range(16)) { + var v = 0ul + for (t in range(4)) { + if ((m & (1 << t)) != 0) { + v |= 0xFFul << uint64(8 * t) + } + } + elems2 |> push(LLVMConstInt(te.types.t_int32, v, 0)) + } + smask = LLVMAddGlobal(mod, LLVMArrayType(te.types.t_int32, 16u), "dasllama.iq3s.smask") + LLVMSetInitializer(smask, LLVMConstArray(te.types.t_int32, array_data_ptr(elems2), 16u)) + LLVMSetGlobalConstant(smask, 1) + LLVMSetLinkage(smask, LLVMLinkage.LLVMPrivateLinkage) + LLVMSetAlignment(smask, 16u) + } + te.iq3s_smask_g = smask +} + +// iq2xs gemv: gather superblock `sbi` off the PACKED grp planes into the alloca panel - the +// iq2s gather's two-word u64 form with the u16 qs word's 9-bit index and its high-7-bit +// KSIGNS lookup in place of explicit sign bytes (the iq3xxs sign path). +def private emit_iq2xs_gather(var te : TileEmit; var sbi : LLVMOpaqueValue?) { + let b = te.builder + let mr = te.interleave + var wbP = LLVMBuildMul(b, sbi, te.types->ConstI64(uint64(mr * 64)), "iq2x.wb") + var pre = LLVMGetInsertBlock(b) + var head = LLVMAppendBasicBlockInContext(te.gctx, te.gfn, "iq2x.gath") + var cont = LLVMAppendBasicBlockInContext(te.gctx, te.gfn, "iq2x.cont") + LLVMBuildBr(b, head) + LLVMPositionBuilderAtEnd(b, head) + var rPhi = LLVMBuildPhi(b, te.types.t_int64, "iq2x.r") + var preVals <- [te.types->ConstI64(0ul)] + var preBlocks <- [pre] + LLVMAddIncoming(rPhi, preVals, preBlocks) + var rT4 = LLVMBuildMul(b, rPhi, te.types->ConstI64(4ul), "iq2x.r4") + var base = LLVMBuildAdd(b, wbP, rT4, "iq2x.base") + let c511 = LLVMConstInt(te.types.t_int32, 511ul, 0) + let c15 = LLVMConstInt(te.types.t_int32, 15ul, 0) + let cneg = LLVMConstInt(te.types.t_int32, 0x01010101ul, 0) + for (blk in range(8)) { + for (l in range(4)) { + let lp = (blk * 4 + l) * 2 + var qlp = LLVMBuildGEP2(b, te.types.t_int8, te.wg, LLVMBuildAdd(b, base, te.types->ConstI64(uint64((lp / 4) * mr * 4 + lp % 4)), ""), "") + var qlo = LLVMBuildZExt(b, LLVMBuildLoad2Aligned(b, te.types.t_int8, qlp, 1u, ""), te.types.t_int32, "") + var qhp = LLVMBuildGEP2(b, te.types.t_int8, te.wg, LLVMBuildAdd(b, base, te.types->ConstI64(uint64((lp / 4) * mr * 4 + lp % 4 + 1)), ""), "") + var qhi = LLVMBuildZExt(b, LLVMBuildLoad2Aligned(b, te.types.t_int8, qhp, 1u, ""), te.types.t_int32, "") + var w16 = LLVMBuildOr(b, qlo, LLVMBuildShl(b, qhi, LLVMConstInt(te.types.t_int32, 8ul, 0), ""), "iq2x.w16{blk}_{l}") + var idx = LLVMBuildAnd(b, w16, c511, "iq2x.i{blk}_{l}") + var idx2 = LLVMBuildShl(b, idx, LLVMConstInt(te.types.t_int32, 1ul, 0), "") + var sidx = LLVMBuildLShr(b, w16, LLVMConstInt(te.types.t_int32, 9ul, 0), "") + var ksp2 = LLVMBuildGEP2(b, te.types.t_int32, te.iq3xxs_ksigns_g, sidx, "") + var sgn = LLVMBuildLoad2Aligned(b, te.types.t_int32, ksp2, 4u, "iq2x.sg{blk}_{l}") + var mv : LLVMOpaqueValue? [2] + mv[0] = LLVMBuildAnd(b, sgn, c15, "") + mv[1] = LLVMBuildAnd(b, LLVMBuildLShr(b, sgn, LLVMConstInt(te.types.t_int32, 4ul, 0), ""), c15, "") + for (h in range(2)) { + var gi = h == 0 ? idx2 : LLVMBuildAdd(b, idx2, LLVMConstInt(te.types.t_int32, 1ul, 0), "") + var gp = LLVMBuildGEP2(b, te.types.t_int32, te.iq3s_grid_g, gi, "") + var gw = LLVMBuildLoad2Aligned(b, te.types.t_int32, gp, 4u, "iq2x.g{blk}_{l}_{h}") + var mp = LLVMBuildGEP2(b, te.types.t_int32, te.iq3s_smask_g, mv[h], "") + var mw = LLVMBuildLoad2Aligned(b, te.types.t_int32, mp, 4u, "") + var w = LLVMBuildAdd(b, LLVMBuildXor(b, gw, mw, ""), LLVMBuildAnd(b, mw, cneg, ""), "iq2x.w{blk}_{l}_{h}") + let e = l * 8 + h * 4 + let dd = (e < 16 ? 0 : 128 * mr) + (blk * 4 + (e % 16) / 4) * mr * 4 + var dp = LLVMBuildGEP2(b, te.types.t_int8, te.iq3s_panel, LLVMBuildAdd(b, rT4, te.types->ConstI64(uint64(dd)), ""), "") + LLVMBuildStore(b, w, dp) + } + } + } + var rNext = LLVMBuildAdd(b, rPhi, te.types->ConstI64(1ul), "iq2x.rn") + var more = LLVMBuildICmp(b, LLVMIntPredicate.LLVMIntSLT, rNext, te.types->ConstI64(uint64(mr)), "iq2x.more") + LLVMBuildCondBr(b, more, head, cont) + var loopVals <- [rNext] + var loopBlocks <- [head] + LLVMAddIncoming(rPhi, loopVals, loopBlocks) + LLVMPositionBuilderAtEnd(b, cont) +} + +// iq2xxs gemv: gather superblock `sbi` off the PACKED grp planes into the alloca panel - block +// b's grid BYTES (column 2b) double into the u64 grid's word pair, its aux32 (column 2b+1) +// carries the 7-bit KSIGNS indices (the per-32 scale nibble rides the scale row, not the gather). +def private emit_iq2xxs_gather(var te : TileEmit; var sbi : LLVMOpaqueValue?) { + let b = te.builder + let mr = te.interleave + var wbP = LLVMBuildMul(b, sbi, te.types->ConstI64(uint64(mr * 64)), "iq2xx.wb") + var pre = LLVMGetInsertBlock(b) + var head = LLVMAppendBasicBlockInContext(te.gctx, te.gfn, "iq2xx.gath") + var cont = LLVMAppendBasicBlockInContext(te.gctx, te.gfn, "iq2xx.cont") + LLVMBuildBr(b, head) + LLVMPositionBuilderAtEnd(b, head) + var rPhi = LLVMBuildPhi(b, te.types.t_int64, "iq2xx.r") + var preVals <- [te.types->ConstI64(0ul)] + var preBlocks <- [pre] + LLVMAddIncoming(rPhi, preVals, preBlocks) + var rT4 = LLVMBuildMul(b, rPhi, te.types->ConstI64(4ul), "iq2xx.r4") + var base = LLVMBuildAdd(b, wbP, rT4, "iq2xx.base") + let c15 = LLVMConstInt(te.types.t_int32, 15ul, 0) + let c127 = LLVMConstInt(te.types.t_int32, 127ul, 0) + let cneg = LLVMConstInt(te.types.t_int32, 0x01010101ul, 0) + for (blk in range(8)) { + var a0p = LLVMBuildGEP2(b, te.types.t_int8, te.wg, LLVMBuildAdd(b, base, te.types->ConstI64(uint64((2 * blk + 1) * mr * 4)), ""), "") + var aux = LLVMBuildZExt(b, LLVMBuildLoad2Aligned(b, te.types.t_int8, a0p, 1u, ""), te.types.t_int32, "") + for (k in range(1, 4)) { + var akp = LLVMBuildGEP2(b, te.types.t_int8, te.wg, LLVMBuildAdd(b, base, te.types->ConstI64(uint64((2 * blk + 1) * mr * 4 + k)), ""), "") + var ab = LLVMBuildZExt(b, LLVMBuildLoad2Aligned(b, te.types.t_int8, akp, 1u, ""), te.types.t_int32, "") + aux = LLVMBuildOr(b, aux, LLVMBuildShl(b, ab, LLVMConstInt(te.types.t_int32, uint64(8 * k), 0), ""), "iq2xx.aux{blk}_{k}") + } + for (l in range(4)) { + var ip = LLVMBuildGEP2(b, te.types.t_int8, te.wg, LLVMBuildAdd(b, base, te.types->ConstI64(uint64(2 * blk * mr * 4 + l)), ""), "") + var idx = LLVMBuildZExt(b, LLVMBuildLoad2Aligned(b, te.types.t_int8, ip, 1u, ""), te.types.t_int32, "iq2xx.i{blk}_{l}") + var idx2 = LLVMBuildShl(b, idx, LLVMConstInt(te.types.t_int32, 1ul, 0), "") + var sidx = LLVMBuildAnd(b, LLVMBuildLShr(b, aux, LLVMConstInt(te.types.t_int32, uint64(7 * l), 0), ""), c127, "") + var ksp2 = LLVMBuildGEP2(b, te.types.t_int32, te.iq3xxs_ksigns_g, sidx, "") + var sgn = LLVMBuildLoad2Aligned(b, te.types.t_int32, ksp2, 4u, "iq2xx.sg{blk}_{l}") + var mv : LLVMOpaqueValue? [2] + mv[0] = LLVMBuildAnd(b, sgn, c15, "") + mv[1] = LLVMBuildAnd(b, LLVMBuildLShr(b, sgn, LLVMConstInt(te.types.t_int32, 4ul, 0), ""), c15, "") + for (h in range(2)) { + var gi = h == 0 ? idx2 : LLVMBuildAdd(b, idx2, LLVMConstInt(te.types.t_int32, 1ul, 0), "") + var gp = LLVMBuildGEP2(b, te.types.t_int32, te.iq3s_grid_g, gi, "") + var gw = LLVMBuildLoad2Aligned(b, te.types.t_int32, gp, 4u, "iq2xx.g{blk}_{l}_{h}") + var mp = LLVMBuildGEP2(b, te.types.t_int32, te.iq3s_smask_g, mv[h], "") + var mw = LLVMBuildLoad2Aligned(b, te.types.t_int32, mp, 4u, "") + var w = LLVMBuildAdd(b, LLVMBuildXor(b, gw, mw, ""), LLVMBuildAnd(b, mw, cneg, ""), "iq2xx.w{blk}_{l}_{h}") + let e = l * 8 + h * 4 + let dd = (e < 16 ? 0 : 128 * mr) + (blk * 4 + (e % 16) / 4) * mr * 4 + var dp = LLVMBuildGEP2(b, te.types.t_int8, te.iq3s_panel, LLVMBuildAdd(b, rT4, te.types->ConstI64(uint64(dd)), ""), "") + LLVMBuildStore(b, w, dp) + } + } + } + var rNext = LLVMBuildAdd(b, rPhi, te.types->ConstI64(1ul), "iq2xx.rn") + var more = LLVMBuildICmp(b, LLVMIntPredicate.LLVMIntSLT, rNext, te.types->ConstI64(uint64(mr)), "iq2xx.more") + LLVMBuildCondBr(b, more, head, cont) + var loopVals <- [rNext] + var loopBlocks <- [head] + LLVMAddIncoming(rPhi, loopVals, loopBlocks) + LLVMPositionBuilderAtEnd(b, cont) +} + +// One 256-weight SUPERBLOCK, IQ4_XS/iq3s grp form (te.kq = 44/33): mx4's LUT decode (44) +// or the byte-expanded signed panel (33; the gemv gathers it per superblock, the tile gets the +// runtime unpack) — dot_lane's sign-trick lattice, k4's fold with no min term. +def private emit_block_iq4xs(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f : LLVMOpaqueValue? [8]; tokBase, tokCount : int) { // nolint:STYLE037,STYLE038 — one register-coupled superblock loop nest (the kqv2 emitters' shape) + let b = te.builder + let rq = te.rq + let mr = te.interleave + let w8 = te.width / 8 + let panel = te.kq == 33 || te.kq == 34 || te.kq == 23 || te.kq == 24 || te.kq == 25 // iq3s/iq3xxs/iq2s/iq2xs/iq2xxs: the byte-expanded signed panel replaces the nibble+LUT decode + let iq2 = te.kq == 23 || te.kq == 24 // iq2s/iq2xs: per-16 UNSIGNED strips - lo/hi halves keep split accumulators + let nl = te.kq == 45 // iq4nl: the 44 LUT decode with q40's PER-BLOCK f16 d fold (no sub-scales, no min term) + let gather = panel && te.iq3s_panel != null // gemv mode: gather this superblock off the packed planes first + var wbase = te.wg + var wb : LLVMOpaqueValue? + if (gather) { + if (te.kq == 33) { + emit_iq3s_gather(te, sbi) + } elif (te.kq == 23) { + emit_iq2s_gather(te, sbi) + } elif (te.kq == 24) { + emit_iq2xs_gather(te, sbi) + } elif (te.kq == 25) { + emit_iq2xxs_gather(te, sbi) + } else { + emit_iq3xxs_gather(te, sbi) + } + wbase = te.iq3s_panel + wb = te.types->ConstI64(0ul) + } else { + wb = LLVMBuildMul(b, sbi, te.types->ConstI64(uint64(mr * (panel ? 256 : 128))), "wb") + } + var sb = LLVMBuildMul(b, sbi, te.types->ConstI64(uint64(mr * (nl ? 16 : 20))), "sb") + var xb = LLVMBuildMul(b, sbi, te.types->ConstI64(0x100ul), "xb") + var vri8 = LLVMVectorType(te.types.t_int8, uint(te.rv)) + var dv : LLVMOpaqueValue? [2] + if (!nl) { // one superblock d per row lane; iq4nl loads its per-block d inside the walk + for (qd in range(rq)) { + dv[qd] = load_f16_vec_at(te, te.sg, LLVMBuildAdd(b, sb, te.types->ConstI64(uint64(2 * (qd * te.rv))), ""), "d{qd}") + } + } + var d8nl : LLVMOpaqueValue? [4] + if (nl) { // hoist the per-token activation scales - the per-block fold consumes them 8x + for (i in range(tokCount)) { + let tk = tokBase + i + var qp0 = LLVMBuildGEP2(b, te.types.t_float, te.xs[tk], sbi, "qp{tk}") + d8nl[i] = splat_f32(te, LLVMBuildLoad2Aligned(b, te.types.t_float, qp0, 4u, "d8{tk}"), "d8v{tk}") + } + } + var iacc : LLVMOpaqueValue? [8] + for (i in range(tokCount * rq)) { + iacc[i] = LLVMConstNull(te.vni32) + } + for (blk in range(8)) { + var xoff0 = LLVMBuildAdd(b, xb, te.types->ConstI64(uint64(blk * 32)), "") + var xoff1 = LLVMBuildAdd(b, xb, te.types->ConstI64(uint64(blk * 32 + 16)), "") + var xv0 : LLVMOpaqueValue? [4] + var xv1 : LLVMOpaqueValue? [4] + for (i in range(tokCount)) { + xv0[i] = load_v16i8(te, te.x[tokBase + i], xoff0, "x{tokBase + i}_{blk}lo") + xv1[i] = load_v16i8(te, te.x[tokBase + i], xoff1, "x{tokBase + i}_{blk}hi") + } + var a : LLVMOpaqueValue? [8] // [token * rq + qd]; tokCount*rq <= 8 (nrsplit*vq rail) + var a1 : LLVMOpaqueValue? [8] // iq2s: the hi-16 half's own accumulator (per-16 strips) + for (i in range(tokCount * rq)) { + a[i] = LLVMConstNull(te.vni32) + a1[i] = LLVMConstNull(te.vni32) + } + for (j in range(4)) { + for (qd in range(rq)) { + var noff = LLVMBuildAdd(b, wb, te.types->ConstI64(uint64((blk * 16 + j * 4) * mr + qd * w8)), "") + var wlo : LLVMOpaqueValue? + var whi : LLVMOpaqueValue? + if (panel) { + var hoff = LLVMBuildAdd(b, noff, te.types->ConstI64(uint64(128 * mr)), "") + wlo = load_vec(te, te.vwi8, wbase, noff, "wlo{blk}_{j * rq + qd}") + whi = load_vec(te, te.vwi8, wbase, hoff, "whi{blk}_{j * rq + qd}") + } else { + var nv = load_vec(te, te.vwi8, te.wg, noff, "nv{blk}_{j * rq + qd}") + var loIdx = LLVMBuildAnd(b, nv, splat_i8w(te, 15), "nib.lo") + var hiIdx = LLVMBuildLShr(b, nv, splat_i8w(te, 4), "nib.hi") + wlo = lut_lookup(te, loIdx, "wlo{blk}_{j * rq + qd}") + whi = lut_lookup(te, hiIdx, "whi{blk}_{j * rq + qd}") + } + var ulo = abs_w(te, wlo, "ulo{blk}_{j * rq + qd}") + var uhi = abs_w(te, whi, "uhi{blk}_{j * rq + qd}") + for (i in range(tokCount)) { + if (iq2) { + a[i * rq + qd] = dot_lane(te, a[i * rq + qd], wlo, ulo, xv0[i], j) + a1[i * rq + qd] = dot_lane(te, a1[i * rq + qd], whi, uhi, xv1[i], j) + } else { + a[i * rq + qd] = dot_lane(te, dot_lane(te, a[i * rq + qd], wlo, ulo, xv0[i], j), whi, uhi, xv1[i], j) + } + } + } + } + if (nl) { + // per-block fold: f += sitofp(idot) * (d_blk * d8) - q40's shape, straight to float + var dblk : LLVMOpaqueValue? [2] + for (qd in range(rq)) { + dblk[qd] = load_f16_vec_at(te, te.sg, LLVMBuildAdd(b, sb, te.types->ConstI64(uint64(blk * 2 * mr + 2 * (qd * te.rv))), ""), "dnl{blk}_{qd}") + } + for (i in range(tokCount)) { + for (qd in range(rq)) { + var dsn = LLVMBuildFMul(b, dblk[qd], d8nl[i], "dsn{tokBase + i}_{blk}_{qd}") + f[i * rq + qd] = fold_fma(te, LLVMBuildSIToFP(b, a[i * rq + qd], te.vnf32, ""), dsn, f[i * rq + qd], "f{tokBase + i}_{blk}_{qd}n") + } + } + } else { + var scv : LLVMOpaqueValue? [2] + var scv1 : LLVMOpaqueValue? [2] // iq2s: the hi half's strip + for (qd in range(rq)) { + if (iq2) { + var s0p = LLVMBuildGEP2(b, te.types.t_int8, te.sg, LLVMBuildAdd(b, sb, te.types->ConstI64(uint64(4 * mr + 2 * blk * mr + qd * te.rv)), ""), "s0p{blk}_{qd}") + var s1p = LLVMBuildGEP2(b, te.types.t_int8, te.sg, LLVMBuildAdd(b, sb, te.types->ConstI64(uint64(4 * mr + (2 * blk + 1) * mr + qd * te.rv)), ""), "s1p{blk}_{qd}") + scv[qd] = LLVMBuildZExt(b, LLVMBuildLoad2Aligned(b, vri8, s0p, 1u, ""), te.vni32, "sc{blk}_{qd}a") + scv1[qd] = LLVMBuildZExt(b, LLVMBuildLoad2Aligned(b, vri8, s1p, 1u, ""), te.vni32, "sc{blk}_{qd}b") + continue + } + var scp = LLVMBuildGEP2(b, te.types.t_int8, te.sg, LLVMBuildAdd(b, sb, te.types->ConstI64(uint64(4 * mr + blk * mr + qd * te.rv)), ""), "scp{blk}_{qd}") + scv[qd] = LLVMBuildSExt(b, LLVMBuildLoad2Aligned(b, vri8, scp, 1u, ""), te.vni32, "sc{blk}_{qd}") + } + for (i in range(tokCount)) { + for (qd in range(rq)) { + var ia = LLVMBuildMul(b, scv[qd], a[i * rq + qd], "") + if (iq2) { + ia = LLVMBuildAdd(b, ia, LLVMBuildMul(b, scv1[qd], a1[i * rq + qd], ""), "") + } + iacc[i * rq + qd] = LLVMBuildAdd(b, iacc[i * rq + qd], ia, "ia{tokBase + i}_{blk}_{qd}") + } + } + } + } + if (nl) { + return // the per-block fold already landed everything in f + } + for (i in range(tokCount)) { + let tk = tokBase + i + var qp = LLVMBuildGEP2(b, te.types.t_float, te.xs[tk], sbi, "qp{tk}") + var d8 = LLVMBuildLoad2Aligned(b, te.types.t_float, qp, 4u, "d8{tk}") + var d8v = splat_f32(te, d8, "d8v{tk}") + for (qd in range(rq)) { + var ds = LLVMBuildFMul(b, dv[qd], d8v, "ds{tk}_{qd}") + f[i * rq + qd] = fold_fma(te, LLVMBuildSIToFP(b, iacc[i * rq + qd], te.vnf32, ""), ds, f[i * rq + qd], "f{tk}_{qd}s") + } + } +} + // Block-emitter dispatch: the slice/loop machinery is quant-agnostic. The MMA form needs a // token PAIR — single-token slices ride the sdot lattice instead. kq blocks are SUPERBLOCKS // (the kq gemv/tile drive emit_slice with nb = n/256). def private emit_one_block(var te : TileEmit; var bi : LLVMOpaqueValue?; var f : LLVMOpaqueValue? [8]; tokBase, tokCount : int) { - if (te.kq != 0) { + if (te.kq == 44 || te.kq == 33 || te.kq == 34 || te.kq == 45 || te.kq == 23 || te.kq == 24 || te.kq == 25) { + emit_block_iq4xs(te, bi, f, tokBase, tokCount) + } elif (te.kq != 0) { emit_block_kqv2(te, bi, f, tokBase, tokCount) } elif (te.q51) { emit_block_q51(te, bi, f, tokBase, tokCount) @@ -1288,13 +2047,15 @@ def private emit_slice(var te : TileEmit; var sa : SliceArgs; var pred : LLVMOpa var bij = LLVMBuildAdd(b, biPhi, te.types->ConstI64(uint64(j)), "") emit_one_block(te, bij, fCur, tokBase, tokCount) } + // the back edge starts wherever the block emitters LEFT the builder (a splitting emitter ends in its continuation block) + var bodyEnd = LLVMGetInsertBlock(b) LLVMBuildBr(b, head) var bodyValsBi <- [biPK] - var bodyBlocks <- [body] + var bodyBlocks <- [bodyEnd] LLVMAddIncoming(biPhi, bodyValsBi, bodyBlocks) for (i in range(nacc)) { var bv <- [fCur[i]] - var bb <- [body] + var bb <- [bodyEnd] LLVMAddIncoming(fPhi[i], bv, bb) } @@ -1324,8 +2085,9 @@ def private emit_slice(var te : TileEmit; var sa : SliceArgs; var pred : LLVMOpa chainBi = LLVMBuildAdd(b, chainBi, te.types->ConstI64(1ul), "") LLVMBuildBr(b, checks[j + 1]) } else { + var tailEnd = LLVMGetInsertBlock(b) // same rule: the tail body may have split its block LLVMBuildBr(b, after) - mergeBlocks |> push(tails[j]) + mergeBlocks |> push(tailEnd) mergeF |> push(chainF) } } @@ -1459,13 +2221,15 @@ def private setup_tile_emit(var te : TileEmit; var gc : LlvmCodeCtx; p : TilePer } } if (needMx4) { - // the doubled-e2m1 LUT as a constant vector — no runtime table pointer; broadcast per 16-byte lane on x64 - let kv = fixed_array(0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12) + // the nibble codebook as a constant vector (mx4: doubled e2m1; kq 44: kvalues_iq4nl) - no runtime table pointer; broadcast per 16-byte lane on x64 + let e2m1 = fixed_array(0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12) + let iq4nl = iq4nl_lut() let lanes = te.dotKind == DOT_SDOT ? 1 : te.width / 128 var lutElems : array lutElems |> reserve(16 * lanes) for (_lane in range(lanes)) { - for (v in kv) { + for (i in range(16)) { + let v = te.kq == 44 || te.kq == 45 ? int(iq4nl[i]) : e2m1[i] lutElems |> push(LLVMConstInt(te.types.t_int8, uint64(v & 255), 0)) } } @@ -2290,7 +3054,7 @@ def private kq_gemv_gen_impl(var gc : LlvmCodeCtx; fmt : int) : bool { let p = companion_perm(p0) var te = TileEmit(kq = fmt) - if (!setup_tile_emit(te, gc, p, false)) return false + if (!setup_tile_emit(te, gc, p, fmt == 44 || fmt == 45)) return false let b = gc.jit.builder var entry = LLVMAppendBasicBlockInContext(gc.jit.ctx, gc.impl, "entry") @@ -2299,6 +3063,24 @@ def private kq_gemv_gen_impl(var gc : LlvmCodeCtx; fmt : int) : bool { // entry: group range off the row range; strides are disk footprints (quants 128/160/192B, scales 20/20/18B) LLVMPositionBuilderAtEnd(b, entry) + if (fmt == 33 || fmt == 34 || fmt == 23 || fmt == 24 || fmt == 25) { + // iq3s/iq3xxs/iq2s/iq2xs: the gemv gathers each superblock into an alloca panel (the tile gets the runtime unpack) + te.gfn = gc.impl + te.gctx = gc.jit.ctx + if (fmt == 33) { + iq3s_emit_globals(te, LLVMGetGlobalParent(gc.impl)) + } elif (fmt == 23) { + iq2s_emit_globals(te, LLVMGetGlobalParent(gc.impl)) + } elif (fmt == 24) { + iq2xs_emit_globals(te, LLVMGetGlobalParent(gc.impl)) + } elif (fmt == 25) { + iq2xxs_emit_globals(te, LLVMGetGlobalParent(gc.impl)) + } else { + iq3xxs_emit_globals(te, LLVMGetGlobalParent(gc.impl)) + } + te.iq3s_panel = LLVMBuildAlloca(b, LLVMArrayType(gc.jit.types.t_int8, uint(256 * te.interleave)), "iq3s.panel") + LLVMSetAlignment(te.iq3s_panel, 32u) + } var sa = SliceArgs(gc_impl = gc.impl, ctx = gc.jit.ctx, kstep = 1) sa.yp = LLVMGetParam(gc.impl, 0u) let kqp = LLVMGetParam(gc.impl, 1u) @@ -2421,8 +3203,8 @@ def private kq_tile_gen_impl(var gc : LlvmCodeCtx; fmt : int) : bool { if (perm_declines(gc, p0)) return false let p = companion_perm(p0) - var te = TileEmit(kq = fmt, kqBytes = fmt == 5 || fmt == 6) // k4/q40 tiles read the packed planes - if (!setup_tile_emit(te, gc, p, false)) return false + var te = TileEmit(kq = fmt, kqBytes = fmt == 5 || fmt == 6) // only the k5/k6 tiles get the byte-expanded panel + if (!setup_tile_emit(te, gc, p, fmt == 44 || fmt == 45)) return false let b = gc.jit.builder var entry = LLVMAppendBasicBlockInContext(gc.jit.ctx, gc.impl, "entry") @@ -2462,6 +3244,45 @@ def private kq_tile_gen_impl(var gc : LlvmCodeCtx; fmt : int) : bool { return true } +// iq4xs rides the kq gemv/tile walks with emit_block_iq4xs as the block body (LUT decode + +// sign-trick dots); the tbl1/pshufb rail is the mx4 companion's, already in perm_declines. +def private iq4xs_gemv_gen(var gc : LlvmCodeCtx) : bool => kq_gemv_gen_impl(gc, 44) +def private iq4xs_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 44) + +// k3 rides the kq walks through emit_block_kqv2's k6 arms (the k3 flag swaps the quant compose and the offset) +def private k3_gemv_gen(var gc : LlvmCodeCtx) : bool => kq_gemv_gen_impl(gc, 3) +def private k3_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 3) + +// iq3s: the gemv gathers per superblock into an alloca panel (emit_iq3s_gather); the tile +// reads the runtime-unpacked panel — both then ride the iq4xs block lattice. +def private iq3s_gemv_gen(var gc : LlvmCodeCtx) : bool => kq_gemv_gen_impl(gc, 33) +def private iq3s_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 33) + +// iq3xxs: the gemv gathers per superblock into the alloca panel (emit_iq3xxs_gather - the +// halved grid + ksigns); the tile reads the runtime-unpacked panel - the iq3s route exactly. +def private iq3xxs_gemv_gen(var gc : LlvmCodeCtx) : bool => kq_gemv_gen_impl(gc, 34) +def private iq3xxs_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 34) + +// iq4nl rides the iq4xs LUT block body with q40's per-block f16 d fold (the `nl` mode) over +// the packed grp planes - no panel, no gather, no sub-scales. +def private iq4nl_gemv_gen(var gc : LlvmCodeCtx) : bool => kq_gemv_gen_impl(gc, 45) +def private iq4nl_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 45) + +// k2 rides the kq walks through emit_block_kqv2's per-16 arms (the k2 flag swaps the 2-bit +// compose, the nibble scale extracts, and the epilogue to k4/k5's dmin fold) +def private k2_gemv_gen(var gc : LlvmCodeCtx) : bool => kq_gemv_gen_impl(gc, 2) +def private k2_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 2) + +// iq2s rides the iq4xs walk through the panel route: emit_iq2s_gather feeds the shared +// byte-expanded panel; per-16 UNSIGNED strips keep split lo/hi accumulators (the iq2 flag) +def private iq2s_gemv_gen(var gc : LlvmCodeCtx) : bool => kq_gemv_gen_impl(gc, 23) +def private iq2s_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 23) + +// iq2xs rides the iq4xs walk through the panel route: emit_iq2xs_gather feeds the shared +// byte-expanded panel; the iq2 flag's split accumulators and per-16 strips are iq2s's. +def private iq2xs_gemv_gen(var gc : LlvmCodeCtx) : bool => kq_gemv_gen_impl(gc, 24) +def private iq2xs_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 24) + def private k4_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 4) def private k5_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 5) def private k6_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 6) @@ -2516,6 +3337,13 @@ def private q51_tile_gen(var gc : LlvmCodeCtx) : bool { // Called from llvm_user_modules::register_user_llvm_code_generators — the JIT invokes that in the // context that reads the registry, so the direct call lands in the right table copy. + +// iq2xxs rides the iq4xs walk through the panel route: emit_iq2xxs_gather feeds the shared +// byte-expanded panel; per-32 UNSIGNED strips ride the iq3s/iq3xxs scale fold (the strip +// bytes stay under 128, so the shared SExt load reads them exactly). +def private iq2xxs_gemv_gen(var gc : LlvmCodeCtx) : bool => kq_gemv_gen_impl(gc, 25) +def private iq2xxs_tile_gen(var gc : LlvmCodeCtx) : bool => kq_tile_gen_impl(gc, 25) + [macro_function] def public register_dasllama_gemm_generators { register_llvm_code_generator("dasllama_gemm_gen::q8q8_tile", @@tile_gen) @@ -2539,5 +3367,23 @@ def public register_dasllama_gemm_generators { register_llvm_code_generator("dasllama_gemm_gen::k6_tile", @@k6_tile_gen) register_llvm_code_generator("dasllama_gemm_gen::q40_gemv", @@q40_gemv_gen) register_llvm_code_generator("dasllama_gemm_gen::q40_tile", @@q40_tile_gen) + register_llvm_code_generator("dasllama_gemm_gen::iq4xs_gemv", @@iq4xs_gemv_gen) + register_llvm_code_generator("dasllama_gemm_gen::iq4xs_tile", @@iq4xs_tile_gen) + register_llvm_code_generator("dasllama_gemm_gen::k3_gemv", @@k3_gemv_gen) + register_llvm_code_generator("dasllama_gemm_gen::k3_tile", @@k3_tile_gen) + register_llvm_code_generator("dasllama_gemm_gen::iq3s_gemv", @@iq3s_gemv_gen) + register_llvm_code_generator("dasllama_gemm_gen::iq3s_tile", @@iq3s_tile_gen) + register_llvm_code_generator("dasllama_gemm_gen::iq3xxs_gemv", @@iq3xxs_gemv_gen) + register_llvm_code_generator("dasllama_gemm_gen::iq3xxs_tile", @@iq3xxs_tile_gen) + register_llvm_code_generator("dasllama_gemm_gen::iq4nl_gemv", @@iq4nl_gemv_gen) + register_llvm_code_generator("dasllama_gemm_gen::iq4nl_tile", @@iq4nl_tile_gen) + register_llvm_code_generator("dasllama_gemm_gen::k2_gemv", @@k2_gemv_gen) + register_llvm_code_generator("dasllama_gemm_gen::k2_tile", @@k2_tile_gen) + register_llvm_code_generator("dasllama_gemm_gen::iq2s_gemv", @@iq2s_gemv_gen) + register_llvm_code_generator("dasllama_gemm_gen::iq2s_tile", @@iq2s_tile_gen) + register_llvm_code_generator("dasllama_gemm_gen::iq2xs_gemv", @@iq2xs_gemv_gen) + register_llvm_code_generator("dasllama_gemm_gen::iq2xs_tile", @@iq2xs_tile_gen) + register_llvm_code_generator("dasllama_gemm_gen::iq2xxs_gemv", @@iq2xxs_gemv_gen) + register_llvm_code_generator("dasllama_gemm_gen::iq2xxs_tile", @@iq2xxs_tile_gen) register_llvm_code_generator("dasllama_gemm_gen::q8q8_witness", @@witness_gen) } diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_schema.das b/modules/dasLLAMA/dasllama/dasllama_gemm_schema.das index 9a2f769760..f476357100 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_schema.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_schema.das @@ -40,7 +40,7 @@ def q8q8_repack_type(mr : int; wbias : int = 0; kgroup : int = 4) : Q8RepackType //! The mr of the REFERENCE tier's layout (q8q8_repack_type(4), the old hand arm64-laneq layout //! the reference stub bodies read). Non-reference mr runs the generated grp kernel/layout pair, //! stamped from one manifest entry so both decline in lockstep (M4). -let GEMM_REFERENCE_MR = 4 +def GEMM_REFERENCE_MR : int => 4 //! GPU-tier device-plane shape of the Q8 rail: 32-weight blocks, 32B quants + one f16 scale //! halfword per block. Every q8 plane sizing (gathers, arena/stack math) derives from these. @@ -58,24 +58,49 @@ let Q51_QPB = 20l let Q51_SPB = 4l //! Quant-plane bytes per 256-weight superblock per row for a kq format id (4/5/6 = Q4_K/Q5_K/ -//! Q6_K, 40 = Q4_0). The ONE stride source for every fmt-branched kq walker/kernel/repack — -//! an unknown id panics instead of silently walking another format's stride. +//! Q6_K, 40 = Q4_0, 44 = IQ4_XS, 3 = Q3_K, 33 = IQ3_S, 34 = IQ3_XXS, 45 = IQ4_NL, 2 = Q2_K, 23 = IQ2_S, 24 = IQ2_XS, 25 = IQ2_XXS). The ONE stride source for every fmt-branched kq walker/ +//! kernel/repack — an unknown id panics instead of silently walking another format's stride. def kq_qsb(fmt : int) : int64 { if (fmt == 4) return kq_qsb(KqFmt.k4) if (fmt == 5) return kq_qsb(KqFmt.k5) if (fmt == 6) return kq_qsb(KqFmt.k6) if (fmt == 40) return kq_qsb(KqFmt.q40) + if (fmt == 44) return kq_qsb(KqFmt.iq4xs) + if (fmt == 3) return kq_qsb(KqFmt.k3) + if (fmt == 33) return kq_qsb(KqFmt.iq3s) + if (fmt == 34) return kq_qsb(KqFmt.iq3xxs) + if (fmt == 45) return kq_qsb(KqFmt.iq4nl) + if (fmt == 2) return kq_qsb(KqFmt.k2) + if (fmt == 23) return kq_qsb(KqFmt.iq2s) + if (fmt == 24) return kq_qsb(KqFmt.iq2xs) + if (fmt == 25) return kq_qsb(KqFmt.iq2xxs) panic("kq_qsb: unknown kq format id {fmt}") return 0l } -//! Scale-plane bytes per 256-weight superblock per row (see kq_qsb): k4/k5 20 (16B disk block -//! + 4B pad, decoded in place at repack), k6 18 (native), q40 16 (8 x f16 d, verbatim). +//! Scale-plane bytes per 256-weight superblock per row (see kq_qsb): k4/k5 20, k6/k3 18, +//! q40/iq4nl 16 (8 x f16 d), iq4xs/iq3s/iq3xxs 20 (the decoded k4 row shape) - the per-format +//! layouts live on the kq_ssb(KqFmt) arms and the transcode docs. def kq_ssb(fmt : int) : int64 { if (fmt == 4) return kq_ssb(KqFmt.k4) if (fmt == 5) return kq_ssb(KqFmt.k5) if (fmt == 6) return kq_ssb(KqFmt.k6) if (fmt == 40) return kq_ssb(KqFmt.q40) + if (fmt == 44) return kq_ssb(KqFmt.iq4xs) + if (fmt == 3) return kq_ssb(KqFmt.k3) + if (fmt == 33) return kq_ssb(KqFmt.iq3s) + if (fmt == 34) return kq_ssb(KqFmt.iq3xxs) + if (fmt == 45) return kq_ssb(KqFmt.iq4nl) + if (fmt == 2) return kq_ssb(KqFmt.k2) + if (fmt == 23) return kq_ssb(KqFmt.iq2s) + if (fmt == 24) return kq_ssb(KqFmt.iq2xs) + if (fmt == 25) return kq_ssb(KqFmt.iq2xxs) panic("kq_ssb: unknown kq format id {fmt}") return 0l } + +//! True for the kq ids whose tile reads the packed quant plane directly; the others (k5/k6 and +//! the grid formats) read a byte-expanded panel unpacked once per (group, token-block). +def kq_reads_packed_planes(fmt : int64) : bool { + return fmt == 4l || fmt == 40l || fmt == 44l || fmt == 3l || fmt == 45l || fmt == 2l +} diff --git a/modules/dasLLAMA/dasllama/dasllama_gguf.das b/modules/dasLLAMA/dasllama/dasllama_gguf.das index 9ce5b4b1ca..80cbae4abd 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gguf.das +++ b/modules/dasLLAMA/dasllama/dasllama_gguf.das @@ -40,16 +40,25 @@ let private GT_U64 = 10 let private GT_I64 = 11 let private GT_F64 = 12 -// ggml tensor data types (subset we read) +// upstream tensor data types (subset we read) let GGML_TYPE_F32 = 0 let GGML_TYPE_F16 = 1 let GGML_TYPE_Q4_0 = 2 let GGML_TYPE_Q5_0 = 6 let GGML_TYPE_Q5_1 = 7 let GGML_TYPE_Q8_0 = 8 +let GGML_TYPE_Q3_K = 11 +let GGML_TYPE_Q2_K = 10 +let GGML_TYPE_IQ3_S = 21 +let GGML_TYPE_IQ2_S = 22 +let GGML_TYPE_IQ2_XS = 17 +let GGML_TYPE_IQ2_XXS = 16 +let GGML_TYPE_IQ3_XXS = 18 +let GGML_TYPE_IQ4_NL = 20 let GGML_TYPE_Q4_K = 12 let GGML_TYPE_Q5_K = 13 let GGML_TYPE_Q6_K = 14 +let GGML_TYPE_IQ4_XS = 23 let GGML_TYPE_BF16 = 30 let GGML_TYPE_MXFP4 = 39 @@ -394,7 +403,7 @@ def gguf_str(m : GGUFMeta; bytes : array | #; key : string) : string { return rd_str(bytes, o) } -//! Read a string array (e.g. tokenizer.ggml.tokens). +//! Read a string array (e.g. tokenizer.upstream.tokens). def gguf_str_array(m : GGUFMeta; bytes : array | #; key : string) : array { if (!key_exists(m.kv, key)) { panic("gguf: missing key '{key}'") @@ -412,7 +421,7 @@ def gguf_str_array(m : GGUFMeta; bytes : array | #; key : string) : array return <- out } -//! Read a float32 array (e.g. tokenizer.ggml.scores). +//! Read a float32 array (e.g. tokenizer.upstream.scores). def gguf_f32_array(m : GGUFMeta; bytes : array | #; key : string) : array { if (!key_exists(m.kv, key)) { panic("gguf: missing key '{key}'") @@ -431,7 +440,7 @@ def gguf_f32_array(m : GGUFMeta; bytes : array | #; key : string) : array return <- out } -//! Read an int32 array (e.g. tokenizer.ggml.token_type), widened to int64. +//! Read an int32 array (e.g. tokenizer.upstream.token_type), widened to int64. def gguf_i32_array(m : GGUFMeta; bytes : array | #; key : string) : array { if (!key_exists(m.kv, key)) { panic("gguf: missing key '{key}'") @@ -701,6 +710,359 @@ def gguf_transcode_q40(m : GGUFMeta; srcbytes : array | #; name : string; } } +//! Transcode an IQ4_XS tensor into the iq4xs planes (see gguf_transcode_q4k; strides 128/20, +//! exact): the 136B disk superblock splits into the verbatim 128 nibble bytes and a DECODED +//! 20B scale row (f16 d, 2 pad, 8 x int8 (ls - 32), 8 pad — transcode_iq4xs_superblock). +def gguf_transcode_iq4xs(m : GGUFMeta; srcbytes : array | #; name : string; var kq : array; var ks : array; eloff, expect_n : int64; src_off : int64 = 0l) { + let ti = kq_transcode_check(m, name, GGML_TYPE_IQ4_XS, "IQ4_XS", src_off, expect_n) + let nb = expect_n / 256l + if (nb <= 0l) { + return + } + guard_dst(name, "iq4xs quant plane", (eloff / 256l) * IQ4XS_QSB, nb * IQ4XS_QSB, long_length(kq)) + guard_dst(name, "iq4xs scale plane", (eloff / 256l) * IQ4XS_SSB, nb * IQ4XS_SSB, long_length(ks)) + with_tensor_view(m, srcbytes, ti) $(bytes, tbase) { + let bo = tbase + (src_off / 256l) * 136l + unsafe { + let srcp = addr(bytes[bo]) + var kqp = addr(kq[(eloff / 256l) * IQ4XS_QSB]) + var ksp = addr(ks[(eloff / 256l) * IQ4XS_SSB]) + maybe_parallel_for(0, int(nb), transcode_jobs(nb, 136l)) $(rb, re) { + unsafe { + for (sb in range64(int64(rb), int64(re))) { + let src = srcp + sb * 136l + var row = ksp + sb * IQ4XS_SSB + row[0] = src[0] + row[1] = src[1] + row[2] = uint8(0) + row[3] = uint8(0) + let sh = uint(src[2]) | (uint(src[3]) << 8u) + for (blk in range64(8l)) { + let lo = (uint(src[4l + blk / 2l]) >> uint(4l * (blk % 2l))) & 15u + let ls = int(lo | (((sh >> uint(2l * blk)) & 3u) << 4u)) + row[4l + blk] = uint8(ls - 32) + } + for (i in range64(8l)) { + row[12l + i] = uint8(0) + } + bcopy(kqp + sb * IQ4XS_QSB, src + 8l, 128l) + } + } + } + } + } +} + +//! Transcode a Q2_K tensor into the k2 planes (strides 64/20, exact): the 84B disk superblock +//! splits into qs verbatim and the 20B scale row [f16 d][f16 dmin][16 sc/min bytes] - all +//! bytes verbatim, header moved first (transcode_q2k_superblock). +def gguf_transcode_q2k(m : GGUFMeta; srcbytes : array | #; name : string; var kq : array; var ks : array; eloff, expect_n : int64; src_off : int64 = 0l) { + let ti = kq_transcode_check(m, name, GGML_TYPE_Q2_K, "Q2_K", src_off, expect_n) + let nb = expect_n / 256l + if (nb <= 0l) { + return + } + guard_dst(name, "k2 quant plane", (eloff / 256l) * K2_QSB, nb * K2_QSB, long_length(kq)) + guard_dst(name, "k2 scale plane", (eloff / 256l) * K2_SSB, nb * K2_SSB, long_length(ks)) + with_tensor_view(m, srcbytes, ti) $(bytes, tbase) { + let bo = tbase + (src_off / 256l) * 84l + unsafe { + let srcp = addr(bytes[bo]) + var kqp = addr(kq[(eloff / 256l) * K2_QSB]) + var ksp = addr(ks[(eloff / 256l) * K2_SSB]) + maybe_parallel_for(0, int(nb), transcode_jobs(nb, 84l)) $(rb, re) { + unsafe { + for (sb in range64(int64(rb), int64(re))) { + let src = srcp + sb * 84l + var row = ksp + sb * K2_SSB + row[0l] = src[80l] + row[1l] = src[81l] + row[2l] = src[82l] + row[3l] = src[83l] + bcopy(row + 4l, src, 16l) + bcopy(kqp + sb * K2_QSB, src + 16l, 64l) + } + } + } + } + } +} + +//! Transcode a Q3_K tensor into the k3 planes (see gguf_transcode_q4k; strides 96/18, exact): the +//! 110B disk superblock splits into [qs][hmask] verbatim and the k6-shaped scale row with the 16 +//! packed 6-bit sub-scales DECODED to int8 (scale - 32) — transcode_q3k_superblock. +def gguf_transcode_q3k(m : GGUFMeta; srcbytes : array | #; name : string; var kq : array; var ks : array; eloff, expect_n : int64; src_off : int64 = 0l) { + let ti = kq_transcode_check(m, name, GGML_TYPE_Q3_K, "Q3_K", src_off, expect_n) + let nb = expect_n / 256l + if (nb <= 0l) { + return + } + guard_dst(name, "k3 quant plane", (eloff / 256l) * K3_QSB, nb * K3_QSB, long_length(kq)) + guard_dst(name, "k3 scale plane", (eloff / 256l) * K3_SSB, nb * K3_SSB, long_length(ks)) + with_tensor_view(m, srcbytes, ti) $(bytes, tbase) { + let bo = tbase + (src_off / 256l) * 110l + unsafe { + let srcp = addr(bytes[bo]) + var kqp = addr(kq[(eloff / 256l) * K3_QSB]) + var ksp = addr(ks[(eloff / 256l) * K3_SSB]) + maybe_parallel_for(0, int(nb), transcode_jobs(nb, 110l)) $(rb, re) { + unsafe { + for (sb in range64(int64(rb), int64(re))) { + let src = srcp + sb * 110l + var row = ksp + sb * K3_SSB + for (i in range64(16l)) { + let lo = i < 8l ? int(src[96l + i]) & 15 : int(src[96l + i - 8l]) >> 4 + let hi = int(src[104l + i % 4l]) >> int(2l * (i / 4l)) + row[i] = uint8((lo | ((hi & 3) << 4)) - 32) + } + row[16l] = src[108l] + row[17l] = src[109l] + bcopy(kqp + sb * K3_QSB, src + 32l, 64l) + bcopy(kqp + sb * K3_QSB + 64l, src, 32l) + } + } + } + } + } +} + +//! Transcode an IQ3_S tensor into the iq3s planes (strides 104/20, exact): the 110B disk +//! superblock splits into [qs 64][qh 8][signs 32] verbatim and the k4-shaped scale row with the +//! 8 packed 4-bit sub-scales DECODED to int8 (1 + 2s) — transcode_iq3s_superblock. +def gguf_transcode_iq3s(m : GGUFMeta; srcbytes : array | #; name : string; var kq : array; var ks : array; eloff, expect_n : int64; src_off : int64 = 0l) { + let ti = kq_transcode_check(m, name, GGML_TYPE_IQ3_S, "IQ3_S", src_off, expect_n) + let nb = expect_n / 256l + if (nb <= 0l) { + return + } + guard_dst(name, "iq3s quant plane", (eloff / 256l) * IQ3S_QSB, nb * IQ3S_QSB, long_length(kq)) + guard_dst(name, "iq3s scale plane", (eloff / 256l) * IQ3S_SSB, nb * IQ3S_SSB, long_length(ks)) + with_tensor_view(m, srcbytes, ti) $(bytes, tbase) { + let bo = tbase + (src_off / 256l) * 110l + unsafe { + let srcp = addr(bytes[bo]) + var kqp = addr(kq[(eloff / 256l) * IQ3S_QSB]) + var ksp = addr(ks[(eloff / 256l) * IQ3S_SSB]) + maybe_parallel_for(0, int(nb), transcode_jobs(nb, 110l)) $(rb, re) { + unsafe { + for (sb in range64(int64(rb), int64(re))) { + let src = srcp + sb * 110l + var row = ksp + sb * IQ3S_SSB + row[0] = src[0] + row[1] = src[1] + row[2] = uint8(0) + row[3] = uint8(0) + for (b in range64(8l)) { + let nib = b % 2l == 0l ? int(src[106l + b / 2l]) & 15 : int(src[106l + b / 2l]) >> 4 + row[4l + b] = uint8(1 + 2 * nib) + } + for (i in range64(12l, 20l)) { + row[i] = uint8(0) + } + bcopy(kqp + sb * IQ3S_QSB, src + 2l, 104l) + } + } + } + } + } +} + + +//! Transcode an IQ3_XXS tensor into the iq3xxs planes (strides 96/20, exact): qs+aux copy +//! verbatim off the 98B disk block, the iq3s-shaped scale row carries d HALVED (f16_half_bits) +//! and strips (2*ls + 1) — with the halved grid this absorbs upstream's 0.25 exactly. +def gguf_transcode_iq3xxs(m : GGUFMeta; srcbytes : array | #; name : string; var kq : array; var ks : array; eloff, expect_n : int64; src_off : int64 = 0l) { + let ti = kq_transcode_check(m, name, GGML_TYPE_IQ3_XXS, "IQ3_XXS", src_off, expect_n) + let nb = expect_n / 256l + if (nb <= 0l) { + return + } + guard_dst(name, "iq3xxs quant plane", (eloff / 256l) * IQ3XXS_QSB, nb * IQ3XXS_QSB, long_length(kq)) + guard_dst(name, "iq3xxs scale plane", (eloff / 256l) * IQ3XXS_SSB, nb * IQ3XXS_SSB, long_length(ks)) + with_tensor_view(m, srcbytes, ti) $(bytes, tbase) { + let bo = tbase + (src_off / 256l) * 98l + unsafe { + let srcp = addr(bytes[bo]) + var kqp = addr(kq[(eloff / 256l) * IQ3XXS_QSB]) + var ksp = addr(ks[(eloff / 256l) * IQ3XXS_SSB]) + maybe_parallel_for(0, int(nb), transcode_jobs(nb, 98l)) $(rb, re) { + unsafe { + for (sb in range64(int64(rb), int64(re))) { + let src = srcp + sb * 98l + var row = ksp + sb * IQ3XXS_SSB + let dh = f16_half_bits(uint(src[0]) | (uint(src[1]) << 8u)) + row[0] = uint8(dh & 0xFFu) + row[1] = uint8(dh >> 8u) + row[2] = uint8(0) + row[3] = uint8(0) + for (b in range64(8l)) { + row[4l + b] = uint8(1 + 2 * (int(src[66l + b * 4l + 3l]) >> 4)) + } + for (i in range64(12l, 20l)) { + row[i] = uint8(0) + } + bcopy(kqp + sb * IQ3XXS_QSB, src + 2l, 96l) + } + } + } + } + } +} + + +//! Transcode an IQ4_NL tensor into the iq4nl planes (q40's strides 128/16, exact): the disk +//! bytes ARE q40's shape - per 32-block f16 d + 16 nibble bytes, split verbatim; only the +//! nibble semantics differ (IQ4NL_LUT at decode, no -8 offset, no bsum term). +def gguf_transcode_iq4nl(m : GGUFMeta; srcbytes : array | #; name : string; var kq : array; var ks : array; eloff, expect_n : int64; src_off : int64 = 0l) { + let ti = kq_transcode_check(m, name, GGML_TYPE_IQ4_NL, "IQ4_NL", src_off, expect_n) + let nb = expect_n / 256l + if (nb <= 0l) { + return + } + guard_dst(name, "iq4nl quant plane", (eloff / 256l) * Q40_QSB, nb * Q40_QSB, long_length(kq)) + guard_dst(name, "iq4nl scale plane", (eloff / 256l) * Q40_SSB, nb * Q40_SSB, long_length(ks)) + with_tensor_view(m, srcbytes, ti) $(bytes, tbase) { + let bo = tbase + (src_off / 256l) * 144l // 8 x 18B disk blocks per superblock + unsafe { + let srcp = addr(bytes[bo]) + var kqp = addr(kq[(eloff / 256l) * Q40_QSB]) + var ksp = addr(ks[(eloff / 256l) * Q40_SSB]) + maybe_parallel_for(0, int(nb), transcode_jobs(nb, 144l)) $(rb, re) { + unsafe { + for (sb in range64(int64(rb), int64(re))) { + for (blk in range64(8l)) { + let src = srcp + sb * 144l + blk * 18l + ksp[sb * Q40_SSB + blk * 2l] = src[0] // f16 d + ksp[sb * Q40_SSB + blk * 2l + 1l] = src[1] + bcopy(kqp + sb * Q40_QSB + blk * 16l, src + 2l, 16l) + } + } + } + } + } + } +} + +//! Transcode an IQ2_S tensor into the iq2s planes (strides 72/20, exact): the 82B disk +//! superblock splits into [idx][signs][qh] verbatim, d eighth-ed, the 16 strips decoded to +//! (1 + 2s) - transcode_iq2s_superblock. +def gguf_transcode_iq2s(m : GGUFMeta; srcbytes : array | #; name : string; var kq : array; var ks : array; eloff, expect_n : int64; src_off : int64 = 0l) { + let ti = kq_transcode_check(m, name, GGML_TYPE_IQ2_S, "IQ2_S", src_off, expect_n) + let nb = expect_n / 256l + if (nb <= 0l) { + return + } + guard_dst(name, "iq2s quant plane", (eloff / 256l) * IQ2S_QSB, nb * IQ2S_QSB, long_length(kq)) + guard_dst(name, "iq2s scale plane", (eloff / 256l) * IQ2S_SSB, nb * IQ2S_SSB, long_length(ks)) + with_tensor_view(m, srcbytes, ti) $(bytes, tbase) { + let bo = tbase + (src_off / 256l) * 82l + unsafe { + let srcp = addr(bytes[bo]) + var kqp = addr(kq[(eloff / 256l) * IQ2S_QSB]) + var ksp = addr(ks[(eloff / 256l) * IQ2S_SSB]) + maybe_parallel_for(0, int(nb), transcode_jobs(nb, 82l)) $(rb, re) { + unsafe { + for (sb in range64(int64(rb), int64(re))) { + let src = srcp + sb * 82l + var row = ksp + sb * IQ2S_SSB + let d8 = f16_half_bits(f16_half_bits(f16_half_bits(uint(src[0l]) | (uint(src[1l]) << 8u)))) + row[0l] = uint8(d8 & 0xFFu) + row[1l] = uint8(d8 >> 8u) + row[2l] = uint8(0) + row[3l] = uint8(0) + for (g in range64(16l)) { + let nib = g % 2l == 0l ? int(src[74l + g / 2l]) & 15 : int(src[74l + g / 2l]) >> 4 + row[4l + g] = uint8(1 + 2 * nib) + } + bcopy(kqp + sb * IQ2S_QSB, src + 2l, 72l) + } + } + } + } + } +} + +//! Transcode an IQ2_XS tensor into the iq2xs planes (strides 64/20, exact): the 74B disk +//! superblock's 32 u16 qs words go verbatim, d eighth-ed, the 16 strips decoded to (1 + 2s) - +//! transcode_iq2xs_superblock. +def gguf_transcode_iq2xs(m : GGUFMeta; srcbytes : array | #; name : string; var kq : array; var ks : array; eloff, expect_n : int64; src_off : int64 = 0l) { + let ti = kq_transcode_check(m, name, GGML_TYPE_IQ2_XS, "IQ2_XS", src_off, expect_n) + let nb = expect_n / 256l + if (nb <= 0l) { + return + } + guard_dst(name, "iq2xs quant plane", (eloff / 256l) * IQ2XS_QSB, nb * IQ2XS_QSB, long_length(kq)) + guard_dst(name, "iq2xs scale plane", (eloff / 256l) * IQ2XS_SSB, nb * IQ2XS_SSB, long_length(ks)) + with_tensor_view(m, srcbytes, ti) $(bytes, tbase) { + let bo = tbase + (src_off / 256l) * 74l + unsafe { + let srcp = addr(bytes[bo]) + var kqp = addr(kq[(eloff / 256l) * IQ2XS_QSB]) + var ksp = addr(ks[(eloff / 256l) * IQ2XS_SSB]) + maybe_parallel_for(0, int(nb), transcode_jobs(nb, 74l)) $(rb, re) { + unsafe { + for (sb in range64(int64(rb), int64(re))) { + let src = srcp + sb * 74l + var row = ksp + sb * IQ2XS_SSB + let d8 = f16_half_bits(f16_half_bits(f16_half_bits(uint(src[0l]) | (uint(src[1l]) << 8u)))) + row[0l] = uint8(d8 & 0xFFu) + row[1l] = uint8(d8 >> 8u) + row[2l] = uint8(0) + row[3l] = uint8(0) + for (g in range64(16l)) { + let nib = g % 2l == 0l ? int(src[66l + g / 2l]) & 15 : int(src[66l + g / 2l]) >> 4 + row[4l + g] = uint8(1 + 2 * nib) + } + bcopy(kqp + sb * IQ2XS_QSB, src + 2l, 64l) + } + } + } + } + } +} + + +//! Transcode an IQ2_XXS tensor into the iq2xxs planes (strides 64/20, exact): the 66B disk +//! superblock's qs region goes verbatim, d eighth-ed, the 8 per-32 strips decoded to +//! (1 + 2s) - transcode_iq2xxs_superblock. +def gguf_transcode_iq2xxs(m : GGUFMeta; srcbytes : array | #; name : string; var kq : array; var ks : array; eloff, expect_n : int64; src_off : int64 = 0l) { + let ti = kq_transcode_check(m, name, GGML_TYPE_IQ2_XXS, "IQ2_XXS", src_off, expect_n) + let nb = expect_n / 256l + if (nb <= 0l) { + return + } + guard_dst(name, "iq2xxs quant plane", (eloff / 256l) * IQ2XXS_QSB, nb * IQ2XXS_QSB, long_length(kq)) + guard_dst(name, "iq2xxs scale plane", (eloff / 256l) * IQ2XXS_SSB, nb * IQ2XXS_SSB, long_length(ks)) + with_tensor_view(m, srcbytes, ti) $(bytes, tbase) { + let bo = tbase + (src_off / 256l) * 66l + unsafe { + let srcp = addr(bytes[bo]) + var kqp = addr(kq[(eloff / 256l) * IQ2XXS_QSB]) + var ksp = addr(ks[(eloff / 256l) * IQ2XXS_SSB]) + maybe_parallel_for(0, int(nb), transcode_jobs(nb, 66l)) $(rb, re) { + unsafe { + for (sb in range64(int64(rb), int64(re))) { + let src = srcp + sb * 66l + var row = ksp + sb * IQ2XXS_SSB + let d8 = f16_half_bits(f16_half_bits(f16_half_bits(uint(src[0l]) | (uint(src[1l]) << 8u)))) + row[0l] = uint8(d8 & 0xFFu) + row[1l] = uint8(d8 >> 8u) + row[2l] = uint8(0) + row[3l] = uint8(0) + for (b in range64(8l)) { + row[4l + b] = uint8(1 + 2 * (int(src[2l + b * 8l + 7l]) >> 4)) + } + for (i in range64(12l, 20l)) { + row[i] = uint8(0) + } + bcopy(kqp + sb * IQ2XXS_QSB, src + 2l, 64l) + } + } + } + } + } +} + //! Transcode a Q5_1 tensor into the q51 planes (per 32-BLOCK strides 20/4 — see Q51_QB): each //! 24B disk block splits into 2B d + 2B m (scale plane) and 16B nibbles + 4B qh (quant plane), //! all verbatim. Exact; `eloff`/`src_off`/`expect_n` are element offsets, % 32. @@ -735,10 +1097,63 @@ def gguf_transcode_q51(m : GGUFMeta; srcbytes : array | #; name : string; } } -//! Decode tensor `name` (F32 or F16 on disk) into dst[dst_off ..] as fp32. Panics if the -//! tensor is missing, has a different element count than expected, or uses a type we don't -//! yet read (quantized types are wired in a later step). -def gguf_read_tensor_f32(m : GGUFMeta; srcbytes : array | #; name : string; var dst : array; dst_off, expect_n : int64; src_off : int64 = 0l) { // nolint:STYLE038 — one arm per on-disk ggml type; each arm is a self-contained decode kernel +//! The kq format tag of a superblock i-quant / Q2_K / Q3_K GGML type (q8 = not one of them). +def gguf_kq_fmt_of_type(gtype : int) : KqFmt { + if (gtype == GGML_TYPE_IQ4_XS) return KqFmt.iq4xs + if (gtype == GGML_TYPE_Q3_K) return KqFmt.k3 + if (gtype == GGML_TYPE_IQ3_S) return KqFmt.iq3s + if (gtype == GGML_TYPE_IQ3_XXS) return KqFmt.iq3xxs + if (gtype == GGML_TYPE_IQ4_NL) return KqFmt.iq4nl + if (gtype == GGML_TYPE_Q2_K) return KqFmt.k2 + if (gtype == GGML_TYPE_IQ2_S) return KqFmt.iq2s + if (gtype == GGML_TYPE_IQ2_XS) return KqFmt.iq2xs + if (gtype == GGML_TYPE_IQ2_XXS) return KqFmt.iq2xxs + return KqFmt.q8 +} + +//! Disk bytes per 256-superblock of the i-quant / Q2_K / Q3_K GGML types the native tier reads +//! (0 = not one of them). IQ4_NL is eight 18B blocks of 32. +def gguf_kq_disk_sbb(gtype : int) : int64 { + if (gtype == GGML_TYPE_IQ4_XS) return 136l + if (gtype == GGML_TYPE_Q3_K || gtype == GGML_TYPE_IQ3_S) return 110l + if (gtype == GGML_TYPE_IQ3_XXS) return 98l + if (gtype == GGML_TYPE_IQ4_NL) return 144l + if (gtype == GGML_TYPE_Q2_K) return 84l + if (gtype == GGML_TYPE_IQ2_S) return 82l + if (gtype == GGML_TYPE_IQ2_XS) return 74l + if (gtype == GGML_TYPE_IQ2_XXS) return 66l + return 0l +} + +//! One superblock's disk -> plane transcode, keyed by format - the array form the tests and the +//! f32 fallback drive (the bulk loaders are the per-format `gguf_transcode_*`). +def kq_transcode_superblock(fmt : KqFmt; bytes : array | #; bo : int64; var kq : array; kqo : int64; var ks : array; kso : int64) { // nolint:STYLE037 — the format ladder + if (fmt == KqFmt.iq4xs) { + transcode_iq4xs_superblock(bytes, bo, kq, kqo, ks, kso) + } elif (fmt == KqFmt.k3) { + transcode_q3k_superblock(bytes, bo, kq, kqo, ks, kso) + } elif (fmt == KqFmt.iq3s) { + transcode_iq3s_superblock(bytes, bo, kq, kqo, ks, kso) + } elif (fmt == KqFmt.iq3xxs) { + transcode_iq3xxs_superblock(bytes, bo, kq, kqo, ks, kso) + } elif (fmt == KqFmt.iq4nl) { + transcode_iq4nl_superblock(bytes, bo, kq, kqo, ks, kso) + } elif (fmt == KqFmt.k2) { + transcode_q2k_superblock(bytes, bo, kq, kqo, ks, kso) + } elif (fmt == KqFmt.iq2s) { + transcode_iq2s_superblock(bytes, bo, kq, kqo, ks, kso) + } elif (fmt == KqFmt.iq2xs) { + transcode_iq2xs_superblock(bytes, bo, kq, kqo, ks, kso) + } elif (fmt == KqFmt.iq2xxs) { + transcode_iq2xxs_superblock(bytes, bo, kq, kqo, ks, kso) + } else { + panic("kq_transcode_superblock: '{fmt}' has no superblock transcode") + } +} + +//! Decode tensor `name` into dst[dst_off ..] as fp32 from any on-disk type the native tier reads. +//! Panics if the tensor is missing, its element count differs, or its type has no decoder here. +def gguf_read_tensor_f32(m : GGUFMeta; srcbytes : array | #; name : string; var dst : array; dst_off, expect_n : int64; src_off : int64 = 0l) { // nolint:STYLE038 — one arm per on-disk upstream type; each arm is a self-contained decode kernel let ti = gguf_find_tensor(m, name) if (ti < 0) { panic("gguf: tensor '{name}' not found") @@ -752,7 +1167,7 @@ def gguf_read_tensor_f32(m : GGUFMeta; srcbytes : array | #; name : strin } let gtype = m.tensors[ti].ggml_type guard_dst(name, "f32 dst", dst_off, expect_n, long_length(dst)) - with_tensor_view(m, srcbytes, ti) $(bytes, base) { // nolint:STYLE037 — the ggml-type ladder; the arms share only the borrowed view + with_tensor_view(m, srcbytes, ti) $(bytes, base) { // nolint:STYLE037 — the upstream-type ladder; the arms share only the borrowed view if (gtype == GGML_TYPE_F32) { unsafe { // raw copy, threaded in u64 words (+ one odd-float tail) let sp = addr(bytes[base + src_off * 4l]) @@ -891,6 +1306,26 @@ def gguf_read_tensor_f32(m : GGUFMeta; srcbytes : array | #; name : strin } } } + } elif (gguf_kq_disk_sbb(gtype) > 0l) { + // transcode one superblock into scratch planes + its plane dequant: the native load's own two steps, value for value + if (src_off % 256l != 0l || expect_n % 256l != 0l) { + panic("gguf: tensor '{name}' superblock slice [{src_off}, +{expect_n}) is not superblock-aligned") + } + let sbb = gguf_kq_disk_sbb(gtype) + let fmt = gguf_kq_fmt_of_type(gtype) + var kq : array + var ks : array + kq |> resize(int(kq_qsb(fmt))) + ks |> resize(int(kq_ssb(fmt))) + let nb = expect_n / 256l + for (blk in range64(nb)) { + let bo = base + (src_off / 256l ++ blk) * sbb + kq_transcode_superblock(fmt, bytes, bo, kq, 0l, ks, 0l) + dequant_kq_plane_sb(fmt, kq, ks, 0l, false, dst, dst_off + blk * 256l) + } + delete kq + delete ks } elif (gtype == GGML_TYPE_MXFP4) { // block of 32: 1 E8M0 scale byte + 16 nibble bytes (low nibbles = elems 0..15, high = 16..31); // x = e2m1[nibble] * scale — both sides carry the ×2/÷2 on-disk convention (see MXFP4_KVALUES) @@ -935,7 +1370,7 @@ def gguf_transcode_bf16(m : GGUFMeta; srcbytes : array | #; name : string } } -//! The on-disk ggml type of tensor `name` (GGML_TYPE_*), or -1 if the tensor is absent. Lets a +//! The on-disk upstream type of tensor `name` (GGML_TYPE_*), or -1 if the tensor is absent. Lets a //! caller pick the direct-transcode path when the disk type already matches the target precision. def gguf_tensor_type(m : GGUFMeta; name : string) : int { let ti = gguf_find_tensor(m, name) diff --git a/modules/dasLLAMA/dasllama/dasllama_gpu_resident.das b/modules/dasLLAMA/dasllama/dasllama_gpu_resident.das index 48b7a198b7..5b848fe4be 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gpu_resident.das +++ b/modules/dasLLAMA/dasllama/dasllama_gpu_resident.das @@ -227,7 +227,7 @@ def private moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, row //! P3 trim (generate-then-free): pack the emb/cls region into embq/embs, mark the model //! trimmed, and FREE the big CPU weight families — the save then writes them as empty planes. //! Arena-only plans qualify (pure resident-driver dense); false = declined, nothing changed. -def trim_model_planes(var t : Model) : bool { +def trim_model_planes(var t : Model) : bool { // nolint:STYLE037,STYLE038 — the flat per-format plane-trim ladder (one arm per KqFmt) is the honest shape if (empty(t.vkplan) || t.config.n_layer_nextn > 0l) { return false } @@ -278,6 +278,53 @@ def trim_model_planes(var t : Model) : bool { t.embs |> reserve_resize(nsb * Q40_SSB) memcpy(addr(t.embq[0]), addr(t.q40q[sb0 * Q40_QSB]), nsb * Q40_QSB) memcpy(addr(t.embs[0]), addr(t.q40s[sb0 * Q40_SSB]), nsb * Q40_SSB) + } elif (t.emb_fmt == KqFmt.iq4xs) { + t.embq |> reserve_resize(nsb * IQ4XS_QSB) + t.embs |> reserve_resize(nsb * IQ4XS_SSB) + memcpy(addr(t.embq[0]), addr(t.iq4xsq[sb0 * IQ4XS_QSB]), nsb * IQ4XS_QSB) + memcpy(addr(t.embs[0]), addr(t.iq4xss[sb0 * IQ4XS_SSB]), nsb * IQ4XS_SSB) + } elif (t.emb_fmt == KqFmt.k3) { + t.embq |> reserve_resize(nsb * K3_QSB) + t.embs |> reserve_resize(nsb * K3_SSB) + memcpy(addr(t.embq[0]), addr(t.k3q[sb0 * K3_QSB]), nsb * K3_QSB) + memcpy(addr(t.embs[0]), addr(t.k3s[sb0 * K3_SSB]), nsb * K3_SSB) + } elif (t.emb_fmt == KqFmt.iq4nl) { + t.embq |> reserve(nsb * Q40_QSB) + t.embq |> resize(nsb * Q40_QSB) + t.embs |> reserve(nsb * Q40_SSB) + t.embs |> resize(nsb * Q40_SSB) + memcpy(addr(t.embq[0]), addr(t.iq4nlq[sb0 * Q40_QSB]), nsb * Q40_QSB) + memcpy(addr(t.embs[0]), addr(t.iq4nls[sb0 * Q40_SSB]), nsb * Q40_SSB) + } elif (t.emb_fmt == KqFmt.k2) { + t.embq |> reserve_resize(nsb * K2_QSB) + t.embs |> reserve_resize(nsb * K2_SSB) + memcpy(addr(t.embq[0]), addr(t.k2q[sb0 * K2_QSB]), nsb * K2_QSB) + memcpy(addr(t.embs[0]), addr(t.k2s[sb0 * K2_SSB]), nsb * K2_SSB) + } elif (t.emb_fmt == KqFmt.iq2s) { + t.embq |> reserve_resize(nsb * IQ2S_QSB) + t.embs |> reserve_resize(nsb * IQ2S_SSB) + memcpy(addr(t.embq[0]), addr(t.iq2sq[sb0 * IQ2S_QSB]), nsb * IQ2S_QSB) + memcpy(addr(t.embs[0]), addr(t.iq2ss[sb0 * IQ2S_SSB]), nsb * IQ2S_SSB) + } elif (t.emb_fmt == KqFmt.iq2xs) { + t.embq |> reserve_resize(nsb * IQ2XS_QSB) + t.embs |> reserve_resize(nsb * IQ2XS_SSB) + memcpy(addr(t.embq[0]), addr(t.iq2xsq[sb0 * IQ2XS_QSB]), nsb * IQ2XS_QSB) + memcpy(addr(t.embs[0]), addr(t.iq2xss[sb0 * IQ2XS_SSB]), nsb * IQ2XS_SSB) + } elif (t.emb_fmt == KqFmt.iq2xxs) { + t.embq |> reserve_resize(nsb * IQ2XXS_QSB) + t.embs |> reserve_resize(nsb * IQ2XXS_SSB) + memcpy(addr(t.embq[0]), addr(t.iq2xxsq[sb0 * IQ2XXS_QSB]), nsb * IQ2XXS_QSB) + memcpy(addr(t.embs[0]), addr(t.iq2xxss[sb0 * IQ2XXS_SSB]), nsb * IQ2XXS_SSB) + } elif (t.emb_fmt == KqFmt.iq3xxs) { + t.embq |> reserve_resize(nsb * IQ3XXS_QSB) + t.embs |> reserve_resize(nsb * IQ3XXS_SSB) + memcpy(addr(t.embq[0]), addr(t.iq3xxsq[sb0 * IQ3XXS_QSB]), nsb * IQ3XXS_QSB) + memcpy(addr(t.embs[0]), addr(t.iq3xxss[sb0 * IQ3XXS_SSB]), nsb * IQ3XXS_SSB) + } elif (t.emb_fmt == KqFmt.iq3s) { + t.embq |> reserve_resize(nsb * IQ3S_QSB) + t.embs |> reserve_resize(nsb * IQ3S_SSB) + memcpy(addr(t.embq[0]), addr(t.iq3sq[sb0 * IQ3S_QSB]), nsb * IQ3S_QSB) + memcpy(addr(t.embs[0]), addr(t.iq3ss[sb0 * IQ3S_SSB]), nsb * IQ3S_SSB) } else { to_log(LOG_WARNING, "dasLLAMA trim: emb fmt '{t.emb_fmt}' has no kq plane pair - declining\n") return false @@ -296,6 +343,24 @@ def trim_model_planes(var t : Model) : bool { delete t.k5s delete t.k6q delete t.k6s + delete t.iq4xsq + delete t.iq4xss + delete t.k3q + delete t.k3s + delete t.iq3sq + delete t.iq3ss + delete t.iq3xxsq + delete t.iq3xxss + delete t.iq4nlq + delete t.iq4nls + delete t.k2q + delete t.k2s + delete t.iq2sq + delete t.iq2ss + delete t.iq2xsq + delete t.iq2xss + delete t.iq2xxsq + delete t.iq2xxss delete t.q40q delete t.q40s delete t.q51q diff --git a/modules/dasLLAMA/dasllama/dasllama_image.das b/modules/dasLLAMA/dasllama/dasllama_image.das index 4329231b91..88d36e48b9 100644 --- a/modules/dasLLAMA/dasllama/dasllama_image.das +++ b/modules/dasLLAMA/dasllama/dasllama_image.das @@ -38,7 +38,7 @@ require dasllama/dasllama_load // WhisperModel.enc) contribute their planes under dotted names ("enc.fblob"); string-array // fields ride the meta blob via serialize_strings — raw string pointers can't be planes. -let IMAGE_VERSION = 17 // 17: merged lineages - master's 14 and this branch's 14-16 (qwen3v f16 twin plane, devwf16 bake) numbered independently +let IMAGE_VERSION = 26 // the iq2xxs plane pair (IQ2_XXS native tier) //! The metal (blob-only) flavor's identity tag: q8 planes ride the 34B block_q8_0 blob and the //! kq scale planes their GPU forms (convert_model_to_metal_blob) — flavors are per-config and @@ -57,7 +57,14 @@ struct ImgSection { [arch(at = "../ARCHITECTURE_IMAGE.md#image-identity-backend-order")] def image_identity(tag : string = ""; quant : string = "q8") : string { - return dlim_identity(dlim_config_current(quant), IMAGE_VERSION, tag) + return image_identity_of(dlim_config_current(quant), tag) +} + +//! The identity of an explicit configuration — the ONE spelling every identity consumer (the +//! loader, the peek verdicts, the converter's want/have compare, the tests) formats through, so +//! the schema version and the pack-code version can never be folded in two different ways. +def image_identity_of(c : DlimConfiguration; tag : string = "") : string { + return dlim_identity(c, IMAGE_VERSION, tag, PACK_VERSION) } // a 64-bit identity hash per file: a collision would leave two identities re-saving over one path forever @@ -634,6 +641,15 @@ def serialize_image_meta(var arch : Archive; var t : Model) { arch |> serialize_raw(t.kq_repack_mr5) arch |> serialize_raw(t.kq_repack_mr6) arch |> serialize_raw(t.kq_repack_mr40) + arch |> serialize_raw(t.kq_repack_mr44) + arch |> serialize_raw(t.kq_repack_mr3) + arch |> serialize_raw(t.kq_repack_mr33) + arch |> serialize_raw(t.kq_repack_mr34) + arch |> serialize_raw(t.kq_repack_mr45) + arch |> serialize_raw(t.kq_repack_mr2) + arch |> serialize_raw(t.kq_repack_mr23) + arch |> serialize_raw(t.kq_repack_mr24) + arch |> serialize_raw(t.kq_repack_mr25) arch |> serialize_raw(t.wcls_fmt) arch |> serialize_raw(t.emb_fmt) arch |> serialize_raw(t.ple_emb_fmt) @@ -672,8 +688,8 @@ def serialize_image_meta(var arch : Archive; var t : Model) { arch |> serialize_raw(t.mtp_headnorm_off) } -// 66 serialized fields + 3 deliberate skips (blocks, image_map, image_bytes) -let IMAGE_META_FIELDS = 66 + 3 +// the 3 deliberate skips: blocks, image_map, image_bytes +let IMAGE_META_FIELDS = 75 + 3 //! Count of meta-carried fields of any struct: non-arrays plus string arrays (those cannot //! be raw planes — see serialize_strings). Pair with a per-type field-count constant to @@ -766,7 +782,7 @@ def private build_image(var t; dst : string; in_mem : bool; extra : uint64; plane : block<(name : string; var w : ImgWriter; var sections : array) : bool>) : ImgWriter { // one config snapshot serves the meta embed AND the header hash — they must agree let dcfg = dlim_config_current(quant) - let ident = dlim_identity(dcfg, IMAGE_VERSION, tag) + let ident = image_identity_of(dcfg, tag) var fend = 0ul var meta_off = 0ul var meta_bytes = 0ul @@ -957,7 +973,7 @@ def save_model_image(var t : Model; path : string; tag : string = ""; quant : st // streamed planes size as empty, so their bytes plus a page each are added here; over-reserve is free (dwrite_close truncates) def private stream_extra_bytes(t : Model; jobs : array) : uint64 { var extra = 0ul - for (fname in ["qblob", "mblob", "k4q", "k5q", "k6q", "q40q", "q51q"]) { + for (fname in ["qblob", "mblob", "k4q", "k5q", "k6q", "q40q", "q51q", "iq4xsq", "k3q", "iq3sq", "iq3xxsq", "iq4nlq", "k2q", "iq2sq", "iq2xsq", "iq2xxsq"]) { let sb = stream_plane_bytes(t, jobs, fname) if (sb > 0l) { extra += uint64(sb) + uint64(IMAGE_PAGE) diff --git a/modules/dasLLAMA/dasllama/dasllama_kqformat.das b/modules/dasLLAMA/dasllama/dasllama_kqformat.das index 2cfacb9033..8312aed05d 100644 --- a/modules/dasLLAMA/dasllama/dasllama_kqformat.das +++ b/modules/dasLLAMA/dasllama/dasllama_kqformat.das @@ -13,7 +13,8 @@ module dasllama_kqformat shared public require dasllama/dasllama_lint public //! Per-weight storage format under a q8-mode load with native planes: q8 = qblob/qscales; -//! k4/k5/k6/q40 = superblock plane pairs (mixed PER TENSOR); q51 = native Q5_1 per-32 planes. +//! k4/k5/k6/q40/iq4xs/k3/iq3s/iq3xxs/iq4nl/k2/iq2s/iq2xs/iq2xxs = superblock plane pairs (mixed PER TENSOR); q51 = native Q5_1 per-32 +//! planes. New members append — the int value is the device stack tag and the image plane id. enum KqFmt : uint8 { q8 k4 @@ -21,19 +22,28 @@ enum KqFmt : uint8 { k6 q40 q51 + iq4xs + k3 + iq3s + iq3xxs + iq4nl + k2 + iq2s + iq2xs + iq2xxs } //! The superblock-lattice formats (Q8_K-form activations, % 256 rows, repack + stamped kq //! kernels). q51 deliberately fails this: it rides per-32 planes with Q8_0-form activations — //! a `fmt != KqFmt.q8` test does not imply the kq lattice; branch on kq_sb where it does. -def kq_sb(f : KqFmt) : bool => f == KqFmt.k4 || f == KqFmt.k5 || f == KqFmt.k6 || f == KqFmt.q40 +def kq_sb(f : KqFmt) : bool => f == KqFmt.k4 || f == KqFmt.k5 || f == KqFmt.k6 || f == KqFmt.q40 || f == KqFmt.iq4xs || f == KqFmt.k3 || f == KqFmt.iq3s || f == KqFmt.iq3xxs || f == KqFmt.iq4nl || f == KqFmt.k2 || f == KqFmt.iq2s || f == KqFmt.iq2xs || f == KqFmt.iq2xxs //! The same predicate over the int id space the GPU drivers carry per stack/plane. -def kq_sb(fmt : int) : bool => fmt == int(KqFmt.k4) || fmt == int(KqFmt.k5) || fmt == int(KqFmt.k6) || fmt == int(KqFmt.q40) +def kq_sb(fmt : int) : bool => fmt == int(KqFmt.k4) || fmt == int(KqFmt.k5) || fmt == int(KqFmt.k6) || fmt == int(KqFmt.q40) || fmt == int(KqFmt.iq4xs) || fmt == int(KqFmt.k3) || fmt == int(KqFmt.iq3s) || fmt == int(KqFmt.iq3xxs) || fmt == int(KqFmt.iq4nl) || fmt == int(KqFmt.k2) || fmt == int(KqFmt.iq2s) || fmt == int(KqFmt.iq2xs) || fmt == int(KqFmt.iq2xxs) -//! KqFmt -> the kernel/IR format id space (4/5/6/40) — the ONE bridge between the enum and the -//! int ids the generated kernels take as runtime params. q8/q51 are not kq superblock -//! formats and panic. +//! KqFmt -> the kernel/IR format id space — the ONE bridge between the enum and the int ids the +//! generated kernels take as runtime params. Mnemonic ids: a K-quant is its bit width (3/4/5/6), +//! Q4_0 is 40, an i-quant is bit width x 10 + a variant digit (IQ4_XS = 44, IQ3_S = 33). q8/q51 panic. def kq_schema_id(f : KqFmt) : int { if (f == KqFmt.k4) { return 4 @@ -47,6 +57,33 @@ def kq_schema_id(f : KqFmt) : int { if (f == KqFmt.q40) { return 40 } + if (f == KqFmt.iq4xs) { + return 44 + } + if (f == KqFmt.k3) { + return 3 + } + if (f == KqFmt.iq3s) { + return 33 + } + if (f == KqFmt.iq3xxs) { + return 34 + } + if (f == KqFmt.iq4nl) { + return 45 + } + if (f == KqFmt.k2) { + return 2 + } + if (f == KqFmt.iq2s) { + return 23 + } + if (f == KqFmt.iq2xs) { + return 24 + } + if (f == KqFmt.iq2xxs) { + return 25 + } panic("kq_schema_id: not a kq superblock format") return 0 } @@ -63,12 +100,670 @@ let Q40_QSB = 128l // Q4_0 quant plane: 128 nibble bytes (k4 tiling) let Q40_SSB = 16l // Q4_0 scale plane: 8 x f16 d, verbatim let Q51_QB = 20l // q5_1 quant plane PER 32-BLOCK: 16 nibble bytes (k/k+16 pairing) + 4 qh let Q51_SB = 4l // q5_1 scale plane PER 32-BLOCK: f16 d + f16 m +let IQ4XS_QSB = 128l // IQ4_XS quant plane: 128 nibble bytes, the disk k/k+16 pairing (q40 tiling) +let IQ4XS_SSB = 20l // IQ4_XS scale plane: f16 d, 2 pad, 8 x int8 (ls - 32), 8 pad — the k4 decoded-row shape +let K3_QSB = 96l // Q3_K quant plane: 64 qs bytes (four 2-bit lanes each) + 32 hmask bytes, verbatim disk order +let K3_SSB = 18l // Q3_K scale plane: 16 x int8 (6-bit scale - 32, decoded at transcode) + f16 d — the k6 row +let IQ3S_QSB = 104l // IQ3_S quant plane: [64 grid-index qs][8 qh ninth bits][32 sign bytes], verbatim disk order +let IQ3S_SSB = 20l // IQ3_S scale plane: f16 d, 2 pad, 8 x int8 (1 + 2s), 8 pad — the iq4xs/k4 row shape +let IQ3XXS_QSB = 96l // IQ3_XXS quant plane: [64 grid-index qs][32 aux bytes: per block 4x7-bit sign indices + 4-bit scale], verbatim disk order +let IQ3XXS_SSB = 20l // IQ3_XXS scale plane: f16 d HALVED, 2 pad, 8 x uint8 (2*ls + 1), 8 pad — the iq3s row shape and fold (the format's global 0.25 = the halved d x the halved grid) +let K2_QSB = 64l // Q2_K quant plane: 64 qs bytes (four 2-bit lanes each), verbatim disk order +let K2_SSB = 20l // Q2_K scale plane: [f16 d][f16 dmin][16 sc/min nibble-pair bytes] - the disk scale block, header first +let IQ2S_QSB = 72l // IQ2_S quant plane: [32 grid-index low bytes][32 sign bytes][8 qh], verbatim disk order +let IQ2S_SSB = 20l // IQ2_S scale plane: f16 d EIGHTH-ed (the disk (0.5+ls)*0.25 = (2ls+1)/8), 2 pad, 16 x uint8 (2ls+1) - per-16 strips in the 20B row +let IQ2XS_QSB = 64l // IQ2_XS quant plane: 32 u16 words ([9-bit grid index][7-bit ksigns index]), verbatim disk order +let IQ2XS_SSB = 20l // IQ2_XS scale plane: the iq2s row exactly - f16 d EIGHTH-ed, 2 pad, 16 x uint8 (2ls+1) per-16 strips +let IQ2XXS_QSB = 64l // IQ2_XXS quant plane: 8 blocks x [4 iq2xxs_grid byte indices][aux32: 4 x 7-bit ksigns + 4-bit ls], verbatim disk order +let IQ2XXS_SSB = 20l // IQ2_XXS scale plane: the iq3xxs row shape - f16 d EIGHTH-ed, 2 pad, 8 x uint8 (2ls+1) per-32 strips, 8 pad +// IQ4_NL reuses Q40_QSB/Q40_SSB outright: the disk bytes are q40's (f16 d + 16 nibble bytes +// per 32-block, the k/k+16 pairing) — only the nibble SEMANTICS differ (IQ4NL_LUT, no -8). + +//! The IQ4_NL / IQ4_XS nibble codebook: weight = scale x IQ4NL_LUT[nibble] (upstream's kvalues_iq4nl). +//! Main-context use only (tests, oracles, the emitter's constant bake): a kernel that can run on a +//! forked worker reads a module global as zeros - kernels take `iq4nl_lut()` as a local instead. +let IQ4NL_LUT = fixed_array(-127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113) + +//! The same codebook as a per-call local (16 bytes on the caller's frame) - the form every +//! worker-run kernel reads. +def iq4nl_lut() : int8[16] => fixed_array(-127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113) + +//! The IQ3_S grid (upstream's iq3s_grid): entry qs | (qh bit << 8) packs FOUR weight magnitudes as +//! bytes; the per-element sign rides the block's own sign bytes. Per-call local (2 KB on the +//! caller's frame) - the form every worker-run kernel reads; IQ3S_GRID is the main-context copy. +def iq3s_grid() : uint[512] { + return fixed_array( + 0x01010101u, 0x01010103u, 0x01010105u, 0x0101010bu, 0x0101010fu, 0x01010301u, 0x01010303u, 0x01010305u, + 0x01010309u, 0x0101030du, 0x01010501u, 0x01010503u, 0x0101050bu, 0x01010707u, 0x01010901u, 0x01010905u, + 0x0101090bu, 0x0101090fu, 0x01010b03u, 0x01010b07u, 0x01010d01u, 0x01010d05u, 0x01010f03u, 0x01010f09u, + 0x01010f0fu, 0x01030101u, 0x01030103u, 0x01030105u, 0x01030109u, 0x01030301u, 0x01030303u, 0x0103030bu, + 0x01030501u, 0x01030507u, 0x0103050fu, 0x01030703u, 0x0103070bu, 0x01030909u, 0x01030d03u, 0x01030d0bu, + 0x01030f05u, 0x01050101u, 0x01050103u, 0x0105010bu, 0x0105010fu, 0x01050301u, 0x01050307u, 0x0105030du, + 0x01050503u, 0x0105050bu, 0x01050701u, 0x01050709u, 0x01050905u, 0x0105090bu, 0x0105090fu, 0x01050b03u, + 0x01050b07u, 0x01050f01u, 0x01050f07u, 0x01070107u, 0x01070303u, 0x0107030bu, 0x01070501u, 0x01070505u, + 0x01070703u, 0x01070707u, 0x0107070du, 0x01070909u, 0x01070b01u, 0x01070b05u, 0x01070d0fu, 0x01070f03u, + 0x01070f0bu, 0x01090101u, 0x01090307u, 0x0109030fu, 0x01090503u, 0x01090509u, 0x01090705u, 0x01090901u, + 0x01090907u, 0x01090b03u, 0x01090f01u, 0x010b0105u, 0x010b0109u, 0x010b0501u, 0x010b0505u, 0x010b050du, + 0x010b0707u, 0x010b0903u, 0x010b090bu, 0x010b090fu, 0x010b0d0du, 0x010b0f07u, 0x010d010du, 0x010d0303u, + 0x010d0307u, 0x010d0703u, 0x010d0b05u, 0x010d0f03u, 0x010f0101u, 0x010f0105u, 0x010f0109u, 0x010f0501u, + 0x010f0505u, 0x010f050du, 0x010f0707u, 0x010f0b01u, 0x010f0b09u, 0x03010101u, 0x03010103u, 0x03010105u, + 0x03010109u, 0x03010301u, 0x03010303u, 0x03010307u, 0x0301030bu, 0x0301030fu, 0x03010501u, 0x03010505u, + 0x03010703u, 0x03010709u, 0x0301070du, 0x03010b09u, 0x03010b0du, 0x03010d03u, 0x03010f05u, 0x03030101u, + 0x03030103u, 0x03030107u, 0x0303010du, 0x03030301u, 0x03030309u, 0x03030503u, 0x03030701u, 0x03030707u, + 0x03030903u, 0x03030b01u, 0x03030b05u, 0x03030f01u, 0x03030f0du, 0x03050101u, 0x03050305u, 0x0305030bu, + 0x0305030fu, 0x03050501u, 0x03050509u, 0x03050705u, 0x03050901u, 0x03050907u, 0x03050b0bu, 0x03050d01u, + 0x03050f05u, 0x03070103u, 0x03070109u, 0x0307010fu, 0x03070301u, 0x03070307u, 0x03070503u, 0x0307050fu, + 0x03070701u, 0x03070709u, 0x03070903u, 0x03070d05u, 0x03070f01u, 0x03090107u, 0x0309010bu, 0x03090305u, + 0x03090309u, 0x03090703u, 0x03090707u, 0x03090905u, 0x0309090du, 0x03090b01u, 0x03090b09u, 0x030b0103u, + 0x030b0301u, 0x030b0307u, 0x030b0503u, 0x030b0701u, 0x030b0705u, 0x030b0b03u, 0x030d0501u, 0x030d0509u, + 0x030d050fu, 0x030d0909u, 0x030d090du, 0x030f0103u, 0x030f0107u, 0x030f0301u, 0x030f0305u, 0x030f0503u, + 0x030f070bu, 0x030f0903u, 0x030f0d05u, 0x030f0f01u, 0x05010101u, 0x05010103u, 0x05010107u, 0x0501010bu, + 0x0501010fu, 0x05010301u, 0x05010305u, 0x05010309u, 0x0501030du, 0x05010503u, 0x05010507u, 0x0501050fu, + 0x05010701u, 0x05010705u, 0x05010903u, 0x05010907u, 0x0501090bu, 0x05010b01u, 0x05010b05u, 0x05010d0fu, + 0x05010f01u, 0x05010f07u, 0x05010f0bu, 0x05030101u, 0x05030105u, 0x05030301u, 0x05030307u, 0x0503030fu, + 0x05030505u, 0x0503050bu, 0x05030703u, 0x05030709u, 0x05030905u, 0x05030b03u, 0x05050103u, 0x05050109u, + 0x0505010fu, 0x05050503u, 0x05050507u, 0x05050701u, 0x0505070fu, 0x05050903u, 0x05050b07u, 0x05050b0fu, + 0x05050f03u, 0x05050f09u, 0x05070101u, 0x05070105u, 0x0507010bu, 0x05070303u, 0x05070505u, 0x05070509u, + 0x05070703u, 0x05070707u, 0x05070905u, 0x05070b01u, 0x05070d0du, 0x05090103u, 0x0509010fu, 0x05090501u, + 0x05090507u, 0x05090705u, 0x0509070bu, 0x05090903u, 0x05090f05u, 0x05090f0bu, 0x050b0109u, 0x050b0303u, + 0x050b0505u, 0x050b070fu, 0x050b0901u, 0x050b0b07u, 0x050b0f01u, 0x050d0101u, 0x050d0105u, 0x050d010fu, + 0x050d0503u, 0x050d0b0bu, 0x050d0d03u, 0x050f010bu, 0x050f0303u, 0x050f050du, 0x050f0701u, 0x050f0907u, + 0x050f0b01u, 0x07010105u, 0x07010303u, 0x07010307u, 0x0701030bu, 0x0701030fu, 0x07010505u, 0x07010703u, + 0x07010707u, 0x0701070bu, 0x07010905u, 0x07010909u, 0x0701090fu, 0x07010b03u, 0x07010d07u, 0x07010f03u, + 0x07030103u, 0x07030107u, 0x0703010bu, 0x07030309u, 0x07030503u, 0x07030507u, 0x07030901u, 0x07030d01u, + 0x07030f05u, 0x07030f0du, 0x07050101u, 0x07050305u, 0x07050501u, 0x07050705u, 0x07050709u, 0x07050b01u, + 0x07070103u, 0x07070301u, 0x07070309u, 0x07070503u, 0x07070507u, 0x0707050fu, 0x07070701u, 0x07070903u, + 0x07070907u, 0x0707090fu, 0x07070b0bu, 0x07070f07u, 0x07090107u, 0x07090303u, 0x0709030du, 0x07090505u, + 0x07090703u, 0x07090b05u, 0x07090d01u, 0x07090d09u, 0x070b0103u, 0x070b0301u, 0x070b0305u, 0x070b050bu, + 0x070b0705u, 0x070b0909u, 0x070b0b0du, 0x070b0f07u, 0x070d030du, 0x070d0903u, 0x070f0103u, 0x070f0107u, + 0x070f0501u, 0x070f0505u, 0x070f070bu, 0x09010101u, 0x09010109u, 0x09010305u, 0x09010501u, 0x09010509u, + 0x0901050fu, 0x09010705u, 0x09010903u, 0x09010b01u, 0x09010f01u, 0x09030105u, 0x0903010fu, 0x09030303u, + 0x09030307u, 0x09030505u, 0x09030701u, 0x0903070bu, 0x09030907u, 0x09030b03u, 0x09030b0bu, 0x09050103u, + 0x09050107u, 0x09050301u, 0x0905030bu, 0x09050503u, 0x09050707u, 0x09050901u, 0x09050b0fu, 0x09050d05u, + 0x09050f01u, 0x09070109u, 0x09070303u, 0x09070307u, 0x09070501u, 0x09070505u, 0x09070703u, 0x0907070bu, + 0x09090101u, 0x09090105u, 0x09090509u, 0x0909070fu, 0x09090901u, 0x09090f03u, 0x090b010bu, 0x090b010fu, + 0x090b0503u, 0x090b0d05u, 0x090d0307u, 0x090d0709u, 0x090d0d01u, 0x090f0301u, 0x090f030bu, 0x090f0701u, + 0x090f0907u, 0x090f0b03u, 0x0b010105u, 0x0b010301u, 0x0b010309u, 0x0b010505u, 0x0b010901u, 0x0b010909u, + 0x0b01090fu, 0x0b010b05u, 0x0b010d0du, 0x0b010f09u, 0x0b030103u, 0x0b030107u, 0x0b03010bu, 0x0b030305u, + 0x0b030503u, 0x0b030705u, 0x0b030f05u, 0x0b050101u, 0x0b050303u, 0x0b050507u, 0x0b050701u, 0x0b05070du, + 0x0b050b07u, 0x0b070105u, 0x0b07010fu, 0x0b070301u, 0x0b07050fu, 0x0b070909u, 0x0b070b03u, 0x0b070d0bu, + 0x0b070f07u, 0x0b090103u, 0x0b090109u, 0x0b090501u, 0x0b090705u, 0x0b09090du, 0x0b0b0305u, 0x0b0b050du, + 0x0b0b0b03u, 0x0b0b0b07u, 0x0b0d0905u, 0x0b0f0105u, 0x0b0f0109u, 0x0b0f0505u, 0x0d010303u, 0x0d010307u, + 0x0d01030bu, 0x0d010703u, 0x0d010707u, 0x0d010d01u, 0x0d030101u, 0x0d030501u, 0x0d03050fu, 0x0d030d09u, + 0x0d050305u, 0x0d050709u, 0x0d050905u, 0x0d050b0bu, 0x0d050d05u, 0x0d050f01u, 0x0d070101u, 0x0d070309u, + 0x0d070503u, 0x0d070901u, 0x0d09050bu, 0x0d090907u, 0x0d090d05u, 0x0d0b0101u, 0x0d0b0107u, 0x0d0b0709u, + 0x0d0b0d01u, 0x0d0d010bu, 0x0d0d0901u, 0x0d0f0303u, 0x0d0f0307u, 0x0f010101u, 0x0f010109u, 0x0f01010fu, + 0x0f010501u, 0x0f010505u, 0x0f01070du, 0x0f010901u, 0x0f010b09u, 0x0f010d05u, 0x0f030105u, 0x0f030303u, + 0x0f030509u, 0x0f030907u, 0x0f03090bu, 0x0f050103u, 0x0f050109u, 0x0f050301u, 0x0f05030du, 0x0f050503u, + 0x0f050701u, 0x0f050b03u, 0x0f070105u, 0x0f070705u, 0x0f07070bu, 0x0f070b07u, 0x0f090103u, 0x0f09010bu, + 0x0f090307u, 0x0f090501u, 0x0f090b01u, 0x0f0b0505u, 0x0f0b0905u, 0x0f0d0105u, 0x0f0d0703u, 0x0f0f0101u) +} + +//! Main-context use only (tests, oracles, the emitter's constant bake) - a forked worker reads +//! a module global as zeros; kernels take `iq3s_grid()` instead. +let IQ3S_GRID = iq3s_grid() + +//! The IQ3_XXS grid, every byte HALVED (upstream levels are even, 4..62 -> 2..31): one qs byte = +//! FOUR magnitudes; the halved grid + halved stored d absorb upstream's 0.25 exactly, so the fold +//! is iq3s's d x strip x byte. Per-call local (1 KB) - the worker-safe form kernels read. +def iq3xxs_grid() : uint[256] { + return fixed_array( + 0x02020202u, 0x0202020au, 0x02020212u, 0x02020606u, 0x0202060eu, 0x0202061fu, 0x02020a02u, 0x02020a0au, + 0x02020e06u, 0x0202120au, 0x02021f0eu, 0x02021f16u, 0x02060206u, 0x0206020eu, 0x02060602u, 0x0206060au, + 0x02060a06u, 0x02060a16u, 0x02060e02u, 0x02060e0au, 0x02061206u, 0x02061612u, 0x02061f02u, 0x020a0202u, + 0x020a020au, 0x020a0212u, 0x020a0606u, 0x020a0a02u, 0x020a0a0au, 0x020a0e06u, 0x020a0e0eu, 0x020a0e1fu, + 0x020a1606u, 0x020a161fu, 0x020a1f16u, 0x020e0206u, 0x020e021fu, 0x020e0602u, 0x020e060au, 0x020e0a16u, + 0x020e1f02u, 0x0212060eu, 0x02120e1fu, 0x02121212u, 0x0212161fu, 0x02121f0eu, 0x02121f16u, 0x02160206u, + 0x0216021fu, 0x02160e0au, 0x0216160au, 0x021a0e16u, 0x021a1a12u, 0x021f0602u, 0x021f0612u, 0x021f061au, + 0x021f120eu, 0x021f1a06u, 0x06020206u, 0x0602020eu, 0x06020602u, 0x0602060au, 0x06020a06u, 0x06020a0eu, + 0x06020e02u, 0x06020e0au, 0x06020e12u, 0x0602121fu, 0x06021602u, 0x06060202u, 0x0606020au, 0x06060606u, + 0x06060a02u, 0x06060a0au, 0x060a0206u, 0x060a020eu, 0x060a0602u, 0x060a060au, 0x060a0a06u, 0x060a0e02u, + 0x060a1f0au, 0x060e0202u, 0x060e020au, 0x060e0a02u, 0x060e0e06u, 0x060e121au, 0x060e1a1au, 0x06120206u, + 0x06120216u, 0x06121602u, 0x06160a02u, 0x06160a12u, 0x0616121au, 0x06161f06u, 0x061a0216u, 0x061f0a0au, + 0x061f1202u, 0x0a020202u, 0x0a02020au, 0x0a020606u, 0x0a02060eu, 0x0a020a02u, 0x0a020a0au, 0x0a020a1au, + 0x0a020e06u, 0x0a02120au, 0x0a060206u, 0x0a06020eu, 0x0a060216u, 0x0a060602u, 0x0a06060au, 0x0a060a06u, + 0x0a060e02u, 0x0a061a0eu, 0x0a061a1fu, 0x0a061f02u, 0x0a0a0202u, 0x0a0a020au, 0x0a0a0606u, 0x0a0a061fu, + 0x0a0a0a02u, 0x0a0a0a0au, 0x0a0a0e1fu, 0x0a0a1202u, 0x0a0a1616u, 0x0a0e0206u, 0x0a0e0602u, 0x0a0e0612u, + 0x0a0e1f02u, 0x0a0e1f12u, 0x0a120e16u, 0x0a12160eu, 0x0a16020eu, 0x0a160a1fu, 0x0a161206u, 0x0a161f12u, + 0x0a1f0206u, 0x0a1f020eu, 0x0a1f061au, 0x0a1f1216u, 0x0e020206u, 0x0e020602u, 0x0e02060au, 0x0e020a06u, + 0x0e020a0eu, 0x0e021602u, 0x0e021a16u, 0x0e021f0au, 0x0e060202u, 0x0e06020au, 0x0e060a02u, 0x0e060e06u, + 0x0e061212u, 0x0e06121au, 0x0e0a0206u, 0x0e0a020eu, 0x0e0a0602u, 0x0e0a0a16u, 0x0e0a160au, 0x0e0a1f0au, + 0x0e0e0606u, 0x0e0e0e0eu, 0x0e120e02u, 0x0e12121fu, 0x0e121f0au, 0x0e160202u, 0x0e16021au, 0x0e160a0au, + 0x0e161616u, 0x0e1a0612u, 0x0e1a0e1au, 0x0e1a1a0eu, 0x0e1f0e0eu, 0x0e1f1a02u, 0x12020212u, 0x1202061fu, + 0x12020e16u, 0x12020e1fu, 0x1202160eu, 0x1202161fu, 0x12061f12u, 0x120a0a02u, 0x120a0e1fu, 0x120a1202u, + 0x120a1a02u, 0x120a1a1au, 0x120e021fu, 0x120e1216u, 0x12120212u, 0x12121606u, 0x12121a12u, 0x12160a16u, + 0x1216120eu, 0x12161f02u, 0x121f0216u, 0x121f0602u, 0x121f060au, 0x121f0e02u, 0x1602060au, 0x16021206u, + 0x16021f02u, 0x16060202u, 0x1606021au, 0x16060a1au, 0x16061616u, 0x160a0612u, 0x160a0e0au, 0x160a1f0au, + 0x160e020au, 0x160e160eu, 0x16120602u, 0x16120a0eu, 0x16120a1fu, 0x16121f0au, 0x1616020au, 0x16160e06u, + 0x161a1602u, 0x161f0a12u, 0x161f120au, 0x1a020a12u, 0x1a021212u, 0x1a02121au, 0x1a021a12u, 0x1a060a06u, + 0x1a061a06u, 0x1a0a061fu, 0x1a0a1a12u, 0x1a0e0e02u, 0x1a0e0e1au, 0x1a121212u, 0x1a160216u, 0x1a16160au, + 0x1a1a0e0eu, 0x1a1f020eu, 0x1a1f0a06u, 0x1f02020eu, 0x1f020216u, 0x1f02021fu, 0x1f020602u, 0x1f020e0au, + 0x1f02160au, 0x1f060a1au, 0x1f061202u, 0x1f0a060au, 0x1f0a1216u, 0x1f0a160au, 0x1f0e0202u, 0x1f0e0616u, + 0x1f0e0e0eu, 0x1f0e1a02u, 0x1f120a06u, 0x1f121206u, 0x1f160202u, 0x1f16020au, 0x1f160a12u, 0x1f1a0e02u) +} + +//! Main-context copy of the halved IQ3_XXS grid (tests, oracles, emitter bakes). +let IQ3XXS_GRID = iq3xxs_grid() + +//! upstream's ksigns_iq2xs: 7-bit sign index -> 8 sign bits (bit 7 = even parity). Shared by the +//! IQ3_XXS and IQ2 families. Per-call local - the worker-safe form. +def ksigns_iq2xs() : uint8[128] { + return fixed_array( + 0x00, 0x81, 0x82, 0x03, 0x84, 0x05, 0x06, 0x87, 0x88, 0x09, 0x0a, 0x8b, 0x0c, 0x8d, 0x8e, 0x0f, + 0x90, 0x11, 0x12, 0x93, 0x14, 0x95, 0x96, 0x17, 0x18, 0x99, 0x9a, 0x1b, 0x9c, 0x1d, 0x1e, 0x9f, + 0xa0, 0x21, 0x22, 0xa3, 0x24, 0xa5, 0xa6, 0x27, 0x28, 0xa9, 0xaa, 0x2b, 0xac, 0x2d, 0x2e, 0xaf, + 0x30, 0xb1, 0xb2, 0x33, 0xb4, 0x35, 0x36, 0xb7, 0xb8, 0x39, 0x3a, 0xbb, 0x3c, 0xbd, 0xbe, 0x3f, + 0xc0, 0x41, 0x42, 0xc3, 0x44, 0xc5, 0xc6, 0x47, 0x48, 0xc9, 0xca, 0x4b, 0xcc, 0x4d, 0x4e, 0xcf, + 0x50, 0xd1, 0xd2, 0x53, 0xd4, 0x55, 0x56, 0xd7, 0xd8, 0x59, 0x5a, 0xdb, 0x5c, 0xdd, 0xde, 0x5f, + 0x60, 0xe1, 0xe2, 0x63, 0xe4, 0x65, 0x66, 0xe7, 0xe8, 0x69, 0x6a, 0xeb, 0x6c, 0xed, 0xee, 0x6f, + 0xf0, 0x71, 0x72, 0xf3, 0x74, 0xf5, 0xf6, 0x77, 0x78, 0xf9, 0xfa, 0x7b, 0xfc, 0x7d, 0x7e, 0xff) +} + +//! Main-context copy of ksigns_iq2xs. +let KSIGNS_IQ2XS = ksigns_iq2xs() + +//! Halve an f16 bit pattern exactly (exponent decrement; subnormal results truncate-halve). +//! The IQ3_XXS transcode folds upstream's 0.25 into the stored d (one half) and the grid (the other). +def f16_half_bits(u : uint) : uint { + let e = (u >> 10u) & 0x1Fu + if (e >= 2u) { + return u - 0x400u + } + if (e == 1u) { + return (u & 0x8000u) | ((0x400u | (u & 0x3FFu)) >> 1u) + } + return (u & 0x8000u) | ((u & 0x3FFu) >> 1u) +} + + +//! The IQ2_S grid (upstream's iq2s_grid, 1024 u64 entries as low/high uint pairs): a 10-bit index +//! (qs byte | qh 2 bits) selects EIGHT magnitudes ({8, 25, 43}); signs ride the block's own +//! sign bytes. Per-call local (8 KB) - the worker-safe form; IQ2S_GRID is the main-context copy. +def iq2s_grid2() : uint[2048] { // nolint:STYLE038 - a 2048-word data table, not splittable + return fixed_array( + 0x08080808, 0x08080808, 0x0808082b, 0x08080808, 0x08081919, 0x08080808, 0x08082b08, 0x08080808, + 0x08082b2b, 0x08080808, 0x08190819, 0x08080808, 0x08191908, 0x08080808, 0x0819192b, 0x08080808, + 0x08192b19, 0x08080808, 0x082b0808, 0x08080808, 0x082b082b, 0x08080808, 0x082b1919, 0x08080808, + 0x082b2b08, 0x08080808, 0x19080819, 0x08080808, 0x19081908, 0x08080808, 0x1908192b, 0x08080808, + 0x19082b19, 0x08080808, 0x19190808, 0x08080808, 0x1919082b, 0x08080808, 0x19191919, 0x08080808, + 0x19192b08, 0x08080808, 0x192b0819, 0x08080808, 0x192b1908, 0x08080808, 0x192b192b, 0x08080808, + 0x192b2b19, 0x08080808, 0x2b080808, 0x08080808, 0x2b08082b, 0x08080808, 0x2b081919, 0x08080808, + 0x2b082b08, 0x08080808, 0x2b190819, 0x08080808, 0x2b191908, 0x08080808, 0x2b2b0808, 0x08080808, + 0x2b2b1919, 0x08080808, 0x2b2b2b2b, 0x08080808, 0x08080819, 0x08080819, 0x08081908, 0x08080819, + 0x0808192b, 0x08080819, 0x08082b19, 0x08080819, 0x08190808, 0x08080819, 0x0819082b, 0x08080819, + 0x08191919, 0x08080819, 0x08192b08, 0x08080819, 0x082b0819, 0x08080819, 0x082b1908, 0x08080819, + 0x19080808, 0x08080819, 0x1908082b, 0x08080819, 0x19081919, 0x08080819, 0x19082b08, 0x08080819, + 0x19190819, 0x08080819, 0x19191908, 0x08080819, 0x1919192b, 0x08080819, 0x19192b19, 0x08080819, + 0x192b0808, 0x08080819, 0x192b1919, 0x08080819, 0x192b2b08, 0x08080819, 0x2b080819, 0x08080819, + 0x2b081908, 0x08080819, 0x2b190808, 0x08080819, 0x2b19082b, 0x08080819, 0x2b191919, 0x08080819, + 0x2b2b0819, 0x08080819, 0x2b2b1908, 0x08080819, 0x08080808, 0x0808082b, 0x0808082b, 0x0808082b, + 0x08081919, 0x0808082b, 0x08082b08, 0x0808082b, 0x08190819, 0x0808082b, 0x08191908, 0x0808082b, + 0x082b0808, 0x0808082b, 0x082b2b2b, 0x0808082b, 0x19080819, 0x0808082b, 0x19081908, 0x0808082b, + 0x1908192b, 0x0808082b, 0x19082b19, 0x0808082b, 0x19190808, 0x0808082b, 0x19191919, 0x0808082b, + 0x2b080808, 0x0808082b, 0x2b081919, 0x0808082b, 0x2b082b2b, 0x0808082b, 0x2b191908, 0x0808082b, + 0x2b2b082b, 0x0808082b, 0x08080819, 0x08081908, 0x08081908, 0x08081908, 0x0808192b, 0x08081908, + 0x08082b19, 0x08081908, 0x08190808, 0x08081908, 0x0819082b, 0x08081908, 0x08191919, 0x08081908, + 0x08192b08, 0x08081908, 0x082b0819, 0x08081908, 0x082b1908, 0x08081908, 0x082b192b, 0x08081908, + 0x082b2b19, 0x08081908, 0x19080808, 0x08081908, 0x1908082b, 0x08081908, 0x19081919, 0x08081908, + 0x19082b08, 0x08081908, 0x19082b2b, 0x08081908, 0x19190819, 0x08081908, 0x19191908, 0x08081908, + 0x1919192b, 0x08081908, 0x19192b19, 0x08081908, 0x192b0808, 0x08081908, 0x192b082b, 0x08081908, + 0x192b1919, 0x08081908, 0x2b080819, 0x08081908, 0x2b081908, 0x08081908, 0x2b08192b, 0x08081908, + 0x2b082b19, 0x08081908, 0x2b190808, 0x08081908, 0x2b191919, 0x08081908, 0x2b192b08, 0x08081908, + 0x2b2b0819, 0x08081908, 0x2b2b1908, 0x08081908, 0x08080808, 0x08081919, 0x0808082b, 0x08081919, + 0x08081919, 0x08081919, 0x08082b08, 0x08081919, 0x08082b2b, 0x08081919, 0x08190819, 0x08081919, + 0x08191908, 0x08081919, 0x0819192b, 0x08081919, 0x08192b19, 0x08081919, 0x082b0808, 0x08081919, + 0x082b1919, 0x08081919, 0x082b2b08, 0x08081919, 0x19080819, 0x08081919, 0x19081908, 0x08081919, + 0x1908192b, 0x08081919, 0x19082b19, 0x08081919, 0x19190808, 0x08081919, 0x1919082b, 0x08081919, + 0x19191919, 0x08081919, 0x19192b08, 0x08081919, 0x192b0819, 0x08081919, 0x192b1908, 0x08081919, + 0x2b080808, 0x08081919, 0x2b08082b, 0x08081919, 0x2b081919, 0x08081919, 0x2b082b08, 0x08081919, + 0x2b190819, 0x08081919, 0x2b191908, 0x08081919, 0x2b2b0808, 0x08081919, 0x08080819, 0x0808192b, + 0x08081908, 0x0808192b, 0x0808192b, 0x0808192b, 0x08082b19, 0x0808192b, 0x08190808, 0x0808192b, + 0x08191919, 0x0808192b, 0x19080808, 0x0808192b, 0x19081919, 0x0808192b, 0x19082b08, 0x0808192b, + 0x19190819, 0x0808192b, 0x19191908, 0x0808192b, 0x192b0808, 0x0808192b, 0x2b080819, 0x0808192b, + 0x2b081908, 0x0808192b, 0x2b190808, 0x0808192b, 0x08080808, 0x08082b08, 0x0808082b, 0x08082b08, + 0x08081919, 0x08082b08, 0x08082b08, 0x08082b08, 0x08190819, 0x08082b08, 0x08191908, 0x08082b08, + 0x0819192b, 0x08082b08, 0x08192b19, 0x08082b08, 0x082b0808, 0x08082b08, 0x082b1919, 0x08082b08, + 0x082b2b2b, 0x08082b08, 0x19080819, 0x08082b08, 0x19081908, 0x08082b08, 0x1908192b, 0x08082b08, + 0x19082b19, 0x08082b08, 0x19190808, 0x08082b08, 0x1919082b, 0x08082b08, 0x19191919, 0x08082b08, + 0x19192b08, 0x08082b08, 0x192b0819, 0x08082b08, 0x192b1908, 0x08082b08, 0x2b080808, 0x08082b08, + 0x2b081919, 0x08082b08, 0x2b191908, 0x08082b08, 0x2b2b2b2b, 0x08082b08, 0x08080819, 0x08082b19, + 0x08081908, 0x08082b19, 0x08190808, 0x08082b19, 0x0819082b, 0x08082b19, 0x08191919, 0x08082b19, + 0x08192b08, 0x08082b19, 0x082b0819, 0x08082b19, 0x19080808, 0x08082b19, 0x19081919, 0x08082b19, + 0x19082b08, 0x08082b19, 0x19190819, 0x08082b19, 0x19191908, 0x08082b19, 0x192b0808, 0x08082b19, + 0x2b080819, 0x08082b19, 0x2b190808, 0x08082b19, 0x08080808, 0x08082b2b, 0x08190819, 0x08082b2b, + 0x08191908, 0x08082b2b, 0x082b082b, 0x08082b2b, 0x082b2b08, 0x08082b2b, 0x082b2b2b, 0x08082b2b, + 0x19190808, 0x08082b2b, 0x2b192b19, 0x08082b2b, 0x08080819, 0x08190808, 0x08081908, 0x08190808, + 0x0808192b, 0x08190808, 0x08082b19, 0x08190808, 0x08190808, 0x08190808, 0x0819082b, 0x08190808, + 0x08191919, 0x08190808, 0x08192b08, 0x08190808, 0x082b0819, 0x08190808, 0x082b1908, 0x08190808, + 0x082b192b, 0x08190808, 0x19080808, 0x08190808, 0x1908082b, 0x08190808, 0x19081919, 0x08190808, + 0x19082b08, 0x08190808, 0x19190819, 0x08190808, 0x19191908, 0x08190808, 0x1919192b, 0x08190808, + 0x19192b19, 0x08190808, 0x192b0808, 0x08190808, 0x192b082b, 0x08190808, 0x192b1919, 0x08190808, + 0x192b2b08, 0x08190808, 0x2b080819, 0x08190808, 0x2b081908, 0x08190808, 0x2b08192b, 0x08190808, + 0x2b190808, 0x08190808, 0x2b191919, 0x08190808, 0x2b192b08, 0x08190808, 0x2b2b0819, 0x08190808, + 0x2b2b1908, 0x08190808, 0x08080808, 0x08190819, 0x0808082b, 0x08190819, 0x08081919, 0x08190819, + 0x08082b08, 0x08190819, 0x08082b2b, 0x08190819, 0x08190819, 0x08190819, 0x08191908, 0x08190819, + 0x0819192b, 0x08190819, 0x08192b19, 0x08190819, 0x082b0808, 0x08190819, 0x082b082b, 0x08190819, + 0x082b1919, 0x08190819, 0x082b2b08, 0x08190819, 0x19080819, 0x08190819, 0x19081908, 0x08190819, + 0x1908192b, 0x08190819, 0x19082b19, 0x08190819, 0x19190808, 0x08190819, 0x1919082b, 0x08190819, + 0x19191919, 0x08190819, 0x19192b08, 0x08190819, 0x192b0819, 0x08190819, 0x192b1908, 0x08190819, + 0x2b080808, 0x08190819, 0x2b08082b, 0x08190819, 0x2b081919, 0x08190819, 0x2b082b08, 0x08190819, + 0x2b190819, 0x08190819, 0x2b191908, 0x08190819, 0x08080819, 0x0819082b, 0x08081908, 0x0819082b, + 0x08082b19, 0x0819082b, 0x08190808, 0x0819082b, 0x08191919, 0x0819082b, 0x082b0819, 0x0819082b, + 0x082b1908, 0x0819082b, 0x19080808, 0x0819082b, 0x19081919, 0x0819082b, 0x19190819, 0x0819082b, + 0x19191908, 0x0819082b, 0x2b080819, 0x0819082b, 0x2b081908, 0x0819082b, 0x2b190808, 0x0819082b, + 0x08080808, 0x08191908, 0x0808082b, 0x08191908, 0x08081919, 0x08191908, 0x08082b08, 0x08191908, + 0x08190819, 0x08191908, 0x08191908, 0x08191908, 0x0819192b, 0x08191908, 0x08192b19, 0x08191908, + 0x082b0808, 0x08191908, 0x082b1919, 0x08191908, 0x082b2b08, 0x08191908, 0x19080819, 0x08191908, + 0x19081908, 0x08191908, 0x1908192b, 0x08191908, 0x19082b19, 0x08191908, 0x19190808, 0x08191908, + 0x1919082b, 0x08191908, 0x19191919, 0x08191908, 0x19192b08, 0x08191908, 0x192b0819, 0x08191908, + 0x192b1908, 0x08191908, 0x2b080808, 0x08191908, 0x2b08082b, 0x08191908, 0x2b081919, 0x08191908, + 0x2b082b08, 0x08191908, 0x2b190819, 0x08191908, 0x2b191908, 0x08191908, 0x2b2b0808, 0x08191908, + 0x08080819, 0x08191919, 0x08081908, 0x08191919, 0x0808192b, 0x08191919, 0x08082b19, 0x08191919, + 0x08190808, 0x08191919, 0x0819082b, 0x08191919, 0x08191919, 0x08191919, 0x08192b08, 0x08191919, + 0x082b0819, 0x08191919, 0x082b1908, 0x08191919, 0x19080808, 0x08191919, 0x1908082b, 0x08191919, + 0x19081919, 0x08191919, 0x19082b08, 0x08191919, 0x19190819, 0x08191919, 0x19191908, 0x08191919, + 0x192b0808, 0x08191919, 0x2b080819, 0x08191919, 0x2b081908, 0x08191919, 0x2b190808, 0x08191919, + 0x08080808, 0x0819192b, 0x08081919, 0x0819192b, 0x08082b08, 0x0819192b, 0x08190819, 0x0819192b, + 0x08191908, 0x0819192b, 0x082b0808, 0x0819192b, 0x19080819, 0x0819192b, 0x19081908, 0x0819192b, + 0x19190808, 0x0819192b, 0x2b080808, 0x0819192b, 0x2b2b2b2b, 0x0819192b, 0x08080819, 0x08192b08, + 0x08081908, 0x08192b08, 0x0808192b, 0x08192b08, 0x08082b19, 0x08192b08, 0x08190808, 0x08192b08, + 0x08191919, 0x08192b08, 0x08192b08, 0x08192b08, 0x082b0819, 0x08192b08, 0x19080808, 0x08192b08, + 0x1908082b, 0x08192b08, 0x19081919, 0x08192b08, 0x19082b08, 0x08192b08, 0x19190819, 0x08192b08, + 0x19191908, 0x08192b08, 0x192b0808, 0x08192b08, 0x2b080819, 0x08192b08, 0x2b081908, 0x08192b08, + 0x08080808, 0x08192b19, 0x0808082b, 0x08192b19, 0x08081919, 0x08192b19, 0x08082b08, 0x08192b19, + 0x08190819, 0x08192b19, 0x08191908, 0x08192b19, 0x082b0808, 0x08192b19, 0x19080819, 0x08192b19, + 0x19081908, 0x08192b19, 0x19190808, 0x08192b19, 0x192b2b19, 0x08192b19, 0x2b2b082b, 0x08192b19, + 0x08081908, 0x08192b2b, 0x08190808, 0x08192b2b, 0x19080808, 0x08192b2b, 0x1919192b, 0x08192b2b, + 0x08080808, 0x082b0808, 0x0808082b, 0x082b0808, 0x08081919, 0x082b0808, 0x08082b08, 0x082b0808, + 0x08190819, 0x082b0808, 0x08191908, 0x082b0808, 0x0819192b, 0x082b0808, 0x08192b19, 0x082b0808, + 0x082b0808, 0x082b0808, 0x082b1919, 0x082b0808, 0x082b2b2b, 0x082b0808, 0x19080819, 0x082b0808, + 0x19081908, 0x082b0808, 0x19190808, 0x082b0808, 0x1919082b, 0x082b0808, 0x19191919, 0x082b0808, + 0x192b1908, 0x082b0808, 0x2b080808, 0x082b0808, 0x2b082b2b, 0x082b0808, 0x2b191908, 0x082b0808, + 0x2b2b2b2b, 0x082b0808, 0x08080819, 0x082b0819, 0x08081908, 0x082b0819, 0x08190808, 0x082b0819, + 0x0819082b, 0x082b0819, 0x08191919, 0x082b0819, 0x082b0819, 0x082b0819, 0x19080808, 0x082b0819, + 0x1908082b, 0x082b0819, 0x19081919, 0x082b0819, 0x19190819, 0x082b0819, 0x19191908, 0x082b0819, + 0x192b0808, 0x082b0819, 0x2b080819, 0x082b0819, 0x2b081908, 0x082b0819, 0x2b190808, 0x082b0819, + 0x08080808, 0x082b082b, 0x08082b2b, 0x082b082b, 0x082b082b, 0x082b082b, 0x082b2b08, 0x082b082b, + 0x082b2b2b, 0x082b082b, 0x19081908, 0x082b082b, 0x19190808, 0x082b082b, 0x2b082b08, 0x082b082b, + 0x2b082b2b, 0x082b082b, 0x2b2b2b08, 0x082b082b, 0x08080819, 0x082b1908, 0x08081908, 0x082b1908, + 0x0808192b, 0x082b1908, 0x08082b19, 0x082b1908, 0x08190808, 0x082b1908, 0x08191919, 0x082b1908, + 0x08192b08, 0x082b1908, 0x082b0819, 0x082b1908, 0x082b1908, 0x082b1908, 0x19080808, 0x082b1908, + 0x1908082b, 0x082b1908, 0x19081919, 0x082b1908, 0x19082b08, 0x082b1908, 0x19190819, 0x082b1908, + 0x19191908, 0x082b1908, 0x192b0808, 0x082b1908, 0x2b080819, 0x082b1908, 0x2b081908, 0x082b1908, + 0x2b190808, 0x082b1908, 0x08080808, 0x082b1919, 0x08081919, 0x082b1919, 0x08082b08, 0x082b1919, + 0x08190819, 0x082b1919, 0x08191908, 0x082b1919, 0x082b0808, 0x082b1919, 0x19080819, 0x082b1919, + 0x19081908, 0x082b1919, 0x19190808, 0x082b1919, 0x192b192b, 0x082b1919, 0x2b080808, 0x082b1919, + 0x08080819, 0x082b192b, 0x08081908, 0x082b192b, 0x08190808, 0x082b192b, 0x19080808, 0x082b192b, + 0x19192b19, 0x082b192b, 0x08080808, 0x082b2b08, 0x08081919, 0x082b2b08, 0x08190819, 0x082b2b08, + 0x08191908, 0x082b2b08, 0x19080819, 0x082b2b08, 0x19081908, 0x082b2b08, 0x19190808, 0x082b2b08, + 0x2b082b2b, 0x082b2b08, 0x2b2b2b2b, 0x082b2b08, 0x08080819, 0x082b2b19, 0x08081908, 0x082b2b19, + 0x08190808, 0x082b2b19, 0x2b191919, 0x082b2b19, 0x08082b2b, 0x082b2b2b, 0x082b082b, 0x082b2b2b, + 0x192b1908, 0x082b2b2b, 0x2b082b08, 0x082b2b2b, 0x2b082b2b, 0x082b2b2b, 0x08080819, 0x19080808, + 0x08081908, 0x19080808, 0x0808192b, 0x19080808, 0x08082b19, 0x19080808, 0x08190808, 0x19080808, + 0x0819082b, 0x19080808, 0x08191919, 0x19080808, 0x08192b08, 0x19080808, 0x08192b2b, 0x19080808, + 0x082b0819, 0x19080808, 0x082b1908, 0x19080808, 0x082b192b, 0x19080808, 0x19080808, 0x19080808, + 0x1908082b, 0x19080808, 0x19081919, 0x19080808, 0x19082b08, 0x19080808, 0x19082b2b, 0x19080808, + 0x19190819, 0x19080808, 0x19191908, 0x19080808, 0x1919192b, 0x19080808, 0x19192b19, 0x19080808, + 0x192b0808, 0x19080808, 0x192b082b, 0x19080808, 0x192b1919, 0x19080808, 0x2b080819, 0x19080808, + 0x2b081908, 0x19080808, 0x2b190808, 0x19080808, 0x2b191919, 0x19080808, 0x2b192b08, 0x19080808, + 0x2b2b0819, 0x19080808, 0x2b2b1908, 0x19080808, 0x08080808, 0x19080819, 0x0808082b, 0x19080819, + 0x08081919, 0x19080819, 0x08082b08, 0x19080819, 0x08190819, 0x19080819, 0x08191908, 0x19080819, + 0x0819192b, 0x19080819, 0x08192b19, 0x19080819, 0x082b0808, 0x19080819, 0x082b082b, 0x19080819, + 0x082b1919, 0x19080819, 0x19080819, 0x19080819, 0x19081908, 0x19080819, 0x1908192b, 0x19080819, + 0x19082b19, 0x19080819, 0x19190808, 0x19080819, 0x1919082b, 0x19080819, 0x19191919, 0x19080819, + 0x19192b08, 0x19080819, 0x192b0819, 0x19080819, 0x192b1908, 0x19080819, 0x2b080808, 0x19080819, + 0x2b08082b, 0x19080819, 0x2b081919, 0x19080819, 0x2b082b08, 0x19080819, 0x2b190819, 0x19080819, + 0x2b191908, 0x19080819, 0x2b2b0808, 0x19080819, 0x08080819, 0x1908082b, 0x08081908, 0x1908082b, + 0x08190808, 0x1908082b, 0x0819082b, 0x1908082b, 0x08191919, 0x1908082b, 0x08192b08, 0x1908082b, + 0x082b1908, 0x1908082b, 0x19080808, 0x1908082b, 0x19081919, 0x1908082b, 0x19082b08, 0x1908082b, + 0x19190819, 0x1908082b, 0x19191908, 0x1908082b, 0x192b0808, 0x1908082b, 0x2b080819, 0x1908082b, + 0x2b081908, 0x1908082b, 0x08080808, 0x19081908, 0x0808082b, 0x19081908, 0x08081919, 0x19081908, + 0x08082b08, 0x19081908, 0x08082b2b, 0x19081908, 0x08190819, 0x19081908, 0x08191908, 0x19081908, + 0x0819192b, 0x19081908, 0x08192b19, 0x19081908, 0x082b0808, 0x19081908, 0x082b082b, 0x19081908, + 0x082b1919, 0x19081908, 0x082b2b08, 0x19081908, 0x19080819, 0x19081908, 0x19081908, 0x19081908, + 0x1908192b, 0x19081908, 0x19082b19, 0x19081908, 0x19190808, 0x19081908, 0x1919082b, 0x19081908, + 0x19191919, 0x19081908, 0x19192b08, 0x19081908, 0x192b0819, 0x19081908, 0x192b1908, 0x19081908, + 0x2b080808, 0x19081908, 0x2b08082b, 0x19081908, 0x2b081919, 0x19081908, 0x2b082b08, 0x19081908, + 0x2b190819, 0x19081908, 0x2b191908, 0x19081908, 0x2b2b0808, 0x19081908, 0x08080819, 0x19081919, + 0x08081908, 0x19081919, 0x0808192b, 0x19081919, 0x08082b19, 0x19081919, 0x08190808, 0x19081919, + 0x0819082b, 0x19081919, 0x08191919, 0x19081919, 0x08192b08, 0x19081919, 0x082b0819, 0x19081919, + 0x082b1908, 0x19081919, 0x19080808, 0x19081919, 0x1908082b, 0x19081919, 0x19081919, 0x19081919, + 0x19082b08, 0x19081919, 0x19190819, 0x19081919, 0x19191908, 0x19081919, 0x192b0808, 0x19081919, + 0x192b2b2b, 0x19081919, 0x2b080819, 0x19081919, 0x2b081908, 0x19081919, 0x2b190808, 0x19081919, + 0x08080808, 0x1908192b, 0x0808082b, 0x1908192b, 0x08081919, 0x1908192b, 0x08082b08, 0x1908192b, + 0x08190819, 0x1908192b, 0x08191908, 0x1908192b, 0x082b0808, 0x1908192b, 0x19080819, 0x1908192b, + 0x19081908, 0x1908192b, 0x19190808, 0x1908192b, 0x2b080808, 0x1908192b, 0x2b2b1919, 0x1908192b, + 0x08080819, 0x19082b08, 0x08081908, 0x19082b08, 0x08082b19, 0x19082b08, 0x08190808, 0x19082b08, + 0x0819082b, 0x19082b08, 0x08191919, 0x19082b08, 0x08192b08, 0x19082b08, 0x082b0819, 0x19082b08, + 0x082b1908, 0x19082b08, 0x19080808, 0x19082b08, 0x1908082b, 0x19082b08, 0x19081919, 0x19082b08, + 0x19082b08, 0x19082b08, 0x19190819, 0x19082b08, 0x19191908, 0x19082b08, 0x192b0808, 0x19082b08, + 0x2b081908, 0x19082b08, 0x2b190808, 0x19082b08, 0x08080808, 0x19082b19, 0x0808082b, 0x19082b19, + 0x08081919, 0x19082b19, 0x08082b08, 0x19082b19, 0x08190819, 0x19082b19, 0x08191908, 0x19082b19, + 0x082b0808, 0x19082b19, 0x19080819, 0x19082b19, 0x19081908, 0x19082b19, 0x19190808, 0x19082b19, + 0x2b080808, 0x19082b19, 0x2b19192b, 0x19082b19, 0x08080819, 0x19082b2b, 0x08081908, 0x19082b2b, + 0x08190808, 0x19082b2b, 0x19080808, 0x19082b2b, 0x08080808, 0x19190808, 0x0808082b, 0x19190808, + 0x08081919, 0x19190808, 0x08082b08, 0x19190808, 0x08190819, 0x19190808, 0x08191908, 0x19190808, + 0x0819192b, 0x19190808, 0x08192b19, 0x19190808, 0x082b0808, 0x19190808, 0x082b082b, 0x19190808, + 0x082b1919, 0x19190808, 0x082b2b08, 0x19190808, 0x19080819, 0x19190808, 0x19081908, 0x19190808, + 0x1908192b, 0x19190808, 0x19082b19, 0x19190808, 0x19190808, 0x19190808, 0x1919082b, 0x19190808, + 0x19191919, 0x19190808, 0x19192b08, 0x19190808, 0x192b0819, 0x19190808, 0x192b1908, 0x19190808, + 0x2b080808, 0x19190808, 0x2b08082b, 0x19190808, 0x2b081919, 0x19190808, 0x2b082b08, 0x19190808, + 0x2b190819, 0x19190808, 0x2b191908, 0x19190808, 0x08080819, 0x19190819, 0x08081908, 0x19190819, + 0x0808192b, 0x19190819, 0x08082b19, 0x19190819, 0x08190808, 0x19190819, 0x0819082b, 0x19190819, + 0x08191919, 0x19190819, 0x08192b08, 0x19190819, 0x082b0819, 0x19190819, 0x082b1908, 0x19190819, + 0x19080808, 0x19190819, 0x1908082b, 0x19190819, 0x19081919, 0x19190819, 0x19082b08, 0x19190819, + 0x19190819, 0x19190819, 0x19191908, 0x19190819, 0x192b0808, 0x19190819, 0x2b080819, 0x19190819, + 0x2b081908, 0x19190819, 0x2b190808, 0x19190819, 0x08080808, 0x1919082b, 0x08081919, 0x1919082b, + 0x08082b08, 0x1919082b, 0x08190819, 0x1919082b, 0x08191908, 0x1919082b, 0x082b0808, 0x1919082b, + 0x19080819, 0x1919082b, 0x19081908, 0x1919082b, 0x19190808, 0x1919082b, 0x192b2b19, 0x1919082b, + 0x2b080808, 0x1919082b, 0x08080819, 0x19191908, 0x08081908, 0x19191908, 0x0808192b, 0x19191908, + 0x08082b19, 0x19191908, 0x08190808, 0x19191908, 0x0819082b, 0x19191908, 0x08191919, 0x19191908, + 0x08192b08, 0x19191908, 0x082b0819, 0x19191908, 0x082b1908, 0x19191908, 0x19080808, 0x19191908, + 0x1908082b, 0x19191908, 0x19081919, 0x19191908, 0x19082b08, 0x19191908, 0x19190819, 0x19191908, + 0x19191908, 0x19191908, 0x192b0808, 0x19191908, 0x2b080819, 0x19191908, 0x2b081908, 0x19191908, + 0x2b190808, 0x19191908, 0x08080808, 0x19191919, 0x0808082b, 0x19191919, 0x08081919, 0x19191919, + 0x08082b08, 0x19191919, 0x08190819, 0x19191919, 0x08191908, 0x19191919, 0x082b0808, 0x19191919, + 0x19080819, 0x19191919, 0x19081908, 0x19191919, 0x19190808, 0x19191919, 0x2b080808, 0x19191919, + 0x08080819, 0x1919192b, 0x08081908, 0x1919192b, 0x08190808, 0x1919192b, 0x082b192b, 0x1919192b, + 0x19080808, 0x1919192b, 0x08080808, 0x19192b08, 0x0808082b, 0x19192b08, 0x08081919, 0x19192b08, + 0x08082b08, 0x19192b08, 0x08190819, 0x19192b08, 0x08191908, 0x19192b08, 0x082b0808, 0x19192b08, + 0x19080819, 0x19192b08, 0x19081908, 0x19192b08, 0x19190808, 0x19192b08, 0x19192b2b, 0x19192b08, + 0x2b080808, 0x19192b08, 0x08080819, 0x19192b19, 0x08081908, 0x19192b19, 0x08190808, 0x19192b19, + 0x19080808, 0x19192b19, 0x08080808, 0x19192b2b, 0x08192b19, 0x19192b2b, 0x2b081919, 0x19192b2b, + 0x2b2b2b08, 0x19192b2b, 0x08080819, 0x192b0808, 0x08081908, 0x192b0808, 0x0808192b, 0x192b0808, + 0x08190808, 0x192b0808, 0x0819082b, 0x192b0808, 0x08191919, 0x192b0808, 0x08192b08, 0x192b0808, + 0x082b0819, 0x192b0808, 0x082b1908, 0x192b0808, 0x19080808, 0x192b0808, 0x19081919, 0x192b0808, + 0x19082b08, 0x192b0808, 0x19190819, 0x192b0808, 0x19191908, 0x192b0808, 0x192b0808, 0x192b0808, + 0x2b081908, 0x192b0808, 0x2b190808, 0x192b0808, 0x08080808, 0x192b0819, 0x0808082b, 0x192b0819, + 0x08081919, 0x192b0819, 0x08082b08, 0x192b0819, 0x08190819, 0x192b0819, 0x08191908, 0x192b0819, + 0x082b0808, 0x192b0819, 0x19080819, 0x192b0819, 0x19081908, 0x192b0819, 0x19190808, 0x192b0819, + 0x2b080808, 0x192b0819, 0x2b192b19, 0x192b0819, 0x08081908, 0x192b082b, 0x08190808, 0x192b082b, + 0x19080808, 0x192b082b, 0x1919192b, 0x192b082b, 0x2b2b0819, 0x192b082b, 0x08080808, 0x192b1908, + 0x08081919, 0x192b1908, 0x08082b08, 0x192b1908, 0x08190819, 0x192b1908, 0x08191908, 0x192b1908, + 0x082b0808, 0x192b1908, 0x19080819, 0x192b1908, 0x19081908, 0x192b1908, 0x19190808, 0x192b1908, + 0x2b080808, 0x192b1908, 0x08080819, 0x192b1919, 0x08081908, 0x192b1919, 0x08190808, 0x192b1919, + 0x19080808, 0x192b1919, 0x19082b2b, 0x192b1919, 0x192b2b08, 0x192b1919, 0x2b19082b, 0x192b1919, + 0x08080808, 0x192b192b, 0x2b191908, 0x192b192b, 0x08080819, 0x192b2b08, 0x08081908, 0x192b2b08, + 0x08190808, 0x192b2b08, 0x192b1919, 0x192b2b08, 0x2b192b08, 0x192b2b08, 0x08080808, 0x192b2b19, + 0x082b2b2b, 0x192b2b19, 0x1908082b, 0x192b2b2b, 0x2b2b0819, 0x192b2b2b, 0x08080808, 0x2b080808, + 0x0808082b, 0x2b080808, 0x08081919, 0x2b080808, 0x08082b08, 0x2b080808, 0x08190819, 0x2b080808, + 0x08191908, 0x2b080808, 0x08192b19, 0x2b080808, 0x082b0808, 0x2b080808, 0x082b1919, 0x2b080808, + 0x19080819, 0x2b080808, 0x19081908, 0x2b080808, 0x19190808, 0x2b080808, 0x1919082b, 0x2b080808, + 0x19191919, 0x2b080808, 0x19192b08, 0x2b080808, 0x192b0819, 0x2b080808, 0x2b080808, 0x2b080808, + 0x2b081919, 0x2b080808, 0x2b190819, 0x2b080808, 0x2b191908, 0x2b080808, 0x08080819, 0x2b080819, + 0x08081908, 0x2b080819, 0x08082b19, 0x2b080819, 0x08190808, 0x2b080819, 0x0819082b, 0x2b080819, + 0x08191919, 0x2b080819, 0x08192b08, 0x2b080819, 0x082b0819, 0x2b080819, 0x082b1908, 0x2b080819, + 0x19080808, 0x2b080819, 0x1908082b, 0x2b080819, 0x19081919, 0x2b080819, 0x19082b08, 0x2b080819, + 0x19190819, 0x2b080819, 0x19191908, 0x2b080819, 0x2b080819, 0x2b080819, 0x2b081908, 0x2b080819, + 0x2b190808, 0x2b080819, 0x2b2b2b19, 0x2b080819, 0x08080808, 0x2b08082b, 0x08081919, 0x2b08082b, + 0x08082b2b, 0x2b08082b, 0x08190819, 0x2b08082b, 0x08191908, 0x2b08082b, 0x19080819, 0x2b08082b, + 0x19081908, 0x2b08082b, 0x19190808, 0x2b08082b, 0x08080819, 0x2b081908, 0x08081908, 0x2b081908, + 0x0808192b, 0x2b081908, 0x08082b19, 0x2b081908, 0x08190808, 0x2b081908, 0x0819082b, 0x2b081908, + 0x08191919, 0x2b081908, 0x08192b08, 0x2b081908, 0x082b0819, 0x2b081908, 0x19080808, 0x2b081908, + 0x1908082b, 0x2b081908, 0x19081919, 0x2b081908, 0x19082b08, 0x2b081908, 0x19190819, 0x2b081908, + 0x19191908, 0x2b081908, 0x192b0808, 0x2b081908, 0x2b080819, 0x2b081908, 0x2b081908, 0x2b081908, + 0x2b190808, 0x2b081908, 0x08080808, 0x2b081919, 0x0808082b, 0x2b081919, 0x08081919, 0x2b081919, + 0x08082b08, 0x2b081919, 0x08190819, 0x2b081919, 0x08191908, 0x2b081919, 0x082b0808, 0x2b081919, + 0x19080819, 0x2b081919, 0x19081908, 0x2b081919, 0x19190808, 0x2b081919, 0x2b080808, 0x2b081919, + 0x2b082b2b, 0x2b081919, 0x08080819, 0x2b08192b, 0x08081908, 0x2b08192b, 0x08190808, 0x2b08192b, + 0x082b2b19, 0x2b08192b, 0x19080808, 0x2b08192b, 0x08080808, 0x2b082b08, 0x08081919, 0x2b082b08, + 0x08190819, 0x2b082b08, 0x08191908, 0x2b082b08, 0x19080819, 0x2b082b08, 0x19081908, 0x2b082b08, + 0x19190808, 0x2b082b08, 0x2b2b082b, 0x2b082b08, 0x08080819, 0x2b082b19, 0x08081908, 0x2b082b19, + 0x19080808, 0x2b082b19, 0x192b1919, 0x2b082b19, 0x082b082b, 0x2b082b2b, 0x19192b08, 0x2b082b2b, + 0x19192b2b, 0x2b082b2b, 0x2b08082b, 0x2b082b2b, 0x2b2b082b, 0x2b082b2b, 0x08080819, 0x2b190808, + 0x08081908, 0x2b190808, 0x08082b19, 0x2b190808, 0x08190808, 0x2b190808, 0x0819082b, 0x2b190808, + 0x08191919, 0x2b190808, 0x08192b08, 0x2b190808, 0x082b1908, 0x2b190808, 0x19080808, 0x2b190808, + 0x1908082b, 0x2b190808, 0x19081919, 0x2b190808, 0x19082b08, 0x2b190808, 0x19190819, 0x2b190808, + 0x19191908, 0x2b190808, 0x192b0808, 0x2b190808, 0x2b080819, 0x2b190808, 0x2b081908, 0x2b190808, + 0x2b190808, 0x2b190808, 0x08080808, 0x2b190819, 0x08081919, 0x2b190819, 0x08190819, 0x2b190819, + 0x08191908, 0x2b190819, 0x19080819, 0x2b190819, 0x19081908, 0x2b190819, 0x19190808, 0x2b190819, + 0x19192b2b, 0x2b190819, 0x08080819, 0x2b19082b, 0x08081908, 0x2b19082b, 0x08190808, 0x2b19082b, + 0x19080808, 0x2b19082b, 0x2b2b192b, 0x2b19082b, 0x08080808, 0x2b191908, 0x0808082b, 0x2b191908, + 0x08081919, 0x2b191908, 0x08082b08, 0x2b191908, 0x08190819, 0x2b191908, 0x08191908, 0x2b191908, + 0x082b0808, 0x2b191908, 0x19080819, 0x2b191908, 0x19081908, 0x2b191908, 0x19190808, 0x2b191908, + 0x2b080808, 0x2b191908, 0x2b19192b, 0x2b191908, 0x08080819, 0x2b191919, 0x08081908, 0x2b191919, + 0x08190808, 0x2b191919, 0x19080808, 0x2b191919, 0x2b192b08, 0x2b191919, 0x2b2b0819, 0x2b191919, + 0x08080808, 0x2b19192b, 0x1908192b, 0x2b19192b, 0x192b1908, 0x2b19192b, 0x08080819, 0x2b192b08, + 0x08081908, 0x2b192b08, 0x08190808, 0x2b192b08, 0x082b192b, 0x2b192b08, 0x19080808, 0x2b192b08, + 0x2b2b2b19, 0x2b192b08, 0x08080808, 0x2b192b19, 0x19082b19, 0x2b192b19, 0x1919082b, 0x2b192b19, + 0x2b190808, 0x2b192b2b, 0x08080808, 0x2b2b0808, 0x08081919, 0x2b2b0808, 0x08082b2b, 0x2b2b0808, + 0x08191908, 0x2b2b0808, 0x082b082b, 0x2b2b0808, 0x082b2b2b, 0x2b2b0808, 0x19080819, 0x2b2b0808, + 0x19081908, 0x2b2b0808, 0x19190808, 0x2b2b0808, 0x2b2b082b, 0x2b2b0808, 0x2b2b2b2b, 0x2b2b0808, + 0x19080808, 0x2b2b0819, 0x192b1919, 0x2b2b0819, 0x0808082b, 0x2b2b082b, 0x08082b2b, 0x2b2b082b, + 0x082b082b, 0x2b2b082b, 0x082b2b08, 0x2b2b082b, 0x082b2b2b, 0x2b2b082b, 0x2b08082b, 0x2b2b082b, + 0x2b082b08, 0x2b2b082b, 0x2b082b2b, 0x2b2b082b, 0x2b2b2b08, 0x2b2b082b, 0x08080819, 0x2b2b1908, + 0x08081908, 0x2b2b1908, 0x08190808, 0x2b2b1908, 0x19080808, 0x2b2b1908, 0x2b082b19, 0x2b2b1908, + 0x2b2b1908, 0x2b2b1908, 0x08080808, 0x2b2b1919, 0x08192b19, 0x2b2b1919, 0x19190819, 0x2b2b192b, + 0x08082b2b, 0x2b2b2b08, 0x082b2b08, 0x2b2b2b08, 0x2b2b082b, 0x2b2b2b08, 0x19191908, 0x2b2b2b19, + 0x2b08192b, 0x2b2b2b19, 0x08082b08, 0x2b2b2b2b, 0x08082b2b, 0x2b2b2b2b, 0x082b0808, 0x2b2b2b2b, + 0x082b082b, 0x2b2b2b2b, 0x082b2b08, 0x2b2b2b2b, 0x2b082b08, 0x2b2b2b2b, 0x2b2b2b2b, 0x2b2b2b2b) +} + +//! Main-context use only (tests, oracles, the emitter's constant bake) - a forked worker reads +//! a module global as zeros; kernels take `iq2s_grid2()` instead. +let IQ2S_GRID = iq2s_grid2() + + +//! The IQ2_XS grid (upstream's iq2xs_grid, 512 u64 entries as low/high uint pairs): the u16 word's +//! low 9 bits select EIGHT magnitudes ({8, 25, 43}), its high 7 bits the KSIGNS_IQ2XS sign +//! byte (bit j flips element j). Per-call local (4 KB), worker-safe; IQ2XS_GRID = main-context copy. +def iq2xs_grid2() : uint[1024] { // nolint:STYLE038 - a 1024-word data table, not splittable + return fixed_array( + 0x08080808, 0x08080808, 0x0808082b, 0x08080808, 0x08081919, 0x08080808, 0x08082b08, 0x08080808, + 0x08082b2b, 0x08080808, 0x08190819, 0x08080808, 0x08191908, 0x08080808, 0x0819192b, 0x08080808, + 0x08192b19, 0x08080808, 0x082b0808, 0x08080808, 0x082b082b, 0x08080808, 0x082b1919, 0x08080808, + 0x082b2b08, 0x08080808, 0x19080819, 0x08080808, 0x19081908, 0x08080808, 0x1908192b, 0x08080808, + 0x19082b19, 0x08080808, 0x19190808, 0x08080808, 0x1919082b, 0x08080808, 0x19191919, 0x08080808, + 0x19192b08, 0x08080808, 0x192b0819, 0x08080808, 0x192b1908, 0x08080808, 0x2b080808, 0x08080808, + 0x2b08082b, 0x08080808, 0x2b081919, 0x08080808, 0x2b082b08, 0x08080808, 0x2b190819, 0x08080808, + 0x2b191908, 0x08080808, 0x2b192b19, 0x08080808, 0x2b2b0808, 0x08080808, 0x08080819, 0x08080819, + 0x08081908, 0x08080819, 0x0808192b, 0x08080819, 0x08082b19, 0x08080819, 0x08190808, 0x08080819, + 0x0819082b, 0x08080819, 0x08191919, 0x08080819, 0x08192b08, 0x08080819, 0x08192b2b, 0x08080819, + 0x082b0819, 0x08080819, 0x082b1908, 0x08080819, 0x19080808, 0x08080819, 0x1908082b, 0x08080819, + 0x19081919, 0x08080819, 0x19082b08, 0x08080819, 0x19190819, 0x08080819, 0x19191908, 0x08080819, + 0x192b0808, 0x08080819, 0x192b2b08, 0x08080819, 0x2b080819, 0x08080819, 0x2b081908, 0x08080819, + 0x2b190808, 0x08080819, 0x08080808, 0x0808082b, 0x0808082b, 0x0808082b, 0x08081919, 0x0808082b, + 0x08082b08, 0x0808082b, 0x08190819, 0x0808082b, 0x08191908, 0x0808082b, 0x082b0808, 0x0808082b, + 0x19080819, 0x0808082b, 0x19081908, 0x0808082b, 0x19190808, 0x0808082b, 0x19191919, 0x0808082b, + 0x2b080808, 0x0808082b, 0x2b082b2b, 0x0808082b, 0x08080819, 0x08081908, 0x08081908, 0x08081908, + 0x0808192b, 0x08081908, 0x08082b19, 0x08081908, 0x08190808, 0x08081908, 0x0819082b, 0x08081908, + 0x08191919, 0x08081908, 0x08192b08, 0x08081908, 0x082b0819, 0x08081908, 0x082b1908, 0x08081908, + 0x19080808, 0x08081908, 0x1908082b, 0x08081908, 0x19081919, 0x08081908, 0x19082b08, 0x08081908, + 0x19190819, 0x08081908, 0x19191908, 0x08081908, 0x1919192b, 0x08081908, 0x192b0808, 0x08081908, + 0x2b080819, 0x08081908, 0x2b081908, 0x08081908, 0x2b190808, 0x08081908, 0x08080808, 0x08081919, + 0x0808082b, 0x08081919, 0x08081919, 0x08081919, 0x08082b08, 0x08081919, 0x08190819, 0x08081919, + 0x08191908, 0x08081919, 0x082b0808, 0x08081919, 0x19080819, 0x08081919, 0x19081908, 0x08081919, + 0x19190808, 0x08081919, 0x192b0819, 0x08081919, 0x2b080808, 0x08081919, 0x08080819, 0x0808192b, + 0x08081908, 0x0808192b, 0x08190808, 0x0808192b, 0x082b192b, 0x0808192b, 0x19080808, 0x0808192b, + 0x1908082b, 0x0808192b, 0x2b081908, 0x0808192b, 0x08080808, 0x08082b08, 0x0808082b, 0x08082b08, + 0x08081919, 0x08082b08, 0x08082b08, 0x08082b08, 0x08082b2b, 0x08082b08, 0x08190819, 0x08082b08, + 0x08191908, 0x08082b08, 0x082b0808, 0x08082b08, 0x082b1919, 0x08082b08, 0x19080819, 0x08082b08, + 0x19081908, 0x08082b08, 0x19190808, 0x08082b08, 0x19192b08, 0x08082b08, 0x2b080808, 0x08082b08, + 0x2b2b0808, 0x08082b08, 0x2b2b2b2b, 0x08082b08, 0x08080819, 0x08082b19, 0x08081908, 0x08082b19, + 0x08190808, 0x08082b19, 0x19080808, 0x08082b19, 0x2b080819, 0x08082b19, 0x2b082b19, 0x08082b19, + 0x08080808, 0x08082b2b, 0x082b0808, 0x08082b2b, 0x082b2b08, 0x08082b2b, 0x2b19192b, 0x08082b2b, + 0x2b2b0808, 0x08082b2b, 0x08080819, 0x08190808, 0x08081908, 0x08190808, 0x0808192b, 0x08190808, + 0x08082b19, 0x08190808, 0x08190808, 0x08190808, 0x0819082b, 0x08190808, 0x08191919, 0x08190808, + 0x08192b08, 0x08190808, 0x082b0819, 0x08190808, 0x082b1908, 0x08190808, 0x19080808, 0x08190808, + 0x1908082b, 0x08190808, 0x19081919, 0x08190808, 0x19082b08, 0x08190808, 0x19190819, 0x08190808, + 0x19191908, 0x08190808, 0x192b0808, 0x08190808, 0x192b2b2b, 0x08190808, 0x2b080819, 0x08190808, + 0x2b081908, 0x08190808, 0x2b190808, 0x08190808, 0x08080808, 0x08190819, 0x0808082b, 0x08190819, + 0x08081919, 0x08190819, 0x08082b08, 0x08190819, 0x08190819, 0x08190819, 0x08191908, 0x08190819, + 0x082b0808, 0x08190819, 0x19080819, 0x08190819, 0x19081908, 0x08190819, 0x19190808, 0x08190819, + 0x2b080808, 0x08190819, 0x2b191908, 0x08190819, 0x2b19192b, 0x08190819, 0x08080819, 0x0819082b, + 0x08081908, 0x0819082b, 0x0808192b, 0x0819082b, 0x08190808, 0x0819082b, 0x19080808, 0x0819082b, + 0x192b0808, 0x0819082b, 0x08080808, 0x08191908, 0x0808082b, 0x08191908, 0x08081919, 0x08191908, + 0x08082b08, 0x08191908, 0x08190819, 0x08191908, 0x08191908, 0x08191908, 0x082b0808, 0x08191908, + 0x19080819, 0x08191908, 0x19081908, 0x08191908, 0x19082b19, 0x08191908, 0x19190808, 0x08191908, + 0x192b1908, 0x08191908, 0x2b080808, 0x08191908, 0x08080819, 0x08191919, 0x08081908, 0x08191919, + 0x08190808, 0x08191919, 0x19080808, 0x08191919, 0x08080808, 0x0819192b, 0x08191908, 0x0819192b, + 0x19082b19, 0x0819192b, 0x08080819, 0x08192b08, 0x08081908, 0x08192b08, 0x08190808, 0x08192b08, + 0x0819082b, 0x08192b08, 0x19080808, 0x08192b08, 0x19191908, 0x08192b08, 0x2b08192b, 0x08192b08, + 0x08080808, 0x08192b19, 0x08081919, 0x08192b19, 0x192b192b, 0x08192b19, 0x19190819, 0x08192b2b, + 0x2b2b2b19, 0x08192b2b, 0x08080808, 0x082b0808, 0x0808082b, 0x082b0808, 0x08081919, 0x082b0808, + 0x08082b08, 0x082b0808, 0x08082b2b, 0x082b0808, 0x08190819, 0x082b0808, 0x08191908, 0x082b0808, + 0x082b0808, 0x082b0808, 0x19080819, 0x082b0808, 0x19081908, 0x082b0808, 0x19190808, 0x082b0808, + 0x2b080808, 0x082b0808, 0x2b2b0808, 0x082b0808, 0x08080819, 0x082b0819, 0x08081908, 0x082b0819, + 0x08190808, 0x082b0819, 0x19080808, 0x082b0819, 0x19082b08, 0x082b0819, 0x192b1919, 0x082b0819, + 0x08080808, 0x082b082b, 0x082b082b, 0x082b082b, 0x2b080808, 0x082b082b, 0x2b2b2b08, 0x082b082b, + 0x08080819, 0x082b1908, 0x08081908, 0x082b1908, 0x08190808, 0x082b1908, 0x082b2b19, 0x082b1908, + 0x19080808, 0x082b1908, 0x08080808, 0x082b1919, 0x19080819, 0x082b1919, 0x1919082b, 0x082b1919, + 0x2b192b19, 0x082b1919, 0x08080819, 0x082b192b, 0x08192b2b, 0x082b192b, 0x2b2b192b, 0x082b192b, + 0x08080808, 0x082b2b08, 0x08082b08, 0x082b2b08, 0x08082b2b, 0x082b2b08, 0x082b0808, 0x082b2b08, + 0x19191919, 0x082b2b08, 0x2b082b08, 0x082b2b08, 0x2b2b082b, 0x082b2b08, 0x192b2b08, 0x082b2b19, + 0x2b190808, 0x082b2b19, 0x08082b08, 0x082b2b2b, 0x082b0808, 0x082b2b2b, 0x2b08082b, 0x082b2b2b, + 0x2b082b08, 0x082b2b2b, 0x2b082b2b, 0x082b2b2b, 0x08080819, 0x19080808, 0x08081908, 0x19080808, + 0x0808192b, 0x19080808, 0x08082b19, 0x19080808, 0x08190808, 0x19080808, 0x0819082b, 0x19080808, + 0x08191919, 0x19080808, 0x08192b08, 0x19080808, 0x082b0819, 0x19080808, 0x082b1908, 0x19080808, + 0x19080808, 0x19080808, 0x1908082b, 0x19080808, 0x19081919, 0x19080808, 0x19082b08, 0x19080808, + 0x19082b2b, 0x19080808, 0x19190819, 0x19080808, 0x19191908, 0x19080808, 0x192b0808, 0x19080808, + 0x192b1919, 0x19080808, 0x2b080819, 0x19080808, 0x2b081908, 0x19080808, 0x2b190808, 0x19080808, + 0x08080808, 0x19080819, 0x0808082b, 0x19080819, 0x08081919, 0x19080819, 0x08082b08, 0x19080819, + 0x08190819, 0x19080819, 0x08191908, 0x19080819, 0x082b0808, 0x19080819, 0x19080819, 0x19080819, + 0x19081908, 0x19080819, 0x19190808, 0x19080819, 0x2b080808, 0x19080819, 0x2b081919, 0x19080819, + 0x2b2b082b, 0x19080819, 0x08080819, 0x1908082b, 0x08081908, 0x1908082b, 0x08190808, 0x1908082b, + 0x0819082b, 0x1908082b, 0x082b2b19, 0x1908082b, 0x19080808, 0x1908082b, 0x08080808, 0x19081908, + 0x0808082b, 0x19081908, 0x08081919, 0x19081908, 0x08082b08, 0x19081908, 0x08190819, 0x19081908, + 0x08191908, 0x19081908, 0x08192b19, 0x19081908, 0x082b0808, 0x19081908, 0x19080819, 0x19081908, + 0x19081908, 0x19081908, 0x19190808, 0x19081908, 0x2b080808, 0x19081908, 0x2b191908, 0x19081908, + 0x08080819, 0x19081919, 0x08081908, 0x19081919, 0x08190808, 0x19081919, 0x082b1908, 0x19081919, + 0x19080808, 0x19081919, 0x2b192b2b, 0x19081919, 0x08080808, 0x1908192b, 0x08082b2b, 0x1908192b, + 0x19081908, 0x1908192b, 0x19190808, 0x1908192b, 0x08080819, 0x19082b08, 0x08081908, 0x19082b08, + 0x08190808, 0x19082b08, 0x19080808, 0x19082b08, 0x19081919, 0x19082b08, 0x19191908, 0x19082b08, + 0x192b082b, 0x19082b08, 0x08080808, 0x19082b19, 0x08190819, 0x19082b19, 0x19081908, 0x19082b19, + 0x19190808, 0x19082b19, 0x192b2b19, 0x19082b19, 0x08081908, 0x19082b2b, 0x08080808, 0x19190808, + 0x0808082b, 0x19190808, 0x08081919, 0x19190808, 0x08082b08, 0x19190808, 0x08190819, 0x19190808, + 0x08191908, 0x19190808, 0x082b0808, 0x19190808, 0x082b2b08, 0x19190808, 0x19080819, 0x19190808, + 0x19081908, 0x19190808, 0x19190808, 0x19190808, 0x2b080808, 0x19190808, 0x08080819, 0x19190819, + 0x08081908, 0x19190819, 0x08190808, 0x19190819, 0x08191919, 0x19190819, 0x19080808, 0x19190819, + 0x1908082b, 0x19190819, 0x08080808, 0x1919082b, 0x19081908, 0x1919082b, 0x2b2b2b2b, 0x1919082b, + 0x08080819, 0x19191908, 0x08081908, 0x19191908, 0x08190808, 0x19191908, 0x082b0819, 0x19191908, + 0x19080808, 0x19191908, 0x192b0808, 0x19191908, 0x2b080819, 0x19191908, 0x2b2b0819, 0x19191908, + 0x08080808, 0x19191919, 0x08082b08, 0x19191919, 0x2b080808, 0x19191919, 0x2b082b08, 0x19191919, + 0x082b0819, 0x1919192b, 0x192b2b08, 0x1919192b, 0x2b2b0819, 0x1919192b, 0x08080808, 0x19192b08, + 0x08191908, 0x19192b08, 0x19080819, 0x19192b08, 0x19190808, 0x19192b08, 0x2b192b19, 0x19192b08, + 0x08192b2b, 0x19192b19, 0x19080808, 0x19192b19, 0x1908082b, 0x19192b19, 0x2b081919, 0x19192b2b, + 0x08080819, 0x192b0808, 0x08081908, 0x192b0808, 0x08190808, 0x192b0808, 0x19080808, 0x192b0808, + 0x19191908, 0x192b0808, 0x192b082b, 0x192b0808, 0x2b08192b, 0x192b0808, 0x2b2b2b19, 0x192b0808, + 0x08080808, 0x192b0819, 0x082b1908, 0x192b082b, 0x19082b2b, 0x192b082b, 0x2b19082b, 0x192b082b, + 0x08080808, 0x192b1908, 0x0819192b, 0x192b1908, 0x08190808, 0x192b1919, 0x19080808, 0x192b1919, + 0x19081919, 0x192b1919, 0x2b2b1908, 0x192b1919, 0x08080819, 0x192b2b08, 0x192b2b2b, 0x192b2b08, + 0x082b1919, 0x192b2b19, 0x0808192b, 0x192b2b2b, 0x19191908, 0x192b2b2b, 0x192b082b, 0x192b2b2b, + 0x08080808, 0x2b080808, 0x0808082b, 0x2b080808, 0x08081919, 0x2b080808, 0x08082b08, 0x2b080808, + 0x08190819, 0x2b080808, 0x08191908, 0x2b080808, 0x082b0808, 0x2b080808, 0x082b2b2b, 0x2b080808, + 0x19080819, 0x2b080808, 0x19081908, 0x2b080808, 0x19190808, 0x2b080808, 0x2b080808, 0x2b080808, + 0x2b08082b, 0x2b080808, 0x2b2b2b08, 0x2b080808, 0x2b2b2b2b, 0x2b080808, 0x08080819, 0x2b080819, + 0x08081908, 0x2b080819, 0x0808192b, 0x2b080819, 0x08190808, 0x2b080819, 0x19080808, 0x2b080819, + 0x19190819, 0x2b080819, 0x19192b19, 0x2b080819, 0x08080808, 0x2b08082b, 0x082b0808, 0x2b08082b, + 0x2b080808, 0x2b08082b, 0x2b08082b, 0x2b08082b, 0x2b2b0808, 0x2b08082b, 0x2b2b2b08, 0x2b08082b, + 0x08080819, 0x2b081908, 0x08081908, 0x2b081908, 0x08190808, 0x2b081908, 0x0819082b, 0x2b081908, + 0x08191919, 0x2b081908, 0x19080808, 0x2b081908, 0x192b0808, 0x2b081908, 0x2b082b19, 0x2b081908, + 0x08080808, 0x2b081919, 0x19081908, 0x2b081919, 0x2b2b1919, 0x2b081919, 0x08192b08, 0x2b08192b, + 0x192b2b2b, 0x2b08192b, 0x08080808, 0x2b082b08, 0x08082b08, 0x2b082b08, 0x082b1919, 0x2b082b08, + 0x19192b2b, 0x2b082b08, 0x2b080808, 0x2b082b08, 0x2b08082b, 0x2b082b08, 0x2b2b2b08, 0x2b082b08, + 0x0808192b, 0x2b082b19, 0x082b082b, 0x2b082b2b, 0x2b080808, 0x2b082b2b, 0x2b082b08, 0x2b082b2b, + 0x2b19192b, 0x2b082b2b, 0x2b2b2b08, 0x2b082b2b, 0x08080819, 0x2b190808, 0x08081908, 0x2b190808, + 0x08190808, 0x2b190808, 0x19080808, 0x2b190808, 0x1919192b, 0x2b190808, 0x2b081908, 0x2b190808, + 0x08080808, 0x2b190819, 0x082b082b, 0x2b190819, 0x192b1908, 0x2b190819, 0x1919192b, 0x2b19082b, + 0x2b082b19, 0x2b19082b, 0x08080808, 0x2b191908, 0x08081919, 0x2b191908, 0x19081908, 0x2b191908, + 0x19190808, 0x2b191908, 0x19192b08, 0x2b191908, 0x082b2b19, 0x2b191919, 0x2b190808, 0x2b191919, + 0x2b19082b, 0x2b191919, 0x19080819, 0x2b19192b, 0x19190819, 0x2b192b08, 0x2b2b192b, 0x2b192b08, + 0x19082b19, 0x2b192b19, 0x08191919, 0x2b192b2b, 0x192b0808, 0x2b192b2b, 0x08080808, 0x2b2b0808, + 0x0808082b, 0x2b2b0808, 0x08082b08, 0x2b2b0808, 0x08082b2b, 0x2b2b0808, 0x082b0808, 0x2b2b0808, + 0x082b2b2b, 0x2b2b0808, 0x2b2b0808, 0x2b2b0808, 0x19190819, 0x2b2b0819, 0x19192b19, 0x2b2b0819, + 0x2b2b192b, 0x2b2b0819, 0x08080808, 0x2b2b082b, 0x0808082b, 0x2b2b082b, 0x08082b08, 0x2b2b082b, + 0x082b2b2b, 0x2b2b082b, 0x2b080808, 0x2b2b082b, 0x2b2b0808, 0x2b2b082b, 0x19080808, 0x2b2b1908, + 0x2b191919, 0x2b2b1908, 0x192b1919, 0x2b2b192b, 0x2b192b08, 0x2b2b192b, 0x08082b2b, 0x2b2b2b08, + 0x082b0808, 0x2b2b2b08, 0x082b082b, 0x2b2b2b08, 0x082b2b08, 0x2b2b2b08, 0x2b2b0808, 0x2b2b2b08, + 0x2b2b2b08, 0x2b2b2b08, 0x08081908, 0x2b2b2b19, 0x2b081908, 0x2b2b2b19, 0x2b08192b, 0x2b2b2b19, + 0x082b2b08, 0x2b2b2b2b, 0x082b2b2b, 0x2b2b2b2b, 0x2b190819, 0x2b2b2b2b, 0x2b2b2b2b, 0x2b2b2b2b) +} + +//! Main-context use only (tests, oracles, the emitter's constant bake) - a forked worker reads +//! a module global as zeros; kernels take `iq2xs_grid2()` instead. +let IQ2XS_GRID = iq2xs_grid2() + + +//! The IQ2_XXS grid (upstream's iq2xxs_grid, 256 u64 entries as low/high uint pairs): a block's +//! aux32[0] BYTES each select EIGHT magnitudes ({8, 25, 43}), flipped through the shared +//! ksigns_iq2xs table. Per-call local (2 KB), worker-safe; IQ2XXS_GRID = main-context copy. +def iq2xxs_grid2() : uint[512] { + return fixed_array( + 0x08080808, 0x08080808, 0x0808082b, 0x08080808, 0x08081919, 0x08080808, 0x08082b08, 0x08080808, + 0x08082b2b, 0x08080808, 0x08190819, 0x08080808, 0x08191908, 0x08080808, 0x082b0808, 0x08080808, + 0x082b082b, 0x08080808, 0x082b2b08, 0x08080808, 0x082b2b2b, 0x08080808, 0x19080819, 0x08080808, + 0x19081908, 0x08080808, 0x19190808, 0x08080808, 0x19192b08, 0x08080808, 0x192b0819, 0x08080808, + 0x192b1908, 0x08080808, 0x2b080808, 0x08080808, 0x2b08082b, 0x08080808, 0x2b082b2b, 0x08080808, + 0x2b2b082b, 0x08080808, 0x08080819, 0x08080819, 0x08081908, 0x08080819, 0x08190808, 0x08080819, + 0x08191919, 0x08080819, 0x19080808, 0x08080819, 0x2b081908, 0x08080819, 0x2b192b08, 0x08080819, + 0x08080808, 0x0808082b, 0x0808082b, 0x0808082b, 0x082b082b, 0x0808082b, 0x2b08082b, 0x0808082b, + 0x08080819, 0x08081908, 0x08081908, 0x08081908, 0x08190808, 0x08081908, 0x082b0819, 0x08081908, + 0x082b1908, 0x08081908, 0x19080808, 0x08081908, 0x1908082b, 0x08081908, 0x19082b08, 0x08081908, + 0x192b0808, 0x08081908, 0x2b080819, 0x08081908, 0x2b081908, 0x08081908, 0x2b190808, 0x08081908, + 0x2b2b1908, 0x08081908, 0x08080808, 0x08081919, 0x0808082b, 0x08081919, 0x08082b08, 0x08081919, + 0x082b0808, 0x08081919, 0x1908192b, 0x08081919, 0x192b2b19, 0x08081919, 0x2b080808, 0x08081919, + 0x2b190819, 0x08081919, 0x08082b19, 0x0808192b, 0x08190808, 0x0808192b, 0x19080808, 0x0808192b, + 0x2b081908, 0x0808192b, 0x2b2b1908, 0x0808192b, 0x08080808, 0x08082b08, 0x08081919, 0x08082b08, + 0x08082b08, 0x08082b08, 0x08191908, 0x08082b08, 0x082b2b08, 0x08082b08, 0x19080819, 0x08082b08, + 0x19081908, 0x08082b08, 0x19190808, 0x08082b08, 0x1919082b, 0x08082b08, 0x2b082b08, 0x08082b08, + 0x08081908, 0x08082b19, 0x19080808, 0x08082b19, 0x0808082b, 0x08082b2b, 0x08191908, 0x08082b2b, + 0x08080819, 0x08190808, 0x08081908, 0x08190808, 0x08190808, 0x08190808, 0x082b0819, 0x08190808, + 0x19080808, 0x08190808, 0x192b0808, 0x08190808, 0x2b081908, 0x08190808, 0x2b190808, 0x08190808, + 0x2b191919, 0x08190808, 0x08080808, 0x08190819, 0x08082b08, 0x08190819, 0x082b0808, 0x08190819, + 0x19190808, 0x08190819, 0x19192b2b, 0x08190819, 0x2b080808, 0x08190819, 0x082b1908, 0x0819082b, + 0x19081919, 0x0819082b, 0x08080808, 0x08191908, 0x08082b08, 0x08191908, 0x082b0808, 0x08191908, + 0x082b1919, 0x08191908, 0x19082b19, 0x08191908, 0x2b080808, 0x08191908, 0x08192b08, 0x08191919, + 0x192b082b, 0x08191919, 0x08080808, 0x0819192b, 0x0819192b, 0x0819192b, 0x08080819, 0x08192b08, + 0x08081908, 0x08192b08, 0x08190808, 0x08192b08, 0x19080808, 0x08192b08, 0x2b080819, 0x08192b08, + 0x08080808, 0x08192b19, 0x08081919, 0x08192b19, 0x2b2b0808, 0x08192b19, 0x19190819, 0x08192b2b, + 0x08080808, 0x082b0808, 0x0808082b, 0x082b0808, 0x08082b2b, 0x082b0808, 0x19081908, 0x082b0808, + 0x192b0819, 0x082b0808, 0x2b080808, 0x082b0808, 0x2b08082b, 0x082b0808, 0x082b2b19, 0x082b0819, + 0x19082b08, 0x082b0819, 0x08080808, 0x082b082b, 0x0808082b, 0x082b082b, 0x08080819, 0x082b1908, + 0x08081908, 0x082b1908, 0x08190808, 0x082b1908, 0x19080808, 0x082b1908, 0x1919192b, 0x082b1908, + 0x08080808, 0x082b1919, 0x19080819, 0x082b1919, 0x192b1908, 0x082b1919, 0x2b190808, 0x082b192b, + 0x08082b08, 0x082b2b08, 0x082b0808, 0x082b2b08, 0x2b191908, 0x082b2b08, 0x19081908, 0x082b2b2b, + 0x08080819, 0x19080808, 0x08081908, 0x19080808, 0x08190808, 0x19080808, 0x08192b08, 0x19080808, + 0x082b0819, 0x19080808, 0x082b1908, 0x19080808, 0x19080808, 0x19080808, 0x19082b08, 0x19080808, + 0x1919192b, 0x19080808, 0x192b0808, 0x19080808, 0x2b080819, 0x19080808, 0x2b081908, 0x19080808, + 0x2b190808, 0x19080808, 0x08080808, 0x19080819, 0x082b0808, 0x19080819, 0x192b0819, 0x19080819, + 0x2b080808, 0x19080819, 0x2b081919, 0x19080819, 0x08080819, 0x1908082b, 0x08190808, 0x1908082b, + 0x19082b08, 0x1908082b, 0x1919192b, 0x1908082b, 0x192b2b08, 0x1908082b, 0x08080808, 0x19081908, + 0x08082b08, 0x19081908, 0x082b0808, 0x19081908, 0x2b080808, 0x19081908, 0x2b192b19, 0x19081908, + 0x0819082b, 0x19081919, 0x082b1908, 0x19081919, 0x08080808, 0x1908192b, 0x08080819, 0x19082b08, + 0x08081908, 0x19082b08, 0x08190808, 0x19082b08, 0x19080808, 0x19082b08, 0x19081919, 0x19082b08, + 0x08080808, 0x19082b19, 0x19192b08, 0x19082b19, 0x192b0819, 0x19082b19, 0x2b08082b, 0x19082b19, + 0x19081919, 0x19082b2b, 0x2b190808, 0x19082b2b, 0x08080808, 0x19190808, 0x08082b08, 0x19190808, + 0x08190819, 0x19190808, 0x08192b19, 0x19190808, 0x082b0808, 0x19190808, 0x2b080808, 0x19190808, + 0x2b082b08, 0x19190808, 0x08081908, 0x19190819, 0x1908082b, 0x19190819, 0x2b2b1908, 0x19190819, + 0x2b190819, 0x1919082b, 0x2b190808, 0x19191908, 0x2b19082b, 0x19191908, 0x08082b2b, 0x19191919, + 0x08080819, 0x1919192b, 0x19191908, 0x1919192b, 0x08080808, 0x19192b08, 0x08190819, 0x19192b08, + 0x08192b19, 0x19192b08, 0x192b1908, 0x19192b08, 0x19080808, 0x19192b19, 0x08082b08, 0x19192b2b, + 0x08081908, 0x192b0808, 0x08190808, 0x192b0808, 0x19080808, 0x192b0808, 0x192b2b08, 0x192b0808, + 0x08080808, 0x192b0819, 0x19191919, 0x192b0819, 0x08192b08, 0x192b082b, 0x192b0808, 0x192b082b, + 0x08080808, 0x192b1908, 0x08081919, 0x192b1908, 0x08190808, 0x192b1919, 0x0819082b, 0x192b1919, + 0x2b081908, 0x192b1919, 0x1908082b, 0x192b2b08, 0x08080808, 0x2b080808, 0x0808082b, 0x2b080808, + 0x08082b2b, 0x2b080808, 0x19080819, 0x2b080808, 0x2b08082b, 0x2b080808, 0x08081908, 0x2b080819, + 0x08192b08, 0x2b080819, 0x19080808, 0x2b080819, 0x08190819, 0x2b08082b, 0x08080819, 0x2b081908, + 0x08081908, 0x2b081908, 0x08190808, 0x2b081908, 0x08191919, 0x2b081908, 0x19080808, 0x2b081908, + 0x192b0808, 0x2b081908, 0x08080808, 0x2b081919, 0x1908192b, 0x2b081919, 0x2b191908, 0x2b081919, + 0x08082b19, 0x2b08192b, 0x19080808, 0x2b08192b, 0x192b0808, 0x2b08192b, 0x0808082b, 0x2b082b08, + 0x08081908, 0x2b082b19, 0x08190819, 0x2b082b2b, 0x08081908, 0x2b190808, 0x08190808, 0x2b190808, + 0x082b1908, 0x2b190808, 0x19080808, 0x2b190808, 0x2b2b0819, 0x2b190808, 0x0819192b, 0x2b190819, + 0x2b080808, 0x2b190819, 0x19081919, 0x2b19082b, 0x08080808, 0x2b191908, 0x082b082b, 0x2b191908, + 0x19081908, 0x2b191908, 0x19190819, 0x2b191919, 0x2b080819, 0x2b192b08, 0x082b0808, 0x2b192b19, + 0x0808082b, 0x2b2b0808, 0x19190808, 0x2b2b0808, 0x2b081919, 0x2b2b0808, 0x08082b19, 0x2b2b0819, + 0x08080808, 0x2b2b082b, 0x08192b08, 0x2b2b1908, 0x19190808, 0x2b2b2b08, 0x08081908, 0x2b2b2b19) +} + +//! Main-context use only (tests, oracles, the emitter's constant bake) - a forked worker reads +//! a module global as zeros; kernels take `iq2xxs_grid2()` instead. +let IQ2XXS_GRID = iq2xxs_grid2() //! Quant-plane bytes per stride unit — the typed stride surface. The stride unit is one //! 256-weight superblock for the kq lattice and one 32-weight block for q51 (see kq_elems). //! q8 panics (its planes are byte-per-elem, not stride-walked here). def kq_qsb(f : KqFmt) : int64 { - if (f == KqFmt.k4 || f == KqFmt.q40) { + if (f == KqFmt.k4 || f == KqFmt.q40 || f == KqFmt.iq4xs || f == KqFmt.iq4nl) { return K4_QSB } if (f == KqFmt.k5) { @@ -80,19 +775,40 @@ def kq_qsb(f : KqFmt) : int64 { if (f == KqFmt.q51) { return Q51_QB } + if (f == KqFmt.k3) { + return K3_QSB + } + if (f == KqFmt.iq3s) { + return IQ3S_QSB + } + if (f == KqFmt.iq3xxs) { + return IQ3XXS_QSB + } + if (f == KqFmt.k2) { + return K2_QSB + } + if (f == KqFmt.iq2s) { + return IQ2S_QSB + } + if (f == KqFmt.iq2xs) { + return IQ2XS_QSB + } + if (f == KqFmt.iq2xxs) { + return IQ2XXS_QSB + } panic("kq_qsb: no plane stride for this format") return 0l } //! Scale-plane bytes per stride unit (see kq_qsb for the unit). def kq_ssb(f : KqFmt) : int64 { - if (f == KqFmt.k4 || f == KqFmt.k5) { + if (f == KqFmt.k4 || f == KqFmt.k5 || f == KqFmt.iq4xs || f == KqFmt.iq3s || f == KqFmt.iq3xxs || f == KqFmt.k2 || f == KqFmt.iq2s || f == KqFmt.iq2xs || f == KqFmt.iq2xxs) { return K4_SSB } - if (f == KqFmt.k6) { + if (f == KqFmt.k6 || f == KqFmt.k3) { return K6_SSB } - if (f == KqFmt.q40) { + if (f == KqFmt.q40 || f == KqFmt.iq4nl) { return Q40_SSB } if (f == KqFmt.q51) { @@ -107,6 +823,10 @@ def kq_ssb(f : KqFmt) : int64 { def kq_elems(f : KqFmt) : int64 => kq_sb(f) ? 256l : 32l //! The loader's stream/repack region tag for a format (stream_repack_one / push_repack codes). +//! k2's stream-region code: kernel id 2 IS q51's stream tag, so k2 streams under 20 and the +//! repack dispatchers translate it back to kernel id 2. +let K2_STREAM_TAG = 20 + def kq_stream_code(f : KqFmt) : int { if (f == KqFmt.q8) { return 0 @@ -114,5 +834,8 @@ def kq_stream_code(f : KqFmt) : int { if (f == KqFmt.q51) { return 2 } + if (f == KqFmt.k2) { + return K2_STREAM_TAG + } return kq_schema_id(f) } diff --git a/modules/dasLLAMA/dasllama/dasllama_layout.das b/modules/dasLLAMA/dasllama/dasllama_layout.das index aa0e5e6766..920696275b 100644 --- a/modules/dasLLAMA/dasllama/dasllama_layout.das +++ b/modules/dasLLAMA/dasllama/dasllama_layout.das @@ -21,15 +21,21 @@ require math // walkers, and the GPU tier gathers. Requires dasllama_common back for Model/KqFmt; the loader // reaches the entry points through the hooks registered at [init] (register_model_layout). +//! The pack-code version folded into every .dlim identity: bump it with ANY edit to a pack or +//! repack walker in this file, so a cached image baked by the old code re-bakes instead of +//! being mapped and served (an image's filename hash carries model + schema + this). +let PACK_VERSION = 1 + // ===== metal-flavor blob transform (the blob-only .dlim redesign) ===== // A q8 region's blob bind offset (off/32)*34 lands 16B-aligned iff off % 256 == 0; a k6 // region's d-plane bind (NSB*16 + (off/256)*2) needs 4B alignment iff off % 512 == 0. // k4/k5/q40 quant + scale binds are 16B-multiples by construction. +[arch(at = "../ARCHITECTURE_GPU.md#metal-kq-split-scale-plane")] def private metal_blob_off_ok(off : int64; fmt : KqFmt) : bool { if (off < 0l) return true if (fmt == KqFmt.q8) return off % 256l == 0l - if (fmt == KqFmt.k6) return off % 512l == 0l + if (fmt == KqFmt.k6 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s || fmt == KqFmt.iq3xxs || fmt == KqFmt.iq2s || fmt == KqFmt.iq2xs || fmt == KqFmt.iq2xxs) return off % 512l == 0l if (fmt == KqFmt.q51) return off % 128l == 0l // (off/32)*20 and (off/32)*4 both 16B-aligned return true } @@ -154,6 +160,7 @@ def metal_blob_fill_blocks(t : Model; b0, nblk : int64; var dst : array) //! Put ONE kq scale plane into its metal device form, named by its Model field; false = not a kq //! scale plane. Per-plane because the streamed image walk applies each as its own turn comes, //! which is the only point where that plane's fills are done and its bytes are not yet written. +[arch(at = "../ARCHITECTURE_GPU.md#metal-kq-split-scale-plane")] def metal_blob_scale_plane(var t : Model; name : string) : bool { // k4/k5: the 16B verbatim disk scale block per superblock — the 4B pad is CPU-repack // scratch (see the plane-layout comment in dasllama_gguf) and a blob model never repacks @@ -169,6 +176,151 @@ def metal_blob_scale_plane(var t : Model; name : string) : bool { } return true } + if (name == "iq4xss") { + if (!empty(t.iq4xss)) { + var @exact_size xc : array + let nsb = long_length(t.iq4xss) / IQ4XS_SSB + xc |> reserve_resize(nsb * 18l) + unsafe { + var dp = addr(xc[0]) + let sp = addr < uint8 const? >(t.iq4xss[0]) + for (sb in range64(nsb)) { + memcpy(reinterpret(dp + sb * 16l), reinterpret(sp + sb * IQ4XS_SSB + 4l), 16) + dp[nsb * 16l + sb * 2l] = sp[sb * IQ4XS_SSB] + dp[nsb * 16l + sb * 2l + 1l] = sp[sb * IQ4XS_SSB + 1l] + } + } + t.iq4xss <- xc + } + return true + } + if (name == "iq3ss") { + if (!empty(t.iq3ss)) { + var @exact_size i3c : array + let nsb = long_length(t.iq3ss) / IQ3S_SSB + i3c |> reserve_resize(nsb * 18l) + unsafe { + var dp = addr(i3c[0]) + let sp = addr < uint8 const? >(t.iq3ss[0]) + for (sb in range64(nsb)) { + memcpy(reinterpret(dp + sb * 16l), reinterpret(sp + sb * IQ3S_SSB + 4l), 16) + dp[nsb * 16l + sb * 2l] = sp[sb * IQ3S_SSB] + dp[nsb * 16l + sb * 2l + 1l] = sp[sb * IQ3S_SSB + 1l] + } + } + t.iq3ss <- i3c + } + return true + } + if (name == "iq3xxss") { + if (!empty(t.iq3xxss)) { + var @exact_size i3xc : array + let nsb = long_length(t.iq3xxss) / IQ3XXS_SSB + i3xc |> reserve_resize(nsb * 18l) + unsafe { + var dp = addr(i3xc[0]) + let sp = addr < uint8 const? >(t.iq3xxss[0]) + for (sb in range64(nsb)) { + memcpy(reinterpret(dp + sb * 16l), reinterpret(sp + sb * IQ3XXS_SSB + 4l), 16) + dp[nsb * 16l + sb * 2l] = sp[sb * IQ3XXS_SSB] + dp[nsb * 16l + sb * 2l + 1l] = sp[sb * IQ3XXS_SSB + 1l] + } + } + t.iq3xxss <- i3xc + } + return true + } + if (name == "iq2ss") { + if (!empty(t.iq2ss)) { + var @exact_size i2c : array + let nsb = long_length(t.iq2ss) / IQ2S_SSB + i2c |> reserve_resize(nsb * 18l) + unsafe { + var dp = addr(i2c[0]) + let sp = addr < uint8 const? >(t.iq2ss[0]) + for (sb in range64(nsb)) { + memcpy(reinterpret(dp + sb * 16l), reinterpret(sp + sb * IQ2S_SSB + 4l), 16) + dp[nsb * 16l + sb * 2l] = sp[sb * IQ2S_SSB] + dp[nsb * 16l + sb * 2l + 1l] = sp[sb * IQ2S_SSB + 1l] + } + } + t.iq2ss <- i2c + } + return true + } + if (name == "iq2xss") { + if (!empty(t.iq2xss)) { + var @exact_size i2xc : array + let nsb = long_length(t.iq2xss) / IQ2XS_SSB + i2xc |> reserve_resize(nsb * 18l) + unsafe { + var dp = addr(i2xc[0]) + let sp = addr < uint8 const? >(t.iq2xss[0]) + for (sb in range64(nsb)) { + memcpy(reinterpret(dp + sb * 16l), reinterpret(sp + sb * IQ2XS_SSB + 4l), 16) + dp[nsb * 16l + sb * 2l] = sp[sb * IQ2XS_SSB] + dp[nsb * 16l + sb * 2l + 1l] = sp[sb * IQ2XS_SSB + 1l] + } + } + t.iq2xss <- i2xc + } + return true + } + if (name == "iq2xxss") { + if (!empty(t.iq2xxss)) { + var @exact_size i2xxc : array + let nsb = long_length(t.iq2xxss) / IQ2XXS_SSB + i2xxc |> reserve_resize(nsb * 18l) + unsafe { + var dp = addr(i2xxc[0]) + let sp = addr < uint8 const? >(t.iq2xxss[0]) + for (sb in range64(nsb)) { + memcpy(reinterpret(dp + sb * 16l), reinterpret(sp + sb * IQ2XXS_SSB + 4l), 16) + dp[nsb * 16l + sb * 2l] = sp[sb * IQ2XXS_SSB] + dp[nsb * 16l + sb * 2l + 1l] = sp[sb * IQ2XXS_SSB + 1l] + } + } + t.iq2xxss <- i2xxc + } + return true + } + if (name == "k2s") { + if (!empty(t.k2s)) { + var @exact_size k2c : array + let nsb = long_length(t.k2s) / K2_SSB + k2c |> reserve_resize(nsb * K2_SSB) + unsafe { + var dp = addr(k2c[0]) + let sp = addr < uint8 const? >(t.k2s[0]) + for (sb in range64(nsb)) { + memcpy(reinterpret(dp + sb * 16l), reinterpret(sp + sb * K2_SSB + 4l), 16) + for (h in range64(4l)) { + dp[nsb * 16l + sb * 4l + h] = sp[sb * K2_SSB + h] + } + } + } + t.k2s <- k2c + } + return true + } + if (name == "k3s") { + if (!empty(t.k3s)) { + var @exact_size k3c : array + let nsb = long_length(t.k3s) / K3_SSB + k3c |> reserve_resize(nsb * K3_SSB) + unsafe { + var dp = addr(k3c[0]) + let sp = addr < uint8 const? >(t.k3s[0]) + for (sb in range64(nsb)) { + memcpy(reinterpret(dp + sb * 16l), reinterpret(sp + sb * 18l), 16) + dp[nsb * 16l + sb * 2l] = sp[sb * 18l + 16l] + dp[nsb * 16l + sb * 2l + 1l] = sp[sb * 18l + 17l] + } + } + t.k3s <- k3c + } + return true + } if (name != "k6s") { return false } @@ -198,7 +350,7 @@ def metal_blob_commit(var t : Model) : bool { if (!metal_blob_eligible(t)) { return false } - for (nm in ["k4s", "k5s", "k6s"]) { + for (nm in ["k4s", "k5s", "k6s", "iq4xss", "k3s", "iq3ss", "iq3xxss", "k2s", "iq2ss", "iq2xss", "iq2xxss"]) { metal_blob_scale_plane(t, nm) } t.metal_blob = true @@ -292,6 +444,42 @@ def private repack_regions(var t : Model; regs : array) { var q40sp : uint8? = null var q51qp : uint8? = null var q51sp : uint8? = null + var iq4xsqp : uint8? = null + var iq4xssp : uint8? = null + if (!empty(t.iq4xsq)) { iq4xsqp = addr(t.iq4xsq[0]) } + if (!empty(t.iq4xss)) { iq4xssp = addr(t.iq4xss[0]) } + var k3qp : uint8? = null + var k3sp : uint8? = null + if (!empty(t.k3q)) { k3qp = addr(t.k3q[0]) } + if (!empty(t.k3s)) { k3sp = addr(t.k3s[0]) } + var iq3sqp : uint8? = null + var iq3ssp : uint8? = null + if (!empty(t.iq3sq)) { iq3sqp = addr(t.iq3sq[0]) } + if (!empty(t.iq3ss)) { iq3ssp = addr(t.iq3ss[0]) } + var iq3xxsqp : uint8? = null + var iq3xxssp : uint8? = null + if (!empty(t.iq3xxsq)) { iq3xxsqp = addr(t.iq3xxsq[0]) } + if (!empty(t.iq3xxss)) { iq3xxssp = addr(t.iq3xxss[0]) } + var iq4nlqp : uint8? = null + var iq4nlsp : uint8? = null + if (!empty(t.iq4nlq)) { iq4nlqp = addr(t.iq4nlq[0]) } + if (!empty(t.iq4nls)) { iq4nlsp = addr(t.iq4nls[0]) } + var k2qp : uint8? = null + var k2sp : uint8? = null + if (!empty(t.k2q)) { k2qp = addr(t.k2q[0]) } + if (!empty(t.k2s)) { k2sp = addr(t.k2s[0]) } + var iq2sqp : uint8? = null + var iq2ssp : uint8? = null + if (!empty(t.iq2sq)) { iq2sqp = addr(t.iq2sq[0]) } + if (!empty(t.iq2ss)) { iq2ssp = addr(t.iq2ss[0]) } + var iq2xsqp : uint8? = null + var iq2xssp : uint8? = null + if (!empty(t.iq2xsq)) { iq2xsqp = addr(t.iq2xsq[0]) } + if (!empty(t.iq2xss)) { iq2xssp = addr(t.iq2xss[0]) } + var iq2xxsqp : uint8? = null + var iq2xxssp : uint8? = null + if (!empty(t.iq2xxsq)) { iq2xxsqp = addr(t.iq2xxsq[0]) } + if (!empty(t.iq2xxss)) { iq2xxssp = addr(t.iq2xxss[0]) } if (!empty(t.qblob)) { qbp = addr(t.qblob[0]) } if (!empty(t.qscales)) { qsp = addr(t.qscales[0]) } if (!empty(t.mxq)) { mxqp = addr(t.mxq[0]) } @@ -321,11 +509,12 @@ def private repack_regions(var t : Model; regs : array) { invoke(rq51, q51qp + (rp[i].off / 32l) * Q51_QPB, q51sp + (rp[i].off / 32l) * Q51_SPB, rp[i].n, rp[i].d) } else { let sb = rp[i].off / 256l - let qsb = kq_qsb(f) - let ssb = kq_ssb(f) - var kqp = f == 4 ? k4qp : (f == 5 ? k5qp : (f == 6 ? k6qp : q40qp)) - var ksp = f == 4 ? k4sp : (f == 5 ? k5sp : (f == 6 ? k6sp : q40sp)) - invoke(rkq, f, kqp + sb * qsb, ksp + sb * ssb, rp[i].n, rp[i].d) + let fk = f == K2_STREAM_TAG ? 2 : f + let qsb = kq_qsb(fk) + let ssb = kq_ssb(fk) + var kqp = f == 4 ? k4qp : (f == 5 ? k5qp : (f == 6 ? k6qp : (f == 40 ? q40qp : (f == 44 ? iq4xsqp : (f == 3 ? k3qp : (f == 33 ? iq3sqp : (f == 34 ? iq3xxsqp : (f == 45 ? iq4nlqp : (fk == 2 ? k2qp : (f == 23 ? iq2sqp : (f == 24 ? iq2xsqp : iq2xxsqp))))))))))) + var ksp = f == 4 ? k4sp : (f == 5 ? k5sp : (f == 6 ? k6sp : (f == 40 ? q40sp : (f == 44 ? iq4xssp : (f == 3 ? k3sp : (f == 33 ? iq3ssp : (f == 34 ? iq3xxssp : (f == 45 ? iq4nlsp : (fk == 2 ? k2sp : (f == 23 ? iq2ssp : (f == 24 ? iq2xssp : iq2xxssp))))))))))) + invoke(rkq, fk, kqp + sb * qsb, ksp + sb * ssb, rp[i].n, rp[i].d) } } } @@ -450,6 +639,24 @@ def private push_repack_kq(var regs : array; fmt : KqFmt; woff, n, d push_repack(regs, 6, woff, n, d) } elif (fmt == KqFmt.q40) { push_repack(regs, 40, woff, n, d) + } elif (fmt == KqFmt.iq4xs) { + push_repack(regs, 44, woff, n, d) + } elif (fmt == KqFmt.k3) { + push_repack(regs, 3, woff, n, d) + } elif (fmt == KqFmt.iq3s) { + push_repack(regs, 33, woff, n, d) + } elif (fmt == KqFmt.iq3xxs) { + push_repack(regs, 34, woff, n, d) + } elif (fmt == KqFmt.iq4nl) { + push_repack(regs, 45, woff, n, d) + } elif (fmt == KqFmt.k2) { + push_repack(regs, 20, woff, n, d) + } elif (fmt == KqFmt.iq2s) { + push_repack(regs, 23, woff, n, d) + } elif (fmt == KqFmt.iq2xs) { + push_repack(regs, 24, woff, n, d) + } elif (fmt == KqFmt.iq2xxs) { + push_repack(regs, 25, woff, n, d) } } @@ -585,6 +792,8 @@ def private stream_repack_one(fmt : int; qp, sp : void?; n, d : int64) { invoke(active_repack_q8q8(), reinterpret(qp), reinterpret(sp), n, d) } elif (fmt == 2) { invoke(active_repack_q51(), reinterpret(qp), reinterpret(sp), n, d) + } elif (fmt == K2_STREAM_TAG) { + invoke(active_repack_kq(), 2, reinterpret(qp), reinterpret(sp), n, d) } else { invoke(active_repack_kq(), fmt, reinterpret(qp), reinterpret(sp), n, d) } @@ -722,10 +931,16 @@ def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice let sb0 = woff / KQ_SUPERBLOCK_ELEMS let qp = (fmt == KqFmt.k4 ? addr(t.k4q[sb0 * kq_qsb(4)]) : (fmt == KqFmt.k5 ? addr(t.k5q[sb0 * kq_qsb(5)]) - : (fmt == KqFmt.k6 ? addr(t.k6q[sb0 * kq_qsb(6)]) : addr(t.q40q[sb0 * kq_qsb(40)])))) + : (fmt == KqFmt.k6 ? addr(t.k6q[sb0 * kq_qsb(6)]) + : (fmt == KqFmt.q40 ? addr(t.q40q[sb0 * kq_qsb(40)]) + : (fmt == KqFmt.iq4xs ? addr(t.iq4xsq[sb0 * kq_qsb(44)]) + : (fmt == KqFmt.k3 ? addr(t.k3q[sb0 * kq_qsb(3)]) : (fmt == KqFmt.iq3s ? addr(t.iq3sq[sb0 * kq_qsb(33)]) : (fmt == KqFmt.iq3xxs ? addr(t.iq3xxsq[sb0 * kq_qsb(34)]) : (fmt == KqFmt.iq4nl ? addr(t.iq4nlq[sb0 * kq_qsb(45)]) : (fmt == KqFmt.k2 ? addr(t.k2q[sb0 * kq_qsb(2)]) : (fmt == KqFmt.iq2s ? addr(t.iq2sq[sb0 * kq_qsb(23)]) : (fmt == KqFmt.iq2xs ? addr(t.iq2xsq[sb0 * kq_qsb(24)]) : addr(t.iq2xxsq[sb0 * kq_qsb(25)]))))))))))))) let sp = (fmt == KqFmt.k4 ? addr(t.k4s[sb0 * kq_ssb(4)]) : (fmt == KqFmt.k5 ? addr(t.k5s[sb0 * kq_ssb(5)]) - : (fmt == KqFmt.k6 ? addr(t.k6s[sb0 * kq_ssb(6)]) : addr(t.q40s[sb0 * kq_ssb(40)])))) + : (fmt == KqFmt.k6 ? addr(t.k6s[sb0 * kq_ssb(6)]) + : (fmt == KqFmt.q40 ? addr(t.q40s[sb0 * kq_ssb(40)]) + : (fmt == KqFmt.iq4xs ? addr(t.iq4xss[sb0 * kq_ssb(44)]) + : (fmt == KqFmt.k3 ? addr(t.k3s[sb0 * kq_ssb(3)]) : (fmt == KqFmt.iq3s ? addr(t.iq3ss[sb0 * kq_ssb(33)]) : (fmt == KqFmt.iq3xxs ? addr(t.iq3xxss[sb0 * kq_ssb(34)]) : (fmt == KqFmt.iq4nl ? addr(t.iq4nls[sb0 * kq_ssb(45)]) : (fmt == KqFmt.k2 ? addr(t.k2s[sb0 * kq_ssb(2)]) : (fmt == KqFmt.iq2s ? addr(t.iq2ss[sb0 * kq_ssb(23)]) : (fmt == KqFmt.iq2xs ? addr(t.iq2xss[sb0 * kq_ssb(24)]) : addr(t.iq2xxss[sb0 * kq_ssb(25)]))))))))))))) var wqp = addr(wq[0]) var wsp = addr(ws[0]) let njobs = is_job_que_available() ? min(nslices, 4 * (get_total_hw_jobs() + 1)) : 1 @@ -745,8 +960,41 @@ def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice if (grouped) { let gq = qp + sliceQ + g * mr * qrow + sbi * qsb * mr let gs = sp + sliceS + g * mr * srow + sbi * dssb * mr - for (m in range64(128l)) { // nibble region: verbatim per (blk, j, t) - dq[m] = gq[((m / 4l) * mr + r) * 4l + m % 4l] + if (fmt == KqFmt.iq3s) { // 26 uniform 4-byte columns over [qs][qh][signs] + for (m in range64(104l)) { + dq[m] = gq[((m / 4l) * mr + r) * 4l + m % 4l] + } + } elif (fmt == KqFmt.iq3xxs) { // 24 uniform 4-byte columns over [qs][aux] + for (m in range64(96l)) { + dq[m] = gq[((m / 4l) * mr + r) * 4l + m % 4l] + } + } elif (fmt == KqFmt.k3) { // 16 qs columns, then the 8 hmask columns at 64*mr + for (m in range64(64l)) { + dq[m] = gq[((m / 4l) * mr + r) * 4l + m % 4l] + } + for (m in range64(32l)) { + dq[64l + m] = gq[64l * mr + ((m / 4l) * mr + r) * 4l + m % 4l] + } + } elif (fmt == KqFmt.k2) { // 16 qs columns, nothing else + for (m in range64(64l)) { + dq[m] = gq[((m / 4l) * mr + r) * 4l + m % 4l] + } + } elif (fmt == KqFmt.iq2s) { // 18 uniform 4-byte columns over [idx][signs][qh] + for (m in range64(72l)) { + dq[m] = gq[((m / 4l) * mr + r) * 4l + m % 4l] + } + } elif (fmt == KqFmt.iq2xs) { // 16 uniform 4-byte columns over the u16 qs words + for (m in range64(64l)) { + dq[m] = gq[((m / 4l) * mr + r) * 4l + m % 4l] + } + } elif (fmt == KqFmt.iq2xxs) { // 16 uniform 4-byte columns over the u16 qs words + for (m in range64(64l)) { + dq[m] = gq[((m / 4l) * mr + r) * 4l + m % 4l] + } + } else { + for (m in range64(128l)) { // nibble region: verbatim per (blk, j, t) + dq[m] = gq[((m / 4l) * mr + r) * 4l + m % 4l] + } } if (fmt == KqFmt.k5) { for (bj in range64(32l)) { @@ -757,7 +1005,7 @@ def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice dq[128l + p] = gq[128l * mr + ((p / 4l) * mr + r) * 4l + p % 4l] } } - if (fmt == KqFmt.k6) { + if (fmt == KqFmt.k6 || fmt == KqFmt.k3) { for (idx in range64(16l)) { dsc[idx] = gs[idx * mr + r] } @@ -765,14 +1013,22 @@ def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice dsc[17] = gs[16l * mr + 2l * r + 1l] dsc[18] = uint8(0) dsc[19] = uint8(0) - } elif (fmt == KqFmt.q40) { - for (blk in range64(8l)) { // 8 per-block f16 d, mr-interleaved + } elif (fmt == KqFmt.q40 || fmt == KqFmt.iq4nl) { + for (blk in range64(8l)) { // 8 per-block f16 d, mr-interleaved (iq4nl repacks with q40's interleave) dsc[blk * 2l] = gs[blk * 2l * mr + 2l * r] dsc[blk * 2l + 1l] = gs[blk * 2l * mr + 2l * r + 1l] } for (idx in range64(16l, 20l)) { dsc[idx] = uint8(0) } + } elif (fmt == KqFmt.k2) { // the field-major grp header re-packs to [d][dmin][16 pair bytes] + dsc[0] = gs[16l * mr + 2l * r] + dsc[1] = gs[16l * mr + 2l * r + 1l] + dsc[2] = gs[18l * mr + 2l * r] + dsc[3] = gs[18l * mr + 2l * r + 1l] + for (idx in range64(16l)) { + dsc[4l + idx] = gs[idx * mr + r] + } } else { dsc[0] = gs[2l * r] dsc[1] = gs[2l * r + 1l] @@ -786,8 +1042,8 @@ def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice } else { let rq = qp + sliceQ + ri * qrow + sbi * qsb let rs = sp + sliceS + ri * srow + sbi * dssb - if (fmt == KqFmt.q40) { - for (m in range64(128l)) { // Q4_0 bytes already pair k / k+16 + if (fmt == KqFmt.q40 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s || fmt == KqFmt.iq3xxs || fmt == KqFmt.iq4nl || fmt == KqFmt.k2 || fmt == KqFmt.iq2s || fmt == KqFmt.iq2xs || fmt == KqFmt.iq2xxs) { + for (m in range64(qsb)) { // disk bytes already in the device form (k/k+16 pairs; k3's lanes) dq[m] = rq[m] } } else { @@ -813,7 +1069,7 @@ def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice dq[128l + p] = rq[128l + p] } } - if (fmt == KqFmt.k6 || fmt == KqFmt.q40) { + if (fmt == KqFmt.k6 || fmt == KqFmt.q40 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s || fmt == KqFmt.iq3xxs || fmt == KqFmt.iq4nl || fmt == KqFmt.k2 || fmt == KqFmt.iq2s || fmt == KqFmt.iq2xs || fmt == KqFmt.iq2xxs) { // rows already in the decoded device form for (idx in range64(dssb)) { dsc[idx] = rs[idx] } diff --git a/modules/dasLLAMA/dasllama/dasllama_load.das b/modules/dasLLAMA/dasllama/dasllama_load.das index 8b07acb97d..6fcd08ae21 100644 --- a/modules/dasLLAMA/dasllama/dasllama_load.das +++ b/modules/dasLLAMA/dasllama/dasllama_load.das @@ -41,6 +41,15 @@ struct private LayoutSizes { k6_n : int64 q40_n : int64 q51_n : int64 + iq4xs_n : int64 + k3_n : int64 + iq3s_n : int64 + iq3xxs_n : int64 + iq4nl_n : int64 + k2_n : int64 + iq2s_n : int64 + iq2xs_n : int64 + iq2xxs_n : int64 pleq8_n : int64 // the dedicated q8 plane holding ONLY the PLE table (fp32/q4 serving) } @@ -53,6 +62,15 @@ struct private KqCursors { k6 : int64 q40 : int64 q51 : int64 + iq4xs : int64 + k3 : int64 + iq3s : int64 + iq3xxs : int64 + iq4nl : int64 + k2 : int64 + iq2s : int64 + iq2xs : int64 + iq2xxs : int64 } def private kq_take(var cur : KqCursors; f : KqFmt; n : int64) : int64 { @@ -81,6 +99,51 @@ def private kq_take(var cur : KqCursors; f : KqFmt; n : int64) : int64 { cur.q51 += n return o } + if (f == KqFmt.iq4xs) { + let o = cur.iq4xs + cur.iq4xs += n + return o + } + if (f == KqFmt.k3) { + let o = cur.k3 + cur.k3 += n + return o + } + if (f == KqFmt.iq3s) { + let o = cur.iq3s + cur.iq3s += n + return o + } + if (f == KqFmt.iq3xxs) { + let o = cur.iq3xxs + cur.iq3xxs += n + return o + } + if (f == KqFmt.iq4nl) { + let o = cur.iq4nl + cur.iq4nl += n + return o + } + if (f == KqFmt.k2) { + let o = cur.k2 + cur.k2 += n + return o + } + if (f == KqFmt.iq2s) { + let o = cur.iq2s + cur.iq2s += n + return o + } + if (f == KqFmt.iq2xs) { + let o = cur.iq2xs + cur.iq2xs += n + return o + } + if (f == KqFmt.iq2xxs) { + let o = cur.iq2xxs + cur.iq2xxs += n + return o + } let o = cur.wo cur.wo += n return o @@ -380,7 +443,7 @@ def private layout_offsets(var t : Model) : LayoutSizes { // nolint:STYLE037,S fo += dim } return LayoutSizes(fblob_n = fo, wblob_n = cur.wo, mx_n = mx, bf16_n = bf16, - k4_n = cur.k4, k5_n = cur.k5, k6_n = cur.k6, q40_n = cur.q40, q51_n = cur.q51, pleq8_n = pleq8) + k4_n = cur.k4, k5_n = cur.k5, k6_n = cur.k6, q40_n = cur.q40, q51_n = cur.q51, iq4xs_n = cur.iq4xs, k3_n = cur.k3, iq3s_n = cur.iq3s, iq3xxs_n = cur.iq3xxs, iq4nl_n = cur.iq4nl, k2_n = cur.k2, iq2s_n = cur.iq2s, iq2xs_n = cur.iq2xs, iq2xxs_n = cur.iq2xxs, pleq8_n = pleq8) } @@ -639,6 +702,24 @@ def stream_field_of(t : Model; fmt : KqFmt) : string { return "q40q" } elif (fmt == KqFmt.q51) { return "q51q" + } elif (fmt == KqFmt.iq4xs) { + return "iq4xsq" + } elif (fmt == KqFmt.k3) { + return "k3q" + } elif (fmt == KqFmt.iq3s) { + return "iq3sq" + } elif (fmt == KqFmt.iq3xxs) { + return "iq3xxsq" + } elif (fmt == KqFmt.iq4nl) { + return "iq4nlq" + } elif (fmt == KqFmt.k2) { + return "k2q" + } elif (fmt == KqFmt.iq2s) { + return "iq2sq" + } elif (fmt == KqFmt.iq2xs) { + return "iq2xsq" + } elif (fmt == KqFmt.iq2xxs) { + return "iq2xxsq" } elif (t.quant == QuantMode.q8) { // the blob flavor carries q8 as gguf-native 34B blocks in ONE plane, never the split pair return t.metal_blob ? "mblob" : "qblob" @@ -755,6 +836,24 @@ def fill_stream_plane(m : GGUFMeta; bytes : array | #; var t : Model; fie memcpy(addr(t.k6s[(j.woff / 256l) * K6_SSB]), addr(temp_s[0]), sh.sb) } elif (j.fmt == KqFmt.q40) { memcpy(addr(t.q40s[(j.woff / 256l) * Q40_SSB]), addr(temp_s[0]), sh.sb) + } elif (j.fmt == KqFmt.iq4xs) { + memcpy(addr(t.iq4xss[(j.woff / 256l) * IQ4XS_SSB]), addr(temp_s[0]), sh.sb) + } elif (j.fmt == KqFmt.k3) { + memcpy(addr(t.k3s[(j.woff / 256l) * K3_SSB]), addr(temp_s[0]), sh.sb) + } elif (j.fmt == KqFmt.iq3s) { + memcpy(addr(t.iq3ss[(j.woff / 256l) * IQ3S_SSB]), addr(temp_s[0]), sh.sb) + } elif (j.fmt == KqFmt.iq3xxs) { + memcpy(addr(t.iq3xxss[(j.woff / 256l) * IQ3XXS_SSB]), addr(temp_s[0]), sh.sb) + } elif (j.fmt == KqFmt.iq4nl) { + memcpy(addr(t.iq4nls[(j.woff / 256l) * Q40_SSB]), addr(temp_s[0]), sh.sb) + } elif (j.fmt == KqFmt.k2) { + memcpy(addr(t.k2s[(j.woff / 256l) * K2_SSB]), addr(temp_s[0]), sh.sb) + } elif (j.fmt == KqFmt.iq2s) { + memcpy(addr(t.iq2ss[(j.woff / 256l) * IQ2S_SSB]), addr(temp_s[0]), sh.sb) + } elif (j.fmt == KqFmt.iq2xs) { + memcpy(addr(t.iq2xss[(j.woff / 256l) * IQ2XS_SSB]), addr(temp_s[0]), sh.sb) + } elif (j.fmt == KqFmt.iq2xxs) { + memcpy(addr(t.iq2xxss[(j.woff / 256l) * IQ2XXS_SSB]), addr(temp_s[0]), sh.sb) } elif (j.fmt == KqFmt.q51) { memcpy(addr(t.q51s[(j.woff / 32l) * Q51_SB]), addr(temp_s[0]), sh.sb) } elif (sh.sb == 0l && j.n > 0l && !t.metal_blob) { @@ -786,6 +885,33 @@ def fill_stream_plane(m : GGUFMeta; bytes : array | #; var t : Model; fie } elif (r.fmt == 40) { invoke(g_stream_repack, 40, addr(temp_q[((r.off - j.woff) / 256l) * Q40_QSB]), addr(t.q40s[(r.off / 256l) * Q40_SSB]), r.n, r.d) + } elif (r.fmt == 44) { + invoke(g_stream_repack, 44, addr(temp_q[((r.off - j.woff) / 256l) * IQ4XS_QSB]), + addr(t.iq4xss[(r.off / 256l) * IQ4XS_SSB]), r.n, r.d) + } elif (r.fmt == 3) { + invoke(g_stream_repack, 3, addr(temp_q[((r.off - j.woff) / 256l) * K3_QSB]), + addr(t.k3s[(r.off / 256l) * K3_SSB]), r.n, r.d) + } elif (r.fmt == 33) { + invoke(g_stream_repack, 33, addr(temp_q[((r.off - j.woff) / 256l) * IQ3S_QSB]), + addr(t.iq3ss[(r.off / 256l) * IQ3S_SSB]), r.n, r.d) + } elif (r.fmt == 34) { + invoke(g_stream_repack, 34, addr(temp_q[((r.off - j.woff) / 256l) * IQ3XXS_QSB]), + addr(t.iq3xxss[(r.off / 256l) * IQ3XXS_SSB]), r.n, r.d) + } elif (r.fmt == 45) { + invoke(g_stream_repack, 45, addr(temp_q[((r.off - j.woff) / 256l) * Q40_QSB]), + addr(t.iq4nls[(r.off / 256l) * Q40_SSB]), r.n, r.d) + } elif (r.fmt == 20) { + invoke(g_stream_repack, 20, addr(temp_q[((r.off - j.woff) / 256l) * K2_QSB]), + addr(t.k2s[(r.off / 256l) * K2_SSB]), r.n, r.d) + } elif (r.fmt == 23) { + invoke(g_stream_repack, 23, addr(temp_q[((r.off - j.woff) / 256l) * IQ2S_QSB]), + addr(t.iq2ss[(r.off / 256l) * IQ2S_SSB]), r.n, r.d) + } elif (r.fmt == 24) { + invoke(g_stream_repack, 24, addr(temp_q[((r.off - j.woff) / 256l) * IQ2XS_QSB]), + addr(t.iq2xss[(r.off / 256l) * IQ2XS_SSB]), r.n, r.d) + } elif (r.fmt == 25) { + invoke(g_stream_repack, 25, addr(temp_q[((r.off - j.woff) / 256l) * IQ2XXS_QSB]), + addr(t.iq2xxss[(r.off / 256l) * IQ2XXS_SSB]), r.n, r.d) } } } @@ -998,6 +1124,33 @@ def private transcode_kq_tensor(m : GGUFMeta; bytes : array | #; name : s } elif (fmt == KqFmt.q40) { gguf_transcode_q40(m, bytes, name, kq, ks, eloff, n, src_off) return "q40 transcode (Q4_0)" + } elif (fmt == KqFmt.iq4xs) { + gguf_transcode_iq4xs(m, bytes, name, kq, ks, eloff, n, src_off) + return "iq4xs transcode (IQ4_XS)" + } elif (fmt == KqFmt.k3) { + gguf_transcode_q3k(m, bytes, name, kq, ks, eloff, n, src_off) + return "k3 transcode (Q3_K)" + } elif (fmt == KqFmt.iq3s) { + gguf_transcode_iq3s(m, bytes, name, kq, ks, eloff, n, src_off) + return "iq3s transcode (IQ3_S)" + } elif (fmt == KqFmt.iq3xxs) { + gguf_transcode_iq3xxs(m, bytes, name, kq, ks, eloff, n, src_off) + return "iq3xxs transcode (IQ3_XXS)" + } elif (fmt == KqFmt.iq4nl) { + gguf_transcode_iq4nl(m, bytes, name, kq, ks, eloff, n, src_off) + return "iq4nl transcode (IQ4_NL)" + } elif (fmt == KqFmt.k2) { + gguf_transcode_q2k(m, bytes, name, kq, ks, eloff, n, src_off) + return "k2 transcode (Q2_K)" + } elif (fmt == KqFmt.iq2s) { + gguf_transcode_iq2s(m, bytes, name, kq, ks, eloff, n, src_off) + return "iq2s transcode (IQ2_S)" + } elif (fmt == KqFmt.iq2xs) { + gguf_transcode_iq2xs(m, bytes, name, kq, ks, eloff, n, src_off) + return "iq2xs transcode (IQ2_XS)" + } elif (fmt == KqFmt.iq2xxs) { + gguf_transcode_iq2xxs(m, bytes, name, kq, ks, eloff, n, src_off) + return "iq2xxs transcode (IQ2_XXS)" } gguf_transcode_q51(m, bytes, name, kq, ks, eloff, n, src_off) return "q51 transcode (Q5_1)" @@ -1036,6 +1189,24 @@ def private load_big(m : GGUFMeta; bytes : array | #; name : string; var kind = transcode_kq_tensor(m, bytes, name, fmt, t.k6q, t.k6s, woff, n, src_off, scratch) } elif (fmt == KqFmt.q40) { kind = transcode_kq_tensor(m, bytes, name, fmt, t.q40q, t.q40s, woff, n, src_off, scratch) + } elif (fmt == KqFmt.iq4xs) { + kind = transcode_kq_tensor(m, bytes, name, fmt, t.iq4xsq, t.iq4xss, woff, n, src_off, scratch) + } elif (fmt == KqFmt.k3) { + kind = transcode_kq_tensor(m, bytes, name, fmt, t.k3q, t.k3s, woff, n, src_off, scratch) + } elif (fmt == KqFmt.iq3s) { + kind = transcode_kq_tensor(m, bytes, name, fmt, t.iq3sq, t.iq3ss, woff, n, src_off, scratch) + } elif (fmt == KqFmt.iq3xxs) { + kind = transcode_kq_tensor(m, bytes, name, fmt, t.iq3xxsq, t.iq3xxss, woff, n, src_off, scratch) + } elif (fmt == KqFmt.iq4nl) { + kind = transcode_kq_tensor(m, bytes, name, fmt, t.iq4nlq, t.iq4nls, woff, n, src_off, scratch) + } elif (fmt == KqFmt.k2) { + kind = transcode_kq_tensor(m, bytes, name, fmt, t.k2q, t.k2s, woff, n, src_off, scratch) + } elif (fmt == KqFmt.iq2s) { + kind = transcode_kq_tensor(m, bytes, name, fmt, t.iq2sq, t.iq2ss, woff, n, src_off, scratch) + } elif (fmt == KqFmt.iq2xs) { + kind = transcode_kq_tensor(m, bytes, name, fmt, t.iq2xsq, t.iq2xss, woff, n, src_off, scratch) + } elif (fmt == KqFmt.iq2xxs) { + kind = transcode_kq_tensor(m, bytes, name, fmt, t.iq2xxsq, t.iq2xxss, woff, n, src_off, scratch) } elif (fmt == KqFmt.q51) { kind = transcode_kq_tensor(m, bytes, name, fmt, t.q51q, t.q51s, woff, n, src_off, scratch) } elif (t.quant == QuantMode.q8) { @@ -1135,8 +1306,8 @@ def private log_load_report(t : Model) { to_log(LOG_INFO, "dasLLAMA noisy: backend '{active_kernel_backend()}' | kquant_native {get_kquant_native()} q40_native {get_kq_q40_native()} q50_native {get_kq_q50_native()}\n") // the tier arms at [init], before a --noisy CLI flag can land — restate it here so the flag alone suffices to_log(LOG_INFO, "dasLLAMA noisy: GPU tier installed {moe_gpu_tier_installed()} | want auto {gpu_want_auto()}, moe layers {gpu_want_moe_layers()}, stream {gpu_want_moe_stream()}\n") - to_log(LOG_INFO, "dasLLAMA noisy: format tags - q8 {tally_fmt(t, KqFmt.q8)}, k4 {tally_fmt(t, KqFmt.k4)}, k5 {tally_fmt(t, KqFmt.k5)}, k6 {tally_fmt(t, KqFmt.k6)}, q40 {tally_fmt(t, KqFmt.q40)}, q51 {tally_fmt(t, KqFmt.q51)}\n") - to_log(LOG_INFO, "dasLLAMA noisy: plane MB - q8 {float(long_length(t.qblob)) / mb} (+scales {float(long_length(t.qscales) * 4l) / mb}), k4 {float(long_length(t.k4q) + long_length(t.k4s)) / mb}, k5 {float(long_length(t.k5q) + long_length(t.k5s)) / mb}, k6 {float(long_length(t.k6q) + long_length(t.k6s)) / mb}, q40 {float(long_length(t.q40q) + long_length(t.q40s)) / mb}, q51 {float(long_length(t.q51q) + long_length(t.q51s)) / mb}, f32 {float(long_length(t.wblob) * 4l) / mb}\n") + to_log(LOG_INFO, "dasLLAMA noisy: format tags - q8 {tally_fmt(t, KqFmt.q8)}, k4 {tally_fmt(t, KqFmt.k4)}, k5 {tally_fmt(t, KqFmt.k5)}, k6 {tally_fmt(t, KqFmt.k6)}, q40 {tally_fmt(t, KqFmt.q40)}, q51 {tally_fmt(t, KqFmt.q51)}, iq4xs {tally_fmt(t, KqFmt.iq4xs)}, k3 {tally_fmt(t, KqFmt.k3)}, iq3s {tally_fmt(t, KqFmt.iq3s)}, iq3xxs {tally_fmt(t, KqFmt.iq3xxs)}, iq4nl {tally_fmt(t, KqFmt.iq4nl)}, k2 {tally_fmt(t, KqFmt.k2)}, iq2s {tally_fmt(t, KqFmt.iq2s)}, iq2xs {tally_fmt(t, KqFmt.iq2xs)}, iq2xxs {tally_fmt(t, KqFmt.iq2xxs)}\n") + to_log(LOG_INFO, "dasLLAMA noisy: plane MB - q8 {float(long_length(t.qblob)) / mb} (+scales {float(long_length(t.qscales) * 4l) / mb}), k4 {float(long_length(t.k4q) + long_length(t.k4s)) / mb}, k5 {float(long_length(t.k5q) + long_length(t.k5s)) / mb}, k6 {float(long_length(t.k6q) + long_length(t.k6s)) / mb}, q40 {float(long_length(t.q40q) + long_length(t.q40s)) / mb}, q51 {float(long_length(t.q51q) + long_length(t.q51s)) / mb}, iq4xs {float(long_length(t.iq4xsq) + long_length(t.iq4xss)) / mb}, k3 {float(long_length(t.k3q) + long_length(t.k3s)) / mb}, iq3s {float(long_length(t.iq3sq) + long_length(t.iq3ss)) / mb}, iq3xxs {float(long_length(t.iq3xxsq) + long_length(t.iq3xxss)) / mb}, iq4nl {float(long_length(t.iq4nlq) + long_length(t.iq4nls)) / mb}, k2 {float(long_length(t.k2q) + long_length(t.k2s)) / mb}, iq2s {float(long_length(t.iq2sq) + long_length(t.iq2ss)) / mb}, iq2xs {float(long_length(t.iq2xsq) + long_length(t.iq2xss)) / mb}, iq2xxs {float(long_length(t.iq2xxsq) + long_length(t.iq2xxss)) / mb}, f32 {float(long_length(t.wblob) * 4l) / mb}\n") } // GGML disk type -> plane format tag (non-K-quant types ride the classic q8 path) @@ -1150,6 +1321,33 @@ def private kq_fmt_of(gt : int) : KqFmt { if (gt == GGML_TYPE_Q6_K) { return KqFmt.k6 } + if (gt == GGML_TYPE_IQ4_XS) { + return KqFmt.iq4xs + } + if (gt == GGML_TYPE_Q3_K) { + return KqFmt.k3 + } + if (gt == GGML_TYPE_IQ3_S) { + return KqFmt.iq3s + } + if (gt == GGML_TYPE_IQ3_XXS) { + return KqFmt.iq3xxs + } + if (gt == GGML_TYPE_IQ4_NL) { + return KqFmt.iq4nl + } + if (gt == GGML_TYPE_Q2_K) { + return KqFmt.k2 + } + if (gt == GGML_TYPE_IQ2_S) { + return KqFmt.iq2s + } + if (gt == GGML_TYPE_IQ2_XS) { + return KqFmt.iq2xs + } + if (gt == GGML_TYPE_IQ2_XXS) { + return KqFmt.iq2xxs + } if (gt == GGML_TYPE_Q4_0 && get_kq_q40_native()) { return KqFmt.q40 } @@ -1983,6 +2181,15 @@ def private load_gguf_parsed(var t : Model; m : GGUFMeta; bytes : array | g_stream_plane_total["k6q"] = (sz.k6_n / 256l) * K6_QSB g_stream_plane_total["q40q"] = (sz.q40_n / 256l) * Q40_QSB g_stream_plane_total["q51q"] = (sz.q51_n / 32l) * Q51_QB + g_stream_plane_total["iq4xsq"] = (sz.iq4xs_n / 256l) * IQ4XS_QSB + g_stream_plane_total["k3q"] = (sz.k3_n / 256l) * K3_QSB + g_stream_plane_total["iq3sq"] = (sz.iq3s_n / 256l) * IQ3S_QSB + g_stream_plane_total["iq3xxsq"] = (sz.iq3xxs_n / 256l) * IQ3XXS_QSB + g_stream_plane_total["iq4nlq"] = (sz.iq4nl_n / 256l) * Q40_QSB + g_stream_plane_total["k2q"] = (sz.k2_n / 256l) * K2_QSB + g_stream_plane_total["iq2sq"] = (sz.iq2s_n / 256l) * IQ2S_QSB + g_stream_plane_total["iq2xsq"] = (sz.iq2xs_n / 256l) * IQ2XS_QSB + g_stream_plane_total["iq2xxsq"] = (sz.iq2xxs_n / 256l) * IQ2XXS_QSB let wb = mode == QuantMode.q8 ? sz.wblob_n : 0l g_stream_plane_total[t.metal_blob ? "mblob" : "qblob"] = (t.metal_blob ? (wb / Q8_BLOCK_ELEMS) * (Q8_QPB + Q8_SPB) : wb) @@ -2027,6 +2234,78 @@ def private load_gguf_parsed(var t : Model; m : GGUFMeta; bytes : array | t.q51s |> reserve((sz.q51_n / 32l) * Q51_SB) t.q51s |> resize((sz.q51_n / 32l) * Q51_SB) } + if (sz.iq4xs_n > 0l) { + if (!stream_q) { + t.iq4xsq |> reserve((sz.iq4xs_n / 256l) * IQ4XS_QSB) + t.iq4xsq |> resize((sz.iq4xs_n / 256l) * IQ4XS_QSB) + } + t.iq4xss |> reserve((sz.iq4xs_n / 256l) * IQ4XS_SSB) + t.iq4xss |> resize((sz.iq4xs_n / 256l) * IQ4XS_SSB) + } + if (sz.k3_n > 0l) { + if (!stream_q) { + t.k3q |> reserve((sz.k3_n / 256l) * K3_QSB) + t.k3q |> resize((sz.k3_n / 256l) * K3_QSB) + } + t.k3s |> reserve((sz.k3_n / 256l) * K3_SSB) + t.k3s |> resize((sz.k3_n / 256l) * K3_SSB) + } + if (sz.iq3s_n > 0l) { + if (!stream_q) { + t.iq3sq |> reserve((sz.iq3s_n / 256l) * IQ3S_QSB) + t.iq3sq |> resize((sz.iq3s_n / 256l) * IQ3S_QSB) + } + t.iq3ss |> reserve((sz.iq3s_n / 256l) * IQ3S_SSB) + t.iq3ss |> resize((sz.iq3s_n / 256l) * IQ3S_SSB) + } + if (sz.iq3xxs_n > 0l) { + if (!stream_q) { + t.iq3xxsq |> reserve((sz.iq3xxs_n / 256l) * IQ3XXS_QSB) + t.iq3xxsq |> resize((sz.iq3xxs_n / 256l) * IQ3XXS_QSB) + } + t.iq3xxss |> reserve((sz.iq3xxs_n / 256l) * IQ3XXS_SSB) + t.iq3xxss |> resize((sz.iq3xxs_n / 256l) * IQ3XXS_SSB) + } + if (sz.iq4nl_n > 0l) { + if (!stream_q) { + t.iq4nlq |> reserve((sz.iq4nl_n / 256l) * Q40_QSB) + t.iq4nlq |> resize((sz.iq4nl_n / 256l) * Q40_QSB) + } + t.iq4nls |> reserve((sz.iq4nl_n / 256l) * Q40_SSB) + t.iq4nls |> resize((sz.iq4nl_n / 256l) * Q40_SSB) + } + if (sz.k2_n > 0l) { + if (!stream_q) { + t.k2q |> reserve((sz.k2_n / 256l) * K2_QSB) + t.k2q |> resize((sz.k2_n / 256l) * K2_QSB) + } + t.k2s |> reserve((sz.k2_n / 256l) * K2_SSB) + t.k2s |> resize((sz.k2_n / 256l) * K2_SSB) + } + if (sz.iq2s_n > 0l) { + if (!stream_q) { + t.iq2sq |> reserve((sz.iq2s_n / 256l) * IQ2S_QSB) + t.iq2sq |> resize((sz.iq2s_n / 256l) * IQ2S_QSB) + } + t.iq2ss |> reserve((sz.iq2s_n / 256l) * IQ2S_SSB) + t.iq2ss |> resize((sz.iq2s_n / 256l) * IQ2S_SSB) + } + if (sz.iq2xs_n > 0l) { + if (!stream_q) { + t.iq2xsq |> reserve((sz.iq2xs_n / 256l) * IQ2XS_QSB) + t.iq2xsq |> resize((sz.iq2xs_n / 256l) * IQ2XS_QSB) + } + t.iq2xss |> reserve((sz.iq2xs_n / 256l) * IQ2XS_SSB) + t.iq2xss |> resize((sz.iq2xs_n / 256l) * IQ2XS_SSB) + } + if (sz.iq2xxs_n > 0l) { + if (!stream_q) { + t.iq2xxsq |> reserve((sz.iq2xxs_n / 256l) * IQ2XXS_QSB) + t.iq2xxsq |> resize((sz.iq2xxs_n / 256l) * IQ2XXS_QSB) + } + t.iq2xxss |> reserve((sz.iq2xxs_n / 256l) * IQ2XXS_SSB) + t.iq2xxss |> resize((sz.iq2xxs_n / 256l) * IQ2XXS_SSB) + } if (mode == QuantMode.q8) { if (!stream_q) { t.qblob |> reserve(sz.wblob_n) @@ -2333,6 +2612,15 @@ def private load_gguf_parsed(var t : Model; m : GGUFMeta; bytes : array | t.kq_repack_mr5 = active_kq_layout_mr(5) t.kq_repack_mr6 = active_kq_layout_mr(6) t.kq_repack_mr40 = active_kq_layout_mr(40) + t.kq_repack_mr44 = active_kq_layout_mr(44) + t.kq_repack_mr3 = active_kq_layout_mr(3) + t.kq_repack_mr33 = active_kq_layout_mr(33) + t.kq_repack_mr34 = active_kq_layout_mr(34) + t.kq_repack_mr45 = active_kq_layout_mr(45) + t.kq_repack_mr2 = active_kq_layout_mr(2) + t.kq_repack_mr23 = active_kq_layout_mr(23) + t.kq_repack_mr24 = active_kq_layout_mr(24) + t.kq_repack_mr25 = active_kq_layout_mr(25) } } else { layout_repack_q8(t) @@ -2349,7 +2637,16 @@ def private load_gguf_parsed(var t : Model; m : GGUFMeta; bytes : array | t.kq_repack_mr5 = active_kq_layout_mr(5) t.kq_repack_mr6 = active_kq_layout_mr(6) t.kq_repack_mr40 = active_kq_layout_mr(40) - to_log(LOG_INFO, "dasLLAMA: K-quant planes repacked to the '{active_kernel_backend()}' layouts (k4 grp{t.kq_repack_mr4}, k5 grp{t.kq_repack_mr5}, k6 grp{t.kq_repack_mr6}, q40 grp{t.kq_repack_mr40})\n") + t.kq_repack_mr44 = active_kq_layout_mr(44) + t.kq_repack_mr3 = active_kq_layout_mr(3) + t.kq_repack_mr33 = active_kq_layout_mr(33) + t.kq_repack_mr34 = active_kq_layout_mr(34) + t.kq_repack_mr45 = active_kq_layout_mr(45) + t.kq_repack_mr2 = active_kq_layout_mr(2) + t.kq_repack_mr23 = active_kq_layout_mr(23) + t.kq_repack_mr24 = active_kq_layout_mr(24) + t.kq_repack_mr25 = active_kq_layout_mr(25) + to_log(LOG_INFO, "dasLLAMA: K-quant planes repacked to the '{active_kernel_backend()}' layouts (k4 grp{t.kq_repack_mr4}, k5 grp{t.kq_repack_mr5}, k6 grp{t.kq_repack_mr6}, q40 grp{t.kq_repack_mr40}, iq4xs grp{t.kq_repack_mr44}, k3 grp{t.kq_repack_mr3}, iq3s grp{t.kq_repack_mr33}, iq3xxs grp{t.kq_repack_mr34}, iq4nl grp{t.kq_repack_mr45}, k2 grp{t.kq_repack_mr2}, iq2s grp{t.kq_repack_mr23}, iq2xs grp{t.kq_repack_mr24}, iq2xxs grp{t.kq_repack_mr25})\n") } } } diff --git a/modules/dasLLAMA/dasllama/dasllama_math.das b/modules/dasLLAMA/dasllama/dasllama_math.das index 3dbe5e8d37..868c319512 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math.das +++ b/modules/dasLLAMA/dasllama/dasllama_math.das @@ -1084,6 +1084,15 @@ struct KernelBackend { kq_rows_k5 : MatmulKqRowsFn = @@kq_unset_rows kq_rows_k6 : MatmulKqRowsFn = @@kq_unset_rows kq_rows_q40 : MatmulKqRowsFn = @@kq_unset_rows + kq_rows_iq4xs : MatmulKqRowsFn = @@kq_unset_rows + kq_rows_k3 : MatmulKqRowsFn = @@kq_unset_rows + kq_rows_iq3s : MatmulKqRowsFn = @@kq_unset_rows + kq_rows_iq3xxs : MatmulKqRowsFn = @@kq_unset_rows + kq_rows_iq4nl : MatmulKqRowsFn = @@kq_unset_rows + kq_rows_k2 : MatmulKqRowsFn = @@kq_unset_rows + kq_rows_iq2s : MatmulKqRowsFn = @@kq_unset_rows + kq_rows_iq2xs : MatmulKqRowsFn = @@kq_unset_rows + kq_rows_iq2xxs : MatmulKqRowsFn = @@kq_unset_rows kq_batch : MatmulKqBatchFn = @@kq_unset_batch kq_groupn : MatmulKqGroupNFn = @@kq_unset_groupn kq_batch_groupn : MatmulKqBatchGroupNFn = @@kq_unset_batch_groupn @@ -1146,6 +1155,15 @@ var g_kq_rows_k4 = @@kq_unset_rows var g_kq_rows_k5 = @@kq_unset_rows var g_kq_rows_k6 = @@kq_unset_rows var g_kq_rows_q40 = @@kq_unset_rows +var g_kq_rows_iq4xs = @@kq_unset_rows +var g_kq_rows_k3 = @@kq_unset_rows +var g_kq_rows_iq3s = @@kq_unset_rows +var g_kq_rows_iq3xxs = @@kq_unset_rows +var g_kq_rows_iq4nl = @@kq_unset_rows +var g_kq_rows_k2 = @@kq_unset_rows +var g_kq_rows_iq2s = @@kq_unset_rows +var g_kq_rows_iq2xs = @@kq_unset_rows +var g_kq_rows_iq2xxs = @@kq_unset_rows var g_mm_kq_batch = @@kq_unset_batch var g_mm_kq_groupn = @@kq_unset_groupn var g_mm_kq_batch_groupn = @@kq_unset_batch_groupn @@ -1473,7 +1491,7 @@ def active_q8_layout_mr() : int64 => g_bake_cpu_override ? g_bake_cpu.q8_mr : g_ //! family split; the loader freezes these onto the Model at repack time. def active_kq_layout_mr(fmt : int) : int64 { if (g_bake_cpu_override) { - return fmt == 4 ? g_bake_cpu.kq_mr4 : (fmt == 5 ? g_bake_cpu.kq_mr5 : (fmt == 40 ? g_bake_cpu.kq_mr40 : g_bake_cpu.kq_mr6)) + return fmt == 4 ? g_bake_cpu.kq_mr4 : (fmt == 5 ? g_bake_cpu.kq_mr5 : (fmt == 40 ? g_bake_cpu.kq_mr40 : (fmt == 44 ? g_bake_cpu.kq_mr44 : (fmt == 3 ? g_bake_cpu.kq_mr3 : (fmt == 33 ? g_bake_cpu.kq_mr33 : (fmt == 34 ? g_bake_cpu.kq_mr34 : (fmt == 45 ? g_bake_cpu.kq_mr45 : (fmt == 2 ? g_bake_cpu.kq_mr2 : (fmt == 23 ? g_bake_cpu.kq_mr23 : (fmt == 24 ? g_bake_cpu.kq_mr24 : (fmt == 25 ? g_bake_cpu.kq_mr25 : g_bake_cpu.kq_mr6))))))))))) } return int64(invoke(g_active_kq_layout, fmt)) } @@ -1649,6 +1667,15 @@ def private activate(be : KernelBackend) { g_kq_rows_k5 = be.kq_rows_k5 g_kq_rows_k6 = be.kq_rows_k6 g_kq_rows_q40 = be.kq_rows_q40 + g_kq_rows_iq4xs = be.kq_rows_iq4xs + g_kq_rows_k3 = be.kq_rows_k3 + g_kq_rows_iq3s = be.kq_rows_iq3s + g_kq_rows_iq3xxs = be.kq_rows_iq3xxs + g_kq_rows_iq4nl = be.kq_rows_iq4nl + g_kq_rows_k2 = be.kq_rows_k2 + g_kq_rows_iq2s = be.kq_rows_iq2s + g_kq_rows_iq2xs = be.kq_rows_iq2xs + g_kq_rows_iq2xxs = be.kq_rows_iq2xxs g_mm_kq_batch = be.kq_batch g_mm_kq_groupn = be.kq_groupn g_mm_kq_batch_groupn = be.kq_batch_groupn @@ -1851,6 +1878,15 @@ def kq_rows_fn(fmt : int) : MatmulKqRowsFn { if (fmt == 5) return g_kq_rows_k5 if (fmt == 6) return g_kq_rows_k6 if (fmt == 40) return g_kq_rows_q40 + if (fmt == 44) return g_kq_rows_iq4xs + if (fmt == 3) return g_kq_rows_k3 + if (fmt == 33) return g_kq_rows_iq3s + if (fmt == 34) return g_kq_rows_iq3xxs + if (fmt == 45) return g_kq_rows_iq4nl + if (fmt == 2) return g_kq_rows_k2 + if (fmt == 23) return g_kq_rows_iq2s + if (fmt == 24) return g_kq_rows_iq2xs + if (fmt == 25) return g_kq_rows_iq2xxs return @@kq_unset_rows } @@ -3211,6 +3247,33 @@ def private restore_math_fn_defaults { // nolint:STYLE037,STYLE038 — flat seam if (g_kq_rows_q40 == null) { g_kq_rows_q40 = @@kq_unset_rows } + if (g_kq_rows_iq4xs == null) { + g_kq_rows_iq4xs = @@kq_unset_rows + } + if (g_kq_rows_k3 == null) { + g_kq_rows_k3 = @@kq_unset_rows + } + if (g_kq_rows_iq3s == null) { + g_kq_rows_iq3s = @@kq_unset_rows + } + if (g_kq_rows_iq3xxs == null) { + g_kq_rows_iq3xxs = @@kq_unset_rows + } + if (g_kq_rows_iq4nl == null) { + g_kq_rows_iq4nl = @@kq_unset_rows + } + if (g_kq_rows_k2 == null) { + g_kq_rows_k2 = @@kq_unset_rows + } + if (g_kq_rows_iq2s == null) { + g_kq_rows_iq2s = @@kq_unset_rows + } + if (g_kq_rows_iq2xs == null) { + g_kq_rows_iq2xs = @@kq_unset_rows + } + if (g_kq_rows_iq2xxs == null) { + g_kq_rows_iq2xxs = @@kq_unset_rows + } if (g_mm_kq_batch == null) { g_mm_kq_batch = @@kq_unset_batch } diff --git a/modules/dasLLAMA/dasllama/dasllama_math_default.das b/modules/dasLLAMA/dasllama/dasllama_math_default.das index e90019a36f..126d1f932b 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_default.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_default.das @@ -546,6 +546,225 @@ def dot_q40q8(kqrow : uint8 const?; ksrow : uint8 const?; xqp : int8 const?; xsp return acc } +//! IQ4_XS row dot off the DISK-ORDER planes: per 32-block idot = sum IQ4NL_LUT[q] * x (signed, +//! exact int), folded as sum sc_blk * idot_blk over the superblock, then * (d * xs). No bsum +//! term — the codebook carries the sign, so xbsp is unused. +[unused_argument(xbsp)] +def dot_iq4xs_q8(kqrow : uint8 const?; ksrow : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n : int64) : float { + var acc = 0.0 + let lut = iq4nl_lut() + unsafe { + for (sb in range64(n / 256l)) { + let kqo = sb * 128l + let kso = sb * 20l + let dd = f16_to_f32(uint(ksrow[kso]) | (uint(ksrow[kso + 1l]) << 8u)) + var isum = 0 + for (blk in range64(8l)) { + let ab = (sb * 8l + blk) * 32l + var idot = 0 + for (l in range64(16l)) { + let q = int(kqrow[kqo + blk * 16l + l]) + idot += int(lut[q & 15]) * int(xqp[ab + l]) + idot += int(lut[q >> 4]) * int(xqp[ab + 16l + l]) + } + let u = int(ksrow[kso + 4l + blk]) + isum += (u < 128 ? u : u - 256) * idot + } + acc += float(isum) * (dd * xsp[sb]) + } + } + return acc +} + +//! Q3_K row dot off the DISK-ORDER planes: block b = 4h + j takes the 2-bit lane of qs byte +//! h*32 + l at shift 2j plus hmask bit b of byte l (q in 0..7); k6's fold with the -4 offset: +//! isum = sum sc16 * dot, bsum = sum sc16 * sum a16, then (isum - 4 * bsum) * d * xs per superblock. +[hint(unsafe_range_check, noalias = kqrow, noalias = ksrow, noalias = xqp, noalias = xsp, noalias = xbsp)] +def dot_k3q8(kqrow : uint8 const?; ksrow : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n : int64) : float { + var acc = 0.0 + unsafe { + for (sb in range64(n / 256l)) { + let kqo = sb * 96l + let kso = sb * 18l + let d = f16_to_f32(uint(ksrow[kso + 16l]) | (uint(ksrow[kso + 17l]) << 8u)) + var isum = 0 + var bsum = 0 + for (blk in range64(8l)) { + let qb = kqo + (blk / 4l) * 32l + let hb = kqo + 64l + let sh = int((blk % 4l) * 2l) + let b = sb * 8l + blk + let ab = b * 32l + var i0 = 0 + var i1 = 0 + for (l in range64(16l)) { + let q0 = ((int(kqrow[qb + l]) >> sh) & 3) | (((int(kqrow[hb + l]) >> int(blk)) & 1) << 2) + let q1 = ((int(kqrow[qb + 16l + l]) >> sh) & 3) | (((int(kqrow[hb + 16l + l]) >> int(blk)) & 1) << 2) + i0 += q0 * int(xqp[ab + l]) + i1 += q1 * int(xqp[ab + 16l + l]) + } + let u0 = int(ksrow[kso + 2l * blk]) + let u1 = int(ksrow[kso + 2l * blk + 1l]) + let s0 = u0 < 128 ? u0 : u0 - 256 + let s1 = u1 < 128 ? u1 : u1 - 256 + isum += s0 * i0 + s1 * i1 + bsum += s0 * xbsp[b * 2l] + s1 * xbsp[b * 2l + 1l] + } + acc += float(isum - 4 * bsum) * (d * xsp[sb]) + } + } + return acc +} + +//! IQ3_S row dot off the DISK-ORDER planes: per 32-block, grid entry qs[b*8 + 2l + half] | +//! (qh[b] bit << 8) yields four magnitudes, the block's sign bytes flip them, sub-scale +//! (1 + 2s) folds per block; signed weights, so no activation block-sum term. +[unused_argument(xbsp), hint(unsafe_range_check, noalias = kqrow, noalias = ksrow, noalias = xqp, noalias = xsp, noalias = xbsp)] +def dot_iq3s_q8(kqrow : uint8 const?; ksrow : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n : int64) : float { + var acc = 0.0 + let grid = iq3s_grid() + unsafe { + for (sb in range64(n / 256l)) { + let kqo = sb * 104l + let kso = sb * 20l + let d = f16_to_f32(uint(ksrow[kso]) | (uint(ksrow[kso + 1l]) << 8u)) + var isum = 0 + for (blk in range64(8l)) { + let qh = int(kqrow[kqo + 64l + blk]) + let ab = (sb * 8l + blk) * 32l + var ib = 0 + for (l in range64(4l)) { + let sgn = int(kqrow[kqo + 72l + blk * 4l + l]) + let g1 = grid[int(kqrow[kqo + blk * 8l + 2l * l]) | ((qh << int(8l - 2l * l)) & 256)] + let g2 = grid[int(kqrow[kqo + blk * 8l + 2l * l + 1l]) | ((qh << int(7l - 2l * l)) & 256)] + for (j in range64(4l)) { + let w1 = int((g1 >> uint(8l * j)) & 0xFFu) + let w2 = int((g2 >> uint(8l * j)) & 0xFFu) + ib += (((sgn >> int(j)) & 1) != 0 ? -w1 : w1) * int(xqp[ab + l * 8l + j]) + ib += (((sgn >> int(4l + j)) & 1) != 0 ? -w2 : w2) * int(xqp[ab + l * 8l + 4l + j]) + } + } + isum += int(int8(ksrow[kso + 4l + blk])) * ib + } + acc += float(isum) * (d * xsp[sb]) + } + } + return acc +} + +//! IQ2_S row dot off the DISK-ORDER planes: per 32-block, grid entry idx | qh<<8 bits yields +//! EIGHT magnitudes (two uint words), the block's sign bytes flip them, per-16 strips (1 + 2s) +//! fold on the eighth-ed d; signed weights, so no activation block-sum term. +[unused_argument(xbsp), hint(unsafe_range_check, noalias = kqrow, noalias = ksrow, noalias = xqp, noalias = xsp, noalias = xbsp)] +def dot_iq2s_q8(kqrow : uint8 const?; ksrow : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n : int64) : float { + var acc = 0.0 + let grid = iq2s_grid2() + unsafe { + for (sb in range64(n / 256l)) { + let kqo = sb * 72l + let kso = sb * 20l + let d8 = f16_to_f32(uint(ksrow[kso]) | (uint(ksrow[kso + 1l]) << 8u)) + var isum = 0 + for (b in range64(8l)) { + let qh = int(kqrow[kqo + 64l + b]) + let ab = (sb * 8l + b) * 32l + for (l in range64(4l)) { + let sgn = int(kqrow[kqo + 32l + b * 4l + l]) + let idx = int(kqrow[kqo + b * 4l + l]) | ((qh << int(8l - 2l * l)) & 0x300) + let g1 = grid[idx * 2] + let g2 = grid[idx * 2 + 1] + var ig = 0 + for (j in range64(4l)) { + let w1 = int((g1 >> uint(8l * j)) & 0xFFu) + let w2 = int((g2 >> uint(8l * j)) & 0xFFu) + ig += (((sgn >> int(j)) & 1) != 0 ? -w1 : w1) * int(xqp[ab + l * 8l + j]) + ig += (((sgn >> int(4l + j)) & 1) != 0 ? -w2 : w2) * int(xqp[ab + l * 8l + 4l + j]) + } + isum += int(ksrow[kso + 4l + 2l * b + l / 2l]) * ig + } + } + acc += float(isum) * (d8 * xsp[sb]) + } + } + return acc +} + +//! IQ2_XS row dot off the DISK-ORDER planes: each u16 qs word's low 9 bits pick a u64 grid +//! entry (EIGHT magnitudes, two uint words), its high 7 bits pick the ksigns sign byte; per-16 +//! strips (1 + 2s) fold on the eighth-ed d; signed weights, so no activation block-sum term. +[unused_argument(xbsp), hint(unsafe_range_check, noalias = kqrow, noalias = ksrow, noalias = xqp, noalias = xsp, noalias = xbsp)] +def dot_iq2xs_q8(kqrow : uint8 const?; ksrow : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n : int64) : float { + var acc = 0.0 + let grid = iq2xs_grid2() + let ksg = ksigns_iq2xs() + unsafe { + for (sb in range64(n / 256l)) { + let kqo = sb * 64l + let kso = sb * 20l + let d8 = f16_to_f32(uint(ksrow[kso]) | (uint(ksrow[kso + 1l]) << 8u)) + var isum = 0 + for (b in range64(8l)) { + let ab = (sb * 8l + b) * 32l + for (l in range64(4l)) { + let w16 = uint(kqrow[kqo + (b * 4l + l) * 2l]) | (uint(kqrow[kqo + (b * 4l + l) * 2l + 1l]) << 8u) + let sgn = int(ksg[int(w16 >> 9u)]) + let idx = int(w16 & 511u) + let g1 = grid[idx * 2] + let g2 = grid[idx * 2 + 1] + var ig = 0 + for (j in range64(4l)) { + let w1 = int((g1 >> uint(8l * j)) & 0xFFu) + let w2 = int((g2 >> uint(8l * j)) & 0xFFu) + ig += (((sgn >> int(j)) & 1) != 0 ? -w1 : w1) * int(xqp[ab + l * 8l + j]) + ig += (((sgn >> int(4l + j)) & 1) != 0 ? -w2 : w2) * int(xqp[ab + l * 8l + 4l + j]) + } + isum += int(ksrow[kso + 4l + 2l * b + l / 2l]) * ig + } + } + acc += float(isum) * (d8 * xsp[sb]) + } + } + return acc +} + +//! IQ2_XXS row dot off the DISK-ORDER planes: per block the aux32[0] bytes pick u64 grid +//! entries (two words each), the aux32[1]'s 7-bit ksigns indices flip signs, its top nibble +//! is the per-32 strip (1 + 2s) on the eighth-ed d - decoded to the strip byte at transcode. +[unused_argument(xbsp)] +def dot_iq2xxs_q8(kqrow : uint8 const?; ksrow : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n : int64) : float { + var acc = 0.0 + let grid = iq2xxs_grid2() + let signs = ksigns_iq2xs() + unsafe { + for (sb in range64(n / 256l)) { + let kqo = sb * 64l + let kso = sb * 20l + let d8 = f16_to_f32(uint(ksrow[kso]) | (uint(ksrow[kso + 1l]) << 8u)) + var isum = 0 + for (blk in range64(8l)) { + let aux = uint(kqrow[kqo + blk * 8l + 4l]) | (uint(kqrow[kqo + blk * 8l + 5l]) << 8u) | (uint(kqrow[kqo + blk * 8l + 6l]) << 16u) | (uint(kqrow[kqo + blk * 8l + 7l]) << 24u) + let ab = (sb * 8l + blk) * 32l + var ib = 0 + for (l in range64(4l)) { + let sgn = int(signs[int((aux >> uint(7l * l)) & 127u)]) + let idx = int(kqrow[kqo + blk * 8l + l]) + let g1 = grid[idx * 2] + let g2 = grid[idx * 2 + 1] + for (j in range64(4l)) { + let w1 = int((g1 >> uint(8l * j)) & 0xFFu) + let w2 = int((g2 >> uint(8l * j)) & 0xFFu) + ib += (((sgn >> int(j)) & 1) != 0 ? -w1 : w1) * int(xqp[ab + l * 8l + j]) + ib += (((sgn >> int(4l + j)) & 1) != 0 ? -w2 : w2) * int(xqp[ab + l * 8l + 4l + j]) + } + } + isum += int(ksrow[kso + 4l + blk]) * ib + } + acc += float(isum) * (d8 * xsp[sb]) + } + } + return acc +} + // Row-range cores + full GEMVs per format. Not `private`: invoked through hoisted function // pointers from lifted worker lambdas (the fused chains) and the dispatch wrappers in common. def k4_rows_kernel(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, rb, re : int64) { @@ -584,6 +803,181 @@ def q40_rows_kernel(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp } } +def iq4xs_rows_kernel(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, rb, re : int64) { + let nsb = n / 256l + unsafe { + for (i in range64(rb, re)) { + yp[i] = dot_iq4xs_q8(kqp + i * nsb * 128l, ksp + i * nsb * 20l, xqp, xsp, xbsp, n) + } + } +} + +def k3_rows_kernel(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, rb, re : int64) { + let nsb = n / 256l + unsafe { + for (i in range64(rb, re)) { + yp[i] = dot_k3q8(kqp + i * nsb * 96l, ksp + i * nsb * 18l, xqp, xsp, xbsp, n) + } + } +} + +[unused_argument(xbsp)] +def dot_iq3xxs_q8(kqrow : uint8 const?; ksrow : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n : int64) : float { + var acc = 0.0 + let grid = iq3xxs_grid() + let signs = ksigns_iq2xs() + unsafe { + for (sb in range64(n / 256l)) { + let kqo = sb * 96l + let kso = sb * 20l + let d = f16_to_f32(uint(ksrow[kso]) | (uint(ksrow[kso + 1l]) << 8u)) + var isum = 0 + for (blk in range64(8l)) { + let aux = uint(kqrow[kqo + 64l + blk * 4l]) | (uint(kqrow[kqo + 64l + blk * 4l + 1l]) << 8u) | (uint(kqrow[kqo + 64l + blk * 4l + 2l]) << 16u) | (uint(kqrow[kqo + 64l + blk * 4l + 3l]) << 24u) + let ab = (sb * 8l + blk) * 32l + var ib = 0 + for (l in range64(4l)) { + let sgn = int(signs[int((aux >> uint(7l * l)) & 127u)]) + let g1 = grid[int(kqrow[kqo + blk * 8l + 2l * l])] + let g2 = grid[int(kqrow[kqo + blk * 8l + 2l * l + 1l])] + for (j in range64(4l)) { + let w1 = int((g1 >> uint(8l * j)) & 0xFFu) + let w2 = int((g2 >> uint(8l * j)) & 0xFFu) + ib += (((sgn >> int(j)) & 1) != 0 ? -w1 : w1) * int(xqp[ab + l * 8l + j]) + ib += (((sgn >> int(4l + j)) & 1) != 0 ? -w2 : w2) * int(xqp[ab + l * 8l + 4l + j]) + } + } + isum += int(int8(ksrow[kso + 4l + blk])) * ib + } + acc += float(isum) * (d * xsp[sb]) + } + } + return acc +} + +[unused_argument(xbsp)] +def dot_iq4nl_q8(kqrow : uint8 const?; ksrow : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n : int64) : float { + var acc = 0.0 + let lut = iq4nl_lut() + unsafe { + for (sb in range64(n / 256l)) { + let kqo = sb * 128l + let kso = sb * 16l + var facc = 0.0 + for (blk in range64(8l)) { + let ab = (sb * 8l + blk) * 32l + var idot = 0 + for (l in range64(16l)) { + let q = int(kqrow[kqo + blk * 16l + l]) + idot += int(lut[q & 15]) * int(xqp[ab + l]) + idot += int(lut[q >> 4]) * int(xqp[ab + 16l + l]) + } + let d = f16_to_f32(uint(ksrow[kso + 2l * blk]) | (uint(ksrow[kso + 2l * blk + 1l]) << 8u)) + facc += float(idot) * d + } + acc += facc * xsp[sb] + } + } + return acc +} + +def iq3s_rows_kernel(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, rb, re : int64) { + let nsb = n / 256l + unsafe { + for (i in range64(rb, re)) { + yp[i] = dot_iq3s_q8(kqp + i * nsb * 104l, ksp + i * nsb * 20l, xqp, xsp, xbsp, n) + } + } +} + +def iq3xxs_rows_kernel(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, rb, re : int64) { + let nsb = n / 256l + unsafe { + for (i in range64(rb, re)) { + yp[i] = dot_iq3xxs_q8(kqp + i * nsb * 96l, ksp + i * nsb * 20l, xqp, xsp, xbsp, n) + } + } +} + +def iq4nl_rows_kernel(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, rb, re : int64) { + let nsb = n / 256l + unsafe { + for (i in range64(rb, re)) { + yp[i] = dot_iq4nl_q8(kqp + i * nsb * 128l, ksp + i * nsb * 16l, xqp, xsp, xbsp, n) + } + } +} + +//! Q2_K row dot off the DISK-ORDER planes: per 16-elem group g, sc/mn = the group's nibble +//! pair, q = the unsigned 2-bit lane; the k4/k5 fold - isum on d*sc, the min term through the +//! activation 16-sums (bsum on dmin*mn). +[hint(unsafe_range_check, noalias = kqrow, noalias = ksrow, noalias = xqp, noalias = xsp, noalias = xbsp)] +def dot_k2q8(kqrow : uint8 const?; ksrow : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n : int64) : float { + var acc = 0.0 + unsafe { + for (sb in range64(n / 256l)) { + let kqo = sb * 64l + let kso = sb * 20l + let dd = f16_to_f32(uint(ksrow[kso]) | (uint(ksrow[kso + 1l]) << 8u)) + let dm = f16_to_f32(uint(ksrow[kso + 2l]) | (uint(ksrow[kso + 3l]) << 8u)) + var isum = 0 + var bsum = 0 + for (g in range64(16l)) { + let scb = int(ksrow[kso + 4l + g]) + let qb = kqo + (g / 8l) * 32l + (g % 2l) * 16l + let sh = int(((g % 8l) / 2l) * 2l) + let ab = sb * 256l + g * 16l + var ig = 0 + for (l in range64(16l)) { + ig += ((int(kqrow[qb + l]) >> sh) & 3) * int(xqp[ab + l]) + } + isum += (scb & 15) * ig + bsum += (scb >> 4) * xbsp[sb * 16l + g] + } + let d8 = xsp[sb] + acc += float(isum) * (dd * d8) + acc -= float(bsum) * (dm * d8) + } + } + return acc +} + +def k2_rows_kernel(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, rb, re : int64) { + let nsb = n / 256l + unsafe { + for (i in range64(rb, re)) { + yp[i] = dot_k2q8(kqp + i * nsb * 64l, ksp + i * nsb * 20l, xqp, xsp, xbsp, n) + } + } +} + +def iq2s_rows_kernel(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, rb, re : int64) { + let nsb = n / 256l + unsafe { + for (i in range64(rb, re)) { + yp[i] = dot_iq2s_q8(kqp + i * nsb * 72l, ksp + i * nsb * 20l, xqp, xsp, xbsp, n) + } + } +} + +def iq2xs_rows_kernel(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, rb, re : int64) { + let nsb = n / 256l + unsafe { + for (i in range64(rb, re)) { + yp[i] = dot_iq2xs_q8(kqp + i * nsb * 64l, ksp + i * nsb * 20l, xqp, xsp, xbsp, n) + } + } +} + +def iq2xxs_rows_kernel(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, rb, re : int64) { + let nsb = n / 256l + unsafe { + for (i in range64(rb, re)) { + yp[i] = dot_iq2xxs_q8(kqp + i * nsb * 64l, ksp + i * nsb * 20l, xqp, xsp, xbsp, n) + } + } +} + def private kq_gemv_kernel(fmt : int; var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d : int64) { var myp = yp maybe_parallel_for(0, int(d), matmul_chunks_gemv(int(d), 1, n * d)) $(rb, re) { @@ -593,38 +987,354 @@ def private kq_gemv_kernel(fmt : int; var yp : float?; kqp : uint8 const?; ksp : k5_rows_kernel(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb), int64(re)) } elif (fmt == 40) { q40_rows_kernel(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb), int64(re)) + } elif (fmt == 44) { + iq4xs_rows_kernel(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb), int64(re)) + } elif (fmt == 3) { + k3_rows_kernel(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb), int64(re)) + } elif (fmt == 33) { + iq3s_rows_kernel(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb), int64(re)) + } elif (fmt == 34) { + iq3xxs_rows_kernel(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb), int64(re)) + } elif (fmt == 45) { + iq4nl_rows_kernel(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb), int64(re)) + } elif (fmt == 2) { + k2_rows_kernel(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb), int64(re)) + } elif (fmt == 23) { + iq2s_rows_kernel(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb), int64(re)) + } elif (fmt == 24) { + iq2xs_rows_kernel(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb), int64(re)) + } elif (fmt == 25) { + iq2xxs_rows_kernel(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb), int64(re)) } else { k6_rows_kernel(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb), int64(re)) } } } +//! q40 grp row dequant: the k4 nibble tiling with a per-block f16 d plane — w = d·(q − 8). +def private dequant_q40_row_grp(kqg : uint8 const?; ksg : uint8 const?; r, mr, nsb : int64; var dst : float?) { + unsafe { + for (sbi in range64(nsb)) { + let qb = sbi * 128l * mr + let sb = sbi * 16l * mr + for (blk in range64(8l)) { + let kb = sbi * 256l + blk * 32l + let d = f16_to_f32(uint(ksg[sb + blk * 2l * mr + 2l * r]) | (uint(ksg[sb + blk * 2l * mr + 2l * r + 1l]) << 8u)) + for (j in range64(4l)) { + for (t in range64(4l)) { + let nib = uint(kqg[qb + ((blk * 4l + j) * mr + r) * 4l + t]) + dst[kb + j * 4l + t] = d * float(int(nib & 15u) - 8) + dst[kb + 16l + j * 4l + t] = d * float(int(nib >> 4u) - 8) + } + } + } + } + } +} + +//! iq4xs grp row dequant: the k4 tiling + the k4 scale-row interleave (d, signed int8 sc, no +//! min) — w = (d·sc)·IQ4NL_LUT[q]. +def private dequant_iq4xs_row_grp(kqg : uint8 const?; ksg : uint8 const?; r, mr, nsb : int64; var dst : float?) { + let lut = iq4nl_lut() + unsafe { + for (sbi in range64(nsb)) { + let qb = sbi * 128l * mr + let sb = sbi * 20l * mr + let d = f16_to_f32(uint(ksg[sb + 2l * r]) | (uint(ksg[sb + 2l * r + 1l]) << 8u)) + for (blk in range64(8l)) { + let kb = sbi * 256l + blk * 32l + let u = int(ksg[sb + 4l * mr + blk * mr + r]) + let dl = d * float(u < 128 ? u : u - 256) + for (j in range64(4l)) { + for (t in range64(4l)) { + let nib = int(kqg[qb + ((blk * 4l + j) * mr + r) * 4l + t]) + dst[kb + j * 4l + t] = dl * float(lut[nib & 15]) + dst[kb + 16l + j * 4l + t] = dl * float(lut[nib >> 4]) + } + } + } + } + } +} + +//! k3 grp row dequant: the 2-bit lanes and hmask bits stay in place per (h, column) — w = (d * sc16) * (q - 4). +def private dequant_k3_row_grp(kqg : uint8 const?; ksg : uint8 const?; r, mr, nsb : int64; var dst : float?) { + unsafe { + for (sbi in range64(nsb)) { + let qb = sbi * 96l * mr + let sb = sbi * 18l * mr + let d = f16_to_f32(uint(ksg[sb + 16l * mr + 2l * r]) | (uint(ksg[sb + 16l * mr + 2l * r + 1l]) << 8u)) + for (blk in range64(8l)) { + let kb = sbi * 256l + blk * 32l + let h = blk / 4l + let sh = int((blk % 4l) * 2l) + let s0 = d * float(int(int8(ksg[sb + 2l * blk * mr + r]))) + let s1 = d * float(int(int8(ksg[sb + (2l * blk + 1l) * mr + r]))) + for (l4 in range64(8l)) { + for (t in range64(4l)) { + let q2 = (int(kqg[qb + ((h * 8l + l4) * mr + r) * 4l + t]) >> sh) & 3 + let hb = (int(kqg[qb + 64l * mr + (l4 * mr + r) * 4l + t]) >> int(blk)) & 1 + dst[kb + l4 * 4l + t] = (l4 < 4l ? s0 : s1) * float((q2 | (hb << 2)) - 4) + } + } + } + } + } +} + +//! iq3s grp row dequant: 26 uniform 4-byte columns per superblock row ([qs][qh][signs] in +//! place), the k4-shaped scale interleave — w = (d * (1 + 2s)) * grid_byte * sign. +def private dequant_iq3s_row_grp(kqg : uint8 const?; ksg : uint8 const?; r, mr, nsb : int64; var dst : float?) { + let grid = iq3s_grid() + unsafe { + for (sbi in range64(nsb)) { + let qb = sbi * 104l * mr + let sb = sbi * 20l * mr + let d = f16_to_f32(uint(ksg[sb + 2l * r]) | (uint(ksg[sb + 2l * r + 1l]) << 8u)) + for (blk in range64(8l)) { + let db = d * float(int(int8(ksg[sb + 4l * mr + blk * mr + r]))) + let hp = 64l + blk + let qh = int(kqg[qb + ((hp / 4l) * mr + r) * 4l + hp % 4l]) + for (l in range64(4l)) { + let sp = 72l + blk * 4l + l + let sgn = int(kqg[qb + ((sp / 4l) * mr + r) * 4l + sp % 4l]) + let q1p = blk * 8l + 2l * l + let q2p = q1p + 1l + let g1 = grid[int(kqg[qb + ((q1p / 4l) * mr + r) * 4l + q1p % 4l]) | ((qh << int(8l - 2l * l)) & 256)] + let g2 = grid[int(kqg[qb + ((q2p / 4l) * mr + r) * 4l + q2p % 4l]) | ((qh << int(7l - 2l * l)) & 256)] + let kb = sbi * 256l + blk * 32l + l * 8l + for (j in range64(4l)) { + dst[kb + j] = db * float((g1 >> uint(8l * j)) & 0xFFu) * (((sgn >> int(j)) & 1) != 0 ? -1.0 : 1.0) + dst[kb + 4l + j] = db * float((g2 >> uint(8l * j)) & 0xFFu) * (((sgn >> int(4l + j)) & 1) != 0 ? -1.0 : 1.0) + } + } + } + } + } +} + +//! iq3xxs grp row dequant: 24 uniform 4-byte columns per superblock row ([qs][aux] in place), +//! the iq3s scale interleave — w = (d_halved * (2ls + 1)) * halved_grid_byte * sign. +def private dequant_iq3xxs_row_grp(kqg : uint8 const?; ksg : uint8 const?; r, mr, nsb : int64; var dst : float?) { + let grid = iq3xxs_grid() + let signs = ksigns_iq2xs() + unsafe { + for (sbi in range64(nsb)) { + let qb = sbi * 96l * mr + let sb = sbi * 20l * mr + let d = f16_to_f32(uint(ksg[sb + 2l * r]) | (uint(ksg[sb + 2l * r + 1l]) << 8u)) + for (blk in range64(8l)) { + let db = d * float(int(int8(ksg[sb + 4l * mr + blk * mr + r]))) + var aux = 0u + for (t in range64(4l)) { + let ap = 64l + blk * 4l + t + aux |= uint(kqg[qb + ((ap / 4l) * mr + r) * 4l + ap % 4l]) << uint(8l * t) + } + for (l in range64(4l)) { + let sgn = int(signs[int((aux >> uint(7l * l)) & 127u)]) + let q1p = blk * 8l + 2l * l + let q2p = q1p + 1l + let g1 = grid[int(kqg[qb + ((q1p / 4l) * mr + r) * 4l + q1p % 4l])] + let g2 = grid[int(kqg[qb + ((q2p / 4l) * mr + r) * 4l + q2p % 4l])] + let kb = sbi * 256l + blk * 32l + l * 8l + for (j in range64(4l)) { + dst[kb + j] = db * float((g1 >> uint(8l * j)) & 0xFFu) * (((sgn >> int(j)) & 1) != 0 ? -1.0 : 1.0) + dst[kb + 4l + j] = db * float((g2 >> uint(8l * j)) & 0xFFu) * (((sgn >> int(4l + j)) & 1) != 0 ? -1.0 : 1.0) + } + } + } + } + } +} + +//! iq4nl grp row dequant: q40's tiling and scale interleave, the codebook in place of (q - 8). +def private dequant_iq4nl_row_grp(kqg : uint8 const?; ksg : uint8 const?; r, mr, nsb : int64; var dst : float?) { + let lut = iq4nl_lut() + unsafe { + for (sbi in range64(nsb)) { + let qb = sbi * 128l * mr + let sb = sbi * 16l * mr + for (blk in range64(8l)) { + let kb = sbi * 256l + blk * 32l + let d = f16_to_f32(uint(ksg[sb + blk * 2l * mr + 2l * r]) | (uint(ksg[sb + blk * 2l * mr + 2l * r + 1l]) << 8u)) + for (j in range64(4l)) { + for (t in range64(4l)) { + let nib = uint(kqg[qb + ((blk * 4l + j) * mr + r) * 4l + t]) + dst[kb + j * 4l + t] = d * float(int(lut[int(nib & 15u)])) + dst[kb + 16l + j * 4l + t] = d * float(int(lut[int(nib >> 4u)])) + } + } + } + } + } +} + +//! k2 grp row dequant: the 2-bit lanes stay in place as 16 four-byte columns x mr; scale row = +//! [16 sc/min bytes interleaved x mr][mr x f16 d][mr x f16 dmin]. w = (d*sc)*q - (dmin*mn). +def private dequant_k2_row_grp(kqg : uint8 const?; ksg : uint8 const?; r, mr, nsb : int64; var dst : float?) { + unsafe { + for (sbi in range64(nsb)) { + let qb = sbi * 64l * mr + let sb = sbi * 20l * mr + let hd = sb + 16l * mr + 2l * r + let d = f16_to_f32(uint(ksg[hd]) | (uint(ksg[hd + 1l]) << 8u)) + let hm = sb + 18l * mr + 2l * r + let dmin = f16_to_f32(uint(ksg[hm]) | (uint(ksg[hm + 1l]) << 8u)) + for (g in range64(16l)) { + let scb = int(ksg[sb + g * mr + r]) + let dl = d * float(scb & 15) + let ml = dmin * float(scb >> 4) + let base = (g / 8l) * 8l + (g % 2l) * 4l + let sh = int(((g % 8l) / 2l) * 2l) + for (c4 in range64(4l)) { + for (t in range64(4l)) { + let q = (int(kqg[qb + ((base + c4) * mr + r) * 4l + t]) >> sh) & 3 + dst[sbi * 256l + g * 16l + c4 * 4l + t] = dl * float(q) - ml + } + } + } + } + } +} + +//! iq2s grp row dequant: 18 uniform 4-byte columns per superblock row ([idx][signs][qh] in +//! place), the k4-shaped scale interleave with 16 strips - w = (d8 * (1 + 2s)) * grid * sign. +def private dequant_iq2s_row_grp(kqg : uint8 const?; ksg : uint8 const?; r, mr, nsb : int64; var dst : float?) { + let grid = iq2s_grid2() + unsafe { + for (sbi in range64(nsb)) { + let qb = sbi * 72l * mr + let sb = sbi * 20l * mr + let d8 = f16_to_f32(uint(ksg[sb + 2l * r]) | (uint(ksg[sb + 2l * r + 1l]) << 8u)) + for (b in range64(8l)) { + let hp = 64l + b + let qh = int(kqg[qb + ((hp / 4l) * mr + r) * 4l + hp % 4l]) + for (l in range64(4l)) { + let db = d8 * float(int(ksg[sb + 4l * mr + (2l * b + l / 2l) * mr + r])) + let sp = 32l + b * 4l + l + let sgn = int(kqg[qb + ((sp / 4l) * mr + r) * 4l + sp % 4l]) + let ip = b * 4l + l + let idx = int(kqg[qb + ((ip / 4l) * mr + r) * 4l + ip % 4l]) | ((qh << int(8l - 2l * l)) & 0x300) + let g1 = grid[idx * 2] + let g2 = grid[idx * 2 + 1] + let kb = sbi * 256l + b * 32l + l * 8l + for (j in range64(4l)) { + dst[kb + j] = db * float((g1 >> uint(8l * j)) & 0xFFu) * (((sgn >> int(j)) & 1) != 0 ? -1.0 : 1.0) + dst[kb + 4l + j] = db * float((g2 >> uint(8l * j)) & 0xFFu) * (((sgn >> int(4l + j)) & 1) != 0 ? -1.0 : 1.0) + } + } + } + } + } +} + +//! iq2xs grp row dequant: 16 uniform 4-byte columns per superblock row (the u16 qs words in +//! place), the k4-shaped scale interleave with 16 strips - w = (d8 * (1 + 2s)) * grid * sign. +def private dequant_iq2xs_row_grp(kqg : uint8 const?; ksg : uint8 const?; r, mr, nsb : int64; var dst : float?) { + let grid = iq2xs_grid2() + let ksgn = ksigns_iq2xs() + unsafe { + for (sbi in range64(nsb)) { + let qb = sbi * 64l * mr + let sb = sbi * 20l * mr + let d8 = f16_to_f32(uint(ksg[sb + 2l * r]) | (uint(ksg[sb + 2l * r + 1l]) << 8u)) + for (b in range64(8l)) { + for (l in range64(4l)) { + let db = d8 * float(int(ksg[sb + 4l * mr + (2l * b + l / 2l) * mr + r])) + let lp = (b * 4l + l) * 2l + let w16 = uint(kqg[qb + ((lp / 4l) * mr + r) * 4l + lp % 4l]) | (uint(kqg[qb + ((lp / 4l) * mr + r) * 4l + lp % 4l + 1l]) << 8u) + let sgn = int(ksgn[int(w16 >> 9u)]) + let idx = int(w16 & 511u) + let g1 = grid[idx * 2] + let g2 = grid[idx * 2 + 1] + let kb = sbi * 256l + b * 32l + l * 8l + for (j in range64(4l)) { + dst[kb + j] = db * float((g1 >> uint(8l * j)) & 0xFFu) * (((sgn >> int(j)) & 1) != 0 ? -1.0 : 1.0) + dst[kb + 4l + j] = db * float((g2 >> uint(8l * j)) & 0xFFu) * (((sgn >> int(4l + j)) & 1) != 0 ? -1.0 : 1.0) + } + } + } + } + } +} + +//! iq2xxs grp row dequant: 16 uniform 4-byte columns per superblock row - column 2b carries +//! block b's four grid bytes, column 2b+1 its aux32 (signs + the per-32 scale nibble, whose +//! strip byte the scale row already carries). +def private dequant_iq2xxs_row_grp(kqg : uint8 const?; ksg : uint8 const?; r, mr, nsb : int64; var dst : float?) { + let grid = iq2xxs_grid2() + let ksgn = ksigns_iq2xs() + unsafe { + for (sbi in range64(nsb)) { + let qb = sbi * 64l * mr + let sb = sbi * 20l * mr + let d8 = f16_to_f32(uint(ksg[sb + 2l * r]) | (uint(ksg[sb + 2l * r + 1l]) << 8u)) + for (b in range64(8l)) { + let db = d8 * float(int(ksg[sb + 4l * mr + b * mr + r])) + let ac = (2l * b + 1l) * mr + r + let aux = uint(kqg[qb + ac * 4l]) | (uint(kqg[qb + ac * 4l + 1l]) << 8u) | (uint(kqg[qb + ac * 4l + 2l]) << 16u) | (uint(kqg[qb + ac * 4l + 3l]) << 24u) + let gc = (2l * b) * mr + r + for (l in range64(4l)) { + let sgn = int(ksgn[int((aux >> uint(7l * l)) & 127u)]) + let idx = int(kqg[qb + gc * 4l + l]) + let g1 = grid[idx * 2] + let g2 = grid[idx * 2 + 1] + let kb = sbi * 256l + b * 32l + l * 8l + for (j in range64(4l)) { + dst[kb + j] = db * float((g1 >> uint(8l * j)) & 0xFFu) * (((sgn >> int(j)) & 1) != 0 ? -1.0 : 1.0) + dst[kb + 4l + j] = db * float((g2 >> uint(8l * j)) & 0xFFu) * (((sgn >> int(4l + j)) & 1) != 0 ? -1.0 : 1.0) + } + } + } + } + } +} + //! Dequant one row off the grp-REPACKED K-quant planes (repack_k4/k5/k6_grp layout). //! kqg/ksg = row's GROUP plane base (group g = row/mr); r = row % mr. //! embed_row's kq_repacked path — per-token, so scalar is fine. -def dequant_kq_row_grp(fmt : int64; kqg : uint8 const?; ksg : uint8 const?; r, mr, n : int64; var dst : float?) { +def dequant_kq_row_grp(fmt : int64; kqg : uint8 const?; ksg : uint8 const?; r, mr, n : int64; var dst : float?) { // nolint:STYLE037,STYLE038 — the flat per-format dispatch (one arm per kq format) is the honest shape let nsb = n / 256l let qsb = kq_qsb(int(fmt)) let ssb = kq_ssb(int(fmt)) if (fmt == 40l) { - // q40 grp: the k4 nibble tiling with a per-block f16 d plane — w = d·(q − 8) - unsafe { - for (sbi in range64(nsb)) { - let qb = sbi * qsb * mr - let sb = sbi * ssb * mr - for (blk in range64(8l)) { - let kb = sbi * 256l + blk * 32l - let d = f16_to_f32(uint(ksg[sb + blk * 2l * mr + 2l * r]) | (uint(ksg[sb + blk * 2l * mr + 2l * r + 1l]) << 8u)) - for (j in range64(4l)) { - for (t in range64(4l)) { - let nib = uint(kqg[qb + ((blk * 4l + j) * mr + r) * 4l + t]) - dst[kb + j * 4l + t] = d * float(int(nib & 15u) - 8) - dst[kb + 16l + j * 4l + t] = d * float(int(nib >> 4u) - 8) - } - } - } - } - } + dequant_q40_row_grp(kqg, ksg, r, mr, nsb, dst) + return + } + if (fmt == 44l) { + dequant_iq4xs_row_grp(kqg, ksg, r, mr, nsb, dst) + return + } + if (fmt == 3l) { + dequant_k3_row_grp(kqg, ksg, r, mr, nsb, dst) + return + } + if (fmt == 33l) { + dequant_iq3s_row_grp(kqg, ksg, r, mr, nsb, dst) + return + } + if (fmt == 34l) { + dequant_iq3xxs_row_grp(kqg, ksg, r, mr, nsb, dst) + return + } + if (fmt == 45l) { + dequant_iq4nl_row_grp(kqg, ksg, r, mr, nsb, dst) + return + } + if (fmt == 2l) { + dequant_k2_row_grp(kqg, ksg, r, mr, nsb, dst) + return + } + if (fmt == 23l) { + dequant_iq2s_row_grp(kqg, ksg, r, mr, nsb, dst) + return + } + if (fmt == 24l) { + dequant_iq2xs_row_grp(kqg, ksg, r, mr, nsb, dst) + return + } + if (fmt == 25l) { + dequant_iq2xxs_row_grp(kqg, ksg, r, mr, nsb, dst) return } unsafe { @@ -721,6 +1431,24 @@ def matmul_kq_groupn(fmt : int; var y : array | #; kq : array | #; myp[ii] = dot_k5q8(kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) } elif (fmt == 40) { myp[ii] = dot_q40q8(kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) + } elif (fmt == 44) { + myp[ii] = dot_iq4xs_q8(kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) + } elif (fmt == 3) { + myp[ii] = dot_k3q8(kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) + } elif (fmt == 33) { + myp[ii] = dot_iq3s_q8(kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) + } elif (fmt == 34) { + myp[ii] = dot_iq3xxs_q8(kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) + } elif (fmt == 45) { + myp[ii] = dot_iq4nl_q8(kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) + } elif (fmt == 2) { + myp[ii] = dot_k2q8(kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) + } elif (fmt == 23) { + myp[ii] = dot_iq2s_q8(kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) + } elif (fmt == 24) { + myp[ii] = dot_iq2xs_q8(kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) + } elif (fmt == 25) { + myp[ii] = dot_iq2xxs_q8(kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) } else { myp[ii] = dot_k6q8(kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) } diff --git a/modules/dasLLAMA/dasllama/dasllama_math_gen.das b/modules/dasLLAMA/dasllama/dasllama_math_gen.das index c7e75be3ed..27f4e2932d 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_gen.das @@ -56,28 +56,29 @@ require math // entries for (the completeness set — drift shows up as a loud "missing:" list at startup). [tune_scope(name = "dasllama", tuner = "../harness/dasllama_tuner.das", covers = "dasllama_math;dasllama_math_default;dasllama_math_aarch64_neon;dasllama_kv_codec;dasllama_convert", - version_of = "dasllama_version/DASLLAMA_RELEASE", version_key = "dasllama_version")] + version_of = "dasllama_version/DASLLAMA_RELEASE", version_key = "dasllama_version", + defaults = "../performance/defaults")] struct private DasllamaTuneScope {} //! The layout companion (six-function stamp): the repack interleave (mr) the stamped tile reads. //! Reference body = the reference tile's grp4; generator declines in lockstep with the tile //! generator, so a perm the tile can't emit keeps the grp4 reference pair on both sides. def q8q8_layout_gen() : int { - return q8q8_repack_type(GEMM_REFERENCE_MR).interleave + return q8q8_repack_type(GEMM_REFERENCE_MR()).interleave } //! The wbias companion (slice H): additive bias baked into the grp plane's GROUP-row bytes — //! 128 for bias128 vpdpbusd perms (w^0x80, −128·Σx acc-init correction), 0 otherwise. Reference //! body = the unbiased grp4 pair, declining in lockstep so plane bytes and kernels never disagree. def q8q8_wbias_gen() : int { - return q8q8_repack_type(GEMM_REFERENCE_MR).wbias + return q8q8_repack_type(GEMM_REFERENCE_MR()).wbias } //! The kgroup companion (slice J): k-bytes per row per interleave group — 8 for smmla perms //! (row-pair × 8-k MMA operands), 4 for every vector-dot perm. Reference body = the kg4 grp4 //! pair, declining in lockstep so plane byte order and kernels never disagree. def q8q8_kgroup_gen() : int { - return q8q8_repack_type(GEMM_REFERENCE_MR).kgroup + return q8q8_repack_type(GEMM_REFERENCE_MR()).kgroup } //! The tokstep companion (slice I): tokens one stamped tile call covers — 16·nrsplit for amx @@ -228,19 +229,55 @@ def kq_grp_row_dot(fmt : int64; kqg : uint8 const?; ksg : uint8 const?; r, mr : //! read. Each format tunes separately and takes its own winner's mr — k5/k6 tiles prefer grp4 //! while q8 crowns grp8 (M1, 2026-07-12). Reference bodies = grp4, declining in lockstep. def k4q8_layout_gen() : int { - return q8q8_repack_type(GEMM_REFERENCE_MR).interleave + return q8q8_repack_type(GEMM_REFERENCE_MR()).interleave } def k5q8_layout_gen() : int { - return q8q8_repack_type(GEMM_REFERENCE_MR).interleave + return q8q8_repack_type(GEMM_REFERENCE_MR()).interleave } def k6q8_layout_gen() : int { - return q8q8_repack_type(GEMM_REFERENCE_MR).interleave + return q8q8_repack_type(GEMM_REFERENCE_MR()).interleave } def q40q8_layout_gen() : int { - return q8q8_repack_type(GEMM_REFERENCE_MR).interleave + return q8q8_repack_type(GEMM_REFERENCE_MR()).interleave +} + +def iq4xsq8_layout_gen() : int { + return q8q8_repack_type(GEMM_REFERENCE_MR()).interleave +} + +def k3q8_layout_gen() : int { + return q8q8_repack_type(GEMM_REFERENCE_MR()).interleave +} + +def iq3sq8_layout_gen() : int { + return q8q8_repack_type(GEMM_REFERENCE_MR()).interleave +} + +def iq3xxsq8_layout_gen() : int { + return q8q8_repack_type(GEMM_REFERENCE_MR()).interleave +} + +def iq4nlq8_layout_gen() : int { + return q8q8_repack_type(GEMM_REFERENCE_MR()).interleave +} + +def k2q8_layout_gen() : int { + return q8q8_repack_type(GEMM_REFERENCE_MR()).interleave +} + +def iq2sq8_layout_gen() : int { + return q8q8_repack_type(GEMM_REFERENCE_MR()).interleave +} + +def iq2xsq8_layout_gen() : int { + return q8q8_repack_type(GEMM_REFERENCE_MR()).interleave +} + +def iq2xxsq8_layout_gen() : int { + return q8q8_repack_type(GEMM_REFERENCE_MR()).interleave } // the KernelBackend.kq_layout slot form of kq_layout_of (registered on both gen backends) @@ -258,6 +295,33 @@ def kq_layout_of(fmt : int) : int64 { if (fmt == 40) { return int64(q40q8_layout_gen()) } + if (fmt == 44) { + return int64(iq4xsq8_layout_gen()) + } + if (fmt == 3) { + return int64(k3q8_layout_gen()) + } + if (fmt == 33) { + return int64(iq3sq8_layout_gen()) + } + if (fmt == 34) { + return int64(iq3xxsq8_layout_gen()) + } + if (fmt == 45) { + return int64(iq4nlq8_layout_gen()) + } + if (fmt == 2) { + return int64(k2q8_layout_gen()) + } + if (fmt == 23) { + return int64(iq2sq8_layout_gen()) + } + if (fmt == 24) { + return int64(iq2xsq8_layout_gen()) + } + if (fmt == 25) { + return int64(iq2xxsq8_layout_gen()) + } return int64(k6q8_layout_gen()) } @@ -293,6 +357,121 @@ def q40_grp_row_dot(kqg : uint8 const?; ksg : uint8 const?; r, mr : int64; xqp : return acc } +//! One row's dot off the grp iq4xs planes, scalar — the iq4xs stubs' reference body and the +//! repack oracle: per 32-block idot = sum IQ4NL_LUT[q] * x, folded sum sc·idot over the +//! superblock (sc = the row's signed int8 sub-scale), then * (d * xs). No bsum term. +[unused_argument(xbsp)] +def iq4xs_grp_row_dot(kqg : uint8 const?; ksg : uint8 const?; r, mr : int64; xqp : int8 const?; xsp : float const?; xbsp : int const?; n : int64) : float { + var acc = 0.0 + let nsb = n / 256l + let lut = iq4nl_lut() + unsafe { + for (sbi in range64(nsb)) { + let qb = sbi * 128l * mr + let sb = sbi * 20l * mr + let d = f16_to_f32(uint(ksg[sb + 2l * r]) | (uint(ksg[sb + 2l * r + 1l]) << 8u)) + var isum = 0 + for (blk in range64(8l)) { + let b = sbi * 8l + blk + var idot = 0 + for (j in range64(4l)) { + for (t in range64(4l)) { + let nib = int(kqg[qb + ((blk * 4l + j) * mr + r) * 4l + t]) + idot += int(lut[nib & 15]) * int(xqp[b * 32l + j * 4l + t]) + idot += int(lut[nib >> 4]) * int(xqp[b * 32l + 16l + j * 4l + t]) + } + } + let u = int(ksg[sb + 4l * mr + blk * mr + r]) + isum += (u < 128 ? u : u - 256) * idot + } + acc += float(isum) * (d * xsp[sbi]) + } + } + return acc +} + +//! One row's dot off the grp k3 planes, scalar — the k3 stubs' reference body and the repack +//! oracle: k6's fold (per-16 signed sub-scales, the -4 offset through the activation sums) over +//! the 2-bit lane + hmask compose. Same float fold order as dot_k3q8. +def k3_grp_row_dot(kqg : uint8 const?; ksg : uint8 const?; r, mr : int64; xqp : int8 const?; xsp : float const?; xbsp : int const?; n : int64) : float { + var acc = 0.0 + let nsb = n / 256l + unsafe { + for (sbi in range64(nsb)) { + let qb = sbi * 96l * mr + let sb = sbi * 18l * mr + var isum = 0 + var bsum = 0 + for (blk in range64(8l)) { + let b = sbi * 8l + blk + let h = blk / 4l + let sh = int((blk % 4l) * 2l) + var ilo = 0 + var ihi = 0 + for (l4 in range64(8l)) { + for (t in range64(4l)) { + let q2 = (int(kqg[qb + ((h * 8l + l4) * mr + r) * 4l + t]) >> sh) & 3 + let hb = (int(kqg[qb + 64l * mr + (l4 * mr + r) * 4l + t]) >> int(blk)) & 1 + let q = q2 | (hb << 2) + if (l4 < 4l) { + ilo += q * int(xqp[b * 32l + l4 * 4l + t]) + } else { + ihi += q * int(xqp[b * 32l + l4 * 4l + t]) + } + } + } + let s0 = int(int8(ksg[sb + 2l * blk * mr + r])) + let s1 = int(int8(ksg[sb + (2l * blk + 1l) * mr + r])) + isum += s0 * ilo + s1 * ihi + bsum += s0 * xbsp[b * 2l] + s1 * xbsp[b * 2l + 1l] + } + let dk = f16_to_f32(uint(ksg[sb + 16l * mr + 2l * r]) | (uint(ksg[sb + 16l * mr + 2l * r + 1l]) << 8u)) + acc += float(isum - 4 * bsum) * (dk * xsp[sbi]) + } + } + return acc +} + +//! One row's dot off the grp iq3s planes, scalar — the iq3s stubs' reference body and the +//! repack oracle: the 26-column layout's per-block grid gather + sign flip, sub-scale (1 + 2s) +//! per block, no activation-sum term. Same integer regroup and float fold as dot_iq3s_q8. +[unused_argument(xbsp)] +def iq3s_grp_row_dot(kqg : uint8 const?; ksg : uint8 const?; r, mr : int64; xqp : int8 const?; xsp : float const?; xbsp : int const?; n : int64) : float { + var acc = 0.0 + let nsb = n / 256l + let grid = iq3s_grid() + unsafe { + for (sbi in range64(nsb)) { + let qb = sbi * 104l * mr + let sb = sbi * 20l * mr + var isum = 0 + for (blk in range64(8l)) { + let hp = 64l + blk + let qh = int(kqg[qb + ((hp / 4l) * mr + r) * 4l + hp % 4l]) + var ib = 0 + for (l in range64(4l)) { + let sp = 72l + blk * 4l + l + let sgn = int(kqg[qb + ((sp / 4l) * mr + r) * 4l + sp % 4l]) + let q1p = blk * 8l + 2l * l + let q2p = q1p + 1l + let g1 = grid[int(kqg[qb + ((q1p / 4l) * mr + r) * 4l + q1p % 4l]) | ((qh << int(8l - 2l * l)) & 256)] + let g2 = grid[int(kqg[qb + ((q2p / 4l) * mr + r) * 4l + q2p % 4l]) | ((qh << int(7l - 2l * l)) & 256)] + for (j in range64(4l)) { + let w1 = int((g1 >> uint(8l * j)) & 0xFFu) + let w2 = int((g2 >> uint(8l * j)) & 0xFFu) + ib += (((sgn >> int(j)) & 1) != 0 ? -w1 : w1) * int(xqp[(sbi * 8l + blk) * 32l + l * 8l + j]) + ib += (((sgn >> int(4l + j)) & 1) != 0 ? -w2 : w2) * int(xqp[(sbi * 8l + blk) * 32l + l * 8l + 4l + j]) + } + } + isum += int(int8(ksg[sb + 4l * mr + blk * mr + r])) * ib + } + let dk = f16_to_f32(uint(ksg[sb + 2l * r]) | (uint(ksg[sb + 2l * r + 1l]) << 8u)) + acc += float(isum) * (dk * xsp[sbi]) + } + } + return acc +} + //! The K-quant GEMV kernels (kq stage 4): rows [rb, re) of one plane-region pair off the grp //! kq planes (each format's OWN layout companion). Reference bodies = the scalar grp walk above, //! declining in lockstep. Not `private`: they ARE the kq_rows_* slots. @@ -344,21 +523,21 @@ def q40q8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp } } -// The kq tune grids (one [tune] family per format): kstep pinned 1, grid is (mr, nrsplit) per ISA -// tier. nrsplit token-splits the 4-token tile; nrsplit-4 (single unpack) wins wherever q-reg -// budget admits it. bias128/smmla/amx rows are absent — kq planes are natively unsigned. +// The kq tune grids (one [tune] family per format): kstep pinned 1, ONE seat per ISA tier — +// mr4/nrsplit clones pruned (no box ever crowned one outside noise; the race exists only where +// an ISA unlock changes the kernel). bias128/smmla/amx rows absent — kq planes are natively unsigned. //! The K-quant TILE kernels (kq batch family): 4 tokens x mr rows per call — kqg is the group's //! QUANT panel (k4 the packed grp plane; k5/k6 a BYTE-EXPANDED scratch unpacked once per //! (group, token-block)). One call is bit-exact vs 4 per-token GEMVs over the same weights. -[tune_perm(mr = 4), tune_perm(mr = 4, nrsplit = 2), tune_perm(mr = 8), tune_perm(mr = 8, nrsplit = 2), - tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "maddubs", width = 256, mr = 8, nrsplit = 2, requires = "avx2"), - tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, nrsplit = 2, requires = "avxvnni|avx512vnni"), - tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, nrsplit = 2, requires = "avx512vnni,avx512bw"), +[tune_perm(mr = 8), + tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), tune_companion(fn = "k4q8_gemv_gen", gen = "dasllama_gemm_gen::k4_gemv"), tune_companion(fn = "k4q8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::k4_tile", - fallback = "dot_vpdpbusd_width512_mr16;dot_vpdpbusd_width256_mr8;dot_maddubs_width256_mr8;mr4"), + fallback = "dot_vpdpbusd_width512_mr16;dot_vpdpbusd_width256_mr8;dot_maddubs_width256_mr8;mr8"), hint(unsafe_range_check, noalias = kqg, noalias = ksg, noalias = xqp, noalias = xsp, noalias = xbsp)] def k4q8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d, g, t0 : int64) : void { let mr = int64(k4q8_layout_gen()) @@ -371,14 +550,14 @@ def k4q8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp : } } -[tune_perm(mr = 4), tune_perm(mr = 4, nrsplit = 2), tune_perm(mr = 8), tune_perm(mr = 8, nrsplit = 2), - tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "maddubs", width = 256, mr = 8, nrsplit = 2, requires = "avx2"), - tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, nrsplit = 2, requires = "avxvnni|avx512vnni"), - tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, nrsplit = 2, requires = "avx512vnni,avx512bw"), +[tune_perm(mr = 8), + tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), tune_companion(fn = "k5q8_gemv_gen", gen = "dasllama_gemm_gen::k5_gemv"), tune_companion(fn = "k5q8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::k5_tile", - fallback = "dot_vpdpbusd_width512_mr16;dot_vpdpbusd_width256_mr8;dot_maddubs_width256_mr8;mr4"), + fallback = "dot_vpdpbusd_width512_mr16;dot_vpdpbusd_width256_mr8;dot_maddubs_width256_mr8;mr8"), hint(unsafe_range_check, noalias = kqg, noalias = ksg, noalias = xqp, noalias = xsp, noalias = xbsp)] def k5q8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d, g, t0 : int64) : void { let mr = int64(k5q8_layout_gen()) @@ -391,14 +570,14 @@ def k5q8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp : } } -[tune_perm(mr = 4), tune_perm(mr = 4, nrsplit = 2), tune_perm(mr = 8), tune_perm(mr = 8, nrsplit = 2), - tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "maddubs", width = 256, mr = 8, nrsplit = 2, requires = "avx2"), - tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, nrsplit = 2, requires = "avxvnni|avx512vnni"), - tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, nrsplit = 2, requires = "avx512vnni,avx512bw"), +[tune_perm(mr = 8), + tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), tune_companion(fn = "k6q8_gemv_gen", gen = "dasllama_gemm_gen::k6_gemv"), tune_companion(fn = "k6q8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::k6_tile", - fallback = "dot_vpdpbusd_width512_mr16;dot_vpdpbusd_width256_mr8;dot_maddubs_width256_mr8;mr4"), + fallback = "dot_vpdpbusd_width512_mr16;dot_vpdpbusd_width256_mr8;dot_maddubs_width256_mr8;mr8"), hint(unsafe_range_check, noalias = kqg, noalias = ksg, noalias = xqp, noalias = xsp, noalias = xbsp)] def k6q8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d, g, t0 : int64) : void { let mr = int64(k6q8_layout_gen()) @@ -411,14 +590,14 @@ def k6q8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp : } } -[tune_perm(mr = 4), tune_perm(mr = 4, nrsplit = 2), tune_perm(mr = 8), tune_perm(mr = 8, nrsplit = 2), - tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "maddubs", width = 256, mr = 8, nrsplit = 2, requires = "avx2"), - tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, nrsplit = 2, requires = "avxvnni|avx512vnni"), - tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, nrsplit = 2, requires = "avx512vnni,avx512bw"), +[tune_perm(mr = 8), + tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), tune_companion(fn = "q40q8_gemv_gen", gen = "dasllama_gemm_gen::q40_gemv"), tune_companion(fn = "q40q8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::q40_tile", - fallback = "dot_vpdpbusd_width512_mr16;dot_vpdpbusd_width256_mr8;dot_maddubs_width256_mr8;mr4"), + fallback = "dot_vpdpbusd_width512_mr16;dot_vpdpbusd_width256_mr8;dot_maddubs_width256_mr8;mr8"), hint(unsafe_range_check, noalias = kqg, noalias = ksg, noalias = xqp, noalias = xsp, noalias = xbsp)] def q40q8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d, g, t0 : int64) : void { let mr = int64(q40q8_layout_gen()) @@ -431,6 +610,525 @@ def q40q8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp } } +[hint(unsafe_range_check, noalias = kqp, noalias = ksp, noalias = xqp, noalias = xsp, noalias = xbsp)] +def iq4xsq8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, rb, re : int64) : void { + let mr = int64(iq4xsq8_layout_gen()) + let nsb = n / 256l + unsafe { + for (i in range64(rb, re)) { + let g = i / mr + yp[i] = iq4xs_grp_row_dot(kqp + g * mr * nsb * 128l, ksp + g * mr * nsb * 20l, i % mr, mr, xqp, xsp, xbsp, n) + } + } +} + +[tune_perm(mr = 8), + tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), + tune_companion(fn = "iq4xsq8_gemv_gen", gen = "dasllama_gemm_gen::iq4xs_gemv"), + tune_companion(fn = "iq4xsq8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), + tune(gen = "dasllama_gemm_gen::iq4xs_tile", + fallback = "dot_vpdpbusd_width512_mr16;dot_vpdpbusd_width256_mr8;dot_maddubs_width256_mr8;mr8"), + hint(unsafe_range_check, noalias = kqg, noalias = ksg, noalias = xqp, noalias = xsp, noalias = xbsp)] +def iq4xsq8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d, g, t0 : int64) : void { + let mr = int64(iq4xsq8_layout_gen()) + unsafe { + for (t in range64(4l)) { + for (r in range64(mr)) { + yp[(t0 + t) * d + g * mr + r] = iq4xs_grp_row_dot(kqg, ksg, r, mr, xqp + (t0 + t) * n, xsp + (t0 + t) * (n / 256l), xbsp + (t0 + t) * (n / 16l), n) + } + } + } +} + +[hint(unsafe_range_check, noalias = kqp, noalias = ksp, noalias = xqp, noalias = xsp, noalias = xbsp)] +def k3q8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, rb, re : int64) : void { + let mr = int64(k3q8_layout_gen()) + let nsb = n / 256l + unsafe { + for (i in range64(rb, re)) { + let g = i / mr + yp[i] = k3_grp_row_dot(kqp + g * mr * nsb * 96l, ksp + g * mr * nsb * 18l, i % mr, mr, xqp, xsp, xbsp, n) + } + } +} + +[tune_perm(mr = 8), + tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), + tune_companion(fn = "k3q8_gemv_gen", gen = "dasllama_gemm_gen::k3_gemv"), + tune_companion(fn = "k3q8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), + tune(gen = "dasllama_gemm_gen::k3_tile", + fallback = "dot_vpdpbusd_width512_mr16;dot_vpdpbusd_width256_mr8;dot_maddubs_width256_mr8;mr8"), + hint(unsafe_range_check, noalias = kqg, noalias = ksg, noalias = xqp, noalias = xsp, noalias = xbsp)] +def k3q8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d, g, t0 : int64) : void { + let mr = int64(k3q8_layout_gen()) + unsafe { + for (t in range64(4l)) { + for (r in range64(mr)) { + yp[(t0 + t) * d + g * mr + r] = k3_grp_row_dot(kqg, ksg, r, mr, xqp + (t0 + t) * n, xsp + (t0 + t) * (n / 256l), xbsp + (t0 + t) * (n / 16l), n) + } + } + } +} + +[hint(unsafe_range_check, noalias = kqp, noalias = ksp, noalias = xqp, noalias = xsp, noalias = xbsp)] +def iq3sq8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, rb, re : int64) : void { + let mr = int64(iq3sq8_layout_gen()) + let nsb = n / 256l + unsafe { + for (i in range64(rb, re)) { + let g = i / mr + yp[i] = iq3s_grp_row_dot(kqp + g * mr * nsb * 104l, ksp + g * mr * nsb * 20l, i % mr, mr, xqp, xsp, xbsp, n) + } + } +} + +[tune_perm(mr = 8), + tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), + tune_companion(fn = "iq3sq8_gemv_gen", gen = "dasllama_gemm_gen::iq3s_gemv"), + tune_companion(fn = "iq3sq8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), + tune(gen = "dasllama_gemm_gen::iq3s_tile", + fallback = "dot_vpdpbusd_width512_mr16;dot_vpdpbusd_width256_mr8;dot_maddubs_width256_mr8;mr8"), + hint(unsafe_range_check, noalias = kqg, noalias = ksg, noalias = xqp, noalias = xsp, noalias = xbsp)] +def iq3sq8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d, g, t0 : int64) : void { + let mr = int64(iq3sq8_layout_gen()) + unsafe { + for (t in range64(4l)) { + for (r in range64(mr)) { + yp[(t0 + t) * d + g * mr + r] = kq_grp_row_dot_b(33l, kqg, ksg, r, mr, xqp + (t0 + t) * n, xsp + (t0 + t) * (n / 256l), xbsp + (t0 + t) * (n / 16l), n) + } + } + } +} + +//! One row's dot off the grp iq3xxs planes, scalar — the iq3xxs stubs' reference body and +//! the tile oracle. d_halved x (2ls + 1) per block, signs off the aux 7-bit ksigns indices, +//! no activation-sum term. Same integer regroup and float fold as dot_iq3xxs_q8. +[unused_argument(xbsp), hint(unsafe_range_check, noalias = kqg, noalias = ksg, noalias = xqp, noalias = xsp, noalias = xbsp)] +def iq3xxs_grp_row_dot(kqg : uint8 const?; ksg : uint8 const?; r, mr : int64; xqp : int8 const?; xsp : float const?; xbsp : int const?; n : int64) : float { + var acc = 0.0 + let nsb = n / 256l + let grid = iq3xxs_grid() + let signs = ksigns_iq2xs() + unsafe { + for (sbi in range64(nsb)) { + let qb = sbi * 96l * mr + let sb = sbi * 20l * mr + var isum = 0 + for (blk in range64(8l)) { + var aux = 0u + for (t in range64(4l)) { + let ap = 64l + blk * 4l + t + aux |= uint(kqg[qb + ((ap / 4l) * mr + r) * 4l + ap % 4l]) << uint(8l * t) + } + var ib = 0 + for (l in range64(4l)) { + let sgn = int(signs[int((aux >> uint(7l * l)) & 127u)]) + let q1p = blk * 8l + 2l * l + let q2p = q1p + 1l + let g1 = grid[int(kqg[qb + ((q1p / 4l) * mr + r) * 4l + q1p % 4l])] + let g2 = grid[int(kqg[qb + ((q2p / 4l) * mr + r) * 4l + q2p % 4l])] + for (j in range64(4l)) { + let w1 = int((g1 >> uint(8l * j)) & 0xFFu) + let w2 = int((g2 >> uint(8l * j)) & 0xFFu) + ib += (((sgn >> int(j)) & 1) != 0 ? -w1 : w1) * int(xqp[(sbi * 8l + blk) * 32l + l * 8l + j]) + ib += (((sgn >> int(4l + j)) & 1) != 0 ? -w2 : w2) * int(xqp[(sbi * 8l + blk) * 32l + l * 8l + 4l + j]) + } + } + isum += int(int8(ksg[sb + 4l * mr + blk * mr + r])) * ib + } + let dk = f16_to_f32(uint(ksg[sb + 2l * r]) | (uint(ksg[sb + 2l * r + 1l]) << 8u)) + acc += float(isum) * (dk * xsp[sbi]) + } + } + return acc +} + +[hint(unsafe_range_check, noalias = kqp, noalias = ksp, noalias = xqp, noalias = xsp, noalias = xbsp)] +def iq3xxsq8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, rb, re : int64) : void { + let mr = int64(iq3xxsq8_layout_gen()) + let nsb = n / 256l + unsafe { + for (i in range64(rb, re)) { + let g = i / mr + yp[i] = iq3xxs_grp_row_dot(kqp + g * mr * nsb * 96l, ksp + g * mr * nsb * 20l, i % mr, mr, xqp, xsp, xbsp, n) + } + } +} + +[tune_perm(mr = 8), + tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), + tune_companion(fn = "iq3xxsq8_gemv_gen", gen = "dasllama_gemm_gen::iq3xxs_gemv"), + tune_companion(fn = "iq3xxsq8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), + tune(gen = "dasllama_gemm_gen::iq3xxs_tile", + fallback = "dot_vpdpbusd_width512_mr16;dot_vpdpbusd_width256_mr8;dot_maddubs_width256_mr8;mr8"), + hint(unsafe_range_check, noalias = kqg, noalias = ksg, noalias = xqp, noalias = xsp, noalias = xbsp)] +def iq3xxsq8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d, g, t0 : int64) : void { + let mr = int64(iq3xxsq8_layout_gen()) + unsafe { + for (t in range64(4l)) { + for (r in range64(mr)) { + yp[(t0 + t) * d + g * mr + r] = kq_grp_row_dot_b(34l, kqg, ksg, r, mr, xqp + (t0 + t) * n, xsp + (t0 + t) * (n / 256l), xbsp + (t0 + t) * (n / 16l), n) + } + } + } +} + +//! One row's dot off the grp iq4nl planes, scalar - q40's walk with the codebook and no +//! bsum term (the LUT values carry no offset). The iq4nl stubs' reference body and tile oracle. +[unused_argument(xbsp), hint(unsafe_range_check, noalias = kqg, noalias = ksg, noalias = xqp, noalias = xsp, noalias = xbsp)] +def iq4nl_grp_row_dot(kqg : uint8 const?; ksg : uint8 const?; r, mr : int64; xqp : int8 const?; xsp : float const?; xbsp : int const?; n : int64) : float { + var acc = 0.0 + let nsb = n / 256l + let lut = iq4nl_lut() + unsafe { + for (sbi in range64(nsb)) { + let qb = sbi * 128l * mr + let sb = sbi * 16l * mr + var facc = 0.0 + for (blk in range64(8l)) { + let b = sbi * 8l + blk + var idot = 0 + for (j in range64(4l)) { + for (t in range64(4l)) { + let nib = uint(kqg[qb + ((blk * 4l + j) * mr + r) * 4l + t]) + idot += int(lut[int(nib & 15u)]) * int(xqp[b * 32l + j * 4l + t]) + idot += int(lut[int(nib >> 4u)]) * int(xqp[b * 32l + 16l + j * 4l + t]) + } + } + let d = f16_to_f32(uint(ksg[sb + blk * 2l * mr + 2l * r]) | (uint(ksg[sb + blk * 2l * mr + 2l * r + 1l]) << 8u)) + facc += float(idot) * d + } + acc += facc * xsp[sbi] + } + } + return acc +} + +[unused_argument(xbsp), hint(unsafe_range_check, noalias = kqp, noalias = ksp, noalias = xqp, noalias = xsp, noalias = xbsp)] +def iq4nlq8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, rb, re : int64) : void { + let mr = int64(iq4nlq8_layout_gen()) + let nsb = n / 256l + unsafe { + for (i in range64(rb, re)) { + let g = i / mr + yp[i] = iq4nl_grp_row_dot(kqp + g * mr * nsb * 128l, ksp + g * mr * nsb * 16l, i % mr, mr, xqp, xsp, xbsp, n) + } + } +} + +[tune_perm(mr = 8), + tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), + tune_companion(fn = "iq4nlq8_gemv_gen", gen = "dasllama_gemm_gen::iq4nl_gemv"), + tune_companion(fn = "iq4nlq8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), + tune(gen = "dasllama_gemm_gen::iq4nl_tile", + fallback = "dot_vpdpbusd_width512_mr16;dot_vpdpbusd_width256_mr8;dot_maddubs_width256_mr8;mr8"), + hint(unsafe_range_check, noalias = kqg, noalias = ksg, noalias = xqp, noalias = xsp, noalias = xbsp)] +def iq4nlq8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d, g, t0 : int64) : void { + let mr = int64(iq4nlq8_layout_gen()) + unsafe { + for (t in range64(4l)) { + for (r in range64(mr)) { + yp[(t0 + t) * d + g * mr + r] = iq4nl_grp_row_dot(kqg, ksg, r, mr, xqp + (t0 + t) * n, xsp + (t0 + t) * (n / 256l), xbsp + (t0 + t) * (n / 16l), n) + } + } + } +} + +//! One row's dot off the grp k2 planes, scalar - the k2 stubs' reference body and the +//! repack oracle: the k4/k5 fold (per-16 nibble sc/min pairs, the min term through the +//! activation sums) over the in-place 2-bit lanes. Same float fold order as dot_k2q8. +def k2_grp_row_dot(kqg : uint8 const?; ksg : uint8 const?; r, mr : int64; xqp : int8 const?; xsp : float const?; xbsp : int const?; n : int64) : float { + var acc = 0.0 + let nsb = n / 256l + unsafe { + for (sbi in range64(nsb)) { + let qb = sbi * 64l * mr + let sb = sbi * 20l * mr + var isum = 0 + var bsum = 0 + for (g in range64(16l)) { + let scb = int(ksg[sb + g * mr + r]) + let base = (g / 8l) * 8l + (g % 2l) * 4l + let sh = int(((g % 8l) / 2l) * 2l) + var ig = 0 + for (c4 in range64(4l)) { + for (t in range64(4l)) { + let q = (int(kqg[qb + ((base + c4) * mr + r) * 4l + t]) >> sh) & 3 + ig += q * int(xqp[sbi * 256l + g * 16l + c4 * 4l + t]) + } + } + isum += (scb & 15) * ig + bsum += (scb >> 4) * xbsp[sbi * 16l + g] + } + let hd = sb + 16l * mr + 2l * r + let dk = f16_to_f32(uint(ksg[hd]) | (uint(ksg[hd + 1l]) << 8u)) + let hm = sb + 18l * mr + 2l * r + let dm = f16_to_f32(uint(ksg[hm]) | (uint(ksg[hm + 1l]) << 8u)) + acc += float(isum) * (dk * xsp[sbi]) + acc -= float(bsum) * (dm * xsp[sbi]) + } + } + return acc +} + +[hint(unsafe_range_check, noalias = kqp, noalias = ksp, noalias = xqp, noalias = xsp, noalias = xbsp)] +def k2q8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, rb, re : int64) : void { + let mr = int64(k2q8_layout_gen()) + let nsb = n / 256l + unsafe { + for (i in range64(rb, re)) { + let g = i / mr + yp[i] = k2_grp_row_dot(kqp + g * mr * nsb * 64l, ksp + g * mr * nsb * 20l, i % mr, mr, xqp, xsp, xbsp, n) + } + } +} + +[tune_perm(mr = 8), + tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), + tune_companion(fn = "k2q8_gemv_gen", gen = "dasllama_gemm_gen::k2_gemv"), + tune_companion(fn = "k2q8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), + tune(gen = "dasllama_gemm_gen::k2_tile", + fallback = "dot_vpdpbusd_width512_mr16;dot_vpdpbusd_width256_mr8;dot_maddubs_width256_mr8;mr8"), + hint(unsafe_range_check, noalias = kqg, noalias = ksg, noalias = xqp, noalias = xsp, noalias = xbsp)] +def k2q8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d, g, t0 : int64) : void { + let mr = int64(k2q8_layout_gen()) + unsafe { + for (t in range64(4l)) { + for (r in range64(mr)) { + yp[(t0 + t) * d + g * mr + r] = k2_grp_row_dot(kqg, ksg, r, mr, xqp + (t0 + t) * n, xsp + (t0 + t) * (n / 256l), xbsp + (t0 + t) * (n / 16l), n) + } + } + } +} + +//! One row's dot off the grp iq2s planes, scalar - the iq2s stubs' reference body and the +//! repack oracle: the 18-column layout's per-block grid gather + sign flip, per-16 strips +//! (1 + 2s) on the eighth-ed d, no activation-sum term. Same fold order as dot_iq2s_q8. +[unused_argument(xbsp)] +def iq2s_grp_row_dot(kqg : uint8 const?; ksg : uint8 const?; r, mr : int64; xqp : int8 const?; xsp : float const?; xbsp : int const?; n : int64) : float { + var acc = 0.0 + let nsb = n / 256l + let grid = iq2s_grid2() + unsafe { + for (sbi in range64(nsb)) { + let qb = sbi * 72l * mr + let sb = sbi * 20l * mr + var isum = 0 + for (b in range64(8l)) { + let hp = 64l + b + let qh = int(kqg[qb + ((hp / 4l) * mr + r) * 4l + hp % 4l]) + for (l in range64(4l)) { + let sp = 32l + b * 4l + l + let sgn = int(kqg[qb + ((sp / 4l) * mr + r) * 4l + sp % 4l]) + let ip = b * 4l + l + let idx = int(kqg[qb + ((ip / 4l) * mr + r) * 4l + ip % 4l]) | ((qh << int(8l - 2l * l)) & 0x300) + let g1 = grid[idx * 2] + let g2 = grid[idx * 2 + 1] + var ig = 0 + for (j in range64(4l)) { + let w1 = int((g1 >> uint(8l * j)) & 0xFFu) + let w2 = int((g2 >> uint(8l * j)) & 0xFFu) + ig += (((sgn >> int(j)) & 1) != 0 ? -w1 : w1) * int(xqp[(sbi * 8l + b) * 32l + l * 8l + j]) + ig += (((sgn >> int(4l + j)) & 1) != 0 ? -w2 : w2) * int(xqp[(sbi * 8l + b) * 32l + l * 8l + 4l + j]) + } + isum += int(ksg[sb + 4l * mr + (2l * b + l / 2l) * mr + r]) * ig + } + } + let dk = f16_to_f32(uint(ksg[sb + 2l * r]) | (uint(ksg[sb + 2l * r + 1l]) << 8u)) + acc += float(isum) * (dk * xsp[sbi]) + } + } + return acc +} + +[hint(unsafe_range_check, noalias = kqp, noalias = ksp, noalias = xqp, noalias = xsp, noalias = xbsp)] +def iq2sq8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, rb, re : int64) : void { + let mr = int64(iq2sq8_layout_gen()) + let nsb = n / 256l + unsafe { + for (i in range64(rb, re)) { + let g = i / mr + yp[i] = iq2s_grp_row_dot(kqp + g * mr * nsb * 72l, ksp + g * mr * nsb * 20l, i % mr, mr, xqp, xsp, xbsp, n) + } + } +} + +[tune_perm(mr = 8), + tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), + tune_companion(fn = "iq2sq8_gemv_gen", gen = "dasllama_gemm_gen::iq2s_gemv"), + tune_companion(fn = "iq2sq8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), + tune(gen = "dasllama_gemm_gen::iq2s_tile", + fallback = "dot_vpdpbusd_width512_mr16;dot_vpdpbusd_width256_mr8;dot_maddubs_width256_mr8;mr8"), + hint(unsafe_range_check, noalias = kqg, noalias = ksg, noalias = xqp, noalias = xsp, noalias = xbsp)] +def iq2sq8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d, g, t0 : int64) : void { + let mr = int64(iq2sq8_layout_gen()) + unsafe { + for (t in range64(4l)) { + for (r in range64(mr)) { + yp[(t0 + t) * d + g * mr + r] = kq_grp_row_dot_b(23l, kqg, ksg, r, mr, xqp + (t0 + t) * n, xsp + (t0 + t) * (n / 256l), xbsp + (t0 + t) * (n / 16l), n) + } + } + } +} + +//! One row's dot off the grp iq2xs planes, scalar - the iq2xs stubs' reference body and +//! the repack oracle: per u16 qs word the 9-bit grid gather + ksigns sign flip, per-16 strips +//! (1 + 2s) on the eighth-ed d, no activation-sum term. Same fold order as dot_iq2xs_q8. +[unused_argument(xbsp)] +def iq2xs_grp_row_dot(kqg : uint8 const?; ksg : uint8 const?; r, mr : int64; xqp : int8 const?; xsp : float const?; xbsp : int const?; n : int64) : float { + var acc = 0.0 + let nsb = n / 256l + let grid = iq2xs_grid2() + let ksgn = ksigns_iq2xs() + unsafe { + for (sbi in range64(nsb)) { + let qb = sbi * 64l * mr + let sb = sbi * 20l * mr + var isum = 0 + for (b in range64(8l)) { + for (l in range64(4l)) { + let lp = (b * 4l + l) * 2l + let w16 = uint(kqg[qb + ((lp / 4l) * mr + r) * 4l + lp % 4l]) | (uint(kqg[qb + ((lp / 4l) * mr + r) * 4l + lp % 4l + 1l]) << 8u) + let sgn = int(ksgn[int(w16 >> 9u)]) + let idx = int(w16 & 511u) + let g1 = grid[idx * 2] + let g2 = grid[idx * 2 + 1] + var ig = 0 + for (j in range64(4l)) { + let w1 = int((g1 >> uint(8l * j)) & 0xFFu) + let w2 = int((g2 >> uint(8l * j)) & 0xFFu) + ig += (((sgn >> int(j)) & 1) != 0 ? -w1 : w1) * int(xqp[(sbi * 8l + b) * 32l + l * 8l + j]) + ig += (((sgn >> int(4l + j)) & 1) != 0 ? -w2 : w2) * int(xqp[(sbi * 8l + b) * 32l + l * 8l + 4l + j]) + } + isum += int(ksg[sb + 4l * mr + (2l * b + l / 2l) * mr + r]) * ig + } + } + let dk = f16_to_f32(uint(ksg[sb + 2l * r]) | (uint(ksg[sb + 2l * r + 1l]) << 8u)) + acc += float(isum) * (dk * xsp[sbi]) + } + } + return acc +} + +[hint(unsafe_range_check, noalias = kqp, noalias = ksp, noalias = xqp, noalias = xsp, noalias = xbsp)] +def iq2xsq8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, rb, re : int64) : void { + let mr = int64(iq2xsq8_layout_gen()) + let nsb = n / 256l + unsafe { + for (i in range64(rb, re)) { + let g = i / mr + yp[i] = iq2xs_grp_row_dot(kqp + g * mr * nsb * 64l, ksp + g * mr * nsb * 20l, i % mr, mr, xqp, xsp, xbsp, n) + } + } +} + +[tune_perm(mr = 8), + tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), + tune_companion(fn = "iq2xsq8_gemv_gen", gen = "dasllama_gemm_gen::iq2xs_gemv"), + tune_companion(fn = "iq2xsq8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), + tune(gen = "dasllama_gemm_gen::iq2xs_tile", + fallback = "dot_vpdpbusd_width512_mr16;dot_vpdpbusd_width256_mr8;dot_maddubs_width256_mr8;mr8"), + hint(unsafe_range_check, noalias = kqg, noalias = ksg, noalias = xqp, noalias = xsp, noalias = xbsp)] +def iq2xsq8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d, g, t0 : int64) : void { + let mr = int64(iq2xsq8_layout_gen()) + unsafe { + for (t in range64(4l)) { + for (r in range64(mr)) { + yp[(t0 + t) * d + g * mr + r] = kq_grp_row_dot_b(24l, kqg, ksg, r, mr, xqp + (t0 + t) * n, xsp + (t0 + t) * (n / 256l), xbsp + (t0 + t) * (n / 16l), n) + } + } + } +} + + +//! One row's dot off the grp iq2xxs planes, scalar - the iq2xxs stubs' reference body and +//! the repack oracle: column 2b carries block b's grid bytes, column 2b+1 its aux32 (ksigns + +//! per-32 scale, mirrored into the strip row). Same fold order as dot_iq2xxs_q8. +[unused_argument(xbsp)] +def iq2xxs_grp_row_dot(kqg : uint8 const?; ksg : uint8 const?; r, mr : int64; xqp : int8 const?; xsp : float const?; xbsp : int const?; n : int64) : float { + var acc = 0.0 + let nsb = n / 256l + let grid = iq2xxs_grid2() + let ksgn = ksigns_iq2xs() + unsafe { + for (sbi in range64(nsb)) { + let qb = sbi * 64l * mr + let sb = sbi * 20l * mr + var isum = 0 + for (b in range64(8l)) { + let ac = (2l * b + 1l) * mr + r + let aux = uint(kqg[qb + ac * 4l]) | (uint(kqg[qb + ac * 4l + 1l]) << 8u) | (uint(kqg[qb + ac * 4l + 2l]) << 16u) | (uint(kqg[qb + ac * 4l + 3l]) << 24u) + let gc = (2l * b) * mr + r + var ibb = 0 + for (l in range64(4l)) { + let sgn = int(ksgn[int((aux >> uint(7l * l)) & 127u)]) + let idx = int(kqg[qb + gc * 4l + l]) + let g1 = grid[idx * 2] + let g2 = grid[idx * 2 + 1] + for (j in range64(4l)) { + let w1 = int((g1 >> uint(8l * j)) & 0xFFu) + let w2 = int((g2 >> uint(8l * j)) & 0xFFu) + ibb += (((sgn >> int(j)) & 1) != 0 ? -w1 : w1) * int(xqp[(sbi * 8l + b) * 32l + l * 8l + j]) + ibb += (((sgn >> int(4l + j)) & 1) != 0 ? -w2 : w2) * int(xqp[(sbi * 8l + b) * 32l + l * 8l + 4l + j]) + } + } + isum += int(ksg[sb + 4l * mr + b * mr + r]) * ibb + } + let dk = f16_to_f32(uint(ksg[sb + 2l * r]) | (uint(ksg[sb + 2l * r + 1l]) << 8u)) + acc += float(isum) * (dk * xsp[sbi]) + } + } + return acc +} + +[hint(unsafe_range_check, noalias = kqp, noalias = ksp, noalias = xqp, noalias = xsp, noalias = xbsp)] +def iq2xxsq8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, rb, re : int64) : void { + let mr = int64(iq2xxsq8_layout_gen()) + let nsb = n / 256l + unsafe { + for (i in range64(rb, re)) { + let g = i / mr + yp[i] = iq2xxs_grp_row_dot(kqp + g * mr * nsb * 64l, ksp + g * mr * nsb * 20l, i % mr, mr, xqp, xsp, xbsp, n) + } + } +} + +[tune_perm(mr = 8), + tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), + tune_companion(fn = "iq2xxsq8_gemv_gen", gen = "dasllama_gemm_gen::iq2xxs_gemv"), + tune_companion(fn = "iq2xxsq8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), + tune(gen = "dasllama_gemm_gen::iq2xxs_tile", + fallback = "dot_vpdpbusd_width512_mr16;dot_vpdpbusd_width256_mr8;dot_maddubs_width256_mr8;mr8"), + hint(unsafe_range_check, noalias = kqg, noalias = ksg, noalias = xqp, noalias = xsp, noalias = xbsp)] +def iq2xxsq8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d, g, t0 : int64) : void { + let mr = int64(iq2xxsq8_layout_gen()) + unsafe { + for (t in range64(4l)) { + for (r in range64(mr)) { + yp[(t0 + t) * d + g * mr + r] = kq_grp_row_dot_b(25l, kqg, ksg, r, mr, xqp + (t0 + t) * n, xsp + (t0 + t) * (n / 256l), xbsp + (t0 + t) * (n / 16l), n) + } + } + } +} + //! The f16-scale GEMV companion (wscale_f16 rail): q8q8_gemv_gen's twin over the raw binary16 //! group-scale plane. Same rows-range contract; the generated body differs only in the scale //! fold's widen (fcvtl/vcvtph2ps). Not `private`: it IS the mm_rows_s16 slot. @@ -469,21 +1167,13 @@ def q8q8_tile_s16_gen(var yp : float?; wg : int8 const?; sg : uint16 const?; xqp //! The generated tile: mr interleaved rows x 4 tokens (the q8q8_repack_type(mr) layout). Reference //! body = 4 single-token laneq4 dots, runs on interp/AOT/non-arm64 JIT/generator-declined; ignores //! xbsp unless a biased stamp reads it. dot=/width= rows are the x64 legs (slice F), one grid per ISA. -[tune_perm(kstep = 1), tune_perm(kstep = 2), tune_perm(kstep = 4), tune_perm(kstep = 2, nrsplit = 2), - tune_perm(kstep = 1, nrsplit = 2), tune_perm(kstep = 4, nrsplit = 2), - tune_perm(kstep = 1, nrsplit = 2, mr = 8), tune_perm(kstep = 2, nrsplit = 2, mr = 8), tune_perm(kstep = 4, nrsplit = 2, mr = 8), - tune_perm(kstep = 2, gkstep = 2), tune_perm(kstep = 2, gkstep = 4), - tune_perm(kstep = 4, nrsplit = 2, mr = 8, gkstep = 2), tune_perm(kstep = 4, nrsplit = 2, mr = 8, gkstep = 4), - tune_perm(dot = "maddubs", width = 256, mr = 8, kstep = 2, requires = "avx2"), tune_perm(dot = "maddubs", width = 256, mr = 8, kstep = 4, nrsplit = 2), - tune_perm(dot = "vpdpbusd", width = 256, mr = 8, kstep = 2), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, kstep = 4, nrsplit = 2), - tune_perm(dot = "vpdpbusd", width = 256, mr = 16, kstep = 2, nrsplit = 2), tune_perm(dot = "vpdpbssd", width = 256, mr = 8, kstep = 2), - tune_perm(dot = "maddubs", width = 512, mr = 16, kstep = 2), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, kstep = 2), - tune_perm(dot = "vpdpbusd", width = 512, mr = 16, kstep = 4, nrsplit = 2), tune_perm(dot = "vpdpbusd", width = 512, mr = 32, kstep = 2, nrsplit = 2), - tune_perm(dot = "vpdpbusd", width = 256, mr = 8, kstep = 2, gkstep = 2), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, kstep = 2, gkstep = 2), - tune_perm(dot = "vpdpbusd", width = 256, mr = 8, kstep = 2, bias = 128), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, kstep = 2, gkstep = 2, bias = 128, requires = "avxvnni|avx512vnni"), - tune_perm(dot = "vpdpbusd", width = 512, mr = 16, kstep = 2, bias = 128), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, kstep = 2, gkstep = 2, bias = 128, requires = "avx512vnni,avx512bw"), - tune_perm(dot = "vpdpbusd", width = 512, mr = 16, kstep = 4, nrsplit = 2, bias = 128), - tune_perm(dot = "maddubs", width = 256, mr = 8, kstep = 2, bias = 128), +[tune_perm(kstep = 2), tune_perm(suffix = "mr8_budget", mr = 8), + tune_perm(dot = "maddubs", width = 256, mr = 8, kstep = 2, requires = "avx2"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 8, kstep = 2, requires = "avxvnni|avx512vnni"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 8, kstep = 2, gkstep = 2, bias = 128, requires = "avxvnni|avx512vnni"), + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, kstep = 2, requires = "avx512vnni,avx512bw"), + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, kstep = 2, gkstep = 2, bias = 128, requires = "avx512vnni,avx512bw"), + tune_perm(dot = "vpdpbssd", width = 256, mr = 8, kstep = 2, requires = "avxvnniint8"), tune_perm(dot = "smmla", mr = 4, kstep = 2, requires = "i8mm"), tune_perm(dot = "smmla", mr = 4, kstep = 2, nrsplit = 2, requires = "i8mm"), tune_perm(dot = "smmla", mr = 8, kstep = 2, nrsplit = 2, requires = "i8mm"), tune_perm(dot = "smmla", mr = 8, kstep = 4, nrsplit = 2, requires = "i8mm"), tune_perm(dot = "smmla", mr = 8, kstep = 2, nrsplit = 2, gkstep = 2, requires = "i8mm"), @@ -497,7 +1187,6 @@ def q8q8_tile_s16_gen(var yp : float?; wg : int8 const?; sg : uint16 const?; xqp tune_perm(dot = "amx_int8", width = 512, mr = 16, kstep = 1, nrsplit = 2, pipe = 1, bias = 128, requires = "amx-int8"), tune_perm(dot = "amx_int8", width = 512, mr = 16, kstep = 1, nrsplit = 2, latch = 1, bias = 128, requires = "amx-int8"), tune_perm(dot = "amx_int8", width = 512, mr = 16, kstep = 1, nrsplit = 2, pipe = 1, latch = 1, bias = 128, requires = "amx-int8"), - tune_perm(suffix = "mr8_budget", mr = 8), tune_perm(suffix = "mr2_lanes", mr = 2), tune_perm(dot = "vpdpbusd"), tune_companion(fn = "q8q8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune_companion(fn = "q8q8_wbias_gen", gen = "dasllama_gemm_gen::q8q8_wbias"), tune_companion(fn = "q8q8_kgroup_gen", gen = "dasllama_gemm_gen::q8q8_kgroup"), @@ -616,14 +1305,14 @@ def q51q8_gemv_gen(var yp : float?; qgp : uint8 const?; sgp : uint8 const?; xqp //! The q51 TILE kernel (the family anchor): 4 tokens x mr rows per call off the group's packed //! grp plane pair — the kq tile contract with the 32-weight block unit (Q8_0-form //! activations: per-32 xs and per-32 asum planes). Bit-exact vs 4 per-token GEMVs. -[tune_perm(mr = 4), tune_perm(mr = 4, nrsplit = 2), tune_perm(mr = 8), tune_perm(mr = 8, nrsplit = 2), - tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "maddubs", width = 256, mr = 8, nrsplit = 2, requires = "avx2"), - tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, nrsplit = 2, requires = "avxvnni|avx512vnni"), - tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, nrsplit = 2, requires = "avx512vnni,avx512bw"), +[tune_perm(mr = 8), + tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), tune_companion(fn = "q51q8_gemv_gen", gen = "dasllama_gemm_gen::q51_gemv"), tune_companion(fn = "q51q8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::q51_tile", - fallback = "dot_vpdpbusd_width512_mr16;dot_vpdpbusd_width256_mr8;dot_maddubs_width256_mr8;mr4"), + fallback = "dot_vpdpbusd_width512_mr16;dot_vpdpbusd_width256_mr8;dot_maddubs_width256_mr8;mr8"), hint(unsafe_range_check, noalias = qgp, noalias = sgp, noalias = xqp, noalias = xsp, noalias = xbsp)] def q51q8_tile_gen(var yp : float?; qgp : uint8 const?; sgp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d, g, t0 : int64) : void { let mr = int64(q51q8_layout_gen()) @@ -687,9 +1376,9 @@ def private q51q8_groupn_gen(var yp : float?; wq : uint8 const?; ws : uint8 cons -//! kq_grp_row_dot's twin over a BYTE-EXPANDED panel (unpack_kq_panel_grp's output) — the k5/k6 -//! tile wrappers' reference body, bit-exact against the packed-plane dot. fmt picks the scale -//! form only (5 = u8 sc/mn + d/dmin, 6 = signed per-16 sub-scales + d). +//! kq_grp_row_dot's twin over a BYTE-EXPANDED panel (unpack_kq_panel_grp's output) - bit-exact +//! against the packed-plane dot. fmt picks the scale form only: 5 = u8 sc/mn + d/dmin, 6 = signed +//! per-16 sub-scales + d, 33/34 = per-32 (1 + 2s) + d, 23 = per-16 UNSIGNED strips + d/8; no bsum for 33/34/23. def kq_grp_row_dot_b(fmt : int64; kqg : uint8 const?; ksg : uint8 const?; r, mr : int64; xqp : int8 const?; xsp : float const?; xbsp : int const?; n : int64) : float { var acc = 0.0 let nsb = n / 256l @@ -706,13 +1395,23 @@ def kq_grp_row_dot_b(fmt : int64; kqg : uint8 const?; ksg : uint8 const?; r, mr var ihi = 0 for (j in range64(4l)) { for (t in range64(4l)) { - let qlo = int(kqg[qb + ((blk * 4l + j) * mr + r) * 4l + t]) - let qhi = int(kqg[qb + 128l * mr + ((blk * 4l + j) * mr + r) * 4l + t]) + let qlo = fmt == 33l || fmt == 34l || fmt == 23l || fmt == 24l || fmt == 25l ? int(int8(kqg[qb + ((blk * 4l + j) * mr + r) * 4l + t])) : int(kqg[qb + ((blk * 4l + j) * mr + r) * 4l + t]) + let qhi = fmt == 33l || fmt == 34l || fmt == 23l || fmt == 24l || fmt == 25l ? int(int8(kqg[qb + 128l * mr + ((blk * 4l + j) * mr + r) * 4l + t])) : int(kqg[qb + 128l * mr + ((blk * 4l + j) * mr + r) * 4l + t]) ilo += qlo * int(xqp[b * 32l + j * 4l + t]) ihi += qhi * int(xqp[b * 32l + 16l + j * 4l + t]) } } - if (fmt == 6l) { + if (fmt == 33l || fmt == 34l) { + let sc = int(int8(ksg[sb + 4l * mr + blk * mr + r])) + isum += sc * (ilo + ihi) + } elif (fmt == 23l || fmt == 24l) { + let s0 = int(ksg[sb + 4l * mr + 2l * blk * mr + r]) + let s1 = int(ksg[sb + 4l * mr + (2l * blk + 1l) * mr + r]) + isum += s0 * ilo + s1 * ihi + } elif (fmt == 25l) { + let sc = int(ksg[sb + 4l * mr + blk * mr + r]) // per-32 UNSIGNED strip + isum += sc * (ilo + ihi) + } elif (fmt == 6l) { let s0 = int(int8(ksg[sb + 2l * blk * mr + r])) let s1 = int(int8(ksg[sb + (2l * blk + 1l) * mr + r])) isum += s0 * ilo + s1 * ihi @@ -724,7 +1423,10 @@ def kq_grp_row_dot_b(fmt : int64; kqg : uint8 const?; ksg : uint8 const?; r, mr bsum += mn * (xbsp[b * 2l] + xbsp[b * 2l + 1l]) } } - if (fmt == 6l) { + if (fmt == 33l || fmt == 34l || fmt == 23l || fmt == 24l || fmt == 25l) { + let di = f16_to_f32(uint(ksg[sb + 2l * r]) | (uint(ksg[sb + 2l * r + 1l]) << 8u)) + acc += float(isum) * (di * xsp[sbi]) + } elif (fmt == 6l) { let dk6 = f16_to_f32(uint(ksg[sb + 16l * mr + 2l * r]) | (uint(ksg[sb + 16l * mr + 2l * r + 1l]) << 8u)) acc += float(isum - 32 * bsum) * (dk6 * xsp[sbi]) } else { @@ -748,6 +1450,24 @@ def private repack_kq_gen(fmt : int; var kq : uint8?; var ks : uint8?; n, d : in repack_k5_grp(kq, ks, n, d, kq_layout_of(5)) } elif (fmt == 40) { repack_q40_grp(kq, ks, n, d, kq_layout_of(40)) + } elif (fmt == 44) { + repack_iq4xs_grp(kq, ks, n, d, kq_layout_of(44)) + } elif (fmt == 3) { + repack_k3_grp(kq, ks, n, d, kq_layout_of(3)) + } elif (fmt == 33) { + repack_iq3s_grp(kq, ks, n, d, kq_layout_of(33)) + } elif (fmt == 34) { + repack_iq3xxs_grp(kq, ks, n, d, kq_layout_of(34)) + } elif (fmt == 45) { + repack_q40_grp(kq, ks, n, d, kq_layout_of(45)) // byte-identical plane shapes + } elif (fmt == 2) { + repack_k2_grp(kq, ks, n, d, kq_layout_of(2)) + } elif (fmt == 23) { + repack_iq2s_grp(kq, ks, n, d, kq_layout_of(23)) + } elif (fmt == 24) { + repack_iq2xs_grp(kq, ks, n, d, kq_layout_of(24)) + } elif (fmt == 25) { + repack_iq2xxs_grp(kq, ks, n, d, kq_layout_of(25)) } else { repack_k6_grp(kq, ks, n, d, kq_layout_of(6)) } @@ -760,6 +1480,24 @@ def private repack_kq_bake(fmt : int; var kq : uint8?; var ks : uint8?; n, d : i repack_k5_grp(kq, ks, n, d, active_kq_layout_mr(5)) } elif (fmt == 40) { repack_q40_grp(kq, ks, n, d, active_kq_layout_mr(40)) + } elif (fmt == 44) { + repack_iq4xs_grp(kq, ks, n, d, active_kq_layout_mr(44)) + } elif (fmt == 3) { + repack_k3_grp(kq, ks, n, d, active_kq_layout_mr(3)) + } elif (fmt == 33) { + repack_iq3s_grp(kq, ks, n, d, active_kq_layout_mr(33)) + } elif (fmt == 34) { + repack_iq3xxs_grp(kq, ks, n, d, active_kq_layout_mr(34)) + } elif (fmt == 45) { + repack_q40_grp(kq, ks, n, d, active_kq_layout_mr(45)) // byte-identical plane shapes + } elif (fmt == 2) { + repack_k2_grp(kq, ks, n, d, active_kq_layout_mr(2)) + } elif (fmt == 23) { + repack_iq2s_grp(kq, ks, n, d, active_kq_layout_mr(23)) + } elif (fmt == 24) { + repack_iq2xs_grp(kq, ks, n, d, active_kq_layout_mr(24)) + } elif (fmt == 25) { + repack_iq2xxs_grp(kq, ks, n, d, active_kq_layout_mr(25)) } else { repack_k6_grp(kq, ks, n, d, active_kq_layout_mr(6)) } @@ -1270,6 +2008,24 @@ def private kq_kernel_gen(fmt : int; var yp : float?; kqp : uint8 const?; ksp : k5q8_gemv_gen(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb) * mr, int64(re) * mr) } elif (fmt == 40) { q40q8_gemv_gen(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb) * mr, int64(re) * mr) + } elif (fmt == 44) { + iq4xsq8_gemv_gen(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb) * mr, int64(re) * mr) + } elif (fmt == 3) { + k3q8_gemv_gen(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb) * mr, int64(re) * mr) + } elif (fmt == 33) { + iq3sq8_gemv_gen(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb) * mr, int64(re) * mr) + } elif (fmt == 34) { + iq3xxsq8_gemv_gen(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb) * mr, int64(re) * mr) + } elif (fmt == 45) { + iq4nlq8_gemv_gen(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb) * mr, int64(re) * mr) + } elif (fmt == 2) { + k2q8_gemv_gen(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb) * mr, int64(re) * mr) + } elif (fmt == 23) { + iq2sq8_gemv_gen(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb) * mr, int64(re) * mr) + } elif (fmt == 24) { + iq2xsq8_gemv_gen(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb) * mr, int64(re) * mr) + } elif (fmt == 25) { + iq2xxsq8_gemv_gen(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb) * mr, int64(re) * mr) } else { k6q8_gemv_gen(myp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb) * mr, int64(re) * mr) } @@ -1282,6 +2038,24 @@ def private kq_kernel_gen(fmt : int; var yp : float?; kqp : uint8 const?; ksp : myp[i] = dot_k5q8(kqp + i * nsb * 160l, ksp + i * nsb * 20l, xqp, xsp, xbsp, n) } elif (fmt == 40) { myp[i] = dot_q40q8(kqp + i * nsb * 128l, ksp + i * nsb * 16l, xqp, xsp, xbsp, n) + } elif (fmt == 44) { + myp[i] = dot_iq4xs_q8(kqp + i * nsb * 128l, ksp + i * nsb * 20l, xqp, xsp, xbsp, n) + } elif (fmt == 3) { + myp[i] = dot_k3q8(kqp + i * nsb * 96l, ksp + i * nsb * 18l, xqp, xsp, xbsp, n) + } elif (fmt == 33) { + myp[i] = dot_iq3s_q8(kqp + i * nsb * 104l, ksp + i * nsb * 20l, xqp, xsp, xbsp, n) + } elif (fmt == 34) { + myp[i] = dot_iq3xxs_q8(kqp + i * nsb * 96l, ksp + i * nsb * 20l, xqp, xsp, xbsp, n) + } elif (fmt == 45) { + myp[i] = dot_iq4nl_q8(kqp + i * nsb * 128l, ksp + i * nsb * 16l, xqp, xsp, xbsp, n) + } elif (fmt == 2) { + myp[i] = dot_k2q8(kqp + i * nsb * 64l, ksp + i * nsb * 20l, xqp, xsp, xbsp, n) + } elif (fmt == 23) { + myp[i] = dot_iq2s_q8(kqp + i * nsb * 72l, ksp + i * nsb * 20l, xqp, xsp, xbsp, n) + } elif (fmt == 24) { + myp[i] = dot_iq2xs_q8(kqp + i * nsb * 64l, ksp + i * nsb * 20l, xqp, xsp, xbsp, n) + } elif (fmt == 25) { + myp[i] = dot_iq2xxs_q8(kqp + i * nsb * 64l, ksp + i * nsb * 20l, xqp, xsp, xbsp, n) } else { myp[i] = dot_k6q8(kqp + i * nsb * 192l, ksp + i * nsb * 18l, xqp, xsp, xbsp, n) } @@ -1292,12 +2066,12 @@ def private kq_kernel_gen(fmt : int; var yp : float?; kqp : uint8 const?; ksp : // One (group range x token range) cell of the kq batch walk: TB token blocks outer, groups // inner, 4-token tile calls + per-token gemv tails (ts pinned 4 — no TMUL kq tile exists). k5/k6 // tiles unpack a BYTE-EXPANDED scratch panel once per (group, token-block), amortized over TB. -def private kq_batch_cell_gen(fmt : int; var myp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d, mr, TB : int64; ub, ue : int; tb0, tend : int64) { +def private kq_batch_cell_gen(fmt : int; var myp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d, mr, TB : int64; ub, ue : int; tb0, tend : int64) { // nolint:STYLE037 — the flat per-format ladder (one arm per kq format) is the honest shape let nsb = n / 256l let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) let nb16 = n / 16l - let packed = fmt == 4 || fmt == 40 // pure-nibble planes: the tile reads them directly + let packed = kq_reads_packed_planes(int64(fmt)) var scratch : array if (!packed) { scratch |> resize(mr * nsb * 256l) @@ -1323,6 +2097,24 @@ def private kq_batch_cell_gen(fmt : int; var myp : float?; kqp : uint8 const?; k k5q8_tile_gen(myp, scp, ksg, xqp, xsp, xbsp, n, d, g, tk) } elif (fmt == 40) { q40q8_tile_gen(myp, kqg, ksg, xqp, xsp, xbsp, n, d, g, tk) + } elif (fmt == 44) { + iq4xsq8_tile_gen(myp, kqg, ksg, xqp, xsp, xbsp, n, d, g, tk) + } elif (fmt == 3) { + k3q8_tile_gen(myp, kqg, ksg, xqp, xsp, xbsp, n, d, g, tk) + } elif (fmt == 33) { + iq3sq8_tile_gen(myp, scp, ksg, xqp, xsp, xbsp, n, d, g, tk) + } elif (fmt == 34) { + iq3xxsq8_tile_gen(myp, scp, ksg, xqp, xsp, xbsp, n, d, g, tk) + } elif (fmt == 45) { + iq4nlq8_tile_gen(myp, kqg, ksg, xqp, xsp, xbsp, n, d, g, tk) + } elif (fmt == 2) { + k2q8_tile_gen(myp, kqg, ksg, xqp, xsp, xbsp, n, d, g, tk) + } elif (fmt == 25) { + iq2xxsq8_tile_gen(myp, scp, ksg, xqp, xsp, xbsp, n, d, g, tk) + } elif (fmt == 23) { + iq2sq8_tile_gen(myp, scp, ksg, xqp, xsp, xbsp, n, d, g, tk) + } elif (fmt == 24) { + iq2xsq8_tile_gen(myp, scp, ksg, xqp, xsp, xbsp, n, d, g, tk) } else { k6q8_tile_gen(myp, scp, ksg, xqp, xsp, xbsp, n, d, g, tk) } @@ -1335,6 +2127,24 @@ def private kq_batch_cell_gen(fmt : int; var myp : float?; kqp : uint8 const?; k k5q8_gemv_gen(myp + tk * d, kqp, ksp, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n, g * mr, (g + 1l) * mr) } elif (fmt == 40) { q40q8_gemv_gen(myp + tk * d, kqp, ksp, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n, g * mr, (g + 1l) * mr) + } elif (fmt == 44) { + iq4xsq8_gemv_gen(myp + tk * d, kqp, ksp, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n, g * mr, (g + 1l) * mr) + } elif (fmt == 3) { + k3q8_gemv_gen(myp + tk * d, kqp, ksp, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n, g * mr, (g + 1l) * mr) + } elif (fmt == 33) { + iq3sq8_gemv_gen(myp + tk * d, kqp, ksp, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n, g * mr, (g + 1l) * mr) + } elif (fmt == 34) { + iq3xxsq8_gemv_gen(myp + tk * d, kqp, ksp, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n, g * mr, (g + 1l) * mr) + } elif (fmt == 45) { + iq4nlq8_gemv_gen(myp + tk * d, kqp, ksp, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n, g * mr, (g + 1l) * mr) + } elif (fmt == 2) { + k2q8_gemv_gen(myp + tk * d, kqp, ksp, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n, g * mr, (g + 1l) * mr) + } elif (fmt == 23) { + iq2sq8_gemv_gen(myp + tk * d, kqp, ksp, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n, g * mr, (g + 1l) * mr) + } elif (fmt == 24) { + iq2xsq8_gemv_gen(myp + tk * d, kqp, ksp, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n, g * mr, (g + 1l) * mr) + } elif (fmt == 25) { + iq2xxsq8_gemv_gen(myp + tk * d, kqp, ksp, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n, g * mr, (g + 1l) * mr) } else { k6q8_gemv_gen(myp + tk * d, kqp, ksp, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n, g * mr, (g + 1l) * mr) } @@ -1368,6 +2178,24 @@ def private kq_batch_kernel_gen(fmt : int; var yp : float?; kqp : uint8 const?; myp[tk * d + i] = dot_k5q8(kqp + i * nsb * 160l, ksp + i * nsb * 20l, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * (n / 16l), n) } elif (fmt == 40) { myp[tk * d + i] = dot_q40q8(kqp + i * nsb * 128l, ksp + i * nsb * 16l, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * (n / 16l), n) + } elif (fmt == 44) { + myp[tk * d + i] = dot_iq4xs_q8(kqp + i * nsb * 128l, ksp + i * nsb * 20l, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * (n / 16l), n) + } elif (fmt == 3) { + myp[tk * d + i] = dot_k3q8(kqp + i * nsb * 96l, ksp + i * nsb * 18l, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * (n / 16l), n) + } elif (fmt == 33) { + myp[tk * d + i] = dot_iq3s_q8(kqp + i * nsb * 104l, ksp + i * nsb * 20l, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * (n / 16l), n) + } elif (fmt == 34) { + myp[tk * d + i] = dot_iq3xxs_q8(kqp + i * nsb * 96l, ksp + i * nsb * 20l, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * (n / 16l), n) + } elif (fmt == 45) { + myp[tk * d + i] = dot_iq4nl_q8(kqp + i * nsb * 128l, ksp + i * nsb * 16l, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * (n / 16l), n) + } elif (fmt == 2) { + myp[tk * d + i] = dot_k2q8(kqp + i * nsb * 64l, ksp + i * nsb * 20l, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * (n / 16l), n) + } elif (fmt == 23) { + myp[tk * d + i] = dot_iq2s_q8(kqp + i * nsb * 72l, ksp + i * nsb * 20l, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * (n / 16l), n) + } elif (fmt == 24) { + myp[tk * d + i] = dot_iq2xs_q8(kqp + i * nsb * 64l, ksp + i * nsb * 20l, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * (n / 16l), n) + } elif (fmt == 25) { + myp[tk * d + i] = dot_iq2xxs_q8(kqp + i * nsb * 64l, ksp + i * nsb * 20l, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * (n / 16l), n) } else { myp[tk * d + i] = dot_k6q8(kqp + i * nsb * 192l, ksp + i * nsb * 18l, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * (n / 16l), n) } @@ -1416,6 +2244,24 @@ def private kq_batch_groupn_gen(fmt : int; var yp : float?; kqp : uint8 const?; myp[tk * d + i] = dot_k5q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n) } elif (fmt == 40) { myp[tk * d + i] = dot_q40q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n) + } elif (fmt == 44) { + myp[tk * d + i] = dot_iq4xs_q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n) + } elif (fmt == 3) { + myp[tk * d + i] = dot_k3q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n) + } elif (fmt == 33) { + myp[tk * d + i] = dot_iq3s_q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n) + } elif (fmt == 34) { + myp[tk * d + i] = dot_iq3xxs_q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n) + } elif (fmt == 45) { + myp[tk * d + i] = dot_iq4nl_q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n) + } elif (fmt == 2) { + myp[tk * d + i] = dot_k2q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n) + } elif (fmt == 23) { + myp[tk * d + i] = dot_iq2s_q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n) + } elif (fmt == 24) { + myp[tk * d + i] = dot_iq2xs_q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n) + } elif (fmt == 25) { + myp[tk * d + i] = dot_iq2xxs_q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n) } else { myp[tk * d + i] = dot_k6q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + tk * n, xsp + tk * (n / 256l), xbsp + tk * nb16, n) } @@ -1573,6 +2419,24 @@ def kq_groupn_gen(fmt : int; var yp : float?; kqp : uint8 const?; ksp : uint8 co k5q8_gemv_gen(myp + r * d, kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n, gLocal * mr, gEnd * mr) } elif (fmt == 40) { q40q8_gemv_gen(myp + r * d, kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n, gLocal * mr, gEnd * mr) + } elif (fmt == 44) { + iq4xsq8_gemv_gen(myp + r * d, kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n, gLocal * mr, gEnd * mr) + } elif (fmt == 3) { + k3q8_gemv_gen(myp + r * d, kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n, gLocal * mr, gEnd * mr) + } elif (fmt == 33) { + iq3sq8_gemv_gen(myp + r * d, kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n, gLocal * mr, gEnd * mr) + } elif (fmt == 34) { + iq3xxsq8_gemv_gen(myp + r * d, kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n, gLocal * mr, gEnd * mr) + } elif (fmt == 45) { + iq4nlq8_gemv_gen(myp + r * d, kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n, gLocal * mr, gEnd * mr) + } elif (fmt == 2) { + k2q8_gemv_gen(myp + r * d, kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n, gLocal * mr, gEnd * mr) + } elif (fmt == 23) { + iq2sq8_gemv_gen(myp + r * d, kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n, gLocal * mr, gEnd * mr) + } elif (fmt == 24) { + iq2xsq8_gemv_gen(myp + r * d, kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n, gLocal * mr, gEnd * mr) + } elif (fmt == 25) { + iq2xxsq8_gemv_gen(myp + r * d, kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n, gLocal * mr, gEnd * mr) } else { k6q8_gemv_gen(myp + r * d, kqp + sb * qsb, ksp + sb * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n, gLocal * mr, gEnd * mr) } @@ -1591,6 +2455,24 @@ def kq_groupn_gen(fmt : int; var yp : float?; kqp : uint8 const?; ksp : uint8 co myp[r * d + i] = dot_k5q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) } elif (fmt == 40) { myp[r * d + i] = dot_q40q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) + } elif (fmt == 44) { + myp[r * d + i] = dot_iq4xs_q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) + } elif (fmt == 3) { + myp[r * d + i] = dot_k3q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) + } elif (fmt == 33) { + myp[r * d + i] = dot_iq3s_q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) + } elif (fmt == 34) { + myp[r * d + i] = dot_iq3xxs_q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) + } elif (fmt == 45) { + myp[r * d + i] = dot_iq4nl_q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) + } elif (fmt == 2) { + myp[r * d + i] = dot_k2q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) + } elif (fmt == 23) { + myp[r * d + i] = dot_iq2s_q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) + } elif (fmt == 24) { + myp[r * d + i] = dot_iq2xs_q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) + } elif (fmt == 25) { + myp[r * d + i] = dot_iq2xxs_q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) } else { myp[r * d + i] = dot_k6q8(kqp + (sb + i * nsb) * qsb, ksp + (sb + i * nsb) * ssb, xqp + xoff, xsp + xoff / 256l, xbsp + xoff / 16l, n) } @@ -1884,7 +2766,7 @@ def dasllama_math_gen_register() { // the no-witness stance above covers kq too mm_kq = @@kq_kernel_gen, repack_kq = @@repack_kq_gen, kq_rows_k4 = @@k4q8_gemv_gen, kq_rows_k5 = @@k5q8_gemv_gen, - kq_rows_k6 = @@k6q8_gemv_gen, kq_rows_q40 = @@q40q8_gemv_gen, + kq_rows_k6 = @@k6q8_gemv_gen, kq_rows_q40 = @@q40q8_gemv_gen, kq_rows_iq4xs = @@iq4xsq8_gemv_gen, kq_rows_k3 = @@k3q8_gemv_gen, kq_rows_iq3s = @@iq3sq8_gemv_gen, kq_rows_iq3xxs = @@iq3xxsq8_gemv_gen, kq_rows_iq4nl = @@iq4nlq8_gemv_gen, kq_rows_k2 = @@k2q8_gemv_gen, kq_rows_iq2s = @@iq2sq8_gemv_gen, kq_rows_iq2xs = @@iq2xsq8_gemv_gen, kq_rows_iq2xxs = @@iq2xxsq8_gemv_gen, kq_batch = @@kq_batch_kernel_gen, kq_groupn = @@kq_groupn_gen, kq_batch_groupn = @@kq_batch_groupn_gen, // q51 family: tile + GEMV off the grp planes (the tile family's companions stamp @@ -1913,7 +2795,7 @@ def dasllama_math_gen_register() { mm_rows_s16 = @@q8q8_gemv_s16_gen, mm_kq = @@kq_kernel_gen, repack_kq = @@repack_kq_gen, kq_rows_k4 = @@k4q8_gemv_gen, kq_rows_k5 = @@k5q8_gemv_gen, - kq_rows_k6 = @@k6q8_gemv_gen, kq_rows_q40 = @@q40q8_gemv_gen, + kq_rows_k6 = @@k6q8_gemv_gen, kq_rows_q40 = @@q40q8_gemv_gen, kq_rows_iq4xs = @@iq4xsq8_gemv_gen, kq_rows_k3 = @@k3q8_gemv_gen, kq_rows_iq3s = @@iq3sq8_gemv_gen, kq_rows_iq3xxs = @@iq3xxsq8_gemv_gen, kq_rows_iq4nl = @@iq4nlq8_gemv_gen, kq_rows_k2 = @@k2q8_gemv_gen, kq_rows_iq2s = @@iq2sq8_gemv_gen, kq_rows_iq2xs = @@iq2xsq8_gemv_gen, kq_rows_iq2xxs = @@iq2xxsq8_gemv_gen, kq_batch = @@kq_batch_kernel_gen, kq_groupn = @@kq_groupn_gen, kq_batch_groupn = @@kq_batch_groupn_gen, groupn_q51 = @@q51q8_groupn_gen, repack_q51 = @@repack_q51_gen, diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_common.das b/modules/dasLLAMA/dasllama/dasllama_metal_common.das index 9e564dabb0..8df3f0d33e 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_common.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_common.das @@ -244,6 +244,8 @@ var g_pso_kq_k4 : MetalComputePipeline? var g_pso_kq_k5 : MetalComputePipeline? var g_pso_kq_k5c : MetalComputePipeline? // the select-form nr0=1 twin — n < 3072 / small-d sites var g_pso_kq_k6 : MetalComputePipeline? +var g_pso_kq_iq4xs : MetalComputePipeline? +var g_pso_kq_k3 : MetalComputePipeline? var g_pso_kq_mvb2_k4 : MetalComputePipeline? // batch B=2..8: the kq ext small-batch mv twins var g_pso_kq_mvb4_k4 : MetalComputePipeline? var g_pso_kq_mvb2_k5 : MetalComputePipeline? @@ -253,9 +255,52 @@ var g_pso_kq_mvb4_k6 : MetalComputePipeline? var g_pso_kq_mvb8_k4 : MetalComputePipeline? // B=5..8 single-pass twins (in-graph A/B vs 2 x B4 groups) var g_pso_kq_mvb8_k5 : MetalComputePipeline? var g_pso_kq_mvb8_k6 : MetalComputePipeline? +var g_pso_kq_mvb2_iq4xs : MetalComputePipeline? +var g_pso_kq_mvb4_iq4xs : MetalComputePipeline? +var g_pso_kq_mvb8_iq4xs : MetalComputePipeline? +var g_pso_kq_mvb2_k3 : MetalComputePipeline? +var g_pso_kq_mvb4_k3 : MetalComputePipeline? +var g_pso_kq_mvb8_k3 : MetalComputePipeline? +var g_pso_kq_iq3s : MetalComputePipeline? +var g_pso_kq_mvb2_iq3s : MetalComputePipeline? +var g_pso_kq_mvb4_iq3s : MetalComputePipeline? +var g_pso_kq_mvb8_iq3s : MetalComputePipeline? +var g_pso_kq_mm_b33 : MetalComputePipeline? +var g_pso_kq_iq3xxs : MetalComputePipeline? +var g_pso_kq_iq4nl : MetalComputePipeline? +var g_pso_kq_k2 : MetalComputePipeline? +var g_pso_kq_mvb2_k2 : MetalComputePipeline? +var g_pso_kq_mvb4_k2 : MetalComputePipeline? +var g_pso_kq_mvb8_k2 : MetalComputePipeline? +var g_pso_kq_mm_b2 : MetalComputePipeline? +var g_pso_kq_mvb2_iq4nl : MetalComputePipeline? +var g_pso_kq_mvb4_iq4nl : MetalComputePipeline? +var g_pso_kq_mvb8_iq4nl : MetalComputePipeline? +var g_pso_kq_mm_b45 : MetalComputePipeline? +var g_pso_kq_iq2s : MetalComputePipeline? +var g_pso_kq_mvb2_iq2s : MetalComputePipeline? +var g_pso_kq_mvb4_iq2s : MetalComputePipeline? +var g_pso_kq_mvb8_iq2s : MetalComputePipeline? +var g_pso_kq_mm_b23 : MetalComputePipeline? +var g_pso_kq_iq2xs : MetalComputePipeline? +var g_pso_kq_mvb2_iq2xs : MetalComputePipeline? +var g_pso_kq_mvb4_iq2xs : MetalComputePipeline? +var g_pso_kq_mvb8_iq2xs : MetalComputePipeline? +var g_pso_kq_mm_b24 : MetalComputePipeline? +var g_pso_kq_iq2xxs : MetalComputePipeline? +var g_pso_kq_mvb2_iq2xxs : MetalComputePipeline? +var g_pso_kq_mvb4_iq2xxs : MetalComputePipeline? +var g_pso_kq_mvb8_iq2xxs : MetalComputePipeline? +var g_pso_kq_mm_b25 : MetalComputePipeline? +var g_pso_kq_mvb2_iq3xxs : MetalComputePipeline? +var g_pso_kq_mvb4_iq3xxs : MetalComputePipeline? +var g_pso_kq_mvb8_iq3xxs : MetalComputePipeline? +var g_pso_kq_mm_b34 : MetalComputePipeline? var g_pso_kq_mm_b4 : MetalComputePipeline? // batch B>=9: the prefill-owned kq mul_mm twins var g_pso_kq_mm_b5 : MetalComputePipeline? -var g_pso_kq_mm_b6 : MetalComputePipeline? // fused W1|W3+swiglu (s16 path) +var g_pso_kq_mm_b6 : MetalComputePipeline? +var g_pso_kq_mm_b44 : MetalComputePipeline? +var g_pso_kq_mm_b3 : MetalComputePipeline? // the batched-step set (P4): fixed-B GEMVs + the M-pad-32 GEMM twin (f32 X) + row-table kernels var g_pso_gemv_b2 : MetalComputePipeline? var g_pso_gemv_b4 : MetalComputePipeline? @@ -828,12 +873,48 @@ def kq_quants_of(dev : MetalDevice?; t : Model; fmt : KqFmt; off : int64) : tupl unsafe { if (fmt == KqFmt.k4) { return (buf = plane_buffer(dev, addr < void? >(t.k4q[0]), uint64(long_length(t.k4q)), t.image_map != null), - qoff = uint64(sb0 * 128l)) + qoff = uint64(sb0 * kq_qsb(fmt))) } if (fmt == KqFmt.k5) { return (buf = plane_buffer(dev, addr < void? >(t.k5q[0]), uint64(long_length(t.k5q)), t.image_map != null), qoff = uint64(sb0 * 160l)) } + if (fmt == KqFmt.iq4xs) { + return (buf = plane_buffer(dev, addr < void? >(t.iq4xsq[0]), uint64(long_length(t.iq4xsq)), t.image_map != null), + qoff = uint64(sb0 * kq_qsb(fmt))) + } + if (fmt == KqFmt.k3) { + return (buf = plane_buffer(dev, addr < void? >(t.k3q[0]), uint64(long_length(t.k3q)), t.image_map != null), + qoff = uint64(sb0 * kq_qsb(fmt))) + } + if (fmt == KqFmt.iq3s) { + return (buf = plane_buffer(dev, addr < void? >(t.iq3sq[0]), uint64(long_length(t.iq3sq)), t.image_map != null), + qoff = uint64(sb0 * kq_qsb(fmt))) + } + if (fmt == KqFmt.iq3xxs) { + return (buf = plane_buffer(dev, addr < void? >(t.iq3xxsq[0]), uint64(long_length(t.iq3xxsq)), t.image_map != null), + qoff = uint64(sb0 * kq_qsb(fmt))) + } + if (fmt == KqFmt.iq4nl) { + return (buf = plane_buffer(dev, addr < void? >(t.iq4nlq[0]), uint64(long_length(t.iq4nlq)), t.image_map != null), + qoff = uint64(sb0 * kq_qsb(fmt))) + } + if (fmt == KqFmt.k2) { + return (buf = plane_buffer(dev, addr < void? >(t.k2q[0]), uint64(long_length(t.k2q)), t.image_map != null), + qoff = uint64(sb0 * kq_qsb(fmt))) + } + if (fmt == KqFmt.iq2s) { + return (buf = plane_buffer(dev, addr < void? >(t.iq2sq[0]), uint64(long_length(t.iq2sq)), t.image_map != null), + qoff = uint64(sb0 * kq_qsb(fmt))) + } + if (fmt == KqFmt.iq2xs) { + return (buf = plane_buffer(dev, addr < void? >(t.iq2xsq[0]), uint64(long_length(t.iq2xsq)), t.image_map != null), + qoff = uint64(sb0 * kq_qsb(fmt))) + } + if (fmt == KqFmt.iq2xxs) { + return (buf = plane_buffer(dev, addr < void? >(t.iq2xxsq[0]), uint64(long_length(t.iq2xxsq)), t.image_map != null), + qoff = uint64(sb0 * kq_qsb(fmt))) + } return (buf = plane_buffer(dev, addr < void? >(t.k6q[0]), uint64(long_length(t.k6q)), t.image_map != null), qoff = uint64(sb0 * 192l)) } @@ -842,6 +923,7 @@ def kq_quants_of(dev : MetalDevice?; t : Model; fmt : KqFmt; off : int64) : tupl //! One kq tensor's GPU-form scale bind (the gemv lab's layouts, pre-baked by the blob //! transform): k4/k5 = 16B compact blocks at soff; k6 = the split plane — sub-scale strips at //! soff, the packed f16 d plane at doff (the kernels bind the buffer twice, once per offset). +[arch(at = "../ARCHITECTURE_GPU.md#metal-kq-split-scale-plane")] def kq_scales_of(dev : MetalDevice?; t : Model; fmt : KqFmt; off : int64) : tuple { assert(t.metal_blob) let sb0 = off / 256l @@ -855,6 +937,50 @@ def kq_scales_of(dev : MetalDevice?; t : Model; fmt : KqFmt; off : int64) : tupl return (buf = plane_buffer(dev, addr < void? >(t.k4s[0]), uint64(long_length(t.k4s)), t.image_map != null), soff = uint64(sb0 * 16l), doff = 0ul) } + if (fmt == KqFmt.k3) { + let plane_sb = long_length(t.k3s) / 18l + return (buf = plane_buffer(dev, addr < void? >(t.k3s[0]), uint64(long_length(t.k3s)), t.image_map != null), + soff = uint64(sb0 * 16l), doff = uint64(plane_sb * 16l + sb0 * 2l)) + } + if (fmt == KqFmt.iq3s) { + let plane_sb = long_length(t.iq3ss) / 18l + return (buf = plane_buffer(dev, addr < void? >(t.iq3ss[0]), uint64(long_length(t.iq3ss)), t.image_map != null), + soff = uint64(sb0 * 16l), doff = uint64(plane_sb * 16l + sb0 * 2l)) + } + if (fmt == KqFmt.iq3xxs) { + let plane_sb = long_length(t.iq3xxss) / 18l + return (buf = plane_buffer(dev, addr < void? >(t.iq3xxss[0]), uint64(long_length(t.iq3xxss)), t.image_map != null), + soff = uint64(sb0 * 16l), doff = uint64(plane_sb * 16l + sb0 * 2l)) + } + if (fmt == KqFmt.iq4xs) { + let plane_sb = long_length(t.iq4xss) / 18l + return (buf = plane_buffer(dev, addr < void? >(t.iq4xss[0]), uint64(long_length(t.iq4xss)), t.image_map != null), + soff = uint64(sb0 * 16l), doff = uint64(plane_sb * 16l + sb0 * 2l)) + } + if (fmt == KqFmt.iq4nl) { // q40's 16B/sb plane of 8 f16 d - verbatim, one bind (doff unused) + return (buf = plane_buffer(dev, addr < void? >(t.iq4nls[0]), uint64(long_length(t.iq4nls)), t.image_map != null), + soff = uint64(sb0 * 16l), doff = 0ul) + } + if (fmt == KqFmt.k2) { // the k6 split shape with a 4B d+dmin tail per superblock + let plane_sb = long_length(t.k2s) / 20l + return (buf = plane_buffer(dev, addr < void? >(t.k2s[0]), uint64(long_length(t.k2s)), t.image_map != null), + soff = uint64(sb0 * 16l), doff = uint64(plane_sb * 16l + sb0 * 4l)) + } + if (fmt == KqFmt.iq2s) { // the k6 split form over the 18B/sb device plane (d8 tail) + let plane_sb = long_length(t.iq2ss) / 18l + return (buf = plane_buffer(dev, addr < void? >(t.iq2ss[0]), uint64(long_length(t.iq2ss)), t.image_map != null), + soff = uint64(sb0 * 16l), doff = uint64(plane_sb * 16l + sb0 * 2l)) + } + if (fmt == KqFmt.iq2xs) { + let plane_sb = long_length(t.iq2xss) / 18l + return (buf = plane_buffer(dev, addr < void? >(t.iq2xss[0]), uint64(long_length(t.iq2xss)), t.image_map != null), + soff = uint64(sb0 * 16l), doff = uint64(plane_sb * 16l + sb0 * 2l)) + } + if (fmt == KqFmt.iq2xxs) { + let plane_sb = long_length(t.iq2xxss) / 18l + return (buf = plane_buffer(dev, addr < void? >(t.iq2xxss[0]), uint64(long_length(t.iq2xxss)), t.image_map != null), + soff = uint64(sb0 * 16l), doff = uint64(plane_sb * 16l + sb0 * 2l)) + } return (buf = plane_buffer(dev, addr < void? >(t.k5s[0]), uint64(long_length(t.k5s)), t.image_map != null), soff = uint64(sb0 * 16l), doff = 0ul) } diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das index be4c0c32d6..e9d00cff8d 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das @@ -3636,6 +3636,51 @@ def metal_decode_init : bool { // nolint:STYLE038 — a flat compile_pso per k g_pso_kq_mm_b4 = compile_pso(MetalKqMulMmK4_metal_kq_mulmm_msl, MetalKqMulMmK4_metal_kq_mulmm_msl_entry, MetalKqMulMmK4_metal_kq_mulmm_msl_fastmath, ok) g_pso_kq_mm_b5 = compile_pso(MetalKqMulMmK5_metal_kq_mulmm_msl, MetalKqMulMmK5_metal_kq_mulmm_msl_entry, MetalKqMulMmK5_metal_kq_mulmm_msl_fastmath, ok) g_pso_kq_mm_b6 = compile_pso(MetalKqMulMmK6_metal_kq_mulmm_msl, MetalKqMulMmK6_metal_kq_mulmm_msl_entry, MetalKqMulMmK6_metal_kq_mulmm_msl_fastmath, ok) + g_pso_kq_iq4xs = compile_pso(metal_kq_gemv_iq4xs_msl, metal_kq_gemv_iq4xs_msl_entry, metal_kq_gemv_iq4xs_msl_fastmath, ok) + g_pso_kq_mvb2_iq4xs = compile_pso(MetalKqMvB2Iq4xs_metal_kq_mv_iq4xs_msl, MetalKqMvB2Iq4xs_metal_kq_mv_iq4xs_msl_entry, MetalKqMvB2Iq4xs_metal_kq_mv_iq4xs_msl_fastmath, ok) + g_pso_kq_mvb4_iq4xs = compile_pso(MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl, MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl_entry, MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl_fastmath, ok) + g_pso_kq_mvb8_iq4xs = compile_pso(metal_kq_mvb8_iq4xs_msl, metal_kq_mvb8_iq4xs_msl_entry, metal_kq_mvb8_iq4xs_msl_fastmath, ok) + g_pso_kq_mm_b44 = compile_pso(MetalKqMulMmIq4xs_metal_kq_mulmm_msl, MetalKqMulMmIq4xs_metal_kq_mulmm_msl_entry, MetalKqMulMmIq4xs_metal_kq_mulmm_msl_fastmath, ok) + g_pso_kq_k3 = compile_pso(metal_kq_gemv_k3_msl, metal_kq_gemv_k3_msl_entry, metal_kq_gemv_k3_msl_fastmath, ok) + g_pso_kq_mvb2_k3 = compile_pso(MetalKqMvB2K3_metal_kq_mv_k3_msl, MetalKqMvB2K3_metal_kq_mv_k3_msl_entry, MetalKqMvB2K3_metal_kq_mv_k3_msl_fastmath, ok) + g_pso_kq_mvb4_k3 = compile_pso(MetalKqMvB4K3_metal_kq_mv_k3_msl, MetalKqMvB4K3_metal_kq_mv_k3_msl_entry, MetalKqMvB4K3_metal_kq_mv_k3_msl_fastmath, ok) + g_pso_kq_mvb8_k3 = compile_pso(metal_kq_mvb8_k3_msl, metal_kq_mvb8_k3_msl_entry, metal_kq_mvb8_k3_msl_fastmath, ok) + g_pso_kq_mm_b3 = compile_pso(MetalKqMulMmK3_metal_kq_mulmm_msl, MetalKqMulMmK3_metal_kq_mulmm_msl_entry, MetalKqMulMmK3_metal_kq_mulmm_msl_fastmath, ok) + g_pso_kq_iq3s = compile_pso(metal_kq_gemv_iq3s_msl, metal_kq_gemv_iq3s_msl_entry, metal_kq_gemv_iq3s_msl_fastmath, ok) + g_pso_kq_mvb2_iq3s = compile_pso(MetalKqMvB2Iq3s_metal_kq_mv_iq3s_msl, MetalKqMvB2Iq3s_metal_kq_mv_iq3s_msl_entry, MetalKqMvB2Iq3s_metal_kq_mv_iq3s_msl_fastmath, ok) + g_pso_kq_mvb4_iq3s = compile_pso(MetalKqMvB4Iq3s_metal_kq_mv_iq3s_msl, MetalKqMvB4Iq3s_metal_kq_mv_iq3s_msl_entry, MetalKqMvB4Iq3s_metal_kq_mv_iq3s_msl_fastmath, ok) + g_pso_kq_mvb8_iq3s = compile_pso(metal_kq_mvb8_iq3s_msl, metal_kq_mvb8_iq3s_msl_entry, metal_kq_mvb8_iq3s_msl_fastmath, ok) + g_pso_kq_mm_b33 = compile_pso(MetalKqMulMmIq3s_metal_kq_mulmm_msl, MetalKqMulMmIq3s_metal_kq_mulmm_msl_entry, MetalKqMulMmIq3s_metal_kq_mulmm_msl_fastmath, ok) + g_pso_kq_iq3xxs = compile_pso(metal_kq_gemv_iq3xxs_msl, metal_kq_gemv_iq3xxs_msl_entry, metal_kq_gemv_iq3xxs_msl_fastmath, ok) + g_pso_kq_mvb2_iq3xxs = compile_pso(MetalKqMvB2Iq3xxs_metal_kq_mv_iq3xxs_msl, MetalKqMvB2Iq3xxs_metal_kq_mv_iq3xxs_msl_entry, MetalKqMvB2Iq3xxs_metal_kq_mv_iq3xxs_msl_fastmath, ok) + g_pso_kq_mvb4_iq3xxs = compile_pso(MetalKqMvB4Iq3xxs_metal_kq_mv_iq3xxs_msl, MetalKqMvB4Iq3xxs_metal_kq_mv_iq3xxs_msl_entry, MetalKqMvB4Iq3xxs_metal_kq_mv_iq3xxs_msl_fastmath, ok) + g_pso_kq_mvb8_iq3xxs = compile_pso(metal_kq_mvb8_iq3xxs_msl, metal_kq_mvb8_iq3xxs_msl_entry, metal_kq_mvb8_iq3xxs_msl_fastmath, ok) + g_pso_kq_mm_b34 = compile_pso(MetalKqMulMmIq3xxs_metal_kq_mulmm_msl, MetalKqMulMmIq3xxs_metal_kq_mulmm_msl_entry, MetalKqMulMmIq3xxs_metal_kq_mulmm_msl_fastmath, ok) + g_pso_kq_iq4nl = compile_pso(metal_kq_gemv_iq4nl_msl, metal_kq_gemv_iq4nl_msl_entry, metal_kq_gemv_iq4nl_msl_fastmath, ok) + g_pso_kq_mvb2_iq4nl = compile_pso(MetalKqMvB2Iq4nl_metal_kq_mv_iq4nl_msl, MetalKqMvB2Iq4nl_metal_kq_mv_iq4nl_msl_entry, MetalKqMvB2Iq4nl_metal_kq_mv_iq4nl_msl_fastmath, ok) + g_pso_kq_mvb4_iq4nl = compile_pso(MetalKqMvB4Iq4nl_metal_kq_mv_iq4nl_msl, MetalKqMvB4Iq4nl_metal_kq_mv_iq4nl_msl_entry, MetalKqMvB4Iq4nl_metal_kq_mv_iq4nl_msl_fastmath, ok) + g_pso_kq_mvb8_iq4nl = compile_pso(metal_kq_mvb8_iq4nl_msl, metal_kq_mvb8_iq4nl_msl_entry, metal_kq_mvb8_iq4nl_msl_fastmath, ok) + g_pso_kq_iq2s = compile_pso(metal_kq_gemv_iq2s_msl, metal_kq_gemv_iq2s_msl_entry, metal_kq_gemv_iq2s_msl_fastmath, ok) + g_pso_kq_mvb2_iq2s = compile_pso(MetalKqMvB2Iq2s_metal_kq_mv_iq2s_msl, MetalKqMvB2Iq2s_metal_kq_mv_iq2s_msl_entry, MetalKqMvB2Iq2s_metal_kq_mv_iq2s_msl_fastmath, ok) + g_pso_kq_mvb4_iq2s = compile_pso(MetalKqMvB4Iq2s_metal_kq_mv_iq2s_msl, MetalKqMvB4Iq2s_metal_kq_mv_iq2s_msl_entry, MetalKqMvB4Iq2s_metal_kq_mv_iq2s_msl_fastmath, ok) + g_pso_kq_mvb8_iq2s = compile_pso(metal_kq_mvb8_iq2s_msl, metal_kq_mvb8_iq2s_msl_entry, metal_kq_mvb8_iq2s_msl_fastmath, ok) + g_pso_kq_mm_b23 = compile_pso(MetalKqMulMmIq2s_metal_kq_mulmm_msl, MetalKqMulMmIq2s_metal_kq_mulmm_msl_entry, MetalKqMulMmIq2s_metal_kq_mulmm_msl_fastmath, ok) + g_pso_kq_iq2xs = compile_pso(metal_kq_gemv_iq2xs_msl, metal_kq_gemv_iq2xs_msl_entry, metal_kq_gemv_iq2xs_msl_fastmath, ok) + g_pso_kq_mvb2_iq2xs = compile_pso(MetalKqMvB2Iq2xs_metal_kq_mv_iq2xs_msl, MetalKqMvB2Iq2xs_metal_kq_mv_iq2xs_msl_entry, MetalKqMvB2Iq2xs_metal_kq_mv_iq2xs_msl_fastmath, ok) + g_pso_kq_mvb4_iq2xs = compile_pso(MetalKqMvB4Iq2xs_metal_kq_mv_iq2xs_msl, MetalKqMvB4Iq2xs_metal_kq_mv_iq2xs_msl_entry, MetalKqMvB4Iq2xs_metal_kq_mv_iq2xs_msl_fastmath, ok) + g_pso_kq_mvb8_iq2xs = compile_pso(metal_kq_mvb8_iq2xs_msl, metal_kq_mvb8_iq2xs_msl_entry, metal_kq_mvb8_iq2xs_msl_fastmath, ok) + g_pso_kq_mm_b24 = compile_pso(MetalKqMulMmIq2xs_metal_kq_mulmm_msl, MetalKqMulMmIq2xs_metal_kq_mulmm_msl_entry, MetalKqMulMmIq2xs_metal_kq_mulmm_msl_fastmath, ok) + g_pso_kq_iq2xxs = compile_pso(metal_kq_gemv_iq2xxs_msl, metal_kq_gemv_iq2xxs_msl_entry, metal_kq_gemv_iq2xxs_msl_fastmath, ok) + g_pso_kq_mvb2_iq2xxs = compile_pso(MetalKqMvB2Iq2xxs_metal_kq_mv_iq2xxs_msl, MetalKqMvB2Iq2xxs_metal_kq_mv_iq2xxs_msl_entry, MetalKqMvB2Iq2xxs_metal_kq_mv_iq2xxs_msl_fastmath, ok) + g_pso_kq_mvb4_iq2xxs = compile_pso(MetalKqMvB4Iq2xxs_metal_kq_mv_iq2xxs_msl, MetalKqMvB4Iq2xxs_metal_kq_mv_iq2xxs_msl_entry, MetalKqMvB4Iq2xxs_metal_kq_mv_iq2xxs_msl_fastmath, ok) + g_pso_kq_mvb8_iq2xxs = compile_pso(metal_kq_mvb8_iq2xxs_msl, metal_kq_mvb8_iq2xxs_msl_entry, metal_kq_mvb8_iq2xxs_msl_fastmath, ok) + g_pso_kq_mm_b25 = compile_pso(MetalKqMulMmIq2xxs_metal_kq_mulmm_msl, MetalKqMulMmIq2xxs_metal_kq_mulmm_msl_entry, MetalKqMulMmIq2xxs_metal_kq_mulmm_msl_fastmath, ok) + g_pso_kq_mm_b45 = compile_pso(MetalKqMulMmIq4nl_metal_kq_mulmm_msl, MetalKqMulMmIq4nl_metal_kq_mulmm_msl_entry, MetalKqMulMmIq4nl_metal_kq_mulmm_msl_fastmath, ok) + g_pso_kq_k2 = compile_pso(metal_kq_gemv_k2_msl, metal_kq_gemv_k2_msl_entry, metal_kq_gemv_k2_msl_fastmath, ok) + g_pso_kq_mvb2_k2 = compile_pso(MetalKqMvB2K2_metal_kq_mv_k2_msl, MetalKqMvB2K2_metal_kq_mv_k2_msl_entry, MetalKqMvB2K2_metal_kq_mv_k2_msl_fastmath, ok) + g_pso_kq_mvb4_k2 = compile_pso(MetalKqMvB4K2_metal_kq_mv_k2_msl, MetalKqMvB4K2_metal_kq_mv_k2_msl_entry, MetalKqMvB4K2_metal_kq_mv_k2_msl_fastmath, ok) + g_pso_kq_mvb8_k2 = compile_pso(metal_kq_mvb8_k2_msl, metal_kq_mvb8_k2_msl_entry, metal_kq_mvb8_k2_msl_fastmath, ok) + g_pso_kq_mm_b2 = compile_pso(MetalKqMulMmK2_metal_kq_mulmm_msl, MetalKqMulMmK2_metal_kq_mulmm_msl_entry, MetalKqMulMmK2_metal_kq_mulmm_msl_fastmath, ok) // the batched-step set (P4) g_pso_gemv_b2 = compile_pso(MetalGemvB2_metal_gemv_msl, MetalGemvB2_metal_gemv_msl_entry, MetalGemvB2_metal_gemv_msl_fastmath, ok) g_pso_gemv_b4 = compile_pso(MetalGemvB4_metal_gemv_msl, MetalGemvB4_metal_gemv_msl_entry, MetalGemvB4_metal_gemv_msl_fastmath, ok) @@ -3702,7 +3747,25 @@ def enc_gemv(enc : MetalComputeEncoder?; bw : MetalBuffer?; boff : uint64; bx, b def enc_kq_gemv(enc : MetalComputeEncoder?; fmt : KqFmt; bq : MetalBuffer?; qoff : uint64; bs : MetalBuffer?; soff, doff : uint64; bx, by, bn, bd : MetalBuffer?; rows, n : int64; yoff : uint64 = 0ul; xoff : uint64 = 0ul) { - if (fmt == KqFmt.k6) { + if (fmt == KqFmt.iq4xs) { + enc_kq_iq4xs_c(enc, bs, doff, bs, soff, bq, qoff, bx, xoff, by, yoff, bn, bd, rows, n) + } elif (fmt == KqFmt.k3) { + enc_kq_k3_c(enc, bs, doff, bs, soff, bq, qoff, bx, xoff, by, yoff, bn, bd, rows, n) + } elif (fmt == KqFmt.iq3s) { + enc_kq_iq3s_c(enc, bs, doff, bs, soff, bq, qoff, bx, xoff, by, yoff, bn, bd, rows, n) + } elif (fmt == KqFmt.iq3xxs) { + enc_kq_iq3xxs_c(enc, bs, doff, bs, soff, bq, qoff, bx, xoff, by, yoff, bn, bd, rows, n) + } elif (fmt == KqFmt.iq4nl) { + enc_kq_iq4nl_c(enc, bs, soff, bs, soff, bq, qoff, bx, xoff, by, yoff, bn, bd, rows, n) + } elif (fmt == KqFmt.k2) { + enc_kq_k2_c(enc, bs, doff, bs, soff, bq, qoff, bx, xoff, by, yoff, bn, bd, rows, n) + } elif (fmt == KqFmt.iq2s) { + enc_kq_iq2s_c(enc, bs, doff, bs, soff, bq, qoff, bx, xoff, by, yoff, bn, bd, rows, n) + } elif (fmt == KqFmt.iq2xs) { + enc_kq_iq2xs_c(enc, bs, doff, bs, soff, bq, qoff, bx, xoff, by, yoff, bn, bd, rows, n) + } elif (fmt == KqFmt.iq2xxs) { + enc_kq_iq2xxs_c(enc, bs, doff, bs, soff, bq, qoff, bx, xoff, by, yoff, bn, bd, rows, n) + } elif (fmt == KqFmt.k6) { enc_kq_k6_c(enc, bs, doff, bs, soff, bq, qoff, bx, xoff, by, yoff, bn, bd, rows, n) } elif (fmt == KqFmt.k5 && (n < 3072l || rows <= 1024l)) { // the select-form nr0=1 twin: wins every d at n < 3072 (gemv lab, 4B round) and the @@ -3732,13 +3795,85 @@ def private kq_b8_on : bool { return g_kq_b8_mode == 1 } -def enc_kq_mvb(enc : MetalComputeEncoder?; fmt : KqFmt; bq : MetalBuffer?; qoff : uint64; - bs : MetalBuffer?; soff, doff : uint64; bx, by : MetalBuffer?; yoff : uint64; +def enc_kq_mvb(enc : MetalComputeEncoder?; fmt : KqFmt; bq : MetalBuffer?; qoff : uint64; // nolint:STYLE037 — the flat per-format ladder (one arm per kq format) is the honest shape + bs : MetalBuffer ?; soff, doff : uint64; bx, by : MetalBuffer?; yoff : uint64; var ka : KqMvArgs; rows, nlive : int64) { let two = nlive == 2l let b8 = nlive >= 5l && kq_b8_on() let gcols = (nlive + 3l) / 4l - if (fmt == KqFmt.k6) { + if (fmt == KqFmt.iq4xs) { + if (b8) { + enc_kq_mvb8_iq4xs_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows) + } elif (two) { + enc_kq_mvb2_iq4xs_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows) + } else { + enc_kq_mvb4_iq4xs_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows, gcols) + } + } elif (fmt == KqFmt.k3) { + if (b8) { + enc_kq_mvb8_k3_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows) + } elif (two) { + enc_kq_mvb2_k3_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows) + } else { + enc_kq_mvb4_k3_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows, gcols) + } + } elif (fmt == KqFmt.iq3s) { + if (b8) { + enc_kq_mvb8_iq3s_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows) + } elif (two) { + enc_kq_mvb2_iq3s_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows) + } else { + enc_kq_mvb4_iq3s_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows, gcols) + } + } elif (fmt == KqFmt.iq3xxs) { + if (b8) { + enc_kq_mvb8_iq3xxs_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows) + } elif (two) { + enc_kq_mvb2_iq3xxs_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows) + } else { + enc_kq_mvb4_iq3xxs_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows, gcols) + } + } elif (fmt == KqFmt.iq4nl) { + if (b8) { + enc_kq_mvb8_iq4nl_c(enc, bs, soff, bs, soff, bq, qoff, bx, by, yoff, ka, rows) + } elif (two) { + enc_kq_mvb2_iq4nl_c(enc, bs, soff, bs, soff, bq, qoff, bx, by, yoff, ka, rows) + } else { + enc_kq_mvb4_iq4nl_c(enc, bs, soff, bs, soff, bq, qoff, bx, by, yoff, ka, rows, gcols) + } + } elif (fmt == KqFmt.k2) { + if (b8) { + enc_kq_mvb8_k2_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows) + } elif (two) { + enc_kq_mvb2_k2_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows) + } else { + enc_kq_mvb4_k2_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows, gcols) + } + } elif (fmt == KqFmt.iq2s) { + if (b8) { + enc_kq_mvb8_iq2s_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows) + } elif (two) { + enc_kq_mvb2_iq2s_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows) + } else { + enc_kq_mvb4_iq2s_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows, gcols) + } + } elif (fmt == KqFmt.iq2xs) { + if (b8) { + enc_kq_mvb8_iq2xs_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows) + } elif (two) { + enc_kq_mvb2_iq2xs_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows) + } else { + enc_kq_mvb4_iq2xs_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows, gcols) + } + } elif (fmt == KqFmt.iq2xxs) { + if (b8) { + enc_kq_mvb8_iq2xxs_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows) + } elif (two) { + enc_kq_mvb2_iq2xxs_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows) + } else { + enc_kq_mvb4_iq2xxs_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows, gcols) + } + } elif (fmt == KqFmt.k6) { if (b8) { enc_kq_mvb8_k6_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, ka, rows) } elif (two) { @@ -3770,7 +3905,25 @@ def enc_kq_mvb(enc : MetalComputeEncoder?; fmt : KqFmt; bq : MetalBuffer?; qoff def enc_kq_gemm_mm_b(enc : MetalComputeEncoder?; fmt : KqFmt; bq : MetalBuffer?; qoff : uint64; bs : MetalBuffer?; soff, doff : uint64; bx, by : MetalBuffer?; yoff : uint64; bk, bn : MetalBuffer?; mp, rows : int64) { - if (fmt == KqFmt.k6) { + if (fmt == KqFmt.iq4xs) { + enc_kq_mm_iq4xs_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, bk, bn, mp, rows) + } elif (fmt == KqFmt.k3) { + enc_kq_mm_k3_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, bk, bn, mp, rows) + } elif (fmt == KqFmt.iq3s) { + enc_kq_mm_iq3s_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, bk, bn, mp, rows) + } elif (fmt == KqFmt.iq3xxs) { + enc_kq_mm_iq3xxs_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, bk, bn, mp, rows) + } elif (fmt == KqFmt.iq4nl) { + enc_kq_mm_iq4nl_c(enc, bs, soff, bs, soff, bq, qoff, bx, by, yoff, bk, bn, mp, rows) + } elif (fmt == KqFmt.k2) { + enc_kq_mm_k2_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, bk, bn, mp, rows) + } elif (fmt == KqFmt.iq2s) { + enc_kq_mm_iq2s_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, bk, bn, mp, rows) + } elif (fmt == KqFmt.iq2xs) { + enc_kq_mm_iq2xs_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, bk, bn, mp, rows) + } elif (fmt == KqFmt.iq2xxs) { + enc_kq_mm_iq2xxs_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, bk, bn, mp, rows) + } elif (fmt == KqFmt.k6) { enc_kq_mm_k6_c(enc, bs, doff, bs, soff, bq, qoff, bx, by, yoff, bk, bn, mp, rows) } elif (fmt == KqFmt.k4) { enc_kq_mm_k4_c(enc, bs, soff, bs, soff, bq, qoff, bx, by, yoff, bk, bn, mp, rows) @@ -4196,6 +4349,51 @@ def metal_kernels_release { // nolint:STYLE037,STYLE038 — a flat release_pso release_pso(g_pso_kq_mm_b4) release_pso(g_pso_kq_mm_b5) release_pso(g_pso_kq_mm_b6) + release_pso(g_pso_kq_iq4xs) + release_pso(g_pso_kq_mvb2_iq4xs) + release_pso(g_pso_kq_mvb4_iq4xs) + release_pso(g_pso_kq_mvb8_iq4xs) + release_pso(g_pso_kq_mm_b44) + release_pso(g_pso_kq_k3) + release_pso(g_pso_kq_mvb2_k3) + release_pso(g_pso_kq_mvb4_k3) + release_pso(g_pso_kq_mvb8_k3) + release_pso(g_pso_kq_mm_b3) + release_pso(g_pso_kq_iq3s) + release_pso(g_pso_kq_mvb2_iq3s) + release_pso(g_pso_kq_mvb4_iq3s) + release_pso(g_pso_kq_mvb8_iq3s) + release_pso(g_pso_kq_mm_b33) + release_pso(g_pso_kq_iq3xxs) + release_pso(g_pso_kq_mvb2_iq3xxs) + release_pso(g_pso_kq_mvb4_iq3xxs) + release_pso(g_pso_kq_mvb8_iq3xxs) + release_pso(g_pso_kq_mm_b34) + release_pso(g_pso_kq_iq4nl) + release_pso(g_pso_kq_iq2s) + release_pso(g_pso_kq_mvb2_iq2s) + release_pso(g_pso_kq_mvb4_iq2s) + release_pso(g_pso_kq_mvb8_iq2s) + release_pso(g_pso_kq_mm_b23) + release_pso(g_pso_kq_iq2xs) + release_pso(g_pso_kq_mvb2_iq2xs) + release_pso(g_pso_kq_mvb4_iq2xs) + release_pso(g_pso_kq_mvb8_iq2xs) + release_pso(g_pso_kq_mm_b24) + release_pso(g_pso_kq_iq2xxs) + release_pso(g_pso_kq_mvb2_iq2xxs) + release_pso(g_pso_kq_mvb4_iq2xxs) + release_pso(g_pso_kq_mvb8_iq2xxs) + release_pso(g_pso_kq_mm_b25) + release_pso(g_pso_kq_mvb2_iq4nl) + release_pso(g_pso_kq_mvb4_iq4nl) + release_pso(g_pso_kq_mvb8_iq4nl) + release_pso(g_pso_kq_mm_b45) + release_pso(g_pso_kq_k2) + release_pso(g_pso_kq_mvb2_k2) + release_pso(g_pso_kq_mvb4_k2) + release_pso(g_pso_kq_mvb8_k2) + release_pso(g_pso_kq_mm_b2) if (g_pso_rms != null) { metal_release(g_pso_rms) g_pso_rms = null @@ -5421,10 +5619,42 @@ class template MetalKqMulMmK45T : MetalMmTileBase { @template_constant BLK : uint = 32u // superblock quant-plane stride in uints (unused in the SIXBIT arm — k6 strides are hardcoded 48/16/8) @template_constant QH : bool = false // q5_K: the 5th-bit overlay plane rides after the nibbles @template_constant SIXBIT : bool = false // q6_K: int8 sub-scales x super-d, ql/qh compose + @template_constant IQ4XS : bool = false // iq4xs: the k6 split scale form, the q40 pairing, codebook decode + @template_constant K3 : bool = false // q3_K: the k6 split scale form, 2-bit lane + hmask compose, offset 4 + @template_constant IQ3S : bool = false // iq3s: the k6 split scale form, threadgroup grid gather + sign flip + @template_constant IQ3XXS : bool = false // iq3xxs: the same form over the halved grid + parity signs + @template_constant IQ4NL : bool = false // iq4nl: the IQ4XS nibble walk with the per-32-block f16 d plane (no sub-scale; s0off = soff) + @template_constant K2 : bool = false // q2_K: per-16 sc/min nibble pairs (one uint4 strip), the split 4B d+dmin tail at s0off + @template_constant IQ2S : bool = false // iq2s: the u64-grid gather DIRECT off the hoisted constant table, per-16 UNSIGNED strips + @template_constant IQ2XS : bool = false // iq2xs: iq2s's walk with ksigns-by-parity off each u16 qs word (no sign plane, no qh) + @template_constant IQ2XXS : bool = false // iq2xxs: byte grid indices + aux32 parity ksigns, per-32 UNSIGNED strips + @workgroup @template_gate = IQ4XS iq4lut : float[16] // kvalues_iq4nl as floats, staged once per threadgroup + @workgroup @template_gate = IQ3S gridw3 : uint[512] // the 2 KB iq3s grid, staged once per threadgroup + @workgroup @template_gate = IQ3XXS gridw3x : uint[256] // the 1 KB halved iq3xxs grid, staged once per threadgroup [metal_kernel] - def metal_kq_mulmm { // nolint:STYLE038 — MSL kernel body; the mul_mm tiles are simdgroup-barrier coupled + def metal_kq_mulmm { // nolint:STYLE037,STYLE038 — MSL kernel body; the mul_mm tiles are simdgroup-barrier coupled let lid = gl_LocalInvocationID.x + static_if (IQ4XS) { + if (lid < 16u) { + iq4lut[lid] = iq4_lut(lid) + } + barrier() + } + static_if (IQ3S) { + for [unroll_full] (gk in range(4)) { + let gi = lid + uint(gk) * 128u + gridw3[gi] = iq3s_gw(int(gi)) + } + barrier() + } + static_if (IQ3XXS) { + for [unroll_full] (gk in range(2)) { + let gi = lid + uint(gk) * 128u + gridw3x[gi] = iq3xxs_gw(int(gi)) + } + barrier() + } let sg = gl_SubgroupID let mBase = gl_WorkGroupID.x * 32u let nBase = gl_WorkGroupID.y * 64u @@ -5489,40 +5719,218 @@ class template MetalKqMulMmK45T : MetalMmTileBase { va[k * 4 + 3] = float16(dsc3 * float(qv & 4278190080u) - dmn) } } else { - let sv = ks4[blk] - var sc = 0u - var mn = 0u - if (js < 4u) { - sc = (sv.y >> (8u * js)) & 63u - mn = (sv.z >> (8u * js)) & 63u - } else { - let j2 = js - 4u - let hi = sv.w >> (8u * j2) - sc = (hi & 15u) | (((sv.y >> (8u * j2 + 6u)) & 3u) << 4u) - mn = ((hi >> 4u) & 15u) | (((sv.z >> (8u * j2 + 6u)) & 3u) << 4u) - } - let dsc = float(ksh[blk * 8u]) * float(sc) - let dmn = float(ksh[blk * 8u + 1u]) * float(mn) - let qb = blk * BLK + (js / 2u) * 8u + il0 * 4u - static_if (QH) { - let hb = blk * BLK + 32u + il0 * 4u - let nsh = (js & 1u) * 4u + static_if (K3) { + // q3_K: strip byte si is the SIGNED sub-scale (k6's split form); block js of the + // superblock = qs uints (js/4)*8 + il0*4.. at shift 2*(js%4) | hmask uints 16 + il0*4.. bit js + let hf3 = js / 4u + let gg3 = js % 4u + let sv = ks4[blk] + let dall = float(ksh[blk]) + let si = hf3 * 8u + gg3 * 2u + il0 + let scw = si < 8u ? (si < 4u ? sv.x : sv.y) : (si < 12u ? sv.z : sv.w) + let s3 = float(((int(scw >> ((si & 3u) * 8u)) & 255) ^ 128) - 128) + let dsc = dall * s3 + let dmn = dsc * 4.0 + let dsc1 = dsc * 0.00390625 + let dsc2 = dsc * 0.0000152587890625 + let dsc3 = dsc * 0.000000059604644775390625 + let qb = blk * 24u + hf3 * 8u + il0 * 4u + let hb = blk * 24u + 16u + il0 * 4u + let nsh = gg3 * 2u + let hsh = hf3 * 4u + gg3 for [unroll_full] (k in range(4)) { let u = kqu[qb + uint(k)] let hu = kqu[hb + uint(k)] - for [unroll_full] (c in range(4)) { - let q = ((u >> (8u * uint(c) + nsh)) & 15u) | (((hu >> (8u * uint(c) + js)) & 1u) << 4u) - va[k * 4 + c] = float16(dsc * float(q) - dmn) - } + let qv = ((u >> nsh) & 0x03030303) | (((hu >> hsh) & 0x01010101) << 2u) + va[k * 4] = float16(dsc * float(qv & 255u) - dmn) + va[k * 4 + 1] = float16(dsc1 * float(qv & 65280u) - dmn) + va[k * 4 + 2] = float16(dsc2 * float(qv & 16711680u) - dmn) + va[k * 4 + 3] = float16(dsc3 * float(qv & 4278190080u) - dmn) } } else { - let nsh = (js & 1u) * 4u - for [unroll_full] (k in range(4)) { - let u = kqu[qb + uint(k)] - for [unroll_full] (c in range(4)) { - let q = (u >> (8u * uint(c) + nsh)) & 15u - va[k * 4 + c] = float16(dsc * float(q) - dmn) + static_if (IQ2XXS) { + // iq2xxs: per-32 UNSIGNED strip byte js; block js's grid word carries + // four byte indices, its aux32 the parity-derived ksigns + let sv = ks4[blk] + let scw = js < 4u ? sv.x : sv.y + let dsc = float(ksh[blk]) * float((scw >> ((js & 3u) * 8u)) & 255u) + let gw2 = kqu[blk * 16u + js * 2u] + let aux2 = kqu[blk * 16u + js * 2u + 1u] + for [unroll_full] (k in range(4)) { + let q3 = il0 * 2u + uint(k) / 2u + let bidx = (gw2 >> (8u * q3)) & 255u + let sgb2 = ksign7m((aux2 >> (7u * q3)) & 127u) + let nib = uint(k) % 2u == 0u ? sgb2 & 15u : sgb2 >> 4u + let sw = iq3s_sw(iq2xxs_gw(int(bidx * 2u + uint(k) % 2u)), nib) + for [unroll_full] (c in range(4)) { + va[k * 4 + c] = float16(dsc * iq3s_sx(sw, uint(c))) + } + } + } else { + static_if (IQ2XS) { + // iq2xs: the iq2s strip/scale reads; each u16 qs word = 9-bit grid + // index + 7-bit ksigns index whose sign byte ksign7m recomputes + let sv = ks4[blk] + let si2 = js * 2u + il0 + let scw = si2 < 8u ? (si2 < 4u ? sv.x : sv.y) : (si2 < 12u ? sv.z : sv.w) + let dsc = float(ksh[blk]) * float((scw >> ((si2 & 3u) * 8u)) & 255u) + let qw2 = kqu[blk * 16u + js * 2u + il0] + for [unroll_full] (k in range(4)) { + let w16 = (qw2 >> (16u * (uint(k) / 2u))) & 0xFFFFu + let idx2 = (w16 & 511u) * 2u + uint(k) % 2u + let sgb2 = ksign7m(w16 >> 9u) + let nib = uint(k) % 2u == 0u ? sgb2 & 15u : sgb2 >> 4u + let sw = iq3s_sw(iq2xs_gw(int(idx2)), nib) + for [unroll_full] (c in range(4)) { + va[k * 4 + c] = float16(dsc * iq3s_sx(sw, uint(c))) + } + } + } else { + static_if (IQ2S) { + // iq2s: strip byte js*2+il0 is the UNSIGNED per-16 sub-scale; idx byte l of + // block js doubles into the u64 grid's word pair, signs off the block's bytes + let sv = ks4[blk] + let si2 = js * 2u + il0 + let scw = si2 < 8u ? (si2 < 4u ? sv.x : sv.y) : (si2 < 12u ? sv.z : sv.w) + let dsc = float(ksh[blk]) * float((scw >> ((si2 & 3u) * 8u)) & 255u) + let qw2 = kqu[blk * 18u + js] + let sgw2 = kqu[blk * 18u + 8u + js] + let qh2 = (kqu[blk * 18u + 16u + js / 4u] >> ((js % 4u) * 8u)) & 255u + for [unroll_full] (k in range(4)) { + let l2 = il0 * 2u + uint(k) / 2u + let idx2 = (((qw2 >> (8u * l2)) & 255u) | ((qh2 << (8u - 2u * l2)) & 0x300u)) * 2u + uint(k) % 2u + let sgb2 = (sgw2 >> (8u * l2)) & 255u + let nib = uint(k) % 2u == 0u ? sgb2 & 15u : sgb2 >> 4u + let sw = iq3s_sw(iq2s_gw(int(idx2)), nib) + for [unroll_full] (c in range(4)) { + va[k * 4 + c] = float16(dsc * iq3s_sx(sw, uint(c))) + } + } + } else { + static_if (IQ3XXS) { + // iq3xxs: the iq3s arm over the halved grid - bare byte index, parity + // signs off the block's aux32 (no table) + let sv = ks4[blk] + let scw = js < 4u ? sv.x : sv.y + let dsc = float(ksh[blk]) * float(((int(scw >> ((js & 3u) * 8u)) & 255) ^ 128) - 128) + let qsw = kqu[blk * 24u + js * 2u + il0] + let aux3 = kqu[blk * 24u + 16u + js] + for [unroll_full] (k in range(4)) { + let q3 = il0 * 2u + uint(k) / 2u + let sgb = ksign7m((aux3 >> (7u * q3)) & 127u) + let nib = uint(k) % 2u == 0u ? sgb & 15u : sgb >> 4u + let sw = iq3s_sw(gridw3x[(qsw >> (8u * uint(k))) & 255u], nib) + for [unroll_full] (c in range(4)) { + va[k * 4 + c] = float16(dsc * iq3s_sx(sw, uint(c))) + } + } + } else { + static_if (IQ3S) { + // iq3s: strip byte js is the SIGNED sub-scale (the k6 split form); block js's + // qs word (js*2 + il0) = 4 grid words = 16 elems, signs off the block's own bytes + let sv = ks4[blk] + let scw = js < 4u ? sv.x : sv.y + let dsc = float(ksh[blk]) * float(((int(scw >> ((js & 3u) * 8u)) & 255) ^ 128) - 128) + let qsw = kqu[blk * 26u + js * 2u + il0] + let qh3 = (kqu[blk * 26u + 16u + js / 4u] >> ((js % 4u) * 8u)) & 255u + let sgw3 = kqu[blk * 26u + 18u + js] + for [unroll_full] (k in range(4)) { + let b3 = il0 * 4u + uint(k) + let idx = ((qsw >> (8u * uint(k))) & 255u) | ((qh3 << (8u - b3)) & 256u) + let nib = (sgw3 >> ((b3 / 2u) * 8u + (b3 % 2u) * 4u)) & 15u + let sw = iq3s_sw(gridw3[idx], nib) + for [unroll_full] (c in range(4)) { + va[k * 4 + c] = float16(dsc * iq3s_sx(sw, uint(c))) + } + } + } else { + static_if (IQ4XS) { + // iq4xs: strip byte js is the SIGNED sub-scale, super-d off the d plane (the k6 split + // form); block js = 4 uints of the q40 pairing, il0 picks the nibble half; the codebook + // off threadgroup memory. iq4nl: same walk, dsc = block js's own f16 d + var dsc = 0.0 + static_if (IQ4NL) { + dsc = float(ksh[blk * 8u + js]) + } else { + let sv = ks4[blk] + let scw = js < 4u ? sv.x : sv.y + dsc = float(ksh[blk]) * float(((int(scw >> ((js & 3u) * 8u)) & 255) ^ 128) - 128) + } + let qb = blk * 32u + js * 4u + let nsh = il0 * 4u + for [unroll_full] (k in range(4)) { + let u = kqu[qb + uint(k)] + for [unroll_full] (c in range(4)) { + va[k * 4 + c] = float16(dsc * iq4lut[(u >> (8u * uint(c) + nsh)) & 15u]) + } + } + } else { + static_if (K2) { + // q2_K: pair byte si = the group's sc|mn nibbles; d/dmin off the split tail + let hf3 = js / 4u + let gg3 = js % 4u + let sv = ks4[blk] + let si = hf3 * 8u + gg3 * 2u + il0 + let pw = si < 8u ? (si < 4u ? sv.x : sv.y) : (si < 12u ? sv.z : sv.w) + let pb = (pw >> ((si & 3u) * 8u)) & 255u + let dsc = float(ksh[blk * 2u]) * float(pb & 15u) + let dmn = float(ksh[blk * 2u + 1u]) * float(pb >> 4u) + let dsc1 = dsc * 0.00390625 + let dsc2 = dsc * 0.0000152587890625 + let dsc3 = dsc * 0.000000059604644775390625 + let qb = blk * 16u + hf3 * 8u + il0 * 4u + let nsh = gg3 * 2u + for [unroll_full] (k in range(4)) { + let u = kqu[qb + uint(k)] + let qv = (u >> nsh) & 0x03030303 + va[k * 4] = float16(dsc * float(qv & 255u) - dmn) + va[k * 4 + 1] = float16(dsc1 * float(qv & 65280u) - dmn) + va[k * 4 + 2] = float16(dsc2 * float(qv & 16711680u) - dmn) + va[k * 4 + 3] = float16(dsc3 * float(qv & 4278190080u) - dmn) + } + } else { + let sv = ks4[blk] + var sc = 0u + var mn = 0u + if (js < 4u) { + sc = (sv.y >> (8u * js)) & 63u + mn = (sv.z >> (8u * js)) & 63u + } else { + let j2 = js - 4u + let hi = sv.w >> (8u * j2) + sc = (hi & 15u) | (((sv.y >> (8u * j2 + 6u)) & 3u) << 4u) + mn = ((hi >> 4u) & 15u) | (((sv.z >> (8u * j2 + 6u)) & 3u) << 4u) + } + let dsc = float(ksh[blk * 8u]) * float(sc) + let dmn = float(ksh[blk * 8u + 1u]) * float(mn) + let qb = blk * BLK + (js / 2u) * 8u + il0 * 4u + static_if (QH) { + let hb = blk * BLK + 32u + il0 * 4u + let nsh = (js & 1u) * 4u + for [unroll_full] (k in range(4)) { + let u = kqu[qb + uint(k)] + let hu = kqu[hb + uint(k)] + for [unroll_full] (c in range(4)) { + let q = ((u >> (8u * uint(c) + nsh)) & 15u) | (((hu >> (8u * uint(c) + js)) & 1u) << 4u) + va[k * 4 + c] = float16(dsc * float(q) - dmn) + } + } + } else { + let nsh = (js & 1u) * 4u + for [unroll_full] (k in range(4)) { + let u = kqu[qb + uint(k)] + for [unroll_full] (c in range(4)) { + let q = (u >> (8u * uint(c) + nsh)) & 15u + va[k * 4 + c] = float16(dsc * float(q) - dmn) + } + } + } + } } + } + } + } + } } } } @@ -5564,6 +5972,53 @@ class MetalKqMulMmK6 : MetalKqMulMmK45T { override SIXBIT = true } +[metal_dispatch(name = "enc_kq_mm_iq4xs_c", pso = "g_pso_kq_mm_b44", tgmem = "MetalKqMulMmIq4xs_metal_kq_mulmm_msl_tgmem", tg = 128, grid = "mp/32, rows/64", params = "mp : int64, rows : int64")] +class MetalKqMulMmIq4xs : MetalKqMulMmK45T { + override IQ4XS = true +} + +[metal_dispatch(name = "enc_kq_mm_k3_c", pso = "g_pso_kq_mm_b3", tgmem = "MetalKqMulMmK3_metal_kq_mulmm_msl_tgmem", tg = 128, grid = "mp/32, rows/64", params = "mp : int64, rows : int64")] +class MetalKqMulMmK3 : MetalKqMulMmK45T { + override K3 = true +} + +[metal_dispatch(name = "enc_kq_mm_iq3s_c", pso = "g_pso_kq_mm_b33", tgmem = "MetalKqMulMmIq3s_metal_kq_mulmm_msl_tgmem", tg = 128, grid = "mp/32, rows/64", params = "mp : int64, rows : int64")] +class MetalKqMulMmIq3s : MetalKqMulMmK45T { + override IQ3S = true +} + +[metal_dispatch(name = "enc_kq_mm_iq3xxs_c", pso = "g_pso_kq_mm_b34", tgmem = "MetalKqMulMmIq3xxs_metal_kq_mulmm_msl_tgmem", tg = 128, grid = "mp/32, rows/64", params = "mp : int64, rows : int64")] +class MetalKqMulMmIq3xxs : MetalKqMulMmK45T { + override IQ3XXS = true +} + +[metal_dispatch(name = "enc_kq_mm_k2_c", pso = "g_pso_kq_mm_b2", tgmem = "MetalKqMulMmK2_metal_kq_mulmm_msl_tgmem", tg = 128, grid = "mp/32, rows/64", params = "mp : int64, rows : int64")] +class MetalKqMulMmK2 : MetalKqMulMmK45T { + override K2 = true +} + +[metal_dispatch(name = "enc_kq_mm_iq2s_c", pso = "g_pso_kq_mm_b23", tgmem = "MetalKqMulMmIq2s_metal_kq_mulmm_msl_tgmem", tg = 128, grid = "mp/32, rows/64", params = "mp : int64, rows : int64")] +class MetalKqMulMmIq2s : MetalKqMulMmK45T { + override IQ2S = true +} + +[metal_dispatch(name = "enc_kq_mm_iq2xs_c", pso = "g_pso_kq_mm_b24", tgmem = "MetalKqMulMmIq2xs_metal_kq_mulmm_msl_tgmem", tg = 128, grid = "mp/32, rows/64", params = "mp : int64, rows : int64")] +class MetalKqMulMmIq2xs : MetalKqMulMmK45T { + override IQ2XS = true +} + +[metal_dispatch(name = "enc_kq_mm_iq2xxs_c", pso = "g_pso_kq_mm_b25", tgmem = "MetalKqMulMmIq2xxs_metal_kq_mulmm_msl_tgmem", tg = 128, grid = "mp/32, rows/64", params = "mp : int64, rows : int64")] +class MetalKqMulMmIq2xxs : MetalKqMulMmK45T { + override IQ2XXS = true +} + +// IQ4NL flips the scale read to q40's per-32-block d plane +[metal_dispatch(name = "enc_kq_mm_iq4nl_c", pso = "g_pso_kq_mm_b45", tgmem = "MetalKqMulMmIq4nl_metal_kq_mulmm_msl_tgmem", tg = 128, grid = "mp/32, rows/64", params = "mp : int64, rows : int64")] +class MetalKqMulMmIq4nl : MetalKqMulMmK45T { + override IQ4XS = true + override IQ4NL = true +} + // Router GEMV over the uploaded fp32 router slab: y[ne] = W[ne x dim] . x per stream. ne <= 256 // rows. Each sg serves NR positions per weight load (slab SLC traffic /NR); BATCHED stamps // clamp tails via ns; singles fold the clamp away and the gate erases ns outright. @@ -5841,6 +6296,612 @@ def private kmask_scales(sv : uint4; iq : uint) : uint4 { ((hw4 >> 4u) & 0x0F0F) | ((hw2 & 0xC0C0) >> 2u)) } +// IQ4_XS codebook (kvalues_iq4nl) as four packed words, selected + byte-extracted — +// no table memory; the k6 sign trick widens the byte +def private iq4_lut(q : uint) : float { + let w = q < 8u ? (q < 4u ? 3215825025u : 4142587343u) : (q < 12u ? 639175937u : 1901675829u) + return float((int((w >> ((q & 3u) * 8u)) & 255u) ^ 128) - 128) +} + +// iq3s_grid, one word per call - the GEMV/mul_mm stage it into threadgroup memory once +// per group; the mv twins read it directly. +def private iq3s_gw(i : int) : uint { + let tbl = fixed_array( + 0x01010101u, 0x01010103u, 0x01010105u, 0x0101010bu, 0x0101010fu, 0x01010301u, 0x01010303u, 0x01010305u, + 0x01010309u, 0x0101030du, 0x01010501u, 0x01010503u, 0x0101050bu, 0x01010707u, 0x01010901u, 0x01010905u, + 0x0101090bu, 0x0101090fu, 0x01010b03u, 0x01010b07u, 0x01010d01u, 0x01010d05u, 0x01010f03u, 0x01010f09u, + 0x01010f0fu, 0x01030101u, 0x01030103u, 0x01030105u, 0x01030109u, 0x01030301u, 0x01030303u, 0x0103030bu, + 0x01030501u, 0x01030507u, 0x0103050fu, 0x01030703u, 0x0103070bu, 0x01030909u, 0x01030d03u, 0x01030d0bu, + 0x01030f05u, 0x01050101u, 0x01050103u, 0x0105010bu, 0x0105010fu, 0x01050301u, 0x01050307u, 0x0105030du, + 0x01050503u, 0x0105050bu, 0x01050701u, 0x01050709u, 0x01050905u, 0x0105090bu, 0x0105090fu, 0x01050b03u, + 0x01050b07u, 0x01050f01u, 0x01050f07u, 0x01070107u, 0x01070303u, 0x0107030bu, 0x01070501u, 0x01070505u, + 0x01070703u, 0x01070707u, 0x0107070du, 0x01070909u, 0x01070b01u, 0x01070b05u, 0x01070d0fu, 0x01070f03u, + 0x01070f0bu, 0x01090101u, 0x01090307u, 0x0109030fu, 0x01090503u, 0x01090509u, 0x01090705u, 0x01090901u, + 0x01090907u, 0x01090b03u, 0x01090f01u, 0x010b0105u, 0x010b0109u, 0x010b0501u, 0x010b0505u, 0x010b050du, + 0x010b0707u, 0x010b0903u, 0x010b090bu, 0x010b090fu, 0x010b0d0du, 0x010b0f07u, 0x010d010du, 0x010d0303u, + 0x010d0307u, 0x010d0703u, 0x010d0b05u, 0x010d0f03u, 0x010f0101u, 0x010f0105u, 0x010f0109u, 0x010f0501u, + 0x010f0505u, 0x010f050du, 0x010f0707u, 0x010f0b01u, 0x010f0b09u, 0x03010101u, 0x03010103u, 0x03010105u, + 0x03010109u, 0x03010301u, 0x03010303u, 0x03010307u, 0x0301030bu, 0x0301030fu, 0x03010501u, 0x03010505u, + 0x03010703u, 0x03010709u, 0x0301070du, 0x03010b09u, 0x03010b0du, 0x03010d03u, 0x03010f05u, 0x03030101u, + 0x03030103u, 0x03030107u, 0x0303010du, 0x03030301u, 0x03030309u, 0x03030503u, 0x03030701u, 0x03030707u, + 0x03030903u, 0x03030b01u, 0x03030b05u, 0x03030f01u, 0x03030f0du, 0x03050101u, 0x03050305u, 0x0305030bu, + 0x0305030fu, 0x03050501u, 0x03050509u, 0x03050705u, 0x03050901u, 0x03050907u, 0x03050b0bu, 0x03050d01u, + 0x03050f05u, 0x03070103u, 0x03070109u, 0x0307010fu, 0x03070301u, 0x03070307u, 0x03070503u, 0x0307050fu, + 0x03070701u, 0x03070709u, 0x03070903u, 0x03070d05u, 0x03070f01u, 0x03090107u, 0x0309010bu, 0x03090305u, + 0x03090309u, 0x03090703u, 0x03090707u, 0x03090905u, 0x0309090du, 0x03090b01u, 0x03090b09u, 0x030b0103u, + 0x030b0301u, 0x030b0307u, 0x030b0503u, 0x030b0701u, 0x030b0705u, 0x030b0b03u, 0x030d0501u, 0x030d0509u, + 0x030d050fu, 0x030d0909u, 0x030d090du, 0x030f0103u, 0x030f0107u, 0x030f0301u, 0x030f0305u, 0x030f0503u, + 0x030f070bu, 0x030f0903u, 0x030f0d05u, 0x030f0f01u, 0x05010101u, 0x05010103u, 0x05010107u, 0x0501010bu, + 0x0501010fu, 0x05010301u, 0x05010305u, 0x05010309u, 0x0501030du, 0x05010503u, 0x05010507u, 0x0501050fu, + 0x05010701u, 0x05010705u, 0x05010903u, 0x05010907u, 0x0501090bu, 0x05010b01u, 0x05010b05u, 0x05010d0fu, + 0x05010f01u, 0x05010f07u, 0x05010f0bu, 0x05030101u, 0x05030105u, 0x05030301u, 0x05030307u, 0x0503030fu, + 0x05030505u, 0x0503050bu, 0x05030703u, 0x05030709u, 0x05030905u, 0x05030b03u, 0x05050103u, 0x05050109u, + 0x0505010fu, 0x05050503u, 0x05050507u, 0x05050701u, 0x0505070fu, 0x05050903u, 0x05050b07u, 0x05050b0fu, + 0x05050f03u, 0x05050f09u, 0x05070101u, 0x05070105u, 0x0507010bu, 0x05070303u, 0x05070505u, 0x05070509u, + 0x05070703u, 0x05070707u, 0x05070905u, 0x05070b01u, 0x05070d0du, 0x05090103u, 0x0509010fu, 0x05090501u, + 0x05090507u, 0x05090705u, 0x0509070bu, 0x05090903u, 0x05090f05u, 0x05090f0bu, 0x050b0109u, 0x050b0303u, + 0x050b0505u, 0x050b070fu, 0x050b0901u, 0x050b0b07u, 0x050b0f01u, 0x050d0101u, 0x050d0105u, 0x050d010fu, + 0x050d0503u, 0x050d0b0bu, 0x050d0d03u, 0x050f010bu, 0x050f0303u, 0x050f050du, 0x050f0701u, 0x050f0907u, + 0x050f0b01u, 0x07010105u, 0x07010303u, 0x07010307u, 0x0701030bu, 0x0701030fu, 0x07010505u, 0x07010703u, + 0x07010707u, 0x0701070bu, 0x07010905u, 0x07010909u, 0x0701090fu, 0x07010b03u, 0x07010d07u, 0x07010f03u, + 0x07030103u, 0x07030107u, 0x0703010bu, 0x07030309u, 0x07030503u, 0x07030507u, 0x07030901u, 0x07030d01u, + 0x07030f05u, 0x07030f0du, 0x07050101u, 0x07050305u, 0x07050501u, 0x07050705u, 0x07050709u, 0x07050b01u, + 0x07070103u, 0x07070301u, 0x07070309u, 0x07070503u, 0x07070507u, 0x0707050fu, 0x07070701u, 0x07070903u, + 0x07070907u, 0x0707090fu, 0x07070b0bu, 0x07070f07u, 0x07090107u, 0x07090303u, 0x0709030du, 0x07090505u, + 0x07090703u, 0x07090b05u, 0x07090d01u, 0x07090d09u, 0x070b0103u, 0x070b0301u, 0x070b0305u, 0x070b050bu, + 0x070b0705u, 0x070b0909u, 0x070b0b0du, 0x070b0f07u, 0x070d030du, 0x070d0903u, 0x070f0103u, 0x070f0107u, + 0x070f0501u, 0x070f0505u, 0x070f070bu, 0x09010101u, 0x09010109u, 0x09010305u, 0x09010501u, 0x09010509u, + 0x0901050fu, 0x09010705u, 0x09010903u, 0x09010b01u, 0x09010f01u, 0x09030105u, 0x0903010fu, 0x09030303u, + 0x09030307u, 0x09030505u, 0x09030701u, 0x0903070bu, 0x09030907u, 0x09030b03u, 0x09030b0bu, 0x09050103u, + 0x09050107u, 0x09050301u, 0x0905030bu, 0x09050503u, 0x09050707u, 0x09050901u, 0x09050b0fu, 0x09050d05u, + 0x09050f01u, 0x09070109u, 0x09070303u, 0x09070307u, 0x09070501u, 0x09070505u, 0x09070703u, 0x0907070bu, + 0x09090101u, 0x09090105u, 0x09090509u, 0x0909070fu, 0x09090901u, 0x09090f03u, 0x090b010bu, 0x090b010fu, + 0x090b0503u, 0x090b0d05u, 0x090d0307u, 0x090d0709u, 0x090d0d01u, 0x090f0301u, 0x090f030bu, 0x090f0701u, + 0x090f0907u, 0x090f0b03u, 0x0b010105u, 0x0b010301u, 0x0b010309u, 0x0b010505u, 0x0b010901u, 0x0b010909u, + 0x0b01090fu, 0x0b010b05u, 0x0b010d0du, 0x0b010f09u, 0x0b030103u, 0x0b030107u, 0x0b03010bu, 0x0b030305u, + 0x0b030503u, 0x0b030705u, 0x0b030f05u, 0x0b050101u, 0x0b050303u, 0x0b050507u, 0x0b050701u, 0x0b05070du, + 0x0b050b07u, 0x0b070105u, 0x0b07010fu, 0x0b070301u, 0x0b07050fu, 0x0b070909u, 0x0b070b03u, 0x0b070d0bu, + 0x0b070f07u, 0x0b090103u, 0x0b090109u, 0x0b090501u, 0x0b090705u, 0x0b09090du, 0x0b0b0305u, 0x0b0b050du, + 0x0b0b0b03u, 0x0b0b0b07u, 0x0b0d0905u, 0x0b0f0105u, 0x0b0f0109u, 0x0b0f0505u, 0x0d010303u, 0x0d010307u, + 0x0d01030bu, 0x0d010703u, 0x0d010707u, 0x0d010d01u, 0x0d030101u, 0x0d030501u, 0x0d03050fu, 0x0d030d09u, + 0x0d050305u, 0x0d050709u, 0x0d050905u, 0x0d050b0bu, 0x0d050d05u, 0x0d050f01u, 0x0d070101u, 0x0d070309u, + 0x0d070503u, 0x0d070901u, 0x0d09050bu, 0x0d090907u, 0x0d090d05u, 0x0d0b0101u, 0x0d0b0107u, 0x0d0b0709u, + 0x0d0b0d01u, 0x0d0d010bu, 0x0d0d0901u, 0x0d0f0303u, 0x0d0f0307u, 0x0f010101u, 0x0f010109u, 0x0f01010fu, + 0x0f010501u, 0x0f010505u, 0x0f01070du, 0x0f010901u, 0x0f010b09u, 0x0f010d05u, 0x0f030105u, 0x0f030303u, + 0x0f030509u, 0x0f030907u, 0x0f03090bu, 0x0f050103u, 0x0f050109u, 0x0f050301u, 0x0f05030du, 0x0f050503u, + 0x0f050701u, 0x0f050b03u, 0x0f070105u, 0x0f070705u, 0x0f07070bu, 0x0f070b07u, 0x0f090103u, 0x0f09010bu, + 0x0f090307u, 0x0f090501u, 0x0f090b01u, 0x0f0b0505u, 0x0f0b0905u, 0x0f0d0105u, 0x0f0d0703u, 0x0f0f0101u) + return tbl[i] +} + +// one signed grid word: the 4-bit sign nibble expands to a byte mask; magnitudes are odd +// 1..15, so the negate's +1 never carries across a byte lane +def private iq3s_sw(g, nib : uint) : uint { + let m1 = (nib * 0x00204081u) & 0x01010101u + return (g ^ (m1 * 255u)) + m1 +} + +def private iq3s_sx(sw, t : uint) : float { + return float((int((sw >> (t * 8u)) & 255u) ^ 128) - 128) +} + +// the HALVED iq3xxs grid (iq3xxs_grid, every byte / 2 - the plane convention), one +// word per call; the GEMV and mul_mm stage it, the mv twins read it directly. +def private iq3xxs_gw(i : int) : uint { + let tbl = fixed_array( + 0x02020202u, 0x0202020au, 0x02020212u, 0x02020606u, 0x0202060eu, 0x0202061fu, 0x02020a02u, 0x02020a0au, + 0x02020e06u, 0x0202120au, 0x02021f0eu, 0x02021f16u, 0x02060206u, 0x0206020eu, 0x02060602u, 0x0206060au, + 0x02060a06u, 0x02060a16u, 0x02060e02u, 0x02060e0au, 0x02061206u, 0x02061612u, 0x02061f02u, 0x020a0202u, + 0x020a020au, 0x020a0212u, 0x020a0606u, 0x020a0a02u, 0x020a0a0au, 0x020a0e06u, 0x020a0e0eu, 0x020a0e1fu, + 0x020a1606u, 0x020a161fu, 0x020a1f16u, 0x020e0206u, 0x020e021fu, 0x020e0602u, 0x020e060au, 0x020e0a16u, + 0x020e1f02u, 0x0212060eu, 0x02120e1fu, 0x02121212u, 0x0212161fu, 0x02121f0eu, 0x02121f16u, 0x02160206u, + 0x0216021fu, 0x02160e0au, 0x0216160au, 0x021a0e16u, 0x021a1a12u, 0x021f0602u, 0x021f0612u, 0x021f061au, + 0x021f120eu, 0x021f1a06u, 0x06020206u, 0x0602020eu, 0x06020602u, 0x0602060au, 0x06020a06u, 0x06020a0eu, + 0x06020e02u, 0x06020e0au, 0x06020e12u, 0x0602121fu, 0x06021602u, 0x06060202u, 0x0606020au, 0x06060606u, + 0x06060a02u, 0x06060a0au, 0x060a0206u, 0x060a020eu, 0x060a0602u, 0x060a060au, 0x060a0a06u, 0x060a0e02u, + 0x060a1f0au, 0x060e0202u, 0x060e020au, 0x060e0a02u, 0x060e0e06u, 0x060e121au, 0x060e1a1au, 0x06120206u, + 0x06120216u, 0x06121602u, 0x06160a02u, 0x06160a12u, 0x0616121au, 0x06161f06u, 0x061a0216u, 0x061f0a0au, + 0x061f1202u, 0x0a020202u, 0x0a02020au, 0x0a020606u, 0x0a02060eu, 0x0a020a02u, 0x0a020a0au, 0x0a020a1au, + 0x0a020e06u, 0x0a02120au, 0x0a060206u, 0x0a06020eu, 0x0a060216u, 0x0a060602u, 0x0a06060au, 0x0a060a06u, + 0x0a060e02u, 0x0a061a0eu, 0x0a061a1fu, 0x0a061f02u, 0x0a0a0202u, 0x0a0a020au, 0x0a0a0606u, 0x0a0a061fu, + 0x0a0a0a02u, 0x0a0a0a0au, 0x0a0a0e1fu, 0x0a0a1202u, 0x0a0a1616u, 0x0a0e0206u, 0x0a0e0602u, 0x0a0e0612u, + 0x0a0e1f02u, 0x0a0e1f12u, 0x0a120e16u, 0x0a12160eu, 0x0a16020eu, 0x0a160a1fu, 0x0a161206u, 0x0a161f12u, + 0x0a1f0206u, 0x0a1f020eu, 0x0a1f061au, 0x0a1f1216u, 0x0e020206u, 0x0e020602u, 0x0e02060au, 0x0e020a06u, + 0x0e020a0eu, 0x0e021602u, 0x0e021a16u, 0x0e021f0au, 0x0e060202u, 0x0e06020au, 0x0e060a02u, 0x0e060e06u, + 0x0e061212u, 0x0e06121au, 0x0e0a0206u, 0x0e0a020eu, 0x0e0a0602u, 0x0e0a0a16u, 0x0e0a160au, 0x0e0a1f0au, + 0x0e0e0606u, 0x0e0e0e0eu, 0x0e120e02u, 0x0e12121fu, 0x0e121f0au, 0x0e160202u, 0x0e16021au, 0x0e160a0au, + 0x0e161616u, 0x0e1a0612u, 0x0e1a0e1au, 0x0e1a1a0eu, 0x0e1f0e0eu, 0x0e1f1a02u, 0x12020212u, 0x1202061fu, + 0x12020e16u, 0x12020e1fu, 0x1202160eu, 0x1202161fu, 0x12061f12u, 0x120a0a02u, 0x120a0e1fu, 0x120a1202u, + 0x120a1a02u, 0x120a1a1au, 0x120e021fu, 0x120e1216u, 0x12120212u, 0x12121606u, 0x12121a12u, 0x12160a16u, + 0x1216120eu, 0x12161f02u, 0x121f0216u, 0x121f0602u, 0x121f060au, 0x121f0e02u, 0x1602060au, 0x16021206u, + 0x16021f02u, 0x16060202u, 0x1606021au, 0x16060a1au, 0x16061616u, 0x160a0612u, 0x160a0e0au, 0x160a1f0au, + 0x160e020au, 0x160e160eu, 0x16120602u, 0x16120a0eu, 0x16120a1fu, 0x16121f0au, 0x1616020au, 0x16160e06u, + 0x161a1602u, 0x161f0a12u, 0x161f120au, 0x1a020a12u, 0x1a021212u, 0x1a02121au, 0x1a021a12u, 0x1a060a06u, + 0x1a061a06u, 0x1a0a061fu, 0x1a0a1a12u, 0x1a0e0e02u, 0x1a0e0e1au, 0x1a121212u, 0x1a160216u, 0x1a16160au, + 0x1a1a0e0eu, 0x1a1f020eu, 0x1a1f0a06u, 0x1f02020eu, 0x1f020216u, 0x1f02021fu, 0x1f020602u, 0x1f020e0au, + 0x1f02160au, 0x1f060a1au, 0x1f061202u, 0x1f0a060au, 0x1f0a1216u, 0x1f0a160au, 0x1f0e0202u, 0x1f0e0616u, + 0x1f0e0e0eu, 0x1f0e1a02u, 0x1f120a06u, 0x1f121206u, 0x1f160202u, 0x1f16020au, 0x1f160a12u, 0x1f1a0e02u) + return tbl[i] +} + +// ksigns_iq2xs without the table: bit 7 = the even parity of the 7-bit index. +def private ksign7m(v : uint) : uint { + var t = v ^ (v >> 4u) + t = t ^ (t >> 2u) + t = t ^ (t >> 1u) + return v | ((t & 1u) << 7u) +} + +// iq2xxs_grid as low/high word pairs (word 2i = magnitudes 0..3 of u64 entry i, word +// 2i+1 = 4..7); the same all-literal program-scope-constant form as its grid siblings. +def private iq2xxs_gw(i : int) : uint { + let tbl = fixed_array( + 0x08080808, 0x08080808, 0x0808082b, 0x08080808, 0x08081919, 0x08080808, 0x08082b08, 0x08080808, + 0x08082b2b, 0x08080808, 0x08190819, 0x08080808, 0x08191908, 0x08080808, 0x082b0808, 0x08080808, + 0x082b082b, 0x08080808, 0x082b2b08, 0x08080808, 0x082b2b2b, 0x08080808, 0x19080819, 0x08080808, + 0x19081908, 0x08080808, 0x19190808, 0x08080808, 0x19192b08, 0x08080808, 0x192b0819, 0x08080808, + 0x192b1908, 0x08080808, 0x2b080808, 0x08080808, 0x2b08082b, 0x08080808, 0x2b082b2b, 0x08080808, + 0x2b2b082b, 0x08080808, 0x08080819, 0x08080819, 0x08081908, 0x08080819, 0x08190808, 0x08080819, + 0x08191919, 0x08080819, 0x19080808, 0x08080819, 0x2b081908, 0x08080819, 0x2b192b08, 0x08080819, + 0x08080808, 0x0808082b, 0x0808082b, 0x0808082b, 0x082b082b, 0x0808082b, 0x2b08082b, 0x0808082b, + 0x08080819, 0x08081908, 0x08081908, 0x08081908, 0x08190808, 0x08081908, 0x082b0819, 0x08081908, + 0x082b1908, 0x08081908, 0x19080808, 0x08081908, 0x1908082b, 0x08081908, 0x19082b08, 0x08081908, + 0x192b0808, 0x08081908, 0x2b080819, 0x08081908, 0x2b081908, 0x08081908, 0x2b190808, 0x08081908, + 0x2b2b1908, 0x08081908, 0x08080808, 0x08081919, 0x0808082b, 0x08081919, 0x08082b08, 0x08081919, + 0x082b0808, 0x08081919, 0x1908192b, 0x08081919, 0x192b2b19, 0x08081919, 0x2b080808, 0x08081919, + 0x2b190819, 0x08081919, 0x08082b19, 0x0808192b, 0x08190808, 0x0808192b, 0x19080808, 0x0808192b, + 0x2b081908, 0x0808192b, 0x2b2b1908, 0x0808192b, 0x08080808, 0x08082b08, 0x08081919, 0x08082b08, + 0x08082b08, 0x08082b08, 0x08191908, 0x08082b08, 0x082b2b08, 0x08082b08, 0x19080819, 0x08082b08, + 0x19081908, 0x08082b08, 0x19190808, 0x08082b08, 0x1919082b, 0x08082b08, 0x2b082b08, 0x08082b08, + 0x08081908, 0x08082b19, 0x19080808, 0x08082b19, 0x0808082b, 0x08082b2b, 0x08191908, 0x08082b2b, + 0x08080819, 0x08190808, 0x08081908, 0x08190808, 0x08190808, 0x08190808, 0x082b0819, 0x08190808, + 0x19080808, 0x08190808, 0x192b0808, 0x08190808, 0x2b081908, 0x08190808, 0x2b190808, 0x08190808, + 0x2b191919, 0x08190808, 0x08080808, 0x08190819, 0x08082b08, 0x08190819, 0x082b0808, 0x08190819, + 0x19190808, 0x08190819, 0x19192b2b, 0x08190819, 0x2b080808, 0x08190819, 0x082b1908, 0x0819082b, + 0x19081919, 0x0819082b, 0x08080808, 0x08191908, 0x08082b08, 0x08191908, 0x082b0808, 0x08191908, + 0x082b1919, 0x08191908, 0x19082b19, 0x08191908, 0x2b080808, 0x08191908, 0x08192b08, 0x08191919, + 0x192b082b, 0x08191919, 0x08080808, 0x0819192b, 0x0819192b, 0x0819192b, 0x08080819, 0x08192b08, + 0x08081908, 0x08192b08, 0x08190808, 0x08192b08, 0x19080808, 0x08192b08, 0x2b080819, 0x08192b08, + 0x08080808, 0x08192b19, 0x08081919, 0x08192b19, 0x2b2b0808, 0x08192b19, 0x19190819, 0x08192b2b, + 0x08080808, 0x082b0808, 0x0808082b, 0x082b0808, 0x08082b2b, 0x082b0808, 0x19081908, 0x082b0808, + 0x192b0819, 0x082b0808, 0x2b080808, 0x082b0808, 0x2b08082b, 0x082b0808, 0x082b2b19, 0x082b0819, + 0x19082b08, 0x082b0819, 0x08080808, 0x082b082b, 0x0808082b, 0x082b082b, 0x08080819, 0x082b1908, + 0x08081908, 0x082b1908, 0x08190808, 0x082b1908, 0x19080808, 0x082b1908, 0x1919192b, 0x082b1908, + 0x08080808, 0x082b1919, 0x19080819, 0x082b1919, 0x192b1908, 0x082b1919, 0x2b190808, 0x082b192b, + 0x08082b08, 0x082b2b08, 0x082b0808, 0x082b2b08, 0x2b191908, 0x082b2b08, 0x19081908, 0x082b2b2b, + 0x08080819, 0x19080808, 0x08081908, 0x19080808, 0x08190808, 0x19080808, 0x08192b08, 0x19080808, + 0x082b0819, 0x19080808, 0x082b1908, 0x19080808, 0x19080808, 0x19080808, 0x19082b08, 0x19080808, + 0x1919192b, 0x19080808, 0x192b0808, 0x19080808, 0x2b080819, 0x19080808, 0x2b081908, 0x19080808, + 0x2b190808, 0x19080808, 0x08080808, 0x19080819, 0x082b0808, 0x19080819, 0x192b0819, 0x19080819, + 0x2b080808, 0x19080819, 0x2b081919, 0x19080819, 0x08080819, 0x1908082b, 0x08190808, 0x1908082b, + 0x19082b08, 0x1908082b, 0x1919192b, 0x1908082b, 0x192b2b08, 0x1908082b, 0x08080808, 0x19081908, + 0x08082b08, 0x19081908, 0x082b0808, 0x19081908, 0x2b080808, 0x19081908, 0x2b192b19, 0x19081908, + 0x0819082b, 0x19081919, 0x082b1908, 0x19081919, 0x08080808, 0x1908192b, 0x08080819, 0x19082b08, + 0x08081908, 0x19082b08, 0x08190808, 0x19082b08, 0x19080808, 0x19082b08, 0x19081919, 0x19082b08, + 0x08080808, 0x19082b19, 0x19192b08, 0x19082b19, 0x192b0819, 0x19082b19, 0x2b08082b, 0x19082b19, + 0x19081919, 0x19082b2b, 0x2b190808, 0x19082b2b, 0x08080808, 0x19190808, 0x08082b08, 0x19190808, + 0x08190819, 0x19190808, 0x08192b19, 0x19190808, 0x082b0808, 0x19190808, 0x2b080808, 0x19190808, + 0x2b082b08, 0x19190808, 0x08081908, 0x19190819, 0x1908082b, 0x19190819, 0x2b2b1908, 0x19190819, + 0x2b190819, 0x1919082b, 0x2b190808, 0x19191908, 0x2b19082b, 0x19191908, 0x08082b2b, 0x19191919, + 0x08080819, 0x1919192b, 0x19191908, 0x1919192b, 0x08080808, 0x19192b08, 0x08190819, 0x19192b08, + 0x08192b19, 0x19192b08, 0x192b1908, 0x19192b08, 0x19080808, 0x19192b19, 0x08082b08, 0x19192b2b, + 0x08081908, 0x192b0808, 0x08190808, 0x192b0808, 0x19080808, 0x192b0808, 0x192b2b08, 0x192b0808, + 0x08080808, 0x192b0819, 0x19191919, 0x192b0819, 0x08192b08, 0x192b082b, 0x192b0808, 0x192b082b, + 0x08080808, 0x192b1908, 0x08081919, 0x192b1908, 0x08190808, 0x192b1919, 0x0819082b, 0x192b1919, + 0x2b081908, 0x192b1919, 0x1908082b, 0x192b2b08, 0x08080808, 0x2b080808, 0x0808082b, 0x2b080808, + 0x08082b2b, 0x2b080808, 0x19080819, 0x2b080808, 0x2b08082b, 0x2b080808, 0x08081908, 0x2b080819, + 0x08192b08, 0x2b080819, 0x19080808, 0x2b080819, 0x08190819, 0x2b08082b, 0x08080819, 0x2b081908, + 0x08081908, 0x2b081908, 0x08190808, 0x2b081908, 0x08191919, 0x2b081908, 0x19080808, 0x2b081908, + 0x192b0808, 0x2b081908, 0x08080808, 0x2b081919, 0x1908192b, 0x2b081919, 0x2b191908, 0x2b081919, + 0x08082b19, 0x2b08192b, 0x19080808, 0x2b08192b, 0x192b0808, 0x2b08192b, 0x0808082b, 0x2b082b08, + 0x08081908, 0x2b082b19, 0x08190819, 0x2b082b2b, 0x08081908, 0x2b190808, 0x08190808, 0x2b190808, + 0x082b1908, 0x2b190808, 0x19080808, 0x2b190808, 0x2b2b0819, 0x2b190808, 0x0819192b, 0x2b190819, + 0x2b080808, 0x2b190819, 0x19081919, 0x2b19082b, 0x08080808, 0x2b191908, 0x082b082b, 0x2b191908, + 0x19081908, 0x2b191908, 0x19190819, 0x2b191919, 0x2b080819, 0x2b192b08, 0x082b0808, 0x2b192b19, + 0x0808082b, 0x2b2b0808, 0x19190808, 0x2b2b0808, 0x2b081919, 0x2b2b0808, 0x08082b19, 0x2b2b0819, + 0x08080808, 0x2b2b082b, 0x08192b08, 0x2b2b1908, 0x19190808, 0x2b2b2b08, 0x08081908, 0x2b2b2b19) + return tbl[i] +} + +// iq2xs_grid as low/high word pairs (word 2i = magnitudes 0..3 of u64 entry i, word +// 2i+1 = 4..7); the same all-literal program-scope-constant form as iq2s_gw below. +def private iq2xs_gw(i : int) : uint { // nolint:STYLE038 - a 1024-word data table, not splittable + let tbl = fixed_array( + 0x08080808, 0x08080808, 0x0808082b, 0x08080808, 0x08081919, 0x08080808, 0x08082b08, 0x08080808, + 0x08082b2b, 0x08080808, 0x08190819, 0x08080808, 0x08191908, 0x08080808, 0x0819192b, 0x08080808, + 0x08192b19, 0x08080808, 0x082b0808, 0x08080808, 0x082b082b, 0x08080808, 0x082b1919, 0x08080808, + 0x082b2b08, 0x08080808, 0x19080819, 0x08080808, 0x19081908, 0x08080808, 0x1908192b, 0x08080808, + 0x19082b19, 0x08080808, 0x19190808, 0x08080808, 0x1919082b, 0x08080808, 0x19191919, 0x08080808, + 0x19192b08, 0x08080808, 0x192b0819, 0x08080808, 0x192b1908, 0x08080808, 0x2b080808, 0x08080808, + 0x2b08082b, 0x08080808, 0x2b081919, 0x08080808, 0x2b082b08, 0x08080808, 0x2b190819, 0x08080808, + 0x2b191908, 0x08080808, 0x2b192b19, 0x08080808, 0x2b2b0808, 0x08080808, 0x08080819, 0x08080819, + 0x08081908, 0x08080819, 0x0808192b, 0x08080819, 0x08082b19, 0x08080819, 0x08190808, 0x08080819, + 0x0819082b, 0x08080819, 0x08191919, 0x08080819, 0x08192b08, 0x08080819, 0x08192b2b, 0x08080819, + 0x082b0819, 0x08080819, 0x082b1908, 0x08080819, 0x19080808, 0x08080819, 0x1908082b, 0x08080819, + 0x19081919, 0x08080819, 0x19082b08, 0x08080819, 0x19190819, 0x08080819, 0x19191908, 0x08080819, + 0x192b0808, 0x08080819, 0x192b2b08, 0x08080819, 0x2b080819, 0x08080819, 0x2b081908, 0x08080819, + 0x2b190808, 0x08080819, 0x08080808, 0x0808082b, 0x0808082b, 0x0808082b, 0x08081919, 0x0808082b, + 0x08082b08, 0x0808082b, 0x08190819, 0x0808082b, 0x08191908, 0x0808082b, 0x082b0808, 0x0808082b, + 0x19080819, 0x0808082b, 0x19081908, 0x0808082b, 0x19190808, 0x0808082b, 0x19191919, 0x0808082b, + 0x2b080808, 0x0808082b, 0x2b082b2b, 0x0808082b, 0x08080819, 0x08081908, 0x08081908, 0x08081908, + 0x0808192b, 0x08081908, 0x08082b19, 0x08081908, 0x08190808, 0x08081908, 0x0819082b, 0x08081908, + 0x08191919, 0x08081908, 0x08192b08, 0x08081908, 0x082b0819, 0x08081908, 0x082b1908, 0x08081908, + 0x19080808, 0x08081908, 0x1908082b, 0x08081908, 0x19081919, 0x08081908, 0x19082b08, 0x08081908, + 0x19190819, 0x08081908, 0x19191908, 0x08081908, 0x1919192b, 0x08081908, 0x192b0808, 0x08081908, + 0x2b080819, 0x08081908, 0x2b081908, 0x08081908, 0x2b190808, 0x08081908, 0x08080808, 0x08081919, + 0x0808082b, 0x08081919, 0x08081919, 0x08081919, 0x08082b08, 0x08081919, 0x08190819, 0x08081919, + 0x08191908, 0x08081919, 0x082b0808, 0x08081919, 0x19080819, 0x08081919, 0x19081908, 0x08081919, + 0x19190808, 0x08081919, 0x192b0819, 0x08081919, 0x2b080808, 0x08081919, 0x08080819, 0x0808192b, + 0x08081908, 0x0808192b, 0x08190808, 0x0808192b, 0x082b192b, 0x0808192b, 0x19080808, 0x0808192b, + 0x1908082b, 0x0808192b, 0x2b081908, 0x0808192b, 0x08080808, 0x08082b08, 0x0808082b, 0x08082b08, + 0x08081919, 0x08082b08, 0x08082b08, 0x08082b08, 0x08082b2b, 0x08082b08, 0x08190819, 0x08082b08, + 0x08191908, 0x08082b08, 0x082b0808, 0x08082b08, 0x082b1919, 0x08082b08, 0x19080819, 0x08082b08, + 0x19081908, 0x08082b08, 0x19190808, 0x08082b08, 0x19192b08, 0x08082b08, 0x2b080808, 0x08082b08, + 0x2b2b0808, 0x08082b08, 0x2b2b2b2b, 0x08082b08, 0x08080819, 0x08082b19, 0x08081908, 0x08082b19, + 0x08190808, 0x08082b19, 0x19080808, 0x08082b19, 0x2b080819, 0x08082b19, 0x2b082b19, 0x08082b19, + 0x08080808, 0x08082b2b, 0x082b0808, 0x08082b2b, 0x082b2b08, 0x08082b2b, 0x2b19192b, 0x08082b2b, + 0x2b2b0808, 0x08082b2b, 0x08080819, 0x08190808, 0x08081908, 0x08190808, 0x0808192b, 0x08190808, + 0x08082b19, 0x08190808, 0x08190808, 0x08190808, 0x0819082b, 0x08190808, 0x08191919, 0x08190808, + 0x08192b08, 0x08190808, 0x082b0819, 0x08190808, 0x082b1908, 0x08190808, 0x19080808, 0x08190808, + 0x1908082b, 0x08190808, 0x19081919, 0x08190808, 0x19082b08, 0x08190808, 0x19190819, 0x08190808, + 0x19191908, 0x08190808, 0x192b0808, 0x08190808, 0x192b2b2b, 0x08190808, 0x2b080819, 0x08190808, + 0x2b081908, 0x08190808, 0x2b190808, 0x08190808, 0x08080808, 0x08190819, 0x0808082b, 0x08190819, + 0x08081919, 0x08190819, 0x08082b08, 0x08190819, 0x08190819, 0x08190819, 0x08191908, 0x08190819, + 0x082b0808, 0x08190819, 0x19080819, 0x08190819, 0x19081908, 0x08190819, 0x19190808, 0x08190819, + 0x2b080808, 0x08190819, 0x2b191908, 0x08190819, 0x2b19192b, 0x08190819, 0x08080819, 0x0819082b, + 0x08081908, 0x0819082b, 0x0808192b, 0x0819082b, 0x08190808, 0x0819082b, 0x19080808, 0x0819082b, + 0x192b0808, 0x0819082b, 0x08080808, 0x08191908, 0x0808082b, 0x08191908, 0x08081919, 0x08191908, + 0x08082b08, 0x08191908, 0x08190819, 0x08191908, 0x08191908, 0x08191908, 0x082b0808, 0x08191908, + 0x19080819, 0x08191908, 0x19081908, 0x08191908, 0x19082b19, 0x08191908, 0x19190808, 0x08191908, + 0x192b1908, 0x08191908, 0x2b080808, 0x08191908, 0x08080819, 0x08191919, 0x08081908, 0x08191919, + 0x08190808, 0x08191919, 0x19080808, 0x08191919, 0x08080808, 0x0819192b, 0x08191908, 0x0819192b, + 0x19082b19, 0x0819192b, 0x08080819, 0x08192b08, 0x08081908, 0x08192b08, 0x08190808, 0x08192b08, + 0x0819082b, 0x08192b08, 0x19080808, 0x08192b08, 0x19191908, 0x08192b08, 0x2b08192b, 0x08192b08, + 0x08080808, 0x08192b19, 0x08081919, 0x08192b19, 0x192b192b, 0x08192b19, 0x19190819, 0x08192b2b, + 0x2b2b2b19, 0x08192b2b, 0x08080808, 0x082b0808, 0x0808082b, 0x082b0808, 0x08081919, 0x082b0808, + 0x08082b08, 0x082b0808, 0x08082b2b, 0x082b0808, 0x08190819, 0x082b0808, 0x08191908, 0x082b0808, + 0x082b0808, 0x082b0808, 0x19080819, 0x082b0808, 0x19081908, 0x082b0808, 0x19190808, 0x082b0808, + 0x2b080808, 0x082b0808, 0x2b2b0808, 0x082b0808, 0x08080819, 0x082b0819, 0x08081908, 0x082b0819, + 0x08190808, 0x082b0819, 0x19080808, 0x082b0819, 0x19082b08, 0x082b0819, 0x192b1919, 0x082b0819, + 0x08080808, 0x082b082b, 0x082b082b, 0x082b082b, 0x2b080808, 0x082b082b, 0x2b2b2b08, 0x082b082b, + 0x08080819, 0x082b1908, 0x08081908, 0x082b1908, 0x08190808, 0x082b1908, 0x082b2b19, 0x082b1908, + 0x19080808, 0x082b1908, 0x08080808, 0x082b1919, 0x19080819, 0x082b1919, 0x1919082b, 0x082b1919, + 0x2b192b19, 0x082b1919, 0x08080819, 0x082b192b, 0x08192b2b, 0x082b192b, 0x2b2b192b, 0x082b192b, + 0x08080808, 0x082b2b08, 0x08082b08, 0x082b2b08, 0x08082b2b, 0x082b2b08, 0x082b0808, 0x082b2b08, + 0x19191919, 0x082b2b08, 0x2b082b08, 0x082b2b08, 0x2b2b082b, 0x082b2b08, 0x192b2b08, 0x082b2b19, + 0x2b190808, 0x082b2b19, 0x08082b08, 0x082b2b2b, 0x082b0808, 0x082b2b2b, 0x2b08082b, 0x082b2b2b, + 0x2b082b08, 0x082b2b2b, 0x2b082b2b, 0x082b2b2b, 0x08080819, 0x19080808, 0x08081908, 0x19080808, + 0x0808192b, 0x19080808, 0x08082b19, 0x19080808, 0x08190808, 0x19080808, 0x0819082b, 0x19080808, + 0x08191919, 0x19080808, 0x08192b08, 0x19080808, 0x082b0819, 0x19080808, 0x082b1908, 0x19080808, + 0x19080808, 0x19080808, 0x1908082b, 0x19080808, 0x19081919, 0x19080808, 0x19082b08, 0x19080808, + 0x19082b2b, 0x19080808, 0x19190819, 0x19080808, 0x19191908, 0x19080808, 0x192b0808, 0x19080808, + 0x192b1919, 0x19080808, 0x2b080819, 0x19080808, 0x2b081908, 0x19080808, 0x2b190808, 0x19080808, + 0x08080808, 0x19080819, 0x0808082b, 0x19080819, 0x08081919, 0x19080819, 0x08082b08, 0x19080819, + 0x08190819, 0x19080819, 0x08191908, 0x19080819, 0x082b0808, 0x19080819, 0x19080819, 0x19080819, + 0x19081908, 0x19080819, 0x19190808, 0x19080819, 0x2b080808, 0x19080819, 0x2b081919, 0x19080819, + 0x2b2b082b, 0x19080819, 0x08080819, 0x1908082b, 0x08081908, 0x1908082b, 0x08190808, 0x1908082b, + 0x0819082b, 0x1908082b, 0x082b2b19, 0x1908082b, 0x19080808, 0x1908082b, 0x08080808, 0x19081908, + 0x0808082b, 0x19081908, 0x08081919, 0x19081908, 0x08082b08, 0x19081908, 0x08190819, 0x19081908, + 0x08191908, 0x19081908, 0x08192b19, 0x19081908, 0x082b0808, 0x19081908, 0x19080819, 0x19081908, + 0x19081908, 0x19081908, 0x19190808, 0x19081908, 0x2b080808, 0x19081908, 0x2b191908, 0x19081908, + 0x08080819, 0x19081919, 0x08081908, 0x19081919, 0x08190808, 0x19081919, 0x082b1908, 0x19081919, + 0x19080808, 0x19081919, 0x2b192b2b, 0x19081919, 0x08080808, 0x1908192b, 0x08082b2b, 0x1908192b, + 0x19081908, 0x1908192b, 0x19190808, 0x1908192b, 0x08080819, 0x19082b08, 0x08081908, 0x19082b08, + 0x08190808, 0x19082b08, 0x19080808, 0x19082b08, 0x19081919, 0x19082b08, 0x19191908, 0x19082b08, + 0x192b082b, 0x19082b08, 0x08080808, 0x19082b19, 0x08190819, 0x19082b19, 0x19081908, 0x19082b19, + 0x19190808, 0x19082b19, 0x192b2b19, 0x19082b19, 0x08081908, 0x19082b2b, 0x08080808, 0x19190808, + 0x0808082b, 0x19190808, 0x08081919, 0x19190808, 0x08082b08, 0x19190808, 0x08190819, 0x19190808, + 0x08191908, 0x19190808, 0x082b0808, 0x19190808, 0x082b2b08, 0x19190808, 0x19080819, 0x19190808, + 0x19081908, 0x19190808, 0x19190808, 0x19190808, 0x2b080808, 0x19190808, 0x08080819, 0x19190819, + 0x08081908, 0x19190819, 0x08190808, 0x19190819, 0x08191919, 0x19190819, 0x19080808, 0x19190819, + 0x1908082b, 0x19190819, 0x08080808, 0x1919082b, 0x19081908, 0x1919082b, 0x2b2b2b2b, 0x1919082b, + 0x08080819, 0x19191908, 0x08081908, 0x19191908, 0x08190808, 0x19191908, 0x082b0819, 0x19191908, + 0x19080808, 0x19191908, 0x192b0808, 0x19191908, 0x2b080819, 0x19191908, 0x2b2b0819, 0x19191908, + 0x08080808, 0x19191919, 0x08082b08, 0x19191919, 0x2b080808, 0x19191919, 0x2b082b08, 0x19191919, + 0x082b0819, 0x1919192b, 0x192b2b08, 0x1919192b, 0x2b2b0819, 0x1919192b, 0x08080808, 0x19192b08, + 0x08191908, 0x19192b08, 0x19080819, 0x19192b08, 0x19190808, 0x19192b08, 0x2b192b19, 0x19192b08, + 0x08192b2b, 0x19192b19, 0x19080808, 0x19192b19, 0x1908082b, 0x19192b19, 0x2b081919, 0x19192b2b, + 0x08080819, 0x192b0808, 0x08081908, 0x192b0808, 0x08190808, 0x192b0808, 0x19080808, 0x192b0808, + 0x19191908, 0x192b0808, 0x192b082b, 0x192b0808, 0x2b08192b, 0x192b0808, 0x2b2b2b19, 0x192b0808, + 0x08080808, 0x192b0819, 0x082b1908, 0x192b082b, 0x19082b2b, 0x192b082b, 0x2b19082b, 0x192b082b, + 0x08080808, 0x192b1908, 0x0819192b, 0x192b1908, 0x08190808, 0x192b1919, 0x19080808, 0x192b1919, + 0x19081919, 0x192b1919, 0x2b2b1908, 0x192b1919, 0x08080819, 0x192b2b08, 0x192b2b2b, 0x192b2b08, + 0x082b1919, 0x192b2b19, 0x0808192b, 0x192b2b2b, 0x19191908, 0x192b2b2b, 0x192b082b, 0x192b2b2b, + 0x08080808, 0x2b080808, 0x0808082b, 0x2b080808, 0x08081919, 0x2b080808, 0x08082b08, 0x2b080808, + 0x08190819, 0x2b080808, 0x08191908, 0x2b080808, 0x082b0808, 0x2b080808, 0x082b2b2b, 0x2b080808, + 0x19080819, 0x2b080808, 0x19081908, 0x2b080808, 0x19190808, 0x2b080808, 0x2b080808, 0x2b080808, + 0x2b08082b, 0x2b080808, 0x2b2b2b08, 0x2b080808, 0x2b2b2b2b, 0x2b080808, 0x08080819, 0x2b080819, + 0x08081908, 0x2b080819, 0x0808192b, 0x2b080819, 0x08190808, 0x2b080819, 0x19080808, 0x2b080819, + 0x19190819, 0x2b080819, 0x19192b19, 0x2b080819, 0x08080808, 0x2b08082b, 0x082b0808, 0x2b08082b, + 0x2b080808, 0x2b08082b, 0x2b08082b, 0x2b08082b, 0x2b2b0808, 0x2b08082b, 0x2b2b2b08, 0x2b08082b, + 0x08080819, 0x2b081908, 0x08081908, 0x2b081908, 0x08190808, 0x2b081908, 0x0819082b, 0x2b081908, + 0x08191919, 0x2b081908, 0x19080808, 0x2b081908, 0x192b0808, 0x2b081908, 0x2b082b19, 0x2b081908, + 0x08080808, 0x2b081919, 0x19081908, 0x2b081919, 0x2b2b1919, 0x2b081919, 0x08192b08, 0x2b08192b, + 0x192b2b2b, 0x2b08192b, 0x08080808, 0x2b082b08, 0x08082b08, 0x2b082b08, 0x082b1919, 0x2b082b08, + 0x19192b2b, 0x2b082b08, 0x2b080808, 0x2b082b08, 0x2b08082b, 0x2b082b08, 0x2b2b2b08, 0x2b082b08, + 0x0808192b, 0x2b082b19, 0x082b082b, 0x2b082b2b, 0x2b080808, 0x2b082b2b, 0x2b082b08, 0x2b082b2b, + 0x2b19192b, 0x2b082b2b, 0x2b2b2b08, 0x2b082b2b, 0x08080819, 0x2b190808, 0x08081908, 0x2b190808, + 0x08190808, 0x2b190808, 0x19080808, 0x2b190808, 0x1919192b, 0x2b190808, 0x2b081908, 0x2b190808, + 0x08080808, 0x2b190819, 0x082b082b, 0x2b190819, 0x192b1908, 0x2b190819, 0x1919192b, 0x2b19082b, + 0x2b082b19, 0x2b19082b, 0x08080808, 0x2b191908, 0x08081919, 0x2b191908, 0x19081908, 0x2b191908, + 0x19190808, 0x2b191908, 0x19192b08, 0x2b191908, 0x082b2b19, 0x2b191919, 0x2b190808, 0x2b191919, + 0x2b19082b, 0x2b191919, 0x19080819, 0x2b19192b, 0x19190819, 0x2b192b08, 0x2b2b192b, 0x2b192b08, + 0x19082b19, 0x2b192b19, 0x08191919, 0x2b192b2b, 0x192b0808, 0x2b192b2b, 0x08080808, 0x2b2b0808, + 0x0808082b, 0x2b2b0808, 0x08082b08, 0x2b2b0808, 0x08082b2b, 0x2b2b0808, 0x082b0808, 0x2b2b0808, + 0x082b2b2b, 0x2b2b0808, 0x2b2b0808, 0x2b2b0808, 0x19190819, 0x2b2b0819, 0x19192b19, 0x2b2b0819, + 0x2b2b192b, 0x2b2b0819, 0x08080808, 0x2b2b082b, 0x0808082b, 0x2b2b082b, 0x08082b08, 0x2b2b082b, + 0x082b2b2b, 0x2b2b082b, 0x2b080808, 0x2b2b082b, 0x2b2b0808, 0x2b2b082b, 0x19080808, 0x2b2b1908, + 0x2b191919, 0x2b2b1908, 0x192b1919, 0x2b2b192b, 0x2b192b08, 0x2b2b192b, 0x08082b2b, 0x2b2b2b08, + 0x082b0808, 0x2b2b2b08, 0x082b082b, 0x2b2b2b08, 0x082b2b08, 0x2b2b2b08, 0x2b2b0808, 0x2b2b2b08, + 0x2b2b2b08, 0x2b2b2b08, 0x08081908, 0x2b2b2b19, 0x2b081908, 0x2b2b2b19, 0x2b08192b, 0x2b2b2b19, + 0x082b2b08, 0x2b2b2b2b, 0x082b2b2b, 0x2b2b2b2b, 0x2b190819, 0x2b2b2b2b, 0x2b2b2b2b, 0x2b2b2b2b) + return tbl[i] +} + +// iq2s_grid as low/high word pairs (word 2i = magnitudes 0..3 of u64 entry i, word 2i+1 +// = 4..7); an all-literal fixed_array local, so it lowers to a program-scope constant table - +// every kernel reads it DIRECT (an f4 slab of 1024 entries would be 32 KB, past the tg budget). +def private iq2s_gw(i : int) : uint { // nolint:STYLE038 - a 2048-word data table, not splittable + let tbl = fixed_array( + 0x08080808, 0x08080808, 0x0808082b, 0x08080808, 0x08081919, 0x08080808, 0x08082b08, 0x08080808, + 0x08082b2b, 0x08080808, 0x08190819, 0x08080808, 0x08191908, 0x08080808, 0x0819192b, 0x08080808, + 0x08192b19, 0x08080808, 0x082b0808, 0x08080808, 0x082b082b, 0x08080808, 0x082b1919, 0x08080808, + 0x082b2b08, 0x08080808, 0x19080819, 0x08080808, 0x19081908, 0x08080808, 0x1908192b, 0x08080808, + 0x19082b19, 0x08080808, 0x19190808, 0x08080808, 0x1919082b, 0x08080808, 0x19191919, 0x08080808, + 0x19192b08, 0x08080808, 0x192b0819, 0x08080808, 0x192b1908, 0x08080808, 0x192b192b, 0x08080808, + 0x192b2b19, 0x08080808, 0x2b080808, 0x08080808, 0x2b08082b, 0x08080808, 0x2b081919, 0x08080808, + 0x2b082b08, 0x08080808, 0x2b190819, 0x08080808, 0x2b191908, 0x08080808, 0x2b2b0808, 0x08080808, + 0x2b2b1919, 0x08080808, 0x2b2b2b2b, 0x08080808, 0x08080819, 0x08080819, 0x08081908, 0x08080819, + 0x0808192b, 0x08080819, 0x08082b19, 0x08080819, 0x08190808, 0x08080819, 0x0819082b, 0x08080819, + 0x08191919, 0x08080819, 0x08192b08, 0x08080819, 0x082b0819, 0x08080819, 0x082b1908, 0x08080819, + 0x19080808, 0x08080819, 0x1908082b, 0x08080819, 0x19081919, 0x08080819, 0x19082b08, 0x08080819, + 0x19190819, 0x08080819, 0x19191908, 0x08080819, 0x1919192b, 0x08080819, 0x19192b19, 0x08080819, + 0x192b0808, 0x08080819, 0x192b1919, 0x08080819, 0x192b2b08, 0x08080819, 0x2b080819, 0x08080819, + 0x2b081908, 0x08080819, 0x2b190808, 0x08080819, 0x2b19082b, 0x08080819, 0x2b191919, 0x08080819, + 0x2b2b0819, 0x08080819, 0x2b2b1908, 0x08080819, 0x08080808, 0x0808082b, 0x0808082b, 0x0808082b, + 0x08081919, 0x0808082b, 0x08082b08, 0x0808082b, 0x08190819, 0x0808082b, 0x08191908, 0x0808082b, + 0x082b0808, 0x0808082b, 0x082b2b2b, 0x0808082b, 0x19080819, 0x0808082b, 0x19081908, 0x0808082b, + 0x1908192b, 0x0808082b, 0x19082b19, 0x0808082b, 0x19190808, 0x0808082b, 0x19191919, 0x0808082b, + 0x2b080808, 0x0808082b, 0x2b081919, 0x0808082b, 0x2b082b2b, 0x0808082b, 0x2b191908, 0x0808082b, + 0x2b2b082b, 0x0808082b, 0x08080819, 0x08081908, 0x08081908, 0x08081908, 0x0808192b, 0x08081908, + 0x08082b19, 0x08081908, 0x08190808, 0x08081908, 0x0819082b, 0x08081908, 0x08191919, 0x08081908, + 0x08192b08, 0x08081908, 0x082b0819, 0x08081908, 0x082b1908, 0x08081908, 0x082b192b, 0x08081908, + 0x082b2b19, 0x08081908, 0x19080808, 0x08081908, 0x1908082b, 0x08081908, 0x19081919, 0x08081908, + 0x19082b08, 0x08081908, 0x19082b2b, 0x08081908, 0x19190819, 0x08081908, 0x19191908, 0x08081908, + 0x1919192b, 0x08081908, 0x19192b19, 0x08081908, 0x192b0808, 0x08081908, 0x192b082b, 0x08081908, + 0x192b1919, 0x08081908, 0x2b080819, 0x08081908, 0x2b081908, 0x08081908, 0x2b08192b, 0x08081908, + 0x2b082b19, 0x08081908, 0x2b190808, 0x08081908, 0x2b191919, 0x08081908, 0x2b192b08, 0x08081908, + 0x2b2b0819, 0x08081908, 0x2b2b1908, 0x08081908, 0x08080808, 0x08081919, 0x0808082b, 0x08081919, + 0x08081919, 0x08081919, 0x08082b08, 0x08081919, 0x08082b2b, 0x08081919, 0x08190819, 0x08081919, + 0x08191908, 0x08081919, 0x0819192b, 0x08081919, 0x08192b19, 0x08081919, 0x082b0808, 0x08081919, + 0x082b1919, 0x08081919, 0x082b2b08, 0x08081919, 0x19080819, 0x08081919, 0x19081908, 0x08081919, + 0x1908192b, 0x08081919, 0x19082b19, 0x08081919, 0x19190808, 0x08081919, 0x1919082b, 0x08081919, + 0x19191919, 0x08081919, 0x19192b08, 0x08081919, 0x192b0819, 0x08081919, 0x192b1908, 0x08081919, + 0x2b080808, 0x08081919, 0x2b08082b, 0x08081919, 0x2b081919, 0x08081919, 0x2b082b08, 0x08081919, + 0x2b190819, 0x08081919, 0x2b191908, 0x08081919, 0x2b2b0808, 0x08081919, 0x08080819, 0x0808192b, + 0x08081908, 0x0808192b, 0x0808192b, 0x0808192b, 0x08082b19, 0x0808192b, 0x08190808, 0x0808192b, + 0x08191919, 0x0808192b, 0x19080808, 0x0808192b, 0x19081919, 0x0808192b, 0x19082b08, 0x0808192b, + 0x19190819, 0x0808192b, 0x19191908, 0x0808192b, 0x192b0808, 0x0808192b, 0x2b080819, 0x0808192b, + 0x2b081908, 0x0808192b, 0x2b190808, 0x0808192b, 0x08080808, 0x08082b08, 0x0808082b, 0x08082b08, + 0x08081919, 0x08082b08, 0x08082b08, 0x08082b08, 0x08190819, 0x08082b08, 0x08191908, 0x08082b08, + 0x0819192b, 0x08082b08, 0x08192b19, 0x08082b08, 0x082b0808, 0x08082b08, 0x082b1919, 0x08082b08, + 0x082b2b2b, 0x08082b08, 0x19080819, 0x08082b08, 0x19081908, 0x08082b08, 0x1908192b, 0x08082b08, + 0x19082b19, 0x08082b08, 0x19190808, 0x08082b08, 0x1919082b, 0x08082b08, 0x19191919, 0x08082b08, + 0x19192b08, 0x08082b08, 0x192b0819, 0x08082b08, 0x192b1908, 0x08082b08, 0x2b080808, 0x08082b08, + 0x2b081919, 0x08082b08, 0x2b191908, 0x08082b08, 0x2b2b2b2b, 0x08082b08, 0x08080819, 0x08082b19, + 0x08081908, 0x08082b19, 0x08190808, 0x08082b19, 0x0819082b, 0x08082b19, 0x08191919, 0x08082b19, + 0x08192b08, 0x08082b19, 0x082b0819, 0x08082b19, 0x19080808, 0x08082b19, 0x19081919, 0x08082b19, + 0x19082b08, 0x08082b19, 0x19190819, 0x08082b19, 0x19191908, 0x08082b19, 0x192b0808, 0x08082b19, + 0x2b080819, 0x08082b19, 0x2b190808, 0x08082b19, 0x08080808, 0x08082b2b, 0x08190819, 0x08082b2b, + 0x08191908, 0x08082b2b, 0x082b082b, 0x08082b2b, 0x082b2b08, 0x08082b2b, 0x082b2b2b, 0x08082b2b, + 0x19190808, 0x08082b2b, 0x2b192b19, 0x08082b2b, 0x08080819, 0x08190808, 0x08081908, 0x08190808, + 0x0808192b, 0x08190808, 0x08082b19, 0x08190808, 0x08190808, 0x08190808, 0x0819082b, 0x08190808, + 0x08191919, 0x08190808, 0x08192b08, 0x08190808, 0x082b0819, 0x08190808, 0x082b1908, 0x08190808, + 0x082b192b, 0x08190808, 0x19080808, 0x08190808, 0x1908082b, 0x08190808, 0x19081919, 0x08190808, + 0x19082b08, 0x08190808, 0x19190819, 0x08190808, 0x19191908, 0x08190808, 0x1919192b, 0x08190808, + 0x19192b19, 0x08190808, 0x192b0808, 0x08190808, 0x192b082b, 0x08190808, 0x192b1919, 0x08190808, + 0x192b2b08, 0x08190808, 0x2b080819, 0x08190808, 0x2b081908, 0x08190808, 0x2b08192b, 0x08190808, + 0x2b190808, 0x08190808, 0x2b191919, 0x08190808, 0x2b192b08, 0x08190808, 0x2b2b0819, 0x08190808, + 0x2b2b1908, 0x08190808, 0x08080808, 0x08190819, 0x0808082b, 0x08190819, 0x08081919, 0x08190819, + 0x08082b08, 0x08190819, 0x08082b2b, 0x08190819, 0x08190819, 0x08190819, 0x08191908, 0x08190819, + 0x0819192b, 0x08190819, 0x08192b19, 0x08190819, 0x082b0808, 0x08190819, 0x082b082b, 0x08190819, + 0x082b1919, 0x08190819, 0x082b2b08, 0x08190819, 0x19080819, 0x08190819, 0x19081908, 0x08190819, + 0x1908192b, 0x08190819, 0x19082b19, 0x08190819, 0x19190808, 0x08190819, 0x1919082b, 0x08190819, + 0x19191919, 0x08190819, 0x19192b08, 0x08190819, 0x192b0819, 0x08190819, 0x192b1908, 0x08190819, + 0x2b080808, 0x08190819, 0x2b08082b, 0x08190819, 0x2b081919, 0x08190819, 0x2b082b08, 0x08190819, + 0x2b190819, 0x08190819, 0x2b191908, 0x08190819, 0x08080819, 0x0819082b, 0x08081908, 0x0819082b, + 0x08082b19, 0x0819082b, 0x08190808, 0x0819082b, 0x08191919, 0x0819082b, 0x082b0819, 0x0819082b, + 0x082b1908, 0x0819082b, 0x19080808, 0x0819082b, 0x19081919, 0x0819082b, 0x19190819, 0x0819082b, + 0x19191908, 0x0819082b, 0x2b080819, 0x0819082b, 0x2b081908, 0x0819082b, 0x2b190808, 0x0819082b, + 0x08080808, 0x08191908, 0x0808082b, 0x08191908, 0x08081919, 0x08191908, 0x08082b08, 0x08191908, + 0x08190819, 0x08191908, 0x08191908, 0x08191908, 0x0819192b, 0x08191908, 0x08192b19, 0x08191908, + 0x082b0808, 0x08191908, 0x082b1919, 0x08191908, 0x082b2b08, 0x08191908, 0x19080819, 0x08191908, + 0x19081908, 0x08191908, 0x1908192b, 0x08191908, 0x19082b19, 0x08191908, 0x19190808, 0x08191908, + 0x1919082b, 0x08191908, 0x19191919, 0x08191908, 0x19192b08, 0x08191908, 0x192b0819, 0x08191908, + 0x192b1908, 0x08191908, 0x2b080808, 0x08191908, 0x2b08082b, 0x08191908, 0x2b081919, 0x08191908, + 0x2b082b08, 0x08191908, 0x2b190819, 0x08191908, 0x2b191908, 0x08191908, 0x2b2b0808, 0x08191908, + 0x08080819, 0x08191919, 0x08081908, 0x08191919, 0x0808192b, 0x08191919, 0x08082b19, 0x08191919, + 0x08190808, 0x08191919, 0x0819082b, 0x08191919, 0x08191919, 0x08191919, 0x08192b08, 0x08191919, + 0x082b0819, 0x08191919, 0x082b1908, 0x08191919, 0x19080808, 0x08191919, 0x1908082b, 0x08191919, + 0x19081919, 0x08191919, 0x19082b08, 0x08191919, 0x19190819, 0x08191919, 0x19191908, 0x08191919, + 0x192b0808, 0x08191919, 0x2b080819, 0x08191919, 0x2b081908, 0x08191919, 0x2b190808, 0x08191919, + 0x08080808, 0x0819192b, 0x08081919, 0x0819192b, 0x08082b08, 0x0819192b, 0x08190819, 0x0819192b, + 0x08191908, 0x0819192b, 0x082b0808, 0x0819192b, 0x19080819, 0x0819192b, 0x19081908, 0x0819192b, + 0x19190808, 0x0819192b, 0x2b080808, 0x0819192b, 0x2b2b2b2b, 0x0819192b, 0x08080819, 0x08192b08, + 0x08081908, 0x08192b08, 0x0808192b, 0x08192b08, 0x08082b19, 0x08192b08, 0x08190808, 0x08192b08, + 0x08191919, 0x08192b08, 0x08192b08, 0x08192b08, 0x082b0819, 0x08192b08, 0x19080808, 0x08192b08, + 0x1908082b, 0x08192b08, 0x19081919, 0x08192b08, 0x19082b08, 0x08192b08, 0x19190819, 0x08192b08, + 0x19191908, 0x08192b08, 0x192b0808, 0x08192b08, 0x2b080819, 0x08192b08, 0x2b081908, 0x08192b08, + 0x08080808, 0x08192b19, 0x0808082b, 0x08192b19, 0x08081919, 0x08192b19, 0x08082b08, 0x08192b19, + 0x08190819, 0x08192b19, 0x08191908, 0x08192b19, 0x082b0808, 0x08192b19, 0x19080819, 0x08192b19, + 0x19081908, 0x08192b19, 0x19190808, 0x08192b19, 0x192b2b19, 0x08192b19, 0x2b2b082b, 0x08192b19, + 0x08081908, 0x08192b2b, 0x08190808, 0x08192b2b, 0x19080808, 0x08192b2b, 0x1919192b, 0x08192b2b, + 0x08080808, 0x082b0808, 0x0808082b, 0x082b0808, 0x08081919, 0x082b0808, 0x08082b08, 0x082b0808, + 0x08190819, 0x082b0808, 0x08191908, 0x082b0808, 0x0819192b, 0x082b0808, 0x08192b19, 0x082b0808, + 0x082b0808, 0x082b0808, 0x082b1919, 0x082b0808, 0x082b2b2b, 0x082b0808, 0x19080819, 0x082b0808, + 0x19081908, 0x082b0808, 0x19190808, 0x082b0808, 0x1919082b, 0x082b0808, 0x19191919, 0x082b0808, + 0x192b1908, 0x082b0808, 0x2b080808, 0x082b0808, 0x2b082b2b, 0x082b0808, 0x2b191908, 0x082b0808, + 0x2b2b2b2b, 0x082b0808, 0x08080819, 0x082b0819, 0x08081908, 0x082b0819, 0x08190808, 0x082b0819, + 0x0819082b, 0x082b0819, 0x08191919, 0x082b0819, 0x082b0819, 0x082b0819, 0x19080808, 0x082b0819, + 0x1908082b, 0x082b0819, 0x19081919, 0x082b0819, 0x19190819, 0x082b0819, 0x19191908, 0x082b0819, + 0x192b0808, 0x082b0819, 0x2b080819, 0x082b0819, 0x2b081908, 0x082b0819, 0x2b190808, 0x082b0819, + 0x08080808, 0x082b082b, 0x08082b2b, 0x082b082b, 0x082b082b, 0x082b082b, 0x082b2b08, 0x082b082b, + 0x082b2b2b, 0x082b082b, 0x19081908, 0x082b082b, 0x19190808, 0x082b082b, 0x2b082b08, 0x082b082b, + 0x2b082b2b, 0x082b082b, 0x2b2b2b08, 0x082b082b, 0x08080819, 0x082b1908, 0x08081908, 0x082b1908, + 0x0808192b, 0x082b1908, 0x08082b19, 0x082b1908, 0x08190808, 0x082b1908, 0x08191919, 0x082b1908, + 0x08192b08, 0x082b1908, 0x082b0819, 0x082b1908, 0x082b1908, 0x082b1908, 0x19080808, 0x082b1908, + 0x1908082b, 0x082b1908, 0x19081919, 0x082b1908, 0x19082b08, 0x082b1908, 0x19190819, 0x082b1908, + 0x19191908, 0x082b1908, 0x192b0808, 0x082b1908, 0x2b080819, 0x082b1908, 0x2b081908, 0x082b1908, + 0x2b190808, 0x082b1908, 0x08080808, 0x082b1919, 0x08081919, 0x082b1919, 0x08082b08, 0x082b1919, + 0x08190819, 0x082b1919, 0x08191908, 0x082b1919, 0x082b0808, 0x082b1919, 0x19080819, 0x082b1919, + 0x19081908, 0x082b1919, 0x19190808, 0x082b1919, 0x192b192b, 0x082b1919, 0x2b080808, 0x082b1919, + 0x08080819, 0x082b192b, 0x08081908, 0x082b192b, 0x08190808, 0x082b192b, 0x19080808, 0x082b192b, + 0x19192b19, 0x082b192b, 0x08080808, 0x082b2b08, 0x08081919, 0x082b2b08, 0x08190819, 0x082b2b08, + 0x08191908, 0x082b2b08, 0x19080819, 0x082b2b08, 0x19081908, 0x082b2b08, 0x19190808, 0x082b2b08, + 0x2b082b2b, 0x082b2b08, 0x2b2b2b2b, 0x082b2b08, 0x08080819, 0x082b2b19, 0x08081908, 0x082b2b19, + 0x08190808, 0x082b2b19, 0x2b191919, 0x082b2b19, 0x08082b2b, 0x082b2b2b, 0x082b082b, 0x082b2b2b, + 0x192b1908, 0x082b2b2b, 0x2b082b08, 0x082b2b2b, 0x2b082b2b, 0x082b2b2b, 0x08080819, 0x19080808, + 0x08081908, 0x19080808, 0x0808192b, 0x19080808, 0x08082b19, 0x19080808, 0x08190808, 0x19080808, + 0x0819082b, 0x19080808, 0x08191919, 0x19080808, 0x08192b08, 0x19080808, 0x08192b2b, 0x19080808, + 0x082b0819, 0x19080808, 0x082b1908, 0x19080808, 0x082b192b, 0x19080808, 0x19080808, 0x19080808, + 0x1908082b, 0x19080808, 0x19081919, 0x19080808, 0x19082b08, 0x19080808, 0x19082b2b, 0x19080808, + 0x19190819, 0x19080808, 0x19191908, 0x19080808, 0x1919192b, 0x19080808, 0x19192b19, 0x19080808, + 0x192b0808, 0x19080808, 0x192b082b, 0x19080808, 0x192b1919, 0x19080808, 0x2b080819, 0x19080808, + 0x2b081908, 0x19080808, 0x2b190808, 0x19080808, 0x2b191919, 0x19080808, 0x2b192b08, 0x19080808, + 0x2b2b0819, 0x19080808, 0x2b2b1908, 0x19080808, 0x08080808, 0x19080819, 0x0808082b, 0x19080819, + 0x08081919, 0x19080819, 0x08082b08, 0x19080819, 0x08190819, 0x19080819, 0x08191908, 0x19080819, + 0x0819192b, 0x19080819, 0x08192b19, 0x19080819, 0x082b0808, 0x19080819, 0x082b082b, 0x19080819, + 0x082b1919, 0x19080819, 0x19080819, 0x19080819, 0x19081908, 0x19080819, 0x1908192b, 0x19080819, + 0x19082b19, 0x19080819, 0x19190808, 0x19080819, 0x1919082b, 0x19080819, 0x19191919, 0x19080819, + 0x19192b08, 0x19080819, 0x192b0819, 0x19080819, 0x192b1908, 0x19080819, 0x2b080808, 0x19080819, + 0x2b08082b, 0x19080819, 0x2b081919, 0x19080819, 0x2b082b08, 0x19080819, 0x2b190819, 0x19080819, + 0x2b191908, 0x19080819, 0x2b2b0808, 0x19080819, 0x08080819, 0x1908082b, 0x08081908, 0x1908082b, + 0x08190808, 0x1908082b, 0x0819082b, 0x1908082b, 0x08191919, 0x1908082b, 0x08192b08, 0x1908082b, + 0x082b1908, 0x1908082b, 0x19080808, 0x1908082b, 0x19081919, 0x1908082b, 0x19082b08, 0x1908082b, + 0x19190819, 0x1908082b, 0x19191908, 0x1908082b, 0x192b0808, 0x1908082b, 0x2b080819, 0x1908082b, + 0x2b081908, 0x1908082b, 0x08080808, 0x19081908, 0x0808082b, 0x19081908, 0x08081919, 0x19081908, + 0x08082b08, 0x19081908, 0x08082b2b, 0x19081908, 0x08190819, 0x19081908, 0x08191908, 0x19081908, + 0x0819192b, 0x19081908, 0x08192b19, 0x19081908, 0x082b0808, 0x19081908, 0x082b082b, 0x19081908, + 0x082b1919, 0x19081908, 0x082b2b08, 0x19081908, 0x19080819, 0x19081908, 0x19081908, 0x19081908, + 0x1908192b, 0x19081908, 0x19082b19, 0x19081908, 0x19190808, 0x19081908, 0x1919082b, 0x19081908, + 0x19191919, 0x19081908, 0x19192b08, 0x19081908, 0x192b0819, 0x19081908, 0x192b1908, 0x19081908, + 0x2b080808, 0x19081908, 0x2b08082b, 0x19081908, 0x2b081919, 0x19081908, 0x2b082b08, 0x19081908, + 0x2b190819, 0x19081908, 0x2b191908, 0x19081908, 0x2b2b0808, 0x19081908, 0x08080819, 0x19081919, + 0x08081908, 0x19081919, 0x0808192b, 0x19081919, 0x08082b19, 0x19081919, 0x08190808, 0x19081919, + 0x0819082b, 0x19081919, 0x08191919, 0x19081919, 0x08192b08, 0x19081919, 0x082b0819, 0x19081919, + 0x082b1908, 0x19081919, 0x19080808, 0x19081919, 0x1908082b, 0x19081919, 0x19081919, 0x19081919, + 0x19082b08, 0x19081919, 0x19190819, 0x19081919, 0x19191908, 0x19081919, 0x192b0808, 0x19081919, + 0x192b2b2b, 0x19081919, 0x2b080819, 0x19081919, 0x2b081908, 0x19081919, 0x2b190808, 0x19081919, + 0x08080808, 0x1908192b, 0x0808082b, 0x1908192b, 0x08081919, 0x1908192b, 0x08082b08, 0x1908192b, + 0x08190819, 0x1908192b, 0x08191908, 0x1908192b, 0x082b0808, 0x1908192b, 0x19080819, 0x1908192b, + 0x19081908, 0x1908192b, 0x19190808, 0x1908192b, 0x2b080808, 0x1908192b, 0x2b2b1919, 0x1908192b, + 0x08080819, 0x19082b08, 0x08081908, 0x19082b08, 0x08082b19, 0x19082b08, 0x08190808, 0x19082b08, + 0x0819082b, 0x19082b08, 0x08191919, 0x19082b08, 0x08192b08, 0x19082b08, 0x082b0819, 0x19082b08, + 0x082b1908, 0x19082b08, 0x19080808, 0x19082b08, 0x1908082b, 0x19082b08, 0x19081919, 0x19082b08, + 0x19082b08, 0x19082b08, 0x19190819, 0x19082b08, 0x19191908, 0x19082b08, 0x192b0808, 0x19082b08, + 0x2b081908, 0x19082b08, 0x2b190808, 0x19082b08, 0x08080808, 0x19082b19, 0x0808082b, 0x19082b19, + 0x08081919, 0x19082b19, 0x08082b08, 0x19082b19, 0x08190819, 0x19082b19, 0x08191908, 0x19082b19, + 0x082b0808, 0x19082b19, 0x19080819, 0x19082b19, 0x19081908, 0x19082b19, 0x19190808, 0x19082b19, + 0x2b080808, 0x19082b19, 0x2b19192b, 0x19082b19, 0x08080819, 0x19082b2b, 0x08081908, 0x19082b2b, + 0x08190808, 0x19082b2b, 0x19080808, 0x19082b2b, 0x08080808, 0x19190808, 0x0808082b, 0x19190808, + 0x08081919, 0x19190808, 0x08082b08, 0x19190808, 0x08190819, 0x19190808, 0x08191908, 0x19190808, + 0x0819192b, 0x19190808, 0x08192b19, 0x19190808, 0x082b0808, 0x19190808, 0x082b082b, 0x19190808, + 0x082b1919, 0x19190808, 0x082b2b08, 0x19190808, 0x19080819, 0x19190808, 0x19081908, 0x19190808, + 0x1908192b, 0x19190808, 0x19082b19, 0x19190808, 0x19190808, 0x19190808, 0x1919082b, 0x19190808, + 0x19191919, 0x19190808, 0x19192b08, 0x19190808, 0x192b0819, 0x19190808, 0x192b1908, 0x19190808, + 0x2b080808, 0x19190808, 0x2b08082b, 0x19190808, 0x2b081919, 0x19190808, 0x2b082b08, 0x19190808, + 0x2b190819, 0x19190808, 0x2b191908, 0x19190808, 0x08080819, 0x19190819, 0x08081908, 0x19190819, + 0x0808192b, 0x19190819, 0x08082b19, 0x19190819, 0x08190808, 0x19190819, 0x0819082b, 0x19190819, + 0x08191919, 0x19190819, 0x08192b08, 0x19190819, 0x082b0819, 0x19190819, 0x082b1908, 0x19190819, + 0x19080808, 0x19190819, 0x1908082b, 0x19190819, 0x19081919, 0x19190819, 0x19082b08, 0x19190819, + 0x19190819, 0x19190819, 0x19191908, 0x19190819, 0x192b0808, 0x19190819, 0x2b080819, 0x19190819, + 0x2b081908, 0x19190819, 0x2b190808, 0x19190819, 0x08080808, 0x1919082b, 0x08081919, 0x1919082b, + 0x08082b08, 0x1919082b, 0x08190819, 0x1919082b, 0x08191908, 0x1919082b, 0x082b0808, 0x1919082b, + 0x19080819, 0x1919082b, 0x19081908, 0x1919082b, 0x19190808, 0x1919082b, 0x192b2b19, 0x1919082b, + 0x2b080808, 0x1919082b, 0x08080819, 0x19191908, 0x08081908, 0x19191908, 0x0808192b, 0x19191908, + 0x08082b19, 0x19191908, 0x08190808, 0x19191908, 0x0819082b, 0x19191908, 0x08191919, 0x19191908, + 0x08192b08, 0x19191908, 0x082b0819, 0x19191908, 0x082b1908, 0x19191908, 0x19080808, 0x19191908, + 0x1908082b, 0x19191908, 0x19081919, 0x19191908, 0x19082b08, 0x19191908, 0x19190819, 0x19191908, + 0x19191908, 0x19191908, 0x192b0808, 0x19191908, 0x2b080819, 0x19191908, 0x2b081908, 0x19191908, + 0x2b190808, 0x19191908, 0x08080808, 0x19191919, 0x0808082b, 0x19191919, 0x08081919, 0x19191919, + 0x08082b08, 0x19191919, 0x08190819, 0x19191919, 0x08191908, 0x19191919, 0x082b0808, 0x19191919, + 0x19080819, 0x19191919, 0x19081908, 0x19191919, 0x19190808, 0x19191919, 0x2b080808, 0x19191919, + 0x08080819, 0x1919192b, 0x08081908, 0x1919192b, 0x08190808, 0x1919192b, 0x082b192b, 0x1919192b, + 0x19080808, 0x1919192b, 0x08080808, 0x19192b08, 0x0808082b, 0x19192b08, 0x08081919, 0x19192b08, + 0x08082b08, 0x19192b08, 0x08190819, 0x19192b08, 0x08191908, 0x19192b08, 0x082b0808, 0x19192b08, + 0x19080819, 0x19192b08, 0x19081908, 0x19192b08, 0x19190808, 0x19192b08, 0x19192b2b, 0x19192b08, + 0x2b080808, 0x19192b08, 0x08080819, 0x19192b19, 0x08081908, 0x19192b19, 0x08190808, 0x19192b19, + 0x19080808, 0x19192b19, 0x08080808, 0x19192b2b, 0x08192b19, 0x19192b2b, 0x2b081919, 0x19192b2b, + 0x2b2b2b08, 0x19192b2b, 0x08080819, 0x192b0808, 0x08081908, 0x192b0808, 0x0808192b, 0x192b0808, + 0x08190808, 0x192b0808, 0x0819082b, 0x192b0808, 0x08191919, 0x192b0808, 0x08192b08, 0x192b0808, + 0x082b0819, 0x192b0808, 0x082b1908, 0x192b0808, 0x19080808, 0x192b0808, 0x19081919, 0x192b0808, + 0x19082b08, 0x192b0808, 0x19190819, 0x192b0808, 0x19191908, 0x192b0808, 0x192b0808, 0x192b0808, + 0x2b081908, 0x192b0808, 0x2b190808, 0x192b0808, 0x08080808, 0x192b0819, 0x0808082b, 0x192b0819, + 0x08081919, 0x192b0819, 0x08082b08, 0x192b0819, 0x08190819, 0x192b0819, 0x08191908, 0x192b0819, + 0x082b0808, 0x192b0819, 0x19080819, 0x192b0819, 0x19081908, 0x192b0819, 0x19190808, 0x192b0819, + 0x2b080808, 0x192b0819, 0x2b192b19, 0x192b0819, 0x08081908, 0x192b082b, 0x08190808, 0x192b082b, + 0x19080808, 0x192b082b, 0x1919192b, 0x192b082b, 0x2b2b0819, 0x192b082b, 0x08080808, 0x192b1908, + 0x08081919, 0x192b1908, 0x08082b08, 0x192b1908, 0x08190819, 0x192b1908, 0x08191908, 0x192b1908, + 0x082b0808, 0x192b1908, 0x19080819, 0x192b1908, 0x19081908, 0x192b1908, 0x19190808, 0x192b1908, + 0x2b080808, 0x192b1908, 0x08080819, 0x192b1919, 0x08081908, 0x192b1919, 0x08190808, 0x192b1919, + 0x19080808, 0x192b1919, 0x19082b2b, 0x192b1919, 0x192b2b08, 0x192b1919, 0x2b19082b, 0x192b1919, + 0x08080808, 0x192b192b, 0x2b191908, 0x192b192b, 0x08080819, 0x192b2b08, 0x08081908, 0x192b2b08, + 0x08190808, 0x192b2b08, 0x192b1919, 0x192b2b08, 0x2b192b08, 0x192b2b08, 0x08080808, 0x192b2b19, + 0x082b2b2b, 0x192b2b19, 0x1908082b, 0x192b2b2b, 0x2b2b0819, 0x192b2b2b, 0x08080808, 0x2b080808, + 0x0808082b, 0x2b080808, 0x08081919, 0x2b080808, 0x08082b08, 0x2b080808, 0x08190819, 0x2b080808, + 0x08191908, 0x2b080808, 0x08192b19, 0x2b080808, 0x082b0808, 0x2b080808, 0x082b1919, 0x2b080808, + 0x19080819, 0x2b080808, 0x19081908, 0x2b080808, 0x19190808, 0x2b080808, 0x1919082b, 0x2b080808, + 0x19191919, 0x2b080808, 0x19192b08, 0x2b080808, 0x192b0819, 0x2b080808, 0x2b080808, 0x2b080808, + 0x2b081919, 0x2b080808, 0x2b190819, 0x2b080808, 0x2b191908, 0x2b080808, 0x08080819, 0x2b080819, + 0x08081908, 0x2b080819, 0x08082b19, 0x2b080819, 0x08190808, 0x2b080819, 0x0819082b, 0x2b080819, + 0x08191919, 0x2b080819, 0x08192b08, 0x2b080819, 0x082b0819, 0x2b080819, 0x082b1908, 0x2b080819, + 0x19080808, 0x2b080819, 0x1908082b, 0x2b080819, 0x19081919, 0x2b080819, 0x19082b08, 0x2b080819, + 0x19190819, 0x2b080819, 0x19191908, 0x2b080819, 0x2b080819, 0x2b080819, 0x2b081908, 0x2b080819, + 0x2b190808, 0x2b080819, 0x2b2b2b19, 0x2b080819, 0x08080808, 0x2b08082b, 0x08081919, 0x2b08082b, + 0x08082b2b, 0x2b08082b, 0x08190819, 0x2b08082b, 0x08191908, 0x2b08082b, 0x19080819, 0x2b08082b, + 0x19081908, 0x2b08082b, 0x19190808, 0x2b08082b, 0x08080819, 0x2b081908, 0x08081908, 0x2b081908, + 0x0808192b, 0x2b081908, 0x08082b19, 0x2b081908, 0x08190808, 0x2b081908, 0x0819082b, 0x2b081908, + 0x08191919, 0x2b081908, 0x08192b08, 0x2b081908, 0x082b0819, 0x2b081908, 0x19080808, 0x2b081908, + 0x1908082b, 0x2b081908, 0x19081919, 0x2b081908, 0x19082b08, 0x2b081908, 0x19190819, 0x2b081908, + 0x19191908, 0x2b081908, 0x192b0808, 0x2b081908, 0x2b080819, 0x2b081908, 0x2b081908, 0x2b081908, + 0x2b190808, 0x2b081908, 0x08080808, 0x2b081919, 0x0808082b, 0x2b081919, 0x08081919, 0x2b081919, + 0x08082b08, 0x2b081919, 0x08190819, 0x2b081919, 0x08191908, 0x2b081919, 0x082b0808, 0x2b081919, + 0x19080819, 0x2b081919, 0x19081908, 0x2b081919, 0x19190808, 0x2b081919, 0x2b080808, 0x2b081919, + 0x2b082b2b, 0x2b081919, 0x08080819, 0x2b08192b, 0x08081908, 0x2b08192b, 0x08190808, 0x2b08192b, + 0x082b2b19, 0x2b08192b, 0x19080808, 0x2b08192b, 0x08080808, 0x2b082b08, 0x08081919, 0x2b082b08, + 0x08190819, 0x2b082b08, 0x08191908, 0x2b082b08, 0x19080819, 0x2b082b08, 0x19081908, 0x2b082b08, + 0x19190808, 0x2b082b08, 0x2b2b082b, 0x2b082b08, 0x08080819, 0x2b082b19, 0x08081908, 0x2b082b19, + 0x19080808, 0x2b082b19, 0x192b1919, 0x2b082b19, 0x082b082b, 0x2b082b2b, 0x19192b08, 0x2b082b2b, + 0x19192b2b, 0x2b082b2b, 0x2b08082b, 0x2b082b2b, 0x2b2b082b, 0x2b082b2b, 0x08080819, 0x2b190808, + 0x08081908, 0x2b190808, 0x08082b19, 0x2b190808, 0x08190808, 0x2b190808, 0x0819082b, 0x2b190808, + 0x08191919, 0x2b190808, 0x08192b08, 0x2b190808, 0x082b1908, 0x2b190808, 0x19080808, 0x2b190808, + 0x1908082b, 0x2b190808, 0x19081919, 0x2b190808, 0x19082b08, 0x2b190808, 0x19190819, 0x2b190808, + 0x19191908, 0x2b190808, 0x192b0808, 0x2b190808, 0x2b080819, 0x2b190808, 0x2b081908, 0x2b190808, + 0x2b190808, 0x2b190808, 0x08080808, 0x2b190819, 0x08081919, 0x2b190819, 0x08190819, 0x2b190819, + 0x08191908, 0x2b190819, 0x19080819, 0x2b190819, 0x19081908, 0x2b190819, 0x19190808, 0x2b190819, + 0x19192b2b, 0x2b190819, 0x08080819, 0x2b19082b, 0x08081908, 0x2b19082b, 0x08190808, 0x2b19082b, + 0x19080808, 0x2b19082b, 0x2b2b192b, 0x2b19082b, 0x08080808, 0x2b191908, 0x0808082b, 0x2b191908, + 0x08081919, 0x2b191908, 0x08082b08, 0x2b191908, 0x08190819, 0x2b191908, 0x08191908, 0x2b191908, + 0x082b0808, 0x2b191908, 0x19080819, 0x2b191908, 0x19081908, 0x2b191908, 0x19190808, 0x2b191908, + 0x2b080808, 0x2b191908, 0x2b19192b, 0x2b191908, 0x08080819, 0x2b191919, 0x08081908, 0x2b191919, + 0x08190808, 0x2b191919, 0x19080808, 0x2b191919, 0x2b192b08, 0x2b191919, 0x2b2b0819, 0x2b191919, + 0x08080808, 0x2b19192b, 0x1908192b, 0x2b19192b, 0x192b1908, 0x2b19192b, 0x08080819, 0x2b192b08, + 0x08081908, 0x2b192b08, 0x08190808, 0x2b192b08, 0x082b192b, 0x2b192b08, 0x19080808, 0x2b192b08, + 0x2b2b2b19, 0x2b192b08, 0x08080808, 0x2b192b19, 0x19082b19, 0x2b192b19, 0x1919082b, 0x2b192b19, + 0x2b190808, 0x2b192b2b, 0x08080808, 0x2b2b0808, 0x08081919, 0x2b2b0808, 0x08082b2b, 0x2b2b0808, + 0x08191908, 0x2b2b0808, 0x082b082b, 0x2b2b0808, 0x082b2b2b, 0x2b2b0808, 0x19080819, 0x2b2b0808, + 0x19081908, 0x2b2b0808, 0x19190808, 0x2b2b0808, 0x2b2b082b, 0x2b2b0808, 0x2b2b2b2b, 0x2b2b0808, + 0x19080808, 0x2b2b0819, 0x192b1919, 0x2b2b0819, 0x0808082b, 0x2b2b082b, 0x08082b2b, 0x2b2b082b, + 0x082b082b, 0x2b2b082b, 0x082b2b08, 0x2b2b082b, 0x082b2b2b, 0x2b2b082b, 0x2b08082b, 0x2b2b082b, + 0x2b082b08, 0x2b2b082b, 0x2b082b2b, 0x2b2b082b, 0x2b2b2b08, 0x2b2b082b, 0x08080819, 0x2b2b1908, + 0x08081908, 0x2b2b1908, 0x08190808, 0x2b2b1908, 0x19080808, 0x2b2b1908, 0x2b082b19, 0x2b2b1908, + 0x2b2b1908, 0x2b2b1908, 0x08080808, 0x2b2b1919, 0x08192b19, 0x2b2b1919, 0x19190819, 0x2b2b192b, + 0x08082b2b, 0x2b2b2b08, 0x082b2b08, 0x2b2b2b08, 0x2b2b082b, 0x2b2b2b08, 0x19191908, 0x2b2b2b19, + 0x2b08192b, 0x2b2b2b19, 0x08082b08, 0x2b2b2b2b, 0x08082b2b, 0x2b2b2b2b, 0x082b0808, 0x2b2b2b2b, + 0x082b082b, 0x2b2b2b2b, 0x082b2b08, 0x2b2b2b2b, 0x2b082b08, 0x2b2b2b2b, 0x2b2b2b2b, 0x2b2b2b2b) + return tbl[i] +} + // Expert-indexed Q4_K GEMV — MetalKqGemvK4's dot; the superblock index shifts by sel[slot] * esb. // x rides a float4 view: the 32-scalar-x-load form was load-issue-bound at the routed expert // shapes (nfe-row planes) — the vector view is 2.25x there (lab: bench_metal_moe_lab, 142→321 wGB/s). @@ -6835,6 +7896,2072 @@ class MetalKqGemvK6 { } } +// Q2_K: the k3 GEMV's lane map minus the hmask - unsigned 2-bit lanes, per-16 sc/min NIBBLE +// pairs off the split strips (sc scales the dot, mn folds on the lane's x sums against dmin), +// d + dmin off the 4B tail plane. +[metal_dispatch(name = "enc_kq_k2_c", pso = "g_pso_kq_k2", tg = 64, grid = "rows/4", params = "rows : int64, n : int64")] +class MetalKqGemvK2 { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d+dmin tail - the scale buffer bound at byte nsb*16 (s0off = the caller's doff) + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // 16B pair-byte strips, byte view (same buffer at 0) + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqb : array // k2 quant plane, byte view (qs at 64*blk) + @ssbo @binding = 3 @role = "read" @off = "xoff" @span = "n*4" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" @span = "rows*4" y : array + @uniform @binding = 5 ndim : uint + @uniform @binding = 6 ddim : uint + + [metal_kernel(name="metal_kq_gemv_k2_msl")] + def metal_kq_gemv_k2 { + let lane = gl_SubgroupInvocationID + let tid = lane / 2u + let ix = lane % 2u + let ip = tid / 8u + let il = tid % 8u + let l0 = il * 4u + let is0 = ip * 8u + l0 / 16u + let nb = ndim / 256u + let first_row = (gl_WorkGroupID.x * 2u + gl_SubgroupID) * 2u + var yl : float[16] + var sumf : float[2] + var ib = ix + while (ib < nb) { + let yb = ib * 256u + ip * 128u + l0 + var ysum : float[4] + for [unroll_full] (l in range(4)) { + yl[4 * l] = x[yb + uint(l)] + yl[4 * l + 1] = x[yb + 32u + uint(l)] + yl[4 * l + 2] = x[yb + 64u + uint(l)] + yl[4 * l + 3] = x[yb + 96u + uint(l)] + ysum[0] += yl[4 * l] + ysum[1] += yl[4 * l + 1] + ysum[2] += yl[4 * l + 2] + ysum[3] += yl[4 * l + 3] + } + for [unroll_full] (r in range(2)) { + let blk = (first_row + uint(r)) * nb + ib + let qs0 = blk * 64u + ip * 32u + l0 + let sb = blk * 16u + is0 + var sums : float[4] + for [unroll_full] (l in range(4)) { + let q = uint(kqb[qs0 + uint(l)]) + sums[0] += yl[4 * l] * float(q & 3u) + sums[1] += yl[4 * l + 1] * float((q >> 2u) & 3u) + sums[2] += yl[4 * l + 2] * float((q >> 4u) & 3u) + sums[3] += yl[4 * l + 3] * float(q >> 6u) + } + let b0 = uint(kscb[sb]) + let b2 = uint(kscb[sb + 2u]) + let b4 = uint(kscb[sb + 4u]) + let b6 = uint(kscb[sb + 6u]) + let dd = float(kdh[blk * 2u]) + let dmn = float(kdh[blk * 2u + 1u]) + sumf[r] += dd * (sums[0] * float(b0 & 15u) + sums[1] * float(b2 & 15u) + sums[2] * float(b4 & 15u) + sums[3] * float(b6 & 15u)) + sumf[r] -= dmn * (ysum[0] * float(b0 >> 4u) + ysum[1] * float(b2 >> 4u) + ysum[2] * float(b4 >> 4u) + ysum[3] * float(b6 >> 4u)) + } + ib += 2u + } + for [unroll_full] (r in range(2)) { + let s = simd_sum(sumf[r]) + if (lane == 0u && first_row + uint(r) < ddim) { + y[first_row + uint(r)] = s + } + } + } +} + +// The k2 B2/B4 pair: the K3T shell minus the hmask - w = q*d*sc - dmin*mn per element, the +// pair byte picked per (slot, half). +[ |> template_struct_instance] +class template MetalKqMvK2T { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d+dmin tail - the scale buffer bound at byte nsb*16 + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // 16B pair-byte strips, byte view (same buffer at 0) + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array // k2 quant plane, uint view (16 qs words per superblock) + @ssbo @binding = 3 @role = "read" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" y : array + @uniform @binding = 5 ka : KqMvArgs + @template_constant NR : int = 2 + @template_constant NRU : uint = 2u // NR's uint spelling (colbase math) — override BOTH together + @template_constant TILED : bool = false + + [metal_kernel] + def metal_kq_mv_k2 { + let lane = gl_SubgroupInvocationID + let tx = lane % 8u + let ty = lane / 8u + let row = gl_WorkGroupID.x * 8u + gl_SubgroupID * 4u + ty + let colbase = gl_WorkGroupID.y * NRU // nolint:LINT002 dead in the TILED=false stamp — measured free + let nb = ka.ndim / 256u + let s = tx / 2u + let o4 = (tx % 2u) * 4u + let shr = 2u * s + let nb4 = ka.ndim / 4u + var sumf : float[NR] + var sb = 0u + while (sb < nb) { + let blk = row * nb + sb + let dd = float(kdh[blk * 2u]) + let dmn = float(kdh[blk * 2u + 1u]) + for [unroll_full] (ch in range(2)) { + let ip = uint(ch) + let qb = blk * 16u + ip * 8u + o4 + let g = ip * 8u + s * 2u + (tx % 2u) + let pb = uint(kscb[blk * 16u + g]) + let dl = dd * float(pb & 15u) + let ml = dmn * float(pb >> 4u) + var w : float[16] + for [unroll_full] (u in range(4)) { + let qu = kqu[qb + uint(u)] + let qv = (qu >> shr) & 0x03030303 + for [unroll_full] (e in range(4)) { + w[4 * u + e] = float((qv >> (8u * uint(e))) & 255u) * dl - ml + } + } + let cx4 = sb * 64u + ip * 32u + s * 8u + o4 + static_if (TILED) { + for [unroll_full] (b in range(NR)) { + let xb4 = (colbase + uint(b)) * nb4 + cx4 + var acc = 0.0 + for [unroll_full] (u in range(4)) { + let v = x[xb4 + uint(u)] + acc += v.x * w[4 * u] + v.y * w[4 * u + 1] + v.z * w[4 * u + 2] + v.w * w[4 * u + 3] + } + sumf[b] += acc + } + } else { + for [unroll_full] (b in range(NR)) { + let xb4 = uint(b) * nb4 + cx4 + var acc = 0.0 + for [unroll_full] (u in range(4)) { + let v = x[xb4 + uint(u)] + acc += v.x * w[4 * u] + v.y * w[4 * u + 1] + v.z * w[4 * u + 2] + v.w * w[4 * u + 3] + } + sumf[b] += acc + } + } + } + sb++ + } + static_if (TILED) { + for [unroll_full] (b in range(NR)) { + var sm = sumf[b] + sm += simd_shuffle_down(sm, 4u) + sm += simd_shuffle_down(sm, 2u) + sm += simd_shuffle_down(sm, 1u) + if (tx == 0u && row < ka.ddim && colbase + uint(b) < ka.nr) { + y[(colbase + uint(b)) * ka.ys + row] = sm + } + } + } else { + for [unroll_full] (b in range(NR)) { + var sm = sumf[b] + sm += simd_shuffle_down(sm, 4u) + sm += simd_shuffle_down(sm, 2u) + sm += simd_shuffle_down(sm, 1u) + if (tx == 0u && row < ka.ddim && uint(b) < ka.nr) { + y[uint(b) * ka.ys + row] = sm + } + } + } + } +} + +[metal_dispatch(name = "enc_kq_mvb2_k2_c", pso = "g_pso_kq_mvb2_k2", tg = 64, grid = "rows/8", params = "rows : int64")] +class MetalKqMvB2K2 : MetalKqMvK2T { + override NR = 2 + override NRU = 2u + override TILED = false +} + +[metal_dispatch(name = "enc_kq_mvb4_k2_c", pso = "g_pso_kq_mvb4_k2", tg = 64, grid = "rows/8, gcols", params = "rows : int64, gcols : int64")] +class MetalKqMvB4K2 : MetalKqMvK2T { + override NR = 4 + override NRU = 4u + override TILED = true +} + +// the k2 B8 twin: the k3 B8 shell minus the hmask, per-element w = q*d*sc - dmin*mn +[metal_dispatch(name = "enc_kq_mvb8_k2_c", pso = "g_pso_kq_mvb8_k2", tgmem = "metal_kq_mvb8_k2_msl_tgmem", tg = 64, grid = "rows/8", params = "rows : int64")] +class MetalKqMvB8K2 { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d+dmin tail - the scale buffer bound at byte nsb*16 + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // 16B pair-byte strips, byte view (same buffer at 0) + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array // k2 quant plane, uint view + @ssbo @binding = 3 @role = "read" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" y : array + @uniform @binding = 5 ka : KqMvArgs + @workgroup txp : float4[512] + + [metal_kernel(name="metal_kq_mvb8_k2_msl")] + def metal_kq_mvb8_k2 { + let lid = gl_LocalInvocationID.x + let lane = gl_SubgroupInvocationID + let tx = lane % 8u + let ty = lane / 8u + let row = gl_WorkGroupID.x * 8u + gl_SubgroupID * 4u + ty + let nb = ka.ndim / 256u + let s = tx / 2u + let o4 = (tx % 2u) * 4u + let shr = 2u * s + let nb4 = ka.ndim / 4u + var acc : float4[8] + var sb = 0u + while (sb < nb) { + barrier() + for [unroll_full] (j in range(8)) { + let slot = uint(j) * 64u + lid + txp[slot] = x[(slot / 64u) * nb4 + sb * 64u + (slot % 64u)] + } + barrier() + let blk = row * nb + sb + let dd = float(kdh[blk * 2u]) + let dmn = float(kdh[blk * 2u + 1u]) + for [unroll_full] (ch in range(2)) { + let ip = uint(ch) + let qb = blk * 16u + ip * 8u + o4 + let g = ip * 8u + s * 2u + (tx % 2u) + let pb = uint(kscb[blk * 16u + g]) + let dl = dd * float(pb & 15u) + let ml = dmn * float(pb >> 4u) + var w4 : float4[4] + for [unroll_full] (u in range(4)) { + let qvc = (kqu[qb + uint(u)] >> shr) & 0x03030303 + w4[u] = float4(float(qvc & 255u), float((qvc >> 8u) & 255u), + float((qvc >> 16u) & 255u), float((qvc >> 24u) & 255u)) * dl - float4(ml) + } + let cxl = ip * 32u + s * 8u + o4 + for [unroll_full] (b in range(8)) { + for [unroll_full] (u in range(4)) { + acc[b] += txp[uint(b) * 64u + cxl + uint(u)] * w4[u] + } + } + } + sb++ + } + for [unroll_full] (b in range(8)) { + var sm = acc[b].x + acc[b].y + acc[b].z + acc[b].w + sm += simd_shuffle_down(sm, 4u) + sm += simd_shuffle_down(sm, 2u) + sm += simd_shuffle_down(sm, 1u) + if (tx == 0u && row < ka.ddim && uint(b) < ka.nr) { + y[uint(b) * ka.ys + row] = sm + } + } + } +} + +// Q3_K: k6's lane map (a lane's 4 elements of each of a half's 4 blocks) over the k3 planes — the +// 2-bit lane of qs byte ip*32 + l at shift 2j | hmask bit 4ip + j of byte l, minus 4; per-16 SIGNED +// sub-scales off the split strips, d off the d plane. +[metal_dispatch(name = "enc_kq_k3_c", pso = "g_pso_kq_k3", tg = 64, grid = "rows/4", params = "rows : int64, n : int64")] +class MetalKqGemvK3 { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d plane — the scale buffer bound at byte nsb*16 (s0off = the caller's doff) + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // 16B sub-scale strips, byte view (same buffer at 0) + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqb : array // k3 quant plane, byte view (qs at 96*blk, hmask at +64) + @ssbo @binding = 3 @role = "read" @off = "xoff" @span = "n*4" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" @span = "rows*4" y : array + @uniform @binding = 5 ndim : uint + @uniform @binding = 6 ddim : uint + + [metal_kernel(name="metal_kq_gemv_k3_msl")] + def metal_kq_gemv_k3 { + let lane = gl_SubgroupInvocationID + let tid = lane / 2u + let ix = lane % 2u + let ip = tid / 8u + let il = tid % 8u + let l0 = il * 4u + let is0 = ip * 8u + l0 / 16u + let hb0 = ip * 4u + let nb = ndim / 256u + let first_row = (gl_WorkGroupID.x * 2u + gl_SubgroupID) * 2u + var yl : float[16] + var sumf : float[2] + var ib = ix + while (ib < nb) { + let yb = ib * 256u + ip * 128u + l0 + for [unroll_full] (l in range(4)) { + yl[4 * l] = x[yb + uint(l)] + yl[4 * l + 1] = x[yb + 32u + uint(l)] + yl[4 * l + 2] = x[yb + 64u + uint(l)] + yl[4 * l + 3] = x[yb + 96u + uint(l)] + } + for [unroll_full] (r in range(2)) { + let blk = (first_row + uint(r)) * nb + ib + let qs0 = blk * 96u + ip * 32u + l0 + let hm0 = blk * 96u + 64u + l0 + let sb = blk * 16u + is0 + var sums : float[4] + for [unroll_full] (l in range(4)) { + let q = uint(kqb[qs0 + uint(l)]) + let h = uint(kqb[hm0 + uint(l)]) >> hb0 + sums[0] += yl[4 * l] * (float((q & 3u) | ((h & 1u) << 2u)) - 4.0) + sums[1] += yl[4 * l + 1] * (float(((q >> 2u) & 3u) | ((h & 2u) << 1u)) - 4.0) + sums[2] += yl[4 * l + 2] * (float(((q >> 4u) & 3u) | (h & 4u)) - 4.0) + sums[3] += yl[4 * l + 3] * (float((q >> 6u) | ((h & 8u) >> 1u)) - 4.0) + } + let s0 = float((int(kscb[sb]) ^ 128) - 128) + let s2 = float((int(kscb[sb + 2u]) ^ 128) - 128) + let s4 = float((int(kscb[sb + 4u]) ^ 128) - 128) + let s6 = float((int(kscb[sb + 6u]) ^ 128) - 128) + let dd = float(kdh[blk]) + sumf[r] += dd * (sums[0] * s0 + sums[1] * s2 + sums[2] * s4 + sums[3] * s6) + } + ib += 2u + } + for [unroll_full] (r in range(2)) { + let s = simd_sum(sumf[r]) + if (lane == 0u && first_row + uint(r) < ddim) { + y[first_row + uint(r)] = s + } + } + } +} + +// The k3 B2/B4 pair: the K6T shell (slot s = block of the half, o4 = its 16-half) with the +// lane + mask compose off the 24-uint superblock (qs 0..15, hmask 16..23), offset 4. +[ |> template_struct_instance] +class template MetalKqMvK3T { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d plane — the scale buffer bound at byte nsb*16 + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // 16B sub-scale strips, byte view (same buffer at 0) + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array // k3 quant plane, uint view (qs at 24*blk, hmask at 24*blk+16) + @ssbo @binding = 3 @role = "read" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" y : array + @uniform @binding = 5 ka : KqMvArgs + @template_constant NR : int = 2 + @template_constant NRU : uint = 2u // NR's uint spelling (colbase math) — override BOTH together + @template_constant TILED : bool = false + + [metal_kernel] + def metal_kq_mv_k3 { // nolint:STYLE038 — the TILED twin loops are byte-identity-driven, not splittable + let lane = gl_SubgroupInvocationID + let tx = lane % 8u + let ty = lane / 8u + let row = gl_WorkGroupID.x * 8u + gl_SubgroupID * 4u + ty + let colbase = gl_WorkGroupID.y * NRU // nolint:LINT002 dead in the TILED=false stamp — measured free + let nb = ka.ndim / 256u + let s = tx / 2u + let o4 = (tx % 2u) * 4u + let shr = 2u * s + let nb4 = ka.ndim / 4u + var sumf : float[NR] + var sb = 0u + while (sb < nb) { + let blk = row * nb + sb + let dd = float(kdh[blk]) + for [unroll_full] (ch in range(2)) { + let ip = uint(ch) + let qb = blk * 24u + ip * 8u + o4 + let hb = blk * 24u + 16u + o4 + let hsh = ip * 4u + s + let g = ip * 8u + s * 2u + (tx % 2u) + let sc = float((int(kscb[blk * 16u + g]) ^ 128) - 128) + let dl = dd * sc + let ml = dl * 4.0 + var w : float[16] + for [unroll_full] (u in range(4)) { + let qu = kqu[qb + uint(u)] + let hu = kqu[hb + uint(u)] + let qv = ((qu >> shr) & 0x03030303) | (((hu >> hsh) & 0x01010101) << 2u) + for [unroll_full] (e in range(4)) { + w[4 * u + e] = float((qv >> (8u * uint(e))) & 255u) * dl - ml + } + } + let cx4 = sb * 64u + ip * 32u + s * 8u + o4 + static_if (TILED) { + for [unroll_full] (b in range(NR)) { + let xb4 = (colbase + uint(b)) * nb4 + cx4 + var acc = 0.0 + for [unroll_full] (u in range(4)) { + let v = x[xb4 + uint(u)] + acc += v.x * w[4 * u] + v.y * w[4 * u + 1] + v.z * w[4 * u + 2] + v.w * w[4 * u + 3] + } + sumf[b] += acc + } + } else { + for [unroll_full] (b in range(NR)) { + let xb4 = uint(b) * nb4 + cx4 + var acc = 0.0 + for [unroll_full] (u in range(4)) { + let v = x[xb4 + uint(u)] + acc += v.x * w[4 * u] + v.y * w[4 * u + 1] + v.z * w[4 * u + 2] + v.w * w[4 * u + 3] + } + sumf[b] += acc + } + } + } + sb++ + } + static_if (TILED) { + for [unroll_full] (b in range(NR)) { + var sm = sumf[b] + sm += simd_shuffle_down(sm, 4u) + sm += simd_shuffle_down(sm, 2u) + sm += simd_shuffle_down(sm, 1u) + if (tx == 0u && row < ka.ddim && colbase + uint(b) < ka.nr) { + y[(colbase + uint(b)) * ka.ys + row] = sm + } + } + } else { + for [unroll_full] (b in range(NR)) { + var sm = sumf[b] + sm += simd_shuffle_down(sm, 4u) + sm += simd_shuffle_down(sm, 2u) + sm += simd_shuffle_down(sm, 1u) + if (tx == 0u && row < ka.ddim && uint(b) < ka.nr) { + y[uint(b) * ka.ys + row] = sm + } + } + } + } +} + +[metal_dispatch(name = "enc_kq_mvb2_k3_c", pso = "g_pso_kq_mvb2_k3", tg = 64, grid = "rows/8", params = "rows : int64")] +class MetalKqMvB2K3 : MetalKqMvK3T { + override NR = 2 + override NRU = 2u + override TILED = false +} + +[metal_dispatch(name = "enc_kq_mvb4_k3_c", pso = "g_pso_kq_mvb4_k3", tg = 64, grid = "rows/8, gcols", params = "rows : int64, gcols : int64")] +class MetalKqMvB4K3 : MetalKqMvK3T { + override NR = 4 + override NRU = 4u + override TILED = true +} + +// the k3 B8 twin: the k6 B8 shell (one superblock's X panel staged per threadgroup) with the +// lane + mask compose +[metal_dispatch(name = "enc_kq_mvb8_k3_c", pso = "g_pso_kq_mvb8_k3", tgmem = "metal_kq_mvb8_k3_msl_tgmem", tg = 64, grid = "rows/8", params = "rows : int64")] +class MetalKqMvB8K3 { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d plane — the scale buffer bound at byte nsb*16 + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // 16B sub-scale strips, byte view (same buffer at 0) + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array // k3 quant plane, uint view (qs at 24*blk, hmask at 24*blk+16) + @ssbo @binding = 3 @role = "read" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" y : array + @uniform @binding = 5 ka : KqMvArgs + @workgroup txp : float4[512] + + [metal_kernel(name="metal_kq_mvb8_k3_msl")] + def metal_kq_mvb8_k3 { + let lid = gl_LocalInvocationID.x + let lane = gl_SubgroupInvocationID + let tx = lane % 8u + let ty = lane / 8u + let row = gl_WorkGroupID.x * 8u + gl_SubgroupID * 4u + ty + let nb = ka.ndim / 256u + let s = tx / 2u + let o4 = (tx % 2u) * 4u + let shr = 2u * s + let nb4 = ka.ndim / 4u + var acc : float4[8] + var sb = 0u + while (sb < nb) { + barrier() + for [unroll_full] (j in range(8)) { + let slot = uint(j) * 64u + lid + txp[slot] = x[(slot / 64u) * nb4 + sb * 64u + (slot % 64u)] + } + barrier() + let blk = row * nb + sb + let dd = float(kdh[blk]) + for [unroll_full] (ch in range(2)) { + let ip = uint(ch) + let qb = blk * 24u + ip * 8u + o4 + let hb = blk * 24u + 16u + o4 + let hsh = ip * 4u + s + var qu : uint[4] + var hu : uint[4] + for [unroll_full] (u in range(4)) { + qu[u] = kqu[qb + uint(u)] + hu[u] = kqu[hb + uint(u)] + } + let g = ip * 8u + s * 2u + (tx % 2u) + let sc = float((int(kscb[blk * 16u + g]) ^ 128) - 128) + let dl = dd * sc + let ml = dl * 4.0 + var w4 : float4[4] + for [unroll_full] (u in range(4)) { + let qvc = ((qu[u] >> shr) & 0x03030303) | (((hu[u] >> hsh) & 0x01010101) << 2u) + w4[u] = float4(float(qvc & 255u), float((qvc >> 8u) & 255u), + float((qvc >> 16u) & 255u), float((qvc >> 24u) & 255u)) * dl - float4(ml) + } + let cxl = ip * 32u + s * 8u + o4 + for [unroll_full] (b in range(8)) { + for [unroll_full] (u in range(4)) { + acc[b] += txp[uint(b) * 64u + cxl + uint(u)] * w4[u] + } + } + } + sb++ + } + for [unroll_full] (b in range(8)) { + var sm = acc[b].x + acc[b].y + acc[b].z + acc[b].w + sm += simd_shuffle_down(sm, 4u) + sm += simd_shuffle_down(sm, 2u) + sm += simd_shuffle_down(sm, 1u) + if (tx == 0u && row < ka.ddim && uint(b) < ka.nr) { + y[uint(b) * ka.ys + row] = sm + } + } + } +} + +// IQ4_XS: w = d*sc*LUT[q], sc the SIGNED strip byte, no min plane; the codebook in threadgroup +// floats, a 16-lane half-simdgroup per superblock, lane (bu, il) = block bu's 8-byte half il +// (two uint loads, four float4 x loads; the q40 pairing). Scale plane = the k6 split form. +[metal_dispatch(name = "enc_kq_iq4xs_c", pso = "g_pso_kq_iq4xs", tgmem = "metal_kq_gemv_iq4xs_msl_tgmem", tg = 64, grid = "rows/4", params = "rows : int64, n : int64")] +class MetalKqGemvIq4xs { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d plane — the scale buffer bound at byte nsb*16 (s0off = the caller's doff) + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // 16B strips: 8 signed sub-scales + 8 pad, byte view (same buffer at 0) + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array // iq4xs quant plane, uint view (block bu of blk at 32*blk + 4*bu) + @ssbo @binding = 3 @role = "read" @off = "xoff" @span = "n*4" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" @span = "rows*4" y : array + @uniform @binding = 5 ndim : uint + @uniform @binding = 6 ddim : uint + @workgroup lut : float[64] // kvalues_iq4nl as floats, one slot per thread of the 64-wide group + + [metal_kernel(name="metal_kq_gemv_iq4xs_msl")] + def metal_kq_gemv_iq4xs { + let lid = gl_LocalInvocationID.x + lut[lid] = iq4_lut(lid & 15u) + barrier() + let lane = gl_SubgroupInvocationID + let ix = lane / 16u + let it = lane % 16u + let bu = it / 2u + let il = it % 2u + let nb = ndim / 256u + let first_row = (gl_WorkGroupID.x * 2u + gl_SubgroupID) * 2u + var sumf : float[2] + var ib = ix + while (ib < nb) { + let yb4 = ib * 64u + bu * 8u + il * 2u + let y0 = x[yb4] + let y1 = x[yb4 + 1u] + let y2 = x[yb4 + 4u] + let y3 = x[yb4 + 5u] + for [unroll_full] (r in range(2)) { + let blk = (first_row + uint(r)) * nb + ib + let qb = blk * 32u + bu * 4u + il * 2u + let q0 = kqu[qb] + let q1 = kqu[qb + 1u] + var acc = y0 * float4(lut[q0 & 15u], lut[(q0 >> 8u) & 15u], lut[(q0 >> 16u) & 15u], lut[(q0 >> 24u) & 15u]) + acc += y2 * float4(lut[(q0 >> 4u) & 15u], lut[(q0 >> 12u) & 15u], lut[(q0 >> 20u) & 15u], lut[q0 >> 28u]) + acc += y1 * float4(lut[q1 & 15u], lut[(q1 >> 8u) & 15u], lut[(q1 >> 16u) & 15u], lut[(q1 >> 24u) & 15u]) + acc += y3 * float4(lut[(q1 >> 4u) & 15u], lut[(q1 >> 12u) & 15u], lut[(q1 >> 20u) & 15u], lut[q1 >> 28u]) + let sc = float((int(kscb[blk * 16u + bu]) ^ 128) - 128) + sumf[r] += float(kdh[blk]) * sc * (acc.x + acc.y + acc.z + acc.w) + } + ib += 2u + } + for [unroll_full] (r in range(2)) { + let s = simd_sum(sumf[r]) + if (lane == 0u && first_row + uint(r) < ddim) { + y[first_row + uint(r)] = s + } + } + } +} + +// IQ4_NL: w = d*LUT[q] with a PER-32-BLOCK f16 d (q40's 16B/sb scale plane verbatim, one bind, +// no sub-scale, no split plane); the quant walk is the iq4xs GEMV's q40 pairing over the same +// threadgroup codebook. kscb keeps the shared kq binding shape (iq4nl has no strip bytes). +[metal_dispatch(name = "enc_kq_iq4nl_c", pso = "g_pso_kq_iq4nl", tgmem = "metal_kq_gemv_iq4nl_msl_tgmem", tg = 64, grid = "rows/4", params = "rows : int64, n : int64")] +class MetalKqGemvIq4nl { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the 16B/sb d plane, half view - block bu's d at blk*8+bu + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // the same plane, byte view (binding-shape twin; unused) + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array // iq4nl quant plane, uint view (q40's 128B/sb nibble tiling) + @ssbo @binding = 3 @role = "read" @off = "xoff" @span = "n*4" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" @span = "rows*4" y : array + @uniform @binding = 5 ndim : uint + @uniform @binding = 6 ddim : uint + @workgroup lut : float[64] // kvalues_iq4nl as floats, one slot per thread of the 64-wide group + + [metal_kernel(name="metal_kq_gemv_iq4nl_msl")] + def metal_kq_gemv_iq4nl { + let lid = gl_LocalInvocationID.x + lut[lid] = iq4_lut(lid & 15u) + barrier() + let lane = gl_SubgroupInvocationID + let ix = lane / 16u + let it = lane % 16u + let bu = it / 2u + let il = it % 2u + let nb = ndim / 256u + let first_row = (gl_WorkGroupID.x * 2u + gl_SubgroupID) * 2u + var sumf : float[2] + var ib = ix + while (ib < nb) { + let yb4 = ib * 64u + bu * 8u + il * 2u + let y0 = x[yb4] + let y1 = x[yb4 + 1u] + let y2 = x[yb4 + 4u] + let y3 = x[yb4 + 5u] + for [unroll_full] (r in range(2)) { + let blk = (first_row + uint(r)) * nb + ib + let qb = blk * 32u + bu * 4u + il * 2u + let q0 = kqu[qb] + let q1 = kqu[qb + 1u] + var acc = y0 * float4(lut[q0 & 15u], lut[(q0 >> 8u) & 15u], lut[(q0 >> 16u) & 15u], lut[(q0 >> 24u) & 15u]) + acc += y2 * float4(lut[(q0 >> 4u) & 15u], lut[(q0 >> 12u) & 15u], lut[(q0 >> 20u) & 15u], lut[q0 >> 28u]) + acc += y1 * float4(lut[q1 & 15u], lut[(q1 >> 8u) & 15u], lut[(q1 >> 16u) & 15u], lut[(q1 >> 24u) & 15u]) + acc += y3 * float4(lut[(q1 >> 4u) & 15u], lut[(q1 >> 12u) & 15u], lut[(q1 >> 20u) & 15u], lut[q1 >> 28u]) + sumf[r] += float(kdh[blk * 8u + bu]) * (acc.x + acc.y + acc.z + acc.w) + } + ib += 2u + } + for [unroll_full] (r in range(2)) { + let s = simd_sum(sumf[r]) + if (lane == 0u && first_row + uint(r) < ddim) { + y[first_row + uint(r)] = s + } + } + } +} + +// IQ3_S: w = d*sc*(+-grid byte); the 2 KB grid in threadgroup memory, a +// 16-lane half-simdgroup per superblock, lane (bu, il) = block bu's half il: one qs word = 4 +// grid words = 16 elems = 4 CONSECUTIVE float4 x loads. Scale plane = the iq4xs/k6 split form. +[metal_dispatch(name = "enc_kq_iq3s_c", pso = "g_pso_kq_iq3s", tgmem = "metal_kq_gemv_iq3s_msl_tgmem", tg = 64, grid = "rows/8", params = "rows : int64, n : int64")] +class MetalKqGemvIq3s { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d plane — the scale buffer bound at byte nsb*16 (s0off = the caller's doff) + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // 16B strips: 8 signed sub-scales + 8 pad, byte view (same buffer at 0) + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array // iq3s quant plane, uint view (26 words per superblock: 16 qs, 2 qh, 8 signs) + @ssbo @binding = 3 @role = "read" @off = "xoff" @span = "n*4" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" @span = "rows*4" y : array + @uniform @binding = 5 ndim : uint + @uniform @binding = 6 ddim : uint + @workgroup gridf : float4[512] // magnitudes pre-expanded at stage time: the inner loop reads one float4 and flips signs, no byte extracts (the best of the eight raced forms, the reference exe's geometry included) + + [metal_kernel(name="metal_kq_gemv_iq3s_msl")] + def metal_kq_gemv_iq3s { + let lid = gl_LocalInvocationID.x + for [unroll_full] (k in range(8)) { + let gi = lid + uint(k) * 64u + let gw = iq3s_gw(int(gi)) + gridf[gi] = float4(float(gw & 255u), float((gw >> 8u) & 255u), float((gw >> 16u) & 255u), float(gw >> 24u)) + } + barrier() + let lane = gl_SubgroupInvocationID + let ix = lane / 16u + let it = lane % 16u + let bu = it / 2u + let il = it % 2u + let nb = ndim / 256u + let first_row = (gl_WorkGroupID.x * 2u + gl_SubgroupID) * 4u // 4 rows per simdgroup - y loads and grid gathers amortize x4 + var sumf : float[4] + var ib = ix + while (ib < nb) { + let yb4 = ib * 64u + bu * 8u + il * 4u + let y0 = x[yb4] + let y1 = x[yb4 + 1u] + let y2 = x[yb4 + 2u] + let y3 = x[yb4 + 3u] + for [unroll_full] (r in range(4)) { + let blk = (first_row + uint(r)) * nb + ib + let qsw = kqu[blk * 26u + bu * 2u + il] + let qh = (kqu[blk * 26u + 16u + bu / 4u] >> ((bu % 4u) * 8u)) & 255u + let sgw = kqu[blk * 26u + 18u + bu] + var accv = float4(0.0) + for [unroll_full] (wi in range(4)) { + let b = il * 4u + uint(wi) + let idx = ((qsw >> (8u * uint(wi))) & 255u) | ((qh << (8u - b)) & 256u) + let nib = (sgw >> ((b / 2u) * 8u + (b % 2u) * 4u)) & 15u + let fs = float4((nib & 1u) != 0u ? -1.0 : 1.0, (nib & 2u) != 0u ? -1.0 : 1.0, + (nib & 4u) != 0u ? -1.0 : 1.0, (nib & 8u) != 0u ? -1.0 : 1.0) + let yv = wi == 0 ? y0 : (wi == 1 ? y1 : (wi == 2 ? y2 : y3)) + accv += yv * (gridf[idx] * fs) + } + let sc = float((int(kscb[blk * 16u + bu]) ^ 128) - 128) + sumf[r] += float(kdh[blk]) * sc * (accv.x + accv.y + accv.z + accv.w) + } + ib += 2u + } + for [unroll_full] (r in range(4)) { + let s = simd_sum(sumf[r]) + if (lane == 0u && first_row + uint(r) < ddim) { + y[first_row + uint(r)] = s + } + } + } +} + +// IQ2_S: the iq3s GEMV geometry (4 rows per simdgroup, dispatch rows/8) over the u64 grid read +// DIRECT off the hoisted constant table; per-16 UNSIGNED strips scale each 16-element half, +// d pre-eighth-ed at transcode - the fold is d8 * strip * signed_grid_byte. +[metal_dispatch(name = "enc_kq_iq2s_c", pso = "g_pso_kq_iq2s", tg = 64, grid = "rows/8", params = "rows : int64, n : int64")] +class MetalKqGemvIq2s { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d8 plane - the scale buffer bound at byte nsb*16 (s0off = the caller's doff) + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // 16B strips: 16 UNSIGNED per-16 sub-scales, byte view (same buffer at 0) + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array // iq2s quant plane, uint view (18 words per superblock: 8 idx, 8 signs, 2 qh) + @ssbo @binding = 3 @role = "read" @off = "xoff" @span = "n*4" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" @span = "rows*4" y : array + @uniform @binding = 5 ndim : uint + @uniform @binding = 6 ddim : uint + + [metal_kernel(name="metal_kq_gemv_iq2s_msl")] + def metal_kq_gemv_iq2s { + let lane = gl_SubgroupInvocationID + let ix = lane / 16u + let it = lane % 16u + let bu = it / 2u + let il = it % 2u + let nb = ndim / 256u + let first_row = (gl_WorkGroupID.x * 2u + gl_SubgroupID) * 4u + var sumf : float[4] + var ib = ix + while (ib < nb) { + let yb4 = ib * 64u + bu * 8u + il * 4u + let y0 = x[yb4] + let y1 = x[yb4 + 1u] + let y2 = x[yb4 + 2u] + let y3 = x[yb4 + 3u] + for [unroll_full] (r in range(4)) { + let blk = (first_row + uint(r)) * nb + ib + let qw = kqu[blk * 18u + bu] + let sgw = kqu[blk * 18u + 8u + bu] + let qh = (kqu[blk * 18u + 16u + bu / 4u] >> ((bu % 4u) * 8u)) & 255u + var accv = float4(0.0) + for [unroll_full] (wi in range(2)) { + let l = il * 2u + uint(wi) + let idx2 = (((qw >> (8u * l)) & 255u) | ((qh << (8u - 2u * l)) & 0x300u)) * 2u + let sgn = (sgw >> (8u * l)) & 255u + let w0 = iq3s_sw(iq2s_gw(int(idx2)), sgn & 15u) + let w1 = iq3s_sw(iq2s_gw(int(idx2 + 1u)), sgn >> 4u) + let ya = wi == 0 ? y0 : y2 + let yb = wi == 0 ? y1 : y3 + accv += ya * float4(iq3s_sx(w0, 0u), iq3s_sx(w0, 1u), iq3s_sx(w0, 2u), iq3s_sx(w0, 3u)) + accv += yb * float4(iq3s_sx(w1, 0u), iq3s_sx(w1, 1u), iq3s_sx(w1, 2u), iq3s_sx(w1, 3u)) + } + let sc = float(kscb[blk * 16u + bu * 2u + il]) + sumf[r] += float(kdh[blk]) * sc * (accv.x + accv.y + accv.z + accv.w) + } + ib += 2u + } + for [unroll_full] (r in range(4)) { + let s = simd_sum(sumf[r]) + if (lane == 0u && first_row + uint(r) < ddim) { + y[first_row + uint(r)] = s + } + } + } +} + +// IQ2_XS: the iq2s GEMV geometry over the parity-signed u16 words - per (block, half) one u32 +// carries two u16 qs words; 9-bit index doubles into the grid pair, ksign7m recomputes the +// sign byte (no sign plane, no qh); per-16 UNSIGNED strips, d pre-eighth-ed. +[metal_dispatch(name = "enc_kq_iq2xs_c", pso = "g_pso_kq_iq2xs", tg = 64, grid = "rows/8", params = "rows : int64, n : int64")] +class MetalKqGemvIq2xs { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d8 plane - the scale buffer bound at byte nsb*16 (s0off = the caller's doff) + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // 16B strips: 16 UNSIGNED per-16 sub-scales, byte view (same buffer at 0) + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array // iq2xs quant plane, uint view (16 words per superblock: 32 u16 qs words) + @ssbo @binding = 3 @role = "read" @off = "xoff" @span = "n*4" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" @span = "rows*4" y : array + @uniform @binding = 5 ndim : uint + @uniform @binding = 6 ddim : uint + + [metal_kernel(name="metal_kq_gemv_iq2xs_msl")] + def metal_kq_gemv_iq2xs { + let lane = gl_SubgroupInvocationID + let ix = lane / 16u + let it = lane % 16u + let bu = it / 2u + let il = it % 2u + let nb = ndim / 256u + let first_row = (gl_WorkGroupID.x * 2u + gl_SubgroupID) * 4u + var sumf : float[4] + var ib = ix + while (ib < nb) { + let yb4 = ib * 64u + bu * 8u + il * 4u + let y0 = x[yb4] + let y1 = x[yb4 + 1u] + let y2 = x[yb4 + 2u] + let y3 = x[yb4 + 3u] + for [unroll_full] (r in range(4)) { + let blk = (first_row + uint(r)) * nb + ib + let qw = kqu[blk * 16u + bu * 2u + il] + var accv = float4(0.0) + for [unroll_full] (wi in range(2)) { + let w16 = (qw >> (16u * uint(wi))) & 0xFFFFu + let idx2 = (w16 & 511u) * 2u + let sgn = ksign7m(w16 >> 9u) + let w0 = iq3s_sw(iq2xs_gw(int(idx2)), sgn & 15u) + let w1 = iq3s_sw(iq2xs_gw(int(idx2 + 1u)), sgn >> 4u) + let ya = wi == 0 ? y0 : y2 + let yb = wi == 0 ? y1 : y3 + accv += ya * float4(iq3s_sx(w0, 0u), iq3s_sx(w0, 1u), iq3s_sx(w0, 2u), iq3s_sx(w0, 3u)) + accv += yb * float4(iq3s_sx(w1, 0u), iq3s_sx(w1, 1u), iq3s_sx(w1, 2u), iq3s_sx(w1, 3u)) + } + let sc = float(kscb[blk * 16u + bu * 2u + il]) + sumf[r] += float(kdh[blk]) * sc * (accv.x + accv.y + accv.z + accv.w) + } + ib += 2u + } + for [unroll_full] (r in range(4)) { + let s = simd_sum(sumf[r]) + if (lane == 0u && first_row + uint(r) < ddim) { + y[first_row + uint(r)] = s + } + } + } +} + +// IQ2_XXS: the iq2xs GEMV geometry over the aux32 form - per (block, half) the block's grid +// word supplies two BYTE indices (each doubling into the u64 grid pair), its aux32 the +// parity ksigns; ONE per-32 UNSIGNED strip scales the whole block on the eighth-ed d. +[metal_dispatch(name = "enc_kq_iq2xxs_c", pso = "g_pso_kq_iq2xxs", tg = 64, grid = "rows/8", params = "rows : int64, n : int64")] +class MetalKqGemvIq2xxs { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d8 plane - the scale buffer bound at byte nsb*16 (s0off = the caller's doff) + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // strip region: 8 UNSIGNED per-32 sub-scales + 8 pad, byte view (same buffer at 0) + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array // iq2xxs quant plane, uint view (16 words per superblock: [grid bytes][aux32] per block) + @ssbo @binding = 3 @role = "read" @off = "xoff" @span = "n*4" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" @span = "rows*4" y : array + @uniform @binding = 5 ndim : uint + @uniform @binding = 6 ddim : uint + + [metal_kernel(name="metal_kq_gemv_iq2xxs_msl")] + def metal_kq_gemv_iq2xxs { + let lane = gl_SubgroupInvocationID + let ix = lane / 16u + let it = lane % 16u + let bu = it / 2u + let il = it % 2u + let nb = ndim / 256u + let first_row = (gl_WorkGroupID.x * 2u + gl_SubgroupID) * 4u + var sumf : float[4] + var ib = ix + while (ib < nb) { + let yb4 = ib * 64u + bu * 8u + il * 4u + let y0 = x[yb4] + let y1 = x[yb4 + 1u] + let y2 = x[yb4 + 2u] + let y3 = x[yb4 + 3u] + for [unroll_full] (r in range(4)) { + let blk = (first_row + uint(r)) * nb + ib + let gw = kqu[blk * 16u + bu * 2u] + let aux = kqu[blk * 16u + bu * 2u + 1u] + var accv = float4(0.0) + for [unroll_full] (wi in range(2)) { + let q3 = il * 2u + uint(wi) + let bidx = (gw >> (8u * q3)) & 255u + let sgn = ksign7m((aux >> (7u * q3)) & 127u) + let w0 = iq3s_sw(iq2xxs_gw(int(bidx * 2u)), sgn & 15u) + let w1 = iq3s_sw(iq2xxs_gw(int(bidx * 2u + 1u)), sgn >> 4u) + let ya = wi == 0 ? y0 : y2 + let yb = wi == 0 ? y1 : y3 + accv += ya * float4(iq3s_sx(w0, 0u), iq3s_sx(w0, 1u), iq3s_sx(w0, 2u), iq3s_sx(w0, 3u)) + accv += yb * float4(iq3s_sx(w1, 0u), iq3s_sx(w1, 1u), iq3s_sx(w1, 2u), iq3s_sx(w1, 3u)) + } + let sc = float(kscb[blk * 16u + bu]) + sumf[r] += float(kdh[blk]) * sc * (accv.x + accv.y + accv.z + accv.w) + } + ib += 2u + } + for [unroll_full] (r in range(4)) { + let s = simd_sum(sumf[r]) + if (lane == 0u && first_row + uint(r) < ddim) { + y[first_row + uint(r)] = s + } + } + } +} + +// IQ3_XXS: the iq3s GEMV shape verbatim over the 24-word rows - the 1 KB halved grid as a +// float4 magnitude slab, signs from the block's aux32 through the parity helper (no table), +// the same (2ls+1)-strip x halved-d fold. 4 rows per simdgroup, dispatch rows/8. +[metal_dispatch(name = "enc_kq_iq3xxs_c", pso = "g_pso_kq_iq3xxs", tgmem = "metal_kq_gemv_iq3xxs_msl_tgmem", tg = 64, grid = "rows/8", params = "rows : int64, n : int64")] +class MetalKqGemvIq3xxs { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d plane — the scale buffer bound at byte nsb*16 (s0off = the caller's doff) + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // 16B strips: 8 sub-scale bytes (2ls+1) + 8 pad, byte view (same buffer at 0) + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array // iq3xxs quant plane, uint view (24 words per superblock: 16 qs, 8 aux) + @ssbo @binding = 3 @role = "read" @off = "xoff" @span = "n*4" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" @span = "rows*4" y : array + @uniform @binding = 5 ndim : uint + @uniform @binding = 6 ddim : uint + @workgroup gridf : float4[256] // magnitudes pre-expanded at stage time (the iq3s GEMV's winning form) + + [metal_kernel(name="metal_kq_gemv_iq3xxs_msl")] + def metal_kq_gemv_iq3xxs { + let lid = gl_LocalInvocationID.x + for [unroll_full] (k in range(4)) { + let gi = lid + uint(k) * 64u + let gw = iq3xxs_gw(int(gi)) + gridf[gi] = float4(float(gw & 255u), float((gw >> 8u) & 255u), float((gw >> 16u) & 255u), float(gw >> 24u)) + } + barrier() + let lane = gl_SubgroupInvocationID + let ix = lane / 16u + let it = lane % 16u + let bu = it / 2u + let il = it % 2u + let nb = ndim / 256u + let first_row = (gl_WorkGroupID.x * 2u + gl_SubgroupID) * 4u + var sumf : float[4] + var ib = ix + while (ib < nb) { + let yb4 = ib * 64u + bu * 8u + il * 4u + let y0 = x[yb4] + let y1 = x[yb4 + 1u] + let y2 = x[yb4 + 2u] + let y3 = x[yb4 + 3u] + for [unroll_full] (r in range(4)) { + let blk = (first_row + uint(r)) * nb + ib + let qsw = kqu[blk * 24u + bu * 2u + il] + let aux = kqu[blk * 24u + 16u + bu] + var accv = float4(0.0) + for [unroll_full] (wi in range(4)) { + let q3 = il * 2u + uint(wi) / 2u + let sgb = ksign7m((aux >> (7u * q3)) & 127u) + let nib = uint(wi) % 2u == 0u ? sgb & 15u : sgb >> 4u + let idx = (qsw >> (8u * uint(wi))) & 255u + let fs = float4((nib & 1u) != 0u ? -1.0 : 1.0, (nib & 2u) != 0u ? -1.0 : 1.0, + (nib & 4u) != 0u ? -1.0 : 1.0, (nib & 8u) != 0u ? -1.0 : 1.0) + let yv = wi == 0 ? y0 : (wi == 1 ? y1 : (wi == 2 ? y2 : y3)) + accv += yv * (gridf[idx] * fs) + } + let sc = float((int(kscb[blk * 16u + bu]) ^ 128) - 128) + sumf[r] += float(kdh[blk]) * sc * (accv.x + accv.y + accv.z + accv.w) + } + ib += 2u + } + for [unroll_full] (r in range(4)) { + let s = simd_sum(sumf[r]) + if (lane == 0u && first_row + uint(r) < ddim) { + y[first_row + uint(r)] = s + } + } + } +} + +// The iq3xxs B2/B4 pair: the iq3s shells over the 24-word rows - direct halved-grid reads, +// parity signs. +[ |> template_struct_instance] +class template MetalKqMvIq3xxsT { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d plane — the scale buffer bound at byte nsb*16 + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // 16B strips: 8 sub-scale bytes (2ls+1) + 8 pad (same buffer at 0) + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array // iq3xxs quant plane, uint view + @ssbo @binding = 3 @role = "read" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" y : array + @uniform @binding = 5 ka : KqMvArgs + @template_constant NR : int = 2 + @template_constant NRU : uint = 2u // NR's uint spelling (colbase math) — override BOTH together + @template_constant TILED : bool = false + + [metal_kernel] + def metal_kq_mv_iq3xxs { + let lane = gl_SubgroupInvocationID + let tx = lane % 8u + let ty = lane / 8u + let row = gl_WorkGroupID.x * 8u + gl_SubgroupID * 4u + ty + let colbase = gl_WorkGroupID.y * NRU // nolint:LINT002 dead in the TILED=false stamp — measured free + let nb = ka.ndim / 256u + let nb4 = ka.ndim / 4u + var sumf : float[NR] + var sb = 0u + while (sb < nb) { + let blk = row * nb + sb + let dl = float(kdh[blk]) * float((int(kscb[blk * 16u + tx]) ^ 128) - 128) + let aux = kqu[blk * 24u + 16u + tx] + var w : float4[8] + for [unroll_full] (h in range(2)) { + let qsw = kqu[blk * 24u + tx * 2u + uint(h)] + for [unroll_full] (wi in range(4)) { + let q3 = uint(h) * 2u + uint(wi) / 2u + let sgb = ksign7m((aux >> (7u * q3)) & 127u) + let nib = uint(wi) % 2u == 0u ? sgb & 15u : sgb >> 4u + let sw = iq3s_sw(iq3xxs_gw(int((qsw >> (8u * uint(wi))) & 255u)), nib) + w[h * 4 + wi] = float4(iq3s_sx(sw, 0u), iq3s_sx(sw, 1u), iq3s_sx(sw, 2u), iq3s_sx(sw, 3u)) * dl + } + } + let cx4 = sb * 64u + tx * 8u + static_if (TILED) { + for [unroll_full] (b in range(NR)) { + let xb4 = (colbase + uint(b)) * nb4 + cx4 + var acc = 0.0 + for [unroll_full] (u in range(8)) { + let v = x[xb4 + uint(u)] + acc += v.x * w[u].x + v.y * w[u].y + v.z * w[u].z + v.w * w[u].w + } + sumf[b] += acc + } + } else { + for [unroll_full] (b in range(NR)) { + let xb4 = uint(b) * nb4 + cx4 + var acc = 0.0 + for [unroll_full] (u in range(8)) { + let v = x[xb4 + uint(u)] + acc += v.x * w[u].x + v.y * w[u].y + v.z * w[u].z + v.w * w[u].w + } + sumf[b] += acc + } + } + sb++ + } + static_if (TILED) { + for [unroll_full] (b in range(NR)) { + var s = sumf[b] + s += simd_shuffle_down(s, 4u) + s += simd_shuffle_down(s, 2u) + s += simd_shuffle_down(s, 1u) + if (tx == 0u && row < ka.ddim && colbase + uint(b) < ka.nr) { + y[(colbase + uint(b)) * ka.ys + row] = s + } + } + } else { + for [unroll_full] (b in range(NR)) { + var s = sumf[b] + s += simd_shuffle_down(s, 4u) + s += simd_shuffle_down(s, 2u) + s += simd_shuffle_down(s, 1u) + if (tx == 0u && row < ka.ddim && uint(b) < ka.nr) { + y[uint(b) * ka.ys + row] = s + } + } + } + } +} + +[metal_dispatch(name = "enc_kq_mvb2_iq3xxs_c", pso = "g_pso_kq_mvb2_iq3xxs", tg = 64, grid = "rows/8", params = "rows : int64")] +class MetalKqMvB2Iq3xxs : MetalKqMvIq3xxsT { + override NR = 2 + override NRU = 2u + override TILED = false +} + +[metal_dispatch(name = "enc_kq_mvb4_iq3xxs_c", pso = "g_pso_kq_mvb4_iq3xxs", tg = 64, grid = "rows/8, gcols", params = "rows : int64, gcols : int64")] +class MetalKqMvB4Iq3xxs : MetalKqMvIq3xxsT { + override NR = 4 + override NRU = 4u + override TILED = true +} + +// the iq3xxs B8 twin: the iq3s B8 shell (one superblock's X panel staged per threadgroup) +// over the 24-word rows; the grid read direct. +[metal_dispatch(name = "enc_kq_mvb8_iq3xxs_c", pso = "g_pso_kq_mvb8_iq3xxs", tgmem = "metal_kq_mvb8_iq3xxs_msl_tgmem", tg = 64, grid = "rows/8", params = "rows : int64")] +class MetalKqMvB8Iq3xxs { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d plane — the scale buffer bound at byte nsb*16 + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // 16B strips: 8 sub-scale bytes (2ls+1) + 8 pad (same buffer at 0) + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array // iq3xxs quant plane, uint view + @ssbo @binding = 3 @role = "read" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" y : array + @uniform @binding = 5 ka : KqMvArgs + @workgroup txp : float4[512] // [8 streams x 64 float4] one superblock's X panel + + [metal_kernel(name="metal_kq_mvb8_iq3xxs_msl")] + def metal_kq_mvb8_iq3xxs { + let lid = gl_LocalInvocationID.x + let lane = gl_SubgroupInvocationID + let tx = lane % 8u + let ty = lane / 8u + let row = gl_WorkGroupID.x * 8u + gl_SubgroupID * 4u + ty + let nb = ka.ndim / 256u + let nb4 = ka.ndim / 4u + var acc : float4[8] + var sb = 0u + while (sb < nb) { + barrier() + for [unroll_full] (j in range(8)) { + let slot = uint(j) * 64u + lid + txp[slot] = x[(slot / 64u) * nb4 + sb * 64u + (slot % 64u)] + } + barrier() + let blk = row * nb + sb + let dl = float(kdh[blk]) * float((int(kscb[blk * 16u + tx]) ^ 128) - 128) + let aux = kqu[blk * 24u + 16u + tx] + var w : float4[8] + for [unroll_full] (h in range(2)) { + let qsw = kqu[blk * 24u + tx * 2u + uint(h)] + for [unroll_full] (wi in range(4)) { + let q3 = uint(h) * 2u + uint(wi) / 2u + let sgb = ksign7m((aux >> (7u * q3)) & 127u) + let nib = uint(wi) % 2u == 0u ? sgb & 15u : sgb >> 4u + let sw = iq3s_sw(iq3xxs_gw(int((qsw >> (8u * uint(wi))) & 255u)), nib) + w[h * 4 + wi] = float4(iq3s_sx(sw, 0u), iq3s_sx(sw, 1u), iq3s_sx(sw, 2u), iq3s_sx(sw, 3u)) * dl + } + } + let cxl = tx * 8u + for [unroll_full] (b in range(8)) { + for [unroll_full] (u in range(8)) { + acc[b] += txp[uint(b) * 64u + cxl + uint(u)] * w[u] + } + } + sb++ + } + for [unroll_full] (b in range(8)) { + var s = acc[b].x + acc[b].y + acc[b].z + acc[b].w + s += simd_shuffle_down(s, 4u) + s += simd_shuffle_down(s, 2u) + s += simd_shuffle_down(s, 1u) + if (tx == 0u && row < ka.ddim && uint(b) < ka.nr) { + y[uint(b) * ka.ys + row] = s + } + } + } +} + +// The iq3s B2/B4 pair: the iq4xs shell over the per-32-block lane map — lane tx owns block tx, +// 8 grid words = 32 consecutive elems as float4s; the grid read direct (unmeasured batch shapes). +[ |> template_struct_instance] +class template MetalKqMvIq3sT { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d plane — the scale buffer bound at byte nsb*16 + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // 16B strips: 8 signed sub-scales + 8 pad (same buffer at 0) + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array // iq3s quant plane, uint view + @ssbo @binding = 3 @role = "read" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" y : array + @uniform @binding = 5 ka : KqMvArgs + @template_constant NR : int = 2 + @template_constant NRU : uint = 2u // NR's uint spelling (colbase math) — override BOTH together + @template_constant TILED : bool = false + + [metal_kernel] + def metal_kq_mv_iq3s { + let lane = gl_SubgroupInvocationID + let tx = lane % 8u + let ty = lane / 8u + let row = gl_WorkGroupID.x * 8u + gl_SubgroupID * 4u + ty + let colbase = gl_WorkGroupID.y * NRU // nolint:LINT002 dead in the TILED=false stamp — measured free + let nb = ka.ndim / 256u + let nb4 = ka.ndim / 4u + var sumf : float[NR] + var sb = 0u + while (sb < nb) { + let blk = row * nb + sb + let dl = float(kdh[blk]) * float((int(kscb[blk * 16u + tx]) ^ 128) - 128) + let qh = (kqu[blk * 26u + 16u + tx / 4u] >> ((tx % 4u) * 8u)) & 255u + let sgw = kqu[blk * 26u + 18u + tx] + var w : float4[8] + for [unroll_full] (h in range(2)) { + let qsw = kqu[blk * 26u + tx * 2u + uint(h)] + for [unroll_full] (wi in range(4)) { + let b = uint(h) * 4u + uint(wi) + let idx = ((qsw >> (8u * uint(wi))) & 255u) | ((qh << (8u - b)) & 256u) + let nib = (sgw >> ((b / 2u) * 8u + (b % 2u) * 4u)) & 15u + let sw = iq3s_sw(iq3s_gw(int(idx)), nib) + w[h * 4 + wi] = float4(iq3s_sx(sw, 0u), iq3s_sx(sw, 1u), iq3s_sx(sw, 2u), iq3s_sx(sw, 3u)) * dl + } + } + let cx4 = sb * 64u + tx * 8u + static_if (TILED) { + for [unroll_full] (b in range(NR)) { + let xb4 = (colbase + uint(b)) * nb4 + cx4 + var acc = 0.0 + for [unroll_full] (u in range(8)) { + let v = x[xb4 + uint(u)] + acc += v.x * w[u].x + v.y * w[u].y + v.z * w[u].z + v.w * w[u].w + } + sumf[b] += acc + } + } else { + for [unroll_full] (b in range(NR)) { + let xb4 = uint(b) * nb4 + cx4 + var acc = 0.0 + for [unroll_full] (u in range(8)) { + let v = x[xb4 + uint(u)] + acc += v.x * w[u].x + v.y * w[u].y + v.z * w[u].z + v.w * w[u].w + } + sumf[b] += acc + } + } + sb++ + } + static_if (TILED) { + for [unroll_full] (b in range(NR)) { + var s = sumf[b] + s += simd_shuffle_down(s, 4u) + s += simd_shuffle_down(s, 2u) + s += simd_shuffle_down(s, 1u) + if (tx == 0u && row < ka.ddim && colbase + uint(b) < ka.nr) { + y[(colbase + uint(b)) * ka.ys + row] = s + } + } + } else { + for [unroll_full] (b in range(NR)) { + var s = sumf[b] + s += simd_shuffle_down(s, 4u) + s += simd_shuffle_down(s, 2u) + s += simd_shuffle_down(s, 1u) + if (tx == 0u && row < ka.ddim && uint(b) < ka.nr) { + y[uint(b) * ka.ys + row] = s + } + } + } + } +} + +[metal_dispatch(name = "enc_kq_mvb2_iq3s_c", pso = "g_pso_kq_mvb2_iq3s", tg = 64, grid = "rows/8", params = "rows : int64")] +class MetalKqMvB2Iq3s : MetalKqMvIq3sT { + override NR = 2 + override NRU = 2u + override TILED = false +} + +[metal_dispatch(name = "enc_kq_mvb4_iq3s_c", pso = "g_pso_kq_mvb4_iq3s", tg = 64, grid = "rows/8, gcols", params = "rows : int64, gcols : int64")] +class MetalKqMvB4Iq3s : MetalKqMvIq3sT { + override NR = 4 + override NRU = 4u + override TILED = true +} + +// the iq3s B8 twin: the iq4xs B8 shell (one superblock's X panel staged per threadgroup) over +// the per-32-block lane map; the grid read direct. +[metal_dispatch(name = "enc_kq_mvb8_iq3s_c", pso = "g_pso_kq_mvb8_iq3s", tgmem = "metal_kq_mvb8_iq3s_msl_tgmem", tg = 64, grid = "rows/8", params = "rows : int64")] +class MetalKqMvB8Iq3s { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d plane — the scale buffer bound at byte nsb*16 + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // 16B strips: 8 signed sub-scales + 8 pad (same buffer at 0) + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array // iq3s quant plane, uint view + @ssbo @binding = 3 @role = "read" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" y : array + @uniform @binding = 5 ka : KqMvArgs + @workgroup txp : float4[512] // [8 streams x 64 float4] one superblock's X panel + + [metal_kernel(name="metal_kq_mvb8_iq3s_msl")] + def metal_kq_mvb8_iq3s { + let lid = gl_LocalInvocationID.x + let lane = gl_SubgroupInvocationID + let tx = lane % 8u + let ty = lane / 8u + let row = gl_WorkGroupID.x * 8u + gl_SubgroupID * 4u + ty + let nb = ka.ndim / 256u + let nb4 = ka.ndim / 4u + var acc : float4[8] + var sb = 0u + while (sb < nb) { + barrier() + for [unroll_full] (j in range(8)) { + let slot = uint(j) * 64u + lid + txp[slot] = x[(slot / 64u) * nb4 + sb * 64u + (slot % 64u)] + } + barrier() + let blk = row * nb + sb + let dl = float(kdh[blk]) * float((int(kscb[blk * 16u + tx]) ^ 128) - 128) + let qh = (kqu[blk * 26u + 16u + tx / 4u] >> ((tx % 4u) * 8u)) & 255u + let sgw = kqu[blk * 26u + 18u + tx] + var w : float4[8] + for [unroll_full] (h in range(2)) { + let qsw = kqu[blk * 26u + tx * 2u + uint(h)] + for [unroll_full] (wi in range(4)) { + let b = uint(h) * 4u + uint(wi) + let idx = ((qsw >> (8u * uint(wi))) & 255u) | ((qh << (8u - b)) & 256u) + let nib = (sgw >> ((b / 2u) * 8u + (b % 2u) * 4u)) & 15u + let sw = iq3s_sw(iq3s_gw(int(idx)), nib) + w[h * 4 + wi] = float4(iq3s_sx(sw, 0u), iq3s_sx(sw, 1u), iq3s_sx(sw, 2u), iq3s_sx(sw, 3u)) * dl + } + } + let cxl = tx * 8u + for [unroll_full] (b in range(8)) { + for [unroll_full] (u in range(8)) { + acc[b] += txp[uint(b) * 64u + cxl + uint(u)] * w[u] + } + } + sb++ + } + for [unroll_full] (b in range(8)) { + var s = acc[b].x + acc[b].y + acc[b].z + acc[b].w + s += simd_shuffle_down(s, 4u) + s += simd_shuffle_down(s, 2u) + s += simd_shuffle_down(s, 1u) + if (tx == 0u && row < ka.ddim && uint(b) < ka.nr) { + y[uint(b) * ka.ys + row] = s + } + } + } +} + +// The iq2s B2/B4 pair: the iq3s shell over the u64 grid read direct - lane tx owns block tx, +// TWO grid words per 10-bit index; per-16 UNSIGNED strips give each half its own d8*strip. +[ |> template_struct_instance] +class template MetalKqMvIq2sT { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d8 plane - the scale buffer bound at byte nsb*16 + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // 16B strips: 16 UNSIGNED per-16 sub-scales (same buffer at 0) + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array // iq2s quant plane, uint view + @ssbo @binding = 3 @role = "read" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" y : array + @uniform @binding = 5 ka : KqMvArgs + @template_constant NR : int = 2 + @template_constant NRU : uint = 2u // NR's uint spelling (colbase math) - override BOTH together + @template_constant TILED : bool = false + + [metal_kernel] + def metal_kq_mv_iq2s { + let lane = gl_SubgroupInvocationID + let tx = lane % 8u + let ty = lane / 8u + let row = gl_WorkGroupID.x * 8u + gl_SubgroupID * 4u + ty + let colbase = gl_WorkGroupID.y * NRU // nolint:LINT002 dead in the TILED=false stamp - measured free + let nb = ka.ndim / 256u + let nb4 = ka.ndim / 4u + var sumf : float[NR] + var sb = 0u + while (sb < nb) { + let blk = row * nb + sb + let dl0 = float(kdh[blk]) * float(kscb[blk * 16u + tx * 2u]) + let dl1 = float(kdh[blk]) * float(kscb[blk * 16u + tx * 2u + 1u]) + let qw = kqu[blk * 18u + tx] + let sgw = kqu[blk * 18u + 8u + tx] + let qh = (kqu[blk * 18u + 16u + tx / 4u] >> ((tx % 4u) * 8u)) & 255u + var w : float4[8] + for [unroll_full] (l in range(4)) { + let lu = uint(l) + let idx2 = (((qw >> (8u * lu)) & 255u) | ((qh << (8u - 2u * lu)) & 0x300u)) * 2u + let sgn = (sgw >> (8u * lu)) & 255u + let dl = l < 2 ? dl0 : dl1 + let sw0 = iq3s_sw(iq2s_gw(int(idx2)), sgn & 15u) + let sw1 = iq3s_sw(iq2s_gw(int(idx2 + 1u)), sgn >> 4u) + w[l * 2] = float4(iq3s_sx(sw0, 0u), iq3s_sx(sw0, 1u), iq3s_sx(sw0, 2u), iq3s_sx(sw0, 3u)) * dl + w[l * 2 + 1] = float4(iq3s_sx(sw1, 0u), iq3s_sx(sw1, 1u), iq3s_sx(sw1, 2u), iq3s_sx(sw1, 3u)) * dl + } + let cx4 = sb * 64u + tx * 8u + static_if (TILED) { + for [unroll_full] (b in range(NR)) { + let xb4 = (colbase + uint(b)) * nb4 + cx4 + var acc = 0.0 + for [unroll_full] (u in range(8)) { + let v = x[xb4 + uint(u)] + acc += v.x * w[u].x + v.y * w[u].y + v.z * w[u].z + v.w * w[u].w + } + sumf[b] += acc + } + } else { + for [unroll_full] (b in range(NR)) { + let xb4 = uint(b) * nb4 + cx4 + var acc = 0.0 + for [unroll_full] (u in range(8)) { + let v = x[xb4 + uint(u)] + acc += v.x * w[u].x + v.y * w[u].y + v.z * w[u].z + v.w * w[u].w + } + sumf[b] += acc + } + } + sb++ + } + for [unroll_full] (b in range(NR)) { + var s = sumf[b] + s += simd_shuffle_down(s, 4u) + s += simd_shuffle_down(s, 2u) + s += simd_shuffle_down(s, 1u) + if (tx == 0u && row < ka.ddim && uint(b) < ka.nr) { + y[uint(b) * ka.ys + row] = s + } + } + } +} + +[metal_dispatch(name = "enc_kq_mvb2_iq2s_c", pso = "g_pso_kq_mvb2_iq2s", tg = 64, grid = "rows/8", params = "rows : int64")] +class MetalKqMvB2Iq2s : MetalKqMvIq2sT { + override NR = 2 + override NRU = 2u + override TILED = false +} + +[metal_dispatch(name = "enc_kq_mvb4_iq2s_c", pso = "g_pso_kq_mvb4_iq2s", tg = 64, grid = "rows/8, gcols", params = "rows : int64, gcols : int64")] +class MetalKqMvB4Iq2s : MetalKqMvIq2sT { + override NR = 4 + override NRU = 4u + override TILED = true +} + +// the iq2s B8 twin: the iq3s B8 shell (one superblock's X panel staged per threadgroup) with +// the u64 two-word compose; the grid read direct. +[metal_dispatch(name = "enc_kq_mvb8_iq2s_c", pso = "g_pso_kq_mvb8_iq2s", tgmem = "metal_kq_mvb8_iq2s_msl_tgmem", tg = 64, grid = "rows/8", params = "rows : int64")] +class MetalKqMvB8Iq2s { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d8 plane - the scale buffer bound at byte nsb*16 + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // 16B strips: 16 UNSIGNED per-16 sub-scales (same buffer at 0) + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array // iq2s quant plane, uint view + @ssbo @binding = 3 @role = "read" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" y : array + @uniform @binding = 5 ka : KqMvArgs + @workgroup txp : float4[512] // [8 streams x 64 float4] one superblock's X panel + + [metal_kernel(name="metal_kq_mvb8_iq2s_msl")] + def metal_kq_mvb8_iq2s { + let lid = gl_LocalInvocationID.x + let lane = gl_SubgroupInvocationID + let tx = lane % 8u + let ty = lane / 8u + let row = gl_WorkGroupID.x * 8u + gl_SubgroupID * 4u + ty + let nb = ka.ndim / 256u + let nb4 = ka.ndim / 4u + var acc : float4[8] + var sb = 0u + while (sb < nb) { + barrier() + for [unroll_full] (j in range(8)) { + let slot = uint(j) * 64u + lid + txp[slot] = x[(slot / 64u) * nb4 + sb * 64u + (slot % 64u)] + } + barrier() + let blk = row * nb + sb + let dl0 = float(kdh[blk]) * float(kscb[blk * 16u + tx * 2u]) + let dl1 = float(kdh[blk]) * float(kscb[blk * 16u + tx * 2u + 1u]) + let qw = kqu[blk * 18u + tx] + let sgw = kqu[blk * 18u + 8u + tx] + let qh = (kqu[blk * 18u + 16u + tx / 4u] >> ((tx % 4u) * 8u)) & 255u + var w : float4[8] + for [unroll_full] (l in range(4)) { + let lu = uint(l) + let idx2 = (((qw >> (8u * lu)) & 255u) | ((qh << (8u - 2u * lu)) & 0x300u)) * 2u + let sgn = (sgw >> (8u * lu)) & 255u + let dl = l < 2 ? dl0 : dl1 + let sw0 = iq3s_sw(iq2s_gw(int(idx2)), sgn & 15u) + let sw1 = iq3s_sw(iq2s_gw(int(idx2 + 1u)), sgn >> 4u) + w[l * 2] = float4(iq3s_sx(sw0, 0u), iq3s_sx(sw0, 1u), iq3s_sx(sw0, 2u), iq3s_sx(sw0, 3u)) * dl + w[l * 2 + 1] = float4(iq3s_sx(sw1, 0u), iq3s_sx(sw1, 1u), iq3s_sx(sw1, 2u), iq3s_sx(sw1, 3u)) * dl + } + let cxl = tx * 8u + for [unroll_full] (b in range(8)) { + for [unroll_full] (u in range(8)) { + acc[b] += txp[uint(b) * 64u + cxl + uint(u)] * w[u] + } + } + sb++ + } + for [unroll_full] (b in range(8)) { + var s = acc[b].x + acc[b].y + acc[b].z + acc[b].w + s += simd_shuffle_down(s, 4u) + s += simd_shuffle_down(s, 2u) + s += simd_shuffle_down(s, 1u) + if (tx == 0u && row < ka.ddim && uint(b) < ka.nr) { + y[uint(b) * ka.ys + row] = s + } + } + } +} + +// The iq2xs B2/B4 pair: the iq2s shell with the parity-signed u16 walk - lane tx owns block +// tx; two u32 qs words per block, ksign7m per u16; per-16 UNSIGNED strips as dl0/dl1. +[ |> template_struct_instance] +class template MetalKqMvIq2xsT { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d8 plane - the scale buffer bound at byte nsb*16 + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // 16B strips: 16 UNSIGNED per-16 sub-scales (same buffer at 0) + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array // iq2xs quant plane, uint view + @ssbo @binding = 3 @role = "read" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" y : array + @uniform @binding = 5 ka : KqMvArgs + @template_constant NR : int = 2 + @template_constant NRU : uint = 2u // NR's uint spelling (colbase math) - override BOTH together + @template_constant TILED : bool = false + + [metal_kernel] + def metal_kq_mv_iq2xs { + let lane = gl_SubgroupInvocationID + let tx = lane % 8u + let ty = lane / 8u + let row = gl_WorkGroupID.x * 8u + gl_SubgroupID * 4u + ty + let colbase = gl_WorkGroupID.y * NRU // nolint:LINT002 dead in the TILED=false stamp - measured free + let nb = ka.ndim / 256u + let nb4 = ka.ndim / 4u + var sumf : float[NR] + var sb = 0u + while (sb < nb) { + let blk = row * nb + sb + let dl0 = float(kdh[blk]) * float(kscb[blk * 16u + tx * 2u]) + let dl1 = float(kdh[blk]) * float(kscb[blk * 16u + tx * 2u + 1u]) + let qw0 = kqu[blk * 16u + tx * 2u] + let qw1 = kqu[blk * 16u + tx * 2u + 1u] + var w : float4[8] + for [unroll_full] (l in range(4)) { + let lu = uint(l) + let w16 = ((l < 2 ? qw0 : qw1) >> (16u * (lu & 1u))) & 0xFFFFu + let idx2 = (w16 & 511u) * 2u + let sgn = ksign7m(w16 >> 9u) + let dl = l < 2 ? dl0 : dl1 + let sw0 = iq3s_sw(iq2xs_gw(int(idx2)), sgn & 15u) + let sw1 = iq3s_sw(iq2xs_gw(int(idx2 + 1u)), sgn >> 4u) + w[l * 2] = float4(iq3s_sx(sw0, 0u), iq3s_sx(sw0, 1u), iq3s_sx(sw0, 2u), iq3s_sx(sw0, 3u)) * dl + w[l * 2 + 1] = float4(iq3s_sx(sw1, 0u), iq3s_sx(sw1, 1u), iq3s_sx(sw1, 2u), iq3s_sx(sw1, 3u)) * dl + } + let cx4 = sb * 64u + tx * 8u + static_if (TILED) { + for [unroll_full] (b in range(NR)) { + let xb4 = (colbase + uint(b)) * nb4 + cx4 + var acc = 0.0 + for [unroll_full] (u in range(8)) { + let v = x[xb4 + uint(u)] + acc += v.x * w[u].x + v.y * w[u].y + v.z * w[u].z + v.w * w[u].w + } + sumf[b] += acc + } + } else { + for [unroll_full] (b in range(NR)) { + let xb4 = uint(b) * nb4 + cx4 + var acc = 0.0 + for [unroll_full] (u in range(8)) { + let v = x[xb4 + uint(u)] + acc += v.x * w[u].x + v.y * w[u].y + v.z * w[u].z + v.w * w[u].w + } + sumf[b] += acc + } + } + sb++ + } + for [unroll_full] (b in range(NR)) { + var s = sumf[b] + s += simd_shuffle_down(s, 4u) + s += simd_shuffle_down(s, 2u) + s += simd_shuffle_down(s, 1u) + if (tx == 0u && row < ka.ddim && uint(b) < ka.nr) { + y[uint(b) * ka.ys + row] = s + } + } + } +} + +[metal_dispatch(name = "enc_kq_mvb2_iq2xs_c", pso = "g_pso_kq_mvb2_iq2xs", tg = 64, grid = "rows/8", params = "rows : int64")] +class MetalKqMvB2Iq2xs : MetalKqMvIq2xsT { + override NR = 2 + override NRU = 2u + override TILED = false +} + +[metal_dispatch(name = "enc_kq_mvb4_iq2xs_c", pso = "g_pso_kq_mvb4_iq2xs", tg = 64, grid = "rows/8, gcols", params = "rows : int64, gcols : int64")] +class MetalKqMvB4Iq2xs : MetalKqMvIq2xsT { + override NR = 4 + override NRU = 4u + override TILED = true +} + +// the iq2xs B8 twin: the iq2s B8 shell (one superblock's X panel staged per threadgroup) with +// the parity-signed u16 compose; the grid read direct. +[metal_dispatch(name = "enc_kq_mvb8_iq2xs_c", pso = "g_pso_kq_mvb8_iq2xs", tgmem = "metal_kq_mvb8_iq2xs_msl_tgmem", tg = 64, grid = "rows/8", params = "rows : int64")] +class MetalKqMvB8Iq2xs { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d8 plane - the scale buffer bound at byte nsb*16 + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // 16B strips: 16 UNSIGNED per-16 sub-scales (same buffer at 0) + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array // iq2xs quant plane, uint view + @ssbo @binding = 3 @role = "read" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" y : array + @uniform @binding = 5 ka : KqMvArgs + @workgroup txp : float4[512] // [8 streams x 64 float4] one superblock's X panel + + [metal_kernel(name="metal_kq_mvb8_iq2xs_msl")] + def metal_kq_mvb8_iq2xs { + let lid = gl_LocalInvocationID.x + let lane = gl_SubgroupInvocationID + let tx = lane % 8u + let ty = lane / 8u + let row = gl_WorkGroupID.x * 8u + gl_SubgroupID * 4u + ty + let nb = ka.ndim / 256u + let nb4 = ka.ndim / 4u + var acc : float4[8] + var sb = 0u + while (sb < nb) { + barrier() + for [unroll_full] (j in range(8)) { + let slot = uint(j) * 64u + lid + txp[slot] = x[(slot / 64u) * nb4 + sb * 64u + (slot % 64u)] + } + barrier() + let blk = row * nb + sb + let dl0 = float(kdh[blk]) * float(kscb[blk * 16u + tx * 2u]) + let dl1 = float(kdh[blk]) * float(kscb[blk * 16u + tx * 2u + 1u]) + let qw0 = kqu[blk * 16u + tx * 2u] + let qw1 = kqu[blk * 16u + tx * 2u + 1u] + var w : float4[8] + for [unroll_full] (l in range(4)) { + let lu = uint(l) + let w16 = ((l < 2 ? qw0 : qw1) >> (16u * (lu & 1u))) & 0xFFFFu + let idx2 = (w16 & 511u) * 2u + let sgn = ksign7m(w16 >> 9u) + let dl = l < 2 ? dl0 : dl1 + let sw0 = iq3s_sw(iq2xs_gw(int(idx2)), sgn & 15u) + let sw1 = iq3s_sw(iq2xs_gw(int(idx2 + 1u)), sgn >> 4u) + w[l * 2] = float4(iq3s_sx(sw0, 0u), iq3s_sx(sw0, 1u), iq3s_sx(sw0, 2u), iq3s_sx(sw0, 3u)) * dl + w[l * 2 + 1] = float4(iq3s_sx(sw1, 0u), iq3s_sx(sw1, 1u), iq3s_sx(sw1, 2u), iq3s_sx(sw1, 3u)) * dl + } + let cxl = tx * 8u + for [unroll_full] (b in range(8)) { + for [unroll_full] (u in range(8)) { + acc[b] += txp[uint(b) * 64u + cxl + uint(u)] * w[u] + } + } + sb++ + } + for [unroll_full] (b in range(8)) { + var s = acc[b].x + acc[b].y + acc[b].z + acc[b].w + s += simd_shuffle_down(s, 4u) + s += simd_shuffle_down(s, 2u) + s += simd_shuffle_down(s, 1u) + if (tx == 0u && row < ka.ddim && uint(b) < ka.nr) { + y[uint(b) * ka.ys + row] = s + } + } + } +} + +// The iq2xxs B2/B4 pair: the iq2xs shell over the aux32 walk - lane tx owns block tx; the +// block's grid word supplies four byte indices, ksign7m per index; ONE per-32 strip as dl. +[ |> template_struct_instance] +class template MetalKqMvIq2xxsT { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d8 plane - the scale buffer bound at byte nsb*16 + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // strip region: 8 UNSIGNED per-32 sub-scales + 8 pad (same buffer at 0) + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array // iq2xxs quant plane, uint view + @ssbo @binding = 3 @role = "read" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" y : array + @uniform @binding = 5 ka : KqMvArgs + @template_constant NR : int = 2 + @template_constant NRU : uint = 2u // NR's uint spelling (colbase math) - override BOTH together + @template_constant TILED : bool = false + + [metal_kernel] + def metal_kq_mv_iq2xxs { + let lane = gl_SubgroupInvocationID + let tx = lane % 8u + let ty = lane / 8u + let row = gl_WorkGroupID.x * 8u + gl_SubgroupID * 4u + ty + let colbase = gl_WorkGroupID.y * NRU // nolint:LINT002 dead in the TILED=false stamp - measured free + let nb = ka.ndim / 256u + let nb4 = ka.ndim / 4u + var sumf : float[NR] + var sb = 0u + while (sb < nb) { + let blk = row * nb + sb + let dl = float(kdh[blk]) * float(kscb[blk * 16u + tx]) + let gw = kqu[blk * 16u + tx * 2u] + let aux = kqu[blk * 16u + tx * 2u + 1u] + var w : float4[8] + for [unroll_full] (l in range(4)) { + let lu = uint(l) + let bidx = (gw >> (8u * lu)) & 255u + let sgn = ksign7m((aux >> (7u * lu)) & 127u) + let sw0 = iq3s_sw(iq2xxs_gw(int(bidx * 2u)), sgn & 15u) + let sw1 = iq3s_sw(iq2xxs_gw(int(bidx * 2u + 1u)), sgn >> 4u) + w[l * 2] = float4(iq3s_sx(sw0, 0u), iq3s_sx(sw0, 1u), iq3s_sx(sw0, 2u), iq3s_sx(sw0, 3u)) * dl + w[l * 2 + 1] = float4(iq3s_sx(sw1, 0u), iq3s_sx(sw1, 1u), iq3s_sx(sw1, 2u), iq3s_sx(sw1, 3u)) * dl + } + let cx4 = sb * 64u + tx * 8u + static_if (TILED) { + for [unroll_full] (b in range(NR)) { + let xb4 = (colbase + uint(b)) * nb4 + cx4 + var acc = 0.0 + for [unroll_full] (u in range(8)) { + let v = x[xb4 + uint(u)] + acc += v.x * w[u].x + v.y * w[u].y + v.z * w[u].z + v.w * w[u].w + } + sumf[b] += acc + } + } else { + for [unroll_full] (b in range(NR)) { + let xb4 = uint(b) * nb4 + cx4 + var acc = 0.0 + for [unroll_full] (u in range(8)) { + let v = x[xb4 + uint(u)] + acc += v.x * w[u].x + v.y * w[u].y + v.z * w[u].z + v.w * w[u].w + } + sumf[b] += acc + } + } + sb++ + } + for [unroll_full] (b in range(NR)) { + var s = sumf[b] + s += simd_shuffle_down(s, 4u) + s += simd_shuffle_down(s, 2u) + s += simd_shuffle_down(s, 1u) + if (tx == 0u && row < ka.ddim && uint(b) < ka.nr) { + y[uint(b) * ka.ys + row] = s + } + } + } +} + +[metal_dispatch(name = "enc_kq_mvb2_iq2xxs_c", pso = "g_pso_kq_mvb2_iq2xxs", tg = 64, grid = "rows/8", params = "rows : int64")] +class MetalKqMvB2Iq2xxs : MetalKqMvIq2xxsT { + override NR = 2 + override NRU = 2u + override TILED = false +} + +[metal_dispatch(name = "enc_kq_mvb4_iq2xxs_c", pso = "g_pso_kq_mvb4_iq2xxs", tg = 64, grid = "rows/8, gcols", params = "rows : int64, gcols : int64")] +class MetalKqMvB4Iq2xxs : MetalKqMvIq2xxsT { + override NR = 4 + override NRU = 4u + override TILED = true +} + +// the iq2xxs B8 twin: the iq2xs B8 shell (one superblock's X panel staged per threadgroup) +// with the aux32 compose; the grid read direct. +[metal_dispatch(name = "enc_kq_mvb8_iq2xxs_c", pso = "g_pso_kq_mvb8_iq2xxs", tgmem = "metal_kq_mvb8_iq2xxs_msl_tgmem", tg = 64, grid = "rows/8", params = "rows : int64")] +class MetalKqMvB8Iq2xxs { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d8 plane - the scale buffer bound at byte nsb*16 + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // strip region: 8 UNSIGNED per-32 sub-scales + 8 pad (same buffer at 0) + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array // iq2xxs quant plane, uint view + @ssbo @binding = 3 @role = "read" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" y : array + @uniform @binding = 5 ka : KqMvArgs + @workgroup txp : float4[512] // [8 streams x 64 float4] one superblock's X panel + + [metal_kernel(name="metal_kq_mvb8_iq2xxs_msl")] + def metal_kq_mvb8_iq2xxs { + let lid = gl_LocalInvocationID.x + let lane = gl_SubgroupInvocationID + let tx = lane % 8u + let ty = lane / 8u + let row = gl_WorkGroupID.x * 8u + gl_SubgroupID * 4u + ty + let nb = ka.ndim / 256u + let nb4 = ka.ndim / 4u + var acc : float4[8] + var sb = 0u + while (sb < nb) { + barrier() + for [unroll_full] (j in range(8)) { + let slot = uint(j) * 64u + lid + txp[slot] = x[(slot / 64u) * nb4 + sb * 64u + (slot % 64u)] + } + barrier() + let blk = row * nb + sb + let dl = float(kdh[blk]) * float(kscb[blk * 16u + tx]) + let gw = kqu[blk * 16u + tx * 2u] + let aux = kqu[blk * 16u + tx * 2u + 1u] + var w : float4[8] + for [unroll_full] (l in range(4)) { + let lu = uint(l) + let bidx = (gw >> (8u * lu)) & 255u + let sgn = ksign7m((aux >> (7u * lu)) & 127u) + let sw0 = iq3s_sw(iq2xxs_gw(int(bidx * 2u)), sgn & 15u) + let sw1 = iq3s_sw(iq2xxs_gw(int(bidx * 2u + 1u)), sgn >> 4u) + w[l * 2] = float4(iq3s_sx(sw0, 0u), iq3s_sx(sw0, 1u), iq3s_sx(sw0, 2u), iq3s_sx(sw0, 3u)) * dl + w[l * 2 + 1] = float4(iq3s_sx(sw1, 0u), iq3s_sx(sw1, 1u), iq3s_sx(sw1, 2u), iq3s_sx(sw1, 3u)) * dl + } + let cxl = tx * 8u + for [unroll_full] (b in range(8)) { + for [unroll_full] (u in range(8)) { + acc[b] += txp[uint(b) * 64u + cxl + uint(u)] * w[u] + } + } + sb++ + } + for [unroll_full] (b in range(8)) { + var s = acc[b].x + acc[b].y + acc[b].z + acc[b].w + s += simd_shuffle_down(s, 4u) + s += simd_shuffle_down(s, 2u) + s += simd_shuffle_down(s, 1u) + if (tx == 0u && row < ka.ddim && uint(b) < ka.nr) { + y[uint(b) * ka.ys + row] = s + } + } + } +} + +// The iq4xs B2/B4 pair: the K4T shell (row/colbase/TILED twins) over the per-32-block lane map of +// the GEMV above — lane tx owns block tx of each superblock, 16 lo + 16 hi weights as float4s. +[ |> template_struct_instance] +class template MetalKqMvIq4xsT { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d plane — the scale buffer bound at byte nsb*16 + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // 16B strips: 8 signed sub-scales + 8 pad (same buffer at 0) + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array // iq4xs quant plane, uint view + @ssbo @binding = 3 @role = "read" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" y : array + @uniform @binding = 5 ka : KqMvArgs + @template_constant NR : int = 2 + @template_constant NRU : uint = 2u // NR's uint spelling (colbase math) — override BOTH together + @template_constant TILED : bool = false + + [metal_kernel] + def metal_kq_mv_iq4xs { + let lane = gl_SubgroupInvocationID + let tx = lane % 8u + let ty = lane / 8u + let row = gl_WorkGroupID.x * 8u + gl_SubgroupID * 4u + ty + let colbase = gl_WorkGroupID.y * NRU // nolint:LINT002 dead in the TILED=false stamp — measured free + let nb = ka.ndim / 256u + let nb4 = ka.ndim / 4u + var sumf : float[NR] + var sb = 0u + while (sb < nb) { + let blk = row * nb + sb + let dl = float(kdh[blk]) * float((int(kscb[blk * 16u + tx]) ^ 128) - 128) + let qb = blk * 32u + tx * 4u + var wl : float4[4] + var wh : float4[4] + for [unroll_full] (u in range(4)) { + let qv = kqu[qb + uint(u)] + wl[u] = float4(iq4_lut(qv & 15u), iq4_lut((qv >> 8u) & 15u), iq4_lut((qv >> 16u) & 15u), iq4_lut((qv >> 24u) & 15u)) * dl + wh[u] = float4(iq4_lut((qv >> 4u) & 15u), iq4_lut((qv >> 12u) & 15u), iq4_lut((qv >> 20u) & 15u), iq4_lut(qv >> 28u)) * dl + } + let cx4 = sb * 64u + tx * 8u + static_if (TILED) { + for [unroll_full] (b in range(NR)) { + let xb4 = (colbase + uint(b)) * nb4 + cx4 + var acc = 0.0 + for [unroll_full] (u in range(4)) { + let v = x[xb4 + uint(u)] + let vh = x[xb4 + 4u + uint(u)] + acc += v.x * wl[u].x + v.y * wl[u].y + v.z * wl[u].z + v.w * wl[u].w + acc += vh.x * wh[u].x + vh.y * wh[u].y + vh.z * wh[u].z + vh.w * wh[u].w + } + sumf[b] += acc + } + } else { + for [unroll_full] (b in range(NR)) { + let xb4 = uint(b) * nb4 + cx4 + var acc = 0.0 + for [unroll_full] (u in range(4)) { + let v = x[xb4 + uint(u)] + let vh = x[xb4 + 4u + uint(u)] + acc += v.x * wl[u].x + v.y * wl[u].y + v.z * wl[u].z + v.w * wl[u].w + acc += vh.x * wh[u].x + vh.y * wh[u].y + vh.z * wh[u].z + vh.w * wh[u].w + } + sumf[b] += acc + } + } + sb++ + } + static_if (TILED) { + for [unroll_full] (b in range(NR)) { + var s = sumf[b] + s += simd_shuffle_down(s, 4u) + s += simd_shuffle_down(s, 2u) + s += simd_shuffle_down(s, 1u) + if (tx == 0u && row < ka.ddim && colbase + uint(b) < ka.nr) { + y[(colbase + uint(b)) * ka.ys + row] = s + } + } + } else { + for [unroll_full] (b in range(NR)) { + var s = sumf[b] + s += simd_shuffle_down(s, 4u) + s += simd_shuffle_down(s, 2u) + s += simd_shuffle_down(s, 1u) + if (tx == 0u && row < ka.ddim && uint(b) < ka.nr) { + y[uint(b) * ka.ys + row] = s + } + } + } + } +} + +[metal_dispatch(name = "enc_kq_mvb2_iq4xs_c", pso = "g_pso_kq_mvb2_iq4xs", tg = 64, grid = "rows/8", params = "rows : int64")] +class MetalKqMvB2Iq4xs : MetalKqMvIq4xsT { + override NR = 2 + override NRU = 2u + override TILED = false +} + +[metal_dispatch(name = "enc_kq_mvb4_iq4xs_c", pso = "g_pso_kq_mvb4_iq4xs", tg = 64, grid = "rows/8, gcols", params = "rows : int64, gcols : int64")] +class MetalKqMvB4Iq4xs : MetalKqMvIq4xsT { + override NR = 4 + override NRU = 4u + override TILED = true +} + +// the iq4xs B8 twin: the K45 B8 shell (one superblock's X panel staged per threadgroup) over the +// per-32-block lane map +[metal_dispatch(name = "enc_kq_mvb8_iq4xs_c", pso = "g_pso_kq_mvb8_iq4xs", tgmem = "metal_kq_mvb8_iq4xs_msl_tgmem", tg = 64, grid = "rows/8", params = "rows : int64")] +class MetalKqMvB8Iq4xs { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d plane — the scale buffer bound at byte nsb*16 + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // 16B strips: 8 signed sub-scales + 8 pad (same buffer at 0) + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array // iq4xs quant plane, uint view + @ssbo @binding = 3 @role = "read" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" y : array + @uniform @binding = 5 ka : KqMvArgs + @workgroup txp : float4[512] // [8 streams x 64 float4] one superblock's X panel + + [metal_kernel(name="metal_kq_mvb8_iq4xs_msl")] + def metal_kq_mvb8_iq4xs { + let lid = gl_LocalInvocationID.x + let lane = gl_SubgroupInvocationID + let tx = lane % 8u + let ty = lane / 8u + let row = gl_WorkGroupID.x * 8u + gl_SubgroupID * 4u + ty + let nb = ka.ndim / 256u + let nb4 = ka.ndim / 4u + var acc : float4[8] + var sb = 0u + while (sb < nb) { + barrier() + for [unroll_full] (j in range(8)) { + let slot = uint(j) * 64u + lid + txp[slot] = x[(slot / 64u) * nb4 + sb * 64u + (slot % 64u)] + } + barrier() + let blk = row * nb + sb + let dl = float(kdh[blk]) * float((int(kscb[blk * 16u + tx]) ^ 128) - 128) + let qb = blk * 32u + tx * 4u + var wl : float4[4] + var wh : float4[4] + for [unroll_full] (u in range(4)) { + let qv = kqu[qb + uint(u)] + wl[u] = float4(iq4_lut(qv & 15u), iq4_lut((qv >> 8u) & 15u), iq4_lut((qv >> 16u) & 15u), iq4_lut((qv >> 24u) & 15u)) * dl + wh[u] = float4(iq4_lut((qv >> 4u) & 15u), iq4_lut((qv >> 12u) & 15u), iq4_lut((qv >> 20u) & 15u), iq4_lut(qv >> 28u)) * dl + } + let cxl = tx * 8u + for [unroll_full] (b in range(8)) { + for [unroll_full] (u in range(4)) { + acc[b] += txp[uint(b) * 64u + cxl + uint(u)] * wl[u] + txp[uint(b) * 64u + cxl + 4u + uint(u)] * wh[u] + } + } + sb++ + } + for [unroll_full] (b in range(8)) { + var s = acc[b].x + acc[b].y + acc[b].z + acc[b].w + s += simd_shuffle_down(s, 4u) + s += simd_shuffle_down(s, 2u) + s += simd_shuffle_down(s, 1u) + if (tx == 0u && row < ka.ddim && uint(b) < ka.nr) { + y[uint(b) * ka.ys + row] = s + } + } + } +} + +// The iq4nl B2/B4 pair: the iq4xs shells with the per-32-block d fold (no strip byte). +[ |> template_struct_instance] +class template MetalKqMvIq4nlT { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the 16B/sb d plane, half view + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // binding-shape twin (unused) + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array // iq4nl quant plane, uint view + @ssbo @binding = 3 @role = "read" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" y : array + @uniform @binding = 5 ka : KqMvArgs + @template_constant NR : int = 2 + @template_constant NRU : uint = 2u // NR's uint spelling (colbase math) — override BOTH together + @template_constant TILED : bool = false + + [metal_kernel] + def metal_kq_mv_iq4nl { + let lane = gl_SubgroupInvocationID + let tx = lane % 8u + let ty = lane / 8u + let row = gl_WorkGroupID.x * 8u + gl_SubgroupID * 4u + ty + let colbase = gl_WorkGroupID.y * NRU // nolint:LINT002 dead in the TILED=false stamp — measured free + let nb = ka.ndim / 256u + let nb4 = ka.ndim / 4u + var sumf : float[NR] + var sb = 0u + while (sb < nb) { + let blk = row * nb + sb + let dl = float(kdh[blk * 8u + tx]) + let qb = blk * 32u + tx * 4u + var wl : float4[4] + var wh : float4[4] + for [unroll_full] (u in range(4)) { + let qv = kqu[qb + uint(u)] + wl[u] = float4(iq4_lut(qv & 15u), iq4_lut((qv >> 8u) & 15u), iq4_lut((qv >> 16u) & 15u), iq4_lut((qv >> 24u) & 15u)) * dl + wh[u] = float4(iq4_lut((qv >> 4u) & 15u), iq4_lut((qv >> 12u) & 15u), iq4_lut((qv >> 20u) & 15u), iq4_lut(qv >> 28u)) * dl + } + let cx4 = sb * 64u + tx * 8u + static_if (TILED) { + for [unroll_full] (b in range(NR)) { + let xb4 = (colbase + uint(b)) * nb4 + cx4 + var acc = 0.0 + for [unroll_full] (u in range(4)) { + let v = x[xb4 + uint(u)] + let vh = x[xb4 + 4u + uint(u)] + acc += v.x * wl[u].x + v.y * wl[u].y + v.z * wl[u].z + v.w * wl[u].w + acc += vh.x * wh[u].x + vh.y * wh[u].y + vh.z * wh[u].z + vh.w * wh[u].w + } + sumf[b] += acc + } + } else { + for [unroll_full] (b in range(NR)) { + let xb4 = uint(b) * nb4 + cx4 + var acc = 0.0 + for [unroll_full] (u in range(4)) { + let v = x[xb4 + uint(u)] + let vh = x[xb4 + 4u + uint(u)] + acc += v.x * wl[u].x + v.y * wl[u].y + v.z * wl[u].z + v.w * wl[u].w + acc += vh.x * wh[u].x + vh.y * wh[u].y + vh.z * wh[u].z + vh.w * wh[u].w + } + sumf[b] += acc + } + } + sb++ + } + static_if (TILED) { + for [unroll_full] (b in range(NR)) { + var s = sumf[b] + s += simd_shuffle_down(s, 4u) + s += simd_shuffle_down(s, 2u) + s += simd_shuffle_down(s, 1u) + if (tx == 0u && row < ka.ddim && colbase + uint(b) < ka.nr) { + y[(colbase + uint(b)) * ka.ys + row] = s + } + } + } else { + for [unroll_full] (b in range(NR)) { + var s = sumf[b] + s += simd_shuffle_down(s, 4u) + s += simd_shuffle_down(s, 2u) + s += simd_shuffle_down(s, 1u) + if (tx == 0u && row < ka.ddim && uint(b) < ka.nr) { + y[uint(b) * ka.ys + row] = s + } + } + } + } +} + +[metal_dispatch(name = "enc_kq_mvb2_iq4nl_c", pso = "g_pso_kq_mvb2_iq4nl", tg = 64, grid = "rows/8", params = "rows : int64")] +class MetalKqMvB2Iq4nl : MetalKqMvIq4nlT { + override NR = 2 + override NRU = 2u + override TILED = false +} + +[metal_dispatch(name = "enc_kq_mvb4_iq4nl_c", pso = "g_pso_kq_mvb4_iq4nl", tg = 64, grid = "rows/8, gcols", params = "rows : int64, gcols : int64")] +class MetalKqMvB4Iq4nl : MetalKqMvIq4nlT { + override NR = 4 + override NRU = 4u + override TILED = true +} + +// the iq4nl B8 twin: the iq4xs B8 shell (one superblock's X panel staged per threadgroup) with +// the per-32-block d fold +[metal_dispatch(name = "enc_kq_mvb8_iq4nl_c", pso = "g_pso_kq_mvb8_iq4nl", tgmem = "metal_kq_mvb8_iq4nl_msl_tgmem", tg = 64, grid = "rows/8", params = "rows : int64")] +class MetalKqMvB8Iq4nl { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the 16B/sb d plane, half view + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // binding-shape twin (unused) + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array // iq4nl quant plane, uint view + @ssbo @binding = 3 @role = "read" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" y : array + @uniform @binding = 5 ka : KqMvArgs + @workgroup txp : float4[512] // [8 streams x 64 float4] one superblock's X panel + + [metal_kernel(name="metal_kq_mvb8_iq4nl_msl")] + def metal_kq_mvb8_iq4nl { + let lid = gl_LocalInvocationID.x + let lane = gl_SubgroupInvocationID + let tx = lane % 8u + let ty = lane / 8u + let row = gl_WorkGroupID.x * 8u + gl_SubgroupID * 4u + ty + let nb = ka.ndim / 256u + let nb4 = ka.ndim / 4u + var acc : float4[8] + var sb = 0u + while (sb < nb) { + barrier() + for [unroll_full] (j in range(8)) { + let slot = uint(j) * 64u + lid + txp[slot] = x[(slot / 64u) * nb4 + sb * 64u + (slot % 64u)] + } + barrier() + let blk = row * nb + sb + let dl = float(kdh[blk * 8u + tx]) + let qb = blk * 32u + tx * 4u + var wl : float4[4] + var wh : float4[4] + for [unroll_full] (u in range(4)) { + let qv = kqu[qb + uint(u)] + wl[u] = float4(iq4_lut(qv & 15u), iq4_lut((qv >> 8u) & 15u), iq4_lut((qv >> 16u) & 15u), iq4_lut((qv >> 24u) & 15u)) * dl + wh[u] = float4(iq4_lut((qv >> 4u) & 15u), iq4_lut((qv >> 12u) & 15u), iq4_lut((qv >> 20u) & 15u), iq4_lut(qv >> 28u)) * dl + } + let cxl = tx * 8u + for [unroll_full] (b in range(8)) { + for [unroll_full] (u in range(4)) { + acc[b] += txp[uint(b) * 64u + cxl + uint(u)] * wl[u] + txp[uint(b) * 64u + cxl + 4u + uint(u)] * wh[u] + } + } + sb++ + } + for [unroll_full] (b in range(8)) { + var s = acc[b].x + acc[b].y + acc[b].z + acc[b].w + s += simd_shuffle_down(s, 4u) + s += simd_shuffle_down(s, 2u) + s += simd_shuffle_down(s, 1u) + if (tx == 0u && row < ka.ddim && uint(b) < ka.nr) { + y[uint(b) * ka.ys + row] = s + } + } + } +} + // The B2/B4 width pair as one template per format (the B8 trio below is a DIFFERENT algorithm and // stays hand-written). TILED branch-duplicates the b-loop/writeback because a LIVE colbase on B2 // costs +2% (k4) / +0.5% (k6) at cls (bench_metal_gemv_kernels KqMv round); B4 stamps byte-identical. diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das b/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das index 9fcbf636cb..b6687d0a7e 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das @@ -4411,6 +4411,9 @@ def private pf_devw_panel(enc : MetalComputeEncoder?; bwblob : MetalBuffer?; wbo //! quant superblocks, plus the k6 2B d tail def private pf_devw_panel_kq(enc : MetalComputeEncoder?; t : Model; fmt : KqFmt; woff : int64; bxh, by, bk, bn : MetalBuffer?; rows, d, kdim : int64; yoff : uint64) : bool { + if (fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s || fmt == KqFmt.iq3xxs || fmt == KqFmt.iq4nl || fmt == KqFmt.k2 || fmt == KqFmt.iq2s || fmt == KqFmt.iq2xs || fmt == KqFmt.iq2xxs) { //! no dev-W dequant kernel for these formats + return false + } let dq = fmt == KqFmt.k6 ? g_pf_pso_dq_k6 : (fmt == KqFmt.k4 ? g_pf_pso_dq_k4 : g_pf_pso_dq_k5) if (dq == null) { return false @@ -4573,6 +4576,42 @@ def private pf_enc_kq_site_mm(enc : MetalComputeEncoder?; t : Model; fmt : KqFmt let fmt_tensor = fmt == KqFmt.k6 ? g_pf_kq_mm6_tensor : (fmt == KqFmt.k4 ? g_pf_kq_mm4_tensor : g_pf_kq_mm5_tensor) let bq = kq_quants_of(g_dev, t, fmt, woff) let bs = kq_scales_of(g_dev, t, fmt, woff) + if (fmt == KqFmt.iq4xs) { + enc_kq_mm_iq4xs_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, by, yoff, bk, bn, mp, rows) + return + } + if (fmt == KqFmt.k3) { + enc_kq_mm_k3_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, by, yoff, bk, bn, mp, rows) + return + } + if (fmt == KqFmt.iq3s) { + enc_kq_mm_iq3s_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, by, yoff, bk, bn, mp, rows) + return + } + if (fmt == KqFmt.iq3xxs) { + enc_kq_mm_iq3xxs_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, by, yoff, bk, bn, mp, rows) + return + } + if (fmt == KqFmt.iq4nl) { + enc_kq_mm_iq4nl_c(enc, bs.buf, bs.soff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, by, yoff, bk, bn, mp, rows) + return + } + if (fmt == KqFmt.k2) { + enc_kq_mm_k2_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, by, yoff, bk, bn, mp, rows) + return + } + if (fmt == KqFmt.iq2s) { + enc_kq_mm_iq2s_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, by, yoff, bk, bn, mp, rows) + return + } + if (fmt == KqFmt.iq2xs) { + enc_kq_mm_iq2xs_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, by, yoff, bk, bn, mp, rows) + return + } + if (fmt == KqFmt.iq2xxs) { + enc_kq_mm_iq2xxs_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, by, yoff, bk, bn, mp, rows) + return + } //! deep-class arm: a panel dev-W would split past 8 tiles is DRAM-resident - the tall //! in-kernel-dequant stamp reads the quant plane once per 128-row tile instead of //! materializing and re-streaming a 2B/element f16 panel @@ -4676,7 +4715,25 @@ def private pf_enc_kq_gemv(enc : MetalComputeEncoder?; t : Model; fmt : KqFmt; w bx : MetalBuffer?; xoff : uint64; by, bn, bd : MetalBuffer?) { let bq = kq_quants_of(g_dev, t, fmt, woff) let bs = kq_scales_of(g_dev, t, fmt, woff) - if (fmt == KqFmt.k6) { + if (fmt == KqFmt.iq4xs) { + enc_kq_iq4xs_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, xoff, by, 0ul, bn, bd, rows, n) + } elif (fmt == KqFmt.k3) { + enc_kq_k3_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, xoff, by, 0ul, bn, bd, rows, n) + } elif (fmt == KqFmt.iq3s) { + enc_kq_iq3s_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, xoff, by, 0ul, bn, bd, rows, n) + } elif (fmt == KqFmt.iq3xxs) { + enc_kq_iq3xxs_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, xoff, by, 0ul, bn, bd, rows, n) + } elif (fmt == KqFmt.iq2s) { + enc_kq_iq2s_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, xoff, by, 0ul, bn, bd, rows, n) + } elif (fmt == KqFmt.iq2xs) { + enc_kq_iq2xs_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, xoff, by, 0ul, bn, bd, rows, n) + } elif (fmt == KqFmt.iq2xxs) { + enc_kq_iq2xxs_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, xoff, by, 0ul, bn, bd, rows, n) + } elif (fmt == KqFmt.iq4nl) { + enc_kq_iq4nl_c(enc, bs.buf, bs.soff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, xoff, by, 0ul, bn, bd, rows, n) + } elif (fmt == KqFmt.k2) { + enc_kq_k2_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, xoff, by, 0ul, bn, bd, rows, n) + } elif (fmt == KqFmt.k6) { enc_kq_k6_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, xoff, by, 0ul, bn, bd, rows, n) } elif (fmt == KqFmt.k4) { enc_kq_k4_c(enc, bs.buf, bs.soff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, xoff, by, 0ul, bn, bd, rows, n) diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_shapes.das b/modules/dasLLAMA/dasllama/dasllama_metal_shapes.das index eaf6031e26..08003d490e 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_shapes.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_shapes.das @@ -180,7 +180,7 @@ def record_needs(var tab : table; missing : MetalNeed) { // formats) must DECLINE kquant_native instead of falling into a wrong-layout kernel branch // (the dispatchers below treat "not k4/k6" as k5) def kq_fmt_gpu_supported(f : KqFmt) : bool { - return f == KqFmt.q8 || f == KqFmt.k4 || f == KqFmt.k5 || f == KqFmt.k6 + return f == KqFmt.q8 || f == KqFmt.k4 || f == KqFmt.k5 || f == KqFmt.k6 || f == KqFmt.iq4xs || f == KqFmt.k3 || f == KqFmt.iq3s || f == KqFmt.iq3xxs || f == KqFmt.iq4nl || f == KqFmt.k2 || f == KqFmt.iq2s || f == KqFmt.iq2xs || f == KqFmt.iq2xxs } def private kq_fmts_gpu_supported(a : array) : bool { @@ -236,6 +236,7 @@ def dn_metal_ok(t : Model) : bool { } // (q8 32-blocks, kq 256-superblocks — the expert shift is index math, no base multiple) +[arch(at = "../ARCHITECTURE_GPU.md#metal-kq-split-scale-plane")] def moe_site_ok(fmt : KqFmt; off, ege : int64) : bool { if (fmt == KqFmt.q8) { return off >= 0l && (off % 256l) == 0l && (ege % 32l) == 0l @@ -243,7 +244,7 @@ def moe_site_ok(fmt : KqFmt; off, ege : int64) : bool { if (fmt == KqFmt.k4 || fmt == KqFmt.k5) { return off >= 0l && (off % 256l) == 0l && (ege % 256l) == 0l } - if (fmt == KqFmt.k6) { + if (fmt == KqFmt.k6 || fmt == KqFmt.iq4xs || fmt == KqFmt.k3 || fmt == KqFmt.iq3s || fmt == KqFmt.iq3xxs || fmt == KqFmt.iq2s || fmt == KqFmt.iq2xs || fmt == KqFmt.iq2xxs) { return off >= 0l && (off % 512l) == 0l && (ege % 256l) == 0l } if (fmt == KqFmt.q51) { // per-32 planes: off % 128 keeps the 20B/4B binds 16B-aligned @@ -324,9 +325,9 @@ def moe_metal_ok(t : Model) : bool { // nolint:STYLE038 — flat per-shape ref let f3 = fmt_at(t.we3_fmt, l) // per-site alignment + reduction dims: superblock kq kernels iterate whole 256-rows, // q51 whole 32-blocks (+ the mul_mm's 64-col output tiles) - let sb1 = f1 == KqFmt.k4 || f1 == KqFmt.k5 || f1 == KqFmt.k6 - let sb3 = f3 == KqFmt.k4 || f3 == KqFmt.k5 || f3 == KqFmt.k6 - let sb2 = f2 == KqFmt.k4 || f2 == KqFmt.k5 || f2 == KqFmt.k6 + let sb1 = kq_sb(f1) + let sb3 = kq_sb(f3) + let sb2 = kq_sb(f2) if (!moe_site_ok(f1, t.we1_offs[l], ege) || !moe_site_ok(f2, t.we2_offs[l], ege) || !moe_site_ok(f3, t.we3_offs[l], ege) || diff --git a/modules/dasLLAMA/dasllama/dasllama_ple.das b/modules/dasLLAMA/dasllama/dasllama_ple.das index ddc47b4f1d..2754fea480 100644 --- a/modules/dasLLAMA/dasllama/dasllama_ple.das +++ b/modules/dasLLAMA/dasllama/dasllama_ple.das @@ -62,6 +62,24 @@ def ple_check_table(t : Model; origin : string) { have = (long_length(t.k6q) / K6_QSB) * 256l } elif (t.ple_emb_fmt == KqFmt.q40) { have = (long_length(t.q40q) / Q40_QSB) * 256l + } elif (t.ple_emb_fmt == KqFmt.iq4xs) { + have = (long_length(t.iq4xsq) / IQ4XS_QSB) * 256l + } elif (t.ple_emb_fmt == KqFmt.k3) { + have = (long_length(t.k3q) / K3_QSB) * 256l + } elif (t.ple_emb_fmt == KqFmt.iq3s) { + have = (long_length(t.iq3sq) / IQ3S_QSB) * 256l + } elif (t.ple_emb_fmt == KqFmt.iq3xxs) { + have = (long_length(t.iq3xxsq) / IQ3XXS_QSB) * 256l + } elif (t.ple_emb_fmt == KqFmt.iq4nl) { + have = (long_length(t.iq4nlq) / Q40_QSB) * 256l + } elif (t.ple_emb_fmt == KqFmt.k2) { + have = (long_length(t.k2q) / K2_QSB) * 256l + } elif (t.ple_emb_fmt == KqFmt.iq2s) { + have = (long_length(t.iq2sq) / IQ2S_QSB) * 256l + } elif (t.ple_emb_fmt == KqFmt.iq2xs) { + have = (long_length(t.iq2xsq) / IQ2XS_QSB) * 256l + } elif (t.ple_emb_fmt == KqFmt.iq2xxs) { + have = (long_length(t.iq2xxsq) / IQ2XXS_QSB) * 256l } if (have < need) { panic("dasLLAMA: the PLE token table is not resident ({origin}): fmt {t.ple_emb_fmt} plane holds {have} of {need} elements - regenerate this carrier (it predates the pinned-table rail)") @@ -78,26 +96,12 @@ def private ple_gather_row(t : Model; row : int64; var dst : array; doff dequant_q8_row(t, eloff, all, unsafe(addr(dst[doff]))) return } - // metal-blob carriers compact k4/k5 scale strips to 16B and split k6 (embed_row's strides) - let ssb45 = t.metal_blob ? 16l : K4_SSB - let k6d0 = t.metal_blob ? (long_length(t.k6s) / K6_SSB) * 16l : 0l + var kqp : array const? + var ksp : array const? + kq_planes_of(t, t.ple_emb_fmt, kqp, ksp) for (blk in range64(all / 256l)) { let sb = (eloff + blk * 256l) / 256l - if (t.ple_emb_fmt == KqFmt.k4) { - dequant_k4_plane_superblock(t.k4q, sb * K4_QSB, t.k4s, sb * ssb45, dst, doff + blk * 256l) - } elif (t.ple_emb_fmt == KqFmt.k5) { - dequant_k5_plane_superblock(t.k5q, sb * K5_QSB, t.k5s, sb * ssb45, dst, doff + blk * 256l) - } elif (t.ple_emb_fmt == KqFmt.k6) { - if (t.metal_blob) { - dequant_k6_plane_superblock_at(t.k6q, sb * K6_QSB, t.k6s, sb * 16l, k6d0 + sb * 2l, dst, doff + blk * 256l) - } else { - dequant_k6_plane_superblock(t.k6q, sb * K6_QSB, t.k6s, sb * K6_SSB, dst, doff + blk * 256l) - } - } elif (t.ple_emb_fmt == KqFmt.q40) { - dequant_q40_plane_superblock(t.q40q, sb * Q40_QSB, t.q40s, sb * Q40_SSB, dst, doff + blk * 256l) - } else { - panic("ple_gather_row: '{t.ple_emb_fmt}' has no kq plane pair") - } + dequant_kq_plane_sb(t.ple_emb_fmt, *kqp, *ksp, sb, t.metal_blob, dst, doff + blk * 256l) } } diff --git a/modules/dasLLAMA/dasllama/dasllama_repack.das b/modules/dasLLAMA/dasllama/dasllama_repack.das index 03cf97166b..b44d1470dd 100644 --- a/modules/dasLLAMA/dasllama/dasllama_repack.das +++ b/modules/dasLLAMA/dasllama/dasllama_repack.das @@ -4,6 +4,7 @@ options _dasllama_internal = true module dasllama_repack shared public +require dasllama/dasllama_kqformat // iq3s_grid - the panel unpack gathers through the codebook require dasllama/dasllama_lint public require dasllama/dasllama_math_default // k4_sc_mn — the kq 6-bit sc/mn decoder (stays there: hot in the kq dots, auto-inline is same-module-only) @@ -373,13 +374,603 @@ def repack_q40_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { delete tq delete ts } + +//! iq4xs grp layout: the q40 quant gather (the disk k/k+16 pairing IS the k4 tiling) and the k4 +//! scale-row interleave over the DECODED 20B row — [mr x f16 d][mr x f16 0][8 x mr int8 sc][8 x mr 0]. +def repack_iq4xs_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { + let nsb = n / 256l + let qrow = nsb * 128l + let srow = nsb * 20l + let ng = d / mr + var tq : array + var ts : array + tq |> resize(d * qrow) + ts |> resize(d * srow) + unsafe { + var tqp = addr(tq[0]) + var tsp = addr(ts[0]) + memcpy(tqp, kq, d * qrow) + memcpy(tsp, ks, d * srow) + for (g in range64(ng)) { + for (sbi in range64(nsb)) { + let dq = g * mr * qrow + sbi * 128l * mr + let ds = g * mr * srow + sbi * 20l * mr + for (r in range64(mr)) { + let sq = (g * mr + r) * qrow + sbi * 128l + let ss = (g * mr + r) * srow + sbi * 20l + for (blk in range64(8l)) { + for (j in range64(4l)) { + for (t in range64(4l)) { + kq[dq + ((blk * 4l + j) * mr + r) * 4l + t] = tqp[sq + blk * 16l + j * 4l + t] + } + } + ks[ds + 4l * mr + blk * mr + r] = tsp[ss + 4l + blk] + ks[ds + 12l * mr + blk * mr + r] = uint8(0) + } + ks[ds + 2l * r] = tsp[ss] + ks[ds + 2l * r + 1l] = tsp[ss + 1l] + ks[ds + 2l * mr + 2l * r] = uint8(0) + ks[ds + 2l * mr + 2l * r + 1l] = uint8(0) + } + } + } + } + delete tq + delete ts +} + +//! k3 grp layout: the qs bytes as 16 four-byte columns ([h 0..1][8 l-groups]) x mr rows, then the +//! hmask as 8 columns x mr rows at 64*mr — the 2-bit lanes and mask bits stay in place (the +//! kernels shift per block); scales interleave as k6's row ([16 x mr int8][mr x f16 d]). +def repack_k3_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { + let nsb = n / 256l + let qrow = nsb * 96l + let srow = nsb * 18l + let ng = d / mr + var tq : array + var ts : array + tq |> resize(d * qrow) + ts |> resize(d * srow) + unsafe { + var tqp = addr(tq[0]) + var tsp = addr(ts[0]) + memcpy(tqp, kq, d * qrow) + memcpy(tsp, ks, d * srow) + for (g in range64(ng)) { + for (sbi in range64(nsb)) { + let dq = g * mr * qrow + sbi * 96l * mr + let ds = g * mr * srow + sbi * 18l * mr + for (r in range64(mr)) { + let sq = (g * mr + r) * qrow + sbi * 96l + let ss = (g * mr + r) * srow + sbi * 18l + for (c in range64(16l)) { + for (t in range64(4l)) { + kq[dq + (c * mr + r) * 4l + t] = tqp[sq + c * 4l + t] + } + } + for (c in range64(8l)) { + for (t in range64(4l)) { + kq[dq + 64l * mr + (c * mr + r) * 4l + t] = tqp[sq + 64l + c * 4l + t] + } + } + for (idx in range64(16l)) { + ks[ds + idx * mr + r] = tsp[ss + idx] + } + ks[ds + 16l * mr + 2l * r] = tsp[ss + 16l] + ks[ds + 16l * mr + 2l * r + 1l] = tsp[ss + 17l] + } + } + } + } + delete tq + delete ts +} + +//! iq3s grp layout: the whole 104B row as 26 four-byte columns x mr rows — qs, qh and sign +//! bytes stay in place per column (the kernels compose per block); scales interleave as k4's +//! row ([d f16 x mr][pad x mr][8 sc x mr][8 pad x mr]). +def repack_iq3s_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { + let nsb = n / 256l + let qrow = nsb * 104l + let srow = nsb * 20l + let ng = d / mr + var tq : array + var ts : array + tq |> resize(d * qrow) + ts |> resize(d * srow) + unsafe { + var tqp = addr(tq[0]) + var tsp = addr(ts[0]) + memcpy(tqp, kq, d * qrow) + memcpy(tsp, ks, d * srow) + for (g in range64(ng)) { + for (sbi in range64(nsb)) { + let dq = g * mr * qrow + sbi * 104l * mr + let ds = g * mr * srow + sbi * 20l * mr + for (r in range64(mr)) { + let sq = (g * mr + r) * qrow + sbi * 104l + let ss = (g * mr + r) * srow + sbi * 20l + for (c in range64(26l)) { + for (t in range64(4l)) { + kq[dq + (c * mr + r) * 4l + t] = tqp[sq + c * 4l + t] + } + } + ks[ds + 2l * r] = tsp[ss] + ks[ds + 2l * r + 1l] = tsp[ss + 1l] + ks[ds + 2l * mr + 2l * r] = tsp[ss + 2l] + ks[ds + 2l * mr + 2l * r + 1l] = tsp[ss + 3l] + for (b in range64(8l)) { + ks[ds + 4l * mr + b * mr + r] = tsp[ss + 4l + b] + ks[ds + 12l * mr + b * mr + r] = tsp[ss + 12l + b] + } + } + } + } + } + delete tq + delete ts +} + +//! k2 grp layout: the qs bytes as 16 four-byte columns x mr rows (the 2-bit lanes stay in +//! place - the kernels shift per group); scale row interleaves FIELD-MAJOR as +//! [16 sc/min bytes x mr][mr x f16 d][mr x f16 dmin] (contiguous d rows for the emitter's f16 vector loads). +def repack_k2_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { + let nsb = n / 256l + let qrow = nsb * 64l + let srow = nsb * 20l + let ng = d / mr + var tq : array + var ts : array + tq |> resize(d * qrow) + ts |> resize(d * srow) + unsafe { + var tqp = addr(tq[0]) + var tsp = addr(ts[0]) + memcpy(tqp, kq, d * qrow) + memcpy(tsp, ks, d * srow) + for (g in range64(ng)) { + for (sbi in range64(nsb)) { + let dq = g * mr * qrow + sbi * 64l * mr + let ds = g * mr * srow + sbi * 20l * mr + for (r in range64(mr)) { + let sq = (g * mr + r) * qrow + sbi * 64l + let ss = (g * mr + r) * srow + sbi * 20l + for (c in range64(16l)) { + for (t in range64(4l)) { + kq[dq + (c * mr + r) * 4l + t] = tqp[sq + c * 4l + t] + } + } + for (idx in range64(16l)) { + ks[ds + idx * mr + r] = tsp[ss + 4l + idx] + } + ks[ds + 16l * mr + 2l * r] = tsp[ss] + ks[ds + 16l * mr + 2l * r + 1l] = tsp[ss + 1l] + ks[ds + 18l * mr + 2l * r] = tsp[ss + 2l] + ks[ds + 18l * mr + 2l * r + 1l] = tsp[ss + 3l] + } + } + } + } + delete tq + delete ts +} + +//! iq2s grp layout: the whole 72B row as 18 four-byte columns x mr rows - idx, sign and qh +//! bytes stay in place per column; scales interleave as k4's row with 16 strips +//! ([d8 f16 x mr][pad x mr][16 strips x mr]). +def repack_iq2s_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { + let nsb = n / 256l + let qrow = nsb * 72l + let srow = nsb * 20l + let ng = d / mr + var tq : array + var ts : array + tq |> resize(d * qrow) + ts |> resize(d * srow) + unsafe { + var tqp = addr(tq[0]) + var tsp = addr(ts[0]) + memcpy(tqp, kq, d * qrow) + memcpy(tsp, ks, d * srow) + for (g in range64(ng)) { + for (sbi in range64(nsb)) { + let dq = g * mr * qrow + sbi * 72l * mr + let ds = g * mr * srow + sbi * 20l * mr + for (r in range64(mr)) { + let sq = (g * mr + r) * qrow + sbi * 72l + let ss = (g * mr + r) * srow + sbi * 20l + for (c in range64(18l)) { + for (t in range64(4l)) { + kq[dq + (c * mr + r) * 4l + t] = tqp[sq + c * 4l + t] + } + } + ks[ds + 2l * r] = tsp[ss] + ks[ds + 2l * r + 1l] = tsp[ss + 1l] + ks[ds + 2l * mr + 2l * r] = tsp[ss + 2l] + ks[ds + 2l * mr + 2l * r + 1l] = tsp[ss + 3l] + for (g16 in range64(16l)) { + ks[ds + 4l * mr + g16 * mr + r] = tsp[ss + 4l + g16] + } + } + } + } + } + delete tq + delete ts +} + +//! iq2xs grp layout: the whole 64B row as 16 four-byte columns x mr rows - the u16 qs words +//! stay in place per column (each word's two bytes share a column); scales interleave as k4's +//! row with 16 strips ([d8 f16 x mr][pad x mr][16 strips x mr]). +def repack_iq2xs_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { + let nsb = n / 256l + let qrow = nsb * 64l + let srow = nsb * 20l + let ng = d / mr + var tq : array + var ts : array + tq |> resize(d * qrow) + ts |> resize(d * srow) + unsafe { + var tqp = addr(tq[0]) + var tsp = addr(ts[0]) + memcpy(tqp, kq, d * qrow) + memcpy(tsp, ks, d * srow) + for (g in range64(ng)) { + for (sbi in range64(nsb)) { + let dq = g * mr * qrow + sbi * 64l * mr + let ds = g * mr * srow + sbi * 20l * mr + for (r in range64(mr)) { + let sq = (g * mr + r) * qrow + sbi * 64l + let ss = (g * mr + r) * srow + sbi * 20l + for (c in range64(16l)) { + for (t in range64(4l)) { + kq[dq + (c * mr + r) * 4l + t] = tqp[sq + c * 4l + t] + } + } + ks[ds + 2l * r] = tsp[ss] + ks[ds + 2l * r + 1l] = tsp[ss + 1l] + ks[ds + 2l * mr + 2l * r] = tsp[ss + 2l] + ks[ds + 2l * mr + 2l * r + 1l] = tsp[ss + 3l] + for (g16 in range64(16l)) { + ks[ds + 4l * mr + g16 * mr + r] = tsp[ss + 4l + g16] + } + } + } + } + } + delete tq + delete ts +} + +//! iq2xxs grp layout: the whole 64B row as 16 four-byte columns x mr rows - the u16 qs words +//! stay in place per column (each word's two bytes share a column); scales interleave as k4's +//! row with 16 strips ([d8 f16 x mr][pad x mr][16 strips x mr]). +def repack_iq2xxs_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { + let nsb = n / 256l + let qrow = nsb * 64l + let srow = nsb * 20l + let ng = d / mr + var tq : array + var ts : array + tq |> resize(d * qrow) + ts |> resize(d * srow) + unsafe { + var tqp = addr(tq[0]) + var tsp = addr(ts[0]) + memcpy(tqp, kq, d * qrow) + memcpy(tsp, ks, d * srow) + for (g in range64(ng)) { + for (sbi in range64(nsb)) { + let dq = g * mr * qrow + sbi * 64l * mr + let ds = g * mr * srow + sbi * 20l * mr + for (r in range64(mr)) { + let sq = (g * mr + r) * qrow + sbi * 64l + let ss = (g * mr + r) * srow + sbi * 20l + for (c in range64(16l)) { + for (t in range64(4l)) { + kq[dq + (c * mr + r) * 4l + t] = tqp[sq + c * 4l + t] + } + } + ks[ds + 2l * r] = tsp[ss] + ks[ds + 2l * r + 1l] = tsp[ss + 1l] + ks[ds + 2l * mr + 2l * r] = tsp[ss + 2l] + ks[ds + 2l * mr + 2l * r + 1l] = tsp[ss + 3l] + for (g16 in range64(16l)) { + ks[ds + 4l * mr + g16 * mr + r] = tsp[ss + 4l + g16] + } + } + } + } + } + delete tq + delete ts +} + // ===== grp panel unpack (the byte-expanded tile form the kq stamps load) ===== //! Unpack ONE group's packed grp k5/k6 quant panel into the BYTE-EXPANDED tile form (wlo/whi //! kqBytes stamps load verbatim). SWAR over uint64 lanes: k5 deposit uses an add-carry test //! ((x+0x7F)&0x80) to fold the high bit; k6 tops sit at a uniform shift per block. [hint(unsafe_range_check, noalias = kqg, noalias = dst)] +//! iq3s panel unpack: the grid gather + sign apply, packed grp planes -> the byte-expanded +//! signed panel. One grid word = 4 elements = ONE i32 store; the sign nibble expands through +//! the mask table and negates byte-wise (magnitudes are odd 1..15 - no cross-byte carry). +def unpack_iq3s_panel_grp(kqg : uint8 const?; var dst : uint8?; mr, nsb : int64) { + let grid = iq3s_grid() + let smask = fixed_array(0x00000000u, 0x000000FFu, 0x0000FF00u, 0x0000FFFFu, + 0x00FF0000u, 0x00FF00FFu, 0x00FFFF00u, 0x00FFFFFFu, + 0xFF000000u, 0xFF0000FFu, 0xFF00FF00u, 0xFF00FFFFu, + 0xFFFF0000u, 0xFFFF00FFu, 0xFFFFFF00u, 0xFFFFFFFFu) + unsafe { + for (sbi in range64(nsb)) { + let qb = sbi * 104l * mr + let ob = sbi * 256l * mr + for (r in range64(mr)) { + for (blk in range64(8l)) { + let hp = 64l + blk + let qh = int(kqg[qb + ((hp / 4l) * mr + r) * 4l + hp % 4l]) + for (l in range64(4l)) { + let sp = 72l + blk * 4l + l + let sgn = int(kqg[qb + ((sp / 4l) * mr + r) * 4l + sp % 4l]) + let q1p = blk * 8l + 2l * l + let q2p = q1p + 1l + let i1 = int(kqg[qb + ((q1p / 4l) * mr + r) * 4l + q1p % 4l]) | ((qh << int(8l - 2l * l)) & 256) + let i2 = int(kqg[qb + ((q2p / 4l) * mr + r) * 4l + q2p % 4l]) | ((qh << int(7l - 2l * l)) & 256) + let m1 = smask[sgn & 15] + let m2 = smask[(sgn >> 4) & 15] + let w1 = (grid[i1] ^ m1) + (m1 & 0x01010101u) + let w2 = (grid[i2] ^ m2) + (m2 & 0x01010101u) + // element e = l*8 + half*4 + t: e < 16 lands in the lo panel half, else hi + let e1 = l * 8l + let e2 = l * 8l + 4l + let d1 = (e1 < 16l ? 0l : 128l * mr) + ((blk * 4l + (e1 % 16l) / 4l) * mr + r) * 4l + let d2 = (e2 < 16l ? 0l : 128l * mr) + ((blk * 4l + (e2 % 16l) / 4l) * mr + r) * 4l + var wp1 = reinterpret(dst + (ob + d1)) + var wp2 = reinterpret(dst + (ob + d2)) + wp1[0] = w1 + wp2[0] = w2 + } + } + } + } + } +} + +//! iq3xxs grp layout: the whole 96B row as 24 four-byte columns x mr rows — qs and aux move +//! together; the scale row keeps the iq3s interleave (d pairs, pads, strips, pads). +def repack_iq3xxs_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { + let nsb = n / 256l + let qrow = nsb * 96l + let srow = nsb * 20l + let ng = d / mr + var tq : array + var ts : array + tq |> resize(d * qrow) + ts |> resize(d * srow) + unsafe { + var tqp = addr(tq[0]) + var tsp = addr(ts[0]) + memcpy(tqp, kq, d * qrow) + memcpy(tsp, ks, d * srow) + for (g in range64(ng)) { + for (sbi in range64(nsb)) { + let dq = g * mr * qrow + sbi * 96l * mr + let ds = g * mr * srow + sbi * 20l * mr + for (r in range64(mr)) { + let sq = (g * mr + r) * qrow + sbi * 96l + let ss = (g * mr + r) * srow + sbi * 20l + for (c in range64(24l)) { + for (t in range64(4l)) { + kq[dq + (c * mr + r) * 4l + t] = tqp[sq + c * 4l + t] + } + } + ks[ds + 2l * r] = tsp[ss] + ks[ds + 2l * r + 1l] = tsp[ss + 1l] + ks[ds + 2l * mr + 2l * r] = tsp[ss + 2l] + ks[ds + 2l * mr + 2l * r + 1l] = tsp[ss + 3l] + for (b in range64(8l)) { + ks[ds + 4l * mr + b * mr + r] = tsp[ss + 4l + b] + ks[ds + 12l * mr + b * mr + r] = tsp[ss + 12l + b] + } + } + } + } + } + delete tq + delete ts +} + +//! iq3xxs panel unpack: the halved-grid gather + ksigns sign apply, packed grp planes -> the +//! byte-expanded signed panel (the iq3s panel form — the shared fmt-33/34 tile reads it). +def unpack_iq3xxs_panel_grp(kqg : uint8 const?; var dst : uint8?; mr, nsb : int64) { + let grid = iq3xxs_grid() + let ksg = ksigns_iq2xs() + let smask = fixed_array(0x00000000u, 0x000000FFu, 0x0000FF00u, 0x0000FFFFu, + 0x00FF0000u, 0x00FF00FFu, 0x00FFFF00u, 0x00FFFFFFu, + 0xFF000000u, 0xFF0000FFu, 0xFF00FF00u, 0xFF00FFFFu, + 0xFFFF0000u, 0xFFFF00FFu, 0xFFFFFF00u, 0xFFFFFFFFu) + unsafe { + for (sbi in range64(nsb)) { + let qb = sbi * 96l * mr + let ob = sbi * 256l * mr + for (r in range64(mr)) { + for (blk in range64(8l)) { + var aux = 0u + for (t in range64(4l)) { + let ap = 64l + blk * 4l + t + aux |= uint(kqg[qb + ((ap / 4l) * mr + r) * 4l + ap % 4l]) << uint(8l * t) + } + for (l in range64(4l)) { + let sgn = int(ksg[int((aux >> uint(7l * l)) & 127u)]) + let q1p = blk * 8l + 2l * l + let q2p = q1p + 1l + let i1 = int(kqg[qb + ((q1p / 4l) * mr + r) * 4l + q1p % 4l]) + let i2 = int(kqg[qb + ((q2p / 4l) * mr + r) * 4l + q2p % 4l]) + let m1 = smask[sgn & 15] + let m2 = smask[(sgn >> 4) & 15] + let w1 = (grid[i1] ^ m1) + (m1 & 0x01010101u) + let w2 = (grid[i2] ^ m2) + (m2 & 0x01010101u) + // element e = l*8 + half*4 + t: e < 16 lands in the lo panel half, else hi + let e1 = l * 8l + let e2 = l * 8l + 4l + let d1 = (e1 < 16l ? 0l : 128l * mr) + ((blk * 4l + (e1 % 16l) / 4l) * mr + r) * 4l + let d2 = (e2 < 16l ? 0l : 128l * mr) + ((blk * 4l + (e2 % 16l) / 4l) * mr + r) * 4l + var wp1 = reinterpret(dst + (ob + d1)) + var wp2 = reinterpret(dst + (ob + d2)) + wp1[0] = w1 + wp2[0] = w2 + } + } + } + } + } +} + +//! iq2s panel unpack: the 10-bit qs|qh gather off the u64 grid (TWO words per index - eight +//! magnitudes {8, 25, 43}) + the block's own sign bytes, packed grp planes -> the shared +//! byte-expanded signed panel (the fmt-33/34 panel form; d is pre-eighth-ed at transcode). +def unpack_iq2s_panel_grp(kqg : uint8 const?; var dst : uint8?; mr, nsb : int64) { + let grid = iq2s_grid2() + let smask = fixed_array(0x00000000u, 0x000000FFu, 0x0000FF00u, 0x0000FFFFu, + 0x00FF0000u, 0x00FF00FFu, 0x00FFFF00u, 0x00FFFFFFu, + 0xFF000000u, 0xFF0000FFu, 0xFF00FF00u, 0xFF00FFFFu, + 0xFFFF0000u, 0xFFFF00FFu, 0xFFFFFF00u, 0xFFFFFFFFu) + unsafe { + for (sbi in range64(nsb)) { + let qb = sbi * 72l * mr + let ob = sbi * 256l * mr + for (r in range64(mr)) { + for (blk in range64(8l)) { + let hp = 64l + blk + let qh = int(kqg[qb + ((hp / 4l) * mr + r) * 4l + hp % 4l]) + for (l in range64(4l)) { + let sp = 32l + blk * 4l + l + let sgn = int(kqg[qb + ((sp / 4l) * mr + r) * 4l + sp % 4l]) + let ip = blk * 4l + l + let ix = (int(kqg[qb + ((ip / 4l) * mr + r) * 4l + ip % 4l]) | ((qh << int(8l - 2l * l)) & 0x300)) * 2 + let m1 = smask[sgn & 15] + let m2 = smask[(sgn >> 4) & 15] + let w1 = (grid[ix] ^ m1) + (m1 & 0x01010101u) + let w2 = (grid[ix + 1] ^ m2) + (m2 & 0x01010101u) + // element e = l*8 + half*4 + t: e < 16 lands in the lo panel half, else hi + let e1 = l * 8l + let e2 = l * 8l + 4l + let d1 = (e1 < 16l ? 0l : 128l * mr) + ((blk * 4l + (e1 % 16l) / 4l) * mr + r) * 4l + let d2 = (e2 < 16l ? 0l : 128l * mr) + ((blk * 4l + (e2 % 16l) / 4l) * mr + r) * 4l + var wp1 = reinterpret(dst + (ob + d1)) + var wp2 = reinterpret(dst + (ob + d2)) + wp1[0] = w1 + wp2[0] = w2 + } + } + } + } + } +} + +//! iq2xs panel unpack: the u16 qs word's 9-bit index doubles into the u64 grid's word pair, +//! its high 7 bits pick the KSIGNS sign byte - packed grp planes -> the shared signed panel. +def unpack_iq2xs_panel_grp(kqg : uint8 const?; var dst : uint8?; mr, nsb : int64) { + let grid = iq2xs_grid2() + let ksgn = ksigns_iq2xs() + let smask = fixed_array(0x00000000u, 0x000000FFu, 0x0000FF00u, 0x0000FFFFu, + 0x00FF0000u, 0x00FF00FFu, 0x00FFFF00u, 0x00FFFFFFu, + 0xFF000000u, 0xFF0000FFu, 0xFF00FF00u, 0xFF00FFFFu, + 0xFFFF0000u, 0xFFFF00FFu, 0xFFFFFF00u, 0xFFFFFFFFu) + unsafe { + for (sbi in range64(nsb)) { + let qb = sbi * 64l * mr + let ob = sbi * 256l * mr + for (r in range64(mr)) { + for (blk in range64(8l)) { + for (l in range64(4l)) { + let lp = (blk * 4l + l) * 2l + let w16 = uint(kqg[qb + ((lp / 4l) * mr + r) * 4l + lp % 4l]) | (uint(kqg[qb + ((lp / 4l) * mr + r) * 4l + lp % 4l + 1l]) << 8u) + let sgn = int(ksgn[int(w16 >> 9u)]) + let ix = int(w16 & 511u) * 2 + let m1 = smask[sgn & 15] + let m2 = smask[(sgn >> 4) & 15] + let w1 = (grid[ix] ^ m1) + (m1 & 0x01010101u) + let w2 = (grid[ix + 1] ^ m2) + (m2 & 0x01010101u) + // element e = l*8 + half*4 + t: e < 16 lands in the lo panel half, else hi + let e1 = l * 8l + let e2 = l * 8l + 4l + let d1 = (e1 < 16l ? 0l : 128l * mr) + ((blk * 4l + (e1 % 16l) / 4l) * mr + r) * 4l + let d2 = (e2 < 16l ? 0l : 128l * mr) + ((blk * 4l + (e2 % 16l) / 4l) * mr + r) * 4l + var wp1 = reinterpret(dst + (ob + d1)) + var wp2 = reinterpret(dst + (ob + d2)) + wp1[0] = w1 + wp2[0] = w2 + } + } + } + } + } +} + +//! iq2xxs panel unpack: block b's grid BYTES (column 2b) each double into the u64 grid's word +//! pair, its aux32 (column 2b+1) picks the KSIGNS sign bytes - packed grp planes -> the +//! shared signed panel (the per-32 scale rides the strip row). +def unpack_iq2xxs_panel_grp(kqg : uint8 const?; var dst : uint8?; mr, nsb : int64) { + let grid = iq2xxs_grid2() + let ksgn = ksigns_iq2xs() + let smask = fixed_array(0x00000000u, 0x000000FFu, 0x0000FF00u, 0x0000FFFFu, + 0x00FF0000u, 0x00FF00FFu, 0x00FFFF00u, 0x00FFFFFFu, + 0xFF000000u, 0xFF0000FFu, 0xFF00FF00u, 0xFF00FFFFu, + 0xFFFF0000u, 0xFFFF00FFu, 0xFFFFFF00u, 0xFFFFFFFFu) + unsafe { + for (sbi in range64(nsb)) { + let qb = sbi * 64l * mr + let ob = sbi * 256l * mr + for (r in range64(mr)) { + for (blk in range64(8l)) { + let ac = (2l * blk + 1l) * mr + r + let aux = uint(kqg[qb + ac * 4l]) | (uint(kqg[qb + ac * 4l + 1l]) << 8u) | (uint(kqg[qb + ac * 4l + 2l]) << 16u) | (uint(kqg[qb + ac * 4l + 3l]) << 24u) + let gc = (2l * blk) * mr + r + for (l in range64(4l)) { + let sgn = int(ksgn[int((aux >> uint(7l * l)) & 127u)]) + let ix = int(kqg[qb + gc * 4l + l]) * 2 + let m1 = smask[sgn & 15] + let m2 = smask[(sgn >> 4) & 15] + let w1 = (grid[ix] ^ m1) + (m1 & 0x01010101u) + let w2 = (grid[ix + 1] ^ m2) + (m2 & 0x01010101u) + let e1 = l * 8l + let e2 = l * 8l + 4l + let d1 = (e1 < 16l ? 0l : 128l * mr) + ((blk * 4l + (e1 % 16l) / 4l) * mr + r) * 4l + let d2 = (e2 < 16l ? 0l : 128l * mr) + ((blk * 4l + (e2 % 16l) / 4l) * mr + r) * 4l + var wp1 = reinterpret(dst + (ob + d1)) + var wp2 = reinterpret(dst + (ob + d2)) + wp1[0] = w1 + wp2[0] = w2 + } + } + } + } + } +} + def unpack_kq_panel_grp(fmt : int64; kqg : uint8 const?; var dst : uint8?; mr, nsb : int64) { + if (fmt == 33l) { + unpack_iq3s_panel_grp(kqg, dst, mr, nsb) + return + } + if (fmt == 34l) { + unpack_iq3xxs_panel_grp(kqg, dst, mr, nsb) + return + } + if (fmt == 23l) { + unpack_iq2s_panel_grp(kqg, dst, mr, nsb) + return + } + if (fmt == 25l) { + unpack_iq2xxs_panel_grp(kqg, dst, mr, nsb) + return + } + if (fmt == 24l) { + unpack_iq2xs_panel_grp(kqg, dst, mr, nsb) + return + } let LO4 = 0x0F0F0F0F0F0F0F0Ful let SEL = 0x0804020108040201ul // LE lane t (0..3, twice) selects high-bit t let C7F = 0x7F7F7F7F7F7F7F7Ful diff --git a/modules/dasLLAMA/dasllama/dasllama_tune.das b/modules/dasLLAMA/dasllama/dasllama_tune.das index 5ecd6a3a86..ba83e64231 100644 --- a/modules/dasLLAMA/dasllama/dasllama_tune.das +++ b/modules/dasLLAMA/dasllama/dasllama_tune.das @@ -30,11 +30,11 @@ struct TunePerm { } def private build_grid() : array { - // unroll=1 OMITS unroll_count so each row stays byte-identical to the original hand hints (DEFAULT_PERM vec8_u2 included); width 4/32 rarely win on M1 but x64 AVX-512 picks them. - let WIDTHS = fixed_array(4, 8, 16, 32) - let UNROLLS = fixed_array(2, 4, 8) // unroll-only rows; u1 == plain + // pruned to the seats that win beyond noise (the full 20-seat walk never spread >1.4% winner-vs-runner-up in any box's sidecar race tables - zen2 lcpp_bench/parity, m1, m4, m5); unroll=1 omits unroll_count so rows stay byte-identical to the hand hints + let WIDTHS = fixed_array(8, 16) + let UNROLLS = fixed_array(2) // unroll-only rows; u1 == plain var grid : array - grid |> reserve(1 + length(UNROLLS) + length(WIDTHS) * (1 + length(UNROLLS))) + grid |> reserve(2 + length(UNROLLS) + length(WIDTHS) * (1 + length(UNROLLS))) grid |> emplace(TunePerm(suffix = "plain")) for (u in UNROLLS) { grid |> emplace(TunePerm(suffix = "u{u}", hints <- [TuneHint(tune = 1, unroll = u)])) @@ -46,6 +46,7 @@ def private build_grid() : array { hints <- [TuneHint(tune = 1, vectorize = true, width = w, unroll = u)])) } } + grid |> emplace(TunePerm(suffix = "vec4_u4", hints <- [TuneHint(tune = 1, vectorize = true, width = 4, unroll = 4)])) // dot_q4's pinned NEON nibble-widening winner return <- grid } @@ -323,6 +324,10 @@ def private resolve_perm(func : FunctionPtr; args : AnnotationArgumentList) : st let fname = "{func.name}" let fb = fallback_perm(args) g_fallbacks[fname] = fb + if (tune_policy_env() == "reference") { + // the true kill switch, [tune]'s twin: the template's own loops, no hints stamped + return report_perm(fname, "plain", "reference policy") + } let av = find_arg(args, "perm") if (av is tString) return av as tString let cfg = config_perm_for(fname) diff --git a/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das b/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das index 4289abce11..fd964a8dfe 100644 --- a/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das +++ b/modules/dasLLAMA/dasllama/dasllama_vulkan_classes.das @@ -2464,6 +2464,620 @@ def private k5_dep(hb : uint) : uint { return ((hb & 1u) << 4u) | ((hb & 2u) << 11u) | ((hb & 4u) << 18u) | ((hb & 8u) << 25u) } +// IQ4_XS codebook decode of one word of 4 nibble lanes (bits 7..4 of each byte clear) into 4 +// SIGNED int8 lanes (kvalues_iq4nl) — the sdot4 operand form; batch and gemv share it. +// The codebook is four packed words picked by a dynamic vector index and byte-extracted: pure +// ALU (a fixed_array local was a Function-storage table, 3.4x slower on the decode GEMV - `benchmarks/lcpp_bench.das` tg128, 5060 Ti) +// iq3s_grid, one word per call - the staging loops copy it into workgroup memory once +// per kernel (a few indexed reads per thread), so the local-array storage class (QUIRK 20's +// slow path) never sits on a hot path; a shader function cannot return a fixed array. +def private iq3s_grid_word(i : int) : uint { + let tbl = fixed_array( + 0x01010101u, 0x01010103u, 0x01010105u, 0x0101010bu, 0x0101010fu, 0x01010301u, 0x01010303u, 0x01010305u, + 0x01010309u, 0x0101030du, 0x01010501u, 0x01010503u, 0x0101050bu, 0x01010707u, 0x01010901u, 0x01010905u, + 0x0101090bu, 0x0101090fu, 0x01010b03u, 0x01010b07u, 0x01010d01u, 0x01010d05u, 0x01010f03u, 0x01010f09u, + 0x01010f0fu, 0x01030101u, 0x01030103u, 0x01030105u, 0x01030109u, 0x01030301u, 0x01030303u, 0x0103030bu, + 0x01030501u, 0x01030507u, 0x0103050fu, 0x01030703u, 0x0103070bu, 0x01030909u, 0x01030d03u, 0x01030d0bu, + 0x01030f05u, 0x01050101u, 0x01050103u, 0x0105010bu, 0x0105010fu, 0x01050301u, 0x01050307u, 0x0105030du, + 0x01050503u, 0x0105050bu, 0x01050701u, 0x01050709u, 0x01050905u, 0x0105090bu, 0x0105090fu, 0x01050b03u, + 0x01050b07u, 0x01050f01u, 0x01050f07u, 0x01070107u, 0x01070303u, 0x0107030bu, 0x01070501u, 0x01070505u, + 0x01070703u, 0x01070707u, 0x0107070du, 0x01070909u, 0x01070b01u, 0x01070b05u, 0x01070d0fu, 0x01070f03u, + 0x01070f0bu, 0x01090101u, 0x01090307u, 0x0109030fu, 0x01090503u, 0x01090509u, 0x01090705u, 0x01090901u, + 0x01090907u, 0x01090b03u, 0x01090f01u, 0x010b0105u, 0x010b0109u, 0x010b0501u, 0x010b0505u, 0x010b050du, + 0x010b0707u, 0x010b0903u, 0x010b090bu, 0x010b090fu, 0x010b0d0du, 0x010b0f07u, 0x010d010du, 0x010d0303u, + 0x010d0307u, 0x010d0703u, 0x010d0b05u, 0x010d0f03u, 0x010f0101u, 0x010f0105u, 0x010f0109u, 0x010f0501u, + 0x010f0505u, 0x010f050du, 0x010f0707u, 0x010f0b01u, 0x010f0b09u, 0x03010101u, 0x03010103u, 0x03010105u, + 0x03010109u, 0x03010301u, 0x03010303u, 0x03010307u, 0x0301030bu, 0x0301030fu, 0x03010501u, 0x03010505u, + 0x03010703u, 0x03010709u, 0x0301070du, 0x03010b09u, 0x03010b0du, 0x03010d03u, 0x03010f05u, 0x03030101u, + 0x03030103u, 0x03030107u, 0x0303010du, 0x03030301u, 0x03030309u, 0x03030503u, 0x03030701u, 0x03030707u, + 0x03030903u, 0x03030b01u, 0x03030b05u, 0x03030f01u, 0x03030f0du, 0x03050101u, 0x03050305u, 0x0305030bu, + 0x0305030fu, 0x03050501u, 0x03050509u, 0x03050705u, 0x03050901u, 0x03050907u, 0x03050b0bu, 0x03050d01u, + 0x03050f05u, 0x03070103u, 0x03070109u, 0x0307010fu, 0x03070301u, 0x03070307u, 0x03070503u, 0x0307050fu, + 0x03070701u, 0x03070709u, 0x03070903u, 0x03070d05u, 0x03070f01u, 0x03090107u, 0x0309010bu, 0x03090305u, + 0x03090309u, 0x03090703u, 0x03090707u, 0x03090905u, 0x0309090du, 0x03090b01u, 0x03090b09u, 0x030b0103u, + 0x030b0301u, 0x030b0307u, 0x030b0503u, 0x030b0701u, 0x030b0705u, 0x030b0b03u, 0x030d0501u, 0x030d0509u, + 0x030d050fu, 0x030d0909u, 0x030d090du, 0x030f0103u, 0x030f0107u, 0x030f0301u, 0x030f0305u, 0x030f0503u, + 0x030f070bu, 0x030f0903u, 0x030f0d05u, 0x030f0f01u, 0x05010101u, 0x05010103u, 0x05010107u, 0x0501010bu, + 0x0501010fu, 0x05010301u, 0x05010305u, 0x05010309u, 0x0501030du, 0x05010503u, 0x05010507u, 0x0501050fu, + 0x05010701u, 0x05010705u, 0x05010903u, 0x05010907u, 0x0501090bu, 0x05010b01u, 0x05010b05u, 0x05010d0fu, + 0x05010f01u, 0x05010f07u, 0x05010f0bu, 0x05030101u, 0x05030105u, 0x05030301u, 0x05030307u, 0x0503030fu, + 0x05030505u, 0x0503050bu, 0x05030703u, 0x05030709u, 0x05030905u, 0x05030b03u, 0x05050103u, 0x05050109u, + 0x0505010fu, 0x05050503u, 0x05050507u, 0x05050701u, 0x0505070fu, 0x05050903u, 0x05050b07u, 0x05050b0fu, + 0x05050f03u, 0x05050f09u, 0x05070101u, 0x05070105u, 0x0507010bu, 0x05070303u, 0x05070505u, 0x05070509u, + 0x05070703u, 0x05070707u, 0x05070905u, 0x05070b01u, 0x05070d0du, 0x05090103u, 0x0509010fu, 0x05090501u, + 0x05090507u, 0x05090705u, 0x0509070bu, 0x05090903u, 0x05090f05u, 0x05090f0bu, 0x050b0109u, 0x050b0303u, + 0x050b0505u, 0x050b070fu, 0x050b0901u, 0x050b0b07u, 0x050b0f01u, 0x050d0101u, 0x050d0105u, 0x050d010fu, + 0x050d0503u, 0x050d0b0bu, 0x050d0d03u, 0x050f010bu, 0x050f0303u, 0x050f050du, 0x050f0701u, 0x050f0907u, + 0x050f0b01u, 0x07010105u, 0x07010303u, 0x07010307u, 0x0701030bu, 0x0701030fu, 0x07010505u, 0x07010703u, + 0x07010707u, 0x0701070bu, 0x07010905u, 0x07010909u, 0x0701090fu, 0x07010b03u, 0x07010d07u, 0x07010f03u, + 0x07030103u, 0x07030107u, 0x0703010bu, 0x07030309u, 0x07030503u, 0x07030507u, 0x07030901u, 0x07030d01u, + 0x07030f05u, 0x07030f0du, 0x07050101u, 0x07050305u, 0x07050501u, 0x07050705u, 0x07050709u, 0x07050b01u, + 0x07070103u, 0x07070301u, 0x07070309u, 0x07070503u, 0x07070507u, 0x0707050fu, 0x07070701u, 0x07070903u, + 0x07070907u, 0x0707090fu, 0x07070b0bu, 0x07070f07u, 0x07090107u, 0x07090303u, 0x0709030du, 0x07090505u, + 0x07090703u, 0x07090b05u, 0x07090d01u, 0x07090d09u, 0x070b0103u, 0x070b0301u, 0x070b0305u, 0x070b050bu, + 0x070b0705u, 0x070b0909u, 0x070b0b0du, 0x070b0f07u, 0x070d030du, 0x070d0903u, 0x070f0103u, 0x070f0107u, + 0x070f0501u, 0x070f0505u, 0x070f070bu, 0x09010101u, 0x09010109u, 0x09010305u, 0x09010501u, 0x09010509u, + 0x0901050fu, 0x09010705u, 0x09010903u, 0x09010b01u, 0x09010f01u, 0x09030105u, 0x0903010fu, 0x09030303u, + 0x09030307u, 0x09030505u, 0x09030701u, 0x0903070bu, 0x09030907u, 0x09030b03u, 0x09030b0bu, 0x09050103u, + 0x09050107u, 0x09050301u, 0x0905030bu, 0x09050503u, 0x09050707u, 0x09050901u, 0x09050b0fu, 0x09050d05u, + 0x09050f01u, 0x09070109u, 0x09070303u, 0x09070307u, 0x09070501u, 0x09070505u, 0x09070703u, 0x0907070bu, + 0x09090101u, 0x09090105u, 0x09090509u, 0x0909070fu, 0x09090901u, 0x09090f03u, 0x090b010bu, 0x090b010fu, + 0x090b0503u, 0x090b0d05u, 0x090d0307u, 0x090d0709u, 0x090d0d01u, 0x090f0301u, 0x090f030bu, 0x090f0701u, + 0x090f0907u, 0x090f0b03u, 0x0b010105u, 0x0b010301u, 0x0b010309u, 0x0b010505u, 0x0b010901u, 0x0b010909u, + 0x0b01090fu, 0x0b010b05u, 0x0b010d0du, 0x0b010f09u, 0x0b030103u, 0x0b030107u, 0x0b03010bu, 0x0b030305u, + 0x0b030503u, 0x0b030705u, 0x0b030f05u, 0x0b050101u, 0x0b050303u, 0x0b050507u, 0x0b050701u, 0x0b05070du, + 0x0b050b07u, 0x0b070105u, 0x0b07010fu, 0x0b070301u, 0x0b07050fu, 0x0b070909u, 0x0b070b03u, 0x0b070d0bu, + 0x0b070f07u, 0x0b090103u, 0x0b090109u, 0x0b090501u, 0x0b090705u, 0x0b09090du, 0x0b0b0305u, 0x0b0b050du, + 0x0b0b0b03u, 0x0b0b0b07u, 0x0b0d0905u, 0x0b0f0105u, 0x0b0f0109u, 0x0b0f0505u, 0x0d010303u, 0x0d010307u, + 0x0d01030bu, 0x0d010703u, 0x0d010707u, 0x0d010d01u, 0x0d030101u, 0x0d030501u, 0x0d03050fu, 0x0d030d09u, + 0x0d050305u, 0x0d050709u, 0x0d050905u, 0x0d050b0bu, 0x0d050d05u, 0x0d050f01u, 0x0d070101u, 0x0d070309u, + 0x0d070503u, 0x0d070901u, 0x0d09050bu, 0x0d090907u, 0x0d090d05u, 0x0d0b0101u, 0x0d0b0107u, 0x0d0b0709u, + 0x0d0b0d01u, 0x0d0d010bu, 0x0d0d0901u, 0x0d0f0303u, 0x0d0f0307u, 0x0f010101u, 0x0f010109u, 0x0f01010fu, + 0x0f010501u, 0x0f010505u, 0x0f01070du, 0x0f010901u, 0x0f010b09u, 0x0f010d05u, 0x0f030105u, 0x0f030303u, + 0x0f030509u, 0x0f030907u, 0x0f03090bu, 0x0f050103u, 0x0f050109u, 0x0f050301u, 0x0f05030du, 0x0f050503u, + 0x0f050701u, 0x0f050b03u, 0x0f070105u, 0x0f070705u, 0x0f07070bu, 0x0f070b07u, 0x0f090103u, 0x0f09010bu, + 0x0f090307u, 0x0f090501u, 0x0f090b01u, 0x0f0b0505u, 0x0f0b0905u, 0x0f0d0105u, 0x0f0d0703u, 0x0f0f0101u) + return tbl[i] +} + +// iq2xxs_grid as low/high word pairs - 256 u64 entries, one uint word per call (word +// 2i = magnitudes 0..3 of entry i, word 2i+1 = 4..7); staged into workgroup memory per kernel. +def private iq2xxs_grid_word(i : int) : uint { + let tbl = fixed_array( + 0x08080808, 0x08080808, 0x0808082b, 0x08080808, 0x08081919, 0x08080808, 0x08082b08, 0x08080808, + 0x08082b2b, 0x08080808, 0x08190819, 0x08080808, 0x08191908, 0x08080808, 0x082b0808, 0x08080808, + 0x082b082b, 0x08080808, 0x082b2b08, 0x08080808, 0x082b2b2b, 0x08080808, 0x19080819, 0x08080808, + 0x19081908, 0x08080808, 0x19190808, 0x08080808, 0x19192b08, 0x08080808, 0x192b0819, 0x08080808, + 0x192b1908, 0x08080808, 0x2b080808, 0x08080808, 0x2b08082b, 0x08080808, 0x2b082b2b, 0x08080808, + 0x2b2b082b, 0x08080808, 0x08080819, 0x08080819, 0x08081908, 0x08080819, 0x08190808, 0x08080819, + 0x08191919, 0x08080819, 0x19080808, 0x08080819, 0x2b081908, 0x08080819, 0x2b192b08, 0x08080819, + 0x08080808, 0x0808082b, 0x0808082b, 0x0808082b, 0x082b082b, 0x0808082b, 0x2b08082b, 0x0808082b, + 0x08080819, 0x08081908, 0x08081908, 0x08081908, 0x08190808, 0x08081908, 0x082b0819, 0x08081908, + 0x082b1908, 0x08081908, 0x19080808, 0x08081908, 0x1908082b, 0x08081908, 0x19082b08, 0x08081908, + 0x192b0808, 0x08081908, 0x2b080819, 0x08081908, 0x2b081908, 0x08081908, 0x2b190808, 0x08081908, + 0x2b2b1908, 0x08081908, 0x08080808, 0x08081919, 0x0808082b, 0x08081919, 0x08082b08, 0x08081919, + 0x082b0808, 0x08081919, 0x1908192b, 0x08081919, 0x192b2b19, 0x08081919, 0x2b080808, 0x08081919, + 0x2b190819, 0x08081919, 0x08082b19, 0x0808192b, 0x08190808, 0x0808192b, 0x19080808, 0x0808192b, + 0x2b081908, 0x0808192b, 0x2b2b1908, 0x0808192b, 0x08080808, 0x08082b08, 0x08081919, 0x08082b08, + 0x08082b08, 0x08082b08, 0x08191908, 0x08082b08, 0x082b2b08, 0x08082b08, 0x19080819, 0x08082b08, + 0x19081908, 0x08082b08, 0x19190808, 0x08082b08, 0x1919082b, 0x08082b08, 0x2b082b08, 0x08082b08, + 0x08081908, 0x08082b19, 0x19080808, 0x08082b19, 0x0808082b, 0x08082b2b, 0x08191908, 0x08082b2b, + 0x08080819, 0x08190808, 0x08081908, 0x08190808, 0x08190808, 0x08190808, 0x082b0819, 0x08190808, + 0x19080808, 0x08190808, 0x192b0808, 0x08190808, 0x2b081908, 0x08190808, 0x2b190808, 0x08190808, + 0x2b191919, 0x08190808, 0x08080808, 0x08190819, 0x08082b08, 0x08190819, 0x082b0808, 0x08190819, + 0x19190808, 0x08190819, 0x19192b2b, 0x08190819, 0x2b080808, 0x08190819, 0x082b1908, 0x0819082b, + 0x19081919, 0x0819082b, 0x08080808, 0x08191908, 0x08082b08, 0x08191908, 0x082b0808, 0x08191908, + 0x082b1919, 0x08191908, 0x19082b19, 0x08191908, 0x2b080808, 0x08191908, 0x08192b08, 0x08191919, + 0x192b082b, 0x08191919, 0x08080808, 0x0819192b, 0x0819192b, 0x0819192b, 0x08080819, 0x08192b08, + 0x08081908, 0x08192b08, 0x08190808, 0x08192b08, 0x19080808, 0x08192b08, 0x2b080819, 0x08192b08, + 0x08080808, 0x08192b19, 0x08081919, 0x08192b19, 0x2b2b0808, 0x08192b19, 0x19190819, 0x08192b2b, + 0x08080808, 0x082b0808, 0x0808082b, 0x082b0808, 0x08082b2b, 0x082b0808, 0x19081908, 0x082b0808, + 0x192b0819, 0x082b0808, 0x2b080808, 0x082b0808, 0x2b08082b, 0x082b0808, 0x082b2b19, 0x082b0819, + 0x19082b08, 0x082b0819, 0x08080808, 0x082b082b, 0x0808082b, 0x082b082b, 0x08080819, 0x082b1908, + 0x08081908, 0x082b1908, 0x08190808, 0x082b1908, 0x19080808, 0x082b1908, 0x1919192b, 0x082b1908, + 0x08080808, 0x082b1919, 0x19080819, 0x082b1919, 0x192b1908, 0x082b1919, 0x2b190808, 0x082b192b, + 0x08082b08, 0x082b2b08, 0x082b0808, 0x082b2b08, 0x2b191908, 0x082b2b08, 0x19081908, 0x082b2b2b, + 0x08080819, 0x19080808, 0x08081908, 0x19080808, 0x08190808, 0x19080808, 0x08192b08, 0x19080808, + 0x082b0819, 0x19080808, 0x082b1908, 0x19080808, 0x19080808, 0x19080808, 0x19082b08, 0x19080808, + 0x1919192b, 0x19080808, 0x192b0808, 0x19080808, 0x2b080819, 0x19080808, 0x2b081908, 0x19080808, + 0x2b190808, 0x19080808, 0x08080808, 0x19080819, 0x082b0808, 0x19080819, 0x192b0819, 0x19080819, + 0x2b080808, 0x19080819, 0x2b081919, 0x19080819, 0x08080819, 0x1908082b, 0x08190808, 0x1908082b, + 0x19082b08, 0x1908082b, 0x1919192b, 0x1908082b, 0x192b2b08, 0x1908082b, 0x08080808, 0x19081908, + 0x08082b08, 0x19081908, 0x082b0808, 0x19081908, 0x2b080808, 0x19081908, 0x2b192b19, 0x19081908, + 0x0819082b, 0x19081919, 0x082b1908, 0x19081919, 0x08080808, 0x1908192b, 0x08080819, 0x19082b08, + 0x08081908, 0x19082b08, 0x08190808, 0x19082b08, 0x19080808, 0x19082b08, 0x19081919, 0x19082b08, + 0x08080808, 0x19082b19, 0x19192b08, 0x19082b19, 0x192b0819, 0x19082b19, 0x2b08082b, 0x19082b19, + 0x19081919, 0x19082b2b, 0x2b190808, 0x19082b2b, 0x08080808, 0x19190808, 0x08082b08, 0x19190808, + 0x08190819, 0x19190808, 0x08192b19, 0x19190808, 0x082b0808, 0x19190808, 0x2b080808, 0x19190808, + 0x2b082b08, 0x19190808, 0x08081908, 0x19190819, 0x1908082b, 0x19190819, 0x2b2b1908, 0x19190819, + 0x2b190819, 0x1919082b, 0x2b190808, 0x19191908, 0x2b19082b, 0x19191908, 0x08082b2b, 0x19191919, + 0x08080819, 0x1919192b, 0x19191908, 0x1919192b, 0x08080808, 0x19192b08, 0x08190819, 0x19192b08, + 0x08192b19, 0x19192b08, 0x192b1908, 0x19192b08, 0x19080808, 0x19192b19, 0x08082b08, 0x19192b2b, + 0x08081908, 0x192b0808, 0x08190808, 0x192b0808, 0x19080808, 0x192b0808, 0x192b2b08, 0x192b0808, + 0x08080808, 0x192b0819, 0x19191919, 0x192b0819, 0x08192b08, 0x192b082b, 0x192b0808, 0x192b082b, + 0x08080808, 0x192b1908, 0x08081919, 0x192b1908, 0x08190808, 0x192b1919, 0x0819082b, 0x192b1919, + 0x2b081908, 0x192b1919, 0x1908082b, 0x192b2b08, 0x08080808, 0x2b080808, 0x0808082b, 0x2b080808, + 0x08082b2b, 0x2b080808, 0x19080819, 0x2b080808, 0x2b08082b, 0x2b080808, 0x08081908, 0x2b080819, + 0x08192b08, 0x2b080819, 0x19080808, 0x2b080819, 0x08190819, 0x2b08082b, 0x08080819, 0x2b081908, + 0x08081908, 0x2b081908, 0x08190808, 0x2b081908, 0x08191919, 0x2b081908, 0x19080808, 0x2b081908, + 0x192b0808, 0x2b081908, 0x08080808, 0x2b081919, 0x1908192b, 0x2b081919, 0x2b191908, 0x2b081919, + 0x08082b19, 0x2b08192b, 0x19080808, 0x2b08192b, 0x192b0808, 0x2b08192b, 0x0808082b, 0x2b082b08, + 0x08081908, 0x2b082b19, 0x08190819, 0x2b082b2b, 0x08081908, 0x2b190808, 0x08190808, 0x2b190808, + 0x082b1908, 0x2b190808, 0x19080808, 0x2b190808, 0x2b2b0819, 0x2b190808, 0x0819192b, 0x2b190819, + 0x2b080808, 0x2b190819, 0x19081919, 0x2b19082b, 0x08080808, 0x2b191908, 0x082b082b, 0x2b191908, + 0x19081908, 0x2b191908, 0x19190819, 0x2b191919, 0x2b080819, 0x2b192b08, 0x082b0808, 0x2b192b19, + 0x0808082b, 0x2b2b0808, 0x19190808, 0x2b2b0808, 0x2b081919, 0x2b2b0808, 0x08082b19, 0x2b2b0819, + 0x08080808, 0x2b2b082b, 0x08192b08, 0x2b2b1908, 0x19190808, 0x2b2b2b08, 0x08081908, 0x2b2b2b19) + return tbl[i] +} + +// iq2xs_grid as low/high word pairs - 512 u64 entries, one uint word per call (word +// 2i = magnitudes 0..3 of entry i, word 2i+1 = 4..7); staged into workgroup memory per kernel. +def private iq2xs_grid_word(i : int) : uint { // nolint:STYLE038 - a 1024-word data table, not splittable + let tbl = fixed_array( + 0x08080808, 0x08080808, 0x0808082b, 0x08080808, 0x08081919, 0x08080808, 0x08082b08, 0x08080808, + 0x08082b2b, 0x08080808, 0x08190819, 0x08080808, 0x08191908, 0x08080808, 0x0819192b, 0x08080808, + 0x08192b19, 0x08080808, 0x082b0808, 0x08080808, 0x082b082b, 0x08080808, 0x082b1919, 0x08080808, + 0x082b2b08, 0x08080808, 0x19080819, 0x08080808, 0x19081908, 0x08080808, 0x1908192b, 0x08080808, + 0x19082b19, 0x08080808, 0x19190808, 0x08080808, 0x1919082b, 0x08080808, 0x19191919, 0x08080808, + 0x19192b08, 0x08080808, 0x192b0819, 0x08080808, 0x192b1908, 0x08080808, 0x2b080808, 0x08080808, + 0x2b08082b, 0x08080808, 0x2b081919, 0x08080808, 0x2b082b08, 0x08080808, 0x2b190819, 0x08080808, + 0x2b191908, 0x08080808, 0x2b192b19, 0x08080808, 0x2b2b0808, 0x08080808, 0x08080819, 0x08080819, + 0x08081908, 0x08080819, 0x0808192b, 0x08080819, 0x08082b19, 0x08080819, 0x08190808, 0x08080819, + 0x0819082b, 0x08080819, 0x08191919, 0x08080819, 0x08192b08, 0x08080819, 0x08192b2b, 0x08080819, + 0x082b0819, 0x08080819, 0x082b1908, 0x08080819, 0x19080808, 0x08080819, 0x1908082b, 0x08080819, + 0x19081919, 0x08080819, 0x19082b08, 0x08080819, 0x19190819, 0x08080819, 0x19191908, 0x08080819, + 0x192b0808, 0x08080819, 0x192b2b08, 0x08080819, 0x2b080819, 0x08080819, 0x2b081908, 0x08080819, + 0x2b190808, 0x08080819, 0x08080808, 0x0808082b, 0x0808082b, 0x0808082b, 0x08081919, 0x0808082b, + 0x08082b08, 0x0808082b, 0x08190819, 0x0808082b, 0x08191908, 0x0808082b, 0x082b0808, 0x0808082b, + 0x19080819, 0x0808082b, 0x19081908, 0x0808082b, 0x19190808, 0x0808082b, 0x19191919, 0x0808082b, + 0x2b080808, 0x0808082b, 0x2b082b2b, 0x0808082b, 0x08080819, 0x08081908, 0x08081908, 0x08081908, + 0x0808192b, 0x08081908, 0x08082b19, 0x08081908, 0x08190808, 0x08081908, 0x0819082b, 0x08081908, + 0x08191919, 0x08081908, 0x08192b08, 0x08081908, 0x082b0819, 0x08081908, 0x082b1908, 0x08081908, + 0x19080808, 0x08081908, 0x1908082b, 0x08081908, 0x19081919, 0x08081908, 0x19082b08, 0x08081908, + 0x19190819, 0x08081908, 0x19191908, 0x08081908, 0x1919192b, 0x08081908, 0x192b0808, 0x08081908, + 0x2b080819, 0x08081908, 0x2b081908, 0x08081908, 0x2b190808, 0x08081908, 0x08080808, 0x08081919, + 0x0808082b, 0x08081919, 0x08081919, 0x08081919, 0x08082b08, 0x08081919, 0x08190819, 0x08081919, + 0x08191908, 0x08081919, 0x082b0808, 0x08081919, 0x19080819, 0x08081919, 0x19081908, 0x08081919, + 0x19190808, 0x08081919, 0x192b0819, 0x08081919, 0x2b080808, 0x08081919, 0x08080819, 0x0808192b, + 0x08081908, 0x0808192b, 0x08190808, 0x0808192b, 0x082b192b, 0x0808192b, 0x19080808, 0x0808192b, + 0x1908082b, 0x0808192b, 0x2b081908, 0x0808192b, 0x08080808, 0x08082b08, 0x0808082b, 0x08082b08, + 0x08081919, 0x08082b08, 0x08082b08, 0x08082b08, 0x08082b2b, 0x08082b08, 0x08190819, 0x08082b08, + 0x08191908, 0x08082b08, 0x082b0808, 0x08082b08, 0x082b1919, 0x08082b08, 0x19080819, 0x08082b08, + 0x19081908, 0x08082b08, 0x19190808, 0x08082b08, 0x19192b08, 0x08082b08, 0x2b080808, 0x08082b08, + 0x2b2b0808, 0x08082b08, 0x2b2b2b2b, 0x08082b08, 0x08080819, 0x08082b19, 0x08081908, 0x08082b19, + 0x08190808, 0x08082b19, 0x19080808, 0x08082b19, 0x2b080819, 0x08082b19, 0x2b082b19, 0x08082b19, + 0x08080808, 0x08082b2b, 0x082b0808, 0x08082b2b, 0x082b2b08, 0x08082b2b, 0x2b19192b, 0x08082b2b, + 0x2b2b0808, 0x08082b2b, 0x08080819, 0x08190808, 0x08081908, 0x08190808, 0x0808192b, 0x08190808, + 0x08082b19, 0x08190808, 0x08190808, 0x08190808, 0x0819082b, 0x08190808, 0x08191919, 0x08190808, + 0x08192b08, 0x08190808, 0x082b0819, 0x08190808, 0x082b1908, 0x08190808, 0x19080808, 0x08190808, + 0x1908082b, 0x08190808, 0x19081919, 0x08190808, 0x19082b08, 0x08190808, 0x19190819, 0x08190808, + 0x19191908, 0x08190808, 0x192b0808, 0x08190808, 0x192b2b2b, 0x08190808, 0x2b080819, 0x08190808, + 0x2b081908, 0x08190808, 0x2b190808, 0x08190808, 0x08080808, 0x08190819, 0x0808082b, 0x08190819, + 0x08081919, 0x08190819, 0x08082b08, 0x08190819, 0x08190819, 0x08190819, 0x08191908, 0x08190819, + 0x082b0808, 0x08190819, 0x19080819, 0x08190819, 0x19081908, 0x08190819, 0x19190808, 0x08190819, + 0x2b080808, 0x08190819, 0x2b191908, 0x08190819, 0x2b19192b, 0x08190819, 0x08080819, 0x0819082b, + 0x08081908, 0x0819082b, 0x0808192b, 0x0819082b, 0x08190808, 0x0819082b, 0x19080808, 0x0819082b, + 0x192b0808, 0x0819082b, 0x08080808, 0x08191908, 0x0808082b, 0x08191908, 0x08081919, 0x08191908, + 0x08082b08, 0x08191908, 0x08190819, 0x08191908, 0x08191908, 0x08191908, 0x082b0808, 0x08191908, + 0x19080819, 0x08191908, 0x19081908, 0x08191908, 0x19082b19, 0x08191908, 0x19190808, 0x08191908, + 0x192b1908, 0x08191908, 0x2b080808, 0x08191908, 0x08080819, 0x08191919, 0x08081908, 0x08191919, + 0x08190808, 0x08191919, 0x19080808, 0x08191919, 0x08080808, 0x0819192b, 0x08191908, 0x0819192b, + 0x19082b19, 0x0819192b, 0x08080819, 0x08192b08, 0x08081908, 0x08192b08, 0x08190808, 0x08192b08, + 0x0819082b, 0x08192b08, 0x19080808, 0x08192b08, 0x19191908, 0x08192b08, 0x2b08192b, 0x08192b08, + 0x08080808, 0x08192b19, 0x08081919, 0x08192b19, 0x192b192b, 0x08192b19, 0x19190819, 0x08192b2b, + 0x2b2b2b19, 0x08192b2b, 0x08080808, 0x082b0808, 0x0808082b, 0x082b0808, 0x08081919, 0x082b0808, + 0x08082b08, 0x082b0808, 0x08082b2b, 0x082b0808, 0x08190819, 0x082b0808, 0x08191908, 0x082b0808, + 0x082b0808, 0x082b0808, 0x19080819, 0x082b0808, 0x19081908, 0x082b0808, 0x19190808, 0x082b0808, + 0x2b080808, 0x082b0808, 0x2b2b0808, 0x082b0808, 0x08080819, 0x082b0819, 0x08081908, 0x082b0819, + 0x08190808, 0x082b0819, 0x19080808, 0x082b0819, 0x19082b08, 0x082b0819, 0x192b1919, 0x082b0819, + 0x08080808, 0x082b082b, 0x082b082b, 0x082b082b, 0x2b080808, 0x082b082b, 0x2b2b2b08, 0x082b082b, + 0x08080819, 0x082b1908, 0x08081908, 0x082b1908, 0x08190808, 0x082b1908, 0x082b2b19, 0x082b1908, + 0x19080808, 0x082b1908, 0x08080808, 0x082b1919, 0x19080819, 0x082b1919, 0x1919082b, 0x082b1919, + 0x2b192b19, 0x082b1919, 0x08080819, 0x082b192b, 0x08192b2b, 0x082b192b, 0x2b2b192b, 0x082b192b, + 0x08080808, 0x082b2b08, 0x08082b08, 0x082b2b08, 0x08082b2b, 0x082b2b08, 0x082b0808, 0x082b2b08, + 0x19191919, 0x082b2b08, 0x2b082b08, 0x082b2b08, 0x2b2b082b, 0x082b2b08, 0x192b2b08, 0x082b2b19, + 0x2b190808, 0x082b2b19, 0x08082b08, 0x082b2b2b, 0x082b0808, 0x082b2b2b, 0x2b08082b, 0x082b2b2b, + 0x2b082b08, 0x082b2b2b, 0x2b082b2b, 0x082b2b2b, 0x08080819, 0x19080808, 0x08081908, 0x19080808, + 0x0808192b, 0x19080808, 0x08082b19, 0x19080808, 0x08190808, 0x19080808, 0x0819082b, 0x19080808, + 0x08191919, 0x19080808, 0x08192b08, 0x19080808, 0x082b0819, 0x19080808, 0x082b1908, 0x19080808, + 0x19080808, 0x19080808, 0x1908082b, 0x19080808, 0x19081919, 0x19080808, 0x19082b08, 0x19080808, + 0x19082b2b, 0x19080808, 0x19190819, 0x19080808, 0x19191908, 0x19080808, 0x192b0808, 0x19080808, + 0x192b1919, 0x19080808, 0x2b080819, 0x19080808, 0x2b081908, 0x19080808, 0x2b190808, 0x19080808, + 0x08080808, 0x19080819, 0x0808082b, 0x19080819, 0x08081919, 0x19080819, 0x08082b08, 0x19080819, + 0x08190819, 0x19080819, 0x08191908, 0x19080819, 0x082b0808, 0x19080819, 0x19080819, 0x19080819, + 0x19081908, 0x19080819, 0x19190808, 0x19080819, 0x2b080808, 0x19080819, 0x2b081919, 0x19080819, + 0x2b2b082b, 0x19080819, 0x08080819, 0x1908082b, 0x08081908, 0x1908082b, 0x08190808, 0x1908082b, + 0x0819082b, 0x1908082b, 0x082b2b19, 0x1908082b, 0x19080808, 0x1908082b, 0x08080808, 0x19081908, + 0x0808082b, 0x19081908, 0x08081919, 0x19081908, 0x08082b08, 0x19081908, 0x08190819, 0x19081908, + 0x08191908, 0x19081908, 0x08192b19, 0x19081908, 0x082b0808, 0x19081908, 0x19080819, 0x19081908, + 0x19081908, 0x19081908, 0x19190808, 0x19081908, 0x2b080808, 0x19081908, 0x2b191908, 0x19081908, + 0x08080819, 0x19081919, 0x08081908, 0x19081919, 0x08190808, 0x19081919, 0x082b1908, 0x19081919, + 0x19080808, 0x19081919, 0x2b192b2b, 0x19081919, 0x08080808, 0x1908192b, 0x08082b2b, 0x1908192b, + 0x19081908, 0x1908192b, 0x19190808, 0x1908192b, 0x08080819, 0x19082b08, 0x08081908, 0x19082b08, + 0x08190808, 0x19082b08, 0x19080808, 0x19082b08, 0x19081919, 0x19082b08, 0x19191908, 0x19082b08, + 0x192b082b, 0x19082b08, 0x08080808, 0x19082b19, 0x08190819, 0x19082b19, 0x19081908, 0x19082b19, + 0x19190808, 0x19082b19, 0x192b2b19, 0x19082b19, 0x08081908, 0x19082b2b, 0x08080808, 0x19190808, + 0x0808082b, 0x19190808, 0x08081919, 0x19190808, 0x08082b08, 0x19190808, 0x08190819, 0x19190808, + 0x08191908, 0x19190808, 0x082b0808, 0x19190808, 0x082b2b08, 0x19190808, 0x19080819, 0x19190808, + 0x19081908, 0x19190808, 0x19190808, 0x19190808, 0x2b080808, 0x19190808, 0x08080819, 0x19190819, + 0x08081908, 0x19190819, 0x08190808, 0x19190819, 0x08191919, 0x19190819, 0x19080808, 0x19190819, + 0x1908082b, 0x19190819, 0x08080808, 0x1919082b, 0x19081908, 0x1919082b, 0x2b2b2b2b, 0x1919082b, + 0x08080819, 0x19191908, 0x08081908, 0x19191908, 0x08190808, 0x19191908, 0x082b0819, 0x19191908, + 0x19080808, 0x19191908, 0x192b0808, 0x19191908, 0x2b080819, 0x19191908, 0x2b2b0819, 0x19191908, + 0x08080808, 0x19191919, 0x08082b08, 0x19191919, 0x2b080808, 0x19191919, 0x2b082b08, 0x19191919, + 0x082b0819, 0x1919192b, 0x192b2b08, 0x1919192b, 0x2b2b0819, 0x1919192b, 0x08080808, 0x19192b08, + 0x08191908, 0x19192b08, 0x19080819, 0x19192b08, 0x19190808, 0x19192b08, 0x2b192b19, 0x19192b08, + 0x08192b2b, 0x19192b19, 0x19080808, 0x19192b19, 0x1908082b, 0x19192b19, 0x2b081919, 0x19192b2b, + 0x08080819, 0x192b0808, 0x08081908, 0x192b0808, 0x08190808, 0x192b0808, 0x19080808, 0x192b0808, + 0x19191908, 0x192b0808, 0x192b082b, 0x192b0808, 0x2b08192b, 0x192b0808, 0x2b2b2b19, 0x192b0808, + 0x08080808, 0x192b0819, 0x082b1908, 0x192b082b, 0x19082b2b, 0x192b082b, 0x2b19082b, 0x192b082b, + 0x08080808, 0x192b1908, 0x0819192b, 0x192b1908, 0x08190808, 0x192b1919, 0x19080808, 0x192b1919, + 0x19081919, 0x192b1919, 0x2b2b1908, 0x192b1919, 0x08080819, 0x192b2b08, 0x192b2b2b, 0x192b2b08, + 0x082b1919, 0x192b2b19, 0x0808192b, 0x192b2b2b, 0x19191908, 0x192b2b2b, 0x192b082b, 0x192b2b2b, + 0x08080808, 0x2b080808, 0x0808082b, 0x2b080808, 0x08081919, 0x2b080808, 0x08082b08, 0x2b080808, + 0x08190819, 0x2b080808, 0x08191908, 0x2b080808, 0x082b0808, 0x2b080808, 0x082b2b2b, 0x2b080808, + 0x19080819, 0x2b080808, 0x19081908, 0x2b080808, 0x19190808, 0x2b080808, 0x2b080808, 0x2b080808, + 0x2b08082b, 0x2b080808, 0x2b2b2b08, 0x2b080808, 0x2b2b2b2b, 0x2b080808, 0x08080819, 0x2b080819, + 0x08081908, 0x2b080819, 0x0808192b, 0x2b080819, 0x08190808, 0x2b080819, 0x19080808, 0x2b080819, + 0x19190819, 0x2b080819, 0x19192b19, 0x2b080819, 0x08080808, 0x2b08082b, 0x082b0808, 0x2b08082b, + 0x2b080808, 0x2b08082b, 0x2b08082b, 0x2b08082b, 0x2b2b0808, 0x2b08082b, 0x2b2b2b08, 0x2b08082b, + 0x08080819, 0x2b081908, 0x08081908, 0x2b081908, 0x08190808, 0x2b081908, 0x0819082b, 0x2b081908, + 0x08191919, 0x2b081908, 0x19080808, 0x2b081908, 0x192b0808, 0x2b081908, 0x2b082b19, 0x2b081908, + 0x08080808, 0x2b081919, 0x19081908, 0x2b081919, 0x2b2b1919, 0x2b081919, 0x08192b08, 0x2b08192b, + 0x192b2b2b, 0x2b08192b, 0x08080808, 0x2b082b08, 0x08082b08, 0x2b082b08, 0x082b1919, 0x2b082b08, + 0x19192b2b, 0x2b082b08, 0x2b080808, 0x2b082b08, 0x2b08082b, 0x2b082b08, 0x2b2b2b08, 0x2b082b08, + 0x0808192b, 0x2b082b19, 0x082b082b, 0x2b082b2b, 0x2b080808, 0x2b082b2b, 0x2b082b08, 0x2b082b2b, + 0x2b19192b, 0x2b082b2b, 0x2b2b2b08, 0x2b082b2b, 0x08080819, 0x2b190808, 0x08081908, 0x2b190808, + 0x08190808, 0x2b190808, 0x19080808, 0x2b190808, 0x1919192b, 0x2b190808, 0x2b081908, 0x2b190808, + 0x08080808, 0x2b190819, 0x082b082b, 0x2b190819, 0x192b1908, 0x2b190819, 0x1919192b, 0x2b19082b, + 0x2b082b19, 0x2b19082b, 0x08080808, 0x2b191908, 0x08081919, 0x2b191908, 0x19081908, 0x2b191908, + 0x19190808, 0x2b191908, 0x19192b08, 0x2b191908, 0x082b2b19, 0x2b191919, 0x2b190808, 0x2b191919, + 0x2b19082b, 0x2b191919, 0x19080819, 0x2b19192b, 0x19190819, 0x2b192b08, 0x2b2b192b, 0x2b192b08, + 0x19082b19, 0x2b192b19, 0x08191919, 0x2b192b2b, 0x192b0808, 0x2b192b2b, 0x08080808, 0x2b2b0808, + 0x0808082b, 0x2b2b0808, 0x08082b08, 0x2b2b0808, 0x08082b2b, 0x2b2b0808, 0x082b0808, 0x2b2b0808, + 0x082b2b2b, 0x2b2b0808, 0x2b2b0808, 0x2b2b0808, 0x19190819, 0x2b2b0819, 0x19192b19, 0x2b2b0819, + 0x2b2b192b, 0x2b2b0819, 0x08080808, 0x2b2b082b, 0x0808082b, 0x2b2b082b, 0x08082b08, 0x2b2b082b, + 0x082b2b2b, 0x2b2b082b, 0x2b080808, 0x2b2b082b, 0x2b2b0808, 0x2b2b082b, 0x19080808, 0x2b2b1908, + 0x2b191919, 0x2b2b1908, 0x192b1919, 0x2b2b192b, 0x2b192b08, 0x2b2b192b, 0x08082b2b, 0x2b2b2b08, + 0x082b0808, 0x2b2b2b08, 0x082b082b, 0x2b2b2b08, 0x082b2b08, 0x2b2b2b08, 0x2b2b0808, 0x2b2b2b08, + 0x2b2b2b08, 0x2b2b2b08, 0x08081908, 0x2b2b2b19, 0x2b081908, 0x2b2b2b19, 0x2b08192b, 0x2b2b2b19, + 0x082b2b08, 0x2b2b2b2b, 0x082b2b2b, 0x2b2b2b2b, 0x2b190819, 0x2b2b2b2b, 0x2b2b2b2b, 0x2b2b2b2b) + return tbl[i] +} + +// iq2s_grid as low/high word pairs - 1024 u64 entries, one uint word per call (word 2i = +// magnitudes 0..3 of entry i, word 2i+1 = 4..7); the staging loops copy it into workgroup +// memory once per kernel, so the local-array storage class never sits on a hot path. +def private iq2s_grid_word(i : int) : uint { // nolint:STYLE038 - a 2048-word data table, not splittable + let tbl = fixed_array( + 0x08080808, 0x08080808, 0x0808082b, 0x08080808, 0x08081919, 0x08080808, 0x08082b08, 0x08080808, + 0x08082b2b, 0x08080808, 0x08190819, 0x08080808, 0x08191908, 0x08080808, 0x0819192b, 0x08080808, + 0x08192b19, 0x08080808, 0x082b0808, 0x08080808, 0x082b082b, 0x08080808, 0x082b1919, 0x08080808, + 0x082b2b08, 0x08080808, 0x19080819, 0x08080808, 0x19081908, 0x08080808, 0x1908192b, 0x08080808, + 0x19082b19, 0x08080808, 0x19190808, 0x08080808, 0x1919082b, 0x08080808, 0x19191919, 0x08080808, + 0x19192b08, 0x08080808, 0x192b0819, 0x08080808, 0x192b1908, 0x08080808, 0x192b192b, 0x08080808, + 0x192b2b19, 0x08080808, 0x2b080808, 0x08080808, 0x2b08082b, 0x08080808, 0x2b081919, 0x08080808, + 0x2b082b08, 0x08080808, 0x2b190819, 0x08080808, 0x2b191908, 0x08080808, 0x2b2b0808, 0x08080808, + 0x2b2b1919, 0x08080808, 0x2b2b2b2b, 0x08080808, 0x08080819, 0x08080819, 0x08081908, 0x08080819, + 0x0808192b, 0x08080819, 0x08082b19, 0x08080819, 0x08190808, 0x08080819, 0x0819082b, 0x08080819, + 0x08191919, 0x08080819, 0x08192b08, 0x08080819, 0x082b0819, 0x08080819, 0x082b1908, 0x08080819, + 0x19080808, 0x08080819, 0x1908082b, 0x08080819, 0x19081919, 0x08080819, 0x19082b08, 0x08080819, + 0x19190819, 0x08080819, 0x19191908, 0x08080819, 0x1919192b, 0x08080819, 0x19192b19, 0x08080819, + 0x192b0808, 0x08080819, 0x192b1919, 0x08080819, 0x192b2b08, 0x08080819, 0x2b080819, 0x08080819, + 0x2b081908, 0x08080819, 0x2b190808, 0x08080819, 0x2b19082b, 0x08080819, 0x2b191919, 0x08080819, + 0x2b2b0819, 0x08080819, 0x2b2b1908, 0x08080819, 0x08080808, 0x0808082b, 0x0808082b, 0x0808082b, + 0x08081919, 0x0808082b, 0x08082b08, 0x0808082b, 0x08190819, 0x0808082b, 0x08191908, 0x0808082b, + 0x082b0808, 0x0808082b, 0x082b2b2b, 0x0808082b, 0x19080819, 0x0808082b, 0x19081908, 0x0808082b, + 0x1908192b, 0x0808082b, 0x19082b19, 0x0808082b, 0x19190808, 0x0808082b, 0x19191919, 0x0808082b, + 0x2b080808, 0x0808082b, 0x2b081919, 0x0808082b, 0x2b082b2b, 0x0808082b, 0x2b191908, 0x0808082b, + 0x2b2b082b, 0x0808082b, 0x08080819, 0x08081908, 0x08081908, 0x08081908, 0x0808192b, 0x08081908, + 0x08082b19, 0x08081908, 0x08190808, 0x08081908, 0x0819082b, 0x08081908, 0x08191919, 0x08081908, + 0x08192b08, 0x08081908, 0x082b0819, 0x08081908, 0x082b1908, 0x08081908, 0x082b192b, 0x08081908, + 0x082b2b19, 0x08081908, 0x19080808, 0x08081908, 0x1908082b, 0x08081908, 0x19081919, 0x08081908, + 0x19082b08, 0x08081908, 0x19082b2b, 0x08081908, 0x19190819, 0x08081908, 0x19191908, 0x08081908, + 0x1919192b, 0x08081908, 0x19192b19, 0x08081908, 0x192b0808, 0x08081908, 0x192b082b, 0x08081908, + 0x192b1919, 0x08081908, 0x2b080819, 0x08081908, 0x2b081908, 0x08081908, 0x2b08192b, 0x08081908, + 0x2b082b19, 0x08081908, 0x2b190808, 0x08081908, 0x2b191919, 0x08081908, 0x2b192b08, 0x08081908, + 0x2b2b0819, 0x08081908, 0x2b2b1908, 0x08081908, 0x08080808, 0x08081919, 0x0808082b, 0x08081919, + 0x08081919, 0x08081919, 0x08082b08, 0x08081919, 0x08082b2b, 0x08081919, 0x08190819, 0x08081919, + 0x08191908, 0x08081919, 0x0819192b, 0x08081919, 0x08192b19, 0x08081919, 0x082b0808, 0x08081919, + 0x082b1919, 0x08081919, 0x082b2b08, 0x08081919, 0x19080819, 0x08081919, 0x19081908, 0x08081919, + 0x1908192b, 0x08081919, 0x19082b19, 0x08081919, 0x19190808, 0x08081919, 0x1919082b, 0x08081919, + 0x19191919, 0x08081919, 0x19192b08, 0x08081919, 0x192b0819, 0x08081919, 0x192b1908, 0x08081919, + 0x2b080808, 0x08081919, 0x2b08082b, 0x08081919, 0x2b081919, 0x08081919, 0x2b082b08, 0x08081919, + 0x2b190819, 0x08081919, 0x2b191908, 0x08081919, 0x2b2b0808, 0x08081919, 0x08080819, 0x0808192b, + 0x08081908, 0x0808192b, 0x0808192b, 0x0808192b, 0x08082b19, 0x0808192b, 0x08190808, 0x0808192b, + 0x08191919, 0x0808192b, 0x19080808, 0x0808192b, 0x19081919, 0x0808192b, 0x19082b08, 0x0808192b, + 0x19190819, 0x0808192b, 0x19191908, 0x0808192b, 0x192b0808, 0x0808192b, 0x2b080819, 0x0808192b, + 0x2b081908, 0x0808192b, 0x2b190808, 0x0808192b, 0x08080808, 0x08082b08, 0x0808082b, 0x08082b08, + 0x08081919, 0x08082b08, 0x08082b08, 0x08082b08, 0x08190819, 0x08082b08, 0x08191908, 0x08082b08, + 0x0819192b, 0x08082b08, 0x08192b19, 0x08082b08, 0x082b0808, 0x08082b08, 0x082b1919, 0x08082b08, + 0x082b2b2b, 0x08082b08, 0x19080819, 0x08082b08, 0x19081908, 0x08082b08, 0x1908192b, 0x08082b08, + 0x19082b19, 0x08082b08, 0x19190808, 0x08082b08, 0x1919082b, 0x08082b08, 0x19191919, 0x08082b08, + 0x19192b08, 0x08082b08, 0x192b0819, 0x08082b08, 0x192b1908, 0x08082b08, 0x2b080808, 0x08082b08, + 0x2b081919, 0x08082b08, 0x2b191908, 0x08082b08, 0x2b2b2b2b, 0x08082b08, 0x08080819, 0x08082b19, + 0x08081908, 0x08082b19, 0x08190808, 0x08082b19, 0x0819082b, 0x08082b19, 0x08191919, 0x08082b19, + 0x08192b08, 0x08082b19, 0x082b0819, 0x08082b19, 0x19080808, 0x08082b19, 0x19081919, 0x08082b19, + 0x19082b08, 0x08082b19, 0x19190819, 0x08082b19, 0x19191908, 0x08082b19, 0x192b0808, 0x08082b19, + 0x2b080819, 0x08082b19, 0x2b190808, 0x08082b19, 0x08080808, 0x08082b2b, 0x08190819, 0x08082b2b, + 0x08191908, 0x08082b2b, 0x082b082b, 0x08082b2b, 0x082b2b08, 0x08082b2b, 0x082b2b2b, 0x08082b2b, + 0x19190808, 0x08082b2b, 0x2b192b19, 0x08082b2b, 0x08080819, 0x08190808, 0x08081908, 0x08190808, + 0x0808192b, 0x08190808, 0x08082b19, 0x08190808, 0x08190808, 0x08190808, 0x0819082b, 0x08190808, + 0x08191919, 0x08190808, 0x08192b08, 0x08190808, 0x082b0819, 0x08190808, 0x082b1908, 0x08190808, + 0x082b192b, 0x08190808, 0x19080808, 0x08190808, 0x1908082b, 0x08190808, 0x19081919, 0x08190808, + 0x19082b08, 0x08190808, 0x19190819, 0x08190808, 0x19191908, 0x08190808, 0x1919192b, 0x08190808, + 0x19192b19, 0x08190808, 0x192b0808, 0x08190808, 0x192b082b, 0x08190808, 0x192b1919, 0x08190808, + 0x192b2b08, 0x08190808, 0x2b080819, 0x08190808, 0x2b081908, 0x08190808, 0x2b08192b, 0x08190808, + 0x2b190808, 0x08190808, 0x2b191919, 0x08190808, 0x2b192b08, 0x08190808, 0x2b2b0819, 0x08190808, + 0x2b2b1908, 0x08190808, 0x08080808, 0x08190819, 0x0808082b, 0x08190819, 0x08081919, 0x08190819, + 0x08082b08, 0x08190819, 0x08082b2b, 0x08190819, 0x08190819, 0x08190819, 0x08191908, 0x08190819, + 0x0819192b, 0x08190819, 0x08192b19, 0x08190819, 0x082b0808, 0x08190819, 0x082b082b, 0x08190819, + 0x082b1919, 0x08190819, 0x082b2b08, 0x08190819, 0x19080819, 0x08190819, 0x19081908, 0x08190819, + 0x1908192b, 0x08190819, 0x19082b19, 0x08190819, 0x19190808, 0x08190819, 0x1919082b, 0x08190819, + 0x19191919, 0x08190819, 0x19192b08, 0x08190819, 0x192b0819, 0x08190819, 0x192b1908, 0x08190819, + 0x2b080808, 0x08190819, 0x2b08082b, 0x08190819, 0x2b081919, 0x08190819, 0x2b082b08, 0x08190819, + 0x2b190819, 0x08190819, 0x2b191908, 0x08190819, 0x08080819, 0x0819082b, 0x08081908, 0x0819082b, + 0x08082b19, 0x0819082b, 0x08190808, 0x0819082b, 0x08191919, 0x0819082b, 0x082b0819, 0x0819082b, + 0x082b1908, 0x0819082b, 0x19080808, 0x0819082b, 0x19081919, 0x0819082b, 0x19190819, 0x0819082b, + 0x19191908, 0x0819082b, 0x2b080819, 0x0819082b, 0x2b081908, 0x0819082b, 0x2b190808, 0x0819082b, + 0x08080808, 0x08191908, 0x0808082b, 0x08191908, 0x08081919, 0x08191908, 0x08082b08, 0x08191908, + 0x08190819, 0x08191908, 0x08191908, 0x08191908, 0x0819192b, 0x08191908, 0x08192b19, 0x08191908, + 0x082b0808, 0x08191908, 0x082b1919, 0x08191908, 0x082b2b08, 0x08191908, 0x19080819, 0x08191908, + 0x19081908, 0x08191908, 0x1908192b, 0x08191908, 0x19082b19, 0x08191908, 0x19190808, 0x08191908, + 0x1919082b, 0x08191908, 0x19191919, 0x08191908, 0x19192b08, 0x08191908, 0x192b0819, 0x08191908, + 0x192b1908, 0x08191908, 0x2b080808, 0x08191908, 0x2b08082b, 0x08191908, 0x2b081919, 0x08191908, + 0x2b082b08, 0x08191908, 0x2b190819, 0x08191908, 0x2b191908, 0x08191908, 0x2b2b0808, 0x08191908, + 0x08080819, 0x08191919, 0x08081908, 0x08191919, 0x0808192b, 0x08191919, 0x08082b19, 0x08191919, + 0x08190808, 0x08191919, 0x0819082b, 0x08191919, 0x08191919, 0x08191919, 0x08192b08, 0x08191919, + 0x082b0819, 0x08191919, 0x082b1908, 0x08191919, 0x19080808, 0x08191919, 0x1908082b, 0x08191919, + 0x19081919, 0x08191919, 0x19082b08, 0x08191919, 0x19190819, 0x08191919, 0x19191908, 0x08191919, + 0x192b0808, 0x08191919, 0x2b080819, 0x08191919, 0x2b081908, 0x08191919, 0x2b190808, 0x08191919, + 0x08080808, 0x0819192b, 0x08081919, 0x0819192b, 0x08082b08, 0x0819192b, 0x08190819, 0x0819192b, + 0x08191908, 0x0819192b, 0x082b0808, 0x0819192b, 0x19080819, 0x0819192b, 0x19081908, 0x0819192b, + 0x19190808, 0x0819192b, 0x2b080808, 0x0819192b, 0x2b2b2b2b, 0x0819192b, 0x08080819, 0x08192b08, + 0x08081908, 0x08192b08, 0x0808192b, 0x08192b08, 0x08082b19, 0x08192b08, 0x08190808, 0x08192b08, + 0x08191919, 0x08192b08, 0x08192b08, 0x08192b08, 0x082b0819, 0x08192b08, 0x19080808, 0x08192b08, + 0x1908082b, 0x08192b08, 0x19081919, 0x08192b08, 0x19082b08, 0x08192b08, 0x19190819, 0x08192b08, + 0x19191908, 0x08192b08, 0x192b0808, 0x08192b08, 0x2b080819, 0x08192b08, 0x2b081908, 0x08192b08, + 0x08080808, 0x08192b19, 0x0808082b, 0x08192b19, 0x08081919, 0x08192b19, 0x08082b08, 0x08192b19, + 0x08190819, 0x08192b19, 0x08191908, 0x08192b19, 0x082b0808, 0x08192b19, 0x19080819, 0x08192b19, + 0x19081908, 0x08192b19, 0x19190808, 0x08192b19, 0x192b2b19, 0x08192b19, 0x2b2b082b, 0x08192b19, + 0x08081908, 0x08192b2b, 0x08190808, 0x08192b2b, 0x19080808, 0x08192b2b, 0x1919192b, 0x08192b2b, + 0x08080808, 0x082b0808, 0x0808082b, 0x082b0808, 0x08081919, 0x082b0808, 0x08082b08, 0x082b0808, + 0x08190819, 0x082b0808, 0x08191908, 0x082b0808, 0x0819192b, 0x082b0808, 0x08192b19, 0x082b0808, + 0x082b0808, 0x082b0808, 0x082b1919, 0x082b0808, 0x082b2b2b, 0x082b0808, 0x19080819, 0x082b0808, + 0x19081908, 0x082b0808, 0x19190808, 0x082b0808, 0x1919082b, 0x082b0808, 0x19191919, 0x082b0808, + 0x192b1908, 0x082b0808, 0x2b080808, 0x082b0808, 0x2b082b2b, 0x082b0808, 0x2b191908, 0x082b0808, + 0x2b2b2b2b, 0x082b0808, 0x08080819, 0x082b0819, 0x08081908, 0x082b0819, 0x08190808, 0x082b0819, + 0x0819082b, 0x082b0819, 0x08191919, 0x082b0819, 0x082b0819, 0x082b0819, 0x19080808, 0x082b0819, + 0x1908082b, 0x082b0819, 0x19081919, 0x082b0819, 0x19190819, 0x082b0819, 0x19191908, 0x082b0819, + 0x192b0808, 0x082b0819, 0x2b080819, 0x082b0819, 0x2b081908, 0x082b0819, 0x2b190808, 0x082b0819, + 0x08080808, 0x082b082b, 0x08082b2b, 0x082b082b, 0x082b082b, 0x082b082b, 0x082b2b08, 0x082b082b, + 0x082b2b2b, 0x082b082b, 0x19081908, 0x082b082b, 0x19190808, 0x082b082b, 0x2b082b08, 0x082b082b, + 0x2b082b2b, 0x082b082b, 0x2b2b2b08, 0x082b082b, 0x08080819, 0x082b1908, 0x08081908, 0x082b1908, + 0x0808192b, 0x082b1908, 0x08082b19, 0x082b1908, 0x08190808, 0x082b1908, 0x08191919, 0x082b1908, + 0x08192b08, 0x082b1908, 0x082b0819, 0x082b1908, 0x082b1908, 0x082b1908, 0x19080808, 0x082b1908, + 0x1908082b, 0x082b1908, 0x19081919, 0x082b1908, 0x19082b08, 0x082b1908, 0x19190819, 0x082b1908, + 0x19191908, 0x082b1908, 0x192b0808, 0x082b1908, 0x2b080819, 0x082b1908, 0x2b081908, 0x082b1908, + 0x2b190808, 0x082b1908, 0x08080808, 0x082b1919, 0x08081919, 0x082b1919, 0x08082b08, 0x082b1919, + 0x08190819, 0x082b1919, 0x08191908, 0x082b1919, 0x082b0808, 0x082b1919, 0x19080819, 0x082b1919, + 0x19081908, 0x082b1919, 0x19190808, 0x082b1919, 0x192b192b, 0x082b1919, 0x2b080808, 0x082b1919, + 0x08080819, 0x082b192b, 0x08081908, 0x082b192b, 0x08190808, 0x082b192b, 0x19080808, 0x082b192b, + 0x19192b19, 0x082b192b, 0x08080808, 0x082b2b08, 0x08081919, 0x082b2b08, 0x08190819, 0x082b2b08, + 0x08191908, 0x082b2b08, 0x19080819, 0x082b2b08, 0x19081908, 0x082b2b08, 0x19190808, 0x082b2b08, + 0x2b082b2b, 0x082b2b08, 0x2b2b2b2b, 0x082b2b08, 0x08080819, 0x082b2b19, 0x08081908, 0x082b2b19, + 0x08190808, 0x082b2b19, 0x2b191919, 0x082b2b19, 0x08082b2b, 0x082b2b2b, 0x082b082b, 0x082b2b2b, + 0x192b1908, 0x082b2b2b, 0x2b082b08, 0x082b2b2b, 0x2b082b2b, 0x082b2b2b, 0x08080819, 0x19080808, + 0x08081908, 0x19080808, 0x0808192b, 0x19080808, 0x08082b19, 0x19080808, 0x08190808, 0x19080808, + 0x0819082b, 0x19080808, 0x08191919, 0x19080808, 0x08192b08, 0x19080808, 0x08192b2b, 0x19080808, + 0x082b0819, 0x19080808, 0x082b1908, 0x19080808, 0x082b192b, 0x19080808, 0x19080808, 0x19080808, + 0x1908082b, 0x19080808, 0x19081919, 0x19080808, 0x19082b08, 0x19080808, 0x19082b2b, 0x19080808, + 0x19190819, 0x19080808, 0x19191908, 0x19080808, 0x1919192b, 0x19080808, 0x19192b19, 0x19080808, + 0x192b0808, 0x19080808, 0x192b082b, 0x19080808, 0x192b1919, 0x19080808, 0x2b080819, 0x19080808, + 0x2b081908, 0x19080808, 0x2b190808, 0x19080808, 0x2b191919, 0x19080808, 0x2b192b08, 0x19080808, + 0x2b2b0819, 0x19080808, 0x2b2b1908, 0x19080808, 0x08080808, 0x19080819, 0x0808082b, 0x19080819, + 0x08081919, 0x19080819, 0x08082b08, 0x19080819, 0x08190819, 0x19080819, 0x08191908, 0x19080819, + 0x0819192b, 0x19080819, 0x08192b19, 0x19080819, 0x082b0808, 0x19080819, 0x082b082b, 0x19080819, + 0x082b1919, 0x19080819, 0x19080819, 0x19080819, 0x19081908, 0x19080819, 0x1908192b, 0x19080819, + 0x19082b19, 0x19080819, 0x19190808, 0x19080819, 0x1919082b, 0x19080819, 0x19191919, 0x19080819, + 0x19192b08, 0x19080819, 0x192b0819, 0x19080819, 0x192b1908, 0x19080819, 0x2b080808, 0x19080819, + 0x2b08082b, 0x19080819, 0x2b081919, 0x19080819, 0x2b082b08, 0x19080819, 0x2b190819, 0x19080819, + 0x2b191908, 0x19080819, 0x2b2b0808, 0x19080819, 0x08080819, 0x1908082b, 0x08081908, 0x1908082b, + 0x08190808, 0x1908082b, 0x0819082b, 0x1908082b, 0x08191919, 0x1908082b, 0x08192b08, 0x1908082b, + 0x082b1908, 0x1908082b, 0x19080808, 0x1908082b, 0x19081919, 0x1908082b, 0x19082b08, 0x1908082b, + 0x19190819, 0x1908082b, 0x19191908, 0x1908082b, 0x192b0808, 0x1908082b, 0x2b080819, 0x1908082b, + 0x2b081908, 0x1908082b, 0x08080808, 0x19081908, 0x0808082b, 0x19081908, 0x08081919, 0x19081908, + 0x08082b08, 0x19081908, 0x08082b2b, 0x19081908, 0x08190819, 0x19081908, 0x08191908, 0x19081908, + 0x0819192b, 0x19081908, 0x08192b19, 0x19081908, 0x082b0808, 0x19081908, 0x082b082b, 0x19081908, + 0x082b1919, 0x19081908, 0x082b2b08, 0x19081908, 0x19080819, 0x19081908, 0x19081908, 0x19081908, + 0x1908192b, 0x19081908, 0x19082b19, 0x19081908, 0x19190808, 0x19081908, 0x1919082b, 0x19081908, + 0x19191919, 0x19081908, 0x19192b08, 0x19081908, 0x192b0819, 0x19081908, 0x192b1908, 0x19081908, + 0x2b080808, 0x19081908, 0x2b08082b, 0x19081908, 0x2b081919, 0x19081908, 0x2b082b08, 0x19081908, + 0x2b190819, 0x19081908, 0x2b191908, 0x19081908, 0x2b2b0808, 0x19081908, 0x08080819, 0x19081919, + 0x08081908, 0x19081919, 0x0808192b, 0x19081919, 0x08082b19, 0x19081919, 0x08190808, 0x19081919, + 0x0819082b, 0x19081919, 0x08191919, 0x19081919, 0x08192b08, 0x19081919, 0x082b0819, 0x19081919, + 0x082b1908, 0x19081919, 0x19080808, 0x19081919, 0x1908082b, 0x19081919, 0x19081919, 0x19081919, + 0x19082b08, 0x19081919, 0x19190819, 0x19081919, 0x19191908, 0x19081919, 0x192b0808, 0x19081919, + 0x192b2b2b, 0x19081919, 0x2b080819, 0x19081919, 0x2b081908, 0x19081919, 0x2b190808, 0x19081919, + 0x08080808, 0x1908192b, 0x0808082b, 0x1908192b, 0x08081919, 0x1908192b, 0x08082b08, 0x1908192b, + 0x08190819, 0x1908192b, 0x08191908, 0x1908192b, 0x082b0808, 0x1908192b, 0x19080819, 0x1908192b, + 0x19081908, 0x1908192b, 0x19190808, 0x1908192b, 0x2b080808, 0x1908192b, 0x2b2b1919, 0x1908192b, + 0x08080819, 0x19082b08, 0x08081908, 0x19082b08, 0x08082b19, 0x19082b08, 0x08190808, 0x19082b08, + 0x0819082b, 0x19082b08, 0x08191919, 0x19082b08, 0x08192b08, 0x19082b08, 0x082b0819, 0x19082b08, + 0x082b1908, 0x19082b08, 0x19080808, 0x19082b08, 0x1908082b, 0x19082b08, 0x19081919, 0x19082b08, + 0x19082b08, 0x19082b08, 0x19190819, 0x19082b08, 0x19191908, 0x19082b08, 0x192b0808, 0x19082b08, + 0x2b081908, 0x19082b08, 0x2b190808, 0x19082b08, 0x08080808, 0x19082b19, 0x0808082b, 0x19082b19, + 0x08081919, 0x19082b19, 0x08082b08, 0x19082b19, 0x08190819, 0x19082b19, 0x08191908, 0x19082b19, + 0x082b0808, 0x19082b19, 0x19080819, 0x19082b19, 0x19081908, 0x19082b19, 0x19190808, 0x19082b19, + 0x2b080808, 0x19082b19, 0x2b19192b, 0x19082b19, 0x08080819, 0x19082b2b, 0x08081908, 0x19082b2b, + 0x08190808, 0x19082b2b, 0x19080808, 0x19082b2b, 0x08080808, 0x19190808, 0x0808082b, 0x19190808, + 0x08081919, 0x19190808, 0x08082b08, 0x19190808, 0x08190819, 0x19190808, 0x08191908, 0x19190808, + 0x0819192b, 0x19190808, 0x08192b19, 0x19190808, 0x082b0808, 0x19190808, 0x082b082b, 0x19190808, + 0x082b1919, 0x19190808, 0x082b2b08, 0x19190808, 0x19080819, 0x19190808, 0x19081908, 0x19190808, + 0x1908192b, 0x19190808, 0x19082b19, 0x19190808, 0x19190808, 0x19190808, 0x1919082b, 0x19190808, + 0x19191919, 0x19190808, 0x19192b08, 0x19190808, 0x192b0819, 0x19190808, 0x192b1908, 0x19190808, + 0x2b080808, 0x19190808, 0x2b08082b, 0x19190808, 0x2b081919, 0x19190808, 0x2b082b08, 0x19190808, + 0x2b190819, 0x19190808, 0x2b191908, 0x19190808, 0x08080819, 0x19190819, 0x08081908, 0x19190819, + 0x0808192b, 0x19190819, 0x08082b19, 0x19190819, 0x08190808, 0x19190819, 0x0819082b, 0x19190819, + 0x08191919, 0x19190819, 0x08192b08, 0x19190819, 0x082b0819, 0x19190819, 0x082b1908, 0x19190819, + 0x19080808, 0x19190819, 0x1908082b, 0x19190819, 0x19081919, 0x19190819, 0x19082b08, 0x19190819, + 0x19190819, 0x19190819, 0x19191908, 0x19190819, 0x192b0808, 0x19190819, 0x2b080819, 0x19190819, + 0x2b081908, 0x19190819, 0x2b190808, 0x19190819, 0x08080808, 0x1919082b, 0x08081919, 0x1919082b, + 0x08082b08, 0x1919082b, 0x08190819, 0x1919082b, 0x08191908, 0x1919082b, 0x082b0808, 0x1919082b, + 0x19080819, 0x1919082b, 0x19081908, 0x1919082b, 0x19190808, 0x1919082b, 0x192b2b19, 0x1919082b, + 0x2b080808, 0x1919082b, 0x08080819, 0x19191908, 0x08081908, 0x19191908, 0x0808192b, 0x19191908, + 0x08082b19, 0x19191908, 0x08190808, 0x19191908, 0x0819082b, 0x19191908, 0x08191919, 0x19191908, + 0x08192b08, 0x19191908, 0x082b0819, 0x19191908, 0x082b1908, 0x19191908, 0x19080808, 0x19191908, + 0x1908082b, 0x19191908, 0x19081919, 0x19191908, 0x19082b08, 0x19191908, 0x19190819, 0x19191908, + 0x19191908, 0x19191908, 0x192b0808, 0x19191908, 0x2b080819, 0x19191908, 0x2b081908, 0x19191908, + 0x2b190808, 0x19191908, 0x08080808, 0x19191919, 0x0808082b, 0x19191919, 0x08081919, 0x19191919, + 0x08082b08, 0x19191919, 0x08190819, 0x19191919, 0x08191908, 0x19191919, 0x082b0808, 0x19191919, + 0x19080819, 0x19191919, 0x19081908, 0x19191919, 0x19190808, 0x19191919, 0x2b080808, 0x19191919, + 0x08080819, 0x1919192b, 0x08081908, 0x1919192b, 0x08190808, 0x1919192b, 0x082b192b, 0x1919192b, + 0x19080808, 0x1919192b, 0x08080808, 0x19192b08, 0x0808082b, 0x19192b08, 0x08081919, 0x19192b08, + 0x08082b08, 0x19192b08, 0x08190819, 0x19192b08, 0x08191908, 0x19192b08, 0x082b0808, 0x19192b08, + 0x19080819, 0x19192b08, 0x19081908, 0x19192b08, 0x19190808, 0x19192b08, 0x19192b2b, 0x19192b08, + 0x2b080808, 0x19192b08, 0x08080819, 0x19192b19, 0x08081908, 0x19192b19, 0x08190808, 0x19192b19, + 0x19080808, 0x19192b19, 0x08080808, 0x19192b2b, 0x08192b19, 0x19192b2b, 0x2b081919, 0x19192b2b, + 0x2b2b2b08, 0x19192b2b, 0x08080819, 0x192b0808, 0x08081908, 0x192b0808, 0x0808192b, 0x192b0808, + 0x08190808, 0x192b0808, 0x0819082b, 0x192b0808, 0x08191919, 0x192b0808, 0x08192b08, 0x192b0808, + 0x082b0819, 0x192b0808, 0x082b1908, 0x192b0808, 0x19080808, 0x192b0808, 0x19081919, 0x192b0808, + 0x19082b08, 0x192b0808, 0x19190819, 0x192b0808, 0x19191908, 0x192b0808, 0x192b0808, 0x192b0808, + 0x2b081908, 0x192b0808, 0x2b190808, 0x192b0808, 0x08080808, 0x192b0819, 0x0808082b, 0x192b0819, + 0x08081919, 0x192b0819, 0x08082b08, 0x192b0819, 0x08190819, 0x192b0819, 0x08191908, 0x192b0819, + 0x082b0808, 0x192b0819, 0x19080819, 0x192b0819, 0x19081908, 0x192b0819, 0x19190808, 0x192b0819, + 0x2b080808, 0x192b0819, 0x2b192b19, 0x192b0819, 0x08081908, 0x192b082b, 0x08190808, 0x192b082b, + 0x19080808, 0x192b082b, 0x1919192b, 0x192b082b, 0x2b2b0819, 0x192b082b, 0x08080808, 0x192b1908, + 0x08081919, 0x192b1908, 0x08082b08, 0x192b1908, 0x08190819, 0x192b1908, 0x08191908, 0x192b1908, + 0x082b0808, 0x192b1908, 0x19080819, 0x192b1908, 0x19081908, 0x192b1908, 0x19190808, 0x192b1908, + 0x2b080808, 0x192b1908, 0x08080819, 0x192b1919, 0x08081908, 0x192b1919, 0x08190808, 0x192b1919, + 0x19080808, 0x192b1919, 0x19082b2b, 0x192b1919, 0x192b2b08, 0x192b1919, 0x2b19082b, 0x192b1919, + 0x08080808, 0x192b192b, 0x2b191908, 0x192b192b, 0x08080819, 0x192b2b08, 0x08081908, 0x192b2b08, + 0x08190808, 0x192b2b08, 0x192b1919, 0x192b2b08, 0x2b192b08, 0x192b2b08, 0x08080808, 0x192b2b19, + 0x082b2b2b, 0x192b2b19, 0x1908082b, 0x192b2b2b, 0x2b2b0819, 0x192b2b2b, 0x08080808, 0x2b080808, + 0x0808082b, 0x2b080808, 0x08081919, 0x2b080808, 0x08082b08, 0x2b080808, 0x08190819, 0x2b080808, + 0x08191908, 0x2b080808, 0x08192b19, 0x2b080808, 0x082b0808, 0x2b080808, 0x082b1919, 0x2b080808, + 0x19080819, 0x2b080808, 0x19081908, 0x2b080808, 0x19190808, 0x2b080808, 0x1919082b, 0x2b080808, + 0x19191919, 0x2b080808, 0x19192b08, 0x2b080808, 0x192b0819, 0x2b080808, 0x2b080808, 0x2b080808, + 0x2b081919, 0x2b080808, 0x2b190819, 0x2b080808, 0x2b191908, 0x2b080808, 0x08080819, 0x2b080819, + 0x08081908, 0x2b080819, 0x08082b19, 0x2b080819, 0x08190808, 0x2b080819, 0x0819082b, 0x2b080819, + 0x08191919, 0x2b080819, 0x08192b08, 0x2b080819, 0x082b0819, 0x2b080819, 0x082b1908, 0x2b080819, + 0x19080808, 0x2b080819, 0x1908082b, 0x2b080819, 0x19081919, 0x2b080819, 0x19082b08, 0x2b080819, + 0x19190819, 0x2b080819, 0x19191908, 0x2b080819, 0x2b080819, 0x2b080819, 0x2b081908, 0x2b080819, + 0x2b190808, 0x2b080819, 0x2b2b2b19, 0x2b080819, 0x08080808, 0x2b08082b, 0x08081919, 0x2b08082b, + 0x08082b2b, 0x2b08082b, 0x08190819, 0x2b08082b, 0x08191908, 0x2b08082b, 0x19080819, 0x2b08082b, + 0x19081908, 0x2b08082b, 0x19190808, 0x2b08082b, 0x08080819, 0x2b081908, 0x08081908, 0x2b081908, + 0x0808192b, 0x2b081908, 0x08082b19, 0x2b081908, 0x08190808, 0x2b081908, 0x0819082b, 0x2b081908, + 0x08191919, 0x2b081908, 0x08192b08, 0x2b081908, 0x082b0819, 0x2b081908, 0x19080808, 0x2b081908, + 0x1908082b, 0x2b081908, 0x19081919, 0x2b081908, 0x19082b08, 0x2b081908, 0x19190819, 0x2b081908, + 0x19191908, 0x2b081908, 0x192b0808, 0x2b081908, 0x2b080819, 0x2b081908, 0x2b081908, 0x2b081908, + 0x2b190808, 0x2b081908, 0x08080808, 0x2b081919, 0x0808082b, 0x2b081919, 0x08081919, 0x2b081919, + 0x08082b08, 0x2b081919, 0x08190819, 0x2b081919, 0x08191908, 0x2b081919, 0x082b0808, 0x2b081919, + 0x19080819, 0x2b081919, 0x19081908, 0x2b081919, 0x19190808, 0x2b081919, 0x2b080808, 0x2b081919, + 0x2b082b2b, 0x2b081919, 0x08080819, 0x2b08192b, 0x08081908, 0x2b08192b, 0x08190808, 0x2b08192b, + 0x082b2b19, 0x2b08192b, 0x19080808, 0x2b08192b, 0x08080808, 0x2b082b08, 0x08081919, 0x2b082b08, + 0x08190819, 0x2b082b08, 0x08191908, 0x2b082b08, 0x19080819, 0x2b082b08, 0x19081908, 0x2b082b08, + 0x19190808, 0x2b082b08, 0x2b2b082b, 0x2b082b08, 0x08080819, 0x2b082b19, 0x08081908, 0x2b082b19, + 0x19080808, 0x2b082b19, 0x192b1919, 0x2b082b19, 0x082b082b, 0x2b082b2b, 0x19192b08, 0x2b082b2b, + 0x19192b2b, 0x2b082b2b, 0x2b08082b, 0x2b082b2b, 0x2b2b082b, 0x2b082b2b, 0x08080819, 0x2b190808, + 0x08081908, 0x2b190808, 0x08082b19, 0x2b190808, 0x08190808, 0x2b190808, 0x0819082b, 0x2b190808, + 0x08191919, 0x2b190808, 0x08192b08, 0x2b190808, 0x082b1908, 0x2b190808, 0x19080808, 0x2b190808, + 0x1908082b, 0x2b190808, 0x19081919, 0x2b190808, 0x19082b08, 0x2b190808, 0x19190819, 0x2b190808, + 0x19191908, 0x2b190808, 0x192b0808, 0x2b190808, 0x2b080819, 0x2b190808, 0x2b081908, 0x2b190808, + 0x2b190808, 0x2b190808, 0x08080808, 0x2b190819, 0x08081919, 0x2b190819, 0x08190819, 0x2b190819, + 0x08191908, 0x2b190819, 0x19080819, 0x2b190819, 0x19081908, 0x2b190819, 0x19190808, 0x2b190819, + 0x19192b2b, 0x2b190819, 0x08080819, 0x2b19082b, 0x08081908, 0x2b19082b, 0x08190808, 0x2b19082b, + 0x19080808, 0x2b19082b, 0x2b2b192b, 0x2b19082b, 0x08080808, 0x2b191908, 0x0808082b, 0x2b191908, + 0x08081919, 0x2b191908, 0x08082b08, 0x2b191908, 0x08190819, 0x2b191908, 0x08191908, 0x2b191908, + 0x082b0808, 0x2b191908, 0x19080819, 0x2b191908, 0x19081908, 0x2b191908, 0x19190808, 0x2b191908, + 0x2b080808, 0x2b191908, 0x2b19192b, 0x2b191908, 0x08080819, 0x2b191919, 0x08081908, 0x2b191919, + 0x08190808, 0x2b191919, 0x19080808, 0x2b191919, 0x2b192b08, 0x2b191919, 0x2b2b0819, 0x2b191919, + 0x08080808, 0x2b19192b, 0x1908192b, 0x2b19192b, 0x192b1908, 0x2b19192b, 0x08080819, 0x2b192b08, + 0x08081908, 0x2b192b08, 0x08190808, 0x2b192b08, 0x082b192b, 0x2b192b08, 0x19080808, 0x2b192b08, + 0x2b2b2b19, 0x2b192b08, 0x08080808, 0x2b192b19, 0x19082b19, 0x2b192b19, 0x1919082b, 0x2b192b19, + 0x2b190808, 0x2b192b2b, 0x08080808, 0x2b2b0808, 0x08081919, 0x2b2b0808, 0x08082b2b, 0x2b2b0808, + 0x08191908, 0x2b2b0808, 0x082b082b, 0x2b2b0808, 0x082b2b2b, 0x2b2b0808, 0x19080819, 0x2b2b0808, + 0x19081908, 0x2b2b0808, 0x19190808, 0x2b2b0808, 0x2b2b082b, 0x2b2b0808, 0x2b2b2b2b, 0x2b2b0808, + 0x19080808, 0x2b2b0819, 0x192b1919, 0x2b2b0819, 0x0808082b, 0x2b2b082b, 0x08082b2b, 0x2b2b082b, + 0x082b082b, 0x2b2b082b, 0x082b2b08, 0x2b2b082b, 0x082b2b2b, 0x2b2b082b, 0x2b08082b, 0x2b2b082b, + 0x2b082b08, 0x2b2b082b, 0x2b082b2b, 0x2b2b082b, 0x2b2b2b08, 0x2b2b082b, 0x08080819, 0x2b2b1908, + 0x08081908, 0x2b2b1908, 0x08190808, 0x2b2b1908, 0x19080808, 0x2b2b1908, 0x2b082b19, 0x2b2b1908, + 0x2b2b1908, 0x2b2b1908, 0x08080808, 0x2b2b1919, 0x08192b19, 0x2b2b1919, 0x19190819, 0x2b2b192b, + 0x08082b2b, 0x2b2b2b08, 0x082b2b08, 0x2b2b2b08, 0x2b2b082b, 0x2b2b2b08, 0x19191908, 0x2b2b2b19, + 0x2b08192b, 0x2b2b2b19, 0x08082b08, 0x2b2b2b2b, 0x08082b2b, 0x2b2b2b2b, 0x082b0808, 0x2b2b2b2b, + 0x082b082b, 0x2b2b2b2b, 0x082b2b08, 0x2b2b2b2b, 0x2b082b08, 0x2b2b2b2b, 0x2b2b2b2b, 0x2b2b2b2b) + return tbl[i] +} + +// the HALVED iq3xxs grid (iq3xxs_grid with every byte / 2 - the plane convention that +// absorbs the format's 0.25 into the stored d), one word per call; staging loops copy it into +// workgroup memory (QUIRK 20's slow path stays off the hot loop). +def private iq3xxs_grid_word(i : int) : uint { + let tbl = fixed_array( + 0x02020202u, 0x0202020au, 0x02020212u, 0x02020606u, 0x0202060eu, 0x0202061fu, 0x02020a02u, 0x02020a0au, + 0x02020e06u, 0x0202120au, 0x02021f0eu, 0x02021f16u, 0x02060206u, 0x0206020eu, 0x02060602u, 0x0206060au, + 0x02060a06u, 0x02060a16u, 0x02060e02u, 0x02060e0au, 0x02061206u, 0x02061612u, 0x02061f02u, 0x020a0202u, + 0x020a020au, 0x020a0212u, 0x020a0606u, 0x020a0a02u, 0x020a0a0au, 0x020a0e06u, 0x020a0e0eu, 0x020a0e1fu, + 0x020a1606u, 0x020a161fu, 0x020a1f16u, 0x020e0206u, 0x020e021fu, 0x020e0602u, 0x020e060au, 0x020e0a16u, + 0x020e1f02u, 0x0212060eu, 0x02120e1fu, 0x02121212u, 0x0212161fu, 0x02121f0eu, 0x02121f16u, 0x02160206u, + 0x0216021fu, 0x02160e0au, 0x0216160au, 0x021a0e16u, 0x021a1a12u, 0x021f0602u, 0x021f0612u, 0x021f061au, + 0x021f120eu, 0x021f1a06u, 0x06020206u, 0x0602020eu, 0x06020602u, 0x0602060au, 0x06020a06u, 0x06020a0eu, + 0x06020e02u, 0x06020e0au, 0x06020e12u, 0x0602121fu, 0x06021602u, 0x06060202u, 0x0606020au, 0x06060606u, + 0x06060a02u, 0x06060a0au, 0x060a0206u, 0x060a020eu, 0x060a0602u, 0x060a060au, 0x060a0a06u, 0x060a0e02u, + 0x060a1f0au, 0x060e0202u, 0x060e020au, 0x060e0a02u, 0x060e0e06u, 0x060e121au, 0x060e1a1au, 0x06120206u, + 0x06120216u, 0x06121602u, 0x06160a02u, 0x06160a12u, 0x0616121au, 0x06161f06u, 0x061a0216u, 0x061f0a0au, + 0x061f1202u, 0x0a020202u, 0x0a02020au, 0x0a020606u, 0x0a02060eu, 0x0a020a02u, 0x0a020a0au, 0x0a020a1au, + 0x0a020e06u, 0x0a02120au, 0x0a060206u, 0x0a06020eu, 0x0a060216u, 0x0a060602u, 0x0a06060au, 0x0a060a06u, + 0x0a060e02u, 0x0a061a0eu, 0x0a061a1fu, 0x0a061f02u, 0x0a0a0202u, 0x0a0a020au, 0x0a0a0606u, 0x0a0a061fu, + 0x0a0a0a02u, 0x0a0a0a0au, 0x0a0a0e1fu, 0x0a0a1202u, 0x0a0a1616u, 0x0a0e0206u, 0x0a0e0602u, 0x0a0e0612u, + 0x0a0e1f02u, 0x0a0e1f12u, 0x0a120e16u, 0x0a12160eu, 0x0a16020eu, 0x0a160a1fu, 0x0a161206u, 0x0a161f12u, + 0x0a1f0206u, 0x0a1f020eu, 0x0a1f061au, 0x0a1f1216u, 0x0e020206u, 0x0e020602u, 0x0e02060au, 0x0e020a06u, + 0x0e020a0eu, 0x0e021602u, 0x0e021a16u, 0x0e021f0au, 0x0e060202u, 0x0e06020au, 0x0e060a02u, 0x0e060e06u, + 0x0e061212u, 0x0e06121au, 0x0e0a0206u, 0x0e0a020eu, 0x0e0a0602u, 0x0e0a0a16u, 0x0e0a160au, 0x0e0a1f0au, + 0x0e0e0606u, 0x0e0e0e0eu, 0x0e120e02u, 0x0e12121fu, 0x0e121f0au, 0x0e160202u, 0x0e16021au, 0x0e160a0au, + 0x0e161616u, 0x0e1a0612u, 0x0e1a0e1au, 0x0e1a1a0eu, 0x0e1f0e0eu, 0x0e1f1a02u, 0x12020212u, 0x1202061fu, + 0x12020e16u, 0x12020e1fu, 0x1202160eu, 0x1202161fu, 0x12061f12u, 0x120a0a02u, 0x120a0e1fu, 0x120a1202u, + 0x120a1a02u, 0x120a1a1au, 0x120e021fu, 0x120e1216u, 0x12120212u, 0x12121606u, 0x12121a12u, 0x12160a16u, + 0x1216120eu, 0x12161f02u, 0x121f0216u, 0x121f0602u, 0x121f060au, 0x121f0e02u, 0x1602060au, 0x16021206u, + 0x16021f02u, 0x16060202u, 0x1606021au, 0x16060a1au, 0x16061616u, 0x160a0612u, 0x160a0e0au, 0x160a1f0au, + 0x160e020au, 0x160e160eu, 0x16120602u, 0x16120a0eu, 0x16120a1fu, 0x16121f0au, 0x1616020au, 0x16160e06u, + 0x161a1602u, 0x161f0a12u, 0x161f120au, 0x1a020a12u, 0x1a021212u, 0x1a02121au, 0x1a021a12u, 0x1a060a06u, + 0x1a061a06u, 0x1a0a061fu, 0x1a0a1a12u, 0x1a0e0e02u, 0x1a0e0e1au, 0x1a121212u, 0x1a160216u, 0x1a16160au, + 0x1a1a0e0eu, 0x1a1f020eu, 0x1a1f0a06u, 0x1f02020eu, 0x1f020216u, 0x1f02021fu, 0x1f020602u, 0x1f020e0au, + 0x1f02160au, 0x1f060a1au, 0x1f061202u, 0x1f0a060au, 0x1f0a1216u, 0x1f0a160au, 0x1f0e0202u, 0x1f0e0616u, + 0x1f0e0e0eu, 0x1f0e1a02u, 0x1f120a06u, 0x1f121206u, 0x1f160202u, 0x1f16020au, 0x1f160a12u, 0x1f1a0e02u) + return tbl[i] +} + +// ksigns_iq2xs without the table: bit 7 of the sign byte is the EVEN PARITY of the 7-bit +// index, so the byte is idx | (parity << 7) - five ALU ops, nothing staged. +def private ksign7(v : uint) : uint { + var t = v ^ (v >> 4u) + t = t ^ (t >> 2u) + t = t ^ (t >> 1u) + return v | ((t & 1u) << 7u) +} + +def private iq4_word(n : uint) : uint { + let tbl = uint4(3215825025u, 4142587343u, 639175937u, 1901675829u) // kvalues_iq4nl, 4 bytes per word + let q0 = n & 15u + let q1 = (n >> 8u) & 15u + let q2 = (n >> 16u) & 15u + let q3 = (n >> 24u) & 15u + let b0 = (tbl[int(q0 >> 2u)] >> ((q0 & 3u) * 8u)) & 0xFFu + let b1 = (tbl[int(q1 >> 2u)] >> ((q1 & 3u) * 8u)) & 0xFFu + let b2 = (tbl[int(q2 >> 2u)] >> ((q2 & 3u) * 8u)) & 0xFFu + let b3 = (tbl[int(q3 >> 2u)] >> ((q3 & 3u) * 8u)) & 0xFFu + return b0 | (b1 << 8u) | (b2 << 16u) | (b3 << 24u) +} + +// the iq4xs scale row's sub-scale for block blk: word 1..2 of the 5-word row, SIGNED (ls - 32) +def private iq4_sc(w : uint; blk : uint) : int { + let s4 = int4(unpack8(int(w))) + return blk % 4u == 0u ? s4.x : (blk % 4u == 1u ? s4.y : (blk % 4u == 2u ? s4.z : s4.w)) +} + class KqBatchBase { @ssbo @binding = 0 wq : array // weight quant planes (per-format block layout) @ssbo @binding = 1 wsu : array // scale planes (5-word superblock layout) @@ -2682,6 +3296,209 @@ class KqBatchQ40 : KqBatchK4 { } } +// IQ4_XS tile: the k4 staging with the codebook decode in stage_w (signed weight lanes, so no +// block-sum term) and one signed-scale plane — per block acc += xs * d*sc * idot +[vk_dispatch(name = "kq_batch_iq4xs_cls", kernel = "run", family = "kq_batch_cls", grid = "wgs", params = "wgs : int64")] +class KqBatchIq4xs : KqBatchBase { + def override stage_w(wsb0, wcol, nsb, s, word, grp : uint) { + for (ki in range(2)) { + let k = word + uint(ki) * 8u + let bu = k / 2u + let sh = (k % 2u) * 4u + var wv = uint4(0u) + if (wcol < pa.d) { + let wsb = wsb0 + wcol * nsb + s + let rb4 = wsb * 32u + bu * 4u + wv = uint4(iq4_word((wq[rb4] >> sh) & 0x0F0F0F0F), iq4_word((wq[rb4 + 1u] >> sh) & 0x0F0F0F0F), + iq4_word((wq[rb4 + 2u] >> sh) & 0x0F0F0F0F), iq4_word((wq[rb4 + 3u] >> sh) & 0x0F0F0F0F)) + } + ww4[grp * 17u + k] = wv + } + } + + def override stage_ws(wsb0, wt, nsb, s, tid : uint) { + let slane = tid % 32u + let sblk = tid / 32u + var sa = 0.0 + if (wt * 32u + slane < pa.d) { + let wsb = wsb0 + (wt * 32u + slane) * nsb + s + sa = unpackHalf2x16(wsu[wsb * 5u]).x * float(iq4_sc(wsu[wsb * 5u + 1u + sblk / 4u], sblk)) + } + wsw[sblk * 32u + slane] = sa + } + + def override blk_fma(bu, grp, word : uint; xscl : float) : float4 { + var i = int4(0) + dots(bu, grp, word, i) + let ws = float4(wsw[bu * 32u + word], wsw[bu * 32u + word + 8u], + wsw[bu * 32u + word + 16u], wsw[bu * 32u + word + 24u]) + return xscl * ws * float4(i) + } + + [spirv_kernel(local_size_x = 256, name = "kq_batch_iq4xs_cls_spv")] + def run { + tile_shell() + } +} + +// IQ3_S tile: the iq4xs staging with the GRID GATHER in stage_w - each staged uint4 is four +// grid words sign-applied (mask = ((nibble * 0x00204081) & 0x01010101) * 255; magnitudes odd +// 1..15, no cross-byte carry); scale row and blk_fma are iq4xs's verbatim (inherited). The 2 KB +// grid stages into workgroup memory once, before the tile loop. +[vk_dispatch(name = "kq_batch_iq3s_cls", kernel = "run", family = "kq_batch_cls", grid = "wgs", params = "wgs : int64")] +class KqBatchIq3s : KqBatchIq4xs { + @workgroup gridw : uint[512] + + def iq3s_signed(g, nib : uint) : uint { + let m1 = (nib * 0x00204081u) & 0x01010101u + let mask = m1 * 255u + return (g ^ mask) + m1 + } + + def override stage_w(wsb0, wcol, nsb, s, word, grp : uint) { + for (ki in range(2)) { + let k = word + uint(ki) * 8u + let bu = k / 2u + let hh = k % 2u + var wv = uint4(0u) + if (wcol < pa.d) { + let wsb = wsb0 + wcol * nsb + s + let qsw = wq[wsb * 26u + bu * 2u + hh] + let qh = (wq[wsb * 26u + 16u + bu / 4u] >> ((bu % 4u) * 8u)) & 255u + let sgw = wq[wsb * 26u + 18u + bu] + let l0 = hh * 2u + let sg0 = (sgw >> (l0 * 8u)) & 255u + let sg1 = (sgw >> (l0 * 8u + 8u)) & 255u + let i0 = (qsw & 255u) | ((qh << (8u - 2u * l0)) & 256u) + let i1 = ((qsw >> 8u) & 255u) | ((qh << (7u - 2u * l0)) & 256u) + let i2 = ((qsw >> 16u) & 255u) | ((qh << (8u - 2u * (l0 + 1u))) & 256u) + let i3 = (qsw >> 24u) | ((qh << (7u - 2u * (l0 + 1u))) & 256u) + wv = uint4(iq3s_signed(gridw[i0], sg0 & 15u), iq3s_signed(gridw[i1], sg0 >> 4u), + iq3s_signed(gridw[i2], sg1 & 15u), iq3s_signed(gridw[i3], sg1 >> 4u)) + } + ww4[grp * 17u + k] = wv + } + } + + [spirv_kernel(local_size_x = 256, name = "kq_batch_iq3s_cls_spv")] + def override run { + let tid = gl_LocalInvocationID.x + gridw[tid] = iq3s_grid_word(int(tid)) + gridw[tid + 256u] = iq3s_grid_word(int(tid + 256u)) + barrier() // the grid visible before the first stage_w + tile_shell() + } +} + +// IQ3_XXS tile: the iq3s staging with the halved-grid gather - bare 8-bit indices (no ninth +// bit), signs from the aux32's four 7-bit ksigns indices via the parity helper (no table); +// scale row and blk_fma are iq4xs's verbatim (inherited). The 1 KB grid stages once. +[vk_dispatch(name = "kq_batch_iq3xxs_cls", kernel = "run", family = "kq_batch_cls", grid = "wgs", params = "wgs : int64")] +class KqBatchIq3xxs : KqBatchIq4xs { + @workgroup gridw : uint[256] + + def iq3s_signed(g, nib : uint) : uint { + let m1 = (nib * 0x00204081u) & 0x01010101u + let mask = m1 * 255u + return (g ^ mask) + m1 + } + + def override stage_w(wsb0, wcol, nsb, s, word, grp : uint) { + for (ki in range(2)) { + let k = word + uint(ki) * 8u + let bu = k / 2u + let hh = k % 2u + var wv = uint4(0u) + if (wcol < pa.d) { + let wsb = wsb0 + wcol * nsb + s + let qsw = wq[wsb * 24u + bu * 2u + hh] + let aux = wq[wsb * 24u + 16u + bu] + let l0 = hh * 2u + let sg0 = ksign7((aux >> (7u * l0)) & 127u) + let sg1 = ksign7((aux >> (7u * l0 + 7u)) & 127u) + wv = uint4(iq3s_signed(gridw[qsw & 255u], sg0 & 15u), iq3s_signed(gridw[(qsw >> 8u) & 255u], sg0 >> 4u), + iq3s_signed(gridw[(qsw >> 16u) & 255u], sg1 & 15u), iq3s_signed(gridw[qsw >> 24u], sg1 >> 4u)) + } + ww4[grp * 17u + k] = wv + } + } + + [spirv_kernel(local_size_x = 256, name = "kq_batch_iq3xxs_cls_spv")] + def override run { + let tid = gl_LocalInvocationID.x + gridw[tid] = iq3xxs_grid_word(int(tid)) + barrier() // the grid visible before the first stage_w + tile_shell() + } +} + +// IQ2_XXS tile: the iq3xxs shell over the two-word u64 grid - block b's grid word (column 2b) +// carries four BYTE indices, its aux32 (column 2b+1) the parity-derived ksigns; the per-32 +// strip plane and fma inherit from KqBatchIq4xs (the (2ls+1) strips read exactly under the +// signed extract, staying below 128). +[vk_dispatch(name = "kq_batch_iq2xxs_cls", kernel = "run", family = "kq_batch_cls", grid = "wgs", params = "wgs : int64")] +class KqBatchIq2xxs : KqBatchIq4xs { + @workgroup gridw : uint[512] + + def iq3s_signed(g, nib : uint) : uint { + let m1 = (nib * 0x00204081u) & 0x01010101u + let mask = m1 * 255u + return (g ^ mask) + m1 + } + + def override stage_w(wsb0, wcol, nsb, s, word, grp : uint) { + for (ki in range(2)) { + let k = word + uint(ki) * 8u + let bu = k / 2u + let hh = k % 2u + var wv = uint4(0u) + if (wcol < pa.d) { + let wsb = wsb0 + wcol * nsb + s + let gw = wq[wsb * 16u + bu * 2u] + let aux = wq[wsb * 16u + bu * 2u + 1u] + let b0 = (gw >> (16u * hh)) & 255u + let b1 = (gw >> (16u * hh + 8u)) & 255u + let sg0 = ksign7((aux >> (14u * hh)) & 127u) + let sg1 = ksign7((aux >> (14u * hh + 7u)) & 127u) + wv = uint4(iq3s_signed(gridw[b0 * 2u], sg0 & 15u), iq3s_signed(gridw[b0 * 2u + 1u], sg0 >> 4u), + iq3s_signed(gridw[b1 * 2u], sg1 & 15u), iq3s_signed(gridw[b1 * 2u + 1u], sg1 >> 4u)) + } + ww4[grp * 17u + k] = wv + } + } + + [spirv_kernel(local_size_x = 256, name = "kq_batch_iq2xxs_cls_spv")] + def override run { + let tid = gl_LocalInvocationID.x + gridw[tid] = iq2xxs_grid_word(int(tid)) + gridw[tid + 256u] = iq2xxs_grid_word(int(tid + 256u)) + barrier() // the iq2xxs grid visible before the first stage_w + tile_shell() + } +} + +// IQ4_NL tile: iq4xs's LUT staging (stage_w and the no-min blk_fma inherit) with q40's +// per-block f16 d staged as the scale - no sub-scales, no bsum term anywhere. +[vk_dispatch(name = "kq_batch_iq4nl_cls", kernel = "run", family = "kq_batch_cls", grid = "wgs", params = "wgs : int64")] +class KqBatchIq4nl : KqBatchIq4xs { + def override stage_ws(wsb0, wt, nsb, s, tid : uint) { + let slane = tid % 32u + let sblk = tid / 32u + var sa = 0.0 + if (wt * 32u + slane < pa.d) { + let wsb = wsb0 + (wt * 32u + slane) * nsb + s + let dp = unpackHalf2x16(wsu[wsb * 5u + sblk / 2u]) + sa = sblk % 2u == 0u ? dp.x : dp.y + } + wsw[sblk * 32u + slane] = sa + } + + [spirv_kernel(local_size_x = 256, name = "kq_batch_iq4nl_cls_spv")] + def override run { + tile_shell() + } +} + // Q5_K tile: the k4 compose with the 5th bit OR'd in from the 40B block's qh words [vk_dispatch(name = "kq_batch_k5_cls", kernel = "run", family = "kq_batch_cls", grid = "wgs", params = "wgs : int64")] class KqBatchK5 : KqBatchBase { @@ -2814,93 +3631,421 @@ class KqBatchK6 : KqBatchBase { } } -// ===== the prefill batch GEMMs — cooperative-matrix variants (DASLLAMA_COOPMAT=f16|int8|mm) ===== - -let private CML_SST = 20u // mm shared row stride in uints (16 f16-pair data + 4 pad — bank spread) +// Q3_K tile: k6's split-half fold with the offset 4 over the k3 planes (24 words per superblock: +// 16 qs words, 8 hmask words) — block bu takes qs words (bu/4)*8 + hh*4.. at shift 2*(bu%4) +// OR the hmask words 16 + hh*4.. bit bu, shifted up two +[vk_dispatch(name = "kq_batch_k3_cls", kernel = "run", family = "kq_batch_cls", grid = "wgs", params = "wgs : int64")] +class KqBatchK3 : KqBatchK6 { + def override stage_w(wsb0, wcol, nsb, s, word, grp : uint) { + for (ki in range(2)) { + let k = word + uint(ki) * 8u + let bu = k / 2u + let hh = k % 2u + var wv = uint4(0u) + if (wcol < pa.d) { + let wsb = wsb0 + wcol * nsb + s + let qb4 = wsb * 24u + (bu / 4u) * 8u + hh * 4u + let hb4 = wsb * 24u + 16u + hh * 4u + let shift = (bu % 4u) * 2u + wv = uint4(((wq[qb4] >> shift) & 0x03030303) | (((wq[hb4] >> bu) & 0x01010101) << 2u), + ((wq[qb4 + 1u] >> shift) & 0x03030303) | (((wq[hb4 + 1u] >> bu) & 0x01010101) << 2u), + ((wq[qb4 + 2u] >> shift) & 0x03030303) | (((wq[hb4 + 2u] >> bu) & 0x01010101) << 2u), + ((wq[qb4 + 3u] >> shift) & 0x03030303) | (((wq[hb4 + 3u] >> bu) & 0x01010101) << 2u)) + } + ww4[grp * 17u + k] = wv + } + } -// the MoE region rail the coopmat batch tiles share: sched = 4-word region records -// [wblk0, row0, cnt, wg0] at 0, then the per-wg map at pa.map_off -class MoeCmBase { - @ssbo @binding = 2 sched : array // region records + per-wg map (bulk schedule data) - @push_constant pa : BatchArgs + def override blk_fma(bu, grp, word : uint; xscl : float) : float4 { + let kL = bu * 2u + let kH = bu * 2u + 1u + let xwL = xw4[grp * 17u + kL] + let xwH = xw4[grp * 17u + kH] + let blo = sdot4(xwL.x, 0x01010101) + sdot4(xwL.y, 0x01010101) + sdot4(xwL.z, 0x01010101) + sdot4(xwL.w, 0x01010101) + let bhi = sdot4(xwH.x, 0x01010101) + sdot4(xwH.y, 0x01010101) + sdot4(xwH.z, 0x01010101) + sdot4(xwH.w, 0x01010101) + let wL0 = ww4[word * 17u + kL] + let wL1 = ww4[(word + 8u) * 17u + kL] + let wL2 = ww4[(word + 16u) * 17u + kL] + let wL3 = ww4[(word + 24u) * 17u + kL] + let wH0 = ww4[word * 17u + kH] + let wH1 = ww4[(word + 8u) * 17u + kH] + let wH2 = ww4[(word + 16u) * 17u + kH] + let wH3 = ww4[(word + 24u) * 17u + kH] + let l0 = sdot4(wL0.x, xwL.x) + sdot4(wL0.y, xwL.y) + sdot4(wL0.z, xwL.z) + sdot4(wL0.w, xwL.w) + let l1 = sdot4(wL1.x, xwL.x) + sdot4(wL1.y, xwL.y) + sdot4(wL1.z, xwL.z) + sdot4(wL1.w, xwL.w) + let l2 = sdot4(wL2.x, xwL.x) + sdot4(wL2.y, xwL.y) + sdot4(wL2.z, xwL.z) + sdot4(wL2.w, xwL.w) + let l3 = sdot4(wL3.x, xwL.x) + sdot4(wL3.y, xwL.y) + sdot4(wL3.z, xwL.z) + sdot4(wL3.w, xwL.w) + let h0 = sdot4(wH0.x, xwH.x) + sdot4(wH0.y, xwH.y) + sdot4(wH0.z, xwH.z) + sdot4(wH0.w, xwH.w) + let h1 = sdot4(wH1.x, xwH.x) + sdot4(wH1.y, xwH.y) + sdot4(wH1.z, xwH.z) + sdot4(wH1.w, xwH.w) + let h2 = sdot4(wH2.x, xwH.x) + sdot4(wH2.y, xwH.y) + sdot4(wH2.z, xwH.z) + sdot4(wH2.w, xwH.w) + let h3 = sdot4(wH3.x, xwH.x) + sdot4(wH3.y, xwH.y) + sdot4(wH3.z, xwH.z) + sdot4(wH3.w, xwH.w) + return float4(xscl * (wsw[bu * 32u + word] * float(l0 - 4 * blo) + wsw2[bu * 32u + word] * float(h0 - 4 * bhi)), + xscl * (wsw[bu * 32u + word + 8u] * float(l1 - 4 * blo) + wsw2[bu * 32u + word + 8u] * float(h1 - 4 * bhi)), + xscl * (wsw[bu * 32u + word + 16u] * float(l2 - 4 * blo) + wsw2[bu * 32u + word + 16u] * float(h2 - 4 * bhi)), + xscl * (wsw[bu * 32u + word + 24u] * float(l3 - 4 * blo) + wsw2[bu * 32u + word + 24u] * float(h3 - 4 * bhi))) + } - def region_rec : uint4 { - let rid = sched[pa.map_off + gl_WorkGroupID.x] - let rb = rid * 4u - return uint4(sched[rb], sched[rb + 1u], sched[rb + 2u], gl_WorkGroupID.x - sched[rb + 3u]) + [spirv_kernel(local_size_x = 256, name = "kq_batch_k3_cls_spv")] + def override run { + tile_shell() } } -// f16 tensor-core 32x32 tile: same MoE routing + Q8_0 layout as Q8Batch, but the inner GEMM rides -// the tensor cores — 128 threads / 4 subgroups, one 16x16 subtile each; edge tiles zero-fill -[vk_dispatch(name = "q8_batch_cmf16_cls", grid = "wgs", params = "wgs : int64")] -class CmF16Batch : MoeCmBase { - @ssbo @binding = 0 wq : array // weight quant words - @ssbo @binding = 1 wsh : array // per-block weight scales - @ssbo @binding = 3 xqw : array // activation quant words - @ssbo @binding = 4 axs : array // per-block activation scales - @ssbo @binding = 5 y : array - @workgroup cm_as : float16[8192] // 32 local rows x 256 k, row-major stride 256 (f16 dequant A) - @workgroup cm_bs : float16[8192] // 256 k x 32 cols, row-major stride 32 (f16 dequant B) - @workgroup cm_out : float[1024] // 4 subgroups x 16x16 f32 scratch (bounds-checked write-out) +// Q2_K tile: k3's staging minus the hmask (unsigned 0..3 lanes), FOUR scale planes - d*sc and +// dmin*mn per 16-half - and a split-half fma folding the min sides against the half sums. +[vk_dispatch(name = "kq_batch_k2_cls", kernel = "run", family = "kq_batch_cls", grid = "wgs", params = "wgs : int64")] +class KqBatchK2 : KqBatchK6 { + @workgroup wsw3 : float[256] // dmin*mn, lo half + @workgroup wsw4 : float[256] // dmin*mn, hi half - [spirv_kernel(local_size_x = 128, name = "q8_batch_cmf16_cls_spv")] - def run { // nolint:STYLE038 — the verbatim tile pass, parity-locked to the shipped kernel - let reg = region_rec() - let wblk0 = reg.x - let row0 = reg.y - let cnt = reg.z - let tix = reg.w - let wtiles = (pa.d + 31u) / 32u - let xt = tix / wtiles - let wt = tix % wtiles - let nbb = pa.n / 32u - let nsteps = (nbb + 7u) / 8u - let tid = gl_LocalInvocationID.x - let sg = gl_SubgroupID // 0..3 (subgroupSize 32) - let smi = sg / 2u // which 16-row half of the tile - let sni = sg % 2u // which 16-col half of the tile - var acc : coopmatAcc_f32_16x16 - var s = 0u - while (s < nsteps) { - let b0 = s * 8u - for (i in range(16)) { // stage A: 32x64 = 2048 uints, 16/thread; OOR -> 0 - let u = tid * 16u + uint(i) - let row = u / 64u // 0..31 local token row - let chunk = u % 64u - let blk = chunk / 8u - let w = chunk % 8u - let xrow = xt * 32u + row - let gblk = b0 + blk - var packed = 0u - var xs = 0.0 - if (xrow < cnt && gblk < nbb) { - packed = xqw[((row0 + xrow) * nbb + gblk) * 8u + w] - xs = axs[(row0 + xrow) * nbb + gblk] - } - let qv = int4(unpack8(int(packed))) - let o = row * 256u + blk * 32u + w * 4u - cm_as[o] = float16(float(qv.x) * xs) - cm_as[o + 1u] = float16(float(qv.y) * xs) - cm_as[o + 2u] = float16(float(qv.z) * xs) - cm_as[o + 3u] = float16(float(qv.w) * xs) + def override stage_w(wsb0, wcol, nsb, s, word, grp : uint) { + for (ki in range(2)) { + let k = word + uint(ki) * 8u + let bu = k / 2u + let hh = k % 2u + var wv = uint4(0u) + if (wcol < pa.d) { + let wsb = wsb0 + wcol * nsb + s + let qb4 = wsb * 16u + (bu / 4u) * 8u + hh * 4u + let shift = (bu % 4u) * 2u + wv = uint4((wq[qb4] >> shift) & 0x03030303, (wq[qb4 + 1u] >> shift) & 0x03030303, + (wq[qb4 + 2u] >> shift) & 0x03030303, (wq[qb4 + 3u] >> shift) & 0x03030303) } - for (i in range(16)) { // stage B: 32x64 = 2048 uints, 16/thread; OOR -> 0 - let u = tid * 16u + uint(i) - let col = u / 64u // 0..31 output col - let chunk = u % 64u - let blk = chunk / 8u - let w = chunk % 8u - let wcol = wt * 32u + col - let gblk = b0 + blk - var packed = 0u - var ws = 0.0 - if (wcol < pa.d && gblk < nbb) { - packed = wq[(wblk0 + wcol * nbb + gblk) * 8u + w] - ws = float(wsh[wblk0 + wcol * nbb + gblk]) - } - let qv = int4(unpack8(int(packed))) - let k = blk * 32u + w * 4u + ww4[grp * 17u + k] = wv + } + } + + def override stage_ws(wsb0, wt, nsb, s, tid : uint) { + let slane = tid % 32u + let sblk = tid / 32u + var sa = 0.0 + var sbv = 0.0 + var ma = 0.0 + var mb = 0.0 + if (wt * 32u + slane < pa.d) { + let wsb = wsb0 + (wt * 32u + slane) * nsb + s + let w = wsu[wsb * 5u + 1u + sblk / 2u] + let b0 = (w >> ((sblk % 2u) * 16u)) & 255u + let b1 = (w >> ((sblk % 2u) * 16u + 8u)) & 255u + let dm = unpackHalf2x16(wsu[wsb * 5u]) + sa = dm.x * float(b0 & 15u) + sbv = dm.x * float(b1 & 15u) + ma = dm.y * float(b0 >> 4u) + mb = dm.y * float(b1 >> 4u) + } + wsw[sblk * 32u + slane] = sa + wsw2[sblk * 32u + slane] = sbv + wsw3[sblk * 32u + slane] = ma + wsw4[sblk * 32u + slane] = mb + } + + // the split-half fma with the min planes: xscl * (sc_lo*l + sc_hi*h - mn_lo*blo - mn_hi*bhi) + def override blk_fma(bu, grp, word : uint; xscl : float) : float4 { + let kL = bu * 2u + let kH = bu * 2u + 1u + let xwL = xw4[grp * 17u + kL] + let xwH = xw4[grp * 17u + kH] + let blo = sdot4(xwL.x, 0x01010101) + sdot4(xwL.y, 0x01010101) + sdot4(xwL.z, 0x01010101) + sdot4(xwL.w, 0x01010101) + let bhi = sdot4(xwH.x, 0x01010101) + sdot4(xwH.y, 0x01010101) + sdot4(xwH.z, 0x01010101) + sdot4(xwH.w, 0x01010101) + let wL0 = ww4[word * 17u + kL] + let wL1 = ww4[(word + 8u) * 17u + kL] + let wL2 = ww4[(word + 16u) * 17u + kL] + let wL3 = ww4[(word + 24u) * 17u + kL] + let wH0 = ww4[word * 17u + kH] + let wH1 = ww4[(word + 8u) * 17u + kH] + let wH2 = ww4[(word + 16u) * 17u + kH] + let wH3 = ww4[(word + 24u) * 17u + kH] + let l0 = sdot4(wL0.x, xwL.x) + sdot4(wL0.y, xwL.y) + sdot4(wL0.z, xwL.z) + sdot4(wL0.w, xwL.w) + let l1 = sdot4(wL1.x, xwL.x) + sdot4(wL1.y, xwL.y) + sdot4(wL1.z, xwL.z) + sdot4(wL1.w, xwL.w) + let l2 = sdot4(wL2.x, xwL.x) + sdot4(wL2.y, xwL.y) + sdot4(wL2.z, xwL.z) + sdot4(wL2.w, xwL.w) + let l3 = sdot4(wL3.x, xwL.x) + sdot4(wL3.y, xwL.y) + sdot4(wL3.z, xwL.z) + sdot4(wL3.w, xwL.w) + let h0 = sdot4(wH0.x, xwH.x) + sdot4(wH0.y, xwH.y) + sdot4(wH0.z, xwH.z) + sdot4(wH0.w, xwH.w) + let h1 = sdot4(wH1.x, xwH.x) + sdot4(wH1.y, xwH.y) + sdot4(wH1.z, xwH.z) + sdot4(wH1.w, xwH.w) + let h2 = sdot4(wH2.x, xwH.x) + sdot4(wH2.y, xwH.y) + sdot4(wH2.z, xwH.z) + sdot4(wH2.w, xwH.w) + let h3 = sdot4(wH3.x, xwH.x) + sdot4(wH3.y, xwH.y) + sdot4(wH3.z, xwH.z) + sdot4(wH3.w, xwH.w) + return float4(xscl * (wsw[bu * 32u + word] * float(l0) + wsw2[bu * 32u + word] * float(h0) - wsw3[bu * 32u + word] * float(blo) - wsw4[bu * 32u + word] * float(bhi)), + xscl * (wsw[bu * 32u + word + 8u] * float(l1) + wsw2[bu * 32u + word + 8u] * float(h1) - wsw3[bu * 32u + word + 8u] * float(blo) - wsw4[bu * 32u + word + 8u] * float(bhi)), + xscl * (wsw[bu * 32u + word + 16u] * float(l2) + wsw2[bu * 32u + word + 16u] * float(h2) - wsw3[bu * 32u + word + 16u] * float(blo) - wsw4[bu * 32u + word + 16u] * float(bhi)), + xscl * (wsw[bu * 32u + word + 24u] * float(l3) + wsw2[bu * 32u + word + 24u] * float(h3) - wsw3[bu * 32u + word + 24u] * float(blo) - wsw4[bu * 32u + word + 24u] * float(bhi))) + } + + [spirv_kernel(local_size_x = 256, name = "kq_batch_k2_cls_spv")] + def override run { + tile_shell() + } +} + +// IQ2_S tile: the iq3s grid gather widened to the u64 grid (a 10-bit qs|qh index picks TWO +// words - eight magnitudes) with the block's own sign bytes; the per-16 UNSIGNED strips ride +// k6's wsw/wsw2 planes with a plain split fma (no offset, no min term). The 8 KB grid stages +// once, 8 words per thread. +[vk_dispatch(name = "kq_batch_iq2s_cls", kernel = "run", family = "kq_batch_cls", grid = "wgs", params = "wgs : int64")] +class KqBatchIq2s : KqBatchK6 { + @workgroup gridw : uint[2048] + + def iq3s_signed(g, nib : uint) : uint { + let m1 = (nib * 0x00204081u) & 0x01010101u + let mask = m1 * 255u + return (g ^ mask) + m1 + } + + def override stage_w(wsb0, wcol, nsb, s, word, grp : uint) { + for (ki in range(2)) { + let k = word + uint(ki) * 8u + let bu = k / 2u + let hh = k % 2u + var wv = uint4(0u) + if (wcol < pa.d) { + let wsb = wsb0 + wcol * nsb + s + let qsw = wq[wsb * 18u + bu] + let sgw = wq[wsb * 18u + 8u + bu] + let qh = (wq[wsb * 18u + 16u + bu / 4u] >> ((bu % 4u) * 8u)) & 255u + let l0 = hh * 2u + let sg0 = (sgw >> (l0 * 8u)) & 255u + let sg1 = (sgw >> (l0 * 8u + 8u)) & 255u + let i0 = (((qsw >> (l0 * 8u)) & 255u) | ((qh << (8u - 2u * l0)) & 0x300u)) * 2u + let i1 = (((qsw >> (l0 * 8u + 8u)) & 255u) | ((qh << (8u - 2u * (l0 + 1u))) & 0x300u)) * 2u + wv = uint4(iq3s_signed(gridw[i0], sg0 & 15u), iq3s_signed(gridw[i0 + 1u], sg0 >> 4u), + iq3s_signed(gridw[i1], sg1 & 15u), iq3s_signed(gridw[i1 + 1u], sg1 >> 4u)) + } + ww4[grp * 17u + k] = wv + } + } + + def override stage_ws(wsb0, wt, nsb, s, tid : uint) { + let slane = tid % 32u + let sblk = tid / 32u + var sa = 0.0 + var sbv = 0.0 + if (wt * 32u + slane < pa.d) { + let wsb = wsb0 + (wt * 32u + slane) * nsb + s + let w = wsu[wsb * 5u + 1u + sblk / 2u] + let s0 = (w >> ((sblk % 2u) * 16u)) & 255u + let s1 = (w >> ((sblk % 2u) * 16u + 8u)) & 255u + let dd = unpackHalf2x16(wsu[wsb * 5u]).x + sa = dd * float(s0) + sbv = dd * float(s1) + } + wsw[sblk * 32u + slane] = sa + wsw2[sblk * 32u + slane] = sbv + } + + // the split-half fma with UNSIGNED strips: xscl * (d*s0*l + d*s1*h) - no offset, no min + def override blk_fma(bu, grp, word : uint; xscl : float) : float4 { + let kL = bu * 2u + let kH = bu * 2u + 1u + let xwL = xw4[grp * 17u + kL] + let xwH = xw4[grp * 17u + kH] + let wL0 = ww4[word * 17u + kL] + let wL1 = ww4[(word + 8u) * 17u + kL] + let wL2 = ww4[(word + 16u) * 17u + kL] + let wL3 = ww4[(word + 24u) * 17u + kL] + let wH0 = ww4[word * 17u + kH] + let wH1 = ww4[(word + 8u) * 17u + kH] + let wH2 = ww4[(word + 16u) * 17u + kH] + let wH3 = ww4[(word + 24u) * 17u + kH] + let l0 = sdot4(wL0.x, xwL.x) + sdot4(wL0.y, xwL.y) + sdot4(wL0.z, xwL.z) + sdot4(wL0.w, xwL.w) + let l1 = sdot4(wL1.x, xwL.x) + sdot4(wL1.y, xwL.y) + sdot4(wL1.z, xwL.z) + sdot4(wL1.w, xwL.w) + let l2 = sdot4(wL2.x, xwL.x) + sdot4(wL2.y, xwL.y) + sdot4(wL2.z, xwL.z) + sdot4(wL2.w, xwL.w) + let l3 = sdot4(wL3.x, xwL.x) + sdot4(wL3.y, xwL.y) + sdot4(wL3.z, xwL.z) + sdot4(wL3.w, xwL.w) + let h0 = sdot4(wH0.x, xwH.x) + sdot4(wH0.y, xwH.y) + sdot4(wH0.z, xwH.z) + sdot4(wH0.w, xwH.w) + let h1 = sdot4(wH1.x, xwH.x) + sdot4(wH1.y, xwH.y) + sdot4(wH1.z, xwH.z) + sdot4(wH1.w, xwH.w) + let h2 = sdot4(wH2.x, xwH.x) + sdot4(wH2.y, xwH.y) + sdot4(wH2.z, xwH.z) + sdot4(wH2.w, xwH.w) + let h3 = sdot4(wH3.x, xwH.x) + sdot4(wH3.y, xwH.y) + sdot4(wH3.z, xwH.z) + sdot4(wH3.w, xwH.w) + return float4(xscl * (wsw[bu * 32u + word] * float(l0) + wsw2[bu * 32u + word] * float(h0)), + xscl * (wsw[bu * 32u + word + 8u] * float(l1) + wsw2[bu * 32u + word + 8u] * float(h1)), + xscl * (wsw[bu * 32u + word + 16u] * float(l2) + wsw2[bu * 32u + word + 16u] * float(h2)), + xscl * (wsw[bu * 32u + word + 24u] * float(l3) + wsw2[bu * 32u + word + 24u] * float(h3))) + } + + [spirv_kernel(local_size_x = 256, name = "kq_batch_iq2s_cls_spv")] + def override run { + let tid = gl_LocalInvocationID.x + for (k in range(8)) { + gridw[tid + uint(k) * 256u] = iq2s_grid_word(int(tid + uint(k) * 256u)) + } + barrier() // the grid visible before the first stage_w + tile_shell() + } +} + +// IQ2_XS tile: the iq2s gather with ksigns-by-parity - each u16 qs word carries a 9-bit grid +// index (two staged words) and a 7-bit KSIGNS_IQ2XS index whose sign byte ksign7 recomputes +// (no sign plane, no qh); strips and fma ride the iq2s spellings. The 4 KB grid stages once. +[vk_dispatch(name = "kq_batch_iq2xs_cls", kernel = "run", family = "kq_batch_cls", grid = "wgs", params = "wgs : int64")] +class KqBatchIq2xs : KqBatchK6 { + @workgroup gridw : uint[1024] + + def iq3s_signed(g, nib : uint) : uint { + let m1 = (nib * 0x00204081u) & 0x01010101u + let mask = m1 * 255u + return (g ^ mask) + m1 + } + + def override stage_w(wsb0, wcol, nsb, s, word, grp : uint) { + for (ki in range(2)) { + let k = word + uint(ki) * 8u + var wv = uint4(0u) + if (wcol < pa.d) { + let wsb = wsb0 + wcol * nsb + s + let qw = wq[wsb * 16u + k] + let wa = qw & 0xFFFFu + let wb = qw >> 16u + let sg0 = ksign7(wa >> 9u) + let sg1 = ksign7(wb >> 9u) + let i0 = (wa & 511u) * 2u + let i1 = (wb & 511u) * 2u + wv = uint4(iq3s_signed(gridw[i0], sg0 & 15u), iq3s_signed(gridw[i0 + 1u], sg0 >> 4u), + iq3s_signed(gridw[i1], sg1 & 15u), iq3s_signed(gridw[i1 + 1u], sg1 >> 4u)) + } + ww4[grp * 17u + k] = wv + } + } + + def override stage_ws(wsb0, wt, nsb, s, tid : uint) { + let slane = tid % 32u + let sblk = tid / 32u + var sa = 0.0 + var sbv = 0.0 + if (wt * 32u + slane < pa.d) { + let wsb = wsb0 + (wt * 32u + slane) * nsb + s + let w = wsu[wsb * 5u + 1u + sblk / 2u] + let s0 = (w >> ((sblk % 2u) * 16u)) & 255u + let s1 = (w >> ((sblk % 2u) * 16u + 8u)) & 255u + let dd = unpackHalf2x16(wsu[wsb * 5u]).x + sa = dd * float(s0) + sbv = dd * float(s1) + } + wsw[sblk * 32u + slane] = sa + wsw2[sblk * 32u + slane] = sbv + } + + // the split-half fma with UNSIGNED strips: xscl * (d*s0*l + d*s1*h) - no offset, no min + def override blk_fma(bu, grp, word : uint; xscl : float) : float4 { + let kL = bu * 2u + let kH = bu * 2u + 1u + let xwL = xw4[grp * 17u + kL] + let xwH = xw4[grp * 17u + kH] + let wL0 = ww4[word * 17u + kL] + let wL1 = ww4[(word + 8u) * 17u + kL] + let wL2 = ww4[(word + 16u) * 17u + kL] + let wL3 = ww4[(word + 24u) * 17u + kL] + let wH0 = ww4[word * 17u + kH] + let wH1 = ww4[(word + 8u) * 17u + kH] + let wH2 = ww4[(word + 16u) * 17u + kH] + let wH3 = ww4[(word + 24u) * 17u + kH] + let l0 = sdot4(wL0.x, xwL.x) + sdot4(wL0.y, xwL.y) + sdot4(wL0.z, xwL.z) + sdot4(wL0.w, xwL.w) + let l1 = sdot4(wL1.x, xwL.x) + sdot4(wL1.y, xwL.y) + sdot4(wL1.z, xwL.z) + sdot4(wL1.w, xwL.w) + let l2 = sdot4(wL2.x, xwL.x) + sdot4(wL2.y, xwL.y) + sdot4(wL2.z, xwL.z) + sdot4(wL2.w, xwL.w) + let l3 = sdot4(wL3.x, xwL.x) + sdot4(wL3.y, xwL.y) + sdot4(wL3.z, xwL.z) + sdot4(wL3.w, xwL.w) + let h0 = sdot4(wH0.x, xwH.x) + sdot4(wH0.y, xwH.y) + sdot4(wH0.z, xwH.z) + sdot4(wH0.w, xwH.w) + let h1 = sdot4(wH1.x, xwH.x) + sdot4(wH1.y, xwH.y) + sdot4(wH1.z, xwH.z) + sdot4(wH1.w, xwH.w) + let h2 = sdot4(wH2.x, xwH.x) + sdot4(wH2.y, xwH.y) + sdot4(wH2.z, xwH.z) + sdot4(wH2.w, xwH.w) + let h3 = sdot4(wH3.x, xwH.x) + sdot4(wH3.y, xwH.y) + sdot4(wH3.z, xwH.z) + sdot4(wH3.w, xwH.w) + return float4(xscl * (wsw[bu * 32u + word] * float(l0) + wsw2[bu * 32u + word] * float(h0)), + xscl * (wsw[bu * 32u + word + 8u] * float(l1) + wsw2[bu * 32u + word + 8u] * float(h1)), + xscl * (wsw[bu * 32u + word + 16u] * float(l2) + wsw2[bu * 32u + word + 16u] * float(h2)), + xscl * (wsw[bu * 32u + word + 24u] * float(l3) + wsw2[bu * 32u + word + 24u] * float(h3))) + } + + [spirv_kernel(local_size_x = 256, name = "kq_batch_iq2xs_cls_spv")] + def override run { + let tid = gl_LocalInvocationID.x + for (k in range(4)) { + gridw[tid + uint(k) * 256u] = iq2xs_grid_word(int(tid + uint(k) * 256u)) + } + barrier() // the iq2xs grid visible before the first stage_w + tile_shell() + } +} + +// ===== the prefill batch GEMMs — cooperative-matrix variants (DASLLAMA_COOPMAT=f16|int8|mm) ===== + +let private CML_SST = 20u // mm shared row stride in uints (16 f16-pair data + 4 pad — bank spread) + +// the MoE region rail the coopmat batch tiles share: sched = 4-word region records +// [wblk0, row0, cnt, wg0] at 0, then the per-wg map at pa.map_off +class MoeCmBase { + @ssbo @binding = 2 sched : array // region records + per-wg map (bulk schedule data) + @push_constant pa : BatchArgs + + def region_rec : uint4 { + let rid = sched[pa.map_off + gl_WorkGroupID.x] + let rb = rid * 4u + return uint4(sched[rb], sched[rb + 1u], sched[rb + 2u], gl_WorkGroupID.x - sched[rb + 3u]) + } +} + +// f16 tensor-core 32x32 tile: same MoE routing + Q8_0 layout as Q8Batch, but the inner GEMM rides +// the tensor cores — 128 threads / 4 subgroups, one 16x16 subtile each; edge tiles zero-fill +[vk_dispatch(name = "q8_batch_cmf16_cls", grid = "wgs", params = "wgs : int64")] +class CmF16Batch : MoeCmBase { + @ssbo @binding = 0 wq : array // weight quant words + @ssbo @binding = 1 wsh : array // per-block weight scales + @ssbo @binding = 3 xqw : array // activation quant words + @ssbo @binding = 4 axs : array // per-block activation scales + @ssbo @binding = 5 y : array + @workgroup cm_as : float16[8192] // 32 local rows x 256 k, row-major stride 256 (f16 dequant A) + @workgroup cm_bs : float16[8192] // 256 k x 32 cols, row-major stride 32 (f16 dequant B) + @workgroup cm_out : float[1024] // 4 subgroups x 16x16 f32 scratch (bounds-checked write-out) + + [spirv_kernel(local_size_x = 128, name = "q8_batch_cmf16_cls_spv")] + def run { // nolint:STYLE038 — the verbatim tile pass, parity-locked to the shipped kernel + let reg = region_rec() + let wblk0 = reg.x + let row0 = reg.y + let cnt = reg.z + let tix = reg.w + let wtiles = (pa.d + 31u) / 32u + let xt = tix / wtiles + let wt = tix % wtiles + let nbb = pa.n / 32u + let nsteps = (nbb + 7u) / 8u + let tid = gl_LocalInvocationID.x + let sg = gl_SubgroupID // 0..3 (subgroupSize 32) + let row_half = sg / 2u // which 16-row half of the tile + let col_half = sg % 2u // which 16-col half of the tile + var acc : coopmatAcc_f32_16x16 + var s = 0u + while (s < nsteps) { + let b0 = s * 8u + for (i in range(16)) { // stage A: 32x64 = 2048 uints, 16/thread; OOR -> 0 + let u = tid * 16u + uint(i) + let row = u / 64u // 0..31 local token row + let chunk = u % 64u + let blk = chunk / 8u + let w = chunk % 8u + let xrow = xt * 32u + row + let gblk = b0 + blk + var packed = 0u + var xs = 0.0 + if (xrow < cnt && gblk < nbb) { + packed = xqw[((row0 + xrow) * nbb + gblk) * 8u + w] + xs = axs[(row0 + xrow) * nbb + gblk] + } + let qv = int4(unpack8(int(packed))) + let o = row * 256u + blk * 32u + w * 4u + cm_as[o] = float16(float(qv.x) * xs) + cm_as[o + 1u] = float16(float(qv.y) * xs) + cm_as[o + 2u] = float16(float(qv.z) * xs) + cm_as[o + 3u] = float16(float(qv.w) * xs) + } + for (i in range(16)) { // stage B: 32x64 = 2048 uints, 16/thread; OOR -> 0 + let u = tid * 16u + uint(i) + let col = u / 64u // 0..31 output col + let chunk = u % 64u + let blk = chunk / 8u + let w = chunk % 8u + let wcol = wt * 32u + col + let gblk = b0 + blk + var packed = 0u + var ws = 0.0 + if (wcol < pa.d && gblk < nbb) { + packed = wq[(wblk0 + wcol * nbb + gblk) * 8u + w] + ws = float(wsh[wblk0 + wcol * nbb + gblk]) + } + let qv = int4(unpack8(int(packed))) + let k = blk * 32u + w * 4u cm_bs[k * 32u + col] = float16(float(qv.x) * ws) cm_bs[(k + 1u) * 32u + col] = float16(float(qv.y) * ws) cm_bs[(k + 2u) * 32u + col] = float16(float(qv.z) * ws) @@ -2911,8 +4056,8 @@ class CmF16Batch : MoeCmBase { for (ks in range(2)) { var a : coopmatA_f16_16x16 var b : coopmatB_f16_16x16 - coopmatLoad(a, cm_as, int(smi * 16u * 256u + uint(bb) * 32u + uint(ks) * 16u), 256, 0) - coopmatLoad(b, cm_bs, int((uint(bb) * 32u + uint(ks) * 16u) * 32u + sni * 16u), 32, 0) + coopmatLoad(a, cm_as, int(row_half * 16u * 256u + uint(bb) * 32u + uint(ks) * 16u), 256, 0) + coopmatLoad(b, cm_bs, int((uint(bb) * 32u + uint(ks) * 16u) * 32u + col_half * 16u), 32, 0) acc = coopmatMulAdd(a, b, acc) } } @@ -2926,8 +4071,8 @@ class CmF16Batch : MoeCmBase { let idx = lane * 8u + uint(e) let m = idx / 16u let nn = idx % 16u - let orow = xt * 32u + smi * 16u + m - let ocol = wt * 32u + sni * 16u + nn + let orow = xt * 32u + row_half * 16u + m + let ocol = wt * 32u + col_half * 16u + nn if (orow < cnt && ocol < pa.d) { y[(row0 + orow) * pa.d + ocol] = cm_out[sg * 256u + idx] } @@ -2960,10 +4105,10 @@ class CmI8Batch : MoeCmBase { let nbb = pa.n / 32u let sg = gl_SubgroupID let lane = gl_SubgroupInvocationID - let smi = sg / 2u - let sni = sg % 2u - let arow = xt * 32u + smi * 16u // this subgroup's local token-row base - let bcol = wt * 32u + sni * 16u // this subgroup's output-col base + let row_half = sg / 2u + let col_half = sg % 2u + let arow = xt * 32u + row_half * 16u // this subgroup's local token-row base + let bcol = wt * 32u + col_half * 16u // this subgroup's output-col base for (e in range(8)) { cmi_acc[sg * 256u + lane * 8u + uint(e)] = 0.0 } @@ -3511,8 +4656,8 @@ class KqQ40CmF16 : MoeCmBase { let nsb = pa.n / 256u let tid = gl_LocalInvocationID.x let sg = gl_SubgroupID - let smi = sg / 2u - let sni = sg % 2u + let row_half = sg / 2u + let col_half = sg % 2u var acc : coopmatAcc_f32_16x16 var s = 0u while (s < nsb) { @@ -3570,8 +4715,8 @@ class KqQ40CmF16 : MoeCmBase { for (ks in range(2)) { var a : coopmatA_f16_16x16 var b : coopmatB_f16_16x16 - coopmatLoad(a, cm_as, int(smi * 16u * 256u + uint(bb) * 32u + uint(ks) * 16u), 256, 0) - coopmatLoad(b, cm_bs, int((uint(bb) * 32u + uint(ks) * 16u) * 32u + sni * 16u), 32, 0) + coopmatLoad(a, cm_as, int(row_half * 16u * 256u + uint(bb) * 32u + uint(ks) * 16u), 256, 0) + coopmatLoad(b, cm_bs, int((uint(bb) * 32u + uint(ks) * 16u) * 32u + col_half * 16u), 32, 0) acc = coopmatMulAdd(a, b, acc) } } @@ -3585,8 +4730,8 @@ class KqQ40CmF16 : MoeCmBase { let idx = lane * 8u + uint(e) let m = idx / 16u let nn = idx % 16u - let orow = xt * 32u + smi * 16u + m - let ocol = wt * 32u + sni * 16u + nn + let orow = xt * 32u + row_half * 16u + m + let ocol = wt * 32u + col_half * 16u + nn if (orow < cnt && ocol < pa.d) { y[(row0 + orow) * pa.d + ocol] = cm_out[sg * 256u + idx] } @@ -3594,218 +4739,96 @@ class KqQ40CmF16 : MoeCmBase { } } -// ===== the cm2 (NV_cooperative_matrix2) prefill tiles: native fmt-0 planes, decode-in-load ===== +// ===== the cm2 (NV_cooperative_matrix2) prefill tiles: native planes, decode-in-load ===== struct VkQ8Blk { - qs : int16[16] // one fmt-0 q8 block: 32 packed int8 quants as 16-bit lanes; the scale lives in wsh -} - -// the cm2 quant "l" geometry on the NATIVE fmt-0 two-plane layout: 256 threads, one -// 128-weight x 256-token tile per wg, BK=64 (2 blocks) per step, 8x-unrolled k loop. The decode -// METHOD reads the separate scale plane by block index — the reason the class-method decode form -// exists. Binding 4 unused (sparse) -[vk_dispatch(name = "q8_batch_cm2l_cls", grid = "wgs", params = "wgs : int64")] -class Q8Cm2LBatch : MoeCmBase { - @ssbo @binding = 0 wq : array // fmt-0 weight quant blocks (32 bytes each, read as 16-bit lanes) - @ssbo @binding = 1 wsh : array // per-block weight scales (the second plane) - @ssbo @binding = 3 xf16 : array // f16 activation plane - @ssbo @binding = 5 y : array - @workgroup wg_blk0 : uint // the region's block base, staged for the decode method + qs : int16[16] // one fmt-0 q8 block: 32 packed int8 quants as 16-bit lanes; the scale lives in ws +} - [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] - def decode_q8(blk : VkQ8Blk; bc, cib : uint2) : float16 { - let q = unpack8(blk.qs[int((cib.y & 30u) >> 1u)])[int(cib.y & 1u)] - return wsh[wg_blk0 + bc.x * (pa.n >> 5u) + bc.y] * float16(float(int(q))) - } +struct VkK4Blk { + qs : int16[64] // one Q4_K superblock's 128 nibble bytes as 16-bit lanes; its scale row (d|dmin + 8 sc + 8 mn bytes) lives in ws +} - [spirv_kernel(local_size_x = 256, name = "q8_batch_cm2l_cls_spv"), arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-tile-pick-and-default")] - def run { // nolint:STYLE038 — the fast/edge path pair, barrier- and register-coupled - let reg = region_rec() - let wblk0 = reg.x - let row0 = reg.y - let cnt = reg.z - let ttiles = (cnt + 255u) / 256u - // split-k: wgs stack nsplit planes of (wtiles x ttiles); plane ks covers k [k0, k1) and - // stores f32 partials at ybase (the reduce sums the planes into y). SINGLE-REGION (dense) - // dispatches only — the plane size rides row0 + cnt, so multi-region planes would collide - var tix = reg.w - var ks = 0u - var k1 = pa.n - var ybase = 0u - if (pa.ksplit != 0u) { - let ptiles = ((pa.d + 127u) / 128u) * ttiles - ks = tix / ptiles - tix -= ks * ptiles - k1 = min(pa.n, (ks + 1u) * pa.ksplit) - ybase = ks * (row0 + cnt) * pa.d - } - let k0 = ks * pa.ksplit - let xt = tix % ttiles // token 256-tile (fastest) - let wt = tix / ttiles // weight 128-tile - if (gl_LocalInvocationID.x == 0u) { - wg_blk0 = wblk0 - } - var tv : tensorView2Dt - tensorViewCreate(tv) - barrier() // wg_blk0 visible before the first decode load - var a : coopmatWgA_f16_128x64 - var b : coopmatWgB_f16_64x256 - var acc : coopmatWgAcc_f16_128x256 - let t0 = row0 + xt * 256u - let m0 = wt * 128u - // FAST PATH — every access of this tile provably in bounds and K a whole number of - // BK steps: unclamped (clamp-Undefined) layouts, and the B/out strides masked to a - // multiple of 8 f16 (identity here — pa.n and pa.d are 32-multiples — but the mask makes - // the alignment PROVABLE to the driver's address analysis via stride &= ~7) - if (m0 + 128u <= pa.d && xt * 256u + 256u <= cnt && (pa.n & 63u) == 0u) { - var fla : tensorLayout2D - tensorLayoutCreate(fla) - tensorLayoutSetBlockSize(fla, 1u, 32u) - tensorLayoutSetDimension(fla, pa.d, pa.n) - tensorLayoutSetStride(fla, pa.n >> 5u, 1u) - var flb : tensorLayout2D - tensorLayoutCreate(flb) - tensorLayoutSetDimension(flb, row0 + cnt, pa.n) - tensorLayoutSetStride(flb, pa.n & ~7u, 1u) - var flo : tensorLayout2D - tensorLayoutCreate(flo) - tensorLayoutSetDimension(flo, row0 + cnt, pa.d) - tensorLayoutSetStride(flo, pa.d & ~7u, 1u) - // the no-split arm keeps LITERAL loop bounds and store base: the k0/k1/ybase form - // in this hot loop measured -27% pp on the 5060 Ti even when the values were 0/pa.n/0 - if (pa.ksplit == 0u) { - var k = 0u - for (_i in range(int(pa.n / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_q8) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 256u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < pa.n) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_q8) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 256u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x256 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, 0u, flo, t0, 256u, m0, 128u, tv) - return - } - var k = k0 - for (_i in range(int((k1 - k0) / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_q8) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 256u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < k1) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_q8) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 256u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x256 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, ybase, flo, t0, 256u, m0, 128u, tv) - return - } - // EDGE PATH — clamp-Constant layouts everywhere. A: the region's [d x n] q8 plane, block - // addressing; layout coords are region-local, so the decode method's scale index is - // wg_blk0 + bc.x * nbb + bc.y off the staged base - var tla : tensorLayout2DPad - tensorLayoutCreate(tla) - tensorLayoutSetBlockSize(tla, 1u, 32u) - tensorLayoutSetDimension(tla, pa.d, pa.n) - tensorLayoutSetStride(tla, pa.n / 32u, 1u) - // B: the f16 activation rows, transpose-viewed; the dimension cuts at the region's end - var tlb : tensorLayout2DPad - tensorLayoutCreate(tlb) - tensorLayoutSetDimension(tlb, row0 + cnt, pa.n) - tensorLayoutSetStride(tlb, pa.n, 1u) - // out: token-major y through the transpose view. MUST be the clamp-Constant layout type: - // only clamped layouts DISCARD out-of-bounds stores from a partial tile - var tlo : tensorLayout2DPad - tensorLayoutCreate(tlo) - tensorLayoutSetDimension(tlo, row0 + cnt, pa.d) - tensorLayoutSetStride(tlo, pa.d, 1u) - // the 8x-unrolled main loop (their UNROLL_K = BK*8 = 512), then the sub-512 tail. - // The hint is honored: a manual 4x expansion A/Bed dead even (2719 +- 20 vs 2749 +- 12 - // on the 3B row), so the loop form stays. Same literal-bound specialization as the - // fast path — the no-split arm never reads k0/k1/ybase - if (pa.ksplit == 0u) { - var k = 0u - for (_i in range(int(pa.n / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_q8) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 256u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < pa.n) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_q8) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 256u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x256 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, 0u, tlo, t0, 256u, m0, 128u, tv) - return - } - var k = k0 - for (_i in range(int((k1 - k0) / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_q8) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 256u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < k1) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_q8) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 256u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x256 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, ybase, tlo, t0, 256u, m0, 128u, tv) - } +struct VkK6Blk { + ql : int16[64] // one Q6_K superblock's 128 low-nibble bytes as 16-bit lanes + qh : int16[32] // ... and its 64 high-2-bit bytes; scales (16 int8 + f16 d) live in ws } -// the m-tile sibling (cm2 quant "m": BN=128, same A / BK / thread count); the body mirrors -// Q8Cm2LBatch at 128-token width -[vk_dispatch(name = "q8_batch_cm2m_cls", grid = "wgs", params = "wgs : int64")] -class Q8Cm2MBatch : MoeCmBase { - @ssbo @binding = 0 wq : array // fmt-0 weight quant blocks (32 bytes each, read as 16-bit lanes) - @ssbo @binding = 1 wsh : array // per-block weight scales (the second plane) - @ssbo @binding = 3 xf16 : array // f16 activation plane - @ssbo @binding = 5 y : array - @workgroup wg_blk0 : uint // the region's block base, staged for the decode method +struct VkK5Blk { + qs : int16[64] // one Q5_K superblock's 128 nibble bytes (the k/k+16 device pairing) + qh : int16[16] // ... and its 32 packed 5th-bit bytes: byte g*4+q = bits of elems g*32+4q..+3 (lo) | +16.. (hi) +} - [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] - def decode_q8(blk : VkQ8Blk; bc, cib : uint2) : float16 { - let q = unpack8(blk.qs[int((cib.y & 30u) >> 1u)])[int(cib.y & 1u)] - return wsh[wg_blk0 + bc.x * (pa.n >> 5u) + bc.y] * float16(float(int(q))) - } +struct VkIq3sBlk { + qs : int16[32] // one IQ3_S superblock's 64 grid-index bytes as 16-bit lanes + qh : int16[4] // ... its 8 ninth-bit bytes + sg : int16[16] // ... and its 32 sign bytes; scales (f16 d + 8 int8 (1 + 2s)) live in ws +} + +struct VkIq3xxsBlk { + qs : int16[32] // one IQ3_XXS superblock's 64 grid-index bytes as 16-bit lanes + ax : int16[16] // ... and its 32 aux bytes (per block: 4x7-bit ksigns indices + 4-bit ls); scales (halved f16 d + 8 x (2ls+1)) live in ws +} + +struct VkK3Blk { + qs : int16[32] // one Q3_K superblock's 64 qs bytes, four 2-bit lanes each, verbatim disk order + hm : int16[16] // ... and its 32 hmask bytes; scales (16 int8 (sc-32) + f16 d) live in ws +} + +struct VkK2Blk { + qs : int16[32] // one Q2_K superblock's 64 qs bytes, four 2-bit lanes each; scales ([f16 d][f16 dmin][16 sc/min pair bytes]) live in ws +} + +struct VkIq2sBlk { + qs : int16[16] // one IQ2_S superblock's 32 grid-index bytes as 16-bit lanes + sg : int16[16] // ... its 32 sign bytes; scales (f16 d8 + 16 unsigned (1 + 2s) strips) live in ws + qh : int16[4] // ... and its 8 qh bytes (two index high bits per (block, l) window) +} + +struct VkIq2xsBlk { + qs : int16[32] // one IQ2_XS superblock's 32 u16 qs words as 16-bit lanes (9-bit grid index + 7-bit ksigns index each); scales (f16 d8 + 16 unsigned (1 + 2s) strips) live in ws +} + +struct VkIq2xxsBlk { + qs : int16[32] // one IQ2_XXS superblock's 64B qs as 16-bit lanes: per block 4 grid byte indices + its aux32 (4 x 7-bit ksigns + per-32 ls nibble); scales (f16 d8 + 8 unsigned (1 + 2s) strips) live in ws +} - [spirv_kernel(local_size_x = 256, name = "q8_batch_cm2m_cls_spv"), arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-tile-pick-and-default")] - def run { // nolint:STYLE038 — the fast/edge path pair, barrier- and register-coupled +// The cm2 tile: one class template over the two stamp axes. The token column BN - l 256, m 128, +// s 32 (the expert-bucket geometry) - and the weight format: BLK +// the 16-bit-lane block struct, ST the scale plane's element, BLKW the block's element width, and +// the format template's decode. 256 threads, one 128-weight x BN-token tile per wg, BK=64 per +// step, an 8x-unrolled k loop; the decode METHOD reads the separate scale plane by block index - +// the reason the class-method decode form exists. Binding 4 unused (sparse) +[ |> template_struct_instance] +class template KqCm2BatchT : MoeCmBase { + @ssbo @binding = 0 @role = "weight" wq : array // the quant plane, read as 16-bit lanes + @ssbo @binding = 1 @role = "weight" ws : array // the scale plane (q8: f16 per block; kq: 5 words per superblock) + @ssbo @binding = 3 xf16 : array // f16 activation plane + @ssbo @binding = 5 y : array + @workgroup wg_blk0 : uint // the region's block base, staged for the decode + @template_constant BN : uint = 256u // the token column + @template_constant STILE : bool = false // the 32-row column's partial-column fast path + @template_constant BLKW : uint = 256u // elements per weight block + @template_constant IQLUT : bool = false // stage the iq4nl codebook into workgroup memory + @workgroup @template_gate = IQLUT iq4lut : float16[16] // kvalues_iq4nl as f16 + @template_constant IQ3GRID : bool = false // stage the 2 KB iq3s grid into workgroup memory + @workgroup @template_gate = IQ3GRID iq3s_gridc : uint[512] + @template_constant IQ3XGRID : bool = false // stage the 1 KB halved iq3xxs grid into workgroup memory + @workgroup @template_gate = IQ3XGRID iq3x_gridc : uint[256] + @template_constant IQ2SGRID : bool = false // stage the 8 KB iq2s u64 grid into workgroup memory + @workgroup @template_gate = IQ2SGRID iq2s_gridc : uint[2048] + @template_constant IQ2XSGRID : bool = false // stage the 4 KB iq2xs u64 grid into workgroup memory + @workgroup @template_gate = IQ2XSGRID iq2xs_gridc : uint[1024] + @template_constant IQ2XXSGRID : bool = false // stage the 2 KB iq2xxs u64 grid into workgroup memory + @workgroup @template_gate = IQ2XXSGRID iq2xxs_gridc : uint[512] + + [spirv_kernel(local_size_x = 256), arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-tile-pick-and-default")] + def run { // nolint:STYLE037,STYLE038 — the fast/edge path pair, barrier- and register-coupled let reg = region_rec() let wblk0 = reg.x let row0 = reg.y let cnt = reg.z - let ttiles = (cnt + 127u) / 128u + let ttiles = (cnt + BN - 1u) / BN // split-k: wgs stack nsplit planes of (wtiles x ttiles); plane ks covers k [k0, k1) and // stores f32 partials at ybase (the reduce sums the planes into y). SINGLE-REGION (dense) // dispatches only — the plane size rides row0 + cnt, so multi-region planes would collide @@ -3821,201 +4844,63 @@ class Q8Cm2MBatch : MoeCmBase { ybase = ks * (row0 + cnt) * pa.d } let k0 = ks * pa.ksplit - let xt = tix % ttiles // token 128-tile (fastest) + let xt = tix % ttiles // token BN-tile (fastest) let wt = tix / ttiles // weight 128-tile if (gl_LocalInvocationID.x == 0u) { wg_blk0 = wblk0 } var tv : tensorView2Dt tensorViewCreate(tv) - barrier() // wg_blk0 visible before the first decode load - var a : coopmatWgA_f16_128x64 - var b : coopmatWgB_f16_64x128 - var acc : coopmatWgAcc_f16_128x128 - let t0 = row0 + xt * 128u - let m0 = wt * 128u - // FAST PATH — see Q8Cm2LBatch; identical rule at the m width, same literal-bound - // no-split specialization (the k0/k1/ybase form in the hot loop measured -27% pp) - if (m0 + 128u <= pa.d && xt * 128u + 128u <= cnt && (pa.n & 63u) == 0u) { - var fla : tensorLayout2D - tensorLayoutCreate(fla) - tensorLayoutSetBlockSize(fla, 1u, 32u) - tensorLayoutSetDimension(fla, pa.d, pa.n) - tensorLayoutSetStride(fla, pa.n >> 5u, 1u) - var flb : tensorLayout2D - tensorLayoutCreate(flb) - tensorLayoutSetDimension(flb, row0 + cnt, pa.n) - tensorLayoutSetStride(flb, pa.n & ~7u, 1u) - var flo : tensorLayout2D - tensorLayoutCreate(flo) - tensorLayoutSetDimension(flo, row0 + cnt, pa.d) - tensorLayoutSetStride(flo, pa.d & ~7u, 1u) - if (pa.ksplit == 0u) { - var k = 0u - for (_i in range(int(pa.n / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_q8) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 128u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < pa.n) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_q8) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 128u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x128 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, 0u, flo, t0, 128u, m0, 128u, tv) - return - } - var k = k0 - for (_i in range(int((k1 - k0) / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_q8) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 128u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < k1) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_q8) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 128u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u + static_if (IQLUT) { + if (gl_LocalInvocationID.x < 16u) { + let li = gl_LocalInvocationID.x + let tbl = uint4(3215825025u, 4142587343u, 639175937u, 1901675829u) + iq4lut[li] = float16(float((int((tbl[int(li >> 2u)] >> ((li & 3u) * 8u)) & 255u) ^ 128) - 128)) } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x128 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, ybase, flo, t0, 128u, m0, 128u, tv) - return } - // EDGE PATH — clamp-Constant layouts everywhere (store discard load-bearing) - var tla : tensorLayout2DPad - tensorLayoutCreate(tla) - tensorLayoutSetBlockSize(tla, 1u, 32u) - tensorLayoutSetDimension(tla, pa.d, pa.n) - tensorLayoutSetStride(tla, pa.n / 32u, 1u) - var tlb : tensorLayout2DPad - tensorLayoutCreate(tlb) - tensorLayoutSetDimension(tlb, row0 + cnt, pa.n) - tensorLayoutSetStride(tlb, pa.n, 1u) - var tlo : tensorLayout2DPad - tensorLayoutCreate(tlo) - tensorLayoutSetDimension(tlo, row0 + cnt, pa.d) - tensorLayoutSetStride(tlo, pa.d, 1u) - if (pa.ksplit == 0u) { - var k = 0u - for (_i in range(int(pa.n / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_q8) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 128u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < pa.n) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_q8) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 128u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x128 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, 0u, tlo, t0, 128u, m0, 128u, tv) - return + static_if (IQ3GRID) { + let gi = gl_LocalInvocationID.x + iq3s_gridc[gi] = iq3s_grid_word(int(gi)) + iq3s_gridc[gi + 256u] = iq3s_grid_word(int(gi + 256u)) } - var k = k0 - for (_i in range(int((k1 - k0) / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_q8) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 128u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } + static_if (IQ3XGRID) { + iq3x_gridc[gl_LocalInvocationID.x] = iq3xxs_grid_word(int(gl_LocalInvocationID.x)) } - while (k < k1) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_q8) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 128u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u + static_if (IQ2SGRID) { + let g2 = gl_LocalInvocationID.x + for (k in range(8)) { + iq2s_gridc[g2 + uint(k) * 256u] = iq2s_grid_word(int(g2 + uint(k) * 256u)) + } } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x128 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, ybase, tlo, t0, 128u, m0, 128u, tv) - } -} - -// the s-tile sibling (BN=32): the MoE expert-bucket shape - a region of ~32 rows fills the -// tile instead of padding three quarters of an m column; the body mirrors Q8Cm2MBatch at -// 32-token width -[vk_dispatch(name = "q8_batch_cm2s_cls", grid = "wgs", params = "wgs : int64")] -class Q8Cm2SBatch : MoeCmBase { - @ssbo @binding = 0 wq : array // fmt-0 weight quant blocks (32 bytes each, read as 16-bit lanes) - @ssbo @binding = 1 wsh : array // per-block weight scales (the second plane) - @ssbo @binding = 3 xf16 : array // f16 activation plane - @ssbo @binding = 5 y : array - @workgroup wg_blk0 : uint // the region's block base, staged for the decode method - - [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] - def decode_q8(blk : VkQ8Blk; bc, cib : uint2) : float16 { - let q = unpack8(blk.qs[int((cib.y & 30u) >> 1u)])[int(cib.y & 1u)] - return wsh[wg_blk0 + bc.x * (pa.n >> 5u) + bc.y] * float16(float(int(q))) - } - - [spirv_kernel(local_size_x = 256, name = "q8_batch_cm2s_cls_spv"), arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-tile-pick-and-default")] - def run { // nolint:STYLE038 — the fast/edge path pair, barrier- and register-coupled - let reg = region_rec() - let wblk0 = reg.x - let row0 = reg.y - let cnt = reg.z - let ttiles = (cnt + 31u) / 32u - // split-k: wgs stack nsplit planes of (wtiles x ttiles); plane ks covers k [k0, k1) and - // stores f32 partials at ybase (the reduce sums the planes into y). SINGLE-REGION (dense) - // dispatches only — the plane size rides row0 + cnt, so multi-region planes would collide - var tix = reg.w - var ks = 0u - var k1 = pa.n - var ybase = 0u - if (pa.ksplit != 0u) { - let ptiles = ((pa.d + 127u) / 128u) * ttiles - ks = tix / ptiles - tix -= ks * ptiles - k1 = min(pa.n, (ks + 1u) * pa.ksplit) - ybase = ks * (row0 + cnt) * pa.d + static_if (IQ2XSGRID) { + let g2x = gl_LocalInvocationID.x + for (k in range(4)) { + iq2xs_gridc[g2x + uint(k) * 256u] = iq2xs_grid_word(int(g2x + uint(k) * 256u)) + } } - let k0 = ks * pa.ksplit - let xt = tix % ttiles // token 32-tile (fastest) - let wt = tix / ttiles // weight 128-tile - if (gl_LocalInvocationID.x == 0u) { - wg_blk0 = wblk0 + static_if (IQ2XXSGRID) { + let g2xx = gl_LocalInvocationID.x + for (k in range(2)) { + iq2xxs_gridc[g2xx + uint(k) * 256u] = iq2xxs_grid_word(int(g2xx + uint(k) * 256u)) + } } - var tv : tensorView2Dt - tensorViewCreate(tv) - barrier() // wg_blk0 visible before the first decode load + barrier() // wg_blk0 (and the gated codebook) visible before the first decode load var a : coopmatWgA_f16_128x64 - var b : coopmatWgB_f16_64x32 - var acc : coopmatWgAcc_f16_128x32 - let t0 = row0 + xt * 32u + var b : BT + var acc : ACC + let t0 = row0 + xt * BN let m0 = wt * 128u - // FAST PATH — see Q8Cm2LBatch; identical rule at the s width, same literal-bound - // no-split specialization (the k0/k1/ybase form in the hot loop measured -27% pp) - if (m0 + 128u <= pa.d && (pa.n & 63u) == 0u) { + if (m0 + 128u <= pa.d && (STILE || xt * BN + BN <= cnt) && (pa.n & 63u) == 0u) { var fla : tensorLayout2D tensorLayoutCreate(fla) - tensorLayoutSetBlockSize(fla, 1u, 32u) + tensorLayoutSetBlockSize(fla, 1u, BLKW) tensorLayoutSetDimension(fla, pa.d, pa.n) - tensorLayoutSetStride(fla, pa.n >> 5u, 1u) + tensorLayoutSetStride(fla, pa.n / BLKW, 1u) var flb : tensorLayout2D tensorLayoutCreate(flb) - tensorLayoutSetDimension(flb, row0 + ((cnt + 31u) & ~31u), pa.n) + tensorLayoutSetDimension(flb, row0 + (STILE ? ((cnt + 31u) & ~31u) : cnt), pa.n) tensorLayoutSetStride(flb, pa.n & ~7u, 1u) - var flo : tensorLayout2DPad + var flo : FLO tensorLayoutCreate(flo) tensorLayoutSetDimension(flo, row0 + cnt, pa.d) tensorLayoutSetStride(flo, pa.d & ~7u, 1u) @@ -4023,51 +4908,51 @@ class Q8Cm2SBatch : MoeCmBase { var k = 0u for (_i in range(int(pa.n / 512u))) { for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_q8) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 32u, k, 64u, tv) + coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode) + coopmatLoadTensor(b, xf16, 0u, flb, t0, BN, k, 64u, tv) acc = coopmatMulAdd(a, b, acc) k += 64u } } while (k < pa.n) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_q8) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 32u, k, 64u, tv) + coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode) + coopmatLoadTensor(b, xf16, 0u, flb, t0, BN, k, 64u, tv) acc = coopmatMulAdd(a, b, acc) k += 64u } coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x32 + var accw : ACCW coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, 0u, flo, t0, 32u, m0, 128u, tv) + coopmatStoreTensor(accw, y, 0u, flo, t0, BN, m0, 128u, tv) return } var k = k0 for (_i in range(int((k1 - k0) / 512u))) { for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_q8) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 32u, k, 64u, tv) + coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode) + coopmatLoadTensor(b, xf16, 0u, flb, t0, BN, k, 64u, tv) acc = coopmatMulAdd(a, b, acc) k += 64u } } while (k < k1) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_q8) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 32u, k, 64u, tv) + coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode) + coopmatLoadTensor(b, xf16, 0u, flb, t0, BN, k, 64u, tv) acc = coopmatMulAdd(a, b, acc) k += 64u } coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x32 + var accw : ACCW coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, ybase, flo, t0, 32u, m0, 128u, tv) + coopmatStoreTensor(accw, y, ybase, flo, t0, BN, m0, 128u, tv) return } // EDGE PATH — clamp-Constant layouts everywhere (store discard load-bearing) var tla : tensorLayout2DPad tensorLayoutCreate(tla) - tensorLayoutSetBlockSize(tla, 1u, 32u) + tensorLayoutSetBlockSize(tla, 1u, BLKW) tensorLayoutSetDimension(tla, pa.d, pa.n) - tensorLayoutSetStride(tla, pa.n / 32u, 1u) + tensorLayoutSetStride(tla, pa.n / BLKW, 1u) var tlb : tensorLayout2DPad tensorLayoutCreate(tlb) tensorLayoutSetDimension(tlb, row0 + cnt, pa.n) @@ -4080,1086 +4965,752 @@ class Q8Cm2SBatch : MoeCmBase { var k = 0u for (_i in range(int(pa.n / 512u))) { for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_q8) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 32u, k, 64u, tv) + coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode) + coopmatLoadTensor(b, xf16, 0u, tlb, t0, BN, k, 64u, tv) acc = coopmatMulAdd(a, b, acc) k += 64u } } while (k < pa.n) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_q8) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 32u, k, 64u, tv) + coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode) + coopmatLoadTensor(b, xf16, 0u, tlb, t0, BN, k, 64u, tv) acc = coopmatMulAdd(a, b, acc) k += 64u } coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x32 + var accw : ACCW coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, 0u, tlo, t0, 32u, m0, 128u, tv) + coopmatStoreTensor(accw, y, 0u, tlo, t0, BN, m0, 128u, tv) return } var k = k0 for (_i in range(int((k1 - k0) / 512u))) { for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_q8) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 32u, k, 64u, tv) + coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode) + coopmatLoadTensor(b, xf16, 0u, tlb, t0, BN, k, 64u, tv) acc = coopmatMulAdd(a, b, acc) k += 64u } } while (k < k1) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_q8) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 32u, k, 64u, tv) + coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode) + coopmatLoadTensor(b, xf16, 0u, tlb, t0, BN, k, 64u, tv) acc = coopmatMulAdd(a, b, acc) k += 64u } coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x32 + var accw : ACCW coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, ybase, tlo, t0, 32u, m0, 128u, tv) + coopmatStoreTensor(accw, y, ybase, tlo, t0, BN, m0, 128u, tv) } } -// ===== the cm2 K-quant tiles: Q4_K decode-in-load over the repacked superblock planes ===== +// the format templates: the block struct, the scale element, the block width and the decode +[ |> template_struct_instance] +class template Q8Cm2T : KqCm2BatchT { + typedef BLK = VkQ8Blk + typedef ST = float16 + override BLKW = 32u -struct VkK4Blk { - qs : int16[64] // one Q4_K superblock's 128 nibble bytes as 16-bit lanes; its scale row (d|dmin + 8 sc + 8 mn bytes) lives in wsu + [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] + def decode(blk : VkQ8Blk; bc, cib : uint2) : float16 { + let q = unpack8(blk.qs[int((cib.y & 30u) >> 1u)])[int(cib.y & 1u)] + return ws[wg_blk0 + bc.x * (pa.n >> 5u) + bc.y] * float16(float(int(q))) + } } -// the l-tile Q4_K twin of Q8Cm2LBatch: same geometry, superblock (1, 256) blocks, nibble + per-32-group scale/min decode -[vk_dispatch(name = "kq_batch_k4_cm2l_cls", grid = "wgs", params = "wgs : int64")] -class K4Cm2LBatch : MoeCmBase { - @ssbo @binding = 0 @role = "weight" wq : array // Q4_K quant plane (128-byte superblocks) - @ssbo @binding = 1 @role = "weight" wsu : array // scale plane: 5 words per superblock - @ssbo @binding = 3 xf16 : array // f16 activation plane - @ssbo @binding = 5 y : array - @workgroup wg_blk0 : uint // the region's SUPERBLOCK base, staged for the decode +[ |> template_struct_instance] +class template K4Cm2T : KqCm2BatchT { + typedef BLK = VkK4Blk + typedef ST = uint [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] - def decode_k4(blk : VkK4Blk; bc, cib : uint2) : float16 { + def decode(blk : VkK4Blk; bc, cib : uint2) : float16 { let g = cib.y >> 5u let e = cib.y & 31u let bidx = g * 16u + (e & 15u) let by = uint(int(unpack8(blk.qs[int(bidx >> 1u)])[int(bidx & 1u)])) & 0xFFu let q = (by >> ((e >> 4u) * 4u)) & 0xFu let srow = (wg_blk0 + bc.x * (pa.n >> 8u) + bc.y) * 5u - let dm = unpackHalf2x16(wsu[srow]) - let sc = (wsu[srow + 1u + (g >> 2u)] >> ((g & 3u) * 8u)) & 0xFFu - let mn = (wsu[srow + 3u + (g >> 2u)] >> ((g & 3u) * 8u)) & 0xFFu + let dm = unpackHalf2x16(ws[srow]) + let sc = (ws[srow + 1u + (g >> 2u)] >> ((g & 3u) * 8u)) & 0xFFu + let mn = (ws[srow + 3u + (g >> 2u)] >> ((g & 3u) * 8u)) & 0xFFu return float16(dm.x * float(sc) * float(q) - dm.y * float(mn)) } +} - [spirv_kernel(local_size_x = 256, name = "kq_batch_k4_cm2l_cls_spv"), arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-tile-pick-and-default")] - def run { // nolint:STYLE038 — the fast/edge path pair, barrier- and register-coupled - let reg = region_rec() - let wblk0 = reg.x - let row0 = reg.y - let cnt = reg.z - let ttiles = (cnt + 255u) / 256u - var tix = reg.w - var ks = 0u - var k1 = pa.n - var ybase = 0u - if (pa.ksplit != 0u) { - let ptiles = ((pa.d + 127u) / 128u) * ttiles - ks = tix / ptiles - tix -= ks * ptiles - k1 = min(pa.n, (ks + 1u) * pa.ksplit) - ybase = ks * (row0 + cnt) * pa.d - } - let k0 = ks * pa.ksplit - let xt = tix % ttiles - let wt = tix / ttiles - if (gl_LocalInvocationID.x == 0u) { - wg_blk0 = wblk0 - } - var tv : tensorView2Dt - tensorViewCreate(tv) - barrier() // wg_blk0 visible before the first decode load - var a : coopmatWgA_f16_128x64 - var b : coopmatWgB_f16_64x256 - var acc : coopmatWgAcc_f16_128x256 - let t0 = row0 + xt * 256u - let m0 = wt * 128u - // FAST PATH — see Q8Cm2LBatch (kq n is always a 256-multiple, so only tile bounds gate) - if (m0 + 128u <= pa.d && xt * 256u + 256u <= cnt && (pa.n & 63u) == 0u) { - var fla : tensorLayout2D - tensorLayoutCreate(fla) - tensorLayoutSetBlockSize(fla, 1u, 256u) - tensorLayoutSetDimension(fla, pa.d, pa.n) - tensorLayoutSetStride(fla, pa.n >> 8u, 1u) - var flb : tensorLayout2D - tensorLayoutCreate(flb) - tensorLayoutSetDimension(flb, row0 + cnt, pa.n) - tensorLayoutSetStride(flb, pa.n & ~7u, 1u) - var flo : tensorLayout2D - tensorLayoutCreate(flo) - tensorLayoutSetDimension(flo, row0 + cnt, pa.d) - tensorLayoutSetStride(flo, pa.d & ~7u, 1u) - if (pa.ksplit == 0u) { - var k = 0u - for (_i in range(int(pa.n / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k4) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 256u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < pa.n) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k4) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 256u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x256 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, 0u, flo, t0, 256u, m0, 128u, tv) - return - } - var k = k0 - for (_i in range(int((k1 - k0) / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k4) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 256u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < k1) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k4) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 256u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x256 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, ybase, flo, t0, 256u, m0, 128u, tv) - return - } - // EDGE PATH — clamp-Constant layouts everywhere (store discard load-bearing) - var tla : tensorLayout2DPad - tensorLayoutCreate(tla) - tensorLayoutSetBlockSize(tla, 1u, 256u) - tensorLayoutSetDimension(tla, pa.d, pa.n) - tensorLayoutSetStride(tla, pa.n / 256u, 1u) - var tlb : tensorLayout2DPad - tensorLayoutCreate(tlb) - tensorLayoutSetDimension(tlb, row0 + cnt, pa.n) - tensorLayoutSetStride(tlb, pa.n, 1u) - var tlo : tensorLayout2DPad - tensorLayoutCreate(tlo) - tensorLayoutSetDimension(tlo, row0 + cnt, pa.d) - tensorLayoutSetStride(tlo, pa.d, 1u) - if (pa.ksplit == 0u) { - var k = 0u - for (_i in range(int(pa.n / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k4) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 256u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < pa.n) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k4) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 256u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x256 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, 0u, tlo, t0, 256u, m0, 128u, tv) - return - } - var k = k0 - for (_i in range(int((k1 - k0) / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k4) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 256u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < k1) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k4) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 256u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x256 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, ybase, tlo, t0, 256u, m0, 128u, tv) +[ |> template_struct_instance] +class template K6Cm2T : KqCm2BatchT { + typedef BLK = VkK6Blk + typedef ST = uint + + [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] + def decode(blk : VkK6Blk; bc, cib : uint2) : float16 { + let e = cib.y + let bu = e >> 5u + let hh = (e >> 4u) & 1u + let j = e & 15u + let lb = bu * 16u + j + let lo = uint(int(unpack8(blk.ql[int(lb >> 1u)])[int(lb & 1u)])) & 0xFFu + let qb = (bu >> 2u) * 32u + hh * 16u + j + let hby = uint(int(unpack8(blk.qh[int(qb >> 1u)])[int(qb & 1u)])) & 0xFFu + let q6 = int(((lo >> (hh * 4u)) & 0xFu) | (((hby >> ((bu & 3u) * 2u)) & 3u) << 4u)) - 32 + let srow = (wg_blk0 + bc.x * (pa.n >> 8u) + bc.y) * 5u + let sidx = e >> 4u + let sc = int(ws[srow + (sidx >> 2u)] << ((3u - (sidx & 3u)) * 8u)) >> 24 + let d = unpackHalf2x16(ws[srow + 4u]).x + return float16(d * float(sc) * float(q6)) } } -// the m-tile Q4_K sibling (BN=128) — the k4 decode on the m geometry -[vk_dispatch(name = "kq_batch_k4_cm2m_cls", grid = "wgs", params = "wgs : int64")] -class K4Cm2MBatch : MoeCmBase { - @ssbo @binding = 0 @role = "weight" wq : array - @ssbo @binding = 1 @role = "weight" wsu : array - @ssbo @binding = 3 xf16 : array - @ssbo @binding = 5 y : array - @workgroup wg_blk0 : uint +[ |> template_struct_instance] +class template K5Cm2T : KqCm2BatchT { + typedef BLK = VkK5Blk + typedef ST = uint [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] - def decode_k4(blk : VkK4Blk; bc, cib : uint2) : float16 { + def decode(blk : VkK5Blk; bc, cib : uint2) : float16 { let g = cib.y >> 5u let e = cib.y & 31u - let bidx = g * 16u + (e & 15u) + let j = e & 15u + let h = e >> 4u + let bidx = g * 16u + j let by = uint(int(unpack8(blk.qs[int(bidx >> 1u)])[int(bidx & 1u)])) & 0xFFu - let q = (by >> ((e >> 4u) * 4u)) & 0xFu + let hidx = g * 4u + (j >> 2u) + let hby = uint(int(unpack8(blk.qh[int(hidx >> 1u)])[int(hidx & 1u)])) & 0xFFu + let q = ((by >> (h * 4u)) & 0xFu) | (((hby >> ((j & 3u) + h * 4u)) & 1u) << 4u) let srow = (wg_blk0 + bc.x * (pa.n >> 8u) + bc.y) * 5u - let dm = unpackHalf2x16(wsu[srow]) - let sc = (wsu[srow + 1u + (g >> 2u)] >> ((g & 3u) * 8u)) & 0xFFu - let mn = (wsu[srow + 3u + (g >> 2u)] >> ((g & 3u) * 8u)) & 0xFFu + let dm = unpackHalf2x16(ws[srow]) + let sc = (ws[srow + 1u + (g >> 2u)] >> ((g & 3u) * 8u)) & 0xFFu + let mn = (ws[srow + 3u + (g >> 2u)] >> ((g & 3u) * 8u)) & 0xFFu return float16(dm.x * float(sc) * float(q) - dm.y * float(mn)) } +} - [spirv_kernel(local_size_x = 256, name = "kq_batch_k4_cm2m_cls_spv"), arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-tile-pick-and-default")] - def run { // nolint:STYLE038 — the fast/edge path pair, barrier- and register-coupled - let reg = region_rec() - let wblk0 = reg.x - let row0 = reg.y - let cnt = reg.z - let ttiles = (cnt + 127u) / 128u - var tix = reg.w - var ks = 0u - var k1 = pa.n - var ybase = 0u - if (pa.ksplit != 0u) { - let ptiles = ((pa.d + 127u) / 128u) * ttiles - ks = tix / ptiles - tix -= ks * ptiles - k1 = min(pa.n, (ks + 1u) * pa.ksplit) - ybase = ks * (row0 + cnt) * pa.d - } - let k0 = ks * pa.ksplit - let xt = tix % ttiles - let wt = tix / ttiles - if (gl_LocalInvocationID.x == 0u) { - wg_blk0 = wblk0 - } - var tv : tensorView2Dt - tensorViewCreate(tv) - barrier() - var a : coopmatWgA_f16_128x64 - var b : coopmatWgB_f16_64x128 - var acc : coopmatWgAcc_f16_128x128 - let t0 = row0 + xt * 128u - let m0 = wt * 128u - if (m0 + 128u <= pa.d && xt * 128u + 128u <= cnt && (pa.n & 63u) == 0u) { - var fla : tensorLayout2D - tensorLayoutCreate(fla) - tensorLayoutSetBlockSize(fla, 1u, 256u) - tensorLayoutSetDimension(fla, pa.d, pa.n) - tensorLayoutSetStride(fla, pa.n >> 8u, 1u) - var flb : tensorLayout2D - tensorLayoutCreate(flb) - tensorLayoutSetDimension(flb, row0 + cnt, pa.n) - tensorLayoutSetStride(flb, pa.n & ~7u, 1u) - var flo : tensorLayout2D - tensorLayoutCreate(flo) - tensorLayoutSetDimension(flo, row0 + cnt, pa.d) - tensorLayoutSetStride(flo, pa.d & ~7u, 1u) - if (pa.ksplit == 0u) { - var k = 0u - for (_i in range(int(pa.n / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k4) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 128u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < pa.n) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k4) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 128u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x128 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, 0u, flo, t0, 128u, m0, 128u, tv) - return - } - var k = k0 - for (_i in range(int((k1 - k0) / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k4) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 128u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < k1) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k4) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 128u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x128 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, ybase, flo, t0, 128u, m0, 128u, tv) - return - } - var tla : tensorLayout2DPad - tensorLayoutCreate(tla) - tensorLayoutSetBlockSize(tla, 1u, 256u) - tensorLayoutSetDimension(tla, pa.d, pa.n) - tensorLayoutSetStride(tla, pa.n / 256u, 1u) - var tlb : tensorLayout2DPad - tensorLayoutCreate(tlb) - tensorLayoutSetDimension(tlb, row0 + cnt, pa.n) - tensorLayoutSetStride(tlb, pa.n, 1u) - var tlo : tensorLayout2DPad - tensorLayoutCreate(tlo) - tensorLayoutSetDimension(tlo, row0 + cnt, pa.d) - tensorLayoutSetStride(tlo, pa.d, 1u) - if (pa.ksplit == 0u) { - var k = 0u - for (_i in range(int(pa.n / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k4) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 128u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < pa.n) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k4) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 128u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x128 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, 0u, tlo, t0, 128u, m0, 128u, tv) - return - } - var k = k0 - for (_i in range(int((k1 - k0) / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k4) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 128u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < k1) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k4) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 128u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x128 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, ybase, tlo, t0, 128u, m0, 128u, tv) +[ |> template_struct_instance] +class template Q40Cm2T : KqCm2BatchT { + typedef BLK = VkK4Blk // the same 128-nibble lane view - q40 rows land in the k/k+16 device pairing too + typedef ST = uint + + [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] + def decode(blk : VkK4Blk; bc, cib : uint2) : float16 { + let g = cib.y >> 5u + let e = cib.y & 31u + let bidx = g * 16u + (e & 15u) + let by = uint(int(unpack8(blk.qs[int(bidx >> 1u)])[int(bidx & 1u)])) & 0xFFu + let q = (by >> ((e >> 4u) * 4u)) & 0xFu + let srow = (wg_blk0 + bc.x * (pa.n >> 8u) + bc.y) * 5u + let dp = unpackHalf2x16(ws[srow + (g >> 1u)]) + let d = (g & 1u) == 0u ? dp.x : dp.y + return float16(d * (float(q) - 8.0)) } } -// the s-tile Q4_K sibling (BN=32) - the k4 decode on the expert-bucket geometry (see Q8Cm2SBatch) -[vk_dispatch(name = "kq_batch_k4_cm2s_cls", grid = "wgs", params = "wgs : int64")] -class K4Cm2SBatch : MoeCmBase { - @ssbo @binding = 0 @role = "weight" wq : array - @ssbo @binding = 1 @role = "weight" wsu : array - @ssbo @binding = 3 xf16 : array - @ssbo @binding = 5 y : array - @workgroup wg_blk0 : uint +[ |> template_struct_instance] +class template Iq4xsCm2T : KqCm2BatchT { + typedef BLK = VkK4Blk // the iq4xs plane is the k/k+16 pairing verbatim - the k4 lane view fits + typedef ST = uint + override IQLUT = true [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] - def decode_k4(blk : VkK4Blk; bc, cib : uint2) : float16 { + def decode(blk : VkK4Blk; bc, cib : uint2) : float16 { let g = cib.y >> 5u let e = cib.y & 31u let bidx = g * 16u + (e & 15u) let by = uint(int(unpack8(blk.qs[int(bidx >> 1u)])[int(bidx & 1u)])) & 0xFFu let q = (by >> ((e >> 4u) * 4u)) & 0xFu let srow = (wg_blk0 + bc.x * (pa.n >> 8u) + bc.y) * 5u - let dm = unpackHalf2x16(wsu[srow]) - let sc = (wsu[srow + 1u + (g >> 2u)] >> ((g & 3u) * 8u)) & 0xFFu - let mn = (wsu[srow + 3u + (g >> 2u)] >> ((g & 3u) * 8u)) & 0xFFu - return float16(dm.x * float(sc) * float(q) - dm.y * float(mn)) + let d = unpackHalf2x16(ws[srow]).x + let sc = int(ws[srow + 1u + (g >> 2u)] << ((3u - (g & 3u)) * 8u)) >> 24 + return float16(d * float(sc) * float(iq4lut[int(q)])) } +} - [spirv_kernel(local_size_x = 256, name = "kq_batch_k4_cm2s_cls_spv"), arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-tile-pick-and-default")] - def run { // nolint:STYLE038 — the fast/edge path pair, barrier- and register-coupled - let reg = region_rec() - let wblk0 = reg.x - let row0 = reg.y - let cnt = reg.z - let ttiles = (cnt + 31u) / 32u - var tix = reg.w - var ks = 0u - var k1 = pa.n - var ybase = 0u - if (pa.ksplit != 0u) { - let ptiles = ((pa.d + 127u) / 128u) * ttiles - ks = tix / ptiles - tix -= ks * ptiles - k1 = min(pa.n, (ks + 1u) * pa.ksplit) - ybase = ks * (row0 + cnt) * pa.d - } - let k0 = ks * pa.ksplit - let xt = tix % ttiles - let wt = tix / ttiles - if (gl_LocalInvocationID.x == 0u) { - wg_blk0 = wblk0 - } - var tv : tensorView2Dt - tensorViewCreate(tv) - barrier() - var a : coopmatWgA_f16_128x64 - var b : coopmatWgB_f16_64x32 - var acc : coopmatWgAcc_f16_128x32 - let t0 = row0 + xt * 32u - let m0 = wt * 128u - if (m0 + 128u <= pa.d && (pa.n & 63u) == 0u) { - var fla : tensorLayout2D - tensorLayoutCreate(fla) - tensorLayoutSetBlockSize(fla, 1u, 256u) - tensorLayoutSetDimension(fla, pa.d, pa.n) - tensorLayoutSetStride(fla, pa.n >> 8u, 1u) - var flb : tensorLayout2D - tensorLayoutCreate(flb) - tensorLayoutSetDimension(flb, row0 + ((cnt + 31u) & ~31u), pa.n) - tensorLayoutSetStride(flb, pa.n & ~7u, 1u) - var flo : tensorLayout2DPad - tensorLayoutCreate(flo) - tensorLayoutSetDimension(flo, row0 + cnt, pa.d) - tensorLayoutSetStride(flo, pa.d & ~7u, 1u) - if (pa.ksplit == 0u) { - var k = 0u - for (_i in range(int(pa.n / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k4) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 32u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < pa.n) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k4) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 32u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x32 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, 0u, flo, t0, 32u, m0, 128u, tv) - return - } - var k = k0 - for (_i in range(int((k1 - k0) / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k4) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 32u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < k1) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k4) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 32u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x32 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, ybase, flo, t0, 32u, m0, 128u, tv) - return - } - var tla : tensorLayout2DPad - tensorLayoutCreate(tla) - tensorLayoutSetBlockSize(tla, 1u, 256u) - tensorLayoutSetDimension(tla, pa.d, pa.n) - tensorLayoutSetStride(tla, pa.n / 256u, 1u) - var tlb : tensorLayout2DPad - tensorLayoutCreate(tlb) - tensorLayoutSetDimension(tlb, row0 + cnt, pa.n) - tensorLayoutSetStride(tlb, pa.n, 1u) - var tlo : tensorLayout2DPad - tensorLayoutCreate(tlo) - tensorLayoutSetDimension(tlo, row0 + cnt, pa.d) - tensorLayoutSetStride(tlo, pa.d, 1u) - if (pa.ksplit == 0u) { - var k = 0u - for (_i in range(int(pa.n / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k4) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 32u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < pa.n) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k4) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 32u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x32 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, 0u, tlo, t0, 32u, m0, 128u, tv) - return - } - var k = k0 - for (_i in range(int((k1 - k0) / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k4) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 32u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < k1) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k4) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 32u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x32 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, ybase, tlo, t0, 32u, m0, 128u, tv) - } -} - -struct VkK6Blk { - ql : int16[64] // one Q6_K superblock's 128 low-nibble bytes as 16-bit lanes - qh : int16[32] // ... and its 64 high-2-bit bytes; scales (16 int8 + f16 d) live in wsu -} - -// the l-tile Q6_K twin: 6-bit compose (nibble | qh 2 bits) - 32, per-16 signed sub-scale -[vk_dispatch(name = "kq_batch_k6_cm2l_cls", grid = "wgs", params = "wgs : int64")] -class K6Cm2LBatch : MoeCmBase { - @ssbo @binding = 0 @role = "weight" wq : array // Q6_K quant plane (192-byte superblocks) - @ssbo @binding = 1 @role = "weight" wsu : array // scale plane: 5 words per superblock - @ssbo @binding = 3 xf16 : array // f16 activation plane - @ssbo @binding = 5 y : array - @workgroup wg_blk0 : uint // the region's SUPERBLOCK base, staged for the decode +[ |> template_struct_instance] +class template K3Cm2T : KqCm2BatchT { + typedef BLK = VkK3Blk + typedef ST = uint [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] - def decode_k6(blk : VkK6Blk; bc, cib : uint2) : float16 { + def decode(blk : VkK3Blk; bc, cib : uint2) : float16 { let e = cib.y - let bu = e >> 5u - let hh = (e >> 4u) & 1u - let j = e & 15u - let lb = bu * 16u + j - let lo = uint(int(unpack8(blk.ql[int(lb >> 1u)])[int(lb & 1u)])) & 0xFFu - let qb = (bu >> 2u) * 32u + hh * 16u + j - let hby = uint(int(unpack8(blk.qh[int(qb >> 1u)])[int(qb & 1u)])) & 0xFFu - let q6 = int(((lo >> (hh * 4u)) & 0xFu) | (((hby >> ((bu & 3u) * 2u)) & 3u) << 4u)) - 32 + let g = e >> 5u + let l = e & 31u + let qi = (g >> 2u) * 32u + l + let qby = uint(int(unpack8(blk.qs[int(qi >> 1u)])[int(qi & 1u)])) & 0xFFu + let hby = uint(int(unpack8(blk.hm[int(l >> 1u)])[int(l & 1u)])) & 0xFFu + let q3 = int((qby >> ((g & 3u) * 2u)) & 3u) - (((hby >> g) & 1u) == 0u ? 4 : 0) let srow = (wg_blk0 + bc.x * (pa.n >> 8u) + bc.y) * 5u let sidx = e >> 4u - let sc = int(wsu[srow + (sidx >> 2u)] << ((3u - (sidx & 3u)) * 8u)) >> 24 - let d = unpackHalf2x16(wsu[srow + 4u]).x - return float16(d * float(sc) * float(q6)) - } - - [spirv_kernel(local_size_x = 256, name = "kq_batch_k6_cm2l_cls_spv"), arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-tile-pick-and-default")] - def run { // nolint:STYLE038 — the fast/edge path pair, barrier- and register-coupled - let reg = region_rec() - let wblk0 = reg.x - let row0 = reg.y - let cnt = reg.z - let ttiles = (cnt + 255u) / 256u - var tix = reg.w - var ks = 0u - var k1 = pa.n - var ybase = 0u - if (pa.ksplit != 0u) { - let ptiles = ((pa.d + 127u) / 128u) * ttiles - ks = tix / ptiles - tix -= ks * ptiles - k1 = min(pa.n, (ks + 1u) * pa.ksplit) - ybase = ks * (row0 + cnt) * pa.d - } - let k0 = ks * pa.ksplit - let xt = tix % ttiles - let wt = tix / ttiles - if (gl_LocalInvocationID.x == 0u) { - wg_blk0 = wblk0 - } - var tv : tensorView2Dt - tensorViewCreate(tv) - barrier() // wg_blk0 visible before the first decode load - var a : coopmatWgA_f16_128x64 - var b : coopmatWgB_f16_64x256 - var acc : coopmatWgAcc_f16_128x256 - let t0 = row0 + xt * 256u - let m0 = wt * 128u - if (m0 + 128u <= pa.d && xt * 256u + 256u <= cnt && (pa.n & 63u) == 0u) { - var fla : tensorLayout2D - tensorLayoutCreate(fla) - tensorLayoutSetBlockSize(fla, 1u, 256u) - tensorLayoutSetDimension(fla, pa.d, pa.n) - tensorLayoutSetStride(fla, pa.n >> 8u, 1u) - var flb : tensorLayout2D - tensorLayoutCreate(flb) - tensorLayoutSetDimension(flb, row0 + cnt, pa.n) - tensorLayoutSetStride(flb, pa.n & ~7u, 1u) - var flo : tensorLayout2D - tensorLayoutCreate(flo) - tensorLayoutSetDimension(flo, row0 + cnt, pa.d) - tensorLayoutSetStride(flo, pa.d & ~7u, 1u) - if (pa.ksplit == 0u) { - var k = 0u - for (_i in range(int(pa.n / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k6) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 256u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < pa.n) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k6) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 256u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x256 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, 0u, flo, t0, 256u, m0, 128u, tv) - return - } - var k = k0 - for (_i in range(int((k1 - k0) / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k6) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 256u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < k1) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k6) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 256u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x256 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, ybase, flo, t0, 256u, m0, 128u, tv) - return - } - // EDGE PATH — clamp-Constant layouts everywhere (store discard load-bearing) - var tla : tensorLayout2DPad - tensorLayoutCreate(tla) - tensorLayoutSetBlockSize(tla, 1u, 256u) - tensorLayoutSetDimension(tla, pa.d, pa.n) - tensorLayoutSetStride(tla, pa.n / 256u, 1u) - var tlb : tensorLayout2DPad - tensorLayoutCreate(tlb) - tensorLayoutSetDimension(tlb, row0 + cnt, pa.n) - tensorLayoutSetStride(tlb, pa.n, 1u) - var tlo : tensorLayout2DPad - tensorLayoutCreate(tlo) - tensorLayoutSetDimension(tlo, row0 + cnt, pa.d) - tensorLayoutSetStride(tlo, pa.d, 1u) - if (pa.ksplit == 0u) { - var k = 0u - for (_i in range(int(pa.n / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k6) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 256u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < pa.n) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k6) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 256u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x256 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, 0u, tlo, t0, 256u, m0, 128u, tv) - return - } - var k = k0 - for (_i in range(int((k1 - k0) / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k6) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 256u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < k1) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k6) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 256u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x256 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, ybase, tlo, t0, 256u, m0, 128u, tv) + let sc = int(ws[srow + (sidx >> 2u)] << ((3u - (sidx & 3u)) * 8u)) >> 24 + let d = unpackHalf2x16(ws[srow + 4u]).x + return float16(d * float(sc) * float(q3)) } } -// the m-tile Q6_K sibling (BN=128) -[vk_dispatch(name = "kq_batch_k6_cm2m_cls", grid = "wgs", params = "wgs : int64")] -class K6Cm2MBatch : MoeCmBase { - @ssbo @binding = 0 @role = "weight" wq : array - @ssbo @binding = 1 @role = "weight" wsu : array - @ssbo @binding = 3 xf16 : array - @ssbo @binding = 5 y : array - @workgroup wg_blk0 : uint +[ |> template_struct_instance] +class template K2Cm2T : KqCm2BatchT { + typedef BLK = VkK2Blk + typedef ST = uint [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] - def decode_k6(blk : VkK6Blk; bc, cib : uint2) : float16 { + def decode(blk : VkK2Blk; bc, cib : uint2) : float16 { let e = cib.y - let bu = e >> 5u - let hh = (e >> 4u) & 1u - let j = e & 15u - let lb = bu * 16u + j - let lo = uint(int(unpack8(blk.ql[int(lb >> 1u)])[int(lb & 1u)])) & 0xFFu - let qb = (bu >> 2u) * 32u + hh * 16u + j - let hby = uint(int(unpack8(blk.qh[int(qb >> 1u)])[int(qb & 1u)])) & 0xFFu - let q6 = int(((lo >> (hh * 4u)) & 0xFu) | (((hby >> ((bu & 3u) * 2u)) & 3u) << 4u)) - 32 + let g = e >> 4u + let l = e & 15u + let qi = (g >> 3u) * 32u + (g & 1u) * 16u + l + let qby = uint(int(unpack8(blk.qs[int(qi >> 1u)])[int(qi & 1u)])) & 0xFFu + let q = (qby >> (((g >> 1u) & 3u) * 2u)) & 3u let srow = (wg_blk0 + bc.x * (pa.n >> 8u) + bc.y) * 5u - let sidx = e >> 4u - let sc = int(wsu[srow + (sidx >> 2u)] << ((3u - (sidx & 3u)) * 8u)) >> 24 - let d = unpackHalf2x16(wsu[srow + 4u]).x - return float16(d * float(sc) * float(q6)) + let dm = unpackHalf2x16(ws[srow]) + let pb = (ws[srow + 1u + (g >> 2u)] >> ((g & 3u) * 8u)) & 0xFFu + return float16(dm.x * float(pb & 15u) * float(q) - dm.y * float(pb >> 4u)) } +} - [spirv_kernel(local_size_x = 256, name = "kq_batch_k6_cm2m_cls_spv"), arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-tile-pick-and-default")] - def run { // nolint:STYLE038 — the fast/edge path pair, barrier- and register-coupled - let reg = region_rec() - let wblk0 = reg.x - let row0 = reg.y - let cnt = reg.z - let ttiles = (cnt + 127u) / 128u - var tix = reg.w - var ks = 0u - var k1 = pa.n - var ybase = 0u - if (pa.ksplit != 0u) { - let ptiles = ((pa.d + 127u) / 128u) * ttiles - ks = tix / ptiles - tix -= ks * ptiles - k1 = min(pa.n, (ks + 1u) * pa.ksplit) - ybase = ks * (row0 + cnt) * pa.d - } - let k0 = ks * pa.ksplit - let xt = tix % ttiles - let wt = tix / ttiles - if (gl_LocalInvocationID.x == 0u) { - wg_blk0 = wblk0 - } - var tv : tensorView2Dt - tensorViewCreate(tv) - barrier() - var a : coopmatWgA_f16_128x64 - var b : coopmatWgB_f16_64x128 - var acc : coopmatWgAcc_f16_128x128 - let t0 = row0 + xt * 128u - let m0 = wt * 128u - if (m0 + 128u <= pa.d && xt * 128u + 128u <= cnt && (pa.n & 63u) == 0u) { - var fla : tensorLayout2D - tensorLayoutCreate(fla) - tensorLayoutSetBlockSize(fla, 1u, 256u) - tensorLayoutSetDimension(fla, pa.d, pa.n) - tensorLayoutSetStride(fla, pa.n >> 8u, 1u) - var flb : tensorLayout2D - tensorLayoutCreate(flb) - tensorLayoutSetDimension(flb, row0 + cnt, pa.n) - tensorLayoutSetStride(flb, pa.n & ~7u, 1u) - var flo : tensorLayout2D - tensorLayoutCreate(flo) - tensorLayoutSetDimension(flo, row0 + cnt, pa.d) - tensorLayoutSetStride(flo, pa.d & ~7u, 1u) - if (pa.ksplit == 0u) { - var k = 0u - for (_i in range(int(pa.n / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k6) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 128u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < pa.n) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k6) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 128u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x128 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, 0u, flo, t0, 128u, m0, 128u, tv) - return - } - var k = k0 - for (_i in range(int((k1 - k0) / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k6) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 128u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < k1) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k6) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 128u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x128 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, ybase, flo, t0, 128u, m0, 128u, tv) - return - } - var tla : tensorLayout2DPad - tensorLayoutCreate(tla) - tensorLayoutSetBlockSize(tla, 1u, 256u) - tensorLayoutSetDimension(tla, pa.d, pa.n) - tensorLayoutSetStride(tla, pa.n / 256u, 1u) - var tlb : tensorLayout2DPad - tensorLayoutCreate(tlb) - tensorLayoutSetDimension(tlb, row0 + cnt, pa.n) - tensorLayoutSetStride(tlb, pa.n, 1u) - var tlo : tensorLayout2DPad - tensorLayoutCreate(tlo) - tensorLayoutSetDimension(tlo, row0 + cnt, pa.d) - tensorLayoutSetStride(tlo, pa.d, 1u) - if (pa.ksplit == 0u) { - var k = 0u - for (_i in range(int(pa.n / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k6) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 128u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < pa.n) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k6) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 128u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x128 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, 0u, tlo, t0, 128u, m0, 128u, tv) - return - } - var k = k0 - for (_i in range(int((k1 - k0) / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k6) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 128u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < k1) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k6) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 128u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x128 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, ybase, tlo, t0, 128u, m0, 128u, tv) +[ |> template_struct_instance] +class template Iq3sCm2T : KqCm2BatchT { + typedef BLK = VkIq3sBlk + typedef ST = uint + override IQ3GRID = true + + [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] + def decode(blk : VkIq3sBlk; bc, cib : uint2) : float16 { + let e = cib.y + let g = e >> 5u + let r = e & 31u + let wj = r >> 2u + let qi = g * 8u + wj + let qb = uint(int(unpack8(blk.qs[int(qi >> 1u)])[int(qi & 1u)])) & 0xFFu + let qh = uint(int(unpack8(blk.qh[int(g >> 1u)])[int(g & 1u)])) & 0xFFu + let si = g * 4u + (r >> 3u) + let sgb = uint(int(unpack8(blk.sg[int(si >> 1u)])[int(si & 1u)])) & 0xFFu + let gw = iq3s_gridc[qb | ((qh << (8u - wj)) & 256u)] + let gb = float((gw >> ((r & 3u) * 8u)) & 0xFFu) + let srow = (wg_blk0 + bc.x * (pa.n >> 8u) + bc.y) * 5u + let d = unpackHalf2x16(ws[srow]).x + let sc = int(ws[srow + 1u + (g >> 2u)] << ((3u - (g & 3u)) * 8u)) >> 24 + return float16(d * float(sc) * (((sgb >> (((r >> 2u) & 1u) * 4u + (r & 3u))) & 1u) != 0u ? -gb : gb)) } } -// the s-tile Q6_K sibling (BN=32, see Q8Cm2SBatch) -[vk_dispatch(name = "kq_batch_k6_cm2s_cls", grid = "wgs", params = "wgs : int64")] -class K6Cm2SBatch : MoeCmBase { - @ssbo @binding = 0 @role = "weight" wq : array - @ssbo @binding = 1 @role = "weight" wsu : array - @ssbo @binding = 3 xf16 : array - @ssbo @binding = 5 y : array - @workgroup wg_blk0 : uint +[ |> template_struct_instance] +class template Iq3xxsCm2T : KqCm2BatchT { + typedef BLK = VkIq3xxsBlk + typedef ST = uint + override IQ3XGRID = true [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] - def decode_k6(blk : VkK6Blk; bc, cib : uint2) : float16 { + def decode(blk : VkIq3xxsBlk; bc, cib : uint2) : float16 { let e = cib.y - let bu = e >> 5u - let hh = (e >> 4u) & 1u - let j = e & 15u - let lb = bu * 16u + j - let lo = uint(int(unpack8(blk.ql[int(lb >> 1u)])[int(lb & 1u)])) & 0xFFu - let qb = (bu >> 2u) * 32u + hh * 16u + j - let hby = uint(int(unpack8(blk.qh[int(qb >> 1u)])[int(qb & 1u)])) & 0xFFu - let q6 = int(((lo >> (hh * 4u)) & 0xFu) | (((hby >> ((bu & 3u) * 2u)) & 3u) << 4u)) - 32 + let g = e >> 5u + let r = e & 31u + let wj = r >> 2u + let qi = g * 8u + wj + let qb = uint(int(unpack8(blk.qs[int(qi >> 1u)])[int(qi & 1u)])) & 0xFFu + let l = r >> 3u + let bl = 7u * l + let a0i = g * 4u + (bl >> 3u) + let b0 = uint(int(unpack8(blk.ax[int(a0i >> 1u)])[int(a0i & 1u)])) & 0xFFu + let b1 = uint(int(unpack8(blk.ax[int((a0i + 1u) >> 1u)])[int((a0i + 1u) & 1u)])) & 0xFFu + let sidx = ((b0 | (b1 << 8u)) >> (bl & 7u)) & 127u + var tt = sidx ^ (sidx >> 4u) + tt = tt ^ (tt >> 2u) + tt = tt ^ (tt >> 1u) + let sgb = sidx | ((tt & 1u) << 7u) + let gw = iq3x_gridc[qb] + let gb = float((gw >> ((r & 3u) * 8u)) & 0xFFu) let srow = (wg_blk0 + bc.x * (pa.n >> 8u) + bc.y) * 5u - let sidx = e >> 4u - let sc = int(wsu[srow + (sidx >> 2u)] << ((3u - (sidx & 3u)) * 8u)) >> 24 - let d = unpackHalf2x16(wsu[srow + 4u]).x - return float16(d * float(sc) * float(q6)) + let d = unpackHalf2x16(ws[srow]).x + let sc = int(ws[srow + 1u + (g >> 2u)] << ((3u - (g & 3u)) * 8u)) >> 24 + return float16(d * float(sc) * (((sgb >> (r & 7u)) & 1u) != 0u ? -gb : gb)) } +} - [spirv_kernel(local_size_x = 256, name = "kq_batch_k6_cm2s_cls_spv"), arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-tile-pick-and-default")] - def run { // nolint:STYLE038 — the fast/edge path pair, barrier- and register-coupled - let reg = region_rec() - let wblk0 = reg.x - let row0 = reg.y - let cnt = reg.z - let ttiles = (cnt + 31u) / 32u - var tix = reg.w - var ks = 0u - var k1 = pa.n - var ybase = 0u - if (pa.ksplit != 0u) { - let ptiles = ((pa.d + 127u) / 128u) * ttiles - ks = tix / ptiles - tix -= ks * ptiles - k1 = min(pa.n, (ks + 1u) * pa.ksplit) - ybase = ks * (row0 + cnt) * pa.d - } - let k0 = ks * pa.ksplit - let xt = tix % ttiles - let wt = tix / ttiles - if (gl_LocalInvocationID.x == 0u) { - wg_blk0 = wblk0 - } - var tv : tensorView2Dt - tensorViewCreate(tv) - barrier() - var a : coopmatWgA_f16_128x64 - var b : coopmatWgB_f16_64x32 - var acc : coopmatWgAcc_f16_128x32 - let t0 = row0 + xt * 32u - let m0 = wt * 128u - if (m0 + 128u <= pa.d && (pa.n & 63u) == 0u) { - var fla : tensorLayout2D - tensorLayoutCreate(fla) - tensorLayoutSetBlockSize(fla, 1u, 256u) - tensorLayoutSetDimension(fla, pa.d, pa.n) - tensorLayoutSetStride(fla, pa.n >> 8u, 1u) - var flb : tensorLayout2D - tensorLayoutCreate(flb) - tensorLayoutSetDimension(flb, row0 + ((cnt + 31u) & ~31u), pa.n) - tensorLayoutSetStride(flb, pa.n & ~7u, 1u) - var flo : tensorLayout2DPad - tensorLayoutCreate(flo) - tensorLayoutSetDimension(flo, row0 + cnt, pa.d) - tensorLayoutSetStride(flo, pa.d & ~7u, 1u) - if (pa.ksplit == 0u) { - var k = 0u - for (_i in range(int(pa.n / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k6) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 32u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < pa.n) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k6) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 32u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x32 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, 0u, flo, t0, 32u, m0, 128u, tv) - return - } - var k = k0 - for (_i in range(int((k1 - k0) / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k6) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 32u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < k1) { - coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode_k6) - coopmatLoadTensor(b, xf16, 0u, flb, t0, 32u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x32 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, ybase, flo, t0, 32u, m0, 128u, tv) - return - } - var tla : tensorLayout2DPad - tensorLayoutCreate(tla) - tensorLayoutSetBlockSize(tla, 1u, 256u) - tensorLayoutSetDimension(tla, pa.d, pa.n) - tensorLayoutSetStride(tla, pa.n / 256u, 1u) - var tlb : tensorLayout2DPad - tensorLayoutCreate(tlb) - tensorLayoutSetDimension(tlb, row0 + cnt, pa.n) - tensorLayoutSetStride(tlb, pa.n, 1u) - var tlo : tensorLayout2DPad - tensorLayoutCreate(tlo) - tensorLayoutSetDimension(tlo, row0 + cnt, pa.d) - tensorLayoutSetStride(tlo, pa.d, 1u) - if (pa.ksplit == 0u) { - var k = 0u - for (_i in range(int(pa.n / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k6) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 32u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < pa.n) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k6) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 32u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x32 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, 0u, tlo, t0, 32u, m0, 128u, tv) - return - } - var k = k0 - for (_i in range(int((k1 - k0) / 512u))) { - for [unroll] (_j in range(8)) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k6) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 32u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - } - while (k < k1) { - coopmatLoadTensorDecode(a, wq, wblk0, tla, m0, 128u, k, 64u, self.decode_k6) - coopmatLoadTensor(b, xf16, 0u, tlb, t0, 32u, k, 64u, tv) - acc = coopmatMulAdd(a, b, acc) - k += 64u - } - coopmatClamp(acc, -65504.0, 65504.0) - var accw : coopmatWgAcc_f32_128x32 - coopmatConvert(accw, acc) - coopmatStoreTensor(accw, y, ybase, tlo, t0, 32u, m0, 128u, tv) +[ |> template_struct_instance] +class template Iq4nlCm2T : KqCm2BatchT { + typedef BLK = VkK4Blk // q40's nibble plane verbatim - the k4 lane view fits + typedef ST = uint + override IQLUT = true + + [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] + def decode(blk : VkK4Blk; bc, cib : uint2) : float16 { + let g = cib.y >> 5u + let e = cib.y & 31u + let bidx = g * 16u + (e & 15u) + let by = uint(int(unpack8(blk.qs[int(bidx >> 1u)])[int(bidx & 1u)])) & 0xFFu + let q = (by >> ((e >> 4u) * 4u)) & 0xFu + let srow = (wg_blk0 + bc.x * (pa.n >> 8u) + bc.y) * 5u + let dp = unpackHalf2x16(ws[srow + (g >> 1u)]) + let d = (g & 1u) == 0u ? dp.x : dp.y + return float16(d * float(iq4lut[int(q)])) } } +[ |> template_struct_instance] +class template Iq2sCm2T : KqCm2BatchT { + typedef BLK = VkIq2sBlk + typedef ST = uint + override IQ2SGRID = true + + [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] + def decode(blk : VkIq2sBlk; bc, cib : uint2) : float16 { + let e = cib.y + let g = e >> 5u + let r = e & 31u + let l = r >> 3u + let j = r & 7u + let qi = g * 4u + l + let qb = uint(int(unpack8(blk.qs[int(qi >> 1u)])[int(qi & 1u)])) & 0xFFu + let qh = uint(int(unpack8(blk.qh[int(g >> 1u)])[int(g & 1u)])) & 0xFFu + let sgb = uint(int(unpack8(blk.sg[int(qi >> 1u)])[int(qi & 1u)])) & 0xFFu + let gw = iq2s_gridc[(qb | ((qh << (8u - 2u * l)) & 0x300u)) * 2u + (j >> 2u)] + let gb = float((gw >> ((j & 3u) * 8u)) & 0xFFu) + let srow = (wg_blk0 + bc.x * (pa.n >> 8u) + bc.y) * 5u + let d = unpackHalf2x16(ws[srow]).x + let si = g * 2u + (r >> 4u) + let sc = (ws[srow + 1u + (si >> 2u)] >> ((si & 3u) * 8u)) & 0xFFu + return float16(d * float(sc) * (((sgb >> j) & 1u) != 0u ? -gb : gb)) + } +} + +class template Iq2xsCm2T : KqCm2BatchT { + typedef BLK = VkIq2xsBlk + typedef ST = uint + override IQ2XSGRID = true + + [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] + def decode(blk : VkIq2xsBlk; bc, cib : uint2) : float16 { + let e = cib.y + let wi = e >> 3u + let j = e & 7u + let w16 = uint(int(blk.qs[int(wi)])) & 0xFFFFu + let sidx = w16 >> 9u + var tt = sidx ^ (sidx >> 4u) + tt = tt ^ (tt >> 2u) + tt = tt ^ (tt >> 1u) + let sgb = sidx | ((tt & 1u) << 7u) + let gw = iq2xs_gridc[(w16 & 511u) * 2u + (j >> 2u)] + let gb = float((gw >> ((j & 3u) * 8u)) & 0xFFu) + let srow = (wg_blk0 + bc.x * (pa.n >> 8u) + bc.y) * 5u + let dv = unpackHalf2x16(ws[srow]).x + let si = e >> 4u + let sc = (ws[srow + 1u + (si >> 2u)] >> ((si & 3u) * 8u)) & 0xFFu + return float16(dv * float(sc) * (((sgb >> j) & 1u) != 0u ? -gb : gb)) + } +} + +class template Iq2xxsCm2T : KqCm2BatchT { + typedef BLK = VkIq2xxsBlk + typedef ST = uint + override IQ2XXSGRID = true + + [spirv_decode, arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-decode-16bit-lanes")] + def decode(blk : VkIq2xxsBlk; bc, cib : uint2) : float16 { + let e = cib.y + let g = e >> 5u + let r = e & 31u + let l = r >> 3u + let j = r & 7u + let qi = g * 8u + l + let qb = uint(int(unpack8(blk.qs[int(qi >> 1u)])[int(qi & 1u)])) & 0xFFu + let bl = 7u * l + let a0i = g * 8u + 4u + (bl >> 3u) + let b0 = uint(int(unpack8(blk.qs[int(a0i >> 1u)])[int(a0i & 1u)])) & 0xFFu + let b1 = uint(int(unpack8(blk.qs[int((a0i + 1u) >> 1u)])[int((a0i + 1u) & 1u)])) & 0xFFu + let sidx = ((b0 | (b1 << 8u)) >> (bl & 7u)) & 127u + var tt = sidx ^ (sidx >> 4u) + tt = tt ^ (tt >> 2u) + tt = tt ^ (tt >> 1u) + let sgb = sidx | ((tt & 1u) << 7u) + let gw = iq2xxs_gridc[qb * 2u + (j >> 2u)] + let gb = float((gw >> ((j & 3u) * 8u)) & 0xFFu) + let srow = (wg_blk0 + bc.x * (pa.n >> 8u) + bc.y) * 5u + let dv = unpackHalf2x16(ws[srow]).x + let sc = int(ws[srow + 1u + (g >> 2u)] << ((3u - (g & 3u)) * 8u)) >> 24 + return float16(dv * float(sc) * (((sgb >> j) & 1u) != 0u ? -gb : gb)) + } +} + +// the stamps: the three width columns of each format (the dispatcher keys on (fmt, tc)) +[vk_dispatch(name = "q8_batch_cm2l_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Q8Cm2LBatch : Q8Cm2T { + typedef BT = coopmatWgB_f16_64x256 + typedef ACC = coopmatWgAcc_f16_128x256 + typedef ACCW = coopmatWgAcc_f32_128x256 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "q8_batch_cm2m_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Q8Cm2MBatch : Q8Cm2T { + override BN = 128u + typedef BT = coopmatWgB_f16_64x128 + typedef ACC = coopmatWgAcc_f16_128x128 + typedef ACCW = coopmatWgAcc_f32_128x128 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "q8_batch_cm2s_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Q8Cm2SBatch : Q8Cm2T { + override BN = 32u + override STILE = true + typedef BT = coopmatWgB_f16_64x32 + typedef ACC = coopmatWgAcc_f16_128x32 + typedef ACCW = coopmatWgAcc_f32_128x32 + typedef FLO = tensorLayout2DPad +} + +[vk_dispatch(name = "kq_batch_k4_cm2l_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class K4Cm2LBatch : K4Cm2T { + typedef BT = coopmatWgB_f16_64x256 + typedef ACC = coopmatWgAcc_f16_128x256 + typedef ACCW = coopmatWgAcc_f32_128x256 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_k4_cm2m_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class K4Cm2MBatch : K4Cm2T { + override BN = 128u + typedef BT = coopmatWgB_f16_64x128 + typedef ACC = coopmatWgAcc_f16_128x128 + typedef ACCW = coopmatWgAcc_f32_128x128 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_k4_cm2s_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class K4Cm2SBatch : K4Cm2T { + override BN = 32u + override STILE = true + typedef BT = coopmatWgB_f16_64x32 + typedef ACC = coopmatWgAcc_f16_128x32 + typedef ACCW = coopmatWgAcc_f32_128x32 + typedef FLO = tensorLayout2DPad +} + +[vk_dispatch(name = "kq_batch_k6_cm2l_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class K6Cm2LBatch : K6Cm2T { + typedef BT = coopmatWgB_f16_64x256 + typedef ACC = coopmatWgAcc_f16_128x256 + typedef ACCW = coopmatWgAcc_f32_128x256 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_k6_cm2m_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class K6Cm2MBatch : K6Cm2T { + override BN = 128u + typedef BT = coopmatWgB_f16_64x128 + typedef ACC = coopmatWgAcc_f16_128x128 + typedef ACCW = coopmatWgAcc_f32_128x128 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_k6_cm2s_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class K6Cm2SBatch : K6Cm2T { + override BN = 32u + override STILE = true + typedef BT = coopmatWgB_f16_64x32 + typedef ACC = coopmatWgAcc_f16_128x32 + typedef ACCW = coopmatWgAcc_f32_128x32 + typedef FLO = tensorLayout2DPad +} + +[vk_dispatch(name = "kq_batch_k5_cm2l_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class K5Cm2LBatch : K5Cm2T { + typedef BT = coopmatWgB_f16_64x256 + typedef ACC = coopmatWgAcc_f16_128x256 + typedef ACCW = coopmatWgAcc_f32_128x256 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_k5_cm2m_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class K5Cm2MBatch : K5Cm2T { + override BN = 128u + typedef BT = coopmatWgB_f16_64x128 + typedef ACC = coopmatWgAcc_f16_128x128 + typedef ACCW = coopmatWgAcc_f32_128x128 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_k5_cm2s_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class K5Cm2SBatch : K5Cm2T { + override BN = 32u + override STILE = true + typedef BT = coopmatWgB_f16_64x32 + typedef ACC = coopmatWgAcc_f16_128x32 + typedef ACCW = coopmatWgAcc_f32_128x32 + typedef FLO = tensorLayout2DPad +} + +[vk_dispatch(name = "kq_batch_q40_cm2l_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Q40Cm2LBatch : Q40Cm2T { + typedef BT = coopmatWgB_f16_64x256 + typedef ACC = coopmatWgAcc_f16_128x256 + typedef ACCW = coopmatWgAcc_f32_128x256 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_q40_cm2m_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Q40Cm2MBatch : Q40Cm2T { + override BN = 128u + typedef BT = coopmatWgB_f16_64x128 + typedef ACC = coopmatWgAcc_f16_128x128 + typedef ACCW = coopmatWgAcc_f32_128x128 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_q40_cm2s_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Q40Cm2SBatch : Q40Cm2T { + override BN = 32u + override STILE = true + typedef BT = coopmatWgB_f16_64x32 + typedef ACC = coopmatWgAcc_f16_128x32 + typedef ACCW = coopmatWgAcc_f32_128x32 + typedef FLO = tensorLayout2DPad +} + +[vk_dispatch(name = "kq_batch_iq4xs_cm2l_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Iq4xsCm2LBatch : Iq4xsCm2T { + typedef BT = coopmatWgB_f16_64x256 + typedef ACC = coopmatWgAcc_f16_128x256 + typedef ACCW = coopmatWgAcc_f32_128x256 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_iq4xs_cm2m_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Iq4xsCm2MBatch : Iq4xsCm2T { + override BN = 128u + typedef BT = coopmatWgB_f16_64x128 + typedef ACC = coopmatWgAcc_f16_128x128 + typedef ACCW = coopmatWgAcc_f32_128x128 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_iq4xs_cm2s_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Iq4xsCm2SBatch : Iq4xsCm2T { + override BN = 32u + override STILE = true + typedef BT = coopmatWgB_f16_64x32 + typedef ACC = coopmatWgAcc_f16_128x32 + typedef ACCW = coopmatWgAcc_f32_128x32 + typedef FLO = tensorLayout2DPad +} + +[vk_dispatch(name = "kq_batch_k3_cm2l_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class K3Cm2LBatch : K3Cm2T { + typedef BT = coopmatWgB_f16_64x256 + typedef ACC = coopmatWgAcc_f16_128x256 + typedef ACCW = coopmatWgAcc_f32_128x256 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_k3_cm2m_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class K3Cm2MBatch : K3Cm2T { + override BN = 128u + typedef BT = coopmatWgB_f16_64x128 + typedef ACC = coopmatWgAcc_f16_128x128 + typedef ACCW = coopmatWgAcc_f32_128x128 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_k3_cm2s_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class K3Cm2SBatch : K3Cm2T { + override BN = 32u + override STILE = true + typedef BT = coopmatWgB_f16_64x32 + typedef ACC = coopmatWgAcc_f16_128x32 + typedef ACCW = coopmatWgAcc_f32_128x32 + typedef FLO = tensorLayout2DPad +} + +[vk_dispatch(name = "kq_batch_iq3s_cm2l_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Iq3sCm2LBatch : Iq3sCm2T { + typedef BT = coopmatWgB_f16_64x256 + typedef ACC = coopmatWgAcc_f16_128x256 + typedef ACCW = coopmatWgAcc_f32_128x256 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_iq3s_cm2m_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Iq3sCm2MBatch : Iq3sCm2T { + override BN = 128u + typedef BT = coopmatWgB_f16_64x128 + typedef ACC = coopmatWgAcc_f16_128x128 + typedef ACCW = coopmatWgAcc_f32_128x128 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_iq3s_cm2s_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Iq3sCm2SBatch : Iq3sCm2T { + override BN = 32u + override STILE = true + typedef BT = coopmatWgB_f16_64x32 + typedef ACC = coopmatWgAcc_f16_128x32 + typedef ACCW = coopmatWgAcc_f32_128x32 + typedef FLO = tensorLayout2DPad +} + +[vk_dispatch(name = "kq_batch_iq3xxs_cm2l_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Iq3xxsCm2LBatch : Iq3xxsCm2T { + typedef BT = coopmatWgB_f16_64x256 + typedef ACC = coopmatWgAcc_f16_128x256 + typedef ACCW = coopmatWgAcc_f32_128x256 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_iq3xxs_cm2m_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Iq3xxsCm2MBatch : Iq3xxsCm2T { + override BN = 128u + typedef BT = coopmatWgB_f16_64x128 + typedef ACC = coopmatWgAcc_f16_128x128 + typedef ACCW = coopmatWgAcc_f32_128x128 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_iq3xxs_cm2s_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Iq3xxsCm2SBatch : Iq3xxsCm2T { + override BN = 32u + override STILE = true + typedef BT = coopmatWgB_f16_64x32 + typedef ACC = coopmatWgAcc_f16_128x32 + typedef ACCW = coopmatWgAcc_f32_128x32 + typedef FLO = tensorLayout2DPad +} + +[vk_dispatch(name = "kq_batch_iq4nl_cm2l_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Iq4nlCm2LBatch : Iq4nlCm2T { + typedef BT = coopmatWgB_f16_64x256 + typedef ACC = coopmatWgAcc_f16_128x256 + typedef ACCW = coopmatWgAcc_f32_128x256 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_iq4nl_cm2m_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Iq4nlCm2MBatch : Iq4nlCm2T { + override BN = 128u + typedef BT = coopmatWgB_f16_64x128 + typedef ACC = coopmatWgAcc_f16_128x128 + typedef ACCW = coopmatWgAcc_f32_128x128 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_iq4nl_cm2s_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Iq4nlCm2SBatch : Iq4nlCm2T { + override BN = 32u + override STILE = true + typedef BT = coopmatWgB_f16_64x32 + typedef ACC = coopmatWgAcc_f16_128x32 + typedef ACCW = coopmatWgAcc_f32_128x32 + typedef FLO = tensorLayout2DPad +} + +[vk_dispatch(name = "kq_batch_k2_cm2l_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class K2Cm2LBatch : K2Cm2T { + typedef BT = coopmatWgB_f16_64x256 + typedef ACC = coopmatWgAcc_f16_128x256 + typedef ACCW = coopmatWgAcc_f32_128x256 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_k2_cm2m_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class K2Cm2MBatch : K2Cm2T { + override BN = 128u + typedef BT = coopmatWgB_f16_64x128 + typedef ACC = coopmatWgAcc_f16_128x128 + typedef ACCW = coopmatWgAcc_f32_128x128 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_k2_cm2s_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class K2Cm2SBatch : K2Cm2T { + override BN = 32u + override STILE = true + typedef BT = coopmatWgB_f16_64x32 + typedef ACC = coopmatWgAcc_f16_128x32 + typedef ACCW = coopmatWgAcc_f32_128x32 + typedef FLO = tensorLayout2DPad +} + +[vk_dispatch(name = "kq_batch_iq2s_cm2l_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Iq2sCm2LBatch : Iq2sCm2T { + typedef BT = coopmatWgB_f16_64x256 + typedef ACC = coopmatWgAcc_f16_128x256 + typedef ACCW = coopmatWgAcc_f32_128x256 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_iq2s_cm2m_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Iq2sCm2MBatch : Iq2sCm2T { + override BN = 128u + typedef BT = coopmatWgB_f16_64x128 + typedef ACC = coopmatWgAcc_f16_128x128 + typedef ACCW = coopmatWgAcc_f32_128x128 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_iq2s_cm2s_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Iq2sCm2SBatch : Iq2sCm2T { + override BN = 32u + override STILE = true + typedef BT = coopmatWgB_f16_64x32 + typedef ACC = coopmatWgAcc_f16_128x32 + typedef ACCW = coopmatWgAcc_f32_128x32 + typedef FLO = tensorLayout2DPad +} + +[vk_dispatch(name = "kq_batch_iq2xs_cm2l_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Iq2xsCm2LBatch : Iq2xsCm2T { + typedef BT = coopmatWgB_f16_64x256 + typedef ACC = coopmatWgAcc_f16_128x256 + typedef ACCW = coopmatWgAcc_f32_128x256 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_iq2xs_cm2m_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Iq2xsCm2MBatch : Iq2xsCm2T { + override BN = 128u + typedef BT = coopmatWgB_f16_64x128 + typedef ACC = coopmatWgAcc_f16_128x128 + typedef ACCW = coopmatWgAcc_f32_128x128 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_iq2xs_cm2s_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Iq2xsCm2SBatch : Iq2xsCm2T { + override BN = 32u + override STILE = true + typedef BT = coopmatWgB_f16_64x32 + typedef ACC = coopmatWgAcc_f16_128x32 + typedef ACCW = coopmatWgAcc_f32_128x32 + typedef FLO = tensorLayout2DPad +} + +[vk_dispatch(name = "kq_batch_iq2xxs_cm2l_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Iq2xxsCm2LBatch : Iq2xxsCm2T { + typedef BT = coopmatWgB_f16_64x256 + typedef ACC = coopmatWgAcc_f16_128x256 + typedef ACCW = coopmatWgAcc_f32_128x256 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_iq2xxs_cm2m_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Iq2xxsCm2MBatch : Iq2xxsCm2T { + override BN = 128u + typedef BT = coopmatWgB_f16_64x128 + typedef ACC = coopmatWgAcc_f16_128x128 + typedef ACCW = coopmatWgAcc_f32_128x128 + typedef FLO = tensorLayout2D +} + +[vk_dispatch(name = "kq_batch_iq2xxs_cm2s_cls", kernel = "run", grid = "wgs", params = "wgs : int64")] +class Iq2xxsCm2SBatch : Iq2xxsCm2T { + override BN = 32u + override STILE = true + typedef BT = coopmatWgB_f16_64x32 + typedef ACC = coopmatWgAcc_f16_128x32 + typedef ACCW = coopmatWgAcc_f32_128x32 + typedef FLO = tensorLayout2DPad +} + // ===== the split-k reduce (sums the cm2 partial planes into y) ===== struct SkRedArgs { @@ -5576,7 +6127,7 @@ let private KQC_ONES = 0x01010101u // sdot4 against this sums 4 signed bytes class KqGemvBase { @ssbo @binding = 0 @role = "alias" wq : array // weight quant words (k5's qh rides this view) @ssbo @binding = 0 wq4 : array // vec4 view of the same words - @ssbo @binding = 1 wsu : array // scale planes ([f16 d][f16 dmin][8 sc][8 mn] / q40 half pairs / k6 int8+d) + @ssbo @binding = 1 wsu : array // scale planes ([f16 d][f16 dmin][8 sc][8 mn] / q40 half pairs / k6 int8+d / iq4xs d + 8 signed sc) @ssbo @binding = 2 regions : array // (rel_wblock, xblock) pairs per region — bulk data, stays SSBO @ssbo @binding = 3 xq4 : array // activation quant words, vec4 view @ssbo @binding = 4 axs : array // activation block scales @@ -5725,6 +6276,24 @@ def kq_batch_cls_ensure(fmt : int; q40cm : bool) : bool { return ensure_kq_batch_k6_cls() } elif (fmt == int(KqFmt.q40)) { return ensure_kq_batch_q40_cls() + } elif (fmt == int(KqFmt.iq4xs)) { + return ensure_kq_batch_iq4xs_cls() + } elif (fmt == int(KqFmt.k3)) { + return ensure_kq_batch_k3_cls() + } elif (fmt == int(KqFmt.iq3s)) { + return ensure_kq_batch_iq3s_cls() + } elif (fmt == int(KqFmt.iq3xxs)) { + return ensure_kq_batch_iq3xxs_cls() + } elif (fmt == int(KqFmt.iq4nl)) { + return ensure_kq_batch_iq4nl_cls() + } elif (fmt == int(KqFmt.k2)) { + return ensure_kq_batch_k2_cls() + } elif (fmt == int(KqFmt.iq2s)) { + return ensure_kq_batch_iq2s_cls() + } elif (fmt == int(KqFmt.iq2xs)) { + return ensure_kq_batch_iq2xs_cls() + } elif (fmt == int(KqFmt.iq2xxs)) { + return ensure_kq_batch_iq2xxs_cls() } panic("dasLLAMA vulkan tier: no kq batch class for format {fmt}") return false @@ -5751,6 +6320,24 @@ def kq_batch_cls_enc_for(fmt : int; q40cm : bool; raw : VkCommandBuffer; var h : enc_kq_batch_k6_cls(raw, h, set_, pc, wgs) } elif (fmt == int(KqFmt.q40)) { enc_kq_batch_q40_cls(raw, h, set_, pc, wgs) + } elif (fmt == int(KqFmt.iq4xs)) { + enc_kq_batch_iq4xs_cls(raw, h, set_, pc, wgs) + } elif (fmt == int(KqFmt.k3)) { + enc_kq_batch_k3_cls(raw, h, set_, pc, wgs) + } elif (fmt == int(KqFmt.iq3s)) { + enc_kq_batch_iq3s_cls(raw, h, set_, pc, wgs) + } elif (fmt == int(KqFmt.iq3xxs)) { + enc_kq_batch_iq3xxs_cls(raw, h, set_, pc, wgs) + } elif (fmt == int(KqFmt.iq4nl)) { + enc_kq_batch_iq4nl_cls(raw, h, set_, pc, wgs) + } elif (fmt == int(KqFmt.k2)) { + enc_kq_batch_k2_cls(raw, h, set_, pc, wgs) + } elif (fmt == int(KqFmt.iq2s)) { + enc_kq_batch_iq2s_cls(raw, h, set_, pc, wgs) + } elif (fmt == int(KqFmt.iq2xs)) { + enc_kq_batch_iq2xs_cls(raw, h, set_, pc, wgs) + } elif (fmt == int(KqFmt.iq2xxs)) { + enc_kq_batch_iq2xxs_cls(raw, h, set_, pc, wgs) } else { panic("dasLLAMA vulkan tier: no kq batch class for format {fmt}") } @@ -5771,6 +6358,24 @@ def gemv_cls_ensure(fmt : int) : bool { return ensure_kq_gemv_k6_cls() } elif (fmt == int(KqFmt.q40)) { return ensure_kq_gemv_q40_cls() + } elif (fmt == int(KqFmt.iq4xs)) { + return ensure_kq_gemv_iq4xs_cls() + } elif (fmt == int(KqFmt.k3)) { + return ensure_kq_gemv_k3_cls() + } elif (fmt == int(KqFmt.iq3s)) { + return ensure_kq_gemv_iq3s_cls() + } elif (fmt == int(KqFmt.iq3xxs)) { + return ensure_kq_gemv_iq3xxs_cls() + } elif (fmt == int(KqFmt.iq4nl)) { + return ensure_kq_gemv_iq4nl_cls() + } elif (fmt == int(KqFmt.k2)) { + return ensure_kq_gemv_k2_cls() + } elif (fmt == int(KqFmt.iq2s)) { + return ensure_kq_gemv_iq2s_cls() + } elif (fmt == int(KqFmt.iq2xs)) { + return ensure_kq_gemv_iq2xs_cls() + } elif (fmt == int(KqFmt.iq2xxs)) { + return ensure_kq_gemv_iq2xxs_cls() } panic("dasLLAMA vulkan tier: no GEMV class for format {fmt}") return false @@ -5779,8 +6384,8 @@ def gemv_cls_ensure(fmt : int) : bool { def gemv_cls_set(fmt : int; bufs : uint64 const[6]; sizes : int64 const[6]; gbits : uint const[6]) : VkDescriptorSet { if (fmt == int(KqFmt.q8)) { return set_q8_gemv_cls(bufs, sizes, gbits) - } elif (fmt == int(KqFmt.k4) || fmt == int(KqFmt.k5) || fmt == int(KqFmt.k6) || fmt == int(KqFmt.q40)) { - return set_kq_gemv_cls(bufs, sizes, gbits) // the family shares one layout — one set serves all four + } elif (kq_sb(fmt)) { + return set_kq_gemv_cls(bufs, sizes, gbits) // the family shares one layout — one set serves every kq format } panic("dasLLAMA vulkan tier: no GEMV class for format {fmt}") return VkDescriptorSet() @@ -5797,14 +6402,32 @@ def gemv_cls_enc(fmt : int; raw : VkCommandBuffer; var h : VkHaz; var set_ : VkD enc_kq_gemv_k6_cls(raw, h, set_, pc, wgs) } elif (fmt == int(KqFmt.q40)) { enc_kq_gemv_q40_cls(raw, h, set_, pc, wgs) + } elif (fmt == int(KqFmt.iq4xs)) { + enc_kq_gemv_iq4xs_cls(raw, h, set_, pc, wgs) + } elif (fmt == int(KqFmt.k3)) { + enc_kq_gemv_k3_cls(raw, h, set_, pc, wgs) + } elif (fmt == int(KqFmt.iq3s)) { + enc_kq_gemv_iq3s_cls(raw, h, set_, pc, wgs) + } elif (fmt == int(KqFmt.iq3xxs)) { + enc_kq_gemv_iq3xxs_cls(raw, h, set_, pc, wgs) + } elif (fmt == int(KqFmt.iq4nl)) { + enc_kq_gemv_iq4nl_cls(raw, h, set_, pc, wgs) + } elif (fmt == int(KqFmt.k2)) { + enc_kq_gemv_k2_cls(raw, h, set_, pc, wgs) + } elif (fmt == int(KqFmt.iq2s)) { + enc_kq_gemv_iq2s_cls(raw, h, set_, pc, wgs) + } elif (fmt == int(KqFmt.iq2xs)) { + enc_kq_gemv_iq2xs_cls(raw, h, set_, pc, wgs) + } elif (fmt == int(KqFmt.iq2xxs)) { + enc_kq_gemv_iq2xxs_cls(raw, h, set_, pc, wgs) } else { panic("dasLLAMA vulkan tier: no GEMV class for format {fmt}") } } // Q4_K: w = d*sc*q - dmin*mn*1, q in [0,15] — per 32-block axs*(d*sc*idot - dmin*mn*bsum). -// The four formats share family=kq_gemv_cls: ONE set layout + ONE set_kq_gemv_cls serves all -// four pipelines (identical inherited members), each leaf keeping its own ensure_/enc_ names. +// Every kq format shares family=kq_gemv_cls: ONE set layout + ONE set_kq_gemv_cls serves every +// pipeline (identical inherited members), each leaf keeping its own ensure_/enc_ names. [vk_dispatch(name = "kq_gemv_k4_cls", kernel = "run", family = "kq_gemv_cls", grid = "wgs", params = "wgs : int64")] class KqGemvK4 : KqGemvBase { def override blk_contrib(wsb, blk, xsb : uint) : float { @@ -5843,6 +6466,241 @@ class KqGemvQ40 : KqGemvBase { } } +// IQ4_XS: w = d*sc*LUT[q] — the q40 nibble tiling decoded through the codebook into signed +// lanes (no block-sum term), the k4 scale-row shape with SIGNED (ls - 32) sub-scales and no min +[vk_dispatch(name = "kq_gemv_iq4xs_cls", kernel = "run", family = "kq_gemv_cls", grid = "wgs", params = "wgs : int64")] +class KqGemvIq4xs : KqGemvBase { + def iq4_dot(wv, xl, xh : uint4) : int { + var idot = sdot4(iq4_word(wv.x & KQC_LOW), xl.x) + sdot4(iq4_word(wv.y & KQC_LOW), xl.y) + idot += sdot4(iq4_word(wv.z & KQC_LOW), xl.z) + sdot4(iq4_word(wv.w & KQC_LOW), xl.w) + idot += sdot4(iq4_word((wv.x >> 4u) & KQC_LOW), xh.x) + sdot4(iq4_word((wv.y >> 4u) & KQC_LOW), xh.y) + return idot + sdot4(iq4_word((wv.z >> 4u) & KQC_LOW), xh.z) + sdot4(iq4_word((wv.w >> 4u) & KQC_LOW), xh.w) + } + + def override blk_contrib(wsb, blk, xsb : uint) : float { + let xb4 = xsb * 16u + blk * 2u + let idot = iq4_dot(wq4[wsb * 8u + blk], xq4[xb4], xq4[xb4 + 1u]) + let dd = unpackHalf2x16(wsu[wsb * 5u]).x + return axs[xsb] * dd * float(iq4_sc(wsu[wsb * 5u + 1u + blk / 4u], blk) * idot) + } + + [spirv_kernel(local_size_x = 64, name = "kq_gemv_iq4xs_cls_spv")] + def run { + gemv_shell() + } +} + +// IQ3_S: grid gather (8-bit index | qh ninth bit -> 4 magnitudes per word), the block's own +// sign bytes flip them, per-32 signed (1 + 2s) sub-scale x d - iq4xs's fold (no bsum term) +// over the 26-word rows; the 2 KB grid stages into workgroup memory ahead of the shell. +[vk_dispatch(name = "kq_gemv_iq3s_cls", kernel = "run", family = "kq_gemv_cls", grid = "wgs", params = "wgs : int64")] +class KqGemvIq3s : KqGemvBase { + @workgroup gridw : uint[512] + + def iq3s_signed(g, nib : uint) : uint { + let m1 = (nib * 0x00204081u) & 0x01010101u + let mask = m1 * 255u + return (g ^ mask) + m1 + } + + def override blk_contrib(wsb, blk, xsb : uint) : float { + let qb = wsb * 26u + blk * 2u + let qh = (wq[wsb * 26u + 16u + blk / 4u] >> ((blk % 4u) * 8u)) & 255u + let sgw = wq[wsb * 26u + 18u + blk] + let xb4 = xsb * 16u + blk * 2u + let xl = xq4[xb4] + let xh = xq4[xb4 + 1u] + var isum = 0 + for (l in range(2)) { + let qsw = wq[qb + uint(l)] + let l0 = uint(l) * 2u + let sg0 = (sgw >> (l0 * 8u)) & 255u + let sg1 = (sgw >> (l0 * 8u + 8u)) & 255u + let w0 = iq3s_signed(gridw[(qsw & 255u) | ((qh << (8u - 2u * l0)) & 256u)], sg0 & 15u) + let w1 = iq3s_signed(gridw[((qsw >> 8u) & 255u) | ((qh << (7u - 2u * l0)) & 256u)], sg0 >> 4u) + let w2 = iq3s_signed(gridw[((qsw >> 16u) & 255u) | ((qh << (8u - 2u * (l0 + 1u))) & 256u)], sg1 & 15u) + let w3 = iq3s_signed(gridw[(qsw >> 24u) | ((qh << (7u - 2u * (l0 + 1u))) & 256u)], sg1 >> 4u) + if (l == 0) { + isum += sdot4(w0, xl.x) + sdot4(w1, xl.y) + sdot4(w2, xl.z) + sdot4(w3, xl.w) + } else { + isum += sdot4(w0, xh.x) + sdot4(w1, xh.y) + sdot4(w2, xh.z) + sdot4(w3, xh.w) + } + } + let dd = unpackHalf2x16(wsu[wsb * 5u]).x + return axs[xsb] * dd * float(iq4_sc(wsu[wsb * 5u + 1u + blk / 4u], blk) * isum) + } + + [spirv_kernel(local_size_x = 64, name = "kq_gemv_iq3s_cls_spv")] + def run { + let lid = gl_LocalInvocationID.x + for (k in range(8)) { + let idx = lid + uint(k) * 64u + gridw[idx] = iq3s_grid_word(int(idx)) + } + barrier() // the grid visible before the shell's block walk + gemv_shell() + } +} + +// IQ3_XXS: halved-grid gather (bare 8-bit index -> 4 magnitudes per word), signs from the +// aux32's four 7-bit ksigns indices (bit 7 = parity, computed - no table), per-32 (2ls + 1) +// sub-scale x halved d - iq3s's fold exactly, over the 24-word rows; the 1 KB grid stages +// into workgroup memory ahead of the shell. +[vk_dispatch(name = "kq_gemv_iq3xxs_cls", kernel = "run", family = "kq_gemv_cls", grid = "wgs", params = "wgs : int64")] +class KqGemvIq3xxs : KqGemvBase { + @workgroup gridw : uint[256] + + def iq3s_signed(g, nib : uint) : uint { + let m1 = (nib * 0x00204081u) & 0x01010101u + let mask = m1 * 255u + return (g ^ mask) + m1 + } + + def override blk_contrib(wsb, blk, xsb : uint) : float { + let qb = wsb * 24u + blk * 2u + let aux = wq[wsb * 24u + 16u + blk] + let xb4 = xsb * 16u + blk * 2u + let xl = xq4[xb4] + let xh = xq4[xb4 + 1u] + var isum = 0 + for (l in range(2)) { + let qsw = wq[qb + uint(l)] + let l0 = uint(l) * 2u + let sg0 = ksign7((aux >> (7u * l0)) & 127u) + let sg1 = ksign7((aux >> (7u * l0 + 7u)) & 127u) + let w0 = iq3s_signed(gridw[qsw & 255u], sg0 & 15u) + let w1 = iq3s_signed(gridw[(qsw >> 8u) & 255u], sg0 >> 4u) + let w2 = iq3s_signed(gridw[(qsw >> 16u) & 255u], sg1 & 15u) + let w3 = iq3s_signed(gridw[qsw >> 24u], sg1 >> 4u) + if (l == 0) { + isum += sdot4(w0, xl.x) + sdot4(w1, xl.y) + sdot4(w2, xl.z) + sdot4(w3, xl.w) + } else { + isum += sdot4(w0, xh.x) + sdot4(w1, xh.y) + sdot4(w2, xh.z) + sdot4(w3, xh.w) + } + } + let dd = unpackHalf2x16(wsu[wsb * 5u]).x + return axs[xsb] * dd * float(iq4_sc(wsu[wsb * 5u + 1u + blk / 4u], blk) * isum) + } + + [spirv_kernel(local_size_x = 64, name = "kq_gemv_iq3xxs_cls_spv")] + def run { + let lid = gl_LocalInvocationID.x + for (k in range(4)) { + let idx = lid + uint(k) * 64u + gridw[idx] = iq3xxs_grid_word(int(idx)) + } + barrier() // the grid visible before the shell's block walk + gemv_shell() + } +} + +// IQ4_NL: iq4xs's codebook dot (iq4_word, pure ALU) folded by q40's per-block f16 d - no +// sub-scales, no bsum term; the rows are q40's 32-word planes verbatim. +[vk_dispatch(name = "kq_gemv_iq4nl_cls", kernel = "run", family = "kq_gemv_cls", grid = "wgs", params = "wgs : int64")] +class KqGemvIq4nl : KqGemvBase { + def iq4_dot(wv, xl, xh : uint4) : int { + var idot = sdot4(iq4_word(wv.x & KQC_LOW), xl.x) + sdot4(iq4_word(wv.y & KQC_LOW), xl.y) + idot += sdot4(iq4_word(wv.z & KQC_LOW), xl.z) + sdot4(iq4_word(wv.w & KQC_LOW), xl.w) + idot += sdot4(iq4_word((wv.x >> 4u) & KQC_LOW), xh.x) + sdot4(iq4_word((wv.y >> 4u) & KQC_LOW), xh.y) + return idot + sdot4(iq4_word((wv.z >> 4u) & KQC_LOW), xh.z) + sdot4(iq4_word((wv.w >> 4u) & KQC_LOW), xh.w) + } + + def q40_d(wsb, blk : uint) : float { + let dp = unpackHalf2x16(wsu[wsb * 5u + blk / 2u]) + return blk % 2u == 0u ? dp.x : dp.y + } + + def override blk_contrib(wsb, blk, xsb : uint) : float { + let xb4 = xsb * 16u + blk * 2u + let idot = iq4_dot(wq4[wsb * 8u + blk], xq4[xb4], xq4[xb4 + 1u]) + return axs[xsb] * q40_d(wsb, blk) * float(idot) + } + + [spirv_kernel(local_size_x = 64, name = "kq_gemv_iq4nl_cls_spv")] + def run { + gemv_shell() + } +} + +// Q3_K: q = 2-bit qs lane | hmask bit << 2, minus 4, per-16 SIGNED sub-scales x d — k6's fold with +// the offset 4 over the k3 planes (6 uint4 per superblock: qs 0..3, hmask 4..5) +[vk_dispatch(name = "kq_gemv_k3_cls", kernel = "run", family = "kq_gemv_cls", grid = "wgs", params = "wgs : int64")] +class KqGemvK3 : KqGemvBase { + // (ilo, ihi, blo, bhi) — an int4, since a shader function cannot return a tuple + def k3_dot(ql, qh, xl, xh, hl, hh : uint4; sh, bit : uint) : int4 { + let m2 = 0x03030303u + let m1 = 0x01010101u + let l0 = ((ql.x >> sh) & m2) | (((hl.x >> bit) & m1) << 2u) + let l1 = ((ql.y >> sh) & m2) | (((hl.y >> bit) & m1) << 2u) + let l2 = ((ql.z >> sh) & m2) | (((hl.z >> bit) & m1) << 2u) + let l3 = ((ql.w >> sh) & m2) | (((hl.w >> bit) & m1) << 2u) + let h0 = ((qh.x >> sh) & m2) | (((hh.x >> bit) & m1) << 2u) + let h1 = ((qh.y >> sh) & m2) | (((hh.y >> bit) & m1) << 2u) + let h2 = ((qh.z >> sh) & m2) | (((hh.z >> bit) & m1) << 2u) + let h3 = ((qh.w >> sh) & m2) | (((hh.w >> bit) & m1) << 2u) + return int4(sdot4(l0, xl.x) + sdot4(l1, xl.y) + sdot4(l2, xl.z) + sdot4(l3, xl.w), + sdot4(h0, xh.x) + sdot4(h1, xh.y) + sdot4(h2, xh.z) + sdot4(h3, xh.w), + sdot4(xl.x, KQC_ONES) + sdot4(xl.y, KQC_ONES) + sdot4(xl.z, KQC_ONES) + sdot4(xl.w, KQC_ONES), + sdot4(xh.x, KQC_ONES) + sdot4(xh.y, KQC_ONES) + sdot4(xh.z, KQC_ONES) + sdot4(xh.w, KQC_ONES)) + } + + def override blk_contrib(wsb, blk, xsb : uint) : float { + let xb4 = xsb * 16u + blk * 2u + let q4 = wsb * 6u + (blk / 4u) * 2u + let dot = k3_dot(wq4[q4], wq4[q4 + 1u], xq4[xb4], xq4[xb4 + 1u], + wq4[wsb * 6u + 4u], wq4[wsb * 6u + 5u], (blk % 4u) * 2u, blk) + let s4 = int4(unpack8(int(wsu[wsb * 5u + blk / 2u]))) // 4 signed sub-scales + let slo = (blk % 2u) == 0u ? s4.x : s4.z + let shi = (blk % 2u) == 0u ? s4.y : s4.w + let dd = unpackHalf2x16(wsu[wsb * 5u + 4u]).x + return axs[xsb] * dd * float(slo * (dot.x - 4 * dot.z) + shi * (dot.y - 4 * dot.w)) + } + + [spirv_kernel(local_size_x = 64, name = "kq_gemv_k3_cls_spv")] + def run { + gemv_shell() + } +} + +// Q2_K: q = the unsigned 2-bit qs lane, per-16 sc/min NIBBLE pairs x d/dmin - the k4 d|dmin +// fold at k6's granularity over the 16-word rows (4 uint4 of qs; scale row [d|dmin][16 pair bytes]) +[vk_dispatch(name = "kq_gemv_k2_cls", kernel = "run", family = "kq_gemv_cls", grid = "wgs", params = "wgs : int64")] +class KqGemvK2 : KqGemvBase { + // (ilo, ihi, blo, bhi) - an int4, since a shader function cannot return a tuple + def k2_dot(ql, qh, xl, xh : uint4; sh : uint) : int4 { + let m2 = 0x03030303u + let l0 = (ql.x >> sh) & m2 + let l1 = (ql.y >> sh) & m2 + let l2 = (ql.z >> sh) & m2 + let l3 = (ql.w >> sh) & m2 + let h0 = (qh.x >> sh) & m2 + let h1 = (qh.y >> sh) & m2 + let h2 = (qh.z >> sh) & m2 + let h3 = (qh.w >> sh) & m2 + return int4(sdot4(l0, xl.x) + sdot4(l1, xl.y) + sdot4(l2, xl.z) + sdot4(l3, xl.w), + sdot4(h0, xh.x) + sdot4(h1, xh.y) + sdot4(h2, xh.z) + sdot4(h3, xh.w), + sdot4(xl.x, KQC_ONES) + sdot4(xl.y, KQC_ONES) + sdot4(xl.z, KQC_ONES) + sdot4(xl.w, KQC_ONES), + sdot4(xh.x, KQC_ONES) + sdot4(xh.y, KQC_ONES) + sdot4(xh.z, KQC_ONES) + sdot4(xh.w, KQC_ONES)) + } + + def override blk_contrib(wsb, blk, xsb : uint) : float { + let xb4 = xsb * 16u + blk * 2u + let q4 = wsb * 4u + (blk / 4u) * 2u + let dot = k2_dot(wq4[q4], wq4[q4 + 1u], xq4[xb4], xq4[xb4 + 1u], (blk % 4u) * 2u) + let w = wsu[wsb * 5u + 1u + blk / 2u] + let b0 = (w >> ((blk % 2u) * 16u)) & 255u + let b1 = (w >> ((blk % 2u) * 16u + 8u)) & 255u + let dm = unpackHalf2x16(wsu[wsb * 5u]) + return axs[xsb] * (dm.x * float(int(b0 & 15u) * dot.x + int(b1 & 15u) * dot.y) + - dm.y * float(int(b0 >> 4u) * dot.z + int(b1 >> 4u) * dot.w)) + } + + [spirv_kernel(local_size_x = 64, name = "kq_gemv_k2_cls_spv")] + def run { + gemv_shell() + } +} + // Q5_K: the k4 fold with the 5th bit deposited from the block's qh word [vk_dispatch(name = "kq_gemv_k5_cls", kernel = "run", family = "kq_gemv_cls", grid = "wgs", params = "wgs : int64")] class KqGemvK5 : KqGemvBase { @@ -5913,3 +6771,166 @@ class KqGemvK6 : KqGemvBase { gemv_shell() } } + +// IQ2_S: the iq3s gather widened to the u64 grid - a 10-bit qs|qh index picks TWO grid words +// (eight magnitudes {8, 25, 43}), the block's own sign bytes flip them; per-16 UNSIGNED +// strips (1 + 2s) keep split lo/hi dots under the eighth-ed d. The 8 KB grid stages ahead. +[vk_dispatch(name = "kq_gemv_iq2s_cls", kernel = "run", family = "kq_gemv_cls", grid = "wgs", params = "wgs : int64")] +class KqGemvIq2s : KqGemvBase { + @workgroup gridw : uint[2048] + + def iq3s_signed(g, nib : uint) : uint { + let m1 = (nib * 0x00204081u) & 0x01010101u + let mask = m1 * 255u + return (g ^ mask) + m1 + } + + def override blk_contrib(wsb, blk, xsb : uint) : float { + let qw = wq[wsb * 18u + blk] + let sgw = wq[wsb * 18u + 8u + blk] + let qh = (wq[wsb * 18u + 16u + blk / 4u] >> ((blk % 4u) * 8u)) & 255u + let xb4 = xsb * 16u + blk * 2u + let xl = xq4[xb4] + let xh = xq4[xb4 + 1u] + let sw = wsu[wsb * 5u + 1u + blk / 2u] + let s0 = int((sw >> ((blk % 2u) * 16u)) & 255u) + let s1 = int((sw >> ((blk % 2u) * 16u + 8u)) & 255u) + var isum = 0 + for (h in range(2)) { + let l0 = uint(h) * 2u + let sg0 = (sgw >> (l0 * 8u)) & 255u + let sg1 = (sgw >> (l0 * 8u + 8u)) & 255u + let i0 = (((qw >> (l0 * 8u)) & 255u) | ((qh << (8u - 2u * l0)) & 0x300u)) * 2u + let i1 = (((qw >> (l0 * 8u + 8u)) & 255u) | ((qh << (8u - 2u * (l0 + 1u))) & 0x300u)) * 2u + let w0 = iq3s_signed(gridw[i0], sg0 & 15u) + let w1 = iq3s_signed(gridw[i0 + 1u], sg0 >> 4u) + let w2 = iq3s_signed(gridw[i1], sg1 & 15u) + let w3 = iq3s_signed(gridw[i1 + 1u], sg1 >> 4u) + if (h == 0) { + isum += s0 * (sdot4(w0, xl.x) + sdot4(w1, xl.y) + sdot4(w2, xl.z) + sdot4(w3, xl.w)) + } else { + isum += s1 * (sdot4(w0, xh.x) + sdot4(w1, xh.y) + sdot4(w2, xh.z) + sdot4(w3, xh.w)) + } + } + let dd = unpackHalf2x16(wsu[wsb * 5u]).x + return axs[xsb] * dd * float(isum) + } + + [spirv_kernel(local_size_x = 64, name = "kq_gemv_iq2s_cls_spv")] + def run { + let lid = gl_LocalInvocationID.x + for (k in range(32)) { + let idx = lid + uint(k) * 64u + gridw[idx] = iq2s_grid_word(int(idx)) + } + barrier() // the grid visible before the shell's block walk + gemv_shell() + } +} + +// IQ2_XS: the iq2s u64-grid walk with ksigns-by-parity - each u16 qs word carries a 9-bit +// grid index (TWO staged words) and a 7-bit KSIGNS_IQ2XS index whose sign byte ksign7 +// recomputes (no sign plane, no qh); per-16 UNSIGNED strips under the eighth-ed d. +[vk_dispatch(name = "kq_gemv_iq2xs_cls", kernel = "run", family = "kq_gemv_cls", grid = "wgs", params = "wgs : int64")] +class KqGemvIq2xs : KqGemvBase { + @workgroup gridw : uint[1024] + + def iq3s_signed(g, nib : uint) : uint { + let m1 = (nib * 0x00204081u) & 0x01010101u + let mask = m1 * 255u + return (g ^ mask) + m1 + } + + def override blk_contrib(wsb, blk, xsb : uint) : float { + let xb4 = xsb * 16u + blk * 2u + let xl = xq4[xb4] + let xh = xq4[xb4 + 1u] + let sw = wsu[wsb * 5u + 1u + blk / 2u] + let s0 = int((sw >> ((blk % 2u) * 16u)) & 255u) + let s1 = int((sw >> ((blk % 2u) * 16u + 8u)) & 255u) + var isum = 0 + for (h in range(2)) { + let qw = wq[wsb * 16u + blk * 2u + uint(h)] + let wa = qw & 0xFFFFu + let wb = qw >> 16u + let sg0 = ksign7(wa >> 9u) + let sg1 = ksign7(wb >> 9u) + let i0 = (wa & 511u) * 2u + let i1 = (wb & 511u) * 2u + let w0 = iq3s_signed(gridw[i0], sg0 & 15u) + let w1 = iq3s_signed(gridw[i0 + 1u], sg0 >> 4u) + let w2 = iq3s_signed(gridw[i1], sg1 & 15u) + let w3 = iq3s_signed(gridw[i1 + 1u], sg1 >> 4u) + if (h == 0) { + isum += s0 * (sdot4(w0, xl.x) + sdot4(w1, xl.y) + sdot4(w2, xl.z) + sdot4(w3, xl.w)) + } else { + isum += s1 * (sdot4(w0, xh.x) + sdot4(w1, xh.y) + sdot4(w2, xh.z) + sdot4(w3, xh.w)) + } + } + let dd = unpackHalf2x16(wsu[wsb * 5u]).x + return axs[xsb] * dd * float(isum) + } + + [spirv_kernel(local_size_x = 64, name = "kq_gemv_iq2xs_cls_spv")] + def run { + let lid = gl_LocalInvocationID.x + for (k in range(16)) { + let idx = lid + uint(k) * 64u + gridw[idx] = iq2xs_grid_word(int(idx)) + } + barrier() // the iq2xs grid visible before the shell's block walk + gemv_shell() + } +} + +// IQ2_XXS: the iq3xxs GEMV shell over the two-word u64 grid - block b's grid word carries +// four BYTE indices (two staged words each), its aux32 the parity-derived ksigns; the per-32 +// (1 + 2s) strip under the eighth-ed d reads exactly through the shared signed extract. +[vk_dispatch(name = "kq_gemv_iq2xxs_cls", kernel = "run", family = "kq_gemv_cls", grid = "wgs", params = "wgs : int64")] +class KqGemvIq2xxs : KqGemvBase { + @workgroup gridw : uint[512] + + def iq3s_signed(g, nib : uint) : uint { + let m1 = (nib * 0x00204081u) & 0x01010101u + let mask = m1 * 255u + return (g ^ mask) + m1 + } + + def override blk_contrib(wsb, blk, xsb : uint) : float { + let gw = wq[wsb * 16u + blk * 2u] + let aux = wq[wsb * 16u + blk * 2u + 1u] + let xb4 = xsb * 16u + blk * 2u + let xl = xq4[xb4] + let xh = xq4[xb4 + 1u] + var isum = 0 + for (h in range(2)) { + let hu = uint(h) + let b0 = (gw >> (16u * hu)) & 255u + let b1 = (gw >> (16u * hu + 8u)) & 255u + let sg0 = ksign7((aux >> (14u * hu)) & 127u) + let sg1 = ksign7((aux >> (14u * hu + 7u)) & 127u) + let w0 = iq3s_signed(gridw[b0 * 2u], sg0 & 15u) + let w1 = iq3s_signed(gridw[b0 * 2u + 1u], sg0 >> 4u) + let w2 = iq3s_signed(gridw[b1 * 2u], sg1 & 15u) + let w3 = iq3s_signed(gridw[b1 * 2u + 1u], sg1 >> 4u) + if (h == 0) { + isum += sdot4(w0, xl.x) + sdot4(w1, xl.y) + sdot4(w2, xl.z) + sdot4(w3, xl.w) + } else { + isum += sdot4(w0, xh.x) + sdot4(w1, xh.y) + sdot4(w2, xh.z) + sdot4(w3, xh.w) + } + } + let dd = unpackHalf2x16(wsu[wsb * 5u]).x + return axs[xsb] * dd * float(iq4_sc(wsu[wsb * 5u + 1u + blk / 4u], blk) * isum) + } + + [spirv_kernel(local_size_x = 64, name = "kq_gemv_iq2xxs_cls_spv")] + def run { + let lid = gl_LocalInvocationID.x + for (k in range(8)) { + let idx = lid + uint(k) * 64u + gridw[idx] = iq2xxs_grid_word(int(idx)) + } + barrier() // the iq2xxs grid visible before the shell's block walk + gemv_shell() + } +} diff --git a/modules/dasLLAMA/dasllama/dasllama_vulkan_common.das b/modules/dasLLAMA/dasllama/dasllama_vulkan_common.das index 95018ef0ef..f41b13e29d 100644 --- a/modules/dasLLAMA/dasllama/dasllama_vulkan_common.das +++ b/modules/dasLLAMA/dasllama/dasllama_vulkan_common.das @@ -1709,7 +1709,7 @@ struct ArenaSlab { } struct ArenaFmt { - fmt : int // int(KqFmt): 0 = q8, 1/2/3 = k4/k5/k6, 4 = q40 + fmt : int // int(KqFmt) slabs : array cur : int // slab the bump cursor is in slab_cap : int64 // max blocks one slab may carry (msr over both plane strides) @@ -1740,17 +1740,26 @@ def arena_local_blk(blk : int64) : int64 => blk & ((1l << 32l) - 1l) // per-block plane strides — the block is 32 weights for q8 and a 256-weight superblock for kq. // Mirrors stack_plane_bytes: it computes the same totals as count * these strides. -// the tier's stack fmt space is int(KqFmt) — bridge to the gemm_schema kq id space (4/5/6/40) +// the tier's stack fmt space is int(KqFmt) — bridge to the gemm_schema kq id space def vk_kq_schema_id(fmt : int) : int { - if (fmt == 1) return 4 - if (fmt == 2) return 5 - if (fmt == 3) return 6 - if (fmt == 4) return 40 + if (fmt == int(KqFmt.k4)) return 4 + if (fmt == int(KqFmt.k5)) return 5 + if (fmt == int(KqFmt.k6)) return 6 + if (fmt == int(KqFmt.q40)) return 40 + if (fmt == int(KqFmt.iq4xs)) return 44 + if (fmt == int(KqFmt.k3)) return 3 + if (fmt == int(KqFmt.iq3s)) return 33 + if (fmt == int(KqFmt.iq3xxs)) return 34 + if (fmt == int(KqFmt.iq4nl)) return 45 + if (fmt == int(KqFmt.k2)) return 2 + if (fmt == int(KqFmt.iq2s)) return 23 + if (fmt == int(KqFmt.iq2xs)) return 24 + if (fmt == int(KqFmt.iq2xxs)) return 25 panic("vk_kq_schema_id: not a kq stack fmt") return 0 } -// fmt space here = int(KqFmt): 0 q8 (32-elem blocks), 1-4 the kq lattice + q40 +// fmt space here = int(KqFmt): 0 = q8 (32-elem blocks), every other value the kq lattice def vk_fmt_b32(fmt : int) : bool => fmt == 0 // meta/activation block unit of a stack format — q8 counts 32-weight blocks, the kq diff --git a/modules/dasLLAMA/dasllama/dasllama_vulkan_prefill.das b/modules/dasLLAMA/dasllama/dasllama_vulkan_prefill.das index a9eef68898..3b4cdc501a 100644 --- a/modules/dasLLAMA/dasllama/dasllama_vulkan_prefill.das +++ b/modules/dasLLAMA/dasllama/dasllama_vulkan_prefill.das @@ -40,7 +40,7 @@ var private @scratch g_pf_ids_stage : array [arch(at="../ARCHITECTURE_RUNTIME.md#activation-scale-lattice")] def private pf_f16_feed(f : int) : bool { - return (f == int(KqFmt.q8) || f == int(KqFmt.k4) || f == int(KqFmt.k6)) && g_gpu.coopmat_mode == 4 + return (f == int(KqFmt.q8) || kq_sb(f)) && g_gpu.coopmat_mode == 4 } [arch(at="../ARCHITECTURE_RUNTIME.md#activation-scale-lattice")] @@ -56,14 +56,48 @@ let private PF_CHUNK_MAX = 8l // the overlap ramp doubles 1,2,4 then holds her [arch(at="../ARCHITECTURE_GPU_VULKAN.md#cm2-tile-pick-and-default")] def private cm2_cls_ensure(fmt : int; tc : int64) : bool { - verify(fmt == int(KqFmt.q8) || fmt == int(KqFmt.k4) || fmt == int(KqFmt.k6), "vk prefill: a cm2 tile for a format the f16 feed never admits") + verify(fmt == int(KqFmt.q8) || kq_sb(fmt), "vk prefill: a cm2 tile for a format the f16 feed never admits") verify(tc == 256l || tc == 128l || tc == 32l, "vk prefill: a cm2 tile column the pick never returns") if (fmt == int(KqFmt.k4)) { return tc == 256l ? ensure_kq_batch_k4_cm2l_cls() : (tc == 128l ? ensure_kq_batch_k4_cm2m_cls() : ensure_kq_batch_k4_cm2s_cls()) } + if (fmt == int(KqFmt.k5)) { + return tc == 256l ? ensure_kq_batch_k5_cm2l_cls() : (tc == 128l ? ensure_kq_batch_k5_cm2m_cls() : ensure_kq_batch_k5_cm2s_cls()) + } if (fmt == int(KqFmt.k6)) { return tc == 256l ? ensure_kq_batch_k6_cm2l_cls() : (tc == 128l ? ensure_kq_batch_k6_cm2m_cls() : ensure_kq_batch_k6_cm2s_cls()) } + if (fmt == int(KqFmt.q40)) { + return tc == 256l ? ensure_kq_batch_q40_cm2l_cls() : (tc == 128l ? ensure_kq_batch_q40_cm2m_cls() : ensure_kq_batch_q40_cm2s_cls()) + } + if (fmt == int(KqFmt.iq4xs)) { + return tc == 256l ? ensure_kq_batch_iq4xs_cm2l_cls() : (tc == 128l ? ensure_kq_batch_iq4xs_cm2m_cls() : ensure_kq_batch_iq4xs_cm2s_cls()) + } + if (fmt == int(KqFmt.k3)) { + return tc == 256l ? ensure_kq_batch_k3_cm2l_cls() : (tc == 128l ? ensure_kq_batch_k3_cm2m_cls() : ensure_kq_batch_k3_cm2s_cls()) + } + if (fmt == int(KqFmt.iq3s)) { + return tc == 256l ? ensure_kq_batch_iq3s_cm2l_cls() : (tc == 128l ? ensure_kq_batch_iq3s_cm2m_cls() : ensure_kq_batch_iq3s_cm2s_cls()) + } + if (fmt == int(KqFmt.iq3xxs)) { + return tc == 256l ? ensure_kq_batch_iq3xxs_cm2l_cls() : (tc == 128l ? ensure_kq_batch_iq3xxs_cm2m_cls() : ensure_kq_batch_iq3xxs_cm2s_cls()) + } + if (fmt == int(KqFmt.iq4nl)) { + return tc == 256l ? ensure_kq_batch_iq4nl_cm2l_cls() : (tc == 128l ? ensure_kq_batch_iq4nl_cm2m_cls() : ensure_kq_batch_iq4nl_cm2s_cls()) + } + if (fmt == int(KqFmt.k2)) { + return tc == 256l ? ensure_kq_batch_k2_cm2l_cls() : (tc == 128l ? ensure_kq_batch_k2_cm2m_cls() : ensure_kq_batch_k2_cm2s_cls()) + } + if (fmt == int(KqFmt.iq2s)) { + return tc == 256l ? ensure_kq_batch_iq2s_cm2l_cls() : (tc == 128l ? ensure_kq_batch_iq2s_cm2m_cls() : ensure_kq_batch_iq2s_cm2s_cls()) + } + if (fmt == int(KqFmt.iq2xs)) { + return tc == 256l ? ensure_kq_batch_iq2xs_cm2l_cls() : (tc == 128l ? ensure_kq_batch_iq2xs_cm2m_cls() : ensure_kq_batch_iq2xs_cm2s_cls()) + } + if (fmt == int(KqFmt.iq2xxs)) { + return tc == 256l ? ensure_kq_batch_iq2xxs_cm2l_cls() : (tc == 128l ? ensure_kq_batch_iq2xxs_cm2m_cls() : ensure_kq_batch_iq2xxs_cm2s_cls()) + } + verify(fmt == int(KqFmt.q8), "vk prefill: the cm2 ensure fell through for a kq format - a format the feed admits must own a tile arm") return tc == 256l ? ensure_q8_batch_cm2l_cls() : (tc == 128l ? ensure_q8_batch_cm2m_cls() : ensure_q8_batch_cm2s_cls()) } @@ -75,12 +109,79 @@ def private cm2_cls_set(fmt : int; tc : int64; bufs : uint64 const[5]; sizes : i } return tc == 128l ? set_kq_batch_k4_cm2m_cls(bufs, sizes, gbits) : set_kq_batch_k4_cm2s_cls(bufs, sizes, gbits) } + if (fmt == int(KqFmt.k5)) { + if (tc == 256l) { + return set_kq_batch_k5_cm2l_cls(bufs, sizes, gbits) + } + return tc == 128l ? set_kq_batch_k5_cm2m_cls(bufs, sizes, gbits) : set_kq_batch_k5_cm2s_cls(bufs, sizes, gbits) + } if (fmt == int(KqFmt.k6)) { if (tc == 256l) { return set_kq_batch_k6_cm2l_cls(bufs, sizes, gbits) } return tc == 128l ? set_kq_batch_k6_cm2m_cls(bufs, sizes, gbits) : set_kq_batch_k6_cm2s_cls(bufs, sizes, gbits) } + if (fmt == int(KqFmt.q40)) { + if (tc == 256l) { + return set_kq_batch_q40_cm2l_cls(bufs, sizes, gbits) + } + return tc == 128l ? set_kq_batch_q40_cm2m_cls(bufs, sizes, gbits) : set_kq_batch_q40_cm2s_cls(bufs, sizes, gbits) + } + if (fmt == int(KqFmt.iq4xs)) { + if (tc == 256l) { + return set_kq_batch_iq4xs_cm2l_cls(bufs, sizes, gbits) + } + return tc == 128l ? set_kq_batch_iq4xs_cm2m_cls(bufs, sizes, gbits) : set_kq_batch_iq4xs_cm2s_cls(bufs, sizes, gbits) + } + if (fmt == int(KqFmt.k3)) { + if (tc == 256l) { + return set_kq_batch_k3_cm2l_cls(bufs, sizes, gbits) + } + return tc == 128l ? set_kq_batch_k3_cm2m_cls(bufs, sizes, gbits) : set_kq_batch_k3_cm2s_cls(bufs, sizes, gbits) + } + if (fmt == int(KqFmt.iq3s)) { + if (tc == 256l) { + return set_kq_batch_iq3s_cm2l_cls(bufs, sizes, gbits) + } + return tc == 128l ? set_kq_batch_iq3s_cm2m_cls(bufs, sizes, gbits) : set_kq_batch_iq3s_cm2s_cls(bufs, sizes, gbits) + } + if (fmt == int(KqFmt.iq3xxs)) { + if (tc == 256l) { + return set_kq_batch_iq3xxs_cm2l_cls(bufs, sizes, gbits) + } + return tc == 128l ? set_kq_batch_iq3xxs_cm2m_cls(bufs, sizes, gbits) : set_kq_batch_iq3xxs_cm2s_cls(bufs, sizes, gbits) + } + if (fmt == int(KqFmt.iq4nl)) { + if (tc == 256l) { + return set_kq_batch_iq4nl_cm2l_cls(bufs, sizes, gbits) + } + return tc == 128l ? set_kq_batch_iq4nl_cm2m_cls(bufs, sizes, gbits) : set_kq_batch_iq4nl_cm2s_cls(bufs, sizes, gbits) + } + if (fmt == int(KqFmt.k2)) { + if (tc == 256l) { + return set_kq_batch_k2_cm2l_cls(bufs, sizes, gbits) + } + return tc == 128l ? set_kq_batch_k2_cm2m_cls(bufs, sizes, gbits) : set_kq_batch_k2_cm2s_cls(bufs, sizes, gbits) + } + if (fmt == int(KqFmt.iq2s)) { + if (tc == 256l) { + return set_kq_batch_iq2s_cm2l_cls(bufs, sizes, gbits) + } + return tc == 128l ? set_kq_batch_iq2s_cm2m_cls(bufs, sizes, gbits) : set_kq_batch_iq2s_cm2s_cls(bufs, sizes, gbits) + } + if (fmt == int(KqFmt.iq2xs)) { + if (tc == 256l) { + return set_kq_batch_iq2xs_cm2l_cls(bufs, sizes, gbits) + } + return tc == 128l ? set_kq_batch_iq2xs_cm2m_cls(bufs, sizes, gbits) : set_kq_batch_iq2xs_cm2s_cls(bufs, sizes, gbits) + } + if (fmt == int(KqFmt.iq2xxs)) { + if (tc == 256l) { + return set_kq_batch_iq2xxs_cm2l_cls(bufs, sizes, gbits) + } + return tc == 128l ? set_kq_batch_iq2xxs_cm2m_cls(bufs, sizes, gbits) : set_kq_batch_iq2xxs_cm2s_cls(bufs, sizes, gbits) + } + verify(fmt == int(KqFmt.q8), "vk prefill: the cm2 set fell through for a kq format - a format the feed admits must own a tile arm") if (tc == 256l) { return set_q8_batch_cm2l_cls(bufs, sizes, gbits) } @@ -97,6 +198,14 @@ def private cm2_cls_enc(fmt : int; tc : int64; raw : VkCommandBuffer; var h : Vk } else { enc_kq_batch_k4_cm2s_cls(raw, h, s, pc, groups) } + } elif (fmt == int(KqFmt.k5)) { + if (tc == 256l) { + enc_kq_batch_k5_cm2l_cls(raw, h, s, pc, groups) + } elif (tc == 128l) { + enc_kq_batch_k5_cm2m_cls(raw, h, s, pc, groups) + } else { + enc_kq_batch_k5_cm2s_cls(raw, h, s, pc, groups) + } } elif (fmt == int(KqFmt.k6)) { if (tc == 256l) { enc_kq_batch_k6_cm2l_cls(raw, h, s, pc, groups) @@ -105,11 +214,94 @@ def private cm2_cls_enc(fmt : int; tc : int64; raw : VkCommandBuffer; var h : Vk } else { enc_kq_batch_k6_cm2s_cls(raw, h, s, pc, groups) } + } elif (fmt == int(KqFmt.q40)) { + if (tc == 256l) { + enc_kq_batch_q40_cm2l_cls(raw, h, s, pc, groups) + } elif (tc == 128l) { + enc_kq_batch_q40_cm2m_cls(raw, h, s, pc, groups) + } else { + enc_kq_batch_q40_cm2s_cls(raw, h, s, pc, groups) + } + } elif (fmt == int(KqFmt.iq4xs)) { + if (tc == 256l) { + enc_kq_batch_iq4xs_cm2l_cls(raw, h, s, pc, groups) + } elif (tc == 128l) { + enc_kq_batch_iq4xs_cm2m_cls(raw, h, s, pc, groups) + } else { + enc_kq_batch_iq4xs_cm2s_cls(raw, h, s, pc, groups) + } + } elif (fmt == int(KqFmt.k3)) { + if (tc == 256l) { + enc_kq_batch_k3_cm2l_cls(raw, h, s, pc, groups) + } elif (tc == 128l) { + enc_kq_batch_k3_cm2m_cls(raw, h, s, pc, groups) + } else { + enc_kq_batch_k3_cm2s_cls(raw, h, s, pc, groups) + } + } elif (fmt == int(KqFmt.iq3s)) { + if (tc == 256l) { + enc_kq_batch_iq3s_cm2l_cls(raw, h, s, pc, groups) + } elif (tc == 128l) { + enc_kq_batch_iq3s_cm2m_cls(raw, h, s, pc, groups) + } else { + enc_kq_batch_iq3s_cm2s_cls(raw, h, s, pc, groups) + } + } elif (fmt == int(KqFmt.iq3xxs)) { + if (tc == 256l) { + enc_kq_batch_iq3xxs_cm2l_cls(raw, h, s, pc, groups) + } elif (tc == 128l) { + enc_kq_batch_iq3xxs_cm2m_cls(raw, h, s, pc, groups) + } else { + enc_kq_batch_iq3xxs_cm2s_cls(raw, h, s, pc, groups) + } + } elif (fmt == int(KqFmt.iq4nl)) { + if (tc == 256l) { + enc_kq_batch_iq4nl_cm2l_cls(raw, h, s, pc, groups) + } elif (tc == 128l) { + enc_kq_batch_iq4nl_cm2m_cls(raw, h, s, pc, groups) + } else { + enc_kq_batch_iq4nl_cm2s_cls(raw, h, s, pc, groups) + } + } elif (fmt == int(KqFmt.k2)) { + if (tc == 256l) { + enc_kq_batch_k2_cm2l_cls(raw, h, s, pc, groups) + } elif (tc == 128l) { + enc_kq_batch_k2_cm2m_cls(raw, h, s, pc, groups) + } else { + enc_kq_batch_k2_cm2s_cls(raw, h, s, pc, groups) + } + } elif (fmt == int(KqFmt.iq2s)) { + if (tc == 256l) { + enc_kq_batch_iq2s_cm2l_cls(raw, h, s, pc, groups) + } elif (tc == 128l) { + enc_kq_batch_iq2s_cm2m_cls(raw, h, s, pc, groups) + } else { + enc_kq_batch_iq2s_cm2s_cls(raw, h, s, pc, groups) + } + } elif (fmt == int(KqFmt.iq2xs)) { + if (tc == 256l) { + enc_kq_batch_iq2xs_cm2l_cls(raw, h, s, pc, groups) + } elif (tc == 128l) { + enc_kq_batch_iq2xs_cm2m_cls(raw, h, s, pc, groups) + } else { + enc_kq_batch_iq2xs_cm2s_cls(raw, h, s, pc, groups) + } + } elif (fmt == int(KqFmt.iq2xxs)) { + if (tc == 256l) { + enc_kq_batch_iq2xxs_cm2l_cls(raw, h, s, pc, groups) + } elif (tc == 128l) { + enc_kq_batch_iq2xxs_cm2m_cls(raw, h, s, pc, groups) + } else { + enc_kq_batch_iq2xxs_cm2s_cls(raw, h, s, pc, groups) + } } elif (tc == 256l) { + verify(fmt == int(KqFmt.q8), "vk prefill: the cm2 enc fell through for a kq format - a format the feed admits must own a tile arm") enc_q8_batch_cm2l_cls(raw, h, s, pc, groups) } elif (tc == 128l) { + verify(fmt == int(KqFmt.q8), "vk prefill: the cm2 enc fell through for a kq format - a format the feed admits must own a tile arm") enc_q8_batch_cm2m_cls(raw, h, s, pc, groups) } else { + verify(fmt == int(KqFmt.q8), "vk prefill: the cm2 enc fell through for a kq format - a format the feed admits must own a tile arm") enc_q8_batch_cm2s_cls(raw, h, s, pc, groups) } } diff --git a/modules/dasLLAMA/followup_general.md b/modules/dasLLAMA/followup_general.md index d262d58a86..97abd94787 100644 --- a/modules/dasLLAMA/followup_general.md +++ b/modules/dasLLAMA/followup_general.md @@ -714,3 +714,71 @@ once already. The tests checklist ledgers the residue; the fix is an in-process equivalent of `DASLLAMA_IMAGE=0` (`g_env_engine.image` is a `let` read at load), so such cells can run image-free instead of risking the purge. + +57. **Plane types have no `long_length`.** `length(PlaneF)` / `length(PlaneU16)` return + `int`, so every `uint64(length(t.blob) * 4l)` spelling caps a plane at 2^31 elements + before the widening - headroom-only today (whisper large-v3's twin is ~632M elements). + Done = `long_length` overloads in `dasllama_plane` and the buffer-sizing call sites + moved onto them. + +58. **The M5 pass on the IQ4_XS Metal kernels (Boris, 2026-08-30: "we'll ledger M5 pass on + new kernels for later").** The format's Metal set is correctness-first: `MetalKqGemvIq4xs`, + the `MetalKqMvIq4xsT` B2/B4 pair, `MetalKqMvB8Iq4xs` and the `IQ4XS` arm of + `MetalKqMulMmK45T` copy the k4/k6 lane maps with a per-element `iq4_lut` (four packed + words, select + shift + sign trick) and no measurement behind them; the prefill site takes + the base mul_mm only - no tensor (`_t`), tall (`_th128`), double-buffered (`_thdb`) or dev-W + dequant twins, and no MoE GEMV / mul_mm trio for the format. Done = the twins stamped on + the existing templates, the LUT cost measured against a `constant` table and against a + byte-pair decode on the M5, and `bench_metal_gemv_kernels` / `bench_metal_kq_mm_lab` rows + for the format beside k4's. The same gap now covers every format this arc added: `k3`, + `iq3s`, `iq3xxs`, `iq4nl`, `k2`, `iq2s`, `iq2xs` and `iq2xxs` take the base `mul_mm` at the + prefill site - no tensor (`_t`), tall (`_th128`), double-buffered (`_thdb`) or dev-W dequant + twins - and `pf_devw_panel_kq` declines all nine formats outright. Done = the twins stamped + on the existing templates for whichever formats measure worth it, and that decline list + shortened to match. + +59. **DONE (2026-08-31, the unquirk pass): `--tune-only ` re-mints one family into the + existing sidecar, and the shipped defaults profiles adopt-then-race only the residue.** + Original ask - partial mint (Boris, 2026-08-30, for after the formats arc): "takes existing mint and + mints new kernels only. at least in debug-only mode." Today a sidecar missing ANY demanded + kernel re-tunes the whole scope (the completeness rule), so every new `[tune]` family - one + per format in this arc - re-mints every application sidecar on the box on its next start, + minutes each (HOW_TO_ADD_A_FORMAT.md QUIRK 17), and the same rule keeps a stub-era + `"reference"` pin alive after the emitter lands (QUIRK 11). The partial mint keeps the + existing entries and races only the families the sidecar lacks (or names as reference), + validating the merged file as one. Debug-only is the acceptable first form: a + `DAS_TUNE_PARTIAL=1` (or `--tune-partial`) rail that the auto policy does not take, so a + shipped box still mints whole. Done = the rail exists, a fresh family on a minted box + costs one family's race, and the tuner's status line names the partial mint as such. + +60. **Retroactive llama.cpp CPU-kernel audit for the arc's earlier formats (Boris, + 2026-08-30: "lets make sure we do for all new CPU kernels, and if we skipped for previous + ones - lets ledger towards the end of this arc").** The IQ3_S CPU arm is the first written + against llama.cpp's arch kernel (signs-on-activation, sllv index compose, scalar grid + loads beating hardware gather); IQ4_XS and Q3_K were written from the disk format alone. + Toward the end of the arc: read the reference exe's vec-dot and the reference exe's vec-dot + (x86 + arm), list every technique ours does not measure, land the missing ones as + `[tune_perm]` spellings and let the probe judge. Done = a per-format note naming what was + raced and what won, beside the existing bench rows. + +61. **IQ3_S CPU decode: race a no-panel gemv spelling (the 0.92x tail).** The stamped gemv + gathers each superblock into an alloca panel and then runs the vector dot - a store/load + round trip per superblock that a single token never amortizes; llama.cpp's per-row form + (grid words composed straight into vectors, signs applied to the ACTIVATION via + shuffle+cmpeq/xor-sub, magnitudes kept unsigned for maddubs) edges it 57.0 vs 52.4 tg128 + on the zen2 (pp512 is ours 4.93x - the panel amortizes across the tile). The counter to + race as a [tune_perm]: compose the gathered words directly into the weight vectors + (insertelement per i32 lane, no panel), and/or the signs-on-activation form that drops + the abs+psign pair. Done = a gemv perm that takes tg128 at or past llama.cpp's, crowned + by the probe. + +62. **IQ3_S Metal decode: the ~140 GB/s compose ceiling (tg 0.95x).** Eight GEMV forms raced + at n=2048 d=8192 - gather placement x3, gather deleted, signs deleted, llama.cpp's exact + 1-lane-per-block geometry, row width, f4 magnitude slab - all land in 127-141 GB/s while + k4 does 204 and k6 287 in the same harness; the 9-bit-index + per-nibble-sign compose is + the cost, not any one instruction class (deleting the gather OR the signs moves it under + 10%). The f4 slab ships (+9%, tg128 0.82x -> 0.95x same-run). Candidates beyond kernel + shape: emitted-MSL diffs vs llama.cpp's compiled kernel (half math, function constants), + or fusing the sign flip into the staged slab per SITE via a second indexed table. Done = + a form that clears 180 GB/s in the dispatch-loop probe (QUIRK 22's harness), or a note + proving the ceiling is shared by llama.cpp's own kernel when isolated the same way. diff --git a/modules/dasLLAMA/followup_metal.md b/modules/dasLLAMA/followup_metal.md new file mode 100644 index 0000000000..b6819e92dc --- /dev/null +++ b/modules/dasLLAMA/followup_metal.md @@ -0,0 +1,58 @@ +# Metal / mac-session follow-up - the M5 catch-up ledger + +**Status: OPEN (2026-08-31, unquirk pass).** The ledger for the parallel mac session that +forks after PR-1 of `plans/unquirk_pass.md`. Metal-tier perf items move here from the +per-format notes during the Phase E doc split; the CPU items below are mac-session work too - +the M-series CPU tiers are minted and raced from that box. + +## 1. The dark smmla (i8mm) leg - free race, never run + +`q8q8_tile_gen` ships five `dot = "smmla"` seats gated `requires = "i8mm"` (mr4/mr8 x +kstep/nrsplit/gkstep), and NO box has ever raced them: `LLVMGetHostCPUFeatures()` returns an +EMPTY string on macOS (llvm_jit_common.das documents it beside `g_target_arm64_i8mm`), so +i8mm never detected on Apple Silicon at the EMITTER tier (`g_target_arm64_i8mm`), so the +generators declined and the seats never raced - even though `cpu_supports("i8mm")` answered +correctly via sysctl. M1 lacks i8mm; M2+ has it; the M5 Max additionally has FEAT_SME2p1 + +BF16/EBF16. FIXED in the unquirk pass (PR-1): `g_target_arm64_i8mm` also consults +`cpu_supports`, and the target machine appends `+i8mm` when the host has it - an M5 `--tune` +now races smmla-vs-NEON with zero new kernel work. Mac-session order: + +1. On the M5 (post PR-1): `--tune`, confirm the smmla seats EMIT (they have never been + exercised - treat the emitter arms as unproven, gate with the gen probe TEST mode first) + and report the crowns; export the `arm-i8mm` profile if they win. +2. If smmla wins q8q8: the kq tile families have NO ARM ISA seats at all (mr8 NEON is the + whole grid) - an smmla kq tile emitter arm is the highest-leverage CPU kernel work on + the mac, and it transfers to Graviton3+ (c8g) verbatim. +3. SME/SME2 is the tier after: no seats, no emitter, new kernel design (streaming mode + + ZA tiles) - research first per the standing research-before-kernel-work rule. + +## 2. The fixtures - which GGUF per format, and where to get it + +Every vehicle is a Llama-3.2-1B-Instruct requant unless named otherwise. zen2 holds them at +`D:\Work\llama.cpp\models\`; the M1 worktree box holds the same set at +`~/Work/llama.cpp/models/` together with the baked `.dlim` images beside each; the M5 starts +empty - `scp m1:~/Work/llama.cpp/models/ ~/Work/llama.cpp/models/` (or from zen2) per row. +The `-local` files are our own `llama-quantize --allow-requantize` products (the IQ ones with +`--imatrix`), the rest are bartowski / mradermacher downloads. + +| format | vehicle GGUF | notes | +|---|---|---| +| IQ4_XS | `Llama-3.2-1B-Instruct-IQ4_XS.gguf` | the pilot (bartowski) | +| Q3_K | `Llama-3.2-1B-Instruct-Q3_K_L.gguf` | a third of the bytes are k5/k6 | +| IQ3_S | `Qwen2.5-1.5B-Instruct-IQ3_XS.gguf` (kernels) + `Llama-3.2-1B-Instruct-IQ3_M.gguf` (e2e/bench) | the Qwen file proves kernels, the local IQ3_M carries iq3_s on the bench rows | +| IQ3_XXS | `Llama-3.2-1B-Instruct-IQ3_XXS-local.gguf` | `Llama-3.2-1B-Instruct.i1-IQ3_XXS.gguf` (mradermacher) mixes in IQ2_S attn - the IQ2_S vehicle below | +| IQ4_NL | `Llama-3.2-1B-Instruct-IQ4_NL-local.gguf` | no imatrix needed | +| Q2_K | `Llama-3.2-1B-Instruct-Q2_K-local.gguf` | | +| IQ2_S | `Llama-3.2-1B-Instruct.i1-IQ3_XXS.gguf` | IQ2_S attn x32 + IQ3_XXS/IQ3_S/Q4_K/Q5_K - a MIXED vehicle by construction | +| IQ2_XS | `Llama-3.2-1B-Instruct-IQ2_XS-local.gguf` | carries q2_K on blk.0/1 ffn_down - the mixed-vehicle e2e of QUIRK 26 | +| IQ2_XXS | `Llama-3.2-1B-Instruct-IQ2_XXS-local.gguf` | prompt ids `128000,12805,5304,264,892`; the step-3 0.0267 top-2 tie | + +The reference build for the ratios is llama.cpp b10660 (`~/Work/llama.cpp/build/bin/llama-bench` +/ `llama-simple` on the M1; the same tree serves the `simple_ids` oracle). Parity prompt for +every format: `harness/parity.das -- -m -n 40 --ids 128000,12805,5304,264,892`. + +## 3. Metal tier perf items + +Placeholder - the per-format Metal notes (tg 0.78-0.93x tails, the IQ4_XS lane-map gap of +followup_general #58, the Q22 dispatch-loop probe method) consolidate here in Phase E of +`plans/unquirk_pass.md`; until then they live in `HOW_TO_ADD_A_FORMAT.md`'s per-format notes. diff --git a/modules/dasLLAMA/followup_vulkan.md b/modules/dasLLAMA/followup_vulkan.md index 4bb0e15c69..a185a036fd 100644 --- a/modules/dasLLAMA/followup_vulkan.md +++ b/modules/dasLLAMA/followup_vulkan.md @@ -447,7 +447,14 @@ module) is independent and can land any time - it is pure structure. `def override decode_*` per format - the shape `harness/vk_gemm_probe.das`'s `K6PxBase` already proves. Gate: the nine oracle cells in `tests/test_vulkan_kernels.das` stay 0-off, the probe's l/m rows stay within noise. The k5/q40 stamps (item 11's NEXT) land on the - template, not as more copies. + template, not as more copies. Measured 2026-08-30 on the 1B (`lcpp_bench`, 5060 Ti): the + formats with no cm2 tile prefill at 5161 (iq4xs) and 5174 (k3) t/s against k4's 13144 on + the same box (llama.cpp 17060 / 17509 / 19719) while their decode sits at parity, so the + template is also the lever that puts every new format on the tile path; k4's own 0.67x is + the 1B-shape tier gap, a separate item. DONE 2026-08-30 (the iquant arc): `KqCm2BatchT` + stamps all nine originals (suite + the Q4_K_M e2e row within noise), and k5/q40/iq4xs/k3 + joined as format templates - twelve more stamps, iq4xs pp512 15334 (0.90x llama.cpp, + above the k4 control), k3 14031 (0.80x). Item 11's k5/q40 stamps landed with it. 25. **Try `VK_NV_cooperative_vector` for decode GEMV on real hardware (Boris, 2026-08-28).** cm2 has no matrix-vector op - its seven feature bits are all tile-shaped, minimum tile 16 - @@ -529,3 +536,22 @@ module) is independent and can land any time - it is pure structure. (`keep_hidden`), plus an override-capability row so a whole-plane consumer can test for it; a cell that embeds through the vulkan override and compares against the CPU pool proves it. + +34. **END-OF-ARC: the pp512 tier class (~0.67-0.70x of llama.cpp on 1B shapes).** Every + sb-format cm2 tile lands in the same band (k4 control 0.67x, iq3s 0.70x on the 5060 Ti) + while tg and the CPU tiers hold parity or better - the gap is the shared batch-GEMM + tier, not any one format's decode. Boris 2026-08-30: this one bothers him at 0.7 - + schedule a dedicated pass at the END of the iquant-formats arc (after the last format + lands), not per-format. Start from the followup 29-32 streamed-layer levers and a + kernel-level probe of the cm2 tile vs llama.cpp's mul_mm_cm2 at matched shapes. + +35. **The grid-format GEMV workgroup re-stage is a fixed per-workgroup cost - amplified on + small models.** Every u64-grid gemv (iq2s 8 KB, iq2xs 4 KB) stages the codebook into + workgroup memory per 2-row workgroup, so tg pays a fixed latency the row length must + amortize. On the 3B i1 vehicle iq2s tg landed 0.81x llama.cpp (~350 GB/s effective); + the 1B IQ2_XS vehicle lands 0.54x (188.7 vs 349.9 t/s = ~84 GB/s effective - latency- + bound, while its cm2 pp512 sits at a healthy 0.77x). Levers, in likely order: persist + the staged grid across the row loop (one stage per SM residency, not per workgroup), + widen rows-per-workgroup for grid formats, or fold the grid into a device-buffer read + the L2 serves. Done = 1B-class grid-format tg within the k-format band on the same + vehicle. diff --git a/modules/dasLLAMA/harness/dasllama_tuner.das b/modules/dasLLAMA/harness/dasllama_tuner.das index 59cf4452f7..623ea2932b 100644 --- a/modules/dasLLAMA/harness/dasllama_tuner.das +++ b/modules/dasLLAMA/harness/dasllama_tuner.das @@ -151,6 +151,10 @@ def private run_half(name : string; paranoid : bool) : bool { } let elapsed_ms = get_time_usec(t0) / 1000 print("dasllama_tuner: {name} finished in {elapsed_ms} ms (rc={rc})\n") + if (rc == TUNE_ONLY_NO_MATCH) { + g_no_match_halves++ + return true // the filter named nothing here; main fails the mint only when BOTH halves say so + } if (rc != 0) { // the refusal detail scrolled by minutes ago mid-stream — the closing line carries the // WHY and the escape hatches so a nonzero exit is self-diagnosing where you look. @@ -165,6 +169,10 @@ def private run_half(name : string; paranoid : bool) : bool { return rc == 0 } +// exit code of a half whose --tune-only filter matched nothing (tune_kernels / gen_tune_probe) +let TUNE_ONLY_NO_MATCH = 3 +var private g_no_match_halves = 0 + [export] def main : int { let t0 = ref_time_ticks() @@ -178,6 +186,9 @@ def main : int { if (tune_noise_override()) { print("dasllama_tuner: OVERRIDE DAS_TUNE_NOISE_OVERRIDE - a busy-box refusal mints anyway (sidecar stamped 'noise: overridden')\n") } + if (!empty(tune_only_filter())) { + print("dasllama_tuner: FILTER --tune-only '{tune_only_filter()}' - only matching kernel families race; every other sidecar entry survives the upsert\n") + } // pre-mint snapshot: COPY, not rename — the halves UPSERT, and a rename would drop other // scopes' keys from the shared file. The post-mint diff reads it back. let sidecar = g_env_core.tune_manifest @@ -190,6 +201,10 @@ def main : int { } let ok1 = run_half("gen_tune_probe.das", paranoid) let ok2 = run_half("tune_kernels.das", paranoid) + if (g_no_match_halves == 2) { + print("dasllama_tuner: --tune-only '{tune_only_filter()}' matched no family in either half - a typo'd token; sidecar untouched\n") + return 1 + } if (!empty(sidecar)) { if (ok1 && ok2) { if (!empty(oldText)) { diff --git a/modules/dasLLAMA/harness/export_tune_profile.das b/modules/dasLLAMA/harness/export_tune_profile.das new file mode 100644 index 0000000000..abee9f224f --- /dev/null +++ b/modules/dasLLAMA/harness/export_tune_profile.das @@ -0,0 +1,39 @@ +options gen2 + +require llvm/daslib/llvm_tune +require daslib/clargs +require daslib/fio +require daslib/module_path + +[CommandLineArgs] +struct ExportArgs { + @clarg_doc = "Sidecar to export from (default: the DAS_TUNE_MANIFEST / app-beside resolution)" + sidecar : string + + @clarg_doc = "Output directory (default: modules/dasLLAMA/performance/defaults beside this script)" + out_dir : string + + @clarg_doc = "CPU class name to export as (default: this box's tune_cpu_class())" + klass : string +} + +//! Export a minted sidecar's kernel winners as the shipped defaults profile for this box's +//! CPU class — run on a reference box after a full `--tune` mint, then check the file in. +//! `daslang harness/export_tune_profile.das -- --sidecar benchmarks/lcpp_bench.tune.json` +[export] +def main : int { + var r <- parse_args(type) + if (r |> is_err) { + print("{r |> unwrap_err}\n") + return 1 + } + let cfg <- r |> move_unwrap + if (!empty(cfg.sidecar)) { + set_tune_manifest_runtime_path(cfg.sidecar) + } + let klass = empty(cfg.klass) ? tune_cpu_class() : cfg.klass + let outDir = empty(cfg.out_dir) ? path_join(path_join(get_this_module_dir(), ".."), "performance/defaults") : cfg.out_dir + mkdir(outDir) + let outPath = path_join(outDir, "{klass}.tune-defaults.json") + return tune_profile_export(outPath, klass) ? 0 : 1 +} diff --git a/modules/dasLLAMA/harness/gen_tune_probe.das b/modules/dasLLAMA/harness/gen_tune_probe.das index 329b457e35..3eaa4ea0b1 100644 --- a/modules/dasLLAMA/harness/gen_tune_probe.das +++ b/modules/dasLLAMA/harness/gen_tune_probe.das @@ -21,8 +21,8 @@ options _dasllama_internal = true // equality: variants run under full fast-math, so the backend may legally regroup a // fold per loop shape (~1e-6 relative; the reference holds no cross-kernel bit-parity // either) — real generator bugs (wrong stride/lane/scale) are orders louder and -// still die here. Declining perms (mr8_budget / mr2_lanes / dot_vpdpbusd) run the -// reference bodies. +// still die here. Declining perms (mr8_budget, and every ISA-gated row the box +// fails) run the reference bodies. // tune: iso-bench every variant (interleaved best-of) on its own repacked buffers — the // tile on a batch shape, the gemv core on a single-token decode shape — print both // tables, and tune_manifest_set the merged winner: the row whose TILE knobs win the @@ -93,6 +93,9 @@ def private gen_round_step(r : int; keep : array) { // the previous sidecar's seat for `entry`, read raw (a stale sidecar still names the seats) — // a re-mint's crown only moves when it beats the incumbent by gen_margin() var private g_incumbent : table +// the --tune-only filter matched no generator family here: exit 3, so the wrapper can tell a +// half-that-sat-out from a typo'd token (both halves reporting it fails the mint) +var private g_tune_only_no_match = false var private g_incumbent_loaded = false def private gen_incumbent(entry : string) : string { @@ -440,6 +443,9 @@ struct KqFixture { def kq_qsb(fmt : int64) : int64 => dasllama_gemm_schema::kq_qsb(int(fmt)) def kq_ssb(fmt : int64) : int64 => dasllama_gemm_schema::kq_ssb(int(fmt)) +// the family's sidecar entry = its [tune] function name (the K-quants by bit width - k3 included - q40/iq4xs by tag) +def kq_tile_entry(fmt : int64) : string => fmt == 40l ? "q40q8_tile_gen" : (fmt == 44l ? "iq4xsq8_tile_gen" : (fmt == 33l ? "iq3sq8_tile_gen" : (fmt == 34l ? "iq3xxsq8_tile_gen" : (fmt == 45l ? "iq4nlq8_tile_gen" : (fmt == 23l ? "iq2sq8_tile_gen" : (fmt == 24l ? "iq2xsq8_tile_gen" : (fmt == 25l ? "iq2xxsq8_tile_gen" : "k{fmt}q8_tile_gen"))))))) + // synthetic disk superblocks, varied per (row, superblock) so no two blocks repeat: any byte // is a legal quant, and the small d/dmin keep the f16 (s, o) pairs finite and row sums tame def pack_kq_scale_header(var blkb : array; base : int) { @@ -460,7 +466,7 @@ def pack_kq_scale_header(var blkb : array; base : int) { } } -def pack_kq_disk_block(fmt : int64; var blkb : array; base : int) { +def pack_kq_disk_block(fmt : int64; var blkb : array; base : int) { // nolint:STYLE037,STYLE038 — the flat per-format ladder (one arm per kq format) is the honest shape if (fmt == 40l) { for (blk in range(8)) { // 8 x 18B q4_0 disk blocks: f16 d + 16 nibble bytes let dbits = f32_to_f16(0.002 + 0.0001 * float((base + blk) % 13)) @@ -470,6 +476,94 @@ def pack_kq_disk_block(fmt : int64; var blkb : array; base : int) { blkb[blk * 18 + 2 + i] = uint8((base * 37 + blk * 59 + i * 101 + 29) % 256) } } + } elif (fmt == 44l) { + // 136B IQ4_XS disk block: f16 d, u16 scales_h, 4 scales_l bytes, 128 nibble bytes — every byte pattern is legal + let dbits = f32_to_f16(0.002 + 0.0001 * float(base % 13)) + blkb[0] = uint8(dbits & 0xFF) + blkb[1] = uint8(dbits >> 8u) + blkb[2] = uint8((base * 7 + 3) % 256) + blkb[3] = uint8((base * 11 + 5) % 256) + for (i in range(4)) { + blkb[4 + i] = uint8((base * 13 + i * 17 + 9) % 256) + } + for (i in range(128)) { + blkb[8 + i] = uint8((base * 37 + i * 101 + 29) % 256) + } + } elif (fmt == 3l) { + // 110B Q3_K disk block: 32 hmask, 64 qs, 12 packed 6-bit scales, f16 d — every byte pattern is legal + for (i in range(96)) { + blkb[i] = uint8((base * 37 + i * 101 + 29) % 256) + } + for (i in range(12)) { + blkb[96 + i] = uint8((base * 29 + i * 71 + 5) % 256) + } + let dbits = f32_to_f16(0.002 + 0.0001 * float(base % 13)) + blkb[108] = uint8(dbits & 0xFF) + blkb[109] = uint8(dbits >> 8u) + } elif (fmt == 33l) { + // 110B IQ3_S disk block: f16 d, 64 grid-index qs, 8 qh, 32 signs, 4 packed 4-bit scales — every byte pattern is legal + let dbits = f32_to_f16(0.002 + 0.0001 * float(base % 13)) + blkb[0] = uint8(dbits & 0xFF) + blkb[1] = uint8(dbits >> 8u) + for (i in range(104)) { + blkb[2 + i] = uint8((base * 37 + i * 101 + 29) % 256) + } + for (i in range(4)) { + blkb[106 + i] = uint8((base * 29 + i * 71 + 5) % 256) + } + } elif (fmt == 34l) { + // 98B IQ3_XXS disk block: f16 d, 64 grid-index qs, 8 x aux32 — every byte pattern is legal + let dbits = f32_to_f16(0.002 + 0.0001 * float(base % 13)) + blkb[0] = uint8(dbits & 0xFF) + blkb[1] = uint8(dbits >> 8u) + for (i in range(96)) { + blkb[2 + i] = uint8((base * 37 + i * 101 + 29) % 256) + } + } elif (fmt == 45l) { + // 8 x 18B IQ4_NL disk blocks: per block f16 d + 16 nibble bytes — every pattern is legal + for (b in range(8)) { + let dbits = f32_to_f16(0.002 + 0.0001 * float((base + b) % 13)) + blkb[b * 18] = uint8(dbits & 0xFF) + blkb[b * 18 + 1] = uint8(dbits >> 8u) + for (i in range(16)) { + blkb[b * 18 + 2 + i] = uint8((base * 41 + b * 17 + i * 89 + 7) % 256) + } + } + } elif (fmt == 2l) { + // 84B Q2_K disk block: 16 sc/min bytes, 64 qs, f16 d, f16 dmin - every byte pattern is legal + for (i in range(80)) { + blkb[i] = uint8((base * 37 + i * 101 + 29) % 256) + } + let dbits = f32_to_f16(0.002 + 0.0001 * float(base % 13)) + blkb[80] = uint8(dbits & 0xFF) + blkb[81] = uint8(dbits >> 8u) + let mbits = f32_to_f16(0.001 + 0.0001 * float((base + 5) % 11)) + blkb[82] = uint8(mbits & 0xFF) + blkb[83] = uint8(mbits >> 8u) + } elif (fmt == 24l) { + // 74B IQ2_XS disk block: f16 d, 32 u16 qs words, 8 packed scale nibbles - every byte pattern is legal + let dbits = f32_to_f16(0.002 + 0.0001 * float(base % 13)) + blkb[0] = uint8(dbits & 0xFF) + blkb[1] = uint8(dbits >> 8u) + for (i in range(72)) { + blkb[2 + i] = uint8((base * 41 + i * 97 + 17) % 256) + } + } elif (fmt == 25l) { + // 66B IQ2_XXS disk block: f16 d, then 8 blocks x [4 grid bytes][aux32 signs + ls] - every byte pattern is legal + let dbits = f32_to_f16(0.002 + 0.0001 * float(base % 13)) + blkb[0] = uint8(dbits & 0xFF) + blkb[1] = uint8(dbits >> 8u) + for (i in range(64)) { + blkb[2 + i] = uint8((base * 41 + i * 97 + 17) % 256) + } + } elif (fmt == 23l) { + // 82B IQ2_S disk block: f16 d, 32 idx + 32 sign bytes, 8 qh, 8 packed scale nibbles - every byte pattern is legal + let dbits = f32_to_f16(0.002 + 0.0001 * float(base % 13)) + blkb[0] = uint8(dbits & 0xFF) + blkb[1] = uint8(dbits >> 8u) + for (i in range(80)) { + blkb[2 + i] = uint8((base * 37 + i * 101 + 29) % 256) + } } elif (fmt == 4l) { pack_kq_scale_header(blkb, base) for (i in range(128)) { @@ -506,19 +600,37 @@ def repack_kq_grp_fmt(fmt : int64; var kq : uint8?; var ks : uint8?; n, d, mr : repack_k5_grp(kq, ks, n, d, mr) } elif (fmt == 40l) { repack_q40_grp(kq, ks, n, d, mr) + } elif (fmt == 44l) { + repack_iq4xs_grp(kq, ks, n, d, mr) + } elif (fmt == 3l) { + repack_k3_grp(kq, ks, n, d, mr) + } elif (fmt == 33l) { + repack_iq3s_grp(kq, ks, n, d, mr) + } elif (fmt == 34l) { + repack_iq3xxs_grp(kq, ks, n, d, mr) + } elif (fmt == 45l) { + repack_q40_grp(kq, ks, n, d, mr) + } elif (fmt == 2l) { + repack_k2_grp(kq, ks, n, d, mr) + } elif (fmt == 23l) { + repack_iq2s_grp(kq, ks, n, d, mr) + } elif (fmt == 24l) { + repack_iq2xs_grp(kq, ks, n, d, mr) + } elif (fmt == 25l) { + repack_iq2xxs_grp(kq, ks, n, d, mr) } else { repack_k6_grp(kq, ks, n, d, mr) } } -def build_kq_fixture(fmt : int64; n, d, ntok : int64) : KqFixture { +def build_kq_fixture(fmt : int64; n, d, ntok : int64) : KqFixture { // nolint:STYLE037,STYLE038 — the flat per-format ladder (one arm per kq format) is the honest shape var fx = KqFixture(fmt = fmt, n = n, d = d, ntok = ntok, nsb = n / 256l) let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) fx.kq |> resize(d * fx.nsb * qsb) fx.ks |> resize(d * fx.nsb * ssb) var blkb : array - blkb |> resize(fmt == 4l || fmt == 40l ? 144l : (fmt == 5l ? 176l : 210l)) + blkb |> resize(fmt == 2l ? 84l : (fmt == 23l ? 82l : (fmt == 24l ? 74l : (fmt == 25l ? 66l : (fmt == 4l || fmt == 40l || fmt == 45l ? 144l : (fmt == 5l ? 176l : (fmt == 44l ? 136l : (fmt == 3l || fmt == 33l ? 110l : (fmt == 34l ? 98l : 210l))))))))) for (r in range64(d)) { for (s in range64(fx.nsb)) { let base = int(r * 31l + s * 7l) @@ -530,6 +642,24 @@ def build_kq_fixture(fmt : int64; n, d, ntok : int64) : KqFixture { transcode_q5k_superblock(blkb, 0l, fx.kq, sb * qsb, fx.ks, sb * ssb) } elif (fmt == 40l) { transcode_q40_superblock(blkb, 0l, fx.kq, sb * qsb, fx.ks, sb * ssb) + } elif (fmt == 44l) { + transcode_iq4xs_superblock(blkb, 0l, fx.kq, sb * qsb, fx.ks, sb * ssb) + } elif (fmt == 3l) { + transcode_q3k_superblock(blkb, 0l, fx.kq, sb * qsb, fx.ks, sb * ssb) + } elif (fmt == 33l) { + transcode_iq3s_superblock(blkb, 0l, fx.kq, sb * qsb, fx.ks, sb * ssb) + } elif (fmt == 34l) { + transcode_iq3xxs_superblock(blkb, 0l, fx.kq, sb * qsb, fx.ks, sb * ssb) + } elif (fmt == 45l) { + transcode_iq4nl_superblock(blkb, 0l, fx.kq, sb * qsb, fx.ks, sb * ssb) + } elif (fmt == 2l) { + transcode_q2k_superblock(blkb, 0l, fx.kq, sb * qsb, fx.ks, sb * ssb) + } elif (fmt == 23l) { + transcode_iq2s_superblock(blkb, 0l, fx.kq, sb * qsb, fx.ks, sb * ssb) + } elif (fmt == 24l) { + transcode_iq2xs_superblock(blkb, 0l, fx.kq, sb * qsb, fx.ks, sb * ssb) + } elif (fmt == 25l) { + transcode_iq2xxs_superblock(blkb, 0l, fx.kq, sb * qsb, fx.ks, sb * ssb) } else { transcode_q6k_superblock(blkb, 0l, fx.kq, sb * qsb, fx.ks, sb * ssb) } @@ -564,8 +694,35 @@ def build_kq_fixture(fmt : int64; n, d, ntok : int64) : KqFixture { fx.yref[tk * d + g * 4l + r] = (fmt == 40l ? q40_grp_row_dot(kqg, ksg, r, 4l, addr(fx.xq[0]) + tk * n, addr(fx.xs[0]) + tk * (n / 256l), addr(fx.xbs[0]) + tk * (n / 16l), n) - : kq_grp_row_dot(fmt, kqg, ksg, r, 4l, - addr(fx.xq[0]) + tk * n, addr(fx.xs[0]) + tk * (n / 256l), addr(fx.xbs[0]) + tk * (n / 16l), n)) + : (fmt == 44l + ? iq4xs_grp_row_dot(kqg, ksg, r, 4l, + addr(fx.xq[0]) + tk * n, addr(fx.xs[0]) + tk * (n / 256l), addr(fx.xbs[0]) + tk * (n / 16l), n) + : (fmt == 3l + ? k3_grp_row_dot(kqg, ksg, r, 4l, + addr(fx.xq[0]) + tk * n, addr(fx.xs[0]) + tk * (n / 256l), addr(fx.xbs[0]) + tk * (n / 16l), n) + : (fmt == 33l + ? iq3s_grp_row_dot(kqg, ksg, r, 4l, + addr(fx.xq[0]) + tk * n, addr(fx.xs[0]) + tk * (n / 256l), addr(fx.xbs[0]) + tk * (n / 16l), n) + : (fmt == 34l + ? iq3xxs_grp_row_dot(kqg, ksg, r, 4l, + addr(fx.xq[0]) + tk * n, addr(fx.xs[0]) + tk * (n / 256l), addr(fx.xbs[0]) + tk * (n / 16l), n) + : (fmt == 45l + ? iq4nl_grp_row_dot(kqg, ksg, r, 4l, + addr(fx.xq[0]) + tk * n, addr(fx.xs[0]) + tk * (n / 256l), addr(fx.xbs[0]) + tk * (n / 16l), n) + : (fmt == 25l + ? iq2xxs_grp_row_dot(kqg, ksg, r, 4l, + addr(fx.xq[0]) + tk * n, addr(fx.xs[0]) + tk * (n / 256l), addr(fx.xbs[0]) + tk * (n / 16l), n) + : (fmt == 24l + ? iq2xs_grp_row_dot(kqg, ksg, r, 4l, + addr(fx.xq[0]) + tk * n, addr(fx.xs[0]) + tk * (n / 256l), addr(fx.xbs[0]) + tk * (n / 16l), n) + : (fmt == 23l + ? iq2s_grp_row_dot(kqg, ksg, r, 4l, + addr(fx.xq[0]) + tk * n, addr(fx.xs[0]) + tk * (n / 256l), addr(fx.xbs[0]) + tk * (n / 16l), n) + : (fmt == 2l + ? k2_grp_row_dot(kqg, ksg, r, 4l, + addr(fx.xq[0]) + tk * n, addr(fx.xs[0]) + tk * (n / 256l), addr(fx.xbs[0]) + tk * (n / 16l), n) + : kq_grp_row_dot(fmt, kqg, ksg, r, 4l, + addr(fx.xq[0]) + tk * n, addr(fx.xs[0]) + tk * (n / 256l), addr(fx.xbs[0]) + tk * (n / 16l), n))))))))))) } } } @@ -587,10 +744,37 @@ def kq_tile_variants(fmt : int64) : array> { if (fmt == 40l) { return <- q40q8_tile_gen_variants() } + if (fmt == 44l) { + return <- iq4xsq8_tile_gen_variants() + } + if (fmt == 3l) { + return <- k3q8_tile_gen_variants() + } + if (fmt == 33l) { + return <- iq3sq8_tile_gen_variants() + } + if (fmt == 34l) { + return <- iq3xxsq8_tile_gen_variants() + } + if (fmt == 45l) { + return <- iq4nlq8_tile_gen_variants() + } + if (fmt == 2l) { + return <- k2q8_tile_gen_variants() + } + if (fmt == 23l) { + return <- iq2sq8_tile_gen_variants() + } + if (fmt == 24l) { + return <- iq2xsq8_tile_gen_variants() + } + if (fmt == 25l) { + return <- iq2xxsq8_tile_gen_variants() + } return <- k6q8_tile_gen_variants() } -def kq_gemv_variants_by_suffix(fmt : int64) : table { +def kq_gemv_variants_by_suffix(fmt : int64) : table { // nolint:STYLE037,STYLE038 — the flat per-format ladder (one arm per kq format) is the honest shape var tab : table if (fmt == 4l) { var gvs <- k4q8_gemv_gen_variants() @@ -610,6 +794,60 @@ def kq_gemv_variants_by_suffix(fmt : int64) : table { tab[v._0] = v._1 } delete gvs + } elif (fmt == 44l) { + var gvs <- iq4xsq8_gemv_gen_variants() + for (v in gvs) { + tab[v._0] = v._1 + } + delete gvs + } elif (fmt == 3l) { + var gvs <- k3q8_gemv_gen_variants() + for (v in gvs) { + tab[v._0] = v._1 + } + delete gvs + } elif (fmt == 33l) { + var gvs <- iq3sq8_gemv_gen_variants() + for (v in gvs) { + tab[v._0] = v._1 + } + delete gvs + } elif (fmt == 34l) { + var gvs <- iq3xxsq8_gemv_gen_variants() + for (v in gvs) { + tab[v._0] = v._1 + } + delete gvs + } elif (fmt == 45l) { + var gvs <- iq4nlq8_gemv_gen_variants() + for (v in gvs) { + tab[v._0] = v._1 + } + delete gvs + } elif (fmt == 24l) { + var gvs <- iq2xsq8_gemv_gen_variants() + for (v in gvs) { + tab[v._0] = v._1 + } + delete gvs + } elif (fmt == 25l) { + var gvs <- iq2xxsq8_gemv_gen_variants() + for (v in gvs) { + tab[v._0] = v._1 + } + delete gvs + } elif (fmt == 23l) { + var gvs <- iq2sq8_gemv_gen_variants() + for (v in gvs) { + tab[v._0] = v._1 + } + delete gvs + } elif (fmt == 2l) { + var gvs <- k2q8_gemv_gen_variants() + for (v in gvs) { + tab[v._0] = v._1 + } + delete gvs } else { var gvs <- k6q8_gemv_gen_variants() for (v in gvs) { @@ -622,7 +860,7 @@ def kq_gemv_variants_by_suffix(fmt : int64) : table { // suffix -> plane interleave off the family's OWN layout companion registry (per-format // layouts since the family split — declined rows report the reference grp4) -def kq_layout_mrs(fmt : int64) : table { +def kq_layout_mrs(fmt : int64) : table { // nolint:STYLE037,STYLE038 — the flat per-format ladder (one arm per kq format) is the honest shape var mrs : table if (fmt == 4l) { var lvs <- k4q8_layout_gen_variants() @@ -642,6 +880,60 @@ def kq_layout_mrs(fmt : int64) : table { mrs[v._0] = invoke(v._1) } delete lvs + } elif (fmt == 44l) { + var lvs <- iq4xsq8_layout_gen_variants() + for (v in lvs) { + mrs[v._0] = invoke(v._1) + } + delete lvs + } elif (fmt == 3l) { + var lvs <- k3q8_layout_gen_variants() + for (v in lvs) { + mrs[v._0] = invoke(v._1) + } + delete lvs + } elif (fmt == 33l) { + var lvs <- iq3sq8_layout_gen_variants() + for (v in lvs) { + mrs[v._0] = invoke(v._1) + } + delete lvs + } elif (fmt == 34l) { + var lvs <- iq3xxsq8_layout_gen_variants() + for (v in lvs) { + mrs[v._0] = invoke(v._1) + } + delete lvs + } elif (fmt == 45l) { + var lvs <- iq4nlq8_layout_gen_variants() + for (v in lvs) { + mrs[v._0] = invoke(v._1) + } + delete lvs + } elif (fmt == 24l) { + var lvs <- iq2xsq8_layout_gen_variants() + for (v in lvs) { + mrs[v._0] = invoke(v._1) + } + delete lvs + } elif (fmt == 25l) { + var lvs <- iq2xxsq8_layout_gen_variants() + for (v in lvs) { + mrs[v._0] = invoke(v._1) + } + delete lvs + } elif (fmt == 23l) { + var lvs <- iq2sq8_layout_gen_variants() + for (v in lvs) { + mrs[v._0] = invoke(v._1) + } + delete lvs + } elif (fmt == 2l) { + var lvs <- k2q8_layout_gen_variants() + for (v in lvs) { + mrs[v._0] = invoke(v._1) + } + delete lvs } else { var lvs <- k6q8_layout_gen_variants() for (v in lvs) { @@ -659,7 +951,7 @@ def kq_layout_mrs(fmt : int64) : table { def run_kq_tile(tile; fx : KqFixture; kq : array; ks : array; mr : int64; var y : array) { let ng = fx.d / mr verify(fx.ntok % 4l == 0l) - let packed = fx.fmt == 4l || fx.fmt == 40l // pure-nibble planes: the tile reads them directly + let packed = kq_reads_packed_planes(fx.fmt) var panel : array if (!packed) { panel |> resize(int(mr * fx.nsb * 256l)) @@ -961,7 +1253,7 @@ def kq_test_family(fmt : int64; kfxs : array) : bool { // and the gemv (streamed + hot decode shapes), print the table, and return the tile-best // suffix ("" = a gate failed). The kq gemv is nrsplit-independent — same-mr rows share the // plane and the gemv shape — so the tile bench decides the family entry. -def kq_tune_family(fmt : int64; var rows_out : table) : string { // nolint:STYLE037,STYLE038 — one family race: shared repacked buffers couple gate, bench and pick +def kq_tune_family(fmt : int64; var rows_out : table) : string { // nolint:STYLE038 — one family race: shared repacked buffers couple gate, bench and pick // batch shape: fat STREAMING ffn (weights > the L2 budget, multi-token-block walk) — the shape // class that carries ~90% of prefill time. The old 2048x512x64 kv-projection probe was L2-hot // and crowned hot-shape winners that lose double-digit % at model scale (mr4 vs mr8, M1). @@ -1086,7 +1378,7 @@ def kq_tune_family(fmt : int64; var rows_out : table) : stri tune_detail("k{fmt} {tvs[vi]._0} (mr={vmr[vi]}): tile {macs / best[vi] / 1000.0lf} GMAC/s, gemv stream {gmacs / gbest[vi] / 1000.0lf} GMAC/s ({gwbytes / gbest[vi] / 1000.0lf} GB/s w), hot {hmacs / hbest[vi] / 1000.0lf} GMAC/s (best of {ROUNDS})\n") } var tnames <- [for (t in tvs); t._0] - winner = gen_sticky(fmt == 40l ? "q40q8_tile_gen" : "k{fmt}q8_tile_gen", tnames, best, winner) + winner = gen_sticky(kq_tile_entry(fmt), tnames, best, winner) delete tnames let wname = tvs[winner]._0 delete keep @@ -1139,6 +1431,15 @@ def test_mode : bool { // nolint:STYLE037,STYLE038 — flat per-variant correctn var kfxs5 <- [ <- build_kq_fixture(5l, 256l, 32l, 8l), <- build_kq_fixture(5l, 768l, 32l, 8l), <- build_kq_fixture(5l, 2048l, 32l, 8l)] var kfxs6 <- [ <- build_kq_fixture(6l, 256l, 32l, 8l), <- build_kq_fixture(6l, 768l, 32l, 8l), <- build_kq_fixture(6l, 2048l, 32l, 8l)] var kfxs40 <- [ <- build_kq_fixture(40l, 256l, 32l, 8l), <- build_kq_fixture(40l, 768l, 32l, 8l), <- build_kq_fixture(40l, 2048l, 32l, 8l)] + var kfxs44 <- [ <- build_kq_fixture(44l, 256l, 32l, 8l), <- build_kq_fixture(44l, 768l, 32l, 8l), <- build_kq_fixture(44l, 2048l, 32l, 8l)] + var kfxs3 <- [ <- build_kq_fixture(3l, 256l, 32l, 8l), <- build_kq_fixture(3l, 768l, 32l, 8l), <- build_kq_fixture(3l, 2048l, 32l, 8l)] + var kfxs33 <- [ <- build_kq_fixture(33l, 256l, 32l, 8l), <- build_kq_fixture(33l, 768l, 32l, 8l), <- build_kq_fixture(33l, 2048l, 32l, 8l)] + var kfxs34 <- [ <- build_kq_fixture(34l, 256l, 32l, 8l), <- build_kq_fixture(34l, 768l, 32l, 8l), <- build_kq_fixture(34l, 2048l, 32l, 8l)] + var kfxs45 <- [ <- build_kq_fixture(45l, 256l, 32l, 8l), <- build_kq_fixture(45l, 768l, 32l, 8l), <- build_kq_fixture(45l, 2048l, 32l, 8l)] + var kfxs2 <- [ <- build_kq_fixture(2l, 256l, 32l, 8l), <- build_kq_fixture(2l, 768l, 32l, 8l), <- build_kq_fixture(2l, 2048l, 32l, 8l)] + var kfxs23 <- [ <- build_kq_fixture(23l, 256l, 32l, 8l), <- build_kq_fixture(23l, 768l, 32l, 8l), <- build_kq_fixture(23l, 2048l, 32l, 8l)] + var kfxs24 <- [ <- build_kq_fixture(24l, 256l, 32l, 8l), <- build_kq_fixture(24l, 768l, 32l, 8l), <- build_kq_fixture(24l, 2048l, 32l, 8l)] + var kfxs25 <- [ <- build_kq_fixture(25l, 256l, 32l, 8l), <- build_kq_fixture(25l, 768l, 32l, 8l), <- build_kq_fixture(25l, 2048l, 32l, 8l)] var vs <- q8q8_tile_gen_variants() var mrs <- variant_mrs() var wbs <- variant_wbias() @@ -1166,7 +1467,7 @@ def test_mode : bool { // nolint:STYLE037,STYLE038 — flat per-variant correctn if (v._0 == "reference") { verify(!live) } - if (int(mr) != GEMM_REFERENCE_MR && !live) { + if (int(mr) != GEMM_REFERENCE_MR() && !live) { print(" WITNESS DESYNC {v._0}: layout mr={mr} but witness says not live\n") allok = false } @@ -1263,6 +1564,15 @@ def test_mode : bool { // nolint:STYLE037,STYLE038 — flat per-variant correctn allok = kq_test_family(5l, kfxs5) && allok allok = kq_test_family(6l, kfxs6) && allok allok = kq_test_family(40l, kfxs40) && allok + allok = kq_test_family(44l, kfxs44) && allok + allok = kq_test_family(3l, kfxs3) && allok + allok = kq_test_family(33l, kfxs33) && allok + allok = kq_test_family(34l, kfxs34) && allok + allok = kq_test_family(45l, kfxs45) && allok + allok = kq_test_family(2l, kfxs2) && allok + allok = kq_test_family(23l, kfxs23) && allok + allok = kq_test_family(24l, kfxs24) && allok + allok = kq_test_family(25l, kfxs25) && allok delete wtab delete mtab delete gtab @@ -1278,6 +1588,15 @@ def test_mode : bool { // nolint:STYLE037,STYLE038 — flat per-variant correctn delete kfxs5 delete kfxs6 delete kfxs40 + delete kfxs44 + delete kfxs3 + delete kfxs33 + delete kfxs34 + delete kfxs45 + delete kfxs2 + delete kfxs23 + delete kfxs24 + delete kfxs25 return allok } @@ -1505,18 +1824,11 @@ def private confirm_winner(winner : string) : string { return fb } -def tune_mode_run : bool { // nolint:STYLE037,STYLE038 — flat family-by-family race rail; gates + deferred flush in call order - let tune_total_t0 = ref_time_ticks() - // refuse before burning minutes measuring noise; every winner below queues into `pending` - // and nothing reaches the sidecar until the end-of-run gate passes - if (!noise_gate("start")) { - return false - } - var pending : table - // q8q8 + four kq formats + q51 + the end-to-end confirm - tune_progress_plan("dasllama_generators", 7) - interrupt_gate("q8q8_tile_gen") - tune_progress_kernel_begin("q8q8_tile_gen", ROUNDS) +// The q8q8 family race: the fat-batch tile bench +// picks the tile knobs (and with them the layout), the hot decode bench picks gkstep among the +// rows sharing them, incumbent stickiness last. Returns the merged winner's suffix; "" means a +// correctness gate failed and the tune must abort (the mismatch is printed inside). +def private q8q8_tune_family : string { // nolint:STYLE037,STYLE038 - the q8q8 race rail, one flat pass // batch shape: fat STREAMING ffn like the kq fixture — the old 2048x512x64 kv-projection was // L2-hot; divergent crowns still pass the e2e confirm gate before shipping var fx <- build_fixture(2048l, 8192l, 256l) @@ -1630,13 +1942,13 @@ def tune_mode_run : bool { // nolint:STYLE037,STYLE038 — flat family-by-family var maxdiff = 0.0 if (!close_to_ref(fx, y, maxdiff)) { print("{vs[vi]._0}: tile MISMATCH (maxdiff {maxdiff}) - aborting tune\n") - return false + return "" } run_gemv(gtab[vs[vi]._0], gfx, gbufs[vi].q, gbufs[vi].s, vmr[vi], yg) var gmaxdiff = 0.0 if (!close_to_ref(gfx, yg, gmaxdiff)) { print("{vs[vi]._0}: gemv MISMATCH (maxdiff {gmaxdiff}) - aborting tune\n") - return false + return "" } } // the hot shape runs HOT_REPS back-to-back per timing sample: 1MB of weights stays @@ -1709,26 +2021,87 @@ def tune_mode_run : bool { // nolint:STYLE037,STYLE038 — flat family-by-family merged = gen_sticky("q8q8_tile_gen", vnames, incSameTile ? hbest : best, merged) delete vnames delete keep + let w = vs[merged]._0 + delete y + delete yg + delete yh + delete bufs + delete gbufs + delete hbufs + delete vmr + delete vts + delete vgs + delete best + delete gbest + delete hbest + delete gtab + delete ctab + delete tss + delete kgs + delete wbs + delete mrs + delete vs + delete fx + delete gfx + delete hfx + return w +} + +def tune_mode_run : bool { // nolint:STYLE037,STYLE038 — flat family-by-family race rail; gates + deferred flush in call order + let tune_total_t0 = ref_time_ticks() + // the --tune-only re-mint filter: only matching families race (q8q8 brings its e2e + // confirm along); a skipped family writes no entry, so its previous seat survives the + // upsert - a one-family re-mint after landing a format is seconds, not the full walk + var kqSel <- [for (fmt in fixed_array(4l, 5l, 6l, 40l, 44l, 3l, 33l, 34l, 45l, 2l, 23l, 24l, 25l)); fmt; where tune_family_selected(kq_tile_entry(fmt))] + let wantQ8 = tune_family_selected("q8q8_tile_gen") + let wantQ51 = tune_family_selected("q51q8_tile_gen") + let planTotal = (wantQ8 ? 2 : 0) + length(kqSel) + (wantQ51 ? 1 : 0) + if (planTotal == 0) { + print("tune-only '{tune_only_filter()}': no generator family matches - half skipped, sidecar untouched\n") + delete kqSel + g_tune_only_no_match = true + return true + } + if (!empty(tune_only_filter())) { + tune_detail("tune-only '{tune_only_filter()}': racing {planTotal - (wantQ8 ? 1 : 0)} of 15 generator families\n") + } + // refuse before burning minutes measuring noise; every winner below queues into `pending` + // and nothing reaches the sidecar until the end-of-run gate passes + if (!noise_gate("start")) { + return false + } + var pending : table + tune_progress_plan("dasllama_generators", planTotal) + var q8winner = "" + if (wantQ8) { + interrupt_gate("q8q8_tile_gen") + let q8_t0 = ref_time_ticks() + tune_progress_kernel_begin("q8q8_tile_gen", ROUNDS) + q8winner = q8q8_tune_family() + tune_detail("TUNE_GEN_TIME q8q8_family {get_time_usec(q8_t0) / 1000} ms\n") + tune_progress_kernel_end("q8q8_tile_gen", q8winner, "beats") + if (empty(q8winner)) { + return false // a tile/gemv correctness gate failed - the mismatch is named above + } + } + if (!noise_gate("mid1")) { + print("aborting mid-run - no sidecar written\n") + return false + } // the kq families tune separately — one manifest entry per format, tile-best wins (the // gemv is nrsplit-independent, so same-mr rows share its plane and kernel shape). No // child-process confirm gate here: the kq stamp only moves the kq planes' interleave, // and the e2e exposure (a tile-optimal mr shaving decode kernel rate) is bounded by the // decode path staying DRAM-bound — validated end-to-end when the entries first landed. - tune_detail("TUNE_GEN_TIME q8q8_family {get_time_usec(tune_total_t0) / 1000} ms\n") - tune_progress_kernel_end("q8q8_tile_gen", vs[merged]._0, "beats") - if (!noise_gate("mid1")) { - print("aborting mid-run - no sidecar written\n") - return false - } - for (fmt in fixed_array(4l, 5l, 6l, 40l)) { + for (fmt in kqSel) { interrupt_gate("k{fmt}q8_tile_gen") let kq_t0 = ref_time_ticks() - tune_progress_kernel_begin("k{fmt}q8_tile_gen", ROUNDS) + let entry = kq_tile_entry(fmt) + tune_progress_kernel_begin(entry, ROUNDS) var krows : table let w = kq_tune_family(fmt, krows) - tune_detail("TUNE_GEN_TIME k{fmt}_family {get_time_usec(kq_t0) / 1000} ms\n") - tune_progress_kernel_end("k{fmt}q8_tile_gen", w, empty(w) ? "rejected" : "beats") - let entry = fmt == 40l ? "q40q8_tile_gen" : "k{fmt}q8_tile_gen" + tune_detail("TUNE_GEN_TIME {entry} family {get_time_usec(kq_t0) / 1000} ms\n") + tune_progress_kernel_end(entry, w, empty(w) ? "rejected" : "beats") // gate failure (the broken perm is named above): queue the always-correct reference // body so the scope still COMPLETES — a missing key re-tunes every start pending[entry] = empty(w) ? "reference" : w @@ -1739,30 +2112,35 @@ def tune_mode_run : bool { // nolint:STYLE037,STYLE038 — flat family-by-family print("aborting mid-run - no sidecar written\n") return false } - interrupt_gate("q51q8_tile_gen") - let q51_t0 = ref_time_ticks() - // the manifest key must match the [tune] fn (the TILE) or the stamp misses its own crown; - // the race still benches the gemv shape - tune_progress_kernel_begin("q51q8_tile_gen", ROUNDS) - var qrows : table - let qw = q51_tune_family(qrows) - tune_detail("TUNE_GEN_TIME q51_family {get_time_usec(q51_t0) / 1000} ms\n") - tune_progress_kernel_end("q51q8_tile_gen", qw, empty(qw) ? "rejected" : "beats") - // same gate-failure fallback as the kq loop: complete the scope on the reference body - pending["q51q8_tile_gen"] = empty(qw) ? "reference" : qw - g_rows["q51q8_tile_gen"] = JV(qrows) - print("q51q8_tile_gen winner: {empty(qw) ? "reference (gate-failure fallback)" : qw}\n") - interrupt_gate("confirm_e2e_prefill") - let confirm_t0 = ref_time_ticks() - tune_progress_kernel_begin("confirm_e2e_prefill", 0) - let cw = confirm_winner(vs[merged]._0) - var ok = !empty(cw) - if (ok) { - pending["q8q8_tile_gen"] = cw - print("q8q8_tile_gen winner: {cw}\n") + if (wantQ51) { + interrupt_gate("q51q8_tile_gen") + let q51_t0 = ref_time_ticks() + // the manifest key must match the [tune] fn (the TILE) or the stamp misses its own crown; + // the race still benches the gemv shape + tune_progress_kernel_begin("q51q8_tile_gen", ROUNDS) + var qrows : table + let qw = q51_tune_family(qrows) + tune_detail("TUNE_GEN_TIME q51_family {get_time_usec(q51_t0) / 1000} ms\n") + tune_progress_kernel_end("q51q8_tile_gen", qw, empty(qw) ? "rejected" : "beats") + // same gate-failure fallback as the kq loop: complete the scope on the reference body + pending["q51q8_tile_gen"] = empty(qw) ? "reference" : qw + g_rows["q51q8_tile_gen"] = JV(qrows) + print("q51q8_tile_gen winner: {empty(qw) ? "reference (gate-failure fallback)" : qw}\n") + } + var ok = true + if (wantQ8) { + interrupt_gate("confirm_e2e_prefill") + let confirm_t0 = ref_time_ticks() + tune_progress_kernel_begin("confirm_e2e_prefill", 0) + let cw = confirm_winner(q8winner) + ok = !empty(cw) + if (ok) { + pending["q8q8_tile_gen"] = cw + print("q8q8_tile_gen winner: {cw}\n") + } + tune_detail("TUNE_GEN_TIME confirm {get_time_usec(confirm_t0) / 1000} ms\n") + tune_progress_kernel_end("confirm_e2e_prefill", "", ok ? "holds" : "rejected") } - tune_detail("TUNE_GEN_TIME confirm {get_time_usec(confirm_t0) / 1000} ms\n") - tune_progress_kernel_end("confirm_e2e_prefill", "", ok ? "holds" : "rejected") tune_progress_finish() // the only write, behind the last gate: a noisy end-probe means NO entry is written — the // winners above were raced in conditions the probe just proved untrustworthy @@ -1788,28 +2166,7 @@ def tune_mode_run : bool { // nolint:STYLE037,STYLE038 — flat family-by-family } delete pending tune_detail("TUNE_GEN_TIME total {get_time_usec(tune_total_t0) / 1000} ms\n") - delete y - delete yg - delete yh - delete bufs - delete gbufs - delete hbufs - delete vmr - delete vts - delete vgs - delete best - delete gbest - delete hbest - delete gtab - delete ctab - delete tss - delete kgs - delete wbs - delete mrs - delete vs - delete fx - delete gfx - delete hfx + delete kqSel return ok } @@ -1852,7 +2209,7 @@ def main : int { if (mode == "tune") { let rok = tune_mode_run() print(rok ? "GEN TUNE RUN OK\n" : "GEN TUNE RUN FAIL\n") - return rok ? 0 : 1 + return rok ? (g_tune_only_no_match ? 3 : 0) : 1 } print("set DAS_TUNE_MODE=test or =tune (compile-time env: it stamps the variant grid)\n") return 1 diff --git a/modules/dasLLAMA/harness/tune_kernels.das b/modules/dasLLAMA/harness/tune_kernels.das index 67f19c63fa..f8660d817d 100644 --- a/modules/dasLLAMA/harness/tune_kernels.das +++ b/modules/dasLLAMA/harness/tune_kernels.das @@ -2336,13 +2336,60 @@ def bench_laneq4x4() : string { // nolint:STYLE038 — one bench: fixture, backe return report("dot_q8q8_laneq4x4", names, best, ok) } +// one METAL_TWIN race row: log it and bank the family when the tensor twin won +def private metal_twin_absorb(var wins : array; var ran : bool&; r) { + ran = true + let mt_note = empty(r.note) ? "" : " ({r.note})" + tune_detail("METAL_TWIN {r.family}: base={r.base_ms}ms tensor={r.twin_ms}ms -> {r.winner}{mt_note}\n") + if (r.winner == "tensor") { + wins |> push(r.family) + } +} + // fallback: the tuner rail drives this half explicitly (DAS_TUNE_MODE=tune) — the auto // policy must not fire a nested tuner off this program's own missing per-app sidecar +//! Exit code of a `--tune-only` half whose filter matched nothing here: the wrapper fails the mint +//! only when BOTH halves report it (a filter naming one half's family is not a typo). +let TUNE_ONLY_NO_MATCH = 3 + [export, tune_policy(missing = "fallback")] def main : int { // nolint:STYLE037,STYLE038 — flat one-bench-per-kernel rail; the noise probes are its checkpoints if (tuner_paranoid_requested()) { tune_detail("kernel tuner: --tune-paranoid is retired - one protocol, margin-decided; running it\n") } + // every bench by name, declared up front so the --tune-only filter can prune it before + // the job queue even spins up; the sweep runs the list in order, validation re-runs the + // changed ones + var benches <- [ + ("dot", @@bench_dot), ("axpy", @@bench_axpy), ("dot_f16", @@bench_dot_f16), + ("axpy_f16", @@bench_axpy_f16), ("cvt_f32_to_f16", @@bench_cvt_f32_to_f16), + ("cvt_f16_to_f32", @@bench_cvt_f16_to_f32), ("add_inplace", @@bench_add_inplace), + ("mul_inplace", @@bench_mul_inplace), ("scale_inplace", @@bench_scale), + ("copy_floats", @@bench_copy_floats), ("softmax", @@bench_softmax), ("rmsnorm", @@bench_rmsnorm), + ("dot_q4", @@bench_dot_q4), ("dot_q8q8", @@bench_dot_q8q8), ("dot_q8kv", @@bench_dot_q8kv), + ("dot_q8q8kv", @@bench_dot_q8q8kv), ("axpy_q8kv", @@bench_axpy_q8kv), + ("cvt_q8kv_to_f32", @@bench_cvt_q8kv_to_f32), ("quantize_q8kv_row", @@bench_quantize_q8kv_row), + ("dot_mx4q8", @@bench_dot_mx4q8), ("quantize_q8_0_into_ptr", @@bench_quantize), + ("quantize_q8_0_bs_into_ptr", @@bench_quantize_bs), ("rope_scaled_neox_tab", @@bench_rope_tab), + ("gemm_f32_uk_4x16", @@bench_gemm_tile), + ("dot_q8q8_laneq4x4", @@bench_laneq4x4) + ] + let partialMint = !empty(tune_only_filter()) + if (partialMint) { + let nb = length(benches) + for (k in range(nb)) { + let i = nb - 1 - k + if (!tune_family_selected(benches[i]._0)) { + benches |> erase(i) + } + } + if (empty(benches)) { + print("tune-only '{tune_only_filter()}': no loop-hint kernel matches - half skipped, sidecar untouched\n") + delete benches + return TUNE_ONLY_NO_MATCH + } + tune_detail("tune-only '{tune_only_filter()}': racing {length(benches)} loop-hint kernel(s); the Metal tensor race and the runtime knob snapshot sit out (previous sidecar values survive)\n") + } let affinity_inherited = is_some(g_env_core.jobque_affinity) if (!affinity_inherited) { set_jobque_affinity(2) // JobQue pins its creator (this measurement thread) to physical CPU 0 @@ -2357,24 +2404,9 @@ def main : int { // nolint:STYLE037,STYLE038 — flat one-bench-per-kernel rail; if (!noise_gate("start")) { return 1 } - tune_progress_plan("dasllama_kernels", TUNED_KERNEL_COUNT + VALIDATE_PLAN_ALLOWANCE) + tune_progress_plan("dasllama_kernels", length(benches) + VALIDATE_PLAN_ALLOWANCE) load_incumbents() - // every bench by name: the sweep runs the list in order, validation re-runs the changed ones - var benches <- [ - ("dot", @@bench_dot), ("axpy", @@bench_axpy), ("dot_f16", @@bench_dot_f16), - ("axpy_f16", @@bench_axpy_f16), ("cvt_f32_to_f16", @@bench_cvt_f32_to_f16), - ("cvt_f16_to_f32", @@bench_cvt_f16_to_f32), ("add_inplace", @@bench_add_inplace), - ("mul_inplace", @@bench_mul_inplace), ("scale_inplace", @@bench_scale), - ("copy_floats", @@bench_copy_floats), ("softmax", @@bench_softmax), ("rmsnorm", @@bench_rmsnorm), - ("dot_q4", @@bench_dot_q4), ("dot_q8q8", @@bench_dot_q8q8), ("dot_q8kv", @@bench_dot_q8kv), - ("dot_q8q8kv", @@bench_dot_q8q8kv), ("axpy_q8kv", @@bench_axpy_q8kv), - ("cvt_q8kv_to_f32", @@bench_cvt_q8kv_to_f32), ("quantize_q8kv_row", @@bench_quantize_q8kv_row), - ("dot_mx4q8", @@bench_dot_mx4q8), ("quantize_q8_0_into_ptr", @@bench_quantize), - ("quantize_q8_0_bs_into_ptr", @@bench_quantize_bs), ("rope_scaled_neox_tab", @@bench_rope_tab), - ("gemm_f32_uk_4x16", @@bench_gemm_tile), - ("dot_q8q8_laneq4x4", @@bench_laneq4x4) // last: it pins the repack backend (sticks until process exit) - ] for (b in benches) { let kernel_t0 = kernel_begin(b._0) invoke(b._1) @@ -2424,7 +2456,7 @@ def main : int { // nolint:STYLE037,STYLE038 — flat one-bench-per-kernel rail; ? "\nevery kernel keeps its shipped fallback - the tuner found nothing new on this box.\n" : "\n{changed} kernel(s) beat their shipped fallback on this box.\n") // the one count still written by hand; say so rather than let the bar quietly misreport - if (length(g_results) != TUNED_KERNEL_COUNT) { + if (!partialMint && length(g_results) != TUNED_KERNEL_COUNT) { print("note: TUNED_KERNEL_COUNT is {TUNED_KERNEL_COUNT} but {length(g_results)} kernels ran - the progress bar was scaled wrong; update the constant\n") } @@ -2435,27 +2467,21 @@ def main : int { // nolint:STYLE037,STYLE038 — flat one-bench-per-kernel rail; static_if (typeinfo builtin_module_exists(das_metal)) { var mt_wins : array var mt_ran = false - var mt_all <- metal_tensor_race() - var mt_dec <- metal_tensor_race_decode() - mt_all |> reserve(length(mt_all) + length(mt_dec)) - for (r in mt_dec) { - mt_all |> emplace(r) - } - delete mt_dec - for (r in mt_all) { - mt_ran = true - let mt_note = empty(r.note) ? "" : " ({r.note})" - tune_detail("METAL_TWIN {r.family}: base={r.base_ms}ms tensor={r.twin_ms}ms -> {r.winner}{mt_note}\n") - if (r.winner == "tensor") { - mt_wins |> push(r.family) - } - } - if (!mt_ran) { + if (!partialMint) { // a filtered mint leaves "runtime" untouched - no crowns to record + // the two race halves come through different require paths, so their result element + // types do not unify into one typed local - absorb each returned array where it lands + for (r in metal_tensor_race()) { + metal_twin_absorb(mt_wins, mt_ran, r) + } + for (r in metal_tensor_race_decode()) { + metal_twin_absorb(mt_wins, mt_ran, r) + } + } + if (!mt_ran && !partialMint) { tune_detail("METAL_TWIN: no Metal device - tensor race skipped\n") } metal_crowns = join(mt_wins, ",") delete mt_wins - delete mt_all } // softmax_sink shares softmax's loop shape exactly (one extra scalar joins the reduction), so it @@ -2480,6 +2506,12 @@ def main : int { // nolint:STYLE037,STYLE038 — flat one-bench-per-kernel rail; threads = get_total_hw_jobs() + 1, metal_tensor = metal_crowns, metal_tall_floor = metal_tall_floor(), metal_devw_small_panel_mb = metal_devw_small_panel_mb(), metal_cvt_min_rows = metal_cvt_min_rows())) // total lanes (workers + computing main), the DAS_JOBQUE_THREADS unit + if (partialMint) { + // a filtered mint must not clobber the runtime knobs (hand-edited TB/L2 refinements + // live there): a null runtime leaves the sidecar section untouched + delete_json(runtime) + runtime = null + } // winners come from the results table (validation may have demoted one); the [tuned] // kernels this harness doesn't sweep yet keep their SHIPPED fallback as an explicit entry - // the scope completeness check demands every non-pinned [tuned] kernel @@ -2489,6 +2521,9 @@ def main : int { // nolint:STYLE037,STYLE038 — flat one-bench-per-kernel rail; "dot_q8tq4kv" => "vec16", "quantize_tq4kv_row" => "plain", "dot_q8q8_f16s" => "vec16", "dot_q51e" => "vec16" } + if (partialMint) { // a filtered mint seeds nothing - only the raced kernels write + winners |> clear() + } for (r in g_results) { winners[r.kernel] = r.winner if (r.kernel == "softmax") { diff --git a/modules/dasLLAMA/performance/REVIEW.das b/modules/dasLLAMA/performance/REVIEW.das index 2ecfa66fbc..ef6847311b 100644 --- a/modules/dasLLAMA/performance/REVIEW.das +++ b/modules/dasLLAMA/performance/REVIEW.das @@ -105,6 +105,47 @@ def private check_last_known_good { // historical rows legitimately carry older pins and shas - so they stay out of this sweep; // reachability runs in make-pr's chain, where the orphan hazard appears.) let private RECORDS_DIR = "modules/dasLLAMA/performance/records" +let private DEFAULTS_DIR = "modules/dasLLAMA/performance/defaults" + +// the shipped defaults profiles: a quiet mint, a version pin, and NO commit stamp (a +// profile's validity is its pin + feature class; an engine_sha would be an orphan after +// every rebase and says nothing a reader can act on) +def private check_defaults_profiles { + dir(DEFAULTS_DIR) $(name) { + return if (!(name |> ends_with(".tune-defaults.json"))) + let full = "{DEFAULTS_DIR}/{name}" + let text = fread(full) + if (empty(text)) { + gate_finding(full, "missing or unreadable") + return + } + var jerr = "" + var doc = read_json(text, jerr) + defer() { + unsafe { + delete doc + } + } + if (doc == null) { + gate_finding(full, "does not parse as JSON: {jerr}") + return + } + let noise = "{doc?["provenance"]?["noise"] ?? ""}" + if (noise != "ok") { + gate_finding(full, "provenance.noise is '{noise}' - a shipped profile carries a quiet mint (noise ok)") + } + if (empty("{doc?["provenance"]?["dasllama_version"] ?? ""}")) { + gate_finding(full, "no provenance.dasllama_version - a profile with no version pin adopts on every build") + } + if (!empty("{doc?["provenance"]?["engine_sha"] ?? ""}")) { + gate_finding(full, "carries provenance.engine_sha - a profile ships no commit stamp (tune_profile_export strips it)") + } + let kernels = doc?["kernels"] + if (kernels == null || !(kernels.value is _object) || empty(kernels.value as _object)) { + gate_finding(full, "no kernels section - not a profile") + } + } +} def private check_records_corpus { dir(RECORDS_DIR) $(name) { @@ -156,6 +197,7 @@ def main() : int { return 2 } check_last_known_good() + check_defaults_profiles() check_schema_engine_free() check_single_exchange_client() check_records_corpus() diff --git a/modules/dasLLAMA/performance/REVIEW.md b/modules/dasLLAMA/performance/REVIEW.md index 8cbcd39137..fa6a5bd4f3 100644 --- a/modules/dasLLAMA/performance/REVIEW.md +++ b/modules/dasLLAMA/performance/REVIEW.md @@ -8,29 +8,37 @@ validate through `../dasllama/dasllama_exchange_schema.das` instead.** The engin `dasllama/` require beyond the lint macro module) is `REVIEW.das`'s to enforce; weakening that gate is a defect. -**Weakening `REVIEW.das`'s record-hygiene checks - `hardware.remote_desktop` `off` on every -stored run, `provenance.noise` `ok` on every sidecar - is a defect; the fix for a red is a -re-mint on a quiet, session-free box, never an edit.** +**Weakening `REVIEW.das`'s record-hygiene checks is a defect; the fix for a red is a re-mint +on a quiet, session-free box, never an edit.** What each check enforces is read from the gate +itself; each check's finding text states its own rule. -**A diff that writes a commit stamp anywhere under this folder - `records/`, -`last_known_good_sidecar.json`, any provenance manifest - naming a commit the branch under -review cannot reach is a defect - re-mint, or re-stamp to a reachable commit whose +**A diff that writes a commit stamp anywhere under this folder naming a commit the branch +under review cannot reach is a defect - re-mint, or re-stamp to a reachable commit whose `modules/dasLLAMA/` tree is byte-identical to the tree that was measured, with the PR body -naming the re-stamp.** The commit stamps are a `das` row's `sha` and a sidecar's -`provenance.engine_sha`. A stamp that resolves to no commit at all counts as unreachable. -Re-stamping inside a content-addressed archive (`records/.tune..json`) re-hashes -and renames the file and repoints every row's `tune_sha` in the same change. +naming the re-stamp.** The commit stamps are a `das` row's `sha` and a `provenance.engine_sha` +in any checked-in JSON under this folder. A stamp that resolves to no commit at all counts as +unreachable. + +**A diff that re-stamps an archived sidecar (`records/.tune..json`) re-hashes and +renames the file and repoints every `records/.json` row whose `tune_sha` named the old +name, in the same change.** The archive is content-addressed; a row left on the old name +points at a file that no longer exists. **A diff that writes a reference-engine row to `records/` whose `sha` names anything but the standing ref pin (`DEFAULT_REF_SHA`, `../benchmarks/setup_lcpp_ref.das`) is a defect - re-mint.** A reference row that carries no `sha` (the cli and python reference tools) is pinned by its builder instead - the cli tools by that same ref pin's checkout, the python -legs by `../benchmarks/asr/requirements-*.txt` - and a diff that moves one of those pins -re-mints every row the moved pin's tool minted, or withdraws them. +legs by `../benchmarks/asr/requirements-*.txt`. + +**A diff that changes a reference-tool pin - `DEFAULT_REF_SHA` in +`../benchmarks/setup_lcpp_ref.das`, or a python leg's +`../benchmarks/asr/requirements-*.txt` - re-mints every row that pin's tool minted, or +withdraws them.** -**A diff that writes a sidecar to `records/` whose `provenance.dasllama_version` differs from -`DASLLAMA_RELEASE` (`../dasllama/dasllama_version.das`) is a defect - re-mint.** Read -`DASLLAMA_RELEASE` at the commit the sidecar's `provenance.engine_sha` names. +**A diff that writes a provenance manifest under this folder whose `provenance.dasllama_version` +differs from the `DASLLAMA_RELEASE` string (`../dasllama/dasllama_version.das`) is a defect - +re-mint.** For a sidecar with an `engine_sha`, read the value at that commit; a `defaults/` +profile compares against the branch under review. **A diff that writes a row to `records/.json` mints that row from a board cell.** A board cell is one `gen_bench_records.das` spawns, or a manual `../benchmarks/lcpp_bench.das` cell @@ -91,9 +99,6 @@ the row's canonical values** - the verify never reads the url of a file already **A diff that refreshes `last_known_good_sidecar.json` replaces it with one complete mint from the box its provenance names, at the current `DASLLAMA_RELEASE` - never a hand-edited copy.** -`REVIEW.das` (beside this file) machine-checks the age-independent half (`noise` `ok`, -`validation` `ok`, every `race` winner equal to its `kernels` value); weakening that gate is a -defect. **A diff that changes the exchange consent notice wording updates every checked-in copy in the same change, and a diff that adds a copy names it here in the same change: @@ -101,4 +106,3 @@ the same change, and a diff that adds a copy names it here in the same change: `CONSENT_TEXT` (`utils/watchdog/watchdog.py`, repo root), and the captured `utils/dasllama-server/tests/fixtures/exchange.json` (repo root).** The console prompt and the control page render the served constant, so they are not copies. - diff --git a/modules/dasLLAMA/performance/defaults/arm-neon.tune-defaults.json b/modules/dasLLAMA/performance/defaults/arm-neon.tune-defaults.json new file mode 100644 index 0000000000..7bef73466d --- /dev/null +++ b/modules/dasLLAMA/performance/defaults/arm-neon.tune-defaults.json @@ -0,0 +1,69 @@ +{ + "kernels": { + "add_inplace": "vec8_u2", + "add_scale_inplace": "vec8_u2", + "axpy": "vec8_u2", + "axpy_f16": "vec4_u4", + "axpy_q8kv": "u2", + "axpy_tq4kv": "vec8_u2", + "copy_floats": "vec8_u2", + "cvt_f16_to_f32": "vec16", + "cvt_f32_to_f16": "vec8_u2", + "cvt_q8kv_to_f32": "vec8_u2", + "cvt_tq4kv_to_f32": "vec8_u2", + "dot": "vec8_u2", + "dot_bf16": "vec8_u2", + "dot_f16": "vec8_u2", + "dot_mx4q8": "vec4_u4", + "dot_q4": "vec4_u4", + "dot_q51e": "vec16", + "dot_q8kv": "vec4_u4", + "dot_q8q8": "vec16", + "dot_q8q8_f16s": "vec16", + "dot_q8q8_laneq4x4": "u2", + "dot_q8q8kv": "vec16", + "dot_q8tq4kv": "vec16", + "gemm_f32_uk_4x16": "u2", + "iq2sq8_tile_gen": "mr8", + "iq2xsq8_tile_gen": "mr8", + "iq2xxsq8_tile_gen": "mr8", + "iq3sq8_tile_gen": "mr8", + "iq3xxsq8_tile_gen": "mr8", + "iq4nlq8_tile_gen": "mr8", + "iq4xsq8_tile_gen": "mr8", + "k2q8_tile_gen": "mr8", + "k3q8_tile_gen": "mr8", + "k4q8_tile_gen": "mr8", + "k5q8_tile_gen": "mr8", + "k6q8_tile_gen": "mr8", + "mul_inplace": "vec8_u2", + "q40q8_tile_gen": "mr8", + "q51q8_tile_gen": "mr8", + "q8q8_tile_gen": "mr8_budget", + "quantize_q8_0_bs_into_ptr": "plain", + "quantize_q8_0_into_ptr": "plain", + "quantize_q8kv_row": "plain", + "quantize_tq4kv_row": "plain", + "rmsnorm": "vec8", + "rope_scaled_neox_tab": "vec16", + "scale_inplace": "vec8_u2", + "softmax": "vec8_u2", + "softmax_sink": "vec8_u2" + }, + "provenance": { + "arch": "arm64", + "class": "arm-neon", + "dasllama_version": "15", + "features": "neon;dotprod;fullfp16;lse", + "mode": "normal", + "noise": "ok", + "noise_floor_cv_pct": "0.22", + "noise_probes": "start cv 0.22%; mid1 cv 0.11%; mid2 cv 0.10%; end cv 0.09%", + "origin": "profile", + "platform": "darwin", + "validation": "ok", + "validation_demoted": "0", + "validation_max_drift_pct": "0.13", + "written": "2026-09-01T01:37:39.701Z" + } +} diff --git a/modules/dasLLAMA/performance/defaults/x86-avx2.tune-defaults.json b/modules/dasLLAMA/performance/defaults/x86-avx2.tune-defaults.json new file mode 100644 index 0000000000..3392983b1a --- /dev/null +++ b/modules/dasLLAMA/performance/defaults/x86-avx2.tune-defaults.json @@ -0,0 +1,69 @@ +{ + "kernels": { + "add_inplace": "vec8_u2", + "add_scale_inplace": "vec8_u2", + "axpy": "vec8_u2", + "axpy_f16": "vec8_u2", + "axpy_q8kv": "vec8_u2", + "axpy_tq4kv": "vec8_u2", + "copy_floats": "vec8_u2", + "cvt_f16_to_f32": "vec8_u2", + "cvt_f32_to_f16": "vec8_u2", + "cvt_q8kv_to_f32": "vec8_u2", + "cvt_tq4kv_to_f32": "vec8_u2", + "dot": "vec8_u2", + "dot_bf16": "vec8_u2", + "dot_f16": "vec8_u2", + "dot_mx4q8": "u2", + "dot_q4": "vec8_u2", + "dot_q51e": "vec16", + "dot_q8kv": "vec4_u4", + "dot_q8q8": "vec16", + "dot_q8q8_f16s": "vec16", + "dot_q8q8_laneq4x4": "", + "dot_q8q8kv": "vec16", + "dot_q8tq4kv": "vec16", + "gemm_f32_uk_4x16": "plain", + "iq2sq8_tile_gen": "dot_maddubs_width256_mr8", + "iq2xsq8_tile_gen": "dot_maddubs_width256_mr8", + "iq2xxsq8_tile_gen": "dot_maddubs_width256_mr8", + "iq3sq8_tile_gen": "dot_maddubs_width256_mr8", + "iq3xxsq8_tile_gen": "dot_maddubs_width256_mr8", + "iq4nlq8_tile_gen": "dot_maddubs_width256_mr8", + "iq4xsq8_tile_gen": "dot_maddubs_width256_mr8", + "k2q8_tile_gen": "dot_maddubs_width256_mr8", + "k3q8_tile_gen": "dot_maddubs_width256_mr8", + "k4q8_tile_gen": "dot_maddubs_width256_mr8", + "k5q8_tile_gen": "dot_maddubs_width256_mr8", + "k6q8_tile_gen": "dot_maddubs_width256_mr8", + "mul_inplace": "vec8_u2", + "q40q8_tile_gen": "dot_maddubs_width256_mr8", + "q51q8_tile_gen": "dot_maddubs_width256_mr8", + "q8q8_tile_gen": "dot_maddubs_width256_mr8_kstep2", + "quantize_q8_0_bs_into_ptr": "vec16_u2", + "quantize_q8_0_into_ptr": "plain", + "quantize_q8kv_row": "plain", + "quantize_tq4kv_row": "plain", + "rmsnorm": "vec16_u2", + "rope_scaled_neox_tab": "plain", + "scale_inplace": "vec8_u2", + "softmax": "vec8_u2", + "softmax_sink": "vec8_u2" + }, + "provenance": { + "arch": "x86_64", + "class": "x86-avx2", + "dasllama_version": "15", + "features": "avx2;f16c;fma;sse4.2", + "mode": "normal", + "noise": "ok", + "noise_floor_cv_pct": "0.33", + "noise_probes": "start cv 0.28%; mid1 cv 0.33%; mid2 cv 0.15%; end cv 0.31%", + "origin": "profile", + "platform": "windows", + "validation": "ok", + "validation_demoted": "0", + "validation_max_drift_pct": "19.46", + "written": "2026-09-01T01:06:53.218Z" + } +} diff --git a/modules/dasLLAMA/performance/defaults/x86-vnni512.tune-defaults.json b/modules/dasLLAMA/performance/defaults/x86-vnni512.tune-defaults.json new file mode 100644 index 0000000000..ebedf6abd3 --- /dev/null +++ b/modules/dasLLAMA/performance/defaults/x86-vnni512.tune-defaults.json @@ -0,0 +1,69 @@ +{ + "kernels": { + "add_inplace": "vec8_u2", + "add_scale_inplace": "vec8_u2", + "axpy": "vec8_u2", + "axpy_f16": "vec8_u2", + "axpy_q8kv": "plain", + "axpy_tq4kv": "vec8_u2", + "copy_floats": "vec8_u2", + "cvt_f16_to_f32": "vec8_u2", + "cvt_f32_to_f16": "plain", + "cvt_q8kv_to_f32": "vec8_u2", + "cvt_tq4kv_to_f32": "vec8_u2", + "dot": "vec16", + "dot_bf16": "vec8_u2", + "dot_f16": "vec16", + "dot_mx4q8": "u2", + "dot_q4": "vec8_u2", + "dot_q51e": "vec16", + "dot_q8kv": "vec8_u2", + "dot_q8q8": "u2", + "dot_q8q8_f16s": "vec16", + "dot_q8q8_laneq4x4": "", + "dot_q8q8kv": "u2", + "dot_q8tq4kv": "vec16", + "gemm_f32_uk_4x16": "u2", + "iq2sq8_tile_gen": "dot_vpdpbusd_width512_mr16", + "iq2xsq8_tile_gen": "dot_vpdpbusd_width512_mr16", + "iq2xxsq8_tile_gen": "dot_vpdpbusd_width512_mr16", + "iq3sq8_tile_gen": "dot_vpdpbusd_width512_mr16", + "iq3xxsq8_tile_gen": "dot_vpdpbusd_width512_mr16", + "iq4nlq8_tile_gen": "dot_vpdpbusd_width512_mr16", + "iq4xsq8_tile_gen": "dot_vpdpbusd_width512_mr16", + "k2q8_tile_gen": "dot_vpdpbusd_width512_mr16", + "k3q8_tile_gen": "dot_vpdpbusd_width512_mr16", + "k4q8_tile_gen": "dot_vpdpbusd_width512_mr16", + "k5q8_tile_gen": "dot_vpdpbusd_width512_mr16", + "k6q8_tile_gen": "dot_vpdpbusd_width512_mr16", + "mul_inplace": "vec8_u2", + "q40q8_tile_gen": "dot_vpdpbusd_width512_mr16", + "q51q8_tile_gen": "dot_vpdpbusd_width512_mr16", + "q8q8_tile_gen": "dot_vpdpbusd_width512_mr16_kstep2_gkstep2_bias128", + "quantize_q8_0_bs_into_ptr": "u2", + "quantize_q8_0_into_ptr": "plain", + "quantize_q8kv_row": "plain", + "quantize_tq4kv_row": "plain", + "rmsnorm": "vec8", + "rope_scaled_neox_tab": "vec8_u2", + "scale_inplace": "vec8_u2", + "softmax": "vec8_u2", + "softmax_sink": "vec8_u2" + }, + "provenance": { + "arch": "x86_64", + "class": "x86-vnni512", + "dasllama_version": "15", + "features": "avx2;f16c;fma;sse4.2;avx512f;avx512bw;avx512vl;avx512vnni", + "mode": "normal", + "noise": "ok", + "noise_floor_cv_pct": "0.21", + "noise_probes": "start cv 0.09%; mid1 cv 0.04%; mid2 cv 0.21%; end cv 0.06%", + "origin": "profile", + "platform": "linux", + "validation": "ok", + "validation_demoted": "3", + "validation_max_drift_pct": "21.77", + "written": "2026-09-01T01:30:04.021Z" + } +} diff --git a/modules/dasLLAMA/tests/REVIEW.md b/modules/dasLLAMA/tests/REVIEW.md index 255e718346..7c2e2303ac 100644 --- a/modules/dasLLAMA/tests/REVIEW.md +++ b/modules/dasLLAMA/tests/REVIEW.md @@ -8,15 +8,13 @@ the whole directory.** A change reaches a test when it alters anything the test' depends on - the test file, a shared helper, engine code it exercises, an in-tree fixture or corpus it reads, or a name it asserts on; a comment-only edit reaches none. -**Leaving a test file that no `run.das` model suite lists out of the `model-free` suite is a -defect, when its cells still assert what they claim under `DASLLAMA_CPU_PREFILL=1`.** The model -suites are every suite but `model-free`, and that variable is what the runner arms for -`model-free`. The listing lands in the same change that adds the file, and the file skips -honestly when its models are absent. +**Leaving a test file out of every `run.das` suite is a defect, unless the file's header +states why its cells cannot hold under `DASLLAMA_CPU_PREFILL=1`.** `DASLLAMA_CPU_PREFILL=1` is +what the runner arms for `model-free`. The listing lands in the same change that adds the +file, and the file skips honestly when its models are absent. **Listing a test file that `DASLLAMA_CPU_PREFILL=1` disarms in any `run.das` suite is a defect, -and so is leaving that fact out of the file's header.** `DASLLAMA_CPU_PREFILL=1` is what the -runner arms for `model-free`. +and so is leaving that fact out of the file's header.** **Invoking dastest directly on a test file in a `run.das` model suite (every suite but `model-free`) is a defect; such a file runs only through `run.das`. A `model-free` file runs @@ -136,18 +134,19 @@ a defect.** A resize cap is not evidence. logits-tolerance form. Counting cells stay token-exact. **A diff that adds a GPU kernel class under `../dasllama/` - a `[metal_kernel]` def, a -`[vk_dispatch]` declaration, or a new instance of a template carrying one - covers that class in -`test_kernel_coverage.das`, one of two ways.** Either a census row there dispatches the class, or the diff names it in -that file's `CENSUS_NEVER_DISPATCHED`, with the reason no row can reach it. A -`DASLLAMA_PARITY_FULL`-gated census row counts as the dispatching arm. Naming a -class a census row could dispatch is a defect. +`[vk_dispatch]` declaration, or a new instance of a template carrying one - covers that class +in `test_kernel_coverage.das`, one of two ways.** Either a census row there dispatches the +class, or the diff names it in that file's `CENSUS_NEVER_DISPATCHED`, with the reason no row +can reach it. A `DASLLAMA_PARITY_FULL`-gated census row counts as the dispatching arm. Naming +a class a census row could dispatch is a defect. **A kernel-unit cell - a model-less cell that dispatches one kernel class and asserts on its output - missing a compare against a CPU oracle that can witness the cell's property is a defect.** (A census row dispatches without asserting on output, so it is not one.) -**A kernel-unit cell that bit-compares a buffer the GPU writes prefills its output buffers with -a sentinel.** An unprefilled output can pass a bit compare by staying stale. +**A kernel-unit cell fills a GPU output buffer with a sentinel before every dispatch whose +output it then reads.** An unprefilled output can pass by staying stale - the previous +dispatch's values, or garbage that happens to sit inside the tolerance bar. **A cross-dispatch bit-identity compare - comparing the outputs of two dispatches - runs GPU against GPU.** No CPU oracle can witness that property. @@ -187,15 +186,14 @@ parameter that takes the lane, is a defect.** A family whose knob has no reset t it to the documented default. A runtime decline standing in for a pin measures whichever lane the box's policy picked. -**A cell that loads a media carrier under a lane pin - a `set__q8`-class knob or a -tensor-crown pin - mints in memory through the family's `stage_*` + `mint_*` pair, never -through a `.dlim`-baking loader (`load__tower` / `load__encoder` / -`load_model*`).** A disk bake under a pinned lane GC-purges the serving lane's `.dlim` -beside the model, and the next direct-image load in another suite panics on the wrong -identity. The one residue: a cell whose SUBJECT is a facade lane knob (`load_asr_model` -under `set_asr_tower_fp32`) keeps the facade loader - the image identity folds the pin, so -minting around it would unmake the claim; the purge exposure on a cold box is followup 61's -owed in-process image-off seam. +**A cell that loads a media carrier under a lane pin - a `set__q8`-class knob, or a +`set_metal_tensor_crowns` / `pin_metal_tensor_crowns` pin - and whose subject is not that lane +knob itself mints in memory through the family's `stage_*` + `mint_*` pair, never through a +`.dlim`-baking loader (`load__tower` / `load__encoder` / `load_model*`).** A +disk bake under a pinned lane GC-purges the serving lane's `.dlim` beside the model, and the +next direct-image load in another suite panics on the wrong identity. A cell whose subject is +the lane knob (`load_asr_model` under `set_asr_tower_fp32`) keeps the facade loader: the image +identity folds the pin, so minting around it would unmake the claim. **A CPU-vs-GPU arm that does not run a PLANAR model for its CPU stages, and that model's `blob_twin(t, path, seq_cap)` for override-selected stages, is a defect.** One session spans @@ -207,8 +205,8 @@ model. tag is the token passed to `family_on(t, name)` (`_model_tier.das`). An untagged block silently joins every family's gate. -**No CPU-control batch parity runs against the 70B.** Its batch coverage is ENGAGE-only in the -support matrix. The batched code paths get their parity on small models, through pins. +**No CPU-control batch parity runs against the 70B.** The batched code paths get their parity +on small models, through pins. **Setting a knob a cell can reach only through the environment after the process that reads it starts is a defect - set it before that process starts.** That process is a child the cell @@ -245,10 +243,12 @@ exact fixtures. **An embedding-parity cell that does not name its fixture, or does not log the measured maxdiff on green as well as red, is a defect.** -**A new gate, or a new or loosened tolerance bar, ships a control that reds it - a poison, a -knockout, a disconnected mechanism, or a cross-lane witness - in the same change.** A bar -nothing has ever exceeded is not known to discriminate, and a gate that reads state the same -code path wrote can be a tautology - only the control proves either can fail. +**A new gate, or a new or loosened tolerance bar, ships a control that reds it in the same +change.** A control is a run of the same gate that must RED - a poisoned input, a poisoned +expectation, a disconnected mechanism, or a second independent lane; a gate's own reference is +never its control. A bar nothing has ever exceeded is not known to discriminate, and a gate +that reads state the same code path wrote can be a tautology - only the control proves either +can fail. **A family that gains a live thinking or tool format ships its recognition tests in the same change** - the wire-shape pins, the render pins, and a live server leg gated on the family's @@ -273,8 +273,9 @@ route's to say: a twin-W route reads the baked halfword twin (`wblob`), so poiso plane alone is a valid control there, while a route reading both planes needs both zeroed. A poison the served route never reads passes on a broken kernel. -**An ASR cell comparing transcripts across two serving lanes asserts TOKEN equality; the one -exception is the crowned-lane twin, whose ruled grade is WORD equality (the tensor twins' -rounding legitimately flips tokens - followup 60's resolution).** A cell that cannot hold -its grade converts to the forced-feed logits-tolerance form - never to a looser text -compare. +**An ASR cell comparing transcripts across two serving lanes asserts TOKEN equality; a cell +comparing a crowned lane against its tensor twin asserts WORD equality, because the twins' +rounding legitimately flips tokens.** A crowned lane is the raced kernel form a tune sidecar +arms as the serving one; its tensor twin is the same kernel written on Metal's tensor +primitives. A cell that cannot hold its grade converts to the forced-feed logits-tolerance +form - never to a looser text compare. diff --git a/modules/dasLLAMA/tests/_metal_kernel_common.das b/modules/dasLLAMA/tests/_metal_kernel_common.das index aea4f78ca0..b058f6de6a 100644 --- a/modules/dasLLAMA/tests/_metal_kernel_common.das +++ b/modules/dasLLAMA/tests/_metal_kernel_common.das @@ -171,18 +171,41 @@ def kq_hash_byte(i : int) : uint8 { let KQ_SAFE_SCALES = fixed_array(0.25, 0.5, 1.0, 0.125, 2.0, 0.75, 1.5, 0.375) // quant plane = hash bytes (any bit pattern is valid); scale plane K4/K5 = [d f16][dmin f16] -// [12 packed 6-bit bytes] per superblock, K6 = [16B sub-scales] x nsb + f16 d tail -def kq_fill_planes(fmt, nsb : int; var kq : array; var ks : array) { - let qsb = fmt == 4 ? 128 : (fmt == 5 ? 160 : 192) +// [12 packed 6-bit bytes] per superblock, K6 = [16B sub-scales] x nsb + f16 d tail, IQ4_XS (44) = +// the same split form with [8 signed sub-scales in -31..31][8 zero] strips, Q3_K (3) = the k6 form over 96B quants +def kq_fill_planes(fmt, nsb : int; var kq : array; var ks : array) { // nolint:STYLE037 - the flat per-format fill ladder is the honest shape + let qsb = fmt == 2 || fmt == 24 || fmt == 25 ? 64 : (fmt == 23 ? 72 : (fmt == 4 || fmt == 44 || fmt == 45 ? 128 : (fmt == 5 ? 160 : (fmt == 3 ? 96 : (fmt == 33 ? 104 : (fmt == 34 ? 96 : 192)))))) kq |> resize(nsb * qsb) for (i in range(nsb * qsb)) { kq[i] = kq_hash_byte(i * 3 + fmt) } - if (fmt == 6) { - ks |> resize(nsb * 18) + if (fmt == 2) { + // k2: the metal split form - [16 pair-byte strips x nsb][4B d+dmin tail x nsb] + ks |> resize(nsb * 20) for (i in range(nsb * 16)) { ks[i] = kq_hash_byte(i * 7 + 13) } + for (b in range(nsb)) { + let dbits = f32_to_f16(KQ_SAFE_SCALES[b % 8]) + let mbits = f32_to_f16(KQ_SAFE_SCALES[(b + 3) % 8]) + ks[nsb * 16 + b * 4] = uint8(dbits & 0xFFu) + ks[nsb * 16 + b * 4 + 1] = uint8(dbits >> 8u) + ks[nsb * 16 + b * 4 + 2] = uint8(mbits & 0xFFu) + ks[nsb * 16 + b * 4 + 3] = uint8(mbits >> 8u) + } + } elif (fmt == 45) { + // iq4nl: 16B/sb = 8 per-32-block f16 d's, verbatim (no strips) + ks |> resize(nsb * 16) + for (b in range(nsb * 8)) { + let dbits = f32_to_f16(KQ_SAFE_SCALES[(b + (b / 8) * 3) % 8]) + ks[b * 2] = uint8(dbits & 0xFFu) + ks[b * 2 + 1] = uint8(dbits >> 8u) + } + } elif (fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 23 || fmt == 24 || fmt == 25) { + ks |> resize(nsb * 18) + for (i in range(nsb * 16)) { + ks[i] = fmt == 6 || fmt == 3 || fmt == 23 || fmt == 24 || fmt == 25 ? kq_hash_byte(i * 7 + 13) : uint8(i % 16 < 8 ? int(kq_hash_byte(i * 7 + 13)) % 63 - 31 : 0) // 44/33: signed strip bytes + zero pad; 23: 16 unsigned strips + } for (b in range(nsb)) { let dbits = f32_to_f16(KQ_SAFE_SCALES[b % 8]) ks[nsb * 16 + b * 2] = uint8(dbits & 0xFFu) @@ -244,6 +267,32 @@ def kq_row_ref(fmt, sb_base, nb, nsb_total : int; kq, ks : array; var wro dequant_k4_plane_superblock(kq, int64(sb * 128), ks, int64(sb * 16), wrow, int64(ib * 256)) } elif (fmt == 5) { dequant_k5_plane_superblock(kq, int64(sb * 160), ks, int64(sb * 16), wrow, int64(ib * 256)) + } elif (fmt == 44) { + dequant_iq4xs_plane_superblock_at(kq, int64(sb * 128), ks, int64(sb * 16), + int64(nsb_total * 16 + sb * 2), wrow, int64(ib * 256)) + } elif (fmt == 3) { + dequant_k3_plane_superblock_at(kq, int64(sb * 96), ks, int64(sb * 16), + int64(nsb_total * 16 + sb * 2), wrow, int64(ib * 256)) + } elif (fmt == 33) { + dequant_iq3s_plane_superblock_at(kq, int64(sb * 104), ks, int64(sb * 16), + int64(nsb_total * 16 + sb * 2), wrow, int64(ib * 256)) + } elif (fmt == 34) { + dequant_iq3xxs_plane_superblock_at(kq, int64(sb * 96), ks, int64(sb * 16), + int64(nsb_total * 16 + sb * 2), wrow, int64(ib * 256)) + } elif (fmt == 45) { + dequant_iq4nl_plane_superblock(kq, int64(sb * 128), ks, int64(sb * 16), wrow, int64(ib * 256)) + } elif (fmt == 2) { + dequant_k2_plane_superblock_at(kq, int64(sb * 64), ks, int64(sb * 16), + int64(nsb_total * 16 + sb * 4), wrow, int64(ib * 256)) + } elif (fmt == 23) { + dequant_iq2s_plane_superblock_at(kq, int64(sb * 72), ks, int64(sb * 16), + int64(nsb_total * 16 + sb * 2), wrow, int64(ib * 256)) + } elif (fmt == 24) { + dequant_iq2xs_plane_superblock_at(kq, int64(sb * 64), ks, int64(sb * 16), + int64(nsb_total * 16 + sb * 2), wrow, int64(ib * 256)) + } elif (fmt == 25) { + dequant_iq2xxs_plane_superblock_at(kq, int64(sb * 64), ks, int64(sb * 16), + int64(nsb_total * 16 + sb * 2), wrow, int64(ib * 256)) } else { dequant_k6_plane_superblock_at(kq, int64(sb * 192), ks, int64(sb * 16), int64(nsb_total * 16 + sb * 2), wrow, int64(ib * 256)) diff --git a/modules/dasLLAMA/tests/_vkd_oracles.das b/modules/dasLLAMA/tests/_vkd_oracles.das index 7207ac95cb..3af9753c64 100644 --- a/modules/dasLLAMA/tests/_vkd_oracles.das +++ b/modules/dasLLAMA/tests/_vkd_oracles.das @@ -169,6 +169,44 @@ def kq_cls_ref(fmt : int; wq, ws, xq : array; xs : array) : KqGemvB k = new KqGemvQ40() } elif (fmt == int(KqFmt.k5)) { k = new KqGemvK5() + } elif (fmt == int(KqFmt.iq4xs)) { + k = new KqGemvIq4xs() + } elif (fmt == int(KqFmt.k3)) { + k = new KqGemvK3() + } elif (fmt == int(KqFmt.iq3s)) { + var ki = new KqGemvIq3s() + for (i in range(512)) { // the CPU replay never runs the staging kernel head - fill the workgroup grid here + ki.gridw[i] = IQ3S_GRID[i] + } + k = ki + } elif (fmt == int(KqFmt.iq3xxs)) { + var kx = new KqGemvIq3xxs() + for (i in range(256)) { // same replay rule: the halved grid fills here + kx.gridw[i] = IQ3XXS_GRID[i] + } + k = kx + } elif (fmt == int(KqFmt.iq4nl)) { + k = new KqGemvIq4nl() // pure-ALU codebook - no workgroup state to fill + } elif (fmt == int(KqFmt.k2)) { + k = new KqGemvK2() // pure ALU - no workgroup state to fill + } elif (fmt == int(KqFmt.iq2s)) { + var k2s = new KqGemvIq2s() + for (i in range(2048)) { // same replay rule: the u64 grid's word pairs fill here + k2s.gridw[i] = IQ2S_GRID[i] + } + k = k2s + } elif (fmt == int(KqFmt.iq2xs)) { + var k2x = new KqGemvIq2xs() + for (i in range(1024)) { // same replay rule, the 512-entry grid's word pairs + k2x.gridw[i] = IQ2XS_GRID[i] + } + k = k2x + } elif (fmt == int(KqFmt.iq2xxs)) { + var k2xx = new KqGemvIq2xxs() + for (i in range(512)) { // same replay rule, the 256-entry grid's word pairs + k2xx.gridw[i] = IQ2XXS_GRID[i] + } + k = k2xx } else { k = new KqGemvK6() } @@ -205,6 +243,526 @@ def kq_gemv_oracle(fmt : int; wq, ws, xq : array; xs : array; return <- y } +// IQ4_XS GEMV float oracle, INDEPENDENT of the class bit-math: w = d*sc*LUT[q] straight off the +// plane bytes (q40 nibble tiling, the k4 row shape with signed sc), x off the Q8_K bytes — the +// gate that a wrong codebook pack in iq4_word cannot pass while staying self-consistent +def iq4xs_gemv_float_oracle(wq, ws, xq : array; xs : array; + regions : array; n, d, nreg : int) : array { + let lut = iq4nl_lut() + var y : array + y |> resize(nreg * d) + let nsb = n / 256 + for (r in range(nreg)) { + for (row in range(d)) { + let w0 = int(regions[r * 2]) + row * nsb + let x0 = int(regions[r * 2 + 1]) + var acc = 0.0 + for (s in range(nsb)) { + let sb = w0 + s + let xsb = x0 + s + let dd = unpackHalf2x16(ws[sb * 5]).x + for (blk in range(8)) { + let sc = byte_s8(ws, sb * 20 + 4 + blk) + var bacc = 0.0 + for (m in range(16)) { + let b = byte_u8(wq, sb * 128 + blk * 16 + m) + let xlo = byte_s8(xq, xsb * 256 + blk * 32 + m) + let xhi = byte_s8(xq, xsb * 256 + blk * 32 + 16 + m) + bacc += float(lut[b & 15]) * float(xlo) + float(lut[b >> 4]) * float(xhi) + } + acc += xs[xsb] * dd * float(sc) * bacc + } + } + y[r * d + row] = acc + } + } + return <- y +} + +// Q3_K GEMV float oracle, INDEPENDENT of the class bit-math: w = d * sc16 * (q - 4) straight off +// the plane bytes (qs lane at shift 2*(blk%4), hmask bit blk, k6's 5-word scale row), x off the +// Q8_K bytes — the witness a wrong lane/mask pick in k3_dot cannot pass while staying self-consistent +def k3_gemv_float_oracle(wq, ws, xq : array; xs : array; + regions : array; n, d, nreg : int) : array { + var y : array + y |> resize(nreg * d) + let nsb = n / 256 + for (r in range(nreg)) { + for (row in range(d)) { + let w0 = int(regions[r * 2]) + row * nsb + let x0 = int(regions[r * 2 + 1]) + var acc = 0.0 + for (s in range(nsb)) { + let sb = w0 + s + let xsb = x0 + s + let dd = half_at(ws, sb * 10 + 8) + for (blk in range(8)) { + let sh = 2 * (blk % 4) + let s0 = byte_s8(ws, sb * 20 + 2 * blk) + let s1 = byte_s8(ws, sb * 20 + 2 * blk + 1) + var bacc = 0.0 + for (l in range(32)) { + let q2 = (byte_u8(wq, sb * 96 + (blk / 4) * 32 + l) >> sh) & 3 + let hb = (byte_u8(wq, sb * 96 + 64 + l) >> blk) & 1 + let q = (q2 | (hb << 2)) - 4 + bacc += float(l < 16 ? s0 : s1) * float(q) * float(byte_s8(xq, xsb * 256 + blk * 32 + l)) + } + acc += xs[xsb] * dd * bacc + } + } + y[r * d + row] = acc + } + } + return <- y +} + +// IQ3_S GEMV float oracle, INDEPENDENT of the class bit-math: w = d * (1 + 2s) * grid_byte * +// sign straight off the plane bytes ([qs 64][qh 8][signs 32], the k4-shaped scale row), x off +// the Q8_K bytes - the witness a wrong index/sign pick in the gather cannot pass +def iq3s_gemv_float_oracle(wq, ws, xq : array; xs : array; + regions : array; n, d, nreg : int) : array { + var y : array + y |> resize(nreg * d) + let nsb = n / 256 + for (r in range(nreg)) { + for (row in range(d)) { + let w0 = int(regions[r * 2]) + row * nsb + let x0 = int(regions[r * 2 + 1]) + var acc = 0.0 + for (s in range(nsb)) { + let sb = w0 + s + let xsb = x0 + s + let dd = half_at(ws, sb * 10) + for (blk in range(8)) { + let sc = float(byte_s8(ws, sb * 20 + 4 + blk)) // the plane byte IS the signed (1 + 2s) sub-scale + var bacc = 0.0 + for (e in range(32)) { + let l = e / 8 + let hj = e % 8 + let qi = byte_u8(wq, sb * 104 + blk * 8 + 2 * l + hj / 4) | ((byte_u8(wq, sb * 104 + 64 + blk) << (hj < 4 ? 8 - 2 * l : 7 - 2 * l)) & 256) + let g = float(int(IQ3S_GRID[qi] >> uint(8 * (hj % 4))) & 0xFF) + let sgn = ((byte_u8(wq, sb * 104 + 72 + blk * 4 + l) >> hj) & 1) != 0 ? -1.0 : 1.0 + bacc += g * sgn * float(byte_s8(xq, xsb * 256 + blk * 32 + e)) + } + acc += xs[xsb] * dd * sc * bacc + } + } + y[r * d + row] = acc + } + } + return <- y +} + +// IQ3_XXS GEMV float oracle, INDEPENDENT of the class bit-math: w = d_halved * (2ls + 1) * +// halved_grid_byte * sign straight off the plane bytes ([qs 64][aux 32], the iq3s-shaped +// scale row), signs through the CPU-side ksigns table. +def iq3xxs_gemv_float_oracle(wq, ws, xq : array; xs : array; + regions : array; n, d, nreg : int) : array { + var y : array + y |> resize(nreg * d) + let nsb = n / 256 + for (r in range(nreg)) { + for (row in range(d)) { + let w0 = int(regions[r * 2]) + row * nsb + let x0 = int(regions[r * 2 + 1]) + var acc = 0.0 + for (s in range(nsb)) { + let sb = w0 + s + let xsb = x0 + s + let dd = half_at(ws, sb * 10) + for (blk in range(8)) { + let sc = float(byte_s8(ws, sb * 20 + 4 + blk)) // the plane byte IS the (2ls + 1) sub-scale + let aux = byte_u8(wq, sb * 96 + 64 + blk * 4) | (byte_u8(wq, sb * 96 + 64 + blk * 4 + 1) << 8) | (byte_u8(wq, sb * 96 + 64 + blk * 4 + 2) << 16) | (byte_u8(wq, sb * 96 + 64 + blk * 4 + 3) << 24) + var bacc = 0.0 + for (e in range(32)) { + let l = e / 8 + let hj = e % 8 + let qi = byte_u8(wq, sb * 96 + blk * 8 + 2 * l + hj / 4) + let g = float(int(IQ3XXS_GRID[qi] >> uint(8 * (hj % 4))) & 0xFF) + let sgn = ((int(KSIGNS_IQ2XS[(aux >> (7 * l)) & 127]) >> hj) & 1) != 0 ? -1.0 : 1.0 + bacc += g * sgn * float(byte_s8(xq, xsb * 256 + blk * 32 + e)) + } + acc += xs[xsb] * dd * sc * bacc + } + } + y[r * d + row] = acc + } + } + return <- y +} + +// IQ2_S GEMV float oracle, INDEPENDENT of the class bit-math: w = d8 * (1 + 2s) * grid_byte * +// sign straight off the plane bytes ([idx 32][signs 32][qh 8], the k4-shaped scale row with +// UNSIGNED per-16 strips and the pre-eighth-ed d). +def iq2s_gemv_float_oracle(wq, ws, xq : array; xs : array; + regions : array; n, d, nreg : int) : array { + var y : array + y |> resize(nreg * d) + let nsb = n / 256 + for (r in range(nreg)) { + for (row in range(d)) { + let w0 = int(regions[r * 2]) + row * nsb + let x0 = int(regions[r * 2 + 1]) + var acc = 0.0 + for (s in range(nsb)) { + let sb = w0 + s + let xsb = x0 + s + let dd = half_at(ws, sb * 10) + for (blk in range(8)) { + let qh = byte_u8(wq, sb * 72 + 64 + blk) + for (hh in range(2)) { + let sc = float(byte_u8(ws, sb * 20 + 4 + blk * 2 + hh)) + var bacc = 0.0 + for (e in range(16)) { + let ee = hh * 16 + e + let l = ee / 8 + let hj = ee % 8 + let qi = (byte_u8(wq, sb * 72 + blk * 4 + l) | ((qh << (8 - 2 * l)) & 0x300)) * 2 + hj / 4 + let g = float(int(IQ2S_GRID[qi] >> uint(8 * (hj % 4))) & 0xFF) + let sgn = ((byte_u8(wq, sb * 72 + 32 + blk * 4 + l) >> hj) & 1) != 0 ? -1.0 : 1.0 + bacc += g * sgn * float(byte_s8(xq, xsb * 256 + blk * 32 + ee)) + } + acc += xs[xsb] * dd * sc * bacc + } + } + } + y[r * d + row] = acc + } + } + return <- y +} + +// IQ2_XS GEMV float oracle, INDEPENDENT of the class bit-math: w = d8 * (1 + 2s) * grid_byte +// * ksign straight off the plane words (32 u16 qs words per superblock, no sign/qh planes - +// the 7-bit ksigns index rides bits 9..15; the k4-shaped scale row with UNSIGNED strips). +def iq2xs_gemv_float_oracle(wq, ws, xq : array; xs : array; + regions : array; n, d, nreg : int) : array { + var y : array + y |> resize(nreg * d) + let nsb = n / 256 + for (r in range(nreg)) { + for (row in range(d)) { + let w0 = int(regions[r * 2]) + row * nsb + let x0 = int(regions[r * 2 + 1]) + var acc = 0.0 + for (s in range(nsb)) { + let sb = w0 + s + let xsb = x0 + s + let dd = half_at(ws, sb * 10) + for (wi in range(32)) { + let w16 = int((wq[sb * 16 + wi / 2] >> uint((wi % 2) * 16)) & 0xFFFFu) + let sidx = w16 >> 9 + var tt = sidx ^ (sidx >> 4) + tt = tt ^ (tt >> 2) + tt = tt ^ (tt >> 1) + let sgb = sidx | ((tt & 1) << 7) + let sc = float(byte_u8(ws, sb * 20 + 4 + wi / 2)) + var bacc = 0.0 + for (j in range(8)) { + let gb = float(int(IQ2XS_GRID[(w16 & 511) * 2 + j / 4] >> uint(8 * (j % 4))) & 0xFF) + let sgn = ((sgb >> j) & 1) != 0 ? -1.0 : 1.0 + bacc += gb * sgn * float(byte_s8(xq, xsb * 256 + wi * 8 + j)) + } + acc += xs[xsb] * dd * sc * bacc + } + } + y[r * d + row] = acc + } + } + return <- y +} + +// IQ2_XXS GEMV float oracle, INDEPENDENT of the class bit-math: w = d8 * (1 + 2s) * grid_byte +// * ksign straight off the plane words (per block a grid word of 4 byte indices + its aux32; +// the k4-shaped scale row with 8 UNSIGNED per-32 strips and the pre-eighth-ed d). +def iq2xxs_gemv_float_oracle(wq, ws, xq : array; xs : array; + regions : array; n, d, nreg : int) : array { + var y : array + y |> resize(nreg * d) + let nsb = n / 256 + for (r in range(nreg)) { + for (row in range(d)) { + let w0 = int(regions[r * 2]) + row * nsb + let x0 = int(regions[r * 2 + 1]) + var acc = 0.0 + for (s in range(nsb)) { + let sb = w0 + s + let xsb = x0 + s + let dd = half_at(ws, sb * 10) + for (b in range(8)) { + let gw = wq[sb * 16 + b * 2] + let aux = int(wq[sb * 16 + b * 2 + 1]) + let sc = float(byte_s8(ws, sb * 20 + 4 + b)) // SIGNED like the class/batch stage - real (2ls+1) strips stay positive + var bacc = 0.0 + for (l in range(4)) { + let idx = int((gw >> uint(8 * l)) & 0xFFu) + let sidx = (aux >> (7 * l)) & 127 + var tt = sidx ^ (sidx >> 4) + tt = tt ^ (tt >> 2) + tt = tt ^ (tt >> 1) + let sgb = sidx | ((tt & 1) << 7) + for (j in range(8)) { + let gb = float(int(IQ2XXS_GRID[idx * 2 + j / 4] >> uint(8 * (j % 4))) & 0xFF) + let sgn = ((sgb >> j) & 1) != 0 ? -1.0 : 1.0 + bacc += gb * sgn * float(byte_s8(xq, xsb * 256 + b * 32 + l * 8 + j)) + } + } + acc += xs[xsb] * dd * sc * bacc + } + } + y[r * d + row] = acc + } + } + return <- y +} + +// IQ3_S f16-feed GEMM reference over the device form: grid gather + sign flip per element, +// f16-rounded per weight like its siblings ([qs 64][qh 8][signs 32] rows, the k4-shaped +// scale row with signed (1 + 2s) sub-scales). +def iq3sf16_gemm_oracle(wq : array; wsu : array; xf : array; + recs : array; nrec, n, d : int; var y : array) { + let nsb = n / 256 + for (rr in range(nrec)) { + let wsb0 = int(recs[rr * 4]) + let row0 = int(recs[rr * 4 + 1]) + let cnt = int(recs[rr * 4 + 2]) + for (r in range(cnt)) { + for (c in range(d)) { + var acc = 0.0 + for (s in range(nsb)) { + let sb = wsb0 + c * nsb + s + let dv = unpackHalf2x16(wsu[sb * 5]).x + for (g in range(8)) { + let sc = float(int(wsu[sb * 5 + 1 + g / 4] << uint((3 - g % 4) * 8)) >> 24) + for (e in range(32)) { + let wj = e / 4 + let qi = byte_u8(wq, sb * 104 + g * 8 + wj) | ((byte_u8(wq, sb * 104 + 64 + g) << (8 - wj)) & 256) + let gb = float(int(IQ3S_GRID[qi] >> uint(8 * (e % 4))) & 0xFF) + let sbit = (byte_u8(wq, sb * 104 + 72 + g * 4 + e / 8) >> (((e / 4) % 2) * 4 + e % 4)) & 1 + let w = float(float16(dv * sc * (sbit != 0 ? -gb : gb))) + acc += w * half_at(xf, (row0 + r) * n + s * 256 + g * 32 + e) + } + } + } + y[(row0 + r) * d + c] = acc + } + } + } +} + +// IQ3_XXS f16-feed GEMM reference over the device form: halved-grid gather + ksigns sign per +// element, f16-rounded per weight like its siblings ([qs 64][aux 32] rows, the iq3s-shaped +// scale row with (2ls + 1) sub-scales and the halved d). +def iq3xxsf16_gemm_oracle(wq : array; wsu : array; xf : array; + recs : array; nrec, n, d : int; var y : array) { + let nsb = n / 256 + for (rr in range(nrec)) { + let wsb0 = int(recs[rr * 4]) + let row0 = int(recs[rr * 4 + 1]) + let cnt = int(recs[rr * 4 + 2]) + for (r in range(cnt)) { + for (c in range(d)) { + var acc = 0.0 + for (s in range(nsb)) { + let sb = wsb0 + c * nsb + s + let dv = unpackHalf2x16(wsu[sb * 5]).x + for (g in range(8)) { + let sc = float(int(wsu[sb * 5 + 1 + g / 4] << uint((3 - g % 4) * 8)) >> 24) + let aux = byte_u8(wq, sb * 96 + 64 + g * 4) | (byte_u8(wq, sb * 96 + 64 + g * 4 + 1) << 8) | (byte_u8(wq, sb * 96 + 64 + g * 4 + 2) << 16) | (byte_u8(wq, sb * 96 + 64 + g * 4 + 3) << 24) + for (e in range(32)) { + let qi = byte_u8(wq, sb * 96 + g * 8 + e / 4) + let gb = float(int(IQ3XXS_GRID[qi] >> uint(8 * (e % 4))) & 0xFF) + let sbit = (int(KSIGNS_IQ2XS[(aux >> (7 * (e / 8))) & 127]) >> (e % 8)) & 1 + let w = float(float16(dv * sc * (sbit != 0 ? -gb : gb))) + acc += w * half_at(xf, (row0 + r) * n + s * 256 + g * 32 + e) + } + } + } + y[(row0 + r) * d + c] = acc + } + } + } +} + +// IQ2_S f16-feed GEMM reference over the device form: u64-grid gather (two words per 10-bit +// index) + the block's own sign bytes, f16-rounded per weight like its siblings ([idx 32] +// [signs 32][qh 8] rows, the k4-shaped scale row with UNSIGNED per-16 strips, d pre-eighth-ed). +def iq2sf16_gemm_oracle(wq : array; wsu : array; xf : array; + recs : array; nrec, n, d : int; var y : array) { + let nsb = n / 256 + for (rr in range(nrec)) { + let wsb0 = int(recs[rr * 4]) + let row0 = int(recs[rr * 4 + 1]) + let cnt = int(recs[rr * 4 + 2]) + for (r in range(cnt)) { + for (c in range(d)) { + var acc = 0.0 + for (s in range(nsb)) { + let sb = wsb0 + c * nsb + s + let dv = unpackHalf2x16(wsu[sb * 5]).x + for (g in range(8)) { + let qh = byte_u8(wq, sb * 72 + 64 + g) + for (e in range(32)) { + let l = e / 8 + let si = g * 2 + e / 16 + let sc = float((wsu[sb * 5 + 1 + si / 4] >> uint((si % 4) * 8)) & 0xFFu) + let qi = (byte_u8(wq, sb * 72 + g * 4 + l) | ((qh << (8 - 2 * l)) & 0x300)) * 2 + (e % 8) / 4 + let gb = float(int(IQ2S_GRID[qi] >> uint(8 * (e % 4))) & 0xFF) + let sbit = (byte_u8(wq, sb * 72 + 32 + g * 4 + l) >> (e % 8)) & 1 + let w = float(float16(dv * sc * (sbit != 0 ? -gb : gb))) + acc += w * half_at(xf, (row0 + r) * n + s * 256 + g * 32 + e) + } + } + } + y[(row0 + r) * d + c] = acc + } + } + } +} + +// IQ2_XS f16-feed GEMM reference over the device form: u64-grid gather (two words per 9-bit +// index) + ksigns-by-parity off each u16 qs word, f16-rounded per weight like its siblings +// (32 u16 words per superblock, the k4-shaped scale row with UNSIGNED strips, d pre-eighth-ed). +def iq2xsf16_gemm_oracle(wq : array; wsu : array; xf : array; + recs : array; nrec, n, d : int; var y : array) { + let nsb = n / 256 + for (rr in range(nrec)) { + let wsb0 = int(recs[rr * 4]) + let row0 = int(recs[rr * 4 + 1]) + let cnt = int(recs[rr * 4 + 2]) + for (r in range(cnt)) { + for (c in range(d)) { + var acc = 0.0 + for (s in range(nsb)) { + let sb = wsb0 + c * nsb + s + let dv = unpackHalf2x16(wsu[sb * 5]).x + for (e in range(256)) { + let wi = e / 8 + let j = e % 8 + let w16 = int((wq[sb * 16 + wi / 2] >> uint((wi % 2) * 16)) & 0xFFFFu) + let sidx = w16 >> 9 + var tt = sidx ^ (sidx >> 4) + tt = tt ^ (tt >> 2) + tt = tt ^ (tt >> 1) + let sgb = sidx | ((tt & 1) << 7) + let si = e / 16 + let sc = float((wsu[sb * 5 + 1 + si / 4] >> uint((si % 4) * 8)) & 0xFFu) + let gb = float(int(IQ2XS_GRID[(w16 & 511) * 2 + j / 4] >> uint(8 * (j % 4))) & 0xFF) + let w = float(float16(dv * sc * (((sgb >> j) & 1) != 0 ? -gb : gb))) + acc += w * half_at(xf, (row0 + r) * n + s * 256 + e) + } + } + y[(row0 + r) * d + c] = acc + } + } + } +} + +// IQ2_XXS f16-feed GEMM reference over the device form: per block a grid word of 4 byte +// indices (two u64 words each) + its aux32 (ksigns by parity), f16-rounded per weight like +// its siblings (16 uint words per superblock, the k4-shaped scale row with 8 UNSIGNED per-32 +// strips, d pre-eighth-ed). +def iq2xxsf16_gemm_oracle(wq : array; wsu : array; xf : array; + recs : array; nrec, n, d : int; var y : array) { + let nsb = n / 256 + for (rr in range(nrec)) { + let wsb0 = int(recs[rr * 4]) + let row0 = int(recs[rr * 4 + 1]) + let cnt = int(recs[rr * 4 + 2]) + for (r in range(cnt)) { + for (c in range(d)) { + var acc = 0.0 + for (s in range(nsb)) { + let sb = wsb0 + c * nsb + s + let dv = unpackHalf2x16(wsu[sb * 5]).x + for (e in range(256)) { + let g = e / 32 + let rr2 = e % 32 + let l = rr2 / 8 + let j = rr2 % 8 + let gw = wq[sb * 16 + g * 2] + let aux = int(wq[sb * 16 + g * 2 + 1]) + let idx = int((gw >> uint(8 * l)) & 0xFFu) + let sidx = (aux >> (7 * l)) & 127 + var tt = sidx ^ (sidx >> 4) + tt = tt ^ (tt >> 2) + tt = tt ^ (tt >> 1) + let sgb = sidx | ((tt & 1) << 7) + let sc = float(int(wsu[sb * 5 + 1 + g / 4] << uint((3 - (g % 4)) * 8)) >> 24) + let gb = float(int(IQ2XXS_GRID[idx * 2 + j / 4] >> uint(8 * (j % 4))) & 0xFF) + let w = float(float16(dv * sc * (((sgb >> j) & 1) != 0 ? -gb : gb))) + acc += w * half_at(xf, (row0 + r) * n + s * 256 + e) + } + } + y[(row0 + r) * d + c] = acc + } + } + } +} + +// IQ4_NL f16-feed GEMM reference over the device form: q40's walk with the codebook in place +// of (q - 8), f16-rounded per weight like its siblings (q40's 32-word rows + 8 f16 d). +def iq4nlf16_gemm_oracle(wq : array; wsu : array; xf : array; + recs : array; nrec, n, d : int; var y : array) { + let nsb = n / 256 + for (rr in range(nrec)) { + let wsb0 = int(recs[rr * 4]) + let row0 = int(recs[rr * 4 + 1]) + let cnt = int(recs[rr * 4 + 2]) + for (r in range(cnt)) { + for (c in range(d)) { + var acc = 0.0 + for (s in range(nsb)) { + let sb = wsb0 + c * nsb + s + for (g in range(8)) { + let dp = unpackHalf2x16(wsu[sb * 5 + g / 2]) + let dv = g % 2 == 0 ? dp.x : dp.y + for (e in range(32)) { + let by = byte_u8(wq, sb * 128 + g * 16 + e % 16) + let q = (by >> ((e / 16) * 4)) & 0xF + let w = float(float16(dv * float(int(IQ4NL_LUT[q])))) + acc += w * half_at(xf, (row0 + r) * n + s * 256 + g * 32 + e) + } + } + } + y[(row0 + r) * d + c] = acc + } + } + } +} + +// Q2_K f16-feed GEMM reference over the device form: per-16 sc/min nibble pairs x d/dmin, +// f16-rounded per weight like its siblings (16-word rows + [d|dmin][16 pair bytes] scales). +def k2f16_gemm_oracle(wq : array; wsu : array; xf : array; + recs : array; nrec, n, d : int; var y : array) { + let nsb = n / 256 + for (rr in range(nrec)) { + let wsb0 = int(recs[rr * 4]) + let row0 = int(recs[rr * 4 + 1]) + let cnt = int(recs[rr * 4 + 2]) + for (r in range(cnt)) { + for (c in range(d)) { + var acc = 0.0 + for (s in range(nsb)) { + let sb = wsb0 + c * nsb + s + let dm = unpackHalf2x16(wsu[sb * 5]) + for (g in range(16)) { + let pb = (int(wsu[sb * 5 + 1 + g / 4]) >> ((g % 4) * 8)) & 0xFF + for (l in range(16)) { + let qi = (g / 8) * 32 + (g % 2) * 16 + l + let by = byte_u8(wq, sb * 64 + qi) + let q = (by >> (((g / 2) % 4) * 2)) & 3 + let w = float(float16(dm.x * float(pb & 0xF) * float(q) - dm.y * float(pb >> 4))) + acc += w * half_at(xf, (row0 + r) * n + s * 256 + g * 16 + l) + } + } + } + y[(row0 + r) * d + c] = acc + } + } + } +} + // kq batch GEMM oracle over region records — same blk_contrib per (row, col, block) def kq_gemm_oracle(fmt : int; wq, ws, xq : array; xs : array; recs : array; nrec, n, d : int; var y : array) { @@ -230,3 +788,129 @@ def kq_gemm_oracle(fmt : int; wq, ws, xq : array; xs : array; delete k } } + +// the four remaining kq formats' f16-feed GEMM references, over the same device forms the cm2 +// decode callbacks read (k5 = the k4 row plus the packed 5th-bit plane; q40 = 8 f16 d; iq4xs = +// d + signed sub-scales + the iq4nl codebook; k3 = 2-bit lanes + hmask over the k6 scale row) +def k5f16_gemm_oracle(wq : array; wsu : array; xf : array; + recs : array; nrec, n, d : int; var y : array) { + let nsb = n / 256 + for (rr in range(nrec)) { + let wsb0 = int(recs[rr * 4]) + let row0 = int(recs[rr * 4 + 1]) + let cnt = int(recs[rr * 4 + 2]) + for (r in range(cnt)) { + for (c in range(d)) { + var acc = 0.0 + for (s in range(nsb)) { + let sb = wsb0 + c * nsb + s + let dm = unpackHalf2x16(wsu[sb * 5]) + for (g in range(8)) { + let sc = float(int((wsu[sb * 5 + 1 + g / 4] >> uint((g % 4) * 8)) & 0xFFu)) + let mn = float(int((wsu[sb * 5 + 3 + g / 4] >> uint((g % 4) * 8)) & 0xFFu)) + for (e in range(32)) { + let j = e % 16 + let h = e / 16 + let by = byte_u8(wq, sb * 160 + g * 16 + j) + let hby = byte_u8(wq, sb * 160 + 128 + g * 4 + j / 4) + let q = float(((by >> (h * 4)) & 0xF) | (((hby >> (j % 4 + h * 4)) & 1) << 4)) + let w = float(float16(dm.x * sc * q - dm.y * mn)) + acc += w * half_at(xf, (row0 + r) * n + s * 256 + g * 32 + e) + } + } + } + y[(row0 + r) * d + c] = acc + } + } + } +} + +def q40f16_gemm_oracle(wq : array; wsu : array; xf : array; + recs : array; nrec, n, d : int; var y : array) { + let nsb = n / 256 + for (rr in range(nrec)) { + let wsb0 = int(recs[rr * 4]) + let row0 = int(recs[rr * 4 + 1]) + let cnt = int(recs[rr * 4 + 2]) + for (r in range(cnt)) { + for (c in range(d)) { + var acc = 0.0 + for (s in range(nsb)) { + let sb = wsb0 + c * nsb + s + for (g in range(8)) { + let dp = unpackHalf2x16(wsu[sb * 5 + g / 2]) + let dv = g % 2 == 0 ? dp.x : dp.y + for (e in range(32)) { + let by = byte_u8(wq, sb * 128 + g * 16 + e % 16) + let q = float((by >> ((e / 16) * 4)) & 0xF) + let w = float(float16(dv * (q - 8.0))) + acc += w * half_at(xf, (row0 + r) * n + s * 256 + g * 32 + e) + } + } + } + y[(row0 + r) * d + c] = acc + } + } + } +} + +def iq4xsf16_gemm_oracle(wq : array; wsu : array; xf : array; + recs : array; nrec, n, d : int; var y : array) { + let nsb = n / 256 + let lut = iq4nl_lut() + for (rr in range(nrec)) { + let wsb0 = int(recs[rr * 4]) + let row0 = int(recs[rr * 4 + 1]) + let cnt = int(recs[rr * 4 + 2]) + for (r in range(cnt)) { + for (c in range(d)) { + var acc = 0.0 + for (s in range(nsb)) { + let sb = wsb0 + c * nsb + s + let dv = unpackHalf2x16(wsu[sb * 5]).x + for (g in range(8)) { + let sc = float(int(wsu[sb * 5 + 1 + g / 4] << uint((3 - g % 4) * 8)) >> 24) + for (e in range(32)) { + let by = byte_u8(wq, sb * 128 + g * 16 + e % 16) + let q = (by >> ((e / 16) * 4)) & 0xF + let w = float(float16(dv * sc * float(lut[q]))) + acc += w * half_at(xf, (row0 + r) * n + s * 256 + g * 32 + e) + } + } + } + y[(row0 + r) * d + c] = acc + } + } + } +} + +def k3f16_gemm_oracle(wq : array; wsu : array; xf : array; + recs : array; nrec, n, d : int; var y : array) { + let nsb = n / 256 + for (rr in range(nrec)) { + let wsb0 = int(recs[rr * 4]) + let row0 = int(recs[rr * 4 + 1]) + let cnt = int(recs[rr * 4 + 2]) + for (r in range(cnt)) { + for (c in range(d)) { + var acc = 0.0 + for (s in range(nsb)) { + let sb = wsb0 + c * nsb + s + let dv = unpackHalf2x16(wsu[sb * 5 + 4]).x + for (g in range(8)) { + for (e in range(32)) { + let sidx = g * 2 + e / 16 + let sc = float(int(wsu[sb * 5 + sidx / 4] << uint((3 - sidx % 4) * 8)) >> 24) + let qby = byte_u8(wq, sb * 96 + (g / 4) * 32 + e) + let hby = byte_u8(wq, sb * 96 + 64 + e) + let q3 = float(((qby >> ((g % 4) * 2)) & 3) - (((hby >> g) & 1) == 0 ? 4 : 0)) + let w = float(float16(dv * sc * q3)) + acc += w * half_at(xf, (row0 + r) * n + s * 256 + g * 32 + e) + } + } + } + y[(row0 + r) * d + c] = acc + } + } + } +} diff --git a/modules/dasLLAMA/tests/run.das b/modules/dasLLAMA/tests/run.das index fccaa00427..f9e5f06cf3 100644 --- a/modules/dasLLAMA/tests/run.das +++ b/modules/dasLLAMA/tests/run.das @@ -106,6 +106,7 @@ def private suite_files(name : string) : array { // nolint:STYLE038 - "modules/dasLLAMA/tests/test_kernel_backend.das", "modules/dasLLAMA/tests/test_kgroup_repack.das", "modules/dasLLAMA/tests/test_kqformat.das", + "modules/dasLLAMA/tests/test_kquant.das", "modules/dasLLAMA/tests/test_kv_prefix.das", "modules/dasLLAMA/tests/test_matmul.das", "modules/dasLLAMA/tests/test_matmul_batch.das", diff --git a/modules/dasLLAMA/tests/test_gguf_quant.das b/modules/dasLLAMA/tests/test_gguf_quant.das index f2a1445332..c71dc34289 100644 --- a/modules/dasLLAMA/tests/test_gguf_quant.das +++ b/modules/dasLLAMA/tests/test_gguf_quant.das @@ -4,6 +4,7 @@ options _dasllama_internal = true require dastest/testing_boost public require dasllama/dasllama_convert // the codec layer under test — direct require pins the new module API require dasllama/dasllama_gguf +require dasllama/dasllama_common // dequant_kq_plane_sb - the plane dequant the f32 fallback must match require math // Byte-level GGUF quant-format gates on a synthetic in-memory file: the dequant arm must match an @@ -210,3 +211,63 @@ def test_q5k_encoder(t : T?) { delete ks } } + +// the superblock disk types the native tier reads: the f32 fallback decodes every one of them, +// value for value the way the native load does (transcode, then plane dequant). Random bytes +// are valid input for all nine (grid indices, sign bits and scale nibbles have no invalid values). +def private synth_sb_bytes(sbb : int64; seed : int) : array { + var b : array + b |> reserve(sbb * 2l) + var x = uint(seed) * 2654435761u + 12345u + for (_i in range64(sbb * 2l)) { + x = x * 1664525u + 1013904223u + b |> push(uint8((x >> 24u) & 0xFFu)) + } + return <- b +} + +[test] +def test_gguf_f32_reader_superblock_types(t : T?) { + t |> run("gguf_read_tensor_f32 decodes every superblock i-quant / K-quant the native tier reads") <| @(t : T?) { + let types <- [GGML_TYPE_IQ4_XS, GGML_TYPE_Q3_K, GGML_TYPE_IQ3_S, GGML_TYPE_IQ3_XXS, GGML_TYPE_IQ4_NL, + GGML_TYPE_Q2_K, GGML_TYPE_IQ2_S, GGML_TYPE_IQ2_XS, GGML_TYPE_IQ2_XXS] + var ti = 0 + for (gtype in types) { + let sbb = gguf_kq_disk_sbb(gtype) + t |> success(sbb > 0l, "type {gtype} has a disk stride") + let fmt = gguf_kq_fmt_of_type(gtype) + let data <- synth_sb_bytes(sbb, 7 + ti) + let gg <- synth_gguf(gtype, 512l, data) + var m <- parse_gguf_meta(gg) + var deq : array + deq |> resize(512) + gguf_read_tensor_f32(m, gg, "t", deq, 0l, 512l) + // the oracle: the native two-step over the same bytes + var kq : array + var ks : array + kq |> resize(int(kq_qsb(fmt))) + ks |> resize(int(kq_ssb(fmt))) + var want : array + want |> resize(512) + let base = long_length(gg) - sbb * 2l + for (blk in range64(2l)) { + kq_transcode_superblock(fmt, gg, base + blk * sbb, kq, 0l, ks, 0l) + dequant_kq_plane_sb(fmt, kq, ks, 0l, false, want, blk * 256l) + } + var bad = 0 + for (i in range(512)) { + // random f16 scale bits can be NaN on both sides - a NaN pair is agreement + if (deq[i] != want[i] && !(deq[i] != deq[i] && want[i] != want[i])) { + bad++ + } + } + t |> equal(bad, 0, "type {gtype}: reader == transcode+dequant ({bad} of 512 differ)") + delete m + delete deq + delete want + delete kq + delete ks + ti++ + } + } +} diff --git a/modules/dasLLAMA/tests/test_kernel_coverage.das b/modules/dasLLAMA/tests/test_kernel_coverage.das index 88eabaa228..55d1a10cf2 100644 --- a/modules/dasLLAMA/tests/test_kernel_coverage.das +++ b/modules/dasLLAMA/tests/test_kernel_coverage.das @@ -69,7 +69,21 @@ let private CENSUS_NEVER_DISPATCHED <- [ // the dense tall in-kernel-dequant kq stamps: they engage only when a site's f16 panel // would reach TALLKQ_MIN_PANEL (96 MiB) - the zoo's small carriers top out far below it. // Their coverage is the tall kernel-unit leg in test_metal_gemm_kernels.das (kq_mulmm_gate) - "MetalKqMulMmK4TH128", "MetalKqMulMmK5TH128", "MetalKqMulMmK6TH128" + "MetalKqMulMmK4TH128", "MetalKqMulMmK5TH128", "MetalKqMulMmK6TH128", + // the nine formats the iquant arc added (Q2_K, Q3_K, IQ4_XS, IQ4_NL, IQ3_S, IQ3_XXS, IQ2_S, + // IQ2_XS, IQ2_XXS): no stocked census carrier holds a plane in any of them - the zoo quants + // are Q8_0 / Q5_K_M / Q4_K_M / mxfp4. Their coverage is the kernel-unit gates in + // test_metal_gemv_kernels.das / test_metal_gemm_kernels.das and the per-format e2e parity + // rows recorded in HOW_TO_ADD_A_FORMAT.md + "MetalKqGemvK2", "MetalKqMvB2K2", "MetalKqMvB4K2", "MetalKqMvB8K2", "MetalKqMulMmK2", + "MetalKqGemvK3", "MetalKqMvB2K3", "MetalKqMvB4K3", "MetalKqMvB8K3", "MetalKqMulMmK3", + "MetalKqGemvIq4xs", "MetalKqMvB2Iq4xs", "MetalKqMvB4Iq4xs", "MetalKqMvB8Iq4xs", "MetalKqMulMmIq4xs", + "MetalKqGemvIq4nl", "MetalKqMvB2Iq4nl", "MetalKqMvB4Iq4nl", "MetalKqMvB8Iq4nl", "MetalKqMulMmIq4nl", + "MetalKqGemvIq3s", "MetalKqMvB2Iq3s", "MetalKqMvB4Iq3s", "MetalKqMvB8Iq3s", "MetalKqMulMmIq3s", + "MetalKqGemvIq3xxs", "MetalKqMvB2Iq3xxs", "MetalKqMvB4Iq3xxs", "MetalKqMvB8Iq3xxs", "MetalKqMulMmIq3xxs", + "MetalKqGemvIq2s", "MetalKqMvB2Iq2s", "MetalKqMvB4Iq2s", "MetalKqMvB8Iq2s", "MetalKqMulMmIq2s", + "MetalKqGemvIq2xs", "MetalKqMvB2Iq2xs", "MetalKqMvB4Iq2xs", "MetalKqMvB8Iq2xs", "MetalKqMulMmIq2xs", + "MetalKqGemvIq2xxs", "MetalKqMvB2Iq2xxs", "MetalKqMvB4Iq2xxs", "MetalKqMvB8Iq2xxs", "MetalKqMulMmIq2xxs" ] def private counting_prompt(start, last : int) : string { diff --git a/modules/dasLLAMA/tests/test_kqformat.das b/modules/dasLLAMA/tests/test_kqformat.das index 22c6207011..6037f39254 100644 --- a/modules/dasLLAMA/tests/test_kqformat.das +++ b/modules/dasLLAMA/tests/test_kqformat.das @@ -9,6 +9,7 @@ options _dasllama_internal = true require dastest/testing_boost public require dasllama/dasllama_kqformat +require dasllama/dasllama_common // kq_fmt_row_ok - the dense-row demotion require dasllama/dasllama_gemm_schema // the int-id delegation under test [test] @@ -20,6 +21,15 @@ def test_kqfmt_enum_values(t : T?) { t |> equal(int(KqFmt.k6), 3) t |> equal(int(KqFmt.q40), 4) t |> equal(int(KqFmt.q51), 5) + t |> equal(int(KqFmt.iq4xs), 6) + t |> equal(int(KqFmt.k3), 7) + t |> equal(int(KqFmt.iq3s), 8) + t |> equal(int(KqFmt.iq3xxs), 9) + t |> equal(int(KqFmt.iq4nl), 10) + t |> equal(int(KqFmt.k2), 11) + t |> equal(int(KqFmt.iq2s), 12) + t |> equal(int(KqFmt.iq2xs), 13) + t |> equal(int(KqFmt.iq2xxs), 14) } } @@ -32,6 +42,24 @@ def test_kqfmt_predicates(t : T?) { t |> success(kq_sb(KqFmt.k6), "k6 is superblock") t |> success(kq_sb(KqFmt.q40), "q40 is superblock") t |> success(!kq_sb(KqFmt.q51), "q51 rides per-32 planes") + t |> success(kq_sb(KqFmt.iq4xs), "iq4xs is superblock") + t |> success(kq_sb(int(KqFmt.iq4xs)), "the int-id predicate agrees for iq4xs") + t |> success(kq_sb(KqFmt.k3), "k3 is superblock") + t |> success(kq_sb(int(KqFmt.k3)), "the int-id predicate agrees for k3") + t |> success(kq_sb(KqFmt.iq3s), "iq3s is superblock") + t |> success(kq_sb(KqFmt.iq3xxs), "iq3xxs is superblock") + t |> success(kq_sb(KqFmt.iq4nl), "iq4nl is superblock") + t |> success(kq_sb(KqFmt.k2), "k2 is superblock") + t |> success(kq_sb(int(KqFmt.iq3s)), "the int-id predicate agrees for iq3s") + t |> success(kq_sb(int(KqFmt.iq3xxs)), "the int-id predicate agrees for iq3xxs") + t |> success(kq_sb(int(KqFmt.iq4nl)), "the int-id predicate agrees for iq4nl") + t |> success(kq_sb(int(KqFmt.k2)), "the int-id predicate agrees for k2") + t |> success(kq_sb(KqFmt.iq2s), "iq2s is superblock") + t |> success(kq_sb(int(KqFmt.iq2s)), "the int-id predicate agrees for iq2s") + t |> success(kq_sb(KqFmt.iq2xs), "iq2xs is superblock") + t |> success(kq_sb(int(KqFmt.iq2xs)), "the int-id predicate agrees for iq2xs") + t |> success(kq_sb(KqFmt.iq2xxs), "iq2xxs is superblock") + t |> success(kq_sb(int(KqFmt.iq2xxs)), "the int-id predicate agrees for iq2xxs") } } @@ -48,6 +76,24 @@ def test_kqfmt_strides(t : T?) { t |> equal(kq_ssb(KqFmt.q40), 16l) // 8 x f16 d t |> equal(kq_qsb(KqFmt.q51), 20l) // per 32-BLOCK: 16 nibbles + 4 qh t |> equal(kq_ssb(KqFmt.q51), 4l) // per 32-BLOCK: f16 d + f16 m + t |> equal(kq_qsb(KqFmt.iq4xs), 128l) // the q40 tiling (disk k/k+16 pairing) + t |> equal(kq_ssb(KqFmt.iq4xs), 20l) // f16 d + 8 int8 sub-scales in the k4 row shape + t |> equal(kq_qsb(KqFmt.k3), 96l) // 64 qs + 32 hmask, verbatim + t |> equal(kq_ssb(KqFmt.k3), 18l) // the k6 row: 16 int8 sub-scales + f16 d + t |> equal(kq_qsb(KqFmt.iq3s), 104l) // [64 qs][8 qh][32 signs], verbatim + t |> equal(kq_qsb(KqFmt.iq3xxs), 96l) // [64 qs][32 aux], verbatim + t |> equal(kq_qsb(KqFmt.iq4nl), 128l) // q40's nibble plane, verbatim + t |> equal(kq_ssb(KqFmt.iq3s), 20l) // the k4 row: f16 d + 8 int8 (1 + 2s) + t |> equal(kq_ssb(KqFmt.iq3xxs), 20l) // the same row: f16 d halved + 8 x (2ls + 1) + t |> equal(kq_ssb(KqFmt.iq4nl), 16l) // q40's row: 8 x f16 d + t |> equal(kq_qsb(KqFmt.k2), 64l) // 64 qs bytes, verbatim + t |> equal(kq_ssb(KqFmt.k2), 20l) // [f16 d][f16 dmin][16 sc/min bytes] + t |> equal(kq_qsb(KqFmt.iq2s), 72l) // [32 idx][32 signs][8 qh], verbatim + t |> equal(kq_ssb(KqFmt.iq2s), 20l) // f16 d/8 + 16 x (1 + 2s) strips + t |> equal(kq_qsb(KqFmt.iq2xs), 64l) // 32 u16 [9-bit grid idx][7-bit ksigns idx] words, verbatim + t |> equal(kq_ssb(KqFmt.iq2xs), 20l) // the iq2s row exactly - f16 d/8 + 16 x (1 + 2s) + t |> equal(kq_qsb(KqFmt.iq2xxs), 64l) // 8 blocks x [4 grid bytes][aux32 signs + ls], verbatim + t |> equal(kq_ssb(KqFmt.iq2xxs), 20l) // the iq2s row exactly - f16 d/8 + 16 x (1 + 2s) } t |> run("named constants agree with the literals") @(t : T?) { t |> equal(K4_QSB, 128l) @@ -60,19 +106,62 @@ def test_kqfmt_strides(t : T?) { t |> equal(Q40_SSB, 16l) t |> equal(Q51_QB, 20l) t |> equal(Q51_SB, 4l) + t |> equal(IQ4XS_QSB, 128l) + t |> equal(IQ4XS_SSB, 20l) + t |> equal(K3_QSB, 96l) + t |> equal(K3_SSB, 18l) + t |> equal(K2_QSB, 64l) + t |> equal(K2_SSB, 20l) + t |> equal(IQ2S_QSB, 72l) + t |> equal(IQ2S_SSB, 20l) + t |> equal(IQ2XS_QSB, 64l) + t |> equal(IQ2XS_SSB, 20l) + t |> equal(IQ2XXS_QSB, 64l) + t |> equal(IQ2XXS_SSB, 20l) + t |> equal(IQ3S_QSB, 104l) + t |> equal(IQ3S_SSB, 20l) + } + t |> run("the IQ4_NL codebook is upstream's kvalues_iq4nl") @(t : T?) { + t |> equal(int(IQ4NL_LUT[0]), -127) + t |> equal(int(IQ4NL_LUT[7]), -10) + t |> equal(int(IQ4NL_LUT[8]), 1) + t |> equal(int(IQ4NL_LUT[15]), 113) + for (i in range(15)) { + t |> success(int(IQ4NL_LUT[i]) < int(IQ4NL_LUT[i + 1]), "the codebook is monotonic") + } + } +} + +[test] +def test_kqfmt_grid(t : T?) { + t |> run("the IQ3_S grid is upstream's iq3s_grid, and the worker-safe local agrees") @(t : T?) { + t |> equal(IQ3S_GRID[0], 0x01010101u) + t |> equal(IQ3S_GRID[511], 0x0f0f0101u) + let lut = iq3s_grid() + t |> equal(IQ3S_GRID[256], lut[256]) + t |> equal(IQ3S_GRID[303], lut[303]) } } [test] def test_kqfmt_id_bridge(t : T?) { - t |> run("kq_schema_id is the one enum->kernel-id bridge (4/5/6/40)") @(t : T?) { + t |> run("kq_schema_id is the one enum->kernel-id bridge (3/4/5/6/40/44/33)") @(t : T?) { t |> equal(kq_schema_id(KqFmt.k4), 4) t |> equal(kq_schema_id(KqFmt.k5), 5) t |> equal(kq_schema_id(KqFmt.k6), 6) t |> equal(kq_schema_id(KqFmt.q40), 40) + t |> equal(kq_schema_id(KqFmt.iq4xs), 44) + t |> equal(kq_schema_id(KqFmt.k3), 3) + t |> equal(kq_schema_id(KqFmt.iq3s), 33) + t |> equal(kq_schema_id(KqFmt.iq3xxs), 34) + t |> equal(kq_schema_id(KqFmt.iq4nl), 45) + t |> equal(kq_schema_id(KqFmt.k2), 2) + t |> equal(kq_schema_id(KqFmt.iq2s), 23) + t |> equal(kq_schema_id(KqFmt.iq2xs), 24) + t |> equal(kq_schema_id(KqFmt.iq2xxs), 25) } t |> run("gemm_schema's int-id forms delegate to the same truth") @(t : T?) { - for (f in [KqFmt.k4, KqFmt.k5, KqFmt.k6, KqFmt.q40]) { + for (f in [KqFmt.k4, KqFmt.k5, KqFmt.k6, KqFmt.q40, KqFmt.iq4xs, KqFmt.k3, KqFmt.iq3s, KqFmt.iq3xxs, KqFmt.iq4nl, KqFmt.k2, KqFmt.iq2s, KqFmt.iq2xs, KqFmt.iq2xxs]) { t |> equal(kq_qsb(kq_schema_id(f)), kq_qsb(f)) t |> equal(kq_ssb(kq_schema_id(f)), kq_ssb(f)) } @@ -86,6 +175,15 @@ def test_kqfmt_elems_and_stream(t : T?) { t |> equal(kq_elems(KqFmt.k5), 256l) t |> equal(kq_elems(KqFmt.k6), 256l) t |> equal(kq_elems(KqFmt.q40), 256l) + t |> equal(kq_elems(KqFmt.iq4xs), 256l) + t |> equal(kq_elems(KqFmt.k3), 256l) + t |> equal(kq_elems(KqFmt.iq3s), 256l) + t |> equal(kq_elems(KqFmt.iq3xxs), 256l) + t |> equal(kq_elems(KqFmt.iq4nl), 256l) + t |> equal(kq_elems(KqFmt.k2), 256l) + t |> equal(kq_elems(KqFmt.iq2s), 256l) + t |> equal(kq_elems(KqFmt.iq2xs), 256l) + t |> equal(kq_elems(KqFmt.iq2xxs), 256l) t |> equal(kq_elems(KqFmt.q8), 32l) t |> equal(kq_elems(KqFmt.q51), 32l) } @@ -96,5 +194,57 @@ def test_kqfmt_elems_and_stream(t : T?) { t |> equal(kq_stream_code(KqFmt.k5), 5) t |> equal(kq_stream_code(KqFmt.k6), 6) t |> equal(kq_stream_code(KqFmt.q40), 40) + t |> equal(kq_stream_code(KqFmt.iq4xs), 44) + t |> equal(kq_stream_code(KqFmt.k3), 3) + t |> equal(kq_stream_code(KqFmt.iq3s), 33) + t |> equal(kq_stream_code(KqFmt.iq3xxs), 34) + t |> equal(kq_stream_code(KqFmt.iq4nl), 45) + t |> equal(kq_stream_code(KqFmt.k2), 20) // kernel id 2 = q51's stream tag + t |> equal(kq_stream_code(KqFmt.iq2s), 23) + t |> equal(kq_stream_code(KqFmt.iq2xs), 24) + t |> equal(kq_stream_code(KqFmt.iq2xxs), 25) + } +} + +[test] +def test_iq3xxs_tables(t : T?) { + t |> run("the halved IQ3_XXS grid, ksigns and the f16 halver hold their contracts") @(t : T?) { + let lut = iq3xxs_grid() + for (i in range(256)) { + t |> equal(IQ3XXS_GRID[i], lut[i]) + for (b in range(4)) { + let v = int((lut[i] >> uint(8 * b)) & 0xFFu) + // the disk levels 4..62 halved: even 2..30 plus 31 (the 62 top level) + t |> success(v == 31 || (v >= 2 && v <= 30 && v % 2 == 0), "halved grid byte in range") + } + } + let ks = ksigns_iq2xs() + for (i in range(128)) { + t |> equal(int(KSIGNS_IQ2XS[i]), int(ks[i])) + var parity = 0 + for (b in range(7)) { + parity ^= (i >> b) & 1 + } + t |> equal(int(ks[i]), i | (parity << 7)) // bit 7 = even parity of the 7-bit index + } + // exact halving: a normal f16 halves by exponent decrement (0x3400 = 0.25 -> 0x3000 = + // 0.125), the sign rides along, and the exp==1 edge shifts into the subnormal space + t |> equal(f16_half_bits(0x3400u), 0x3000u) + t |> equal(f16_half_bits(0xB400u), 0xB000u) + t |> equal(f16_half_bits(0x0400u), 0x0200u) + } +} + +[test] +def test_kq_fmt_row_demotion(t : T?) { + t |> run("the 32-block disk formats demote to q8 on rows the superblock walkers cannot tile") <| @(t : T?) { + // 704 = 22 blocks of 32, not a 256-multiple: q40, iq4nl and Q5_0-born k5 all fall to q8 + t |> equal(kq_fmt_row_ok(KqFmt.q40, 704l), KqFmt.q8) + t |> equal(kq_fmt_row_ok(KqFmt.iq4nl, 704l), KqFmt.q8) + t |> equal(kq_fmt_row_ok(KqFmt.k5, 704l), KqFmt.q8) + t |> equal(kq_fmt_row_ok(KqFmt.iq4nl, 768l), KqFmt.iq4nl) + t |> equal(kq_fmt_row_ok(KqFmt.q40, 768l), KqFmt.q40) + // the 256-superblock disk formats never see a non-multiple row (the file cannot hold one) + t |> equal(kq_fmt_row_ok(KqFmt.iq2xxs, 768l), KqFmt.iq2xxs) } } diff --git a/modules/dasLLAMA/tests/test_kquant.das b/modules/dasLLAMA/tests/test_kquant.das index 63a665431d..c1bf709a7a 100644 --- a/modules/dasLLAMA/tests/test_kquant.das +++ b/modules/dasLLAMA/tests/test_kquant.das @@ -1,13 +1,14 @@ options gen2 options stack = 524288 // every dasLLAMA program root takes this budget (options stack does not unify up from libs) options persistent_heap // the layout arm loads a multi-GB model; explicit delete below -options _cyclomatic_complexity = 35 // every gate here is the same flat fmt == 4/5/6/40 ladder repeated per stage -options _function_length = 170 // and each gate carries its whole fixture build + oracle inline +options _cyclomatic_complexity = 76 // every gate here is the same flat per-format ladder (thirteen formats) repeated per stage +options _function_length = 300 // and each gate carries its whole fixture build + oracle inline options _dasllama_internal = true require dastest/testing_boost public require _model_tier require dasllama/dasllama_gguf +require dasllama/dasllama_common // dequant_kq_plane_sb - the carrier-form dispatcher require dasllama/dasllama_convert require dasllama/dasllama_math // pin_kernel_backend — the layout arm needs disk-order planes require dasllama/dasllama_math_default // dot_k4q8/k5/k6/q40q8 — the kernel arms gate them vs the plane dequant @@ -27,6 +28,8 @@ require math def private q4_pat(k : int) : int => (k * 7 + 3) % 16 def private q5_pat(k : int) : int => (k * 3 + 7) % 32 def private q6_pat(k : int) : int => (k * 5 + 11) % 64 +def private q3_pat(k : int) : int => (k * 5 + 3) % 8 +def private q2_pat(k : int) : int => (k * 3 + 1) % 4 // The shared synthetic scale/min sets: 6-bit values > 15 exercise the split high-bit arms of // the 6-bit scale/min unpack (j >= 4); d/dmin are f16-exact so expected-value math uses the f32 value. @@ -102,9 +105,250 @@ def private build_q6k_block() : array { return <- blkb } -// loop-generated test labels: "k4"/"k5"/"k6" for the K-quants, "q40" for fmt 40 — matches the -// kernel/repack function names (dot_q40q8, repack_q40_grp) so failures grep 1:1 -def private kq_tag(fmt : int) : string => fmt == 40 ? "q40" : "k{fmt}" +// loop-generated test labels: "k4"/"k5"/"k6" for the K-quants, "q40" for fmt 40, "iq4xs" for +// fmt 44 — matches the kernel/repack function names (dot_q40q8, repack_q40_grp) so failures grep 1:1 +// iq3xxs synthetic superblock: 98B IQ3_XXS disk block — f16 d, 64 grid-index bytes, 8 x aux32 +// (4x7-bit ksigns indices + a 4-bit scale in the top nibble). Any byte pattern is legal. +def private iq3xxs_qs_pat(i : int) : int => (i * 13 + 5) % 256 +def private iq3xxs_aux_pat(p : int) : int => (p * 9 + 17) % 256 +def private build_iq3xxs_block() : array { + var blkb : array + blkb |> resize(98) + let dbits = f32_to_f16(0.002) + blkb[0] = uint8(dbits & 0xFF) + blkb[1] = uint8(dbits >> 8u) + for (i in range(64)) { + blkb[2 + i] = uint8(iq3xxs_qs_pat(i)) + } + for (p in range(32)) { + blkb[66 + p] = uint8(iq3xxs_aux_pat(p)) + } + return <- blkb +} + +// iq4nl synthetic superblock: 8 x 18B IQ4_NL disk blocks (per block f16 d + 16 nibble bytes, +// q40's exact disk shape). Any byte pattern is legal. +def private iq4nl_qs_pat(i : int) : int => (i * 19 + 7) % 256 +def private build_iq4nl_block() : array { + var blkb : array + blkb |> resize(144) + for (b in range(8)) { + let dbits = f32_to_f16(0.002 + 0.0003 * float(b)) + blkb[b * 18] = uint8(dbits & 0xFF) + blkb[b * 18 + 1] = uint8(dbits >> 8u) + for (i in range(16)) { + blkb[b * 18 + 2 + i] = uint8(iq4nl_qs_pat(b * 16 + i)) + } + } + return <- blkb +} + +def private kq_tag(fmt : int) : string => fmt == 40 ? "q40" : (fmt == 44 ? "iq4xs" : (fmt == 33 ? "iq3s" : (fmt == 34 ? "iq3xxs" : (fmt == 45 ? "iq4nl" : (fmt == 23 ? "iq2s" : (fmt == 24 ? "iq2xs" : (fmt == 25 ? "iq2xxs" : "k{fmt}"))))))) + +// iq4xs synthetic superblock: 136B disk block — f16 d = SYNTH_D, the 8 six-bit sub-scales are +// SYNTH_SC (packed low nibbles into scales_l, top two bits into scales_h), the shared nibble +// pattern in the k/k+16 pairing — weight k = (d * (SYNTH_SC[k/32] - 32)) * IQ4NL_LUT[q4_pat(k)] +def private build_iq4xs_block() : array { + var blkb : array + blkb |> resize(136) + let dbits = f32_to_f16(SYNTH_D) + blkb[0] = uint8(dbits & 0xFF) + blkb[1] = uint8(dbits >> 8u) + var sh = 0 + for (blk in range(8)) { + sh |= ((SYNTH_SC[blk] >> 4) & 3) << (2 * blk) + } + blkb[2] = uint8(sh & 0xFF) + blkb[3] = uint8(sh >> 8) + for (j in range(4)) { // scales_l[j]: block 2j's low nibble, block 2j+1's in the high nibble + blkb[4 + j] = uint8((SYNTH_SC[2 * j] & 15) | ((SYNTH_SC[2 * j + 1] & 15) << 4)) + } + for (blk in range(8)) { + for (j in range(16)) { // byte j: weight blk*32+j in the low nibble, +16 in the high + blkb[8 + blk * 16 + j] = uint8(q4_pat(blk * 32 + j) | (q4_pat(blk * 32 + 16 + j) << 4)) + } + } + return <- blkb +} + +// k3 synthetic superblock: 110B Q3_K disk block — 16 six-bit sub-scales SYNTH_SC[is % 8] packed in +// the disk kmask form (written in the PACK direction), f16 d = SYNTH_D, the 2-bit lanes + hmask bits +// of q3_pat — weight k = (d * (SYNTH_SC[(k/16) % 8] - 32)) * (q3_pat(k) - 4) +def private build_q3k_block() : array { + var blkb : array + blkb |> resize(110) + var hm : array + var qs : array + var sc : array + hm |> resize(32) + qs |> resize(64) + sc |> resize(12) + for (k in range(256)) { // block b = k/32: qs byte (b/4)*32 + l holds the 2-bit lane at shift 2*(b%4); hmask byte l bit b + let b = k / 32 + let l = k % 32 + let q = q3_pat(k) + qs[(b / 4) * 32 + l] |= (q & 3) << (2 * (b % 4)) + hm[l] |= (q >> 2) << b + } + for (i in range(16)) { // scale i: low nibble in byte i (i < 8) / high nibble of byte i-8, top two bits in byte 8 + i%4 at 2*(i/4) + let v = SYNTH_SC[i % 8] + if (i < 8) { + sc[i] |= v & 15 + } else { + sc[i - 8] |= (v & 15) << 4 + } + sc[8 + i % 4] |= ((v >> 4) & 3) << (2 * (i / 4)) + } + for (i in range(32)) { + blkb[i] = uint8(hm[i]) + } + for (i in range(64)) { + blkb[32 + i] = uint8(qs[i]) + } + for (i in range(12)) { + blkb[96 + i] = uint8(sc[i]) + } + let dbits = f32_to_f16(SYNTH_D) + blkb[108] = uint8(dbits & 0xFF) + blkb[109] = uint8(dbits >> 8u) + return <- blkb +} + +// iq3s synthetic superblock: 110B IQ3_S disk block — 4-bit sub-scales SYNTH_SC[b] & 15, f16 d = +// SYNTH_D, grid-index/qh/sign byte patterns (written in the PACK direction); the gate recomposes +// each weight through IQ3S_GRID +def private iq3s_qs_pat(i : int) : int => (i * 11 + 3) % 256 +def private iq3s_qh_pat(b : int) : int => (b * 5 + 1) % 256 +def private iq3s_sg_pat(p : int) : int => (p * 7 + 13) % 256 +def private build_iq3s_block() : array { + var blkb : array + blkb |> resize(110) + let dbits = f32_to_f16(SYNTH_D) + blkb[0] = uint8(dbits & 0xFF) + blkb[1] = uint8(dbits >> 8u) + for (i in range(64)) { + blkb[2 + i] = uint8(iq3s_qs_pat(i)) + } + for (b in range(8)) { + blkb[66 + b] = uint8(iq3s_qh_pat(b)) + } + for (p in range(32)) { + blkb[74 + p] = uint8(iq3s_sg_pat(p)) + } + for (b in range(8)) { // scale nibble b: low nibble of byte b/2 (b even), high nibble (b odd) + let v = SYNTH_SC[b] & 15 + let nib = b % 2 == 0 ? v : v << 4 + blkb[106 + b / 2] = uint8(int(blkb[106 + b / 2]) | nib) + } + return <- blkb +} + +// k2 synthetic superblock: 84B Q2_K disk block - per-group sc/min nibble pairs (SYNTH_SC & 15, +// SYNTH_MN & 15), f16 d/dmin, the 2-bit lanes of q2_pat (written in the PACK direction) - +// weight k = SYNTH_D * (SYNTH_SC[g % 8] & 15) * q2_pat(k) - SYNTH_DMIN * (SYNTH_MN[g % 8] & 15), g = k / 16 +def private build_q2k_block() : array { + var blkb : array + blkb |> resize(84) + var qs : array + qs |> resize(64) + for (k in range(256)) { // group g = k/16: qs byte (g/8)*32 + (g%2)*16 + k%16 holds the 2-bit lane at shift 2*((g%8)/2) + let g = k / 16 + qs[(g / 8) * 32 + (g % 2) * 16 + k % 16] |= q2_pat(k) << (2 * ((g % 8) / 2)) + } + for (g in range(16)) { + blkb[g] = uint8((SYNTH_SC[g % 8] & 15) | ((SYNTH_MN[g % 8] & 15) << 4)) + } + for (i in range(64)) { + blkb[16 + i] = uint8(qs[i]) + } + let dbits = f32_to_f16(SYNTH_D) + blkb[80] = uint8(dbits & 0xFF) + blkb[81] = uint8(dbits >> 8u) + let mbits = f32_to_f16(SYNTH_DMIN) + blkb[82] = uint8(mbits & 0xFF) + blkb[83] = uint8(mbits >> 8u) + return <- blkb +} + +// iq2s synthetic superblock: 82B IQ2_S disk block - per-16 scale nibbles SYNTH_SC & 15, f16 d, +// idx/qh/sign byte patterns (written in the PACK direction); the gate recomposes each weight +// through IQ2S_GRID +def private iq2s_ix_pat(i : int) : int => (i * 13 + 5) % 256 +def private iq2s_qh_pat(b : int) : int => (b * 3 + 1) % 256 +def private iq2s_sg_pat(p : int) : int => (p * 7 + 11) % 256 +def private build_iq2s_block() : array { + var blkb : array + blkb |> resize(82) + let dbits = f32_to_f16(SYNTH_D) + blkb[0] = uint8(dbits & 0xFF) + blkb[1] = uint8(dbits >> 8u) + for (i in range(32)) { + blkb[2 + i] = uint8(iq2s_ix_pat(i)) + } + for (p in range(32)) { + blkb[34 + p] = uint8(iq2s_sg_pat(p)) + } + for (b in range(8)) { + blkb[66 + b] = uint8(iq2s_qh_pat(b)) + } + for (g in range(16)) { // scale nibble g: low nibble of byte g/2 (g even), high (g odd) + let v = SYNTH_SC[g % 8] & 15 + let nib = g % 2 == 0 ? v : v << 4 + blkb[74 + g / 2] = uint8(int(blkb[74 + g / 2]) | nib) + } + return <- blkb +} + +// iq2xs synthetic superblock: 74B IQ2_XS disk block - per-16 scale nibbles SYNTH_SC & 15, +// f16 d, u16 qs words carrying a 9-bit grid index + 7-bit ksigns index pattern (written in the +// PACK direction); the gate recomposes each weight through IQ2XS_GRID + KSIGNS_IQ2XS +def private iq2xs_ix_pat(i : int) : int => (i * 29 + 7) % 512 +def private iq2xs_sg7_pat(i : int) : int => (i * 5 + 3) % 128 +def private build_iq2xs_block() : array { + var blkb : array + blkb |> resize(74) + let dbits = f32_to_f16(SYNTH_D) + blkb[0] = uint8(dbits & 0xFF) + blkb[1] = uint8(dbits >> 8u) + for (w in range(32)) { + let v = iq2xs_ix_pat(w) | (iq2xs_sg7_pat(w) << 9) + blkb[2 + w * 2] = uint8(v & 0xFF) + blkb[3 + w * 2] = uint8(v >> 8) + } + for (g in range(16)) { // scale nibble g: low nibble of byte g/2 (g even), high (g odd) + let v = SYNTH_SC[g % 8] & 15 + let nib = g % 2 == 0 ? v : v << 4 + blkb[66 + g / 2] = uint8(int(blkb[66 + g / 2]) | nib) + } + return <- blkb +} + +// iq2xxs synthetic superblock: 66B IQ2_XXS disk block - f16 d, per block 4 grid byte indices +// + an aux32 packing four 7-bit ksigns indices and the per-32 scale nibble (written in the +// PACK direction); the gate recomposes each weight through IQ2XXS_GRID + the shared KSIGNS_IQ2XS +def private iq2xxs_ix_pat(i : int) : int => (i * 29 + 7) % 256 +def private iq2xxs_sg7_pat(i : int) : int => (i * 5 + 3) % 128 +def private build_iq2xxs_block() : array { + var blkb : array + blkb |> resize(66) + let dbits = f32_to_f16(SYNTH_D) + blkb[0] = uint8(dbits & 0xFF) + blkb[1] = uint8(dbits >> 8u) + for (b in range(8)) { + for (l in range(4)) { + blkb[2 + b * 8 + l] = uint8(iq2xxs_ix_pat(b * 4 + l)) + } + var aux = 0 + for (l in range(4)) { + aux = aux | (iq2xxs_sg7_pat(b * 4 + l) << (7 * l)) + } + aux = aux | ((SYNTH_SC[b % 8] & 15) << 28) + for (k in range(4)) { + blkb[2 + b * 8 + 4 + k] = uint8((aux >> (8 * k)) & 0xFF) + } + } + return <- blkb +} // q40 synthetic superblock: 8 x 18B q4_0 disk blocks, per-block f16-exact d = (blk+1)/16 and // the shared nibble pattern — weight k = d(k/32) * (q4_pat(k) - 8) @@ -259,6 +503,160 @@ def test_kq_transcode_planes(t : T?) { t |> success(dst[k] == expected, "q40 plane element must match hand-packed value exactly") } } + t |> run("iq4xs planes decode the 6-bit sub-scales and the codebook exactly") @(t : T?) { + let blkb <- build_iq4xs_block() + var kq : array + var ks : array + kq |> resize(128) + ks |> resize(20) + transcode_iq4xs_superblock(blkb, 0l, kq, 0l, ks, 0l) + for (blk in range(8)) { + t |> equal(iq4xs_sc(ks, 0l, int64(blk)), SYNTH_SC[blk] - 32, "decoded sub-scale is ls - 32, signed") + } + var dst : array + dst |> resize(256) + dequant_iq4xs_plane_superblock(kq, 0l, ks, 0l, dst, 0l) + for (k in range(256)) { + let expected = (SYNTH_D * float(SYNTH_SC[k / 32] - 32)) * float(IQ4NL_LUT[q4_pat(k)]) + t |> success(dst[k] == expected, "iq4xs plane element must match the upstream dequant exactly") + } + } + t |> run("k3 planes decode the packed 6-bit sub-scales and compose the 3-bit quants exactly") @(t : T?) { + let blkb <- build_q3k_block() + var kq : array + var ks : array + kq |> resize(96) + ks |> resize(18) + transcode_q3k_superblock(blkb, 0l, kq, 0l, ks, 0l) + for (i in range(16)) { + t |> equal(int(int8(ks[i])), SYNTH_SC[i % 8] - 32, "decoded sub-scale is scale - 32, signed") + } + var dst : array + dst |> resize(256) + dequant_k3_plane_superblock(kq, 0l, ks, 0l, dst, 0l) + for (k in range(256)) { + let expected = (SYNTH_D * float(SYNTH_SC[(k / 16) % 8] - 32)) * float(q3_pat(k) - 4) + t |> success(dst[k] == expected, "k3 plane element must match the upstream dequant exactly") + } + } + t |> run("iq3s planes decode the 4-bit sub-scales and gather the grid exactly") @(t : T?) { + let blkb <- build_iq3s_block() + var kq : array + var ks : array + kq |> resize(104) + ks |> resize(20) + transcode_iq3s_superblock(blkb, 0l, kq, 0l, ks, 0l) + for (b in range(8)) { + t |> equal(int(int8(ks[4 + b])), 1 + 2 * (SYNTH_SC[b] & 15), "decoded sub-scale is 1 + 2s") + } + var dst : array + dst |> resize(256) + dequant_iq3s_plane_superblock(kq, 0l, ks, 0l, dst, 0l) + for (k in range(256)) { + let b = k / 32 + let l = (k % 32) / 8 + let hj = k % 8 + let qi = iq3s_qs_pat(b * 8 + 2 * l + hj / 4) | ((iq3s_qh_pat(b) << (hj < 4 ? 8 - 2 * l : 7 - 2 * l)) & 256) + let g = int((IQ3S_GRID[qi] >> uint(8 * (hj % 4))) & 0xFFu) + let sgn = ((iq3s_sg_pat(b * 4 + l) >> hj) & 1) != 0 ? -1.0 : 1.0 + let expected = (SYNTH_D * float(1 + 2 * (SYNTH_SC[b] & 15))) * float(g) * sgn + t |> success(dst[k] == expected, "iq3s plane element must match the upstream dequant exactly") + } + } + t |> run("k2 planes reorder the header and compose the 2-bit quants exactly") @(t : T?) { + let blkb <- build_q2k_block() + var kq : array + var ks : array + kq |> resize(64) + ks |> resize(20) + transcode_q2k_superblock(blkb, 0l, kq, 0l, ks, 0l) + var dst : array + dst |> resize(256) + dequant_k2_plane_superblock(kq, 0l, ks, 0l, dst, 0l) + for (k in range(256)) { + let g = k / 16 + let expected = SYNTH_D * float(SYNTH_SC[g % 8] & 15) * float(q2_pat(k)) - SYNTH_DMIN * float(SYNTH_MN[g % 8] & 15) + t |> success(dst[k] == expected, "k2 plane element must match the upstream dequant exactly") + } + } + t |> run("iq2xs planes eighth the d, decode the strips, and gather the 512-entry grid + ksigns exactly") @(t : T?) { + let blkb <- build_iq2xs_block() + var kq : array + var ks : array + kq |> resize(64) + ks |> resize(20) + transcode_iq2xs_superblock(blkb, 0l, kq, 0l, ks, 0l) + for (g in range(16)) { + t |> equal(int(ks[4 + g]), 1 + 2 * (SYNTH_SC[g % 8] & 15), "decoded strip is 1 + 2s") + } + var dst : array + dst |> resize(256) + dequant_iq2xs_plane_superblock(kq, 0l, ks, 0l, dst, 0l) + let d8 = SYNTH_D * 0.125 + for (k in range(256)) { + let b = k / 32 + let l = (k % 32) / 8 + let j = k % 8 + let idx = iq2xs_ix_pat(b * 4 + l) + let gw = j < 4 ? IQ2XS_GRID[idx * 2] : IQ2XS_GRID[idx * 2 + 1] + let g = int((gw >> uint(8 * (j % 4))) & 0xFFu) + let sgn = ((int(KSIGNS_IQ2XS[iq2xs_sg7_pat(b * 4 + l)]) >> j) & 1) != 0 ? -1.0 : 1.0 + let expected = (d8 * float(1 + 2 * (SYNTH_SC[(2 * b + l / 2) % 8] & 15))) * float(g) * sgn + t |> success(dst[k] == expected, "iq2xs plane element must match the upstream dequant exactly") + } + } + t |> run("iq2xxs planes eighth the d, decode the per-32 strips, and gather the 256-entry grid + ksigns exactly") @(t : T?) { + let blkb <- build_iq2xxs_block() + var kq : array + var ks : array + kq |> resize(64) + ks |> resize(20) + transcode_iq2xxs_superblock(blkb, 0l, kq, 0l, ks, 0l) + for (b in range(8)) { + t |> equal(int(ks[4 + b]), 1 + 2 * (SYNTH_SC[b % 8] & 15), "decoded per-32 strip is 1 + 2s") + } + var dst : array + dst |> resize(256) + dequant_iq2xxs_plane_superblock(kq, 0l, ks, 0l, dst, 0l) + let d8 = SYNTH_D * 0.125 + for (k in range(256)) { + let b = k / 32 + let l = (k % 32) / 8 + let j = k % 8 + let idx = iq2xxs_ix_pat(b * 4 + l) + let gw = j < 4 ? IQ2XXS_GRID[idx * 2] : IQ2XXS_GRID[idx * 2 + 1] + let g = int((gw >> uint(8 * (j % 4))) & 0xFFu) + let sgn = ((int(KSIGNS_IQ2XS[iq2xxs_sg7_pat(b * 4 + l)]) >> j) & 1) != 0 ? -1.0 : 1.0 + let expected = (d8 * float(1 + 2 * (SYNTH_SC[b % 8] & 15))) * float(g) * sgn + t |> success(dst[k] == expected, "iq2xxs plane element must match the upstream dequant exactly") + } + } + t |> run("iq2s planes eighth the d, decode the strips, and gather the u64 grid exactly") @(t : T?) { + let blkb <- build_iq2s_block() + var kq : array + var ks : array + kq |> resize(72) + ks |> resize(20) + transcode_iq2s_superblock(blkb, 0l, kq, 0l, ks, 0l) + for (g in range(16)) { + t |> equal(int(ks[4 + g]), 1 + 2 * (SYNTH_SC[g % 8] & 15), "decoded strip is 1 + 2s") + } + var dst : array + dst |> resize(256) + dequant_iq2s_plane_superblock(kq, 0l, ks, 0l, dst, 0l) + let d8 = SYNTH_D * 0.125 + for (k in range(256)) { + let b = k / 32 + let l = (k % 32) / 8 + let j = k % 8 + let idx = iq2s_ix_pat(b * 4 + l) | ((iq2s_qh_pat(b) << (8 - 2 * l)) & 0x300) + let gw = j < 4 ? IQ2S_GRID[idx * 2] : IQ2S_GRID[idx * 2 + 1] + let g = int((gw >> uint(8 * (j % 4))) & 0xFFu) + let sgn = ((iq2s_sg_pat(b * 4 + l) >> j) & 1) != 0 ? -1.0 : 1.0 + let expected = (d8 * float(1 + 2 * (SYNTH_SC[(2 * b + l / 2) % 8] & 15))) * float(g) * sgn + t |> success(dst[k] == expected, "iq2s plane element must match the upstream dequant exactly") + } + } t |> run("q51 planes unpack the hand-packed Q5_1 blocks exactly") @(t : T?) { for (blk in range(4)) { let blkb <- build_q51_block(blk) @@ -505,7 +903,7 @@ def test_q51_kernels(t : T?) { // elementwise dot over the plane dequant (the kernels' declared oracle). def private kq_dot_gate(t : T?; fmt : int; n : int64 = 512l) { let nsb = n / 256l - let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : build_q6k_block())) + let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : (fmt == 45 ? build_iq4nl_block() : (fmt == 2 ? build_q2k_block() : (fmt == 23 ? build_iq2s_block() : (fmt == 24 ? build_iq2xs_block() : (fmt == 25 ? build_iq2xxs_block() : build_q6k_block()))))))))))) let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) var kq : array @@ -519,6 +917,24 @@ def private kq_dot_gate(t : T?; fmt : int; n : int64 = 512l) { transcode_q5k_superblock(blk1, 0l, kq, s * qsb, ks, s * ssb) } elif (fmt == 40) { transcode_q40_superblock(blk1, 0l, kq, s * qsb, ks, s * ssb) + } elif (fmt == 44) { + transcode_iq4xs_superblock(blk1, 0l, kq, s * qsb, ks, s * ssb) + } elif (fmt == 3) { + transcode_q3k_superblock(blk1, 0l, kq, s * qsb, ks, s * ssb) + } elif (fmt == 33) { + transcode_iq3s_superblock(blk1, 0l, kq, s * qsb, ks, s * ssb) + } elif (fmt == 34) { + transcode_iq3xxs_superblock(blk1, 0l, kq, s * qsb, ks, s * ssb) + } elif (fmt == 45) { + transcode_iq4nl_superblock(blk1, 0l, kq, s * qsb, ks, s * ssb) + } elif (fmt == 2) { + transcode_q2k_superblock(blk1, 0l, kq, s * qsb, ks, s * ssb) + } elif (fmt == 24) { + transcode_iq2xs_superblock(blk1, 0l, kq, s * qsb, ks, s * ssb) + } elif (fmt == 25) { + transcode_iq2xxs_superblock(blk1, 0l, kq, s * qsb, ks, s * ssb) + } elif (fmt == 23) { + transcode_iq2s_superblock(blk1, 0l, kq, s * qsb, ks, s * ssb) } else { transcode_q6k_superblock(blk1, 0l, kq, s * qsb, ks, s * ssb) } @@ -538,6 +954,24 @@ def private kq_dot_gate(t : T?; fmt : int; n : int64 = 512l) { dequant_k5_plane_superblock(kq, s * qsb, ks, s * ssb, w, s * 256l) } elif (fmt == 40) { dequant_q40_plane_superblock(kq, s * qsb, ks, s * ssb, w, s * 256l) + } elif (fmt == 44) { + dequant_iq4xs_plane_superblock(kq, s * qsb, ks, s * ssb, w, s * 256l) + } elif (fmt == 3) { + dequant_k3_plane_superblock(kq, s * qsb, ks, s * ssb, w, s * 256l) + } elif (fmt == 33) { + dequant_iq3s_plane_superblock(kq, s * qsb, ks, s * ssb, w, s * 256l) + } elif (fmt == 34) { + dequant_iq3xxs_plane_superblock(kq, s * qsb, ks, s * ssb, w, s * 256l) + } elif (fmt == 45) { + dequant_iq4nl_plane_superblock(kq, s * qsb, ks, s * ssb, w, s * 256l) + } elif (fmt == 2) { + dequant_k2_plane_superblock(kq, s * qsb, ks, s * ssb, w, s * 256l) + } elif (fmt == 24) { + dequant_iq2xs_plane_superblock(kq, s * qsb, ks, s * ssb, w, s * 256l) + } elif (fmt == 25) { + dequant_iq2xxs_plane_superblock(kq, s * qsb, ks, s * ssb, w, s * 256l) + } elif (fmt == 23) { + dequant_iq2s_plane_superblock(kq, s * qsb, ks, s * ssb, w, s * 256l) } else { dequant_k6_plane_superblock(kq, s * qsb, ks, s * ssb, w, s * 256l) } @@ -569,6 +1003,24 @@ def private kq_dot_gate(t : T?; fmt : int; n : int64 = 512l) { got = dot_k5q8(addr(kq[0]), addr(ks[0]), addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } elif (fmt == 40) { got = dot_q40q8(addr(kq[0]), addr(ks[0]), addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + } elif (fmt == 44) { + got = dot_iq4xs_q8(addr(kq[0]), addr(ks[0]), addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + } elif (fmt == 3) { + got = dot_k3q8(addr(kq[0]), addr(ks[0]), addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + } elif (fmt == 33) { + got = dot_iq3s_q8(addr(kq[0]), addr(ks[0]), addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + } elif (fmt == 34) { + got = dot_iq3xxs_q8(addr(kq[0]), addr(ks[0]), addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + } elif (fmt == 45) { + got = dot_iq4nl_q8(addr(kq[0]), addr(ks[0]), addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + } elif (fmt == 2) { + got = dot_k2q8(addr(kq[0]), addr(ks[0]), addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + } elif (fmt == 24) { + got = dot_iq2xs_q8(addr(kq[0]), addr(ks[0]), addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + } elif (fmt == 25) { + got = dot_iq2xxs_q8(addr(kq[0]), addr(ks[0]), addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + } elif (fmt == 23) { + got = dot_iq2s_q8(addr(kq[0]), addr(ks[0]), addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } else { got = dot_k6q8(addr(kq[0]), addr(ks[0]), addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } @@ -591,9 +1043,18 @@ def test_kq_dots(t : T?) { t |> run("dot_q40q8 matches the fp64 plane-dequant reference") @(t : T?) { kq_dot_gate(t, 40) } + t |> run("dot_iq4xs_q8 matches the fp64 plane-dequant reference") @(t : T?) { + kq_dot_gate(t, 44) + } + t |> run("dot_k3q8 matches the fp64 plane-dequant reference") @(t : T?) { + kq_dot_gate(t, 3) + } + t |> run("dot_iq3s_q8 matches the fp64 plane-dequant reference") @(t : T?) { + kq_dot_gate(t, 33) + } // 122B-class row widths: n=3072 (12 superblocks — qwen35moe-122B gate/up/cls) and n=1024 // (its k5 down_exps). Every earlier kq model kept n at 512..2560. - for (fmt in [4, 5, 6, 40]) { + for (fmt in [4, 5, 6, 40, 44, 3, 33, 34, 45, 2, 23, 24, 25]) { for (nn in [1024l, 3072l]) { t |> run("dot_{kq_tag(fmt)}q8 matches the fp64 reference at n={int(nn)}") @(t : T?) { kq_dot_gate(t, fmt, nn) @@ -612,7 +1073,7 @@ def private kq_gemv_rows_gate(t : T?; fmt : int; n : int64 = 512l) { let nsb = n / 256l let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) - let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : build_q6k_block())) + let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : (fmt == 45 ? build_iq4nl_block() : (fmt == 2 ? build_q2k_block() : (fmt == 23 ? build_iq2s_block() : (fmt == 24 ? build_iq2xs_block() : (fmt == 25 ? build_iq2xxs_block() : build_q6k_block()))))))))))) var kq : array var ks : array kq |> resize(d * nsb * qsb) @@ -625,6 +1086,24 @@ def private kq_gemv_rows_gate(t : T?; fmt : int; n : int64 = 512l) { transcode_q5k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } elif (fmt == 40) { transcode_q40_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 44) { + transcode_iq4xs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 3) { + transcode_q3k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 33) { + transcode_iq3s_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 34) { + transcode_iq3xxs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 45) { + transcode_iq4nl_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 2) { + transcode_q2k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 24) { + transcode_iq2xs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 25) { + transcode_iq2xxs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 23) { + transcode_iq2s_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } else { transcode_q6k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } @@ -663,6 +1142,24 @@ def private kq_gemv_rows_gate(t : T?; fmt : int; n : int64 = 512l) { want = dot_k5q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } elif (fmt == 40) { want = dot_q40q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + } elif (fmt == 44) { + want = dot_iq4xs_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + } elif (fmt == 3) { + want = dot_k3q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + } elif (fmt == 33) { + want = dot_iq3s_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + } elif (fmt == 34) { + want = dot_iq3xxs_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + } elif (fmt == 45) { + want = dot_iq4nl_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + } elif (fmt == 2) { + want = dot_k2q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + } elif (fmt == 24) { + want = dot_iq2xs_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + } elif (fmt == 25) { + want = dot_iq2xxs_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + } elif (fmt == 23) { + want = dot_iq2s_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } else { want = dot_k6q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } @@ -695,7 +1192,16 @@ def test_kq_gemv_rows(t : T?) { t |> run("portable q40 GEMV rows bit-match per-row disk dots") @(t : T?) { kq_gemv_rows_gate(t, 40) } - for (fmt in [4, 5, 6, 40]) { + t |> run("portable iq4xs GEMV rows bit-match per-row disk dots") @(t : T?) { + kq_gemv_rows_gate(t, 44) + } + t |> run("portable k3 GEMV rows bit-match per-row disk dots") @(t : T?) { + kq_gemv_rows_gate(t, 3) + } + t |> run("portable iq3s GEMV rows bit-match per-row disk dots") @(t : T?) { + kq_gemv_rows_gate(t, 33) + } + for (fmt in [4, 5, 6, 40, 44, 3, 33, 34, 45, 2, 23, 24, 25]) { for (nn in [1024l, 3072l]) { t |> run("portable {kq_tag(fmt)} GEMV rows bit-match per-row disk dots at n={int(nn)}") @(t : T?) { kq_gemv_rows_gate(t, fmt, nn) @@ -710,12 +1216,12 @@ def test_kq_gemv_rows(t : T?) { // grp scalar reference (kq_grp_row_dot — the stamped cores' decline body) and the grp row // dequant to reproduce them BIT-EXACTLY (same block order, exact integer sub-sums — only the // plane layout moved). -def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { +def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { // nolint:STYLE037 — the flat per-format ladder (one arm per kq format) is the honest shape let d = 32l let nsb = n / 256l let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) - let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : build_q6k_block())) + let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : (fmt == 45 ? build_iq4nl_block() : (fmt == 2 ? build_q2k_block() : (fmt == 23 ? build_iq2s_block() : (fmt == 24 ? build_iq2xs_block() : (fmt == 25 ? build_iq2xxs_block() : build_q6k_block()))))))))))) var kq : array var ks : array kq |> resize(d * nsb * qsb) @@ -728,6 +1234,24 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { transcode_q5k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } elif (fmt == 40) { transcode_q40_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 44) { + transcode_iq4xs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 3) { + transcode_q3k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 33) { + transcode_iq3s_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 34) { + transcode_iq3xxs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 45) { + transcode_iq4nl_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 2) { + transcode_q2k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 24) { + transcode_iq2xs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 25) { + transcode_iq2xxs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 23) { + transcode_iq2s_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } else { transcode_q6k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } @@ -765,6 +1289,24 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { want[r] = dot_k5q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } elif (fmt == 40) { want[r] = dot_q40q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + } elif (fmt == 44) { + want[r] = dot_iq4xs_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + } elif (fmt == 3) { + want[r] = dot_k3q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + } elif (fmt == 33) { + want[r] = dot_iq3s_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + } elif (fmt == 34) { + want[r] = dot_iq3xxs_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + } elif (fmt == 45) { + want[r] = dot_iq4nl_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + } elif (fmt == 2) { + want[r] = dot_k2q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + } elif (fmt == 24) { + want[r] = dot_iq2xs_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + } elif (fmt == 25) { + want[r] = dot_iq2xxs_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + } elif (fmt == 23) { + want[r] = dot_iq2s_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } else { want[r] = dot_k6q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) } @@ -775,6 +1317,24 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { dequant_k5_plane_superblock(kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb, wrow, r * n + s * 256l) } elif (fmt == 40) { dequant_q40_plane_superblock(kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb, wrow, r * n + s * 256l) + } elif (fmt == 44) { + dequant_iq4xs_plane_superblock(kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb, wrow, r * n + s * 256l) + } elif (fmt == 3) { + dequant_k3_plane_superblock(kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb, wrow, r * n + s * 256l) + } elif (fmt == 33) { + dequant_iq3s_plane_superblock(kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb, wrow, r * n + s * 256l) + } elif (fmt == 34) { + dequant_iq3xxs_plane_superblock(kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb, wrow, r * n + s * 256l) + } elif (fmt == 45) { + dequant_iq4nl_plane_superblock(kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb, wrow, r * n + s * 256l) + } elif (fmt == 2) { + dequant_k2_plane_superblock(kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb, wrow, r * n + s * 256l) + } elif (fmt == 24) { + dequant_iq2xs_plane_superblock(kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb, wrow, r * n + s * 256l) + } elif (fmt == 25) { + dequant_iq2xxs_plane_superblock(kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb, wrow, r * n + s * 256l) + } elif (fmt == 23) { + dequant_iq2s_plane_superblock(kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb, wrow, r * n + s * 256l) } else { dequant_k6_plane_superblock(kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb, wrow, r * n + s * 256l) } @@ -786,6 +1346,24 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { repack_k5_grp(addr(kq[0]), addr(ks[0]), n, d, mr) } elif (fmt == 40) { repack_q40_grp(addr(kq[0]), addr(ks[0]), n, d, mr) + } elif (fmt == 44) { + repack_iq4xs_grp(addr(kq[0]), addr(ks[0]), n, d, mr) + } elif (fmt == 3) { + repack_k3_grp(addr(kq[0]), addr(ks[0]), n, d, mr) + } elif (fmt == 33) { + repack_iq3s_grp(addr(kq[0]), addr(ks[0]), n, d, mr) + } elif (fmt == 34) { + repack_iq3xxs_grp(addr(kq[0]), addr(ks[0]), n, d, mr) + } elif (fmt == 45) { + repack_q40_grp(addr(kq[0]), addr(ks[0]), n, d, mr) + } elif (fmt == 2) { + repack_k2_grp(addr(kq[0]), addr(ks[0]), n, d, mr) + } elif (fmt == 24) { + repack_iq2xs_grp(addr(kq[0]), addr(ks[0]), n, d, mr) + } elif (fmt == 25) { + repack_iq2xxs_grp(addr(kq[0]), addr(ks[0]), n, d, mr) + } elif (fmt == 23) { + repack_iq2s_grp(addr(kq[0]), addr(ks[0]), n, d, mr) } else { repack_k6_grp(addr(kq[0]), addr(ks[0]), n, d, mr) } @@ -799,7 +1377,13 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { let ksg = addr(ks[g * mr * nsb * ssb]) let got = (fmt == 40 ? q40_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) - : kq_grp_row_dot(int64(fmt), kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n)) + : (fmt == 44 + ? iq4xs_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + : (fmt == 3 + ? k3_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + : (fmt == 33 || fmt == 34 || fmt == 45 || fmt == 2 || fmt == 23 || fmt == 24 || fmt == 25 + ? (fmt == 33 ? iq3s_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) : (fmt == 34 ? iq3xxs_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) : (fmt == 45 ? iq4nl_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) : (fmt == 2 ? k2_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) : (fmt == 23 ? iq2s_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) : (fmt == 24 ? iq2xs_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) : iq2xxs_grp_row_dot(kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n))))))) + : kq_grp_row_dot(int64(fmt), kqg, ksg, r % mr, mr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n))))) if (got != want[r]) { dotbad++ } @@ -826,7 +1410,7 @@ def private kq_repack_gate(t : T?; fmt : int; mr : int64; n : int64 = 512l) { [test] def test_kq_repack_grp(t : T?) { - for (fmt in [4, 5, 6, 40]) { + for (fmt in [4, 5, 6, 40, 44, 3, 33, 34, 45, 2, 23, 24, 25]) { for (mr in [4l, 8l, 16l]) { t |> run("repack_{kq_tag(fmt)}_grp mr={mr} preserves dots and row dequants") @(t : T?) { kq_repack_gate(t, fmt, mr) @@ -844,14 +1428,14 @@ def test_kq_repack_grp(t : T?) { // repacked planes (the tile's per-token fold order IS the GEMV's). Repack at the layout // companion's mr — the same value the stubs/kernels read, so this holds on the reference // bodies (interp) AND the stamped kernels (-jit) alike. -def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { +def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { // nolint:STYLE037 — the flat per-format ladder (one arm per kq format) is the honest shape let d = 32l let ntok = 7l // 4-token tile + a 3-token gemv tail let nsb = n / 256l let mr = kq_layout_of(fmt) // the format's OWN layout companion (per-format tune families) let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) - let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : build_q6k_block())) + let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : (fmt == 45 ? build_iq4nl_block() : (fmt == 2 ? build_q2k_block() : (fmt == 23 ? build_iq2s_block() : (fmt == 24 ? build_iq2xs_block() : (fmt == 25 ? build_iq2xxs_block() : build_q6k_block()))))))))))) var kq : array var ks : array kq |> resize(d * nsb * qsb) @@ -864,6 +1448,24 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { transcode_q5k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } elif (fmt == 40) { transcode_q40_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 44) { + transcode_iq4xs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 3) { + transcode_q3k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 33) { + transcode_iq3s_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 34) { + transcode_iq3xxs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 45) { + transcode_iq4nl_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 2) { + transcode_q2k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 24) { + transcode_iq2xs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 25) { + transcode_iq2xxs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 23) { + transcode_iq2s_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } else { transcode_q6k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } @@ -881,6 +1483,24 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { repack_k5_grp(addr(kq[0]), addr(ks[0]), n, d, mr) } elif (fmt == 40) { repack_q40_grp(addr(kq[0]), addr(ks[0]), n, d, mr) + } elif (fmt == 44) { + repack_iq4xs_grp(addr(kq[0]), addr(ks[0]), n, d, mr) + } elif (fmt == 3) { + repack_k3_grp(addr(kq[0]), addr(ks[0]), n, d, mr) + } elif (fmt == 33) { + repack_iq3s_grp(addr(kq[0]), addr(ks[0]), n, d, mr) + } elif (fmt == 34) { + repack_iq3xxs_grp(addr(kq[0]), addr(ks[0]), n, d, mr) + } elif (fmt == 45) { + repack_q40_grp(addr(kq[0]), addr(ks[0]), n, d, mr) + } elif (fmt == 2) { + repack_k2_grp(addr(kq[0]), addr(ks[0]), n, d, mr) + } elif (fmt == 24) { + repack_iq2xs_grp(addr(kq[0]), addr(ks[0]), n, d, mr) + } elif (fmt == 25) { + repack_iq2xxs_grp(addr(kq[0]), addr(ks[0]), n, d, mr) + } elif (fmt == 23) { + repack_iq2s_grp(addr(kq[0]), addr(ks[0]), n, d, mr) } else { repack_k6_grp(addr(kq[0]), addr(ks[0]), n, d, mr) } @@ -909,7 +1529,7 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { // k5/k6 tiles read the byte-expanded panel the batch cell unpacks per (group, token-block) // — the gate covers unpack_kq_panel_grp too (tile-over-panel vs gemv-over-packed planes); // k4/q40 tiles read the packed planes directly - let packed = fmt == 4 || fmt == 40 + let packed = kq_reads_packed_planes(int64(fmt)) var panel : array if (!packed) { panel |> resize(mr * nsb * 256l) @@ -930,6 +1550,24 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { k5q8_tile_gen(addr(ytile[0]), addr(panel[0]), ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) } elif (fmt == 40) { q40q8_tile_gen(addr(ytile[0]), kqg, ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) + } elif (fmt == 44) { + iq4xsq8_tile_gen(addr(ytile[0]), kqg, ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) + } elif (fmt == 3) { + k3q8_tile_gen(addr(ytile[0]), kqg, ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) + } elif (fmt == 33) { + iq3sq8_tile_gen(addr(ytile[0]), addr(panel[0]), ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) + } elif (fmt == 34) { + iq3xxsq8_tile_gen(addr(ytile[0]), addr(panel[0]), ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) + } elif (fmt == 45) { + iq4nlq8_tile_gen(addr(ytile[0]), kqg, ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) + } elif (fmt == 2) { + k2q8_tile_gen(addr(ytile[0]), kqg, ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) + } elif (fmt == 25) { + iq2xxsq8_tile_gen(addr(ytile[0]), addr(panel[0]), ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) + } elif (fmt == 24) { + iq2xsq8_tile_gen(addr(ytile[0]), addr(panel[0]), ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) + } elif (fmt == 23) { + iq2sq8_tile_gen(addr(ytile[0]), addr(panel[0]), ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) } else { k6q8_tile_gen(addr(ytile[0]), addr(panel[0]), ksg, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, d, g, tk) } @@ -942,6 +1580,24 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { k5q8_gemv_gen(addr(ytile[tk * d]), addr(kq[0]), addr(ks[0]), addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, g * mr, (g + 1l) * mr) } elif (fmt == 40) { q40q8_gemv_gen(addr(ytile[tk * d]), addr(kq[0]), addr(ks[0]), addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, g * mr, (g + 1l) * mr) + } elif (fmt == 44) { + iq4xsq8_gemv_gen(addr(ytile[tk * d]), addr(kq[0]), addr(ks[0]), addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, g * mr, (g + 1l) * mr) + } elif (fmt == 3) { + k3q8_gemv_gen(addr(ytile[tk * d]), addr(kq[0]), addr(ks[0]), addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, g * mr, (g + 1l) * mr) + } elif (fmt == 33) { + iq3sq8_gemv_gen(addr(ytile[tk * d]), addr(kq[0]), addr(ks[0]), addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, g * mr, (g + 1l) * mr) + } elif (fmt == 34) { + iq3xxsq8_gemv_gen(addr(ytile[tk * d]), addr(kq[0]), addr(ks[0]), addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, g * mr, (g + 1l) * mr) + } elif (fmt == 45) { + iq4nlq8_gemv_gen(addr(ytile[tk * d]), addr(kq[0]), addr(ks[0]), addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, g * mr, (g + 1l) * mr) + } elif (fmt == 2) { + k2q8_gemv_gen(addr(ytile[tk * d]), addr(kq[0]), addr(ks[0]), addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, g * mr, (g + 1l) * mr) + } elif (fmt == 24) { + iq2xsq8_gemv_gen(addr(ytile[tk * d]), addr(kq[0]), addr(ks[0]), addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, g * mr, (g + 1l) * mr) + } elif (fmt == 25) { + iq2xxsq8_gemv_gen(addr(ytile[tk * d]), addr(kq[0]), addr(ks[0]), addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, g * mr, (g + 1l) * mr) + } elif (fmt == 23) { + iq2sq8_gemv_gen(addr(ytile[tk * d]), addr(kq[0]), addr(ks[0]), addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, g * mr, (g + 1l) * mr) } else { k6q8_gemv_gen(addr(ytile[tk * d]), addr(kq[0]), addr(ks[0]), addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, g * mr, (g + 1l) * mr) } @@ -955,6 +1611,24 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { k5q8_gemv_gen(addr(ygemv[p * d]), addr(kq[0]), addr(ks[0]), addr(xq[p * n]), addr(xs[p * (n / 256l)]), addr(xbs[p * (n / 16l)]), n, 0l, d) } elif (fmt == 40) { q40q8_gemv_gen(addr(ygemv[p * d]), addr(kq[0]), addr(ks[0]), addr(xq[p * n]), addr(xs[p * (n / 256l)]), addr(xbs[p * (n / 16l)]), n, 0l, d) + } elif (fmt == 44) { + iq4xsq8_gemv_gen(addr(ygemv[p * d]), addr(kq[0]), addr(ks[0]), addr(xq[p * n]), addr(xs[p * (n / 256l)]), addr(xbs[p * (n / 16l)]), n, 0l, d) + } elif (fmt == 3) { + k3q8_gemv_gen(addr(ygemv[p * d]), addr(kq[0]), addr(ks[0]), addr(xq[p * n]), addr(xs[p * (n / 256l)]), addr(xbs[p * (n / 16l)]), n, 0l, d) + } elif (fmt == 33) { + iq3sq8_gemv_gen(addr(ygemv[p * d]), addr(kq[0]), addr(ks[0]), addr(xq[p * n]), addr(xs[p * (n / 256l)]), addr(xbs[p * (n / 16l)]), n, 0l, d) + } elif (fmt == 34) { + iq3xxsq8_gemv_gen(addr(ygemv[p * d]), addr(kq[0]), addr(ks[0]), addr(xq[p * n]), addr(xs[p * (n / 256l)]), addr(xbs[p * (n / 16l)]), n, 0l, d) + } elif (fmt == 45) { + iq4nlq8_gemv_gen(addr(ygemv[p * d]), addr(kq[0]), addr(ks[0]), addr(xq[p * n]), addr(xs[p * (n / 256l)]), addr(xbs[p * (n / 16l)]), n, 0l, d) + } elif (fmt == 2) { + k2q8_gemv_gen(addr(ygemv[p * d]), addr(kq[0]), addr(ks[0]), addr(xq[p * n]), addr(xs[p * (n / 256l)]), addr(xbs[p * (n / 16l)]), n, 0l, d) + } elif (fmt == 24) { + iq2xsq8_gemv_gen(addr(ygemv[p * d]), addr(kq[0]), addr(ks[0]), addr(xq[p * n]), addr(xs[p * (n / 256l)]), addr(xbs[p * (n / 16l)]), n, 0l, d) + } elif (fmt == 25) { + iq2xxsq8_gemv_gen(addr(ygemv[p * d]), addr(kq[0]), addr(ks[0]), addr(xq[p * n]), addr(xs[p * (n / 256l)]), addr(xbs[p * (n / 16l)]), n, 0l, d) + } elif (fmt == 23) { + iq2sq8_gemv_gen(addr(ygemv[p * d]), addr(kq[0]), addr(ks[0]), addr(xq[p * n]), addr(xs[p * (n / 256l)]), addr(xbs[p * (n / 16l)]), n, 0l, d) } else { k6q8_gemv_gen(addr(ygemv[p * d]), addr(kq[0]), addr(ks[0]), addr(xq[p * n]), addr(xs[p * (n / 256l)]), addr(xbs[p * (n / 16l)]), n, 0l, d) } @@ -980,7 +1654,7 @@ def private kq_tile_gate(t : T?; fmt : int; n : int64 = 512l) { [test] def test_kq_tile(t : T?) { - for (fmt in [4, 5, 6, 40]) { + for (fmt in [4, 5, 6, 40, 44, 3, 33, 34, 45, 2, 23, 24, 25]) { t |> run("{kq_tag(fmt)} 4-token tile bit-matches per-token GEMVs") @(t : T?) { kq_tile_gate(t, fmt) } @@ -997,11 +1671,11 @@ def test_kq_tile(t : T?) { // per-region grp slices, the per-expert-slice repack shape). Regions carry distinct LCG // payloads; both offs forms run — the shared activation image (gate/up, xoff 0) and // per-region activation rows (down, xoff = r*n). -def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; nreg : int64 = 3l) { +def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; nreg : int64 = 3l) { // nolint:STYLE037 — the flat per-format ladder (one arm per kq format) is the honest shape let nsb = n / 256l let qsb = kq_qsb(fmt) let ssb = kq_ssb(fmt) - let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : build_q6k_block())) + let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : (fmt == 45 ? build_iq4nl_block() : (fmt == 2 ? build_q2k_block() : (fmt == 23 ? build_iq2s_block() : (fmt == 24 ? build_iq2xs_block() : (fmt == 25 ? build_iq2xxs_block() : build_q6k_block()))))))))))) var kq : array var ks : array kq |> resize(nreg * d * nsb * qsb) @@ -1014,6 +1688,24 @@ def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; transcode_q5k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } elif (fmt == 40) { transcode_q40_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 44) { + transcode_iq4xs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 3) { + transcode_q3k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 33) { + transcode_iq3s_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 34) { + transcode_iq3xxs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 45) { + transcode_iq4nl_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 2) { + transcode_q2k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 24) { + transcode_iq2xs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 25) { + transcode_iq2xxs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 23) { + transcode_iq2s_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } else { transcode_q6k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } @@ -1071,6 +1763,33 @@ def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; } elif (fmt == 40) { want_sh[r * d + row] = dot_q40q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) want_pr[r * d + row] = dot_q40q8(kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n) + } elif (fmt == 44) { + want_sh[r * d + row] = dot_iq4xs_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + want_pr[r * d + row] = dot_iq4xs_q8(kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n) + } elif (fmt == 3) { + want_sh[r * d + row] = dot_k3q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + want_pr[r * d + row] = dot_k3q8(kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n) + } elif (fmt == 33) { + want_sh[r * d + row] = dot_iq3s_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + want_pr[r * d + row] = dot_iq3s_q8(kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n) + } elif (fmt == 34) { + want_sh[r * d + row] = dot_iq3xxs_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + want_pr[r * d + row] = dot_iq3xxs_q8(kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n) + } elif (fmt == 45) { + want_sh[r * d + row] = dot_iq4nl_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + want_pr[r * d + row] = dot_iq4nl_q8(kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n) + } elif (fmt == 2) { + want_sh[r * d + row] = dot_k2q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + want_pr[r * d + row] = dot_k2q8(kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n) + } elif (fmt == 24) { + want_sh[r * d + row] = dot_iq2xs_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + want_pr[r * d + row] = dot_iq2xs_q8(kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n) + } elif (fmt == 25) { + want_sh[r * d + row] = dot_iq2xxs_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + want_pr[r * d + row] = dot_iq2xxs_q8(kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n) + } elif (fmt == 23) { + want_sh[r * d + row] = dot_iq2s_q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + want_pr[r * d + row] = dot_iq2s_q8(kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n) } else { want_sh[r * d + row] = dot_k6q8(kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) want_pr[r * d + row] = dot_k6q8(kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n) @@ -1113,6 +1832,24 @@ def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; repack_k5_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) } elif (fmt == 40) { repack_q40_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) + } elif (fmt == 44) { + repack_iq4xs_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) + } elif (fmt == 3) { + repack_k3_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) + } elif (fmt == 33) { + repack_iq3s_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) + } elif (fmt == 34) { + repack_iq3xxs_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) + } elif (fmt == 45) { + repack_q40_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) + } elif (fmt == 2) { + repack_k2_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) + } elif (fmt == 24) { + repack_iq2xs_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) + } elif (fmt == 25) { + repack_iq2xxs_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) + } elif (fmt == 23) { + repack_iq2s_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) } else { repack_k6_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) } @@ -1129,6 +1866,33 @@ def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; } elif (fmt == 40) { q40q8_gemv_gen(addr(want_sh[r * d]), kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, 0l, d) q40q8_gemv_gen(addr(want_pr[r * d]), kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n, 0l, d) + } elif (fmt == 44) { + iq4xsq8_gemv_gen(addr(want_sh[r * d]), kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, 0l, d) + iq4xsq8_gemv_gen(addr(want_pr[r * d]), kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n, 0l, d) + } elif (fmt == 3) { + k3q8_gemv_gen(addr(want_sh[r * d]), kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, 0l, d) + k3q8_gemv_gen(addr(want_pr[r * d]), kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n, 0l, d) + } elif (fmt == 33) { + iq3sq8_gemv_gen(addr(want_sh[r * d]), kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, 0l, d) + iq3sq8_gemv_gen(addr(want_pr[r * d]), kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n, 0l, d) + } elif (fmt == 34) { + iq3xxsq8_gemv_gen(addr(want_sh[r * d]), kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, 0l, d) + iq3xxsq8_gemv_gen(addr(want_pr[r * d]), kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n, 0l, d) + } elif (fmt == 45) { + iq4nlq8_gemv_gen(addr(want_sh[r * d]), kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, 0l, d) + iq4nlq8_gemv_gen(addr(want_pr[r * d]), kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n, 0l, d) + } elif (fmt == 2) { + k2q8_gemv_gen(addr(want_sh[r * d]), kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, 0l, d) + k2q8_gemv_gen(addr(want_pr[r * d]), kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n, 0l, d) + } elif (fmt == 24) { + iq2xsq8_gemv_gen(addr(want_sh[r * d]), kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, 0l, d) + iq2xsq8_gemv_gen(addr(want_pr[r * d]), kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n, 0l, d) + } elif (fmt == 25) { + iq2xxsq8_gemv_gen(addr(want_sh[r * d]), kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, 0l, d) + iq2xxsq8_gemv_gen(addr(want_pr[r * d]), kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n, 0l, d) + } elif (fmt == 23) { + iq2sq8_gemv_gen(addr(want_sh[r * d]), kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, 0l, d) + iq2sq8_gemv_gen(addr(want_pr[r * d]), kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n, 0l, d) } else { k6q8_gemv_gen(addr(want_sh[r * d]), kqr, ksr, addr(xq[0]), addr(xs[0]), addr(xbs[0]), n, 0l, d) k6q8_gemv_gen(addr(want_pr[r * d]), kqr, ksr, addr(xq[r * n]), addr(xs[r * (n / 256l)]), addr(xbs[r * (n / 16l)]), n, 0l, d) @@ -1164,7 +1928,7 @@ def private kq_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 = 32l; [test] def test_kq_groupn(t : T?) { - for (fmt in [4, 5, 6, 40]) { + for (fmt in [4, 5, 6, 40, 44, 3, 33, 34, 45, 2, 23, 24, 25]) { t |> run("{kq_tag(fmt)} region-list GEMV bit-matches per-row dots (disk + grp slices)") @(t : T?) { kq_groupn_gate(t, fmt) } @@ -1208,7 +1972,7 @@ def private kq_batch_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 for (c in cnts) { nk += c } - let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : build_q6k_block())) + let blk1 <- fmt == 4 ? build_q4k_block() : (fmt == 5 ? build_q5k_block() : (fmt == 40 ? build_q40_block() : (fmt == 44 ? build_iq4xs_block() : (fmt == 3 ? build_q3k_block() : (fmt == 33 ? build_iq3s_block() : (fmt == 34 ? build_iq3xxs_block() : (fmt == 45 ? build_iq4nl_block() : (fmt == 2 ? build_q2k_block() : (fmt == 23 ? build_iq2s_block() : (fmt == 24 ? build_iq2xs_block() : (fmt == 25 ? build_iq2xxs_block() : build_q6k_block()))))))))))) var kq : array var ks : array kq |> resize(nreg * d * nsb * qsb) @@ -1221,6 +1985,24 @@ def private kq_batch_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 transcode_q5k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } elif (fmt == 40) { transcode_q40_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 44) { + transcode_iq4xs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 3) { + transcode_q3k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 33) { + transcode_iq3s_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 34) { + transcode_iq3xxs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 45) { + transcode_iq4nl_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 2) { + transcode_q2k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 24) { + transcode_iq2xs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 25) { + transcode_iq2xxs_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 23) { + transcode_iq2s_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } else { transcode_q6k_superblock(blk1, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) } @@ -1239,6 +2021,24 @@ def private kq_batch_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 repack_k5_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) } elif (fmt == 40) { repack_q40_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) + } elif (fmt == 44) { + repack_iq4xs_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) + } elif (fmt == 3) { + repack_k3_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) + } elif (fmt == 33) { + repack_iq3s_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) + } elif (fmt == 34) { + repack_iq3xxs_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) + } elif (fmt == 45) { + repack_q40_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) + } elif (fmt == 2) { + repack_k2_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) + } elif (fmt == 24) { + repack_iq2xs_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) + } elif (fmt == 25) { + repack_iq2xxs_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) + } elif (fmt == 23) { + repack_iq2s_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) } else { repack_k6_grp(addr(kq[r * d * nsb * qsb]), addr(ks[r * d * nsb * ssb]), n, d, mr) } @@ -1283,6 +2083,24 @@ def private kq_batch_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 k5q8_gemv_gen(addr(want[tk * d]), kqr, ksr, addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, 0l, d) } elif (fmt == 40) { q40q8_gemv_gen(addr(want[tk * d]), kqr, ksr, addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, 0l, d) + } elif (fmt == 44) { + iq4xsq8_gemv_gen(addr(want[tk * d]), kqr, ksr, addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, 0l, d) + } elif (fmt == 3) { + k3q8_gemv_gen(addr(want[tk * d]), kqr, ksr, addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, 0l, d) + } elif (fmt == 33) { + iq3sq8_gemv_gen(addr(want[tk * d]), kqr, ksr, addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, 0l, d) + } elif (fmt == 34) { + iq3xxsq8_gemv_gen(addr(want[tk * d]), kqr, ksr, addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, 0l, d) + } elif (fmt == 45) { + iq4nlq8_gemv_gen(addr(want[tk * d]), kqr, ksr, addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, 0l, d) + } elif (fmt == 2) { + k2q8_gemv_gen(addr(want[tk * d]), kqr, ksr, addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, 0l, d) + } elif (fmt == 24) { + iq2xsq8_gemv_gen(addr(want[tk * d]), kqr, ksr, addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, 0l, d) + } elif (fmt == 25) { + iq2xxsq8_gemv_gen(addr(want[tk * d]), kqr, ksr, addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, 0l, d) + } elif (fmt == 23) { + iq2sq8_gemv_gen(addr(want[tk * d]), kqr, ksr, addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, 0l, d) } else { k6q8_gemv_gen(addr(want[tk * d]), kqr, ksr, addr(xq[tk * n]), addr(xs[tk * (n / 256l)]), addr(xbs[tk * (n / 16l)]), n, 0l, d) } @@ -1314,7 +2132,7 @@ def private kq_batch_groupn_gate(t : T?; fmt : int; n : int64 = 512l; d : int64 [test] def test_kq_batch_groupn(t : T?) { - for (fmt in [4, 5, 6, 40]) { + for (fmt in [4, 5, 6, 40, 44, 3, 33, 34, 45, 2, 23, 24, 25]) { t |> run("{kq_tag(fmt)} batch groupn bit-matches per-token rows-core calls") @(t : T?) { kq_batch_groupn_gate(t, fmt) } @@ -1611,3 +2429,138 @@ def private rel_rms(a, b : array) : float { return float(sqrt(num / den)) } + +[test] +def test_iq3xxs_planes(t : T?) { + t |> run("iq3xxs planes fold the 0.25 into d and the grid, and decode exactly") @(t : T?) { + let blkb <- build_iq3xxs_block() + var kq : array + var ks : array + kq |> resize(96) + ks |> resize(20) + transcode_iq3xxs_superblock(blkb, 0l, kq, 0l, ks, 0l) + var dst : array + dst |> resize(256) + dequant_iq3xxs_plane_superblock(kq, 0l, ks, 0l, dst, 0l) + // independent recomputation off the DISK bytes through the identity tables + let dh = f16_to_f32(f16_half_bits(uint(blkb[0]) | (uint(blkb[1]) << 8u))) + let grid = iq3xxs_grid() + let signs = ksigns_iq2xs() + for (k in range(256)) { + let b = k / 32 + let l = (k % 32) / 8 + let hj = k % 8 + var aux = 0u + for (tt in range(4)) { + aux |= uint(iq3xxs_aux_pat(b * 4 + tt)) << uint(8 * tt) + } + let db = dh * float(1 + 2 * int(aux >> 28u)) + let sgnb = int(signs[int((aux >> uint(7 * l)) & 127u)]) + let g = grid[iq3xxs_qs_pat(b * 8 + 2 * l + hj / 4)] + let mag = float((g >> uint(8 * (hj % 4))) & 0xFFu) + let sgn = ((sgnb >> hj) & 1) != 0 ? -1.0 : 1.0 + let expected = db * mag * sgn + t |> success(dst[k] == expected, "iq3xxs plane element must match the table recomputation exactly") + } + delete kq + delete ks + delete dst + } +} + +[test] +def test_iq4nl_planes(t : T?) { + t |> run("iq4nl planes split q40's bytes and decode through the codebook exactly") @(t : T?) { + let blkb <- build_iq4nl_block() + var kq : array + var ks : array + kq |> resize(128) + ks |> resize(16) + transcode_iq4nl_superblock(blkb, 0l, kq, 0l, ks, 0l) + var dst : array + dst |> resize(256) + dequant_iq4nl_plane_superblock(kq, 0l, ks, 0l, dst, 0l) + for (k in range(256)) { + let b = k / 32 + let l = k % 32 + let d = f16_to_f32(uint(blkb[b * 18]) | (uint(blkb[b * 18 + 1]) << 8u)) + let q = iq4nl_qs_pat(b * 16 + (l % 16)) + let nib = l < 16 ? q & 15 : q >> 4 + let expected = d * float(int(IQ4NL_LUT[nib])) + t |> success(dst[k] == expected, "iq4nl plane element must match the codebook recomputation exactly") + } + delete kq + delete ks + delete dst + } +} + +// the Metal-blob carrier splits a format's scale plane into [16B strips of every superblock] +// [the d tails] (ARCHITECTURE_GPU.md sec.2.2y); this rebuilds that form from the CPU row form +// the way metal_blob_scale_plane does for the 20B rows: [f16 d][2 pad][16B strips] +def private split_scale_plane(ks : array; ssb : int64) : array { + let nsb = long_length(ks) / ssb + var out : array + out |> resize(nsb * 18l) + for (sb in range64(nsb)) { + for (i in range64(16l)) { + out[sb * 16l + i] = ks[sb * ssb + 4l + i] + } + out[nsb * 16l + sb * 2l] = ks[sb * ssb] + out[nsb * 16l + sb * 2l + 1l] = ks[sb * ssb + 1l] + } + return <- out +} + +[test] +def test_kq_plane_sb_metal_blob_form(t : T?) { + t |> run("dequant_kq_plane_sb reads the Metal-blob split scale plane to the same values as the row form") <| @(t : T?) { + // one superblock per format from random disk bytes: transcode, dequant the row form, then + // dequant the split form through the metal_blob arm - the values must agree exactly + let gtypes <- [GGML_TYPE_IQ2_XXS, GGML_TYPE_IQ2_XS, GGML_TYPE_IQ2_S, GGML_TYPE_IQ3_XXS, GGML_TYPE_IQ3_S, GGML_TYPE_IQ4_XS] + var seed = 11 + for (gtype in gtypes) { + let fmt = gguf_kq_fmt_of_type(gtype) + let sbb = gguf_kq_disk_sbb(gtype) + var bytes : array + bytes |> reserve(sbb * 3l) + var x = uint(seed) * 2654435761u + for (_i in range64(sbb * 3l)) { + x = x * 1664525u + 1013904223u + bytes |> push(uint8((x >> 24u) & 0xFFu)) + } + let qsb = kq_qsb(fmt) + let ssb = kq_ssb(fmt) + var kq : array + var ks : array + kq |> resize(int(qsb * 3l)) + ks |> resize(int(ssb * 3l)) + for (sb in range64(3l)) { + kq_transcode_superblock(fmt, bytes, sb * sbb, kq, sb * qsb, ks, sb * ssb) + } + var rowform : array + var blobform : array + rowform |> resize(768) + blobform |> resize(768) + let split <- split_scale_plane(ks, ssb) + for (sb in range64(3l)) { + dequant_kq_plane_sb(fmt, kq, ks, sb, false, rowform, sb * 256l) + dequant_kq_plane_sb(fmt, kq, split, sb, true, blobform, sb * 256l) + } + var bad = 0 + for (i in range(768)) { + // random f16 scale bits can be NaN on both sides - a NaN pair is agreement + if (rowform[i] != blobform[i] && !(rowform[i] != rowform[i] && blobform[i] != blobform[i])) { + bad++ + } + } + t |> equal(bad, 0, "{fmt}: split form == row form ({bad} of 768 differ)") + delete bytes + delete kq + delete ks + delete rowform + delete blobform + seed++ + } + } +} diff --git a/modules/dasLLAMA/tests/test_metal_gemm_kernels.das b/modules/dasLLAMA/tests/test_metal_gemm_kernels.das index 106ae00e62..a5d3d0d29c 100644 --- a/modules/dasLLAMA/tests/test_metal_gemm_kernels.das +++ b/modules/dasLLAMA/tests/test_metal_gemm_kernels.das @@ -36,28 +36,28 @@ def private kq_mulmm_gate(t : T?; dev, queue; fmt : int; tensor : bool; m, kdim, : halfx ? (fmt == 4 ? MetalKqMulMmK4TH_metal_kq_mulmm_t_msl : (fmt == 5 ? MetalKqMulMmK5TH_metal_kq_mulmm_t_msl : MetalKqMulMmK6TH_metal_kq_mulmm_k6_t_msl)) : (fmt == 4 ? MetalKqMulMmK4T_metal_kq_mulmm_t_msl : (fmt == 5 ? MetalKqMulMmK5T_metal_kq_mulmm_t_msl : MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl))) - : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl : MetalKqMulMmK6_metal_kq_mulmm_msl))) + : (fmt == 25 ? MetalKqMulMmIq2xxs_metal_kq_mulmm_msl : (fmt == 24 ? MetalKqMulMmIq2xs_metal_kq_mulmm_msl : (fmt == 23 ? MetalKqMulMmIq2s_metal_kq_mulmm_msl : (fmt == 45 ? MetalKqMulMmIq4nl_metal_kq_mulmm_msl : (fmt == 2 ? MetalKqMulMmK2_metal_kq_mulmm_msl : (fmt == 34 ? MetalKqMulMmIq3xxs_metal_kq_mulmm_msl : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl : MetalKqMulMmK6_metal_kq_mulmm_msl)))))))))))) let entry = (tensor ? (tall ? (fmt == 4 ? MetalKqMulMmK4TH128_metal_kq_mulmm_t_msl_entry : (fmt == 5 ? MetalKqMulMmK5TH128_metal_kq_mulmm_t_msl_entry : MetalKqMulMmK6TH128_metal_kq_mulmm_k6_t_msl_entry)) : halfx ? (fmt == 4 ? MetalKqMulMmK4TH_metal_kq_mulmm_t_msl_entry : (fmt == 5 ? MetalKqMulMmK5TH_metal_kq_mulmm_t_msl_entry : MetalKqMulMmK6TH_metal_kq_mulmm_k6_t_msl_entry)) : (fmt == 4 ? MetalKqMulMmK4T_metal_kq_mulmm_t_msl_entry : (fmt == 5 ? MetalKqMulMmK5T_metal_kq_mulmm_t_msl_entry : MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl_entry))) - : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_entry : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_entry : MetalKqMulMmK6_metal_kq_mulmm_msl_entry))) + : (fmt == 25 ? MetalKqMulMmIq2xxs_metal_kq_mulmm_msl_entry : (fmt == 24 ? MetalKqMulMmIq2xs_metal_kq_mulmm_msl_entry : (fmt == 23 ? MetalKqMulMmIq2s_metal_kq_mulmm_msl_entry : (fmt == 45 ? MetalKqMulMmIq4nl_metal_kq_mulmm_msl_entry : (fmt == 2 ? MetalKqMulMmK2_metal_kq_mulmm_msl_entry : (fmt == 34 ? MetalKqMulMmIq3xxs_metal_kq_mulmm_msl_entry : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl_entry : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl_entry : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl_entry : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_entry : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_entry : MetalKqMulMmK6_metal_kq_mulmm_msl_entry)))))))))))) let fm = (tensor ? (tall ? (fmt == 4 ? MetalKqMulMmK4TH128_metal_kq_mulmm_t_msl_fastmath : (fmt == 5 ? MetalKqMulMmK5TH128_metal_kq_mulmm_t_msl_fastmath : MetalKqMulMmK6TH128_metal_kq_mulmm_k6_t_msl_fastmath)) : halfx ? (fmt == 4 ? MetalKqMulMmK4TH_metal_kq_mulmm_t_msl_fastmath : (fmt == 5 ? MetalKqMulMmK5TH_metal_kq_mulmm_t_msl_fastmath : MetalKqMulMmK6TH_metal_kq_mulmm_k6_t_msl_fastmath)) : (fmt == 4 ? MetalKqMulMmK4T_metal_kq_mulmm_t_msl_fastmath : (fmt == 5 ? MetalKqMulMmK5T_metal_kq_mulmm_t_msl_fastmath : MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl_fastmath))) - : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_fastmath : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_fastmath : MetalKqMulMmK6_metal_kq_mulmm_msl_fastmath))) + : (fmt == 25 ? MetalKqMulMmIq2xxs_metal_kq_mulmm_msl_fastmath : (fmt == 24 ? MetalKqMulMmIq2xs_metal_kq_mulmm_msl_fastmath : (fmt == 23 ? MetalKqMulMmIq2s_metal_kq_mulmm_msl_fastmath : (fmt == 45 ? MetalKqMulMmIq4nl_metal_kq_mulmm_msl_fastmath : (fmt == 2 ? MetalKqMulMmK2_metal_kq_mulmm_msl_fastmath : (fmt == 34 ? MetalKqMulMmIq3xxs_metal_kq_mulmm_msl_fastmath : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl_fastmath : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl_fastmath : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl_fastmath : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_fastmath : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_fastmath : MetalKqMulMmK6_metal_kq_mulmm_msl_fastmath)))))))))))) let tgm = (tensor ? (tall ? (fmt == 4 ? MetalKqMulMmK4TH128_metal_kq_mulmm_t_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5TH128_metal_kq_mulmm_t_msl_tgmem : MetalKqMulMmK6TH128_metal_kq_mulmm_k6_t_msl_tgmem)) : halfx ? (fmt == 4 ? MetalKqMulMmK4TH_metal_kq_mulmm_t_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5TH_metal_kq_mulmm_t_msl_tgmem : MetalKqMulMmK6TH_metal_kq_mulmm_k6_t_msl_tgmem)) : (fmt == 4 ? MetalKqMulMmK4T_metal_kq_mulmm_t_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5T_metal_kq_mulmm_t_msl_tgmem : MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl_tgmem))) - : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_tgmem : MetalKqMulMmK6_metal_kq_mulmm_msl_tgmem))) + : (fmt == 25 ? MetalKqMulMmIq2xxs_metal_kq_mulmm_msl_tgmem : (fmt == 24 ? MetalKqMulMmIq2xs_metal_kq_mulmm_msl_tgmem : (fmt == 23 ? MetalKqMulMmIq2s_metal_kq_mulmm_msl_tgmem : (fmt == 45 ? MetalKqMulMmIq4nl_metal_kq_mulmm_msl_tgmem : (fmt == 2 ? MetalKqMulMmK2_metal_kq_mulmm_msl_tgmem : (fmt == 34 ? MetalKqMulMmIq3xxs_metal_kq_mulmm_msl_tgmem : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl_tgmem : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl_tgmem : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl_tgmem : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_tgmem : MetalKqMulMmK6_metal_kq_mulmm_msl_tgmem)))))))))))) var pso = pipeline_from_source(dev, src, entry, fm, err) if (tensor && pso == null && !metal4_tensor_available(dev)) { to_log(LOG_INFO, "{tag}: no Metal-4 tensor toolchain on this box - arm skipped\n") @@ -123,7 +123,7 @@ def private kq_mulmm_gate(t : T?; dev, queue; fmt : int; tensor : bool; m, kdim, let ran = with_compute_encoder(queue, err) $(enc : MetalComputeEncoder?) { metal_set_pipeline(enc, pso) metal_set_threadgroup_memory_length(enc, tgm, 0) - metal_set_buffer(enc, bks, fmt == 6 ? uint64(nsb * 16) : 0ul, 0) // k6: the f16 d tail + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 || fmt == 24 || fmt == 25 ? uint64(nsb * 16) : 0ul, 0) // the split-scale formats: the f16 d tail at nsb*16 metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bx, 0ul, 3) @@ -134,7 +134,7 @@ def private kq_mulmm_gate(t : T?; dev, queue; fmt : int; tensor : bool; m, kdim, if (rem_pso != null) { metal_set_pipeline(enc, rem_pso) metal_set_threadgroup_memory_length(enc, fmt == 4 ? MetalKqMulMmK4TH_metal_kq_mulmm_t_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5TH_metal_kq_mulmm_t_msl_tgmem : MetalKqMulMmK6TH_metal_kq_mulmm_k6_t_msl_tgmem), 0) - metal_set_buffer(enc, bks, fmt == 6 ? uint64(nsb * 16) : 0ul, 0) + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 || fmt == 24 || fmt == 25 ? uint64(nsb * 16) : 0ul, 0) metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bx, uint64(r0 * kdim * 2), 3) @@ -484,7 +484,7 @@ def private kq_devw_gate(t : T?; dev, queue; fmt : int; m, kdim, ndim : int) { var dka = CvtArgs(total = uint(nblk)) let ran = with_compute_encoder(queue, err) $(enc : MetalComputeEncoder?) { metal_set_pipeline(enc, dq_pso) - metal_set_buffer(enc, bks, fmt == 6 ? uint64(nsb * 16) : 0ul, 0) // k6: the f16 d tail + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 || fmt == 24 || fmt == 25 ? uint64(nsb * 16) : 0ul, 0) // the split-scale formats: the f16 d tail at nsb*16 metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bwh, 0ul, 3) @@ -1461,7 +1461,7 @@ def private moe_mulmm_kq_gate(t : T?; dev, queue; fmt : int; kdim, ndim : int; c let ran = with_compute_encoder(queue, err) $(enc : MetalComputeEncoder?) { metal_set_pipeline(enc, pso) metal_set_threadgroup_memory_length(enc, tgm, 0) - metal_set_buffer(enc, bks, fmt == 6 ? uint64(nsb * 16) : 0ul, 0) // k6: the f16 d tail + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 || fmt == 24 || fmt == 25 ? uint64(nsb * 16) : 0ul, 0) // the split-scale formats: the f16 d tail at nsb*16 metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bx, 0ul, 3) @@ -1498,7 +1498,7 @@ def private moe_mulmm_kq_gate(t : T?; dev, queue; fmt : int; kdim, ndim : int; c metal_set_threadgroup_memory_length(enc, fmt == 4 ? MetalMoeMulMmK4TH_metal_moe_mulmm_k45_t_msl_tgmem : (fmt == 5 ? MetalMoeMulMmK5TH_metal_moe_mulmm_k45_t_msl_tgmem : MetalMoeMulMmK6TH_metal_moe_mulmm_k6_t_msl_tgmem), 0) - metal_set_buffer(enc, bks, fmt == 6 ? uint64(nsb * 16) : 0ul, 0) + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 || fmt == 24 || fmt == 25 ? uint64(nsb * 16) : 0ul, 0) metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bxh, 0ul, 3) @@ -1538,7 +1538,7 @@ def private moe_mulmm_kq_gate(t : T?; dev, queue; fmt : int; kdim, ndim : int; c metal_set_threadgroup_memory_length(enc, fmt == 4 ? MetalMoeMulMmK4TH128_metal_moe_mulmm_k45_t_msl_tgmem : (fmt == 5 ? MetalMoeMulMmK5TH128_metal_moe_mulmm_k45_t_msl_tgmem : MetalMoeMulMmK6TH128_metal_moe_mulmm_k6_t_msl_tgmem), 0) - metal_set_buffer(enc, bks, fmt == 6 ? uint64(nsb * 16) : 0ul, 0) + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 || fmt == 24 || fmt == 25 ? uint64(nsb * 16) : 0ul, 0) metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bxh, 0ul, 3) @@ -1551,7 +1551,7 @@ def private moe_mulmm_kq_gate(t : T?; dev, queue; fmt : int; kdim, ndim : int; c metal_set_threadgroup_memory_length(enc, fmt == 4 ? MetalMoeMulMmK4THR_metal_moe_mulmm_k45_t_msl_tgmem : (fmt == 5 ? MetalMoeMulMmK5THR_metal_moe_mulmm_k45_t_msl_tgmem : MetalMoeMulMmK6THR_metal_moe_mulmm_k6_t_msl_tgmem), 0) - metal_set_buffer(enc, bks, fmt == 6 ? uint64(nsb * 16) : 0ul, 0) + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 || fmt == 24 || fmt == 25 ? uint64(nsb * 16) : 0ul, 0) metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bxh, 0ul, 3) @@ -2047,6 +2047,24 @@ def test_metal_gemm_kernels(t : T?) { kq_mulmm_gate(t, dev, queue, fmt, true, 320, 512, 64, halfx = true, tall = true) // off the 128 floor: the remainder pair at nonzero x/y offsets } kq_mulmm_gate(t, dev, queue, 4, false, 32, 256, 128) // second shape: 2 col tiles + kq_mulmm_gate(t, dev, queue, 44, false, 64, 512, 64) + kq_mulmm_gate(t, dev, queue, 44, false, 32, 256, 128) + kq_mulmm_gate(t, dev, queue, 3, false, 64, 512, 64) + kq_mulmm_gate(t, dev, queue, 3, false, 32, 256, 128) + kq_mulmm_gate(t, dev, queue, 33, false, 64, 512, 64) + kq_mulmm_gate(t, dev, queue, 33, false, 32, 256, 128) + kq_mulmm_gate(t, dev, queue, 34, false, 64, 512, 64) + kq_mulmm_gate(t, dev, queue, 34, false, 32, 256, 128) + kq_mulmm_gate(t, dev, queue, 45, false, 64, 512, 64) + kq_mulmm_gate(t, dev, queue, 45, false, 32, 256, 128) + kq_mulmm_gate(t, dev, queue, 2, false, 64, 512, 64) + kq_mulmm_gate(t, dev, queue, 2, false, 32, 256, 128) + kq_mulmm_gate(t, dev, queue, 23, false, 64, 512, 64) + kq_mulmm_gate(t, dev, queue, 23, false, 32, 256, 128) + kq_mulmm_gate(t, dev, queue, 24, false, 64, 512, 64) + kq_mulmm_gate(t, dev, queue, 24, false, 32, 256, 128) + kq_mulmm_gate(t, dev, queue, 25, false, 64, 512, 64) + kq_mulmm_gate(t, dev, queue, 25, false, 32, 256, 128) q8_mulmm_t_gate(t, dev, queue, 64, 256, 64) q8_mulmm_t_gate(t, dev, queue, 64, 256, 64, halfx = true) q8_mulmm_t_gate(t, dev, queue, 64, 160, 64) // kdim % 64 == 32: the bk=64 helper's 32-tail chunk diff --git a/modules/dasLLAMA/tests/test_metal_gemv_kernels.das b/modules/dasLLAMA/tests/test_metal_gemv_kernels.das index 6a9958a8f2..c97620e11c 100644 --- a/modules/dasLLAMA/tests/test_metal_gemv_kernels.das +++ b/modules/dasLLAMA/tests/test_metal_gemv_kernels.das @@ -23,12 +23,12 @@ require math // ===== single-stream kq GEMV (MetalKqGemvK4 / K5 / K5C / K6) ===== def private kq_gemv_gate(t : T?; dev, queue; vform : string; n, d : int) { - let fmt = vform == "k4" ? 4 : (vform == "k6" ? 6 : 5) + let fmt = vform == "k4" ? 4 : (vform == "k6" ? 6 : (vform == "iq4xs" ? 44 : (vform == "k3" ? 3 : (vform == "iq3s" ? 33 : (vform == "iq3xxs" ? 34 : (vform == "iq4nl" ? 45 : (vform == "iq2s" ? 23 : (vform == "iq2xs" ? 24 : (vform == "iq2xxs" ? 25 : (vform == "k2" ? 2 : 5)))))))))) let tag = "kq_gemv_{vform} n={n} d={d}" var err : string - let src = vform == "k4" ? metal_kq_gemv_k4_msl : (vform == "k5" ? metal_kq_gemv_k5_msl : (vform == "k5c" ? metal_kq_gemv_k5c_msl : metal_kq_gemv_k6_msl)) - let entry = vform == "k4" ? metal_kq_gemv_k4_msl_entry : (vform == "k5" ? metal_kq_gemv_k5_msl_entry : (vform == "k5c" ? metal_kq_gemv_k5c_msl_entry : metal_kq_gemv_k6_msl_entry)) - let fm = vform == "k4" ? metal_kq_gemv_k4_msl_fastmath : (vform == "k5" ? metal_kq_gemv_k5_msl_fastmath : (vform == "k5c" ? metal_kq_gemv_k5c_msl_fastmath : metal_kq_gemv_k6_msl_fastmath)) + let src = vform == "iq2xxs" ? metal_kq_gemv_iq2xxs_msl : (vform == "iq2xs" ? metal_kq_gemv_iq2xs_msl : (vform == "iq2s" ? metal_kq_gemv_iq2s_msl : (vform == "k2" ? metal_kq_gemv_k2_msl : (vform == "iq4nl" ? metal_kq_gemv_iq4nl_msl : (vform == "iq3xxs" ? metal_kq_gemv_iq3xxs_msl : (vform == "iq3s" ? metal_kq_gemv_iq3s_msl : (vform == "k3" ? metal_kq_gemv_k3_msl : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl : (vform == "k4" ? metal_kq_gemv_k4_msl : (vform == "k5" ? metal_kq_gemv_k5_msl : (vform == "k5c" ? metal_kq_gemv_k5c_msl : metal_kq_gemv_k6_msl))))))))))) + let entry = vform == "iq2xxs" ? metal_kq_gemv_iq2xxs_msl_entry : (vform == "iq2xs" ? metal_kq_gemv_iq2xs_msl_entry : (vform == "iq2s" ? metal_kq_gemv_iq2s_msl_entry : (vform == "k2" ? metal_kq_gemv_k2_msl_entry : (vform == "iq4nl" ? metal_kq_gemv_iq4nl_msl_entry : (vform == "iq3xxs" ? metal_kq_gemv_iq3xxs_msl_entry : (vform == "iq3s" ? metal_kq_gemv_iq3s_msl_entry : (vform == "k3" ? metal_kq_gemv_k3_msl_entry : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl_entry : (vform == "k4" ? metal_kq_gemv_k4_msl_entry : (vform == "k5" ? metal_kq_gemv_k5_msl_entry : (vform == "k5c" ? metal_kq_gemv_k5c_msl_entry : metal_kq_gemv_k6_msl_entry))))))))))) + let fm = vform == "iq2xxs" ? metal_kq_gemv_iq2xxs_msl_fastmath : (vform == "iq2xs" ? metal_kq_gemv_iq2xs_msl_fastmath : (vform == "iq2s" ? metal_kq_gemv_iq2s_msl_fastmath : (vform == "k2" ? metal_kq_gemv_k2_msl_fastmath : (vform == "iq4nl" ? metal_kq_gemv_iq4nl_msl_fastmath : (vform == "iq3xxs" ? metal_kq_gemv_iq3xxs_msl_fastmath : (vform == "iq3s" ? metal_kq_gemv_iq3s_msl_fastmath : (vform == "k3" ? metal_kq_gemv_k3_msl_fastmath : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl_fastmath : (vform == "k4" ? metal_kq_gemv_k4_msl_fastmath : (vform == "k5" ? metal_kq_gemv_k5_msl_fastmath : (vform == "k5c" ? metal_kq_gemv_k5c_msl_fastmath : metal_kq_gemv_k6_msl_fastmath))))))))))) var pso = pipeline_from_source(dev, src, entry, fm, err) t |> success(pso != null, "{tag}: pipeline: {err}") return if (pso == null) @@ -66,10 +66,22 @@ def private kq_gemv_gate(t : T?; dev, queue; vform : string; n, d : int) { var by = buf_fill(dev, d, -1000.0) var bn = buf_u32(dev, uint(n)) var bd = buf_u32(dev, uint(d)) - let groups = vform == "k5c" ? (d + 1) / 2 : (d + 3) / 4 + let groups = vform == "k5c" ? (d + 1) / 2 : (vform == "iq3s" || vform == "iq3xxs" || vform == "iq2s" || vform == "iq2xs" || vform == "iq2xxs" ? (d + 7) / 8 : (d + 3) / 4) let ran = with_compute_encoder(queue, err) $(enc : MetalComputeEncoder?) { metal_set_pipeline(enc, pso) - metal_set_buffer(enc, bks, fmt == 6 ? uint64(nsb * 16) : 0ul, 0) // k6: the f16 d tail + if (vform == "iq4xs") { + metal_set_threadgroup_memory_length(enc, metal_kq_gemv_iq4xs_msl_tgmem, 0) // the codebook slab + } + if (vform == "iq3s") { + metal_set_threadgroup_memory_length(enc, metal_kq_gemv_iq3s_msl_tgmem, 0) // the grid slab + } + if (vform == "iq3xxs") { + metal_set_threadgroup_memory_length(enc, metal_kq_gemv_iq3xxs_msl_tgmem, 0) // the halved-grid slab + } + if (vform == "iq4nl") { + metal_set_threadgroup_memory_length(enc, metal_kq_gemv_iq4nl_msl_tgmem, 0) // the codebook slab + } + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 || fmt == 24 || fmt == 25 ? uint64(nsb * 16) : 0ul, 0) // the split-scale formats: the f16 d tail at nsb*16 metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bx, 0ul, 3) @@ -102,15 +114,15 @@ def private kq_gemv_gate(t : T?; dev, queue; vform : string; n, d : int) { def private kq_mvb_gate(t : T?; dev, queue; fmt, bwidth, n, d, nr, ys : int) { let tag = "kq_mvb{bwidth}_k{fmt} n={n} d={d} nr={nr} ys={ys}" var err : string - let src = (bwidth == 2 ? (fmt == 4 ? MetalKqMvB2K4_metal_kq_mv_k4_msl : (fmt == 5 ? MetalKqMvB2K5_metal_kq_mv_k5_msl : MetalKqMvB2K6_metal_kq_mv_k6_msl)) - : (bwidth == 4 ? (fmt == 4 ? MetalKqMvB4K4_metal_kq_mv_k4_msl : (fmt == 5 ? MetalKqMvB4K5_metal_kq_mv_k5_msl : MetalKqMvB4K6_metal_kq_mv_k6_msl)) - : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl : metal_kq_mvb8_k6_msl)))) - let entry = (bwidth == 2 ? (fmt == 4 ? MetalKqMvB2K4_metal_kq_mv_k4_msl_entry : (fmt == 5 ? MetalKqMvB2K5_metal_kq_mv_k5_msl_entry : MetalKqMvB2K6_metal_kq_mv_k6_msl_entry)) - : (bwidth == 4 ? (fmt == 4 ? MetalKqMvB4K4_metal_kq_mv_k4_msl_entry : (fmt == 5 ? MetalKqMvB4K5_metal_kq_mv_k5_msl_entry : MetalKqMvB4K6_metal_kq_mv_k6_msl_entry)) - : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_entry : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_entry : metal_kq_mvb8_k6_msl_entry)))) - let fm = (bwidth == 2 ? (fmt == 4 ? MetalKqMvB2K4_metal_kq_mv_k4_msl_fastmath : (fmt == 5 ? MetalKqMvB2K5_metal_kq_mv_k5_msl_fastmath : MetalKqMvB2K6_metal_kq_mv_k6_msl_fastmath)) - : (bwidth == 4 ? (fmt == 4 ? MetalKqMvB4K4_metal_kq_mv_k4_msl_fastmath : (fmt == 5 ? MetalKqMvB4K5_metal_kq_mv_k5_msl_fastmath : MetalKqMvB4K6_metal_kq_mv_k6_msl_fastmath)) - : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_fastmath : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_fastmath : metal_kq_mvb8_k6_msl_fastmath)))) + let src = (bwidth == 2 ? (fmt == 25 ? MetalKqMvB2Iq2xxs_metal_kq_mv_iq2xxs_msl : (fmt == 24 ? MetalKqMvB2Iq2xs_metal_kq_mv_iq2xs_msl : (fmt == 23 ? MetalKqMvB2Iq2s_metal_kq_mv_iq2s_msl : (fmt == 45 ? MetalKqMvB2Iq4nl_metal_kq_mv_iq4nl_msl : (fmt == 2 ? MetalKqMvB2K2_metal_kq_mv_k2_msl : (fmt == 34 ? MetalKqMvB2Iq3xxs_metal_kq_mv_iq3xxs_msl : (fmt == 33 ? MetalKqMvB2Iq3s_metal_kq_mv_iq3s_msl : (fmt == 3 ? MetalKqMvB2K3_metal_kq_mv_k3_msl : (fmt == 44 ? MetalKqMvB2Iq4xs_metal_kq_mv_iq4xs_msl : (fmt == 4 ? MetalKqMvB2K4_metal_kq_mv_k4_msl : (fmt == 5 ? MetalKqMvB2K5_metal_kq_mv_k5_msl : MetalKqMvB2K6_metal_kq_mv_k6_msl))))))))))) + : (bwidth == 4 ? (fmt == 25 ? MetalKqMvB4Iq2xxs_metal_kq_mv_iq2xxs_msl : (fmt == 24 ? MetalKqMvB4Iq2xs_metal_kq_mv_iq2xs_msl : (fmt == 23 ? MetalKqMvB4Iq2s_metal_kq_mv_iq2s_msl : (fmt == 45 ? MetalKqMvB4Iq4nl_metal_kq_mv_iq4nl_msl : (fmt == 2 ? MetalKqMvB4K2_metal_kq_mv_k2_msl : (fmt == 34 ? MetalKqMvB4Iq3xxs_metal_kq_mv_iq3xxs_msl : (fmt == 33 ? MetalKqMvB4Iq3s_metal_kq_mv_iq3s_msl : (fmt == 3 ? MetalKqMvB4K3_metal_kq_mv_k3_msl : (fmt == 44 ? MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl : (fmt == 4 ? MetalKqMvB4K4_metal_kq_mv_k4_msl : (fmt == 5 ? MetalKqMvB4K5_metal_kq_mv_k5_msl : MetalKqMvB4K6_metal_kq_mv_k6_msl))))))))))) + : (fmt == 25 ? metal_kq_mvb8_iq2xxs_msl : (fmt == 24 ? metal_kq_mvb8_iq2xs_msl : (fmt == 23 ? metal_kq_mvb8_iq2s_msl : (fmt == 45 ? metal_kq_mvb8_iq4nl_msl : (fmt == 2 ? metal_kq_mvb8_k2_msl : (fmt == 34 ? metal_kq_mvb8_iq3xxs_msl : (fmt == 33 ? metal_kq_mvb8_iq3s_msl : (fmt == 3 ? metal_kq_mvb8_k3_msl : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl : metal_kq_mvb8_k6_msl))))))))))))) + let entry = (bwidth == 2 ? (fmt == 25 ? MetalKqMvB2Iq2xxs_metal_kq_mv_iq2xxs_msl_entry : (fmt == 24 ? MetalKqMvB2Iq2xs_metal_kq_mv_iq2xs_msl_entry : (fmt == 23 ? MetalKqMvB2Iq2s_metal_kq_mv_iq2s_msl_entry : (fmt == 45 ? MetalKqMvB2Iq4nl_metal_kq_mv_iq4nl_msl_entry : (fmt == 2 ? MetalKqMvB2K2_metal_kq_mv_k2_msl_entry : (fmt == 34 ? MetalKqMvB2Iq3xxs_metal_kq_mv_iq3xxs_msl_entry : (fmt == 33 ? MetalKqMvB2Iq3s_metal_kq_mv_iq3s_msl_entry : (fmt == 3 ? MetalKqMvB2K3_metal_kq_mv_k3_msl_entry : (fmt == 44 ? MetalKqMvB2Iq4xs_metal_kq_mv_iq4xs_msl_entry : (fmt == 4 ? MetalKqMvB2K4_metal_kq_mv_k4_msl_entry : (fmt == 5 ? MetalKqMvB2K5_metal_kq_mv_k5_msl_entry : MetalKqMvB2K6_metal_kq_mv_k6_msl_entry))))))))))) + : (bwidth == 4 ? (fmt == 25 ? MetalKqMvB4Iq2xxs_metal_kq_mv_iq2xxs_msl_entry : (fmt == 24 ? MetalKqMvB4Iq2xs_metal_kq_mv_iq2xs_msl_entry : (fmt == 23 ? MetalKqMvB4Iq2s_metal_kq_mv_iq2s_msl_entry : (fmt == 45 ? MetalKqMvB4Iq4nl_metal_kq_mv_iq4nl_msl_entry : (fmt == 2 ? MetalKqMvB4K2_metal_kq_mv_k2_msl_entry : (fmt == 34 ? MetalKqMvB4Iq3xxs_metal_kq_mv_iq3xxs_msl_entry : (fmt == 33 ? MetalKqMvB4Iq3s_metal_kq_mv_iq3s_msl_entry : (fmt == 3 ? MetalKqMvB4K3_metal_kq_mv_k3_msl_entry : (fmt == 44 ? MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl_entry : (fmt == 4 ? MetalKqMvB4K4_metal_kq_mv_k4_msl_entry : (fmt == 5 ? MetalKqMvB4K5_metal_kq_mv_k5_msl_entry : MetalKqMvB4K6_metal_kq_mv_k6_msl_entry))))))))))) + : (fmt == 25 ? metal_kq_mvb8_iq2xxs_msl_entry : (fmt == 24 ? metal_kq_mvb8_iq2xs_msl_entry : (fmt == 23 ? metal_kq_mvb8_iq2s_msl_entry : (fmt == 45 ? metal_kq_mvb8_iq4nl_msl_entry : (fmt == 2 ? metal_kq_mvb8_k2_msl_entry : (fmt == 34 ? metal_kq_mvb8_iq3xxs_msl_entry : (fmt == 33 ? metal_kq_mvb8_iq3s_msl_entry : (fmt == 3 ? metal_kq_mvb8_k3_msl_entry : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl_entry : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_entry : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_entry : metal_kq_mvb8_k6_msl_entry))))))))))))) + let fm = (bwidth == 2 ? (fmt == 25 ? MetalKqMvB2Iq2xxs_metal_kq_mv_iq2xxs_msl_fastmath : (fmt == 24 ? MetalKqMvB2Iq2xs_metal_kq_mv_iq2xs_msl_fastmath : (fmt == 23 ? MetalKqMvB2Iq2s_metal_kq_mv_iq2s_msl_fastmath : (fmt == 45 ? MetalKqMvB2Iq4nl_metal_kq_mv_iq4nl_msl_fastmath : (fmt == 2 ? MetalKqMvB2K2_metal_kq_mv_k2_msl_fastmath : (fmt == 34 ? MetalKqMvB2Iq3xxs_metal_kq_mv_iq3xxs_msl_fastmath : (fmt == 33 ? MetalKqMvB2Iq3s_metal_kq_mv_iq3s_msl_fastmath : (fmt == 3 ? MetalKqMvB2K3_metal_kq_mv_k3_msl_fastmath : (fmt == 44 ? MetalKqMvB2Iq4xs_metal_kq_mv_iq4xs_msl_fastmath : (fmt == 4 ? MetalKqMvB2K4_metal_kq_mv_k4_msl_fastmath : (fmt == 5 ? MetalKqMvB2K5_metal_kq_mv_k5_msl_fastmath : MetalKqMvB2K6_metal_kq_mv_k6_msl_fastmath))))))))))) + : (bwidth == 4 ? (fmt == 25 ? MetalKqMvB4Iq2xxs_metal_kq_mv_iq2xxs_msl_fastmath : (fmt == 24 ? MetalKqMvB4Iq2xs_metal_kq_mv_iq2xs_msl_fastmath : (fmt == 23 ? MetalKqMvB4Iq2s_metal_kq_mv_iq2s_msl_fastmath : (fmt == 45 ? MetalKqMvB4Iq4nl_metal_kq_mv_iq4nl_msl_fastmath : (fmt == 2 ? MetalKqMvB4K2_metal_kq_mv_k2_msl_fastmath : (fmt == 34 ? MetalKqMvB4Iq3xxs_metal_kq_mv_iq3xxs_msl_fastmath : (fmt == 33 ? MetalKqMvB4Iq3s_metal_kq_mv_iq3s_msl_fastmath : (fmt == 3 ? MetalKqMvB4K3_metal_kq_mv_k3_msl_fastmath : (fmt == 44 ? MetalKqMvB4Iq4xs_metal_kq_mv_iq4xs_msl_fastmath : (fmt == 4 ? MetalKqMvB4K4_metal_kq_mv_k4_msl_fastmath : (fmt == 5 ? MetalKqMvB4K5_metal_kq_mv_k5_msl_fastmath : MetalKqMvB4K6_metal_kq_mv_k6_msl_fastmath))))))))))) + : (fmt == 25 ? metal_kq_mvb8_iq2xxs_msl_fastmath : (fmt == 24 ? metal_kq_mvb8_iq2xs_msl_fastmath : (fmt == 23 ? metal_kq_mvb8_iq2s_msl_fastmath : (fmt == 45 ? metal_kq_mvb8_iq4nl_msl_fastmath : (fmt == 2 ? metal_kq_mvb8_k2_msl_fastmath : (fmt == 34 ? metal_kq_mvb8_iq3xxs_msl_fastmath : (fmt == 33 ? metal_kq_mvb8_iq3s_msl_fastmath : (fmt == 3 ? metal_kq_mvb8_k3_msl_fastmath : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl_fastmath : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_fastmath : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_fastmath : metal_kq_mvb8_k6_msl_fastmath))))))))))))) var pso = pipeline_from_source(dev, src, entry, fm, err) t |> success(pso != null, "{tag}: pipeline: {err}") return if (pso == null) @@ -160,9 +172,9 @@ def private kq_mvb_gate(t : T?; dev, queue; fmt, bwidth, n, d, nr, ys : int) { metal_set_pipeline(enc, pso) if (bwidth == 8) { metal_set_threadgroup_memory_length(enc, - fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_tgmem : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_tgmem : metal_kq_mvb8_k6_msl_tgmem), 0) + fmt == 25 ? metal_kq_mvb8_iq2xxs_msl_tgmem : (fmt == 24 ? metal_kq_mvb8_iq2xs_msl_tgmem : (fmt == 23 ? metal_kq_mvb8_iq2s_msl_tgmem : (fmt == 2 ? metal_kq_mvb8_k2_msl_tgmem : (fmt == 45 ? metal_kq_mvb8_iq4nl_msl_tgmem : (fmt == 34 ? metal_kq_mvb8_iq3xxs_msl_tgmem : (fmt == 33 ? metal_kq_mvb8_iq3s_msl_tgmem : (fmt == 3 ? metal_kq_mvb8_k3_msl_tgmem : (fmt == 44 ? metal_kq_mvb8_iq4xs_msl_tgmem : (fmt == 4 ? MetalKqMvB8K4_metal_kq_mvb8_msl_tgmem : (fmt == 5 ? MetalKqMvB8K5_metal_kq_mvb8_msl_tgmem : metal_kq_mvb8_k6_msl_tgmem)))))))))), 0) } - metal_set_buffer(enc, bks, fmt == 6 ? uint64(nsb * 16) : 0ul, 0) // k6: the f16 d tail + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 || fmt == 24 || fmt == 25 ? uint64(nsb * 16) : 0ul, 0) // the split-scale formats: the f16 d tail at nsb*16 metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bx, 0ul, 3) @@ -251,7 +263,7 @@ def private moe_kq_gemv_gate(t : T?; dev, queue; fmt, n, d, net, k, nst : int; p bxs = uint((perslot ? k : 1) * n), bys = uint(k * d), hasb = 0u) let ran = with_compute_encoder(queue, err) $(enc : MetalComputeEncoder?) { metal_set_pipeline(enc, pso) - metal_set_buffer(enc, bks, fmt == 6 ? uint64(nsb * 16) : 0ul, 0) // k6: the f16 d tail + metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 || fmt == 24 || fmt == 25 ? uint64(nsb * 16) : 0ul, 0) // the split-scale formats: the f16 d tail at nsb*16 metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) metal_set_buffer(enc, bx, 0ul, 3) @@ -721,8 +733,26 @@ def test_metal_kq_gemv_kernels(t : T?) { kq_gemv_gate(t, dev, queue, "k6", 1280, 11) kq_gemv_gate(t, dev, queue, "k4", 512, 30) kq_gemv_gate(t, dev, queue, "k6", 512, 30) + kq_gemv_gate(t, dev, queue, "iq4xs", 1280, 11) + kq_gemv_gate(t, dev, queue, "iq4xs", 512, 30) + kq_gemv_gate(t, dev, queue, "k3", 1280, 11) + kq_gemv_gate(t, dev, queue, "k3", 512, 30) + kq_gemv_gate(t, dev, queue, "iq3s", 1280, 11) + kq_gemv_gate(t, dev, queue, "iq3s", 512, 30) + kq_gemv_gate(t, dev, queue, "iq3xxs", 1280, 11) + kq_gemv_gate(t, dev, queue, "iq3xxs", 512, 30) + kq_gemv_gate(t, dev, queue, "iq4nl", 1280, 11) + kq_gemv_gate(t, dev, queue, "iq4nl", 512, 30) + kq_gemv_gate(t, dev, queue, "k2", 1280, 11) + kq_gemv_gate(t, dev, queue, "k2", 512, 30) + kq_gemv_gate(t, dev, queue, "iq2s", 1280, 11) + kq_gemv_gate(t, dev, queue, "iq2s", 512, 30) + kq_gemv_gate(t, dev, queue, "iq2xs", 1280, 11) + kq_gemv_gate(t, dev, queue, "iq2xs", 512, 30) + kq_gemv_gate(t, dev, queue, "iq2xxs", 1280, 11) + kq_gemv_gate(t, dev, queue, "iq2xxs", 512, 30) // the small-batch twins: nr = 2 / 3 (b4 col guard) / 6 (b8 pad), ys > d - for (fmt in [4, 5, 6]) { + for (fmt in [4, 5, 6, 44, 3, 33, 34, 45, 2, 23, 24, 25]) { kq_mvb_gate(t, dev, queue, fmt, 2, 768, 17, 2, 22) kq_mvb_gate(t, dev, queue, fmt, 4, 768, 17, 3, 22) kq_mvb_gate(t, dev, queue, fmt, 8, 768, 17, 6, 22) diff --git a/modules/dasLLAMA/tests/test_model_image.das b/modules/dasLLAMA/tests/test_model_image.das index 9ed26ae205..11c1673a88 100644 --- a/modules/dasLLAMA/tests/test_model_image.das +++ b/modules/dasLLAMA/tests/test_model_image.das @@ -275,7 +275,7 @@ def test_image_mechanics(t : T?) { t |> equal(pk.identity, image_identity("img-probe"), "peek: baked identity string") var pcfg = DlimConfiguration() t |> success(dlim_config_from_json(pk.config_json, pcfg), "peek: embedded config JSON parses") - t |> equal(dlim_identity(pcfg, IMAGE_VERSION, "img-probe"), pk.identity, "peek: config re-derives the identity") + t |> equal(image_identity_of(pcfg, "img-probe"), pk.identity, "peek: config re-derives the identity") var m3 = ImgProbe() t |> success(!load_image(path, m3, "wrong-tag"), "identity mismatch declines") @@ -378,7 +378,7 @@ def test_image_mechanics(t : T?) { t |> success(!empty(js), "config serializes") var back = DlimConfiguration() t |> success(dlim_config_from_json(js, back), "JSON parses back") - t |> equal(dlim_identity(back, IMAGE_VERSION), dlim_identity(dc, IMAGE_VERSION), "identity survives the round trip") + t |> equal(image_identity_of(back), image_identity_of(dc), "identity survives the round trip") var junk = DlimConfiguration() t |> success(!dlim_config_from_json("not a config", junk), "garbage declines") } @@ -2379,3 +2379,16 @@ def test_layout_fingerprint(t : T?) { remove(path) } } + +[test] +def test_dlim_identity_pack_version(t : T?) { + t |> run("the pack-code version folds into the identity and nothing else moves") <| @(t : T?) { + let c = DlimConfiguration() + let base = dlim_identity(c, IMAGE_VERSION, "", 0) + let packed = dlim_identity(c, IMAGE_VERSION, "", 7) + t |> success(base != packed, "pack_version 7 changes the identity") + t |> success(packed |> starts_with("v{IMAGE_VERSION}p7|"), "the pack version rides the schema version: {packed}") + t |> success(base |> starts_with("v{IMAGE_VERSION}|"), "pack_version 0 is the pre-versioned spelling: {base}") + t |> equal(image_identity_of(c), dlim_identity(c, IMAGE_VERSION, "", PACK_VERSION), "image_identity_of folds the layout's PACK_VERSION") + } +} diff --git a/modules/dasLLAMA/tests/test_vulkan_kernels.das b/modules/dasLLAMA/tests/test_vulkan_kernels.das index 17258b0bff..cb7a23720e 100644 --- a/modules/dasLLAMA/tests/test_vulkan_kernels.das +++ b/modules/dasLLAMA/tests/test_vulkan_kernels.das @@ -379,24 +379,24 @@ def private attn_row_oracle(q : array; qbase : int; kk : array; kb [test] def test_vkd_kq_gemv_family(t0 : T?) { - t0 |> run("kq GEMV class family == the CPU oracle (all 4 formats)") <| @(t : T?) { + t0 |> run("kq GEMV class family == the CPU oracle (all 13 formats)") <| @(t : T?) { static_if (typeinfo builtin_module_exists(vulkan)) { - if (!(ensure_kq_gemv_k4_cls() && ensure_kq_gemv_q40_cls() && ensure_kq_gemv_k5_cls() && ensure_kq_gemv_k6_cls())) { - feint("no Vulkan device - skipping\n") + if (!(ensure_kq_gemv_k4_cls() && ensure_kq_gemv_q40_cls() && ensure_kq_gemv_k5_cls() && ensure_kq_gemv_k6_cls() && ensure_kq_gemv_iq4xs_cls() && ensure_kq_gemv_k3_cls() && ensure_kq_gemv_iq3s_cls() && ensure_kq_gemv_iq3xxs_cls() && ensure_kq_gemv_iq4nl_cls() && ensure_kq_gemv_k2_cls() && ensure_kq_gemv_iq2s_cls() && ensure_kq_gemv_iq2xs_cls() && ensure_kq_gemv_iq2xxs_cls())) { + t |> skip("no Vulkan device, or a kq GEMV class pipeline failed to build") return } let nsb = 2 // superblocks per row reduction (n = 512) let d = 3 // rows per region let nreg = 2 let rows = nreg * d - let nfmt = 4 - let fmts = fixed_array(int(KqFmt.k4), int(KqFmt.q40), int(KqFmt.k5), int(KqFmt.k6)) - let stride4s = fixed_array(8, 8, 10, 12) // uint4s per superblock + let nfmt = 13 + let fmts = fixed_array(int(KqFmt.k4), int(KqFmt.q40), int(KqFmt.k5), int(KqFmt.k6), int(KqFmt.iq4xs), int(KqFmt.k3), int(KqFmt.iq3s), int(KqFmt.iq3xxs), int(KqFmt.iq4nl), int(KqFmt.k2), int(KqFmt.iq2s), int(KqFmt.iq2xs), int(KqFmt.iq2xxs)) + let stride4s = fixed_array(32, 32, 40, 48, 32, 24, 26, 24, 32, 16, 18, 16, 16) // uint WORDS per superblock (iq3s/iq3xxs/iq2s are not uint4-aligned) for (fi in range(nfmt)) { let fmt = fmts[fi] let stride4 = stride4s[fi] let nwsb = rows * nsb // region r's weights at r*d*nsb - let wq_words = nwsb * stride4 * 4 + let wq_words = nwsb * stride4 let ws_words_n = nwsb * 5 var wqh : array var wsh : array @@ -461,6 +461,24 @@ def test_vkd_kq_gemv_family(t0 : T?) { enc_kq_gemv_q40_cls(raw2, h2, s_cls, pc, wgs) } elif (fmt == int(KqFmt.k5)) { enc_kq_gemv_k5_cls(raw2, h2, s_cls, pc, wgs) + } elif (fmt == int(KqFmt.iq4xs)) { + enc_kq_gemv_iq4xs_cls(raw2, h2, s_cls, pc, wgs) + } elif (fmt == int(KqFmt.k3)) { + enc_kq_gemv_k3_cls(raw2, h2, s_cls, pc, wgs) + } elif (fmt == int(KqFmt.iq3s)) { + enc_kq_gemv_iq3s_cls(raw2, h2, s_cls, pc, wgs) + } elif (fmt == int(KqFmt.iq3xxs)) { + enc_kq_gemv_iq3xxs_cls(raw2, h2, s_cls, pc, wgs) + } elif (fmt == int(KqFmt.iq4nl)) { + enc_kq_gemv_iq4nl_cls(raw2, h2, s_cls, pc, wgs) + } elif (fmt == int(KqFmt.k2)) { + enc_kq_gemv_k2_cls(raw2, h2, s_cls, pc, wgs) + } elif (fmt == int(KqFmt.iq2s)) { + enc_kq_gemv_iq2s_cls(raw2, h2, s_cls, pc, wgs) + } elif (fmt == int(KqFmt.iq2xs)) { + enc_kq_gemv_iq2xs_cls(raw2, h2, s_cls, pc, wgs) + } elif (fmt == int(KqFmt.iq2xxs)) { + enc_kq_gemv_iq2xxs_cls(raw2, h2, s_cls, pc, wgs) } else { enc_kq_gemv_k6_cls(raw2, h2, s_cls, pc, wgs) } @@ -473,6 +491,48 @@ def test_vkd_kq_gemv_family(t0 : T?) { var y_ref <- kq_gemv_oracle(fmt, wqh, wsh, xqh, xsh, regs, 512, d, nreg) let bad = mismatch_approx(y_cls, y_ref) t |> success(bad == 0, "fmt {fmt}: class kernel matches the CPU oracle ({bad} of {rows} rows off)") + if (fmt == int(KqFmt.iq4xs)) { // the codebook pack has an independent float witness + var y_flt <- iq4xs_gemv_float_oracle(wqh, wsh, xqh, xsh, regs, 512, d, nreg) + let badf = mismatch_approx(y_ref, y_flt) + t |> success(badf == 0, "iq4xs: the class bit-math matches the float dequant oracle ({badf} of {rows} rows off)") + delete y_flt + } + if (fmt == int(KqFmt.k3)) { // the lane/mask compose has an independent float witness + var y_flt <- k3_gemv_float_oracle(wqh, wsh, xqh, xsh, regs, 512, d, nreg) + let badf = mismatch_approx(y_ref, y_flt) + t |> success(badf == 0, "k3: the class bit-math matches the float dequant oracle ({badf} of {rows} rows off)") + delete y_flt + } + if (fmt == int(KqFmt.iq3s)) { // the grid/sign gather has an independent float witness + var y_flt <- iq3s_gemv_float_oracle(wqh, wsh, xqh, xsh, regs, 512, d, nreg) + let badf = mismatch_approx(y_ref, y_flt) + t |> success(badf == 0, "iq3s: the class bit-math matches the float dequant oracle ({badf} of {rows} rows off)") + delete y_flt + } + if (fmt == int(KqFmt.iq3xxs)) { // the halved-grid/parity-sign gather has its own float witness + var y_flt <- iq3xxs_gemv_float_oracle(wqh, wsh, xqh, xsh, regs, 512, d, nreg) + let badf = mismatch_approx(y_ref, y_flt) + t |> success(badf == 0, "iq3xxs: the class bit-math matches the float dequant oracle ({badf} of {rows} rows off)") + delete y_flt + } + if (fmt == int(KqFmt.iq2s)) { // the u64-grid/sign-byte gather has its own float witness + var y_flt <- iq2s_gemv_float_oracle(wqh, wsh, xqh, xsh, regs, 512, d, nreg) + let badf = mismatch_approx(y_ref, y_flt) + t |> success(badf == 0, "iq2s: the class bit-math matches the float dequant oracle ({badf} of {rows} rows off)") + delete y_flt + } + if (fmt == int(KqFmt.iq2xs)) { // the u64-grid/ksigns-parity gather has its own float witness + var y_flt <- iq2xs_gemv_float_oracle(wqh, wsh, xqh, xsh, regs, 512, d, nreg) + let badf = mismatch_approx(y_ref, y_flt) + t |> success(badf == 0, "iq2xs: the class bit-math matches the float dequant oracle ({badf} of {rows} rows off)") + delete y_flt + } + if (fmt == int(KqFmt.iq2xxs)) { // the byte-grid/aux32 gather has its own float witness + var y_flt <- iq2xxs_gemv_float_oracle(wqh, wsh, xqh, xsh, regs, 512, d, nreg) + let badf = mismatch_approx(y_ref, y_flt) + t |> success(badf == 0, "iq2xxs: the class bit-math matches the float dequant oracle ({badf} of {rows} rows off)") + delete y_flt + } delete regs delete y_ref delete wqh @@ -991,10 +1051,10 @@ def test_vkd_q8_batch_tile(t0 : T?) { [test] def test_vkd_kq_batch_family(t0 : T?) { - t0 |> run("kq batch tile class family == the CPU oracle (all 4 formats)") <| @(t : T?) { + t0 |> run("kq batch tile class family == the CPU oracle (all 13 formats)") <| @(t : T?) { static_if (typeinfo builtin_module_exists(vulkan)) { - if (!(ensure_kq_batch_k4_cls() && ensure_kq_batch_q40_cls() && ensure_kq_batch_k5_cls() && ensure_kq_batch_k6_cls())) { - feint("no Vulkan device - skipping\n") + if (!(ensure_kq_batch_k4_cls() && ensure_kq_batch_q40_cls() && ensure_kq_batch_k5_cls() && ensure_kq_batch_k6_cls() && ensure_kq_batch_iq4xs_cls() && ensure_kq_batch_k3_cls() && ensure_kq_batch_iq3s_cls() && ensure_kq_batch_iq3xxs_cls() && ensure_kq_batch_iq4nl_cls() && ensure_kq_batch_k2_cls() && ensure_kq_batch_iq2s_cls() && ensure_kq_batch_iq2xs_cls() && ensure_kq_batch_iq2xxs_cls())) { + t |> skip("no Vulkan device, or a kq batch class pipeline failed to build") return } let n = 512 // nsb = 2 superblocks @@ -1008,9 +1068,9 @@ def test_vkd_kq_batch_family(t0 : T?) { let wgs1 = ((cnt1 + 31) / 32) * wtiles let wgs = wgs0 + wgs1 let totsb = 2 * d * nsb // superblocks across both regions - let nfmt = 4 - let wstrides = fixed_array(32, 32, 40, 48) // uint words per superblock: k4 q40 k5 k6 - let pipes = fixed_array(int(KqFmt.k4), int(KqFmt.q40), int(KqFmt.k5), int(KqFmt.k6)) + let nfmt = 13 + let wstrides = fixed_array(32, 32, 40, 48, 32, 24, 26, 24, 32, 16, 18, 16, 16) // uint words per superblock: k4 q40 k5 k6 iq4xs k3 iq3s iq3xxs iq4nl k2 iq2s iq2xs iq2xxs + let pipes = fixed_array(int(KqFmt.k4), int(KqFmt.q40), int(KqFmt.k5), int(KqFmt.k6), int(KqFmt.iq4xs), int(KqFmt.k3), int(KqFmt.iq3s), int(KqFmt.iq3xxs), int(KqFmt.iq4nl), int(KqFmt.k2), int(KqFmt.iq2s), int(KqFmt.iq2xs), int(KqFmt.iq2xxs)) for (fi in range(nfmt)) { let fmt = pipes[fi] let wstride = wstrides[fi] @@ -1083,6 +1143,24 @@ def test_vkd_kq_batch_family(t0 : T?) { enc_kq_batch_q40_cls(raw2, h2, sc, pc, int64(wgs)) } elif (fmt == int(KqFmt.k5)) { enc_kq_batch_k5_cls(raw2, h2, sc, pc, int64(wgs)) + } elif (fmt == int(KqFmt.iq4xs)) { + enc_kq_batch_iq4xs_cls(raw2, h2, sc, pc, int64(wgs)) + } elif (fmt == int(KqFmt.k3)) { + enc_kq_batch_k3_cls(raw2, h2, sc, pc, int64(wgs)) + } elif (fmt == int(KqFmt.iq3s)) { + enc_kq_batch_iq3s_cls(raw2, h2, sc, pc, int64(wgs)) + } elif (fmt == int(KqFmt.iq3xxs)) { + enc_kq_batch_iq3xxs_cls(raw2, h2, sc, pc, int64(wgs)) + } elif (fmt == int(KqFmt.iq4nl)) { + enc_kq_batch_iq4nl_cls(raw2, h2, sc, pc, int64(wgs)) + } elif (fmt == int(KqFmt.k2)) { + enc_kq_batch_k2_cls(raw2, h2, sc, pc, int64(wgs)) + } elif (fmt == int(KqFmt.iq2s)) { + enc_kq_batch_iq2s_cls(raw2, h2, sc, pc, int64(wgs)) + } elif (fmt == int(KqFmt.iq2xs)) { + enc_kq_batch_iq2xs_cls(raw2, h2, sc, pc, int64(wgs)) + } elif (fmt == int(KqFmt.iq2xxs)) { + enc_kq_batch_iq2xxs_cls(raw2, h2, sc, pc, int64(wgs)) } else { enc_kq_batch_k6_cls(raw2, h2, sc, pc, int64(wgs)) } @@ -1631,6 +1709,1413 @@ def test_vkd_k4_cm2_batch(t0 : T?) { } } +[test] +def test_vkd_k5_cm2_batch(t0 : T?) { + t0 |> run("cm2 Q5_K l/m/s tiles == the CPU oracle (5th-bit compose decode-in-load)") <| @(t : T?) { + static_if (typeinfo builtin_module_exists(vulkan)) { + if (!ensure_q8_batch_cls()) { // engage the device without touching coopmat rails + t |> skip("no Vulkan device") + return + } + if (!(g_gpu.coopmat_mode == 4 && g_gpu.has_coopmat2)) { + t |> skip("the cm2 k5 tiles serve only in mode 4 on an NV_coopmat2 device") + return + } + verify(ensure_kq_batch_k5_cm2l_cls() && ensure_kq_batch_k5_cm2m_cls() && ensure_kq_batch_k5_cm2s_cls(), "k5 cm2 class rails must engage in cm2 mode") + let n = 768 // one 512-unrolled pass + 4 tail steps crossing superblock bounds + let nsb = n / 256 + let d = 160 // one full 128-tile + a 32 edge + let cnt0 = 300 + let cnt1 = 260 + let rows = cnt0 + cnt1 + let totsb = 2 * d * nsb + var wqh : array + var wsuh : array + var xfh : array + wqh |> resize(totsb * 40) + wsuh |> resize(totsb * 5) + xfh |> resize((rows + 32) * n / 2) // + the s tile's 32-row read slack + for (i in range(totsb * 40)) { + wqh[i] = hash_word(uint(i) + 29u) // arbitrary quant bytes + } + for (sb in range(totsb)) { + let dv = 0.0002 * float(1 + sb % 7) // cool scales: the f16 acc must stay far from 65504 + let dmv = 0.00005 * float(1 + sb % 5) + wsuh[sb * 5] = packHalf2x16(float2(dv, dmv)) + for (wi in range(4)) { + wsuh[sb * 5 + 1 + wi] = hash_word(uint(sb * 4 + wi) + 613u) // sc / mn bytes + } + } + for (i in range((rows + 32) * n / 2)) { + xfh[i] = ws_word(uint(i) * 3u + 5u) // f16 activation pairs + } + let wq_bytes = int64(totsb) * 160l + let ws_bytes = int64(length(wsuh)) * 4l + let xf_bytes = int64((rows + 32) * n) * 2l + let y_bytes = int64(rows * d) * 4l + let wqd = make_device_buf(wq_bytes) + let wsd = make_device_buf(ws_bytes) + let xfd = make_device_buf(xf_bytes) + let yd2 = make_device_buf(y_bytes) + var host = make_host_buf(y_bytes, true, [cached = true]) + var y_cls : array + y_cls |> resize(rows * d) + var y_ref : array + y_ref |> resize(rows * d) + for (ml in range(3)) { + let ttile = ml == 0 ? 256 : (ml == 1 ? 128 : 32) + let wtiles = (d + 127) / 128 + let wgs0 = ((cnt0 + ttile - 1) / ttile) * wtiles + let wgs1 = ((cnt1 + ttile - 1) / ttile) * wtiles + let wgs = wgs0 + wgs1 + var sched : array + sched |> resize(2 * 4 + wgs) + sched[0] = 0u + sched[1] = 0u + sched[2] = uint(cnt0) + sched[3] = 0u + sched[4] = uint(d * nsb) + sched[5] = uint(cnt0) + sched[6] = uint(cnt1) + sched[7] = uint(wgs0) + for (w in range(wgs)) { + sched[8 + w] = w < wgs0 ? 0u : 1u + } + unsafe { + upload_region_at(wqd, 0l, addr(wqh[0]), wq_bytes) + upload_region_at(wsd, 0l, addr(wsuh[0]), ws_bytes) + upload_region_at(xfd, 0l, addr(xfh[0]), xf_bytes) + let sc_bytes = int64(2 * 4 + wgs) * 4l + let scd = make_device_buf(sc_bytes) + upload_region_at(scd, 0l, addr(sched[0]), sc_bytes) + var sc = (ml == 0 + ? set_kq_batch_k5_cm2l_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)) + : (ml == 1 + ? set_kq_batch_k5_cm2m_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)) + : set_kq_batch_k5_cm2s_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)))) + var raw2 = alloc_cmd() + let begin = VkCommandBufferBeginInfo() + vk_check(vkBeginCommandBuffer(raw2, begin), null) + var h2 : VkHaz + var pc = BatchArgs(n = uint(n), d = uint(d), map_off = uint(2 * 4)) + if (ml == 0) { + enc_kq_batch_k5_cm2l_cls(raw2, h2, sc, pc, int64(wgs)) + } elif (ml == 1) { + enc_kq_batch_k5_cm2m_cls(raw2, h2, sc, pc, int64(wgs)) + } else { + enc_kq_batch_k5_cm2s_cls(raw2, h2, sc, pc, int64(wgs)) + } + vhz_dep(raw2, h2, 16u, 0u, true) + cmd_copy_whole(raw2, yd2, host.buf, y_bytes) + vk_check(vkEndCommandBuffer(raw2), null) + submit_wait(raw2) + memcpy(addr(y_cls[0]), host.mapped, y_bytes) + } + k5f16_gemm_oracle(wqh, wsuh, xfh, sched, 2, n, d, y_ref) + let bad = mismatch_bars(y_cls, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) + to_log(LOG_INFO, "cm2 k5 {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile batch: {bad} of {rows * d} off the oracle\n") + t |> success(bad == 0, "cm2 k5 {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile matches the CPU oracle ({bad} of {rows * d} off)") + var y_poison := y_cls // the bar's control: one element pushed past both bars must red + y_poison[0] += 1.0 + 2.0 * max_abs(y_ref) + t |> success(mismatch_bars(y_poison, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) > 0, "the k5 bar reds a poisoned element") + delete y_poison + delete sched + } + delete y_ref + delete wqh + delete wsuh + delete xfh + delete y_cls + } else { + t |> skip("dasVulkan not present") + } + } +} + +[test] +def test_vkd_q40_cm2_batch(t0 : T?) { + t0 |> run("cm2 Q4_0 l/m/s tiles == the CPU oracle (per-block f16 d decode-in-load)") <| @(t : T?) { + static_if (typeinfo builtin_module_exists(vulkan)) { + if (!ensure_q8_batch_cls()) { // engage the device without touching coopmat rails + t |> skip("no Vulkan device") + return + } + if (!(g_gpu.coopmat_mode == 4 && g_gpu.has_coopmat2)) { + t |> skip("the cm2 q40 tiles serve only in mode 4 on an NV_coopmat2 device") + return + } + verify(ensure_kq_batch_q40_cm2l_cls() && ensure_kq_batch_q40_cm2m_cls() && ensure_kq_batch_q40_cm2s_cls(), "q40 cm2 class rails must engage in cm2 mode") + let n = 768 // one 512-unrolled pass + 4 tail steps crossing superblock bounds + let nsb = n / 256 + let d = 160 // one full 128-tile + a 32 edge + let cnt0 = 300 + let cnt1 = 260 + let rows = cnt0 + cnt1 + let totsb = 2 * d * nsb + var wqh : array + var wsuh : array + var xfh : array + wqh |> resize(totsb * 32) + wsuh |> resize(totsb * 5) + xfh |> resize((rows + 32) * n / 2) // + the s tile's 32-row read slack + for (i in range(totsb * 32)) { + wqh[i] = hash_word(uint(i) + 29u) // arbitrary quant bytes + } + for (sb in range(totsb)) { + for (wi in range(4)) { // 8 per-block f16 d - cool, the f16 acc must stay far from 65504 + let p = sb * 4 + wi + wsuh[sb * 5 + wi] = packHalf2x16(float2(0.0002 * float(1 + p % 7), 0.00013 * float(1 + p % 5))) + } + wsuh[sb * 5 + 4] = 0u + } + for (i in range((rows + 32) * n / 2)) { + xfh[i] = ws_word(uint(i) * 3u + 5u) // f16 activation pairs + } + let wq_bytes = int64(totsb) * 128l + let ws_bytes = int64(length(wsuh)) * 4l + let xf_bytes = int64((rows + 32) * n) * 2l + let y_bytes = int64(rows * d) * 4l + let wqd = make_device_buf(wq_bytes) + let wsd = make_device_buf(ws_bytes) + let xfd = make_device_buf(xf_bytes) + let yd2 = make_device_buf(y_bytes) + var host = make_host_buf(y_bytes, true, [cached = true]) + var y_cls : array + y_cls |> resize(rows * d) + var y_ref : array + y_ref |> resize(rows * d) + for (ml in range(3)) { + let ttile = ml == 0 ? 256 : (ml == 1 ? 128 : 32) + let wtiles = (d + 127) / 128 + let wgs0 = ((cnt0 + ttile - 1) / ttile) * wtiles + let wgs1 = ((cnt1 + ttile - 1) / ttile) * wtiles + let wgs = wgs0 + wgs1 + var sched : array + sched |> resize(2 * 4 + wgs) + sched[0] = 0u + sched[1] = 0u + sched[2] = uint(cnt0) + sched[3] = 0u + sched[4] = uint(d * nsb) + sched[5] = uint(cnt0) + sched[6] = uint(cnt1) + sched[7] = uint(wgs0) + for (w in range(wgs)) { + sched[8 + w] = w < wgs0 ? 0u : 1u + } + unsafe { + upload_region_at(wqd, 0l, addr(wqh[0]), wq_bytes) + upload_region_at(wsd, 0l, addr(wsuh[0]), ws_bytes) + upload_region_at(xfd, 0l, addr(xfh[0]), xf_bytes) + let sc_bytes = int64(2 * 4 + wgs) * 4l + let scd = make_device_buf(sc_bytes) + upload_region_at(scd, 0l, addr(sched[0]), sc_bytes) + var sc = (ml == 0 + ? set_kq_batch_q40_cm2l_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)) + : (ml == 1 + ? set_kq_batch_q40_cm2m_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)) + : set_kq_batch_q40_cm2s_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)))) + var raw2 = alloc_cmd() + let begin = VkCommandBufferBeginInfo() + vk_check(vkBeginCommandBuffer(raw2, begin), null) + var h2 : VkHaz + var pc = BatchArgs(n = uint(n), d = uint(d), map_off = uint(2 * 4)) + if (ml == 0) { + enc_kq_batch_q40_cm2l_cls(raw2, h2, sc, pc, int64(wgs)) + } elif (ml == 1) { + enc_kq_batch_q40_cm2m_cls(raw2, h2, sc, pc, int64(wgs)) + } else { + enc_kq_batch_q40_cm2s_cls(raw2, h2, sc, pc, int64(wgs)) + } + vhz_dep(raw2, h2, 16u, 0u, true) + cmd_copy_whole(raw2, yd2, host.buf, y_bytes) + vk_check(vkEndCommandBuffer(raw2), null) + submit_wait(raw2) + memcpy(addr(y_cls[0]), host.mapped, y_bytes) + } + q40f16_gemm_oracle(wqh, wsuh, xfh, sched, 2, n, d, y_ref) + let bad = mismatch_bars(y_cls, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) + to_log(LOG_INFO, "cm2 q40 {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile batch: {bad} of {rows * d} off the oracle\n") + t |> success(bad == 0, "cm2 q40 {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile matches the CPU oracle ({bad} of {rows * d} off)") + var y_poison := y_cls // the bar's control: one element pushed past both bars must red + y_poison[0] += 1.0 + 2.0 * max_abs(y_ref) + t |> success(mismatch_bars(y_poison, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) > 0, "the q40 bar reds a poisoned element") + delete y_poison + delete sched + } + delete y_ref + delete wqh + delete wsuh + delete xfh + delete y_cls + } else { + t |> skip("dasVulkan not present") + } + } +} + +[test] +def test_vkd_k2_cm2_batch(t0 : T?) { + t0 |> run("cm2 Q2_K l/m/s tiles == the CPU oracle (pair-byte nibble scales decode-in-load)") <| @(t : T?) { + static_if (typeinfo builtin_module_exists(vulkan)) { + if (!ensure_q8_batch_cls()) { // engage the device without touching coopmat rails + t |> skip("no Vulkan device") + return + } + if (!(g_gpu.coopmat_mode == 4 && g_gpu.has_coopmat2)) { + t |> skip("the cm2 k2 tiles serve only in mode 4 on an NV_coopmat2 device") + return + } + verify(ensure_kq_batch_k2_cm2l_cls() && ensure_kq_batch_k2_cm2m_cls() && ensure_kq_batch_k2_cm2s_cls(), "k2 cm2 class rails must engage in cm2 mode") + let n = 768 // one 512-unrolled pass + 4 tail steps crossing superblock bounds + let nsb = n / 256 + let d = 160 // one full 128-tile + a 32 edge + let cnt0 = 300 + let cnt1 = 260 + let rows = cnt0 + cnt1 + let totsb = 2 * d * nsb + var wqh : array + var wsuh : array + var xfh : array + wqh |> resize(totsb * 16) + wsuh |> resize(totsb * 5) + xfh |> resize((rows + 32) * n / 2) // + the s tile's 32-row read slack + for (i in range(totsb * 16)) { + wqh[i] = hash_word(uint(i) + 31u) // arbitrary quant bytes + } + for (sb in range(totsb)) { + wsuh[sb * 5] = packHalf2x16(float2(0.0004 * float(1 + sb % 7), 0.0003 * float(1 + sb % 5))) + for (wi in range(4)) { // 16 sc/min nibble-pair bytes + wsuh[sb * 5 + 1 + wi] = hash_word(uint(sb * 4 + wi) + 91u) + } + } + for (i in range((rows + 32) * n / 2)) { + xfh[i] = ws_word(uint(i) * 3u + 7u) // f16 activation pairs + } + let wq_bytes = int64(totsb) * 64l + let ws_bytes = int64(length(wsuh)) * 4l + let xf_bytes = int64((rows + 32) * n) * 2l + let y_bytes = int64(rows * d) * 4l + let wqd = make_device_buf(wq_bytes) + let wsd = make_device_buf(ws_bytes) + let xfd = make_device_buf(xf_bytes) + let yd2 = make_device_buf(y_bytes) + var host = make_host_buf(y_bytes, true, [cached = true]) + var y_cls : array + y_cls |> resize(rows * d) + var y_ref : array + y_ref |> resize(rows * d) + for (ml in range(3)) { + let ttile = ml == 0 ? 256 : (ml == 1 ? 128 : 32) + let wtiles = (d + 127) / 128 + let wgs0 = ((cnt0 + ttile - 1) / ttile) * wtiles + let wgs1 = ((cnt1 + ttile - 1) / ttile) * wtiles + let wgs = wgs0 + wgs1 + var sched : array + sched |> resize(2 * 4 + wgs) + sched[0] = 0u + sched[1] = 0u + sched[2] = uint(cnt0) + sched[3] = 0u + sched[4] = uint(d * nsb) + sched[5] = uint(cnt0) + sched[6] = uint(cnt1) + sched[7] = uint(wgs0) + for (w in range(wgs)) { + sched[8 + w] = w < wgs0 ? 0u : 1u + } + unsafe { + upload_region_at(wqd, 0l, addr(wqh[0]), wq_bytes) + upload_region_at(wsd, 0l, addr(wsuh[0]), ws_bytes) + upload_region_at(xfd, 0l, addr(xfh[0]), xf_bytes) + let sc_bytes = int64(2 * 4 + wgs) * 4l + let scd = make_device_buf(sc_bytes) + upload_region_at(scd, 0l, addr(sched[0]), sc_bytes) + var sc = (ml == 0 + ? set_kq_batch_k2_cm2l_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)) + : (ml == 1 + ? set_kq_batch_k2_cm2m_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)) + : set_kq_batch_k2_cm2s_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)))) + var raw2 = alloc_cmd() + let begin = VkCommandBufferBeginInfo() + vk_check(vkBeginCommandBuffer(raw2, begin), null) + var h2 : VkHaz + var pc = BatchArgs(n = uint(n), d = uint(d), map_off = uint(2 * 4)) + if (ml == 0) { + enc_kq_batch_k2_cm2l_cls(raw2, h2, sc, pc, int64(wgs)) + } elif (ml == 1) { + enc_kq_batch_k2_cm2m_cls(raw2, h2, sc, pc, int64(wgs)) + } else { + enc_kq_batch_k2_cm2s_cls(raw2, h2, sc, pc, int64(wgs)) + } + vhz_dep(raw2, h2, 16u, 0u, true) + cmd_copy_whole(raw2, yd2, host.buf, y_bytes) + vk_check(vkEndCommandBuffer(raw2), null) + submit_wait(raw2) + memcpy(addr(y_cls[0]), host.mapped, y_bytes) + } + k2f16_gemm_oracle(wqh, wsuh, xfh, sched, 2, n, d, y_ref) + let bad = mismatch_bars(y_cls, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) + to_log(LOG_INFO, "cm2 k2 {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile batch: {bad} of {rows * d} off the oracle\n") + t |> success(bad == 0, "cm2 k2 {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile matches the CPU oracle ({bad} of {rows * d} off)") + var y_poison := y_cls // the bar's control: one element pushed past both bars must red + y_poison[0] += 1.0 + 2.0 * max_abs(y_ref) + t |> success(mismatch_bars(y_poison, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) > 0, "the k2 bar reds a poisoned element") + delete y_poison + delete sched + } + delete y_ref + delete wqh + delete wsuh + delete xfh + delete y_cls + } else { + t |> skip("dasVulkan not present") + } + } +} + +[test] +def test_vkd_iq4nl_cm2_batch(t0 : T?) { + t0 |> run("cm2 IQ4_NL l/m/s tiles == the CPU oracle (codebook + per-block f16 d decode-in-load)") <| @(t : T?) { + static_if (typeinfo builtin_module_exists(vulkan)) { + if (!ensure_q8_batch_cls()) { // engage the device without touching coopmat rails + t |> skip("no Vulkan device") + return + } + if (!(g_gpu.coopmat_mode == 4 && g_gpu.has_coopmat2)) { + t |> skip("the cm2 iq4nl tiles serve only in mode 4 on an NV_coopmat2 device") + return + } + verify(ensure_kq_batch_iq4nl_cm2l_cls() && ensure_kq_batch_iq4nl_cm2m_cls() && ensure_kq_batch_iq4nl_cm2s_cls(), "iq4nl cm2 class rails must engage in cm2 mode") + let n = 768 // one 512-unrolled pass + 4 tail steps crossing superblock bounds + let nsb = n / 256 + let d = 160 // one full 128-tile + a 32 edge + let cnt0 = 300 + let cnt1 = 260 + let rows = cnt0 + cnt1 + let totsb = 2 * d * nsb + var wqh : array + var wsuh : array + var xfh : array + wqh |> resize(totsb * 32) + wsuh |> resize(totsb * 5) + xfh |> resize((rows + 32) * n / 2) // + the s tile's 32-row read slack + for (i in range(totsb * 32)) { + wqh[i] = hash_word(uint(i) + 29u) // arbitrary quant bytes + } + for (sb in range(totsb)) { + for (wi in range(4)) { // 8 per-block f16 d - cool, the f16 acc must stay far from 65504 + let p = sb * 4 + wi + wsuh[sb * 5 + wi] = packHalf2x16(float2(0.0002 * float(1 + p % 7), 0.00013 * float(1 + p % 5))) + } + wsuh[sb * 5 + 4] = 0u + } + for (i in range((rows + 32) * n / 2)) { + xfh[i] = ws_word(uint(i) * 3u + 5u) // f16 activation pairs + } + let wq_bytes = int64(totsb) * 128l + let ws_bytes = int64(length(wsuh)) * 4l + let xf_bytes = int64((rows + 32) * n) * 2l + let y_bytes = int64(rows * d) * 4l + let wqd = make_device_buf(wq_bytes) + let wsd = make_device_buf(ws_bytes) + let xfd = make_device_buf(xf_bytes) + let yd2 = make_device_buf(y_bytes) + var host = make_host_buf(y_bytes, true, [cached = true]) + var y_cls : array + y_cls |> resize(rows * d) + var y_ref : array + y_ref |> resize(rows * d) + for (ml in range(3)) { + let ttile = ml == 0 ? 256 : (ml == 1 ? 128 : 32) + let wtiles = (d + 127) / 128 + let wgs0 = ((cnt0 + ttile - 1) / ttile) * wtiles + let wgs1 = ((cnt1 + ttile - 1) / ttile) * wtiles + let wgs = wgs0 + wgs1 + var sched : array + sched |> resize(2 * 4 + wgs) + sched[0] = 0u + sched[1] = 0u + sched[2] = uint(cnt0) + sched[3] = 0u + sched[4] = uint(d * nsb) + sched[5] = uint(cnt0) + sched[6] = uint(cnt1) + sched[7] = uint(wgs0) + for (w in range(wgs)) { + sched[8 + w] = w < wgs0 ? 0u : 1u + } + unsafe { + upload_region_at(wqd, 0l, addr(wqh[0]), wq_bytes) + upload_region_at(wsd, 0l, addr(wsuh[0]), ws_bytes) + upload_region_at(xfd, 0l, addr(xfh[0]), xf_bytes) + let sc_bytes = int64(2 * 4 + wgs) * 4l + let scd = make_device_buf(sc_bytes) + upload_region_at(scd, 0l, addr(sched[0]), sc_bytes) + var sc = (ml == 0 + ? set_kq_batch_iq4nl_cm2l_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)) + : (ml == 1 + ? set_kq_batch_iq4nl_cm2m_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)) + : set_kq_batch_iq4nl_cm2s_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)))) + var raw2 = alloc_cmd() + let begin = VkCommandBufferBeginInfo() + vk_check(vkBeginCommandBuffer(raw2, begin), null) + var h2 : VkHaz + var pc = BatchArgs(n = uint(n), d = uint(d), map_off = uint(2 * 4)) + if (ml == 0) { + enc_kq_batch_iq4nl_cm2l_cls(raw2, h2, sc, pc, int64(wgs)) + } elif (ml == 1) { + enc_kq_batch_iq4nl_cm2m_cls(raw2, h2, sc, pc, int64(wgs)) + } else { + enc_kq_batch_iq4nl_cm2s_cls(raw2, h2, sc, pc, int64(wgs)) + } + vhz_dep(raw2, h2, 16u, 0u, true) + cmd_copy_whole(raw2, yd2, host.buf, y_bytes) + vk_check(vkEndCommandBuffer(raw2), null) + submit_wait(raw2) + memcpy(addr(y_cls[0]), host.mapped, y_bytes) + } + iq4nlf16_gemm_oracle(wqh, wsuh, xfh, sched, 2, n, d, y_ref) + let bad = mismatch_bars(y_cls, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) + to_log(LOG_INFO, "cm2 iq4nl {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile batch: {bad} of {rows * d} off the oracle\n") + t |> success(bad == 0, "cm2 iq4nl {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile matches the CPU oracle ({bad} of {rows * d} off)") + var y_poison := y_cls // the bar's control: one element pushed past both bars must red + y_poison[0] += 1.0 + 2.0 * max_abs(y_ref) + t |> success(mismatch_bars(y_poison, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) > 0, "the iq4nl bar reds a poisoned element") + delete y_poison + delete sched + } + delete y_ref + delete wqh + delete wsuh + delete xfh + delete y_cls + } else { + t |> skip("dasVulkan not present") + } + } +} + +[test] +def test_vkd_iq4xs_cm2_batch(t0 : T?) { + t0 |> run("cm2 IQ4_XS l/m/s tiles == the CPU oracle (workgroup-staged codebook decode-in-load)") <| @(t : T?) { + static_if (typeinfo builtin_module_exists(vulkan)) { + if (!ensure_q8_batch_cls()) { // engage the device without touching coopmat rails + t |> skip("no Vulkan device") + return + } + if (!(g_gpu.coopmat_mode == 4 && g_gpu.has_coopmat2)) { + t |> skip("the cm2 iq4xs tiles serve only in mode 4 on an NV_coopmat2 device") + return + } + verify(ensure_kq_batch_iq4xs_cm2l_cls() && ensure_kq_batch_iq4xs_cm2m_cls() && ensure_kq_batch_iq4xs_cm2s_cls(), "iq4xs cm2 class rails must engage in cm2 mode") + let n = 768 // one 512-unrolled pass + 4 tail steps crossing superblock bounds + let nsb = n / 256 + let d = 160 // one full 128-tile + a 32 edge + let cnt0 = 300 + let cnt1 = 260 + let rows = cnt0 + cnt1 + let totsb = 2 * d * nsb + var wqh : array + var wsuh : array + var xfh : array + wqh |> resize(totsb * 32) + wsuh |> resize(totsb * 5) + xfh |> resize((rows + 32) * n / 2) // + the s tile's 32-row read slack + for (i in range(totsb * 32)) { + wqh[i] = hash_word(uint(i) + 29u) // arbitrary quant bytes + } + for (sb in range(totsb)) { + wsuh[sb * 5] = packHalf2x16(float2(0.0002 * float(1 + sb % 7), 0.0)) // cool d + wsuh[sb * 5 + 1] = hash_word(uint(sb * 2) + 613u) // signed sub-scale bytes + wsuh[sb * 5 + 2] = hash_word(uint(sb * 2 + 1) + 613u) + wsuh[sb * 5 + 3] = 0u + wsuh[sb * 5 + 4] = 0u + } + for (i in range((rows + 32) * n / 2)) { + xfh[i] = ws_word(uint(i) * 3u + 5u) // f16 activation pairs + } + let wq_bytes = int64(totsb) * 128l + let ws_bytes = int64(length(wsuh)) * 4l + let xf_bytes = int64((rows + 32) * n) * 2l + let y_bytes = int64(rows * d) * 4l + let wqd = make_device_buf(wq_bytes) + let wsd = make_device_buf(ws_bytes) + let xfd = make_device_buf(xf_bytes) + let yd2 = make_device_buf(y_bytes) + var host = make_host_buf(y_bytes, true, [cached = true]) + var y_cls : array + y_cls |> resize(rows * d) + var y_ref : array + y_ref |> resize(rows * d) + for (ml in range(3)) { + let ttile = ml == 0 ? 256 : (ml == 1 ? 128 : 32) + let wtiles = (d + 127) / 128 + let wgs0 = ((cnt0 + ttile - 1) / ttile) * wtiles + let wgs1 = ((cnt1 + ttile - 1) / ttile) * wtiles + let wgs = wgs0 + wgs1 + var sched : array + sched |> resize(2 * 4 + wgs) + sched[0] = 0u + sched[1] = 0u + sched[2] = uint(cnt0) + sched[3] = 0u + sched[4] = uint(d * nsb) + sched[5] = uint(cnt0) + sched[6] = uint(cnt1) + sched[7] = uint(wgs0) + for (w in range(wgs)) { + sched[8 + w] = w < wgs0 ? 0u : 1u + } + unsafe { + upload_region_at(wqd, 0l, addr(wqh[0]), wq_bytes) + upload_region_at(wsd, 0l, addr(wsuh[0]), ws_bytes) + upload_region_at(xfd, 0l, addr(xfh[0]), xf_bytes) + let sc_bytes = int64(2 * 4 + wgs) * 4l + let scd = make_device_buf(sc_bytes) + upload_region_at(scd, 0l, addr(sched[0]), sc_bytes) + var sc = (ml == 0 + ? set_kq_batch_iq4xs_cm2l_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)) + : (ml == 1 + ? set_kq_batch_iq4xs_cm2m_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)) + : set_kq_batch_iq4xs_cm2s_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)))) + var raw2 = alloc_cmd() + let begin = VkCommandBufferBeginInfo() + vk_check(vkBeginCommandBuffer(raw2, begin), null) + var h2 : VkHaz + var pc = BatchArgs(n = uint(n), d = uint(d), map_off = uint(2 * 4)) + if (ml == 0) { + enc_kq_batch_iq4xs_cm2l_cls(raw2, h2, sc, pc, int64(wgs)) + } elif (ml == 1) { + enc_kq_batch_iq4xs_cm2m_cls(raw2, h2, sc, pc, int64(wgs)) + } else { + enc_kq_batch_iq4xs_cm2s_cls(raw2, h2, sc, pc, int64(wgs)) + } + vhz_dep(raw2, h2, 16u, 0u, true) + cmd_copy_whole(raw2, yd2, host.buf, y_bytes) + vk_check(vkEndCommandBuffer(raw2), null) + submit_wait(raw2) + memcpy(addr(y_cls[0]), host.mapped, y_bytes) + } + iq4xsf16_gemm_oracle(wqh, wsuh, xfh, sched, 2, n, d, y_ref) + let bad = mismatch_bars(y_cls, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) + to_log(LOG_INFO, "cm2 iq4xs {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile batch: {bad} of {rows * d} off the oracle\n") + t |> success(bad == 0, "cm2 iq4xs {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile matches the CPU oracle ({bad} of {rows * d} off)") + var y_poison := y_cls // the bar's control: one element pushed past both bars must red + y_poison[0] += 1.0 + 2.0 * max_abs(y_ref) + t |> success(mismatch_bars(y_poison, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) > 0, "the iq4xs bar reds a poisoned element") + delete y_poison + delete sched + } + delete y_ref + delete wqh + delete wsuh + delete xfh + delete y_cls + } else { + t |> skip("dasVulkan not present") + } + } +} + +[test] +def test_vkd_k3_cm2_batch(t0 : T?) { + t0 |> run("cm2 Q3_K l/m/s tiles == the CPU oracle (2-bit + hmask compose decode-in-load)") <| @(t : T?) { + static_if (typeinfo builtin_module_exists(vulkan)) { + if (!ensure_q8_batch_cls()) { // engage the device without touching coopmat rails + t |> skip("no Vulkan device") + return + } + if (!(g_gpu.coopmat_mode == 4 && g_gpu.has_coopmat2)) { + t |> skip("the cm2 k3 tiles serve only in mode 4 on an NV_coopmat2 device") + return + } + verify(ensure_kq_batch_k3_cm2l_cls() && ensure_kq_batch_k3_cm2m_cls() && ensure_kq_batch_k3_cm2s_cls(), "k3 cm2 class rails must engage in cm2 mode") + let n = 768 // one 512-unrolled pass + 4 tail steps crossing superblock bounds + let nsb = n / 256 + let d = 160 // one full 128-tile + a 32 edge + let cnt0 = 300 + let cnt1 = 260 + let rows = cnt0 + cnt1 + let totsb = 2 * d * nsb + var wqh : array + var wsuh : array + var xfh : array + wqh |> resize(totsb * 24) + wsuh |> resize(totsb * 5) + xfh |> resize((rows + 32) * n / 2) // + the s tile's 32-row read slack + for (i in range(totsb * 24)) { + wqh[i] = hash_word(uint(i) + 29u) // arbitrary quant bytes + } + for (sb in range(totsb)) { + for (wi in range(4)) { + wsuh[sb * 5 + wi] = hash_word(uint(sb * 4 + wi) + 811u) // signed sub-scale bytes + } + wsuh[sb * 5 + 4] = packHalf2x16(float2(0.0002 * float(1 + sb % 7), 0.0)) // cool d + } + for (i in range((rows + 32) * n / 2)) { + xfh[i] = ws_word(uint(i) * 3u + 5u) // f16 activation pairs + } + let wq_bytes = int64(totsb) * 96l + let ws_bytes = int64(length(wsuh)) * 4l + let xf_bytes = int64((rows + 32) * n) * 2l + let y_bytes = int64(rows * d) * 4l + let wqd = make_device_buf(wq_bytes) + let wsd = make_device_buf(ws_bytes) + let xfd = make_device_buf(xf_bytes) + let yd2 = make_device_buf(y_bytes) + var host = make_host_buf(y_bytes, true, [cached = true]) + var y_cls : array + y_cls |> resize(rows * d) + var y_ref : array + y_ref |> resize(rows * d) + for (ml in range(3)) { + let ttile = ml == 0 ? 256 : (ml == 1 ? 128 : 32) + let wtiles = (d + 127) / 128 + let wgs0 = ((cnt0 + ttile - 1) / ttile) * wtiles + let wgs1 = ((cnt1 + ttile - 1) / ttile) * wtiles + let wgs = wgs0 + wgs1 + var sched : array + sched |> resize(2 * 4 + wgs) + sched[0] = 0u + sched[1] = 0u + sched[2] = uint(cnt0) + sched[3] = 0u + sched[4] = uint(d * nsb) + sched[5] = uint(cnt0) + sched[6] = uint(cnt1) + sched[7] = uint(wgs0) + for (w in range(wgs)) { + sched[8 + w] = w < wgs0 ? 0u : 1u + } + unsafe { + upload_region_at(wqd, 0l, addr(wqh[0]), wq_bytes) + upload_region_at(wsd, 0l, addr(wsuh[0]), ws_bytes) + upload_region_at(xfd, 0l, addr(xfh[0]), xf_bytes) + let sc_bytes = int64(2 * 4 + wgs) * 4l + let scd = make_device_buf(sc_bytes) + upload_region_at(scd, 0l, addr(sched[0]), sc_bytes) + var sc = (ml == 0 + ? set_kq_batch_k3_cm2l_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)) + : (ml == 1 + ? set_kq_batch_k3_cm2m_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)) + : set_kq_batch_k3_cm2s_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)))) + var raw2 = alloc_cmd() + let begin = VkCommandBufferBeginInfo() + vk_check(vkBeginCommandBuffer(raw2, begin), null) + var h2 : VkHaz + var pc = BatchArgs(n = uint(n), d = uint(d), map_off = uint(2 * 4)) + if (ml == 0) { + enc_kq_batch_k3_cm2l_cls(raw2, h2, sc, pc, int64(wgs)) + } elif (ml == 1) { + enc_kq_batch_k3_cm2m_cls(raw2, h2, sc, pc, int64(wgs)) + } else { + enc_kq_batch_k3_cm2s_cls(raw2, h2, sc, pc, int64(wgs)) + } + vhz_dep(raw2, h2, 16u, 0u, true) + cmd_copy_whole(raw2, yd2, host.buf, y_bytes) + vk_check(vkEndCommandBuffer(raw2), null) + submit_wait(raw2) + memcpy(addr(y_cls[0]), host.mapped, y_bytes) + } + k3f16_gemm_oracle(wqh, wsuh, xfh, sched, 2, n, d, y_ref) + let bad = mismatch_bars(y_cls, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) + to_log(LOG_INFO, "cm2 k3 {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile batch: {bad} of {rows * d} off the oracle\n") + t |> success(bad == 0, "cm2 k3 {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile matches the CPU oracle ({bad} of {rows * d} off)") + var y_poison := y_cls // the bar's control: one element pushed past both bars must red + y_poison[0] += 1.0 + 2.0 * max_abs(y_ref) + t |> success(mismatch_bars(y_poison, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) > 0, "the k3 bar reds a poisoned element") + delete y_poison + delete sched + } + delete y_ref + delete wqh + delete wsuh + delete xfh + delete y_cls + } else { + t |> skip("dasVulkan not present") + } + } +} + +[test] +def test_vkd_iq3s_cm2_batch(t0 : T?) { + t0 |> run("cm2 IQ3_S l/m/s tiles == the CPU oracle (workgroup-staged grid decode-in-load)") <| @(t : T?) { + static_if (typeinfo builtin_module_exists(vulkan)) { + if (!ensure_q8_batch_cls()) { // engage the device without touching coopmat rails + t |> skip("no Vulkan device") + return + } + if (!(g_gpu.coopmat_mode == 4 && g_gpu.has_coopmat2)) { + t |> skip("the cm2 iq3s tiles serve only in mode 4 on an NV_coopmat2 device") + return + } + verify(ensure_kq_batch_iq3s_cm2l_cls() && ensure_kq_batch_iq3s_cm2m_cls() && ensure_kq_batch_iq3s_cm2s_cls(), "iq3s cm2 class rails must engage in cm2 mode") + let n = 768 // one 512-unrolled pass + 4 tail steps crossing superblock bounds + let nsb = n / 256 + let d = 160 // one full 128-tile + a 32 edge + let cnt0 = 300 + let cnt1 = 260 + let rows = cnt0 + cnt1 + let totsb = 2 * d * nsb + var wqh : array + var wsuh : array + var xfh : array + wqh |> resize(totsb * 26) + wsuh |> resize(totsb * 5) + xfh |> resize((rows + 32) * n / 2) // + the s tile's 32-row read slack + for (i in range(totsb * 26)) { + wqh[i] = hash_word(uint(i) + 29u) // arbitrary quant bytes + } + for (sb in range(totsb)) { + wsuh[sb * 5] = packHalf2x16(float2(0.0002 * float(1 + sb % 7), 0.0)) // cool d + wsuh[sb * 5 + 1] = hash_word(uint(sb * 2) + 613u) // signed sub-scale bytes + wsuh[sb * 5 + 2] = hash_word(uint(sb * 2 + 1) + 613u) + wsuh[sb * 5 + 3] = 0u + wsuh[sb * 5 + 4] = 0u + } + for (i in range((rows + 32) * n / 2)) { + xfh[i] = ws_word(uint(i) * 3u + 5u) // f16 activation pairs + } + let wq_bytes = int64(totsb) * 104l + let ws_bytes = int64(length(wsuh)) * 4l + let xf_bytes = int64((rows + 32) * n) * 2l + let y_bytes = int64(rows * d) * 4l + let wqd = make_device_buf(wq_bytes) + let wsd = make_device_buf(ws_bytes) + let xfd = make_device_buf(xf_bytes) + let yd2 = make_device_buf(y_bytes) + var host = make_host_buf(y_bytes, true, [cached = true]) + var y_cls : array + y_cls |> resize(rows * d) + var y_ref : array + y_ref |> resize(rows * d) + for (ml in range(3)) { + let ttile = ml == 0 ? 256 : (ml == 1 ? 128 : 32) + let wtiles = (d + 127) / 128 + let wgs0 = ((cnt0 + ttile - 1) / ttile) * wtiles + let wgs1 = ((cnt1 + ttile - 1) / ttile) * wtiles + let wgs = wgs0 + wgs1 + var sched : array + sched |> resize(2 * 4 + wgs) + sched[0] = 0u + sched[1] = 0u + sched[2] = uint(cnt0) + sched[3] = 0u + sched[4] = uint(d * nsb) + sched[5] = uint(cnt0) + sched[6] = uint(cnt1) + sched[7] = uint(wgs0) + for (w in range(wgs)) { + sched[8 + w] = w < wgs0 ? 0u : 1u + } + unsafe { + upload_region_at(wqd, 0l, addr(wqh[0]), wq_bytes) + upload_region_at(wsd, 0l, addr(wsuh[0]), ws_bytes) + upload_region_at(xfd, 0l, addr(xfh[0]), xf_bytes) + let sc_bytes = int64(2 * 4 + wgs) * 4l + let scd = make_device_buf(sc_bytes) + upload_region_at(scd, 0l, addr(sched[0]), sc_bytes) + var sc = (ml == 0 + ? set_kq_batch_iq3s_cm2l_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)) + : (ml == 1 + ? set_kq_batch_iq3s_cm2m_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)) + : set_kq_batch_iq3s_cm2s_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)))) + var raw2 = alloc_cmd() + let begin = VkCommandBufferBeginInfo() + vk_check(vkBeginCommandBuffer(raw2, begin), null) + var h2 : VkHaz + var pc = BatchArgs(n = uint(n), d = uint(d), map_off = uint(2 * 4)) + if (ml == 0) { + enc_kq_batch_iq3s_cm2l_cls(raw2, h2, sc, pc, int64(wgs)) + } elif (ml == 1) { + enc_kq_batch_iq3s_cm2m_cls(raw2, h2, sc, pc, int64(wgs)) + } else { + enc_kq_batch_iq3s_cm2s_cls(raw2, h2, sc, pc, int64(wgs)) + } + vhz_dep(raw2, h2, 16u, 0u, true) + cmd_copy_whole(raw2, yd2, host.buf, y_bytes) + vk_check(vkEndCommandBuffer(raw2), null) + submit_wait(raw2) + memcpy(addr(y_cls[0]), host.mapped, y_bytes) + } + iq3sf16_gemm_oracle(wqh, wsuh, xfh, sched, 2, n, d, y_ref) + let bad = mismatch_bars(y_cls, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) + to_log(LOG_INFO, "cm2 iq3s {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile batch: {bad} of {rows * d} off the oracle\n") + t |> success(bad == 0, "cm2 iq3s {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile matches the CPU oracle ({bad} of {rows * d} off)") + var y_poison := y_cls // the bar's control: one element pushed past both bars must red + y_poison[0] += 1.0 + 2.0 * max_abs(y_ref) + t |> success(mismatch_bars(y_poison, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) > 0, "the iq3s bar reds a poisoned element") + delete y_poison + delete sched + } + delete y_ref + delete wqh + delete wsuh + delete xfh + delete y_cls + } else { + t |> skip("dasVulkan not present") + } + } +} + +[test] +def test_vkd_iq2s_cm2_batch(t0 : T?) { + t0 |> run("cm2 IQ2_S l/m/s tiles == the CPU oracle (workgroup-staged u64 grid decode-in-load)") <| @(t : T?) { + static_if (typeinfo builtin_module_exists(vulkan)) { + if (!ensure_q8_batch_cls()) { // engage the device without touching coopmat rails + t |> skip("no Vulkan device") + return + } + if (!(g_gpu.coopmat_mode == 4 && g_gpu.has_coopmat2)) { + t |> skip("the cm2 iq2s tiles serve only in mode 4 on an NV_coopmat2 device") + return + } + verify(ensure_kq_batch_iq2s_cm2l_cls() && ensure_kq_batch_iq2s_cm2m_cls() && ensure_kq_batch_iq2s_cm2s_cls(), "iq2s cm2 class rails must engage in cm2 mode") + let n = 768 // one 512-unrolled pass + 4 tail steps crossing superblock bounds + let nsb = n / 256 + let d = 160 // one full 128-tile + a 32 edge + let cnt0 = 300 + let cnt1 = 260 + let rows = cnt0 + cnt1 + let totsb = 2 * d * nsb + var wqh : array + var wsuh : array + var xfh : array + wqh |> resize(totsb * 18) + wsuh |> resize(totsb * 5) + xfh |> resize((rows + 32) * n / 2) // + the s tile's 32-row read slack + for (i in range(totsb * 18)) { + wqh[i] = hash_word(uint(i) + 29u) // arbitrary quant bytes + } + for (sb in range(totsb)) { + wsuh[sb * 5] = packHalf2x16(float2(0.0002 * float(1 + sb % 7), 0.0)) // cool d8 + wsuh[sb * 5 + 1] = hash_word(uint(sb * 4) + 613u) // 16 unsigned strip bytes + wsuh[sb * 5 + 2] = hash_word(uint(sb * 4 + 1) + 613u) + wsuh[sb * 5 + 3] = hash_word(uint(sb * 4 + 2) + 613u) + wsuh[sb * 5 + 4] = hash_word(uint(sb * 4 + 3) + 613u) + } + for (i in range((rows + 32) * n / 2)) { + xfh[i] = ws_word(uint(i) * 3u + 5u) // f16 activation pairs + } + let wq_bytes = int64(totsb) * 72l + let ws_bytes = int64(length(wsuh)) * 4l + let xf_bytes = int64((rows + 32) * n) * 2l + let y_bytes = int64(rows * d) * 4l + let wqd = make_device_buf(wq_bytes) + let wsd = make_device_buf(ws_bytes) + let xfd = make_device_buf(xf_bytes) + let yd2 = make_device_buf(y_bytes) + var host = make_host_buf(y_bytes, true, [cached = true]) + var y_cls : array + y_cls |> resize(rows * d) + var y_ref : array + y_ref |> resize(rows * d) + for (ml in range(3)) { + let ttile = ml == 0 ? 256 : (ml == 1 ? 128 : 32) + let wtiles = (d + 127) / 128 + let wgs0 = ((cnt0 + ttile - 1) / ttile) * wtiles + let wgs1 = ((cnt1 + ttile - 1) / ttile) * wtiles + let wgs = wgs0 + wgs1 + var sched : array + sched |> resize(2 * 4 + wgs) + sched[0] = 0u + sched[1] = 0u + sched[2] = uint(cnt0) + sched[3] = 0u + sched[4] = uint(d * nsb) + sched[5] = uint(cnt0) + sched[6] = uint(cnt1) + sched[7] = uint(wgs0) + for (w in range(wgs)) { + sched[8 + w] = w < wgs0 ? 0u : 1u + } + unsafe { + upload_region_at(wqd, 0l, addr(wqh[0]), wq_bytes) + upload_region_at(wsd, 0l, addr(wsuh[0]), ws_bytes) + upload_region_at(xfd, 0l, addr(xfh[0]), xf_bytes) + let sc_bytes = int64(2 * 4 + wgs) * 4l + let scd = make_device_buf(sc_bytes) + upload_region_at(scd, 0l, addr(sched[0]), sc_bytes) + var sc = (ml == 0 + ? set_kq_batch_iq2s_cm2l_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)) + : (ml == 1 + ? set_kq_batch_iq2s_cm2m_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)) + : set_kq_batch_iq2s_cm2s_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)))) + var raw2 = alloc_cmd() + let begin = VkCommandBufferBeginInfo() + vk_check(vkBeginCommandBuffer(raw2, begin), null) + var h2 : VkHaz + var pc = BatchArgs(n = uint(n), d = uint(d), map_off = uint(2 * 4)) + if (ml == 0) { + enc_kq_batch_iq2s_cm2l_cls(raw2, h2, sc, pc, int64(wgs)) + } elif (ml == 1) { + enc_kq_batch_iq2s_cm2m_cls(raw2, h2, sc, pc, int64(wgs)) + } else { + enc_kq_batch_iq2s_cm2s_cls(raw2, h2, sc, pc, int64(wgs)) + } + vhz_dep(raw2, h2, 16u, 0u, true) + cmd_copy_whole(raw2, yd2, host.buf, y_bytes) + vk_check(vkEndCommandBuffer(raw2), null) + submit_wait(raw2) + memcpy(addr(y_cls[0]), host.mapped, y_bytes) + } + iq2sf16_gemm_oracle(wqh, wsuh, xfh, sched, 2, n, d, y_ref) + let bad = mismatch_bars(y_cls, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) + to_log(LOG_INFO, "cm2 iq2s {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile batch: {bad} of {rows * d} off the oracle\n") + t |> success(bad == 0, "cm2 iq2s {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile matches the CPU oracle ({bad} of {rows * d} off)") + var y_poison := y_cls // the bar's control: one element pushed past both bars must red + y_poison[0] += 1.0 + 2.0 * max_abs(y_ref) + t |> success(mismatch_bars(y_poison, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) > 0, "the iq2s bar reds a poisoned element") + delete y_poison + delete sched + } + delete y_ref + delete wqh + delete wsuh + delete xfh + delete y_cls + } else { + t |> skip("dasVulkan not present") + } + } +} + +[test] +def test_vkd_iq2xs_cm2_batch(t0 : T?) { + t0 |> run("cm2 IQ2_XS l/m/s tiles == the CPU oracle (u64 grid + ksigns-by-parity decode-in-load)") <| @(t : T?) { + static_if (typeinfo builtin_module_exists(vulkan)) { + if (!ensure_q8_batch_cls()) { // engage the device without touching coopmat rails + t |> skip("no Vulkan device") + return + } + if (!(g_gpu.coopmat_mode == 4 && g_gpu.has_coopmat2)) { + t |> skip("the cm2 iq2xs tiles serve only in mode 4 on an NV_coopmat2 device") + return + } + verify(ensure_kq_batch_iq2xs_cm2l_cls() && ensure_kq_batch_iq2xs_cm2m_cls() && ensure_kq_batch_iq2xs_cm2s_cls(), "iq2xs cm2 class rails must engage in cm2 mode") + let n = 768 // one 512-unrolled pass + 4 tail steps crossing superblock bounds + let nsb = n / 256 + let d = 160 // one full 128-tile + a 32 edge + let cnt0 = 300 + let cnt1 = 260 + let rows = cnt0 + cnt1 + let totsb = 2 * d * nsb + var wqh : array + var wsuh : array + var xfh : array + wqh |> resize(totsb * 16) + wsuh |> resize(totsb * 5) + xfh |> resize((rows + 32) * n / 2) // + the s tile's 32-row read slack + for (i in range(totsb * 16)) { + wqh[i] = hash_word(uint(i) + 29u) // arbitrary quant bytes + } + for (sb in range(totsb)) { + wsuh[sb * 5] = packHalf2x16(float2(0.0002 * float(1 + sb % 7), 0.0)) // cool d8 + wsuh[sb * 5 + 1] = hash_word(uint(sb * 4) + 613u) // 16 unsigned strip bytes + wsuh[sb * 5 + 2] = hash_word(uint(sb * 4 + 1) + 613u) + wsuh[sb * 5 + 3] = hash_word(uint(sb * 4 + 2) + 613u) + wsuh[sb * 5 + 4] = hash_word(uint(sb * 4 + 3) + 613u) + } + for (i in range((rows + 32) * n / 2)) { + xfh[i] = ws_word(uint(i) * 3u + 5u) // f16 activation pairs + } + let wq_bytes = int64(totsb) * 64l + let ws_bytes = int64(length(wsuh)) * 4l + let xf_bytes = int64((rows + 32) * n) * 2l + let y_bytes = int64(rows * d) * 4l + let wqd = make_device_buf(wq_bytes) + let wsd = make_device_buf(ws_bytes) + let xfd = make_device_buf(xf_bytes) + let yd2 = make_device_buf(y_bytes) + var host = make_host_buf(y_bytes, true, [cached = true]) + var y_cls : array + y_cls |> resize(rows * d) + var y_ref : array + y_ref |> resize(rows * d) + for (ml in range(3)) { + let ttile = ml == 0 ? 256 : (ml == 1 ? 128 : 32) + let wtiles = (d + 127) / 128 + let wgs0 = ((cnt0 + ttile - 1) / ttile) * wtiles + let wgs1 = ((cnt1 + ttile - 1) / ttile) * wtiles + let wgs = wgs0 + wgs1 + var sched : array + sched |> resize(2 * 4 + wgs) + sched[0] = 0u + sched[1] = 0u + sched[2] = uint(cnt0) + sched[3] = 0u + sched[4] = uint(d * nsb) + sched[5] = uint(cnt0) + sched[6] = uint(cnt1) + sched[7] = uint(wgs0) + for (w in range(wgs)) { + sched[8 + w] = w < wgs0 ? 0u : 1u + } + unsafe { + upload_region_at(wqd, 0l, addr(wqh[0]), wq_bytes) + upload_region_at(wsd, 0l, addr(wsuh[0]), ws_bytes) + upload_region_at(xfd, 0l, addr(xfh[0]), xf_bytes) + let sc_bytes = int64(2 * 4 + wgs) * 4l + let scd = make_device_buf(sc_bytes) + upload_region_at(scd, 0l, addr(sched[0]), sc_bytes) + var sc = (ml == 0 + ? set_kq_batch_iq2xs_cm2l_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)) + : (ml == 1 + ? set_kq_batch_iq2xs_cm2m_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)) + : set_kq_batch_iq2xs_cm2s_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)))) + var raw2 = alloc_cmd() + let begin = VkCommandBufferBeginInfo() + vk_check(vkBeginCommandBuffer(raw2, begin), null) + var h2 : VkHaz + var pc = BatchArgs(n = uint(n), d = uint(d), map_off = uint(2 * 4)) + if (ml == 0) { + enc_kq_batch_iq2xs_cm2l_cls(raw2, h2, sc, pc, int64(wgs)) + } elif (ml == 1) { + enc_kq_batch_iq2xs_cm2m_cls(raw2, h2, sc, pc, int64(wgs)) + } else { + enc_kq_batch_iq2xs_cm2s_cls(raw2, h2, sc, pc, int64(wgs)) + } + vhz_dep(raw2, h2, 16u, 0u, true) + cmd_copy_whole(raw2, yd2, host.buf, y_bytes) + vk_check(vkEndCommandBuffer(raw2), null) + submit_wait(raw2) + memcpy(addr(y_cls[0]), host.mapped, y_bytes) + } + iq2xsf16_gemm_oracle(wqh, wsuh, xfh, sched, 2, n, d, y_ref) + let bad = mismatch_bars(y_cls, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) + to_log(LOG_INFO, "cm2 iq2xs {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile batch: {bad} of {rows * d} off the oracle\n") + t |> success(bad == 0, "cm2 iq2xs {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile matches the CPU oracle ({bad} of {rows * d} off)") + var y_poison := y_cls // the bar's control: one element pushed past both bars must red + y_poison[0] += 1.0 + 2.0 * max_abs(y_ref) + t |> success(mismatch_bars(y_poison, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) > 0, "the iq2xs bar reds a poisoned element") + delete y_poison + delete sched + } + delete y_ref + delete wqh + delete wsuh + delete xfh + delete y_cls + } else { + t |> skip("dasVulkan not present") + } + } +} + +[test] +def test_vkd_iq2xxs_cm2_batch(t0 : T?) { + t0 |> run("cm2 IQ2_XXS l/m/s tiles == the CPU oracle (byte-grid + aux32 ksigns decode-in-load)") <| @(t : T?) { + static_if (typeinfo builtin_module_exists(vulkan)) { + if (!ensure_q8_batch_cls()) { // engage the device without touching coopmat rails + t |> skip("no Vulkan device") + return + } + if (!(g_gpu.coopmat_mode == 4 && g_gpu.has_coopmat2)) { + t |> skip("the cm2 iq2xxs tiles serve only in mode 4 on an NV_coopmat2 device") + return + } + verify(ensure_kq_batch_iq2xxs_cm2l_cls() && ensure_kq_batch_iq2xxs_cm2m_cls() && ensure_kq_batch_iq2xxs_cm2s_cls(), "iq2xxs cm2 class rails must engage in cm2 mode") + let n = 768 // one 512-unrolled pass + 4 tail steps crossing superblock bounds + let nsb = n / 256 + let d = 160 // one full 128-tile + a 32 edge + let cnt0 = 300 + let cnt1 = 260 + let rows = cnt0 + cnt1 + let totsb = 2 * d * nsb + var wqh : array + var wsuh : array + var xfh : array + wqh |> resize(totsb * 16) + wsuh |> resize(totsb * 5) + xfh |> resize((rows + 32) * n / 2) // + the s tile's 32-row read slack + for (i in range(totsb * 16)) { + wqh[i] = hash_word(uint(i) + 29u) // arbitrary quant bytes + } + for (sb in range(totsb)) { + wsuh[sb * 5] = packHalf2x16(float2(0.0002 * float(1 + sb % 7), 0.0)) // cool d8 + wsuh[sb * 5 + 1] = hash_word(uint(sb * 4) + 613u) // 16 unsigned strip bytes + wsuh[sb * 5 + 2] = hash_word(uint(sb * 4 + 1) + 613u) + wsuh[sb * 5 + 3] = hash_word(uint(sb * 4 + 2) + 613u) + wsuh[sb * 5 + 4] = hash_word(uint(sb * 4 + 3) + 613u) + } + for (i in range((rows + 32) * n / 2)) { + xfh[i] = ws_word(uint(i) * 3u + 5u) // f16 activation pairs + } + let wq_bytes = int64(totsb) * 64l + let ws_bytes = int64(length(wsuh)) * 4l + let xf_bytes = int64((rows + 32) * n) * 2l + let y_bytes = int64(rows * d) * 4l + let wqd = make_device_buf(wq_bytes) + let wsd = make_device_buf(ws_bytes) + let xfd = make_device_buf(xf_bytes) + let yd2 = make_device_buf(y_bytes) + var host = make_host_buf(y_bytes, true, [cached = true]) + var y_cls : array + y_cls |> resize(rows * d) + var y_ref : array + y_ref |> resize(rows * d) + for (ml in range(3)) { + let ttile = ml == 0 ? 256 : (ml == 1 ? 128 : 32) + let wtiles = (d + 127) / 128 + let wgs0 = ((cnt0 + ttile - 1) / ttile) * wtiles + let wgs1 = ((cnt1 + ttile - 1) / ttile) * wtiles + let wgs = wgs0 + wgs1 + var sched : array + sched |> resize(2 * 4 + wgs) + sched[0] = 0u + sched[1] = 0u + sched[2] = uint(cnt0) + sched[3] = 0u + sched[4] = uint(d * nsb) + sched[5] = uint(cnt0) + sched[6] = uint(cnt1) + sched[7] = uint(wgs0) + for (w in range(wgs)) { + sched[8 + w] = w < wgs0 ? 0u : 1u + } + unsafe { + upload_region_at(wqd, 0l, addr(wqh[0]), wq_bytes) + upload_region_at(wsd, 0l, addr(wsuh[0]), ws_bytes) + upload_region_at(xfd, 0l, addr(xfh[0]), xf_bytes) + let sc_bytes = int64(2 * 4 + wgs) * 4l + let scd = make_device_buf(sc_bytes) + upload_region_at(scd, 0l, addr(sched[0]), sc_bytes) + var sc = (ml == 0 + ? set_kq_batch_iq2xxs_cm2l_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)) + : (ml == 1 + ? set_kq_batch_iq2xxs_cm2m_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)) + : set_kq_batch_iq2xxs_cm2s_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)))) + var raw2 = alloc_cmd() + let begin = VkCommandBufferBeginInfo() + vk_check(vkBeginCommandBuffer(raw2, begin), null) + var h2 : VkHaz + var pc = BatchArgs(n = uint(n), d = uint(d), map_off = uint(2 * 4)) + if (ml == 0) { + enc_kq_batch_iq2xxs_cm2l_cls(raw2, h2, sc, pc, int64(wgs)) + } elif (ml == 1) { + enc_kq_batch_iq2xxs_cm2m_cls(raw2, h2, sc, pc, int64(wgs)) + } else { + enc_kq_batch_iq2xxs_cm2s_cls(raw2, h2, sc, pc, int64(wgs)) + } + vhz_dep(raw2, h2, 16u, 0u, true) + cmd_copy_whole(raw2, yd2, host.buf, y_bytes) + vk_check(vkEndCommandBuffer(raw2), null) + submit_wait(raw2) + memcpy(addr(y_cls[0]), host.mapped, y_bytes) + } + iq2xxsf16_gemm_oracle(wqh, wsuh, xfh, sched, 2, n, d, y_ref) + let bad = mismatch_bars(y_cls, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) + to_log(LOG_INFO, "cm2 iq2xxs {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile batch: {bad} of {rows * d} off the oracle\n") + t |> success(bad == 0, "cm2 iq2xxs {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile matches the CPU oracle ({bad} of {rows * d} off)") + var y_poison := y_cls // the bar's control: one element pushed past both bars must red + y_poison[0] += 1.0 + 2.0 * max_abs(y_ref) + t |> success(mismatch_bars(y_poison, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) > 0, "the iq2xxs bar reds a poisoned element") + delete y_poison + delete sched + } + delete y_ref + delete wqh + delete wsuh + delete xfh + delete y_cls + } else { + t |> skip("dasVulkan not present") + } + } +} + +[test] +def test_vkd_iq3xxs_cm2_batch(t0 : T?) { + t0 |> run("cm2 IQ3_XXS l/m/s tiles == the CPU oracle (halved grid + parity signs decode-in-load)") <| @(t : T?) { + static_if (typeinfo builtin_module_exists(vulkan)) { + if (!ensure_q8_batch_cls()) { // engage the device without touching coopmat rails + t |> skip("no Vulkan device") + return + } + if (!(g_gpu.coopmat_mode == 4 && g_gpu.has_coopmat2)) { + t |> skip("the cm2 iq3xxs tiles serve only in mode 4 on an NV_coopmat2 device") + return + } + verify(ensure_kq_batch_iq3xxs_cm2l_cls() && ensure_kq_batch_iq3xxs_cm2m_cls() && ensure_kq_batch_iq3xxs_cm2s_cls(), "iq3xxs cm2 class rails must engage in cm2 mode") + let n = 768 // one 512-unrolled pass + 4 tail steps crossing superblock bounds + let nsb = n / 256 + let d = 160 // one full 128-tile + a 32 edge + let cnt0 = 300 + let cnt1 = 260 + let rows = cnt0 + cnt1 + let totsb = 2 * d * nsb + var wqh : array + var wsuh : array + var xfh : array + wqh |> resize(totsb * 24) + wsuh |> resize(totsb * 5) + xfh |> resize((rows + 32) * n / 2) // + the s tile's 32-row read slack + for (i in range(totsb * 24)) { + wqh[i] = hash_word(uint(i) + 29u) // arbitrary quant bytes + } + for (sb in range(totsb)) { + wsuh[sb * 5] = packHalf2x16(float2(0.0002 * float(1 + sb % 7), 0.0)) // cool d + wsuh[sb * 5 + 1] = hash_word(uint(sb * 2) + 613u) // signed sub-scale bytes + wsuh[sb * 5 + 2] = hash_word(uint(sb * 2 + 1) + 613u) + wsuh[sb * 5 + 3] = 0u + wsuh[sb * 5 + 4] = 0u + } + for (i in range((rows + 32) * n / 2)) { + xfh[i] = ws_word(uint(i) * 3u + 5u) // f16 activation pairs + } + let wq_bytes = int64(totsb) * 96l + let ws_bytes = int64(length(wsuh)) * 4l + let xf_bytes = int64((rows + 32) * n) * 2l + let y_bytes = int64(rows * d) * 4l + let wqd = make_device_buf(wq_bytes) + let wsd = make_device_buf(ws_bytes) + let xfd = make_device_buf(xf_bytes) + let yd2 = make_device_buf(y_bytes) + var host = make_host_buf(y_bytes, true, [cached = true]) + var y_cls : array + y_cls |> resize(rows * d) + var y_ref : array + y_ref |> resize(rows * d) + for (ml in range(3)) { + let ttile = ml == 0 ? 256 : (ml == 1 ? 128 : 32) + let wtiles = (d + 127) / 128 + let wgs0 = ((cnt0 + ttile - 1) / ttile) * wtiles + let wgs1 = ((cnt1 + ttile - 1) / ttile) * wtiles + let wgs = wgs0 + wgs1 + var sched : array + sched |> resize(2 * 4 + wgs) + sched[0] = 0u + sched[1] = 0u + sched[2] = uint(cnt0) + sched[3] = 0u + sched[4] = uint(d * nsb) + sched[5] = uint(cnt0) + sched[6] = uint(cnt1) + sched[7] = uint(wgs0) + for (w in range(wgs)) { + sched[8 + w] = w < wgs0 ? 0u : 1u + } + unsafe { + upload_region_at(wqd, 0l, addr(wqh[0]), wq_bytes) + upload_region_at(wsd, 0l, addr(wsuh[0]), ws_bytes) + upload_region_at(xfd, 0l, addr(xfh[0]), xf_bytes) + let sc_bytes = int64(2 * 4 + wgs) * 4l + let scd = make_device_buf(sc_bytes) + upload_region_at(scd, 0l, addr(sched[0]), sc_bytes) + var sc = (ml == 0 + ? set_kq_batch_iq3xxs_cm2l_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)) + : (ml == 1 + ? set_kq_batch_iq3xxs_cm2m_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)) + : set_kq_batch_iq3xxs_cm2s_cls(fixed_array(wqd, wsd, scd, xfd, yd2), + fixed_array(wq_bytes, ws_bytes, sc_bytes, xf_bytes, y_bytes), + fixed_array(1u, 2u, 0u, 4u, 16u)))) + var raw2 = alloc_cmd() + let begin = VkCommandBufferBeginInfo() + vk_check(vkBeginCommandBuffer(raw2, begin), null) + var h2 : VkHaz + var pc = BatchArgs(n = uint(n), d = uint(d), map_off = uint(2 * 4)) + if (ml == 0) { + enc_kq_batch_iq3xxs_cm2l_cls(raw2, h2, sc, pc, int64(wgs)) + } elif (ml == 1) { + enc_kq_batch_iq3xxs_cm2m_cls(raw2, h2, sc, pc, int64(wgs)) + } else { + enc_kq_batch_iq3xxs_cm2s_cls(raw2, h2, sc, pc, int64(wgs)) + } + vhz_dep(raw2, h2, 16u, 0u, true) + cmd_copy_whole(raw2, yd2, host.buf, y_bytes) + vk_check(vkEndCommandBuffer(raw2), null) + submit_wait(raw2) + memcpy(addr(y_cls[0]), host.mapped, y_bytes) + } + iq3xxsf16_gemm_oracle(wqh, wsuh, xfh, sched, 2, n, d, y_ref) + let bad = mismatch_bars(y_cls, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) + to_log(LOG_INFO, "cm2 iq3xxs {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile batch: {bad} of {rows * d} off the oracle\n") + t |> success(bad == 0, "cm2 iq3xxs {ml == 0 ? "l" : (ml == 1 ? "m" : "s")}-tile matches the CPU oracle ({bad} of {rows * d} off)") + var y_poison := y_cls // the bar's control: one element pushed past both bars must red + y_poison[0] += 1.0 + 2.0 * max_abs(y_ref) + t |> success(mismatch_bars(y_poison, y_ref, 2e-2, 4e-3 * max_abs(y_ref)) > 0, "the iq3xxs bar reds a poisoned element") + delete y_poison + delete sched + } + delete y_ref + delete wqh + delete wsuh + delete xfh + delete y_cls + } else { + t |> skip("dasVulkan not present") + } + } +} + [test] def test_vkd_k6_cm2_batch(t0 : T?) { t0 |> run("cm2 Q6_K l/m tiles == the CPU oracle (6-bit compose decode-in-load)") <| @(t : T?) { diff --git a/modules/dasLLVM/ARCHITECTURE.md b/modules/dasLLVM/ARCHITECTURE.md index 6789e3eac5..1894290013 100644 --- a/modules/dasLLVM/ARCHITECTURE.md +++ b/modules/dasLLVM/ARCHITECTURE.md @@ -106,7 +106,9 @@ The backend's override knobs - the escapes that change what a run compiles, tune beyond its defaults - are: `DAS_TUNE_POLICY` (replaces the declared/injected tune policy), `DAS_TUNE_MODE` (grid/tuner compile modes), `DAS_TUNE_MANIFEST` (pins the sidecar), `DAS_TUNE_NOISE_CV` (recalibrates the tuner noise gate), `DAS_TUNE_NOISE_OVERRIDE` (mints -through a failing gate), `--tune` (forced re-mint), `--jit-obj-cache=0` (forces every split +through a failing gate), `--tune` (forced re-mint), `--tune-only` / `DAS_TUNE_ONLY` (re-mints +only the named families; the policy guard arms it itself for a profile's residue), +`DAS_TUNE_CONTROL` (a supervisor's stop request - tuners abort between families), `--jit-obj-cache=0` (forces every split partition to re-emit, bypassing the obj cache), `DAS_JIT_PROBE_LTO` (split partitions emit bitcode and the link runs lld LTO - a dev probe artifact), `DAS_JIT_X64_FORCE_FEATURES` / `DAS_JIT_ARM64_FORCE_FEATURES` (force CPU features past detection - emission, the cache keys, @@ -125,3 +127,21 @@ Environment knobs load ONCE, at context init, into the `[EnvConfig]` structs `g_ `g_env_tune` (`llvm_env.das`) - a mid-process `setenv` changes nothing the backend reads. In-process overrides therefore go through the tune setters (`tune_set_verbosity`, `tune_set_noise_cv`, ...), which also arm spawned children by exporting the matching variable. + +## 4. Host CPU feature truth on aarch64 {#aarch64-feature-truth} + +An aarch64 host target reads its CPU features from two sources, because neither answers alone. +`LLVMGetHostCPUFeatures` returns an EMPTY string on macOS - there the CPU name is meant to imply +the features - and a part this LLVM cannot name maps to the generic CPU, where SDOT and SMMLA +have no instruction to select and codegen aborts. `cpu_supports` reads the operating system +instead (sysctl / `AT_HWCAP` / `IsProcessorFeaturePresent`), so it answers for silicon LLVM has +never heard of. Both the tier gates (`init_jit_target_flags`) and the target machine's feature +string (`create_default_target_machine`) therefore take the union of the two: an LLVM host-string +hit OR a `cpu_supports` hit (fullfp16 additionally reads darwin-arm64 as always-on - every +Apple Silicon part has it). A cross-compile triple takes neither - only the force env. + +The two ways a feature reaches the target machine's string license different things. A +detection-derived append - `+dotprod` always, `+i8mm` when `cpu_supports` confirms it - is +EXECUTION-safe: the silicon running this process really has the instruction. A +`DAS_JIT_ARM64_FORCE_FEATURES` append is EMISSION-only: it may name silicon this box does not +have, so the artifact is for another machine and executing it here traps. diff --git a/modules/dasLLVM/ENVIRONMENT.md b/modules/dasLLVM/ENVIRONMENT.md index c50bbaa048..ceed549549 100644 --- a/modules/dasLLVM/ENVIRONMENT.md +++ b/modules/dasLLVM/ENVIRONMENT.md @@ -35,8 +35,9 @@ Loaded once at context init into `g_env_tune`; tuner children inherit the enviro | `DAS_TUNE_NOISE_CV` | number | 2 | The noise probe's note threshold in percent (a louder probe stamps the mint noisy; refusal sits at max(10, this)) - a calibration lever. Garbage or <= 0 reads as unset. | | `DAS_TUNE_NOISE_OVERRIDE` | flag | off | Mint through the busy-box refusal (a probe cv past the hard ceiling). The sidecar is stamped noise overridden, so the escape always leaves a mark. Validation never refuses - a winner that does not hold is demoted per kernel. | | `DAS_TUNE_VERBOSITY` | text | normal | What a tune shows: silent, normal, or verbose. Anything unrecognized reads as normal, so a typo never silences a tune. | -| `DAS_TUNE_POLICY` | text | declared by [tune_policy] | Override the missing-scope policy: fallback, warn, error, auto, or restart. The announce line says when the environment shaped the policy. | +| `DAS_TUNE_POLICY` | text | declared by [tune_policy] | Override the missing-scope policy: fallback, reference, warn, error, auto, or restart. The announce line says when the environment shaped the policy. | | `DAS_TUNE_CONTROL` | path | unset | A supervisor's stop channel: while the named file exists, tune_interrupt_requested() is true and tuners abort at the next kernel-family boundary without minting. The watchdog sets it and owns the file's lifetime; the measurement in flight always completes. | +| `DAS_TUNE_ONLY` | text | unset (every family races) | Comma-separated re-mint filter: a tuner races only kernel families whose name contains one of these tokens, and every skipped family's sidecar entry survives the upsert. Set by --tune-only on the application; the tuner children inherit it. | ## Ambient variables dasLLVM reads but does not own diff --git a/modules/dasLLVM/REVIEW.md b/modules/dasLLVM/REVIEW.md index 1f94d4b62e..030f00d7f8 100644 --- a/modules/dasLLVM/REVIEW.md +++ b/modules/dasLLVM/REVIEW.md @@ -7,13 +7,15 @@ `tests/README.md` here). The suite is outside the core `tests/` sweep, so no other lane covers it. -- **A change gated on the host platform - a `get_platform_name()` test - runs the - module-owned suite on that platform.** - -- **A diff whose new behavior sits under a target-triple branch - one it adds or one already - there - states in its PR body the cross-compile (`write_exe`) for that target that exercised - the new behavior.** The suite runs on the host; a target-triple branch is checked only by - the artifact built for that target. +- **A change whose behavior depends on the host - a branch on `get_platform_name()`, + `get_architecture_name()`, or `cpu_supports()` - runs the module-owned suite on a machine + matching that condition.** + +- **A diff whose new behavior runs only under a set target triple - a cross-compile, not the + host - names in its PR body the cross-compile (`write_exe`) for that target that exercised + the behavior.** This holds whether the diff adds that code path or finds it already there; + the suite runs on the host, so a target-triple branch is checked only by the artifact built + for that target. - **A diff that adds work to, or moves work within, what `run_jit` (`daslib/llvm_jit_run.das`) executes - its own body or any callee - also prints an diff --git a/modules/dasLLVM/daslib/llvm_env.das b/modules/dasLLVM/daslib/llvm_env.das index 46f610d7b2..96c2fea95d 100644 --- a/modules/dasLLVM/daslib/llvm_env.das +++ b/modules/dasLLVM/daslib/llvm_env.das @@ -67,12 +67,16 @@ struct public TuneEnv { tune_verbosity : string = "" @clarg_default_doc = "declared by [tune_policy]" - @clarg_doc = "Override the missing-scope policy: fallback, warn, error, auto, or restart. The announce line says when the environment shaped the policy." + @clarg_doc = "Override the missing-scope policy: fallback, reference, warn, error, auto, or restart. The announce line says when the environment shaped the policy." tune_policy : string = "" @clarg_path @clarg_doc = "A supervisor's stop channel: while the named file exists, tune_interrupt_requested() is true and tuners abort at the next kernel-family boundary without minting. The watchdog sets it and owns the file's lifetime; the measurement in flight always completes." tune_control : string = "" + + @clarg_default_doc = "unset (every family races)" + @clarg_doc = "Comma-separated re-mint filter: a tuner races only kernel families whose name contains one of these tokens, and every skipped family's sidecar entry survives the upsert. Set by --tune-only on the application; the tuner children inherit it." + tune_only : string = "" } let public g_env_tune = env_config(type) diff --git a/modules/dasLLVM/daslib/llvm_jit_common.das b/modules/dasLLVM/daslib/llvm_jit_common.das index dd392a1960..5faf21bebc 100644 --- a/modules/dasLLVM/daslib/llvm_jit_common.das +++ b/modules/dasLLVM/daslib/llvm_jit_common.das @@ -444,8 +444,7 @@ var public g_target_x64_amx = false var public g_target_os_linux = false // ARMv8.6 i8mm (SMMLA — 2×2 s8 matrix-multiply-accumulate; Apple M2/A15+, Graviton3+; M1 has -// DotProd only). Host truth is LLVMGetHostCPUFeatures "+i8mm" — populated on Linux aarch64, EMPTY -// on macOS where the CPU name implies features — so there the force env is the rail on both boxes. +// DotProd only). Host truth is LLVMGetHostCPUFeatures "+i8mm" OR cpu_supports (ARCHITECTURE.md#aarch64-feature-truth). var public g_target_arm64_i8mm = false // ARMv8.2 fullfp16 (native half arithmetic — fadd.4h/8h etc.). Detected as i8mm, except @@ -603,6 +602,7 @@ def public init_jit_clopts() { // Pure target truth, no LLVM state touched: safe and required BEFORE init_jit, whose DisableJitVisitor // consults these gates. init_jit re-runs it — pass the SAME host_features or pass and emission diverge. // host_features=false targets a GENERIC CPU: cpuid gates stay off, since leaked F16C emits libcalls. +[arch(at = "../ARCHITECTURE.md#aarch64-feature-truth")] def public init_jit_target_flags(target_triple : string = ""; host_features : bool = true) { g_target_host_features = host_features g_target_is_wasm = target_triple |> starts_with("wasm") @@ -632,14 +632,12 @@ def public init_jit_target_flags(target_triple : string = ""; host_features : bo g_target_os_linux = (empty(target_triple) ? get_platform_name() == "linux" : find(target_triple, "linux") >= 0) - // aarch64 i8mm tier (see the g_target_arm64_i8mm declaration comment): host targets read - // LLVM's host-features string OR the arm force env; cross aarch64 triples the env only. g_target_arm64_i8mm = g_target_is_aarch64 && (arm64_forced_feature("i8mm") - || (empty(target_triple) && host_features && host_llvm_feature("i8mm"))) + || (empty(target_triple) && host_features && (host_llvm_feature("i8mm") || cpu_supports("i8mm")))) // aarch64 fullfp16 (see declaration comment): darwin-arm64 == Apple Silicon == always. // Generic-exe builds stay off — the promote/narrow path is ARMv8.0-legal, fullfp16 is not. g_target_arm64_fullfp16 = g_target_is_aarch64 && (arm64_forced_feature("fullfp16") - || (empty(target_triple) && host_features && (host_llvm_feature("fullfp16") || get_platform_name() == "darwin"))) + || (empty(target_triple) && host_features && (host_llvm_feature("fullfp16") || cpu_supports("fullfp16") || get_platform_name() == "darwin"))) // fp16 operators lower as promote-compute-narrow IR wherever half converts are hardware // (routing truth for isExprOp2/Op1_Func — lives in llvm_boost). aarch64 needs no feature // gate: f16<->f32 converts are ARMv8.0 baseline, so even generic-exe aarch64 stays native. @@ -893,7 +891,7 @@ def public with_target_machine(triple, cpu, features : string; opt_level : uint; LLVMDisposeTargetMachine(targetMachine) } -[macro_function] +[macro_function, arch(at = "../ARCHITECTURE.md#aarch64-feature-truth")] def public create_default_target_machine(opt_level : uint; use_host_cpu : bool) : LLVMTargetMachineRef { LLVMInitializeAllTargetInfos() LLVMInitializeAllTargets() @@ -915,15 +913,13 @@ def public create_default_target_machine(opt_level : uint; use_host_cpu : bool) let cpu_msg = use_host_cpu ? LLVMGetHostCPUName() : "" let features_msg = use_host_cpu ? LLVMGetHostCPUFeatures() : "" - // JIT host artifact on aarch64: force +dotprod in. LLVMGetHostCPUFeatures is empty on macOS - // (the CPU name implies features), so a part an older LLVM cannot name maps to "generic", - // where SDOT has no instruction to select and codegen aborts. Every aarch64 host has dotprod. var targetMachine : LLVMTargetMachineRef if (use_host_cpu && g_target_is_aarch64) { var feats = empty(features_msg) ? "+dotprod" : "{features_msg},+dotprod" - // DAS_JIT_ARM64_FORCE_FEATURES (e.g. "i8mm"): the x64 rail's twin — a pre-i8mm host EMITS - // AND LINKS an artifact for newer silicon, which the same-env run on the target box then - // cache-hits without a linker. Executing forced instructions here SIGILLs; gate execution. + if (cpu_supports("i8mm")) { + feats = "{feats},+i8mm" + } + // a forced feature's artifact cache-hits on the target box without a linker there let forced_arm <- [for (part in split(g_env_jit.jit_arm64_force_features, ",")); "+{part}"; where !empty(part)] if (!empty(forced_arm)) { feats = "{feats},{join(forced_arm, ",")}" diff --git a/modules/dasLLVM/daslib/llvm_jit_run.das b/modules/dasLLVM/daslib/llvm_jit_run.das index 27832e5e6b..ca7780097e 100644 --- a/modules/dasLLVM/daslib/llvm_jit_run.das +++ b/modules/dasLLVM/daslib/llvm_jit_run.das @@ -36,11 +36,11 @@ var LINK_WHOLE_LIB = false // when true, standalone exe links against the whole // invalidates cached DLLs (e.g. edits to llvm_jit.das, llvm_macro.das, llvm_jit_common.das, // runtime helper ABI, default target triple). Cache filenames fold this in, so a bump // makes every previously written DLL miss the cache on the next run and get GC'd. -let LLVM_JIT_CODEGEN_VERSION : uint64 = 0x58ul // [hint(unsafe_division_check)] drops the sdiv/srem guards (0x57: INT_MIN / -1 and % -1 guards on sdiv/srem) +let LLVM_JIT_CODEGEN_VERSION : uint64 = 0x59ul // aarch64 hosts append +i8mm when cpu_supports says so (0x58: [hint(unsafe_division_check)] drops the sdiv/srem guards) // Read by tests-cpp/small/test_jit_emitter_pin.cpp: FNV-1a64 of the emitter sources // (normalized to LF; file list in the test) -let LLVM_JIT_EMITTER_HASH : uint64 = 0xee1f4cdc69b614f5ul +let LLVM_JIT_EMITTER_HASH : uint64 = 0xfcc23a40234a227ul let JIT_FNV_PRIME : uint64 = 1099511628211ul @@ -176,7 +176,7 @@ def private has_generated_kernel(prog : Program?; jit_all_functions : bool) : bo // The environment/config salt shared by the whole-DLL key AND the per-partition obj cache key. // ONE helper on purpose: a salt component folded into the DLL key but not the partition key // would let a config change reuse stale cached partition objects. -def jit_env_salt(opt_level : int; size_level : int; emit_prologue : bool; debug_info : bool; +def jit_env_salt(opt_level : int; size_level : int; emit_prologue : bool; debug_info : bool; // nolint:STYLE037 - one flat fold per emission input; a split would hide which inputs key the cache target_triple : string; split : bool; use_host_cpu : bool) : uint64 { var h = LLVM_JIT_CODEGEN_VERSION // host/binary identity: object format follows the triple (COFF vs ELF on one checkout), and the emitting binary bakes its Context offsets into every global access - neither is in any AST hash @@ -243,6 +243,8 @@ def jit_env_salt(opt_level : int; size_level : int; emit_prologue : bool; debug_ if (!empty(forced_arm)) { h = (h ^ hash(forced_arm)) * JIT_FNV_PRIME } + // aarch64 i8mm is an emission input (the +i8mm append): a warm pre-i8mm cache must not serve an i8mm host + h = (h ^ (cpu_supports("i8mm") ? 64ul : 0ul)) * JIT_FNV_PRIME // Cross-compile artifacts key on their triple — a --jit-target DLL must not cache-hit a host run (foreign-arch dlopen) if (!empty(target_triple)) { h = (h ^ hash(target_triple)) * JIT_FNV_PRIME diff --git a/modules/dasLLVM/daslib/llvm_tune.das b/modules/dasLLVM/daslib/llvm_tune.das index 9bae3b91f8..cd3aef2f45 100644 --- a/modules/dasLLVM/daslib/llvm_tune.das +++ b/modules/dasLLVM/daslib/llvm_tune.das @@ -421,12 +421,13 @@ def private load_sidecar_doc(path : string) : JsonValue? { // version stamp, and two pinned scopes then invalidate each other (the auto policy loops) def private save_sidecar_doc(path : string; var doc : JsonValue?) : bool { var prov = JV((binary = running_binary(), platform = get_platform_name(), - arch = get_architecture_name(), box = tune_box_identity(), written = iso8601_now())) + arch = get_architecture_name(), box = tune_box_identity(), written = iso8601_now(), + features = tune_feature_fingerprint())) let old = doc?["provenance"] if (old != null && old is _object) { for (k, v in keys(old as _object), values(old as _object)) { continue if (k == "binary" || k == "platform" || k == "arch" || k == "box" || k == "written" - || g_prov_notes |> key_exists(k)) + || k == "features" || g_prov_notes |> key_exists(k)) if (v != null && v is _string) { update(prov, k, JV(v as _string)) } elif (v != null && v is _number) { @@ -539,6 +540,41 @@ def tune_sidecar_section_merge(section : string; var entries : table JV(v as _string); where v != null && v is _string} + var prov : table + let old = doc?["provenance"] + if (old != null && old.value is _object) { + for (k, v in keys(old as _object), values(old as _object)) { + // a profile's validity is its version pin + feature class, never a commit: engine_sha stays behind with the box identity + continue if (k == "binary" || k == "box" || k == "written" || k == "origin" || k == "class" || k == "engine_sha") + if (v != null && v is _string) { + prov[k] = JV(v as _string) + } + } + } + prov["origin"] = JV("profile") + prov["class"] = JV(klass) + prov["written"] = JV(iso8601_now()) + var out = JV({ "kernels" => JV(kernels), "provenance" => JV(prov) }) + let ok = fwrite(outPath, write_json(out)) + print(ok ? "llvm_tune: profile '{klass}' exported -> {outPath} ({length(ks as _object)} kernel entries)\n" + : "llvm_tune: profile export FAILED writing {outPath}\n") + delete_json(out) + delete_json(doc) + return ok +} + // ===== the tune history (~/.tune-history) ===== var private g_history_dir = "" @@ -789,6 +825,7 @@ struct private TuneScope { tunerPath : string // resolved against the declaring file; "" = no tuner declared versionKey : string // provenance key the scope pins ("" = unpinned) versionExpected : string + defaultsPath : string // resolved against the declaring file; "" = no shipped profiles } // `version_of = "module/CONST"` on a [tune_scope]: winners are valid only while the sidecar's @@ -880,6 +917,44 @@ def private scope_fnames_joined(scope : TuneScope) : string { return joined } +// every `requires=` a scope's [tune_perm] rows carry, read off the AST for the runtime guard +def private scope_gated_families_joined(scope : TuneScope) : string { + var wanted : table + wanted |> insert(scope.modName) + for (m in split(scope.covers, ";")) { + if (!empty(m)) { + wanted |> insert(m) + } + } + var rows : array + program_for_each_module(compiling_program()) $(mod) { + if (wanted |> key_exists("{mod.name}")) { + for_each_function(mod, "") $(fn) { + var isTune = false + var exprs : array + for (ann in fn.annotations) { + if (ann.annotation.name == "tune") { + isTune = true + } elif (ann.annotation.name == "tune_perm") { + let rv = find_arg(ann.arguments, "requires") + if (rv is tString && !empty(rv as tString)) { + exprs |> push(rv as tString) + } + } + } + if (isTune && !empty(exprs)) { + rows |> push("{fn.name}={join(exprs, "+")}") + } + delete exprs + } + } + } + let joined = join(rows, ";") + delete wanted + delete rows + return joined +} + //! One `[tune_scope]`'s identity + completeness, as reported by `tune_scopes_status()` — //! the macro-time surface build orchestrators (the `-exe` release-deps JSON, daspkg) read //! to decide whether to run tuners before a rebuild. @@ -1157,6 +1232,84 @@ def tune_pick_fallback(chain : string) : string { return "" } +let private TUNE_KNOWN_FEATURES = "avx2;f16c;fma;sse4.2;avxvnni;avxvnniint8;avx512f;avx512bw;avx512vl;avx512vnni;amx-tile;amx-int8;neon;dotprod;i8mm;bf16;fullfp16;sve;lse" + + //! The ';'-joined subset of the known `requires=` feature names this box passes — stamped + //! into every sidecar's provenance as `features`, which is what lets a shipped profile say + //! exactly which gated seats its minting box could and could not race. +def tune_feature_fingerprint() : string { + var have <- [for (f in split(TUNE_KNOWN_FEATURES, ";")); f; where fallback_feature_ok(f)] + let joined = join(have, ";") + delete have + return joined +} + + //! This box's CPU feature class — the name a shipped defaults profile is keyed by. Classes + //! follow the seats `requires=` can gate, not box models: two boxes of one class stamp the + //! same winners, so one profile serves them all; an arch with no ladder gets a key no profile matches. +def tune_cpu_class() : string { + let arch = get_architecture_name() + if (arch == "x86_64") { + if (fallback_feature_ok("avx512vnni") && fallback_feature_ok("avx512bw")) { + return "x86-vnni512" + } + if (fallback_feature_ok("avxvnni")) { + return "x86-vnni256" + } + return fallback_feature_ok("avx2") ? "x86-avx2" : "x86-base" + } + if (arch == "arm64" || arch == "aarch64") { + return fallback_feature_ok("i8mm") ? "arm-i8mm" : "arm-neon" + } + return "{get_platform_name()}-{arch}" +} + + //! Profile candidates for this box, most specific first — a box above the best shipped + //! profile adopts the nearest lower class and races only what that class could not. +def tune_class_chain() : array { + let ladder = (get_architecture_name() == "x86_64" + ? "x86-vnni512;x86-vnni256;x86-avx2;x86-base" + : "arm-i8mm;arm-neon") + let own = tune_cpu_class() + var chain : array + var seen = false + for (c in split(ladder, ";")) { + seen ||= c == own + if (seen) { + chain |> push(c) + } + } + if (!seen) { + chain |> push(own) + } + return <- chain +} + +// "did the PROFILE's minting box pass this gate" is what separates covered from unlocked seats +def private requires_ok_in_fingerprint(requires : string; fingerprint : string) : bool { + return true if (empty(requires)) + var have : table + for (f in split(fingerprint, ";")) { + if (!empty(f)) { + have |> insert(f) + } + } + var ok = true + for (conj in split(requires, ",")) { + continue if (empty(conj)) + var anyOk = false + for (alt in split(conj, "|")) { + anyOk ||= !empty(alt) && have |> key_exists(alt) + } + if (!anyOk) { + ok = false + break + } + } + delete have + return ok +} + def private perm_requires_ok(p : TunePerm) : bool { return tune_requires_ok(p.requires) } @@ -1456,6 +1609,17 @@ class private TuneAnnotation : AstFunctionAnnotation { delete comps return false } + if (g_env_tune.tune_policy == "reference") { + let noPerms : array + var rok = emit_registry(compiling_module(), func, noPerms, errors) + for (c in comps) { + break if (!rok) + rok = emit_registry(compiling_module(), c.fn, noPerms, errors) + } + delete perms + delete comps + return rok + } // no entry falls back rather than dropping to reference, so a sidecar written before a kernel family landed cannot untune it let fname = "{func.name}" var manifestFound = false @@ -1591,18 +1755,24 @@ class private TuneScopeAnnotation : AstStructureAnnotation { return false } } + var defaults = "" + let dv = find_arg(args, "defaults") + if (dv is tString) { + defaults = resolve_decl_path(string(st.at.fileInfo.name), dv as tString) + } // overwrite, not error: shared-module macro state persists across compiles in one process g_scopes["{compiling_module().name}"] = TuneScope(scopeName = sname, modName = "{compiling_module().name}", covers = covers, manifestPath = tune_manifest_path(), tunerPath = tuner, - versionKey = versionKey, versionExpected = versionExpected) + versionKey = versionKey, versionExpected = versionExpected, + defaultsPath = defaults) return true } } // The tune flags, parsed off the APP's command line — we share an argv we do not own, so this -// is the two-arg parse_args form (clargs skips unknown flag-shaped tokens). Keep every field a -// bool and never read positionals: an app's own `--model foo.gguf` leaves `foo.gguf` in there. +// is the two-arg parse_args form (clargs skips unknown flag-shaped tokens). Never read +// positionals - an app's own `--model foo.gguf` leaves `foo.gguf` in there. [CommandLineArgs] struct private TuneCliArgs { @clarg_doc = "Re-tune this box's kernels before the app runs" @@ -1615,11 +1785,15 @@ struct private TuneCliArgs { @clarg_name = "tune-verbose" @clarg_doc = "Tune with the full per-variant output instead of a progress display" tune_verbose : bool + + @clarg_name = "tune-only" + @clarg_doc = "Comma-separated name filter: re-tune only kernel families whose name contains one of these tokens (implies --tune); every other sidecar entry survives" + tune_only : string } -// Adopts --tune-quiet/--tune-verbose and answers whether --tune was passed. Runs at MACRO -// time, so the verbosity has to travel by env: the runtime guard is a different context and -// would not see this context's global. get_user_args also gets the standalone-exe slice right. +// Adopts --tune-quiet/--tune-verbose/--tune-only and answers whether a forced tune was asked for +// (--tune, or --tune-only alone). Runs at MACRO time, so the verbosity travels by env - the runtime +// guard is another context; get_user_args also gets the standalone-exe slice right. def private apply_cli_tune_flags() : bool { var r <- parse_args(type, get_user_args()) if (r |> is_err) { @@ -1631,7 +1805,10 @@ def private apply_cli_tune_flags() : bool { } elif (cfg.tune_verbose) { tune_set_verbosity("verbose") } - return cfg.tune + if (!empty(cfg.tune_only)) { + set_env_variable("DAS_TUNE_ONLY", cfg.tune_only) + } + return cfg.tune || !empty(cfg.tune_only) } // the exact command a user (or the auto path) runs to tune one scope; the env prefix leads @@ -1907,10 +2084,10 @@ def private announce_tuning(scopeName : string; force, selfRestarts : bool; deta print("llvm_tune: scope '{scopeName}' {why} - tuning now. This takes a while; {tail}.\n") } -// spawn the scope's tuner, relaying its output LIVE — RUNTIME only (the auto and restart -// guards); never from macro context. Streaming is load-bearing: a tune is minutes of work, and -// buffering it to a file left us and any supervisor with no signal until the child exited. -def private run_scope_tuner(scope : TuneScope) : bool { +// spawn the scope's tuner, relaying its output LIVE — RUNTIME only (the auto and restart guards). +// Streaming is load-bearing: buffering minutes of tune output leaves every supervisor blind. +// `onlyFilter` arms DAS_TUNE_ONLY for this one spawn and clears it after (the re-exec child must not inherit it). +def private run_scope_tuner(scope : TuneScope; onlyFilter : string = "") : bool { let cmd = "{tuner_cmd(scope)} 2>&1" if (tune_verbosity() != "silent") { print("llvm_tune: tuning scope '{scope.scopeName}' -> {scope.manifestPath}\n") @@ -1918,6 +2095,9 @@ def private run_scope_tuner(scope : TuneScope) : bool { if (tune_verbosity() == "verbose") { print("llvm_tune: {cmd}\n") } + if (!empty(onlyFilter)) { + set_env_variable("DAS_TUNE_ONLY", onlyFilter) + } var rc : int unsafe { rc = popen(cmd) $(f) { @@ -1928,6 +2108,9 @@ def private run_scope_tuner(scope : TuneScope) : bool { } } } + if (!empty(onlyFilter)) { + set_env_variable("DAS_TUNE_ONLY", "") + } finish_progress() if (rc != 0 && tune_verbosity() != "silent") { print("llvm_tune: tuner for scope '{scope.scopeName}' exited with {rc}\n") @@ -1949,15 +2132,124 @@ def public tune_suppress_mint(knob : string) { g_mint_suppressed = knob } - //! Runtime half of `[tune_policy(missing = "auto")]` / `--tune`, injected at the top of the - //! app's `main` and followed by `tune_auto_reexec()`. Tuning happens at RUNTIME because the - //! winners are compile-time stamps. An empty or corrupt sidecar reads as untuned, so it self-heals. -def tune_auto_prepare(scopeName : string; manifestPath : string; tunerPath : string; force : bool; fnames : string) { +// ===== shipped defaults profiles ([tune_scope(defaults = "dir")]) ===== + +// adopt a shipped class profile into an untuned scope before any race: true = adopted, and +// `onlyFilter` is then the ','-joined families the caller still owes a race ("" = fully covered) +def private profile_try_adopt(defaultsDir, scopeName, manifestPath, fnames, gatedFamilies : string; var onlyFilter : string&) : bool { // nolint:STYLE037,STYLE038 — one straight adopt pipeline: locate, validate, merge, diff + onlyFilter = "" + // a runtime-redirected manifest path declines - adoption never lands in a file this guard was not compiled for + return false if (empty(defaultsDir) || tune_manifest_path() != manifestPath) + var verKey = "" + var verVal = "" + for (tok in split(fnames, ";")) { + if (tok |> starts_with(TUNE_VERSION_TOKEN)) { + let kv = slice(tok, length(TUNE_VERSION_TOKEN)) + let eq = find(kv, "=") + if (eq >= 0) { + verKey = slice(kv, 0, eq) + verVal = slice(kv, eq + 1) + } + } + } + var chain <- tune_class_chain() + var klass = "" + var doc : JsonValue? + for (c in chain) { + let p = path_join(defaultsDir, "{c}.tune-defaults.json") + let text = fread(p) + continue if (empty(text)) + var err = "" + var d = read_json(text, err) + if (d == null || !(d.value is _object)) { + print("llvm_tune: defaults profile {p} is not valid JSON ({err}) - skipped\n") + delete_json(d) + continue + } + if (!empty(verKey) && "{d?["provenance"]?[verKey] ?? ""}" != verVal) { + print("llvm_tune: defaults profile '{c}' predates this build ({verKey} != {verVal}) - skipped\n") + delete_json(d) + continue + } + klass = c + doc = d + break + } + delete chain + return false if (doc == null) + var kernels : table + let ks = doc?["kernels"] + if (ks != null && ks.value is _object) { + for (k, v in keys(ks as _object), values(ks as _object)) { + if (v != null && v is _string) { + kernels[k] = v as _string + } + } + } + if (empty(kernels)) { + print("llvm_tune: defaults profile '{klass}' has no kernel entries - skipped\n") + delete_json(doc) + return false + } + var race : array + for (tok in split(fnames, ";")) { + continue if (empty(tok) || tok |> starts_with(TUNE_VERSION_TOKEN)) + if (!(kernels |> key_exists(tok))) { + race |> push(tok) + } + } + let pfeat = "{doc?["provenance"]?["features"] ?? ""}" + for (row in split(gatedFamilies, ";")) { + continue if (empty(row)) + let eq = find(row, "=") + continue if (eq < 0) + let fam = slice(row, 0, eq) + continue if (race |> find_index(fam) >= 0) + for (expr in split(slice(row, eq + 1), "+")) { + continue if (empty(expr)) + if (tune_requires_ok(expr) && !requires_ok_in_fingerprint(expr, pfeat)) { + race |> push(fam) + break + } + } + } + if (!empty(verKey)) { + tune_provenance_note(verKey, verVal) + } + tune_provenance_note("adopted_profile", klass) + var noRuntime : JsonValue? + let merged = tune_sidecar_merge(kernels, noRuntime) + if (!merged) { + print("llvm_tune: scope '{scopeName}' could not write adopted profile '{klass}' into {manifestPath}\n") + delete kernels + delete race + delete_json(doc) + return false + } + let tail = empty(race) ? "" : " - racing {length(race)} unlocked/new families" + print("llvm_tune: scope '{scopeName}' adopted the '{klass}' defaults profile ({length(kernels)} kernel entries){tail}\n") + onlyFilter = join(race, ",") + delete kernels + delete race + delete_json(doc) + return true +} + + //! Runtime half of `[tune_policy(missing = "auto")]` / `--tune`, injected at the top of `main` + //! and followed by `tune_auto_reexec()`. Adopts a declared defaults profile first, then tunes + //! the residue at RUNTIME (winners are compile-time stamps). A corrupt sidecar self-heals. +def tune_auto_prepare(scopeName : string; manifestPath : string; tunerPath : string; force : bool; fnames : string; + defaultsDir : string = ""; gatedFamilies : string = "") { let v = tune_sidecar_verdict(manifestPath, fnames) if (!force && v.complete) { return } - if (!force && scope_resolved_externally(scopeName, manifestPath, fnames)) { + var onlyFilter = "" + if (!force && profile_try_adopt(defaultsDir, scopeName, manifestPath, fnames, gatedFamilies, onlyFilter) && empty(onlyFilter)) { + g_tuned_scopes++ // nothing raced, yet counted: the re-exec is what stamps the adopted winners + return + } + if (!force && empty(onlyFilter) && scope_resolved_externally(scopeName, manifestPath, fnames)) { g_tuned_scopes++ // the re-exec picks the resolved sidecar up like a fresh tune return } @@ -1968,7 +2260,7 @@ def tune_auto_prepare(scopeName : string; manifestPath : string; tunerPath : str return } announce_tuning(scopeName, force, true, v.detail) - run_scope_tuner(TuneScope(scopeName = scopeName, manifestPath = manifestPath, tunerPath = tunerPath)) + run_scope_tuner(TuneScope(scopeName = scopeName, manifestPath = manifestPath, tunerPath = tunerPath), onlyFilter) let after = tune_sidecar_verdict(manifestPath, fnames) if (after.complete) { g_tuned_scopes++ @@ -1978,7 +2270,7 @@ def tune_auto_prepare(scopeName : string; manifestPath : string; tunerPath : str } //! Relaunch the process so a fresh compile stamps the just-written manifests — they cannot be - //! adopted mid-run. `--tune` is stripped from the relaunch so the child converges. True means + //! adopted mid-run. `--tune` / `--tune-only` are stripped from the relaunch so the child converges. True means //! the child ran and main must return `tune_reexec_code()` immediately. def tune_auto_reexec() : bool { if (g_tuned_scopes == 0) { @@ -1988,10 +2280,18 @@ def tune_auto_reexec() : bool { var argv <- get_command_line_arguments() var args : array args |> reserve(length(argv)) + var skipTuneOnlyValue = false for (a in argv) { - if (a != "--tune") { - args |> push("{a}") + if (skipTuneOnlyValue) { + skipTuneOnlyValue = false + continue } + continue if (a == "--tune" || starts_with(a, "--tune-only=")) + if (a == "--tune-only") { + skipTuneOnlyValue = true + continue + } + args |> push("{a}") } print("llvm_tune: re-launching to apply the fresh manifests\n") var rc : int @@ -2049,15 +2349,50 @@ def tune_interrupt_requested() : bool { return stat(p, fs) } + //! The `DAS_TUNE_POLICY` override, verbatim ("" = none) — how a sibling stamping macro + //! (e.g. a library's `[tuned]` loop-hint rail) honors `reference` and `fallback` too. +def tune_policy_env() : string { + return g_env_tune.tune_policy +} + + //! The `--tune-only` re-mint filter (`DAS_TUNE_ONLY`), verbatim: comma-separated name + //! tokens, empty = race everything. +def tune_only_filter() : string { + return g_env_tune.tune_only +} + + //! True when family `name` should race under the `--tune-only` filter - no filter set, or + //! some comma-separated token is a substring of the name. A harness consults this at each + //! family boundary; a skipped family writes no entry, so its sidecar seat survives. +def tune_family_selected(name : string) : bool { + let f = g_env_tune.tune_only + if (empty(f)) { + return true + } + for (tok in split(f, ",")) { + let t = strip(tok) + if (!empty(t) && find(name, t) >= 0) { + return true + } + } + return false +} + //! Runtime half of `[tune_policy(missing = "restart")]`, injected as a guard at the top of - //! the app's `main`: consults the scope resolver first, else runs the tuner; true = main + //! the app's `main`: adopts a declared defaults profile, then consults the scope resolver, else runs the tuner; true = main //! returns immediately (winners are compile-time stamps), false = scope complete, carry on. -def tune_restart_needed(scopeName : string; manifestPath : string; tunerPath : string; fnames : string) : bool { +def tune_restart_needed(scopeName : string; manifestPath : string; tunerPath : string; fnames : string; + defaultsDir : string = ""; gatedFamilies : string = "") : bool { let v = tune_sidecar_verdict(manifestPath, fnames) if (v.complete) { return false } - if (scope_resolved_externally(scopeName, manifestPath, fnames)) { + var onlyFilter = "" + if (profile_try_adopt(defaultsDir, scopeName, manifestPath, fnames, gatedFamilies, onlyFilter) && empty(onlyFilter)) { + print("llvm_tune: scope '{scopeName}' adopted -> {manifestPath}\nllvm_tune: restart to apply the winners\n") + return true + } + if (empty(onlyFilter) && scope_resolved_externally(scopeName, manifestPath, fnames)) { return true // resolved without a tune; the resolver printed its own restart marker } if (!empty(g_mint_suppressed)) { @@ -2073,7 +2408,7 @@ def tune_restart_needed(scopeName : string; manifestPath : string; tunerPath : s return true } announce_tuning(scopeName, false, false, v.detail) - run_scope_tuner(TuneScope(scopeName = scopeName, manifestPath = manifestPath, tunerPath = tunerPath)) + run_scope_tuner(TuneScope(scopeName = scopeName, manifestPath = manifestPath, tunerPath = tunerPath), onlyFilter) let after = tune_sidecar_verdict(manifestPath, fnames) if (after.complete) { print("llvm_tune: scope '{scopeName}' tuned -> {manifestPath}\nllvm_tune: restart to apply the winners\n") @@ -2186,16 +2521,20 @@ def private apply_tune_policy(var func : FunctionPtr; declared : string; var err var policy = declared { let penv = g_env_tune.tune_policy - // announce deferred to the first INCOMPLETE scope - a fully-tuned start stays silent - var envNote = "" if (!empty(penv)) { policy = penv + // the announce fires where the override changes the outcome - `reference` alters every stamp even on a fully tuned box if (tune_verbosity() != "silent") { - envNote = "llvm_tune: policy '{policy}' set by DAS_TUNE_POLICY (declared/default '{declared}')\n" + print("llvm_tune: policy '{policy}' set by DAS_TUNE_POLICY (declared/default '{declared}')\n") } } - if (policy != "fallback" && policy != "warn" && policy != "error" && policy != "auto" && policy != "restart") { - errors = "[tune_policy] on {func.name}: missing=\"{policy}\" - use fallback | warn | error | auto | restart" + if (policy != "fallback" && policy != "reference" && policy != "warn" && policy != "error" && policy != "auto" && policy != "restart") { + errors = "[tune_policy] on {func.name}: missing=\"{policy}\" - use fallback | reference | warn | error | auto | restart" + return false + } + if (declared == "reference" && penv != "reference") { + // the [tune] stamps in required libraries compile before this root exists and read the env only + errors = "[tune_policy] on {func.name}: missing=\"reference\" is an environment override (DAS_TUNE_POLICY=reference), not a declarable value" return false } let force = apply_cli_tune_flags() @@ -2213,10 +2552,6 @@ def private apply_tune_policy(var func : FunctionPtr; declared : string; var err let scopeFnames = scope_fnames_joined(scope) let v = tune_sidecar_verdict(scope.manifestPath, scopeFnames) continue if (!force && v.complete) - if (!empty(envNote)) { - print(envNote) - envNote = "" - } if (force || policy == "auto") { // a runtime guard tunes and RE-EXECS: restamping a required module's compiled function is a cross-module mutation re-infer cannot survive if (empty(scope.tunerPath)) { @@ -2238,15 +2573,18 @@ def private apply_tune_policy(var func : FunctionPtr; declared : string; var err errors = "[tune_policy]: restart - {func.name} must return void or int" } else { var guard : ExpressionPtr + let scopeGated = scope_gated_families_joined(scope) if (func.result.baseType == Type.tInt) { guard = qmacro_expr() { - if (tune_restart_needed($v(scope.scopeName), $v(scope.manifestPath), $v(scope.tunerPath), $v(scopeFnames))) { + if (tune_restart_needed($v(scope.scopeName), $v(scope.manifestPath), $v(scope.tunerPath), $v(scopeFnames), + $v(scope.defaultsPath), $v(scopeGated))) { return 3 } } } else { guard = qmacro_expr() { - if (tune_restart_needed($v(scope.scopeName), $v(scope.manifestPath), $v(scope.tunerPath), $v(scopeFnames))) { + if (tune_restart_needed($v(scope.scopeName), $v(scope.manifestPath), $v(scope.tunerPath), $v(scopeFnames), + $v(scope.defaultsPath), $v(scopeGated))) { return } } @@ -2272,8 +2610,10 @@ def private apply_tune_policy(var func : FunctionPtr; declared : string; var err guards |> reserve(length(autoScopes) + 1) for (scope in autoScopes) { let fnames = scope_fnames_joined(scope) + let gated = scope_gated_families_joined(scope) guards |> push <| qmacro_expr() { - tune_auto_prepare($v(scope.scopeName), $v(scope.manifestPath), $v(scope.tunerPath), $v(force), $v(fnames)) + tune_auto_prepare($v(scope.scopeName), $v(scope.manifestPath), $v(scope.tunerPath), $v(force), $v(fnames), + $v(scope.defaultsPath), $v(gated)) } } if (func.result.baseType == Type.tInt) { diff --git a/modules/dasLLVM/tests/README.md b/modules/dasLLVM/tests/README.md index f811939c0f..a84240b9eb 100644 --- a/modules/dasLLVM/tests/README.md +++ b/modules/dasLLVM/tests/README.md @@ -11,5 +11,5 @@ Run the suite explicitly from the repository root, on an LLVM-enabled build bin/Release/daslang.exe dastest/dastest.das -jit -- --timing-outliers 10 --test modules/dasLLVM/tests ``` -Files ending in `_client`, `_client_lib`, or `_tuner` are child-process fixtures rather than +Files ending in `_client`, `_client_lib`, `_lib`, or `_tuner` are child-process fixtures rather than standalone test cases. diff --git a/modules/dasLLVM/tests/llvm_tune_profiles.das b/modules/dasLLVM/tests/llvm_tune_profiles.das new file mode 100644 index 0000000000..c528c2d0f2 --- /dev/null +++ b/modules/dasLLVM/tests/llvm_tune_profiles.das @@ -0,0 +1,173 @@ +options gen2 + +require dastest/testing_boost +require daslib/module_path +require daslib/strings_boost +require daslib/fio +require strings +require llvm/daslib/llvm_tune + +// The shipped-defaults-profile rail ([tune_scope(defaults = ...)]) through child daslang +// processes: an untuned start adopts the class profile instead of racing (source=manifest, +// no tuner spawn); a profile whose minting box lacked a feature this box has races ONLY the +// unlocked family (DAS_TUNE_ONLY armed for the tuner, cleared after); no profile file = +// the old full race. The profile files are written HERE — their name carries this box's +// tune_cpu_class(), and their features field is this box's fingerprint (or empty, to fake +// a less-capable minting box). + +def private cleared_tune_env_prefix() : string { + return (get_platform_name() == "windows" ? + "set DAS_TUNE_MODE=&& set DAS_TUNE_MANIFEST=&& set DAS_TUNE_VERBOSITY=verbose&& set DAS_TUNE_POLICY=&& set DAS_TUNE_ONLY=&& " : + "DAS_TUNE_MODE= DAS_TUNE_MANIFEST= DAS_TUNE_VERBOSITY=verbose DAS_TUNE_POLICY= DAS_TUNE_ONLY= ") +} + +def private spawn_child(cmd : string; var lines : array) : int { + var rc : int + unsafe { + rc = popen_timeout("{cmd} 2>&1", 300.0) $(f) { + if (f == null) { + return + } + while (!feof(f)) { + let ln = strip(fgets(f)) + if (!empty(ln)) { + lines |> push("{ln}") + } + } + } + } + return rc +} + +def private count_hits(lines : array; needle : string) : int { + var n = 0 + for (ln in lines) { + if (find(ln, needle) >= 0) { + n++ + } + } + return n +} + +def private has(lines : array; needle : string) : bool { + for (ln in lines) { + if (find(ln, needle) >= 0) { + return true + } + } + return false +} + +def private write_profile(dir, klass, winner, features : string) { + mkdir(dir) + let body = "\{ \"kernels\" : \{ \"prof_add\" : \"{winner}\" \}, \"provenance\" : \{ \"features\" : \"{features}\" \} \}" + fwrite(path_join(dir, "{klass}.tune-defaults.json"), body) +} + +[test] +def test_llvm_tune_profiles(t : T?) { // nolint:STYLE038 - one flat spawn-and-assert rail per scenario; a split would hide the sequence the scenarios build on + if (!jit_enabled()) { + // no generator tier without the jit, so no stamped-tier fingerprint to read + t |> success(true) + return + } + let args <- get_command_line_arguments() // aliases a process-global Array header - do not delete + let bin = args[0] + let client = "{get_this_module_dir()}/llvm_tune_profiles_client.das" + let sidecar = "{get_this_module_dir()}/llvm_tune_profiles_client.tune.json" + let defdir = "{get_this_module_dir()}/llvm_tune_profiles_defaults" + let klass = tune_cpu_class() + let profile = path_join(defdir, "{klass}.tune-defaults.json") + + // 1. no profile file: the plain auto flow - full race (empty filter), the tuner's k2 runs + remove(sidecar) + remove(profile) + var noProfile : array + t |> equal(spawn_child("{cleared_tune_env_prefix()}\"{bin}\" -jit \"{client}\"", noProfile), 0) + t |> success(noProfile |> has("FAKE TUNER")) + t |> success(noProfile |> has("ONLY=''")) + t |> success(!(noProfile |> has("defaults profile"))) + t |> success(noProfile |> has("RESULT 7")) + delete noProfile + + // 2. full-cover profile (this box's own fingerprint): adopted, no tuner, its winner stamps + remove(sidecar) + write_profile(defdir, klass, "k0", tune_feature_fingerprint()) + var fullCover : array + t |> equal(spawn_child("{cleared_tune_env_prefix()}\"{bin}\" -jit \"{client}\"", fullCover), 0) + t |> success(fullCover |> has("adopted the '{klass}' defaults profile")) + t |> success(!(fullCover |> has("FAKE TUNER"))) + t |> success(fullCover |> has("RESULT 5")) + t |> success(fullCover |> has("STATUS prof_add|llvm_tune_profiles_probe|k0|manifest")) + let adoptedSidecar = fread(sidecar) + t |> success(find(adoptedSidecar, "\"adopted_profile\"") >= 0) + delete fullCover + + // 3. a profile minted with no features leaves the k3 requires= seat unlocked here: adopt AND race just that family - the tuner's k2 outranks the adopted k0 + remove(sidecar) + write_profile(defdir, klass, "k0", "") + var unlockedSeat : array + t |> equal(spawn_child("{cleared_tune_env_prefix()}\"{bin}\" -jit \"{client}\"", unlockedSeat), 0) + t |> success(unlockedSeat |> has("racing 1 unlocked/new families")) + t |> success(unlockedSeat |> has("ONLY='prof_add'")) + t |> success(unlockedSeat |> has("RESULT 7")) + t |> success(unlockedSeat |> has("STATUS prof_add|llvm_tune_profiles_probe|k2|manifest")) + delete unlockedSeat + + // 4. --tune outranks the profile: full re-race, no adoption + var forcedTune : array + t |> equal(spawn_child("{cleared_tune_env_prefix()}\"{bin}\" -jit \"{client}\" -- --tune", forcedTune), 0) + t |> success(!(forcedTune |> has("defaults profile"))) + t |> success(forcedTune |> has("FAKE TUNER")) + delete forcedTune + + // 5. --tune-only converges: the relaunch strips both spellings, so exactly one tuner runs and + // the child does not re-tune forever (the strip arms are what keep it from looping) + var onlyOne : array + t |> equal(spawn_child("{cleared_tune_env_prefix()}\"{bin}\" -jit \"{client}\" -- --tune-only prof_add", onlyOne), 0) + t |> equal(count_hits(onlyOne, "FAKE TUNER"), 1) + t |> success(onlyOne |> has("ONLY='prof_add'")) + t |> success(onlyOne |> has("RESULT 7")) + delete onlyOne + var onlyEq : array + t |> equal(spawn_child("{cleared_tune_env_prefix()}\"{bin}\" -jit \"{client}\" -- --tune-only=prof_add", onlyEq), 0) + t |> equal(count_hits(onlyEq, "FAKE TUNER"), 1) + delete onlyEq + + // 6. the class chain: only a LOWER class's profile on disk, and the box adopts it (the + // residue race then covers whatever its own class unlocks) + var chain <- tune_class_chain() + if (length(chain) > 1) { + let lower = chain[length(chain) - 1] + remove(sidecar) + remove(profile) + write_profile(defdir, lower, "k0", tune_feature_fingerprint()) + var lowerAdopt : array + t |> equal(spawn_child("{cleared_tune_env_prefix()}\"{bin}\" -jit \"{client}\"", lowerAdopt), 0) + t |> success(lowerAdopt |> has("adopted the '{lower}' defaults profile")) + t |> success(!(lowerAdopt |> has("FAKE TUNER"))) + delete lowerAdopt + remove(path_join(defdir, "{lower}.tune-defaults.json")) + } + delete chain + + // 7. DAS_TUNE_POLICY=reference serves the original bodies and announces itself even on a + // fully tuned box (the sidecar from case 5 is complete) + var refRun : array + let refEnv = (get_platform_name() == "windows" ? + "set DAS_TUNE_MODE=&& set DAS_TUNE_MANIFEST=&& set DAS_TUNE_VERBOSITY=verbose&& set DAS_TUNE_POLICY=reference&& set DAS_TUNE_ONLY=&& " : + "DAS_TUNE_MODE= DAS_TUNE_MANIFEST= DAS_TUNE_VERBOSITY=verbose DAS_TUNE_POLICY=reference DAS_TUNE_ONLY= ") + t |> equal(spawn_child("{refEnv}\"{bin}\" -jit \"{client}\"", refRun), 0) + t |> success(refRun |> has("policy 'reference' set by DAS_TUNE_POLICY")) + t |> success(refRun |> has("STATUS prof_add|llvm_tune_profiles_probe||reference")) + t |> success(refRun |> has("RESULT 5")) + delete refRun + + remove(sidecar) + remove(profile) +} + +[export] +def main { + return +} diff --git a/modules/dasLLVM/tests/llvm_tune_profiles_client.das b/modules/dasLLVM/tests/llvm_tune_profiles_client.das new file mode 100644 index 0000000000..1af5b55381 --- /dev/null +++ b/modules/dasLLVM/tests/llvm_tune_profiles_client.das @@ -0,0 +1,18 @@ +options gen2 + +require llvm_tune_profiles_lib + +// App-face fixture for llvm_tune_profiles.das (no [test] on purpose — the suite just +// compiles it): default policy (auto), so an untuned start walks adopt -> filtered race -> +// re-exec. RESULT is the k-fingerprint of the stamped tier (jit: 2+3+k; interp reference: 5). +[export] +def main : int { + var n = 2 // nolint:LINT003 — var defeats const-fold so the call survives to codegen + print("RESULT {prof_add(n, 3)}\n") + var st <- tune_status() + for (s in st) { + print("STATUS {s.fname}|{s.scope}|{s.suffix}|{s.source}\n") + } + delete st + return 0 +} diff --git a/modules/dasLLVM/tests/llvm_tune_profiles_lib.das b/modules/dasLLVM/tests/llvm_tune_profiles_lib.das new file mode 100644 index 0000000000..102da66cd2 --- /dev/null +++ b/modules/dasLLVM/tests/llvm_tune_profiles_lib.das @@ -0,0 +1,21 @@ +options gen2 + +module llvm_tune_profiles_lib public + +require llvm/daslib/llvm_tune public + +// Library-side fixture for llvm_tune_profiles.das (no [test] on purpose — the suite just +// compiles it): a scope that also ships defaults profiles. The defaults dir is created and +// filled BY THE TEST (the profile file name carries this box's tune_cpu_class(), which the +// test computes in-process). The k3 row's requires= passes on every test box (avx2 on x64, +// neon on arm) — the "unlocked seat" whose absence from a profile's features forces a race. +[tune_scope(name = "llvm_tune_profiles_probe", tuner = "./llvm_tune_profiles_tuner.das", + defaults = "./llvm_tune_profiles_defaults")] +struct private ScopeDecl {} + +[tune_perm(k = 0), tune_perm(k = 1), tune_perm(k = 2), + tune_perm(k = 3, requires = "avx2|neon"), + tune(gen = "llvm_code_selftest::add_plus_k", fallback = "k1")] +def prof_add(a, b : int) : int { + return a + b +} diff --git a/modules/dasLLVM/tests/llvm_tune_profiles_tuner.das b/modules/dasLLVM/tests/llvm_tune_profiles_tuner.das new file mode 100644 index 0000000000..77f9a1085a --- /dev/null +++ b/modules/dasLLVM/tests/llvm_tune_profiles_tuner.das @@ -0,0 +1,15 @@ +options gen2 + +require llvm/daslib/llvm_tune + +// Instant fake tuner for llvm_tune_profiles.das (no [test] on purpose): prints the +// DAS_TUNE_ONLY filter the guard armed (the race-on-unlock proof) and records k2 — a winner +// distinct from both the fallback (k1) and the profiles the test writes, so RESULT +// fingerprints which path stamped. +[export] +def main { + print("FAKE TUNER -> {tune_manifest_path()} ONLY='{tune_only_filter()}'\n") + if (tune_family_selected("prof_add")) { + tune_manifest_set("prof_add", "k2") + } +} diff --git a/modules/dasLLVM/tests/test_grid.das b/modules/dasLLVM/tests/test_grid.das index 041fdef558..5ba22564bd 100644 --- a/modules/dasLLVM/tests/test_grid.das +++ b/modules/dasLLVM/tests/test_grid.das @@ -34,7 +34,7 @@ def test_grid_registry(t : T?) { let want = reference(a, b) let vs <- grid_dot_template_variants() - t |> equal(length(vs), 20, "grid emits all 20 permutations") + t |> equal(length(vs), 7, "grid emits all 7 permutations (plain, u2, vec8, vec8_u2, vec16, vec16_u2, vec4_u4)") // Every emitted variant is callable and numerically correct (vectorize reassociates FP, // so compare within tolerance rather than bit-exact). @@ -65,7 +65,7 @@ struct GridScale {} [test] def test_grid_void_registry(t : T?) { let vs <- grid_scale_template_variants() - t |> equal(length(vs), 20, "void-template grid emits all 20 permutations") + t |> equal(length(vs), 7, "void-template grid emits all 7 permutations") var d : array d |> resize(8) diff --git a/modules/dasLLVM/tests/test_tuned.das b/modules/dasLLVM/tests/test_tuned.das index 899da4b1d6..30fd2952a0 100644 --- a/modules/dasLLVM/tests/test_tuned.das +++ b/modules/dasLLVM/tests/test_tuned.das @@ -23,8 +23,8 @@ def sample_dot(a, b : array) : float {} // an unknown fallback name fails the compile with "tuned: unknown perm" (negative-probed), so this // compiling proves the fallback names a valid grid perm — the one consulted whenever no profile // entry / perm pin applies (and no profile carries this test-local kernel's key). -[tuned(fallback = "u4", src = "sample_dot_template")] -def sample_dot_u4(a, b : array) : float {} +[tuned(fallback = "u2", src = "sample_dot_template")] +def sample_dot_u2(a, b : array) : float {} def private reference(a, b : array) : float { var s = 0f @@ -39,5 +39,5 @@ def test_tuned_reconstitutes(t : T?) { let a = [for (i in range(64)); float(i)] let b = [for (_i in range(64)); 1.0f] t |> equal(sample_dot(a, b), reference(a, b), "tuned sample_dot == reference") - t |> equal(sample_dot_u4(a, b), reference(a, b), "tuned default=u4 sample_dot_u4 == reference") + t |> equal(sample_dot_u2(a, b), reference(a, b), "tuned default=u2 sample_dot_u2 == reference") } diff --git a/modules/dasMetal/ARCHITECTURE.md b/modules/dasMetal/ARCHITECTURE.md index cc7a7ef959..aa7ca93c0b 100644 --- a/modules/dasMetal/ARCHITECTURE.md +++ b/modules/dasMetal/ARCHITECTURE.md @@ -170,6 +170,13 @@ than a second GPU. Cross-GPU parity of one source is secondary. Current entries: SPV_NV_cooperative_matrix2 tensor loads. Metal-4 tensors have no decode-callback analogue, so the Metal quant GEMMs stage dequant through threadgroup memory instead. Deliberate, target-specific - not a pending port. +- **Literal fixed-array hoisting is Metal-only; pending, not deliberate.** `msl_emit` lowers a + `let` fixed-array local whose elements are all literals to a program-scope `constant` table + (renamed on a same-name/different-content collision); `spirv_emit` keeps such a local in + Function storage, where a dynamically indexed read spills to private memory - the codebook + and grid quant kernels on Vulkan therefore pack their tables into `uint4` words by hand. + Done = the SPIR-V twin (a constant-storage array for the same shape) or a lint on a + dynamically indexed fixed-array local inside a kernel class. ## 7. `@uniform` structs - the kargs form diff --git a/modules/dasMetal/REVIEW.md b/modules/dasMetal/REVIEW.md index 682e5f1dc0..387306721e 100644 --- a/modules/dasMetal/REVIEW.md +++ b/modules/dasMetal/REVIEW.md @@ -8,11 +8,12 @@ **A CPU-oracle or host-side test file answers to `tests/metal/REVIEW.md` (repo root), wherever the diff puts it.** An emitted-text fixture answers to `tests/msl/REVIEW.md` (repo root). -- **A new emitter capability ships a text fixture under `tests/msl/` (repo root) and its own - census kind, in the same change.** A new emitter capability is a new emit site or a newly - accepted construct, including the path a `[metal_kernel]` takes when it has no `name=` +- **A new emitter capability ships a text fixture under `tests/msl/` (repo root) and a census + kind per emit shape, in the same change.** A new emitter capability is a new emit site or a + newly accepted construct, including the path a `[metal_kernel]` takes when it has no `name=` argument. A census kind is the construct label the emitter records at emit time, declared in - `declared_msl_census` (`tests/msl/_msl_common.das`, repo root). Two emit shapes sharing one + `declared_msl_census` (`tests/msl/_msl_common.das`, repo root); an emit shape is one distinct + text the site can produce (a renamed twin is a second shape). Two emit shapes sharing one kind lets either one go untested. - **A new or changed `[metal_kernel]` annotation argument ships a `tests/msl/` (repo root) @@ -20,17 +21,17 @@ the diff puts it.** An emitted-text fixture answers to `tests/msl/REVIEW.md` (re macro declares for it - a module-level global holding the kernel's MSL text or a compile option - or the difference it makes to the emitted text. -- **A diff that makes the MSL emitter reject at compile time a construct it accepted before - - a new rejection, or a widened condition on an existing one - ships a - `tests/msl/_fail_closed/` (repo root) fixture for that construct in the same change.** An - error needle is the substring of the compile error that names the rejected construct; the - fixture-and-assert pairing itself is `REVIEW.das`'s to hold (`check_fail_closed_sync`, - both directions, `tests/msl/test_msl_fail_closed.das` (repo root)). +- **A diff that adds an MSL emitter rejection, widens an existing one's condition, or changes a + rejection's message text ships a `tests/msl/_fail_closed/` (repo root) fixture asserting that + rejection's needle, in the same change.** An error needle is the substring of the compile + error that names the rejected construct; `REVIEW.das` checks the fixture-and-assert pairing + both directions (`check_fail_closed_sync`, `tests/msl/test_msl_fail_closed.das` (repo root)). - **A kernel behavioral change ships a CPU-oracle test under `tests/metal/` (repo root).** A - kernel behavioral change is a change to what a kernel computes - its emitted arithmetic, - indexing, or synchronization; a CPU-oracle test compares the GPU result against a - CPU-computed expectation. + kernel behavioral change is a change to what an existing kernel computes, or a diff that + makes the emitter produce arithmetic, indexing, or synchronization it did not produce before + - a newly accepted construct that lowers to a read, an index, or a barrier; a CPU-oracle + test compares the GPU result against a CPU-computed expectation. - **A change visible only in the emitted text ships a `tests/msl/` (repo root) fixture.** The fixture asserts the emitted text that the change alters. @@ -40,18 +41,19 @@ the diff puts it.** An emitted-text fixture answers to `tests/msl/REVIEW.md` (re - **A new or changed host extern under `modules/dasMetal/src/` ships a host-side test under `tests/metal/` (repo root) in the same change.** A changed public function in - `metal/das_metal_boost.das` fires this rule too. The test `feint`s when no Metal device is - present. A regression in dasMetal alone must fail here, not in a consumer module's tests. + `metal/das_metal_boost.das` fires this rule too. The test reports a `feint` - dastest's skip + result - when no Metal device is present. A regression in dasMetal alone must fail here, not in a consumer module's tests. - **Weakening `REVIEW.das`'s descriptor check is a defect.** That check requires every `matmul2d_descriptor` written in `metal/msl_emit.das` to set `relaxed_precision = true`. Narrowing the file it scans weakens it. So does making its finding text stop naming what failed. `relaxed_precision = false` keeps the op off the tensor-unit fast path. -- **Weakening `REVIEW.das`'s fail-closed sync check is a defect.** That check holds the - `tests/msl/_fail_closed/` fixture list and `test_msl_fail_closed.das`'s `check_rejects` - asserts equal both directions - a fixture without its assert leaves a rejection unpinned, - and a stranded assert is a rejection the emitter no longer makes. +- **Weakening `REVIEW.das`'s fail-closed sync check is a defect.** That check requires every + `tests/msl/_fail_closed/` fixture to have a `check_rejects` call in + `test_msl_fail_closed.das`, and every such call to have a fixture - a fixture without its + call leaves a rejection unpinned, and a call without its fixture is a rejection the emitter + no longer makes. - **Never zero a cooperative tensor element by element before a `matmul2d` `run` accumulates into it - `get_destination_cooperative_tensor` already hands it back zeroed.** That walk @@ -63,9 +65,14 @@ the diff puts it.** An emitted-text fixture answers to `tests/msl/REVIEW.md` (re das body replays the same tile indexing, so a one-sided change reads the wrong halves with no compile error. -- **A diff that changes the threadgroup row stride `tmm2d_helper_text` writes - its `dstp` index - and the matching tensor extent in `metal/msl_emit.das` - also changes the `wt` size documented - on `tmm2d_q8u_f32` (`metal/metal_builtins.das`) and every `@workgroup` staging array a - `tests/msl/` or `tests/metal/` (repo root) fixture sizes for that stride, in the same change.** - A staged weight tile is the dequantized W the emitter parks in threadgroup memory; nothing - checks the caller's allocation against the stride, so a one-sided change overruns it silently. +- **A diff that changes the threadgroup row stride `tmm2d_helper_text` writes (its `dstp` + index and the matching tensor extent in `metal/msl_emit.das`) changes the `wt` size + documented on `tmm2d_q8u_f32` (`metal/metal_builtins.das`) in the same change.** `wt` is the + staged weight tile - the dequantized W the emitter stores in threadgroup memory - and the + documented size is the caller's only statement of the stride. + +- **A diff that changes the threadgroup row stride `tmm2d_helper_text` writes + (`metal/msl_emit.das`) also resizes every `@workgroup` staging array a `tests/msl/` or + `tests/metal/` (repo root) fixture sizes for that stride, in the same change.** + Nothing checks a fixture's allocation against the stride, so a one-sided change overruns + it silently. diff --git a/modules/dasMetal/metal/msl_emit.das b/modules/dasMetal/metal/msl_emit.das index 66315f1d3b..e85e9ea34b 100644 --- a/modules/dasMetal/metal/msl_emit.das +++ b/modules/dasMetal/metal/msl_emit.das @@ -82,6 +82,7 @@ struct private MslCtx { ustructs : table // @uniform struct name -> its MSL definition (preamble) helpers : table // preamble helper functions referenced by lowered calls gen_helpers : table // GENERATED preamble helpers (tensor ops): mangled name -> text + const_tables : table // hoisted program-scope constant arrays: MSL name -> definition text scan_ptrs : table // scan-time pointer local -> member (write-set resolution) tg_accs : table // staged-GEMM acc local -> "m|n|sgs|kk|at" (decl suppressed on GPU) mpp_used : bool // a tile_tg/rowscale-only kernel still needs the MPP include @@ -2961,6 +2962,83 @@ def private local_msl_name(ctx : MslCtx; name : string) : string { return key_exists(ctx.members, name) ? "{base}_l" : base } +// A `let` fixed-array local with an all-literal `fixed_array(...)` initializer lowers to a +// program-scope `constant` array - the codebook/grid-table shape quant kernels stage from. +// Anything else with an initializer stays rejected: das zero-init is the only local form. +def private hoist_const_table(var ctx : MslCtx; v : VariablePtr) { + let et = v._type.firstType + let n = v._type.fixedDim + if (!v._type.flags.constant) { + err(ctx, v.at, "fixed-array local `{v.name}` initializer hoists to a program-scope constant table - it must be a `let` (a written table has no constant form)") + return + } + if (reject_cpu_only_width(ctx, v.at, et)) { + return + } + let ename = msl_type_name(et) + if (empty(ename)) { + err(ctx, v.at, "local array `{v.name}` element type {describe(et)} has no MSL form") + return + } + var mk : ExprMakeArray? + if (v.init is ExprMakeArray) { + mk = v.init as ExprMakeArray + } elif (v.init is ExprCall) { + let ec = v.init as ExprCall + if (length(ec.arguments) == 1 && ec.arguments[0] is ExprMakeArray) { + mk = ec.arguments[0] as ExprMakeArray + } + } + if (mk == null || length(mk.values) != n) { + err(ctx, v.at, "fixed-array local `{v.name}` hoists to a program-scope constant table - its initializer must be a `fixed_array(...)` of exactly {n} elements") + return + } + var vals : array + vals |> reserve(n) + for (el in mk.values) { + if (!(el is ExprConstInt || el is ExprConstUInt || el is ExprConstInt64 || el is ExprConstUInt64 || + el is ExprConstFloat || el is ExprConstDouble || el is ExprConstInt8 || el is ExprConstUInt8 || + el is ExprConstInt16 || el is ExprConstUInt16)) { + err(ctx, v.at, "fixed-array local `{v.name}` hoists to a program-scope constant table - every element must be an integer or float scalar literal (bool, half and vector literals do not hoist)") + return + } + vals |> push(emit_value(ctx, el)) + } + let vtxt = build_string() $(var w) { + for (i in range(n)) { + if (i % 8 == 0) { + w |> write(" ") + } + w |> write(vals[i]) + if (i != n - 1) { + w |> write(",") + } + if (i % 8 == 7 || i == n - 1) { + w |> write("\n") + } else { + w |> write(" ") + } + } + } + let aname = string(v.name) + var an = local_msl_name(ctx, aname) + var txt = "constant {ename} {an}[{n}] = \{\n{vtxt}\};" + if (key_exists(ctx.const_tables, an) && ctx.const_tables[an] != txt) { + var cand = "{an}_c{ctx.splice_seq}" // a same-named table with different content in another helper + while (key_exists(ctx.const_tables, cand)) { // a helper's own table may already hold the generated name + ctx.splice_seq++ + cand = "{an}_c{ctx.splice_seq}" + } + ctx.splice_seq++ + an = cand + txt = "constant {ename} {an}[{n}] = \{\n{vtxt}\};" + note(ctx, "stmt.let.const_table.renamed") + } + ctx.renames[aname] = an + ctx.const_tables[an] = txt + note(ctx, "stmt.let.const_table") +} + def private emit_stmt(var ctx : MslCtx; e : Expression?; indent : string; var lines : array) { if (e == null) { return @@ -3028,9 +3106,10 @@ def private emit_stmt(var ctx : MslCtx; e : Expression?; indent : string; var li } if (v._type != null && v._type.baseType == Type.tFixedArray && v._type.firstType != null && v._type.firstType.baseType != Type.tFixedArray) { - // single-dim fixed-array local; das zero-init semantics, no initializer form + // single-dim fixed-array local; das zero-init semantics. A `let` with an + // all-literal initializer hoists to a program-scope `constant` table instead. if (v.init != null) { - err(ctx, v.at, "fixed-array local `{v.name}` takes no initializer - das zero-init is the MSL form") + hoist_const_table(ctx, v) continue } let et = v._type.firstType @@ -3669,6 +3748,11 @@ def public generate_msl(fn : FunctionPtr; var errors : array; cfg : MslE hs |> sort() // table iteration order is hash-based — pin the emitted order helper_text = join(hs, "\n") + "\n" } + if (!empty(ctx.const_tables)) { + var cts <- [for (v in values(ctx.const_tables)); v] + cts |> sort() // pinned order, same reason + helper_text = join(cts, "\n") + "\n" + helper_text + } if (!empty(ctx.ustructs)) { // @uniform struct definitions come first — the kernel signature names them. // Sorted: table order is hash-based diff --git a/plans/jit_compile_time.md b/plans/jit_compile_time.md new file mode 100644 index 0000000000..2da927a036 --- /dev/null +++ b/plans/jit_compile_time.md @@ -0,0 +1,55 @@ +# JIT compile-time program - the fast dev loop and its invalidation ledger + +The iquant arc's session cost is dominated by recompilation, not by racing or by the model: +one zen2 session logged ~20 of 32 hours in front-end + LLVM codegen + full-walk mints. The +mints are fixed (`--tune-only`, skills/tune.md); this file carries the compile-time half: +the measured levers, the adopted dev-loop, and the invalidation defects that cap it. + +## Measurements (2026-08-31, zen2, lcpp_bench: 99 das-modules, 7215 fns, 16 threads) + +| lever | cold | warm | after one mid-chain kernel edit | +|---|---|---|---| +| default (monolith JIT, no AST cache) | 196 s | 48 s | 196 s | +| `--jit-split-modules=-1` (after `--`) | 65 s (codegen 19.2 s vs 155.8 s - 8.1x) | 48 s (codegen 1.1 s) | 65 s (72/99 partitions re-emit) | +| `-module-cache ` (front end 39.7 s -> 3.1 s) | +write | 5.2 s combined | no help - full re-parse + re-write | +| front end alone (parse + infer + interp macros) | 39.7 s | - | 39.7 s | + +The combined warm loop (AST cache + split obj cache) is 5.2 s vs 48 s; the edit loop is +59 s vs 196 s. Most tuner/parity/bench invocations recompile UNEDITED programs (mints, +re-execs, A/B reruns), so the warm 9x is where the session hours went. + +## Adopted dev loop (the kernel workflow) + +``` +daslang.exe -jit -module-cache .jitted_scripts/module_cache/.dascache .das \ + -- --jit-split-modules=-1 +``` + +One cache file per app, under `.jitted_scripts/` (already gitignored). Caveats that stay: + +- **Bench t/s rows run the stock monolith invocation** until an A/B proves split-neutral - + split loses cross-module inlining across partitions, so its artifact is not the shipped one. +- The first cache-backed run re-keys the JIT cache once (ledger item 1) - expect one cold + codegen right after a cache write. +- QUIRK 21 (HOW_TO_ADD_A_FORMAT.md) still applies: macro-side emitter edits are invisible + to every key - `rm -rf .jitted_scripts` after `dasllama_gemm_gen.das` changes (which also + drops the module caches stored under it - correct, the AST changed). + +## Invalidation ledger + +1. **Deser-vs-parse AOT-hash divergence.** A `-module-cache` deserialized AST produces + different per-function AOT hashes than the freshly parsed one, so the first run after a + cache write re-keys the whole JIT DLL/obj cache (one wasted cold codegen); hashes are + stable from then on. Done = the same program hashes identically parsed or deserialized, + witnessed by a cache write followed by a JIT cache HIT. +2. **Split obj keys are a chained prefix fold - one edit re-emits the suffix.** Editing + `dasllama_repack` re-emitted 72 of 99 partitions (18.3 s, nearly full cold): every module + AFTER the edited one in program order re-keys, though their IR is unchanged. Done = + per-module keys derived from the module's own (transitive-interface) hash, so a leaf edit + re-emits ~itself; measured target: an early-chain kernel edit under ~5 s of codegen. +3. **The AST module cache is all-or-nothing under an edit.** The same repack edit paid the + full 39.7 s front end and rewrote the whole 148 MB cache - no per-module reuse of the + unedited prefix. Done = an edit re-parses the edited module + dependents only. +4. **Macro-emitter changes are invisible to every cache key** (QUIRK 21; QUIRK 15's unquirk + note wants the generator hash folded into the sidecar identity too). Done = an emitter + edit invalidates exactly the families it generates. diff --git a/plans/unquirk_pass.md b/plans/unquirk_pass.md new file mode 100644 index 0000000000..6d5283f09f --- /dev/null +++ b/plans/unquirk_pass.md @@ -0,0 +1,172 @@ +# Unquirk pass - end of the iquant format ladder arc + +The ladder is complete (9 formats x 4 tiers, ea1f41d2c). This plan covers everything between +that and the arc's PR: the tuner rework, the KqPlanes unquirk, the cache/one-off fixes, the +HOW_TO document split, and two PRs. Order: B + C1 + D1 -> **PR-1** (the ladder ships, the +parallel M5/M1 session forks) -> A + D2 + E -> **PR-2**; every phase ends green and +committed. Kernel catch-up runs on zen2 in parallel with the mac session after PR-1. + +Evidence base (2026-08-31 sidecar audit): a full mint races 39 families / 637 seats. The 16 +tile-gen families crown `dot_maddubs_width256_mr8` on every format on zen2 and mr8-shapes on +ARM; the 22 `[tuned]` vector-utility families have winner-vs-runner-up gaps of 0.0-1.4% on +zen2 AND on M5 (25/25 under 2%). Cross-sidecar winner agreement 34/39; all 5 diffs are noise +ties or a skipped-seat artifact. The race rediscovers ~2 facts for ~20 minutes per app per box. +M5 Max carries FEAT_I8MM + FEAT_SME2/2p1 + BF16 that our ARM seat space does not offer a +single seat for - its mint provably lands on M1's answers today. + +## Phase B - tuner rework (kills QUIRK 11, 15, 17, 18) + +Target: mint on a known box = zero races; full race on new silicon (zen4, M5) = a couple of +minutes. If that target holds, the dasllama.io tune service is removed entirely (Boris's call, +outside this PR). + +**B1. Prune the seat space.** +- Tile gens (`dasllama_math_gen.das`, 16 families x 11 seats): keep `mr8` (generic/ARM + baseline), `maddubs w256 mr8` (requires avx2 - the x86 baseline), `vpdpbusd w256 mr8` + (requires vnni), `vpdpbusd w512 mr16` (requires avx512vnni+bw). Drop mr4 and every nrsplit2 + twin (never won beyond noise on any box). 11 -> 4 seats, only 2 raceable per box class. +- q8q8 (15 seats): keep the observed winners' axes only - `kstep2` (zen2 crown), + `mr8_budget` (M5 crown), plus the maddubs/vpdpbusd kstep2 twins under their gates. ~15 -> 4. +- `[tuned]` utilities (`dasllama_tune.das::build_grid()`, one place): grid 20 -> the union of + every winner ever recorded across zen2/m1/m4/m5 sidecars (~6: plain, u2, vec8_u2, vec16, + vec16_u2, vec4_u2). With B2 they stop racing on known classes anyway; the pruned grid is + what a new-silicon full race walks. + +**B2. Arch default profiles (kernels-only - the sharper form of the CPU/GPU split).** +- Profiles carry KERNEL WINNERS ONLY. The runtime-knob section never travels: its knobs mix + device knobs (metal_*, batch_grid_2d, lane caps) with box-shape knobs (core counts, cache + budgets, thread receipts) - neither is class-portable, so nothing GPU- or box-side can + mis-inherit by construction. Knobs stay code-defaults on a fresh box until a deliberate + `--tune`; existing minted sidecars keep theirs. +- Checked-in profiles: `modules/dasLLAMA/performance/defaults/.tune-defaults.json` + (the extension dodges the `*.tune.json` gitignore), `x86-avx2` minted from zen2, + `arm-neon` from m1. `x86-vnni512` comes from the c7a trial; `arm-i8mm`/`arm-sme` exist + only once those kernels do. `harness/export_tune_profile.das` mints one from a sidecar; + the framework fn is `tune_profile_export`. +- Adoption is FRAMEWORK-side, declared on the scope - `[tune_scope(defaults = "dir")]` - + and runs in the auto/restart guards ahead of the scope resolver: walk `tune_class_chain()` + down from `tune_cpu_class()`, adopt the first matching profile (version-pin checked) into + the app sidecar as a normal local write. A covered scope races NOTHING. +- Race-on-unlock: an adopting start races ONLY families with a seat whose `requires=` the + host passes but the profile's minting box did not (recorded as the `features` fingerprint + every sidecar save now stamps) - plus families the profile predates. The residue rides the + existing `--tune-only` machinery (the guard arms `DAS_TUNE_ONLY` for the tuner it spawns). + No new flag: bare `--tune` already skips adoption entirely - it IS the full new-silicon + walk. `--tune-only` unchanged. + +**B3. `DAS_TUNE_POLICY=reference`** - a policy value that serves the original bodies outright +(QUIRK 18's missing spelling). Small, framework-side. + +**B4. Validation + the measurement Boris's service decision needs.** MEASURED 2026-08-31: +zen2 fresh full mint under the pruned grids 488 s (gen probe 414 + tuned/knobs 66; noise cv +<= 0.33%); zen2 parity cold start against the shipped x86-avx2 profile = 0 races (the 285 s +wall is two monolith JIT compiles, pre-adopt + re-exec), stream identical to the recorded +stamped one; c7a.4xlarge (EPYC 9R14 zen4, 16 vCPU) full new-silicon mint **316 s** (gen probe +253 + tuned 58, noise cv <= 0.21%) - `dot_vpdpbusd_width512_mr16` crowns EVERY tile family +and q8q8 takes the bias128/gkstep2 vnni512 stamp; x86-vnni512 profile exported + checked in. +m1 gen half 734 s (M1, 8 threads; mr8 everywhere) - its `tune_kernels` Metal arm had a +macOS-only compile error (two require paths for MetalTensorRaceResult), fixed in-pass. +- dasLLVM tune tests (test_tuned/test_grid/llvm_tune_*) extended for profile resolution + + the split sidecar; gen probe test+tune modes; parity 40/40; one bench row vs current + numbers (winners are unchanged by construction - prove it, don't assume it). +- zen2: delete sidecar, cold start -> 0 races, numbers match. Wall time recorded. +- M5 (ssh): fresh mint under arm-neon profile -> 0 races. Wall time recorded. +- zen4 (ssh, off-hours - it hosts dasweb): `--tune-full` new-silicon race, wall time + recorded (target <= ~2-3 min), its x86-vnni512 profile checked in. First box where + vpdpbusd can actually win - report the crowns. +- skills/tune.md + tune_for_this_box.md + HOW_TO QUIRK 11/15/17/18 text updated in-phase. + +## Phase A - the KqPlanes table (kills QUIRK 1, 2, 3, 5, 7, 8, 13, 19, 25, 26) + +One per-format record on `Model` - quant plane, scale plane, mr, qsb/ssb strides, stream +code, schema id, device-form flag - indexed by `KqFmt`; every `fmt ==` ladder becomes a +lookup. Staged so each commit is green; the fast dev loop (`-module-cache` + +`--jit-split-modules=-1`) keeps the cycle ~1 min; after EVERY stage that touches layout or +pack, delete `*.dlim` and e2e the MIXED vehicle (the iq2xs requant carrying q2_K - QUIRK +26's lesson), not a pure one. + +- **A1.** The table + accessors (`kq_plane_q/s`, `kq_mr`, ...) + the id-derivation table + (schema id + stream code from `KqFmt` - QUIRK 5/25 become one row each) + `test_kqformat` + pins table<->enum. No consumers move yet. +- **A2.** Collapse the load/common/layout ladders (12 + 13 + 3 arms-per-format sites). +- **A3.** Collapse math_gen/math_default/math (8 + 3 + 4); the repack-mr freeze ladder + (QUIRK 8) becomes a table write; `serialize_image_meta` serializes the table and + `dlim_identity` formats from it (QUIRK 3/7 - the hand list and the count tripwire go). + Image meta layout changes -> images re-bake once; C1 lands before this so stale images + die loudly. +- **A4.** `tests/_kq_fixtures.das`: one per-format dispatch (`kq_transcode_sb`, + `kq_dequant_sb`, `kq_dot`, `kq_repack`, `kq_gemv_gen`, `kq_tile_gen`) shared by + `test_kquant` and `gen_tune_probe`; ONE family array walks both probe modes (QUIRK 2; the + stub-family rule of QUIRK 16 gets a `stub` flag in that array instead of a hand list). +- **A5.** Metal test records (QUIRK 13): per-format `{src, entry, fastmath, tgmem}` table + replaces the four parallel ternary chains. m1 sync + metal gemv/gemm gates. +- **A6.** The device gather's two branches key on the table's device-form flag (QUIRK 19). + +## Phase C - cache invalidation + +- **C1** (lands FIRST, before A3): a `PACK_VERSION` folded into the `.dlim` filename hash + (QUIRK 24) - a layout/pack edit re-bakes instead of serving stale planes. Gate: bump -> + next run logs "baked", not "mapped". +- QUIRK 21 (generator bodies into the JIT DLL hash) STAYS on `plans/jit_compile_time.md` - + cache-keying work, not this pass. + +## Phase D - one-offs + +- **D1**: posix arm for `stage_jit_backend` in `utils/mcp/setup.das` (QUIRK 12). +- **D2**: `static_if ... elif` in the MSL emitter (QUIRK 14) - timeboxed to ~2h, dropped to + the metal followup doc if the emitter fights. +- Ledgered, not done here: QUIRK 6 resolves as an INTRINSIC builtin (Boris's ruling: daslang + has no u8/u16 operator math and will not start - a `byte_or(...)`-shaped builtin, never + `|=` on uint8), SPIR-V constant-storage lowering for literal fixed_arrays (QUIRK 20, + emitter). Both proposed as lint rules too: + module-global read reachable from a worker/team-lane kernel (QUIRK 9/27, bitten twice), + dynamically-indexed fixed_array local in a kernel class (QUIRK 20's cheap half). + +## Phase E - the document split (the detour) + +`HOW_TO_ADD_A_FORMAT.md` (today: recipe + 27 quirks + 9 phase diaries) becomes: + +1. **`HOW_TO_ADD_A_FORMAT.md`** - an actual how-to: steps 0-8 rewritten post-unquirk (the + table shrinks the step count), the E-cluster rules folded in as steps/warnings (stubs + first Q4; worker-table rule Q9/27; stale-binder Q10; stub-out-of-test-list Q16; + kernel-probe-not-e2e Q22; census-the-vehicle Q23), and the quirk ledger cut to the + survivors (whatever A/B/C/D did not kill). +2. **`performance/PERF_LEDGER.md`** - the nine per-format four-tier tables, the three gap + classes, each format's vehicle GGUF, and the box inventory: which model files sit on + zen2 for testing, the m1/m5 paths, the ref llama.cpp builds per box. +3. **`followup_metal.md`** - twin of followup_vulkan.md: the Metal tg items (0.78-0.93x), + the Q22 dispatch-loop probe method, D2 if dropped. +4. The per-format phase diaries -> `/history` (doc_archiving skill), linked from the ledger. + +LAWS.md (modules/dasLLAMA) gets the rulings from this session: prune the families, saved +zen2/M1 defaults, race only on feature unlock, CPU/GPU tune split, the doc split. + +## Phase F - TWO PRs (the mid-plan cut that starts the M5 session) + +- **PR-1** (after B): the 9-format ladder the branch already carries + the tuner rework + + C1 + D1. Full make_pr chain. On merge the parallel M5/M1 session forks from master - + free mints (arm-neon profile), fast dev loop, pack-versioned images. +- **PR-2** (fresh branch off master): A (KqPlanes) + D2 + E (the doc split - it depends on + A's survivor-quirk ledger anyway). Full make_pr chain again. +- Parallel-window rule: the mac session owns kernel BODIES and additive arms (i8mm/SME + seats, Metal levers); the zen2 session's A owns dispatch/tables. A5 (the Metal test + records) runs EARLY in A's sequence to shrink the shared-file window. + +## Open questions (Boris) + +1. The new-silicon trial runs on AWS (us-west-2, CLI configured on this box), not the dasweb + zen4. No UMA + built-in-GPU x86 box exists on AWS; per goal the candidates are + c7a.2xlarge (zen4, avx512-vnni - the vnni512 mint), c7i.2xlarge (Sapphire Rapids, + vnni + AMX), c8g.2xlarge (Graviton4: i8mm + SVE2 - the arm-i8mm class once those + kernels exist), g4ad.xlarge (zen2 + Radeon V520 dGPU - Vulkan-on-AMD-driver, not UMA). + True UMA + Vulkan is Strix Halo territory, outside any cloud. Boris picks the box(es). +2. RESOLVED: profile classes key by the feature set that gates seats (winners follow + instruction sets, not box identity). Host resolves to the highest class it satisfies + that has a shipped profile, falling back down the chain (M5 today -> arm-neon). +3. RESOLVED yes: `performance/PERF_LEDGER.md` + `followup_metal.md`. +4. RESOLVED yes: D2 timeboxed, dropped to followup_metal.md if the emitter fights. + +## Rough wall-clock + +B ~1 day incl. the m5/zen4 trials; A ~1-2 days (the wide mechanical stages, each gated); +C/D hours; E ~half day; F the usual chain. Everything on `bbatkin/iquant-formats`. diff --git a/skills/LAWS.md b/skills/LAWS.md index eabca6ccb9..2d5713a478 100644 --- a/skills/LAWS.md +++ b/skills/LAWS.md @@ -13,3 +13,13 @@ compacted, or cited as rules. - 2026-08-31 - `review_md.md`: review-round flashlight item 4. Boris ruled ("yes") to bless a charter-carrying `README.md` as an architecture doc rather than splitting one out - `site/README.md` is the precedent; both playground checklists keep their pointers. + +## 2026-08-31 - skills/tune.md (shipped defaults profiles, DAS_TUNE_POLICY=reference) + +Boris, on the tune mint being the workflow blocker (20+ minutes per app per box): "we prune the +families. we save M1/zen2 defaults. and if winners are available (vnni), some magic M5 stuff ... +then there is a race. if nothing available - no race." Profiles are keyed by CPU feature class +("profile classes key by the feature set that gates seats" - he asked to elaborate and accepted +it), and separate CPU from GPU: "do u want to separate CPU and GPU tunes?" - resolved as +kernels-only profiles, runtime knobs never travel. He will remove the dasllama.io tune service +if the new-silicon race drops to a couple of minutes. diff --git a/skills/internal/llvm_tune_internals.md b/skills/internal/llvm_tune_internals.md index 9dd9a30d0d..3fe0d1f0b3 100644 --- a/skills/internal/llvm_tune_internals.md +++ b/skills/internal/llvm_tune_internals.md @@ -45,6 +45,9 @@ file is only what maintaining the implementation needs on top of it. - `cpu_supports` must answer on arm64 as well as x86 (sysctl / `AT_HWCAP` / `IsProcessorFeaturePresent`, LLVM target-feature spellings). An x86-only probe that fails closed leaves every arm `requires=` gate silently never firing. +- `DAS_TUNE_POLICY=reference` is env-only by construction: the `[tune]` stamps in required + libraries compile before the root that would declare it, so `missing = "reference"` is + rejected at the annotation. ## The gates @@ -85,4 +88,28 @@ tuner is a seconds-fast fake that upserts through `tune_manifest_set`. Two scope sidecar prove the upsert preserves the other's keys, and that tuned-ness is per-key completeness. `llvm_tune_manifest.das` covers the write->stamp round trip and the staleness rail (back-dating with `set_mtime` + `mktime`). Both run under `-jit` and short-circuit to -pass when `!jit_enabled()`. +pass when `!jit_enabled()`. The defaults-profile rail is `llvm_tune_profiles.das` (+ its +`_client` / `_lib` / `_tuner` fixtures): it writes its profile files at runtime under the +untracked `llvm_tune_profiles_defaults/`, named by this box's own `tune_cpu_class()`, and fakes +a less-capable minting box with an empty `features` field to force the unlocked-seat race. + +## Shipped defaults profiles - where the pieces live + +- `defaults=` resolves against the declaring file at `[tune_scope]` apply and rides + `TuneScope.defaultsPath`; the guard emission bakes it, together with the scope's + `scope_gated_families_joined` string - `family=requires+requires` rows, `;`-joined, only for + families with a gated row, read off the AST - into the `tune_auto_prepare` / + `tune_restart_needed` call, because the runtime has no AST. +- Adoption is runtime-side (`profile_try_adopt`): walk `tune_class_chain()` from + `tune_cpu_class()`, skip a profile whose version pin differs (the walk continues down), + `tune_sidecar_merge` the kernels into the app sidecar as a normal local write, then diff the + gated string against the profile's `features` fingerprint. It returns adopted-or-not and + hands the residue back in its `onlyFilter` out-param; the two signals read together - true + with an empty filter means fully covered, true with a filter means adopted-and-owing, and + an owing adoption skips the scope resolver and rides `run_scope_tuner(scope, onlyFilter)`, + which arms `DAS_TUNE_ONLY` for that one spawn. It declines outright when + `tune_manifest_path()` no longer equals the `manifestPath` the guard was compiled for. +- `tune_profile_export` carries `kernels` plus the string provenance that survives, dropping + `binary` / `box` / `engine_sha` and re-stamping `origin` = `profile`, `class` = the export's + class argument, and `written`: a profile's identity is class + version pin + features, never + a commit. diff --git a/skills/tune.md b/skills/tune.md index 4f67390f57..cb5a7b0fcf 100644 --- a/skills/tune.md +++ b/skills/tune.md @@ -146,7 +146,10 @@ completeness check everywhere it runs - the policy rail, `daspkg release per-app file - every library's tuner **upserts its own keys** and preserves everyone else's (that upsert is the isolation contract; "is this scope tuned" is per-key completeness, not file existence). Reading winners needs no scope -at all - every `[tune]` resolves against the app sidecar. +at all - every `[tune]` resolves against the app sidecar. `defaults=` (optional, +resolved against the declaring file) names the directory of shipped defaults +profiles (the *Shipped defaults profiles* section): an untuned box adopts its +CPU class's profile instead of racing. ```{warning} @@ -158,6 +161,48 @@ public surface (a blanket `public` on a module that also re-exports `jobque_boost` floods requirers with name ambiguities). ``` +## Shipped defaults profiles - `[tune_scope(defaults = "dir")]` + +Kernel winners follow **instruction sets, not boxes**: within one CPU feature +class the same permutation wins on every box - what differs from box to box +inside a class sits in the noise band - so one minted answer serves them all. +(A seat is the permutation slot one kernel family's winner occupies.) A +library ships those answers as checked-in profile files - +`/.tune-defaults.json`, each the `"kernels"` section of the +minting box's full mint plus a provenance recording that box's `features` +fingerprint (and any `version_of=` pin value). Runtime knobs, race tables and +box identity never travel - all three are properties of the box that measured +them, not of its class. + +The class names follow the features `requires=` can gate: `x86-vnni512` / +`x86-vnni256` / `x86-avx2` / `x86-base`, `arm-i8mm` / `arm-neon`; any other +architecture gets `-`, a key no shipped profile matches, so +such a box always races (`tune_cpu_class()` computes this box's, +`tune_class_chain()` its adoption ladder). An untuned auto/restart start walks the ladder from the box's own +class downward, adopts the first profile found into the app sidecar (a normal +local write - staleness, box identity and the JIT DLL cache re-key all behave +as for a mint), and then races only what the profile could not answer: + +* families the profile has no entry for (it predates them), and +* families with an ISA-gated seat this box unlocks that the profile's minting + box could not race - decided by comparing each `requires=` against the + profile's recorded `features`. + +That residue races through the ordinary tuner spawn with the `--tune-only` +filter armed, so a box whose own class is `x86-vnni512`, adopting an +`x86-avx2` profile, races only the families whose `requires=` names an +AVX-512 VNNI feature - and a box whose class profile is exact races +**nothing**. Adoption is skipped entirely under `--tune` - a forced re-race +stays a full local mint. A profile whose pinned version differs is skipped and +the walk continues to the next class down; when nothing in the ladder both +exists and matches, the box races the full grid. + +A maintainer produces a profile on the minting box after a full `--tune` mint +with `tune_profile_export(path, klass)` - it refuses an empty or stale +sidecar. Every sidecar save also stamps the box's `features` fingerprint into +provenance - that is what later lets an adopting box tell the seats the +profile's minting box already raced from the ones its own ISA unlocks. + ## Application policy - `[tune_policy]` and `--tune` **Untuned does not start.** Any application whose program root has a `main` @@ -200,11 +245,29 @@ def main { Programs whose root has no `main` never get the default - dastest-driven test files run `[test]` functions, so the test suite never tunes-on-start. +`DAS_TUNE_POLICY=reference` is the A/B truth tier: it serves the ORIGINAL +bodies - no stamps at all, for `[tune]` families and loop-hint `[tuned]` +kernels alike - on a fully tuned box too, and announces itself once per +compile. `fallback` is not it, because a chain's first viable perm can be the +very stamp under test. It is environment-only: `[tune_policy(missing = +"reference")]` is a compile error, because the `[tune]` stamps in required +libraries compile before the root that would declare it. + `--tune` after `--` on the application's command line forces the tune path even when the sidecar is complete (a re-tune; the flag is stripped from the re-exec so the child converges). `DAS_TUNE_POLICY` overrides the declared value - `DAS_TUNE_POLICY=fallback` is the CI kill switch. +`--tune-only ` (comma-separated; implies `--tune`) re-tunes only the +kernel families whose name contains one of the tokens - after landing one +family's kernels, `--tune-only gemm` re-mints that family in seconds instead +of walking every family the app owns. A skipped family races nothing and +writes nothing, so its sidecar entry survives the upsert. The filter rides +the tuner process chain as `DAS_TUNE_ONLY`, and a harness consults +`tune_family_selected(name)` at each family boundary. On a box with no +complete sidecar a filtered mint leaves the scope incomplete (the startup +warning names the missing kernels) - the first full mint still has to happen. + Two further escapes exist for a run that must not mint. `--jit-opt-level=0` flips the *injected default* to `fallback`, because winners raced under O3 codegen mean nothing at O0; a declared `[tune_policy]` is left alone, and @@ -310,7 +373,10 @@ Two seams let a supervisor or a network service participate: - `tune_set_scope_resolver(fn)` - registered from an `[init]` (which must run before the guard at the top of `main`), consulted by - the auto/restart policy guards before spawning a scope's tuner. A resolver + the auto/restart policy guards before spawning a scope's tuner - and after + the *Shipped defaults profiles* section's adoption, which outranks it: a + partial adoption, one that still owes a race, skips the resolver entirely + and races the residue instead. A resolver that can satisfy the scope another way (dasLLAMA's exchange client downloads a matching per-box sidecar from dasllama.io) returns true; completeness is re-checked, never trusted, and `--tune` never consults it. @@ -503,10 +569,15 @@ tables behind each multi-variant winner under `"race"`, and `"provenance"` { "kernels" : { "gemm" : "kstep4", "gemm_gemv" : "reference" }, "provenance" : { "binary" : "...", "platform" : "windows", "arch" : "x86_64", + "features" : "avx2;f16c;fma;sse4.2", "noise" : "ok", "noise_probes" : "start cv 0.40%; mid1 cv 0.14%; end cv 0.28%" } } ``` +`features` is the box's fingerprint over the known `requires=` feature names, +stamped on every save - it is what a shipped defaults profile exported from +this sidecar uses to tell covered seats from unlocked ones. + It is a per-app, per-box artifact - gitignored (`*.tune.json`), and any change re-keys the JIT DLL cache automatically (the winning permutation's args fold into the DLL basename). diff --git a/tests/metal/test_metal_consttab.das b/tests/metal/test_metal_consttab.das new file mode 100644 index 0000000000..355d2dfba4 --- /dev/null +++ b/tests/metal/test_metal_consttab.das @@ -0,0 +1,112 @@ +options gen2 +options indenting = 4 + +// Real-GPU behavioral gate for the hoisted constant tables: a `let` fixed-array local of literals +// lowers to a program-scope `constant` array (renamed when a helper reuses the name with different +// content), and the kernel's reads through it must return the staged values. The CPU run of the +// same body is the oracle; the GPU half is static_if-gated on das_metal and skips without a device. + +require dastest/testing_boost public +require metal/msl_shader +require _metal_common // nolint:STYLE030 — used inside the das_metal static_if arm only + +let N = 256 +let TPG = 64u + +def private ct_pick(i : uint) : uint { + let tbl = fixed_array(0x11u, 0x22u, 0x33u, 0x44u, 0x55u, 0x66u, 0x77u, 0x88u) + return tbl[i & 7u] +} + +def private ct_pick0(i : uint) : uint { // its own table carries the name the rename would generate + let tbl_c0 = fixed_array(0x100u, 0x200u, 0x300u, 0x400u, 0x500u, 0x600u, 0x700u, 0x800u) + return tbl_c0[i & 7u] +} + +// the same local name with different content: the emitter renames the hoisted twin +def private ct_pick2(i : uint) : uint { + let tbl = fixed_array(0x1u, 0x2u, 0x4u, 0x8u, 0x10u, 0x20u, 0x40u, 0x80u) + return tbl[i & 7u] +} + +class ConstTabGpu { + @ssbo @binding = 0 src : array + @ssbo @binding = 1 dst : array + @ssbo @binding = 2 fdst : array + + [metal_kernel(name="consttab_gpu_msl"), marker(no_coverage)] + def consttab { + let i = gl_GlobalInvocationID.x + let sgn = fixed_array(1.0, 0.5) + dst[i] = src[i] + ct_pick0(i) + ct_pick(i) + ct_pick2(i) + fdst[i] = sgn[i & 1u] * 2.0 + } +} + +[test] +def test_consttab_gpu_vs_cpu(t : T?) { + t |> run("hoisted constant tables: GPU == CPU-reference") <| @(t : T?) { + var oracle = new ConstTabGpu() + oracle.src |> resize(N) + oracle.dst |> resize(N) + oracle.fdst |> resize(N) + for (k in range(N)) { + oracle.src[k] = uint(k * 3) + } + for (k in range(N)) { + gl_GlobalInvocationID = uint3(uint(k), 0u, 0u) + oracle->consttab() + } + t |> equal(oracle.dst[1], 3u + 0x200u + 0x22u + 0x2u) // the oracle itself: src 3, the helper's own tbl_c0[1], tbl[1], the renamed tbl_c1[1] + t |> equal(oracle.fdst[1], 1.0) // sgn[1] * 2 + static_if (typeinfo builtin_module_exists(das_metal)) { + var gpu_bad = -1 + with_metal_device() $(dev : MetalDevice?) { + if (dev == null) { + feint("no Metal device on this box; GPU compare skipped\n") + return + } + var perr : string + var pso = pipeline_from_source(dev, consttab_gpu_msl, consttab_gpu_msl_entry, consttab_gpu_msl_fastmath, perr) + t |> success(pso != null, "pipeline_from_source: {perr}") + if (pso == null) { + return + } + var queue = metal_new_command_queue(dev) + var bsrc = buf_upload(dev, oracle.src) + var bdst = buf_fill(dev, N, 0u) + var bfdst = buf_fill(dev, N, 0.0) + var bufs <- [bsrc, bdst, bfdst] + var rerr : string + let ran = run_compute_1d(queue, pso, bufs, uint(N), TPG, rerr) + t |> success(ran, "run_compute_1d: {rerr}") + if (ran) { + gpu_bad = buf_mismatch_exact(bdst, oracle.dst) + buf_mismatch_exact(bfdst, oracle.fdst) + } + bufs |> clear() // non-owning handles — clear() skips pointee finalize + unsafe { + delete bufs + } + metal_release(bsrc) + metal_release(bdst) + metal_release(bfdst) + metal_release(pso) + metal_release(queue) + } + if (gpu_bad >= 0) { + t |> equal(gpu_bad, 0) + } + t |> equal(metal_live_object_count(), 0l) + } else { + feint("das_metal is not built on this platform; CPU-reference half only\n") + } + unsafe { + delete oracle + } + } +} + +[export] +def main { + return +} diff --git a/tests/msl/REVIEW.md b/tests/msl/REVIEW.md index cf5cf5f07f..e576b04203 100644 --- a/tests/msl/REVIEW.md +++ b/tests/msl/REVIEW.md @@ -11,8 +11,11 @@ from `declared_msl_census` or a fixture's census from `all_msl_censuses` (both i `_msl_common.das`). Emitted text cannot be read back into constructs, so the census is the only measure of coverage. +**A diff that adds a fixture outside `_fail_closed/` adds its census to `all_msl_censuses` +(`_msl_common.das`) in the same change.** A fixture left out of that list emits kinds the +check never sees, so both directions pass with the fixture uncounted. + **Weakening `test_msl_fail_closed.das`'s `check_rejects` is a defect.** It asserts each `_fail_closed/` fixture fails to compile and that the compile error contains the fixture's needle - the substring naming the rejected construct. The fixture-has-its-assert pairing is -machine-checked both directions (`modules/dasMetal/REVIEW.das`'s `check_fail_closed_sync`; -its weakening rule lives with that checklist). +machine-checked both directions (`modules/dasMetal/REVIEW.das`'s `check_fail_closed_sync`). diff --git a/tests/msl/_fail_closed/_fc_array_init_nonconst.das b/tests/msl/_fail_closed/_fc_array_init_nonconst.das new file mode 100644 index 0000000000..e9c4599cca --- /dev/null +++ b/tests/msl/_fail_closed/_fc_array_init_nonconst.das @@ -0,0 +1,20 @@ +// Fail-closed fixture: a fixed-array local initializer with a non-literal element. Only an +// all-literal `let` hoists to a program-scope constant table; anything else has no MSL form. + +expect 50501 + +options gen2 + +require metal/msl_shader + +class FcArrayInitNonconst { + @ssbo @binding = 0 src : array + @ssbo @binding = 1 dst : array + + [metal_kernel(name="fc_array_init_nonconst_msl")] + def kern { + let i = gl_GlobalInvocationID.x + let tbl = fixed_array(1u, src[i], 3u, 4u) + dst[i] = tbl[i & 3u] + } +} diff --git a/tests/msl/_msl_common.das b/tests/msl/_msl_common.das index bff241bb0a..fb3541e83e 100644 --- a/tests/msl/_msl_common.das +++ b/tests/msl/_msl_common.das @@ -692,6 +692,41 @@ class UkStride { } } +// ===== hoisted constant tables ===== +// A `let` fixed-array local with all-literal elements lowers to a program-scope `constant` +// array - the codebook/grid shape quant kernels stage from; reads index it by the local's name. + +// a helper's own table already carries a generated-looking name: the rename below must probe past it +def private ctb_pick0(i : uint) : uint { + let tbl_c0 = fixed_array(0x100u, 0x200u, 0x300u, 0x400u, 0x500u, 0x600u, 0x700u, 0x800u) + return tbl_c0[i & 7u] +} + +def private ctb_pick(i : uint) : uint { + let tbl = fixed_array(0x11u, 0x22u, 0x33u, 0x44u, 0x55u, 0x66u, 0x77u, 0x88u) + return tbl[i & 7u] +} + +// the same local name with different content: the hoist renames the second table (tbl_c) +def private ctb_pick2(i : uint) : uint { + let tbl = fixed_array(0x1u, 0x2u, 0x4u, 0x8u, 0x10u, 0x20u, 0x40u, 0x80u) + return tbl[i & 7u] +} + +class ConstTab { + @ssbo @binding = 0 src : array + @ssbo @binding = 1 dst : array + @ssbo @binding = 2 fdst : array + + [metal_kernel(name="consttab_msl"), marker(no_coverage)] + def consttab { + let i = gl_GlobalInvocationID.x + let sgn = fixed_array(1.0, 0.5) + dst[i] = src[i] + ctb_pick0(i) + ctb_pick(i) + ctb_pick2(i) + fdst[i] = sgn[i & 1u] * 2.0 + } +} + // ===== @uniform structs (the kargs form) ===== // A kernel with a dozen scalars costs a dozen binds; one struct costs one. Fields are 4-byte // scalars only, so the das layout IS the MSL layout and the host writes its struct straight @@ -1293,6 +1328,8 @@ def public declared_msl_census : table { // nolint:STYLE038 — one fl "cvt.i8.f32", // statements "stmt.let", + "stmt.let.const_table", + "stmt.let.const_table.renamed", "stmt.var", "stmt.var.zero", "stmt.assign", @@ -1336,7 +1373,7 @@ def public all_msl_censuses : array { vecarith_msl_census, control_msl_census, loops_msl_census, reduce_msl_census, simd_msl_census, halfarith_msl_census, latconv_msl_census, packed3_msl_census, sgmat_msl_census, sgmat_tiled_msl_census, sgmat_rolled_msl_census, mathcalls_msl_census, userfns_msl_census, - ufxcls_msl_census, ukargs_msl_census, ukstride_msl_census, statearr_msl_census, + ufxcls_msl_census, ukargs_msl_census, ukstride_msl_census, consttab_msl_census, statearr_msl_census, tmmhw_msl_census, tmmhwh_msl_census, tmmq8u_msl_census, tmmq8uh_msl_census, tmmq8uh64_msl_census, tmmq8uh128_msl_census, tmmdeva_msl_census, tmmdevaf_msl_census, tmmnt_msl_census, tmmdevant_msl_census, sgmathalf_msl_census, sgmat_store_tg_msl_census, diff --git a/tests/msl/test_msl_fail_closed.das b/tests/msl/test_msl_fail_closed.das index b3c47f70ff..5ee33b1518 100644 --- a/tests/msl/test_msl_fail_closed.das +++ b/tests/msl/test_msl_fail_closed.das @@ -59,6 +59,7 @@ def test_fail_closed_rejections(t : T?) { check_rejects(t, "_fc_uniform_written", "@uniform member `n` is written") check_rejects(t, "_fc_nan_literal", "non-finite float literal has no MSL form") check_rejects(t, "_fc_uniform_array", "array members are @ssbo buffers") + check_rejects(t, "_fc_array_init_nonconst", "must be an integer or float scalar literal") check_rejects(t, "_fc_return_value", "kernels return void") check_rejects(t, "_fc_recursion", "is recursive - MSL forbids recursion") check_rejects(t, "_fc_ufn_param", "type has no MSL form") diff --git a/tests/msl/test_msl_functions.das b/tests/msl/test_msl_functions.das index 5023e2c97b..216807d8d1 100644 --- a/tests/msl/test_msl_functions.das +++ b/tests/msl/test_msl_functions.das @@ -135,3 +135,33 @@ def test_msl_state_and_naming(t : T?) { "the emitted signature carries the derived entry name") } } + +def private hasq(t : T?; needle : string) { + t |> success(find(consttab_msl, needle) >= 0, "emitted MSL carries `{needle}`") + if (find(consttab_msl, needle) < 0) { + to_log(LOG_ERROR, "MISSING `{needle}` in:\n{consttab_msl}\n") + } +} + +def private count_in_consttab(needle : string) : int { + var n = 0 + var at = find(consttab_msl, needle) + while (at >= 0) { + n++ + at = find(consttab_msl, needle, at + 1) + } + return n +} + +[test] +def test_msl_const_tables(t : T?) { + t |> run("literal fixed-array locals hoist to program-scope constant tables") <| @(t : T?) { + hasq(t, "constant uint tbl[8] = \{") // hoisted from the helper's body + hasq(t, "constant uint tbl_c0[8] = \{") // a helper's OWN table under the generated-looking name + hasq(t, "constant uint tbl_c1[8] = \{") // the same-named different-content table renamed past it, not merged + hasq(t, "constant float sgn[2] = \{") // hoisted from the kernel body + hasq(t, "tbl[") // reads index the hoisted table by name + hasq(t, "tbl_c1[") // the renamed table's reads follow the rename + t |> equal(count_in_consttab("constant uint tbl"), 3, "three distinct tables, none overwritten by a rename") + } +} diff --git a/utils/REVIEW.md b/utils/REVIEW.md index 96962616dc..6d91ccff7e 100644 --- a/utils/REVIEW.md +++ b/utils/REVIEW.md @@ -14,6 +14,11 @@ checklist - not with the checklist of the directory it sits in.** **A file in a `utils/` library directory (`common/`) is reviewed with this checklist and with the checklist of every tool that requires it.** +**A diff under `utils/` that changes how a `.dlim` is built from a gguf, how one is loaded, or +what identifies one - the fields that decide whether two `.dlim`s are the same image - answers +to `modules/dasLLAMA/REVIEW.md` (repo root) too.** A `utils/` diff never opens that checklist +on its own. + **A tool's file outside `utils/` answers to the checklist of the folder that contains it as well as to this one.** @@ -31,21 +36,18 @@ removed entry. **A diff that deletes a tool outright records the decision beside `DAS_UTILS_SHIPPED_EXES` in `CMakeLists.txt` (beside this file), in the same change.** -**A test the diff adds or changes alongside a change under `utils/`, whose load-bearing -assertions a CI lane runs against the change - the assertions that prove it, not a skip-path +**A test the diff adds or changes that covers a change under `utils/`, whose load-bearing +assertions a CI lane can run against the change - the assertions that prove it, not a skip-path assertion - ships with a CI row that executes those assertions, wherever the diff puts the test, added in the same change if no row already covers it.** A row that only compile-checks the test (`dastest --compile-only`) does not execute them. A test whose assertions no row executes never runs again. -**A test whose only executing row runs against an already-deployed artifact (a nightly lane -driving the live site) takes the same obligation as a compile-only row: the PR description -records an executed local run of those assertions against the change.** A lane that tests -production after merge proves nothing about the diff under review. - -**A test the diff adds or changes alongside a change under `utils/`, whose load-bearing +**A test the diff adds or changes that covers a change under `utils/`, whose load-bearing assertions no CI lane can run, ships with a CI row that compile-checks it.** -**A test the diff adds or changes alongside a change under `utils/` that gets a compile-only row records its executed -run in the PR description**: the machine the assertions ran on, what that machine had that CI -lacks, and the pass count. +**A test the diff adds or changes that covers a change under `utils/`, whose only row +compile-checks it or whose only executing row runs against an already-deployed artifact - a +nightly lane driving the live site - records its executed run in the PR description**: the +machine the assertions ran on, what that machine had that CI lacks, and the pass count. A lane +that tests production after merge proves nothing about the diff under review. diff --git a/utils/dasllama-convert/main.das b/utils/dasllama-convert/main.das index fceba938a1..8ddb399f8c 100644 --- a/utils/dasllama-convert/main.das +++ b/utils/dasllama-convert/main.das @@ -358,7 +358,7 @@ def main() : int { } else { print("{js}\n") } - to_log(LOG_INFO, "dasllama-convert: identity {dlim_identity(dc, IMAGE_VERSION)}\n") + to_log(LOG_INFO, "dasllama-convert: identity {image_identity_of(dc)}\n") return rc0 } if (cfg.model == "") { @@ -416,8 +416,8 @@ def main() : int { to_log(LOG_ERROR, "dasllama-convert: --config targets '{want.cpu.backend}' and this host has no repack family to bake it with\n") return 2 } - let want_ident = dlim_identity(want, IMAGE_VERSION) - let have_ident = dlim_identity(dlim_config_current(want.quant), IMAGE_VERSION) + let want_ident = image_identity_of(want) + let have_ident = image_identity_of(dlim_config_current(want.quant)) if (have_ident != want_ident) { to_log(LOG_ERROR, "dasllama-convert: --config could not be fully applied on this host:\n want {want_ident}\n have {have_ident}\n") return 2 diff --git a/utils/mcp/REVIEW.md b/utils/mcp/REVIEW.md index 4e67bfac41..8ac4a30221 100644 --- a/utils/mcp/REVIEW.md +++ b/utils/mcp/REVIEW.md @@ -8,7 +8,9 @@ interpreted through `.mcp.json` instead.** Development runs the server through the python keep-alive supervisor, so an exe form would never be used in development before it ships. -**A diff that adds a top-level `.das` under `utils/mcp/` that `main.das` reaches also adds it to -the `install(FILES ...)` block in `CMakeLists.txt` (repo root), in the same change.** `tools/` -and `subtools/` are globbed; a top-level file left out of the list dies in the shipped SDK on -`error[20605] missing prerequisite` while the in-tree server keeps working. +**A diff that adds a top-level `.das` under `utils/mcp/` that the shipped SDK runs or loads - +`main.das` reaches it, or it has its own `main` that something in the shipped SDK runs - also +adds it to the `install(FILES ...)` block that lists `utils/mcp/main.das` in `CMakeLists.txt` +(repo root), in the same change.** `tools/` and `subtools/` are globbed; a top-level file left +out of the list dies in the shipped SDK on `error[20605] missing prerequisite` while the +in-tree server keeps working. diff --git a/utils/mcp/setup.das b/utils/mcp/setup.das index 3c97812d77..567c51d4f5 100644 --- a/utils/mcp/setup.das +++ b/utils/mcp/setup.das @@ -176,13 +176,12 @@ def ensure_sgconfig(root : string) { // Stage the JIT backend files that a plain `--target daslang` build does NOT produce: // /lib/LLVM.dll (the LLVM codegen backend, loaded by name from dasLLVM's bindings) and -// /bin/lld-link.exe (links each JIT-emitted .dll). Only the dasLLVM shared-lib CMake step +// on Windows /bin/lld-link.exe (links each JIT-emitted .dll). Only the dasLLVM shared-lib CMake step // stages these, and building just the `daslang` target skips it — so without this a fresh // worktree binary silently falls back to the interpreter (`-jit` fails: "can't load LLVM.dll"). // Copy them from the source tree of the daslang running this setup (get_das_root()), which is a // JIT-capable build. When setup itself runs under the target worktree's binary, use DASLANG as a // fallback SDK binary: a worktree-local root cannot supply a backend file that it is missing. -// Windows-only for now (the codegen dll name / linker differ on posix — staging there is TBD). def root_from_daslang_binary(binary : string) : string { let executable_dir = dir_name(get_full_file_name(binary)) let leaf = base_name(executable_dir) @@ -193,9 +192,6 @@ def root_from_daslang_binary(binary : string) : string { } def stage_jit_backend(root : string) { - if (get_platform_name() != "windows") { - return - } let src_root = get_das_root() var fallback_root = "" var fallback_source = "DASLANG is unset" @@ -208,7 +204,12 @@ def stage_jit_backend(root : string) { fallback_source = "DASLANG does not name an existing binary" } } - let jit_files = ["lib/LLVM.dll", "bin/lld-link.exe"] + // lib/LLVM.dll is the codegen backend's name on every platform + // its .version stamp stages with it, or a later cmake configure refetches LLVM + var jit_files <- ["lib/LLVM.dll", "lib/LLVM.dll.version"] + if (get_platform_name() == "windows") { + jit_files |> push("bin/lld-link.exe") + } for (rel in jit_files) { let dst = path_join(root, rel) if (fexist(dst)) {