From 6694d2ecae58b9da00d8556da0111e8c14fb022c Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 00:17:03 -0700 Subject: [PATCH 01/35] bench_metal_kq_race: the per-format Metal KQ race rig + the test-backend-ops ladder patch Rig 1 of the Metal kernel catch-up: races production decode GEMV and prefill mul_mm of all twelve kq formats on synthetic planes (no model, no tuner), dispatch-loop timing per QUIRK 22, whole-output dot-envelope oracle before any timing, arms interleaved round-robin. --ref pins a llama.cpp test-backend-ops row per (fmt, tier, shape); the patch replays the ladder onto the b10659 reference worktree (GGML_KQ_RACE_LADDER=1). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014mq9AKyoAwvD5T1cigTnRw --- .../benchmarks/matmul/bench_metal_kq_race.das | 430 ++++++++++++++++++ .../harness/backend_ops_kq_race.patch | 31 ++ 2 files changed, 461 insertions(+) create mode 100644 modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das create mode 100644 modules/dasLLAMA/harness/backend_ops_kq_race.patch diff --git a/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das b/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das new file mode 100644 index 0000000000..fc9de044f8 --- /dev/null +++ b/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das @@ -0,0 +1,430 @@ +options gen2 +options persistent_heap +options stack = 524288 // every dasLLAMA program root takes this budget (options stack does not unify up from libs) +options _function_length = 0 // the race runner is one flat compile/oracle/time flow per tier +options _cyclomatic_complexity = 0 // per-format record building + the arm ladders follow the format axis +options _dasllama_internal = true + +require daslib/clargs +require strings +require daslib/strings_boost +require daslib/strings_convert +require daslib/fio +require math +require metal/msl_shader +require ?das_metal metal/das_metal_boost +require ?das_metal dasllama/dasllama_metal_kernels +require ?das_metal ../../tests/_metal_kernel_common.das + +// The per-format Metal KQ kernel RACE rig - rig 1 of the kernel catch-up procedure. Races the +// production decode GEMV and prefill mul_mm of every kq format against candidate spellings and +// against a pinned llama.cpp test-backend-ops reference row, on synthetic planes, no model, no +// tuner, seconds per cell. The dispatch-loop probe shape (HOW_TO_ADD_A_FORMAT.md QUIRK 22): +// `reps` back-to-back dispatches per timed encoder (the shared y buffer's hazard serializes +// them), best-of `rounds` encoders, arms interleaved round-robin per round so box drift cannot +// crown a phantom winner. The metric is weight-plane GB/s for GEMV (decode is weight-bound) and +// GMAC/s for mul_mm, plus us/dispatch for the reference ratio. +// +// Correctness before timing, per arm: one dispatch vs the CPU plane-dequant oracle +// (kq_row_ref + double dot) over EVERY output element on the dot-envelope bar +// |gpu-cpu| <= 2e-4 * env + 1e-3 - the report line carries that bound. +// +// bin/daslang -jit modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das -- [--fmts iq3s,iq2] +// [--tiers gemv,mm] [--shapes w13_1b] [--rounds 3] [--reps 50] [--ref _kq_race_ref.tsv] +// +// The --ref tsv pins the llama.cpp side (test-backend-ops -o MUL_MAT perf at these shapes): +// whitespace-separated rows `fmt tier shape us_per_dispatch`; the report prints theirs/ours. + +[CommandLineArgs] +struct Args { + @clarg_doc = "Substring filter over format names, comma list (empty = all)" + fmts : string + + @clarg_doc = "Tiers to race: gemv (decode) and/or mm (512-token prefill), comma list" + tiers : string = "gemv,mm" + + @clarg_doc = "Substring filter over shape names, comma list (empty = all)" + shapes : string + + @clarg_doc = "Best-of timing rounds per cell (round 0 is a discarded warmup)" + rounds : int = 3 + + @clarg_doc = "Dispatches per timed encoder" + reps : int = 50 + + @clarg_doc = "Pinned llama.cpp reference table (tsv: fmt tier shape us_per_dispatch)" + ref : string + + @clarg_short = "?" + @clarg_name = "show-help" + @clarg_doc = "Show this help and exit" + help : bool +} + +struct KqRaceFmt { + name : string + fmt : int // the kernel id (kq_fill_planes / kq_row_ref key) + gemv_gdiv : int // GEMV rows per threadgroup (dispatch groups = ceil(d / gdiv)) + split : bool // split-scale form: the f16 d tail binds at nsb*16 +} + +struct KqRaceArm { + name : string + src : string + entry : string + fastmath : bool + tgmem : uint64 + gdiv : int // GEMV only; a candidate arm may change the row map +} + +def race_formats : array { + return <- [ + KqRaceFmt(name = "k4", fmt = 4, gemv_gdiv = 4, split = false), + KqRaceFmt(name = "k5", fmt = 5, gemv_gdiv = 4, split = false), + KqRaceFmt(name = "k6", fmt = 6, gemv_gdiv = 4, split = true), + KqRaceFmt(name = "iq4xs", fmt = 44, gemv_gdiv = 4, split = true), + KqRaceFmt(name = "iq4nl", fmt = 45, gemv_gdiv = 4, split = false), + KqRaceFmt(name = "k3", fmt = 3, gemv_gdiv = 4, split = true), + KqRaceFmt(name = "iq3s", fmt = 33, gemv_gdiv = 8, split = true), + KqRaceFmt(name = "iq3xxs", fmt = 34, gemv_gdiv = 8, split = true), + KqRaceFmt(name = "k2", fmt = 2, gemv_gdiv = 4, split = true), + KqRaceFmt(name = "iq2s", fmt = 23, gemv_gdiv = 8, split = true), + KqRaceFmt(name = "iq2xs", fmt = 24, gemv_gdiv = 8, split = true), + KqRaceFmt(name = "iq2xxs", fmt = 25, gemv_gdiv = 8, split = true) + ] +} + +// (name, n = reduction dim, d = output rows); n % 256, d % 64. The 1B set carries the vehicle +// GGUFs' geometry, the 3B pair the bigger DRAM-honest shapes. +def race_shapes : array> { + return <- [("kv1b", 2048, 512), ("q1b", 2048, 2048), ("w13_1b", 2048, 8192), + ("w2_1b", 8192, 2048), ("q3b", 3072, 3072), ("w13_3b", 3072, 8192)] +} + +def private want(filter, name : string) : bool { + if (empty(filter)) { + return true + } + for (tok in split(filter, ",")) { + if (!empty(tok) && find(name, tok) >= 0) { + return true + } + } + return false +} + +def private gemv_arms(f : KqRaceFmt) : array { // candidate GEMV spellings register here, after prod + var arms : array + if (f.fmt == 4) { + arms |> push(KqRaceArm(name = "prod", src = metal_kq_gemv_k4_msl, entry = metal_kq_gemv_k4_msl_entry, fastmath = metal_kq_gemv_k4_msl_fastmath, tgmem = 0ul, gdiv = f.gemv_gdiv)) + } elif (f.fmt == 5) { + arms |> push(KqRaceArm(name = "prod", src = metal_kq_gemv_k5_msl, entry = metal_kq_gemv_k5_msl_entry, fastmath = metal_kq_gemv_k5_msl_fastmath, tgmem = 0ul, gdiv = f.gemv_gdiv)) + } elif (f.fmt == 6) { + arms |> push(KqRaceArm(name = "prod", src = metal_kq_gemv_k6_msl, entry = metal_kq_gemv_k6_msl_entry, fastmath = metal_kq_gemv_k6_msl_fastmath, tgmem = 0ul, gdiv = f.gemv_gdiv)) + } elif (f.fmt == 44) { + arms |> push(KqRaceArm(name = "prod", src = metal_kq_gemv_iq4xs_msl, entry = metal_kq_gemv_iq4xs_msl_entry, fastmath = metal_kq_gemv_iq4xs_msl_fastmath, tgmem = metal_kq_gemv_iq4xs_msl_tgmem, gdiv = f.gemv_gdiv)) + } elif (f.fmt == 45) { + arms |> push(KqRaceArm(name = "prod", src = metal_kq_gemv_iq4nl_msl, entry = metal_kq_gemv_iq4nl_msl_entry, fastmath = metal_kq_gemv_iq4nl_msl_fastmath, tgmem = metal_kq_gemv_iq4nl_msl_tgmem, gdiv = f.gemv_gdiv)) + } elif (f.fmt == 3) { + arms |> push(KqRaceArm(name = "prod", src = metal_kq_gemv_k3_msl, entry = metal_kq_gemv_k3_msl_entry, fastmath = metal_kq_gemv_k3_msl_fastmath, tgmem = 0ul, gdiv = f.gemv_gdiv)) + } elif (f.fmt == 33) { + arms |> push(KqRaceArm(name = "prod", src = metal_kq_gemv_iq3s_msl, entry = metal_kq_gemv_iq3s_msl_entry, fastmath = metal_kq_gemv_iq3s_msl_fastmath, tgmem = metal_kq_gemv_iq3s_msl_tgmem, gdiv = f.gemv_gdiv)) + } elif (f.fmt == 34) { + arms |> push(KqRaceArm(name = "prod", src = metal_kq_gemv_iq3xxs_msl, entry = metal_kq_gemv_iq3xxs_msl_entry, fastmath = metal_kq_gemv_iq3xxs_msl_fastmath, tgmem = metal_kq_gemv_iq3xxs_msl_tgmem, gdiv = f.gemv_gdiv)) + } elif (f.fmt == 2) { + arms |> push(KqRaceArm(name = "prod", src = metal_kq_gemv_k2_msl, entry = metal_kq_gemv_k2_msl_entry, fastmath = metal_kq_gemv_k2_msl_fastmath, tgmem = 0ul, gdiv = f.gemv_gdiv)) + } elif (f.fmt == 23) { + arms |> push(KqRaceArm(name = "prod", src = metal_kq_gemv_iq2s_msl, entry = metal_kq_gemv_iq2s_msl_entry, fastmath = metal_kq_gemv_iq2s_msl_fastmath, tgmem = 0ul, gdiv = f.gemv_gdiv)) + } elif (f.fmt == 24) { + arms |> push(KqRaceArm(name = "prod", src = metal_kq_gemv_iq2xs_msl, entry = metal_kq_gemv_iq2xs_msl_entry, fastmath = metal_kq_gemv_iq2xs_msl_fastmath, tgmem = 0ul, gdiv = f.gemv_gdiv)) + } else { + arms |> push(KqRaceArm(name = "prod", src = metal_kq_gemv_iq2xxs_msl, entry = metal_kq_gemv_iq2xxs_msl_entry, fastmath = metal_kq_gemv_iq2xxs_msl_fastmath, tgmem = 0ul, gdiv = f.gemv_gdiv)) + } + return <- arms +} + +def private mm_arms(f : KqRaceFmt) : array { // candidate mul_mm spellings register here, after prod + var arms : array + if (f.fmt == 4) { + arms |> push(KqRaceArm(name = "prod", src = MetalKqMulMmK4_metal_kq_mulmm_msl, entry = MetalKqMulMmK4_metal_kq_mulmm_msl_entry, fastmath = MetalKqMulMmK4_metal_kq_mulmm_msl_fastmath, tgmem = MetalKqMulMmK4_metal_kq_mulmm_msl_tgmem, gdiv = 0)) + } elif (f.fmt == 5) { + arms |> push(KqRaceArm(name = "prod", src = MetalKqMulMmK5_metal_kq_mulmm_msl, entry = MetalKqMulMmK5_metal_kq_mulmm_msl_entry, fastmath = MetalKqMulMmK5_metal_kq_mulmm_msl_fastmath, tgmem = MetalKqMulMmK5_metal_kq_mulmm_msl_tgmem, gdiv = 0)) + } elif (f.fmt == 6) { + arms |> push(KqRaceArm(name = "prod", src = MetalKqMulMmK6_metal_kq_mulmm_msl, entry = MetalKqMulMmK6_metal_kq_mulmm_msl_entry, fastmath = MetalKqMulMmK6_metal_kq_mulmm_msl_fastmath, tgmem = MetalKqMulMmK6_metal_kq_mulmm_msl_tgmem, gdiv = 0)) + } elif (f.fmt == 44) { + arms |> push(KqRaceArm(name = "prod", src = MetalKqMulMmIq4xs_metal_kq_mulmm_msl, entry = MetalKqMulMmIq4xs_metal_kq_mulmm_msl_entry, fastmath = MetalKqMulMmIq4xs_metal_kq_mulmm_msl_fastmath, tgmem = MetalKqMulMmIq4xs_metal_kq_mulmm_msl_tgmem, gdiv = 0)) + } elif (f.fmt == 45) { + arms |> push(KqRaceArm(name = "prod", src = MetalKqMulMmIq4nl_metal_kq_mulmm_msl, entry = MetalKqMulMmIq4nl_metal_kq_mulmm_msl_entry, fastmath = MetalKqMulMmIq4nl_metal_kq_mulmm_msl_fastmath, tgmem = MetalKqMulMmIq4nl_metal_kq_mulmm_msl_tgmem, gdiv = 0)) + } elif (f.fmt == 3) { + arms |> push(KqRaceArm(name = "prod", src = MetalKqMulMmK3_metal_kq_mulmm_msl, entry = MetalKqMulMmK3_metal_kq_mulmm_msl_entry, fastmath = MetalKqMulMmK3_metal_kq_mulmm_msl_fastmath, tgmem = MetalKqMulMmK3_metal_kq_mulmm_msl_tgmem, gdiv = 0)) + } elif (f.fmt == 33) { + arms |> push(KqRaceArm(name = "prod", src = MetalKqMulMmIq3s_metal_kq_mulmm_msl, entry = MetalKqMulMmIq3s_metal_kq_mulmm_msl_entry, fastmath = MetalKqMulMmIq3s_metal_kq_mulmm_msl_fastmath, tgmem = MetalKqMulMmIq3s_metal_kq_mulmm_msl_tgmem, gdiv = 0)) + } elif (f.fmt == 34) { + arms |> push(KqRaceArm(name = "prod", src = MetalKqMulMmIq3xxs_metal_kq_mulmm_msl, entry = MetalKqMulMmIq3xxs_metal_kq_mulmm_msl_entry, fastmath = MetalKqMulMmIq3xxs_metal_kq_mulmm_msl_fastmath, tgmem = MetalKqMulMmIq3xxs_metal_kq_mulmm_msl_tgmem, gdiv = 0)) + } elif (f.fmt == 2) { + arms |> push(KqRaceArm(name = "prod", src = MetalKqMulMmK2_metal_kq_mulmm_msl, entry = MetalKqMulMmK2_metal_kq_mulmm_msl_entry, fastmath = MetalKqMulMmK2_metal_kq_mulmm_msl_fastmath, tgmem = MetalKqMulMmK2_metal_kq_mulmm_msl_tgmem, gdiv = 0)) + } elif (f.fmt == 23) { + arms |> push(KqRaceArm(name = "prod", src = MetalKqMulMmIq2s_metal_kq_mulmm_msl, entry = MetalKqMulMmIq2s_metal_kq_mulmm_msl_entry, fastmath = MetalKqMulMmIq2s_metal_kq_mulmm_msl_fastmath, tgmem = MetalKqMulMmIq2s_metal_kq_mulmm_msl_tgmem, gdiv = 0)) + } elif (f.fmt == 24) { + arms |> push(KqRaceArm(name = "prod", src = MetalKqMulMmIq2xs_metal_kq_mulmm_msl, entry = MetalKqMulMmIq2xs_metal_kq_mulmm_msl_entry, fastmath = MetalKqMulMmIq2xs_metal_kq_mulmm_msl_fastmath, tgmem = MetalKqMulMmIq2xs_metal_kq_mulmm_msl_tgmem, gdiv = 0)) + } else { + arms |> push(KqRaceArm(name = "prod", src = MetalKqMulMmIq2xxs_metal_kq_mulmm_msl, entry = MetalKqMulMmIq2xxs_metal_kq_mulmm_msl_entry, fastmath = MetalKqMulMmIq2xxs_metal_kq_mulmm_msl_fastmath, tgmem = MetalKqMulMmIq2xxs_metal_kq_mulmm_msl_tgmem, gdiv = 0)) + } + return <- arms +} + +let PREFILL_M = 512 // the mm tier's token batch + +def private load_ref_table(path : string; var tab : table) { + return if (empty(path)) + fopen(path, "rb") $(f) { + if (f == null) { + panic("--ref {path}: cannot open") + } + let text = fread(f) + for (ln in split(text, "\n")) { + let cols <- split_by_chars(strip(ln), " \t") + var vals <- [for (c in cols); c; where !empty(c)] + if (length(vals) >= 4) { + let us = try_to_double(vals[3]) ?? -1.0lf + if (us > 0.0lf) { + tab["{vals[0]}/{vals[1]}/{vals[2]}"] = us + } + } + delete vals + } + } +} + +// one timed encoder: bind once, `reps` back-to-back dispatches; per-dispatch GPU us, <0 on failure +def private time_arm_us(queue; pso; arm : KqRaceArm; bks, bkq, bx, by, b5, b6; sc_off : uint64; groups : uint3; tgsz, reps : int) : double { + var err = "" + var gpu_ms = 0.0lf + let ok = with_compute_encoder_timed(queue, err, gpu_ms) $(enc : MetalComputeEncoder?) { + metal_set_pipeline(enc, pso) + if (arm.tgmem != 0ul) { + metal_set_threadgroup_memory_length(enc, arm.tgmem, 0) + } + metal_set_buffer(enc, bks, sc_off, 0) + metal_set_buffer(enc, bks, 0ul, 1) + metal_set_buffer(enc, bkq, 0ul, 2) + metal_set_buffer(enc, bx, 0ul, 3) + metal_set_buffer(enc, by, 0ul, 4) + metal_set_buffer(enc, b5, 0ul, 5) + metal_set_buffer(enc, b6, 0ul, 6) + for (_r in range(reps)) { + metal_dispatch_threadgroups(enc, groups, uint3(uint(tgsz), 1u, 1u)) + } + } + if (!ok) { + print(" {arm.name}: dispatch FAILED: {err}\n") + return -1.0lf + } + return gpu_ms * 1000.0lf / double(reps) +} + +// the whole-output dot-envelope oracle the tests trust: want/env from kq_row_ref + double dot +def private build_oracle(fmt, n, d, nsb : int; kq, ks : array; xf : array; xstride, xrows : int; + var want : array; var env : array) { + var wrow : array + wrow |> resize(n) + let nb = n / 256 + for (r in range(d)) { + kq_row_ref(fmt, r * nb, nb, nsb, kq, ks, wrow) + for (xr in range(xrows)) { + var acc = 0.0lf + var ev = 0.0lf + for (c in range(n)) { + let p = double(wrow[c]) * double(xf[xr * xstride + c]) + acc += p + ev += abs(p) + } + want[xr * d + r] = float(acc) + env[xr * d + r] = float(ev) + } + } + delete wrow +} + +def private race_cell(queue, dev; f : KqRaceFmt; tier, shname : string; n, d : int; + arms : array; args : Args; ref_tab : table) : int { + let is_mm = tier == "mm" + let nb = n / 256 + let nsb = d * nb + var kq : array + var ks : array + kq_fill_planes(f.fmt, nsb, kq, ks) + let xrows = is_mm ? PREFILL_M : 1 + var xf : array + xf |> resize(xrows * n) + for (i in range(xrows * n)) { + xf[i] = 0.11 * float(i % 41) - 2.2 + } + var want : array + var env : array + want |> resize(xrows * d) + env |> resize(xrows * d) + build_oracle(f.fmt, n, d, nsb, kq, ks, xf, n, xrows, want, env) + var bks = buf_upload(dev, ks) + var bkq = buf_upload(dev, kq) + var bx = buf_upload(dev, xf) + var b5 = buf_u32(dev, uint(n)) + var b6 = buf_u32(dev, uint(d)) + let sc_off = f.split ? uint64(nsb * 16) : 0ul + let wbytes = double(length(kq) + length(ks)) + var rows = 0 + print("--- {f.name} {tier} {shname} (n={n} d={d}, weights {wbytes / 1048576.0lf} MB)\n") + // compile + correctness gate, one y per arm + var psos : array + var bys : array + var live : array + psos |> reserve(length(arms)) + bys |> reserve(length(arms)) + live |> reserve(length(arms)) + for (arm in arms) { + var err = "" + var pso = pipeline_from_source(dev, arm.src, arm.entry, arm.fastmath, err) + if (pso == null) { + print(" {arm.name}: pipeline FAILED: {err}\n") + } + psos |> push(pso) + bys |> push(buf_fill(dev, xrows * d, -1000.0)) + live |> push(pso != null) + } + for (arm, lv, pso, by in arms, live, psos, bys) { + continue if (!lv) + let groups = is_mm ? uint3(uint(PREFILL_M / 32), uint(d / 64), 1u) : uint3(uint((d + arm.gdiv - 1) / arm.gdiv), 1u, 1u) + let tgsz = is_mm ? 128 : 64 + let us0 = time_arm_us(queue, pso, arm, bks, bkq, bx, by, b5, b6, sc_off, groups, tgsz, 1) + if (us0 < 0.0lf) { + lv = false + continue + } + let bad = buf_mismatch_env(by, want, env, 2e-4, "{f.name}/{tier}/{shname}/{arm.name}") + if (bad != 0) { + print(" {arm.name}: ORACLE MISMATCH ({bad} of {xrows * d}) - arm disqualified\n") + lv = false + } + } + // interleaved timing: round-robin per round, best-of (round 0 discarded as warmup) + var best : array + best |> resize(length(arms)) + for (i in range(length(arms))) { + best[i] = -1.0lf + } + for (round in range(args.rounds + 1)) { + for (arm, lv, pso, by, bst in arms, live, psos, bys, best) { + continue if (!lv) + let groups = is_mm ? uint3(uint(PREFILL_M / 32), uint(d / 64), 1u) : uint3(uint((d + arm.gdiv - 1) / arm.gdiv), 1u, 1u) + let tgsz = is_mm ? 128 : 64 + let us = time_arm_us(queue, pso, arm, bks, bkq, bx, by, b5, b6, sc_off, groups, tgsz, args.reps) + if (us < 0.0lf) { + lv = false + } elif (round > 0 && (bst < 0.0lf || us < bst)) { + bst = us + } + } + } + var base_us = -1.0lf + for (i in range(length(arms))) { + continue if (!live[i] || best[i] < 0.0lf) + let arm = arms[i] + let us = best[i] + if (i == 0) { + base_us = us + } + let gbs = wbytes / (us * 1000.0lf) // us * 1e3 = ns; bytes/ns = GB/s + var line = " {arm.name}: {us:.2f} us/dispatch {gbs:.1f} wGB/s" + if (is_mm) { + let gmac = double(PREFILL_M) * double(n) * double(d) / (us * 1000.0lf) + line = "{line} {gmac:.0f} GMAC/s" + } + if (i > 0 && base_us > 0.0lf) { + line = "{line} vs prod {base_us / us:.3f}x" + } + let rk = "{f.name}/{tier}/{shname}" + if (ref_tab |> key_exists(rk)) { + line = "{line} vs lcpp {(ref_tab?[rk] ?? 0.0lf) / us:.3f}x" + } + print("{line} [env-bar 2e-4 ok]\n") + rows++ + } + for (p in psos) { + if (p != null) { + metal_release(p) + } + } + for (b in bys) { + metal_release(b) + } + metal_release(bks) + metal_release(bkq) + metal_release(bx) + metal_release(b5) + metal_release(b6) + psos |> clear() // borrowed pointers - clear before delete (delete would free pointees) + bys |> clear() + unsafe { + delete psos + delete bys + } + delete kq + delete ks + delete xf + delete want + delete env + delete live + delete best + return rows +} + +def private main_apple(args : Args) { + var dev = metal_create_system_default_device() + if (dev == null) { + panic("no Metal device on this box") + } + print("device={metal_device_name(dev)} rounds={args.rounds} reps={args.reps}\n") + var queue = metal_new_command_queue(dev) + var ref_tab : table + load_ref_table(args.ref, ref_tab) + var fmts <- race_formats() + var shapes <- race_shapes() + var rows = 0 + for (f in fmts) { + continue if (!want(args.fmts, f.name)) + for (sh in shapes) { + continue if (!want(args.shapes, sh._0)) + if (want(args.tiers, "gemv")) { + var arms <- gemv_arms(f) + rows += race_cell(queue, dev, f, "gemv", sh._0, sh._1, sh._2, arms, args, ref_tab) + delete arms + } + if (want(args.tiers, "mm")) { + var arms <- mm_arms(f) + rows += race_cell(queue, dev, f, "mm", sh._0, sh._1, sh._2, arms, args, ref_tab) + delete arms + } + } + } + metal_release(queue) + metal_release(dev) + delete fmts + delete shapes + delete ref_tab + if (rows == 0) { + panic("zero result rows - wrong filters, failed pipelines, or a declining device") + } + print("DONE: {rows} result rows\n") +} + +[export] +def main { + var args = Args() + let rc = parse_args_with_help(args, "bench_metal_kq_race") + return if (rc >= 0) + static_if (typeinfo builtin_module_exists(das_metal)) { + main_apple(args) + } else { + panic("das_metal is not built on this platform") + } +} diff --git a/modules/dasLLAMA/harness/backend_ops_kq_race.patch b/modules/dasLLAMA/harness/backend_ops_kq_race.patch new file mode 100644 index 0000000000..4befd3b68b --- /dev/null +++ b/modules/dasLLAMA/harness/backend_ops_kq_race.patch @@ -0,0 +1,31 @@ +diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp +index df0b9aa66..06f94d51e 100644 +--- a/tests/test-backend-ops.cpp ++++ b/tests/test-backend-ops.cpp +@@ -10151,6 +10151,26 @@ static std::vector> make_test_cases_eval() { + static std::vector> make_test_cases_perf() { + std::vector> test_cases; + ++ // dasLLAMA Metal kq-race ladder: (m = output rows, k = reduction dim) mirroring ++ // bench_metal_kq_race.das's shapes, every kq format, bs 1 (decode GEMV) and 512 (prefill). ++ // Enabled by GGML_KQ_RACE_LADDER=1 so the stock perf sweep stays untouched. ++ if (getenv("GGML_KQ_RACE_LADDER")) { ++ const std::array kq_shapes[] = { ++ // kv1b q1b w13_1b w2_1b q3b w13_3b ++ {512, 2048}, {2048, 2048}, {8192, 2048}, {2048, 8192}, {3072, 3072}, {8192, 3072}, ++ }; ++ for (int bs : {1, 512}) { ++ for (ggml_type ta : {GGML_TYPE_Q4_K, GGML_TYPE_Q5_K, GGML_TYPE_Q6_K, GGML_TYPE_IQ4_XS, ++ GGML_TYPE_IQ4_NL, GGML_TYPE_Q3_K, GGML_TYPE_IQ3_S, GGML_TYPE_IQ3_XXS, ++ GGML_TYPE_Q2_K, GGML_TYPE_IQ2_S, GGML_TYPE_IQ2_XS, GGML_TYPE_IQ2_XXS}) { ++ for (const auto & s : kq_shapes) { ++ test_cases.emplace_back(new test_mul_mat(ta, GGML_TYPE_F32, s[0], bs, s[1], {1, 1}, {1, 1})); ++ } ++ } ++ } ++ return test_cases; ++ } ++ + // SWIGLU at a 27B-class FFN width, fused [gate|up] vs split operands + // note: same bytes either way, so a backend that indexes them differently shows it here + for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32}) { From 3107e4c12eb810e864930319b3be120abedb2f8e Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 00:24:43 -0700 Subject: [PATCH 02/35] kq race lab: the ptrstore mul_mm arm - llama.cpp's 'massively slower' subscript-store claim, raced Rewrites the production stamp's ta[IDX] = RHS threadgroup store into the pointer form at lab runtime and races it. M5 verdict: FLAT (0.997-1.013x, oracle green both arms) - the claim does not reproduce on this compiler; M1/M4 re-race owed before any production spelling change. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014mq9AKyoAwvD5T1cigTnRw --- .../benchmarks/matmul/bench_metal_kq_race.das | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das b/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das index fc9de044f8..c4ec339b7d 100644 --- a/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das +++ b/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das @@ -170,11 +170,42 @@ def private mm_arms(f : KqRaceFmt) : array { // candidate mul_mm sp } else { arms |> push(KqRaceArm(name = "prod", src = MetalKqMulMmIq2xxs_metal_kq_mulmm_msl, entry = MetalKqMulMmIq2xxs_metal_kq_mulmm_msl_entry, fastmath = MetalKqMulMmIq2xxs_metal_kq_mulmm_msl_fastmath, tgmem = MetalKqMulMmIq2xxs_metal_kq_mulmm_msl_tgmem, gdiv = 0)) } + arms |> push(KqRaceArm(name = "ptrstore", src = rewrite_ta_ptr_store(arms[0].src), entry = arms[0].entry, + fastmath = arms[0].fastmath, tgmem = arms[0].tgmem, gdiv = 0)) return <- arms } let PREFILL_M = 512 // the mm tier's token batch +// Rewrite every threadgroup A-tile subscript store `ta[IDX] = RHS;` into the pointer form +// `*(ta + (IDX)) = RHS;` - llama.cpp's mul_mm carries the subscript form commented out as +// "massively slower.. WTF?" (mul_mm.metal); this arm races that claim on our own template. +def private rewrite_ta_ptr_store(src : string) : string { + var out : array + var n = 0 + for (ln in split(src, "\n")) { + let at = find(ln, "ta[") + var done = false + if (at >= 0 && empty(strip(slice(ln, 0, at)))) { + let eq = find(ln, "] = ") + if (eq > at) { + let idxe = slice(ln, at + 3, eq) + let rhs = slice(ln, eq + 4) + out |> push("{slice(ln, 0, at)}*(ta + ({idxe})) = {rhs}") + n++ + done = true + } + } + if (!done) { + out |> push(ln) + } + } + if (n == 0) { + panic("rewrite_ta_ptr_store: no ta[...] = store found - the template spelling moved") + } + return join(out, "\n") +} + def private load_ref_table(path : string; var tab : table) { return if (empty(path)) fopen(path, "rb") $(f) { From 78ab66866fe2cec575931a4ae5677b9f6ce7b467 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 00:38:58 -0700 Subject: [PATCH 03/35] Metal-4 tensor mul_mm twins for all nine iquant formats - seven now beat llama.cpp on M5 MetalKqMulMmSplitTensorBase: the k6 tensor shell with the per-format decode as ONE overridable stage16 (the MoE tensor family's scaffold shape). Each format's base-template arm ports verbatim - va[] store retargeted to the twb tile. T (f32 X) and TH (f16 panel) stamps per format; iq3s/iq3xxs stage their grids, iq2* read the hoisted constant tables direct, exactly their base arms' choices. Shared decode helpers (iq4_lut -> metal_common; grid tables + ksign7m public) now serve both the kernels file and the prefill twins. M5 lab race (dispatch-loop, oracle-green, vs llama.cpp b10659 tensor mul_mm): k2 1.30x, iq2xxs 1.17-1.21x, iq2xs 1.19x, k3 1.13-1.19x, iq4nl 1.11-1.15x, iq2s 1.12x, iq4xs 1.04-1.08x, iq3s/iq3xxs 0.97-0.98x - from 0.39-0.43x on the base stamp (2.4-3.1x per format). Not yet wired: dispatch/PSO/site gates, crowns, gemm-test tensor arms - the twins are lab-raced only. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014mq9AKyoAwvD5T1cigTnRw --- .../benchmarks/matmul/bench_metal_kq_race.das | 28 ++ .../dasllama/dasllama_metal_common.das | 8 + .../dasllama/dasllama_metal_kernels.das | 277 ++++++----- .../dasllama/dasllama_metal_prefill.das | 437 ++++++++++++++++++ 4 files changed, 608 insertions(+), 142 deletions(-) diff --git a/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das b/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das index c4ec339b7d..076130f0bc 100644 --- a/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das +++ b/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das @@ -14,6 +14,7 @@ require math require metal/msl_shader require ?das_metal metal/das_metal_boost require ?das_metal dasllama/dasllama_metal_kernels +require ?das_metal dasllama/dasllama_metal_prefill // the kq tensor mul_mm twins (Metal-4 boxes) require ?das_metal ../../tests/_metal_kernel_common.das // The per-format Metal KQ kernel RACE rig - rig 1 of the kernel catch-up procedure. Races the @@ -172,6 +173,33 @@ def private mm_arms(f : KqRaceFmt) : array { // candidate mul_mm sp } arms |> push(KqRaceArm(name = "ptrstore", src = rewrite_ta_ptr_store(arms[0].src), entry = arms[0].entry, fastmath = arms[0].fastmath, tgmem = arms[0].tgmem, gdiv = 0)) + // the existing Metal-4 tensor twins (f32-X form) - the ceiling the nine formats' twins chase; + // the pipeline declines cleanly on a box without the tensor toolchain + if (f.fmt == 4) { + arms |> push(KqRaceArm(name = "tensor", src = MetalKqMulMmK4T_metal_kq_mulmm_t_msl, entry = MetalKqMulMmK4T_metal_kq_mulmm_t_msl_entry, fastmath = MetalKqMulMmK4T_metal_kq_mulmm_t_msl_fastmath, tgmem = MetalKqMulMmK4T_metal_kq_mulmm_t_msl_tgmem, gdiv = 0)) + } elif (f.fmt == 5) { + arms |> push(KqRaceArm(name = "tensor", src = MetalKqMulMmK5T_metal_kq_mulmm_t_msl, entry = MetalKqMulMmK5T_metal_kq_mulmm_t_msl_entry, fastmath = MetalKqMulMmK5T_metal_kq_mulmm_t_msl_fastmath, tgmem = MetalKqMulMmK5T_metal_kq_mulmm_t_msl_tgmem, gdiv = 0)) + } elif (f.fmt == 6) { + arms |> push(KqRaceArm(name = "tensor", src = MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl, entry = MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl_entry, fastmath = MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl_fastmath, tgmem = MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl_tgmem, gdiv = 0)) + } elif (f.fmt == 44) { + arms |> push(KqRaceArm(name = "tensor", src = MetalKqMulMmIq4xsT_metal_kq_mulmm_iq4xs_t_msl, entry = MetalKqMulMmIq4xsT_metal_kq_mulmm_iq4xs_t_msl_entry, fastmath = MetalKqMulMmIq4xsT_metal_kq_mulmm_iq4xs_t_msl_fastmath, tgmem = MetalKqMulMmIq4xsT_metal_kq_mulmm_iq4xs_t_msl_tgmem, gdiv = 0)) + } elif (f.fmt == 45) { + arms |> push(KqRaceArm(name = "tensor", src = MetalKqMulMmIq4nlT_metal_kq_mulmm_iq4xs_t_msl, entry = MetalKqMulMmIq4nlT_metal_kq_mulmm_iq4xs_t_msl_entry, fastmath = MetalKqMulMmIq4nlT_metal_kq_mulmm_iq4xs_t_msl_fastmath, tgmem = MetalKqMulMmIq4nlT_metal_kq_mulmm_iq4xs_t_msl_tgmem, gdiv = 0)) + } elif (f.fmt == 3) { + arms |> push(KqRaceArm(name = "tensor", src = MetalKqMulMmK3T_metal_kq_mulmm_k3_t_msl, entry = MetalKqMulMmK3T_metal_kq_mulmm_k3_t_msl_entry, fastmath = MetalKqMulMmK3T_metal_kq_mulmm_k3_t_msl_fastmath, tgmem = MetalKqMulMmK3T_metal_kq_mulmm_k3_t_msl_tgmem, gdiv = 0)) + } elif (f.fmt == 2) { + arms |> push(KqRaceArm(name = "tensor", src = MetalKqMulMmK2T_metal_kq_mulmm_k2_t_msl, entry = MetalKqMulMmK2T_metal_kq_mulmm_k2_t_msl_entry, fastmath = MetalKqMulMmK2T_metal_kq_mulmm_k2_t_msl_fastmath, tgmem = MetalKqMulMmK2T_metal_kq_mulmm_k2_t_msl_tgmem, gdiv = 0)) + } elif (f.fmt == 33) { + arms |> push(KqRaceArm(name = "tensor", src = MetalKqMulMmIq3sT_metal_kq_mulmm_iq3s_t_msl, entry = MetalKqMulMmIq3sT_metal_kq_mulmm_iq3s_t_msl_entry, fastmath = MetalKqMulMmIq3sT_metal_kq_mulmm_iq3s_t_msl_fastmath, tgmem = MetalKqMulMmIq3sT_metal_kq_mulmm_iq3s_t_msl_tgmem, gdiv = 0)) + } elif (f.fmt == 34) { + arms |> push(KqRaceArm(name = "tensor", src = MetalKqMulMmIq3xxsT_metal_kq_mulmm_iq3xxs_t_msl, entry = MetalKqMulMmIq3xxsT_metal_kq_mulmm_iq3xxs_t_msl_entry, fastmath = MetalKqMulMmIq3xxsT_metal_kq_mulmm_iq3xxs_t_msl_fastmath, tgmem = MetalKqMulMmIq3xxsT_metal_kq_mulmm_iq3xxs_t_msl_tgmem, gdiv = 0)) + } elif (f.fmt == 23) { + arms |> push(KqRaceArm(name = "tensor", src = MetalKqMulMmIq2sT_metal_kq_mulmm_iq2s_t_msl, entry = MetalKqMulMmIq2sT_metal_kq_mulmm_iq2s_t_msl_entry, fastmath = MetalKqMulMmIq2sT_metal_kq_mulmm_iq2s_t_msl_fastmath, tgmem = MetalKqMulMmIq2sT_metal_kq_mulmm_iq2s_t_msl_tgmem, gdiv = 0)) + } elif (f.fmt == 24) { + arms |> push(KqRaceArm(name = "tensor", src = MetalKqMulMmIq2xsT_metal_kq_mulmm_iq2xs_t_msl, entry = MetalKqMulMmIq2xsT_metal_kq_mulmm_iq2xs_t_msl_entry, fastmath = MetalKqMulMmIq2xsT_metal_kq_mulmm_iq2xs_t_msl_fastmath, tgmem = MetalKqMulMmIq2xsT_metal_kq_mulmm_iq2xs_t_msl_tgmem, gdiv = 0)) + } elif (f.fmt == 25) { + arms |> push(KqRaceArm(name = "tensor", src = MetalKqMulMmIq2xxsT_metal_kq_mulmm_iq2xxs_t_msl, entry = MetalKqMulMmIq2xxsT_metal_kq_mulmm_iq2xxs_t_msl_entry, fastmath = MetalKqMulMmIq2xxsT_metal_kq_mulmm_iq2xxs_t_msl_fastmath, tgmem = MetalKqMulMmIq2xxsT_metal_kq_mulmm_iq2xxs_t_msl_tgmem, gdiv = 0)) + } return <- arms } diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_common.das b/modules/dasLLAMA/dasllama/dasllama_metal_common.das index 8df3f0d33e..16dfbc0a06 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_common.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_common.das @@ -2164,6 +2164,14 @@ def public metal_prefill_declines() : table { return clone_to_move(g_prefill_declines_by_reason) } +// IQ4_XS codebook (kvalues_iq4nl) as four packed words, selected + byte-extracted — +// no table memory; the k6 sign trick widens the byte. Shared by the kernels file's GEMV/mv +// set and the prefill file's tensor twins. +def public iq4_lut(q : uint) : float { + let w = q < 8u ? (q < 4u ? 3215825025u : 4142587343u) : (q < 12u ? 639175937u : 1901675829u) + return float((int((w >> ((q & 3u) * 8u)) & 255u) ^ 128) - 128) +} + // ===== Metal-4 tensor twin race scaffolding (shared by the per-module race sections) ===== struct public MetalTensorRaceResult { diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das index e9d00cff8d..32dab01dad 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das @@ -6296,16 +6296,9 @@ def private kmask_scales(sv : uint4; iq : uint) : uint4 { ((hw4 >> 4u) & 0x0F0F) | ((hw2 & 0xC0C0) >> 2u)) } -// IQ4_XS codebook (kvalues_iq4nl) as four packed words, selected + byte-extracted — -// no table memory; the k6 sign trick widens the byte -def private iq4_lut(q : uint) : float { - let w = q < 8u ? (q < 4u ? 3215825025u : 4142587343u) : (q < 12u ? 639175937u : 1901675829u) - return float((int((w >> ((q & 3u) * 8u)) & 255u) ^ 128) - 128) -} - // iq3s_grid, one word per call - the GEMV/mul_mm stage it into threadgroup memory once // per group; the mv twins read it directly. -def private iq3s_gw(i : int) : uint { +def public iq3s_gw(i : int) : uint { let tbl = fixed_array( 0x01010101u, 0x01010103u, 0x01010105u, 0x0101010bu, 0x0101010fu, 0x01010301u, 0x01010303u, 0x01010305u, 0x01010309u, 0x0101030du, 0x01010501u, 0x01010503u, 0x0101050bu, 0x01010707u, 0x01010901u, 0x01010905u, @@ -6376,18 +6369,18 @@ def private iq3s_gw(i : int) : uint { // one signed grid word: the 4-bit sign nibble expands to a byte mask; magnitudes are odd // 1..15, so the negate's +1 never carries across a byte lane -def private iq3s_sw(g, nib : uint) : uint { +def public iq3s_sw(g, nib : uint) : uint { let m1 = (nib * 0x00204081u) & 0x01010101u return (g ^ (m1 * 255u)) + m1 } -def private iq3s_sx(sw, t : uint) : float { +def public iq3s_sx(sw, t : uint) : float { return float((int((sw >> (t * 8u)) & 255u) ^ 128) - 128) } // the HALVED iq3xxs grid (iq3xxs_grid, every byte / 2 - the plane convention), one // word per call; the GEMV and mul_mm stage it, the mv twins read it directly. -def private iq3xxs_gw(i : int) : uint { +def public iq3xxs_gw(i : int) : uint { let tbl = fixed_array( 0x02020202u, 0x0202020au, 0x02020212u, 0x02020606u, 0x0202060eu, 0x0202061fu, 0x02020a02u, 0x02020a0au, 0x02020e06u, 0x0202120au, 0x02021f0eu, 0x02021f16u, 0x02060206u, 0x0206020eu, 0x02060602u, 0x0206060au, @@ -6425,7 +6418,7 @@ def private iq3xxs_gw(i : int) : uint { } // ksigns_iq2xs without the table: bit 7 = the even parity of the 7-bit index. -def private ksign7m(v : uint) : uint { +def public ksign7m(v : uint) : uint { var t = v ^ (v >> 4u) t = t ^ (t >> 2u) t = t ^ (t >> 1u) @@ -6434,7 +6427,7 @@ def private ksign7m(v : uint) : uint { // iq2xxs_grid as low/high word pairs (word 2i = magnitudes 0..3 of u64 entry i, word // 2i+1 = 4..7); the same all-literal program-scope-constant form as its grid siblings. -def private iq2xxs_gw(i : int) : uint { +def public iq2xxs_gw(i : int) : uint { let tbl = fixed_array( 0x08080808, 0x08080808, 0x0808082b, 0x08080808, 0x08081919, 0x08080808, 0x08082b08, 0x08080808, 0x08082b2b, 0x08080808, 0x08190819, 0x08080808, 0x08191908, 0x08080808, 0x082b0808, 0x08080808, @@ -6505,135 +6498,135 @@ def private iq2xxs_gw(i : int) : uint { // iq2xs_grid as low/high word pairs (word 2i = magnitudes 0..3 of u64 entry i, word // 2i+1 = 4..7); the same all-literal program-scope-constant form as iq2s_gw below. -def private iq2xs_gw(i : int) : uint { // nolint:STYLE038 - a 1024-word data table, not splittable +def public iq2xs_gw(i : int) : uint { // nolint:STYLE038 - a 1024-word data table, not splittable let tbl = fixed_array( - 0x08080808, 0x08080808, 0x0808082b, 0x08080808, 0x08081919, 0x08080808, 0x08082b08, 0x08080808, - 0x08082b2b, 0x08080808, 0x08190819, 0x08080808, 0x08191908, 0x08080808, 0x0819192b, 0x08080808, - 0x08192b19, 0x08080808, 0x082b0808, 0x08080808, 0x082b082b, 0x08080808, 0x082b1919, 0x08080808, - 0x082b2b08, 0x08080808, 0x19080819, 0x08080808, 0x19081908, 0x08080808, 0x1908192b, 0x08080808, - 0x19082b19, 0x08080808, 0x19190808, 0x08080808, 0x1919082b, 0x08080808, 0x19191919, 0x08080808, - 0x19192b08, 0x08080808, 0x192b0819, 0x08080808, 0x192b1908, 0x08080808, 0x2b080808, 0x08080808, - 0x2b08082b, 0x08080808, 0x2b081919, 0x08080808, 0x2b082b08, 0x08080808, 0x2b190819, 0x08080808, - 0x2b191908, 0x08080808, 0x2b192b19, 0x08080808, 0x2b2b0808, 0x08080808, 0x08080819, 0x08080819, - 0x08081908, 0x08080819, 0x0808192b, 0x08080819, 0x08082b19, 0x08080819, 0x08190808, 0x08080819, - 0x0819082b, 0x08080819, 0x08191919, 0x08080819, 0x08192b08, 0x08080819, 0x08192b2b, 0x08080819, - 0x082b0819, 0x08080819, 0x082b1908, 0x08080819, 0x19080808, 0x08080819, 0x1908082b, 0x08080819, - 0x19081919, 0x08080819, 0x19082b08, 0x08080819, 0x19190819, 0x08080819, 0x19191908, 0x08080819, - 0x192b0808, 0x08080819, 0x192b2b08, 0x08080819, 0x2b080819, 0x08080819, 0x2b081908, 0x08080819, - 0x2b190808, 0x08080819, 0x08080808, 0x0808082b, 0x0808082b, 0x0808082b, 0x08081919, 0x0808082b, - 0x08082b08, 0x0808082b, 0x08190819, 0x0808082b, 0x08191908, 0x0808082b, 0x082b0808, 0x0808082b, - 0x19080819, 0x0808082b, 0x19081908, 0x0808082b, 0x19190808, 0x0808082b, 0x19191919, 0x0808082b, - 0x2b080808, 0x0808082b, 0x2b082b2b, 0x0808082b, 0x08080819, 0x08081908, 0x08081908, 0x08081908, - 0x0808192b, 0x08081908, 0x08082b19, 0x08081908, 0x08190808, 0x08081908, 0x0819082b, 0x08081908, - 0x08191919, 0x08081908, 0x08192b08, 0x08081908, 0x082b0819, 0x08081908, 0x082b1908, 0x08081908, - 0x19080808, 0x08081908, 0x1908082b, 0x08081908, 0x19081919, 0x08081908, 0x19082b08, 0x08081908, - 0x19190819, 0x08081908, 0x19191908, 0x08081908, 0x1919192b, 0x08081908, 0x192b0808, 0x08081908, - 0x2b080819, 0x08081908, 0x2b081908, 0x08081908, 0x2b190808, 0x08081908, 0x08080808, 0x08081919, - 0x0808082b, 0x08081919, 0x08081919, 0x08081919, 0x08082b08, 0x08081919, 0x08190819, 0x08081919, - 0x08191908, 0x08081919, 0x082b0808, 0x08081919, 0x19080819, 0x08081919, 0x19081908, 0x08081919, - 0x19190808, 0x08081919, 0x192b0819, 0x08081919, 0x2b080808, 0x08081919, 0x08080819, 0x0808192b, - 0x08081908, 0x0808192b, 0x08190808, 0x0808192b, 0x082b192b, 0x0808192b, 0x19080808, 0x0808192b, - 0x1908082b, 0x0808192b, 0x2b081908, 0x0808192b, 0x08080808, 0x08082b08, 0x0808082b, 0x08082b08, - 0x08081919, 0x08082b08, 0x08082b08, 0x08082b08, 0x08082b2b, 0x08082b08, 0x08190819, 0x08082b08, - 0x08191908, 0x08082b08, 0x082b0808, 0x08082b08, 0x082b1919, 0x08082b08, 0x19080819, 0x08082b08, - 0x19081908, 0x08082b08, 0x19190808, 0x08082b08, 0x19192b08, 0x08082b08, 0x2b080808, 0x08082b08, - 0x2b2b0808, 0x08082b08, 0x2b2b2b2b, 0x08082b08, 0x08080819, 0x08082b19, 0x08081908, 0x08082b19, - 0x08190808, 0x08082b19, 0x19080808, 0x08082b19, 0x2b080819, 0x08082b19, 0x2b082b19, 0x08082b19, - 0x08080808, 0x08082b2b, 0x082b0808, 0x08082b2b, 0x082b2b08, 0x08082b2b, 0x2b19192b, 0x08082b2b, - 0x2b2b0808, 0x08082b2b, 0x08080819, 0x08190808, 0x08081908, 0x08190808, 0x0808192b, 0x08190808, - 0x08082b19, 0x08190808, 0x08190808, 0x08190808, 0x0819082b, 0x08190808, 0x08191919, 0x08190808, - 0x08192b08, 0x08190808, 0x082b0819, 0x08190808, 0x082b1908, 0x08190808, 0x19080808, 0x08190808, - 0x1908082b, 0x08190808, 0x19081919, 0x08190808, 0x19082b08, 0x08190808, 0x19190819, 0x08190808, - 0x19191908, 0x08190808, 0x192b0808, 0x08190808, 0x192b2b2b, 0x08190808, 0x2b080819, 0x08190808, - 0x2b081908, 0x08190808, 0x2b190808, 0x08190808, 0x08080808, 0x08190819, 0x0808082b, 0x08190819, - 0x08081919, 0x08190819, 0x08082b08, 0x08190819, 0x08190819, 0x08190819, 0x08191908, 0x08190819, - 0x082b0808, 0x08190819, 0x19080819, 0x08190819, 0x19081908, 0x08190819, 0x19190808, 0x08190819, - 0x2b080808, 0x08190819, 0x2b191908, 0x08190819, 0x2b19192b, 0x08190819, 0x08080819, 0x0819082b, - 0x08081908, 0x0819082b, 0x0808192b, 0x0819082b, 0x08190808, 0x0819082b, 0x19080808, 0x0819082b, - 0x192b0808, 0x0819082b, 0x08080808, 0x08191908, 0x0808082b, 0x08191908, 0x08081919, 0x08191908, - 0x08082b08, 0x08191908, 0x08190819, 0x08191908, 0x08191908, 0x08191908, 0x082b0808, 0x08191908, - 0x19080819, 0x08191908, 0x19081908, 0x08191908, 0x19082b19, 0x08191908, 0x19190808, 0x08191908, - 0x192b1908, 0x08191908, 0x2b080808, 0x08191908, 0x08080819, 0x08191919, 0x08081908, 0x08191919, - 0x08190808, 0x08191919, 0x19080808, 0x08191919, 0x08080808, 0x0819192b, 0x08191908, 0x0819192b, - 0x19082b19, 0x0819192b, 0x08080819, 0x08192b08, 0x08081908, 0x08192b08, 0x08190808, 0x08192b08, - 0x0819082b, 0x08192b08, 0x19080808, 0x08192b08, 0x19191908, 0x08192b08, 0x2b08192b, 0x08192b08, - 0x08080808, 0x08192b19, 0x08081919, 0x08192b19, 0x192b192b, 0x08192b19, 0x19190819, 0x08192b2b, - 0x2b2b2b19, 0x08192b2b, 0x08080808, 0x082b0808, 0x0808082b, 0x082b0808, 0x08081919, 0x082b0808, - 0x08082b08, 0x082b0808, 0x08082b2b, 0x082b0808, 0x08190819, 0x082b0808, 0x08191908, 0x082b0808, - 0x082b0808, 0x082b0808, 0x19080819, 0x082b0808, 0x19081908, 0x082b0808, 0x19190808, 0x082b0808, - 0x2b080808, 0x082b0808, 0x2b2b0808, 0x082b0808, 0x08080819, 0x082b0819, 0x08081908, 0x082b0819, - 0x08190808, 0x082b0819, 0x19080808, 0x082b0819, 0x19082b08, 0x082b0819, 0x192b1919, 0x082b0819, - 0x08080808, 0x082b082b, 0x082b082b, 0x082b082b, 0x2b080808, 0x082b082b, 0x2b2b2b08, 0x082b082b, - 0x08080819, 0x082b1908, 0x08081908, 0x082b1908, 0x08190808, 0x082b1908, 0x082b2b19, 0x082b1908, - 0x19080808, 0x082b1908, 0x08080808, 0x082b1919, 0x19080819, 0x082b1919, 0x1919082b, 0x082b1919, - 0x2b192b19, 0x082b1919, 0x08080819, 0x082b192b, 0x08192b2b, 0x082b192b, 0x2b2b192b, 0x082b192b, - 0x08080808, 0x082b2b08, 0x08082b08, 0x082b2b08, 0x08082b2b, 0x082b2b08, 0x082b0808, 0x082b2b08, - 0x19191919, 0x082b2b08, 0x2b082b08, 0x082b2b08, 0x2b2b082b, 0x082b2b08, 0x192b2b08, 0x082b2b19, - 0x2b190808, 0x082b2b19, 0x08082b08, 0x082b2b2b, 0x082b0808, 0x082b2b2b, 0x2b08082b, 0x082b2b2b, - 0x2b082b08, 0x082b2b2b, 0x2b082b2b, 0x082b2b2b, 0x08080819, 0x19080808, 0x08081908, 0x19080808, - 0x0808192b, 0x19080808, 0x08082b19, 0x19080808, 0x08190808, 0x19080808, 0x0819082b, 0x19080808, - 0x08191919, 0x19080808, 0x08192b08, 0x19080808, 0x082b0819, 0x19080808, 0x082b1908, 0x19080808, - 0x19080808, 0x19080808, 0x1908082b, 0x19080808, 0x19081919, 0x19080808, 0x19082b08, 0x19080808, - 0x19082b2b, 0x19080808, 0x19190819, 0x19080808, 0x19191908, 0x19080808, 0x192b0808, 0x19080808, - 0x192b1919, 0x19080808, 0x2b080819, 0x19080808, 0x2b081908, 0x19080808, 0x2b190808, 0x19080808, - 0x08080808, 0x19080819, 0x0808082b, 0x19080819, 0x08081919, 0x19080819, 0x08082b08, 0x19080819, - 0x08190819, 0x19080819, 0x08191908, 0x19080819, 0x082b0808, 0x19080819, 0x19080819, 0x19080819, - 0x19081908, 0x19080819, 0x19190808, 0x19080819, 0x2b080808, 0x19080819, 0x2b081919, 0x19080819, - 0x2b2b082b, 0x19080819, 0x08080819, 0x1908082b, 0x08081908, 0x1908082b, 0x08190808, 0x1908082b, - 0x0819082b, 0x1908082b, 0x082b2b19, 0x1908082b, 0x19080808, 0x1908082b, 0x08080808, 0x19081908, - 0x0808082b, 0x19081908, 0x08081919, 0x19081908, 0x08082b08, 0x19081908, 0x08190819, 0x19081908, - 0x08191908, 0x19081908, 0x08192b19, 0x19081908, 0x082b0808, 0x19081908, 0x19080819, 0x19081908, - 0x19081908, 0x19081908, 0x19190808, 0x19081908, 0x2b080808, 0x19081908, 0x2b191908, 0x19081908, - 0x08080819, 0x19081919, 0x08081908, 0x19081919, 0x08190808, 0x19081919, 0x082b1908, 0x19081919, - 0x19080808, 0x19081919, 0x2b192b2b, 0x19081919, 0x08080808, 0x1908192b, 0x08082b2b, 0x1908192b, - 0x19081908, 0x1908192b, 0x19190808, 0x1908192b, 0x08080819, 0x19082b08, 0x08081908, 0x19082b08, - 0x08190808, 0x19082b08, 0x19080808, 0x19082b08, 0x19081919, 0x19082b08, 0x19191908, 0x19082b08, - 0x192b082b, 0x19082b08, 0x08080808, 0x19082b19, 0x08190819, 0x19082b19, 0x19081908, 0x19082b19, - 0x19190808, 0x19082b19, 0x192b2b19, 0x19082b19, 0x08081908, 0x19082b2b, 0x08080808, 0x19190808, - 0x0808082b, 0x19190808, 0x08081919, 0x19190808, 0x08082b08, 0x19190808, 0x08190819, 0x19190808, - 0x08191908, 0x19190808, 0x082b0808, 0x19190808, 0x082b2b08, 0x19190808, 0x19080819, 0x19190808, - 0x19081908, 0x19190808, 0x19190808, 0x19190808, 0x2b080808, 0x19190808, 0x08080819, 0x19190819, - 0x08081908, 0x19190819, 0x08190808, 0x19190819, 0x08191919, 0x19190819, 0x19080808, 0x19190819, - 0x1908082b, 0x19190819, 0x08080808, 0x1919082b, 0x19081908, 0x1919082b, 0x2b2b2b2b, 0x1919082b, - 0x08080819, 0x19191908, 0x08081908, 0x19191908, 0x08190808, 0x19191908, 0x082b0819, 0x19191908, - 0x19080808, 0x19191908, 0x192b0808, 0x19191908, 0x2b080819, 0x19191908, 0x2b2b0819, 0x19191908, - 0x08080808, 0x19191919, 0x08082b08, 0x19191919, 0x2b080808, 0x19191919, 0x2b082b08, 0x19191919, - 0x082b0819, 0x1919192b, 0x192b2b08, 0x1919192b, 0x2b2b0819, 0x1919192b, 0x08080808, 0x19192b08, - 0x08191908, 0x19192b08, 0x19080819, 0x19192b08, 0x19190808, 0x19192b08, 0x2b192b19, 0x19192b08, - 0x08192b2b, 0x19192b19, 0x19080808, 0x19192b19, 0x1908082b, 0x19192b19, 0x2b081919, 0x19192b2b, - 0x08080819, 0x192b0808, 0x08081908, 0x192b0808, 0x08190808, 0x192b0808, 0x19080808, 0x192b0808, - 0x19191908, 0x192b0808, 0x192b082b, 0x192b0808, 0x2b08192b, 0x192b0808, 0x2b2b2b19, 0x192b0808, - 0x08080808, 0x192b0819, 0x082b1908, 0x192b082b, 0x19082b2b, 0x192b082b, 0x2b19082b, 0x192b082b, - 0x08080808, 0x192b1908, 0x0819192b, 0x192b1908, 0x08190808, 0x192b1919, 0x19080808, 0x192b1919, - 0x19081919, 0x192b1919, 0x2b2b1908, 0x192b1919, 0x08080819, 0x192b2b08, 0x192b2b2b, 0x192b2b08, - 0x082b1919, 0x192b2b19, 0x0808192b, 0x192b2b2b, 0x19191908, 0x192b2b2b, 0x192b082b, 0x192b2b2b, - 0x08080808, 0x2b080808, 0x0808082b, 0x2b080808, 0x08081919, 0x2b080808, 0x08082b08, 0x2b080808, - 0x08190819, 0x2b080808, 0x08191908, 0x2b080808, 0x082b0808, 0x2b080808, 0x082b2b2b, 0x2b080808, - 0x19080819, 0x2b080808, 0x19081908, 0x2b080808, 0x19190808, 0x2b080808, 0x2b080808, 0x2b080808, - 0x2b08082b, 0x2b080808, 0x2b2b2b08, 0x2b080808, 0x2b2b2b2b, 0x2b080808, 0x08080819, 0x2b080819, - 0x08081908, 0x2b080819, 0x0808192b, 0x2b080819, 0x08190808, 0x2b080819, 0x19080808, 0x2b080819, - 0x19190819, 0x2b080819, 0x19192b19, 0x2b080819, 0x08080808, 0x2b08082b, 0x082b0808, 0x2b08082b, - 0x2b080808, 0x2b08082b, 0x2b08082b, 0x2b08082b, 0x2b2b0808, 0x2b08082b, 0x2b2b2b08, 0x2b08082b, - 0x08080819, 0x2b081908, 0x08081908, 0x2b081908, 0x08190808, 0x2b081908, 0x0819082b, 0x2b081908, - 0x08191919, 0x2b081908, 0x19080808, 0x2b081908, 0x192b0808, 0x2b081908, 0x2b082b19, 0x2b081908, - 0x08080808, 0x2b081919, 0x19081908, 0x2b081919, 0x2b2b1919, 0x2b081919, 0x08192b08, 0x2b08192b, - 0x192b2b2b, 0x2b08192b, 0x08080808, 0x2b082b08, 0x08082b08, 0x2b082b08, 0x082b1919, 0x2b082b08, - 0x19192b2b, 0x2b082b08, 0x2b080808, 0x2b082b08, 0x2b08082b, 0x2b082b08, 0x2b2b2b08, 0x2b082b08, - 0x0808192b, 0x2b082b19, 0x082b082b, 0x2b082b2b, 0x2b080808, 0x2b082b2b, 0x2b082b08, 0x2b082b2b, - 0x2b19192b, 0x2b082b2b, 0x2b2b2b08, 0x2b082b2b, 0x08080819, 0x2b190808, 0x08081908, 0x2b190808, - 0x08190808, 0x2b190808, 0x19080808, 0x2b190808, 0x1919192b, 0x2b190808, 0x2b081908, 0x2b190808, - 0x08080808, 0x2b190819, 0x082b082b, 0x2b190819, 0x192b1908, 0x2b190819, 0x1919192b, 0x2b19082b, - 0x2b082b19, 0x2b19082b, 0x08080808, 0x2b191908, 0x08081919, 0x2b191908, 0x19081908, 0x2b191908, - 0x19190808, 0x2b191908, 0x19192b08, 0x2b191908, 0x082b2b19, 0x2b191919, 0x2b190808, 0x2b191919, - 0x2b19082b, 0x2b191919, 0x19080819, 0x2b19192b, 0x19190819, 0x2b192b08, 0x2b2b192b, 0x2b192b08, - 0x19082b19, 0x2b192b19, 0x08191919, 0x2b192b2b, 0x192b0808, 0x2b192b2b, 0x08080808, 0x2b2b0808, - 0x0808082b, 0x2b2b0808, 0x08082b08, 0x2b2b0808, 0x08082b2b, 0x2b2b0808, 0x082b0808, 0x2b2b0808, - 0x082b2b2b, 0x2b2b0808, 0x2b2b0808, 0x2b2b0808, 0x19190819, 0x2b2b0819, 0x19192b19, 0x2b2b0819, - 0x2b2b192b, 0x2b2b0819, 0x08080808, 0x2b2b082b, 0x0808082b, 0x2b2b082b, 0x08082b08, 0x2b2b082b, - 0x082b2b2b, 0x2b2b082b, 0x2b080808, 0x2b2b082b, 0x2b2b0808, 0x2b2b082b, 0x19080808, 0x2b2b1908, - 0x2b191919, 0x2b2b1908, 0x192b1919, 0x2b2b192b, 0x2b192b08, 0x2b2b192b, 0x08082b2b, 0x2b2b2b08, - 0x082b0808, 0x2b2b2b08, 0x082b082b, 0x2b2b2b08, 0x082b2b08, 0x2b2b2b08, 0x2b2b0808, 0x2b2b2b08, - 0x2b2b2b08, 0x2b2b2b08, 0x08081908, 0x2b2b2b19, 0x2b081908, 0x2b2b2b19, 0x2b08192b, 0x2b2b2b19, + 0x08080808, 0x08080808, 0x0808082b, 0x08080808, 0x08081919, 0x08080808, 0x08082b08, 0x08080808, + 0x08082b2b, 0x08080808, 0x08190819, 0x08080808, 0x08191908, 0x08080808, 0x0819192b, 0x08080808, + 0x08192b19, 0x08080808, 0x082b0808, 0x08080808, 0x082b082b, 0x08080808, 0x082b1919, 0x08080808, + 0x082b2b08, 0x08080808, 0x19080819, 0x08080808, 0x19081908, 0x08080808, 0x1908192b, 0x08080808, + 0x19082b19, 0x08080808, 0x19190808, 0x08080808, 0x1919082b, 0x08080808, 0x19191919, 0x08080808, + 0x19192b08, 0x08080808, 0x192b0819, 0x08080808, 0x192b1908, 0x08080808, 0x2b080808, 0x08080808, + 0x2b08082b, 0x08080808, 0x2b081919, 0x08080808, 0x2b082b08, 0x08080808, 0x2b190819, 0x08080808, + 0x2b191908, 0x08080808, 0x2b192b19, 0x08080808, 0x2b2b0808, 0x08080808, 0x08080819, 0x08080819, + 0x08081908, 0x08080819, 0x0808192b, 0x08080819, 0x08082b19, 0x08080819, 0x08190808, 0x08080819, + 0x0819082b, 0x08080819, 0x08191919, 0x08080819, 0x08192b08, 0x08080819, 0x08192b2b, 0x08080819, + 0x082b0819, 0x08080819, 0x082b1908, 0x08080819, 0x19080808, 0x08080819, 0x1908082b, 0x08080819, + 0x19081919, 0x08080819, 0x19082b08, 0x08080819, 0x19190819, 0x08080819, 0x19191908, 0x08080819, + 0x192b0808, 0x08080819, 0x192b2b08, 0x08080819, 0x2b080819, 0x08080819, 0x2b081908, 0x08080819, + 0x2b190808, 0x08080819, 0x08080808, 0x0808082b, 0x0808082b, 0x0808082b, 0x08081919, 0x0808082b, + 0x08082b08, 0x0808082b, 0x08190819, 0x0808082b, 0x08191908, 0x0808082b, 0x082b0808, 0x0808082b, + 0x19080819, 0x0808082b, 0x19081908, 0x0808082b, 0x19190808, 0x0808082b, 0x19191919, 0x0808082b, + 0x2b080808, 0x0808082b, 0x2b082b2b, 0x0808082b, 0x08080819, 0x08081908, 0x08081908, 0x08081908, + 0x0808192b, 0x08081908, 0x08082b19, 0x08081908, 0x08190808, 0x08081908, 0x0819082b, 0x08081908, + 0x08191919, 0x08081908, 0x08192b08, 0x08081908, 0x082b0819, 0x08081908, 0x082b1908, 0x08081908, + 0x19080808, 0x08081908, 0x1908082b, 0x08081908, 0x19081919, 0x08081908, 0x19082b08, 0x08081908, + 0x19190819, 0x08081908, 0x19191908, 0x08081908, 0x1919192b, 0x08081908, 0x192b0808, 0x08081908, + 0x2b080819, 0x08081908, 0x2b081908, 0x08081908, 0x2b190808, 0x08081908, 0x08080808, 0x08081919, + 0x0808082b, 0x08081919, 0x08081919, 0x08081919, 0x08082b08, 0x08081919, 0x08190819, 0x08081919, + 0x08191908, 0x08081919, 0x082b0808, 0x08081919, 0x19080819, 0x08081919, 0x19081908, 0x08081919, + 0x19190808, 0x08081919, 0x192b0819, 0x08081919, 0x2b080808, 0x08081919, 0x08080819, 0x0808192b, + 0x08081908, 0x0808192b, 0x08190808, 0x0808192b, 0x082b192b, 0x0808192b, 0x19080808, 0x0808192b, + 0x1908082b, 0x0808192b, 0x2b081908, 0x0808192b, 0x08080808, 0x08082b08, 0x0808082b, 0x08082b08, + 0x08081919, 0x08082b08, 0x08082b08, 0x08082b08, 0x08082b2b, 0x08082b08, 0x08190819, 0x08082b08, + 0x08191908, 0x08082b08, 0x082b0808, 0x08082b08, 0x082b1919, 0x08082b08, 0x19080819, 0x08082b08, + 0x19081908, 0x08082b08, 0x19190808, 0x08082b08, 0x19192b08, 0x08082b08, 0x2b080808, 0x08082b08, + 0x2b2b0808, 0x08082b08, 0x2b2b2b2b, 0x08082b08, 0x08080819, 0x08082b19, 0x08081908, 0x08082b19, + 0x08190808, 0x08082b19, 0x19080808, 0x08082b19, 0x2b080819, 0x08082b19, 0x2b082b19, 0x08082b19, + 0x08080808, 0x08082b2b, 0x082b0808, 0x08082b2b, 0x082b2b08, 0x08082b2b, 0x2b19192b, 0x08082b2b, + 0x2b2b0808, 0x08082b2b, 0x08080819, 0x08190808, 0x08081908, 0x08190808, 0x0808192b, 0x08190808, + 0x08082b19, 0x08190808, 0x08190808, 0x08190808, 0x0819082b, 0x08190808, 0x08191919, 0x08190808, + 0x08192b08, 0x08190808, 0x082b0819, 0x08190808, 0x082b1908, 0x08190808, 0x19080808, 0x08190808, + 0x1908082b, 0x08190808, 0x19081919, 0x08190808, 0x19082b08, 0x08190808, 0x19190819, 0x08190808, + 0x19191908, 0x08190808, 0x192b0808, 0x08190808, 0x192b2b2b, 0x08190808, 0x2b080819, 0x08190808, + 0x2b081908, 0x08190808, 0x2b190808, 0x08190808, 0x08080808, 0x08190819, 0x0808082b, 0x08190819, + 0x08081919, 0x08190819, 0x08082b08, 0x08190819, 0x08190819, 0x08190819, 0x08191908, 0x08190819, + 0x082b0808, 0x08190819, 0x19080819, 0x08190819, 0x19081908, 0x08190819, 0x19190808, 0x08190819, + 0x2b080808, 0x08190819, 0x2b191908, 0x08190819, 0x2b19192b, 0x08190819, 0x08080819, 0x0819082b, + 0x08081908, 0x0819082b, 0x0808192b, 0x0819082b, 0x08190808, 0x0819082b, 0x19080808, 0x0819082b, + 0x192b0808, 0x0819082b, 0x08080808, 0x08191908, 0x0808082b, 0x08191908, 0x08081919, 0x08191908, + 0x08082b08, 0x08191908, 0x08190819, 0x08191908, 0x08191908, 0x08191908, 0x082b0808, 0x08191908, + 0x19080819, 0x08191908, 0x19081908, 0x08191908, 0x19082b19, 0x08191908, 0x19190808, 0x08191908, + 0x192b1908, 0x08191908, 0x2b080808, 0x08191908, 0x08080819, 0x08191919, 0x08081908, 0x08191919, + 0x08190808, 0x08191919, 0x19080808, 0x08191919, 0x08080808, 0x0819192b, 0x08191908, 0x0819192b, + 0x19082b19, 0x0819192b, 0x08080819, 0x08192b08, 0x08081908, 0x08192b08, 0x08190808, 0x08192b08, + 0x0819082b, 0x08192b08, 0x19080808, 0x08192b08, 0x19191908, 0x08192b08, 0x2b08192b, 0x08192b08, + 0x08080808, 0x08192b19, 0x08081919, 0x08192b19, 0x192b192b, 0x08192b19, 0x19190819, 0x08192b2b, + 0x2b2b2b19, 0x08192b2b, 0x08080808, 0x082b0808, 0x0808082b, 0x082b0808, 0x08081919, 0x082b0808, + 0x08082b08, 0x082b0808, 0x08082b2b, 0x082b0808, 0x08190819, 0x082b0808, 0x08191908, 0x082b0808, + 0x082b0808, 0x082b0808, 0x19080819, 0x082b0808, 0x19081908, 0x082b0808, 0x19190808, 0x082b0808, + 0x2b080808, 0x082b0808, 0x2b2b0808, 0x082b0808, 0x08080819, 0x082b0819, 0x08081908, 0x082b0819, + 0x08190808, 0x082b0819, 0x19080808, 0x082b0819, 0x19082b08, 0x082b0819, 0x192b1919, 0x082b0819, + 0x08080808, 0x082b082b, 0x082b082b, 0x082b082b, 0x2b080808, 0x082b082b, 0x2b2b2b08, 0x082b082b, + 0x08080819, 0x082b1908, 0x08081908, 0x082b1908, 0x08190808, 0x082b1908, 0x082b2b19, 0x082b1908, + 0x19080808, 0x082b1908, 0x08080808, 0x082b1919, 0x19080819, 0x082b1919, 0x1919082b, 0x082b1919, + 0x2b192b19, 0x082b1919, 0x08080819, 0x082b192b, 0x08192b2b, 0x082b192b, 0x2b2b192b, 0x082b192b, + 0x08080808, 0x082b2b08, 0x08082b08, 0x082b2b08, 0x08082b2b, 0x082b2b08, 0x082b0808, 0x082b2b08, + 0x19191919, 0x082b2b08, 0x2b082b08, 0x082b2b08, 0x2b2b082b, 0x082b2b08, 0x192b2b08, 0x082b2b19, + 0x2b190808, 0x082b2b19, 0x08082b08, 0x082b2b2b, 0x082b0808, 0x082b2b2b, 0x2b08082b, 0x082b2b2b, + 0x2b082b08, 0x082b2b2b, 0x2b082b2b, 0x082b2b2b, 0x08080819, 0x19080808, 0x08081908, 0x19080808, + 0x0808192b, 0x19080808, 0x08082b19, 0x19080808, 0x08190808, 0x19080808, 0x0819082b, 0x19080808, + 0x08191919, 0x19080808, 0x08192b08, 0x19080808, 0x082b0819, 0x19080808, 0x082b1908, 0x19080808, + 0x19080808, 0x19080808, 0x1908082b, 0x19080808, 0x19081919, 0x19080808, 0x19082b08, 0x19080808, + 0x19082b2b, 0x19080808, 0x19190819, 0x19080808, 0x19191908, 0x19080808, 0x192b0808, 0x19080808, + 0x192b1919, 0x19080808, 0x2b080819, 0x19080808, 0x2b081908, 0x19080808, 0x2b190808, 0x19080808, + 0x08080808, 0x19080819, 0x0808082b, 0x19080819, 0x08081919, 0x19080819, 0x08082b08, 0x19080819, + 0x08190819, 0x19080819, 0x08191908, 0x19080819, 0x082b0808, 0x19080819, 0x19080819, 0x19080819, + 0x19081908, 0x19080819, 0x19190808, 0x19080819, 0x2b080808, 0x19080819, 0x2b081919, 0x19080819, + 0x2b2b082b, 0x19080819, 0x08080819, 0x1908082b, 0x08081908, 0x1908082b, 0x08190808, 0x1908082b, + 0x0819082b, 0x1908082b, 0x082b2b19, 0x1908082b, 0x19080808, 0x1908082b, 0x08080808, 0x19081908, + 0x0808082b, 0x19081908, 0x08081919, 0x19081908, 0x08082b08, 0x19081908, 0x08190819, 0x19081908, + 0x08191908, 0x19081908, 0x08192b19, 0x19081908, 0x082b0808, 0x19081908, 0x19080819, 0x19081908, + 0x19081908, 0x19081908, 0x19190808, 0x19081908, 0x2b080808, 0x19081908, 0x2b191908, 0x19081908, + 0x08080819, 0x19081919, 0x08081908, 0x19081919, 0x08190808, 0x19081919, 0x082b1908, 0x19081919, + 0x19080808, 0x19081919, 0x2b192b2b, 0x19081919, 0x08080808, 0x1908192b, 0x08082b2b, 0x1908192b, + 0x19081908, 0x1908192b, 0x19190808, 0x1908192b, 0x08080819, 0x19082b08, 0x08081908, 0x19082b08, + 0x08190808, 0x19082b08, 0x19080808, 0x19082b08, 0x19081919, 0x19082b08, 0x19191908, 0x19082b08, + 0x192b082b, 0x19082b08, 0x08080808, 0x19082b19, 0x08190819, 0x19082b19, 0x19081908, 0x19082b19, + 0x19190808, 0x19082b19, 0x192b2b19, 0x19082b19, 0x08081908, 0x19082b2b, 0x08080808, 0x19190808, + 0x0808082b, 0x19190808, 0x08081919, 0x19190808, 0x08082b08, 0x19190808, 0x08190819, 0x19190808, + 0x08191908, 0x19190808, 0x082b0808, 0x19190808, 0x082b2b08, 0x19190808, 0x19080819, 0x19190808, + 0x19081908, 0x19190808, 0x19190808, 0x19190808, 0x2b080808, 0x19190808, 0x08080819, 0x19190819, + 0x08081908, 0x19190819, 0x08190808, 0x19190819, 0x08191919, 0x19190819, 0x19080808, 0x19190819, + 0x1908082b, 0x19190819, 0x08080808, 0x1919082b, 0x19081908, 0x1919082b, 0x2b2b2b2b, 0x1919082b, + 0x08080819, 0x19191908, 0x08081908, 0x19191908, 0x08190808, 0x19191908, 0x082b0819, 0x19191908, + 0x19080808, 0x19191908, 0x192b0808, 0x19191908, 0x2b080819, 0x19191908, 0x2b2b0819, 0x19191908, + 0x08080808, 0x19191919, 0x08082b08, 0x19191919, 0x2b080808, 0x19191919, 0x2b082b08, 0x19191919, + 0x082b0819, 0x1919192b, 0x192b2b08, 0x1919192b, 0x2b2b0819, 0x1919192b, 0x08080808, 0x19192b08, + 0x08191908, 0x19192b08, 0x19080819, 0x19192b08, 0x19190808, 0x19192b08, 0x2b192b19, 0x19192b08, + 0x08192b2b, 0x19192b19, 0x19080808, 0x19192b19, 0x1908082b, 0x19192b19, 0x2b081919, 0x19192b2b, + 0x08080819, 0x192b0808, 0x08081908, 0x192b0808, 0x08190808, 0x192b0808, 0x19080808, 0x192b0808, + 0x19191908, 0x192b0808, 0x192b082b, 0x192b0808, 0x2b08192b, 0x192b0808, 0x2b2b2b19, 0x192b0808, + 0x08080808, 0x192b0819, 0x082b1908, 0x192b082b, 0x19082b2b, 0x192b082b, 0x2b19082b, 0x192b082b, + 0x08080808, 0x192b1908, 0x0819192b, 0x192b1908, 0x08190808, 0x192b1919, 0x19080808, 0x192b1919, + 0x19081919, 0x192b1919, 0x2b2b1908, 0x192b1919, 0x08080819, 0x192b2b08, 0x192b2b2b, 0x192b2b08, + 0x082b1919, 0x192b2b19, 0x0808192b, 0x192b2b2b, 0x19191908, 0x192b2b2b, 0x192b082b, 0x192b2b2b, + 0x08080808, 0x2b080808, 0x0808082b, 0x2b080808, 0x08081919, 0x2b080808, 0x08082b08, 0x2b080808, + 0x08190819, 0x2b080808, 0x08191908, 0x2b080808, 0x082b0808, 0x2b080808, 0x082b2b2b, 0x2b080808, + 0x19080819, 0x2b080808, 0x19081908, 0x2b080808, 0x19190808, 0x2b080808, 0x2b080808, 0x2b080808, + 0x2b08082b, 0x2b080808, 0x2b2b2b08, 0x2b080808, 0x2b2b2b2b, 0x2b080808, 0x08080819, 0x2b080819, + 0x08081908, 0x2b080819, 0x0808192b, 0x2b080819, 0x08190808, 0x2b080819, 0x19080808, 0x2b080819, + 0x19190819, 0x2b080819, 0x19192b19, 0x2b080819, 0x08080808, 0x2b08082b, 0x082b0808, 0x2b08082b, + 0x2b080808, 0x2b08082b, 0x2b08082b, 0x2b08082b, 0x2b2b0808, 0x2b08082b, 0x2b2b2b08, 0x2b08082b, + 0x08080819, 0x2b081908, 0x08081908, 0x2b081908, 0x08190808, 0x2b081908, 0x0819082b, 0x2b081908, + 0x08191919, 0x2b081908, 0x19080808, 0x2b081908, 0x192b0808, 0x2b081908, 0x2b082b19, 0x2b081908, + 0x08080808, 0x2b081919, 0x19081908, 0x2b081919, 0x2b2b1919, 0x2b081919, 0x08192b08, 0x2b08192b, + 0x192b2b2b, 0x2b08192b, 0x08080808, 0x2b082b08, 0x08082b08, 0x2b082b08, 0x082b1919, 0x2b082b08, + 0x19192b2b, 0x2b082b08, 0x2b080808, 0x2b082b08, 0x2b08082b, 0x2b082b08, 0x2b2b2b08, 0x2b082b08, + 0x0808192b, 0x2b082b19, 0x082b082b, 0x2b082b2b, 0x2b080808, 0x2b082b2b, 0x2b082b08, 0x2b082b2b, + 0x2b19192b, 0x2b082b2b, 0x2b2b2b08, 0x2b082b2b, 0x08080819, 0x2b190808, 0x08081908, 0x2b190808, + 0x08190808, 0x2b190808, 0x19080808, 0x2b190808, 0x1919192b, 0x2b190808, 0x2b081908, 0x2b190808, + 0x08080808, 0x2b190819, 0x082b082b, 0x2b190819, 0x192b1908, 0x2b190819, 0x1919192b, 0x2b19082b, + 0x2b082b19, 0x2b19082b, 0x08080808, 0x2b191908, 0x08081919, 0x2b191908, 0x19081908, 0x2b191908, + 0x19190808, 0x2b191908, 0x19192b08, 0x2b191908, 0x082b2b19, 0x2b191919, 0x2b190808, 0x2b191919, + 0x2b19082b, 0x2b191919, 0x19080819, 0x2b19192b, 0x19190819, 0x2b192b08, 0x2b2b192b, 0x2b192b08, + 0x19082b19, 0x2b192b19, 0x08191919, 0x2b192b2b, 0x192b0808, 0x2b192b2b, 0x08080808, 0x2b2b0808, + 0x0808082b, 0x2b2b0808, 0x08082b08, 0x2b2b0808, 0x08082b2b, 0x2b2b0808, 0x082b0808, 0x2b2b0808, + 0x082b2b2b, 0x2b2b0808, 0x2b2b0808, 0x2b2b0808, 0x19190819, 0x2b2b0819, 0x19192b19, 0x2b2b0819, + 0x2b2b192b, 0x2b2b0819, 0x08080808, 0x2b2b082b, 0x0808082b, 0x2b2b082b, 0x08082b08, 0x2b2b082b, + 0x082b2b2b, 0x2b2b082b, 0x2b080808, 0x2b2b082b, 0x2b2b0808, 0x2b2b082b, 0x19080808, 0x2b2b1908, + 0x2b191919, 0x2b2b1908, 0x192b1919, 0x2b2b192b, 0x2b192b08, 0x2b2b192b, 0x08082b2b, 0x2b2b2b08, + 0x082b0808, 0x2b2b2b08, 0x082b082b, 0x2b2b2b08, 0x082b2b08, 0x2b2b2b08, 0x2b2b0808, 0x2b2b2b08, + 0x2b2b2b08, 0x2b2b2b08, 0x08081908, 0x2b2b2b19, 0x2b081908, 0x2b2b2b19, 0x2b08192b, 0x2b2b2b19, 0x082b2b08, 0x2b2b2b2b, 0x082b2b2b, 0x2b2b2b2b, 0x2b190819, 0x2b2b2b2b, 0x2b2b2b2b, 0x2b2b2b2b) return tbl[i] } @@ -6641,7 +6634,7 @@ def private iq2xs_gw(i : int) : uint { // nolint:STYLE038 - a 1024-word data t // iq2s_grid as low/high word pairs (word 2i = magnitudes 0..3 of u64 entry i, word 2i+1 // = 4..7); an all-literal fixed_array local, so it lowers to a program-scope constant table - // every kernel reads it DIRECT (an f4 slab of 1024 entries would be 32 KB, past the tg budget). -def private iq2s_gw(i : int) : uint { // nolint:STYLE038 - a 2048-word data table, not splittable +def public iq2s_gw(i : int) : uint { // nolint:STYLE038 - a 2048-word data table, not splittable let tbl = fixed_array( 0x08080808, 0x08080808, 0x0808082b, 0x08080808, 0x08081919, 0x08080808, 0x08082b08, 0x08080808, 0x08082b2b, 0x08080808, 0x08190819, 0x08080808, 0x08191908, 0x08080808, 0x0819192b, 0x08080808, diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das b/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das index b6687d0a7e..4e5ba88e23 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das @@ -14,6 +14,7 @@ require daslib/typemacro_boost require dasllama/dasllama_common require dasllama/dasllama_math require dasllama/dasllama_metal_common +require dasllama/dasllama_metal_kernels // the shared kq decode helpers (grid tables, ksign7m) for the tensor twins require dasllama/dasllama_metal_kernels require dasllama/dasllama_metal_lens @@ -841,6 +842,442 @@ class MetalKqMulMmK6THDb128 : MetalKqMulMmK6DbT { override MT = 128u } +// The dense-KQ tensor mul_mm scaffold for the iquant/split-scale formats: the k6 tensor +// shell with the per-format decode as ONE overridable stage16 (16 elements per work item - +// the base template's own arm granularity, so an arm ports verbatim, va[] store -> twb). +[ |> template_struct_instance] +class template MetalKqMulMmSplitTensorBase { + @template_constant MT : uint = 32u + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array + @ssbo @binding = 1 @role = "weight" @off = "soff" ksc : array + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array + @ssbo @binding = 3 @off = "xoff" xf : array //!< raw activations (float, or the converted f16 panel) + @ssbo @binding = 4 @off = "yoff" @span = "mp*rows*4" y : array + @uniform @binding = 5 kdim : uint + @uniform @binding = 6 ndim : uint + @workgroup twb : float16[6144] //!< W chunk: 64 wrows x 72-half padded rows + + def abstract stage16(work, kb, nBase, nsb : uint) : void + + def run_shell { + let mBase = gl_WorkGroupID.x * MT + let nBase = gl_WorkGroupID.y * 64u + let nkb = kdim / 32u + let nsb = kdim / 256u + let lid = gl_LocalInvocationID.x + var acc : float[8192] + var cp = unsafe(addr(y[mBase * ndim + nBase])) + tmm2d_tg_begin_deva(acc, unsafe(addr(xf[0])), MT, 64u, 4u, 64u) + var kb = 0u + while (kb < nkb) { + var work = lid + while (work < 256u) { + stage16(work, kb, nBase, nsb) + work += gl_WorkGroupSize.x + } + barrier() + tmm2d_tg_step_deva(acc, unsafe(addr(xf[mBase * kdim + kb * 32u])), kdim, twb, MT, 64u, 64u) + barrier() + kb += 2u + } + tmm2d_tg_store(acc, cp, MT, 64u, ndim) + } +} + +[ |> template_struct_instance] +class template MetalKqMulMmIq4xsTensorT : MetalKqMulMmSplitTensorBase { + @template_constant IQ4NL : bool = false + @workgroup iq4lutt : float[16] //!< kvalues_iq4nl as floats, staged once per threadgroup + + [metal_kernel(float_a_ok=true)] + def metal_kq_mulmm_iq4xs_t { + if (gl_LocalInvocationID.x < 16u) { + iq4lutt[gl_LocalInvocationID.x] = iq4_lut(gl_LocalInvocationID.x) + } + barrier() + run_shell() + } + + def override stage16(work, kb, nBase, nsb : uint) { + let j = work >> 2u + let sub = work & 3u + let jsl = kb + (sub >> 1u) + let sb = jsl / 8u + let js = jsl % 8u + let kh = (sub & 1u) * 16u + let blk = (nBase + j) * nsb + sb + var dsc : float + static_if (IQ4NL) { + dsc = float(kdh[blk * 8u + js]) + } else { + let sv = ksc[blk] + let scw = js < 4u ? sv.x : sv.y + dsc = float(kdh[blk]) * float(((int(scw >> ((js & 3u) * 8u)) & 255) ^ 128) - 128) + } + let qb = blk * 32u + js * 4u + let nsh = (sub & 1u) * 4u + for [unroll_full] (k in range(4)) { + let u = kqu[qb + uint(k)] + for [unroll_full] (c in range(4)) { + twb[j * 72u + (sub >> 1u) * 32u + kh + uint(k) * 4u + uint(c)] = float16(dsc * iq4lutt[(u >> (8u * uint(c) + nsh)) & 15u]) + } + } + } +} + +class MetalKqMulMmIq4xsT : MetalKqMulMmIq4xsTensorT { + typedef XT = float +} + +class MetalKqMulMmIq4xsTH : MetalKqMulMmIq4xsTensorT { + typedef XT = float16 +} + +class MetalKqMulMmIq4nlT : MetalKqMulMmIq4xsTensorT { + typedef XT = float + override IQ4NL = true +} + +class MetalKqMulMmIq4nlTH : MetalKqMulMmIq4xsTensorT { + typedef XT = float16 + override IQ4NL = true +} + +// q3_K / q2_K: the byte-parallel compose of their base arms (masked uint-wide shifts, the +// byte-position shift folded into exact power-of-two pre-scales). +[ |> template_struct_instance] +class template MetalKqMulMmK3TensorT : MetalKqMulMmSplitTensorBase { + [metal_kernel(float_a_ok=true)] + def metal_kq_mulmm_k3_t { + run_shell() + } + + def override stage16(work, kb, nBase, nsb : uint) { + let j = work >> 2u + let sub = work & 3u + let jsl = kb + (sub >> 1u) + let sb = jsl / 8u + let js = jsl % 8u + let kh = (sub & 1u) * 16u + let ile = sub & 1u + let blk = (nBase + j) * nsb + sb + let hf3 = js / 4u + let gg3 = js % 4u + let sv = ksc[blk] + let si = hf3 * 8u + gg3 * 2u + ile + let scw = si < 8u ? (si < 4u ? sv.x : sv.y) : (si < 12u ? sv.z : sv.w) + let s3 = float(((int(scw >> ((si & 3u) * 8u)) & 255) ^ 128) - 128) + let dsc = float(kdh[blk]) * s3 + let dmn = dsc * 4.0 + let dsc1 = dsc * 0.00390625 + let dsc2 = dsc * 0.0000152587890625 + let dsc3 = dsc * 0.000000059604644775390625 + let qb = blk * 24u + hf3 * 8u + ile * 4u + let hb = blk * 24u + 16u + ile * 4u + let nsh = gg3 * 2u + let hsh = hf3 * 4u + gg3 + let tb0 = j * 72u + (sub >> 1u) * 32u + kh + for [unroll_full] (k in range(4)) { + let u = kqu[qb + uint(k)] + let hu = kqu[hb + uint(k)] + let qv = ((u >> nsh) & 0x03030303) | (((hu >> hsh) & 0x01010101) << 2u) + twb[tb0 + uint(k) * 4u] = float16(dsc * float(qv & 255u) - dmn) + twb[tb0 + uint(k) * 4u + 1u] = float16(dsc1 * float(qv & 65280u) - dmn) + twb[tb0 + uint(k) * 4u + 2u] = float16(dsc2 * float(qv & 16711680u) - dmn) + twb[tb0 + uint(k) * 4u + 3u] = float16(dsc3 * float(qv & 4278190080u) - dmn) + } + } +} + +class MetalKqMulMmK3T : MetalKqMulMmK3TensorT { + typedef XT = float +} + +class MetalKqMulMmK3TH : MetalKqMulMmK3TensorT { + typedef XT = float16 +} + +[ |> template_struct_instance] +class template MetalKqMulMmK2TensorT : MetalKqMulMmSplitTensorBase { + [metal_kernel(float_a_ok=true)] + def metal_kq_mulmm_k2_t { + run_shell() + } + + def override stage16(work, kb, nBase, nsb : uint) { + let j = work >> 2u + let sub = work & 3u + let jsl = kb + (sub >> 1u) + let sb = jsl / 8u + let js = jsl % 8u + let kh = (sub & 1u) * 16u + let ile = sub & 1u + let blk = (nBase + j) * nsb + sb + let hf3 = js / 4u + let gg3 = js % 4u + let sv = ksc[blk] + let si = hf3 * 8u + gg3 * 2u + ile + let pw = si < 8u ? (si < 4u ? sv.x : sv.y) : (si < 12u ? sv.z : sv.w) + let pb = (pw >> ((si & 3u) * 8u)) & 255u + let dsc = float(kdh[blk * 2u]) * float(pb & 15u) + let dmn = float(kdh[blk * 2u + 1u]) * float(pb >> 4u) + let dsc1 = dsc * 0.00390625 + let dsc2 = dsc * 0.0000152587890625 + let dsc3 = dsc * 0.000000059604644775390625 + let qb = blk * 16u + hf3 * 8u + ile * 4u + let nsh = gg3 * 2u + let tb0 = j * 72u + (sub >> 1u) * 32u + kh + for [unroll_full] (k in range(4)) { + let qv = (kqu[qb + uint(k)] >> nsh) & 0x03030303 + twb[tb0 + uint(k) * 4u] = float16(dsc * float(qv & 255u) - dmn) + twb[tb0 + uint(k) * 4u + 1u] = float16(dsc1 * float(qv & 65280u) - dmn) + twb[tb0 + uint(k) * 4u + 2u] = float16(dsc2 * float(qv & 16711680u) - dmn) + twb[tb0 + uint(k) * 4u + 3u] = float16(dsc3 * float(qv & 4278190080u) - dmn) + } + } +} + +class MetalKqMulMmK2T : MetalKqMulMmK2TensorT { + typedef XT = float +} + +class MetalKqMulMmK2TH : MetalKqMulMmK2TensorT { + typedef XT = float16 +} + +// iq3s / iq3xxs: the staged-grid gathers of their base arms (2 KB / 1 KB threadgroup slabs). +[ |> template_struct_instance] +class template MetalKqMulMmIq3sTensorT : MetalKqMulMmSplitTensorBase { + @workgroup gridw3t : uint[512] //!< the 2 KB iq3s grid, staged once per threadgroup + + [metal_kernel(float_a_ok=true)] + def metal_kq_mulmm_iq3s_t { + for [unroll_full] (gk in range(4)) { + let gi = gl_LocalInvocationID.x + uint(gk) * 128u + gridw3t[gi] = iq3s_gw(int(gi)) + } + barrier() + run_shell() + } + + def override stage16(work, kb, nBase, nsb : uint) { + let j = work >> 2u + let sub = work & 3u + let jsl = kb + (sub >> 1u) + let sb = jsl / 8u + let js = jsl % 8u + let kh = (sub & 1u) * 16u + let ile = sub & 1u + let blk = (nBase + j) * nsb + sb + let sv = ksc[blk] + let scw = js < 4u ? sv.x : sv.y + let dsc = float(kdh[blk]) * float(((int(scw >> ((js & 3u) * 8u)) & 255) ^ 128) - 128) + let qsw = kqu[blk * 26u + js * 2u + ile] + let qh3 = (kqu[blk * 26u + 16u + js / 4u] >> ((js % 4u) * 8u)) & 255u + let sgw3 = kqu[blk * 26u + 18u + js] + let tb0 = j * 72u + (sub >> 1u) * 32u + kh + for [unroll_full] (k in range(4)) { + let b3 = ile * 4u + uint(k) + let idx = ((qsw >> (8u * uint(k))) & 255u) | ((qh3 << (8u - b3)) & 256u) + let nib = (sgw3 >> ((b3 / 2u) * 8u + (b3 % 2u) * 4u)) & 15u + let sw = iq3s_sw(gridw3t[idx], nib) + for [unroll_full] (c in range(4)) { + twb[tb0 + uint(k) * 4u + uint(c)] = float16(dsc * iq3s_sx(sw, uint(c))) + } + } + } +} + +class MetalKqMulMmIq3sT : MetalKqMulMmIq3sTensorT { + typedef XT = float +} + +class MetalKqMulMmIq3sTH : MetalKqMulMmIq3sTensorT { + typedef XT = float16 +} + +[ |> template_struct_instance] +class template MetalKqMulMmIq3xxsTensorT : MetalKqMulMmSplitTensorBase { + @workgroup gridw3xt : uint[256] //!< the 1 KB halved iq3xxs grid, staged once per threadgroup + + [metal_kernel(float_a_ok=true)] + def metal_kq_mulmm_iq3xxs_t { + for [unroll_full] (gk in range(2)) { + let gi = gl_LocalInvocationID.x + uint(gk) * 128u + gridw3xt[gi] = iq3xxs_gw(int(gi)) + } + barrier() + run_shell() + } + + def override stage16(work, kb, nBase, nsb : uint) { + let j = work >> 2u + let sub = work & 3u + let jsl = kb + (sub >> 1u) + let sb = jsl / 8u + let js = jsl % 8u + let kh = (sub & 1u) * 16u + let ile = sub & 1u + let blk = (nBase + j) * nsb + sb + let sv = ksc[blk] + let scw = js < 4u ? sv.x : sv.y + let dsc = float(kdh[blk]) * float(((int(scw >> ((js & 3u) * 8u)) & 255) ^ 128) - 128) + let qsw = kqu[blk * 24u + js * 2u + ile] + let aux3 = kqu[blk * 24u + 16u + js] + let tb0 = j * 72u + (sub >> 1u) * 32u + kh + for [unroll_full] (k in range(4)) { + let q3 = ile * 2u + uint(k) / 2u + let sgb = ksign7m((aux3 >> (7u * q3)) & 127u) + let nib = uint(k) % 2u == 0u ? sgb & 15u : sgb >> 4u + let sw = iq3s_sw(gridw3xt[(qsw >> (8u * uint(k))) & 255u], nib) + for [unroll_full] (c in range(4)) { + twb[tb0 + uint(k) * 4u + uint(c)] = float16(dsc * iq3s_sx(sw, uint(c))) + } + } + } +} + +class MetalKqMulMmIq3xxsT : MetalKqMulMmIq3xxsTensorT { + typedef XT = float +} + +class MetalKqMulMmIq3xxsTH : MetalKqMulMmIq3xxsTensorT { + typedef XT = float16 +} + +// iq2s / iq2xs / iq2xxs: the u64-grid gathers DIRECT off the hoisted constant tables, +// exactly their base arms' choice. +[ |> template_struct_instance] +class template MetalKqMulMmIq2sTensorT : MetalKqMulMmSplitTensorBase { + [metal_kernel(float_a_ok=true)] + def metal_kq_mulmm_iq2s_t { + run_shell() + } + + def override stage16(work, kb, nBase, nsb : uint) { + let j = work >> 2u + let sub = work & 3u + let jsl = kb + (sub >> 1u) + let sb = jsl / 8u + let js = jsl % 8u + let kh = (sub & 1u) * 16u + let ile = sub & 1u + let blk = (nBase + j) * nsb + sb + let sv = ksc[blk] + let si2 = js * 2u + ile + let scw = si2 < 8u ? (si2 < 4u ? sv.x : sv.y) : (si2 < 12u ? sv.z : sv.w) + let dsc = float(kdh[blk]) * float((scw >> ((si2 & 3u) * 8u)) & 255u) + let qw2 = kqu[blk * 18u + js] + let sgw2 = kqu[blk * 18u + 8u + js] + let qh2 = (kqu[blk * 18u + 16u + js / 4u] >> ((js % 4u) * 8u)) & 255u + let tb0 = j * 72u + (sub >> 1u) * 32u + kh + for [unroll_full] (k in range(4)) { + let l2 = ile * 2u + uint(k) / 2u + let idx2 = (((qw2 >> (8u * l2)) & 255u) | ((qh2 << (8u - 2u * l2)) & 0x300u)) * 2u + uint(k) % 2u + let sgb2 = (sgw2 >> (8u * l2)) & 255u + let nib = uint(k) % 2u == 0u ? sgb2 & 15u : sgb2 >> 4u + let sw = iq3s_sw(iq2s_gw(int(idx2)), nib) + for [unroll_full] (c in range(4)) { + twb[tb0 + uint(k) * 4u + uint(c)] = float16(dsc * iq3s_sx(sw, uint(c))) + } + } + } +} + +class MetalKqMulMmIq2sT : MetalKqMulMmIq2sTensorT { + typedef XT = float +} + +class MetalKqMulMmIq2sTH : MetalKqMulMmIq2sTensorT { + typedef XT = float16 +} + +[ |> template_struct_instance] +class template MetalKqMulMmIq2xsTensorT : MetalKqMulMmSplitTensorBase { + [metal_kernel(float_a_ok=true)] + def metal_kq_mulmm_iq2xs_t { + run_shell() + } + + def override stage16(work, kb, nBase, nsb : uint) { + let j = work >> 2u + let sub = work & 3u + let jsl = kb + (sub >> 1u) + let sb = jsl / 8u + let js = jsl % 8u + let kh = (sub & 1u) * 16u + let ile = sub & 1u + let blk = (nBase + j) * nsb + sb + let sv = ksc[blk] + let si2 = js * 2u + ile + let scw = si2 < 8u ? (si2 < 4u ? sv.x : sv.y) : (si2 < 12u ? sv.z : sv.w) + let dsc = float(kdh[blk]) * float((scw >> ((si2 & 3u) * 8u)) & 255u) + let qw2 = kqu[blk * 16u + js * 2u + ile] + let tb0 = j * 72u + (sub >> 1u) * 32u + kh + for [unroll_full] (k in range(4)) { + let w16 = (qw2 >> (16u * (uint(k) / 2u))) & 0xFFFFu + let idx2 = (w16 & 511u) * 2u + uint(k) % 2u + let sgb2 = ksign7m(w16 >> 9u) + let nib = uint(k) % 2u == 0u ? sgb2 & 15u : sgb2 >> 4u + let sw = iq3s_sw(iq2xs_gw(int(idx2)), nib) + for [unroll_full] (c in range(4)) { + twb[tb0 + uint(k) * 4u + uint(c)] = float16(dsc * iq3s_sx(sw, uint(c))) + } + } + } +} + +class MetalKqMulMmIq2xsT : MetalKqMulMmIq2xsTensorT { + typedef XT = float +} + +class MetalKqMulMmIq2xsTH : MetalKqMulMmIq2xsTensorT { + typedef XT = float16 +} + +[ |> template_struct_instance] +class template MetalKqMulMmIq2xxsTensorT : MetalKqMulMmSplitTensorBase { + [metal_kernel(float_a_ok=true)] + def metal_kq_mulmm_iq2xxs_t { + run_shell() + } + + def override stage16(work, kb, nBase, nsb : uint) { + let j = work >> 2u + let sub = work & 3u + let jsl = kb + (sub >> 1u) + let sb = jsl / 8u + let js = jsl % 8u + let kh = (sub & 1u) * 16u + let ile = sub & 1u + let blk = (nBase + j) * nsb + sb + let sv = ksc[blk] + let scw = js < 4u ? sv.x : sv.y + let dsc = float(kdh[blk]) * float((scw >> ((js & 3u) * 8u)) & 255u) + let gw2 = kqu[blk * 16u + js * 2u] + let aux2 = kqu[blk * 16u + js * 2u + 1u] + let tb0 = j * 72u + (sub >> 1u) * 32u + kh + for [unroll_full] (k in range(4)) { + let q3 = ile * 2u + uint(k) / 2u + let bidx = (gw2 >> (8u * q3)) & 255u + let sgb2 = ksign7m((aux2 >> (7u * q3)) & 127u) + let nib = uint(k) % 2u == 0u ? sgb2 & 15u : sgb2 >> 4u + let sw = iq3s_sw(iq2xxs_gw(int(bidx * 2u + uint(k) % 2u)), nib) + for [unroll_full] (c in range(4)) { + twb[tb0 + uint(k) * 4u + uint(c)] = float16(dsc * iq3s_sx(sw, uint(c))) + } + } + } +} + +class MetalKqMulMmIq2xxsT : MetalKqMulMmIq2xxsTensorT { + typedef XT = float +} + +class MetalKqMulMmIq2xxsTH : MetalKqMulMmIq2xxsTensorT { + typedef XT = float16 +} + struct MoeMmArgs { From 49469457cdefdcc2b5a6991fb1d45c5ee88a243a Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 01:03:01 -0700 Subject: [PATCH 04/35] wire the iquant tensor twins: crowns, PSO ladders, site picks, race rows, test arms Per format: a kq_mulmm_ crown gates T/TH PSO compiles (pf_compile_kq_iquant_tensor_twins), the pf_enc_kq_site_mm early-return arms pick TH/T when crowned, release rows, and a race_kq_mulmm row in metal_tensor_race (the fixture's k6 bool generalized to a scale-form enum: k45 blocks / k6 split / k2 split / iq4nl d-plane) so the box mint crowns them. test_metal_gemm_kernels gains T + halfx-TH cells for all nine (suite 2/2). M5 e2e (lcpp_bench --ngl 99, crowns armed, debug-jit): iq4xs pp512 6364 -> 15147 t/s (0.85x lcpp, was 0.36x); iq2xxs 15165 (0.86x). tg unchanged, sanity tokens unchanged. The ~0.85x residual is graph-level - lcpp's attention also rides tensor on M5 while our attn_avmm race crowns simdgroup - ledgered. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014mq9AKyoAwvD5T1cigTnRw --- .../dasllama/dasllama_metal_prefill.das | 258 ++++++++++++++++-- .../tests/test_metal_gemm_kernels.das | 21 +- 2 files changed, 253 insertions(+), 26 deletions(-) diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das b/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das index 4e5ba88e23..2775ff9188 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das @@ -925,19 +925,23 @@ class template MetalKqMulMmIq4xsTensorT : MetalKqMulMmSplitTensorBase { } } +[metal_dispatch(name = "enc_kq_mm_iq4xs_t_c", pso = "g_pf_pso_kq_mm44_t", tgmem = "MetalKqMulMmIq4xsT_metal_kq_mulmm_iq4xs_t_msl_tgmem", tg = 128, grid = "mp/32, rows/64", params = "mp : int64, rows : int64")] class MetalKqMulMmIq4xsT : MetalKqMulMmIq4xsTensorT { typedef XT = float } +[metal_dispatch(name = "enc_kq_mm_iq4xs_th_c", pso = "g_pf_pso_kq_mm44_th", tgmem = "MetalKqMulMmIq4xsTH_metal_kq_mulmm_iq4xs_t_msl_tgmem", tg = 128, grid = "mp/32, rows/64", params = "mp : int64, rows : int64")] class MetalKqMulMmIq4xsTH : MetalKqMulMmIq4xsTensorT { typedef XT = float16 } +[metal_dispatch(name = "enc_kq_mm_iq4nl_t_c", pso = "g_pf_pso_kq_mm45_t", tgmem = "MetalKqMulMmIq4nlT_metal_kq_mulmm_iq4xs_t_msl_tgmem", tg = 128, grid = "mp/32, rows/64", params = "mp : int64, rows : int64")] class MetalKqMulMmIq4nlT : MetalKqMulMmIq4xsTensorT { typedef XT = float override IQ4NL = true } +[metal_dispatch(name = "enc_kq_mm_iq4nl_th_c", pso = "g_pf_pso_kq_mm45_th", tgmem = "MetalKqMulMmIq4nlTH_metal_kq_mulmm_iq4xs_t_msl_tgmem", tg = 128, grid = "mp/32, rows/64", params = "mp : int64, rows : int64")] class MetalKqMulMmIq4nlTH : MetalKqMulMmIq4xsTensorT { typedef XT = float16 override IQ4NL = true @@ -989,10 +993,12 @@ class template MetalKqMulMmK3TensorT : MetalKqMulMmSplitTensorBase { } } +[metal_dispatch(name = "enc_kq_mm_k3_t_c", pso = "g_pf_pso_kq_mm3_t", tgmem = "MetalKqMulMmK3T_metal_kq_mulmm_k3_t_msl_tgmem", tg = 128, grid = "mp/32, rows/64", params = "mp : int64, rows : int64")] class MetalKqMulMmK3T : MetalKqMulMmK3TensorT { typedef XT = float } +[metal_dispatch(name = "enc_kq_mm_k3_th_c", pso = "g_pf_pso_kq_mm3_th", tgmem = "MetalKqMulMmK3TH_metal_kq_mulmm_k3_t_msl_tgmem", tg = 128, grid = "mp/32, rows/64", params = "mp : int64, rows : int64")] class MetalKqMulMmK3TH : MetalKqMulMmK3TensorT { typedef XT = float16 } @@ -1037,10 +1043,12 @@ class template MetalKqMulMmK2TensorT : MetalKqMulMmSplitTensorBase { } } +[metal_dispatch(name = "enc_kq_mm_k2_t_c", pso = "g_pf_pso_kq_mm2_t", tgmem = "MetalKqMulMmK2T_metal_kq_mulmm_k2_t_msl_tgmem", tg = 128, grid = "mp/32, rows/64", params = "mp : int64, rows : int64")] class MetalKqMulMmK2T : MetalKqMulMmK2TensorT { typedef XT = float } +[metal_dispatch(name = "enc_kq_mm_k2_th_c", pso = "g_pf_pso_kq_mm2_th", tgmem = "MetalKqMulMmK2TH_metal_kq_mulmm_k2_t_msl_tgmem", tg = 128, grid = "mp/32, rows/64", params = "mp : int64, rows : int64")] class MetalKqMulMmK2TH : MetalKqMulMmK2TensorT { typedef XT = float16 } @@ -1088,10 +1096,12 @@ class template MetalKqMulMmIq3sTensorT : MetalKqMulMmSplitTensorBase { } } +[metal_dispatch(name = "enc_kq_mm_iq3s_t_c", pso = "g_pf_pso_kq_mm33_t", tgmem = "MetalKqMulMmIq3sT_metal_kq_mulmm_iq3s_t_msl_tgmem", tg = 128, grid = "mp/32, rows/64", params = "mp : int64, rows : int64")] class MetalKqMulMmIq3sT : MetalKqMulMmIq3sTensorT { typedef XT = float } +[metal_dispatch(name = "enc_kq_mm_iq3s_th_c", pso = "g_pf_pso_kq_mm33_th", tgmem = "MetalKqMulMmIq3sTH_metal_kq_mulmm_iq3s_t_msl_tgmem", tg = 128, grid = "mp/32, rows/64", params = "mp : int64, rows : int64")] class MetalKqMulMmIq3sTH : MetalKqMulMmIq3sTensorT { typedef XT = float16 } @@ -1137,10 +1147,12 @@ class template MetalKqMulMmIq3xxsTensorT : MetalKqMulMmSplitTensorBase { } } +[metal_dispatch(name = "enc_kq_mm_iq3xxs_t_c", pso = "g_pf_pso_kq_mm34_t", tgmem = "MetalKqMulMmIq3xxsT_metal_kq_mulmm_iq3xxs_t_msl_tgmem", tg = 128, grid = "mp/32, rows/64", params = "mp : int64, rows : int64")] class MetalKqMulMmIq3xxsT : MetalKqMulMmIq3xxsTensorT { typedef XT = float } +[metal_dispatch(name = "enc_kq_mm_iq3xxs_th_c", pso = "g_pf_pso_kq_mm34_th", tgmem = "MetalKqMulMmIq3xxsTH_metal_kq_mulmm_iq3xxs_t_msl_tgmem", tg = 128, grid = "mp/32, rows/64", params = "mp : int64, rows : int64")] class MetalKqMulMmIq3xxsTH : MetalKqMulMmIq3xxsTensorT { typedef XT = float16 } @@ -1184,10 +1196,12 @@ class template MetalKqMulMmIq2sTensorT : MetalKqMulMmSplitTensorBase { } } +[metal_dispatch(name = "enc_kq_mm_iq2s_t_c", pso = "g_pf_pso_kq_mm23_t", tgmem = "MetalKqMulMmIq2sT_metal_kq_mulmm_iq2s_t_msl_tgmem", tg = 128, grid = "mp/32, rows/64", params = "mp : int64, rows : int64")] class MetalKqMulMmIq2sT : MetalKqMulMmIq2sTensorT { typedef XT = float } +[metal_dispatch(name = "enc_kq_mm_iq2s_th_c", pso = "g_pf_pso_kq_mm23_th", tgmem = "MetalKqMulMmIq2sTH_metal_kq_mulmm_iq2s_t_msl_tgmem", tg = 128, grid = "mp/32, rows/64", params = "mp : int64, rows : int64")] class MetalKqMulMmIq2sTH : MetalKqMulMmIq2sTensorT { typedef XT = float16 } @@ -1227,10 +1241,12 @@ class template MetalKqMulMmIq2xsTensorT : MetalKqMulMmSplitTensorBase { } } +[metal_dispatch(name = "enc_kq_mm_iq2xs_t_c", pso = "g_pf_pso_kq_mm24_t", tgmem = "MetalKqMulMmIq2xsT_metal_kq_mulmm_iq2xs_t_msl_tgmem", tg = 128, grid = "mp/32, rows/64", params = "mp : int64, rows : int64")] class MetalKqMulMmIq2xsT : MetalKqMulMmIq2xsTensorT { typedef XT = float } +[metal_dispatch(name = "enc_kq_mm_iq2xs_th_c", pso = "g_pf_pso_kq_mm24_th", tgmem = "MetalKqMulMmIq2xsTH_metal_kq_mulmm_iq2xs_t_msl_tgmem", tg = 128, grid = "mp/32, rows/64", params = "mp : int64, rows : int64")] class MetalKqMulMmIq2xsTH : MetalKqMulMmIq2xsTensorT { typedef XT = float16 } @@ -1270,10 +1286,12 @@ class template MetalKqMulMmIq2xxsTensorT : MetalKqMulMmSplitTensorBase { } } +[metal_dispatch(name = "enc_kq_mm_iq2xxs_t_c", pso = "g_pf_pso_kq_mm25_t", tgmem = "MetalKqMulMmIq2xxsT_metal_kq_mulmm_iq2xxs_t_msl_tgmem", tg = 128, grid = "mp/32, rows/64", params = "mp : int64, rows : int64")] class MetalKqMulMmIq2xxsT : MetalKqMulMmIq2xxsTensorT { typedef XT = float } +[metal_dispatch(name = "enc_kq_mm_iq2xxs_th_c", pso = "g_pf_pso_kq_mm25_th", tgmem = "MetalKqMulMmIq2xxsTH_metal_kq_mulmm_iq2xxs_t_msl_tgmem", tg = 128, grid = "mp/32, rows/64", params = "mp : int64, rows : int64")] class MetalKqMulMmIq2xxsTH : MetalKqMulMmIq2xxsTensorT { typedef XT = float16 } @@ -3819,6 +3837,33 @@ var private g_pf_pso_kq_mm6_th : MetalComputePipeline? var private g_pf_pso_kq_mm6_th128 : MetalComputePipeline? var private g_pf_pso_kq_mm6_thdb : MetalComputePipeline? var private g_pf_pso_kq_mm6_thdb128 : MetalComputePipeline? +var private g_pf_kq_mm44_tensor : bool //!< "kq_mulmm_iq4xs" crowned (T + TH stamps only - no tall/db twins yet) +var private g_pf_pso_kq_mm44_t : MetalComputePipeline? +var private g_pf_pso_kq_mm44_th : MetalComputePipeline? +var private g_pf_kq_mm45_tensor : bool //!< "kq_mulmm_iq4nl" crowned +var private g_pf_pso_kq_mm45_t : MetalComputePipeline? +var private g_pf_pso_kq_mm45_th : MetalComputePipeline? +var private g_pf_kq_mm3_tensor : bool //!< "kq_mulmm_k3" crowned +var private g_pf_pso_kq_mm3_t : MetalComputePipeline? +var private g_pf_pso_kq_mm3_th : MetalComputePipeline? +var private g_pf_kq_mm33_tensor : bool //!< "kq_mulmm_iq3s" crowned +var private g_pf_pso_kq_mm33_t : MetalComputePipeline? +var private g_pf_pso_kq_mm33_th : MetalComputePipeline? +var private g_pf_kq_mm34_tensor : bool //!< "kq_mulmm_iq3xxs" crowned +var private g_pf_pso_kq_mm34_t : MetalComputePipeline? +var private g_pf_pso_kq_mm34_th : MetalComputePipeline? +var private g_pf_kq_mm2_tensor : bool //!< "kq_mulmm_k2" crowned +var private g_pf_pso_kq_mm2_t : MetalComputePipeline? +var private g_pf_pso_kq_mm2_th : MetalComputePipeline? +var private g_pf_kq_mm23_tensor : bool //!< "kq_mulmm_iq2s" crowned +var private g_pf_pso_kq_mm23_t : MetalComputePipeline? +var private g_pf_pso_kq_mm23_th : MetalComputePipeline? +var private g_pf_kq_mm24_tensor : bool //!< "kq_mulmm_iq2xs" crowned +var private g_pf_pso_kq_mm24_t : MetalComputePipeline? +var private g_pf_pso_kq_mm24_th : MetalComputePipeline? +var private g_pf_kq_mm25_tensor : bool //!< "kq_mulmm_iq2xxs" crowned +var private g_pf_pso_kq_mm25_t : MetalComputePipeline? +var private g_pf_pso_kq_mm25_th : MetalComputePipeline? var private g_pf_qkmm_tensor : bool //!< "attn_qkmm" crowned var private g_pf_pso_qkmm_t : MetalComputePipeline? var private g_pf_pso_qkmm_th : MetalComputePipeline? @@ -4151,6 +4196,24 @@ def public metal_prefill_shutdown { // nolint:STYLE037,STYLE038 — flat one-r metal_release(g_pf_pso_mm_th128) g_pf_pso_mm_th128 = null } + release_pso(g_pf_pso_kq_mm44_t) + release_pso(g_pf_pso_kq_mm44_th) + release_pso(g_pf_pso_kq_mm45_t) + release_pso(g_pf_pso_kq_mm45_th) + release_pso(g_pf_pso_kq_mm3_t) + release_pso(g_pf_pso_kq_mm3_th) + release_pso(g_pf_pso_kq_mm33_t) + release_pso(g_pf_pso_kq_mm33_th) + release_pso(g_pf_pso_kq_mm34_t) + release_pso(g_pf_pso_kq_mm34_th) + release_pso(g_pf_pso_kq_mm2_t) + release_pso(g_pf_pso_kq_mm2_th) + release_pso(g_pf_pso_kq_mm23_t) + release_pso(g_pf_pso_kq_mm23_th) + release_pso(g_pf_pso_kq_mm24_t) + release_pso(g_pf_pso_kq_mm24_th) + release_pso(g_pf_pso_kq_mm25_t) + release_pso(g_pf_pso_kq_mm25_th) release_pso(g_pf_pso_twr_flash) release_pso(g_pf_pso_twr_kvhc) release_pso(g_pf_pso_twr_win) @@ -4382,6 +4445,7 @@ def private metal_prefill_init : bool { // nolint:STYLE038 — flat one-compil g_pf_pso_kq_mm6_thdb = compile_pso(MetalKqMulMmK6THDb_metal_kq_mulmm_k6_db_msl, MetalKqMulMmK6THDb_metal_kq_mulmm_k6_db_msl_entry, MetalKqMulMmK6THDb_metal_kq_mulmm_k6_db_msl_fastmath, ok) g_pf_pso_kq_mm6_thdb128 = compile_pso(MetalKqMulMmK6THDb128_metal_kq_mulmm_k6_db_msl, MetalKqMulMmK6THDb128_metal_kq_mulmm_k6_db_msl_entry, MetalKqMulMmK6THDb128_metal_kq_mulmm_k6_db_msl_fastmath, ok) } + pf_compile_kq_iquant_tensor_twins(ok) //! dev-W: ONE all-device GEMM for every crowned family; dequant passes compile per family if (g_pf_mm_tensor || g_pf_kq_mm4_tensor || g_pf_kq_mm5_tensor || g_pf_kq_mm6_tensor) { g_pf_pso_hmm_th = compile_pso(MetalHalfMulMmTH_metal_half_mulmm_th_msl, MetalHalfMulMmTH_metal_half_mulmm_th_msl_entry, MetalHalfMulMmTH_metal_half_mulmm_th_msl_fastmath, ok) @@ -5008,45 +5072,166 @@ def tallkq_pick(mp, rows, kdim : int64; env_tall : bool) : bool { } //! k6 binds its split scale plane twice: sub-scales at soff, the f16 d tail at doff +//! the iquant/split-scale formats' tensor twins - compiled only behind each format's crown +def private pf_compile_kq_iquant_tensor_twins(var ok : bool&) { + g_pf_kq_mm44_tensor = metal_tensor_crowned("kq_mulmm_iq4xs") + if (g_pf_kq_mm44_tensor) { + g_pf_pso_kq_mm44_t = compile_pso(MetalKqMulMmIq4xsT_metal_kq_mulmm_iq4xs_t_msl, MetalKqMulMmIq4xsT_metal_kq_mulmm_iq4xs_t_msl_entry, MetalKqMulMmIq4xsT_metal_kq_mulmm_iq4xs_t_msl_fastmath, ok) + g_pf_pso_kq_mm44_th = compile_pso(MetalKqMulMmIq4xsTH_metal_kq_mulmm_iq4xs_t_msl, MetalKqMulMmIq4xsTH_metal_kq_mulmm_iq4xs_t_msl_entry, MetalKqMulMmIq4xsTH_metal_kq_mulmm_iq4xs_t_msl_fastmath, ok) + } + g_pf_kq_mm45_tensor = metal_tensor_crowned("kq_mulmm_iq4nl") + if (g_pf_kq_mm45_tensor) { + g_pf_pso_kq_mm45_t = compile_pso(MetalKqMulMmIq4nlT_metal_kq_mulmm_iq4xs_t_msl, MetalKqMulMmIq4nlT_metal_kq_mulmm_iq4xs_t_msl_entry, MetalKqMulMmIq4nlT_metal_kq_mulmm_iq4xs_t_msl_fastmath, ok) + g_pf_pso_kq_mm45_th = compile_pso(MetalKqMulMmIq4nlTH_metal_kq_mulmm_iq4xs_t_msl, MetalKqMulMmIq4nlTH_metal_kq_mulmm_iq4xs_t_msl_entry, MetalKqMulMmIq4nlTH_metal_kq_mulmm_iq4xs_t_msl_fastmath, ok) + } + g_pf_kq_mm3_tensor = metal_tensor_crowned("kq_mulmm_k3") + if (g_pf_kq_mm3_tensor) { + g_pf_pso_kq_mm3_t = compile_pso(MetalKqMulMmK3T_metal_kq_mulmm_k3_t_msl, MetalKqMulMmK3T_metal_kq_mulmm_k3_t_msl_entry, MetalKqMulMmK3T_metal_kq_mulmm_k3_t_msl_fastmath, ok) + g_pf_pso_kq_mm3_th = compile_pso(MetalKqMulMmK3TH_metal_kq_mulmm_k3_t_msl, MetalKqMulMmK3TH_metal_kq_mulmm_k3_t_msl_entry, MetalKqMulMmK3TH_metal_kq_mulmm_k3_t_msl_fastmath, ok) + } + g_pf_kq_mm33_tensor = metal_tensor_crowned("kq_mulmm_iq3s") + if (g_pf_kq_mm33_tensor) { + g_pf_pso_kq_mm33_t = compile_pso(MetalKqMulMmIq3sT_metal_kq_mulmm_iq3s_t_msl, MetalKqMulMmIq3sT_metal_kq_mulmm_iq3s_t_msl_entry, MetalKqMulMmIq3sT_metal_kq_mulmm_iq3s_t_msl_fastmath, ok) + g_pf_pso_kq_mm33_th = compile_pso(MetalKqMulMmIq3sTH_metal_kq_mulmm_iq3s_t_msl, MetalKqMulMmIq3sTH_metal_kq_mulmm_iq3s_t_msl_entry, MetalKqMulMmIq3sTH_metal_kq_mulmm_iq3s_t_msl_fastmath, ok) + } + g_pf_kq_mm34_tensor = metal_tensor_crowned("kq_mulmm_iq3xxs") + if (g_pf_kq_mm34_tensor) { + g_pf_pso_kq_mm34_t = compile_pso(MetalKqMulMmIq3xxsT_metal_kq_mulmm_iq3xxs_t_msl, MetalKqMulMmIq3xxsT_metal_kq_mulmm_iq3xxs_t_msl_entry, MetalKqMulMmIq3xxsT_metal_kq_mulmm_iq3xxs_t_msl_fastmath, ok) + g_pf_pso_kq_mm34_th = compile_pso(MetalKqMulMmIq3xxsTH_metal_kq_mulmm_iq3xxs_t_msl, MetalKqMulMmIq3xxsTH_metal_kq_mulmm_iq3xxs_t_msl_entry, MetalKqMulMmIq3xxsTH_metal_kq_mulmm_iq3xxs_t_msl_fastmath, ok) + } + g_pf_kq_mm2_tensor = metal_tensor_crowned("kq_mulmm_k2") + if (g_pf_kq_mm2_tensor) { + g_pf_pso_kq_mm2_t = compile_pso(MetalKqMulMmK2T_metal_kq_mulmm_k2_t_msl, MetalKqMulMmK2T_metal_kq_mulmm_k2_t_msl_entry, MetalKqMulMmK2T_metal_kq_mulmm_k2_t_msl_fastmath, ok) + g_pf_pso_kq_mm2_th = compile_pso(MetalKqMulMmK2TH_metal_kq_mulmm_k2_t_msl, MetalKqMulMmK2TH_metal_kq_mulmm_k2_t_msl_entry, MetalKqMulMmK2TH_metal_kq_mulmm_k2_t_msl_fastmath, ok) + } + g_pf_kq_mm23_tensor = metal_tensor_crowned("kq_mulmm_iq2s") + if (g_pf_kq_mm23_tensor) { + g_pf_pso_kq_mm23_t = compile_pso(MetalKqMulMmIq2sT_metal_kq_mulmm_iq2s_t_msl, MetalKqMulMmIq2sT_metal_kq_mulmm_iq2s_t_msl_entry, MetalKqMulMmIq2sT_metal_kq_mulmm_iq2s_t_msl_fastmath, ok) + g_pf_pso_kq_mm23_th = compile_pso(MetalKqMulMmIq2sTH_metal_kq_mulmm_iq2s_t_msl, MetalKqMulMmIq2sTH_metal_kq_mulmm_iq2s_t_msl_entry, MetalKqMulMmIq2sTH_metal_kq_mulmm_iq2s_t_msl_fastmath, ok) + } + g_pf_kq_mm24_tensor = metal_tensor_crowned("kq_mulmm_iq2xs") + if (g_pf_kq_mm24_tensor) { + g_pf_pso_kq_mm24_t = compile_pso(MetalKqMulMmIq2xsT_metal_kq_mulmm_iq2xs_t_msl, MetalKqMulMmIq2xsT_metal_kq_mulmm_iq2xs_t_msl_entry, MetalKqMulMmIq2xsT_metal_kq_mulmm_iq2xs_t_msl_fastmath, ok) + g_pf_pso_kq_mm24_th = compile_pso(MetalKqMulMmIq2xsTH_metal_kq_mulmm_iq2xs_t_msl, MetalKqMulMmIq2xsTH_metal_kq_mulmm_iq2xs_t_msl_entry, MetalKqMulMmIq2xsTH_metal_kq_mulmm_iq2xs_t_msl_fastmath, ok) + } + g_pf_kq_mm25_tensor = metal_tensor_crowned("kq_mulmm_iq2xxs") + if (g_pf_kq_mm25_tensor) { + g_pf_pso_kq_mm25_t = compile_pso(MetalKqMulMmIq2xxsT_metal_kq_mulmm_iq2xxs_t_msl, MetalKqMulMmIq2xxsT_metal_kq_mulmm_iq2xxs_t_msl_entry, MetalKqMulMmIq2xxsT_metal_kq_mulmm_iq2xxs_t_msl_fastmath, ok) + g_pf_pso_kq_mm25_th = compile_pso(MetalKqMulMmIq2xxsTH_metal_kq_mulmm_iq2xxs_t_msl, MetalKqMulMmIq2xxsTH_metal_kq_mulmm_iq2xxs_t_msl_entry, MetalKqMulMmIq2xxsTH_metal_kq_mulmm_iq2xxs_t_msl_fastmath, ok) + } +} + def private pf_enc_kq_site_mm(enc : MetalComputeEncoder?; t : Model; fmt : KqFmt; woff, rows : int64; // nolint:STYLE037,STYLE038 — flat per-format form-pick ladder bx, by, bk, bn : MetalBuffer?; mp : int64; yoff : uint64 = 0ul; bxh : MetalBuffer? = null; kdim : int64 = 0l) { let fmt_tensor = fmt == KqFmt.k6 ? g_pf_kq_mm6_tensor : (fmt == KqFmt.k4 ? g_pf_kq_mm4_tensor : g_pf_kq_mm5_tensor) let bq = kq_quants_of(g_dev, t, fmt, woff) let bs = kq_scales_of(g_dev, t, fmt, woff) if (fmt == KqFmt.iq4xs) { - enc_kq_mm_iq4xs_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, by, yoff, bk, bn, mp, rows) + if (g_pf_kq_mm44_tensor && g_pf_pso_kq_mm44_t != null) { + if (bxh != null && g_pf_pso_kq_mm44_th != null) { + enc_kq_mm_iq4xs_th_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bxh, 0ul, by, yoff, bk, bn, mp, rows) + } else { + enc_kq_mm_iq4xs_t_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, 0ul, by, yoff, bk, bn, mp, rows) + } + } else { + enc_kq_mm_iq4xs_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, by, yoff, bk, bn, mp, rows) + } return } if (fmt == KqFmt.k3) { - enc_kq_mm_k3_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, by, yoff, bk, bn, mp, rows) + if (g_pf_kq_mm3_tensor && g_pf_pso_kq_mm3_t != null) { + if (bxh != null && g_pf_pso_kq_mm3_th != null) { + enc_kq_mm_k3_th_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bxh, 0ul, by, yoff, bk, bn, mp, rows) + } else { + enc_kq_mm_k3_t_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, 0ul, by, yoff, bk, bn, mp, rows) + } + } else { + enc_kq_mm_k3_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, by, yoff, bk, bn, mp, rows) + } return } if (fmt == KqFmt.iq3s) { - enc_kq_mm_iq3s_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, by, yoff, bk, bn, mp, rows) + if (g_pf_kq_mm33_tensor && g_pf_pso_kq_mm33_t != null) { + if (bxh != null && g_pf_pso_kq_mm33_th != null) { + enc_kq_mm_iq3s_th_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bxh, 0ul, by, yoff, bk, bn, mp, rows) + } else { + enc_kq_mm_iq3s_t_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, 0ul, by, yoff, bk, bn, mp, rows) + } + } else { + enc_kq_mm_iq3s_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, by, yoff, bk, bn, mp, rows) + } return } if (fmt == KqFmt.iq3xxs) { - enc_kq_mm_iq3xxs_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, by, yoff, bk, bn, mp, rows) + if (g_pf_kq_mm34_tensor && g_pf_pso_kq_mm34_t != null) { + if (bxh != null && g_pf_pso_kq_mm34_th != null) { + enc_kq_mm_iq3xxs_th_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bxh, 0ul, by, yoff, bk, bn, mp, rows) + } else { + enc_kq_mm_iq3xxs_t_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, 0ul, by, yoff, bk, bn, mp, rows) + } + } else { + enc_kq_mm_iq3xxs_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, by, yoff, bk, bn, mp, rows) + } return } if (fmt == KqFmt.iq4nl) { - enc_kq_mm_iq4nl_c(enc, bs.buf, bs.soff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, by, yoff, bk, bn, mp, rows) + if (g_pf_kq_mm45_tensor && g_pf_pso_kq_mm45_t != null) { + if (bxh != null && g_pf_pso_kq_mm45_th != null) { + enc_kq_mm_iq4nl_th_c(enc, bs.buf, bs.soff, bs.buf, bs.soff, bq.buf, bq.qoff, bxh, 0ul, by, yoff, bk, bn, mp, rows) + } else { + enc_kq_mm_iq4nl_t_c(enc, bs.buf, bs.soff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, 0ul, by, yoff, bk, bn, mp, rows) + } + } else { + enc_kq_mm_iq4nl_c(enc, bs.buf, bs.soff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, by, yoff, bk, bn, mp, rows) + } return } if (fmt == KqFmt.k2) { - enc_kq_mm_k2_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, by, yoff, bk, bn, mp, rows) + if (g_pf_kq_mm2_tensor && g_pf_pso_kq_mm2_t != null) { + if (bxh != null && g_pf_pso_kq_mm2_th != null) { + enc_kq_mm_k2_th_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bxh, 0ul, by, yoff, bk, bn, mp, rows) + } else { + enc_kq_mm_k2_t_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, 0ul, by, yoff, bk, bn, mp, rows) + } + } else { + enc_kq_mm_k2_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, by, yoff, bk, bn, mp, rows) + } return } if (fmt == KqFmt.iq2s) { - enc_kq_mm_iq2s_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, by, yoff, bk, bn, mp, rows) + if (g_pf_kq_mm23_tensor && g_pf_pso_kq_mm23_t != null) { + if (bxh != null && g_pf_pso_kq_mm23_th != null) { + enc_kq_mm_iq2s_th_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bxh, 0ul, by, yoff, bk, bn, mp, rows) + } else { + enc_kq_mm_iq2s_t_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, 0ul, by, yoff, bk, bn, mp, rows) + } + } else { + enc_kq_mm_iq2s_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, by, yoff, bk, bn, mp, rows) + } return } if (fmt == KqFmt.iq2xs) { - enc_kq_mm_iq2xs_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, by, yoff, bk, bn, mp, rows) + if (g_pf_kq_mm24_tensor && g_pf_pso_kq_mm24_t != null) { + if (bxh != null && g_pf_pso_kq_mm24_th != null) { + enc_kq_mm_iq2xs_th_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bxh, 0ul, by, yoff, bk, bn, mp, rows) + } else { + enc_kq_mm_iq2xs_t_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, 0ul, by, yoff, bk, bn, mp, rows) + } + } else { + enc_kq_mm_iq2xs_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, by, yoff, bk, bn, mp, rows) + } return } if (fmt == KqFmt.iq2xxs) { - enc_kq_mm_iq2xxs_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, by, yoff, bk, bn, mp, rows) + if (g_pf_kq_mm25_tensor && g_pf_pso_kq_mm25_t != null) { + if (bxh != null && g_pf_pso_kq_mm25_th != null) { + enc_kq_mm_iq2xxs_th_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bxh, 0ul, by, yoff, bk, bn, mp, rows) + } else { + enc_kq_mm_iq2xxs_t_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, 0ul, by, yoff, bk, bn, mp, rows) + } + } else { + enc_kq_mm_iq2xxs_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, by, yoff, bk, bn, mp, rows) + } return } //! deep-class arm: a panel dev-W would split past 8 tiles is DRAM-resident - the tall @@ -7107,7 +7292,7 @@ def private race_moe_mulmm_mx4(dev : MetalDevice?; queue : MetalCommandQueue?) : def private race_kq_mulmm(dev : MetalDevice?; queue : MetalCommandQueue?; family : string; // nolint:STYLE038 — one-shot A/B race harness; buffers + PSOs stay live to the release tail base_src, base_entry : string; base_fm : bool; base_tgmem : uint64; twin_src, twin_entry : string; twin_fm : bool; twin_tgmem : uint64; - qu_per_sb : int; k6 : bool) : MetalTensorRaceResult { + qu_per_sb : int; sform : int) : MetalTensorRaceResult { var res = MetalTensorRaceResult(family = family, winner = "", base_ms = -1.0lf, twin_ms = -1.0lf, note = "") let m = 512 let kdim = 2048 @@ -7133,17 +7318,27 @@ def private race_kq_mulmm(dev : MetalDevice?; queue : MetalCommandQueue?; family for (i in range(nblk * qu_per_sb)) { qplane[i] = uint(i) * 2654435761u //! deterministic pseudo-random bit pattern } - //! scales: k4/k5 = compact 16B blocks; k6 = int8 sub-scale plane + the f16 d tail + //! scale forms: 0 = k4/k5 compact 16B blocks; 1 = the k6 split (16B strips + 2B f16 d + //! tail - every split-scale iquant reads it too); 2 = the k2 split (16B strips + 4B + //! d+dmin tail); 3 = the iq4nl pure-d plane (8 f16 per sb) var splane : array - let sbytes = k6 ? nblk * 16 + nblk * 2 : nblk * 16 + let sbytes = sform == 1 ? nblk * 18 : (sform == 2 ? nblk * 20 : nblk * 16) splane |> resize(sbytes) - if (k6) { + if (sform == 1 || sform == 2) { for (i in range(nblk * 16)) { splane[i] = uint8(2) } unsafe { var ph = addr(splane[nblk * 16]) - for (b in range(nblk)) { + let per = sform == 2 ? 2 : 1 + for (b in range(nblk * per)) { + ph[b] = float16(0.25) + } + } + } elif (sform == 3) { + unsafe { + var ph = addr(splane[0]) + for (b in range(nblk * 8)) { ph[b] = float16(0.25) } } @@ -7168,7 +7363,7 @@ def private race_kq_mulmm(dev : MetalDevice?; queue : MetalCommandQueue?; family var by_twin = race_buf(dev, uint64(m * ndim * 4), null) var bk = race_uniform_u32(dev, uint(kdim)) var bn = race_uniform_u32(dev, uint(ndim)) - let doff = k6 ? uint64(nblk * 16) : 0ul + let doff = sform == 1 || sform == 2 ? uint64(nblk * 16) : 0ul let grid = uint3(uint(m / 32), uint(ndim / 64), 1u) let tg = uint3(128u, 1u, 1u) race_pair_ms(queue, 5, res.base_ms, res.twin_ms, $(enc) { @@ -7353,13 +7548,40 @@ def metal_tensor_race : array { } results |> emplace(race_kq_mulmm(dev, queue, "kq_mulmm_k4", MetalKqMulMmK4_metal_kq_mulmm_msl, MetalKqMulMmK4_metal_kq_mulmm_msl_entry, MetalKqMulMmK4_metal_kq_mulmm_msl_fastmath, MetalKqMulMmK4_metal_kq_mulmm_msl_tgmem, - MetalKqMulMmK4T_metal_kq_mulmm_t_msl, MetalKqMulMmK4T_metal_kq_mulmm_t_msl_entry, MetalKqMulMmK4T_metal_kq_mulmm_t_msl_fastmath, MetalKqMulMmK4T_metal_kq_mulmm_t_msl_tgmem, 32, false)) + MetalKqMulMmK4T_metal_kq_mulmm_t_msl, MetalKqMulMmK4T_metal_kq_mulmm_t_msl_entry, MetalKqMulMmK4T_metal_kq_mulmm_t_msl_fastmath, MetalKqMulMmK4T_metal_kq_mulmm_t_msl_tgmem, 32, 0)) results |> emplace(race_kq_mulmm(dev, queue, "kq_mulmm_k5", MetalKqMulMmK5_metal_kq_mulmm_msl, MetalKqMulMmK5_metal_kq_mulmm_msl_entry, MetalKqMulMmK5_metal_kq_mulmm_msl_fastmath, MetalKqMulMmK5_metal_kq_mulmm_msl_tgmem, - MetalKqMulMmK5T_metal_kq_mulmm_t_msl, MetalKqMulMmK5T_metal_kq_mulmm_t_msl_entry, MetalKqMulMmK5T_metal_kq_mulmm_t_msl_fastmath, MetalKqMulMmK5T_metal_kq_mulmm_t_msl_tgmem, 40, false)) + MetalKqMulMmK5T_metal_kq_mulmm_t_msl, MetalKqMulMmK5T_metal_kq_mulmm_t_msl_entry, MetalKqMulMmK5T_metal_kq_mulmm_t_msl_fastmath, MetalKqMulMmK5T_metal_kq_mulmm_t_msl_tgmem, 40, 0)) results |> emplace(race_kq_mulmm(dev, queue, "kq_mulmm_k6", MetalKqMulMmK6_metal_kq_mulmm_msl, MetalKqMulMmK6_metal_kq_mulmm_msl_entry, MetalKqMulMmK6_metal_kq_mulmm_msl_fastmath, MetalKqMulMmK6_metal_kq_mulmm_msl_tgmem, - MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl, MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl_entry, MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl_fastmath, MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl_tgmem, 48, true)) + MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl, MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl_entry, MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl_fastmath, MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl_tgmem, 48, 1)) + results |> emplace(race_kq_mulmm(dev, queue, "kq_mulmm_iq4xs", + MetalKqMulMmIq4xs_metal_kq_mulmm_msl, MetalKqMulMmIq4xs_metal_kq_mulmm_msl_entry, MetalKqMulMmIq4xs_metal_kq_mulmm_msl_fastmath, MetalKqMulMmIq4xs_metal_kq_mulmm_msl_tgmem, + MetalKqMulMmIq4xsT_metal_kq_mulmm_iq4xs_t_msl, MetalKqMulMmIq4xsT_metal_kq_mulmm_iq4xs_t_msl_entry, MetalKqMulMmIq4xsT_metal_kq_mulmm_iq4xs_t_msl_fastmath, MetalKqMulMmIq4xsT_metal_kq_mulmm_iq4xs_t_msl_tgmem, 32, 1)) + results |> emplace(race_kq_mulmm(dev, queue, "kq_mulmm_iq4nl", + MetalKqMulMmIq4nl_metal_kq_mulmm_msl, MetalKqMulMmIq4nl_metal_kq_mulmm_msl_entry, MetalKqMulMmIq4nl_metal_kq_mulmm_msl_fastmath, MetalKqMulMmIq4nl_metal_kq_mulmm_msl_tgmem, + MetalKqMulMmIq4nlT_metal_kq_mulmm_iq4xs_t_msl, MetalKqMulMmIq4nlT_metal_kq_mulmm_iq4xs_t_msl_entry, MetalKqMulMmIq4nlT_metal_kq_mulmm_iq4xs_t_msl_fastmath, MetalKqMulMmIq4nlT_metal_kq_mulmm_iq4xs_t_msl_tgmem, 32, 3)) + results |> emplace(race_kq_mulmm(dev, queue, "kq_mulmm_k3", + MetalKqMulMmK3_metal_kq_mulmm_msl, MetalKqMulMmK3_metal_kq_mulmm_msl_entry, MetalKqMulMmK3_metal_kq_mulmm_msl_fastmath, MetalKqMulMmK3_metal_kq_mulmm_msl_tgmem, + MetalKqMulMmK3T_metal_kq_mulmm_k3_t_msl, MetalKqMulMmK3T_metal_kq_mulmm_k3_t_msl_entry, MetalKqMulMmK3T_metal_kq_mulmm_k3_t_msl_fastmath, MetalKqMulMmK3T_metal_kq_mulmm_k3_t_msl_tgmem, 24, 1)) + results |> emplace(race_kq_mulmm(dev, queue, "kq_mulmm_iq3s", + MetalKqMulMmIq3s_metal_kq_mulmm_msl, MetalKqMulMmIq3s_metal_kq_mulmm_msl_entry, MetalKqMulMmIq3s_metal_kq_mulmm_msl_fastmath, MetalKqMulMmIq3s_metal_kq_mulmm_msl_tgmem, + MetalKqMulMmIq3sT_metal_kq_mulmm_iq3s_t_msl, MetalKqMulMmIq3sT_metal_kq_mulmm_iq3s_t_msl_entry, MetalKqMulMmIq3sT_metal_kq_mulmm_iq3s_t_msl_fastmath, MetalKqMulMmIq3sT_metal_kq_mulmm_iq3s_t_msl_tgmem, 26, 1)) + results |> emplace(race_kq_mulmm(dev, queue, "kq_mulmm_iq3xxs", + MetalKqMulMmIq3xxs_metal_kq_mulmm_msl, MetalKqMulMmIq3xxs_metal_kq_mulmm_msl_entry, MetalKqMulMmIq3xxs_metal_kq_mulmm_msl_fastmath, MetalKqMulMmIq3xxs_metal_kq_mulmm_msl_tgmem, + MetalKqMulMmIq3xxsT_metal_kq_mulmm_iq3xxs_t_msl, MetalKqMulMmIq3xxsT_metal_kq_mulmm_iq3xxs_t_msl_entry, MetalKqMulMmIq3xxsT_metal_kq_mulmm_iq3xxs_t_msl_fastmath, MetalKqMulMmIq3xxsT_metal_kq_mulmm_iq3xxs_t_msl_tgmem, 24, 1)) + results |> emplace(race_kq_mulmm(dev, queue, "kq_mulmm_k2", + MetalKqMulMmK2_metal_kq_mulmm_msl, MetalKqMulMmK2_metal_kq_mulmm_msl_entry, MetalKqMulMmK2_metal_kq_mulmm_msl_fastmath, MetalKqMulMmK2_metal_kq_mulmm_msl_tgmem, + MetalKqMulMmK2T_metal_kq_mulmm_k2_t_msl, MetalKqMulMmK2T_metal_kq_mulmm_k2_t_msl_entry, MetalKqMulMmK2T_metal_kq_mulmm_k2_t_msl_fastmath, MetalKqMulMmK2T_metal_kq_mulmm_k2_t_msl_tgmem, 16, 2)) + results |> emplace(race_kq_mulmm(dev, queue, "kq_mulmm_iq2s", + MetalKqMulMmIq2s_metal_kq_mulmm_msl, MetalKqMulMmIq2s_metal_kq_mulmm_msl_entry, MetalKqMulMmIq2s_metal_kq_mulmm_msl_fastmath, MetalKqMulMmIq2s_metal_kq_mulmm_msl_tgmem, + MetalKqMulMmIq2sT_metal_kq_mulmm_iq2s_t_msl, MetalKqMulMmIq2sT_metal_kq_mulmm_iq2s_t_msl_entry, MetalKqMulMmIq2sT_metal_kq_mulmm_iq2s_t_msl_fastmath, MetalKqMulMmIq2sT_metal_kq_mulmm_iq2s_t_msl_tgmem, 18, 1)) + results |> emplace(race_kq_mulmm(dev, queue, "kq_mulmm_iq2xs", + MetalKqMulMmIq2xs_metal_kq_mulmm_msl, MetalKqMulMmIq2xs_metal_kq_mulmm_msl_entry, MetalKqMulMmIq2xs_metal_kq_mulmm_msl_fastmath, MetalKqMulMmIq2xs_metal_kq_mulmm_msl_tgmem, + MetalKqMulMmIq2xsT_metal_kq_mulmm_iq2xs_t_msl, MetalKqMulMmIq2xsT_metal_kq_mulmm_iq2xs_t_msl_entry, MetalKqMulMmIq2xsT_metal_kq_mulmm_iq2xs_t_msl_fastmath, MetalKqMulMmIq2xsT_metal_kq_mulmm_iq2xs_t_msl_tgmem, 16, 1)) + results |> emplace(race_kq_mulmm(dev, queue, "kq_mulmm_iq2xxs", + MetalKqMulMmIq2xxs_metal_kq_mulmm_msl, MetalKqMulMmIq2xxs_metal_kq_mulmm_msl_entry, MetalKqMulMmIq2xxs_metal_kq_mulmm_msl_fastmath, MetalKqMulMmIq2xxs_metal_kq_mulmm_msl_tgmem, + MetalKqMulMmIq2xxsT_metal_kq_mulmm_iq2xxs_t_msl, MetalKqMulMmIq2xxsT_metal_kq_mulmm_iq2xxs_t_msl_entry, MetalKqMulMmIq2xxsT_metal_kq_mulmm_iq2xxs_t_msl_fastmath, MetalKqMulMmIq2xxsT_metal_kq_mulmm_iq2xxs_t_msl_tgmem, 16, 1)) metal_release(queue) metal_release(dev) return <- results diff --git a/modules/dasLLAMA/tests/test_metal_gemm_kernels.das b/modules/dasLLAMA/tests/test_metal_gemm_kernels.das index a5d3d0d29c..16409f9ce5 100644 --- a/modules/dasLLAMA/tests/test_metal_gemm_kernels.das +++ b/modules/dasLLAMA/tests/test_metal_gemm_kernels.das @@ -34,29 +34,29 @@ def private kq_mulmm_gate(t : T?; dev, queue; fmt : int; tensor : bool; m, kdim, ? (tall ? (fmt == 4 ? MetalKqMulMmK4TH128_metal_kq_mulmm_t_msl : (fmt == 5 ? MetalKqMulMmK5TH128_metal_kq_mulmm_t_msl : MetalKqMulMmK6TH128_metal_kq_mulmm_k6_t_msl)) : halfx - ? (fmt == 4 ? MetalKqMulMmK4TH_metal_kq_mulmm_t_msl : (fmt == 5 ? MetalKqMulMmK5TH_metal_kq_mulmm_t_msl : MetalKqMulMmK6TH_metal_kq_mulmm_k6_t_msl)) - : (fmt == 4 ? MetalKqMulMmK4T_metal_kq_mulmm_t_msl : (fmt == 5 ? MetalKqMulMmK5T_metal_kq_mulmm_t_msl : MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl))) + ? (fmt == 25 ? MetalKqMulMmIq2xxsTH_metal_kq_mulmm_iq2xxs_t_msl : (fmt == 24 ? MetalKqMulMmIq2xsTH_metal_kq_mulmm_iq2xs_t_msl : (fmt == 23 ? MetalKqMulMmIq2sTH_metal_kq_mulmm_iq2s_t_msl : (fmt == 45 ? MetalKqMulMmIq4nlTH_metal_kq_mulmm_iq4xs_t_msl : (fmt == 2 ? MetalKqMulMmK2TH_metal_kq_mulmm_k2_t_msl : (fmt == 34 ? MetalKqMulMmIq3xxsTH_metal_kq_mulmm_iq3xxs_t_msl : (fmt == 33 ? MetalKqMulMmIq3sTH_metal_kq_mulmm_iq3s_t_msl : (fmt == 3 ? MetalKqMulMmK3TH_metal_kq_mulmm_k3_t_msl : (fmt == 44 ? MetalKqMulMmIq4xsTH_metal_kq_mulmm_iq4xs_t_msl : (fmt == 4 ? MetalKqMulMmK4TH_metal_kq_mulmm_t_msl : (fmt == 5 ? MetalKqMulMmK5TH_metal_kq_mulmm_t_msl : MetalKqMulMmK6TH_metal_kq_mulmm_k6_t_msl))))))))))) + : (fmt == 25 ? MetalKqMulMmIq2xxsT_metal_kq_mulmm_iq2xxs_t_msl : (fmt == 24 ? MetalKqMulMmIq2xsT_metal_kq_mulmm_iq2xs_t_msl : (fmt == 23 ? MetalKqMulMmIq2sT_metal_kq_mulmm_iq2s_t_msl : (fmt == 45 ? MetalKqMulMmIq4nlT_metal_kq_mulmm_iq4xs_t_msl : (fmt == 2 ? MetalKqMulMmK2T_metal_kq_mulmm_k2_t_msl : (fmt == 34 ? MetalKqMulMmIq3xxsT_metal_kq_mulmm_iq3xxs_t_msl : (fmt == 33 ? MetalKqMulMmIq3sT_metal_kq_mulmm_iq3s_t_msl : (fmt == 3 ? MetalKqMulMmK3T_metal_kq_mulmm_k3_t_msl : (fmt == 44 ? MetalKqMulMmIq4xsT_metal_kq_mulmm_iq4xs_t_msl : (fmt == 4 ? MetalKqMulMmK4T_metal_kq_mulmm_t_msl : (fmt == 5 ? MetalKqMulMmK5T_metal_kq_mulmm_t_msl : MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl)))))))))))) : (fmt == 25 ? MetalKqMulMmIq2xxs_metal_kq_mulmm_msl : (fmt == 24 ? MetalKqMulMmIq2xs_metal_kq_mulmm_msl : (fmt == 23 ? MetalKqMulMmIq2s_metal_kq_mulmm_msl : (fmt == 45 ? MetalKqMulMmIq4nl_metal_kq_mulmm_msl : (fmt == 2 ? MetalKqMulMmK2_metal_kq_mulmm_msl : (fmt == 34 ? MetalKqMulMmIq3xxs_metal_kq_mulmm_msl : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl : MetalKqMulMmK6_metal_kq_mulmm_msl)))))))))))) let entry = (tensor ? (tall ? (fmt == 4 ? MetalKqMulMmK4TH128_metal_kq_mulmm_t_msl_entry : (fmt == 5 ? MetalKqMulMmK5TH128_metal_kq_mulmm_t_msl_entry : MetalKqMulMmK6TH128_metal_kq_mulmm_k6_t_msl_entry)) : halfx - ? (fmt == 4 ? MetalKqMulMmK4TH_metal_kq_mulmm_t_msl_entry : (fmt == 5 ? MetalKqMulMmK5TH_metal_kq_mulmm_t_msl_entry : MetalKqMulMmK6TH_metal_kq_mulmm_k6_t_msl_entry)) - : (fmt == 4 ? MetalKqMulMmK4T_metal_kq_mulmm_t_msl_entry : (fmt == 5 ? MetalKqMulMmK5T_metal_kq_mulmm_t_msl_entry : MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl_entry))) + ? (fmt == 25 ? MetalKqMulMmIq2xxsTH_metal_kq_mulmm_iq2xxs_t_msl_entry : (fmt == 24 ? MetalKqMulMmIq2xsTH_metal_kq_mulmm_iq2xs_t_msl_entry : (fmt == 23 ? MetalKqMulMmIq2sTH_metal_kq_mulmm_iq2s_t_msl_entry : (fmt == 45 ? MetalKqMulMmIq4nlTH_metal_kq_mulmm_iq4xs_t_msl_entry : (fmt == 2 ? MetalKqMulMmK2TH_metal_kq_mulmm_k2_t_msl_entry : (fmt == 34 ? MetalKqMulMmIq3xxsTH_metal_kq_mulmm_iq3xxs_t_msl_entry : (fmt == 33 ? MetalKqMulMmIq3sTH_metal_kq_mulmm_iq3s_t_msl_entry : (fmt == 3 ? MetalKqMulMmK3TH_metal_kq_mulmm_k3_t_msl_entry : (fmt == 44 ? MetalKqMulMmIq4xsTH_metal_kq_mulmm_iq4xs_t_msl_entry : (fmt == 4 ? MetalKqMulMmK4TH_metal_kq_mulmm_t_msl_entry : (fmt == 5 ? MetalKqMulMmK5TH_metal_kq_mulmm_t_msl_entry : MetalKqMulMmK6TH_metal_kq_mulmm_k6_t_msl_entry))))))))))) + : (fmt == 25 ? MetalKqMulMmIq2xxsT_metal_kq_mulmm_iq2xxs_t_msl_entry : (fmt == 24 ? MetalKqMulMmIq2xsT_metal_kq_mulmm_iq2xs_t_msl_entry : (fmt == 23 ? MetalKqMulMmIq2sT_metal_kq_mulmm_iq2s_t_msl_entry : (fmt == 45 ? MetalKqMulMmIq4nlT_metal_kq_mulmm_iq4xs_t_msl_entry : (fmt == 2 ? MetalKqMulMmK2T_metal_kq_mulmm_k2_t_msl_entry : (fmt == 34 ? MetalKqMulMmIq3xxsT_metal_kq_mulmm_iq3xxs_t_msl_entry : (fmt == 33 ? MetalKqMulMmIq3sT_metal_kq_mulmm_iq3s_t_msl_entry : (fmt == 3 ? MetalKqMulMmK3T_metal_kq_mulmm_k3_t_msl_entry : (fmt == 44 ? MetalKqMulMmIq4xsT_metal_kq_mulmm_iq4xs_t_msl_entry : (fmt == 4 ? MetalKqMulMmK4T_metal_kq_mulmm_t_msl_entry : (fmt == 5 ? MetalKqMulMmK5T_metal_kq_mulmm_t_msl_entry : MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl_entry)))))))))))) : (fmt == 25 ? MetalKqMulMmIq2xxs_metal_kq_mulmm_msl_entry : (fmt == 24 ? MetalKqMulMmIq2xs_metal_kq_mulmm_msl_entry : (fmt == 23 ? MetalKqMulMmIq2s_metal_kq_mulmm_msl_entry : (fmt == 45 ? MetalKqMulMmIq4nl_metal_kq_mulmm_msl_entry : (fmt == 2 ? MetalKqMulMmK2_metal_kq_mulmm_msl_entry : (fmt == 34 ? MetalKqMulMmIq3xxs_metal_kq_mulmm_msl_entry : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl_entry : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl_entry : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl_entry : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_entry : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_entry : MetalKqMulMmK6_metal_kq_mulmm_msl_entry)))))))))))) let fm = (tensor ? (tall ? (fmt == 4 ? MetalKqMulMmK4TH128_metal_kq_mulmm_t_msl_fastmath : (fmt == 5 ? MetalKqMulMmK5TH128_metal_kq_mulmm_t_msl_fastmath : MetalKqMulMmK6TH128_metal_kq_mulmm_k6_t_msl_fastmath)) : halfx - ? (fmt == 4 ? MetalKqMulMmK4TH_metal_kq_mulmm_t_msl_fastmath : (fmt == 5 ? MetalKqMulMmK5TH_metal_kq_mulmm_t_msl_fastmath : MetalKqMulMmK6TH_metal_kq_mulmm_k6_t_msl_fastmath)) - : (fmt == 4 ? MetalKqMulMmK4T_metal_kq_mulmm_t_msl_fastmath : (fmt == 5 ? MetalKqMulMmK5T_metal_kq_mulmm_t_msl_fastmath : MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl_fastmath))) + ? (fmt == 25 ? MetalKqMulMmIq2xxsTH_metal_kq_mulmm_iq2xxs_t_msl_fastmath : (fmt == 24 ? MetalKqMulMmIq2xsTH_metal_kq_mulmm_iq2xs_t_msl_fastmath : (fmt == 23 ? MetalKqMulMmIq2sTH_metal_kq_mulmm_iq2s_t_msl_fastmath : (fmt == 45 ? MetalKqMulMmIq4nlTH_metal_kq_mulmm_iq4xs_t_msl_fastmath : (fmt == 2 ? MetalKqMulMmK2TH_metal_kq_mulmm_k2_t_msl_fastmath : (fmt == 34 ? MetalKqMulMmIq3xxsTH_metal_kq_mulmm_iq3xxs_t_msl_fastmath : (fmt == 33 ? MetalKqMulMmIq3sTH_metal_kq_mulmm_iq3s_t_msl_fastmath : (fmt == 3 ? MetalKqMulMmK3TH_metal_kq_mulmm_k3_t_msl_fastmath : (fmt == 44 ? MetalKqMulMmIq4xsTH_metal_kq_mulmm_iq4xs_t_msl_fastmath : (fmt == 4 ? MetalKqMulMmK4TH_metal_kq_mulmm_t_msl_fastmath : (fmt == 5 ? MetalKqMulMmK5TH_metal_kq_mulmm_t_msl_fastmath : MetalKqMulMmK6TH_metal_kq_mulmm_k6_t_msl_fastmath))))))))))) + : (fmt == 25 ? MetalKqMulMmIq2xxsT_metal_kq_mulmm_iq2xxs_t_msl_fastmath : (fmt == 24 ? MetalKqMulMmIq2xsT_metal_kq_mulmm_iq2xs_t_msl_fastmath : (fmt == 23 ? MetalKqMulMmIq2sT_metal_kq_mulmm_iq2s_t_msl_fastmath : (fmt == 45 ? MetalKqMulMmIq4nlT_metal_kq_mulmm_iq4xs_t_msl_fastmath : (fmt == 2 ? MetalKqMulMmK2T_metal_kq_mulmm_k2_t_msl_fastmath : (fmt == 34 ? MetalKqMulMmIq3xxsT_metal_kq_mulmm_iq3xxs_t_msl_fastmath : (fmt == 33 ? MetalKqMulMmIq3sT_metal_kq_mulmm_iq3s_t_msl_fastmath : (fmt == 3 ? MetalKqMulMmK3T_metal_kq_mulmm_k3_t_msl_fastmath : (fmt == 44 ? MetalKqMulMmIq4xsT_metal_kq_mulmm_iq4xs_t_msl_fastmath : (fmt == 4 ? MetalKqMulMmK4T_metal_kq_mulmm_t_msl_fastmath : (fmt == 5 ? MetalKqMulMmK5T_metal_kq_mulmm_t_msl_fastmath : MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl_fastmath)))))))))))) : (fmt == 25 ? MetalKqMulMmIq2xxs_metal_kq_mulmm_msl_fastmath : (fmt == 24 ? MetalKqMulMmIq2xs_metal_kq_mulmm_msl_fastmath : (fmt == 23 ? MetalKqMulMmIq2s_metal_kq_mulmm_msl_fastmath : (fmt == 45 ? MetalKqMulMmIq4nl_metal_kq_mulmm_msl_fastmath : (fmt == 2 ? MetalKqMulMmK2_metal_kq_mulmm_msl_fastmath : (fmt == 34 ? MetalKqMulMmIq3xxs_metal_kq_mulmm_msl_fastmath : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl_fastmath : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl_fastmath : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl_fastmath : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_fastmath : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_fastmath : MetalKqMulMmK6_metal_kq_mulmm_msl_fastmath)))))))))))) let tgm = (tensor ? (tall ? (fmt == 4 ? MetalKqMulMmK4TH128_metal_kq_mulmm_t_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5TH128_metal_kq_mulmm_t_msl_tgmem : MetalKqMulMmK6TH128_metal_kq_mulmm_k6_t_msl_tgmem)) : halfx - ? (fmt == 4 ? MetalKqMulMmK4TH_metal_kq_mulmm_t_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5TH_metal_kq_mulmm_t_msl_tgmem : MetalKqMulMmK6TH_metal_kq_mulmm_k6_t_msl_tgmem)) - : (fmt == 4 ? MetalKqMulMmK4T_metal_kq_mulmm_t_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5T_metal_kq_mulmm_t_msl_tgmem : MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl_tgmem))) + ? (fmt == 25 ? MetalKqMulMmIq2xxsTH_metal_kq_mulmm_iq2xxs_t_msl_tgmem : (fmt == 24 ? MetalKqMulMmIq2xsTH_metal_kq_mulmm_iq2xs_t_msl_tgmem : (fmt == 23 ? MetalKqMulMmIq2sTH_metal_kq_mulmm_iq2s_t_msl_tgmem : (fmt == 45 ? MetalKqMulMmIq4nlTH_metal_kq_mulmm_iq4xs_t_msl_tgmem : (fmt == 2 ? MetalKqMulMmK2TH_metal_kq_mulmm_k2_t_msl_tgmem : (fmt == 34 ? MetalKqMulMmIq3xxsTH_metal_kq_mulmm_iq3xxs_t_msl_tgmem : (fmt == 33 ? MetalKqMulMmIq3sTH_metal_kq_mulmm_iq3s_t_msl_tgmem : (fmt == 3 ? MetalKqMulMmK3TH_metal_kq_mulmm_k3_t_msl_tgmem : (fmt == 44 ? MetalKqMulMmIq4xsTH_metal_kq_mulmm_iq4xs_t_msl_tgmem : (fmt == 4 ? MetalKqMulMmK4TH_metal_kq_mulmm_t_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5TH_metal_kq_mulmm_t_msl_tgmem : MetalKqMulMmK6TH_metal_kq_mulmm_k6_t_msl_tgmem))))))))))) + : (fmt == 25 ? MetalKqMulMmIq2xxsT_metal_kq_mulmm_iq2xxs_t_msl_tgmem : (fmt == 24 ? MetalKqMulMmIq2xsT_metal_kq_mulmm_iq2xs_t_msl_tgmem : (fmt == 23 ? MetalKqMulMmIq2sT_metal_kq_mulmm_iq2s_t_msl_tgmem : (fmt == 45 ? MetalKqMulMmIq4nlT_metal_kq_mulmm_iq4xs_t_msl_tgmem : (fmt == 2 ? MetalKqMulMmK2T_metal_kq_mulmm_k2_t_msl_tgmem : (fmt == 34 ? MetalKqMulMmIq3xxsT_metal_kq_mulmm_iq3xxs_t_msl_tgmem : (fmt == 33 ? MetalKqMulMmIq3sT_metal_kq_mulmm_iq3s_t_msl_tgmem : (fmt == 3 ? MetalKqMulMmK3T_metal_kq_mulmm_k3_t_msl_tgmem : (fmt == 44 ? MetalKqMulMmIq4xsT_metal_kq_mulmm_iq4xs_t_msl_tgmem : (fmt == 4 ? MetalKqMulMmK4T_metal_kq_mulmm_t_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5T_metal_kq_mulmm_t_msl_tgmem : MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl_tgmem)))))))))))) : (fmt == 25 ? MetalKqMulMmIq2xxs_metal_kq_mulmm_msl_tgmem : (fmt == 24 ? MetalKqMulMmIq2xs_metal_kq_mulmm_msl_tgmem : (fmt == 23 ? MetalKqMulMmIq2s_metal_kq_mulmm_msl_tgmem : (fmt == 45 ? MetalKqMulMmIq4nl_metal_kq_mulmm_msl_tgmem : (fmt == 2 ? MetalKqMulMmK2_metal_kq_mulmm_msl_tgmem : (fmt == 34 ? MetalKqMulMmIq3xxs_metal_kq_mulmm_msl_tgmem : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl_tgmem : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl_tgmem : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl_tgmem : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_tgmem : MetalKqMulMmK6_metal_kq_mulmm_msl_tgmem)))))))))))) var pso = pipeline_from_source(dev, src, entry, fm, err) if (tensor && pso == null && !metal4_tensor_available(dev)) { @@ -2065,6 +2065,11 @@ def test_metal_gemm_kernels(t : T?) { kq_mulmm_gate(t, dev, queue, 24, false, 32, 256, 128) kq_mulmm_gate(t, dev, queue, 25, false, 64, 512, 64) kq_mulmm_gate(t, dev, queue, 25, false, 32, 256, 128) + // the iquant tensor twins (T + TH stamps; twins feint without Metal-4) + for (fmt in [44, 45, 3, 33, 34, 2, 23, 24, 25]) { + kq_mulmm_gate(t, dev, queue, fmt, true, 64, 512, 64) + kq_mulmm_gate(t, dev, queue, fmt, true, 64, 512, 64, halfx = true) + } q8_mulmm_t_gate(t, dev, queue, 64, 256, 64) q8_mulmm_t_gate(t, dev, queue, 64, 256, 64, halfx = true) q8_mulmm_t_gate(t, dev, queue, 64, 160, 64) // kdim % 64 == 32: the bk=64 helper's 32-tail chunk From d342a0c6f13f95ac65034798866751e142d8916b Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 01:10:20 -0700 Subject: [PATCH 05/35] kq race lab: the D1 lane-map decode spellings for the iq2 family - all three at/above lcpp on M5 The q4_K lane map (8 lanes/superblock, one whole 32-block per thread, ib += 4) with the f4 magnitude slab where it fits: iq2xxs d1f4 (8 KB slab) 1.22-1.56x over prod -> 0.97-1.16x vs lcpp; iq2xs d1f4 (16 KB) 1.09-1.43x -> 0.95-1.14x; iq2s d1 (constant reads - its slab would be 32 KB, lcpp skips staging it too) 1.12-1.30x -> 0.97-1.13x. Row-count knob raced: 2 rows/sg (the k4 choice) beats 4; the raw packed-word slab is flat (pre-EXPANSION is what pays, not staging). M4 race owed before promotion. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014mq9AKyoAwvD5T1cigTnRw --- .../benchmarks/matmul/bench_metal_kq_race.das | 367 ++++++++++++++++++ 1 file changed, 367 insertions(+) diff --git a/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das b/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das index 076130f0bc..4a2fd6d979 100644 --- a/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das +++ b/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das @@ -136,10 +136,16 @@ def private gemv_arms(f : KqRaceFmt) : array { // candidate GEMV sp arms |> push(KqRaceArm(name = "prod", src = metal_kq_gemv_k2_msl, entry = metal_kq_gemv_k2_msl_entry, fastmath = metal_kq_gemv_k2_msl_fastmath, tgmem = 0ul, gdiv = f.gemv_gdiv)) } elif (f.fmt == 23) { arms |> push(KqRaceArm(name = "prod", src = metal_kq_gemv_iq2s_msl, entry = metal_kq_gemv_iq2s_msl_entry, fastmath = metal_kq_gemv_iq2s_msl_fastmath, tgmem = 0ul, gdiv = f.gemv_gdiv)) + arms |> push(KqRaceArm(name = "d1", src = lab_kq_gemv_iq2s_d1_msl, entry = lab_kq_gemv_iq2s_d1_msl_entry, fastmath = lab_kq_gemv_iq2s_d1_msl_fastmath, tgmem = 0ul, gdiv = 4)) } elif (f.fmt == 24) { arms |> push(KqRaceArm(name = "prod", src = metal_kq_gemv_iq2xs_msl, entry = metal_kq_gemv_iq2xs_msl_entry, fastmath = metal_kq_gemv_iq2xs_msl_fastmath, tgmem = 0ul, gdiv = f.gemv_gdiv)) + arms |> push(KqRaceArm(name = "d1f4", src = lab_kq_gemv_iq2xs_d1f4_msl, entry = lab_kq_gemv_iq2xs_d1f4_msl_entry, fastmath = lab_kq_gemv_iq2xs_d1f4_msl_fastmath, tgmem = lab_kq_gemv_iq2xs_d1f4_msl_tgmem, gdiv = 4)) } else { arms |> push(KqRaceArm(name = "prod", src = metal_kq_gemv_iq2xxs_msl, entry = metal_kq_gemv_iq2xxs_msl_entry, fastmath = metal_kq_gemv_iq2xxs_msl_fastmath, tgmem = 0ul, gdiv = f.gemv_gdiv)) + arms |> push(KqRaceArm(name = "d1r2", src = lab_kq_gemv_iq2xxs_d1r2_msl, entry = lab_kq_gemv_iq2xxs_d1r2_msl_entry, fastmath = lab_kq_gemv_iq2xxs_d1r2_msl_fastmath, tgmem = 0ul, gdiv = 4)) + arms |> push(KqRaceArm(name = "d1r4", src = lab_kq_gemv_iq2xxs_d1r4_msl, entry = lab_kq_gemv_iq2xxs_d1r4_msl_entry, fastmath = lab_kq_gemv_iq2xxs_d1r4_msl_fastmath, tgmem = 0ul, gdiv = 8)) + arms |> push(KqRaceArm(name = "d1sg", src = lab_kq_gemv_iq2xxs_d1sg_msl, entry = lab_kq_gemv_iq2xxs_d1sg_msl_entry, fastmath = lab_kq_gemv_iq2xxs_d1sg_msl_fastmath, tgmem = lab_kq_gemv_iq2xxs_d1sg_msl_tgmem, gdiv = 4)) + arms |> push(KqRaceArm(name = "d1f4", src = lab_kq_gemv_iq2xxs_d1f4_msl, entry = lab_kq_gemv_iq2xxs_d1f4_msl_entry, fastmath = lab_kq_gemv_iq2xxs_d1f4_msl_fastmath, tgmem = lab_kq_gemv_iq2xxs_d1f4_msl_tgmem, gdiv = 4)) } return <- arms } @@ -205,6 +211,367 @@ def private mm_arms(f : KqRaceFmt) : array { // candidate mul_mm sp let PREFILL_M = 512 // the mm tier's token batch +// D1 candidate: the iq2xxs decode GEMV on the q4_K lane map - 8 lanes per superblock (one +// whole 32-block per thread: ONE grid-word/aux/strip load serves 32 elements where the +// shipped 16-lane form pays two), 4 superblock streams, ib += 4. llama.cpp runs all five +// iq2/iq3 grid formats on this map; our at-parity k4 GEMV already uses it. Two row-count +// stamps: 2 rows/simdgroup (the k4 choice) and 4 (the shipped iq2xxs choice). +class LabKqGemvIq2xxsD1R2 { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array + @ssbo @binding = 3 @role = "read" @off = "xoff" @span = "n*4" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" @span = "rows*4" y : array + @uniform @binding = 5 ndim : uint + @uniform @binding = 6 ddim : uint + + [metal_kernel(name="lab_kq_gemv_iq2xxs_d1r2_msl")] + def lab_kq_gemv_iq2xxs_d1r2 { + let lane = gl_SubgroupInvocationID + let ix = lane / 8u + let it = lane % 8u + let nb = ndim / 256u + let first_row = (gl_WorkGroupID.x * 2u + gl_SubgroupID) * 2u + var sumf : float[2] + var ib = ix + while (ib < nb) { + let yb4 = ib * 64u + it * 8u + var yv : float4[8] + for [unroll_full] (i in range(8)) { + yv[i] = x[yb4 + uint(i)] + } + for [unroll_full] (r in range(2)) { + let blk = (first_row + uint(r)) * nb + ib + let gw = kqu[blk * 16u + it * 2u] + let aux = kqu[blk * 16u + it * 2u + 1u] + var accv = float4(0.0) + for [unroll_full] (q3 in range(4)) { + let bidx = (gw >> (8u * uint(q3))) & 255u + let sgn = ksign7m((aux >> (7u * uint(q3))) & 127u) + let w0 = iq3s_sw(iq2xxs_gw(int(bidx * 2u)), sgn & 15u) + let w1 = iq3s_sw(iq2xxs_gw(int(bidx * 2u + 1u)), sgn >> 4u) + accv += yv[q3 * 2] * float4(iq3s_sx(w0, 0u), iq3s_sx(w0, 1u), iq3s_sx(w0, 2u), iq3s_sx(w0, 3u)) + accv += yv[q3 * 2 + 1] * float4(iq3s_sx(w1, 0u), iq3s_sx(w1, 1u), iq3s_sx(w1, 2u), iq3s_sx(w1, 3u)) + } + let sc = float(kscb[blk * 16u + it]) + sumf[r] += float(kdh[blk]) * sc * (accv.x + accv.y + accv.z + accv.w) + } + ib += 4u + } + for [unroll_full] (r in range(2)) { + let s = simd_sum(sumf[r]) + if (lane == 0u && first_row + uint(r) < ddim) { + y[first_row + uint(r)] = s + } + } + } +} + +// D1 + D2: the lane-map winner with the 2 KB grid staged into threadgroup memory (llama.cpp's +// choice for iq2_xxs - our two worst tg ratios are exactly the two grids they stage). +class LabKqGemvIq2xxsD1Sg { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array + @ssbo @binding = 3 @role = "read" @off = "xoff" @span = "n*4" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" @span = "rows*4" y : array + @uniform @binding = 5 ndim : uint + @uniform @binding = 6 ddim : uint + @workgroup gws : uint[512] // the 2 KB iq2xxs grid (u64 low/high word pairs), staged per tg + + [metal_kernel(name="lab_kq_gemv_iq2xxs_d1sg_msl")] + def lab_kq_gemv_iq2xxs_d1sg { + let lid = gl_LocalInvocationID.x + for [unroll_full] (gk in range(8)) { + let gi = lid + uint(gk) * 64u + gws[gi] = iq2xxs_gw(int(gi)) + } + barrier() + let lane = gl_SubgroupInvocationID + let ix = lane / 8u + let it = lane % 8u + let nb = ndim / 256u + let first_row = (gl_WorkGroupID.x * 2u + gl_SubgroupID) * 2u + var sumf : float[2] + var ib = ix + while (ib < nb) { + let yb4 = ib * 64u + it * 8u + var yv : float4[8] + for [unroll_full] (i in range(8)) { + yv[i] = x[yb4 + uint(i)] + } + for [unroll_full] (r in range(2)) { + let blk = (first_row + uint(r)) * nb + ib + let gw = kqu[blk * 16u + it * 2u] + let aux = kqu[blk * 16u + it * 2u + 1u] + var accv = float4(0.0) + for [unroll_full] (q3 in range(4)) { + let bidx = (gw >> (8u * uint(q3))) & 255u + let sgn = ksign7m((aux >> (7u * uint(q3))) & 127u) + let w0 = iq3s_sw(gws[bidx * 2u], sgn & 15u) + let w1 = iq3s_sw(gws[bidx * 2u + 1u], sgn >> 4u) + accv += yv[q3 * 2] * float4(iq3s_sx(w0, 0u), iq3s_sx(w0, 1u), iq3s_sx(w0, 2u), iq3s_sx(w0, 3u)) + accv += yv[q3 * 2 + 1] * float4(iq3s_sx(w1, 0u), iq3s_sx(w1, 1u), iq3s_sx(w1, 2u), iq3s_sx(w1, 3u)) + } + let sc = float(kscb[blk * 16u + it]) + sumf[r] += float(kdh[blk]) * sc * (accv.x + accv.y + accv.z + accv.w) + } + ib += 4u + } + for [unroll_full] (r in range(2)) { + let s = simd_sum(sumf[r]) + if (lane == 0u && first_row + uint(r) < ddim) { + y[first_row + uint(r)] = s + } + } + } +} + +// D1 + the f4 magnitude slab (the crowned iq3s GEMV form): 512 float4 = 8 KB, signs by select. +class LabKqGemvIq2xxsD1F4 { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array + @ssbo @binding = 3 @role = "read" @off = "xoff" @span = "n*4" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" @span = "rows*4" y : array + @uniform @binding = 5 ndim : uint + @uniform @binding = 6 ddim : uint + @workgroup gridf : float4[512] // pre-expanded magnitudes, one float4 per grid word (8 KB) + + [metal_kernel(name="lab_kq_gemv_iq2xxs_d1f4_msl")] + def lab_kq_gemv_iq2xxs_d1f4 { + let lid = gl_LocalInvocationID.x + for [unroll_full] (gk in range(8)) { + let gi = lid + uint(gk) * 64u + let w = iq2xxs_gw(int(gi)) + gridf[gi] = float4(float(w & 255u), float((w >> 8u) & 255u), float((w >> 16u) & 255u), float(w >> 24u)) + } + barrier() + let lane = gl_SubgroupInvocationID + let ix = lane / 8u + let it = lane % 8u + let nb = ndim / 256u + let first_row = (gl_WorkGroupID.x * 2u + gl_SubgroupID) * 2u + var sumf : float[2] + var ib = ix + while (ib < nb) { + let yb4 = ib * 64u + it * 8u + var yv : float4[8] + for [unroll_full] (i in range(8)) { + yv[i] = x[yb4 + uint(i)] + } + for [unroll_full] (r in range(2)) { + let blk = (first_row + uint(r)) * nb + ib + let gw = kqu[blk * 16u + it * 2u] + let aux = kqu[blk * 16u + it * 2u + 1u] + var accv = float4(0.0) + for [unroll_full] (q3 in range(4)) { + let bidx = (gw >> (8u * uint(q3))) & 255u + let sgn = ksign7m((aux >> (7u * uint(q3))) & 127u) + let s0 = sgn & 15u + let s1 = sgn >> 4u + let m0 = gridf[bidx * 2u] + let m1 = gridf[bidx * 2u + 1u] + let f0 = float4((s0 & 1u) != 0u ? -m0.x : m0.x, (s0 & 2u) != 0u ? -m0.y : m0.y, + (s0 & 4u) != 0u ? -m0.z : m0.z, (s0 & 8u) != 0u ? -m0.w : m0.w) + let f1 = float4((s1 & 1u) != 0u ? -m1.x : m1.x, (s1 & 2u) != 0u ? -m1.y : m1.y, + (s1 & 4u) != 0u ? -m1.z : m1.z, (s1 & 8u) != 0u ? -m1.w : m1.w) + accv += yv[q3 * 2] * f0 + accv += yv[q3 * 2 + 1] * f1 + } + let sc = float(kscb[blk * 16u + it]) + sumf[r] += float(kdh[blk]) * sc * (accv.x + accv.y + accv.z + accv.w) + } + ib += 4u + } + for [unroll_full] (r in range(2)) { + let s = simd_sum(sumf[r]) + if (lane == 0u && first_row + uint(r) < ddim) { + y[first_row + uint(r)] = s + } + } + } +} + +// iq2xs on the d1 map + the f4 magnitude slab (512 grid entries x 2 words = 1024 float4, +// 16 KB): thread = one 32-block = two per-16 halves, each one uint (2 u16 words). +class LabKqGemvIq2xsD1F4 { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array + @ssbo @binding = 3 @role = "read" @off = "xoff" @span = "n*4" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" @span = "rows*4" y : array + @uniform @binding = 5 ndim : uint + @uniform @binding = 6 ddim : uint + @workgroup gridf : float4[1024] // pre-expanded magnitudes, one float4 per grid word (16 KB) + + [metal_kernel(name="lab_kq_gemv_iq2xs_d1f4_msl")] + def lab_kq_gemv_iq2xs_d1f4 { + let lid = gl_LocalInvocationID.x + for [unroll_full] (gk in range(16)) { + let gi = lid + uint(gk) * 64u + let w = iq2xs_gw(int(gi)) + gridf[gi] = float4(float(w & 255u), float((w >> 8u) & 255u), float((w >> 16u) & 255u), float(w >> 24u)) + } + barrier() + let lane = gl_SubgroupInvocationID + let ix = lane / 8u + let it = lane % 8u + let nb = ndim / 256u + let first_row = (gl_WorkGroupID.x * 2u + gl_SubgroupID) * 2u + var sumf : float[2] + var ib = ix + while (ib < nb) { + let yb4 = ib * 64u + it * 8u + var yv : float4[8] + for [unroll_full] (i in range(8)) { + yv[i] = x[yb4 + uint(i)] + } + for [unroll_full] (r in range(2)) { + let blk = (first_row + uint(r)) * nb + ib + var half_sum : float[2] + for [unroll_full] (h in range(2)) { + let qw = kqu[blk * 16u + it * 2u + uint(h)] + var accv = float4(0.0) + for [unroll_full] (wi in range(2)) { + let w16 = (qw >> (16u * uint(wi))) & 0xFFFFu + let idx2 = (w16 & 511u) * 2u + let sgn = ksign7m(w16 >> 9u) + let s0 = sgn & 15u + let s1 = sgn >> 4u + let m0 = gridf[idx2] + let m1 = gridf[idx2 + 1u] + let f0 = float4((s0 & 1u) != 0u ? -m0.x : m0.x, (s0 & 2u) != 0u ? -m0.y : m0.y, + (s0 & 4u) != 0u ? -m0.z : m0.z, (s0 & 8u) != 0u ? -m0.w : m0.w) + let f1 = float4((s1 & 1u) != 0u ? -m1.x : m1.x, (s1 & 2u) != 0u ? -m1.y : m1.y, + (s1 & 4u) != 0u ? -m1.z : m1.z, (s1 & 8u) != 0u ? -m1.w : m1.w) + accv += yv[h * 4 + wi * 2] * f0 + accv += yv[h * 4 + wi * 2 + 1] * f1 + } + half_sum[h] = float(kscb[blk * 16u + it * 2u + uint(h)]) * (accv.x + accv.y + accv.z + accv.w) + } + sumf[r] += float(kdh[blk]) * (half_sum[0] + half_sum[1]) + } + ib += 4u + } + for [unroll_full] (r in range(2)) { + let s = simd_sum(sumf[r]) + if (lane == 0u && first_row + uint(r) < ddim) { + y[first_row + uint(r)] = s + } + } + } +} + +// iq2s on the d1 map, grid DIRECT off the hoisted constant table (its f4 slab would be 32 KB; +// llama.cpp deliberately skips staging this one too). +class LabKqGemvIq2sD1 { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array + @ssbo @binding = 3 @role = "read" @off = "xoff" @span = "n*4" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" @span = "rows*4" y : array + @uniform @binding = 5 ndim : uint + @uniform @binding = 6 ddim : uint + + [metal_kernel(name="lab_kq_gemv_iq2s_d1_msl")] + def lab_kq_gemv_iq2s_d1 { + let lane = gl_SubgroupInvocationID + let ix = lane / 8u + let it = lane % 8u + let nb = ndim / 256u + let first_row = (gl_WorkGroupID.x * 2u + gl_SubgroupID) * 2u + var sumf : float[2] + var ib = ix + while (ib < nb) { + let yb4 = ib * 64u + it * 8u + var yv : float4[8] + for [unroll_full] (i in range(8)) { + yv[i] = x[yb4 + uint(i)] + } + for [unroll_full] (r in range(2)) { + let blk = (first_row + uint(r)) * nb + ib + let qwi = kqu[blk * 18u + it] + let sgw = kqu[blk * 18u + 8u + it] + let qh = (kqu[blk * 18u + 16u + it / 4u] >> ((it % 4u) * 8u)) & 255u + var half_sum : float[2] + for [unroll_full] (h in range(2)) { + var accv = float4(0.0) + for [unroll_full] (li in range(2)) { + let l = uint(h) * 2u + uint(li) + let idx2 = (((qwi >> (8u * l)) & 255u) | ((qh << (8u - 2u * l)) & 0x300u)) * 2u + let sgb = (sgw >> (8u * l)) & 255u + let w0 = iq3s_sw(iq2s_gw(int(idx2)), sgb & 15u) + let w1 = iq3s_sw(iq2s_gw(int(idx2 + 1u)), sgb >> 4u) + accv += yv[l * 2u] * float4(iq3s_sx(w0, 0u), iq3s_sx(w0, 1u), iq3s_sx(w0, 2u), iq3s_sx(w0, 3u)) + accv += yv[l * 2u + 1u] * float4(iq3s_sx(w1, 0u), iq3s_sx(w1, 1u), iq3s_sx(w1, 2u), iq3s_sx(w1, 3u)) + } + half_sum[h] = float(kscb[blk * 16u + it * 2u + uint(h)]) * (accv.x + accv.y + accv.z + accv.w) + } + sumf[r] += float(kdh[blk]) * (half_sum[0] + half_sum[1]) + } + ib += 4u + } + for [unroll_full] (r in range(2)) { + let s = simd_sum(sumf[r]) + if (lane == 0u && first_row + uint(r) < ddim) { + y[first_row + uint(r)] = s + } + } + } +} + +class LabKqGemvIq2xxsD1R4 { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array + @ssbo @binding = 3 @role = "read" @off = "xoff" @span = "n*4" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" @span = "rows*4" y : array + @uniform @binding = 5 ndim : uint + @uniform @binding = 6 ddim : uint + + [metal_kernel(name="lab_kq_gemv_iq2xxs_d1r4_msl")] + def lab_kq_gemv_iq2xxs_d1r4 { + let lane = gl_SubgroupInvocationID + let ix = lane / 8u + let it = lane % 8u + let nb = ndim / 256u + let first_row = (gl_WorkGroupID.x * 2u + gl_SubgroupID) * 4u + var sumf : float[4] + var ib = ix + while (ib < nb) { + let yb4 = ib * 64u + it * 8u + var yv : float4[8] + for [unroll_full] (i in range(8)) { + yv[i] = x[yb4 + uint(i)] + } + for [unroll_full] (r in range(4)) { + let blk = (first_row + uint(r)) * nb + ib + let gw = kqu[blk * 16u + it * 2u] + let aux = kqu[blk * 16u + it * 2u + 1u] + var accv = float4(0.0) + for [unroll_full] (q3 in range(4)) { + let bidx = (gw >> (8u * uint(q3))) & 255u + let sgn = ksign7m((aux >> (7u * uint(q3))) & 127u) + let w0 = iq3s_sw(iq2xxs_gw(int(bidx * 2u)), sgn & 15u) + let w1 = iq3s_sw(iq2xxs_gw(int(bidx * 2u + 1u)), sgn >> 4u) + accv += yv[q3 * 2] * float4(iq3s_sx(w0, 0u), iq3s_sx(w0, 1u), iq3s_sx(w0, 2u), iq3s_sx(w0, 3u)) + accv += yv[q3 * 2 + 1] * float4(iq3s_sx(w1, 0u), iq3s_sx(w1, 1u), iq3s_sx(w1, 2u), iq3s_sx(w1, 3u)) + } + let sc = float(kscb[blk * 16u + it]) + sumf[r] += float(kdh[blk]) * sc * (accv.x + accv.y + accv.z + accv.w) + } + ib += 4u + } + for [unroll_full] (r in range(4)) { + let s = simd_sum(sumf[r]) + if (lane == 0u && first_row + uint(r) < ddim) { + y[first_row + uint(r)] = s + } + } + } +} + // Rewrite every threadgroup A-tile subscript store `ta[IDX] = RHS;` into the pointer form // `*(ta + (IDX)) = RHS;` - llama.cpp's mul_mm carries the subscript form commented out as // "massively slower.. WTF?" (mul_mm.metal); this arm races that claim on our own template. From 11d15cfd199217cafa34e00a6fb1693dee3615f8 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 01:18:13 -0700 Subject: [PATCH 06/35] promote the D1 lane-map decode GEMVs for the iq2 family + the iq2xxs f4 crown twin iq2s/iq2xs/iq2xxs move to the q4_K lane map (8 lanes/superblock, one 32-block per thread, 2 rows/simdgroup, ib += 4, dispatch rows/4) - interleaved wins on BOTH boxes (M5 1.07-1.43x, M4 1.07-1.22x over the old 16-lane forms). iq2xs additionally stages its grid as a 16 KB f4 magnitude slab (wins both boxes). iq2xxs's 8 KB f4 slab is generation-split (M5 +12-22%, M4 -4-16%), so it ships as MetalKqGemvIq2xxsF4 behind a NEW box-raced crown 'kq_gemv_iq2xxs_f4' (race_gemv_iq2xxs_f4 in metal_tensor_race_decode; both call sites pick by the compiled-under-crown PSO). gemv suite 2/2 incl. the new f4 twin cells; every lab arm oracle-green. Lab: the promoted forms replace their candidate arms, gdiv 4 for the trio. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014mq9AKyoAwvD5T1cigTnRw --- .../benchmarks/matmul/bench_metal_kq_race.das | 376 +----------------- .../dasllama/dasllama_metal_common.das | 1 + .../dasllama/dasllama_metal_kernels.das | 353 ++++++++++++---- .../dasllama/dasllama_metal_prefill.das | 6 +- .../tests/test_metal_gemv_kernels.das | 18 +- 5 files changed, 290 insertions(+), 464 deletions(-) diff --git a/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das b/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das index 4a2fd6d979..a5f3fcf916 100644 --- a/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das +++ b/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das @@ -89,9 +89,9 @@ def race_formats : array { KqRaceFmt(name = "iq3s", fmt = 33, gemv_gdiv = 8, split = true), KqRaceFmt(name = "iq3xxs", fmt = 34, gemv_gdiv = 8, split = true), KqRaceFmt(name = "k2", fmt = 2, gemv_gdiv = 4, split = true), - KqRaceFmt(name = "iq2s", fmt = 23, gemv_gdiv = 8, split = true), - KqRaceFmt(name = "iq2xs", fmt = 24, gemv_gdiv = 8, split = true), - KqRaceFmt(name = "iq2xxs", fmt = 25, gemv_gdiv = 8, split = true) + KqRaceFmt(name = "iq2s", fmt = 23, gemv_gdiv = 4, split = true), + KqRaceFmt(name = "iq2xs", fmt = 24, gemv_gdiv = 4, split = true), + KqRaceFmt(name = "iq2xxs", fmt = 25, gemv_gdiv = 4, split = true) ] } @@ -136,16 +136,11 @@ def private gemv_arms(f : KqRaceFmt) : array { // candidate GEMV sp arms |> push(KqRaceArm(name = "prod", src = metal_kq_gemv_k2_msl, entry = metal_kq_gemv_k2_msl_entry, fastmath = metal_kq_gemv_k2_msl_fastmath, tgmem = 0ul, gdiv = f.gemv_gdiv)) } elif (f.fmt == 23) { arms |> push(KqRaceArm(name = "prod", src = metal_kq_gemv_iq2s_msl, entry = metal_kq_gemv_iq2s_msl_entry, fastmath = metal_kq_gemv_iq2s_msl_fastmath, tgmem = 0ul, gdiv = f.gemv_gdiv)) - arms |> push(KqRaceArm(name = "d1", src = lab_kq_gemv_iq2s_d1_msl, entry = lab_kq_gemv_iq2s_d1_msl_entry, fastmath = lab_kq_gemv_iq2s_d1_msl_fastmath, tgmem = 0ul, gdiv = 4)) } elif (f.fmt == 24) { - arms |> push(KqRaceArm(name = "prod", src = metal_kq_gemv_iq2xs_msl, entry = metal_kq_gemv_iq2xs_msl_entry, fastmath = metal_kq_gemv_iq2xs_msl_fastmath, tgmem = 0ul, gdiv = f.gemv_gdiv)) - arms |> push(KqRaceArm(name = "d1f4", src = lab_kq_gemv_iq2xs_d1f4_msl, entry = lab_kq_gemv_iq2xs_d1f4_msl_entry, fastmath = lab_kq_gemv_iq2xs_d1f4_msl_fastmath, tgmem = lab_kq_gemv_iq2xs_d1f4_msl_tgmem, gdiv = 4)) + arms |> push(KqRaceArm(name = "prod", src = metal_kq_gemv_iq2xs_msl, entry = metal_kq_gemv_iq2xs_msl_entry, fastmath = metal_kq_gemv_iq2xs_msl_fastmath, tgmem = metal_kq_gemv_iq2xs_msl_tgmem, gdiv = f.gemv_gdiv)) } else { arms |> push(KqRaceArm(name = "prod", src = metal_kq_gemv_iq2xxs_msl, entry = metal_kq_gemv_iq2xxs_msl_entry, fastmath = metal_kq_gemv_iq2xxs_msl_fastmath, tgmem = 0ul, gdiv = f.gemv_gdiv)) - arms |> push(KqRaceArm(name = "d1r2", src = lab_kq_gemv_iq2xxs_d1r2_msl, entry = lab_kq_gemv_iq2xxs_d1r2_msl_entry, fastmath = lab_kq_gemv_iq2xxs_d1r2_msl_fastmath, tgmem = 0ul, gdiv = 4)) - arms |> push(KqRaceArm(name = "d1r4", src = lab_kq_gemv_iq2xxs_d1r4_msl, entry = lab_kq_gemv_iq2xxs_d1r4_msl_entry, fastmath = lab_kq_gemv_iq2xxs_d1r4_msl_fastmath, tgmem = 0ul, gdiv = 8)) - arms |> push(KqRaceArm(name = "d1sg", src = lab_kq_gemv_iq2xxs_d1sg_msl, entry = lab_kq_gemv_iq2xxs_d1sg_msl_entry, fastmath = lab_kq_gemv_iq2xxs_d1sg_msl_fastmath, tgmem = lab_kq_gemv_iq2xxs_d1sg_msl_tgmem, gdiv = 4)) - arms |> push(KqRaceArm(name = "d1f4", src = lab_kq_gemv_iq2xxs_d1f4_msl, entry = lab_kq_gemv_iq2xxs_d1f4_msl_entry, fastmath = lab_kq_gemv_iq2xxs_d1f4_msl_fastmath, tgmem = lab_kq_gemv_iq2xxs_d1f4_msl_tgmem, gdiv = 4)) + arms |> push(KqRaceArm(name = "f4", src = metal_kq_gemv_iq2xxs_f4_msl, entry = metal_kq_gemv_iq2xxs_f4_msl_entry, fastmath = metal_kq_gemv_iq2xxs_f4_msl_fastmath, tgmem = metal_kq_gemv_iq2xxs_f4_msl_tgmem, gdiv = 4)) } return <- arms } @@ -211,367 +206,6 @@ def private mm_arms(f : KqRaceFmt) : array { // candidate mul_mm sp let PREFILL_M = 512 // the mm tier's token batch -// D1 candidate: the iq2xxs decode GEMV on the q4_K lane map - 8 lanes per superblock (one -// whole 32-block per thread: ONE grid-word/aux/strip load serves 32 elements where the -// shipped 16-lane form pays two), 4 superblock streams, ib += 4. llama.cpp runs all five -// iq2/iq3 grid formats on this map; our at-parity k4 GEMV already uses it. Two row-count -// stamps: 2 rows/simdgroup (the k4 choice) and 4 (the shipped iq2xxs choice). -class LabKqGemvIq2xxsD1R2 { - @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array - @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array - @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array - @ssbo @binding = 3 @role = "read" @off = "xoff" @span = "n*4" x : array - @ssbo @binding = 4 @role = "write" @off = "yoff" @span = "rows*4" y : array - @uniform @binding = 5 ndim : uint - @uniform @binding = 6 ddim : uint - - [metal_kernel(name="lab_kq_gemv_iq2xxs_d1r2_msl")] - def lab_kq_gemv_iq2xxs_d1r2 { - let lane = gl_SubgroupInvocationID - let ix = lane / 8u - let it = lane % 8u - let nb = ndim / 256u - let first_row = (gl_WorkGroupID.x * 2u + gl_SubgroupID) * 2u - var sumf : float[2] - var ib = ix - while (ib < nb) { - let yb4 = ib * 64u + it * 8u - var yv : float4[8] - for [unroll_full] (i in range(8)) { - yv[i] = x[yb4 + uint(i)] - } - for [unroll_full] (r in range(2)) { - let blk = (first_row + uint(r)) * nb + ib - let gw = kqu[blk * 16u + it * 2u] - let aux = kqu[blk * 16u + it * 2u + 1u] - var accv = float4(0.0) - for [unroll_full] (q3 in range(4)) { - let bidx = (gw >> (8u * uint(q3))) & 255u - let sgn = ksign7m((aux >> (7u * uint(q3))) & 127u) - let w0 = iq3s_sw(iq2xxs_gw(int(bidx * 2u)), sgn & 15u) - let w1 = iq3s_sw(iq2xxs_gw(int(bidx * 2u + 1u)), sgn >> 4u) - accv += yv[q3 * 2] * float4(iq3s_sx(w0, 0u), iq3s_sx(w0, 1u), iq3s_sx(w0, 2u), iq3s_sx(w0, 3u)) - accv += yv[q3 * 2 + 1] * float4(iq3s_sx(w1, 0u), iq3s_sx(w1, 1u), iq3s_sx(w1, 2u), iq3s_sx(w1, 3u)) - } - let sc = float(kscb[blk * 16u + it]) - sumf[r] += float(kdh[blk]) * sc * (accv.x + accv.y + accv.z + accv.w) - } - ib += 4u - } - for [unroll_full] (r in range(2)) { - let s = simd_sum(sumf[r]) - if (lane == 0u && first_row + uint(r) < ddim) { - y[first_row + uint(r)] = s - } - } - } -} - -// D1 + D2: the lane-map winner with the 2 KB grid staged into threadgroup memory (llama.cpp's -// choice for iq2_xxs - our two worst tg ratios are exactly the two grids they stage). -class LabKqGemvIq2xxsD1Sg { - @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array - @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array - @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array - @ssbo @binding = 3 @role = "read" @off = "xoff" @span = "n*4" x : array - @ssbo @binding = 4 @role = "write" @off = "yoff" @span = "rows*4" y : array - @uniform @binding = 5 ndim : uint - @uniform @binding = 6 ddim : uint - @workgroup gws : uint[512] // the 2 KB iq2xxs grid (u64 low/high word pairs), staged per tg - - [metal_kernel(name="lab_kq_gemv_iq2xxs_d1sg_msl")] - def lab_kq_gemv_iq2xxs_d1sg { - let lid = gl_LocalInvocationID.x - for [unroll_full] (gk in range(8)) { - let gi = lid + uint(gk) * 64u - gws[gi] = iq2xxs_gw(int(gi)) - } - barrier() - let lane = gl_SubgroupInvocationID - let ix = lane / 8u - let it = lane % 8u - let nb = ndim / 256u - let first_row = (gl_WorkGroupID.x * 2u + gl_SubgroupID) * 2u - var sumf : float[2] - var ib = ix - while (ib < nb) { - let yb4 = ib * 64u + it * 8u - var yv : float4[8] - for [unroll_full] (i in range(8)) { - yv[i] = x[yb4 + uint(i)] - } - for [unroll_full] (r in range(2)) { - let blk = (first_row + uint(r)) * nb + ib - let gw = kqu[blk * 16u + it * 2u] - let aux = kqu[blk * 16u + it * 2u + 1u] - var accv = float4(0.0) - for [unroll_full] (q3 in range(4)) { - let bidx = (gw >> (8u * uint(q3))) & 255u - let sgn = ksign7m((aux >> (7u * uint(q3))) & 127u) - let w0 = iq3s_sw(gws[bidx * 2u], sgn & 15u) - let w1 = iq3s_sw(gws[bidx * 2u + 1u], sgn >> 4u) - accv += yv[q3 * 2] * float4(iq3s_sx(w0, 0u), iq3s_sx(w0, 1u), iq3s_sx(w0, 2u), iq3s_sx(w0, 3u)) - accv += yv[q3 * 2 + 1] * float4(iq3s_sx(w1, 0u), iq3s_sx(w1, 1u), iq3s_sx(w1, 2u), iq3s_sx(w1, 3u)) - } - let sc = float(kscb[blk * 16u + it]) - sumf[r] += float(kdh[blk]) * sc * (accv.x + accv.y + accv.z + accv.w) - } - ib += 4u - } - for [unroll_full] (r in range(2)) { - let s = simd_sum(sumf[r]) - if (lane == 0u && first_row + uint(r) < ddim) { - y[first_row + uint(r)] = s - } - } - } -} - -// D1 + the f4 magnitude slab (the crowned iq3s GEMV form): 512 float4 = 8 KB, signs by select. -class LabKqGemvIq2xxsD1F4 { - @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array - @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array - @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array - @ssbo @binding = 3 @role = "read" @off = "xoff" @span = "n*4" x : array - @ssbo @binding = 4 @role = "write" @off = "yoff" @span = "rows*4" y : array - @uniform @binding = 5 ndim : uint - @uniform @binding = 6 ddim : uint - @workgroup gridf : float4[512] // pre-expanded magnitudes, one float4 per grid word (8 KB) - - [metal_kernel(name="lab_kq_gemv_iq2xxs_d1f4_msl")] - def lab_kq_gemv_iq2xxs_d1f4 { - let lid = gl_LocalInvocationID.x - for [unroll_full] (gk in range(8)) { - let gi = lid + uint(gk) * 64u - let w = iq2xxs_gw(int(gi)) - gridf[gi] = float4(float(w & 255u), float((w >> 8u) & 255u), float((w >> 16u) & 255u), float(w >> 24u)) - } - barrier() - let lane = gl_SubgroupInvocationID - let ix = lane / 8u - let it = lane % 8u - let nb = ndim / 256u - let first_row = (gl_WorkGroupID.x * 2u + gl_SubgroupID) * 2u - var sumf : float[2] - var ib = ix - while (ib < nb) { - let yb4 = ib * 64u + it * 8u - var yv : float4[8] - for [unroll_full] (i in range(8)) { - yv[i] = x[yb4 + uint(i)] - } - for [unroll_full] (r in range(2)) { - let blk = (first_row + uint(r)) * nb + ib - let gw = kqu[blk * 16u + it * 2u] - let aux = kqu[blk * 16u + it * 2u + 1u] - var accv = float4(0.0) - for [unroll_full] (q3 in range(4)) { - let bidx = (gw >> (8u * uint(q3))) & 255u - let sgn = ksign7m((aux >> (7u * uint(q3))) & 127u) - let s0 = sgn & 15u - let s1 = sgn >> 4u - let m0 = gridf[bidx * 2u] - let m1 = gridf[bidx * 2u + 1u] - let f0 = float4((s0 & 1u) != 0u ? -m0.x : m0.x, (s0 & 2u) != 0u ? -m0.y : m0.y, - (s0 & 4u) != 0u ? -m0.z : m0.z, (s0 & 8u) != 0u ? -m0.w : m0.w) - let f1 = float4((s1 & 1u) != 0u ? -m1.x : m1.x, (s1 & 2u) != 0u ? -m1.y : m1.y, - (s1 & 4u) != 0u ? -m1.z : m1.z, (s1 & 8u) != 0u ? -m1.w : m1.w) - accv += yv[q3 * 2] * f0 - accv += yv[q3 * 2 + 1] * f1 - } - let sc = float(kscb[blk * 16u + it]) - sumf[r] += float(kdh[blk]) * sc * (accv.x + accv.y + accv.z + accv.w) - } - ib += 4u - } - for [unroll_full] (r in range(2)) { - let s = simd_sum(sumf[r]) - if (lane == 0u && first_row + uint(r) < ddim) { - y[first_row + uint(r)] = s - } - } - } -} - -// iq2xs on the d1 map + the f4 magnitude slab (512 grid entries x 2 words = 1024 float4, -// 16 KB): thread = one 32-block = two per-16 halves, each one uint (2 u16 words). -class LabKqGemvIq2xsD1F4 { - @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array - @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array - @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array - @ssbo @binding = 3 @role = "read" @off = "xoff" @span = "n*4" x : array - @ssbo @binding = 4 @role = "write" @off = "yoff" @span = "rows*4" y : array - @uniform @binding = 5 ndim : uint - @uniform @binding = 6 ddim : uint - @workgroup gridf : float4[1024] // pre-expanded magnitudes, one float4 per grid word (16 KB) - - [metal_kernel(name="lab_kq_gemv_iq2xs_d1f4_msl")] - def lab_kq_gemv_iq2xs_d1f4 { - let lid = gl_LocalInvocationID.x - for [unroll_full] (gk in range(16)) { - let gi = lid + uint(gk) * 64u - let w = iq2xs_gw(int(gi)) - gridf[gi] = float4(float(w & 255u), float((w >> 8u) & 255u), float((w >> 16u) & 255u), float(w >> 24u)) - } - barrier() - let lane = gl_SubgroupInvocationID - let ix = lane / 8u - let it = lane % 8u - let nb = ndim / 256u - let first_row = (gl_WorkGroupID.x * 2u + gl_SubgroupID) * 2u - var sumf : float[2] - var ib = ix - while (ib < nb) { - let yb4 = ib * 64u + it * 8u - var yv : float4[8] - for [unroll_full] (i in range(8)) { - yv[i] = x[yb4 + uint(i)] - } - for [unroll_full] (r in range(2)) { - let blk = (first_row + uint(r)) * nb + ib - var half_sum : float[2] - for [unroll_full] (h in range(2)) { - let qw = kqu[blk * 16u + it * 2u + uint(h)] - var accv = float4(0.0) - for [unroll_full] (wi in range(2)) { - let w16 = (qw >> (16u * uint(wi))) & 0xFFFFu - let idx2 = (w16 & 511u) * 2u - let sgn = ksign7m(w16 >> 9u) - let s0 = sgn & 15u - let s1 = sgn >> 4u - let m0 = gridf[idx2] - let m1 = gridf[idx2 + 1u] - let f0 = float4((s0 & 1u) != 0u ? -m0.x : m0.x, (s0 & 2u) != 0u ? -m0.y : m0.y, - (s0 & 4u) != 0u ? -m0.z : m0.z, (s0 & 8u) != 0u ? -m0.w : m0.w) - let f1 = float4((s1 & 1u) != 0u ? -m1.x : m1.x, (s1 & 2u) != 0u ? -m1.y : m1.y, - (s1 & 4u) != 0u ? -m1.z : m1.z, (s1 & 8u) != 0u ? -m1.w : m1.w) - accv += yv[h * 4 + wi * 2] * f0 - accv += yv[h * 4 + wi * 2 + 1] * f1 - } - half_sum[h] = float(kscb[blk * 16u + it * 2u + uint(h)]) * (accv.x + accv.y + accv.z + accv.w) - } - sumf[r] += float(kdh[blk]) * (half_sum[0] + half_sum[1]) - } - ib += 4u - } - for [unroll_full] (r in range(2)) { - let s = simd_sum(sumf[r]) - if (lane == 0u && first_row + uint(r) < ddim) { - y[first_row + uint(r)] = s - } - } - } -} - -// iq2s on the d1 map, grid DIRECT off the hoisted constant table (its f4 slab would be 32 KB; -// llama.cpp deliberately skips staging this one too). -class LabKqGemvIq2sD1 { - @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array - @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array - @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array - @ssbo @binding = 3 @role = "read" @off = "xoff" @span = "n*4" x : array - @ssbo @binding = 4 @role = "write" @off = "yoff" @span = "rows*4" y : array - @uniform @binding = 5 ndim : uint - @uniform @binding = 6 ddim : uint - - [metal_kernel(name="lab_kq_gemv_iq2s_d1_msl")] - def lab_kq_gemv_iq2s_d1 { - let lane = gl_SubgroupInvocationID - let ix = lane / 8u - let it = lane % 8u - let nb = ndim / 256u - let first_row = (gl_WorkGroupID.x * 2u + gl_SubgroupID) * 2u - var sumf : float[2] - var ib = ix - while (ib < nb) { - let yb4 = ib * 64u + it * 8u - var yv : float4[8] - for [unroll_full] (i in range(8)) { - yv[i] = x[yb4 + uint(i)] - } - for [unroll_full] (r in range(2)) { - let blk = (first_row + uint(r)) * nb + ib - let qwi = kqu[blk * 18u + it] - let sgw = kqu[blk * 18u + 8u + it] - let qh = (kqu[blk * 18u + 16u + it / 4u] >> ((it % 4u) * 8u)) & 255u - var half_sum : float[2] - for [unroll_full] (h in range(2)) { - var accv = float4(0.0) - for [unroll_full] (li in range(2)) { - let l = uint(h) * 2u + uint(li) - let idx2 = (((qwi >> (8u * l)) & 255u) | ((qh << (8u - 2u * l)) & 0x300u)) * 2u - let sgb = (sgw >> (8u * l)) & 255u - let w0 = iq3s_sw(iq2s_gw(int(idx2)), sgb & 15u) - let w1 = iq3s_sw(iq2s_gw(int(idx2 + 1u)), sgb >> 4u) - accv += yv[l * 2u] * float4(iq3s_sx(w0, 0u), iq3s_sx(w0, 1u), iq3s_sx(w0, 2u), iq3s_sx(w0, 3u)) - accv += yv[l * 2u + 1u] * float4(iq3s_sx(w1, 0u), iq3s_sx(w1, 1u), iq3s_sx(w1, 2u), iq3s_sx(w1, 3u)) - } - half_sum[h] = float(kscb[blk * 16u + it * 2u + uint(h)]) * (accv.x + accv.y + accv.z + accv.w) - } - sumf[r] += float(kdh[blk]) * (half_sum[0] + half_sum[1]) - } - ib += 4u - } - for [unroll_full] (r in range(2)) { - let s = simd_sum(sumf[r]) - if (lane == 0u && first_row + uint(r) < ddim) { - y[first_row + uint(r)] = s - } - } - } -} - -class LabKqGemvIq2xxsD1R4 { - @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array - @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array - @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array - @ssbo @binding = 3 @role = "read" @off = "xoff" @span = "n*4" x : array - @ssbo @binding = 4 @role = "write" @off = "yoff" @span = "rows*4" y : array - @uniform @binding = 5 ndim : uint - @uniform @binding = 6 ddim : uint - - [metal_kernel(name="lab_kq_gemv_iq2xxs_d1r4_msl")] - def lab_kq_gemv_iq2xxs_d1r4 { - let lane = gl_SubgroupInvocationID - let ix = lane / 8u - let it = lane % 8u - let nb = ndim / 256u - let first_row = (gl_WorkGroupID.x * 2u + gl_SubgroupID) * 4u - var sumf : float[4] - var ib = ix - while (ib < nb) { - let yb4 = ib * 64u + it * 8u - var yv : float4[8] - for [unroll_full] (i in range(8)) { - yv[i] = x[yb4 + uint(i)] - } - for [unroll_full] (r in range(4)) { - let blk = (first_row + uint(r)) * nb + ib - let gw = kqu[blk * 16u + it * 2u] - let aux = kqu[blk * 16u + it * 2u + 1u] - var accv = float4(0.0) - for [unroll_full] (q3 in range(4)) { - let bidx = (gw >> (8u * uint(q3))) & 255u - let sgn = ksign7m((aux >> (7u * uint(q3))) & 127u) - let w0 = iq3s_sw(iq2xxs_gw(int(bidx * 2u)), sgn & 15u) - let w1 = iq3s_sw(iq2xxs_gw(int(bidx * 2u + 1u)), sgn >> 4u) - accv += yv[q3 * 2] * float4(iq3s_sx(w0, 0u), iq3s_sx(w0, 1u), iq3s_sx(w0, 2u), iq3s_sx(w0, 3u)) - accv += yv[q3 * 2 + 1] * float4(iq3s_sx(w1, 0u), iq3s_sx(w1, 1u), iq3s_sx(w1, 2u), iq3s_sx(w1, 3u)) - } - let sc = float(kscb[blk * 16u + it]) - sumf[r] += float(kdh[blk]) * sc * (accv.x + accv.y + accv.z + accv.w) - } - ib += 4u - } - for [unroll_full] (r in range(4)) { - let s = simd_sum(sumf[r]) - if (lane == 0u && first_row + uint(r) < ddim) { - y[first_row + uint(r)] = s - } - } - } -} - // Rewrite every threadgroup A-tile subscript store `ta[IDX] = RHS;` into the pointer form // `*(ta + (IDX)) = RHS;` - llama.cpp's mul_mm carries the subscript form commented out as // "massively slower.. WTF?" (mul_mm.metal); this arm races that claim on our own template. diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_common.das b/modules/dasLLAMA/dasllama/dasllama_metal_common.das index 16dfbc0a06..08fa70ad8f 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_common.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_common.das @@ -288,6 +288,7 @@ var g_pso_kq_mvb4_iq2xs : MetalComputePipeline? var g_pso_kq_mvb8_iq2xs : MetalComputePipeline? var g_pso_kq_mm_b24 : MetalComputePipeline? var g_pso_kq_iq2xxs : MetalComputePipeline? +var g_pso_kq_iq2xxs_f4 : MetalComputePipeline? //!< the f4-slab twin, compiled only behind its crown var g_pso_kq_mvb2_iq2xxs : MetalComputePipeline? var g_pso_kq_mvb4_iq2xxs : MetalComputePipeline? var g_pso_kq_mvb8_iq2xxs : MetalComputePipeline? diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das index 32dab01dad..56524c4ffd 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das @@ -3671,6 +3671,9 @@ def metal_decode_init : bool { // nolint:STYLE038 — a flat compile_pso per k g_pso_kq_mvb8_iq2xs = compile_pso(metal_kq_mvb8_iq2xs_msl, metal_kq_mvb8_iq2xs_msl_entry, metal_kq_mvb8_iq2xs_msl_fastmath, ok) g_pso_kq_mm_b24 = compile_pso(MetalKqMulMmIq2xs_metal_kq_mulmm_msl, MetalKqMulMmIq2xs_metal_kq_mulmm_msl_entry, MetalKqMulMmIq2xs_metal_kq_mulmm_msl_fastmath, ok) g_pso_kq_iq2xxs = compile_pso(metal_kq_gemv_iq2xxs_msl, metal_kq_gemv_iq2xxs_msl_entry, metal_kq_gemv_iq2xxs_msl_fastmath, ok) + if (metal_tensor_crowned("kq_gemv_iq2xxs_f4")) { + g_pso_kq_iq2xxs_f4 = compile_pso(metal_kq_gemv_iq2xxs_f4_msl, metal_kq_gemv_iq2xxs_f4_msl_entry, metal_kq_gemv_iq2xxs_f4_msl_fastmath, ok) + } g_pso_kq_mvb2_iq2xxs = compile_pso(MetalKqMvB2Iq2xxs_metal_kq_mv_iq2xxs_msl, MetalKqMvB2Iq2xxs_metal_kq_mv_iq2xxs_msl_entry, MetalKqMvB2Iq2xxs_metal_kq_mv_iq2xxs_msl_fastmath, ok) g_pso_kq_mvb4_iq2xxs = compile_pso(MetalKqMvB4Iq2xxs_metal_kq_mv_iq2xxs_msl, MetalKqMvB4Iq2xxs_metal_kq_mv_iq2xxs_msl_entry, MetalKqMvB4Iq2xxs_metal_kq_mv_iq2xxs_msl_fastmath, ok) g_pso_kq_mvb8_iq2xxs = compile_pso(metal_kq_mvb8_iq2xxs_msl, metal_kq_mvb8_iq2xxs_msl_entry, metal_kq_mvb8_iq2xxs_msl_fastmath, ok) @@ -3764,7 +3767,11 @@ def enc_kq_gemv(enc : MetalComputeEncoder?; fmt : KqFmt; bq : MetalBuffer?; qoff } elif (fmt == KqFmt.iq2xs) { enc_kq_iq2xs_c(enc, bs, doff, bs, soff, bq, qoff, bx, xoff, by, yoff, bn, bd, rows, n) } elif (fmt == KqFmt.iq2xxs) { - enc_kq_iq2xxs_c(enc, bs, doff, bs, soff, bq, qoff, bx, xoff, by, yoff, bn, bd, rows, n) + if (g_pso_kq_iq2xxs_f4 != null) { + enc_kq_iq2xxs_f4_c(enc, bs, doff, bs, soff, bq, qoff, bx, xoff, by, yoff, bn, bd, rows, n) + } else { + enc_kq_iq2xxs_c(enc, bs, doff, bs, soff, bq, qoff, bx, xoff, by, yoff, bn, bd, rows, n) + } } elif (fmt == KqFmt.k6) { enc_kq_k6_c(enc, bs, doff, bs, soff, bq, qoff, bx, xoff, by, yoff, bn, bd, rows, n) } elif (fmt == KqFmt.k5 && (n < 3072l || rows <= 1024l)) { @@ -4381,6 +4388,7 @@ def metal_kernels_release { // nolint:STYLE037,STYLE038 — a flat release_pso release_pso(g_pso_kq_mvb8_iq2xs) release_pso(g_pso_kq_mm_b24) release_pso(g_pso_kq_iq2xxs) + release_pso(g_pso_kq_iq2xxs_f4) release_pso(g_pso_kq_mvb2_iq2xxs) release_pso(g_pso_kq_mvb4_iq2xxs) release_pso(g_pso_kq_mvb8_iq2xxs) @@ -4710,11 +4718,106 @@ def metal_tensor_race_decode : array { 64, kdim, ndim, 64 * ndim, uint3(uint(2 * (ndim / 64)), 1u, 1u), rka) $(enc, slot_y) { kn_buffer(enc, slot_y, 0ul, 4) }) + results |> emplace(race_gemv_iq2xxs_f4(dev, queue)) metal_release(queue) metal_release(dev) return <- results } +//! The iq2xxs GEMV twin race: the base (constant-table) form vs the f4-slab twin. The slab +//! wins on M5-class GPUs and REGRESSES on M1/M4-class - a per-box crown, not a heuristic. +//! Winner "tensor" = the twin takes the "kq_gemv_iq2xxs_f4" crown (the absorb convention). +def private race_gemv_iq2xxs_f4(dev : MetalDevice?; queue : MetalCommandQueue?) : MetalTensorRaceResult { // nolint:STYLE038 - one-shot A/B race harness; buffers + PSOs stay live to the release tail + var res = MetalTensorRaceResult(family = "kq_gemv_iq2xxs_f4", winner = "", base_ms = -1.0lf, twin_ms = -1.0lf, note = "") + let kdim = 3072 + let ndim = 4096 + let nsb = kdim / 256 + let nblk = ndim * nsb + var err = "" + var base_pso = pipeline_from_source(dev, metal_kq_gemv_iq2xxs_msl, metal_kq_gemv_iq2xxs_msl_entry, metal_kq_gemv_iq2xxs_msl_fastmath, err) + if (base_pso == null) { + res.note = "base pso: {err}" + return res + } + var twin_pso = pipeline_from_source(dev, metal_kq_gemv_iq2xxs_f4_msl, metal_kq_gemv_iq2xxs_f4_msl_entry, metal_kq_gemv_iq2xxs_f4_msl_fastmath, err) + if (twin_pso == null) { + res.winner = "simdgroup" + res.note = "twin pso: {err}" + metal_release(base_pso) + return res + } + var qplane : array + qplane |> resize(nblk * 16) + for (i in range(nblk * 16)) { + qplane[i] = uint(i) * 2654435761u + } + var splane : array + splane |> resize(nblk * 18) + for (i in range(nblk * 16)) { + splane[i] = uint8(2) + } + unsafe { + var ph = addr(splane[nblk * 16]) + for (b in range(nblk)) { + ph[b] = float16(0.25) + } + } + var xa <- race_x_f32(kdim) + var bq = race_buf(dev, uint64(nblk * 64), unsafe(addr(qplane[0]))) + var bs = race_buf(dev, uint64(nblk * 18), unsafe(addr(splane[0]))) + var bxa = race_buf(dev, uint64(kdim * 4), unsafe(addr(xa[0]))) + var by_base = race_buf(dev, uint64(ndim * 4), null) + var by_twin = race_buf(dev, uint64(ndim * 4), null) + var bn = race_uniform_u32(dev, uint(kdim)) + var bd = race_uniform_u32(dev, uint(ndim)) + let doff = uint64(nblk * 16) + let grid = uint3(uint(ndim / 4), 1u, 1u) + let tg = uint3(64u, 1u, 1u) + race_pair_ms(queue, 5, res.base_ms, res.twin_ms, $(enc) { + kn_pipeline(enc, base_pso) + kn_buffer(enc, bs, doff, 0) + kn_buffer(enc, bs, 0ul, 1) + kn_buffer(enc, bq, 0ul, 2) + kn_buffer(enc, bxa, 0ul, 3) + kn_buffer(enc, by_base, 0ul, 4) + kn_buffer(enc, bn, 0ul, 5) + kn_buffer(enc, bd, 0ul, 6) + kn_dispatch(enc, grid, tg) + }, $(enc) { + kn_pipeline(enc, twin_pso) + kn_tgmem(enc, metal_kq_gemv_iq2xxs_f4_msl_tgmem, 0) + kn_buffer(enc, bs, doff, 0) + kn_buffer(enc, bs, 0ul, 1) + kn_buffer(enc, bq, 0ul, 2) + kn_buffer(enc, bxa, 0ul, 3) + kn_buffer(enc, by_twin, 0ul, 4) + kn_buffer(enc, bn, 0ul, 5) + kn_buffer(enc, bd, 0ul, 6) + kn_dispatch(enc, grid, tg) + }) + if (res.base_ms < 0.0lf || res.twin_ms < 0.0lf) { + res.winner = res.base_ms < 0.0lf ? "" : "simdgroup" + res.note = "dispatch failed" + } elif (!race_envelope_ok(by_base, by_twin, ndim, res.note)) { + res.winner = "simdgroup" + } else { + res.winner = res.twin_ms < res.base_ms ? "tensor" : "simdgroup" + } + metal_release(bq) + metal_release(bs) + metal_release(bxa) + metal_release(by_base) + metal_release(by_twin) + metal_release(bn) + metal_release(bd) + metal_release(base_pso) + metal_release(twin_pso) + delete qplane + delete splane + delete xa + return res +} + // ===== family-shared kernel classes (M1: moved from prefill) ===== // One threadgroup per row: cooperative sum of squares, lane 0 folds rsqrt(mean + eps), every @@ -8580,7 +8683,7 @@ class MetalKqGemvIq3s { // IQ2_S: the iq3s GEMV geometry (4 rows per simdgroup, dispatch rows/8) over the u64 grid read // DIRECT off the hoisted constant table; per-16 UNSIGNED strips scale each 16-element half, // d pre-eighth-ed at transcode - the fold is d8 * strip * signed_grid_byte. -[metal_dispatch(name = "enc_kq_iq2s_c", pso = "g_pso_kq_iq2s", tg = 64, grid = "rows/8", params = "rows : int64, n : int64")] +[metal_dispatch(name = "enc_kq_iq2s_c", pso = "g_pso_kq_iq2s", tg = 64, grid = "rows/4", params = "rows : int64, n : int64")] class MetalKqGemvIq2s { @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d8 plane - the scale buffer bound at byte nsb*16 (s0off = the caller's doff) @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // 16B strips: 16 UNSIGNED per-16 sub-scales, byte view (same buffer at 0) @@ -8593,43 +8696,42 @@ class MetalKqGemvIq2s { [metal_kernel(name="metal_kq_gemv_iq2s_msl")] def metal_kq_gemv_iq2s { let lane = gl_SubgroupInvocationID - let ix = lane / 16u - let it = lane % 16u - let bu = it / 2u - let il = it % 2u + let ix = lane / 8u + let it = lane % 8u let nb = ndim / 256u - let first_row = (gl_WorkGroupID.x * 2u + gl_SubgroupID) * 4u - var sumf : float[4] + let first_row = (gl_WorkGroupID.x * 2u + gl_SubgroupID) * 2u + var sumf : float[2] var ib = ix while (ib < nb) { - let yb4 = ib * 64u + bu * 8u + il * 4u - let y0 = x[yb4] - let y1 = x[yb4 + 1u] - let y2 = x[yb4 + 2u] - let y3 = x[yb4 + 3u] - for [unroll_full] (r in range(4)) { + let yb4 = ib * 64u + it * 8u + var yv : float4[8] + for [unroll_full] (i in range(8)) { + yv[i] = x[yb4 + uint(i)] + } + for [unroll_full] (r in range(2)) { let blk = (first_row + uint(r)) * nb + ib - let qw = kqu[blk * 18u + bu] - let sgw = kqu[blk * 18u + 8u + bu] - let qh = (kqu[blk * 18u + 16u + bu / 4u] >> ((bu % 4u) * 8u)) & 255u - var accv = float4(0.0) - for [unroll_full] (wi in range(2)) { - let l = il * 2u + uint(wi) - let idx2 = (((qw >> (8u * l)) & 255u) | ((qh << (8u - 2u * l)) & 0x300u)) * 2u - let sgn = (sgw >> (8u * l)) & 255u - let w0 = iq3s_sw(iq2s_gw(int(idx2)), sgn & 15u) - let w1 = iq3s_sw(iq2s_gw(int(idx2 + 1u)), sgn >> 4u) - let ya = wi == 0 ? y0 : y2 - let yb = wi == 0 ? y1 : y3 - accv += ya * float4(iq3s_sx(w0, 0u), iq3s_sx(w0, 1u), iq3s_sx(w0, 2u), iq3s_sx(w0, 3u)) - accv += yb * float4(iq3s_sx(w1, 0u), iq3s_sx(w1, 1u), iq3s_sx(w1, 2u), iq3s_sx(w1, 3u)) + let qwi = kqu[blk * 18u + it] + let sgw = kqu[blk * 18u + 8u + it] + let qh = (kqu[blk * 18u + 16u + it / 4u] >> ((it % 4u) * 8u)) & 255u + var half_sum : float[2] + for [unroll_full] (h in range(2)) { + var accv = float4(0.0) + for [unroll_full] (li in range(2)) { + let l = uint(h) * 2u + uint(li) + let idx2 = (((qwi >> (8u * l)) & 255u) | ((qh << (8u - 2u * l)) & 0x300u)) * 2u + let sgb = (sgw >> (8u * l)) & 255u + let w0 = iq3s_sw(iq2s_gw(int(idx2)), sgb & 15u) + let w1 = iq3s_sw(iq2s_gw(int(idx2 + 1u)), sgb >> 4u) + accv += yv[l * 2u] * float4(iq3s_sx(w0, 0u), iq3s_sx(w0, 1u), iq3s_sx(w0, 2u), iq3s_sx(w0, 3u)) + accv += yv[l * 2u + 1u] * float4(iq3s_sx(w1, 0u), iq3s_sx(w1, 1u), iq3s_sx(w1, 2u), iq3s_sx(w1, 3u)) + } + half_sum[h] = float(kscb[blk * 16u + it * 2u + uint(h)]) * (accv.x + accv.y + accv.z + accv.w) } - let sc = float(kscb[blk * 16u + bu * 2u + il]) - sumf[r] += float(kdh[blk]) * sc * (accv.x + accv.y + accv.z + accv.w) + sumf[r] += float(kdh[blk]) * (half_sum[0] + half_sum[1]) } - ib += 2u + ib += 4u } - for [unroll_full] (r in range(4)) { + for [unroll_full] (r in range(2)) { let s = simd_sum(sumf[r]) if (lane == 0u && first_row + uint(r) < ddim) { y[first_row + uint(r)] = s @@ -8641,7 +8743,7 @@ class MetalKqGemvIq2s { // IQ2_XS: the iq2s GEMV geometry over the parity-signed u16 words - per (block, half) one u32 // carries two u16 qs words; 9-bit index doubles into the grid pair, ksign7m recomputes the // sign byte (no sign plane, no qh); per-16 UNSIGNED strips, d pre-eighth-ed. -[metal_dispatch(name = "enc_kq_iq2xs_c", pso = "g_pso_kq_iq2xs", tg = 64, grid = "rows/8", params = "rows : int64, n : int64")] +[metal_dispatch(name = "enc_kq_iq2xs_c", pso = "g_pso_kq_iq2xs", tgmem = "metal_kq_gemv_iq2xs_msl_tgmem", tg = 64, grid = "rows/4", params = "rows : int64, n : int64")] class MetalKqGemvIq2xs { @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d8 plane - the scale buffer bound at byte nsb*16 (s0off = the caller's doff) @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // 16B strips: 16 UNSIGNED per-16 sub-scales, byte view (same buffer at 0) @@ -8650,45 +8752,58 @@ class MetalKqGemvIq2xs { @ssbo @binding = 4 @role = "write" @off = "yoff" @span = "rows*4" y : array @uniform @binding = 5 ndim : uint @uniform @binding = 6 ddim : uint + @workgroup gridf : float4[1024] // pre-expanded magnitudes, one float4 per grid word (16 KB) [metal_kernel(name="metal_kq_gemv_iq2xs_msl")] def metal_kq_gemv_iq2xs { + let lid = gl_LocalInvocationID.x + for [unroll_full] (gk in range(16)) { + let gi = lid + uint(gk) * 64u + let w = iq2xs_gw(int(gi)) + gridf[gi] = float4(float(w & 255u), float((w >> 8u) & 255u), float((w >> 16u) & 255u), float(w >> 24u)) + } + barrier() let lane = gl_SubgroupInvocationID - let ix = lane / 16u - let it = lane % 16u - let bu = it / 2u - let il = it % 2u + let ix = lane / 8u + let it = lane % 8u let nb = ndim / 256u - let first_row = (gl_WorkGroupID.x * 2u + gl_SubgroupID) * 4u - var sumf : float[4] + let first_row = (gl_WorkGroupID.x * 2u + gl_SubgroupID) * 2u + var sumf : float[2] var ib = ix while (ib < nb) { - let yb4 = ib * 64u + bu * 8u + il * 4u - let y0 = x[yb4] - let y1 = x[yb4 + 1u] - let y2 = x[yb4 + 2u] - let y3 = x[yb4 + 3u] - for [unroll_full] (r in range(4)) { + let yb4 = ib * 64u + it * 8u + var yv : float4[8] + for [unroll_full] (i in range(8)) { + yv[i] = x[yb4 + uint(i)] + } + for [unroll_full] (r in range(2)) { let blk = (first_row + uint(r)) * nb + ib - let qw = kqu[blk * 16u + bu * 2u + il] - var accv = float4(0.0) - for [unroll_full] (wi in range(2)) { - let w16 = (qw >> (16u * uint(wi))) & 0xFFFFu - let idx2 = (w16 & 511u) * 2u - let sgn = ksign7m(w16 >> 9u) - let w0 = iq3s_sw(iq2xs_gw(int(idx2)), sgn & 15u) - let w1 = iq3s_sw(iq2xs_gw(int(idx2 + 1u)), sgn >> 4u) - let ya = wi == 0 ? y0 : y2 - let yb = wi == 0 ? y1 : y3 - accv += ya * float4(iq3s_sx(w0, 0u), iq3s_sx(w0, 1u), iq3s_sx(w0, 2u), iq3s_sx(w0, 3u)) - accv += yb * float4(iq3s_sx(w1, 0u), iq3s_sx(w1, 1u), iq3s_sx(w1, 2u), iq3s_sx(w1, 3u)) + var half_sum : float[2] + for [unroll_full] (h in range(2)) { + let qw = kqu[blk * 16u + it * 2u + uint(h)] + var accv = float4(0.0) + for [unroll_full] (wi in range(2)) { + let w16 = (qw >> (16u * uint(wi))) & 0xFFFFu + let idx2 = (w16 & 511u) * 2u + let sgn = ksign7m(w16 >> 9u) + let s0 = sgn & 15u + let s1 = sgn >> 4u + let m0 = gridf[idx2] + let m1 = gridf[idx2 + 1u] + let f0 = float4((s0 & 1u) != 0u ? -m0.x : m0.x, (s0 & 2u) != 0u ? -m0.y : m0.y, + (s0 & 4u) != 0u ? -m0.z : m0.z, (s0 & 8u) != 0u ? -m0.w : m0.w) + let f1 = float4((s1 & 1u) != 0u ? -m1.x : m1.x, (s1 & 2u) != 0u ? -m1.y : m1.y, + (s1 & 4u) != 0u ? -m1.z : m1.z, (s1 & 8u) != 0u ? -m1.w : m1.w) + accv += yv[h * 4 + wi * 2] * f0 + accv += yv[h * 4 + wi * 2 + 1] * f1 + } + half_sum[h] = float(kscb[blk * 16u + it * 2u + uint(h)]) * (accv.x + accv.y + accv.z + accv.w) } - let sc = float(kscb[blk * 16u + bu * 2u + il]) - sumf[r] += float(kdh[blk]) * sc * (accv.x + accv.y + accv.z + accv.w) + sumf[r] += float(kdh[blk]) * (half_sum[0] + half_sum[1]) } - ib += 2u + ib += 4u } - for [unroll_full] (r in range(4)) { + for [unroll_full] (r in range(2)) { let s = simd_sum(sumf[r]) if (lane == 0u && first_row + uint(r) < ddim) { y[first_row + uint(r)] = s @@ -8700,7 +8815,7 @@ class MetalKqGemvIq2xs { // IQ2_XXS: the iq2xs GEMV geometry over the aux32 form - per (block, half) the block's grid // word supplies two BYTE indices (each doubling into the u64 grid pair), its aux32 the // parity ksigns; ONE per-32 UNSIGNED strip scales the whole block on the eighth-ed d. -[metal_dispatch(name = "enc_kq_iq2xxs_c", pso = "g_pso_kq_iq2xxs", tg = 64, grid = "rows/8", params = "rows : int64, n : int64")] +[metal_dispatch(name = "enc_kq_iq2xxs_c", pso = "g_pso_kq_iq2xxs", tg = 64, grid = "rows/4", params = "rows : int64, n : int64")] class MetalKqGemvIq2xxs { @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d8 plane - the scale buffer bound at byte nsb*16 (s0off = the caller's doff) @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // strip region: 8 UNSIGNED per-32 sub-scales + 8 pad, byte view (same buffer at 0) @@ -8713,42 +8828,106 @@ class MetalKqGemvIq2xxs { [metal_kernel(name="metal_kq_gemv_iq2xxs_msl")] def metal_kq_gemv_iq2xxs { let lane = gl_SubgroupInvocationID - let ix = lane / 16u - let it = lane % 16u - let bu = it / 2u - let il = it % 2u + let ix = lane / 8u + let it = lane % 8u let nb = ndim / 256u - let first_row = (gl_WorkGroupID.x * 2u + gl_SubgroupID) * 4u - var sumf : float[4] + let first_row = (gl_WorkGroupID.x * 2u + gl_SubgroupID) * 2u + var sumf : float[2] var ib = ix while (ib < nb) { - let yb4 = ib * 64u + bu * 8u + il * 4u - let y0 = x[yb4] - let y1 = x[yb4 + 1u] - let y2 = x[yb4 + 2u] - let y3 = x[yb4 + 3u] - for [unroll_full] (r in range(4)) { + let yb4 = ib * 64u + it * 8u + var yv : float4[8] + for [unroll_full] (i in range(8)) { + yv[i] = x[yb4 + uint(i)] + } + for [unroll_full] (r in range(2)) { let blk = (first_row + uint(r)) * nb + ib - let gw = kqu[blk * 16u + bu * 2u] - let aux = kqu[blk * 16u + bu * 2u + 1u] + let gw = kqu[blk * 16u + it * 2u] + let aux = kqu[blk * 16u + it * 2u + 1u] var accv = float4(0.0) - for [unroll_full] (wi in range(2)) { - let q3 = il * 2u + uint(wi) - let bidx = (gw >> (8u * q3)) & 255u - let sgn = ksign7m((aux >> (7u * q3)) & 127u) + for [unroll_full] (q3 in range(4)) { + let bidx = (gw >> (8u * uint(q3))) & 255u + let sgn = ksign7m((aux >> (7u * uint(q3))) & 127u) let w0 = iq3s_sw(iq2xxs_gw(int(bidx * 2u)), sgn & 15u) let w1 = iq3s_sw(iq2xxs_gw(int(bidx * 2u + 1u)), sgn >> 4u) - let ya = wi == 0 ? y0 : y2 - let yb = wi == 0 ? y1 : y3 - accv += ya * float4(iq3s_sx(w0, 0u), iq3s_sx(w0, 1u), iq3s_sx(w0, 2u), iq3s_sx(w0, 3u)) - accv += yb * float4(iq3s_sx(w1, 0u), iq3s_sx(w1, 1u), iq3s_sx(w1, 2u), iq3s_sx(w1, 3u)) + accv += yv[q3 * 2] * float4(iq3s_sx(w0, 0u), iq3s_sx(w0, 1u), iq3s_sx(w0, 2u), iq3s_sx(w0, 3u)) + accv += yv[q3 * 2 + 1] * float4(iq3s_sx(w1, 0u), iq3s_sx(w1, 1u), iq3s_sx(w1, 2u), iq3s_sx(w1, 3u)) } - let sc = float(kscb[blk * 16u + bu]) + let sc = float(kscb[blk * 16u + it]) sumf[r] += float(kdh[blk]) * sc * (accv.x + accv.y + accv.z + accv.w) } - ib += 2u + ib += 4u } - for [unroll_full] (r in range(4)) { + for [unroll_full] (r in range(2)) { + let s = simd_sum(sumf[r]) + if (lane == 0u && first_row + uint(r) < ddim) { + y[first_row + uint(r)] = s + } + } + } +} + +// The f4-slab twin: the same lane map with the 2 KB grid pre-expanded to float4 magnitudes +// (8 KB threadgroup) and signs applied by select - crowns "kq_gemv_iq2xxs_f4" where the box +// race prefers it (M5-class GPUs; M1/M4-class regress on the slab). +[metal_dispatch(name = "enc_kq_iq2xxs_f4_c", pso = "g_pso_kq_iq2xxs_f4", tgmem = "metal_kq_gemv_iq2xxs_f4_msl_tgmem", tg = 64, grid = "rows/4", params = "rows : int64, n : int64")] +class MetalKqGemvIq2xxsF4 { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d8 plane - the scale buffer bound at byte nsb*16 (s0off = the caller's doff) + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // strip region: 8 UNSIGNED per-32 sub-scales + 8 pad, byte view (same buffer at 0) + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array // iq2xxs quant plane, uint view (16 words per superblock: [grid bytes][aux32] per block) + @ssbo @binding = 3 @role = "read" @off = "xoff" @span = "n*4" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" @span = "rows*4" y : array + @uniform @binding = 5 ndim : uint + @uniform @binding = 6 ddim : uint + @workgroup gridf : float4[512] // pre-expanded magnitudes, one float4 per grid word (8 KB) + + [metal_kernel(name="metal_kq_gemv_iq2xxs_f4_msl")] + def metal_kq_gemv_iq2xxs_f4 { + let lid = gl_LocalInvocationID.x + for [unroll_full] (gk in range(8)) { + let gi = lid + uint(gk) * 64u + let w = iq2xxs_gw(int(gi)) + gridf[gi] = float4(float(w & 255u), float((w >> 8u) & 255u), float((w >> 16u) & 255u), float(w >> 24u)) + } + barrier() + let lane = gl_SubgroupInvocationID + let ix = lane / 8u + let it = lane % 8u + let nb = ndim / 256u + let first_row = (gl_WorkGroupID.x * 2u + gl_SubgroupID) * 2u + var sumf : float[2] + var ib = ix + while (ib < nb) { + let yb4 = ib * 64u + it * 8u + var yv : float4[8] + for [unroll_full] (i in range(8)) { + yv[i] = x[yb4 + uint(i)] + } + for [unroll_full] (r in range(2)) { + let blk = (first_row + uint(r)) * nb + ib + let gw = kqu[blk * 16u + it * 2u] + let aux = kqu[blk * 16u + it * 2u + 1u] + var accv = float4(0.0) + for [unroll_full] (q3 in range(4)) { + let bidx = (gw >> (8u * uint(q3))) & 255u + let sgn = ksign7m((aux >> (7u * uint(q3))) & 127u) + let s0 = sgn & 15u + let s1 = sgn >> 4u + let m0 = gridf[bidx * 2u] + let m1 = gridf[bidx * 2u + 1u] + let f0 = float4((s0 & 1u) != 0u ? -m0.x : m0.x, (s0 & 2u) != 0u ? -m0.y : m0.y, + (s0 & 4u) != 0u ? -m0.z : m0.z, (s0 & 8u) != 0u ? -m0.w : m0.w) + let f1 = float4((s1 & 1u) != 0u ? -m1.x : m1.x, (s1 & 2u) != 0u ? -m1.y : m1.y, + (s1 & 4u) != 0u ? -m1.z : m1.z, (s1 & 8u) != 0u ? -m1.w : m1.w) + accv += yv[q3 * 2] * f0 + accv += yv[q3 * 2 + 1] * f1 + } + let sc = float(kscb[blk * 16u + it]) + sumf[r] += float(kdh[blk]) * sc * (accv.x + accv.y + accv.z + accv.w) + } + ib += 4u + } + for [unroll_full] (r in range(2)) { let s = simd_sum(sumf[r]) if (lane == 0u && first_row + uint(r) < ddim) { y[first_row + uint(r)] = s diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das b/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das index 2775ff9188..92917e5b41 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das @@ -5350,7 +5350,11 @@ def private pf_enc_kq_gemv(enc : MetalComputeEncoder?; t : Model; fmt : KqFmt; w } elif (fmt == KqFmt.iq2xs) { enc_kq_iq2xs_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, xoff, by, 0ul, bn, bd, rows, n) } elif (fmt == KqFmt.iq2xxs) { - enc_kq_iq2xxs_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, xoff, by, 0ul, bn, bd, rows, n) + if (g_pso_kq_iq2xxs_f4 != null) { + enc_kq_iq2xxs_f4_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, xoff, by, 0ul, bn, bd, rows, n) + } else { + enc_kq_iq2xxs_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, xoff, by, 0ul, bn, bd, rows, n) + } } elif (fmt == KqFmt.iq4nl) { enc_kq_iq4nl_c(enc, bs.buf, bs.soff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, xoff, by, 0ul, bn, bd, rows, n) } elif (fmt == KqFmt.k2) { diff --git a/modules/dasLLAMA/tests/test_metal_gemv_kernels.das b/modules/dasLLAMA/tests/test_metal_gemv_kernels.das index c97620e11c..6c5e41fd85 100644 --- a/modules/dasLLAMA/tests/test_metal_gemv_kernels.das +++ b/modules/dasLLAMA/tests/test_metal_gemv_kernels.das @@ -23,12 +23,12 @@ require math // ===== single-stream kq GEMV (MetalKqGemvK4 / K5 / K5C / K6) ===== def private kq_gemv_gate(t : T?; dev, queue; vform : string; n, d : int) { - let fmt = vform == "k4" ? 4 : (vform == "k6" ? 6 : (vform == "iq4xs" ? 44 : (vform == "k3" ? 3 : (vform == "iq3s" ? 33 : (vform == "iq3xxs" ? 34 : (vform == "iq4nl" ? 45 : (vform == "iq2s" ? 23 : (vform == "iq2xs" ? 24 : (vform == "iq2xxs" ? 25 : (vform == "k2" ? 2 : 5)))))))))) + let fmt = vform == "k4" ? 4 : (vform == "k6" ? 6 : (vform == "iq4xs" ? 44 : (vform == "k3" ? 3 : (vform == "iq3s" ? 33 : (vform == "iq3xxs" ? 34 : (vform == "iq4nl" ? 45 : (vform == "iq2s" ? 23 : (vform == "iq2xs" ? 24 : (vform == "iq2xxs" || vform == "iq2xxsf4" ? 25 : (vform == "k2" ? 2 : 5)))))))))) let tag = "kq_gemv_{vform} n={n} d={d}" var err : string - let src = vform == "iq2xxs" ? metal_kq_gemv_iq2xxs_msl : (vform == "iq2xs" ? metal_kq_gemv_iq2xs_msl : (vform == "iq2s" ? metal_kq_gemv_iq2s_msl : (vform == "k2" ? metal_kq_gemv_k2_msl : (vform == "iq4nl" ? metal_kq_gemv_iq4nl_msl : (vform == "iq3xxs" ? metal_kq_gemv_iq3xxs_msl : (vform == "iq3s" ? metal_kq_gemv_iq3s_msl : (vform == "k3" ? metal_kq_gemv_k3_msl : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl : (vform == "k4" ? metal_kq_gemv_k4_msl : (vform == "k5" ? metal_kq_gemv_k5_msl : (vform == "k5c" ? metal_kq_gemv_k5c_msl : metal_kq_gemv_k6_msl))))))))))) - let entry = vform == "iq2xxs" ? metal_kq_gemv_iq2xxs_msl_entry : (vform == "iq2xs" ? metal_kq_gemv_iq2xs_msl_entry : (vform == "iq2s" ? metal_kq_gemv_iq2s_msl_entry : (vform == "k2" ? metal_kq_gemv_k2_msl_entry : (vform == "iq4nl" ? metal_kq_gemv_iq4nl_msl_entry : (vform == "iq3xxs" ? metal_kq_gemv_iq3xxs_msl_entry : (vform == "iq3s" ? metal_kq_gemv_iq3s_msl_entry : (vform == "k3" ? metal_kq_gemv_k3_msl_entry : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl_entry : (vform == "k4" ? metal_kq_gemv_k4_msl_entry : (vform == "k5" ? metal_kq_gemv_k5_msl_entry : (vform == "k5c" ? metal_kq_gemv_k5c_msl_entry : metal_kq_gemv_k6_msl_entry))))))))))) - let fm = vform == "iq2xxs" ? metal_kq_gemv_iq2xxs_msl_fastmath : (vform == "iq2xs" ? metal_kq_gemv_iq2xs_msl_fastmath : (vform == "iq2s" ? metal_kq_gemv_iq2s_msl_fastmath : (vform == "k2" ? metal_kq_gemv_k2_msl_fastmath : (vform == "iq4nl" ? metal_kq_gemv_iq4nl_msl_fastmath : (vform == "iq3xxs" ? metal_kq_gemv_iq3xxs_msl_fastmath : (vform == "iq3s" ? metal_kq_gemv_iq3s_msl_fastmath : (vform == "k3" ? metal_kq_gemv_k3_msl_fastmath : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl_fastmath : (vform == "k4" ? metal_kq_gemv_k4_msl_fastmath : (vform == "k5" ? metal_kq_gemv_k5_msl_fastmath : (vform == "k5c" ? metal_kq_gemv_k5c_msl_fastmath : metal_kq_gemv_k6_msl_fastmath))))))))))) + let src = vform == "iq2xxsf4" ? metal_kq_gemv_iq2xxs_f4_msl : (vform == "iq2xxs" ? metal_kq_gemv_iq2xxs_msl : (vform == "iq2xs" ? metal_kq_gemv_iq2xs_msl : (vform == "iq2s" ? metal_kq_gemv_iq2s_msl : (vform == "k2" ? metal_kq_gemv_k2_msl : (vform == "iq4nl" ? metal_kq_gemv_iq4nl_msl : (vform == "iq3xxs" ? metal_kq_gemv_iq3xxs_msl : (vform == "iq3s" ? metal_kq_gemv_iq3s_msl : (vform == "k3" ? metal_kq_gemv_k3_msl : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl : (vform == "k4" ? metal_kq_gemv_k4_msl : (vform == "k5" ? metal_kq_gemv_k5_msl : (vform == "k5c" ? metal_kq_gemv_k5c_msl : metal_kq_gemv_k6_msl)))))))))))) + let entry = vform == "iq2xxsf4" ? metal_kq_gemv_iq2xxs_f4_msl_entry : (vform == "iq2xxs" ? metal_kq_gemv_iq2xxs_msl_entry : (vform == "iq2xs" ? metal_kq_gemv_iq2xs_msl_entry : (vform == "iq2s" ? metal_kq_gemv_iq2s_msl_entry : (vform == "k2" ? metal_kq_gemv_k2_msl_entry : (vform == "iq4nl" ? metal_kq_gemv_iq4nl_msl_entry : (vform == "iq3xxs" ? metal_kq_gemv_iq3xxs_msl_entry : (vform == "iq3s" ? metal_kq_gemv_iq3s_msl_entry : (vform == "k3" ? metal_kq_gemv_k3_msl_entry : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl_entry : (vform == "k4" ? metal_kq_gemv_k4_msl_entry : (vform == "k5" ? metal_kq_gemv_k5_msl_entry : (vform == "k5c" ? metal_kq_gemv_k5c_msl_entry : metal_kq_gemv_k6_msl_entry)))))))))))) + let fm = vform == "iq2xxsf4" ? metal_kq_gemv_iq2xxs_f4_msl_fastmath : (vform == "iq2xxs" ? metal_kq_gemv_iq2xxs_msl_fastmath : (vform == "iq2xs" ? metal_kq_gemv_iq2xs_msl_fastmath : (vform == "iq2s" ? metal_kq_gemv_iq2s_msl_fastmath : (vform == "k2" ? metal_kq_gemv_k2_msl_fastmath : (vform == "iq4nl" ? metal_kq_gemv_iq4nl_msl_fastmath : (vform == "iq3xxs" ? metal_kq_gemv_iq3xxs_msl_fastmath : (vform == "iq3s" ? metal_kq_gemv_iq3s_msl_fastmath : (vform == "k3" ? metal_kq_gemv_k3_msl_fastmath : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl_fastmath : (vform == "k4" ? metal_kq_gemv_k4_msl_fastmath : (vform == "k5" ? metal_kq_gemv_k5_msl_fastmath : (vform == "k5c" ? metal_kq_gemv_k5c_msl_fastmath : metal_kq_gemv_k6_msl_fastmath)))))))))))) var pso = pipeline_from_source(dev, src, entry, fm, err) t |> success(pso != null, "{tag}: pipeline: {err}") return if (pso == null) @@ -66,7 +66,7 @@ def private kq_gemv_gate(t : T?; dev, queue; vform : string; n, d : int) { var by = buf_fill(dev, d, -1000.0) var bn = buf_u32(dev, uint(n)) var bd = buf_u32(dev, uint(d)) - let groups = vform == "k5c" ? (d + 1) / 2 : (vform == "iq3s" || vform == "iq3xxs" || vform == "iq2s" || vform == "iq2xs" || vform == "iq2xxs" ? (d + 7) / 8 : (d + 3) / 4) + let groups = vform == "k5c" ? (d + 1) / 2 : (vform == "iq3s" || vform == "iq3xxs" ? (d + 7) / 8 : (d + 3) / 4) let ran = with_compute_encoder(queue, err) $(enc : MetalComputeEncoder?) { metal_set_pipeline(enc, pso) if (vform == "iq4xs") { @@ -81,6 +81,12 @@ def private kq_gemv_gate(t : T?; dev, queue; vform : string; n, d : int) { if (vform == "iq4nl") { metal_set_threadgroup_memory_length(enc, metal_kq_gemv_iq4nl_msl_tgmem, 0) // the codebook slab } + if (vform == "iq2xs") { + metal_set_threadgroup_memory_length(enc, metal_kq_gemv_iq2xs_msl_tgmem, 0) // the f4 magnitude slab + } + if (vform == "iq2xxsf4") { + metal_set_threadgroup_memory_length(enc, metal_kq_gemv_iq2xxs_f4_msl_tgmem, 0) // the f4 magnitude slab + } metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 || fmt == 24 || fmt == 25 ? uint64(nsb * 16) : 0ul, 0) // the split-scale formats: the f16 d tail at nsb*16 metal_set_buffer(enc, bks, 0ul, 1) metal_set_buffer(enc, bkq, 0ul, 2) @@ -751,6 +757,8 @@ def test_metal_kq_gemv_kernels(t : T?) { kq_gemv_gate(t, dev, queue, "iq2xs", 512, 30) kq_gemv_gate(t, dev, queue, "iq2xxs", 1280, 11) kq_gemv_gate(t, dev, queue, "iq2xxs", 512, 30) + kq_gemv_gate(t, dev, queue, "iq2xxsf4", 1280, 11) + kq_gemv_gate(t, dev, queue, "iq2xxsf4", 512, 30) // the small-batch twins: nr = 2 / 3 (b4 col guard) / 6 (b8 pad), ys > d for (fmt in [4, 5, 6, 44, 3, 33, 34, 45, 2, 23, 24, 25]) { kq_mvb_gate(t, dev, queue, fmt, 2, 768, 17, 2, 22) From a40edcc26fe425967dc128fbacbda90940d43043 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 01:20:28 -0700 Subject: [PATCH 07/35] kq race lab: per-cell GPU burn (--burn-ms, default 150) - the idle-clock governor artifact A sparse invocation's encoders start at sunk clocks (the CPU-side oracle gaps between cells let the governor race to idle; a 50-dispatch encoder finishes before it ramps) and read 2-3x slow - the dense full sweep never idles, which is why the baseline was clean and every short rerun read worse. Not thermal: no pmset event, and shorter runs degraded more. The burn spends ~150 ms of GPU time on arm 0 before each cell's timing; a sparse k4 control returns to 1.03x. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014mq9AKyoAwvD5T1cigTnRw --- .../benchmarks/matmul/bench_metal_kq_race.das | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das b/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das index a5f3fcf916..1faf89a8d8 100644 --- a/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das +++ b/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das @@ -53,6 +53,9 @@ struct Args { @clarg_doc = "Dispatches per timed encoder" reps : int = 50 + @clarg_doc = "GPU burn before each cell's timing, in ms - ramps the idle-clock governor (a sparse run's encoders otherwise start at sunk clocks and read 2-3x slow; the dense full sweep never idles)" + burn_ms : int = 150 + @clarg_doc = "Pinned llama.cpp reference table (tsv: fmt tier shape us_per_dispatch)" ref : string @@ -366,6 +369,22 @@ def private race_cell(queue, dev; f : KqRaceFmt; tier, shname : string; n, d : i lv = false } } + // clock ramp: burn GPU time on arm 0 until the budget is spent, so the first timed + // encoder runs at sustained clocks even in a sparse (few-cell) invocation + var burned = 0.0lf + while (burned < double(args.burn_ms) * 1000.0lf) { + var did = false + for (arm, lv, pso, by in arms, live, psos, bys) { + continue if (!lv) + let groups = is_mm ? uint3(uint(PREFILL_M / 32), uint(d / 64), 1u) : uint3(uint((d + arm.gdiv - 1) / arm.gdiv), 1u, 1u) + let us = time_arm_us(queue, pso, arm, bks, bkq, bx, by, b5, b6, sc_off, groups, is_mm ? 128 : 64, args.reps) + break if (us < 0.0lf) + burned += us * double(args.reps) + did = true + break + } + break if (!did) + } // interleaved timing: round-robin per round, best-of (round 0 discarded as warmup) var best : array best |> resize(length(arms)) From 7e7c24d38ff52ef2e295e8e98872a7009acda3ea Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 01:24:29 -0700 Subject: [PATCH 08/35] kq race lab: the k2 d1 arm - llama.cpp's q2_K shape (u16 shift-free masks, prefolds, rows/8) on our planes; M5 1.27-1.44x over prod -> 1.16-1.20x vs lcpp Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014mq9AKyoAwvD5T1cigTnRw --- .../benchmarks/matmul/bench_metal_kq_race.das | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das b/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das index 1faf89a8d8..f6b55ac17c 100644 --- a/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das +++ b/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das @@ -137,6 +137,7 @@ def private gemv_arms(f : KqRaceFmt) : array { // candidate GEMV sp arms |> push(KqRaceArm(name = "prod", src = metal_kq_gemv_iq3xxs_msl, entry = metal_kq_gemv_iq3xxs_msl_entry, fastmath = metal_kq_gemv_iq3xxs_msl_fastmath, tgmem = metal_kq_gemv_iq3xxs_msl_tgmem, gdiv = f.gemv_gdiv)) } elif (f.fmt == 2) { arms |> push(KqRaceArm(name = "prod", src = metal_kq_gemv_k2_msl, entry = metal_kq_gemv_k2_msl_entry, fastmath = metal_kq_gemv_k2_msl_fastmath, tgmem = 0ul, gdiv = f.gemv_gdiv)) + arms |> push(KqRaceArm(name = "d1", src = lab_kq_gemv_k2_d1_msl, entry = lab_kq_gemv_k2_d1_msl_entry, fastmath = lab_kq_gemv_k2_d1_msl_fastmath, tgmem = 0ul, gdiv = 8)) } elif (f.fmt == 23) { arms |> push(KqRaceArm(name = "prod", src = metal_kq_gemv_iq2s_msl, entry = metal_kq_gemv_iq2s_msl_entry, fastmath = metal_kq_gemv_iq2s_msl_fastmath, tgmem = 0ul, gdiv = f.gemv_gdiv)) } elif (f.fmt == 24) { @@ -209,6 +210,87 @@ def private mm_arms(f : KqRaceFmt) : array { // candidate mul_mm sp let PREFILL_M = 512 // the mm tier's token batch +// k2 on llama.cpp's q2_K shape: the k4 lane map (4 superblock streams, 32 elem/thread over +// the four sub-blocks of a 128-half), u16 SHIFT-FREE masked compose with power-of-two +// prefolds, the dmin fold on unshifted high nibbles, 4 rows/simdgroup x 2 sg (dispatch rows/8). +class LabKqGemvK2D1 { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array + @ssbo @binding = 3 @role = "read" @off = "xoff" @span = "n*4" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" @span = "rows*4" y : array + @uniform @binding = 5 ndim : uint + @uniform @binding = 6 ddim : uint + + [metal_kernel(name="lab_kq_gemv_k2_d1_msl")] + def lab_kq_gemv_k2_d1 { + let lane = gl_SubgroupInvocationID + let ix = lane / 8u + let it = lane % 8u + let iq = it / 4u + let ir = it % 4u + let isb = ir / 2u + let nb = ndim / 256u + let first_row = (gl_WorkGroupID.x * 2u + gl_SubgroupID) * 4u + var yl : float[32] + var sumf : float[4] + var ib = ix + while (ib < nb) { + let yb = ib * 256u + iq * 128u + ir * 8u + var sumy : float[4] + for [unroll_full] (i in range(8)) { + yl[i] = x[yb + uint(i)] + sumy[0] += yl[i] + yl[i + 8] = x[yb + 32u + uint(i)] + sumy[1] += yl[i + 8] + yl[i + 16] = x[yb + 64u + uint(i)] + sumy[2] += yl[i + 16] + yl[i + 24] = x[yb + 96u + uint(i)] + sumy[3] += yl[i + 24] + } + for [unroll_full] (r in range(4)) { + let blk = (first_row + uint(r)) * nb + ib + let qb = blk * 16u + iq * 8u + ir * 2u + let qa = kqu[qb] + let qc = kqu[qb + 1u] + var acc1 : float[4] + var acc2 : float[4] + for [unroll_full] (i in range(4)) { + let q16 = ((i < 2 ? qa : qc) >> (16u * (uint(i) & 1u))) & 0xFFFF + acc1[0] += yl[2 * i] * float(q16 & 0x0003) + acc2[0] += yl[2 * i + 1] * float(q16 & 0x0300) + acc1[1] += yl[2 * i + 8] * float(q16 & 0x000C) + acc2[1] += yl[2 * i + 9] * float(q16 & 0x0C00) + acc1[2] += yl[2 * i + 16] * float(q16 & 0x0030) + acc2[2] += yl[2 * i + 17] * float(q16 & 0x3000) + acc1[3] += yl[2 * i + 24] * float(q16 & 0x00C0) + acc2[3] += yl[2 * i + 25] * float(q16 & 0xC000) + } + let sb = blk * 16u + iq * 8u + isb + let b0 = uint(kscb[sb]) + let b2 = uint(kscb[sb + 2u]) + let b4 = uint(kscb[sb + 4u]) + let b6 = uint(kscb[sb + 6u]) + let dall = float(kdh[blk * 2u]) + let dmn = float(kdh[blk * 2u + 1u]) * 0.0625 + sumf[r] += dall * ((acc1[0] + 0.00390625 * acc2[0]) * float(b0 & 15u) + + (acc1[1] + 0.00390625 * acc2[1]) * float(b2 & 15u) * 0.25 + + (acc1[2] + 0.00390625 * acc2[2]) * float(b4 & 15u) * 0.0625 + + (acc1[3] + 0.00390625 * acc2[3]) * float(b6 & 15u) * 0.015625) + sumf[r] -= dmn * (sumy[0] * float(b0 & 0xF0) + sumy[1] * float(b2 & 0xF0) + + sumy[2] * float(b4 & 0xF0) + sumy[3] * float(b6 & 0xF0)) + } + ib += 4u + } + for [unroll_full] (r in range(4)) { + let s = simd_sum(sumf[r]) + if (lane == 0u && first_row + uint(r) < ddim) { + y[first_row + uint(r)] = s + } + } + } +} + // Rewrite every threadgroup A-tile subscript store `ta[IDX] = RHS;` into the pointer form // `*(ta + (IDX)) = RHS;` - llama.cpp's mul_mm carries the subscript form commented out as // "massively slower.. WTF?" (mul_mm.metal); this arm races that claim on our own template. From 39121e35a398836281215ff1c3396faa65f86a97 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 01:26:21 -0700 Subject: [PATCH 09/35] promote the k2 d1 GEMV - llama.cpp's q2_K shape on our planes, both boxes win big The k4 lane map with u16 shift-free masked compose (power-of-two prefolds, the dmin fold on unshifted high nibbles), 4 rows/simdgroup x 2 sg, dispatch rows/8; quant binding moves to the uint view. Interleaved: M5 1.27-1.44x over the old k6-map form (-> 1.16-1.20x vs lcpp), M4 1.60-1.85x (-> 1.18-1.21x). gemv suite 2/2; lab k2 gdiv 8, candidate arm retired. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014mq9AKyoAwvD5T1cigTnRw --- .../benchmarks/matmul/bench_metal_kq_race.das | 84 +----------------- .../dasllama/dasllama_metal_kernels.das | 86 +++++++++++-------- .../tests/test_metal_gemv_kernels.das | 2 +- 3 files changed, 50 insertions(+), 122 deletions(-) diff --git a/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das b/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das index f6b55ac17c..9270bbe4a8 100644 --- a/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das +++ b/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das @@ -91,7 +91,7 @@ def race_formats : array { KqRaceFmt(name = "k3", fmt = 3, gemv_gdiv = 4, split = true), KqRaceFmt(name = "iq3s", fmt = 33, gemv_gdiv = 8, split = true), KqRaceFmt(name = "iq3xxs", fmt = 34, gemv_gdiv = 8, split = true), - KqRaceFmt(name = "k2", fmt = 2, gemv_gdiv = 4, split = true), + KqRaceFmt(name = "k2", fmt = 2, gemv_gdiv = 8, split = true), KqRaceFmt(name = "iq2s", fmt = 23, gemv_gdiv = 4, split = true), KqRaceFmt(name = "iq2xs", fmt = 24, gemv_gdiv = 4, split = true), KqRaceFmt(name = "iq2xxs", fmt = 25, gemv_gdiv = 4, split = true) @@ -137,7 +137,6 @@ def private gemv_arms(f : KqRaceFmt) : array { // candidate GEMV sp arms |> push(KqRaceArm(name = "prod", src = metal_kq_gemv_iq3xxs_msl, entry = metal_kq_gemv_iq3xxs_msl_entry, fastmath = metal_kq_gemv_iq3xxs_msl_fastmath, tgmem = metal_kq_gemv_iq3xxs_msl_tgmem, gdiv = f.gemv_gdiv)) } elif (f.fmt == 2) { arms |> push(KqRaceArm(name = "prod", src = metal_kq_gemv_k2_msl, entry = metal_kq_gemv_k2_msl_entry, fastmath = metal_kq_gemv_k2_msl_fastmath, tgmem = 0ul, gdiv = f.gemv_gdiv)) - arms |> push(KqRaceArm(name = "d1", src = lab_kq_gemv_k2_d1_msl, entry = lab_kq_gemv_k2_d1_msl_entry, fastmath = lab_kq_gemv_k2_d1_msl_fastmath, tgmem = 0ul, gdiv = 8)) } elif (f.fmt == 23) { arms |> push(KqRaceArm(name = "prod", src = metal_kq_gemv_iq2s_msl, entry = metal_kq_gemv_iq2s_msl_entry, fastmath = metal_kq_gemv_iq2s_msl_fastmath, tgmem = 0ul, gdiv = f.gemv_gdiv)) } elif (f.fmt == 24) { @@ -210,87 +209,6 @@ def private mm_arms(f : KqRaceFmt) : array { // candidate mul_mm sp let PREFILL_M = 512 // the mm tier's token batch -// k2 on llama.cpp's q2_K shape: the k4 lane map (4 superblock streams, 32 elem/thread over -// the four sub-blocks of a 128-half), u16 SHIFT-FREE masked compose with power-of-two -// prefolds, the dmin fold on unshifted high nibbles, 4 rows/simdgroup x 2 sg (dispatch rows/8). -class LabKqGemvK2D1 { - @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array - @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array - @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array - @ssbo @binding = 3 @role = "read" @off = "xoff" @span = "n*4" x : array - @ssbo @binding = 4 @role = "write" @off = "yoff" @span = "rows*4" y : array - @uniform @binding = 5 ndim : uint - @uniform @binding = 6 ddim : uint - - [metal_kernel(name="lab_kq_gemv_k2_d1_msl")] - def lab_kq_gemv_k2_d1 { - let lane = gl_SubgroupInvocationID - let ix = lane / 8u - let it = lane % 8u - let iq = it / 4u - let ir = it % 4u - let isb = ir / 2u - let nb = ndim / 256u - let first_row = (gl_WorkGroupID.x * 2u + gl_SubgroupID) * 4u - var yl : float[32] - var sumf : float[4] - var ib = ix - while (ib < nb) { - let yb = ib * 256u + iq * 128u + ir * 8u - var sumy : float[4] - for [unroll_full] (i in range(8)) { - yl[i] = x[yb + uint(i)] - sumy[0] += yl[i] - yl[i + 8] = x[yb + 32u + uint(i)] - sumy[1] += yl[i + 8] - yl[i + 16] = x[yb + 64u + uint(i)] - sumy[2] += yl[i + 16] - yl[i + 24] = x[yb + 96u + uint(i)] - sumy[3] += yl[i + 24] - } - for [unroll_full] (r in range(4)) { - let blk = (first_row + uint(r)) * nb + ib - let qb = blk * 16u + iq * 8u + ir * 2u - let qa = kqu[qb] - let qc = kqu[qb + 1u] - var acc1 : float[4] - var acc2 : float[4] - for [unroll_full] (i in range(4)) { - let q16 = ((i < 2 ? qa : qc) >> (16u * (uint(i) & 1u))) & 0xFFFF - acc1[0] += yl[2 * i] * float(q16 & 0x0003) - acc2[0] += yl[2 * i + 1] * float(q16 & 0x0300) - acc1[1] += yl[2 * i + 8] * float(q16 & 0x000C) - acc2[1] += yl[2 * i + 9] * float(q16 & 0x0C00) - acc1[2] += yl[2 * i + 16] * float(q16 & 0x0030) - acc2[2] += yl[2 * i + 17] * float(q16 & 0x3000) - acc1[3] += yl[2 * i + 24] * float(q16 & 0x00C0) - acc2[3] += yl[2 * i + 25] * float(q16 & 0xC000) - } - let sb = blk * 16u + iq * 8u + isb - let b0 = uint(kscb[sb]) - let b2 = uint(kscb[sb + 2u]) - let b4 = uint(kscb[sb + 4u]) - let b6 = uint(kscb[sb + 6u]) - let dall = float(kdh[blk * 2u]) - let dmn = float(kdh[blk * 2u + 1u]) * 0.0625 - sumf[r] += dall * ((acc1[0] + 0.00390625 * acc2[0]) * float(b0 & 15u) - + (acc1[1] + 0.00390625 * acc2[1]) * float(b2 & 15u) * 0.25 - + (acc1[2] + 0.00390625 * acc2[2]) * float(b4 & 15u) * 0.0625 - + (acc1[3] + 0.00390625 * acc2[3]) * float(b6 & 15u) * 0.015625) - sumf[r] -= dmn * (sumy[0] * float(b0 & 0xF0) + sumy[1] * float(b2 & 0xF0) - + sumy[2] * float(b4 & 0xF0) + sumy[3] * float(b6 & 0xF0)) - } - ib += 4u - } - for [unroll_full] (r in range(4)) { - let s = simd_sum(sumf[r]) - if (lane == 0u && first_row + uint(r) < ddim) { - y[first_row + uint(r)] = s - } - } - } -} - // Rewrite every threadgroup A-tile subscript store `ta[IDX] = RHS;` into the pointer form // `*(ta + (IDX)) = RHS;` - llama.cpp's mul_mm carries the subscript form commented out as // "massively slower.. WTF?" (mul_mm.metal); this arm races that claim on our own template. diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das index 56524c4ffd..152bd2b60a 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das @@ -7995,11 +7995,11 @@ class MetalKqGemvK6 { // Q2_K: the k3 GEMV's lane map minus the hmask - unsigned 2-bit lanes, per-16 sc/min NIBBLE // pairs off the split strips (sc scales the dot, mn folds on the lane's x sums against dmin), // d + dmin off the 4B tail plane. -[metal_dispatch(name = "enc_kq_k2_c", pso = "g_pso_kq_k2", tg = 64, grid = "rows/4", params = "rows : int64, n : int64")] +[metal_dispatch(name = "enc_kq_k2_c", pso = "g_pso_kq_k2", tg = 64, grid = "rows/8", params = "rows : int64, n : int64")] class MetalKqGemvK2 { @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d+dmin tail - the scale buffer bound at byte nsb*16 (s0off = the caller's doff) @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // 16B pair-byte strips, byte view (same buffer at 0) - @ssbo @binding = 2 @role = "weight" @off = "qoff" kqb : array // k2 quant plane, byte view (qs at 64*blk) + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array // k2 quant plane, uint view (qs at 16*blk) @ssbo @binding = 3 @role = "read" @off = "xoff" @span = "n*4" x : array @ssbo @binding = 4 @role = "write" @off = "yoff" @span = "rows*4" y : array @uniform @binding = 5 ndim : uint @@ -8008,54 +8008,64 @@ class MetalKqGemvK2 { [metal_kernel(name="metal_kq_gemv_k2_msl")] def metal_kq_gemv_k2 { let lane = gl_SubgroupInvocationID - let tid = lane / 2u - let ix = lane % 2u - let ip = tid / 8u - let il = tid % 8u - let l0 = il * 4u - let is0 = ip * 8u + l0 / 16u + let ix = lane / 8u + let it = lane % 8u + let iq = it / 4u + let ir = it % 4u + let isb = ir / 2u let nb = ndim / 256u - let first_row = (gl_WorkGroupID.x * 2u + gl_SubgroupID) * 2u - var yl : float[16] - var sumf : float[2] + let first_row = (gl_WorkGroupID.x * 2u + gl_SubgroupID) * 4u + var yl : float[32] + var sumf : float[4] var ib = ix while (ib < nb) { - let yb = ib * 256u + ip * 128u + l0 - var ysum : float[4] - for [unroll_full] (l in range(4)) { - yl[4 * l] = x[yb + uint(l)] - yl[4 * l + 1] = x[yb + 32u + uint(l)] - yl[4 * l + 2] = x[yb + 64u + uint(l)] - yl[4 * l + 3] = x[yb + 96u + uint(l)] - ysum[0] += yl[4 * l] - ysum[1] += yl[4 * l + 1] - ysum[2] += yl[4 * l + 2] - ysum[3] += yl[4 * l + 3] + let yb = ib * 256u + iq * 128u + ir * 8u + var sumy : float[4] + for [unroll_full] (i in range(8)) { + yl[i] = x[yb + uint(i)] + sumy[0] += yl[i] + yl[i + 8] = x[yb + 32u + uint(i)] + sumy[1] += yl[i + 8] + yl[i + 16] = x[yb + 64u + uint(i)] + sumy[2] += yl[i + 16] + yl[i + 24] = x[yb + 96u + uint(i)] + sumy[3] += yl[i + 24] } - for [unroll_full] (r in range(2)) { + for [unroll_full] (r in range(4)) { let blk = (first_row + uint(r)) * nb + ib - let qs0 = blk * 64u + ip * 32u + l0 - let sb = blk * 16u + is0 - var sums : float[4] - for [unroll_full] (l in range(4)) { - let q = uint(kqb[qs0 + uint(l)]) - sums[0] += yl[4 * l] * float(q & 3u) - sums[1] += yl[4 * l + 1] * float((q >> 2u) & 3u) - sums[2] += yl[4 * l + 2] * float((q >> 4u) & 3u) - sums[3] += yl[4 * l + 3] * float(q >> 6u) + let qb = blk * 16u + iq * 8u + ir * 2u + let qa = kqu[qb] + let qc = kqu[qb + 1u] + var acc1 : float[4] + var acc2 : float[4] + for [unroll_full] (i in range(4)) { + let q16 = ((i < 2 ? qa : qc) >> (16u * (uint(i) & 1u))) & 0xFFFF + acc1[0] += yl[2 * i] * float(q16 & 0x0003) + acc2[0] += yl[2 * i + 1] * float(q16 & 0x0300) + acc1[1] += yl[2 * i + 8] * float(q16 & 0x000C) + acc2[1] += yl[2 * i + 9] * float(q16 & 0x0C00) + acc1[2] += yl[2 * i + 16] * float(q16 & 0x0030) + acc2[2] += yl[2 * i + 17] * float(q16 & 0x3000) + acc1[3] += yl[2 * i + 24] * float(q16 & 0x00C0) + acc2[3] += yl[2 * i + 25] * float(q16 & 0xC000) } + let sb = blk * 16u + iq * 8u + isb let b0 = uint(kscb[sb]) let b2 = uint(kscb[sb + 2u]) let b4 = uint(kscb[sb + 4u]) let b6 = uint(kscb[sb + 6u]) - let dd = float(kdh[blk * 2u]) - let dmn = float(kdh[blk * 2u + 1u]) - sumf[r] += dd * (sums[0] * float(b0 & 15u) + sums[1] * float(b2 & 15u) + sums[2] * float(b4 & 15u) + sums[3] * float(b6 & 15u)) - sumf[r] -= dmn * (ysum[0] * float(b0 >> 4u) + ysum[1] * float(b2 >> 4u) + ysum[2] * float(b4 >> 4u) + ysum[3] * float(b6 >> 4u)) + let dall = float(kdh[blk * 2u]) + let dmn = float(kdh[blk * 2u + 1u]) * 0.0625 + sumf[r] += dall * ((acc1[0] + 0.00390625 * acc2[0]) * float(b0 & 15u) + + (acc1[1] + 0.00390625 * acc2[1]) * float(b2 & 15u) * 0.25 + + (acc1[2] + 0.00390625 * acc2[2]) * float(b4 & 15u) * 0.0625 + + (acc1[3] + 0.00390625 * acc2[3]) * float(b6 & 15u) * 0.015625) + sumf[r] -= dmn * (sumy[0] * float(b0 & 0xF0) + sumy[1] * float(b2 & 0xF0) + + sumy[2] * float(b4 & 0xF0) + sumy[3] * float(b6 & 0xF0)) } - ib += 2u + ib += 4u } - for [unroll_full] (r in range(2)) { + for [unroll_full] (r in range(4)) { let s = simd_sum(sumf[r]) if (lane == 0u && first_row + uint(r) < ddim) { y[first_row + uint(r)] = s diff --git a/modules/dasLLAMA/tests/test_metal_gemv_kernels.das b/modules/dasLLAMA/tests/test_metal_gemv_kernels.das index 6c5e41fd85..74be4fd933 100644 --- a/modules/dasLLAMA/tests/test_metal_gemv_kernels.das +++ b/modules/dasLLAMA/tests/test_metal_gemv_kernels.das @@ -66,7 +66,7 @@ def private kq_gemv_gate(t : T?; dev, queue; vform : string; n, d : int) { var by = buf_fill(dev, d, -1000.0) var bn = buf_u32(dev, uint(n)) var bd = buf_u32(dev, uint(d)) - let groups = vform == "k5c" ? (d + 1) / 2 : (vform == "iq3s" || vform == "iq3xxs" ? (d + 7) / 8 : (d + 3) / 4) + let groups = vform == "k5c" ? (d + 1) / 2 : (vform == "iq3s" || vform == "iq3xxs" || vform == "k2" ? (d + 7) / 8 : (d + 3) / 4) let ran = with_compute_encoder(queue, err) $(enc : MetalComputeEncoder?) { metal_set_pipeline(enc, pso) if (vform == "iq4xs") { From 0eb718e1135b04394a97d1864e3d8b09175421b1 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 01:41:45 -0700 Subject: [PATCH 10/35] kq race lab: iq3s/iq3xxs d1 arms - M5 verdict: the shipped f4 forms KEEP the crown (d1f4 0.85-0.99x of prod, d1 0.64-0.85x); M4 race owed Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014mq9AKyoAwvD5T1cigTnRw --- .../benchmarks/matmul/bench_metal_kq_race.das | 239 ++++++++++++++++++ 1 file changed, 239 insertions(+) diff --git a/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das b/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das index 9270bbe4a8..98fa3061f6 100644 --- a/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das +++ b/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das @@ -133,8 +133,12 @@ def private gemv_arms(f : KqRaceFmt) : array { // candidate GEMV sp arms |> push(KqRaceArm(name = "prod", src = metal_kq_gemv_k3_msl, entry = metal_kq_gemv_k3_msl_entry, fastmath = metal_kq_gemv_k3_msl_fastmath, tgmem = 0ul, gdiv = f.gemv_gdiv)) } elif (f.fmt == 33) { arms |> push(KqRaceArm(name = "prod", src = metal_kq_gemv_iq3s_msl, entry = metal_kq_gemv_iq3s_msl_entry, fastmath = metal_kq_gemv_iq3s_msl_fastmath, tgmem = metal_kq_gemv_iq3s_msl_tgmem, gdiv = f.gemv_gdiv)) + arms |> push(KqRaceArm(name = "d1f4", src = lab_kq_gemv_iq3s_d1f4_msl, entry = lab_kq_gemv_iq3s_d1f4_msl_entry, fastmath = lab_kq_gemv_iq3s_d1f4_msl_fastmath, tgmem = lab_kq_gemv_iq3s_d1f4_msl_tgmem, gdiv = 4)) + arms |> push(KqRaceArm(name = "d1", src = lab_kq_gemv_iq3s_d1_msl, entry = lab_kq_gemv_iq3s_d1_msl_entry, fastmath = lab_kq_gemv_iq3s_d1_msl_fastmath, tgmem = 0ul, gdiv = 4)) } elif (f.fmt == 34) { arms |> push(KqRaceArm(name = "prod", src = metal_kq_gemv_iq3xxs_msl, entry = metal_kq_gemv_iq3xxs_msl_entry, fastmath = metal_kq_gemv_iq3xxs_msl_fastmath, tgmem = metal_kq_gemv_iq3xxs_msl_tgmem, gdiv = f.gemv_gdiv)) + arms |> push(KqRaceArm(name = "d1f4", src = lab_kq_gemv_iq3xxs_d1f4_msl, entry = lab_kq_gemv_iq3xxs_d1f4_msl_entry, fastmath = lab_kq_gemv_iq3xxs_d1f4_msl_fastmath, tgmem = lab_kq_gemv_iq3xxs_d1f4_msl_tgmem, gdiv = 4)) + arms |> push(KqRaceArm(name = "d1", src = lab_kq_gemv_iq3xxs_d1_msl, entry = lab_kq_gemv_iq3xxs_d1_msl_entry, fastmath = lab_kq_gemv_iq3xxs_d1_msl_fastmath, tgmem = 0ul, gdiv = 4)) } elif (f.fmt == 2) { arms |> push(KqRaceArm(name = "prod", src = metal_kq_gemv_k2_msl, entry = metal_kq_gemv_k2_msl_entry, fastmath = metal_kq_gemv_k2_msl_fastmath, tgmem = 0ul, gdiv = f.gemv_gdiv)) } elif (f.fmt == 23) { @@ -209,6 +213,241 @@ def private mm_arms(f : KqRaceFmt) : array { // candidate mul_mm sp let PREFILL_M = 512 // the mm tier's token batch +// iq3s on the d1 lane map (one 32-block per thread, ib += 4, 2 rows/simdgroup), f4-slab and +// constant-grid variants - the shipped form wins on M5 but loses to llama.cpp on M4-class. +class LabKqGemvIq3sD1F4 { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array + @ssbo @binding = 3 @role = "read" @off = "xoff" @span = "n*4" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" @span = "rows*4" y : array + @uniform @binding = 5 ndim : uint + @uniform @binding = 6 ddim : uint + @workgroup gridf : float4[512] + + [metal_kernel(name="lab_kq_gemv_iq3s_d1f4_msl")] + def lab_kq_gemv_iq3s_d1f4 { + let lid = gl_LocalInvocationID.x + for [unroll_full] (gk in range(8)) { + let gi = lid + uint(gk) * 64u + let w = iq3s_gw(int(gi)) + gridf[gi] = float4(float(w & 255u), float((w >> 8u) & 255u), float((w >> 16u) & 255u), float(w >> 24u)) + } + barrier() + let lane = gl_SubgroupInvocationID + let ix = lane / 8u + let it = lane % 8u + let nb = ndim / 256u + let first_row = (gl_WorkGroupID.x * 2u + gl_SubgroupID) * 2u + var sumf : float[2] + var ib = ix + while (ib < nb) { + let yb4 = ib * 64u + it * 8u + var yv : float4[8] + for [unroll_full] (i in range(8)) { + yv[i] = x[yb4 + uint(i)] + } + for [unroll_full] (r in range(2)) { + let blk = (first_row + uint(r)) * nb + ib + let qh = (kqu[blk * 26u + 16u + it / 4u] >> ((it % 4u) * 8u)) & 255u + let sgw = kqu[blk * 26u + 18u + it] + var accv = float4(0.0) + for [unroll_full] (h in range(2)) { + let qsw = kqu[blk * 26u + it * 2u + uint(h)] + for [unroll_full] (k in range(4)) { + let b3 = uint(h) * 4u + uint(k) + let idx = ((qsw >> (8u * uint(k))) & 255u) | ((qh << (8u - b3)) & 256u) + let nib = (sgw >> ((b3 / 2u) * 8u + (b3 % 2u) * 4u)) & 15u + let m = gridf[idx] + let f = float4((nib & 1u) != 0u ? -m.x : m.x, (nib & 2u) != 0u ? -m.y : m.y, + (nib & 4u) != 0u ? -m.z : m.z, (nib & 8u) != 0u ? -m.w : m.w) + accv += yv[h * 4 + k] * f + } + } + let sc = float(((int(kscb[blk * 16u + it]) & 255) ^ 128) - 128) + sumf[r] += float(kdh[blk]) * sc * (accv.x + accv.y + accv.z + accv.w) + } + ib += 4u + } + for [unroll_full] (r in range(2)) { + let s = simd_sum(sumf[r]) + if (lane == 0u && first_row + uint(r) < ddim) { + y[first_row + uint(r)] = s + } + } + } +} + +class LabKqGemvIq3sD1 { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array + @ssbo @binding = 3 @role = "read" @off = "xoff" @span = "n*4" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" @span = "rows*4" y : array + @uniform @binding = 5 ndim : uint + @uniform @binding = 6 ddim : uint + + [metal_kernel(name="lab_kq_gemv_iq3s_d1_msl")] + def lab_kq_gemv_iq3s_d1 { + let lane = gl_SubgroupInvocationID + let ix = lane / 8u + let it = lane % 8u + let nb = ndim / 256u + let first_row = (gl_WorkGroupID.x * 2u + gl_SubgroupID) * 2u + var sumf : float[2] + var ib = ix + while (ib < nb) { + let yb4 = ib * 64u + it * 8u + var yv : float4[8] + for [unroll_full] (i in range(8)) { + yv[i] = x[yb4 + uint(i)] + } + for [unroll_full] (r in range(2)) { + let blk = (first_row + uint(r)) * nb + ib + let qh = (kqu[blk * 26u + 16u + it / 4u] >> ((it % 4u) * 8u)) & 255u + let sgw = kqu[blk * 26u + 18u + it] + var accv = float4(0.0) + for [unroll_full] (h in range(2)) { + let qsw = kqu[blk * 26u + it * 2u + uint(h)] + for [unroll_full] (k in range(4)) { + let b3 = uint(h) * 4u + uint(k) + let idx = ((qsw >> (8u * uint(k))) & 255u) | ((qh << (8u - b3)) & 256u) + let nib = (sgw >> ((b3 / 2u) * 8u + (b3 % 2u) * 4u)) & 15u + let sw = iq3s_sw(iq3s_gw(int(idx)), nib) + accv += yv[h * 4 + k] * float4(iq3s_sx(sw, 0u), iq3s_sx(sw, 1u), iq3s_sx(sw, 2u), iq3s_sx(sw, 3u)) + } + } + let sc = float(((int(kscb[blk * 16u + it]) & 255) ^ 128) - 128) + sumf[r] += float(kdh[blk]) * sc * (accv.x + accv.y + accv.z + accv.w) + } + ib += 4u + } + for [unroll_full] (r in range(2)) { + let s = simd_sum(sumf[r]) + if (lane == 0u && first_row + uint(r) < ddim) { + y[first_row + uint(r)] = s + } + } + } +} + +// iq3xxs on the d1 map: the halved grid + parity signs off the block's aux word. +class LabKqGemvIq3xxsD1F4 { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array + @ssbo @binding = 3 @role = "read" @off = "xoff" @span = "n*4" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" @span = "rows*4" y : array + @uniform @binding = 5 ndim : uint + @uniform @binding = 6 ddim : uint + @workgroup gridf : float4[256] + + [metal_kernel(name="lab_kq_gemv_iq3xxs_d1f4_msl")] + def lab_kq_gemv_iq3xxs_d1f4 { + let lid = gl_LocalInvocationID.x + for [unroll_full] (gk in range(4)) { + let gi = lid + uint(gk) * 64u + if (gi < 256u) { + let w = iq3xxs_gw(int(gi)) + gridf[gi] = float4(float(w & 255u), float((w >> 8u) & 255u), float((w >> 16u) & 255u), float(w >> 24u)) + } + } + barrier() + let lane = gl_SubgroupInvocationID + let ix = lane / 8u + let it = lane % 8u + let nb = ndim / 256u + let first_row = (gl_WorkGroupID.x * 2u + gl_SubgroupID) * 2u + var sumf : float[2] + var ib = ix + while (ib < nb) { + let yb4 = ib * 64u + it * 8u + var yv : float4[8] + for [unroll_full] (i in range(8)) { + yv[i] = x[yb4 + uint(i)] + } + for [unroll_full] (r in range(2)) { + let blk = (first_row + uint(r)) * nb + ib + let aux = kqu[blk * 24u + 16u + it] + var accv = float4(0.0) + for [unroll_full] (h in range(2)) { + let qsw = kqu[blk * 24u + it * 2u + uint(h)] + for [unroll_full] (k in range(4)) { + let e8 = uint(h) * 2u + uint(k) / 2u + let sgb = ksign7m((aux >> (7u * e8)) & 127u) + let nib = uint(k) % 2u == 0u ? sgb & 15u : sgb >> 4u + let m = gridf[(qsw >> (8u * uint(k))) & 255u] + let f = float4((nib & 1u) != 0u ? -m.x : m.x, (nib & 2u) != 0u ? -m.y : m.y, + (nib & 4u) != 0u ? -m.z : m.z, (nib & 8u) != 0u ? -m.w : m.w) + accv += yv[h * 4 + k] * f + } + } + let sc = float(((int(kscb[blk * 16u + it]) & 255) ^ 128) - 128) + sumf[r] += float(kdh[blk]) * sc * (accv.x + accv.y + accv.z + accv.w) + } + ib += 4u + } + for [unroll_full] (r in range(2)) { + let s = simd_sum(sumf[r]) + if (lane == 0u && first_row + uint(r) < ddim) { + y[first_row + uint(r)] = s + } + } + } +} + +class LabKqGemvIq3xxsD1 { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array + @ssbo @binding = 3 @role = "read" @off = "xoff" @span = "n*4" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" @span = "rows*4" y : array + @uniform @binding = 5 ndim : uint + @uniform @binding = 6 ddim : uint + + [metal_kernel(name="lab_kq_gemv_iq3xxs_d1_msl")] + def lab_kq_gemv_iq3xxs_d1 { + let lane = gl_SubgroupInvocationID + let ix = lane / 8u + let it = lane % 8u + let nb = ndim / 256u + let first_row = (gl_WorkGroupID.x * 2u + gl_SubgroupID) * 2u + var sumf : float[2] + var ib = ix + while (ib < nb) { + let yb4 = ib * 64u + it * 8u + var yv : float4[8] + for [unroll_full] (i in range(8)) { + yv[i] = x[yb4 + uint(i)] + } + for [unroll_full] (r in range(2)) { + let blk = (first_row + uint(r)) * nb + ib + let aux = kqu[blk * 24u + 16u + it] + var accv = float4(0.0) + for [unroll_full] (h in range(2)) { + let qsw = kqu[blk * 24u + it * 2u + uint(h)] + for [unroll_full] (k in range(4)) { + let e8 = uint(h) * 2u + uint(k) / 2u + let sgb = ksign7m((aux >> (7u * e8)) & 127u) + let nib = uint(k) % 2u == 0u ? sgb & 15u : sgb >> 4u + let sw = iq3s_sw(iq3xxs_gw(int((qsw >> (8u * uint(k))) & 255u)), nib) + accv += yv[h * 4 + k] * float4(iq3s_sx(sw, 0u), iq3s_sx(sw, 1u), iq3s_sx(sw, 2u), iq3s_sx(sw, 3u)) + } + } + let sc = float(((int(kscb[blk * 16u + it]) & 255) ^ 128) - 128) + sumf[r] += float(kdh[blk]) * sc * (accv.x + accv.y + accv.z + accv.w) + } + ib += 4u + } + for [unroll_full] (r in range(2)) { + let s = simd_sum(sumf[r]) + if (lane == 0u && first_row + uint(r) < ddim) { + y[first_row + uint(r)] = s + } + } + } +} + // Rewrite every threadgroup A-tile subscript store `ta[IDX] = RHS;` into the pointer form // `*(ta + (IDX)) = RHS;` - llama.cpp's mul_mm carries the subscript form commented out as // "massively slower.. WTF?" (mul_mm.metal); this arm races that claim on our own template. From 84c7044fed8fdffbb870e2847ca6b1914d142774 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 01:46:24 -0700 Subject: [PATCH 11/35] iq3s/iq3xxs: the d1 constant form becomes the base, the f4-slab forms become crowned twins The generation split repeats the iq2xxs pattern in reverse: the shipped f4-slab GEMVs win on M5 (1.26-1.36x vs lcpp) but lose to the d1 lane-map constant form on M4-class (d1 1.18-1.26x over them). So the base is now the d1 form (dispatch rows/4, no slab - lifts M4 iq3s 0.71->0.89x, iq3xxs 0.56->0.74x vs lcpp) and the previous kernels ship verbatim as MetalKqGemvIq3sF4/Iq3xxsF4 behind box-raced crowns kq_gemv_iq3s_f4/iq3xxs_f4. race_gemv_iq2xxs_f4 generalizes to race_gemv_f4_twin (three crown rows); M5 races all three to the twin (iq3s +27%, iq3xxs +46% over d1). Suite 2/2 with the twin cells; the iq3xxs d1 walk also recorded the signed-strip trap (unsigned read = 10x-scale garbage, oracle-caught). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014mq9AKyoAwvD5T1cigTnRw --- .../benchmarks/matmul/bench_metal_kq_race.das | 245 +----------------- .../dasllama/dasllama_metal_common.das | 2 + .../dasllama/dasllama_metal_kernels.das | 179 +++++++++++-- .../dasllama/dasllama_metal_prefill.das | 12 +- .../tests/test_metal_gemv_kernels.das | 22 +- 5 files changed, 187 insertions(+), 273 deletions(-) diff --git a/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das b/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das index 98fa3061f6..489ccb7d87 100644 --- a/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das +++ b/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das @@ -89,8 +89,8 @@ def race_formats : array { KqRaceFmt(name = "iq4xs", fmt = 44, gemv_gdiv = 4, split = true), KqRaceFmt(name = "iq4nl", fmt = 45, gemv_gdiv = 4, split = false), KqRaceFmt(name = "k3", fmt = 3, gemv_gdiv = 4, split = true), - KqRaceFmt(name = "iq3s", fmt = 33, gemv_gdiv = 8, split = true), - KqRaceFmt(name = "iq3xxs", fmt = 34, gemv_gdiv = 8, split = true), + KqRaceFmt(name = "iq3s", fmt = 33, gemv_gdiv = 4, split = true), + KqRaceFmt(name = "iq3xxs", fmt = 34, gemv_gdiv = 4, split = true), KqRaceFmt(name = "k2", fmt = 2, gemv_gdiv = 8, split = true), KqRaceFmt(name = "iq2s", fmt = 23, gemv_gdiv = 4, split = true), KqRaceFmt(name = "iq2xs", fmt = 24, gemv_gdiv = 4, split = true), @@ -133,12 +133,10 @@ def private gemv_arms(f : KqRaceFmt) : array { // candidate GEMV sp arms |> push(KqRaceArm(name = "prod", src = metal_kq_gemv_k3_msl, entry = metal_kq_gemv_k3_msl_entry, fastmath = metal_kq_gemv_k3_msl_fastmath, tgmem = 0ul, gdiv = f.gemv_gdiv)) } elif (f.fmt == 33) { arms |> push(KqRaceArm(name = "prod", src = metal_kq_gemv_iq3s_msl, entry = metal_kq_gemv_iq3s_msl_entry, fastmath = metal_kq_gemv_iq3s_msl_fastmath, tgmem = metal_kq_gemv_iq3s_msl_tgmem, gdiv = f.gemv_gdiv)) - arms |> push(KqRaceArm(name = "d1f4", src = lab_kq_gemv_iq3s_d1f4_msl, entry = lab_kq_gemv_iq3s_d1f4_msl_entry, fastmath = lab_kq_gemv_iq3s_d1f4_msl_fastmath, tgmem = lab_kq_gemv_iq3s_d1f4_msl_tgmem, gdiv = 4)) - arms |> push(KqRaceArm(name = "d1", src = lab_kq_gemv_iq3s_d1_msl, entry = lab_kq_gemv_iq3s_d1_msl_entry, fastmath = lab_kq_gemv_iq3s_d1_msl_fastmath, tgmem = 0ul, gdiv = 4)) + arms |> push(KqRaceArm(name = "f4", src = metal_kq_gemv_iq3s_f4_msl, entry = metal_kq_gemv_iq3s_f4_msl_entry, fastmath = metal_kq_gemv_iq3s_f4_msl_fastmath, tgmem = metal_kq_gemv_iq3s_f4_msl_tgmem, gdiv = 8)) } elif (f.fmt == 34) { arms |> push(KqRaceArm(name = "prod", src = metal_kq_gemv_iq3xxs_msl, entry = metal_kq_gemv_iq3xxs_msl_entry, fastmath = metal_kq_gemv_iq3xxs_msl_fastmath, tgmem = metal_kq_gemv_iq3xxs_msl_tgmem, gdiv = f.gemv_gdiv)) - arms |> push(KqRaceArm(name = "d1f4", src = lab_kq_gemv_iq3xxs_d1f4_msl, entry = lab_kq_gemv_iq3xxs_d1f4_msl_entry, fastmath = lab_kq_gemv_iq3xxs_d1f4_msl_fastmath, tgmem = lab_kq_gemv_iq3xxs_d1f4_msl_tgmem, gdiv = 4)) - arms |> push(KqRaceArm(name = "d1", src = lab_kq_gemv_iq3xxs_d1_msl, entry = lab_kq_gemv_iq3xxs_d1_msl_entry, fastmath = lab_kq_gemv_iq3xxs_d1_msl_fastmath, tgmem = 0ul, gdiv = 4)) + arms |> push(KqRaceArm(name = "f4", src = metal_kq_gemv_iq3xxs_f4_msl, entry = metal_kq_gemv_iq3xxs_f4_msl_entry, fastmath = metal_kq_gemv_iq3xxs_f4_msl_fastmath, tgmem = metal_kq_gemv_iq3xxs_f4_msl_tgmem, gdiv = 8)) } elif (f.fmt == 2) { arms |> push(KqRaceArm(name = "prod", src = metal_kq_gemv_k2_msl, entry = metal_kq_gemv_k2_msl_entry, fastmath = metal_kq_gemv_k2_msl_fastmath, tgmem = 0ul, gdiv = f.gemv_gdiv)) } elif (f.fmt == 23) { @@ -213,241 +211,6 @@ def private mm_arms(f : KqRaceFmt) : array { // candidate mul_mm sp let PREFILL_M = 512 // the mm tier's token batch -// iq3s on the d1 lane map (one 32-block per thread, ib += 4, 2 rows/simdgroup), f4-slab and -// constant-grid variants - the shipped form wins on M5 but loses to llama.cpp on M4-class. -class LabKqGemvIq3sD1F4 { - @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array - @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array - @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array - @ssbo @binding = 3 @role = "read" @off = "xoff" @span = "n*4" x : array - @ssbo @binding = 4 @role = "write" @off = "yoff" @span = "rows*4" y : array - @uniform @binding = 5 ndim : uint - @uniform @binding = 6 ddim : uint - @workgroup gridf : float4[512] - - [metal_kernel(name="lab_kq_gemv_iq3s_d1f4_msl")] - def lab_kq_gemv_iq3s_d1f4 { - let lid = gl_LocalInvocationID.x - for [unroll_full] (gk in range(8)) { - let gi = lid + uint(gk) * 64u - let w = iq3s_gw(int(gi)) - gridf[gi] = float4(float(w & 255u), float((w >> 8u) & 255u), float((w >> 16u) & 255u), float(w >> 24u)) - } - barrier() - let lane = gl_SubgroupInvocationID - let ix = lane / 8u - let it = lane % 8u - let nb = ndim / 256u - let first_row = (gl_WorkGroupID.x * 2u + gl_SubgroupID) * 2u - var sumf : float[2] - var ib = ix - while (ib < nb) { - let yb4 = ib * 64u + it * 8u - var yv : float4[8] - for [unroll_full] (i in range(8)) { - yv[i] = x[yb4 + uint(i)] - } - for [unroll_full] (r in range(2)) { - let blk = (first_row + uint(r)) * nb + ib - let qh = (kqu[blk * 26u + 16u + it / 4u] >> ((it % 4u) * 8u)) & 255u - let sgw = kqu[blk * 26u + 18u + it] - var accv = float4(0.0) - for [unroll_full] (h in range(2)) { - let qsw = kqu[blk * 26u + it * 2u + uint(h)] - for [unroll_full] (k in range(4)) { - let b3 = uint(h) * 4u + uint(k) - let idx = ((qsw >> (8u * uint(k))) & 255u) | ((qh << (8u - b3)) & 256u) - let nib = (sgw >> ((b3 / 2u) * 8u + (b3 % 2u) * 4u)) & 15u - let m = gridf[idx] - let f = float4((nib & 1u) != 0u ? -m.x : m.x, (nib & 2u) != 0u ? -m.y : m.y, - (nib & 4u) != 0u ? -m.z : m.z, (nib & 8u) != 0u ? -m.w : m.w) - accv += yv[h * 4 + k] * f - } - } - let sc = float(((int(kscb[blk * 16u + it]) & 255) ^ 128) - 128) - sumf[r] += float(kdh[blk]) * sc * (accv.x + accv.y + accv.z + accv.w) - } - ib += 4u - } - for [unroll_full] (r in range(2)) { - let s = simd_sum(sumf[r]) - if (lane == 0u && first_row + uint(r) < ddim) { - y[first_row + uint(r)] = s - } - } - } -} - -class LabKqGemvIq3sD1 { - @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array - @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array - @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array - @ssbo @binding = 3 @role = "read" @off = "xoff" @span = "n*4" x : array - @ssbo @binding = 4 @role = "write" @off = "yoff" @span = "rows*4" y : array - @uniform @binding = 5 ndim : uint - @uniform @binding = 6 ddim : uint - - [metal_kernel(name="lab_kq_gemv_iq3s_d1_msl")] - def lab_kq_gemv_iq3s_d1 { - let lane = gl_SubgroupInvocationID - let ix = lane / 8u - let it = lane % 8u - let nb = ndim / 256u - let first_row = (gl_WorkGroupID.x * 2u + gl_SubgroupID) * 2u - var sumf : float[2] - var ib = ix - while (ib < nb) { - let yb4 = ib * 64u + it * 8u - var yv : float4[8] - for [unroll_full] (i in range(8)) { - yv[i] = x[yb4 + uint(i)] - } - for [unroll_full] (r in range(2)) { - let blk = (first_row + uint(r)) * nb + ib - let qh = (kqu[blk * 26u + 16u + it / 4u] >> ((it % 4u) * 8u)) & 255u - let sgw = kqu[blk * 26u + 18u + it] - var accv = float4(0.0) - for [unroll_full] (h in range(2)) { - let qsw = kqu[blk * 26u + it * 2u + uint(h)] - for [unroll_full] (k in range(4)) { - let b3 = uint(h) * 4u + uint(k) - let idx = ((qsw >> (8u * uint(k))) & 255u) | ((qh << (8u - b3)) & 256u) - let nib = (sgw >> ((b3 / 2u) * 8u + (b3 % 2u) * 4u)) & 15u - let sw = iq3s_sw(iq3s_gw(int(idx)), nib) - accv += yv[h * 4 + k] * float4(iq3s_sx(sw, 0u), iq3s_sx(sw, 1u), iq3s_sx(sw, 2u), iq3s_sx(sw, 3u)) - } - } - let sc = float(((int(kscb[blk * 16u + it]) & 255) ^ 128) - 128) - sumf[r] += float(kdh[blk]) * sc * (accv.x + accv.y + accv.z + accv.w) - } - ib += 4u - } - for [unroll_full] (r in range(2)) { - let s = simd_sum(sumf[r]) - if (lane == 0u && first_row + uint(r) < ddim) { - y[first_row + uint(r)] = s - } - } - } -} - -// iq3xxs on the d1 map: the halved grid + parity signs off the block's aux word. -class LabKqGemvIq3xxsD1F4 { - @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array - @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array - @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array - @ssbo @binding = 3 @role = "read" @off = "xoff" @span = "n*4" x : array - @ssbo @binding = 4 @role = "write" @off = "yoff" @span = "rows*4" y : array - @uniform @binding = 5 ndim : uint - @uniform @binding = 6 ddim : uint - @workgroup gridf : float4[256] - - [metal_kernel(name="lab_kq_gemv_iq3xxs_d1f4_msl")] - def lab_kq_gemv_iq3xxs_d1f4 { - let lid = gl_LocalInvocationID.x - for [unroll_full] (gk in range(4)) { - let gi = lid + uint(gk) * 64u - if (gi < 256u) { - let w = iq3xxs_gw(int(gi)) - gridf[gi] = float4(float(w & 255u), float((w >> 8u) & 255u), float((w >> 16u) & 255u), float(w >> 24u)) - } - } - barrier() - let lane = gl_SubgroupInvocationID - let ix = lane / 8u - let it = lane % 8u - let nb = ndim / 256u - let first_row = (gl_WorkGroupID.x * 2u + gl_SubgroupID) * 2u - var sumf : float[2] - var ib = ix - while (ib < nb) { - let yb4 = ib * 64u + it * 8u - var yv : float4[8] - for [unroll_full] (i in range(8)) { - yv[i] = x[yb4 + uint(i)] - } - for [unroll_full] (r in range(2)) { - let blk = (first_row + uint(r)) * nb + ib - let aux = kqu[blk * 24u + 16u + it] - var accv = float4(0.0) - for [unroll_full] (h in range(2)) { - let qsw = kqu[blk * 24u + it * 2u + uint(h)] - for [unroll_full] (k in range(4)) { - let e8 = uint(h) * 2u + uint(k) / 2u - let sgb = ksign7m((aux >> (7u * e8)) & 127u) - let nib = uint(k) % 2u == 0u ? sgb & 15u : sgb >> 4u - let m = gridf[(qsw >> (8u * uint(k))) & 255u] - let f = float4((nib & 1u) != 0u ? -m.x : m.x, (nib & 2u) != 0u ? -m.y : m.y, - (nib & 4u) != 0u ? -m.z : m.z, (nib & 8u) != 0u ? -m.w : m.w) - accv += yv[h * 4 + k] * f - } - } - let sc = float(((int(kscb[blk * 16u + it]) & 255) ^ 128) - 128) - sumf[r] += float(kdh[blk]) * sc * (accv.x + accv.y + accv.z + accv.w) - } - ib += 4u - } - for [unroll_full] (r in range(2)) { - let s = simd_sum(sumf[r]) - if (lane == 0u && first_row + uint(r) < ddim) { - y[first_row + uint(r)] = s - } - } - } -} - -class LabKqGemvIq3xxsD1 { - @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array - @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array - @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array - @ssbo @binding = 3 @role = "read" @off = "xoff" @span = "n*4" x : array - @ssbo @binding = 4 @role = "write" @off = "yoff" @span = "rows*4" y : array - @uniform @binding = 5 ndim : uint - @uniform @binding = 6 ddim : uint - - [metal_kernel(name="lab_kq_gemv_iq3xxs_d1_msl")] - def lab_kq_gemv_iq3xxs_d1 { - let lane = gl_SubgroupInvocationID - let ix = lane / 8u - let it = lane % 8u - let nb = ndim / 256u - let first_row = (gl_WorkGroupID.x * 2u + gl_SubgroupID) * 2u - var sumf : float[2] - var ib = ix - while (ib < nb) { - let yb4 = ib * 64u + it * 8u - var yv : float4[8] - for [unroll_full] (i in range(8)) { - yv[i] = x[yb4 + uint(i)] - } - for [unroll_full] (r in range(2)) { - let blk = (first_row + uint(r)) * nb + ib - let aux = kqu[blk * 24u + 16u + it] - var accv = float4(0.0) - for [unroll_full] (h in range(2)) { - let qsw = kqu[blk * 24u + it * 2u + uint(h)] - for [unroll_full] (k in range(4)) { - let e8 = uint(h) * 2u + uint(k) / 2u - let sgb = ksign7m((aux >> (7u * e8)) & 127u) - let nib = uint(k) % 2u == 0u ? sgb & 15u : sgb >> 4u - let sw = iq3s_sw(iq3xxs_gw(int((qsw >> (8u * uint(k))) & 255u)), nib) - accv += yv[h * 4 + k] * float4(iq3s_sx(sw, 0u), iq3s_sx(sw, 1u), iq3s_sx(sw, 2u), iq3s_sx(sw, 3u)) - } - } - let sc = float(((int(kscb[blk * 16u + it]) & 255) ^ 128) - 128) - sumf[r] += float(kdh[blk]) * sc * (accv.x + accv.y + accv.z + accv.w) - } - ib += 4u - } - for [unroll_full] (r in range(2)) { - let s = simd_sum(sumf[r]) - if (lane == 0u && first_row + uint(r) < ddim) { - y[first_row + uint(r)] = s - } - } - } -} - // Rewrite every threadgroup A-tile subscript store `ta[IDX] = RHS;` into the pointer form // `*(ta + (IDX)) = RHS;` - llama.cpp's mul_mm carries the subscript form commented out as // "massively slower.. WTF?" (mul_mm.metal); this arm races that claim on our own template. diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_common.das b/modules/dasLLAMA/dasllama/dasllama_metal_common.das index 08fa70ad8f..d80ed5c425 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_common.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_common.das @@ -289,6 +289,8 @@ var g_pso_kq_mvb8_iq2xs : MetalComputePipeline? var g_pso_kq_mm_b24 : MetalComputePipeline? var g_pso_kq_iq2xxs : MetalComputePipeline? var g_pso_kq_iq2xxs_f4 : MetalComputePipeline? //!< the f4-slab twin, compiled only behind its crown +var g_pso_kq_iq3s_f4 : MetalComputePipeline? //!< the iq3s f4-slab twin (crown "kq_gemv_iq3s_f4") +var g_pso_kq_iq3xxs_f4 : MetalComputePipeline? //!< the iq3xxs f4-slab twin (crown "kq_gemv_iq3xxs_f4") var g_pso_kq_mvb2_iq2xxs : MetalComputePipeline? var g_pso_kq_mvb4_iq2xxs : MetalComputePipeline? var g_pso_kq_mvb8_iq2xxs : MetalComputePipeline? diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das index 152bd2b60a..2636576cf4 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das @@ -3671,6 +3671,12 @@ def metal_decode_init : bool { // nolint:STYLE038 — a flat compile_pso per k g_pso_kq_mvb8_iq2xs = compile_pso(metal_kq_mvb8_iq2xs_msl, metal_kq_mvb8_iq2xs_msl_entry, metal_kq_mvb8_iq2xs_msl_fastmath, ok) g_pso_kq_mm_b24 = compile_pso(MetalKqMulMmIq2xs_metal_kq_mulmm_msl, MetalKqMulMmIq2xs_metal_kq_mulmm_msl_entry, MetalKqMulMmIq2xs_metal_kq_mulmm_msl_fastmath, ok) g_pso_kq_iq2xxs = compile_pso(metal_kq_gemv_iq2xxs_msl, metal_kq_gemv_iq2xxs_msl_entry, metal_kq_gemv_iq2xxs_msl_fastmath, ok) + if (metal_tensor_crowned("kq_gemv_iq3s_f4")) { + g_pso_kq_iq3s_f4 = compile_pso(metal_kq_gemv_iq3s_f4_msl, metal_kq_gemv_iq3s_f4_msl_entry, metal_kq_gemv_iq3s_f4_msl_fastmath, ok) + } + if (metal_tensor_crowned("kq_gemv_iq3xxs_f4")) { + g_pso_kq_iq3xxs_f4 = compile_pso(metal_kq_gemv_iq3xxs_f4_msl, metal_kq_gemv_iq3xxs_f4_msl_entry, metal_kq_gemv_iq3xxs_f4_msl_fastmath, ok) + } if (metal_tensor_crowned("kq_gemv_iq2xxs_f4")) { g_pso_kq_iq2xxs_f4 = compile_pso(metal_kq_gemv_iq2xxs_f4_msl, metal_kq_gemv_iq2xxs_f4_msl_entry, metal_kq_gemv_iq2xxs_f4_msl_fastmath, ok) } @@ -3755,9 +3761,17 @@ def enc_kq_gemv(enc : MetalComputeEncoder?; fmt : KqFmt; bq : MetalBuffer?; qoff } elif (fmt == KqFmt.k3) { enc_kq_k3_c(enc, bs, doff, bs, soff, bq, qoff, bx, xoff, by, yoff, bn, bd, rows, n) } elif (fmt == KqFmt.iq3s) { - enc_kq_iq3s_c(enc, bs, doff, bs, soff, bq, qoff, bx, xoff, by, yoff, bn, bd, rows, n) + if (g_pso_kq_iq3s_f4 != null) { + enc_kq_iq3s_f4_c(enc, bs, doff, bs, soff, bq, qoff, bx, xoff, by, yoff, bn, bd, rows, n) + } else { + enc_kq_iq3s_c(enc, bs, doff, bs, soff, bq, qoff, bx, xoff, by, yoff, bn, bd, rows, n) + } } elif (fmt == KqFmt.iq3xxs) { - enc_kq_iq3xxs_c(enc, bs, doff, bs, soff, bq, qoff, bx, xoff, by, yoff, bn, bd, rows, n) + if (g_pso_kq_iq3xxs_f4 != null) { + enc_kq_iq3xxs_f4_c(enc, bs, doff, bs, soff, bq, qoff, bx, xoff, by, yoff, bn, bd, rows, n) + } else { + enc_kq_iq3xxs_c(enc, bs, doff, bs, soff, bq, qoff, bx, xoff, by, yoff, bn, bd, rows, n) + } } elif (fmt == KqFmt.iq4nl) { enc_kq_iq4nl_c(enc, bs, soff, bs, soff, bq, qoff, bx, xoff, by, yoff, bn, bd, rows, n) } elif (fmt == KqFmt.k2) { @@ -4389,6 +4403,8 @@ def metal_kernels_release { // nolint:STYLE037,STYLE038 — a flat release_pso release_pso(g_pso_kq_mm_b24) release_pso(g_pso_kq_iq2xxs) release_pso(g_pso_kq_iq2xxs_f4) + release_pso(g_pso_kq_iq3s_f4) + release_pso(g_pso_kq_iq3xxs_f4) release_pso(g_pso_kq_mvb2_iq2xxs) release_pso(g_pso_kq_mvb4_iq2xxs) release_pso(g_pso_kq_mvb8_iq2xxs) @@ -4718,28 +4734,39 @@ def metal_tensor_race_decode : array { 64, kdim, ndim, 64 * ndim, uint3(uint(2 * (ndim / 64)), 1u, 1u), rka) $(enc, slot_y) { kn_buffer(enc, slot_y, 0ul, 4) }) - results |> emplace(race_gemv_iq2xxs_f4(dev, queue)) + results |> emplace(race_gemv_f4_twin(dev, queue, "kq_gemv_iq2xxs_f4", + metal_kq_gemv_iq2xxs_msl, metal_kq_gemv_iq2xxs_msl_entry, metal_kq_gemv_iq2xxs_msl_fastmath, + metal_kq_gemv_iq2xxs_f4_msl, metal_kq_gemv_iq2xxs_f4_msl_entry, metal_kq_gemv_iq2xxs_f4_msl_fastmath, metal_kq_gemv_iq2xxs_f4_msl_tgmem, 16, 4, 4)) + results |> emplace(race_gemv_f4_twin(dev, queue, "kq_gemv_iq3s_f4", + metal_kq_gemv_iq3s_msl, metal_kq_gemv_iq3s_msl_entry, metal_kq_gemv_iq3s_msl_fastmath, + metal_kq_gemv_iq3s_f4_msl, metal_kq_gemv_iq3s_f4_msl_entry, metal_kq_gemv_iq3s_f4_msl_fastmath, metal_kq_gemv_iq3s_f4_msl_tgmem, 26, 4, 8)) + results |> emplace(race_gemv_f4_twin(dev, queue, "kq_gemv_iq3xxs_f4", + metal_kq_gemv_iq3xxs_msl, metal_kq_gemv_iq3xxs_msl_entry, metal_kq_gemv_iq3xxs_msl_fastmath, + metal_kq_gemv_iq3xxs_f4_msl, metal_kq_gemv_iq3xxs_f4_msl_entry, metal_kq_gemv_iq3xxs_f4_msl_fastmath, metal_kq_gemv_iq3xxs_f4_msl_tgmem, 24, 4, 8)) metal_release(queue) metal_release(dev) return <- results } -//! The iq2xxs GEMV twin race: the base (constant-table) form vs the f4-slab twin. The slab -//! wins on M5-class GPUs and REGRESSES on M1/M4-class - a per-box crown, not a heuristic. -//! Winner "tensor" = the twin takes the "kq_gemv_iq2xxs_f4" crown (the absorb convention). -def private race_gemv_iq2xxs_f4(dev : MetalDevice?; queue : MetalCommandQueue?) : MetalTensorRaceResult { // nolint:STYLE038 - one-shot A/B race harness; buffers + PSOs stay live to the release tail - var res = MetalTensorRaceResult(family = "kq_gemv_iq2xxs_f4", winner = "", base_ms = -1.0lf, twin_ms = -1.0lf, note = "") +//! A GEMV f4-slab twin race over the split-scale fixture: base (constant-table, rows/4) +//! vs the f4-slab twin. The slab wins on M5-class GPUs and REGRESSES on M1/M4-class - a +//! per-box crown, not a heuristic. Winner "tensor" = the twin takes the family crown. +def private race_gemv_f4_twin(dev : MetalDevice?; queue : MetalCommandQueue?; family : string; // nolint:STYLE038 - one-shot A/B race harness; buffers + PSOs stay live to the release tail + base_src, base_entry : string; base_fm : bool; + twin_src, twin_entry : string; twin_fm : bool; twin_tgmem : uint64; + qu_per_sb : int; base_gdiv, twin_gdiv : int) : MetalTensorRaceResult { + var res = MetalTensorRaceResult(family = family, winner = "", base_ms = -1.0lf, twin_ms = -1.0lf, note = "") let kdim = 3072 let ndim = 4096 let nsb = kdim / 256 let nblk = ndim * nsb var err = "" - var base_pso = pipeline_from_source(dev, metal_kq_gemv_iq2xxs_msl, metal_kq_gemv_iq2xxs_msl_entry, metal_kq_gemv_iq2xxs_msl_fastmath, err) + var base_pso = pipeline_from_source(dev, base_src, base_entry, base_fm, err) if (base_pso == null) { res.note = "base pso: {err}" return res } - var twin_pso = pipeline_from_source(dev, metal_kq_gemv_iq2xxs_f4_msl, metal_kq_gemv_iq2xxs_f4_msl_entry, metal_kq_gemv_iq2xxs_f4_msl_fastmath, err) + var twin_pso = pipeline_from_source(dev, twin_src, twin_entry, twin_fm, err) if (twin_pso == null) { res.winner = "simdgroup" res.note = "twin pso: {err}" @@ -4747,8 +4774,8 @@ def private race_gemv_iq2xxs_f4(dev : MetalDevice?; queue : MetalCommandQueue?) return res } var qplane : array - qplane |> resize(nblk * 16) - for (i in range(nblk * 16)) { + qplane |> resize(nblk * qu_per_sb) + for (i in range(nblk * qu_per_sb)) { qplane[i] = uint(i) * 2654435761u } var splane : array @@ -4763,7 +4790,7 @@ def private race_gemv_iq2xxs_f4(dev : MetalDevice?; queue : MetalCommandQueue?) } } var xa <- race_x_f32(kdim) - var bq = race_buf(dev, uint64(nblk * 64), unsafe(addr(qplane[0]))) + var bq = race_buf(dev, uint64(nblk * qu_per_sb * 4), unsafe(addr(qplane[0]))) var bs = race_buf(dev, uint64(nblk * 18), unsafe(addr(splane[0]))) var bxa = race_buf(dev, uint64(kdim * 4), unsafe(addr(xa[0]))) var by_base = race_buf(dev, uint64(ndim * 4), null) @@ -4771,7 +4798,6 @@ def private race_gemv_iq2xxs_f4(dev : MetalDevice?; queue : MetalCommandQueue?) var bn = race_uniform_u32(dev, uint(kdim)) var bd = race_uniform_u32(dev, uint(ndim)) let doff = uint64(nblk * 16) - let grid = uint3(uint(ndim / 4), 1u, 1u) let tg = uint3(64u, 1u, 1u) race_pair_ms(queue, 5, res.base_ms, res.twin_ms, $(enc) { kn_pipeline(enc, base_pso) @@ -4782,10 +4808,10 @@ def private race_gemv_iq2xxs_f4(dev : MetalDevice?; queue : MetalCommandQueue?) kn_buffer(enc, by_base, 0ul, 4) kn_buffer(enc, bn, 0ul, 5) kn_buffer(enc, bd, 0ul, 6) - kn_dispatch(enc, grid, tg) + kn_dispatch(enc, uint3(uint(ndim / base_gdiv), 1u, 1u), tg) }, $(enc) { kn_pipeline(enc, twin_pso) - kn_tgmem(enc, metal_kq_gemv_iq2xxs_f4_msl_tgmem, 0) + kn_tgmem(enc, twin_tgmem, 0) kn_buffer(enc, bs, doff, 0) kn_buffer(enc, bs, 0ul, 1) kn_buffer(enc, bq, 0ul, 2) @@ -4793,7 +4819,7 @@ def private race_gemv_iq2xxs_f4(dev : MetalDevice?; queue : MetalCommandQueue?) kn_buffer(enc, by_twin, 0ul, 4) kn_buffer(enc, bn, 0ul, 5) kn_buffer(enc, bd, 0ul, 6) - kn_dispatch(enc, grid, tg) + kn_dispatch(enc, uint3(uint(ndim / twin_gdiv), 1u, 1u), tg) }) if (res.base_ms < 0.0lf || res.twin_ms < 0.0lf) { res.winner = res.base_ms < 0.0lf ? "" : "simdgroup" @@ -8626,7 +8652,7 @@ class MetalKqGemvIq4nl { // IQ3_S: w = d*sc*(+-grid byte); the 2 KB grid in threadgroup memory, a // 16-lane half-simdgroup per superblock, lane (bu, il) = block bu's half il: one qs word = 4 // grid words = 16 elems = 4 CONSECUTIVE float4 x loads. Scale plane = the iq4xs/k6 split form. -[metal_dispatch(name = "enc_kq_iq3s_c", pso = "g_pso_kq_iq3s", tgmem = "metal_kq_gemv_iq3s_msl_tgmem", tg = 64, grid = "rows/8", params = "rows : int64, n : int64")] +[metal_dispatch(name = "enc_kq_iq3s_c", pso = "g_pso_kq_iq3s", tg = 64, grid = "rows/4", params = "rows : int64, n : int64")] class MetalKqGemvIq3s { @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d plane — the scale buffer bound at byte nsb*16 (s0off = the caller's doff) @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // 16B strips: 8 signed sub-scales + 8 pad, byte view (same buffer at 0) @@ -8635,10 +8661,66 @@ class MetalKqGemvIq3s { @ssbo @binding = 4 @role = "write" @off = "yoff" @span = "rows*4" y : array @uniform @binding = 5 ndim : uint @uniform @binding = 6 ddim : uint - @workgroup gridf : float4[512] // magnitudes pre-expanded at stage time: the inner loop reads one float4 and flips signs, no byte extracts (the best of the eight raced forms, the reference exe's geometry included) [metal_kernel(name="metal_kq_gemv_iq3s_msl")] def metal_kq_gemv_iq3s { + let lane = gl_SubgroupInvocationID + let ix = lane / 8u + let it = lane % 8u + let nb = ndim / 256u + let first_row = (gl_WorkGroupID.x * 2u + gl_SubgroupID) * 2u + var sumf : float[2] + var ib = ix + while (ib < nb) { + let yb4 = ib * 64u + it * 8u + var yv : float4[8] + for [unroll_full] (i in range(8)) { + yv[i] = x[yb4 + uint(i)] + } + for [unroll_full] (r in range(2)) { + let blk = (first_row + uint(r)) * nb + ib + let qh = (kqu[blk * 26u + 16u + it / 4u] >> ((it % 4u) * 8u)) & 255u + let sgw = kqu[blk * 26u + 18u + it] + var accv = float4(0.0) + for [unroll_full] (h in range(2)) { + let qsw = kqu[blk * 26u + it * 2u + uint(h)] + for [unroll_full] (k in range(4)) { + let b3 = uint(h) * 4u + uint(k) + let idx = ((qsw >> (8u * uint(k))) & 255u) | ((qh << (8u - b3)) & 256u) + let nib = (sgw >> ((b3 / 2u) * 8u + (b3 % 2u) * 4u)) & 15u + let sw = iq3s_sw(iq3s_gw(int(idx)), nib) + accv += yv[h * 4 + k] * float4(iq3s_sx(sw, 0u), iq3s_sx(sw, 1u), iq3s_sx(sw, 2u), iq3s_sx(sw, 3u)) + } + } + let sc = float(((int(kscb[blk * 16u + it]) & 255) ^ 128) - 128) + sumf[r] += float(kdh[blk]) * sc * (accv.x + accv.y + accv.z + accv.w) + } + ib += 4u + } + for [unroll_full] (r in range(2)) { + let s = simd_sum(sumf[r]) + if (lane == 0u && first_row + uint(r) < ddim) { + y[first_row + uint(r)] = s + } + } + } +} + +// The f4-slab twin (the shipped pre-d1 form verbatim) - crowns "kq_gemv_iq3s_f4" where the +// box race prefers it (M5-class); M1/M4-class regress on the slab and keep the d1 base. +[metal_dispatch(name = "enc_kq_iq3s_f4_c", pso = "g_pso_kq_iq3s_f4", tgmem = "metal_kq_gemv_iq3s_f4_msl_tgmem", tg = 64, grid = "rows/8", params = "rows : int64, n : int64")] +class MetalKqGemvIq3sF4 { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d plane — the scale buffer bound at byte nsb*16 (s0off = the caller's doff) + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // 16B strips: 8 signed sub-scales + 8 pad, byte view (same buffer at 0) + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array // iq3s quant plane, uint view (26 words per superblock: 16 qs, 2 qh, 8 signs) + @ssbo @binding = 3 @role = "read" @off = "xoff" @span = "n*4" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" @span = "rows*4" y : array + @uniform @binding = 5 ndim : uint + @uniform @binding = 6 ddim : uint + @workgroup gridf : float4[512] // magnitudes pre-expanded at stage time: the inner loop reads one float4 and flips signs, no byte extracts (the best of the eight raced forms, the reference exe's geometry included) + + [metal_kernel(name="metal_kq_gemv_iq3s_f4_msl")] + def metal_kq_gemv_iq3s_f4 { let lid = gl_LocalInvocationID.x for [unroll_full] (k in range(8)) { let gi = lid + uint(k) * 64u @@ -8949,7 +9031,7 @@ class MetalKqGemvIq2xxsF4 { // IQ3_XXS: the iq3s GEMV shape verbatim over the 24-word rows - the 1 KB halved grid as a // float4 magnitude slab, signs from the block's aux32 through the parity helper (no table), // the same (2ls+1)-strip x halved-d fold. 4 rows per simdgroup, dispatch rows/8. -[metal_dispatch(name = "enc_kq_iq3xxs_c", pso = "g_pso_kq_iq3xxs", tgmem = "metal_kq_gemv_iq3xxs_msl_tgmem", tg = 64, grid = "rows/8", params = "rows : int64, n : int64")] +[metal_dispatch(name = "enc_kq_iq3xxs_c", pso = "g_pso_kq_iq3xxs", tg = 64, grid = "rows/4", params = "rows : int64, n : int64")] class MetalKqGemvIq3xxs { @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d plane — the scale buffer bound at byte nsb*16 (s0off = the caller's doff) @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // 16B strips: 8 sub-scale bytes (2ls+1) + 8 pad, byte view (same buffer at 0) @@ -8958,10 +9040,65 @@ class MetalKqGemvIq3xxs { @ssbo @binding = 4 @role = "write" @off = "yoff" @span = "rows*4" y : array @uniform @binding = 5 ndim : uint @uniform @binding = 6 ddim : uint - @workgroup gridf : float4[256] // magnitudes pre-expanded at stage time (the iq3s GEMV's winning form) [metal_kernel(name="metal_kq_gemv_iq3xxs_msl")] def metal_kq_gemv_iq3xxs { + let lane = gl_SubgroupInvocationID + let ix = lane / 8u + let it = lane % 8u + let nb = ndim / 256u + let first_row = (gl_WorkGroupID.x * 2u + gl_SubgroupID) * 2u + var sumf : float[2] + var ib = ix + while (ib < nb) { + let yb4 = ib * 64u + it * 8u + var yv : float4[8] + for [unroll_full] (i in range(8)) { + yv[i] = x[yb4 + uint(i)] + } + for [unroll_full] (r in range(2)) { + let blk = (first_row + uint(r)) * nb + ib + let aux = kqu[blk * 24u + 16u + it] + var accv = float4(0.0) + for [unroll_full] (h in range(2)) { + let qsw = kqu[blk * 24u + it * 2u + uint(h)] + for [unroll_full] (k in range(4)) { + let e8 = uint(h) * 2u + uint(k) / 2u + let sgb = ksign7m((aux >> (7u * e8)) & 127u) + let nib = uint(k) % 2u == 0u ? sgb & 15u : sgb >> 4u + let sw = iq3s_sw(iq3xxs_gw(int((qsw >> (8u * uint(k))) & 255u)), nib) + accv += yv[h * 4 + k] * float4(iq3s_sx(sw, 0u), iq3s_sx(sw, 1u), iq3s_sx(sw, 2u), iq3s_sx(sw, 3u)) + } + } + let sc = float(((int(kscb[blk * 16u + it]) & 255) ^ 128) - 128) + sumf[r] += float(kdh[blk]) * sc * (accv.x + accv.y + accv.z + accv.w) + } + ib += 4u + } + for [unroll_full] (r in range(2)) { + let s = simd_sum(sumf[r]) + if (lane == 0u && first_row + uint(r) < ddim) { + y[first_row + uint(r)] = s + } + } + } +} + +// The f4-slab twin (the shipped pre-d1 form verbatim) - crowns "kq_gemv_iq3xxs_f4" where the +// box race prefers it (M5-class); M1/M4-class regress on the slab and keep the d1 base. +[metal_dispatch(name = "enc_kq_iq3xxs_f4_c", pso = "g_pso_kq_iq3xxs_f4", tgmem = "metal_kq_gemv_iq3xxs_f4_msl_tgmem", tg = 64, grid = "rows/8", params = "rows : int64, n : int64")] +class MetalKqGemvIq3xxsF4 { + @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d plane — the scale buffer bound at byte nsb*16 (s0off = the caller's doff) + @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // 16B strips: 8 sub-scale bytes (2ls+1) + 8 pad, byte view (same buffer at 0) + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array // iq3xxs quant plane, uint view (24 words per superblock: 16 qs, 8 aux) + @ssbo @binding = 3 @role = "read" @off = "xoff" @span = "n*4" x : array + @ssbo @binding = 4 @role = "write" @off = "yoff" @span = "rows*4" y : array + @uniform @binding = 5 ndim : uint + @uniform @binding = 6 ddim : uint + @workgroup gridf : float4[256] // magnitudes pre-expanded at stage time (the iq3s GEMV's winning form) + + [metal_kernel(name="metal_kq_gemv_iq3xxs_f4_msl")] + def metal_kq_gemv_iq3xxs_f4 { let lid = gl_LocalInvocationID.x for [unroll_full] (k in range(4)) { let gi = lid + uint(k) * 64u diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das b/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das index 92917e5b41..ec2b13327c 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das @@ -5342,9 +5342,17 @@ def private pf_enc_kq_gemv(enc : MetalComputeEncoder?; t : Model; fmt : KqFmt; w } elif (fmt == KqFmt.k3) { enc_kq_k3_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, xoff, by, 0ul, bn, bd, rows, n) } elif (fmt == KqFmt.iq3s) { - enc_kq_iq3s_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, xoff, by, 0ul, bn, bd, rows, n) + if (g_pso_kq_iq3s_f4 != null) { + enc_kq_iq3s_f4_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, xoff, by, 0ul, bn, bd, rows, n) + } else { + enc_kq_iq3s_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, xoff, by, 0ul, bn, bd, rows, n) + } } elif (fmt == KqFmt.iq3xxs) { - enc_kq_iq3xxs_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, xoff, by, 0ul, bn, bd, rows, n) + if (g_pso_kq_iq3xxs_f4 != null) { + enc_kq_iq3xxs_f4_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, xoff, by, 0ul, bn, bd, rows, n) + } else { + enc_kq_iq3xxs_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, xoff, by, 0ul, bn, bd, rows, n) + } } elif (fmt == KqFmt.iq2s) { enc_kq_iq2s_c(enc, bs.buf, bs.doff, bs.buf, bs.soff, bq.buf, bq.qoff, bx, xoff, by, 0ul, bn, bd, rows, n) } elif (fmt == KqFmt.iq2xs) { diff --git a/modules/dasLLAMA/tests/test_metal_gemv_kernels.das b/modules/dasLLAMA/tests/test_metal_gemv_kernels.das index 74be4fd933..1fe7820947 100644 --- a/modules/dasLLAMA/tests/test_metal_gemv_kernels.das +++ b/modules/dasLLAMA/tests/test_metal_gemv_kernels.das @@ -23,12 +23,12 @@ require math // ===== single-stream kq GEMV (MetalKqGemvK4 / K5 / K5C / K6) ===== def private kq_gemv_gate(t : T?; dev, queue; vform : string; n, d : int) { - let fmt = vform == "k4" ? 4 : (vform == "k6" ? 6 : (vform == "iq4xs" ? 44 : (vform == "k3" ? 3 : (vform == "iq3s" ? 33 : (vform == "iq3xxs" ? 34 : (vform == "iq4nl" ? 45 : (vform == "iq2s" ? 23 : (vform == "iq2xs" ? 24 : (vform == "iq2xxs" || vform == "iq2xxsf4" ? 25 : (vform == "k2" ? 2 : 5)))))))))) + let fmt = vform == "k4" ? 4 : (vform == "k6" ? 6 : (vform == "iq4xs" ? 44 : (vform == "k3" ? 3 : (vform == "iq3s" || vform == "iq3sf4" ? 33 : (vform == "iq3xxs" || vform == "iq3xxsf4" ? 34 : (vform == "iq4nl" ? 45 : (vform == "iq2s" ? 23 : (vform == "iq2xs" ? 24 : (vform == "iq2xxs" || vform == "iq2xxsf4" ? 25 : (vform == "k2" ? 2 : 5)))))))))) let tag = "kq_gemv_{vform} n={n} d={d}" var err : string - let src = vform == "iq2xxsf4" ? metal_kq_gemv_iq2xxs_f4_msl : (vform == "iq2xxs" ? metal_kq_gemv_iq2xxs_msl : (vform == "iq2xs" ? metal_kq_gemv_iq2xs_msl : (vform == "iq2s" ? metal_kq_gemv_iq2s_msl : (vform == "k2" ? metal_kq_gemv_k2_msl : (vform == "iq4nl" ? metal_kq_gemv_iq4nl_msl : (vform == "iq3xxs" ? metal_kq_gemv_iq3xxs_msl : (vform == "iq3s" ? metal_kq_gemv_iq3s_msl : (vform == "k3" ? metal_kq_gemv_k3_msl : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl : (vform == "k4" ? metal_kq_gemv_k4_msl : (vform == "k5" ? metal_kq_gemv_k5_msl : (vform == "k5c" ? metal_kq_gemv_k5c_msl : metal_kq_gemv_k6_msl)))))))))))) - let entry = vform == "iq2xxsf4" ? metal_kq_gemv_iq2xxs_f4_msl_entry : (vform == "iq2xxs" ? metal_kq_gemv_iq2xxs_msl_entry : (vform == "iq2xs" ? metal_kq_gemv_iq2xs_msl_entry : (vform == "iq2s" ? metal_kq_gemv_iq2s_msl_entry : (vform == "k2" ? metal_kq_gemv_k2_msl_entry : (vform == "iq4nl" ? metal_kq_gemv_iq4nl_msl_entry : (vform == "iq3xxs" ? metal_kq_gemv_iq3xxs_msl_entry : (vform == "iq3s" ? metal_kq_gemv_iq3s_msl_entry : (vform == "k3" ? metal_kq_gemv_k3_msl_entry : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl_entry : (vform == "k4" ? metal_kq_gemv_k4_msl_entry : (vform == "k5" ? metal_kq_gemv_k5_msl_entry : (vform == "k5c" ? metal_kq_gemv_k5c_msl_entry : metal_kq_gemv_k6_msl_entry)))))))))))) - let fm = vform == "iq2xxsf4" ? metal_kq_gemv_iq2xxs_f4_msl_fastmath : (vform == "iq2xxs" ? metal_kq_gemv_iq2xxs_msl_fastmath : (vform == "iq2xs" ? metal_kq_gemv_iq2xs_msl_fastmath : (vform == "iq2s" ? metal_kq_gemv_iq2s_msl_fastmath : (vform == "k2" ? metal_kq_gemv_k2_msl_fastmath : (vform == "iq4nl" ? metal_kq_gemv_iq4nl_msl_fastmath : (vform == "iq3xxs" ? metal_kq_gemv_iq3xxs_msl_fastmath : (vform == "iq3s" ? metal_kq_gemv_iq3s_msl_fastmath : (vform == "k3" ? metal_kq_gemv_k3_msl_fastmath : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl_fastmath : (vform == "k4" ? metal_kq_gemv_k4_msl_fastmath : (vform == "k5" ? metal_kq_gemv_k5_msl_fastmath : (vform == "k5c" ? metal_kq_gemv_k5c_msl_fastmath : metal_kq_gemv_k6_msl_fastmath)))))))))))) + let src = vform == "iq2xxsf4" ? metal_kq_gemv_iq2xxs_f4_msl : (vform == "iq2xxs" ? metal_kq_gemv_iq2xxs_msl : (vform == "iq2xs" ? metal_kq_gemv_iq2xs_msl : (vform == "iq2s" ? metal_kq_gemv_iq2s_msl : (vform == "k2" ? metal_kq_gemv_k2_msl : (vform == "iq4nl" ? metal_kq_gemv_iq4nl_msl : (vform == "iq3xxsf4" ? metal_kq_gemv_iq3xxs_f4_msl : (vform == "iq3sf4" ? metal_kq_gemv_iq3s_f4_msl : (vform == "iq3xxs" ? metal_kq_gemv_iq3xxs_msl : (vform == "iq3s" ? metal_kq_gemv_iq3s_msl : (vform == "k3" ? metal_kq_gemv_k3_msl : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl : (vform == "k4" ? metal_kq_gemv_k4_msl : (vform == "k5" ? metal_kq_gemv_k5_msl : (vform == "k5c" ? metal_kq_gemv_k5c_msl : metal_kq_gemv_k6_msl)))))))))))))) + let entry = vform == "iq2xxsf4" ? metal_kq_gemv_iq2xxs_f4_msl_entry : (vform == "iq2xxs" ? metal_kq_gemv_iq2xxs_msl_entry : (vform == "iq2xs" ? metal_kq_gemv_iq2xs_msl_entry : (vform == "iq2s" ? metal_kq_gemv_iq2s_msl_entry : (vform == "k2" ? metal_kq_gemv_k2_msl_entry : (vform == "iq4nl" ? metal_kq_gemv_iq4nl_msl_entry : (vform == "iq3xxsf4" ? metal_kq_gemv_iq3xxs_f4_msl_entry : (vform == "iq3sf4" ? metal_kq_gemv_iq3s_f4_msl_entry : (vform == "iq3xxs" ? metal_kq_gemv_iq3xxs_msl_entry : (vform == "iq3s" ? metal_kq_gemv_iq3s_msl_entry : (vform == "k3" ? metal_kq_gemv_k3_msl_entry : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl_entry : (vform == "k4" ? metal_kq_gemv_k4_msl_entry : (vform == "k5" ? metal_kq_gemv_k5_msl_entry : (vform == "k5c" ? metal_kq_gemv_k5c_msl_entry : metal_kq_gemv_k6_msl_entry)))))))))))))) + let fm = vform == "iq2xxsf4" ? metal_kq_gemv_iq2xxs_f4_msl_fastmath : (vform == "iq2xxs" ? metal_kq_gemv_iq2xxs_msl_fastmath : (vform == "iq2xs" ? metal_kq_gemv_iq2xs_msl_fastmath : (vform == "iq2s" ? metal_kq_gemv_iq2s_msl_fastmath : (vform == "k2" ? metal_kq_gemv_k2_msl_fastmath : (vform == "iq4nl" ? metal_kq_gemv_iq4nl_msl_fastmath : (vform == "iq3xxsf4" ? metal_kq_gemv_iq3xxs_f4_msl_fastmath : (vform == "iq3sf4" ? metal_kq_gemv_iq3s_f4_msl_fastmath : (vform == "iq3xxs" ? metal_kq_gemv_iq3xxs_msl_fastmath : (vform == "iq3s" ? metal_kq_gemv_iq3s_msl_fastmath : (vform == "k3" ? metal_kq_gemv_k3_msl_fastmath : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl_fastmath : (vform == "k4" ? metal_kq_gemv_k4_msl_fastmath : (vform == "k5" ? metal_kq_gemv_k5_msl_fastmath : (vform == "k5c" ? metal_kq_gemv_k5c_msl_fastmath : metal_kq_gemv_k6_msl_fastmath)))))))))))))) var pso = pipeline_from_source(dev, src, entry, fm, err) t |> success(pso != null, "{tag}: pipeline: {err}") return if (pso == null) @@ -66,17 +66,17 @@ def private kq_gemv_gate(t : T?; dev, queue; vform : string; n, d : int) { var by = buf_fill(dev, d, -1000.0) var bn = buf_u32(dev, uint(n)) var bd = buf_u32(dev, uint(d)) - let groups = vform == "k5c" ? (d + 1) / 2 : (vform == "iq3s" || vform == "iq3xxs" || vform == "k2" ? (d + 7) / 8 : (d + 3) / 4) + let groups = vform == "k5c" ? (d + 1) / 2 : (vform == "iq3sf4" || vform == "iq3xxsf4" || vform == "k2" ? (d + 7) / 8 : (d + 3) / 4) let ran = with_compute_encoder(queue, err) $(enc : MetalComputeEncoder?) { metal_set_pipeline(enc, pso) if (vform == "iq4xs") { metal_set_threadgroup_memory_length(enc, metal_kq_gemv_iq4xs_msl_tgmem, 0) // the codebook slab } - if (vform == "iq3s") { - metal_set_threadgroup_memory_length(enc, metal_kq_gemv_iq3s_msl_tgmem, 0) // the grid slab + if (vform == "iq3sf4") { + metal_set_threadgroup_memory_length(enc, metal_kq_gemv_iq3s_f4_msl_tgmem, 0) // the grid slab } - if (vform == "iq3xxs") { - metal_set_threadgroup_memory_length(enc, metal_kq_gemv_iq3xxs_msl_tgmem, 0) // the halved-grid slab + if (vform == "iq3xxsf4") { + metal_set_threadgroup_memory_length(enc, metal_kq_gemv_iq3xxs_f4_msl_tgmem, 0) // the halved-grid slab } if (vform == "iq4nl") { metal_set_threadgroup_memory_length(enc, metal_kq_gemv_iq4nl_msl_tgmem, 0) // the codebook slab @@ -747,6 +747,10 @@ def test_metal_kq_gemv_kernels(t : T?) { kq_gemv_gate(t, dev, queue, "iq3s", 512, 30) kq_gemv_gate(t, dev, queue, "iq3xxs", 1280, 11) kq_gemv_gate(t, dev, queue, "iq3xxs", 512, 30) + kq_gemv_gate(t, dev, queue, "iq3sf4", 1280, 11) + kq_gemv_gate(t, dev, queue, "iq3sf4", 512, 30) + kq_gemv_gate(t, dev, queue, "iq3xxsf4", 1280, 11) + kq_gemv_gate(t, dev, queue, "iq3xxsf4", 512, 30) kq_gemv_gate(t, dev, queue, "iq4nl", 1280, 11) kq_gemv_gate(t, dev, queue, "iq4nl", 512, 30) kq_gemv_gate(t, dev, queue, "k2", 1280, 11) From 60c15b2a219b642d32c6fd9a43ef381d8d240dec Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 01:48:00 -0700 Subject: [PATCH 12/35] densify the gemv crown-race encoders - 32 back-to-back dispatches per block A single sparse dispatch reads idle-clock times and flipped the iq2xxs f4 crown on M4 (the fixture ran at ~17 GB/s where warm kernels do ~90); the dense block keeps the governor ramped so the race verdict matches the lab's interleaved one. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014mq9AKyoAwvD5T1cigTnRw --- modules/dasLLAMA/dasllama/dasllama_metal_kernels.das | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das index 2636576cf4..b373a5822a 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das @@ -4808,7 +4808,9 @@ def private race_gemv_f4_twin(dev : MetalDevice?; queue : MetalCommandQueue?; fa kn_buffer(enc, by_base, 0ul, 4) kn_buffer(enc, bn, 0ul, 5) kn_buffer(enc, bd, 0ul, 6) - kn_dispatch(enc, uint3(uint(ndim / base_gdiv), 1u, 1u), tg) + for (_r in range(32)) { //! back-to-back dispatches keep the clock governor ramped - a single sparse dispatch reads idle-clock times and can flip the verdict + kn_dispatch(enc, uint3(uint(ndim / base_gdiv), 1u, 1u), tg) + } }, $(enc) { kn_pipeline(enc, twin_pso) kn_tgmem(enc, twin_tgmem, 0) @@ -4819,7 +4821,9 @@ def private race_gemv_f4_twin(dev : MetalDevice?; queue : MetalCommandQueue?; fa kn_buffer(enc, by_twin, 0ul, 4) kn_buffer(enc, bn, 0ul, 5) kn_buffer(enc, bd, 0ul, 6) - kn_dispatch(enc, uint3(uint(ndim / twin_gdiv), 1u, 1u), tg) + for (_r in range(32)) { + kn_dispatch(enc, uint3(uint(ndim / twin_gdiv), 1u, 1u), tg) + } }) if (res.base_ms < 0.0lf || res.twin_ms < 0.0lf) { res.winner = res.base_ms < 0.0lf ? "" : "simdgroup" From 312a92cc992627bc25438a9becee6f171d33b4a1 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 01:49:37 -0700 Subject: [PATCH 13/35] gemv crown racer: mint on the dominant decode shape (2048x8192, the ffn-up class) The mid-size square fixture crowned the M4 iq2xxs f4 twin that the served shapes reject in the lab's interleaved races; the crown must be minted on what the graph actually dispatches. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014mq9AKyoAwvD5T1cigTnRw --- modules/dasLLAMA/dasllama/dasllama_metal_kernels.das | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das index b373a5822a..8246bd7920 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das @@ -4756,8 +4756,11 @@ def private race_gemv_f4_twin(dev : MetalDevice?; queue : MetalCommandQueue?; fa twin_src, twin_entry : string; twin_fm : bool; twin_tgmem : uint64; qu_per_sb : int; base_gdiv, twin_gdiv : int) : MetalTensorRaceResult { var res = MetalTensorRaceResult(family = family, winner = "", base_ms = -1.0lf, twin_ms = -1.0lf, note = "") - let kdim = 3072 - let ndim = 4096 + //! the dominant decode site shape (ffn-up class) - the crown must be minted on what the + //! served graph dispatches; a mid-size square fixture crowned the M4 iq2xxs twin that + //! the real shapes reject + let kdim = 2048 + let ndim = 8192 let nsb = kdim / 256 let nblk = ndim * nsb var err = "" From d7fba05907ebd4c7abd86788d1d10078ef1a6585 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 01:51:39 -0700 Subject: [PATCH 14/35] gemv crown racer: burn before the alternation - race_pair's base-first order pays the clock ramp the twin rides Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014mq9AKyoAwvD5T1cigTnRw --- .../dasllama/dasllama_metal_kernels.das | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das index 8246bd7920..f5ce7aa78d 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das @@ -4802,6 +4802,29 @@ def private race_gemv_f4_twin(dev : MetalDevice?; queue : MetalCommandQueue?; fa var bd = race_uniform_u32(dev, uint(ndim)) let doff = uint64(nblk * 16) let tg = uint3(64u, 1u, 1u) + //! clock-governor burn: race_pair runs base-then-twin per round, so on a ramp-prone GPU + //! the first side pays the ramp the second rides - ~150ms of GPU work first makes the + //! alternation fair (the M4 iq2xxs crown flipped on exactly this bias) + var burn_ms = 0.0lf + var burn_err = "" + while (burn_ms < 150.0lf) { + var g = 0.0lf + let okb = with_compute_encoder_timed(queue, burn_err, g) $(enc) { + kn_pipeline(enc, base_pso) + kn_buffer(enc, bs, doff, 0) + kn_buffer(enc, bs, 0ul, 1) + kn_buffer(enc, bq, 0ul, 2) + kn_buffer(enc, bxa, 0ul, 3) + kn_buffer(enc, by_base, 0ul, 4) + kn_buffer(enc, bn, 0ul, 5) + kn_buffer(enc, bd, 0ul, 6) + for (_r in range(32)) { + kn_dispatch(enc, uint3(uint(ndim / base_gdiv), 1u, 1u), tg) + } + } + break if (!okb) + burn_ms += g + } race_pair_ms(queue, 5, res.base_ms, res.twin_ms, $(enc) { kn_pipeline(enc, base_pso) kn_buffer(enc, bs, doff, 0) From 6fd6d5ad88d567dd460cafb5206d486cb7efc859 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 02:07:56 -0700 Subject: [PATCH 15/35] gemv crown racer: overlap the y buffers - the served graph's regime, settled by e2e The same-y serialized chain crowned the M4 iq2xxs slab twin (+9% raced) that serving rejects (tg128 189.6 -> 173.3, -8.6%, measured); alternating two y buffers per side lets dispatches overlap like the real graph. Also the ptrstore lab comment loses the quoted upstream aside. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014mq9AKyoAwvD5T1cigTnRw --- .../benchmarks/matmul/bench_metal_kq_race.das | 77 ++++++++++++++++++- .../dasllama/dasllama_metal_kernels.das | 12 ++- 2 files changed, 84 insertions(+), 5 deletions(-) diff --git a/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das b/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das index 489ccb7d87..aa472e6b3f 100644 --- a/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das +++ b/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das @@ -185,6 +185,7 @@ def private mm_arms(f : KqRaceFmt) : array { // candidate mul_mm sp arms |> push(KqRaceArm(name = "tensor", src = MetalKqMulMmK4T_metal_kq_mulmm_t_msl, entry = MetalKqMulMmK4T_metal_kq_mulmm_t_msl_entry, fastmath = MetalKqMulMmK4T_metal_kq_mulmm_t_msl_fastmath, tgmem = MetalKqMulMmK4T_metal_kq_mulmm_t_msl_tgmem, gdiv = 0)) } elif (f.fmt == 5) { arms |> push(KqRaceArm(name = "tensor", src = MetalKqMulMmK5T_metal_kq_mulmm_t_msl, entry = MetalKqMulMmK5T_metal_kq_mulmm_t_msl_entry, fastmath = MetalKqMulMmK5T_metal_kq_mulmm_t_msl_fastmath, tgmem = MetalKqMulMmK5T_metal_kq_mulmm_t_msl_tgmem, gdiv = 0)) + arms |> push(KqRaceArm(name = "tbyte", src = lab_kq_mm_k5t_byte_msl, entry = lab_kq_mm_k5t_byte_msl_entry, fastmath = lab_kq_mm_k5t_byte_msl_fastmath, tgmem = lab_kq_mm_k5t_byte_msl_tgmem, gdiv = 0)) } elif (f.fmt == 6) { arms |> push(KqRaceArm(name = "tensor", src = MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl, entry = MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl_entry, fastmath = MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl_fastmath, tgmem = MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl_tgmem, gdiv = 0)) } elif (f.fmt == 44) { @@ -211,9 +212,81 @@ def private mm_arms(f : KqRaceFmt) : array { // candidate mul_mm sp let PREFILL_M = 512 // the mm tier's token batch +// k5 tensor mul_mm with the BYTE-PARALLEL QH compose (the k2/k3/k6 mul_mv trick): one masked +// uint-wide qv per 4 elements + prescaled per-byte extracts, vs the shipped per-element +// bit-compose (its tensor twin is the one mm form still at 0.78-0.82x vs llama.cpp on M5). +class LabKqMmK5TByte { + @ssbo @binding = 0 @role = "weight" @off = "s0off" ksh : array + @ssbo @binding = 1 @role = "weight" @off = "soff" ks4 : array + @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array + @ssbo @binding = 3 @off = "xoff" xf : array + @ssbo @binding = 4 @off = "yoff" y : array + @uniform @binding = 5 kdim : uint + @uniform @binding = 6 ndim : uint + @workgroup twb : float16[6144] + + [metal_kernel(name="lab_kq_mm_k5t_byte_msl", float_a_ok=true)] + def lab_kq_mm_k5t_byte { + let mBase = gl_WorkGroupID.x * 32u + let nBase = gl_WorkGroupID.y * 64u + let nkb = kdim / 32u + let nsb = kdim / 256u + let lid = gl_LocalInvocationID.x + var acc : float[8192] + var cp = unsafe(addr(y[mBase * ndim + nBase])) + tmm2d_tg_begin_deva(acc, unsafe(addr(xf[0])), 32u, 64u, 4u, 64u) + var kb = 0u + while (kb < nkb) { + var work = lid + while (work < 256u) { + let j = work >> 2u + let sub = work & 3u + let jsl = kb + (sub >> 1u) + let sb = jsl / 8u + let js = jsl % 8u + let kh = (sub & 1u) * 16u + let blk = (nBase + j) * nsb + sb + let sv = ks4[blk] + var sc = 0u + var mn = 0u + if (js < 4u) { + sc = (sv.y >> (8u * js)) & 63u + mn = (sv.z >> (8u * js)) & 63u + } else { + let j2 = js - 4u + let hi = sv.w >> (8u * j2) + sc = (hi & 15u) | (((sv.y >> (8u * j2 + 6u)) & 3u) << 4u) + mn = ((hi >> 4u) & 15u) | (((sv.z >> (8u * j2 + 6u)) & 3u) << 4u) + } + let dsc = float(ksh[blk * 8u]) * float(sc) + let dmn = float(ksh[blk * 8u + 1u]) * float(mn) + let nsh = (js & 1u) * 4u + let qb = blk * 40u + (js / 2u) * 8u + kh / 4u + let hb = blk * 40u + 32u + kh / 4u + let tb0 = j * 72u + (sub >> 1u) * 32u + kh + for [unroll_full] (k in range(4)) { + let u = kqu[qb + uint(k)] + let hu = kqu[hb + uint(k)] + let qv = ((u >> nsh) & 0x0F0F0F0F) | (((hu >> js) & 0x01010101) << 4u) + for [unroll_full] (c in range(4)) { + twb[tb0 + uint(k) * 4u + uint(c)] = float16(dsc * float((qv >> (8u * uint(c))) & 255u) - dmn) + } + } + work += gl_WorkGroupSize.x + } + barrier() + tmm2d_tg_step_deva(acc, unsafe(addr(xf[mBase * kdim + kb * 32u])), kdim, twb, 32u, 64u, 64u) + barrier() + kb += 2u + } + tmm2d_tg_store(acc, cp, 32u, 64u, ndim) + } +} + // Rewrite every threadgroup A-tile subscript store `ta[IDX] = RHS;` into the pointer form -// `*(ta + (IDX)) = RHS;` - llama.cpp's mul_mm carries the subscript form commented out as -// "massively slower.. WTF?" (mul_mm.metal); this arm races that claim on our own template. +// `*(ta + (IDX)) = RHS;` - llama.cpp's mul_mm ships the pointer form and rejects the +// subscript form as much slower; this arm races that claim on our own template +// (M5 verdict: flat - the claim does not reproduce on this compiler). def private rewrite_ta_ptr_store(src : string) : string { var out : array var n = 0 diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das index f5ce7aa78d..312c7506be 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das @@ -4798,6 +4798,10 @@ def private race_gemv_f4_twin(dev : MetalDevice?; queue : MetalCommandQueue?; fa var bxa = race_buf(dev, uint64(kdim * 4), unsafe(addr(xa[0]))) var by_base = race_buf(dev, uint64(ndim * 4), null) var by_twin = race_buf(dev, uint64(ndim * 4), null) + //! overlap ys: the served graph's consecutive gemvs write DIFFERENT buffers and overlap; + //! a same-y serialized chain misjudged the M4 iq2xxs slab twin (+9% raced, -8.6% e2e) + var by_base2 = race_buf(dev, uint64(ndim * 4), null) + var by_twin2 = race_buf(dev, uint64(ndim * 4), null) var bn = race_uniform_u32(dev, uint(kdim)) var bd = race_uniform_u32(dev, uint(ndim)) let doff = uint64(nblk * 16) @@ -4815,10 +4819,10 @@ def private race_gemv_f4_twin(dev : MetalDevice?; queue : MetalCommandQueue?; fa kn_buffer(enc, bs, 0ul, 1) kn_buffer(enc, bq, 0ul, 2) kn_buffer(enc, bxa, 0ul, 3) - kn_buffer(enc, by_base, 0ul, 4) kn_buffer(enc, bn, 0ul, 5) kn_buffer(enc, bd, 0ul, 6) for (_r in range(32)) { + kn_buffer(enc, (_r & 1) == 0 ? by_base : by_base2, 0ul, 4) kn_dispatch(enc, uint3(uint(ndim / base_gdiv), 1u, 1u), tg) } } @@ -4831,10 +4835,10 @@ def private race_gemv_f4_twin(dev : MetalDevice?; queue : MetalCommandQueue?; fa kn_buffer(enc, bs, 0ul, 1) kn_buffer(enc, bq, 0ul, 2) kn_buffer(enc, bxa, 0ul, 3) - kn_buffer(enc, by_base, 0ul, 4) kn_buffer(enc, bn, 0ul, 5) kn_buffer(enc, bd, 0ul, 6) for (_r in range(32)) { //! back-to-back dispatches keep the clock governor ramped - a single sparse dispatch reads idle-clock times and can flip the verdict + kn_buffer(enc, (_r & 1) == 0 ? by_base : by_base2, 0ul, 4) kn_dispatch(enc, uint3(uint(ndim / base_gdiv), 1u, 1u), tg) } }, $(enc) { @@ -4844,10 +4848,10 @@ def private race_gemv_f4_twin(dev : MetalDevice?; queue : MetalCommandQueue?; fa kn_buffer(enc, bs, 0ul, 1) kn_buffer(enc, bq, 0ul, 2) kn_buffer(enc, bxa, 0ul, 3) - kn_buffer(enc, by_twin, 0ul, 4) kn_buffer(enc, bn, 0ul, 5) kn_buffer(enc, bd, 0ul, 6) for (_r in range(32)) { + kn_buffer(enc, (_r & 1) == 0 ? by_twin : by_twin2, 0ul, 4) kn_dispatch(enc, uint3(uint(ndim / twin_gdiv), 1u, 1u), tg) } }) @@ -4864,6 +4868,8 @@ def private race_gemv_f4_twin(dev : MetalDevice?; queue : MetalCommandQueue?; fa metal_release(bxa) metal_release(by_base) metal_release(by_twin) + metal_release(by_base2) + metal_release(by_twin2) metal_release(bn) metal_release(bd) metal_release(base_pso) From e94c92110d11145274b94c64c731cbc5da0009a3 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 02:11:08 -0700 Subject: [PATCH 16/35] kq_gemv_iq2xxs_f4 leaves the automatic crown race - e2e is its only valid mint Every isolated regime (dense, burned, overlapped ys, the served shape) crowns the M4 twin that serving rejects (tg128 -8.6% measured); M5 serving confirms its crown (+12.6%). A mixed-stream occupancy effect no kernel race sees - the crown is minted by an e2e tg A/B per box (M5 box profile carries it). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014mq9AKyoAwvD5T1cigTnRw --- modules/dasLLAMA/dasllama/dasllama_metal_kernels.das | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das index 312c7506be..b7b4b0ab7e 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das @@ -4734,9 +4734,9 @@ def metal_tensor_race_decode : array { 64, kdim, ndim, 64 * ndim, uint3(uint(2 * (ndim / 64)), 1u, 1u), rka) $(enc, slot_y) { kn_buffer(enc, slot_y, 0ul, 4) }) - results |> emplace(race_gemv_f4_twin(dev, queue, "kq_gemv_iq2xxs_f4", - metal_kq_gemv_iq2xxs_msl, metal_kq_gemv_iq2xxs_msl_entry, metal_kq_gemv_iq2xxs_msl_fastmath, - metal_kq_gemv_iq2xxs_f4_msl, metal_kq_gemv_iq2xxs_f4_msl_entry, metal_kq_gemv_iq2xxs_f4_msl_fastmath, metal_kq_gemv_iq2xxs_f4_msl_tgmem, 16, 4, 4)) + //! kq_gemv_iq2xxs_f4 is NOT auto-raced: every isolated regime tried crowns the twin on + //! M4-class (+9%) while serving loses 8.6% with it (a mixed-stream occupancy effect no + //! kernel race sees); M5 serving gains +12.6%. That crown is minted by an e2e tg A/B only. results |> emplace(race_gemv_f4_twin(dev, queue, "kq_gemv_iq3s_f4", metal_kq_gemv_iq3s_msl, metal_kq_gemv_iq3s_msl_entry, metal_kq_gemv_iq3s_msl_fastmath, metal_kq_gemv_iq3s_f4_msl, metal_kq_gemv_iq3s_f4_msl_entry, metal_kq_gemv_iq3s_f4_msl_fastmath, metal_kq_gemv_iq3s_f4_msl_tgmem, 26, 4, 8)) From 296c5330fd0fe791179576ae79da70d51ec714eb Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 02:13:21 -0700 Subject: [PATCH 17/35] k5 tensor twins: byte-parallel QH compose - 1.4x, the last lagging mm format closes One masked uint-wide qv per 4 elements replaces the per-element bit-compose in the K45 tensor template's QH arm (both the single-tile and Db templates; the k4 non-QH path unchanged). Raced in the kq lab: 1.40-1.48x over the shipped twin -> 1.13-1.16x vs llama.cpp on M5 (was 0.78-0.82). gemm suite 2/2; the promoted twin equals the lab candidate, candidate retired. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014mq9AKyoAwvD5T1cigTnRw --- .../benchmarks/matmul/bench_metal_kq_race.das | 72 ------------------- .../dasllama/dasllama_metal_prefill.das | 50 ++++++++----- 2 files changed, 34 insertions(+), 88 deletions(-) diff --git a/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das b/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das index aa472e6b3f..1b5d4ec5a6 100644 --- a/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das +++ b/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das @@ -185,7 +185,6 @@ def private mm_arms(f : KqRaceFmt) : array { // candidate mul_mm sp arms |> push(KqRaceArm(name = "tensor", src = MetalKqMulMmK4T_metal_kq_mulmm_t_msl, entry = MetalKqMulMmK4T_metal_kq_mulmm_t_msl_entry, fastmath = MetalKqMulMmK4T_metal_kq_mulmm_t_msl_fastmath, tgmem = MetalKqMulMmK4T_metal_kq_mulmm_t_msl_tgmem, gdiv = 0)) } elif (f.fmt == 5) { arms |> push(KqRaceArm(name = "tensor", src = MetalKqMulMmK5T_metal_kq_mulmm_t_msl, entry = MetalKqMulMmK5T_metal_kq_mulmm_t_msl_entry, fastmath = MetalKqMulMmK5T_metal_kq_mulmm_t_msl_fastmath, tgmem = MetalKqMulMmK5T_metal_kq_mulmm_t_msl_tgmem, gdiv = 0)) - arms |> push(KqRaceArm(name = "tbyte", src = lab_kq_mm_k5t_byte_msl, entry = lab_kq_mm_k5t_byte_msl_entry, fastmath = lab_kq_mm_k5t_byte_msl_fastmath, tgmem = lab_kq_mm_k5t_byte_msl_tgmem, gdiv = 0)) } elif (f.fmt == 6) { arms |> push(KqRaceArm(name = "tensor", src = MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl, entry = MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl_entry, fastmath = MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl_fastmath, tgmem = MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl_tgmem, gdiv = 0)) } elif (f.fmt == 44) { @@ -212,77 +211,6 @@ def private mm_arms(f : KqRaceFmt) : array { // candidate mul_mm sp let PREFILL_M = 512 // the mm tier's token batch -// k5 tensor mul_mm with the BYTE-PARALLEL QH compose (the k2/k3/k6 mul_mv trick): one masked -// uint-wide qv per 4 elements + prescaled per-byte extracts, vs the shipped per-element -// bit-compose (its tensor twin is the one mm form still at 0.78-0.82x vs llama.cpp on M5). -class LabKqMmK5TByte { - @ssbo @binding = 0 @role = "weight" @off = "s0off" ksh : array - @ssbo @binding = 1 @role = "weight" @off = "soff" ks4 : array - @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array - @ssbo @binding = 3 @off = "xoff" xf : array - @ssbo @binding = 4 @off = "yoff" y : array - @uniform @binding = 5 kdim : uint - @uniform @binding = 6 ndim : uint - @workgroup twb : float16[6144] - - [metal_kernel(name="lab_kq_mm_k5t_byte_msl", float_a_ok=true)] - def lab_kq_mm_k5t_byte { - let mBase = gl_WorkGroupID.x * 32u - let nBase = gl_WorkGroupID.y * 64u - let nkb = kdim / 32u - let nsb = kdim / 256u - let lid = gl_LocalInvocationID.x - var acc : float[8192] - var cp = unsafe(addr(y[mBase * ndim + nBase])) - tmm2d_tg_begin_deva(acc, unsafe(addr(xf[0])), 32u, 64u, 4u, 64u) - var kb = 0u - while (kb < nkb) { - var work = lid - while (work < 256u) { - let j = work >> 2u - let sub = work & 3u - let jsl = kb + (sub >> 1u) - let sb = jsl / 8u - let js = jsl % 8u - let kh = (sub & 1u) * 16u - let blk = (nBase + j) * nsb + sb - let sv = ks4[blk] - var sc = 0u - var mn = 0u - if (js < 4u) { - sc = (sv.y >> (8u * js)) & 63u - mn = (sv.z >> (8u * js)) & 63u - } else { - let j2 = js - 4u - let hi = sv.w >> (8u * j2) - sc = (hi & 15u) | (((sv.y >> (8u * j2 + 6u)) & 3u) << 4u) - mn = ((hi >> 4u) & 15u) | (((sv.z >> (8u * j2 + 6u)) & 3u) << 4u) - } - let dsc = float(ksh[blk * 8u]) * float(sc) - let dmn = float(ksh[blk * 8u + 1u]) * float(mn) - let nsh = (js & 1u) * 4u - let qb = blk * 40u + (js / 2u) * 8u + kh / 4u - let hb = blk * 40u + 32u + kh / 4u - let tb0 = j * 72u + (sub >> 1u) * 32u + kh - for [unroll_full] (k in range(4)) { - let u = kqu[qb + uint(k)] - let hu = kqu[hb + uint(k)] - let qv = ((u >> nsh) & 0x0F0F0F0F) | (((hu >> js) & 0x01010101) << 4u) - for [unroll_full] (c in range(4)) { - twb[tb0 + uint(k) * 4u + uint(c)] = float16(dsc * float((qv >> (8u * uint(c))) & 255u) - dmn) - } - } - work += gl_WorkGroupSize.x - } - barrier() - tmm2d_tg_step_deva(acc, unsafe(addr(xf[mBase * kdim + kb * 32u])), kdim, twb, 32u, 64u, 64u) - barrier() - kb += 2u - } - tmm2d_tg_store(acc, cp, 32u, 64u, ndim) - } -} - // Rewrite every threadgroup A-tile subscript store `ta[IDX] = RHS;` into the pointer form // `*(ta + (IDX)) = RHS;` - llama.cpp's mul_mm ships the pointer form and rejects the // subscript form as much slower; this arm races that claim on our own template diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das b/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das index ec2b13327c..9d59b33921 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das @@ -518,14 +518,24 @@ class template MetalKqMulMmK45TensorT { } let dsc = float(ksh[blk * 8u]) * float(sc) let dmn = float(ksh[blk * 8u + 1u]) * float(mn) - for [unroll_full] (t in range(16)) { - let e = kh + uint(t) - static_if (QH) { - let u = kqu[blk * BLK + (js / 2u) * 8u + e / 4u] - let hu = kqu[blk * BLK + 32u + e / 4u] - let q = ((u >> (8u * (e % 4u) + (js & 1u) * 4u)) & 15u) | (((hu >> (8u * (e % 4u) + js)) & 1u) << 4u) - twb[j * 72u + (sub >> 1u) * 32u + e] = float16(dsc * float(q) - dmn) - } else { + static_if (QH) { + //! byte-parallel q5 compose: one masked uint-wide qv per 4 elements + //! (1.4x the per-element bit-compose it replaced, raced in the kq lab) + let nsh = (js & 1u) * 4u + let qb = blk * BLK + (js / 2u) * 8u + kh / 4u + let hb = blk * BLK + 32u + kh / 4u + let tb0 = j * 72u + (sub >> 1u) * 32u + kh + for [unroll_full] (k in range(4)) { + let u = kqu[qb + uint(k)] + let hu = kqu[hb + uint(k)] + let qv = ((u >> nsh) & 0x0F0F0F0F) | (((hu >> js) & 0x01010101) << 4u) + for [unroll_full] (c in range(4)) { + twb[tb0 + uint(k) * 4u + uint(c)] = float16(dsc * float((qv >> (8u * uint(c))) & 255u) - dmn) + } + } + } else { + for [unroll_full] (t in range(16)) { + let e = kh + uint(t) let q = (kqu[blk * BLK + (js / 2u) * 8u + e / 4u] >> (8u * (e % 4u) + (js & 1u) * 4u)) & 15u twb[j * 72u + (sub >> 1u) * 32u + e] = float16(dsc * float(q) - dmn) } @@ -619,14 +629,22 @@ class template MetalKqMulMmK45DbT { } let dsc = float(ksh[blk * 8u]) * float(sc) let dmn = float(ksh[blk * 8u + 1u]) * float(mn) - for [unroll_full] (t in range(16)) { - let e = kh + uint(t) - static_if (QH) { - let u = kqu[blk * BLK + (js / 2u) * 8u + e / 4u] - let hu = kqu[blk * BLK + 32u + e / 4u] - let q = ((u >> (8u * (e % 4u) + (js & 1u) * 4u)) & 15u) | (((hu >> (8u * (e % 4u) + js)) & 1u) << 4u) - twb[tbase + j * 72u + (sub >> 1u) * 32u + e] = float16(dsc * float(q) - dmn) - } else { + static_if (QH) { + let nsh = (js & 1u) * 4u + let qb = blk * BLK + (js / 2u) * 8u + kh / 4u + let hb = blk * BLK + 32u + kh / 4u + let tb0 = tbase + j * 72u + (sub >> 1u) * 32u + kh + for [unroll_full] (k in range(4)) { + let u = kqu[qb + uint(k)] + let hu = kqu[hb + uint(k)] + let qv = ((u >> nsh) & 0x0F0F0F0F) | (((hu >> js) & 0x01010101) << 4u) + for [unroll_full] (c in range(4)) { + twb[tb0 + uint(k) * 4u + uint(c)] = float16(dsc * float((qv >> (8u * uint(c))) & 255u) - dmn) + } + } + } else { + for [unroll_full] (t in range(16)) { + let e = kh + uint(t) let q = (kqu[blk * BLK + (js / 2u) * 8u + e / 4u] >> (8u * (e % 4u) + (js & 1u) * 4u)) & 15u twb[tbase + j * 72u + (sub >> 1u) * 32u + e] = float16(dsc * float(q) - dmn) } From bd042d0f3f5aa8c2a64fb8449f2f3186fe5838d2 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 02:39:51 -0700 Subject: [PATCH 18/35] iq2/iq3 gemv compose: float-expand + sign-select replaces the integer sw/sx chain - the whole family beats llama.cpp on M4-class The lab decomposed llama.cpp's iq3xxs recipe one lever at a time on the d1 map: threadgroup-staged grid words (flat), a staged ksigns table (+5%), their nr0=4 geometry (-4%) - and none of it was the gap. The gap was our compose: iq3s_sw/sx spend ~20 integer ops + 4 sign-extending extracts per 4 elements, where the f4 twin's arithmetic (float-expand the grid word, multiply a select-built sign vector) does it in half. Feeding that arithmetic straight from the constant table - no slab, no barrier - wins everywhere the chain ran: iq3s, iq3xxs, iq2s, iq2xxs, and iq2xs (which also drops its 16 KB threadgroup slab and its tgmem dispatch row). Race deltas vs the old bases at w13_1b (2048x8192): M4 +32/+41/+37/+43/+23%, M5 +16/+24/+1/+26/+21% (iq3s/iq3xxs/iq2s/iq2xxs/iq2xs). Every iq2/iq3 format now clears llama.cpp on M4-class at the gemv tier (1.03-1.45x); M4 iq2xxs serving confirms e2e: tg128 189.6 -> 235.1 (+24%). Crown court after the promotion (M5): iq3s_f4 +5.5% and iq3xxs_f4 +20.5% keep their crowns; the iq2xxs_f4 e2e A/B re-ran at 519.8 vs 516.5 - the crown holds by a thread and stays e2e-minted. gemv+gemm suites 2/2 on both boxes (via DAS_TUNE_POLICY=reference: the committed m5.tune.json pins dot_q8kv=vec4_u2, a perm the current grid no longer offers - pre-existing, owed to the box re-mint deliverable). Lab probe classes retired after promotion; the stale pre-d1 class comments and the crown racer's _r lint fixed in passing. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014mq9AKyoAwvD5T1cigTnRw --- .../dasllama/dasllama_metal_kernels.das | 98 +++++++++++-------- .../tests/test_metal_gemv_kernels.das | 3 - 2 files changed, 56 insertions(+), 45 deletions(-) diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das index b7b4b0ab7e..5cceab480a 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das @@ -3816,7 +3816,7 @@ def private kq_b8_on : bool { return g_kq_b8_mode == 1 } -def enc_kq_mvb(enc : MetalComputeEncoder?; fmt : KqFmt; bq : MetalBuffer?; qoff : uint64; // nolint:STYLE037 — the flat per-format ladder (one arm per kq format) is the honest shape +def enc_kq_mvb(enc : MetalComputeEncoder?; fmt : KqFmt; bq : MetalBuffer?; qoff : uint64; // nolint:STYLE037,STYLE038 — the flat per-format ladder (one arm per kq format) is the honest shape bs : MetalBuffer ?; soff, doff : uint64; bx, by : MetalBuffer?; yoff : uint64; var ka : KqMvArgs; rows, nlive : int64) { let two = nlive == 2l @@ -4821,8 +4821,8 @@ def private race_gemv_f4_twin(dev : MetalDevice?; queue : MetalCommandQueue?; fa kn_buffer(enc, bxa, 0ul, 3) kn_buffer(enc, bn, 0ul, 5) kn_buffer(enc, bd, 0ul, 6) - for (_r in range(32)) { - kn_buffer(enc, (_r & 1) == 0 ? by_base : by_base2, 0ul, 4) + for (r in range(32)) { + kn_buffer(enc, (r & 1) == 0 ? by_base : by_base2, 0ul, 4) kn_dispatch(enc, uint3(uint(ndim / base_gdiv), 1u, 1u), tg) } } @@ -4837,8 +4837,8 @@ def private race_gemv_f4_twin(dev : MetalDevice?; queue : MetalCommandQueue?; fa kn_buffer(enc, bxa, 0ul, 3) kn_buffer(enc, bn, 0ul, 5) kn_buffer(enc, bd, 0ul, 6) - for (_r in range(32)) { //! back-to-back dispatches keep the clock governor ramped - a single sparse dispatch reads idle-clock times and can flip the verdict - kn_buffer(enc, (_r & 1) == 0 ? by_base : by_base2, 0ul, 4) + for (r in range(32)) { //! back-to-back dispatches keep the clock governor ramped - a single sparse dispatch reads idle-clock times and can flip the verdict + kn_buffer(enc, (r & 1) == 0 ? by_base : by_base2, 0ul, 4) kn_dispatch(enc, uint3(uint(ndim / base_gdiv), 1u, 1u), tg) } }, $(enc) { @@ -4850,8 +4850,8 @@ def private race_gemv_f4_twin(dev : MetalDevice?; queue : MetalCommandQueue?; fa kn_buffer(enc, bxa, 0ul, 3) kn_buffer(enc, bn, 0ul, 5) kn_buffer(enc, bd, 0ul, 6) - for (_r in range(32)) { - kn_buffer(enc, (_r & 1) == 0 ? by_twin : by_twin2, 0ul, 4) + for (r in range(32)) { + kn_buffer(enc, (r & 1) == 0 ? by_twin : by_twin2, 0ul, 4) kn_dispatch(enc, uint3(uint(ndim / twin_gdiv), 1u, 1u), tg) } }) @@ -8685,9 +8685,9 @@ class MetalKqGemvIq4nl { } } -// IQ3_S: w = d*sc*(+-grid byte); the 2 KB grid in threadgroup memory, a -// 16-lane half-simdgroup per superblock, lane (bu, il) = block bu's half il: one qs word = 4 -// grid words = 16 elems = 4 CONSECUTIVE float4 x loads. Scale plane = the iq4xs/k6 split form. +// IQ3_S: w = d*sc*(+-grid byte); constant-grid words float-expanded in place, sign nibbles +// from the block's sign plane, 2 rows per simdgroup, dispatch rows/4. Scale plane = the +// iq4xs/k6 split form. [metal_dispatch(name = "enc_kq_iq3s_c", pso = "g_pso_kq_iq3s", tg = 64, grid = "rows/4", params = "rows : int64, n : int64")] class MetalKqGemvIq3s { @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d plane — the scale buffer bound at byte nsb*16 (s0off = the caller's doff) @@ -8724,8 +8724,10 @@ class MetalKqGemvIq3s { let b3 = uint(h) * 4u + uint(k) let idx = ((qsw >> (8u * uint(k))) & 255u) | ((qh << (8u - b3)) & 256u) let nib = (sgw >> ((b3 / 2u) * 8u + (b3 % 2u) * 4u)) & 15u - let sw = iq3s_sw(iq3s_gw(int(idx)), nib) - accv += yv[h * 4 + k] * float4(iq3s_sx(sw, 0u), iq3s_sx(sw, 1u), iq3s_sx(sw, 2u), iq3s_sx(sw, 3u)) + let gw = iq3s_gw(int(idx)) + let fs = float4((nib & 1u) != 0u ? -1.0 : 1.0, (nib & 2u) != 0u ? -1.0 : 1.0, + (nib & 4u) != 0u ? -1.0 : 1.0, (nib & 8u) != 0u ? -1.0 : 1.0) + accv += yv[h * 4 + k] * (float4(float(gw & 255u), float((gw >> 8u) & 255u), float((gw >> 16u) & 255u), float(gw >> 24u)) * fs) } } let sc = float(((int(kscb[blk * 16u + it]) & 255) ^ 128) - 128) @@ -8808,9 +8810,9 @@ class MetalKqGemvIq3sF4 { } } -// IQ2_S: the iq3s GEMV geometry (4 rows per simdgroup, dispatch rows/8) over the u64 grid read -// DIRECT off the hoisted constant table; per-16 UNSIGNED strips scale each 16-element half, -// d pre-eighth-ed at transcode - the fold is d8 * strip * signed_grid_byte. +// IQ2_S: the iq3s GEMV geometry (2 rows per simdgroup, dispatch rows/4) over the u64 grid pair +// read DIRECT off the constant table and float-expanded in place; per-16 UNSIGNED strips scale +// each 16-element half, d pre-eighth-ed at transcode - the fold is d8 * strip * signed_grid_byte. [metal_dispatch(name = "enc_kq_iq2s_c", pso = "g_pso_kq_iq2s", tg = 64, grid = "rows/4", params = "rows : int64, n : int64")] class MetalKqGemvIq2s { @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d8 plane - the scale buffer bound at byte nsb*16 (s0off = the caller's doff) @@ -8848,10 +8850,18 @@ class MetalKqGemvIq2s { let l = uint(h) * 2u + uint(li) let idx2 = (((qwi >> (8u * l)) & 255u) | ((qh << (8u - 2u * l)) & 0x300u)) * 2u let sgb = (sgw >> (8u * l)) & 255u - let w0 = iq3s_sw(iq2s_gw(int(idx2)), sgb & 15u) - let w1 = iq3s_sw(iq2s_gw(int(idx2 + 1u)), sgb >> 4u) - accv += yv[l * 2u] * float4(iq3s_sx(w0, 0u), iq3s_sx(w0, 1u), iq3s_sx(w0, 2u), iq3s_sx(w0, 3u)) - accv += yv[l * 2u + 1u] * float4(iq3s_sx(w1, 0u), iq3s_sx(w1, 1u), iq3s_sx(w1, 2u), iq3s_sx(w1, 3u)) + let s0 = sgb & 15u + let s1 = sgb >> 4u + let w0 = iq2s_gw(int(idx2)) + let w1 = iq2s_gw(int(idx2 + 1u)) + let m0 = float4(float(w0 & 255u), float((w0 >> 8u) & 255u), float((w0 >> 16u) & 255u), float(w0 >> 24u)) + let m1 = float4(float(w1 & 255u), float((w1 >> 8u) & 255u), float((w1 >> 16u) & 255u), float(w1 >> 24u)) + let f0 = float4((s0 & 1u) != 0u ? -m0.x : m0.x, (s0 & 2u) != 0u ? -m0.y : m0.y, + (s0 & 4u) != 0u ? -m0.z : m0.z, (s0 & 8u) != 0u ? -m0.w : m0.w) + let f1 = float4((s1 & 1u) != 0u ? -m1.x : m1.x, (s1 & 2u) != 0u ? -m1.y : m1.y, + (s1 & 4u) != 0u ? -m1.z : m1.z, (s1 & 8u) != 0u ? -m1.w : m1.w) + accv += yv[l * 2u] * f0 + accv += yv[l * 2u + 1u] * f1 } half_sum[h] = float(kscb[blk * 16u + it * 2u + uint(h)]) * (accv.x + accv.y + accv.z + accv.w) } @@ -8868,10 +8878,10 @@ class MetalKqGemvIq2s { } } -// IQ2_XS: the iq2s GEMV geometry over the parity-signed u16 words - per (block, half) one u32 -// carries two u16 qs words; 9-bit index doubles into the grid pair, ksign7m recomputes the -// sign byte (no sign plane, no qh); per-16 UNSIGNED strips, d pre-eighth-ed. -[metal_dispatch(name = "enc_kq_iq2xs_c", pso = "g_pso_kq_iq2xs", tgmem = "metal_kq_gemv_iq2xs_msl_tgmem", tg = 64, grid = "rows/4", params = "rows : int64, n : int64")] +// IQ2_XS: the iq2s GEMV geometry over the parity-signed u16 words - one u32 = two u16 qs +// words; 9-bit index doubles into the constant grid pair (float-expanded in place, no slab), +// ksign7m recomputes the sign byte (no sign plane, no qh); per-16 UNSIGNED strips, d eighth-ed. +[metal_dispatch(name = "enc_kq_iq2xs_c", pso = "g_pso_kq_iq2xs", tg = 64, grid = "rows/4", params = "rows : int64, n : int64")] class MetalKqGemvIq2xs { @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d8 plane - the scale buffer bound at byte nsb*16 (s0off = the caller's doff) @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // 16B strips: 16 UNSIGNED per-16 sub-scales, byte view (same buffer at 0) @@ -8880,17 +8890,9 @@ class MetalKqGemvIq2xs { @ssbo @binding = 4 @role = "write" @off = "yoff" @span = "rows*4" y : array @uniform @binding = 5 ndim : uint @uniform @binding = 6 ddim : uint - @workgroup gridf : float4[1024] // pre-expanded magnitudes, one float4 per grid word (16 KB) [metal_kernel(name="metal_kq_gemv_iq2xs_msl")] def metal_kq_gemv_iq2xs { - let lid = gl_LocalInvocationID.x - for [unroll_full] (gk in range(16)) { - let gi = lid + uint(gk) * 64u - let w = iq2xs_gw(int(gi)) - gridf[gi] = float4(float(w & 255u), float((w >> 8u) & 255u), float((w >> 16u) & 255u), float(w >> 24u)) - } - barrier() let lane = gl_SubgroupInvocationID let ix = lane / 8u let it = lane % 8u @@ -8916,8 +8918,10 @@ class MetalKqGemvIq2xs { let sgn = ksign7m(w16 >> 9u) let s0 = sgn & 15u let s1 = sgn >> 4u - let m0 = gridf[idx2] - let m1 = gridf[idx2 + 1u] + let w0 = iq2xs_gw(int(idx2)) + let w1 = iq2xs_gw(int(idx2 + 1u)) + let m0 = float4(float(w0 & 255u), float((w0 >> 8u) & 255u), float((w0 >> 16u) & 255u), float(w0 >> 24u)) + let m1 = float4(float(w1 & 255u), float((w1 >> 8u) & 255u), float((w1 >> 16u) & 255u), float(w1 >> 24u)) let f0 = float4((s0 & 1u) != 0u ? -m0.x : m0.x, (s0 & 2u) != 0u ? -m0.y : m0.y, (s0 & 4u) != 0u ? -m0.z : m0.z, (s0 & 8u) != 0u ? -m0.w : m0.w) let f1 = float4((s1 & 1u) != 0u ? -m1.x : m1.x, (s1 & 2u) != 0u ? -m1.y : m1.y, @@ -8976,10 +8980,18 @@ class MetalKqGemvIq2xxs { for [unroll_full] (q3 in range(4)) { let bidx = (gw >> (8u * uint(q3))) & 255u let sgn = ksign7m((aux >> (7u * uint(q3))) & 127u) - let w0 = iq3s_sw(iq2xxs_gw(int(bidx * 2u)), sgn & 15u) - let w1 = iq3s_sw(iq2xxs_gw(int(bidx * 2u + 1u)), sgn >> 4u) - accv += yv[q3 * 2] * float4(iq3s_sx(w0, 0u), iq3s_sx(w0, 1u), iq3s_sx(w0, 2u), iq3s_sx(w0, 3u)) - accv += yv[q3 * 2 + 1] * float4(iq3s_sx(w1, 0u), iq3s_sx(w1, 1u), iq3s_sx(w1, 2u), iq3s_sx(w1, 3u)) + let s0 = sgn & 15u + let s1 = sgn >> 4u + let w0 = iq2xxs_gw(int(bidx * 2u)) + let w1 = iq2xxs_gw(int(bidx * 2u + 1u)) + let m0 = float4(float(w0 & 255u), float((w0 >> 8u) & 255u), float((w0 >> 16u) & 255u), float(w0 >> 24u)) + let m1 = float4(float(w1 & 255u), float((w1 >> 8u) & 255u), float((w1 >> 16u) & 255u), float(w1 >> 24u)) + let f0 = float4((s0 & 1u) != 0u ? -m0.x : m0.x, (s0 & 2u) != 0u ? -m0.y : m0.y, + (s0 & 4u) != 0u ? -m0.z : m0.z, (s0 & 8u) != 0u ? -m0.w : m0.w) + let f1 = float4((s1 & 1u) != 0u ? -m1.x : m1.x, (s1 & 2u) != 0u ? -m1.y : m1.y, + (s1 & 4u) != 0u ? -m1.z : m1.z, (s1 & 8u) != 0u ? -m1.w : m1.w) + accv += yv[q3 * 2] * f0 + accv += yv[q3 * 2 + 1] * f1 } let sc = float(kscb[blk * 16u + it]) sumf[r] += float(kdh[blk]) * sc * (accv.x + accv.y + accv.z + accv.w) @@ -9064,9 +9076,9 @@ class MetalKqGemvIq2xxsF4 { } } -// IQ3_XXS: the iq3s GEMV shape verbatim over the 24-word rows - the 1 KB halved grid as a -// float4 magnitude slab, signs from the block's aux32 through the parity helper (no table), -// the same (2ls+1)-strip x halved-d fold. 4 rows per simdgroup, dispatch rows/8. +// IQ3_XXS: the iq3s GEMV shape verbatim over the 24-word rows - constant-grid words +// float-expanded in place, signs from the block's aux32 through the parity helper (no table), +// the same (2ls+1)-strip x halved-d fold. 2 rows per simdgroup, dispatch rows/4. [metal_dispatch(name = "enc_kq_iq3xxs_c", pso = "g_pso_kq_iq3xxs", tg = 64, grid = "rows/4", params = "rows : int64, n : int64")] class MetalKqGemvIq3xxs { @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d plane — the scale buffer bound at byte nsb*16 (s0off = the caller's doff) @@ -9102,8 +9114,10 @@ class MetalKqGemvIq3xxs { let e8 = uint(h) * 2u + uint(k) / 2u let sgb = ksign7m((aux >> (7u * e8)) & 127u) let nib = uint(k) % 2u == 0u ? sgb & 15u : sgb >> 4u - let sw = iq3s_sw(iq3xxs_gw(int((qsw >> (8u * uint(k))) & 255u)), nib) - accv += yv[h * 4 + k] * float4(iq3s_sx(sw, 0u), iq3s_sx(sw, 1u), iq3s_sx(sw, 2u), iq3s_sx(sw, 3u)) + let gw = iq3xxs_gw(int((qsw >> (8u * uint(k))) & 255u)) + let fs = float4((nib & 1u) != 0u ? -1.0 : 1.0, (nib & 2u) != 0u ? -1.0 : 1.0, + (nib & 4u) != 0u ? -1.0 : 1.0, (nib & 8u) != 0u ? -1.0 : 1.0) + accv += yv[h * 4 + k] * (float4(float(gw & 255u), float((gw >> 8u) & 255u), float((gw >> 16u) & 255u), float(gw >> 24u)) * fs) } } let sc = float(((int(kscb[blk * 16u + it]) & 255) ^ 128) - 128) diff --git a/modules/dasLLAMA/tests/test_metal_gemv_kernels.das b/modules/dasLLAMA/tests/test_metal_gemv_kernels.das index 1fe7820947..02afd49769 100644 --- a/modules/dasLLAMA/tests/test_metal_gemv_kernels.das +++ b/modules/dasLLAMA/tests/test_metal_gemv_kernels.das @@ -81,9 +81,6 @@ def private kq_gemv_gate(t : T?; dev, queue; vform : string; n, d : int) { if (vform == "iq4nl") { metal_set_threadgroup_memory_length(enc, metal_kq_gemv_iq4nl_msl_tgmem, 0) // the codebook slab } - if (vform == "iq2xs") { - metal_set_threadgroup_memory_length(enc, metal_kq_gemv_iq2xs_msl_tgmem, 0) // the f4 magnitude slab - } if (vform == "iq2xxsf4") { metal_set_threadgroup_memory_length(enc, metal_kq_gemv_iq2xxs_f4_msl_tgmem, 0) // the f4 magnitude slab } From d641eecd70e60651b7f76e00ea5a12949d8061d1 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 03:01:54 -0700 Subject: [PATCH 19/35] mvb batch-decode twins get the cvt compose; the race lab grows an mvb tier The gemv-promoted float-expand + sign-select compose lands in the ten batch-decode kernels of the five grid formats - the B2/B4 templates and the B8 X-panel kernels for iq3s, iq3xxs, iq2s, iq2xs, iq2xxs. Raced at B2 (the most compose-sensitive width - the w vector amortizes over only two columns, and the MTP-served one): M5 +11-19%, M4 +3-7%, all ten cells oracle-clean; B4/B8 share the identical per-superblock compose and only amortize it wider. The lab's new mvb tier races the 2-column batch kernels on the same plane/oracle plumbing - gdiv 8 reuses the gemv dispatch shape, KqMvArgs rides binding 5 as a 4-word buffer, and the 2-column oracle layout matches y[col*ys+row] verbatim. No lcpp reference rows by design: their bs 2-8 runs the GEMV once per column for these formats, so there is nothing to pin. Also closes the fastmath-asymmetry ledger item: the k4 gemv raced fastmath=false (llama.cpp's compile mode) vs our default on both boxes - 0.99-1.00x flat; the asymmetry neither hides the M4 k4 tail nor inflates any win. Lab cvt twins retired after promotion; gemv suite (incl. the B2/B4/B8 oracle cells) 2/2 on both boxes. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014mq9AKyoAwvD5T1cigTnRw --- .../benchmarks/matmul/bench_metal_kq_race.das | 32 ++++- .../dasllama/dasllama_metal_kernels.das | 110 ++++++++++++------ 2 files changed, 106 insertions(+), 36 deletions(-) diff --git a/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das b/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das index 1b5d4ec5a6..b0bb102da2 100644 --- a/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das +++ b/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das @@ -41,7 +41,7 @@ struct Args { @clarg_doc = "Substring filter over format names, comma list (empty = all)" fmts : string - @clarg_doc = "Tiers to race: gemv (decode) and/or mm (512-token prefill), comma list" + @clarg_doc = "Tiers to race: gemv (decode), mm (512-token prefill), mvb (2-column batch decode; grid formats only, no lcpp rows - their bs 2-8 runs the GEMV per column), comma list" tiers : string = "gemv,mm" @clarg_doc = "Substring filter over shape names, comma list (empty = all)" @@ -150,6 +150,22 @@ def private gemv_arms(f : KqRaceFmt) : array { // candidate GEMV sp return <- arms } +def private mvb_arms(f : KqRaceFmt) : array { // the B2 batch-decode pair per grid format: prod vs the cvt-compose candidate + var arms : array + if (f.fmt == 33) { + arms |> push(KqRaceArm(name = "prod", src = MetalKqMvB2Iq3s_metal_kq_mv_iq3s_msl, entry = MetalKqMvB2Iq3s_metal_kq_mv_iq3s_msl_entry, fastmath = MetalKqMvB2Iq3s_metal_kq_mv_iq3s_msl_fastmath, tgmem = 0ul, gdiv = 8)) + } elif (f.fmt == 34) { + arms |> push(KqRaceArm(name = "prod", src = MetalKqMvB2Iq3xxs_metal_kq_mv_iq3xxs_msl, entry = MetalKqMvB2Iq3xxs_metal_kq_mv_iq3xxs_msl_entry, fastmath = MetalKqMvB2Iq3xxs_metal_kq_mv_iq3xxs_msl_fastmath, tgmem = 0ul, gdiv = 8)) + } elif (f.fmt == 23) { + arms |> push(KqRaceArm(name = "prod", src = MetalKqMvB2Iq2s_metal_kq_mv_iq2s_msl, entry = MetalKqMvB2Iq2s_metal_kq_mv_iq2s_msl_entry, fastmath = MetalKqMvB2Iq2s_metal_kq_mv_iq2s_msl_fastmath, tgmem = 0ul, gdiv = 8)) + } elif (f.fmt == 24) { + arms |> push(KqRaceArm(name = "prod", src = MetalKqMvB2Iq2xs_metal_kq_mv_iq2xs_msl, entry = MetalKqMvB2Iq2xs_metal_kq_mv_iq2xs_msl_entry, fastmath = MetalKqMvB2Iq2xs_metal_kq_mv_iq2xs_msl_fastmath, tgmem = 0ul, gdiv = 8)) + } elif (f.fmt == 25) { + arms |> push(KqRaceArm(name = "prod", src = MetalKqMvB2Iq2xxs_metal_kq_mv_iq2xxs_msl, entry = MetalKqMvB2Iq2xxs_metal_kq_mv_iq2xxs_msl_entry, fastmath = MetalKqMvB2Iq2xxs_metal_kq_mv_iq2xxs_msl_fastmath, tgmem = 0ul, gdiv = 8)) + } + return <- arms +} + def private mm_arms(f : KqRaceFmt) : array { // candidate mul_mm spellings register here, after prod var arms : array if (f.fmt == 4) { @@ -315,12 +331,13 @@ def private build_oracle(fmt, n, d, nsb : int; kq, ks : array; xf : array def private race_cell(queue, dev; f : KqRaceFmt; tier, shname : string; n, d : int; arms : array; args : Args; ref_tab : table) : int { let is_mm = tier == "mm" + let is_mvb = tier == "mvb" // 2-column batch decode: same dispatch shape as gemv at gdiv 8, ka args at binding 5 let nb = n / 256 let nsb = d * nb var kq : array var ks : array kq_fill_planes(f.fmt, nsb, kq, ks) - let xrows = is_mm ? PREFILL_M : 1 + let xrows = is_mm ? PREFILL_M : (is_mvb ? 2 : 1) var xf : array xf |> resize(xrows * n) for (i in range(xrows * n)) { @@ -334,7 +351,8 @@ def private race_cell(queue, dev; f : KqRaceFmt; tier, shname : string; n, d : i var bks = buf_upload(dev, ks) var bkq = buf_upload(dev, kq) var bx = buf_upload(dev, xf) - var b5 = buf_u32(dev, uint(n)) + var kav <- [uint(n), uint(d), uint(d), 2u] // KqMvArgs{ndim, ddim, ys, nr} as its raw words + var b5 = is_mvb ? buf_upload(dev, kav) : buf_u32(dev, uint(n)) var b6 = buf_u32(dev, uint(d)) let sc_off = f.split ? uint64(nsb * 16) : 0ul let wbytes = double(length(kq) + length(ks)) @@ -453,6 +471,7 @@ def private race_cell(queue, dev; f : KqRaceFmt; tier, shname : string; n, d : i delete kq delete ks delete xf + delete kav delete want delete env delete live @@ -486,6 +505,13 @@ def private main_apple(args : Args) { rows += race_cell(queue, dev, f, "mm", sh._0, sh._1, sh._2, arms, args, ref_tab) delete arms } + if (want(args.tiers, "mvb")) { + var arms <- mvb_arms(f) + if (!empty(arms)) { + rows += race_cell(queue, dev, f, "mvb", sh._0, sh._1, sh._2, arms, args, ref_tab) + } + delete arms + } } } metal_release(queue) diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das index 5cceab480a..571ba43d23 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das @@ -9236,8 +9236,10 @@ class template MetalKqMvIq3xxsT { let q3 = uint(h) * 2u + uint(wi) / 2u let sgb = ksign7m((aux >> (7u * q3)) & 127u) let nib = uint(wi) % 2u == 0u ? sgb & 15u : sgb >> 4u - let sw = iq3s_sw(iq3xxs_gw(int((qsw >> (8u * uint(wi))) & 255u)), nib) - w[h * 4 + wi] = float4(iq3s_sx(sw, 0u), iq3s_sx(sw, 1u), iq3s_sx(sw, 2u), iq3s_sx(sw, 3u)) * dl + let gw = iq3xxs_gw(int((qsw >> (8u * uint(wi))) & 255u)) + let fsd = float4((nib & 1u) != 0u ? -dl : dl, (nib & 2u) != 0u ? -dl : dl, + (nib & 4u) != 0u ? -dl : dl, (nib & 8u) != 0u ? -dl : dl) + w[h * 4 + wi] = float4(float(gw & 255u), float((gw >> 8u) & 255u), float((gw >> 16u) & 255u), float(gw >> 24u)) * fsd } } let cx4 = sb * 64u + tx * 8u @@ -9342,8 +9344,10 @@ class MetalKqMvB8Iq3xxs { let q3 = uint(h) * 2u + uint(wi) / 2u let sgb = ksign7m((aux >> (7u * q3)) & 127u) let nib = uint(wi) % 2u == 0u ? sgb & 15u : sgb >> 4u - let sw = iq3s_sw(iq3xxs_gw(int((qsw >> (8u * uint(wi))) & 255u)), nib) - w[h * 4 + wi] = float4(iq3s_sx(sw, 0u), iq3s_sx(sw, 1u), iq3s_sx(sw, 2u), iq3s_sx(sw, 3u)) * dl + let gw = iq3xxs_gw(int((qsw >> (8u * uint(wi))) & 255u)) + let fsd = float4((nib & 1u) != 0u ? -dl : dl, (nib & 2u) != 0u ? -dl : dl, + (nib & 4u) != 0u ? -dl : dl, (nib & 8u) != 0u ? -dl : dl) + w[h * 4 + wi] = float4(float(gw & 255u), float((gw >> 8u) & 255u), float((gw >> 16u) & 255u), float(gw >> 24u)) * fsd } } let cxl = tx * 8u @@ -9367,7 +9371,7 @@ class MetalKqMvB8Iq3xxs { } // The iq3s B2/B4 pair: the iq4xs shell over the per-32-block lane map — lane tx owns block tx, -// 8 grid words = 32 consecutive elems as float4s; the grid read direct (unmeasured batch shapes). +// 8 grid words = 32 consecutive elems as float4s; the grid read direct. [ |> template_struct_instance] class template MetalKqMvIq3sT { @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d plane — the scale buffer bound at byte nsb*16 @@ -9403,8 +9407,10 @@ class template MetalKqMvIq3sT { let b = uint(h) * 4u + uint(wi) let idx = ((qsw >> (8u * uint(wi))) & 255u) | ((qh << (8u - b)) & 256u) let nib = (sgw >> ((b / 2u) * 8u + (b % 2u) * 4u)) & 15u - let sw = iq3s_sw(iq3s_gw(int(idx)), nib) - w[h * 4 + wi] = float4(iq3s_sx(sw, 0u), iq3s_sx(sw, 1u), iq3s_sx(sw, 2u), iq3s_sx(sw, 3u)) * dl + let gw = iq3s_gw(int(idx)) + let fsd = float4((nib & 1u) != 0u ? -dl : dl, (nib & 2u) != 0u ? -dl : dl, + (nib & 4u) != 0u ? -dl : dl, (nib & 8u) != 0u ? -dl : dl) + w[h * 4 + wi] = float4(float(gw & 255u), float((gw >> 8u) & 255u), float((gw >> 16u) & 255u), float(gw >> 24u)) * fsd } } let cx4 = sb * 64u + tx * 8u @@ -9510,8 +9516,10 @@ class MetalKqMvB8Iq3s { let b = uint(h) * 4u + uint(wi) let idx = ((qsw >> (8u * uint(wi))) & 255u) | ((qh << (8u - b)) & 256u) let nib = (sgw >> ((b / 2u) * 8u + (b % 2u) * 4u)) & 15u - let sw = iq3s_sw(iq3s_gw(int(idx)), nib) - w[h * 4 + wi] = float4(iq3s_sx(sw, 0u), iq3s_sx(sw, 1u), iq3s_sx(sw, 2u), iq3s_sx(sw, 3u)) * dl + let gw = iq3s_gw(int(idx)) + let fsd = float4((nib & 1u) != 0u ? -dl : dl, (nib & 2u) != 0u ? -dl : dl, + (nib & 4u) != 0u ? -dl : dl, (nib & 8u) != 0u ? -dl : dl) + w[h * 4 + wi] = float4(float(gw & 255u), float((gw >> 8u) & 255u), float((gw >> 16u) & 255u), float(gw >> 24u)) * fsd } } let cxl = tx * 8u @@ -9572,10 +9580,16 @@ class template MetalKqMvIq2sT { let idx2 = (((qw >> (8u * lu)) & 255u) | ((qh << (8u - 2u * lu)) & 0x300u)) * 2u let sgn = (sgw >> (8u * lu)) & 255u let dl = l < 2 ? dl0 : dl1 - let sw0 = iq3s_sw(iq2s_gw(int(idx2)), sgn & 15u) - let sw1 = iq3s_sw(iq2s_gw(int(idx2 + 1u)), sgn >> 4u) - w[l * 2] = float4(iq3s_sx(sw0, 0u), iq3s_sx(sw0, 1u), iq3s_sx(sw0, 2u), iq3s_sx(sw0, 3u)) * dl - w[l * 2 + 1] = float4(iq3s_sx(sw1, 0u), iq3s_sx(sw1, 1u), iq3s_sx(sw1, 2u), iq3s_sx(sw1, 3u)) * dl + let s0 = sgn & 15u + let s1 = sgn >> 4u + let w0 = iq2s_gw(int(idx2)) + let w1 = iq2s_gw(int(idx2 + 1u)) + let m0 = float4(float(w0 & 255u), float((w0 >> 8u) & 255u), float((w0 >> 16u) & 255u), float(w0 >> 24u)) + let m1 = float4(float(w1 & 255u), float((w1 >> 8u) & 255u), float((w1 >> 16u) & 255u), float(w1 >> 24u)) + w[l * 2] = float4((s0 & 1u) != 0u ? -m0.x : m0.x, (s0 & 2u) != 0u ? -m0.y : m0.y, + (s0 & 4u) != 0u ? -m0.z : m0.z, (s0 & 8u) != 0u ? -m0.w : m0.w) * dl + w[l * 2 + 1] = float4((s1 & 1u) != 0u ? -m1.x : m1.x, (s1 & 2u) != 0u ? -m1.y : m1.y, + (s1 & 4u) != 0u ? -m1.z : m1.z, (s1 & 8u) != 0u ? -m1.w : m1.w) * dl } let cx4 = sb * 64u + tx * 8u static_if (TILED) { @@ -9669,10 +9683,16 @@ class MetalKqMvB8Iq2s { let idx2 = (((qw >> (8u * lu)) & 255u) | ((qh << (8u - 2u * lu)) & 0x300u)) * 2u let sgn = (sgw >> (8u * lu)) & 255u let dl = l < 2 ? dl0 : dl1 - let sw0 = iq3s_sw(iq2s_gw(int(idx2)), sgn & 15u) - let sw1 = iq3s_sw(iq2s_gw(int(idx2 + 1u)), sgn >> 4u) - w[l * 2] = float4(iq3s_sx(sw0, 0u), iq3s_sx(sw0, 1u), iq3s_sx(sw0, 2u), iq3s_sx(sw0, 3u)) * dl - w[l * 2 + 1] = float4(iq3s_sx(sw1, 0u), iq3s_sx(sw1, 1u), iq3s_sx(sw1, 2u), iq3s_sx(sw1, 3u)) * dl + let s0 = sgn & 15u + let s1 = sgn >> 4u + let w0 = iq2s_gw(int(idx2)) + let w1 = iq2s_gw(int(idx2 + 1u)) + let m0 = float4(float(w0 & 255u), float((w0 >> 8u) & 255u), float((w0 >> 16u) & 255u), float(w0 >> 24u)) + let m1 = float4(float(w1 & 255u), float((w1 >> 8u) & 255u), float((w1 >> 16u) & 255u), float(w1 >> 24u)) + w[l * 2] = float4((s0 & 1u) != 0u ? -m0.x : m0.x, (s0 & 2u) != 0u ? -m0.y : m0.y, + (s0 & 4u) != 0u ? -m0.z : m0.z, (s0 & 8u) != 0u ? -m0.w : m0.w) * dl + w[l * 2 + 1] = float4((s1 & 1u) != 0u ? -m1.x : m1.x, (s1 & 2u) != 0u ? -m1.y : m1.y, + (s1 & 4u) != 0u ? -m1.z : m1.z, (s1 & 8u) != 0u ? -m1.w : m1.w) * dl } let cxl = tx * 8u for [unroll_full] (b in range(8)) { @@ -9732,10 +9752,16 @@ class template MetalKqMvIq2xsT { let idx2 = (w16 & 511u) * 2u let sgn = ksign7m(w16 >> 9u) let dl = l < 2 ? dl0 : dl1 - let sw0 = iq3s_sw(iq2xs_gw(int(idx2)), sgn & 15u) - let sw1 = iq3s_sw(iq2xs_gw(int(idx2 + 1u)), sgn >> 4u) - w[l * 2] = float4(iq3s_sx(sw0, 0u), iq3s_sx(sw0, 1u), iq3s_sx(sw0, 2u), iq3s_sx(sw0, 3u)) * dl - w[l * 2 + 1] = float4(iq3s_sx(sw1, 0u), iq3s_sx(sw1, 1u), iq3s_sx(sw1, 2u), iq3s_sx(sw1, 3u)) * dl + let s0 = sgn & 15u + let s1 = sgn >> 4u + let w0 = iq2xs_gw(int(idx2)) + let w1 = iq2xs_gw(int(idx2 + 1u)) + let m0 = float4(float(w0 & 255u), float((w0 >> 8u) & 255u), float((w0 >> 16u) & 255u), float(w0 >> 24u)) + let m1 = float4(float(w1 & 255u), float((w1 >> 8u) & 255u), float((w1 >> 16u) & 255u), float(w1 >> 24u)) + w[l * 2] = float4((s0 & 1u) != 0u ? -m0.x : m0.x, (s0 & 2u) != 0u ? -m0.y : m0.y, + (s0 & 4u) != 0u ? -m0.z : m0.z, (s0 & 8u) != 0u ? -m0.w : m0.w) * dl + w[l * 2 + 1] = float4((s1 & 1u) != 0u ? -m1.x : m1.x, (s1 & 2u) != 0u ? -m1.y : m1.y, + (s1 & 4u) != 0u ? -m1.z : m1.z, (s1 & 8u) != 0u ? -m1.w : m1.w) * dl } let cx4 = sb * 64u + tx * 8u static_if (TILED) { @@ -9829,10 +9855,16 @@ class MetalKqMvB8Iq2xs { let idx2 = (w16 & 511u) * 2u let sgn = ksign7m(w16 >> 9u) let dl = l < 2 ? dl0 : dl1 - let sw0 = iq3s_sw(iq2xs_gw(int(idx2)), sgn & 15u) - let sw1 = iq3s_sw(iq2xs_gw(int(idx2 + 1u)), sgn >> 4u) - w[l * 2] = float4(iq3s_sx(sw0, 0u), iq3s_sx(sw0, 1u), iq3s_sx(sw0, 2u), iq3s_sx(sw0, 3u)) * dl - w[l * 2 + 1] = float4(iq3s_sx(sw1, 0u), iq3s_sx(sw1, 1u), iq3s_sx(sw1, 2u), iq3s_sx(sw1, 3u)) * dl + let s0 = sgn & 15u + let s1 = sgn >> 4u + let w0 = iq2xs_gw(int(idx2)) + let w1 = iq2xs_gw(int(idx2 + 1u)) + let m0 = float4(float(w0 & 255u), float((w0 >> 8u) & 255u), float((w0 >> 16u) & 255u), float(w0 >> 24u)) + let m1 = float4(float(w1 & 255u), float((w1 >> 8u) & 255u), float((w1 >> 16u) & 255u), float(w1 >> 24u)) + w[l * 2] = float4((s0 & 1u) != 0u ? -m0.x : m0.x, (s0 & 2u) != 0u ? -m0.y : m0.y, + (s0 & 4u) != 0u ? -m0.z : m0.z, (s0 & 8u) != 0u ? -m0.w : m0.w) * dl + w[l * 2 + 1] = float4((s1 & 1u) != 0u ? -m1.x : m1.x, (s1 & 2u) != 0u ? -m1.y : m1.y, + (s1 & 4u) != 0u ? -m1.z : m1.z, (s1 & 8u) != 0u ? -m1.w : m1.w) * dl } let cxl = tx * 8u for [unroll_full] (b in range(8)) { @@ -9889,10 +9921,16 @@ class template MetalKqMvIq2xxsT { let lu = uint(l) let bidx = (gw >> (8u * lu)) & 255u let sgn = ksign7m((aux >> (7u * lu)) & 127u) - let sw0 = iq3s_sw(iq2xxs_gw(int(bidx * 2u)), sgn & 15u) - let sw1 = iq3s_sw(iq2xxs_gw(int(bidx * 2u + 1u)), sgn >> 4u) - w[l * 2] = float4(iq3s_sx(sw0, 0u), iq3s_sx(sw0, 1u), iq3s_sx(sw0, 2u), iq3s_sx(sw0, 3u)) * dl - w[l * 2 + 1] = float4(iq3s_sx(sw1, 0u), iq3s_sx(sw1, 1u), iq3s_sx(sw1, 2u), iq3s_sx(sw1, 3u)) * dl + let s0 = sgn & 15u + let s1 = sgn >> 4u + let w0 = iq2xxs_gw(int(bidx * 2u)) + let w1 = iq2xxs_gw(int(bidx * 2u + 1u)) + let m0 = float4(float(w0 & 255u), float((w0 >> 8u) & 255u), float((w0 >> 16u) & 255u), float(w0 >> 24u)) + let m1 = float4(float(w1 & 255u), float((w1 >> 8u) & 255u), float((w1 >> 16u) & 255u), float(w1 >> 24u)) + w[l * 2] = float4((s0 & 1u) != 0u ? -m0.x : m0.x, (s0 & 2u) != 0u ? -m0.y : m0.y, + (s0 & 4u) != 0u ? -m0.z : m0.z, (s0 & 8u) != 0u ? -m0.w : m0.w) * dl + w[l * 2 + 1] = float4((s1 & 1u) != 0u ? -m1.x : m1.x, (s1 & 2u) != 0u ? -m1.y : m1.y, + (s1 & 4u) != 0u ? -m1.z : m1.z, (s1 & 8u) != 0u ? -m1.w : m1.w) * dl } let cx4 = sb * 64u + tx * 8u static_if (TILED) { @@ -9983,10 +10021,16 @@ class MetalKqMvB8Iq2xxs { let lu = uint(l) let bidx = (gw >> (8u * lu)) & 255u let sgn = ksign7m((aux >> (7u * lu)) & 127u) - let sw0 = iq3s_sw(iq2xxs_gw(int(bidx * 2u)), sgn & 15u) - let sw1 = iq3s_sw(iq2xxs_gw(int(bidx * 2u + 1u)), sgn >> 4u) - w[l * 2] = float4(iq3s_sx(sw0, 0u), iq3s_sx(sw0, 1u), iq3s_sx(sw0, 2u), iq3s_sx(sw0, 3u)) * dl - w[l * 2 + 1] = float4(iq3s_sx(sw1, 0u), iq3s_sx(sw1, 1u), iq3s_sx(sw1, 2u), iq3s_sx(sw1, 3u)) * dl + let s0 = sgn & 15u + let s1 = sgn >> 4u + let w0 = iq2xxs_gw(int(bidx * 2u)) + let w1 = iq2xxs_gw(int(bidx * 2u + 1u)) + let m0 = float4(float(w0 & 255u), float((w0 >> 8u) & 255u), float((w0 >> 16u) & 255u), float(w0 >> 24u)) + let m1 = float4(float(w1 & 255u), float((w1 >> 8u) & 255u), float((w1 >> 16u) & 255u), float(w1 >> 24u)) + w[l * 2] = float4((s0 & 1u) != 0u ? -m0.x : m0.x, (s0 & 2u) != 0u ? -m0.y : m0.y, + (s0 & 4u) != 0u ? -m0.z : m0.z, (s0 & 8u) != 0u ? -m0.w : m0.w) * dl + w[l * 2 + 1] = float4((s1 & 1u) != 0u ? -m1.x : m1.x, (s1 & 2u) != 0u ? -m1.y : m1.y, + (s1 & 4u) != 0u ? -m1.z : m1.z, (s1 & 8u) != 0u ? -m1.w : m1.w) * dl } let cxl = tx * 8u for [unroll_full] (b in range(8)) { From e0b18aeafde7dc8370d25621d73ff1420d67f61d Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 03:07:12 -0700 Subject: [PATCH 20/35] iq3s/iq3xxs tensor twins: the cvt compose in stage16 - the last two sub-1.0 M5 mm formats cross the line The staged grid slab stays; only the per-16 compose changes - the integer sw/sx sign-apply-and-extract chain becomes float-expand of the staged word times a select-built +-dsc vector, four half stores from its lanes (the k5 byte-parallel precedent, iq3-shaped). Raced on M5 at w13_1b: iq3s 0.973 -> 1.014x lcpp, iq3xxs 0.968 -> 1.052x. With this every kq format's crowned mm form clears llama.cpp on M5. gemm suite 2/2 on both boxes; lab candidates retired after promotion. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014mq9AKyoAwvD5T1cigTnRw --- .../dasllama/dasllama_metal_prefill.das | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das b/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das index 9d59b33921..92116093c6 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das @@ -1106,10 +1106,14 @@ class template MetalKqMulMmIq3sTensorT : MetalKqMulMmSplitTensorBase { let b3 = ile * 4u + uint(k) let idx = ((qsw >> (8u * uint(k))) & 255u) | ((qh3 << (8u - b3)) & 256u) let nib = (sgw3 >> ((b3 / 2u) * 8u + (b3 % 2u) * 4u)) & 15u - let sw = iq3s_sw(gridw3t[idx], nib) - for [unroll_full] (c in range(4)) { - twb[tb0 + uint(k) * 4u + uint(c)] = float16(dsc * iq3s_sx(sw, uint(c))) - } + let gw = gridw3t[idx] + let fsd = float4((nib & 1u) != 0u ? -dsc : dsc, (nib & 2u) != 0u ? -dsc : dsc, + (nib & 4u) != 0u ? -dsc : dsc, (nib & 8u) != 0u ? -dsc : dsc) + let mv = float4(float(gw & 255u), float((gw >> 8u) & 255u), float((gw >> 16u) & 255u), float(gw >> 24u)) * fsd + twb[tb0 + uint(k) * 4u] = float16(mv.x) + twb[tb0 + uint(k) * 4u + 1u] = float16(mv.y) + twb[tb0 + uint(k) * 4u + 2u] = float16(mv.z) + twb[tb0 + uint(k) * 4u + 3u] = float16(mv.w) } } } @@ -1157,10 +1161,14 @@ class template MetalKqMulMmIq3xxsTensorT : MetalKqMulMmSplitTensorBase { let q3 = ile * 2u + uint(k) / 2u let sgb = ksign7m((aux3 >> (7u * q3)) & 127u) let nib = uint(k) % 2u == 0u ? sgb & 15u : sgb >> 4u - let sw = iq3s_sw(gridw3xt[(qsw >> (8u * uint(k))) & 255u], nib) - for [unroll_full] (c in range(4)) { - twb[tb0 + uint(k) * 4u + uint(c)] = float16(dsc * iq3s_sx(sw, uint(c))) - } + let gw = gridw3xt[(qsw >> (8u * uint(k))) & 255u] + let fsd = float4((nib & 1u) != 0u ? -dsc : dsc, (nib & 2u) != 0u ? -dsc : dsc, + (nib & 4u) != 0u ? -dsc : dsc, (nib & 8u) != 0u ? -dsc : dsc) + let mv = float4(float(gw & 255u), float((gw >> 8u) & 255u), float((gw >> 16u) & 255u), float(gw >> 24u)) * fsd + twb[tb0 + uint(k) * 4u] = float16(mv.x) + twb[tb0 + uint(k) * 4u + 1u] = float16(mv.y) + twb[tb0 + uint(k) * 4u + 2u] = float16(mv.z) + twb[tb0 + uint(k) * 4u + 3u] = float16(mv.w) } } } From 455758a960cca8f78cdfee3d1f390b593a033384 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 03:15:09 -0700 Subject: [PATCH 21/35] race lab: the ptrstore arm retires - flat 1.000x on both boxes, the claim is settled The threadgroup A-tile pointer-store spelling raced dead flat against the subscript form on M5 and M4 across every mm format; this compiler emits the same code for both. The source-rewrite arm and its narration go; the verdict lives in the arc record. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014mq9AKyoAwvD5T1cigTnRw --- .../benchmarks/matmul/bench_metal_kq_race.das | 32 ------------------- 1 file changed, 32 deletions(-) diff --git a/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das b/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das index b0bb102da2..bc39a3c658 100644 --- a/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das +++ b/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das @@ -193,8 +193,6 @@ def private mm_arms(f : KqRaceFmt) : array { // candidate mul_mm sp } else { arms |> push(KqRaceArm(name = "prod", src = MetalKqMulMmIq2xxs_metal_kq_mulmm_msl, entry = MetalKqMulMmIq2xxs_metal_kq_mulmm_msl_entry, fastmath = MetalKqMulMmIq2xxs_metal_kq_mulmm_msl_fastmath, tgmem = MetalKqMulMmIq2xxs_metal_kq_mulmm_msl_tgmem, gdiv = 0)) } - arms |> push(KqRaceArm(name = "ptrstore", src = rewrite_ta_ptr_store(arms[0].src), entry = arms[0].entry, - fastmath = arms[0].fastmath, tgmem = arms[0].tgmem, gdiv = 0)) // the existing Metal-4 tensor twins (f32-X form) - the ceiling the nine formats' twins chase; // the pipeline declines cleanly on a box without the tensor toolchain if (f.fmt == 4) { @@ -227,36 +225,6 @@ def private mm_arms(f : KqRaceFmt) : array { // candidate mul_mm sp let PREFILL_M = 512 // the mm tier's token batch -// Rewrite every threadgroup A-tile subscript store `ta[IDX] = RHS;` into the pointer form -// `*(ta + (IDX)) = RHS;` - llama.cpp's mul_mm ships the pointer form and rejects the -// subscript form as much slower; this arm races that claim on our own template -// (M5 verdict: flat - the claim does not reproduce on this compiler). -def private rewrite_ta_ptr_store(src : string) : string { - var out : array - var n = 0 - for (ln in split(src, "\n")) { - let at = find(ln, "ta[") - var done = false - if (at >= 0 && empty(strip(slice(ln, 0, at)))) { - let eq = find(ln, "] = ") - if (eq > at) { - let idxe = slice(ln, at + 3, eq) - let rhs = slice(ln, eq + 4) - out |> push("{slice(ln, 0, at)}*(ta + ({idxe})) = {rhs}") - n++ - done = true - } - } - if (!done) { - out |> push(ln) - } - } - if (n == 0) { - panic("rewrite_ta_ptr_store: no ta[...] = store found - the template spelling moved") - } - return join(out, "\n") -} - def private load_ref_table(path : string; var tab : table) { return if (empty(path)) fopen(path, "rb") $(f) { From ab130cf822b4c5b9b3da538418a28bc85284526a Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 07:09:51 -0700 Subject: [PATCH 22/35] followup_metal: the smmla leg closes - first race on M5, NEON keeps the q8q8 crown The probe gate passed (correctness OK, 2.02x MAC ceiling over sdot4), and the tile race buried it: best smmla seat ~21% behind NEON mr8_budget once the kg8 re-layout and memory traffic price in. No arm-i8mm defaults export; the smmla kq-tile emitter idea moves from mac-leverage to a Graviton-raced candidate. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014mq9AKyoAwvD5T1cigTnRw --- modules/dasLLAMA/followup_metal.md | 36 +++++++++++++----------------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/modules/dasLLAMA/followup_metal.md b/modules/dasLLAMA/followup_metal.md index b6819e92dc..4d982bb838 100644 --- a/modules/dasLLAMA/followup_metal.md +++ b/modules/dasLLAMA/followup_metal.md @@ -5,26 +5,22 @@ forks after PR-1 of `plans/unquirk_pass.md`. Metal-tier perf items move here fro per-format notes during the Phase E doc split; the CPU items below are mac-session work too - the M-series CPU tiers are minted and raced from that box. -## 1. The dark smmla (i8mm) leg - free race, never run - -`q8q8_tile_gen` ships five `dot = "smmla"` seats gated `requires = "i8mm"` (mr4/mr8 x -kstep/nrsplit/gkstep), and NO box has ever raced them: `LLVMGetHostCPUFeatures()` returns an -EMPTY string on macOS (llvm_jit_common.das documents it beside `g_target_arm64_i8mm`), so -i8mm never detected on Apple Silicon at the EMITTER tier (`g_target_arm64_i8mm`), so the -generators declined and the seats never raced - even though `cpu_supports("i8mm")` answered -correctly via sysctl. M1 lacks i8mm; M2+ has it; the M5 Max additionally has FEAT_SME2p1 + -BF16/EBF16. FIXED in the unquirk pass (PR-1): `g_target_arm64_i8mm` also consults -`cpu_supports`, and the target machine appends `+i8mm` when the host has it - an M5 `--tune` -now races smmla-vs-NEON with zero new kernel work. Mac-session order: - -1. On the M5 (post PR-1): `--tune`, confirm the smmla seats EMIT (they have never been - exercised - treat the emitter arms as unproven, gate with the gen probe TEST mode first) - and report the crowns; export the `arm-i8mm` profile if they win. -2. If smmla wins q8q8: the kq tile families have NO ARM ISA seats at all (mr8 NEON is the - whole grid) - an smmla kq tile emitter arm is the highest-leverage CPU kernel work on - the mac, and it transfers to Graviton3+ (c8g) verbatim. -3. SME/SME2 is the tier after: no seats, no emitter, new kernel design (streaming mode + - ZA tiles) - research first per the standing research-before-kernel-work rule. +## 1. The smmla (i8mm) leg - RACED on M5, NEON keeps the crown + +The five `dot = "smmla"` seats of `q8q8_tile_gen` (mr4/mr8 x kstep/nrsplit/gkstep, +`requires = "i8mm"`) raced for the first time in the 2026-09-01 M5 box mint, post the +PR-1 detection fix (`g_target_arm64_i8mm` consults `cpu_supports`, `+i8mm` appended to the +target machine). The `harness/smmla_probe.das` gate passed first: correctness OK, 2.02x +register-resident MAC throughput over sdot4 - the nominal ceiling. In the tile race that +ceiling does not survive the memory traffic plus the kg8 re-layout: best smmla seat +`mr8_kstep2_nrsplit2` at 25893 us vs NEON `mr8_budget` at 21349 us (~21% behind; full table +in the m5 sidecar's race section and `~/.tune-history/m5/`). No `arm-i8mm` defaults export - +the export was conditional on a win. + +Consequences: an smmla kq tile emitter arm is NOT mac-leverage (the q8q8 verdict transfers - +the kq tiles are more memory-bound, not less); it remains a Graviton3+/c8g candidate raced on +that silicon, not built speculatively. SME/SME2 stays the researched-first tier: no seats, no +emitter, new kernel design (streaming mode + ZA tiles). ## 2. The fixtures - which GGUF per format, and where to get it From 72739ebb827b0e1a55678330501c6ad1ce581d59 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 08:08:48 -0700 Subject: [PATCH 23/35] the tune sidecar stops being able to lie: cache-keyed, fallback-safe, and crown-confirmed on real shapes Three seams the M5 re-mint exposed, closed together: 1) Macro file dependencies in the module cache (core). A macro that reads a file beyond the sources - the [tuned]/[tune_policy] sidecar read - now pins it via add_module_cache_dependency (ast builtin): the record stores (path, size, content hash) in its header and the reader re-validates before trusting the payload, so a re-mint invalidates cached modules instead of serving stamps minted against the old file (the empty-tune_status refusal, or worse, silently stale winners). Content-hashed, not mtime'd: lcpp_bench rewrites its sidecar byte-identically on exit, and mtime gating would repay the full parse every run. Deps round-trip through deserialized records - the stream is rewritten from parsedModules, and a kept record must not shed them. Serializer version 117 -> 118. llvm_tune's read_manifest registers the pin before its staleness gate, so a sidecar that first APPEARS also invalidates. Gate test: tests/module_cache/test_macro_dep_invalidate.das (cold / warm / byte-identical rewrite stays cached / content change reparses), driven main-line because a dastest child compiles outside the module cache. 2) A stale sidecar perm falls back instead of bricking the compile. The manifest is measured data, not law: [tuned] on a sidecar-sourced perm the grid no longer offers (dot_q8kv=vec4_u2 after the grid rework) stamps the fallback with a loud line - the tuner that would re-race the entry can always run. An explicit perm= pin keeps the hard refusal (cant_tuned_bad_pin.das); the fallback arm is test_tuned's stale-sidecar cell via a spawned client. 3) The undecidable crown is minted on real shapes. kq_gemv_iq2xxs_f4 - the crown every isolated kernel race gets wrong (M4-class: +9% raced, -8.6% served) - joins the mint through confirm_serving_crown in tune_kernels: two temp manifests differing only in runtime.metal_tensor membership, lcpp_bench tg128 per arm under DAS_TUNE_MANIFEST, margin 1.005, best-of-3. First live run on M5: 521.66 vs 517.21 tok/s - crowned, matching the hand e2e verdict. No vehicle on the box = no crown (base is the safe side) plus a provision hint. This also retires the absorb-drop hazard: the crown is re-decided by every mint rather than preserved by hand. dasVulkan's shared_module rebuilt against the new Program layout (the vintage guard flagged the drift); in-tree shared modules rebuild with the tree. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014mq9AKyoAwvD5T1cigTnRw --- include/daScript/ast/ast.h | 10 ++ include/daScript/ast/ast_serializer.h | 2 +- .../dasllama/dasllama_metal_kernels.das | 6 +- modules/dasLLAMA/dasllama/dasllama_tune.das | 18 ++- modules/dasLLAMA/harness/tune_kernels.das | 110 ++++++++++++++++++ modules/dasLLVM/daslib/llvm_tune.das | 7 +- modules/dasLLVM/tests/cant_tuned_bad_pin.das | 24 ++++ modules/dasLLVM/tests/test_tuned.das | 39 +++++++ .../dasLLVM/tests/tuned_stale_perm_client.das | 23 ++++ src/ast/ast_parse.cpp | 67 ++++++++++- src/builtin/module_builtin_ast.cpp | 44 +++++++ tests/module_cache/_fixtures/mc_dep_drv.das | 11 ++ tests/module_cache/_fixtures/mc_dep_mod.das | 19 +++ tests/module_cache/_fixtures/mc_dep_used.das | 13 +++ .../test_macro_dep_invalidate.das | 81 +++++++++++++ 15 files changed, 467 insertions(+), 7 deletions(-) create mode 100644 modules/dasLLVM/tests/cant_tuned_bad_pin.das create mode 100644 modules/dasLLVM/tests/tuned_stale_perm_client.das create mode 100644 tests/module_cache/_fixtures/mc_dep_drv.das create mode 100644 tests/module_cache/_fixtures/mc_dep_mod.das create mode 100644 tests/module_cache/_fixtures/mc_dep_used.das create mode 100644 tests/module_cache/test_macro_dep_invalidate.das diff --git a/include/daScript/ast/ast.h b/include/daScript/ast/ast.h index 3e85751b73..8504780009 100644 --- a/include/daScript/ast/ast.h +++ b/include/daScript/ast/ast.h @@ -1691,8 +1691,18 @@ namespace das CodeOfPolicies policies; vector> allRequireDecl; das_hash_map astTypeInfo; + // compile-time file inputs a macro consumed beyond the sources (a tune sidecar, a config): + // (path, size, content hash) captured at registration via add_module_cache_dependency. The + // module cache stores and re-validates them, so a changed input invalidates the cached + // record instead of serving stamps minted against the old file. Content-hashed, not + // mtime'd: apps that rewrite their sidecar byte-identically on exit must not churn the cache. + vector> moduleCacheDependencies; }; + // the shared capture/validate stat for Program::moduleCacheDependencies: size -1 + hash 0 + // when the file is absent or unreadable, else (byte size, hash_block64 of the content) + DAS_API void statAndHashFileDependency ( const string & path, int64_t & size, uint64_t & hash ); + // access function from class adapter DAS_API int adapt_field_offset ( const char * fName, const StructInfo * info ); DAS_API int adapt_field_offset_ex ( const char * fName, const StructInfo * info, uint32_t & i ); diff --git a/include/daScript/ast/ast_serializer.h b/include/daScript/ast/ast_serializer.h index 642b24fe7d..f1cd6fc756 100644 --- a/include/daScript/ast/ast_serializer.h +++ b/include/daScript/ast/ast_serializer.h @@ -252,7 +252,7 @@ namespace das { AstSerializer & serializeModule ( Module & module, bool already_exists ); static constexpr uint32_t getVersion () { - return 117; // 117: Function::fromGeneric round-trips (generic-instance origin) + return 118; // 118: per-record macro file dependencies (module cache re-validates them) } void serializeProgram ( ProgramPtr program, ModuleGroup & libGroup ) noexcept; diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das index 571ba43d23..bc9123555a 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das @@ -4734,9 +4734,9 @@ def metal_tensor_race_decode : array { 64, kdim, ndim, 64 * ndim, uint3(uint(2 * (ndim / 64)), 1u, 1u), rka) $(enc, slot_y) { kn_buffer(enc, slot_y, 0ul, 4) }) - //! kq_gemv_iq2xxs_f4 is NOT auto-raced: every isolated regime tried crowns the twin on - //! M4-class (+9%) while serving loses 8.6% with it (a mixed-stream occupancy effect no - //! kernel race sees); M5 serving gains +12.6%. That crown is minted by an e2e tg A/B only. + //! kq_gemv_iq2xxs_f4 is NOT auto-raced (every isolated regime crowns the M4-class twin + //! that serving rejects at -8.6% - a mixed-stream occupancy effect no kernel race sees); + //! the mint's real-model tg A/B decides it instead (tune_kernels' confirm_serving_crown). results |> emplace(race_gemv_f4_twin(dev, queue, "kq_gemv_iq3s_f4", metal_kq_gemv_iq3s_msl, metal_kq_gemv_iq3s_msl_entry, metal_kq_gemv_iq3s_msl_fastmath, metal_kq_gemv_iq3s_f4_msl, metal_kq_gemv_iq3s_f4_msl_entry, metal_kq_gemv_iq3s_f4_msl_fastmath, metal_kq_gemv_iq3s_f4_msl_tgmem, 26, 4, 8)) diff --git a/modules/dasLLAMA/dasllama/dasllama_tune.das b/modules/dasLLAMA/dasllama/dasllama_tune.das index ba83e64231..7f759d6721 100644 --- a/modules/dasLLAMA/dasllama/dasllama_tune.das +++ b/modules/dasLLAMA/dasllama/dasllama_tune.das @@ -375,7 +375,23 @@ class private TunedMacro : AstFunctionAnnotation { return true } } - errors := "tuned: unknown perm '{suffix}' (not in grid)" + // an explicit `perm=` pin names the row the author wants - a miss is an authoring error + let av = find_arg(args, "perm") + if (av is tString) { + errors := "tuned: unknown perm '{suffix}' (not in grid)" + return false + } + // sidecar-sourced: measured data, not law - a perm the grid no longer offers falls back instead of bricking the very tuner that would re-race it + let fname = "{func.name}" + let fb = g_fallbacks?[fname] ?? DEFAULT_PERM + print("dasllama_tune: {fname}: sidecar perm '{suffix}' is not in the grid (stale mint?) - stamping the fallback '{fb}'\n") + for (perm in grid) { + if (perm.suffix == fb) { + apply_perm(func, perm) + return true + } + } + errors := "tuned: fallback perm '{fb}' not in grid" return false } } diff --git a/modules/dasLLAMA/harness/tune_kernels.das b/modules/dasLLAMA/harness/tune_kernels.das index f8660d817d..37eae1e03b 100644 --- a/modules/dasLLAMA/harness/tune_kernels.das +++ b/modules/dasLLAMA/harness/tune_kernels.das @@ -33,6 +33,9 @@ require daslib/json require daslib/json_boost require daslib/fio require daslib/strings_boost // nolint:STYLE030,LINT019 — join for the metal_tensor crown list, only inside the das_metal static_if half +require daslib/strings_convert // try_to_double - the serving-confirm tg parse +require daslib/command_line // get_das_exe - the serving-confirm child spawn +require strings require dasllama/dasllama_env // The shipped-fallback registry: dasllama_tuned_fallbacks() maps every [tuned] kernel to the @@ -2346,6 +2349,109 @@ def private metal_twin_absorb(var wins : array; var ran : bool&; r) { } } +// half a percent, best-of-3: the crown's real serving win where it wins is small (+0.6% M5) +// and its real serving loss where it loses is large (-8.6% M4) - the margin only has to +// separate those from run noise, and a spuriously minted crown costs a re-mint, not a board +let CONFIRM_TG_MARGIN = 1.005lf + +// The serving-truth crown confirm: some GEMV crowns are UNDECIDABLE by any isolated kernel +// race (kq_gemv_iq2xxs_f4: every raced regime crowns the M4-class twin that serving rejects - +// a mixed-stream occupancy effect only the real decode graph shows). Those crowns are minted +// HERE, on the real shapes: two temp manifests differing only in runtime.metal_tensor +// membership, lcpp_bench tg128 per arm under DAS_TUNE_MANIFEST, margin-decided. No vehicle +// on the box = no crown (the base kernel is the safe side) with a provision hint. +def private confirm_serving_crown(crown : string; var wins : array) { + let model = confirm_decode_model() + if (empty(model)) { + print("METAL_TWIN {crown}: serving confirm SKIPPED - no IQ2_XXS vehicle in '{models_dir_resolved()}' (DASLLAMA_CONFIRM_IQ2XXS= provisions one) - the base kernel serves\n") + return + } + print("METAL_TWIN {crown}: serving confirm on {model}\n") + let base_tg = confirm_tg_run(model, wins, "") + let twin_tg = confirm_tg_run(model, wins, crown) + if (base_tg <= 0.0lf || twin_tg <= 0.0lf) { + print("METAL_TWIN {crown}: a serving confirm arm produced no tg number (base {base_tg}, twin {twin_tg}) - the base kernel serves\n") + return + } + if (twin_tg > base_tg * CONFIRM_TG_MARGIN) { + print("METAL_TWIN {crown}: serving {twin_tg} tok/s BEATS base {base_tg} tok/s - crowned\n") + wins |> push(crown) + } else { + print("METAL_TWIN {crown}: serving {twin_tg} tok/s does not beat base {base_tg} tok/s x{CONFIRM_TG_MARGIN} - the base kernel serves\n") + } +} + +def private confirm_decode_model() : string { + let envm = get_env_variable("DASLLAMA_CONFIRM_IQ2XXS") + if (!empty(envm)) { + return envm + } + let md = models_dir_resolved() + var best = "" + dir(md) $(name : string) { + return if (find(name, "IQ2_XXS") < 0 || !ends_with(name, ".gguf") || find(name, "mmproj") >= 0) + if (empty(best) || name < best) { + best = name + } + } + return empty(best) ? "" : "{md}{best}" +} + +// one confirm arm: a minimal manifest (empty kernels - both arms stamp identical fallbacks, +// so the tg DELTA isolates the crown) whose runtime.metal_tensor carries the raced wins plus +// optionally the crown under test; lcpp_bench serves it via DAS_TUNE_MANIFEST and its tg128 +// line is the verdict +def private confirm_tg_run(model : string; wins : array; extra : string) : double { + var crowns = join(wins, ",") + if (!empty(extra)) { + crowns = empty(crowns) ? extra : "{crowns},{extra}" + } + let tmpm = "{get_das_root()}/_tune_confirm_decode_manifest.json" + if (!fwrite(tmpm, "\{\"kernels\": \{\}, \"runtime\": \{\"metal_tensor\": \"{crowns}\"\}, \"provenance\": \{\"box\": \"{tune_box_identity()}\"\}\}")) { + print("confirm: cannot write {tmpm} - is das_root writable?\n") + return 0.0lf + } + let envp = (get_platform_name() == "windows" + ? "set DAS_TUNE_MANIFEST={tmpm}&& " + : "DAS_TUNE_MANIFEST={tmpm} ") + let tmpo = "{get_das_root()}/_tune_confirm_decode_out.txt" + let cmd = "{envp}\"{get_das_exe()}\" -jit \"{get_das_root()}/modules/dasLLAMA/benchmarks/lcpp_bench.das\" -dasroot \"{get_das_root()}\" -- --for-debug-purposes -m \"{model}\" --ngl 99 -p 32 -n 128 -r 3 > \"{tmpo}\" 2>&1" + unsafe(popen(cmd) $(f) { + if (f != null) { + fread(f) + } + }) + var outp = "" + if (fexist(tmpo)) { + fopen(tmpo, "rb") $(f) { + if (f != null) { + outp = fread(f) + } + } + remove(tmpo) + } + remove(tmpm) + let tag = "tg128: " + for (ln in split(outp, "\n")) { + let ix = find(ln, tag) + if (ix >= 0) { + let rest = slice(ln, ix + length(tag)) + let sp = find(rest, " ") + return try_to_double(sp > 0 ? slice(rest, 0, sp) : rest) ?? 0.0lf + } + } + print("confirm arm ({empty(extra) ? "base" : extra}): no tg128 line - child output tail:\n") + var lns <- split(outp, "\n") + let n = length(lns) + for (i in range(max(0, n - 5), n)) { + if (!empty(lns[i])) { + print("confirm: | {lns[i]}\n") + } + } + delete lns + return 0.0lf +} + // fallback: the tuner rail drives this half explicitly (DAS_TUNE_MODE=tune) — the auto // policy must not fire a nested tuner off this program's own missing per-app sidecar //! Exit code of a `--tune-only` half whose filter matched nothing here: the wrapper fails the mint @@ -2476,6 +2582,10 @@ def main : int { // nolint:STYLE037,STYLE038 — flat one-bench-per-kernel rail; for (r in metal_tensor_race_decode()) { metal_twin_absorb(mt_wins, mt_ran, r) } + if (mt_ran) { + // the crown no isolated race can decide - minted on the real shapes instead + confirm_serving_crown("kq_gemv_iq2xxs_f4", mt_wins) + } } if (!mt_ran && !partialMint) { tune_detail("METAL_TWIN: no Metal device - tensor race skipped\n") diff --git a/modules/dasLLVM/daslib/llvm_tune.das b/modules/dasLLVM/daslib/llvm_tune.das index cd3aef2f45..3e5d46b128 100644 --- a/modules/dasLLVM/daslib/llvm_tune.das +++ b/modules/dasLLVM/daslib/llvm_tune.das @@ -356,7 +356,12 @@ def tune_sidecar_stale(path : string) : bool { def private read_manifest(path : string; var found : bool&) : table { found = false var tab : table - if (empty(path) || tune_sidecar_stale(path)) { + if (empty(path)) { + return <- tab + } + // pins the sidecar into the module cache record BEFORE the staleness gate (no-op at runtime): a re-mint or a first mint must invalidate cached [tuned]/[tune_policy] stamps + add_module_cache_dependency(path) + if (tune_sidecar_stale(path)) { return <- tab } let text = fread(path) diff --git a/modules/dasLLVM/tests/cant_tuned_bad_pin.das b/modules/dasLLVM/tests/cant_tuned_bad_pin.das new file mode 100644 index 0000000000..450d1072f4 --- /dev/null +++ b/modules/dasLLVM/tests/cant_tuned_bad_pin.das @@ -0,0 +1,24 @@ +// an explicit `perm=` pin naming a perm outside the grid is an authoring error and keeps the +// hard compile refusal - the stale-SIDECAR fallback in dasllama_tune's [tuned] must not eat it +// (the sidecar arm is the positive twin: test_tuned's stale-sidecar cell) +options gen2 +expect 20800:1 +options _dasllama_internal = true + +require dasllama/dasllama_tune + +def template bad_pin_dot_template(a, b : array) : float { + var s = 0f + for [tune = 1] (i in range(length(a))) { // nolint:PERF029 - the tuned kernel: perm hints stamp this index loop + s += a[i] * b[i] + } + return s +} + +[tuned(perm = "no_such_perm")] +def bad_pin_dot(a, b : array) : float {} + +[export] +def main { + print("never compiles\n") +} diff --git a/modules/dasLLVM/tests/test_tuned.das b/modules/dasLLVM/tests/test_tuned.das index 30fd2952a0..16000f8b67 100644 --- a/modules/dasLLVM/tests/test_tuned.das +++ b/modules/dasLLVM/tests/test_tuned.das @@ -3,6 +3,10 @@ options _dasllama_internal = true require dastest/testing_boost public require dasllama/dasllama_tune +require daslib/fio +require daslib/module_path +require strings +require llvm/daslib/llvm_tune // tune_box_identity - the stale-sidecar cell's manifest must pass the box gate // Part 2 (final assembly): one source of truth, reconstituted into the real symbol. def template sample_dot_template(a, b : array) : float { @@ -41,3 +45,38 @@ def test_tuned_reconstitutes(t : T?) { t |> equal(sample_dot(a, b), reference(a, b), "tuned sample_dot == reference") t |> equal(sample_dot_u2(a, b), reference(a, b), "tuned default=u2 sample_dot_u2 == reference") } + +// The sidecar is measured data, not law: a manifest entry naming a perm the grid no longer +// offers (a grid rework since the mint) must not brick the compile - the macro stamps the +// fallback and says so, so the tuner that would re-race the entry can always run. The hard +// refusal survives only for an explicit perm= pin (cant_tuned_bad_pin.das). +[test] +def test_tuned_stale_sidecar_perm(t : T?) { + let args <- get_command_line_arguments() // locked view - do not delete + let bin = args[0] + let tmpRes = create_temp_file_result("tuned_stale_perm_", ".json") + if (tmpRes is error) { + t |> failure("could not create a temp file: {tmpRes as error}") + return + } + let manifest = tmpRes as value + fwrite(manifest, "\{\"kernels\": \{\"stale_probe_dot\": \"no_such_perm\"\}, \"provenance\": \{\"box\": \"{tune_box_identity()}\"\}\}") + let client = "{get_this_module_dir()}/tuned_stale_perm_client.das" + let env = (get_platform_name() == "windows" + ? "set DAS_TUNE_MANIFEST={manifest}&& " + : "DAS_TUNE_MANIFEST={manifest} ") + let cmd = "{env}\"{bin}\" \"{client}\"" + var outp = "" + var rc : int + unsafe { + rc = popen_timeout(cmd, 300.0) $(f) { + if (f != null) { + outp = fread(f) + } + } + } + t |> equal(rc, 0, "the stale-sidecar client compiles and runs") + t |> success(find(outp, "stale mint?") >= 0, "the fallback stamp is announced (got: {outp})") + t |> success(find(outp, "STALE_CLIENT_OK") >= 0, "the client ran to completion (got: {outp})") + remove(manifest) +} diff --git a/modules/dasLLVM/tests/tuned_stale_perm_client.das b/modules/dasLLVM/tests/tuned_stale_perm_client.das new file mode 100644 index 0000000000..8b83568930 --- /dev/null +++ b/modules/dasLLVM/tests/tuned_stale_perm_client.das @@ -0,0 +1,23 @@ +options gen2 +options _dasllama_internal = true + +require dasllama/dasllama_tune + +// the stale-sidecar client: DAS_TUNE_MANIFEST points it at a manifest whose entry for +// stale_probe_dot names a perm no grid offers - the compile must survive on the fallback +def template stale_probe_dot_template(a, b : array) : float { + var s = 0f + for [tune = 1] (i in range(length(a))) { // nolint:PERF029 - the tuned kernel: perm hints stamp this index loop + s += a[i] * b[i] + } + return s +} + +[tuned] +def stale_probe_dot(a, b : array) : float {} + +[export] +def main { + let a = [for (i in range(8)); float(i)] + print("STALE_CLIENT_OK {stale_probe_dot(a, a)}\n") +} diff --git a/src/ast/ast_parse.cpp b/src/ast/ast_parse.cpp index 0ae6fc11de..b7ef966f4e 100644 --- a/src/ast/ast_parse.cpp +++ b/src/ast/ast_parse.cpp @@ -585,6 +585,31 @@ namespace das { // reader reject a cache written by a different protocol/version cleanly, instead of // desyncing on a layout difference mid-record. static constexpr uint32_t SER_MODULE_STREAM_MAGIC = 0x4D534144u; // 'DASM' + // sanity bound on a record's macro-dependency count: a real program registers a handful + // (one per consumed sidecar/config); anything past this reads as stream corruption + static constexpr uint32_t SER_MAX_MACRO_DEPS = 4096u; + + void statAndHashFileDependency ( const string & path, int64_t & size, uint64_t & hash ) { + size = -1; + hash = 0; +#if !defined(DAS_NO_FILEIO) + FILE * f = fopen(path.c_str(), "rb"); + if ( !f ) return; + fseek(f, 0, SEEK_END); + long fsize = ftell(f); + fseek(f, 0, SEEK_SET); + if ( fsize >= 0 ) { + vector bytes; + bytes.resize(size_t(fsize)); + size_t got = fsize ? fread(bytes.data(), 1, size_t(fsize), f) : 0; + if ( got == size_t(fsize) ) { + size = int64_t(fsize); + hash = fsize ? hash_block64(bytes.data(), size_t(fsize)) : 14695981039346656037ul; // fnv64a seed for the empty file + } + } + fclose(f); +#endif + } bool trySerializeProgramModule ( ProgramPtr & program, @@ -627,17 +652,46 @@ namespace das { int64_t saved_size = -1; string saved_filename{}; uint64_t payload_size = 0; + uint32_t depCount = 0; + vector> savedDeps; if ( !serializer_read->trySerialize([&](AstSerializer & serializer) { serializer << saved_mtime; serializer << saved_size; serializer << saved_filename; + // macro file dependencies (Program::moduleCacheDependencies) ride the record + // header, not the payload: they must be validated BEFORE the payload is trusted + serializer << depCount; + if ( depCount <= SER_MAX_MACRO_DEPS ) { + savedDeps.resize(depCount); + for ( auto & dep : savedDeps ) { + serializer << get<0>(dep); + serializer << get<1>(dep); + serializer << get<2>(dep); + } + } serializer << payload_size; - }) ) { + }) || depCount > SER_MAX_MACRO_DEPS ) { serializer_read->seenNewModule = true; + serializer_read->failed = depCount > SER_MAX_MACRO_DEPS; logs << "ser: read failed '" << fileName << "'\n"; return false; } + // a compile-time input a macro consumed (a tune sidecar) changed since the record was + // written: the cached stamps were minted against the old file - same prefix cutoff as a + // changed source. Compared by CONTENT (size + hash), not mtime: apps rewrite their + // sidecar byte-identically on exit, and that must not churn the cache. + for ( auto & dep : savedDeps ) { + int64_t depSize = -1; + uint64_t depHash = 0; + statAndHashFileDependency(get<0>(dep), depSize, depHash); + if ( depSize != get<1>(dep) || depHash != get<2>(dep) ) { + serializer_read->seenNewModule = true; + serializer_read->failed = true; + logs << "ser: macro dependency changed '" << get<0>(dep) << "' (e.g. a re-minted tune sidecar)\n"; + return false; + } + } // mtime alone is 1-second granular - a same-second rewrite would serve the stale // AST with no diagnostic, so the size rides beside it in the header if ( saved_filename != fileName || file_mtime != saved_mtime || file_size != saved_size ) { @@ -668,6 +722,9 @@ namespace das { if ( read_ok && !program->failed() && !serializer_read->failed ) { program->thisModuleGroup = &libGroup; + // the stream is rewritten every run from parsedModules, so a kept record's deps + // must round-trip through the deserialized program or the next write drops them + program->moduleCacheDependencies = das::move(savedDeps); if ( serializer_write != nullptr ) { serializer_write->parsedModules.push_back({fileName, file_mtime, file_size, program, program->thisModule.get()}); } @@ -1380,6 +1437,14 @@ namespace das { *serializer_write << fileMtime; *serializer_write << fileSize; *serializer_write << const_cast(fileName); + // macro file dependencies: header-resident, validated by the reader before the payload + uint32_t depCount = uint32_t(program->moduleCacheDependencies.size()); + *serializer_write << depCount; + for ( auto & dep : program->moduleCacheDependencies ) { + *serializer_write << get<0>(dep); + *serializer_write << get<1>(dep); + *serializer_write << get<2>(dep); + } // record length, backpatched after the payload: lets the reader skip a record // that fails to deserialize for an UNCHANGED file and keep serving later ones. // Fixed-width u64 on purpose - an adaptive size could not be patched in place. diff --git a/src/builtin/module_builtin_ast.cpp b/src/builtin/module_builtin_ast.cpp index 4b843f2c5f..9ec84ff9ed 100644 --- a/src/builtin/module_builtin_ast.cpp +++ b/src/builtin/module_builtin_ast.cpp @@ -1,5 +1,13 @@ #include "daScript/misc/platform.h" +#if defined(_MSC_VER) + #include + #define das_dep_getcwd _getcwd +#else + #include + #define das_dep_getcwd getcwd +#endif + #include "module_builtin_rtti.h" #include "daScript/simulate/simulate_visit_op.h" @@ -259,6 +267,39 @@ namespace das { return program; } + // Pin an extra compile-time file input (a tune sidecar, a config a macro read) into the + // module cache: the record stores (path, mtime, size) and the reader re-validates them, so + // a change to the file invalidates the cached module instead of serving stale macro output. + // Deliberately a NO-OP outside compilation - manifest readers run at runtime too, and the + // call site stays unconditional. + void addModuleCacheDependency ( const char * path, Context *, LineInfoArg * ) { + auto program = daScriptEnvironment::getBound()->g_Program; + if ( !program || !path || !path[0] ) return; + // absolutize: the validating reader may run from a different cwd, and a relative + // path there would stat the wrong (or no) file + string fullPath = path; +#if !defined(DAS_NO_FILEIO) + bool isAbs = path[0]=='/' || path[0]=='\\' || (path[0] && path[1]==':'); + if ( !isAbs ) { + char cwd[4096]; + if ( das_dep_getcwd(cwd, sizeof(cwd)) ) { + fullPath = string(cwd) + "/" + path; + } + } +#endif + int64_t size = -1; + uint64_t hash = 0; + statAndHashFileDependency(fullPath, size, hash); + for ( auto & dep : program->moduleCacheDependencies ) { + if ( get<0>(dep) == fullPath ) { + get<1>(dep) = size; + get<2>(dep) = hash; + return; + } + } + program->moduleCacheDependencies.emplace_back(fullPath, size, hash); + } + char * ast_describe_typedecl ( TypeDecl * t, bool d_extra, bool d_contracts, bool d_module, Context * context, LineInfoArg * at ) { if ( !t ) context->throw_error_at(at, "expecting type, not null"); return context->allocateString(t->describe( @@ -1389,6 +1430,9 @@ namespace das { addExtern(*this, lib, "compiling_program", SideEffects::accessExternal, "compileProgram") ->args({"context","at"}); + addExtern(*this, lib, "add_module_cache_dependency", + SideEffects::modifyExternal, "addModuleCacheDependency") + ->args({"path","context","at"}); addExtern(*this, lib, "compiling_module", SideEffects::accessExternal, "compileModule") ->args({"context","at"}); diff --git a/tests/module_cache/_fixtures/mc_dep_drv.das b/tests/module_cache/_fixtures/mc_dep_drv.das new file mode 100644 index 0000000000..41f15f6d4c --- /dev/null +++ b/tests/module_cache/_fixtures/mc_dep_drv.das @@ -0,0 +1,11 @@ +options gen2 +options indenting = 4 + +require mc_dep_used + +// plain main-line driver (NOT a dastest child): the module cache serves the main compile's +// require chain, and only there does mc_dep_used land in a cached record +[export] +def main { + print("PROBED={probed()}\n") +} diff --git a/tests/module_cache/_fixtures/mc_dep_mod.das b/tests/module_cache/_fixtures/mc_dep_mod.das new file mode 100644 index 0000000000..e7b7dbe049 --- /dev/null +++ b/tests/module_cache/_fixtures/mc_dep_mod.das @@ -0,0 +1,19 @@ +options gen2 +options indenting = 4 + +module mc_dep_mod shared private + +require daslib/ast +require daslib/ast_boost +require daslib/fio + +// The module-cache dependency gate's fixture macro: [dep_probe] registers the file named by +// MC_DEP_FILE as a compile-time input of the compiling module, exactly the way a tune-sidecar +// read does (add_module_cache_dependency in llvm_tune's read_manifest). +[function_macro(name = "dep_probe")] +class private DepProbe : AstFunctionAnnotation { + def override apply(var func : FunctionPtr; var group : ModuleGroup; args : AnnotationArgumentList; var errors : das_string) : bool { + add_module_cache_dependency(get_env_variable("MC_DEP_FILE")) + return true + } +} diff --git a/tests/module_cache/_fixtures/mc_dep_used.das b/tests/module_cache/_fixtures/mc_dep_used.das new file mode 100644 index 0000000000..b1dd2ee4b1 --- /dev/null +++ b/tests/module_cache/_fixtures/mc_dep_used.das @@ -0,0 +1,13 @@ +options gen2 +options indenting = 4 + +module mc_dep_used shared private + +require mc_dep_mod + +// the [dep_probe] carrier must live in a REQUIRED module: only required modules land in the +// module cache, so only here does the registered dependency reach a cached record +[dep_probe] +def public probed : int { + return 42 +} diff --git a/tests/module_cache/test_macro_dep_invalidate.das b/tests/module_cache/test_macro_dep_invalidate.das new file mode 100644 index 0000000000..59a152c98c --- /dev/null +++ b/tests/module_cache/test_macro_dep_invalidate.das @@ -0,0 +1,81 @@ +options gen2 +options indenting = 4 + +require dastest/testing_boost public + +require strings +require daslib/fio + +// argv[0] is the running interpreter; dastest is invoked as +// `daslang(.exe) dastest/dastest.das ...`, so argv[0] is the daslang binary to spawn. +def das_exe() : string { + let args <- get_command_line_arguments() + return empty(args) ? "" : args[0] +} + +// child env rides the command string (the portable set/prefix split popen_argv cannot carry) +def env_run(dep, cmd : string; var output : string&) : int { + let win = get_platform_name() == "windows" + let full = win ? "set MC_DEP_FILE={dep}&& {cmd}" : "MC_DEP_FILE={dep} {cmd}" + return unsafe(popen_timeout(full, 300.0, $(f) { + if (f != null) { + output = fread(f) + } + })) +} + +def report_child(t : T?; phase : string; rc : int; out : string; marker : string) : bool { + let ok = rc == 0 && find(out, marker) >= 0 + if (!ok) { + t |> failure("{phase} child: rc={rc}, marker '{marker}' {find(out, marker) >= 0 ? "found" : "MISSING"}") + t |> failure("{phase} child output follows:\n{out}") + } + return ok +} + +// A macro-registered file dependency (add_module_cache_dependency - the tune-sidecar shape) +// must gate the module cache by CONTENT: an untouched or byte-identically rewritten input +// serves from cache, a content change invalidates the record and reparses - otherwise the +// cache serves macro output minted against the old file (stale tune stamps, empty +// tune_status, winners nobody raced). +[test] +def test_macro_dep_invalidate(t : T?) { + var terr : string + let tmp = create_temp_directory("das_mc_dep", terr) + if (empty(tmp)) { + t |> failure("create_temp_directory: {terr}") + return + } + let dep = "{tmp}/dep_input.txt" + fwrite(dep, "content v1") + let root = get_das_root() + // the driver runs as the child's MAIN program (not a dastest child): a nested runtime + // compile bypasses the module cache, so only the main-line require chain proves the gate + let cmd = "\"{das_exe()}\" -dasroot \"{root}\" -module-cache \"{tmp}/mc.bin\" \"{root}/tests/module_cache/_fixtures/mc_dep_drv.das\"" + var cold : string + let coldRc = env_run(dep, cmd, cold) + var coldOk = report_child(t, "cold", coldRc, cold, "ser: wrote") + coldOk = coldOk && report_child(t, "cold", coldRc, cold, "PROBED=42") + t |> success(coldOk, "cold run wrote the cache and passed") + var warm : string + let warmRc = env_run(dep, cmd, warm) + t |> success(report_child(t, "warm", warmRc, warm, "deser: clean"), "warm run served every module from the cache") + // byte-identical rewrite (fresh mtime): content gating must keep the cache warm - the + // bench sidecar is rewritten on every exit, and mtime gating would repay the parse each run + fwrite(dep, "content v1") + var rewrite : string + let rewriteRc = env_run(dep, cmd, rewrite) + t |> success(report_child(t, "rewrite", rewriteRc, rewrite, "deser: clean"), "byte-identical rewrite stays cached") + // content change: the record must invalidate, name the file, and the reparse still pass + fwrite(dep, "content v2 - re-minted") + var changed : string + let changedRc = env_run(dep, cmd, changed) + var chOk = report_child(t, "changed", changedRc, changed, "ser: macro dependency changed") + if (find(changed, "deser: clean") >= 0) { + t |> failure("changed run still served the stale record from the cache") + chOk = false + } + t |> success(chOk, "content change invalidated the cached record and the reparse passed") + var err : string + rmdir_rec(tmp, err) +} From 3479b4b4c4a2a5ea9102b60317ad278661acf3f5 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 08:23:50 -0700 Subject: [PATCH 24/35] followup_metal: the elementwise/activation lane, sized honestly The producer-fused _hx family already banks most of the fusion rung (rms_hx, add_rms_bhx, swiglu/geglu_hx, qk_rope_hx); the act_cvt knockout measures ALL remaining converts at +0.65% pp512 on M5. What's left: the attention-out _hx (the one live dense-path cvt, ~0.3% M5 / more on M1-class, with the q_gated sigmul ordering caveat), the per-model cvt arms, and the real gap - f16 activation planes end-to-end, its own arc. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014mq9AKyoAwvD5T1cigTnRw --- modules/dasLLAMA/followup_metal.md | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/modules/dasLLAMA/followup_metal.md b/modules/dasLLAMA/followup_metal.md index 4d982bb838..7477694ad6 100644 --- a/modules/dasLLAMA/followup_metal.md +++ b/modules/dasLLAMA/followup_metal.md @@ -52,3 +52,33 @@ every format: `harness/parity.das -- -m -n 40 --ids 128000,12805,5304,264 Placeholder - the per-format Metal notes (tg 0.78-0.93x tails, the IQ4_XS lane-map gap of followup_general #58, the Q22 dispatch-loop probe method) consolidate here in Phase E of `plans/unquirk_pass.md`; until then they live in `HOW_TO_ADD_A_FORMAT.md`'s per-format notes. + +## 4. The elementwise / activation-precision lane (the last M5 pp residual) + +Attribution (M5, 1B iq2xxs, pp512 = 31 ms encode): mm 27.7 ms at measured tensor-twin rates, +attention 0.16 ms - the ~3 ms remainder is every non-matmul pass over the activation planes +(norms, residual adds, rope, swiglu, activation converts, glue). llama.cpp's slice: ~1.7 ms. + +**What already exists** - the producer-fused f16 twin family (`_hx`): `pf_enc_rms_hx`, +`pf_enc_add_rms_bhx` (add+norm+half-emit in one), `enc_swiglu_hx`/`enc_geglu_hx`, +`enc_qk_rope_hx`. On the dense path these cover the norm and activation producers; the +standalone `enc_cvt_half` fires only through `pf_cvt_panel` fallbacks and at the sites below. +Measured ceiling of ALL remaining converts (`DASLLAMA_METAL_PREFILL_SKIP=act_cvt` knockout): +**+0.65% pp512 on M5** (15334 vs 15235 tok/s) - the fusion rung is mostly banked already. + +- **4a. The attention-out `_hx`** (the one live dense-path cvt): the AV kernels + (`MetalAttnAV`, `MetalAttnAVMm`, the tensor `MetalAttnAVMmTensorT`) write only f32 `xb`; + wo's X half twin comes from `pf_cvt_panel` at the `bxh_av` site. An HX store beside the + simdgroup/tensor stores kills that pass. Caveat: `q_gated` models rewrite `bxb` through + `enc_sigmul` AFTER attention - there the twin must come from sigmul (give it an `_hx`), + or the site keeps the cvt. Prize: ~0.3% pp on M5, larger on M1/M4-class (same bytes, + a third of the bandwidth). Three kernel variants + the gated ordering = half a day. +- **4b. Per-model cvt arms** still on `pf_cvt_panel`: the deltanet out (`bdno`), the MoE + hidden (`bxh_mg`) and shexp, PLE gather/project (gemma4e), the embedder/cat legs. Same + `_hx` pattern where the producer is ours; size per model class before building. +- **4c. THE BIG RUNG - f16 activation planes end-to-end for prefill.** The remaining ~1 ms/ + prefill vs llama.cpp is the f32 elementwise traffic itself (their graph moves half the + bytes through every norm/add/act pass and needs no cvt at all). Touches plane formats, + every elementwise kernel, attention, the CPU-fallback paths, logits/readback - and + re-opens the numerics bars across the parity and prefill suites. ITS OWN ARC, planned; + the acceptance bar is the existing parity suites plus a pp/tg board A/B per class. From 3cbfc2126a3cb1aa2c0e8d6984bde9b48fb9a444 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 08:52:57 -0700 Subject: [PATCH 25/35] PR comment harvest: three mechanisms become architecture, two rules bind the race rig, two renames dissolve their comments The diff's working comments settle per the harvest contract. Three FACT sets land as anchored architecture sections with LINT026-proven citations: the kernel-race fidelity conditions and the serving-confirm crown (ARCHITECTURE_MEASUREMENT.md 2.21, five citers), the iquant GEMV grid read and its per-box f4-slab twin (ARCHITECTURE_GPU.md 2.2z), and the dense-KQ tensor mul_mm scaffold (ARCHITECTURE_GPU_PREFILL.md 2.2aa, two citers). The dasLLVM side lands the tune-sidecar cache-pin mechanism (ARCHITECTURE.md 5) with its citer and a DAS_ARCH_EXTRACT_SPECS row so the citation survives in an installed SDK. Two rules join REVIEW_GPU.md: race chains bind distinct output buffers, and a race burns the clock governor before its first timed round. Two renames replace comments outright: race_shapes becomes a named tuple (name/n/d), and the race fixture's scale-form legend becomes the KqRaceScaleForm enum. The locked-view argv contract hoists to get_command_line_arguments' own doc in daslib/builtin.das. A pre-existing k6 doc line orphaned by an insertion rehomes onto pf_enc_kq_site_mm; a stale mvb_arms comment and a comment that mis-stated the dep tuple are fixed; build_and_debug.md's module-cache invalidation claim catches up with the content-hash pin. followup_metal gains the missing tall/db tensor-stamp ledger row, tied to the arc matrix's deep-K w2 column. The arm-i8mm class defaults profile ships from the fresh full M5 mint (49 kernels, the i8mm seats raced-and-covered), serving every M2+ Apple box including m4. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014mq9AKyoAwvD5T1cigTnRw --- daslib/builtin.das | 2 + include/daScript/ast/ast.h | 8 +-- modules/dasLLAMA/ARCHITECTURE.md | 11 +-- modules/dasLLAMA/ARCHITECTURE_GPU.md | 19 +++++ modules/dasLLAMA/ARCHITECTURE_GPU_PREFILL.md | 29 ++++++++ modules/dasLLAMA/ARCHITECTURE_MEASUREMENT.md | 36 ++++++++++ modules/dasLLAMA/REVIEW_GPU.md | 9 +++ .../benchmarks/matmul/bench_metal_kq_race.das | 57 ++++++++------- .../dasllama/dasllama_metal_common.das | 6 +- .../dasllama/dasllama_metal_kernels.das | 52 +++++--------- .../dasllama/dasllama_metal_prefill.das | 61 ++++++++-------- modules/dasLLAMA/dasllama/dasllama_tune.das | 4 +- modules/dasLLAMA/followup_metal.md | 9 +++ modules/dasLLAMA/harness/tune_kernels.das | 16 +---- .../defaults/arm-i8mm.tune-defaults.json | 69 +++++++++++++++++++ .../tests/test_metal_gemm_kernels.das | 2 +- modules/dasLLVM/ARCHITECTURE.md | 16 +++++ modules/dasLLVM/CMakeLists.txt | 4 ++ modules/dasLLVM/daslib/llvm_tune.das | 2 +- modules/dasLLVM/tests/test_tuned.das | 11 ++- skills/internal/build_and_debug.md | 2 +- src/ast/ast_parse.cpp | 11 ++- src/builtin/module_builtin_ast.cpp | 8 +-- tests/module_cache/_fixtures/mc_dep_drv.das | 2 - tests/module_cache/_fixtures/mc_dep_mod.das | 5 +- tests/module_cache/_fixtures/mc_dep_used.das | 4 +- .../test_macro_dep_invalidate.das | 17 ++--- 27 files changed, 308 insertions(+), 164 deletions(-) create mode 100644 modules/dasLLAMA/performance/defaults/arm-i8mm.tune-defaults.json diff --git a/daslib/builtin.das b/daslib/builtin.das index cff3fd97b0..ee0455dffc 100644 --- a/daslib/builtin.das +++ b/daslib/builtin.das @@ -2123,6 +2123,8 @@ def add_ptr_ref(src : smart_ptr) : smart_ptr { } } +//! The process argv as an array of strings. The result is a LOCKED VIEW over the host's own +//! argv storage - keep it in a `let` and never `delete` it or move elements out of it. [generic] def get_command_line_arguments : array { var args : array diff --git a/include/daScript/ast/ast.h b/include/daScript/ast/ast.h index 8504780009..92eab8f5f7 100644 --- a/include/daScript/ast/ast.h +++ b/include/daScript/ast/ast.h @@ -1691,11 +1691,9 @@ namespace das CodeOfPolicies policies; vector> allRequireDecl; das_hash_map astTypeInfo; - // compile-time file inputs a macro consumed beyond the sources (a tune sidecar, a config): - // (path, size, content hash) captured at registration via add_module_cache_dependency. The - // module cache stores and re-validates them, so a changed input invalidates the cached - // record instead of serving stamps minted against the old file. Content-hashed, not - // mtime'd: apps that rewrite their sidecar byte-identically on exit must not churn the cache. + // compile-time file inputs a macro pinned via add_module_cache_dependency: (path, byte + // size, content hash). The module cache stores and re-validates them, so a changed input + // invalidates the cached record instead of serving stamps minted against the old file. vector> moduleCacheDependencies; }; diff --git a/modules/dasLLAMA/ARCHITECTURE.md b/modules/dasLLAMA/ARCHITECTURE.md index dc4ea42a2f..2bbde3e57c 100644 --- a/modules/dasLLAMA/ARCHITECTURE.md +++ b/modules/dasLLAMA/ARCHITECTURE.md @@ -38,11 +38,12 @@ re-transcoding `$LCPP/src/unicode-data.cpp`). - `ARCHITECTURE_IMAGE.md` - sec.2.1-2.1i: the prepared-image rail, the baked dev-W f16 plane, and the baked tower twin-W plane. -- `ARCHITECTURE_GPU.md` - sec.2.2b, 2.2w-2.2y: the tensor-GEMM and fused-attention shapes that - measured out, the tower attention routes, and the tower driver's encode chains. -- `ARCHITECTURE_GPU_PREFILL.md` - sec.2.2c-2.2i, 2.2u-2.2v: the Metal prefill driver's GEMM form - ladder, dev-W knee map, attention slab, MoE bucket rail, chunked submission, the f16 twin - dual-store, and the last-layer FFN tail. +- `ARCHITECTURE_GPU.md` - sec.2.2b, 2.2w-2.2z: the tensor-GEMM and fused-attention shapes that + measured out, the tower attention routes, the tower driver's encode chains, and the iquant + GEMV grid read with its f4-slab twin. +- `ARCHITECTURE_GPU_PREFILL.md` - sec.2.2c-2.2i, 2.2u-2.2v, 2.2aa: the Metal prefill driver's + GEMM form ladder, dev-W knee map, attention slab, MoE bucket rail, chunked submission, the + f16 twin dual-store, the last-layer FFN tail, and the dense-KQ tensor mul_mm scaffold. - `ARCHITECTURE_GPU_VULKAN.md` - sec.2.2j-2.2q: the Vulkan resident driver - the prefill window chain, the cm2 decode lanes and tile pick, the class-pipeline build seat, the residency plan, the GPU-slot marks swap, the Q8 requant byte store, and the MoE expert chain on the cm2 tiles. diff --git a/modules/dasLLAMA/ARCHITECTURE_GPU.md b/modules/dasLLAMA/ARCHITECTURE_GPU.md index bd9eac0cbe..9f6274d9ee 100644 --- a/modules/dasLLAMA/ARCHITECTURE_GPU.md +++ b/modules/dasLLAMA/ARCHITECTURE_GPU.md @@ -272,3 +272,22 @@ a region's bind offset must be a multiple of 512 elements - the `(off/256)*2` d- the exception: it reuses q40's 16-byte plane of eight f16 d per superblock, binds once, and ignores `doff`. The Vulkan tier does not use this form - it binds the decoded 20-byte row as five uints per superblock. + +### 2.2z The iquant GEMV grid read and its f4-slab twin {#metal-iquant-gemv-grid} + +Every iquant Metal GEMV - iq3s, iq3xxs, iq2s, iq2xs, iq2xxs - reads its codebook grid DIRECT +off the module's hoisted constant tables and float-expands each word in place. That is the base +form and the one that ships everywhere. + +Three of them carry a second form, the f4-slab twin (`MetalKqGemvIq3sF4`, +`MetalKqGemvIq3xxsF4`, `MetalKqGemvIq2xxsF4`): the whole grid staged into threadgroup memory +once per threadgroup, pre-expanded to `float4` magnitudes, signs applied by select, so the +inner loop does no byte extraction. The twin is a PER-BOX CROWN, never a heuristic and never a +default - it wins on M5-class GPUs and REGRESSES on M1/M4-class, which have less threadgroup +bandwidth to trade for the byte work. Its PSO compiles only when the box's tune manifest +carries the crown (`metal_tensor_crowned("kq_gemv__f4")`), and `enc_kq_gemv` picks the +twin exactly when that PSO is non-null, so a box with no crown never compiles it. + +The `kq_gemv_iq3s_f4` and `kq_gemv_iq3xxs_f4` crowns are raced (`race_gemv_f4_twin`). +`kq_gemv_iq2xxs_f4` cannot be settled by an isolated race at all and is minted from a serving +A/B instead - `ARCHITECTURE_MEASUREMENT.md` sec.2.21. diff --git a/modules/dasLLAMA/ARCHITECTURE_GPU_PREFILL.md b/modules/dasLLAMA/ARCHITECTURE_GPU_PREFILL.md index 882c966448..69f17f4c16 100644 --- a/modules/dasLLAMA/ARCHITECTURE_GPU_PREFILL.md +++ b/modules/dasLLAMA/ARCHITECTURE_GPU_PREFILL.md @@ -263,3 +263,32 @@ takes it. `DASLLAMA_METAL_LASTROW=0` pins the full-panel tail. A caller that consumes the whole `x_b` plane afterwards - embedding pooling, a plane-compare probe - sets `Session.keep_hidden` and the prefill keeps every row; the flag is zero-init, so narrowing is the default. + +### 2.2aa The dense-KQ tensor mul_mm scaffold {#prefill-kq-tensor-scaffold} + +Nine iquant and split-scale formats - iq4xs, iq4nl, k3, iq3s, iq3xxs, k2, iq2s, iq2xs, iq2xxs - +share ONE tensor mul_mm body, `MetalKqMulMmSplitTensorBase`. The base holds the k6 tensor +shell: the `tmm2d_tg_*` accumulate loop, the 6144-half `twb` W chunk, the store. It exposes +exactly one overridable stage, `stage16` - 16 elements per work item, decoded into `twb`. A +format derives, binds its own weight views, and overrides `stage16` alone. 16 is not an +arbitrary granularity: it is the base GEMV arm's own granularity, so each format's decode ports +into its `stage16` verbatim, the arm's `va[]` store becoming a `twb` store. This is the shape +sec.2.2g's MoE tensor twins ride, applied to the dense sites; as there, the q8 twin stays its +own template because its body is a different staging mechanism. + +Each format's `stage16` takes one of three forms, inherited from that format's base GEMV arm: + +- **byte-parallel compose** (k3, k2, and the q5 arm of k45) - masked uint-wide shifts produce + one `qv` per four elements, with the byte-position shift folded into exact power-of-two + pre-scales. It replaced a per-element bit compose and measures 1.4x it + (`benchmarks/matmul/bench_metal_kq_race.das --tiers mm`, m5). +- **staged-grid slab** (iq3s, iq3xxs) - the 2 KB / 1 KB grid staged into threadgroup memory + once per threadgroup. +- **direct constant-table gather** (iq2s, iq2xs, iq2xxs) - the u64 grid pair read straight off + the hoisted tables, no slab. + +Each format stamps two instances, `T` (`XT = float`) and `TH` (`XT = float16`); this family has +no tall or double-buffered twins. Every stamp compiles only behind its own crown +(`metal_tensor_crowned("kq_mulmm_")`) in `pf_compile_kq_iquant_tensor_twins`, and +`pf_enc_kq_site_mm` dispatches a twin only when both the crown flag and the PSO are live, so a +box with no tensor toolchain never leaves the base kernels. diff --git a/modules/dasLLAMA/ARCHITECTURE_MEASUREMENT.md b/modules/dasLLAMA/ARCHITECTURE_MEASUREMENT.md index c5a2d7897b..8be2087a2a 100644 --- a/modules/dasLLAMA/ARCHITECTURE_MEASUREMENT.md +++ b/modules/dasLLAMA/ARCHITECTURE_MEASUREMENT.md @@ -117,3 +117,39 @@ commit (the remedy `performance/REVIEW.md` allows when the measured `modules/das byte-identical) therefore re-hashes and renames the file, and every `records/.json` row whose `tune_sha` named the old file is repointed in the same change - a row left on the old name points at a file that no longer exists. + +### 2.21 An isolated kernel race is only as good as the graph it imitates {#kernel-race-fidelity} + +A kernel A/B race times two spellings of one compute on a synthetic fixture. Three conditions +decide whether its winner is the winner the served graph would pick, and a race missing any of +them crowns confidently and wrongly: + +- **Overlap.** The served graph's consecutive GEMVs write DIFFERENT output buffers, so the + scheduler overlaps them. A race chaining every dispatch through one output buffer serializes + on the write-after-read hazard; on the iq2xxs f4-slab twin that one difference read +9% for + an arm the served graph rejects at -8.6%. +- **A warmed clock.** An Apple GPU's clock governor ramps under load, and `race_pair_ms` runs + base-then-twin per round, so the first side pays the ramp the second rides. About 150 ms of + GPU work before the first timed round removes the bias; back-to-back dispatches inside each + timed encoder hold the clock there. A sparse invocation without the burn reads idle-clock + times and can flip a verdict outright. +- **A real site shape.** `REVIEW_GPU.md` binds this one. + +Even a race meeting all three can be structurally blind. `kq_gemv_iq2xxs_f4` is the standing +case: every isolated regime crowns the twin and the served decode graph loses 8.6% with it, +because the effect is mixed-stream occupancy - it exists only when the GEMV runs beside the +rest of the token step. That crown is therefore NOT auto-raced. It is minted on the real shapes +by `harness/tune_kernels.das`'s serving confirm: two temporary manifests differing only in +`runtime.metal_tensor` membership, each served to +`benchmarks/lcpp_bench.das --for-debug-purposes --ngl 99 -p 32 -n 128 -r 3` under +`DAS_TUNE_MANIFEST`, the tg128 line the verdict. Both arms carry an EMPTY `kernels` section, so +they stamp identical fallbacks and the tg delta isolates the crown under test. The margin is +`CONFIRM_TG_MARGIN` = 1.005, best-of-3: the crown's serving win where it wins is +0.6% (m5) and +its serving loss where it loses is -8.6% (m4), so the margin only has to clear run noise, and a +spuriously minted crown costs a re-mint rather than a board row. No IQ2_XXS vehicle on the box +means no crown - the base kernel is the safe side - and the run says so with a provisioning +hint. + +The per-format isolated rig is `benchmarks/matmul/bench_metal_kq_race.das`: synthetic planes, +no model, no tuner, every arm gated against a CPU plane-dequant oracle before it is timed, and +`--burn-ms` (default 150) spent on GPU work before each cell's first timed round. diff --git a/modules/dasLLAMA/REVIEW_GPU.md b/modules/dasLLAMA/REVIEW_GPU.md index ceaba1e621..808bfd2e5a 100644 --- a/modules/dasLLAMA/REVIEW_GPU.md +++ b/modules/dasLLAMA/REVIEW_GPU.md @@ -177,6 +177,15 @@ times both kernels on one queue and compares their outputs. small enough to sit in cache ranks the kernels by an effect production never sees, and the race then picks the slower kernel. +**A race arm that dispatches a chain of one kernel binds a DIFFERENT output buffer for +consecutive dispatches - never one shared output across the chain.** One shared output +serializes the chain on its write-after-read hazard while the served graph overlaps +consecutive dispatches, so the race ranks the arms on a shape production never runs. + +**A Metal race spends GPU work on its own arms before its first timed round, and keeps the +dispatches inside each timed encoder back to back.** The arms alternate, so a first round on +an idle clock bills the governor's ramp to one side. + **A string-typed Metal decline reason is a defect - a Metal decline reason is an enum value in `dasllama/dasllama_metal_shapes.das`, one enum per driver.** diff --git a/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das b/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das index bc39a3c658..a857d2eb9f 100644 --- a/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das +++ b/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das @@ -22,9 +22,8 @@ require ?das_metal ../../tests/_metal_kernel_common.das // against a pinned llama.cpp test-backend-ops reference row, on synthetic planes, no model, no // tuner, seconds per cell. The dispatch-loop probe shape (HOW_TO_ADD_A_FORMAT.md QUIRK 22): // `reps` back-to-back dispatches per timed encoder (the shared y buffer's hazard serializes -// them), best-of `rounds` encoders, arms interleaved round-robin per round so box drift cannot -// crown a phantom winner. The metric is weight-plane GB/s for GEMV (decode is weight-bound) and -// GMAC/s for mul_mm, plus us/dispatch for the reference ratio. +// them), best-of `rounds` encoders. The metric is weight-plane GB/s for GEMV (decode is +// weight-bound) and GMAC/s for mul_mm, plus us/dispatch for the reference ratio. // // Correctness before timing, per arm: one dispatch vs the CPU plane-dequant oracle // (kq_row_ref + double dot) over EVERY output element on the dot-envelope bar @@ -67,9 +66,9 @@ struct Args { struct KqRaceFmt { name : string - fmt : int // the kernel id (kq_fill_planes / kq_row_ref key) - gemv_gdiv : int // GEMV rows per threadgroup (dispatch groups = ceil(d / gdiv)) - split : bool // split-scale form: the f16 d tail binds at nsb*16 + fmt : int //!< the kernel id (kq_fill_planes / kq_row_ref key) + gemv_gdiv : int //!< GEMV rows per threadgroup (dispatch groups = ceil(d / gdiv)) + split : bool //!< split-scale form: the f16 d tail binds at nsb*16 } struct KqRaceArm { @@ -78,7 +77,7 @@ struct KqRaceArm { entry : string fastmath : bool tgmem : uint64 - gdiv : int // GEMV only; a candidate arm may change the row map + gdiv : int //!< GEMV only; a candidate arm may change the row map } def race_formats : array { @@ -98,11 +97,12 @@ def race_formats : array { ] } -// (name, n = reduction dim, d = output rows); n % 256, d % 64. The 1B set carries the vehicle -// GGUFs' geometry, the 3B pair the bigger DRAM-honest shapes. -def race_shapes : array> { - return <- [("kv1b", 2048, 512), ("q1b", 2048, 2048), ("w13_1b", 2048, 8192), - ("w2_1b", 8192, 2048), ("q3b", 3072, 3072), ("w13_3b", 3072, 8192)] +//! every shape keeps n % 256 (the kq superblock) and d % 64 (the mul_mm n-tile); the 1B set is +//! the vehicle GGUFs' geometry, the 3B pair the DRAM-honest ones +def race_shapes : array> { + return <- [(name = "kv1b", n = 2048, d = 512), (name = "q1b", n = 2048, d = 2048), + (name = "w13_1b", n = 2048, d = 8192), (name = "w2_1b", n = 8192, d = 2048), + (name = "q3b", n = 3072, d = 3072), (name = "w13_3b", n = 3072, d = 8192)] } def private want(filter, name : string) : bool { @@ -150,7 +150,7 @@ def private gemv_arms(f : KqRaceFmt) : array { // candidate GEMV sp return <- arms } -def private mvb_arms(f : KqRaceFmt) : array { // the B2 batch-decode pair per grid format: prod vs the cvt-compose candidate +def private mvb_arms(f : KqRaceFmt) : array { // the B2 batch-decode arm per grid format; candidate spellings register here, after prod var arms : array if (f.fmt == 33) { arms |> push(KqRaceArm(name = "prod", src = MetalKqMvB2Iq3s_metal_kq_mv_iq3s_msl, entry = MetalKqMvB2Iq3s_metal_kq_mv_iq3s_msl_entry, fastmath = MetalKqMvB2Iq3s_metal_kq_mv_iq3s_msl_fastmath, tgmem = 0ul, gdiv = 8)) @@ -193,8 +193,7 @@ def private mm_arms(f : KqRaceFmt) : array { // candidate mul_mm sp } else { arms |> push(KqRaceArm(name = "prod", src = MetalKqMulMmIq2xxs_metal_kq_mulmm_msl, entry = MetalKqMulMmIq2xxs_metal_kq_mulmm_msl_entry, fastmath = MetalKqMulMmIq2xxs_metal_kq_mulmm_msl_fastmath, tgmem = MetalKqMulMmIq2xxs_metal_kq_mulmm_msl_tgmem, gdiv = 0)) } - // the existing Metal-4 tensor twins (f32-X form) - the ceiling the nine formats' twins chase; - // the pipeline declines cleanly on a box without the tensor toolchain + //! the Metal-4 tensor twins (f32-X form) are the ceiling the per-format twins chase; their pipeline build declines cleanly on a box with no tensor toolchain if (f.fmt == 4) { arms |> push(KqRaceArm(name = "tensor", src = MetalKqMulMmK4T_metal_kq_mulmm_t_msl, entry = MetalKqMulMmK4T_metal_kq_mulmm_t_msl_entry, fastmath = MetalKqMulMmK4T_metal_kq_mulmm_t_msl_fastmath, tgmem = MetalKqMulMmK4T_metal_kq_mulmm_t_msl_tgmem, gdiv = 0)) } elif (f.fmt == 5) { @@ -223,7 +222,7 @@ def private mm_arms(f : KqRaceFmt) : array { // candidate mul_mm sp return <- arms } -let PREFILL_M = 512 // the mm tier's token batch +let PREFILL_M = 512 def private load_ref_table(path : string; var tab : table) { return if (empty(path)) @@ -246,7 +245,7 @@ def private load_ref_table(path : string; var tab : table) { } } -// one timed encoder: bind once, `reps` back-to-back dispatches; per-dispatch GPU us, <0 on failure +//! one timed encoder: bind once, `reps` back-to-back dispatches; per-dispatch GPU us, <0 on failure def private time_arm_us(queue; pso; arm : KqRaceArm; bks, bkq, bx, by, b5, b6; sc_off : uint64; groups : uint3; tgsz, reps : int) : double { var err = "" var gpu_ms = 0.0lf @@ -273,7 +272,7 @@ def private time_arm_us(queue; pso; arm : KqRaceArm; bks, bkq, bx, by, b5, b6; s return gpu_ms * 1000.0lf / double(reps) } -// the whole-output dot-envelope oracle the tests trust: want/env from kq_row_ref + double dot +//! the whole-output dot-envelope oracle the tests trust: want/env from kq_row_ref + double dot def private build_oracle(fmt, n, d, nsb : int; kq, ks : array; xf : array; xstride, xrows : int; var want : array; var env : array) { var wrow : array @@ -296,10 +295,11 @@ def private build_oracle(fmt, n, d, nsb : int; kq, ks : array; xf : array delete wrow } +[arch(at="../../ARCHITECTURE_MEASUREMENT.md#kernel-race-fidelity")] def private race_cell(queue, dev; f : KqRaceFmt; tier, shname : string; n, d : int; arms : array; args : Args; ref_tab : table) : int { let is_mm = tier == "mm" - let is_mvb = tier == "mvb" // 2-column batch decode: same dispatch shape as gemv at gdiv 8, ka args at binding 5 + let is_mvb = tier == "mvb" //! 2-column batch decode: gemv's dispatch shape at gdiv 8, ka args at binding 5 let nb = n / 256 let nsb = d * nb var kq : array @@ -319,14 +319,14 @@ def private race_cell(queue, dev; f : KqRaceFmt; tier, shname : string; n, d : i var bks = buf_upload(dev, ks) var bkq = buf_upload(dev, kq) var bx = buf_upload(dev, xf) - var kav <- [uint(n), uint(d), uint(d), 2u] // KqMvArgs{ndim, ddim, ys, nr} as its raw words + var kav <- [uint(n), uint(d), uint(d), 2u] //! KqMvArgs{ndim, ddim, ys, nr} as its raw words var b5 = is_mvb ? buf_upload(dev, kav) : buf_u32(dev, uint(n)) var b6 = buf_u32(dev, uint(d)) let sc_off = f.split ? uint64(nsb * 16) : 0ul let wbytes = double(length(kq) + length(ks)) var rows = 0 print("--- {f.name} {tier} {shname} (n={n} d={d}, weights {wbytes / 1048576.0lf} MB)\n") - // compile + correctness gate, one y per arm + //! compile + correctness gate, one y buffer per arm var psos : array var bys : array var live : array @@ -358,8 +358,6 @@ def private race_cell(queue, dev; f : KqRaceFmt; tier, shname : string; n, d : i lv = false } } - // clock ramp: burn GPU time on arm 0 until the budget is spent, so the first timed - // encoder runs at sustained clocks even in a sparse (few-cell) invocation var burned = 0.0lf while (burned < double(args.burn_ms) * 1000.0lf) { var did = false @@ -374,7 +372,6 @@ def private race_cell(queue, dev; f : KqRaceFmt; tier, shname : string; n, d : i } break if (!did) } - // interleaved timing: round-robin per round, best-of (round 0 discarded as warmup) var best : array best |> resize(length(arms)) for (i in range(length(arms))) { @@ -401,7 +398,7 @@ def private race_cell(queue, dev; f : KqRaceFmt; tier, shname : string; n, d : i if (i == 0) { base_us = us } - let gbs = wbytes / (us * 1000.0lf) // us * 1e3 = ns; bytes/ns = GB/s + let gbs = wbytes / (us * 1000.0lf) //! us * 1e3 = ns; bytes/ns = GB/s var line = " {arm.name}: {us:.2f} us/dispatch {gbs:.1f} wGB/s" if (is_mm) { let gmac = double(PREFILL_M) * double(n) * double(d) / (us * 1000.0lf) @@ -430,7 +427,7 @@ def private race_cell(queue, dev; f : KqRaceFmt; tier, shname : string; n, d : i metal_release(bx) metal_release(b5) metal_release(b6) - psos |> clear() // borrowed pointers - clear before delete (delete would free pointees) + psos |> clear() //! borrowed pointers - clear before delete (delete would free the pointees) bys |> clear() unsafe { delete psos @@ -462,21 +459,21 @@ def private main_apple(args : Args) { for (f in fmts) { continue if (!want(args.fmts, f.name)) for (sh in shapes) { - continue if (!want(args.shapes, sh._0)) + continue if (!want(args.shapes, sh.name)) if (want(args.tiers, "gemv")) { var arms <- gemv_arms(f) - rows += race_cell(queue, dev, f, "gemv", sh._0, sh._1, sh._2, arms, args, ref_tab) + rows += race_cell(queue, dev, f, "gemv", sh.name, sh.n, sh.d, arms, args, ref_tab) delete arms } if (want(args.tiers, "mm")) { var arms <- mm_arms(f) - rows += race_cell(queue, dev, f, "mm", sh._0, sh._1, sh._2, arms, args, ref_tab) + rows += race_cell(queue, dev, f, "mm", sh.name, sh.n, sh.d, arms, args, ref_tab) delete arms } if (want(args.tiers, "mvb")) { var arms <- mvb_arms(f) if (!empty(arms)) { - rows += race_cell(queue, dev, f, "mvb", sh._0, sh._1, sh._2, arms, args, ref_tab) + rows += race_cell(queue, dev, f, "mvb", sh.name, sh.n, sh.d, arms, args, ref_tab) } delete arms } diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_common.das b/modules/dasLLAMA/dasllama/dasllama_metal_common.das index d80ed5c425..b46e7f1276 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_common.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_common.das @@ -2167,9 +2167,9 @@ def public metal_prefill_declines() : table { return clone_to_move(g_prefill_declines_by_reason) } -// IQ4_XS codebook (kvalues_iq4nl) as four packed words, selected + byte-extracted — -// no table memory; the k6 sign trick widens the byte. Shared by the kernels file's GEMV/mv -// set and the prefill file's tensor twins. +//! The IQ4_XS codebook (kvalues_iq4nl) as four packed words, selected + byte-extracted - no +//! table memory, the k6 sign trick widens the byte. Shared by the kernels file's GEMV/mv set +//! and the prefill file's tensor twins, which is why it lives here. def public iq4_lut(q : uint) : float { let w = q < 8u ? (q < 4u ? 3215825025u : 4142587343u) : (q < 12u ? 639175937u : 1901675829u) return float((int((w >> ((q & 3u) * 8u)) & 255u) ^ 128) - 128) diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das index bc9123555a..88a5a25850 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das @@ -3753,6 +3753,7 @@ def enc_gemv(enc : MetalComputeEncoder?; bw : MetalBuffer?; boff : uint64; bx, b // K-quant site GEMV — the per-format twin of enc_gemv over the kq plane pair // (kq_upload_quants + kq_upload_scales). k6 binds the scale buffer twice: the trailing packed // f16 d plane at byte nsb*16, the 16B sub-scale blocks at 0. Geometry: 64-thread tg, 4 rows. +[arch(at="../ARCHITECTURE_GPU.md#metal-iquant-gemv-grid")] def enc_kq_gemv(enc : MetalComputeEncoder?; fmt : KqFmt; bq : MetalBuffer?; qoff : uint64; bs : MetalBuffer?; soff, doff : uint64; bx, by, bn, bd : MetalBuffer?; rows, n : int64; yoff : uint64 = 0ul; xoff : uint64 = 0ul) { @@ -4703,6 +4704,7 @@ def private race_decode_family(dev : MetalDevice?; queue : MetalCommandQueue?; f } //! Decode-side tuner entry (the runtime.metal_tensor crown): the batch q8 GEMM families. +[arch(at="../ARCHITECTURE_MEASUREMENT.md#kernel-race-fidelity")] def metal_tensor_race_decode : array { var results : array var dev = metal_create_system_default_device() @@ -4734,9 +4736,6 @@ def metal_tensor_race_decode : array { 64, kdim, ndim, 64 * ndim, uint3(uint(2 * (ndim / 64)), 1u, 1u), rka) $(enc, slot_y) { kn_buffer(enc, slot_y, 0ul, 4) }) - //! kq_gemv_iq2xxs_f4 is NOT auto-raced (every isolated regime crowns the M4-class twin - //! that serving rejects at -8.6% - a mixed-stream occupancy effect no kernel race sees); - //! the mint's real-model tg A/B decides it instead (tune_kernels' confirm_serving_crown). results |> emplace(race_gemv_f4_twin(dev, queue, "kq_gemv_iq3s_f4", metal_kq_gemv_iq3s_msl, metal_kq_gemv_iq3s_msl_entry, metal_kq_gemv_iq3s_msl_fastmath, metal_kq_gemv_iq3s_f4_msl, metal_kq_gemv_iq3s_f4_msl_entry, metal_kq_gemv_iq3s_f4_msl_fastmath, metal_kq_gemv_iq3s_f4_msl_tgmem, 26, 4, 8)) @@ -4748,17 +4747,14 @@ def metal_tensor_race_decode : array { return <- results } -//! A GEMV f4-slab twin race over the split-scale fixture: base (constant-table, rows/4) -//! vs the f4-slab twin. The slab wins on M5-class GPUs and REGRESSES on M1/M4-class - a -//! per-box crown, not a heuristic. Winner "tensor" = the twin takes the family crown. +//! A GEMV f4-slab twin race over the split-scale fixture: base (constant-table, rows/4) vs +//! the f4-slab twin. Winner "tensor" = the twin takes the family crown. +[arch(at="../ARCHITECTURE_MEASUREMENT.md#kernel-race-fidelity")] def private race_gemv_f4_twin(dev : MetalDevice?; queue : MetalCommandQueue?; family : string; // nolint:STYLE038 - one-shot A/B race harness; buffers + PSOs stay live to the release tail base_src, base_entry : string; base_fm : bool; twin_src, twin_entry : string; twin_fm : bool; twin_tgmem : uint64; qu_per_sb : int; base_gdiv, twin_gdiv : int) : MetalTensorRaceResult { var res = MetalTensorRaceResult(family = family, winner = "", base_ms = -1.0lf, twin_ms = -1.0lf, note = "") - //! the dominant decode site shape (ffn-up class) - the crown must be minted on what the - //! served graph dispatches; a mid-size square fixture crowned the M4 iq2xxs twin that - //! the real shapes reject let kdim = 2048 let ndim = 8192 let nsb = kdim / 256 @@ -4798,17 +4794,12 @@ def private race_gemv_f4_twin(dev : MetalDevice?; queue : MetalCommandQueue?; fa var bxa = race_buf(dev, uint64(kdim * 4), unsafe(addr(xa[0]))) var by_base = race_buf(dev, uint64(ndim * 4), null) var by_twin = race_buf(dev, uint64(ndim * 4), null) - //! overlap ys: the served graph's consecutive gemvs write DIFFERENT buffers and overlap; - //! a same-y serialized chain misjudged the M4 iq2xxs slab twin (+9% raced, -8.6% e2e) var by_base2 = race_buf(dev, uint64(ndim * 4), null) var by_twin2 = race_buf(dev, uint64(ndim * 4), null) var bn = race_uniform_u32(dev, uint(kdim)) var bd = race_uniform_u32(dev, uint(ndim)) let doff = uint64(nblk * 16) let tg = uint3(64u, 1u, 1u) - //! clock-governor burn: race_pair runs base-then-twin per round, so on a ramp-prone GPU - //! the first side pays the ramp the second rides - ~150ms of GPU work first makes the - //! alternation fair (the M4 iq2xxs crown flipped on exactly this bias) var burn_ms = 0.0lf var burn_err = "" while (burn_ms < 150.0lf) { @@ -4837,7 +4828,7 @@ def private race_gemv_f4_twin(dev : MetalDevice?; queue : MetalCommandQueue?; fa kn_buffer(enc, bxa, 0ul, 3) kn_buffer(enc, bn, 0ul, 5) kn_buffer(enc, bd, 0ul, 6) - for (r in range(32)) { //! back-to-back dispatches keep the clock governor ramped - a single sparse dispatch reads idle-clock times and can flip the verdict + for (r in range(32)) { kn_buffer(enc, (r & 1) == 0 ? by_base : by_base2, 0ul, 4) kn_dispatch(enc, uint3(uint(ndim / base_gdiv), 1u, 1u), tg) } @@ -8685,9 +8676,8 @@ class MetalKqGemvIq4nl { } } -// IQ3_S: w = d*sc*(+-grid byte); constant-grid words float-expanded in place, sign nibbles -// from the block's sign plane, 2 rows per simdgroup, dispatch rows/4. Scale plane = the -// iq4xs/k6 split form. +// IQ3_S: w = d*sc*(+-grid byte), the sign nibbles from the block's own sign plane; +// 2 rows per simdgroup. [metal_dispatch(name = "enc_kq_iq3s_c", pso = "g_pso_kq_iq3s", tg = 64, grid = "rows/4", params = "rows : int64, n : int64")] class MetalKqGemvIq3s { @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d plane — the scale buffer bound at byte nsb*16 (s0off = the caller's doff) @@ -8744,8 +8734,6 @@ class MetalKqGemvIq3s { } } -// The f4-slab twin (the shipped pre-d1 form verbatim) - crowns "kq_gemv_iq3s_f4" where the -// box race prefers it (M5-class); M1/M4-class regress on the slab and keep the d1 base. [metal_dispatch(name = "enc_kq_iq3s_f4_c", pso = "g_pso_kq_iq3s_f4", tgmem = "metal_kq_gemv_iq3s_f4_msl_tgmem", tg = 64, grid = "rows/8", params = "rows : int64, n : int64")] class MetalKqGemvIq3sF4 { @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d plane — the scale buffer bound at byte nsb*16 (s0off = the caller's doff) @@ -8755,7 +8743,7 @@ class MetalKqGemvIq3sF4 { @ssbo @binding = 4 @role = "write" @off = "yoff" @span = "rows*4" y : array @uniform @binding = 5 ndim : uint @uniform @binding = 6 ddim : uint - @workgroup gridf : float4[512] // magnitudes pre-expanded at stage time: the inner loop reads one float4 and flips signs, no byte extracts (the best of the eight raced forms, the reference exe's geometry included) + @workgroup gridf : float4[512] // magnitudes pre-expanded at stage time: the inner loop reads one float4 and flips signs, no byte extracts [metal_kernel(name="metal_kq_gemv_iq3s_f4_msl")] def metal_kq_gemv_iq3s_f4 { @@ -8810,9 +8798,9 @@ class MetalKqGemvIq3sF4 { } } -// IQ2_S: the iq3s GEMV geometry (2 rows per simdgroup, dispatch rows/4) over the u64 grid pair -// read DIRECT off the constant table and float-expanded in place; per-16 UNSIGNED strips scale -// each 16-element half, d pre-eighth-ed at transcode - the fold is d8 * strip * signed_grid_byte. +// IQ2_S: the iq3s GEMV geometry over the u64 grid pair; per-16 UNSIGNED strips scale each +// 16-element half and d is pre-eighth-ed at transcode, so the fold is +// d8 * strip * signed_grid_byte. [metal_dispatch(name = "enc_kq_iq2s_c", pso = "g_pso_kq_iq2s", tg = 64, grid = "rows/4", params = "rows : int64, n : int64")] class MetalKqGemvIq2s { @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d8 plane - the scale buffer bound at byte nsb*16 (s0off = the caller's doff) @@ -8879,8 +8867,8 @@ class MetalKqGemvIq2s { } // IQ2_XS: the iq2s GEMV geometry over the parity-signed u16 words - one u32 = two u16 qs -// words; 9-bit index doubles into the constant grid pair (float-expanded in place, no slab), -// ksign7m recomputes the sign byte (no sign plane, no qh); per-16 UNSIGNED strips, d eighth-ed. +// words, the 9-bit index doubles into the grid pair, and ksign7m recomputes the sign byte +// (no sign plane, no qh); per-16 UNSIGNED strips, d eighth-ed. [metal_dispatch(name = "enc_kq_iq2xs_c", pso = "g_pso_kq_iq2xs", tg = 64, grid = "rows/4", params = "rows : int64, n : int64")] class MetalKqGemvIq2xs { @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d8 plane - the scale buffer bound at byte nsb*16 (s0off = the caller's doff) @@ -9008,8 +8996,7 @@ class MetalKqGemvIq2xxs { } // The f4-slab twin: the same lane map with the 2 KB grid pre-expanded to float4 magnitudes -// (8 KB threadgroup) and signs applied by select - crowns "kq_gemv_iq2xxs_f4" where the box -// race prefers it (M5-class GPUs; M1/M4-class regress on the slab). +// (8 KB threadgroup) and signs applied by select. [metal_dispatch(name = "enc_kq_iq2xxs_f4_c", pso = "g_pso_kq_iq2xxs_f4", tgmem = "metal_kq_gemv_iq2xxs_f4_msl_tgmem", tg = 64, grid = "rows/4", params = "rows : int64, n : int64")] class MetalKqGemvIq2xxsF4 { @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d8 plane - the scale buffer bound at byte nsb*16 (s0off = the caller's doff) @@ -9076,9 +9063,8 @@ class MetalKqGemvIq2xxsF4 { } } -// IQ3_XXS: the iq3s GEMV shape verbatim over the 24-word rows - constant-grid words -// float-expanded in place, signs from the block's aux32 through the parity helper (no table), -// the same (2ls+1)-strip x halved-d fold. 2 rows per simdgroup, dispatch rows/4. +// IQ3_XXS: the iq3s GEMV shape over the 24-word rows - signs from the block's aux32 through +// the parity helper (no table), the same (2ls+1)-strip x halved-d fold. [metal_dispatch(name = "enc_kq_iq3xxs_c", pso = "g_pso_kq_iq3xxs", tg = 64, grid = "rows/4", params = "rows : int64, n : int64")] class MetalKqGemvIq3xxs { @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d plane — the scale buffer bound at byte nsb*16 (s0off = the caller's doff) @@ -9134,8 +9120,6 @@ class MetalKqGemvIq3xxs { } } -// The f4-slab twin (the shipped pre-d1 form verbatim) - crowns "kq_gemv_iq3xxs_f4" where the -// box race prefers it (M5-class); M1/M4-class regress on the slab and keep the d1 base. [metal_dispatch(name = "enc_kq_iq3xxs_f4_c", pso = "g_pso_kq_iq3xxs_f4", tgmem = "metal_kq_gemv_iq3xxs_f4_msl_tgmem", tg = 64, grid = "rows/8", params = "rows : int64, n : int64")] class MetalKqGemvIq3xxsF4 { @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d plane — the scale buffer bound at byte nsb*16 (s0off = the caller's doff) @@ -9371,7 +9355,7 @@ class MetalKqMvB8Iq3xxs { } // The iq3s B2/B4 pair: the iq4xs shell over the per-32-block lane map — lane tx owns block tx, -// 8 grid words = 32 consecutive elems as float4s; the grid read direct. +// 8 grid words = 32 consecutive elems as float4s. [ |> template_struct_instance] class template MetalKqMvIq3sT { @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d plane — the scale buffer bound at byte nsb*16 diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das b/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das index 92116093c6..26930ff662 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das @@ -519,8 +519,6 @@ class template MetalKqMulMmK45TensorT { let dsc = float(ksh[blk * 8u]) * float(sc) let dmn = float(ksh[blk * 8u + 1u]) * float(mn) static_if (QH) { - //! byte-parallel q5 compose: one masked uint-wide qv per 4 elements - //! (1.4x the per-element bit-compose it replaced, raced in the kq lab) let nsh = (js & 1u) * 4u let qb = blk * BLK + (js / 2u) * 8u + kh / 4u let hb = blk * BLK + 32u + kh / 4u @@ -860,9 +858,6 @@ class MetalKqMulMmK6THDb128 : MetalKqMulMmK6DbT { override MT = 128u } -// The dense-KQ tensor mul_mm scaffold for the iquant/split-scale formats: the k6 tensor -// shell with the per-format decode as ONE overridable stage16 (16 elements per work item - -// the base template's own arm granularity, so an arm ports verbatim, va[] store -> twb). [ |> template_struct_instance] class template MetalKqMulMmSplitTensorBase { @template_constant MT : uint = 32u @@ -965,8 +960,6 @@ class MetalKqMulMmIq4nlTH : MetalKqMulMmIq4xsTensorT { override IQ4NL = true } -// q3_K / q2_K: the byte-parallel compose of their base arms (masked uint-wide shifts, the -// byte-position shift folded into exact power-of-two pre-scales). [ |> template_struct_instance] class template MetalKqMulMmK3TensorT : MetalKqMulMmSplitTensorBase { [metal_kernel(float_a_ok=true)] @@ -1071,7 +1064,6 @@ class MetalKqMulMmK2TH : MetalKqMulMmK2TensorT { typedef XT = float16 } -// iq3s / iq3xxs: the staged-grid gathers of their base arms (2 KB / 1 KB threadgroup slabs). [ |> template_struct_instance] class template MetalKqMulMmIq3sTensorT : MetalKqMulMmSplitTensorBase { @workgroup gridw3t : uint[512] //!< the 2 KB iq3s grid, staged once per threadgroup @@ -1183,8 +1175,6 @@ class MetalKqMulMmIq3xxsTH : MetalKqMulMmIq3xxsTensorT { typedef XT = float16 } -// iq2s / iq2xs / iq2xxs: the u64-grid gathers DIRECT off the hoisted constant tables, -// exactly their base arms' choice. [ |> template_struct_instance] class template MetalKqMulMmIq2sTensorT : MetalKqMulMmSplitTensorBase { [metal_kernel(float_a_ok=true)] @@ -5097,8 +5087,8 @@ def tallkq_pick(mp, rows, kdim : int64; env_tall : bool) : bool { return env_tall && kdim > 0l && mp >= 128l && uint64(rows * kdim * 2l) >= TALLKQ_MIN_PANEL } -//! k6 binds its split scale plane twice: sub-scales at soff, the f16 d tail at doff //! the iquant/split-scale formats' tensor twins - compiled only behind each format's crown +[arch(at="../ARCHITECTURE_GPU_PREFILL.md#prefill-kq-tensor-scaffold")] def private pf_compile_kq_iquant_tensor_twins(var ok : bool&) { g_pf_kq_mm44_tensor = metal_tensor_crowned("kq_mulmm_iq4xs") if (g_pf_kq_mm44_tensor) { @@ -5147,6 +5137,8 @@ def private pf_compile_kq_iquant_tensor_twins(var ok : bool&) { } } +//! k6 binds its split scale plane twice: sub-scales at soff, the f16 d tail at doff +[arch(at="../ARCHITECTURE_GPU_PREFILL.md#prefill-kq-tensor-scaffold")] def private pf_enc_kq_site_mm(enc : MetalComputeEncoder?; t : Model; fmt : KqFmt; woff, rows : int64; // nolint:STYLE037,STYLE038 — flat per-format form-pick ladder bx, by, bk, bn : MetalBuffer?; mp : int64; yoff : uint64 = 0ul; bxh : MetalBuffer? = null; kdim : int64 = 0l) { let fmt_tensor = fmt == KqFmt.k6 ? g_pf_kq_mm6_tensor : (fmt == KqFmt.k4 ? g_pf_kq_mm4_tensor : g_pf_kq_mm5_tensor) @@ -7325,12 +7317,20 @@ def private race_moe_mulmm_mx4(dev : MetalDevice?; queue : MetalCommandQueue?) : return res } +// the race fixture's scale-plane layout, one value per on-disk scale form +enum KqRaceScaleForm { + compact //!< k4/k5: 16B kmask blocks, d+dmin leading halves + split_d //!< k6 + every split-scale iquant: 16B strips + a 2B f16 d tail + split_dmin //!< k2: 16B strips + a 4B d+dmin tail + pure_d //!< iq4nl: a pure-d plane, 8 f16 per superblock +} + //! random quant planes + constant f16 scales: any bit pattern is a valid plane, so the //! twin-vs-base envelope doubles as a layout-agreement check def private race_kq_mulmm(dev : MetalDevice?; queue : MetalCommandQueue?; family : string; // nolint:STYLE038 — one-shot A/B race harness; buffers + PSOs stay live to the release tail base_src, base_entry : string; base_fm : bool; base_tgmem : uint64; twin_src, twin_entry : string; twin_fm : bool; twin_tgmem : uint64; - qu_per_sb : int; sform : int) : MetalTensorRaceResult { + qu_per_sb : int; sform : KqRaceScaleForm) : MetalTensorRaceResult { var res = MetalTensorRaceResult(family = family, winner = "", base_ms = -1.0lf, twin_ms = -1.0lf, note = "") let m = 512 let kdim = 2048 @@ -7356,24 +7356,21 @@ def private race_kq_mulmm(dev : MetalDevice?; queue : MetalCommandQueue?; family for (i in range(nblk * qu_per_sb)) { qplane[i] = uint(i) * 2654435761u //! deterministic pseudo-random bit pattern } - //! scale forms: 0 = k4/k5 compact 16B blocks; 1 = the k6 split (16B strips + 2B f16 d - //! tail - every split-scale iquant reads it too); 2 = the k2 split (16B strips + 4B - //! d+dmin tail); 3 = the iq4nl pure-d plane (8 f16 per sb) var splane : array - let sbytes = sform == 1 ? nblk * 18 : (sform == 2 ? nblk * 20 : nblk * 16) + let sbytes = sform == KqRaceScaleForm.split_d ? nblk * 18 : (sform == KqRaceScaleForm.split_dmin ? nblk * 20 : nblk * 16) splane |> resize(sbytes) - if (sform == 1 || sform == 2) { + if (sform == KqRaceScaleForm.split_d || sform == KqRaceScaleForm.split_dmin) { for (i in range(nblk * 16)) { splane[i] = uint8(2) } unsafe { var ph = addr(splane[nblk * 16]) - let per = sform == 2 ? 2 : 1 + let per = sform == KqRaceScaleForm.split_dmin ? 2 : 1 for (b in range(nblk * per)) { ph[b] = float16(0.25) } } - } elif (sform == 3) { + } elif (sform == KqRaceScaleForm.pure_d) { unsafe { var ph = addr(splane[0]) for (b in range(nblk * 8)) { @@ -7401,7 +7398,7 @@ def private race_kq_mulmm(dev : MetalDevice?; queue : MetalCommandQueue?; family var by_twin = race_buf(dev, uint64(m * ndim * 4), null) var bk = race_uniform_u32(dev, uint(kdim)) var bn = race_uniform_u32(dev, uint(ndim)) - let doff = sform == 1 || sform == 2 ? uint64(nblk * 16) : 0ul + let doff = sform == KqRaceScaleForm.split_d || sform == KqRaceScaleForm.split_dmin ? uint64(nblk * 16) : 0ul let grid = uint3(uint(m / 32), uint(ndim / 64), 1u) let tg = uint3(128u, 1u, 1u) race_pair_ms(queue, 5, res.base_ms, res.twin_ms, $(enc) { @@ -7586,40 +7583,40 @@ def metal_tensor_race : array { } results |> emplace(race_kq_mulmm(dev, queue, "kq_mulmm_k4", MetalKqMulMmK4_metal_kq_mulmm_msl, MetalKqMulMmK4_metal_kq_mulmm_msl_entry, MetalKqMulMmK4_metal_kq_mulmm_msl_fastmath, MetalKqMulMmK4_metal_kq_mulmm_msl_tgmem, - MetalKqMulMmK4T_metal_kq_mulmm_t_msl, MetalKqMulMmK4T_metal_kq_mulmm_t_msl_entry, MetalKqMulMmK4T_metal_kq_mulmm_t_msl_fastmath, MetalKqMulMmK4T_metal_kq_mulmm_t_msl_tgmem, 32, 0)) + MetalKqMulMmK4T_metal_kq_mulmm_t_msl, MetalKqMulMmK4T_metal_kq_mulmm_t_msl_entry, MetalKqMulMmK4T_metal_kq_mulmm_t_msl_fastmath, MetalKqMulMmK4T_metal_kq_mulmm_t_msl_tgmem, 32, KqRaceScaleForm.compact)) results |> emplace(race_kq_mulmm(dev, queue, "kq_mulmm_k5", MetalKqMulMmK5_metal_kq_mulmm_msl, MetalKqMulMmK5_metal_kq_mulmm_msl_entry, MetalKqMulMmK5_metal_kq_mulmm_msl_fastmath, MetalKqMulMmK5_metal_kq_mulmm_msl_tgmem, - MetalKqMulMmK5T_metal_kq_mulmm_t_msl, MetalKqMulMmK5T_metal_kq_mulmm_t_msl_entry, MetalKqMulMmK5T_metal_kq_mulmm_t_msl_fastmath, MetalKqMulMmK5T_metal_kq_mulmm_t_msl_tgmem, 40, 0)) + MetalKqMulMmK5T_metal_kq_mulmm_t_msl, MetalKqMulMmK5T_metal_kq_mulmm_t_msl_entry, MetalKqMulMmK5T_metal_kq_mulmm_t_msl_fastmath, MetalKqMulMmK5T_metal_kq_mulmm_t_msl_tgmem, 40, KqRaceScaleForm.compact)) results |> emplace(race_kq_mulmm(dev, queue, "kq_mulmm_k6", MetalKqMulMmK6_metal_kq_mulmm_msl, MetalKqMulMmK6_metal_kq_mulmm_msl_entry, MetalKqMulMmK6_metal_kq_mulmm_msl_fastmath, MetalKqMulMmK6_metal_kq_mulmm_msl_tgmem, - MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl, MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl_entry, MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl_fastmath, MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl_tgmem, 48, 1)) + MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl, MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl_entry, MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl_fastmath, MetalKqMulMmK6T_metal_kq_mulmm_k6_t_msl_tgmem, 48, KqRaceScaleForm.split_d)) results |> emplace(race_kq_mulmm(dev, queue, "kq_mulmm_iq4xs", MetalKqMulMmIq4xs_metal_kq_mulmm_msl, MetalKqMulMmIq4xs_metal_kq_mulmm_msl_entry, MetalKqMulMmIq4xs_metal_kq_mulmm_msl_fastmath, MetalKqMulMmIq4xs_metal_kq_mulmm_msl_tgmem, - MetalKqMulMmIq4xsT_metal_kq_mulmm_iq4xs_t_msl, MetalKqMulMmIq4xsT_metal_kq_mulmm_iq4xs_t_msl_entry, MetalKqMulMmIq4xsT_metal_kq_mulmm_iq4xs_t_msl_fastmath, MetalKqMulMmIq4xsT_metal_kq_mulmm_iq4xs_t_msl_tgmem, 32, 1)) + MetalKqMulMmIq4xsT_metal_kq_mulmm_iq4xs_t_msl, MetalKqMulMmIq4xsT_metal_kq_mulmm_iq4xs_t_msl_entry, MetalKqMulMmIq4xsT_metal_kq_mulmm_iq4xs_t_msl_fastmath, MetalKqMulMmIq4xsT_metal_kq_mulmm_iq4xs_t_msl_tgmem, 32, KqRaceScaleForm.split_d)) results |> emplace(race_kq_mulmm(dev, queue, "kq_mulmm_iq4nl", MetalKqMulMmIq4nl_metal_kq_mulmm_msl, MetalKqMulMmIq4nl_metal_kq_mulmm_msl_entry, MetalKqMulMmIq4nl_metal_kq_mulmm_msl_fastmath, MetalKqMulMmIq4nl_metal_kq_mulmm_msl_tgmem, - MetalKqMulMmIq4nlT_metal_kq_mulmm_iq4xs_t_msl, MetalKqMulMmIq4nlT_metal_kq_mulmm_iq4xs_t_msl_entry, MetalKqMulMmIq4nlT_metal_kq_mulmm_iq4xs_t_msl_fastmath, MetalKqMulMmIq4nlT_metal_kq_mulmm_iq4xs_t_msl_tgmem, 32, 3)) + MetalKqMulMmIq4nlT_metal_kq_mulmm_iq4xs_t_msl, MetalKqMulMmIq4nlT_metal_kq_mulmm_iq4xs_t_msl_entry, MetalKqMulMmIq4nlT_metal_kq_mulmm_iq4xs_t_msl_fastmath, MetalKqMulMmIq4nlT_metal_kq_mulmm_iq4xs_t_msl_tgmem, 32, KqRaceScaleForm.pure_d)) results |> emplace(race_kq_mulmm(dev, queue, "kq_mulmm_k3", MetalKqMulMmK3_metal_kq_mulmm_msl, MetalKqMulMmK3_metal_kq_mulmm_msl_entry, MetalKqMulMmK3_metal_kq_mulmm_msl_fastmath, MetalKqMulMmK3_metal_kq_mulmm_msl_tgmem, - MetalKqMulMmK3T_metal_kq_mulmm_k3_t_msl, MetalKqMulMmK3T_metal_kq_mulmm_k3_t_msl_entry, MetalKqMulMmK3T_metal_kq_mulmm_k3_t_msl_fastmath, MetalKqMulMmK3T_metal_kq_mulmm_k3_t_msl_tgmem, 24, 1)) + MetalKqMulMmK3T_metal_kq_mulmm_k3_t_msl, MetalKqMulMmK3T_metal_kq_mulmm_k3_t_msl_entry, MetalKqMulMmK3T_metal_kq_mulmm_k3_t_msl_fastmath, MetalKqMulMmK3T_metal_kq_mulmm_k3_t_msl_tgmem, 24, KqRaceScaleForm.split_d)) results |> emplace(race_kq_mulmm(dev, queue, "kq_mulmm_iq3s", MetalKqMulMmIq3s_metal_kq_mulmm_msl, MetalKqMulMmIq3s_metal_kq_mulmm_msl_entry, MetalKqMulMmIq3s_metal_kq_mulmm_msl_fastmath, MetalKqMulMmIq3s_metal_kq_mulmm_msl_tgmem, - MetalKqMulMmIq3sT_metal_kq_mulmm_iq3s_t_msl, MetalKqMulMmIq3sT_metal_kq_mulmm_iq3s_t_msl_entry, MetalKqMulMmIq3sT_metal_kq_mulmm_iq3s_t_msl_fastmath, MetalKqMulMmIq3sT_metal_kq_mulmm_iq3s_t_msl_tgmem, 26, 1)) + MetalKqMulMmIq3sT_metal_kq_mulmm_iq3s_t_msl, MetalKqMulMmIq3sT_metal_kq_mulmm_iq3s_t_msl_entry, MetalKqMulMmIq3sT_metal_kq_mulmm_iq3s_t_msl_fastmath, MetalKqMulMmIq3sT_metal_kq_mulmm_iq3s_t_msl_tgmem, 26, KqRaceScaleForm.split_d)) results |> emplace(race_kq_mulmm(dev, queue, "kq_mulmm_iq3xxs", MetalKqMulMmIq3xxs_metal_kq_mulmm_msl, MetalKqMulMmIq3xxs_metal_kq_mulmm_msl_entry, MetalKqMulMmIq3xxs_metal_kq_mulmm_msl_fastmath, MetalKqMulMmIq3xxs_metal_kq_mulmm_msl_tgmem, - MetalKqMulMmIq3xxsT_metal_kq_mulmm_iq3xxs_t_msl, MetalKqMulMmIq3xxsT_metal_kq_mulmm_iq3xxs_t_msl_entry, MetalKqMulMmIq3xxsT_metal_kq_mulmm_iq3xxs_t_msl_fastmath, MetalKqMulMmIq3xxsT_metal_kq_mulmm_iq3xxs_t_msl_tgmem, 24, 1)) + MetalKqMulMmIq3xxsT_metal_kq_mulmm_iq3xxs_t_msl, MetalKqMulMmIq3xxsT_metal_kq_mulmm_iq3xxs_t_msl_entry, MetalKqMulMmIq3xxsT_metal_kq_mulmm_iq3xxs_t_msl_fastmath, MetalKqMulMmIq3xxsT_metal_kq_mulmm_iq3xxs_t_msl_tgmem, 24, KqRaceScaleForm.split_d)) results |> emplace(race_kq_mulmm(dev, queue, "kq_mulmm_k2", MetalKqMulMmK2_metal_kq_mulmm_msl, MetalKqMulMmK2_metal_kq_mulmm_msl_entry, MetalKqMulMmK2_metal_kq_mulmm_msl_fastmath, MetalKqMulMmK2_metal_kq_mulmm_msl_tgmem, - MetalKqMulMmK2T_metal_kq_mulmm_k2_t_msl, MetalKqMulMmK2T_metal_kq_mulmm_k2_t_msl_entry, MetalKqMulMmK2T_metal_kq_mulmm_k2_t_msl_fastmath, MetalKqMulMmK2T_metal_kq_mulmm_k2_t_msl_tgmem, 16, 2)) + MetalKqMulMmK2T_metal_kq_mulmm_k2_t_msl, MetalKqMulMmK2T_metal_kq_mulmm_k2_t_msl_entry, MetalKqMulMmK2T_metal_kq_mulmm_k2_t_msl_fastmath, MetalKqMulMmK2T_metal_kq_mulmm_k2_t_msl_tgmem, 16, KqRaceScaleForm.split_dmin)) results |> emplace(race_kq_mulmm(dev, queue, "kq_mulmm_iq2s", MetalKqMulMmIq2s_metal_kq_mulmm_msl, MetalKqMulMmIq2s_metal_kq_mulmm_msl_entry, MetalKqMulMmIq2s_metal_kq_mulmm_msl_fastmath, MetalKqMulMmIq2s_metal_kq_mulmm_msl_tgmem, - MetalKqMulMmIq2sT_metal_kq_mulmm_iq2s_t_msl, MetalKqMulMmIq2sT_metal_kq_mulmm_iq2s_t_msl_entry, MetalKqMulMmIq2sT_metal_kq_mulmm_iq2s_t_msl_fastmath, MetalKqMulMmIq2sT_metal_kq_mulmm_iq2s_t_msl_tgmem, 18, 1)) + MetalKqMulMmIq2sT_metal_kq_mulmm_iq2s_t_msl, MetalKqMulMmIq2sT_metal_kq_mulmm_iq2s_t_msl_entry, MetalKqMulMmIq2sT_metal_kq_mulmm_iq2s_t_msl_fastmath, MetalKqMulMmIq2sT_metal_kq_mulmm_iq2s_t_msl_tgmem, 18, KqRaceScaleForm.split_d)) results |> emplace(race_kq_mulmm(dev, queue, "kq_mulmm_iq2xs", MetalKqMulMmIq2xs_metal_kq_mulmm_msl, MetalKqMulMmIq2xs_metal_kq_mulmm_msl_entry, MetalKqMulMmIq2xs_metal_kq_mulmm_msl_fastmath, MetalKqMulMmIq2xs_metal_kq_mulmm_msl_tgmem, - MetalKqMulMmIq2xsT_metal_kq_mulmm_iq2xs_t_msl, MetalKqMulMmIq2xsT_metal_kq_mulmm_iq2xs_t_msl_entry, MetalKqMulMmIq2xsT_metal_kq_mulmm_iq2xs_t_msl_fastmath, MetalKqMulMmIq2xsT_metal_kq_mulmm_iq2xs_t_msl_tgmem, 16, 1)) + MetalKqMulMmIq2xsT_metal_kq_mulmm_iq2xs_t_msl, MetalKqMulMmIq2xsT_metal_kq_mulmm_iq2xs_t_msl_entry, MetalKqMulMmIq2xsT_metal_kq_mulmm_iq2xs_t_msl_fastmath, MetalKqMulMmIq2xsT_metal_kq_mulmm_iq2xs_t_msl_tgmem, 16, KqRaceScaleForm.split_d)) results |> emplace(race_kq_mulmm(dev, queue, "kq_mulmm_iq2xxs", MetalKqMulMmIq2xxs_metal_kq_mulmm_msl, MetalKqMulMmIq2xxs_metal_kq_mulmm_msl_entry, MetalKqMulMmIq2xxs_metal_kq_mulmm_msl_fastmath, MetalKqMulMmIq2xxs_metal_kq_mulmm_msl_tgmem, - MetalKqMulMmIq2xxsT_metal_kq_mulmm_iq2xxs_t_msl, MetalKqMulMmIq2xxsT_metal_kq_mulmm_iq2xxs_t_msl_entry, MetalKqMulMmIq2xxsT_metal_kq_mulmm_iq2xxs_t_msl_fastmath, MetalKqMulMmIq2xxsT_metal_kq_mulmm_iq2xxs_t_msl_tgmem, 16, 1)) + MetalKqMulMmIq2xxsT_metal_kq_mulmm_iq2xxs_t_msl, MetalKqMulMmIq2xxsT_metal_kq_mulmm_iq2xxs_t_msl_entry, MetalKqMulMmIq2xxsT_metal_kq_mulmm_iq2xxs_t_msl_fastmath, MetalKqMulMmIq2xxsT_metal_kq_mulmm_iq2xxs_t_msl_tgmem, 16, KqRaceScaleForm.split_d)) metal_release(queue) metal_release(dev) return <- results diff --git a/modules/dasLLAMA/dasllama/dasllama_tune.das b/modules/dasLLAMA/dasllama/dasllama_tune.das index 7f759d6721..aa84a2b90a 100644 --- a/modules/dasLLAMA/dasllama/dasllama_tune.das +++ b/modules/dasLLAMA/dasllama/dasllama_tune.das @@ -375,13 +375,13 @@ class private TunedMacro : AstFunctionAnnotation { return true } } - // an explicit `perm=` pin names the row the author wants - a miss is an authoring error + //! an explicit `perm=` pin names the row the author wants - a miss is an authoring error let av = find_arg(args, "perm") if (av is tString) { errors := "tuned: unknown perm '{suffix}' (not in grid)" return false } - // sidecar-sourced: measured data, not law - a perm the grid no longer offers falls back instead of bricking the very tuner that would re-race it + //! a sidecar perm is measured data, not law - one the grid dropped falls back rather than brick the tuner that would re-race it let fname = "{func.name}" let fb = g_fallbacks?[fname] ?? DEFAULT_PERM print("dasllama_tune: {fname}: sidecar perm '{suffix}' is not in the grid (stale mint?) - stamping the fallback '{fb}'\n") diff --git a/modules/dasLLAMA/followup_metal.md b/modules/dasLLAMA/followup_metal.md index 7477694ad6..0bb43f8a04 100644 --- a/modules/dasLLAMA/followup_metal.md +++ b/modules/dasLLAMA/followup_metal.md @@ -82,3 +82,12 @@ Measured ceiling of ALL remaining converts (`DASLLAMA_METAL_PREFILL_SKIP=act_cvt every elementwise kernel, attention, the CPU-fallback paths, logits/readback - and re-opens the numerics bars across the parity and prefill suites. ITS OWN ARC, planned; the acceptance bar is the existing parity suites plus a pp/tg board A/B per class. + +## 5. The dense-KQ tensor twins' missing stamps + +The nine iquant/split-scale tensor mul_mm twins stamp `T` and `TH` only. k4/k5/k6 additionally +carry the tall (`TH128`) and double-buffered (`THDb`, `THDb128`) stamps, and that is where the +tall in-kernel-dequant win lives (`ARCHITECTURE_GPU_PREFILL.md` sec.2.2c form 1). Race a tall +stamp for the iquant scaffold before assuming the k6 result transfers; the arc-end matrix's +soft spot is the deep-K w2 column (k6 0.91x / iq3s 0.88x / iq3xxs 0.92x vs llama.cpp on m5), +which is exactly the column a tall stamp serves. diff --git a/modules/dasLLAMA/harness/tune_kernels.das b/modules/dasLLAMA/harness/tune_kernels.das index 37eae1e03b..10e77f53ae 100644 --- a/modules/dasLLAMA/harness/tune_kernels.das +++ b/modules/dasLLAMA/harness/tune_kernels.das @@ -2349,17 +2349,9 @@ def private metal_twin_absorb(var wins : array; var ran : bool&; r) { } } -// half a percent, best-of-3: the crown's real serving win where it wins is small (+0.6% M5) -// and its real serving loss where it loses is large (-8.6% M4) - the margin only has to -// separate those from run noise, and a spuriously minted crown costs a re-mint, not a board let CONFIRM_TG_MARGIN = 1.005lf -// The serving-truth crown confirm: some GEMV crowns are UNDECIDABLE by any isolated kernel -// race (kq_gemv_iq2xxs_f4: every raced regime crowns the M4-class twin that serving rejects - -// a mixed-stream occupancy effect only the real decode graph shows). Those crowns are minted -// HERE, on the real shapes: two temp manifests differing only in runtime.metal_tensor -// membership, lcpp_bench tg128 per arm under DAS_TUNE_MANIFEST, margin-decided. No vehicle -// on the box = no crown (the base kernel is the safe side) with a provision hint. +[arch(at="../ARCHITECTURE_MEASUREMENT.md#kernel-race-fidelity")] def private confirm_serving_crown(crown : string; var wins : array) { let model = confirm_decode_model() if (empty(model)) { @@ -2397,10 +2389,7 @@ def private confirm_decode_model() : string { return empty(best) ? "" : "{md}{best}" } -// one confirm arm: a minimal manifest (empty kernels - both arms stamp identical fallbacks, -// so the tg DELTA isolates the crown) whose runtime.metal_tensor carries the raced wins plus -// optionally the crown under test; lcpp_bench serves it via DAS_TUNE_MANIFEST and its tg128 -// line is the verdict +[arch(at="../ARCHITECTURE_MEASUREMENT.md#kernel-race-fidelity")] def private confirm_tg_run(model : string; wins : array; extra : string) : double { var crowns = join(wins, ",") if (!empty(extra)) { @@ -2583,7 +2572,6 @@ def main : int { // nolint:STYLE037,STYLE038 — flat one-bench-per-kernel rail; metal_twin_absorb(mt_wins, mt_ran, r) } if (mt_ran) { - // the crown no isolated race can decide - minted on the real shapes instead confirm_serving_crown("kq_gemv_iq2xxs_f4", mt_wins) } } diff --git a/modules/dasLLAMA/performance/defaults/arm-i8mm.tune-defaults.json b/modules/dasLLAMA/performance/defaults/arm-i8mm.tune-defaults.json new file mode 100644 index 0000000000..c0cf3da8a0 --- /dev/null +++ b/modules/dasLLAMA/performance/defaults/arm-i8mm.tune-defaults.json @@ -0,0 +1,69 @@ +{ + "kernels" : { + "add_inplace" : "vec16_u2", + "cvt_f32_to_f16" : "vec16", + "rope_scaled_neox_tab" : "vec8_u2", + "softmax" : "vec8_u2", + "q51q8_tile_gen" : "mr8", + "mul_inplace" : "vec8_u2", + "quantize_q8_0_bs_into_ptr" : "plain", + "dot_q8kv" : "vec4_u4", + "iq2xsq8_tile_gen" : "mr8", + "dot_q8q8" : "vec16", + "quantize_q8kv_row" : "plain", + "iq4xsq8_tile_gen" : "mr8", + "iq4nlq8_tile_gen" : "mr8", + "axpy_f16" : "vec8_u2", + "q40q8_tile_gen" : "mr8", + "axpy_tq4kv" : "vec8_u2", + "cvt_tq4kv_to_f32" : "vec8_u2", + "axpy" : "vec8_u2", + "dot_q8q8kv" : "plain", + "dot_mx4q8" : "u2", + "softmax_sink" : "vec8_u2", + "iq2xxsq8_tile_gen" : "mr8", + "dot_q8q8_laneq4x4" : "u2", + "dot_bf16" : "vec8_u2", + "iq2sq8_tile_gen" : "mr8", + "add_scale_inplace" : "vec8_u2", + "cvt_q8kv_to_f32" : "vec8_u2", + "axpy_q8kv" : "vec8_u2", + "dot_q8q8_f16s" : "vec16", + "q8q8_tile_gen" : "mr8_budget", + "quantize_q8_0_into_ptr" : "plain", + "k4q8_tile_gen" : "mr8", + "gemm_f32_uk_4x16" : "u2", + "dot_q51e" : "vec16", + "k5q8_tile_gen" : "mr8", + "k6q8_tile_gen" : "mr8", + "k3q8_tile_gen" : "mr8", + "k2q8_tile_gen" : "mr8", + "dot_f16" : "vec8_u2", + "cvt_f16_to_f32" : "vec8_u2", + "dot" : "vec8_u2", + "iq3xxsq8_tile_gen" : "mr8", + "iq3sq8_tile_gen" : "mr8", + "dot_q8tq4kv" : "vec16", + "scale_inplace" : "vec8_u2", + "dot_q4" : "vec4_u4", + "quantize_tq4kv_row" : "plain", + "copy_floats" : "vec8_u2", + "rmsnorm" : "vec8" + }, + "provenance" : { + "validation" : "ok", + "noise_probes" : "start cv 0.36%; mid1 cv 0.38%; mid2 cv 0.48%; end cv 0.27%", + "platform" : "darwin", + "noise_floor_cv_pct" : "0.48", + "features" : "neon;dotprod;i8mm;bf16;fullfp16;lse", + "class" : "arm-i8mm", + "written" : "2026-09-01T15:43:02.269Z", + "validation_demoted" : "0", + "mode" : "normal", + "dasllama_version" : "15", + "origin" : "profile", + "noise" : "ok", + "arch" : "arm64", + "validation_max_drift_pct" : "1.09" + } +} \ No newline at end of file diff --git a/modules/dasLLAMA/tests/test_metal_gemm_kernels.das b/modules/dasLLAMA/tests/test_metal_gemm_kernels.das index 16409f9ce5..ccaf9e4524 100644 --- a/modules/dasLLAMA/tests/test_metal_gemm_kernels.das +++ b/modules/dasLLAMA/tests/test_metal_gemm_kernels.das @@ -2065,7 +2065,7 @@ def test_metal_gemm_kernels(t : T?) { kq_mulmm_gate(t, dev, queue, 24, false, 32, 256, 128) kq_mulmm_gate(t, dev, queue, 25, false, 64, 512, 64) kq_mulmm_gate(t, dev, queue, 25, false, 32, 256, 128) - // the iquant tensor twins (T + TH stamps; twins feint without Metal-4) + //! the iquant tensor twins (T + TH stamps; twins feint without Metal-4) for (fmt in [44, 45, 3, 33, 34, 2, 23, 24, 25]) { kq_mulmm_gate(t, dev, queue, fmt, true, 64, 512, 64) kq_mulmm_gate(t, dev, queue, fmt, true, 64, 512, 64, halfx = true) diff --git a/modules/dasLLVM/ARCHITECTURE.md b/modules/dasLLVM/ARCHITECTURE.md index 1894290013..59a878dba9 100644 --- a/modules/dasLLVM/ARCHITECTURE.md +++ b/modules/dasLLVM/ARCHITECTURE.md @@ -145,3 +145,19 @@ detection-derived append - `+dotprod` always, `+i8mm` when `cpu_supports` confir EXECUTION-safe: the silicon running this process really has the instruction. A `DAS_JIT_ARM64_FORCE_FEATURES` append is EMISSION-only: it may name silicon this box does not have, so the artifact is for another machine and executing it here traps. + +## 5. The tune sidecar is a module-cache dependency {#tune-sidecar-cache-pin} + +`[tuned]` and `[tune_policy]` stamp a function's hints at macro time out of the tune sidecar, +and the module cache stores the stamped AST. A re-mint therefore has to invalidate the cached +record, or a later run serves stamps minted against the old sidecar until some source file +changes. `read_manifest` (`daslib/llvm_tune.das`) registers the sidecar path with +`add_module_cache_dependency` on every read; the record carries the path with the file's byte +size and content hash, and the reader re-validates both before it trusts the payload. Content, +not mtime: an app that rewrites its sidecar byte-identically on exit must not churn the cache. + +The registration runs before the staleness gate, and for a path that does not exist yet, +because the mints that matter most produce no successful read - the first mint has no sidecar, +and a re-mint replaces one the gate rejected. An absent file registers as size -1 and hash 0, +which the next run's re-validation sees change. Registering is a no-op outside compilation, so +the manifest's runtime readers reach the same call unconditionally. diff --git a/modules/dasLLVM/CMakeLists.txt b/modules/dasLLVM/CMakeLists.txt index 10526526ad..0f26296b24 100644 --- a/modules/dasLLVM/CMakeLists.txt +++ b/modules/dasLLVM/CMakeLists.txt @@ -149,6 +149,10 @@ IF ((NOT DAS_LLVM_INCLUDED) AND ((NOT ${DAS_LLVM_DISABLED}) OR (NOT DEFINED DAS_ DESTINATION ${DAS_INSTALL_MODULESDIR}/dasLLVM/daslib ) file(GLOB DASLLVM_BINDINGS ${DAS_LLVM_DIR}/bindings/*.das) + # daslib/ carries [arch] citations; the bundle gets generated excerpts of the cited + # sections (registry processed by the root CMakeLists) + set_property(GLOBAL APPEND PROPERTY DAS_ARCH_EXTRACT_SPECS + "modules/dasLLVM/daslib|modules/dasLLVM|${DAS_INSTALL_MODULESDIR}/dasLLVM") install(FILES ${DASLLVM_BINDINGS} DESTINATION ${DAS_INSTALL_MODULESDIR}/dasLLVM/bindings) install(FILES ${DAS_LLVM_DIR}/.das_module DESTINATION ${DAS_INSTALL_MODULESDIR}/dasLLVM/) install(FILES ${DAS_LLVM_DIR}/ENVIRONMENT.md DESTINATION ${DAS_INSTALL_MODULESDIR}/dasLLVM/) diff --git a/modules/dasLLVM/daslib/llvm_tune.das b/modules/dasLLVM/daslib/llvm_tune.das index 3e5d46b128..10eb711604 100644 --- a/modules/dasLLVM/daslib/llvm_tune.das +++ b/modules/dasLLVM/daslib/llvm_tune.das @@ -353,13 +353,13 @@ def tune_sidecar_stale(path : string) : bool { // the sidecar's "kernels" section as a flat { function name : perm suffix } map; // found = the file existed, parsed to an object, and is NOT stale vs the running binary // (a "kernels"-less but fresh sidecar counts as found-empty: runtime-knob-only files) +[arch(at="../ARCHITECTURE.md#tune-sidecar-cache-pin")] def private read_manifest(path : string; var found : bool&) : table { found = false var tab : table if (empty(path)) { return <- tab } - // pins the sidecar into the module cache record BEFORE the staleness gate (no-op at runtime): a re-mint or a first mint must invalidate cached [tuned]/[tune_policy] stamps add_module_cache_dependency(path) if (tune_sidecar_stale(path)) { return <- tab diff --git a/modules/dasLLVM/tests/test_tuned.das b/modules/dasLLVM/tests/test_tuned.das index 16000f8b67..fd6c527272 100644 --- a/modules/dasLLVM/tests/test_tuned.das +++ b/modules/dasLLVM/tests/test_tuned.das @@ -6,7 +6,7 @@ require dasllama/dasllama_tune require daslib/fio require daslib/module_path require strings -require llvm/daslib/llvm_tune // tune_box_identity - the stale-sidecar cell's manifest must pass the box gate +require llvm/daslib/llvm_tune // Part 2 (final assembly): one source of truth, reconstituted into the real symbol. def template sample_dot_template(a, b : array) : float { @@ -24,7 +24,7 @@ def sample_dot(a, b : array) : float {} // `fallback=` supplies a kernel's own hand hints when they aren't the global vec8_u2; the box // profile still overrides it (precedence: perm > profile > fallback > vec8_u2). Validation proof: -// an unknown fallback name fails the compile with "tuned: unknown perm" (negative-probed), so this +// an unknown fallback name fails the compile with "tuned: fallback perm ... not in grid", so this // compiling proves the fallback names a valid grid perm — the one consulted whenever no profile // entry / perm pin applies (and no profile carries this test-local kernel's key). [tuned(fallback = "u2", src = "sample_dot_template")] @@ -46,13 +46,10 @@ def test_tuned_reconstitutes(t : T?) { t |> equal(sample_dot_u2(a, b), reference(a, b), "tuned default=u2 sample_dot_u2 == reference") } -// The sidecar is measured data, not law: a manifest entry naming a perm the grid no longer -// offers (a grid rework since the mint) must not brick the compile - the macro stamps the -// fallback and says so, so the tuner that would re-race the entry can always run. The hard -// refusal survives only for an explicit perm= pin (cant_tuned_bad_pin.das). +//! the sidecar arm: an out-of-grid sidecar perm stamps the fallback instead of failing the compile - the explicit perm= pin twin is cant_tuned_bad_pin.das [test] def test_tuned_stale_sidecar_perm(t : T?) { - let args <- get_command_line_arguments() // locked view - do not delete + let args <- get_command_line_arguments() let bin = args[0] let tmpRes = create_temp_file_result("tuned_stale_perm_", ".json") if (tmpRes is error) { diff --git a/skills/internal/build_and_debug.md b/skills/internal/build_and_debug.md index 3186ae1723..1704299185 100644 --- a/skills/internal/build_and_debug.md +++ b/skills/internal/build_and_debug.md @@ -92,7 +92,7 @@ Dev-tier rails that cut the edit-compile-run loop; never benchmark through them. - **`--jit-opt-level=0`** (after the script separator) disables the LLVM IR pass pipeline AND drives the DLL path's codegen-side target machine (fast isel) - a true end-to-end O0 for `-jit` runs, several-fold faster to build. Exe (`-exe`) and AOT-object emission still pin codegen level 3 deliberately (shipped artifacts). At level 0 the injected tune-policy default becomes `fallback` (winners are raced under O3 codegen); `DAS_TUNE_POLICY` still overrides. - **`--jit-split-modules=-1`** (after the `--` separator) partitions codegen per das-module - parallel emit makes the cold build several-fold faster, and the per-module obj cache (on by default under split; `--jit-obj-cache=0` disables) re-emits only from the FIRST CHANGED MODULE onward on a warm edit. **Invalidation is positional, so require order is the cache layout: place the module you are actively editing as LATE in the require chain as its dependencies allow** - a hot-edit module required early drags everything after it into every rebuild, one required last rebuilds nearly alone (canonical: the dasLLAMA umbrella requires the vulkan drivers last for exactly this reason). The cache holds ONE generation - reverting an edit re-emits from the reverted module on, same as the edit did. Split partitions lose cross-module inlining, so never benchmark through a split DLL. -- **Front-end cache: `-module-cache `** (host flag, before the script; also on `daslang-live`) caches the compiled AST module graph via the env-serializer rail: the first run writes the file, later runs deserialize post-infer modules instead of parsing them - measured ~12x on the dasLLAMA graph (12.2s -> 1.0s compile-only). Invalidation is mtime-based with the same positional model as the obj cache: an edited module recompiles itself and everything after it, and the cache rewrites itself. A record that can never deserialize in a cold process (`llvm_func.das` - its `[dasbind]` externs register into the dasbind builtin module during compile) reparses in place (~0.03s) without cutting the stream - the run reports `deser: partial`. `-ser ` / `-deser ` are the explicit write/read halves (the round-trip test instrument; `deser: clean` vs `deser: FALLBACK` is the honest verdict). Composes with the split obj cache: a warm `-jit` edit deserializes the unchanged prefix and re-emits only the tail partitions. The cache is one file you own - delete it when in doubt. +- **Front-end cache: `-module-cache `** (host flag, before the script; also on `daslang-live`) caches the compiled AST module graph via the env-serializer rail: the first run writes the file, later runs deserialize post-infer modules instead of parsing them - measured ~12x on the dasLLAMA graph (12.2s -> 1.0s compile-only). Invalidation is mtime+size on the sources and content-hash on the extra compile-time inputs a macro pins (`add_module_cache_dependency` - a tune sidecar, a config), with the same positional model as the obj cache: an edited module recompiles itself and everything after it, and the cache rewrites itself. The pinned inputs are content-hashed, not mtime'd, so an app that rewrites its sidecar byte-identically on exit does not churn the cache. A record that can never deserialize in a cold process (`llvm_func.das` - its `[dasbind]` externs register into the dasbind builtin module during compile) reparses in place (~0.03s) without cutting the stream - the run reports `deser: partial`. `-ser ` / `-deser ` are the explicit write/read halves (the round-trip test instrument; `deser: clean` vs `deser: FALLBACK` is the honest verdict). Composes with the split obj cache: a warm `-jit` edit deserializes the unchanged prefix and re-emits only the tail partitions. The cache is one file you own - delete it when in doubt. ## Build configurations (module flags) diff --git a/src/ast/ast_parse.cpp b/src/ast/ast_parse.cpp index b7ef966f4e..bf0377596c 100644 --- a/src/ast/ast_parse.cpp +++ b/src/ast/ast_parse.cpp @@ -588,6 +588,7 @@ namespace das { // sanity bound on a record's macro-dependency count: a real program registers a handful // (one per consumed sidecar/config); anything past this reads as stream corruption static constexpr uint32_t SER_MAX_MACRO_DEPS = 4096u; + static constexpr uint64_t FNV64A_SEED = 14695981039346656037ul; // an EXISTING empty file must hash unlike an absent one (hash 0) void statAndHashFileDependency ( const string & path, int64_t & size, uint64_t & hash ) { size = -1; @@ -604,7 +605,7 @@ namespace das { size_t got = fsize ? fread(bytes.data(), 1, size_t(fsize), f) : 0; if ( got == size_t(fsize) ) { size = int64_t(fsize); - hash = fsize ? hash_block64(bytes.data(), size_t(fsize)) : 14695981039346656037ul; // fnv64a seed for the empty file + hash = fsize ? hash_block64(bytes.data(), size_t(fsize)) : FNV64A_SEED; } } fclose(f); @@ -677,10 +678,9 @@ namespace das { return false; } - // a compile-time input a macro consumed (a tune sidecar) changed since the record was - // written: the cached stamps were minted against the old file - same prefix cutoff as a - // changed source. Compared by CONTENT (size + hash), not mtime: apps rewrite their - // sidecar byte-identically on exit, and that must not churn the cache. + // a compile-time input a macro consumed (a tune sidecar) changed since this record was + // written, so its cached stamps are stale. Compared by CONTENT (size + hash), not mtime: + // apps rewrite their sidecar byte-identically on exit, and that must not churn the cache. for ( auto & dep : savedDeps ) { int64_t depSize = -1; uint64_t depHash = 0; @@ -1437,7 +1437,6 @@ namespace das { *serializer_write << fileMtime; *serializer_write << fileSize; *serializer_write << const_cast(fileName); - // macro file dependencies: header-resident, validated by the reader before the payload uint32_t depCount = uint32_t(program->moduleCacheDependencies.size()); *serializer_write << depCount; for ( auto & dep : program->moduleCacheDependencies ) { diff --git a/src/builtin/module_builtin_ast.cpp b/src/builtin/module_builtin_ast.cpp index 9ec84ff9ed..bcbc16a0c4 100644 --- a/src/builtin/module_builtin_ast.cpp +++ b/src/builtin/module_builtin_ast.cpp @@ -267,11 +267,9 @@ namespace das { return program; } - // Pin an extra compile-time file input (a tune sidecar, a config a macro read) into the - // module cache: the record stores (path, mtime, size) and the reader re-validates them, so - // a change to the file invalidates the cached module instead of serving stale macro output. - // Deliberately a NO-OP outside compilation - manifest readers run at runtime too, and the - // call site stays unconditional. + // pins a file a macro read (a tune sidecar) as a compile-time input of the compiling module, + // so a change to it invalidates the cached record. A deliberate NO-OP outside compilation - + // manifest readers also run at runtime, and their call site stays unconditional. void addModuleCacheDependency ( const char * path, Context *, LineInfoArg * ) { auto program = daScriptEnvironment::getBound()->g_Program; if ( !program || !path || !path[0] ) return; diff --git a/tests/module_cache/_fixtures/mc_dep_drv.das b/tests/module_cache/_fixtures/mc_dep_drv.das index 41f15f6d4c..7d42cd1982 100644 --- a/tests/module_cache/_fixtures/mc_dep_drv.das +++ b/tests/module_cache/_fixtures/mc_dep_drv.das @@ -3,8 +3,6 @@ options indenting = 4 require mc_dep_used -// plain main-line driver (NOT a dastest child): the module cache serves the main compile's -// require chain, and only there does mc_dep_used land in a cached record [export] def main { print("PROBED={probed()}\n") diff --git a/tests/module_cache/_fixtures/mc_dep_mod.das b/tests/module_cache/_fixtures/mc_dep_mod.das index e7b7dbe049..5d1e80be79 100644 --- a/tests/module_cache/_fixtures/mc_dep_mod.das +++ b/tests/module_cache/_fixtures/mc_dep_mod.das @@ -7,9 +7,8 @@ require daslib/ast require daslib/ast_boost require daslib/fio -// The module-cache dependency gate's fixture macro: [dep_probe] registers the file named by -// MC_DEP_FILE as a compile-time input of the compiling module, exactly the way a tune-sidecar -// read does (add_module_cache_dependency in llvm_tune's read_manifest). +//! the module-cache dependency gate's fixture macro - it pins MC_DEP_FILE exactly the way a +//! tune-sidecar read does (llvm_tune's read_manifest calls add_module_cache_dependency) [function_macro(name = "dep_probe")] class private DepProbe : AstFunctionAnnotation { def override apply(var func : FunctionPtr; var group : ModuleGroup; args : AnnotationArgumentList; var errors : das_string) : bool { diff --git a/tests/module_cache/_fixtures/mc_dep_used.das b/tests/module_cache/_fixtures/mc_dep_used.das index b1dd2ee4b1..a5533854d1 100644 --- a/tests/module_cache/_fixtures/mc_dep_used.das +++ b/tests/module_cache/_fixtures/mc_dep_used.das @@ -5,8 +5,8 @@ module mc_dep_used shared private require mc_dep_mod -// the [dep_probe] carrier must live in a REQUIRED module: only required modules land in the -// module cache, so only here does the registered dependency reach a cached record +//! the [dep_probe] carrier must sit in a REQUIRED module - only required modules land in the +//! module cache, so only here does the registered dependency reach a cached record [dep_probe] def public probed : int { return 42 diff --git a/tests/module_cache/test_macro_dep_invalidate.das b/tests/module_cache/test_macro_dep_invalidate.das index 59a152c98c..2d2a1f80cf 100644 --- a/tests/module_cache/test_macro_dep_invalidate.das +++ b/tests/module_cache/test_macro_dep_invalidate.das @@ -6,14 +6,14 @@ require dastest/testing_boost public require strings require daslib/fio -// argv[0] is the running interpreter; dastest is invoked as -// `daslang(.exe) dastest/dastest.das ...`, so argv[0] is the daslang binary to spawn. +//! the daslang binary to spawn - dastest runs as `daslang(.exe) dastest/dastest.das ...`, +//! so argv[0] is the interpreter, not the test script def das_exe() : string { let args <- get_command_line_arguments() return empty(args) ? "" : args[0] } -// child env rides the command string (the portable set/prefix split popen_argv cannot carry) +//! child env rides the command string (the portable set/prefix split popen_argv cannot carry) def env_run(dep, cmd : string; var output : string&) : int { let win = get_platform_name() == "windows" let full = win ? "set MC_DEP_FILE={dep}&& {cmd}" : "MC_DEP_FILE={dep} {cmd}" @@ -33,11 +33,9 @@ def report_child(t : T?; phase : string; rc : int; out : string; marker : string return ok } -// A macro-registered file dependency (add_module_cache_dependency - the tune-sidecar shape) -// must gate the module cache by CONTENT: an untouched or byte-identically rewritten input -// serves from cache, a content change invalidates the record and reparses - otherwise the -// cache serves macro output minted against the old file (stale tune stamps, empty -// tune_status, winners nobody raced). +//! the module cache must gate a macro-registered file input by CONTENT, not mtime: a tune +//! sidecar is rewritten byte-identically on every exit, and a record served stale replays +//! macro output minted against the old file - stale stamps, empty tune_status, unraced winners. [test] def test_macro_dep_invalidate(t : T?) { var terr : string @@ -60,13 +58,10 @@ def test_macro_dep_invalidate(t : T?) { var warm : string let warmRc = env_run(dep, cmd, warm) t |> success(report_child(t, "warm", warmRc, warm, "deser: clean"), "warm run served every module from the cache") - // byte-identical rewrite (fresh mtime): content gating must keep the cache warm - the - // bench sidecar is rewritten on every exit, and mtime gating would repay the parse each run fwrite(dep, "content v1") var rewrite : string let rewriteRc = env_run(dep, cmd, rewrite) t |> success(report_child(t, "rewrite", rewriteRc, rewrite, "deser: clean"), "byte-identical rewrite stays cached") - // content change: the record must invalidate, name the file, and the reparse still pass fwrite(dep, "content v2 - re-minted") var changed : string let changedRc = env_run(dep, cmd, changed) From f37bdb0c98ea394f751b0acd939dcf0e0529e686 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 09:31:33 -0700 Subject: [PATCH 26/35] the audit round lands: four test-gate repairs, ten checklist repairs, three new tests, one detwinned kernel Code, from the seven checklist audits and the TDD round: the 21 crown-gated kernel classes join the census blind-spot ledger; the twelve crown names join the support matrix's CROWNED witness plus a skip-aware tensor-twins/iq2xxs row on the arc vehicle (a local requant no fetch provisions - absence skips, never reds); both shared kernel gates gain a poisoned-expectation control that must RED; the Metal-4 capability decline becomes a real skip; the clock-governor burn moves into race_pair_ms so all eight race callers inherit it; the iq2xxs GEMV pair collapses into one template with an F4 stamp axis (oracle-proven); DASLLAMA_CONFIRM_IQ2XXS becomes a declared, announcing knob; plus the clock-control marker, the iq4_lut charter row, figure provenance in followup_metal 4, and the falsified arm-i8mm ledger line rewritten. Tests, closing the TDD round's top gaps: test_tuned gains the wiring cell - a [tuned] client under -module-cache with a real tune sidecar must reparse when the sidecar's content changes, through the production read_manifest pin; test_macro_dep_invalidate gains the absent-then-present phase through a RELATIVE dep path (the first-mint case the architecture doc calls the one that matters, plus the cwd-absolutize arm); cant_tuned_bad_fallback pins the fallback-not-in-grid refusal. Checklists, per their own audits: the two race rules this PR added rebuilt property-scoped with the serialized-rig exception ledgered; homeless rules moved to folders whose walks surface them (census to REVIEW_GPU, bench-list and harness routing to the dasLLAMA root, ref-pin merged into benchmarks, consent to REVIEW_EXCHANGE, kept-comments to utils/mcp); the macro-file-pin duty guards the new cache mechanism; the bit-exact discriminator becomes bit-identity; the third-party-walls rule splits into its ban and its scratch property; seven weakening rules collapse into one; and a dozen wording repairs across tests, dasLLVM, daslib, src/builtin and review_md. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014mq9AKyoAwvD5T1cigTnRw --- daslib/REVIEW.md | 34 +++-- modules/dasLLAMA/ARCHITECTURE_GPU.md | 2 +- modules/dasLLAMA/REVIEW.md | 18 ++- modules/dasLLAMA/REVIEW_EXCHANGE.md | 7 ++ modules/dasLLAMA/REVIEW_GPU.md | 45 ++++--- modules/dasLLAMA/benchmarks/REVIEW.md | 57 +++++---- .../benchmarks/matmul/bench_metal_kq_race.das | 6 +- modules/dasLLAMA/dasllama/dasllama_env.das | 5 + .../dasllama/dasllama_metal_common.das | 16 ++- .../dasllama/dasllama_metal_kernels.das | 116 ++++++------------ .../dasllama/dasllama_metal_prefill.das | 1 - modules/dasLLAMA/followup_metal.md | 24 ++-- modules/dasLLAMA/harness/tune_kernels.das | 3 +- modules/dasLLAMA/performance/REVIEW.md | 13 +- modules/dasLLAMA/tests/REVIEW.md | 67 ++++------ .../dasLLAMA/tests/test_kernel_coverage.das | 12 +- .../tests/test_metal_gemm_kernels.das | 24 ++-- .../tests/test_metal_gemv_kernels.das | 14 ++- .../tests/test_metal_support_matrix.das | 19 ++- modules/dasLLVM/REVIEW.md | 17 +-- .../dasLLVM/tests/cant_tuned_bad_fallback.das | 25 ++++ modules/dasLLVM/tests/test_tuned.das | 56 +++++++++ skills/review_md.md | 3 +- src/ast/ast_parse.cpp | 2 +- src/builtin/REVIEW.md | 5 +- .../test_macro_dep_invalidate.das | 20 +++ utils/mcp/REVIEW.md | 4 + 27 files changed, 376 insertions(+), 239 deletions(-) create mode 100644 modules/dasLLVM/tests/cant_tuned_bad_fallback.das diff --git a/daslib/REVIEW.md b/daslib/REVIEW.md index 9138d33277..2b66f2e2a4 100644 --- a/daslib/REVIEW.md +++ b/daslib/REVIEW.md @@ -1,8 +1,8 @@ # daslib Code Review Checklist -**Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist.** Architecture doc: -`daslib/ARCHITECTURE.md` and its companions `ARCHITECTURE_LINT.md`, `ARCHITECTURE_EMIT.md`, -`ARCHITECTURE_LINQ.md`. Planned work: `plans/daslib_comment_sweep.md` (repo root). +**Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist.** Architecture +docs: `ARCHITECTURE.md`, `ARCHITECTURE_LINT.md`, `ARCHITECTURE_EMIT.md`, `ARCHITECTURE_LINQ.md`. +Planned work: `plans/daslib_comment_sweep.md` (repo root). A diff touching the linq family - `linq*.das`, `sql_*.das` - applies `REVIEW_LINQ.md` too. **A diff that adds or changes a lint report path (`perf_warning` / `style_warning` / @@ -20,7 +20,8 @@ the actionable line is an intermediate call site. inside a lint visitor method is a defect.** The guard's position decides which rules apply inside a lambda; the per-rule policy is in `ARCHITECTURE_LINT.md` sec. 1. -**Never move `add_ptr_ref`'s depth tracking behind the `in_closure` gate.** The rules block +**Never move perf_lint's PERF023 `add_ptr_ref` splice-depth tracking (`perf_lint.das`) behind +the `in_closure` gate.** The rules block is itself a closure, so a gated tracker never sees the splice. **A lint warning that anchors anywhere but the code its fix rewrites is a defect.** A remedy @@ -34,9 +35,9 @@ construct's visit. **Never keep per-loop visitor state in a bare scalar - keep it in a stack.** A scalar survives into the sibling loop's exit path and unbalances its counter. -**A diff that makes a daslib predicate or emitted identifier depend on a C++-side -definition records the pair in the architecture doc, in its module's section, naming both -sides.** Nothing catches it when one side later moves alone. +**A diff that adds or changes a daslib fact - code or a `//!` contract - whose truth is +decided by a C++-side definition records the pair in the architecture doc, in its module's +section, naming both sides.** Nothing catches it when one side later moves alone. **When a diff changes one side of a recorded daslib/C++ pair so the two no longer match, it changes the other side and updates the pair's architecture-doc entry in the same @@ -45,12 +46,11 @@ diff.** **A diff that adds an architecture-doc entry adds it only where no name, shape, or test can carry the fact.** -**Weakening `tests/lint/test_nolint_suppression.das` is a defect** - it pins that a string +**A diff that changes daslib's nolint scanning - suppression parsing, staleness detection, or +the header window - keeps `tests/lint/test_nolint_suppression.das` and +`tests/lint/test_stale_nolint.das` green without weakening them.** They pin that a string literal, a URL, and a mid-comment `nolint:` do not suppress while a first-token directive -after `//` or `//!` does. - -**Weakening `tests/lint/test_stale_nolint.das` is a defect** - it pins that a -`lint-skip-file` past the header window is prose. +after `//` or `//!` does, and that a `lint-skip-file` past the header window is prose. **A diff that changes `build_lint_macro_disabled` keeps its four sources layered in this order: defaults, repo `off`, repo `on`, environment.** Env last lets a one-run @@ -60,12 +60,10 @@ order: defaults, repo `off`, repo `on`, environment.** Env last lets a one-run seeding and the repo `off` directives.** Repo policy must not silence the rule a fixture exists to exercise. -**Weakening `tests/lint/test_lint_config.das` is a defect** - it pins that each `[format]` -key resolves nearest-wins independently up to the `.git` root, so a nearer `.lint_config` -declaring one key does not reset the other. - -**Weakening the kept-comment cases in `utils/mcp/test_tools.das` is a defect** - they pin -the formatter's kept set. +**A diff that changes daslib's `.lint_config` resolution keeps `tests/lint/test_lint_config.das` +green without weakening it** - it pins that each `[format]` key resolves nearest-wins +independently up to the `.git` root, so a nearer `.lint_config` declaring one key does not +reset the other. **A daslib module that emits a lint rule id joins `RULE_MODULES` in `utils/lint/REVIEW.das` in the same change.** An unlisted module's ids are never scanned, so its fixture-and-rst diff --git a/modules/dasLLAMA/ARCHITECTURE_GPU.md b/modules/dasLLAMA/ARCHITECTURE_GPU.md index 9f6274d9ee..5f2a285b12 100644 --- a/modules/dasLLAMA/ARCHITECTURE_GPU.md +++ b/modules/dasLLAMA/ARCHITECTURE_GPU.md @@ -10,7 +10,7 @@ that a question answered for one backend has an obvious address in the other. Th | role | holds | must not hold | |---|---|---| | the kernel home
`dasllama_metal_kernels`, `dasllama_vulkan_classes` | kernel source, the derived-access/PSO census | device state, engine types | -| `dasllama__common`
`dasllama_metal_common`, `dasllama_vulkan_common` | device state, buffer/command plumbing, hazard + capture rail, profiler | driver policy | +| `dasllama__common`
`dasllama_metal_common`, `dasllama_vulkan_common` | device state, buffer/command plumbing, hazard + capture rail, profiler, shared quant-decode helpers (`iq4_lut`) | driver policy | | `dasllama__decode`
`dasllama_metal_decode`, `dasllama_vulkan_decode` | the resident token-step driver + decode-time arms | kernel bodies | | `dasllama__prefill`
`dasllama_metal_prefill`, `dasllama_vulkan_prefill` | the batched prefill driver + batch arms | kernel bodies | | `dasllama__shapes`
`dasllama_metal_shapes` | PORTABLE servability gates - no GPU C++ require, so any box can bake | device calls | diff --git a/modules/dasLLAMA/REVIEW.md b/modules/dasLLAMA/REVIEW.md index 0b87b0991b..b565ca2ddb 100644 --- a/modules/dasLLAMA/REVIEW.md +++ b/modules/dasLLAMA/REVIEW.md @@ -32,6 +32,14 @@ opens that checklist on its own. **A GPU kernel, driver, dispatch-class, or K/V-mirror change applies `REVIEW_GPU.md`.** +**A `harness/` instrument that times or measures applies `../benchmarks/`'s instrument rules +(`benchmarks/REVIEW.md`) as if it lived there.** The harness folder carries no checklist of +its own, and a timing rig is the same instrument wherever it parks. + +**A diff that adds or moves a row in `harness/tune_kernels.das`'s bench list keeps +`dot_q8q8_laneq4x4` last.** That bench pins the repack backend for the rest of the process, so +a row after it is timed against the pinned backend instead of the backend it would select. + **A change to the image rail - `dasllama/dasllama_image.das`, or, wherever the diff puts it, a `.dlim` mint (building a `.dlim` from a gguf), a `.dlim` load, an image identity, or a flavor (the backend-and-layout variant an image is baked for, one part of its identity) - @@ -197,11 +205,11 @@ third-party engine this module measures itself against - the checkout `benchmarks/setup_lcpp_ref.das` pins. A symbol the file carrying that prose calls or holds as a value is its own name, not attribution. -**A line whose job is to locate or reproduce work against the reference build names that -build's binary outright** - a path naming where checked-in data is regenerated FROM, an -env-knob row whose value locates the reference binary, a command line or flag list in a -methodology, profile, bring-up, or how-to document, and a follow-up ledger's row naming the -build its numbers compare against. +**A file or line whose job is to locate, patch, or reproduce work against the reference build +names that build's binaries and symbols outright.** The job decides, not the artifact kind - a +regeneration path, an env-knob row, a command line in a methodology or how-to document, a +ledger row naming the compared build, and a source patch applied TO the reference build all +qualify. **Legal attribution never appears in prose - it lives in `THIRD_PARTY_NOTICES.md` and the `LICENSE.*` files.** diff --git a/modules/dasLLAMA/REVIEW_EXCHANGE.md b/modules/dasLLAMA/REVIEW_EXCHANGE.md index e46c88685b..d4399ffa44 100644 --- a/modules/dasLLAMA/REVIEW_EXCHANGE.md +++ b/modules/dasLLAMA/REVIEW_EXCHANGE.md @@ -32,3 +32,10 @@ falls through to the local sidecar and the baked winners.** `exchange_boot_submit_check`) consent question with no terminal to ask on also emits that question as a `@sidecar` event, in the same change** - the watchdog dialog and the control page are the answer surfaces a supervised boot has. + +**A diff that changes the exchange consent notice wording updates every checked-in copy in +the same change, and a diff that adds a copy names it here in the same change: +`EXCHANGE_CONSENT_NOTICE` (`../dasllama/dasllama_exchange.das`), `CONSENT_TITLE` / +`CONSENT_TEXT` (`utils/watchdog/watchdog.py`, repo root), and the captured +`utils/dasllama-server/tests/fixtures/exchange.json` (repo root).** The console prompt and the +control page render the served constant, so they are not copies. diff --git a/modules/dasLLAMA/REVIEW_GPU.md b/modules/dasLLAMA/REVIEW_GPU.md index 808bfd2e5a..c52c4a6979 100644 --- a/modules/dasLLAMA/REVIEW_GPU.md +++ b/modules/dasLLAMA/REVIEW_GPU.md @@ -119,6 +119,12 @@ an `upload_region` upload never written after arming - is a defect unless it car defect; a per-encode field either omits `@role` or names the access its body performs.** `weight` drops the hazard staging. +**A diff that adds a GPU kernel class under `dasllama/` - a `[metal_kernel]` def, a +`[vk_dispatch]` declaration, or a new instance of a template carrying one - covers that class +in `tests/test_kernel_coverage.das`, one of two ways.** Either a census row there dispatches +the class, or the diff names it in that file's `CENSUS_NEVER_DISPATCHED` with the reason no +row can reach it - an unledgered class turns the census's blind-spot list into a lie. + **A new kernel class declared in `dasllama/` carries `[metal_dispatch]` / `[vk_dispatch]` with every annotation that backend's generated builder reads - per-field `@binding` / `@role` / `@off` / `@default`.** A field carrying none of them is dropped from the bind list @@ -138,13 +144,17 @@ default-filling wrapper, or a composite over generated builders.** dispatch through the kernel's `enc_*` builder instead.** A race or knockout arm hand-binds by construction and answers to the hand-binding-arm rules instead. -**A hand-binding arm that binds a field at another field's declared number is a defect - and -so is any bind in an arm the machine check cannot see.** The undeclared-number half is -`REVIEW.das`'s `check_race_bind_numbers` where the arm names its kernel class (a `kn_tgmem` -constant or an in-function `pipeline_from_source`); an arm binding through a pso or tgmem -passed as a function parameter is invisible to it, so BOTH halves of that arm stay the -reviewer's. A mis-numbered arm dispatches, reads the wrong buffer, and its timing crowns the -wrong kernel silently. +**A hand-binding arm that binds a field at another field's declared number is a defect.** +A mis-numbered arm dispatches, reads the wrong buffer, and its timing crowns the wrong +kernel silently. + +**A hand-binding arm whose diff cannot be checked by `REVIEW.das`'s `check_race_bind_numbers` +- its pipeline source or threadgroup-memory size arrives as a function parameter instead of a +literal MSL global near the binds - states in the PR that its bind numbers were verified by +hand against the class declaration.** The machine check resolves the kernel class only from a +literal global (a `kn_tgmem` constant or an in-function `pipeline_from_source`); a +parameter-shaped arm is invisible to it, and an unverified invisible arm is the mis-numbered +crown's front door. **A value that reaches the kernel twice device-side - a scalar bound both as a uniform buffer and as a kargs field - is a defect.** A `params=` value that the `grid=`/`tg=` spec consumes @@ -177,14 +187,19 @@ times both kernels on one queue and compares their outputs. small enough to sit in cache ranks the kernels by an effect production never sees, and the race then picks the slower kernel. -**A race arm that dispatches a chain of one kernel binds a DIFFERENT output buffer for -consecutive dispatches - never one shared output across the chain.** One shared output -serializes the chain on its write-after-read hazard while the served graph overlaps -consecutive dispatches, so the race ranks the arms on a shape production never runs. - -**A Metal race spends GPU work on its own arms before its first timed round, and keeps the -dispatches inside each timed encoder back to back.** The arms alternate, so a first round on -an idle clock bills the governor's ramp to one side. +**A kernel A/B race arm whose verdict crowns or ships a kernel - wherever the race lives - +binds a DIFFERENT output buffer for consecutive dispatches of its chain, never one shared +output.** One shared output serializes the chain on its write-after-read hazard while the +served graph overlaps consecutive dispatches, so the race ranks the arms on a shape +production never runs. A comparison rig that deliberately matches a serialized reference +instrument's shape is the exception, and it says so where `ARCHITECTURE_MEASUREMENT.md` +sec.2.21 ledgers it - its verdicts never crown alone. + +**Weakening `race_pair_ms`'s burn phase is a defect, and a diff that adds a crowning race arm +OUTSIDE `race_pair_ms` burns GPU work on its own arms before its first timed round, in the +same change.** The arms alternate, so a first round on an idle clock bills the governor's +ramp to one side. The dispatches inside each timed encoder stay back to back - a sparse +encoder reads idle-clock times. **A string-typed Metal decline reason is a defect - a Metal decline reason is an enum value in `dasllama/dasllama_metal_shapes.das`, one enum per driver.** diff --git a/modules/dasLLAMA/benchmarks/REVIEW.md b/modules/dasLLAMA/benchmarks/REVIEW.md index c5e5c6b742..cb2f791a71 100644 --- a/modules/dasLLAMA/benchmarks/REVIEW.md +++ b/modules/dasLLAMA/benchmarks/REVIEW.md @@ -1,9 +1,10 @@ # dasLLAMA benchmarks Code Review Checklist **Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist.** Architecture -doc: `../ARCHITECTURE_MEASUREMENT.md` (the benchmark rig: `#one-benchmark-rig`). Planned work: -`../followup_vulkan.md` for anything about the Vulkan backend, `../PERF_LEDGER.md` for any -other performance followup, `../followup_general.md` for everything else. +doc: `../ARCHITECTURE_MEASUREMENT.md`. Planned work: `../followup_metal.md` for anything about +the Metal backend, `../followup_vulkan.md` for anything about the Vulkan backend, +`../PERF_LEDGER.md` for any other performance followup, `../followup_general.md` for +everything else. **A diff that adds or changes an instrument whose timed body runs a forward pass through a pipeline the dasLLAMA engine selected also calls `tune_gate()` @@ -11,17 +12,20 @@ pipeline the dasLLAMA engine selected also calls `tune_gate()` the instrument compiled itself - or a reference tool's own runtime - is not one the engine selected. Without the gate the instrument measures fallback kernels silently. -**A diff that adds or changes a race times both arms interleaved in one process - a Metal race -through `race_pair_ms`.** A race is an instrument that compares two implementations. Two -separate runs measure the box's drift between them as much as they measure the arms. +**A diff that adds or changes a race alternates its arms within one process - one timed round +per arm, best-of across rounds, GPU work burned before the first timed round.** A race is an +instrument that compares two implementations. Two separate runs measure the box's drift +between them as much as they measure the arms; `race_pair_ms` +(`../dasllama/dasllama_metal_common.das`) is the engine-side helper that carries all three +properties. **A diff that adds or changes a race arm offered as the reason to adopt a change proves the arm's output on its report line, by what the arm computes:** an arm producing no comparable -output carries the literal token `timing-only`; an arm computing the baseline's result in a -different precision prints a bounded-difference compare (against the baseline arm or the CPU -reference) plus the bound it passed; an arm computing it in the same precision prints the -bit-exact compare over the sampled region - the set of output elements the run compares - on -the report's "bit-exact vs ..." line. +output carries the literal token `timing-only`; an arm whose result is bit-identical to the +baseline's prints the bit-exact compare over the sampled region - the set of output elements +the run compares - on the report's "bit-exact vs ..." line; every other arm prints a +bounded-difference compare (against the baseline arm or the CPU reference) plus the bound it +passed. Reduction order and fma contraction decide bit-identity, not declared precision. **A diff that adds or changes a race also checks the race's baseline arm against a CPU reference.** The baseline arm is the arm running the implementation already in use. The @@ -34,10 +38,12 @@ is one whose arms attribute cost across stages instead of selecting between two implementations. The same header line also names what its arms attribute. Without that text a reader takes the sweep's arms for an adoption decision it never made. -**A new instrument that times a served turn is a defect - add a board cell to -`../performance/gen_bench_records.das`, or a `lcpp_bench.das` cell with its own -`../PROFILE.md` section, instead.** A served turn is a whole prefill-plus-decode run. A -second instrument's numbers cannot be compared to any row the board already carries. +**A new instrument that times a served turn - wherever it lives under this module - is a +defect: add a board cell to `../performance/gen_bench_records.das`, or a `lcpp_bench.das` +cell with its own `../PROFILE.md` section, instead.** A served turn is a whole +prefill-plus-decode run. A second instrument's numbers cannot be compared to any row the +board already carries. A tuner confirm that SPAWNS `lcpp_bench.das` and reads its standard +report line adds no instrument and conforms (`../ARCHITECTURE_MEASUREMENT.md` sec.2.21). **An out-of-process observer never measures what the benchmark process can measure about itself - that measurement goes inside the process instead.** An out-of-process observer is a @@ -45,11 +51,14 @@ script that measures a benchmark process from outside. **A timing instrument this checklist governs never writes the wall time of a binary this repository does not build - a third-party reference tool - into -`../performance/records/.json` or `../PERF_LEDGER.md`.** Such walls enter only through -the reference cells of `../performance/gen_bench_records.das`, the cells that time such a -tool on a board workload. A recovery file a reference leg writes (`asr/_pybench_rows.txt`) -is that leg's scratch - untracked, truncated before each spawn, never read by any other -tool's cell. +`../performance/records/.json` or `../PERF_LEDGER.md`.** Board walls for such a tool +enter only through the reference cells of `../performance/gen_bench_records.das`, the cells +that time it on a board workload. + +**A file holding third-party walls outside those two - a reference leg's recovery file, a +pinned reference tsv an instrument reads back - is scratch: untracked, owned by exactly one +instrument, re-derivable from a documented command, and never an input to a board cell.** A +tracked or shared copy of a third-party wall becomes a stale baseline nobody re-derives. **A diff that adds or changes an instrument that prints a number formed by subtracting one wall it also reports from another also prints both raw walls on that report line.** A plain @@ -61,11 +70,9 @@ body.** A board cell is a timed cell of the published results board: one `../performance/gen_bench_records.das` spawns, or a manual `lcpp_bench.das` cell with its own `../PROFILE.md` section. What a cell times changes when a change to its code, to its input corpus, or to the pinned reference build - `DEFAULT_REF_SHA` in `setup_lcpp_ref.das`, the -targets it builds, or a patch it applies - moves the measured quantity. - -**A diff that adds or moves a row in `../harness/tune_kernels.das`'s bench list keeps -`dot_q8q8_laneq4x4` last.** That bench pins the repack backend for the rest of the process, so -a row after it is timed against the pinned backend instead of the backend it would select. +targets it builds, a patch it applies, or a python reference leg's +`asr/requirements-*.txt` - moves the measured quantity. The re-mint or withdrawal lands in +`../performance/records/.json`, the file the affected rows live in. **A diff that adds or changes a timing instrument makes it exit non-zero on a run that ends with zero result rows - wrong flags, failed load, a device that declines.** A run that matched diff --git a/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das b/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das index a857d2eb9f..8fac563e32 100644 --- a/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das +++ b/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das @@ -27,7 +27,7 @@ require ?das_metal ../../tests/_metal_kernel_common.das // // Correctness before timing, per arm: one dispatch vs the CPU plane-dequant oracle // (kq_row_ref + double dot) over EVERY output element on the dot-envelope bar -// |gpu-cpu| <= 2e-4 * env + 1e-3 - the report line carries that bound. +// |gpu-cpu| <= max(2e-4 * env, 1e-3) - the report line carries the relative factor. // // bin/daslang -jit modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das -- [--fmts iq3s,iq2] // [--tiers gemv,mm] [--shapes w13_1b] [--rounds 3] [--reps 50] [--ref _kq_race_ref.tsv] @@ -144,8 +144,8 @@ def private gemv_arms(f : KqRaceFmt) : array { // candidate GEMV sp } elif (f.fmt == 24) { arms |> push(KqRaceArm(name = "prod", src = metal_kq_gemv_iq2xs_msl, entry = metal_kq_gemv_iq2xs_msl_entry, fastmath = metal_kq_gemv_iq2xs_msl_fastmath, tgmem = metal_kq_gemv_iq2xs_msl_tgmem, gdiv = f.gemv_gdiv)) } else { - arms |> push(KqRaceArm(name = "prod", src = metal_kq_gemv_iq2xxs_msl, entry = metal_kq_gemv_iq2xxs_msl_entry, fastmath = metal_kq_gemv_iq2xxs_msl_fastmath, tgmem = 0ul, gdiv = f.gemv_gdiv)) - arms |> push(KqRaceArm(name = "f4", src = metal_kq_gemv_iq2xxs_f4_msl, entry = metal_kq_gemv_iq2xxs_f4_msl_entry, fastmath = metal_kq_gemv_iq2xxs_f4_msl_fastmath, tgmem = metal_kq_gemv_iq2xxs_f4_msl_tgmem, gdiv = 4)) + arms |> push(KqRaceArm(name = "prod", src = MetalKqGemvIq2xxs_metal_kq_gemv_iq2xxs_msl, entry = MetalKqGemvIq2xxs_metal_kq_gemv_iq2xxs_msl_entry, fastmath = MetalKqGemvIq2xxs_metal_kq_gemv_iq2xxs_msl_fastmath, tgmem = 0ul, gdiv = f.gemv_gdiv)) + arms |> push(KqRaceArm(name = "f4", src = MetalKqGemvIq2xxsF4_metal_kq_gemv_iq2xxs_msl, entry = MetalKqGemvIq2xxsF4_metal_kq_gemv_iq2xxs_msl_entry, fastmath = MetalKqGemvIq2xxsF4_metal_kq_gemv_iq2xxs_msl_fastmath, tgmem = MetalKqGemvIq2xxsF4_metal_kq_gemv_iq2xxs_msl_tgmem, gdiv = 4)) } return <- arms } diff --git a/modules/dasLLAMA/dasllama/dasllama_env.das b/modules/dasLLAMA/dasllama/dasllama_env.das index a6184d432a..092b412676 100644 --- a/modules/dasLLAMA/dasllama/dasllama_env.das +++ b/modules/dasLLAMA/dasllama/dasllama_env.das @@ -430,6 +430,11 @@ struct public HarnessEnv { @clarg_doc = "Model used by the tuner's confirm gate (FULL path, not a bare filename). Unset: the gate auto-resolves from the models dir - the preferred confirm carrier, else the largest present q8 gguf; the fallback pins only when the box has no q8 model at all." confirm_model : string = "" + @clarg_path + @clarg_default_doc = "auto-resolved from the models dir" + @clarg_doc = "IQ2_XXS vehicle for the tuner's serving-crown confirm (FULL path). Unset: the confirm auto-resolves the first *IQ2_XXS*.gguf in the models dir; none found = no crown (the base kernel serves)." + confirm_iq2xxs : string = "" + @clarg_doc = "Override the batched-dispatch chunk count in the 1-core GEMM probe." batch_chunks : string = "" diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_common.das b/modules/dasLLAMA/dasllama/dasllama_metal_common.das index b46e7f1276..62b2af6bd2 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_common.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_common.das @@ -2282,7 +2282,8 @@ def race_envelope_ok(ba, bb : MetalBuffer?; count : int; var note : string&) : b } // interleaved adjacent A/B pair — separate per-side blocks read the box's thermal/clock ramp -// as a phantom winner (reds-dig2; sequential blocks INVERTED m4's kq verdicts). Best-of/side. +// as a phantom winner (reds-dig2; sequential blocks INVERTED m4's kq verdicts). Best-of/side; +// ~150ms of burn first, else round 1 bills the governor's ramp to the base side alone. def race_pair_ms(queue : MetalCommandQueue?; reps : int; var base_ms, twin_ms : double&; base_blk : block<(enc : MetalComputeEncoder?) : void>; twin_blk : block<(enc : MetalComputeEncoder?) : void>) { @@ -2291,6 +2292,19 @@ def race_pair_ms(queue : MetalCommandQueue?; reps : int; var base_ms, twin_ms : var err = "" var best_b = -1.0lf var best_t = -1.0lf + var burn_ms = 0.0lf // clock: control — GPU burn budget, ramps the clock governor before round 1 + while (burn_ms < 150.0lf) { + var gb = 0.0lf + let okb = with_compute_encoder_timed(queue, err, gb) $(enc) { + invoke(base_blk, enc) + } + var gt = 0.0lf + let okt = okb && with_compute_encoder_timed(queue, err, gt) $(enc) { + invoke(twin_blk, enc) + } + return if (!okb || !okt) + burn_ms += gb + gt + } for (r in range(reps + 1)) { var g1 = 0.0lf var g2 = 0.0lf diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das index 88a5a25850..1d191f5fd1 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das @@ -3670,7 +3670,7 @@ def metal_decode_init : bool { // nolint:STYLE038 — a flat compile_pso per k g_pso_kq_mvb4_iq2xs = compile_pso(MetalKqMvB4Iq2xs_metal_kq_mv_iq2xs_msl, MetalKqMvB4Iq2xs_metal_kq_mv_iq2xs_msl_entry, MetalKqMvB4Iq2xs_metal_kq_mv_iq2xs_msl_fastmath, ok) g_pso_kq_mvb8_iq2xs = compile_pso(metal_kq_mvb8_iq2xs_msl, metal_kq_mvb8_iq2xs_msl_entry, metal_kq_mvb8_iq2xs_msl_fastmath, ok) g_pso_kq_mm_b24 = compile_pso(MetalKqMulMmIq2xs_metal_kq_mulmm_msl, MetalKqMulMmIq2xs_metal_kq_mulmm_msl_entry, MetalKqMulMmIq2xs_metal_kq_mulmm_msl_fastmath, ok) - g_pso_kq_iq2xxs = compile_pso(metal_kq_gemv_iq2xxs_msl, metal_kq_gemv_iq2xxs_msl_entry, metal_kq_gemv_iq2xxs_msl_fastmath, ok) + g_pso_kq_iq2xxs = compile_pso(MetalKqGemvIq2xxs_metal_kq_gemv_iq2xxs_msl, MetalKqGemvIq2xxs_metal_kq_gemv_iq2xxs_msl_entry, MetalKqGemvIq2xxs_metal_kq_gemv_iq2xxs_msl_fastmath, ok) if (metal_tensor_crowned("kq_gemv_iq3s_f4")) { g_pso_kq_iq3s_f4 = compile_pso(metal_kq_gemv_iq3s_f4_msl, metal_kq_gemv_iq3s_f4_msl_entry, metal_kq_gemv_iq3s_f4_msl_fastmath, ok) } @@ -3678,7 +3678,7 @@ def metal_decode_init : bool { // nolint:STYLE038 — a flat compile_pso per k g_pso_kq_iq3xxs_f4 = compile_pso(metal_kq_gemv_iq3xxs_f4_msl, metal_kq_gemv_iq3xxs_f4_msl_entry, metal_kq_gemv_iq3xxs_f4_msl_fastmath, ok) } if (metal_tensor_crowned("kq_gemv_iq2xxs_f4")) { - g_pso_kq_iq2xxs_f4 = compile_pso(metal_kq_gemv_iq2xxs_f4_msl, metal_kq_gemv_iq2xxs_f4_msl_entry, metal_kq_gemv_iq2xxs_f4_msl_fastmath, ok) + g_pso_kq_iq2xxs_f4 = compile_pso(MetalKqGemvIq2xxsF4_metal_kq_gemv_iq2xxs_msl, MetalKqGemvIq2xxsF4_metal_kq_gemv_iq2xxs_msl_entry, MetalKqGemvIq2xxsF4_metal_kq_gemv_iq2xxs_msl_fastmath, ok) } g_pso_kq_mvb2_iq2xxs = compile_pso(MetalKqMvB2Iq2xxs_metal_kq_mv_iq2xxs_msl, MetalKqMvB2Iq2xxs_metal_kq_mv_iq2xxs_msl_entry, MetalKqMvB2Iq2xxs_metal_kq_mv_iq2xxs_msl_fastmath, ok) g_pso_kq_mvb4_iq2xxs = compile_pso(MetalKqMvB4Iq2xxs_metal_kq_mv_iq2xxs_msl, MetalKqMvB4Iq2xxs_metal_kq_mv_iq2xxs_msl_entry, MetalKqMvB4Iq2xxs_metal_kq_mv_iq2xxs_msl_fastmath, ok) @@ -4800,7 +4800,7 @@ def private race_gemv_f4_twin(dev : MetalDevice?; queue : MetalCommandQueue?; fa var bd = race_uniform_u32(dev, uint(ndim)) let doff = uint64(nblk * 16) let tg = uint3(64u, 1u, 1u) - var burn_ms = 0.0lf + var burn_ms = 0.0lf // clock: control — GPU burn budget, ramps the clock governor var burn_err = "" while (burn_ms < 150.0lf) { var g = 0.0lf @@ -8932,11 +8932,11 @@ class MetalKqGemvIq2xs { } } -// IQ2_XXS: the iq2xs GEMV geometry over the aux32 form - per (block, half) the block's grid -// word supplies two BYTE indices (each doubling into the u64 grid pair), its aux32 the -// parity ksigns; ONE per-32 UNSIGNED strip scales the whole block on the eighth-ed d. -[metal_dispatch(name = "enc_kq_iq2xxs_c", pso = "g_pso_kq_iq2xxs", tg = 64, grid = "rows/4", params = "rows : int64, n : int64")] -class MetalKqGemvIq2xxs { +// IQ2_XXS over the aux32 form: grid-word byte indices double into the u64 grid pair, aux32 +// carries the parity ksigns, ONE per-32 UNSIGNED strip scales the block on the eighth-ed d. +// Two stamps on the F4 axis: in-place constant expand vs the crown-gated threadgroup slab. +[ |> template_struct_instance] +class template MetalKqGemvIq2xxsFam { @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d8 plane - the scale buffer bound at byte nsb*16 (s0off = the caller's doff) @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // strip region: 8 UNSIGNED per-32 sub-scales + 8 pad, byte view (same buffer at 0) @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array // iq2xxs quant plane, uint view (16 words per superblock: [grid bytes][aux32] per block) @@ -8944,9 +8944,20 @@ class MetalKqGemvIq2xxs { @ssbo @binding = 4 @role = "write" @off = "yoff" @span = "rows*4" y : array @uniform @binding = 5 ndim : uint @uniform @binding = 6 ddim : uint + @template_constant F4 : bool = false + @workgroup @template_gate = F4 gridf : float4[512] // pre-expanded magnitudes, one float4 per grid word (8 KB) - [metal_kernel(name="metal_kq_gemv_iq2xxs_msl")] + [metal_kernel] def metal_kq_gemv_iq2xxs { + static_if (F4) { + let lid = gl_LocalInvocationID.x + for [unroll_full] (gk in range(8)) { + let gi = lid + uint(gk) * 64u + let w = iq2xxs_gw(int(gi)) + gridf[gi] = float4(float(w & 255u), float((w >> 8u) & 255u), float((w >> 16u) & 255u), float(w >> 24u)) + } + barrier() + } let lane = gl_SubgroupInvocationID let ix = lane / 8u let it = lane % 8u @@ -8970,10 +8981,17 @@ class MetalKqGemvIq2xxs { let sgn = ksign7m((aux >> (7u * uint(q3))) & 127u) let s0 = sgn & 15u let s1 = sgn >> 4u - let w0 = iq2xxs_gw(int(bidx * 2u)) - let w1 = iq2xxs_gw(int(bidx * 2u + 1u)) - let m0 = float4(float(w0 & 255u), float((w0 >> 8u) & 255u), float((w0 >> 16u) & 255u), float(w0 >> 24u)) - let m1 = float4(float(w1 & 255u), float((w1 >> 8u) & 255u), float((w1 >> 16u) & 255u), float(w1 >> 24u)) + var m0 : float4 + var m1 : float4 + static_if (F4) { + m0 = gridf[bidx * 2u] + m1 = gridf[bidx * 2u + 1u] + } else { + let w0 = iq2xxs_gw(int(bidx * 2u)) + let w1 = iq2xxs_gw(int(bidx * 2u + 1u)) + m0 = float4(float(w0 & 255u), float((w0 >> 8u) & 255u), float((w0 >> 16u) & 255u), float(w0 >> 24u)) + m1 = float4(float(w1 & 255u), float((w1 >> 8u) & 255u), float((w1 >> 16u) & 255u), float(w1 >> 24u)) + } let f0 = float4((s0 & 1u) != 0u ? -m0.x : m0.x, (s0 & 2u) != 0u ? -m0.y : m0.y, (s0 & 4u) != 0u ? -m0.z : m0.z, (s0 & 8u) != 0u ? -m0.w : m0.w) let f1 = float4((s1 & 1u) != 0u ? -m1.x : m1.x, (s1 & 2u) != 0u ? -m1.y : m1.y, @@ -8995,72 +9013,14 @@ class MetalKqGemvIq2xxs { } } -// The f4-slab twin: the same lane map with the 2 KB grid pre-expanded to float4 magnitudes -// (8 KB threadgroup) and signs applied by select. -[metal_dispatch(name = "enc_kq_iq2xxs_f4_c", pso = "g_pso_kq_iq2xxs_f4", tgmem = "metal_kq_gemv_iq2xxs_f4_msl_tgmem", tg = 64, grid = "rows/4", params = "rows : int64, n : int64")] -class MetalKqGemvIq2xxsF4 { - @ssbo @binding = 0 @role = "weight" @off = "s0off" kdh : array // the d8 plane - the scale buffer bound at byte nsb*16 (s0off = the caller's doff) - @ssbo @binding = 1 @role = "weight" @off = "soff" kscb : array // strip region: 8 UNSIGNED per-32 sub-scales + 8 pad, byte view (same buffer at 0) - @ssbo @binding = 2 @role = "weight" @off = "qoff" kqu : array // iq2xxs quant plane, uint view (16 words per superblock: [grid bytes][aux32] per block) - @ssbo @binding = 3 @role = "read" @off = "xoff" @span = "n*4" x : array - @ssbo @binding = 4 @role = "write" @off = "yoff" @span = "rows*4" y : array - @uniform @binding = 5 ndim : uint - @uniform @binding = 6 ddim : uint - @workgroup gridf : float4[512] // pre-expanded magnitudes, one float4 per grid word (8 KB) +[metal_dispatch(name = "enc_kq_iq2xxs_c", pso = "g_pso_kq_iq2xxs", tg = 64, grid = "rows/4", params = "rows : int64, n : int64")] +class MetalKqGemvIq2xxs : MetalKqGemvIq2xxsFam { +} - [metal_kernel(name="metal_kq_gemv_iq2xxs_f4_msl")] - def metal_kq_gemv_iq2xxs_f4 { - let lid = gl_LocalInvocationID.x - for [unroll_full] (gk in range(8)) { - let gi = lid + uint(gk) * 64u - let w = iq2xxs_gw(int(gi)) - gridf[gi] = float4(float(w & 255u), float((w >> 8u) & 255u), float((w >> 16u) & 255u), float(w >> 24u)) - } - barrier() - let lane = gl_SubgroupInvocationID - let ix = lane / 8u - let it = lane % 8u - let nb = ndim / 256u - let first_row = (gl_WorkGroupID.x * 2u + gl_SubgroupID) * 2u - var sumf : float[2] - var ib = ix - while (ib < nb) { - let yb4 = ib * 64u + it * 8u - var yv : float4[8] - for [unroll_full] (i in range(8)) { - yv[i] = x[yb4 + uint(i)] - } - for [unroll_full] (r in range(2)) { - let blk = (first_row + uint(r)) * nb + ib - let gw = kqu[blk * 16u + it * 2u] - let aux = kqu[blk * 16u + it * 2u + 1u] - var accv = float4(0.0) - for [unroll_full] (q3 in range(4)) { - let bidx = (gw >> (8u * uint(q3))) & 255u - let sgn = ksign7m((aux >> (7u * uint(q3))) & 127u) - let s0 = sgn & 15u - let s1 = sgn >> 4u - let m0 = gridf[bidx * 2u] - let m1 = gridf[bidx * 2u + 1u] - let f0 = float4((s0 & 1u) != 0u ? -m0.x : m0.x, (s0 & 2u) != 0u ? -m0.y : m0.y, - (s0 & 4u) != 0u ? -m0.z : m0.z, (s0 & 8u) != 0u ? -m0.w : m0.w) - let f1 = float4((s1 & 1u) != 0u ? -m1.x : m1.x, (s1 & 2u) != 0u ? -m1.y : m1.y, - (s1 & 4u) != 0u ? -m1.z : m1.z, (s1 & 8u) != 0u ? -m1.w : m1.w) - accv += yv[q3 * 2] * f0 - accv += yv[q3 * 2 + 1] * f1 - } - let sc = float(kscb[blk * 16u + it]) - sumf[r] += float(kdh[blk]) * sc * (accv.x + accv.y + accv.z + accv.w) - } - ib += 4u - } - for [unroll_full] (r in range(2)) { - let s = simd_sum(sumf[r]) - if (lane == 0u && first_row + uint(r) < ddim) { - y[first_row + uint(r)] = s - } - } - } +// the F4 twin serves only under the box's serving-confirm crown (ARCHITECTURE_MEASUREMENT.md sec.2.21) +[metal_dispatch(name = "enc_kq_iq2xxs_f4_c", pso = "g_pso_kq_iq2xxs_f4", tgmem = "MetalKqGemvIq2xxsF4_metal_kq_gemv_iq2xxs_msl_tgmem", tg = 64, grid = "rows/4", params = "rows : int64, n : int64")] +class MetalKqGemvIq2xxsF4 : MetalKqGemvIq2xxsFam { + override F4 = true } // IQ3_XXS: the iq3s GEMV shape over the 24-word rows - signs from the block's aux32 through diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das b/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das index 26930ff662..98d69568d1 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_prefill.das @@ -15,7 +15,6 @@ require dasllama/dasllama_common require dasllama/dasllama_math require dasllama/dasllama_metal_common require dasllama/dasllama_metal_kernels // the shared kq decode helpers (grid tables, ksign7m) for the tensor twins -require dasllama/dasllama_metal_kernels require dasllama/dasllama_metal_lens // Phase-6 dasMetal: the full-GPU-resident prefill kernel set — everything between the GEMMs: diff --git a/modules/dasLLAMA/followup_metal.md b/modules/dasLLAMA/followup_metal.md index 0bb43f8a04..b67ebd9275 100644 --- a/modules/dasLLAMA/followup_metal.md +++ b/modules/dasLLAMA/followup_metal.md @@ -12,10 +12,13 @@ The five `dot = "smmla"` seats of `q8q8_tile_gen` (mr4/mr8 x kstep/nrsplit/gkste PR-1 detection fix (`g_target_arm64_i8mm` consults `cpu_supports`, `+i8mm` appended to the target machine). The `harness/smmla_probe.das` gate passed first: correctness OK, 2.02x register-resident MAC throughput over sdot4 - the nominal ceiling. In the tile race that -ceiling does not survive the memory traffic plus the kg8 re-layout: best smmla seat +ceiling does not survive the memory traffic plus the kg8 re-layout (the tile race: +`DAS_TUNE_MODE=tune harness/dasllama_tuner.das`'s q8q8_tile_gen family bench, m5): best smmla seat `mr8_kstep2_nrsplit2` at 25893 us vs NEON `mr8_budget` at 21349 us (~21% behind; full table -in the m5 sidecar's race section and `~/.tune-history/m5/`). No `arm-i8mm` defaults export - -the export was conditional on a win. +in the m5 sidecar's race section and `~/.tune-history/m5/`). The `arm-i8mm` defaults profile +SHIPS regardless (`performance/defaults/arm-i8mm.tune-defaults.json`, exported from the full +m5 mint): its winners are the NEON ones, and its `i8mm` features fingerprint marks the smmla +seats raced-and-covered, so no M2+ box re-races them at adoption. Consequences: an smmla kq tile emitter arm is NOT mac-leverage (the q8q8 verdict transfers - the kq tiles are more memory-bound, not less); it remains a Graviton3+/c8g candidate raced on @@ -55,16 +58,21 @@ followup_general #58, the Q22 dispatch-loop probe method) consolidate here in Ph ## 4. The elementwise / activation-precision lane (the last M5 pp residual) -Attribution (M5, 1B iq2xxs, pp512 = 31 ms encode): mm 27.7 ms at measured tensor-twin rates, -attention 0.16 ms - the ~3 ms remainder is every non-matmul pass over the activation planes -(norms, residual adds, rope, swiglu, activation converts, glue). llama.cpp's slice: ~1.7 ms. +Attribution (M5, 1B iq2xxs, pp512 = 31 ms encode; `benchmarks/lcpp_bench.das +--for-debug-purposes --ngl 99 -p 512`, the prefill stage log line + per-kernel lab rates from +`benchmarks/matmul/bench_metal_kq_race.das`; the llama.cpp slice from its `llama-bench -p 512` +wall minus the same mm/attention accounting at test-backend-ops rates): mm 27.7 ms at measured +tensor-twin rates, attention 0.16 ms - the ~3 ms remainder is every non-matmul pass over the +activation planes (norms, residual adds, rope, swiglu, activation converts, glue). llama.cpp's +slice: ~1.7 ms. **What already exists** - the producer-fused f16 twin family (`_hx`): `pf_enc_rms_hx`, `pf_enc_add_rms_bhx` (add+norm+half-emit in one), `enc_swiglu_hx`/`enc_geglu_hx`, `enc_qk_rope_hx`. On the dense path these cover the norm and activation producers; the standalone `enc_cvt_half` fires only through `pf_cvt_panel` fallbacks and at the sites below. -Measured ceiling of ALL remaining converts (`DASLLAMA_METAL_PREFILL_SKIP=act_cvt` knockout): -**+0.65% pp512 on M5** (15334 vs 15235 tok/s) - the fusion rung is mostly banked already. +Measured ceiling of ALL remaining converts (`DASLLAMA_METAL_PREFILL_SKIP=act_cvt` knockout on +`benchmarks/lcpp_bench.das --for-debug-purposes --ngl 99 -p 512 -n 16 -r 3`, m5): +**+0.65% pp512** (15334 vs 15235 tok/s) - the fusion rung is mostly banked already. - **4a. The attention-out `_hx`** (the one live dense-path cvt): the AV kernels (`MetalAttnAV`, `MetalAttnAVMm`, the tensor `MetalAttnAVMmTensorT`) write only f32 `xb`; diff --git a/modules/dasLLAMA/harness/tune_kernels.das b/modules/dasLLAMA/harness/tune_kernels.das index 10e77f53ae..09955cf1c2 100644 --- a/modules/dasLLAMA/harness/tune_kernels.das +++ b/modules/dasLLAMA/harness/tune_kernels.das @@ -2374,8 +2374,9 @@ def private confirm_serving_crown(crown : string; var wins : array) { } def private confirm_decode_model() : string { - let envm = get_env_variable("DASLLAMA_CONFIRM_IQ2XXS") + let envm = g_env_harness.confirm_iq2xxs if (!empty(envm)) { + print("confirm: DASLLAMA_CONFIRM_IQ2XXS={envm} - the operator pins the crown-confirm vehicle\n") return envm } let md = models_dir_resolved() diff --git a/modules/dasLLAMA/performance/REVIEW.md b/modules/dasLLAMA/performance/REVIEW.md index fa6a5bd4f3..3228fcfb5e 100644 --- a/modules/dasLLAMA/performance/REVIEW.md +++ b/modules/dasLLAMA/performance/REVIEW.md @@ -30,12 +30,7 @@ re-mint.** A reference row that carries no `sha` (the cli and python reference t pinned by its builder instead - the cli tools by that same ref pin's checkout, the python legs by `../benchmarks/asr/requirements-*.txt`. -**A diff that changes a reference-tool pin - `DEFAULT_REF_SHA` in -`../benchmarks/setup_lcpp_ref.das`, or a python leg's -`../benchmarks/asr/requirements-*.txt` - re-mints every row that pin's tool minted, or -withdraws them.** - -**A diff that writes a provenance manifest under this folder whose `provenance.dasllama_version` +**A diff that writes a records row, sidecar archive, or `defaults/` profile under this folder whose `provenance.dasllama_version` differs from the `DASLLAMA_RELEASE` string (`../dasllama/dasllama_version.das`) is a defect - re-mint.** For a sidecar with an `engine_sha`, read the value at that commit; a `defaults/` profile compares against the branch under review. @@ -100,9 +95,3 @@ the row's canonical values** - the verify never reads the url of a file already **A diff that refreshes `last_known_good_sidecar.json` replaces it with one complete mint from the box its provenance names, at the current `DASLLAMA_RELEASE` - never a hand-edited copy.** -**A diff that changes the exchange consent notice wording updates every checked-in copy in -the same change, and a diff that adds a copy names it here in the same change: -`EXCHANGE_CONSENT_NOTICE` (`../dasllama/dasllama_exchange.das`), `CONSENT_TITLE` / -`CONSENT_TEXT` (`utils/watchdog/watchdog.py`, repo root), and the captured -`utils/dasllama-server/tests/fixtures/exchange.json` (repo root).** The console prompt and the -control page render the served constant, so they are not copies. diff --git a/modules/dasLLAMA/tests/REVIEW.md b/modules/dasLLAMA/tests/REVIEW.md index 7c2e2303ac..21d8cc2176 100644 --- a/modules/dasLLAMA/tests/REVIEW.md +++ b/modules/dasLLAMA/tests/REVIEW.md @@ -29,8 +29,9 @@ defect.** Out-of-folder instances are ledgered in `CLAUDE.md`'s "Out-of-folder t **A diff that registers a test file in this folder in a `CMakeLists.txt` is a defect.** -**A diff that adds, removes or re-lanes a gate in a file with a `CLAUDE.md` entry updates that -entry in the same change.** +**A diff that adds, removes or re-lanes a gate in a file whose `CLAUDE.md` paragraph LISTS +its gates updates that paragraph in the same change.** A paragraph that only names the file +(a brace list, a suite roster) carries nothing to update and does not fire this rule. **A new test file listed in `run.das`'s `model-free` suite, or in no `run.das` suite at all, whose name does not say what it covers, gets a `CLAUDE.md` entry in the same change** - @@ -42,31 +43,19 @@ partial. filter mechanics" section in the same change** - an arm the census does not name is unreachable to whoever is choosing what to run. -**Weakening `test_program_roots.das` - dropping a directory from its `ROOT_DIRS` sweep, -loosening its `options stack = 524288` assert, or relaxing its prefill-intent assert - is a -defect.** - -**Weakening `test_env_registry.das` is a defect.** It enforces the knob contract that -`../ENVIRONMENT.md` describes. - -**Weakening `test_model_specs.das` is a defect.** It is the gate on the model-set table -(`../performance/model_specs.das`). - -**Weakening the softcap, sink and span cells of `test_metal_prefill_kernels.das` - dropping -or loosening a softcap, sink (`hass`) or span arm, or dropping a span shape - is a defect** - -they are what fails when `pf_p_weight` and `metal_attn_rowstat` drift apart. - -**Weakening `test_site_records.das` - the gate that byte-compares +**Weakening a contract-pinning gate - dropping an assert, loosening a bound, or narrowing the +corpus or sweep it covers - is a defect.** The pinned set, each with what its gate holds: +`test_program_roots.das` (the `ROOT_DIRS` sweep, `options stack = 524288`, prefill intent); +`test_env_registry.das` (the `../ENVIRONMENT.md` knob contract); `test_model_specs.das` (the +model-set table, `../performance/model_specs.das`); the softcap/sink (`hass`)/span cells of +`test_metal_prefill_kernels.das` (what fails when `pf_p_weight` and `metal_attn_rowstat` +drift apart); `test_site_records.das` (the byte-compare of `site/files/dasllama/bench_records.json` (repo root) against a fresh `merge_site_records` -run - is a defect.** - -**Weakening `test_exchange_schema.das` or `test_bench_records_schema.das` - loosening an -assert, dropping one, or narrowing the corpus either one sweeps - is a defect** - they gate -the real `write_bench_records` output. - -**Weakening `test_scheduler.das`'s media-stream bypass check - a media stream attaches no -cached hit at admit (`prefix_attach`) and donates no pages at reap (`donate_stream`) - is a -defect.** Cache keys are token ids, and the KV past the splice does not follow from them. +run); `test_exchange_schema.das` and `test_bench_records_schema.das` (the real +`write_bench_records` output, corpus sweeps included); `test_scheduler.das`'s media-stream +bypass check (a media stream attaches no cached hit at admit (`prefix_attach`) and donates no +pages at reap (`donate_stream`) - cache keys are token ids, and the KV past the splice does +not follow from them). **A test that silently vanishes on one platform is a defect, and so is a zero-assertion pass - a test passes or skips explicitly on every platform.** A cell that returns without asserting - @@ -106,11 +95,14 @@ registry, and never calls it directly.** The registries this governs: the arch r registry, and the format/backend dispatch tables. A new registry joins that list in the same change. A `[metal_dispatch]` declaration is not one of them. -**A diff that changes anything a kernel dispatches with, binds, or reads from its kargs - the -kernel's kernel-argument struct - updates every gate that hand-binds that kernel, in the same -change.** A hand-bound gate dispatches the geometry and threadgroup memory its production -encoder does. The mechanism - why a missed threadgroup-memory length fails silently - is -`CLAUDE.md`'s "Metal kernel gates" section. +**A diff that changes a kernel's dispatch geometry - its grid divisor, threadgroup size, or +threadgroup-memory length - updates every gate that hand-dispatches that kernel, in the same +change.** A hand-dispatched gate encodes the geometry itself, so a moved divisor leaves it +racing the wrong shape silently. + +**A diff that changes a kernel's kargs - the kernel-argument struct, or any buffer binding - +updates every gate that hand-binds that kernel, in the same change.** A stale hand bind reads +the wrong buffer and passes on garbage that happens to compare. **A kernel that gains an in-body branch keyed on a kargs field ships, in the same change, a gate cell that sets that field to the value selecting the new branch.** @@ -133,13 +125,6 @@ a defect.** A resize cap is not evidence. **A freeform token-parity cell is a defect.** Freeform coverage uses the forced-feed logits-tolerance form. Counting cells stay token-exact. -**A diff that adds a GPU kernel class under `../dasllama/` - a `[metal_kernel]` def, a -`[vk_dispatch]` declaration, or a new instance of a template carrying one - covers that class -in `test_kernel_coverage.das`, one of two ways.** Either a census row there dispatches the -class, or the diff names it in that file's `CENSUS_NEVER_DISPATCHED`, with the reason no row -can reach it. A `DASLLAMA_PARITY_FULL`-gated census row counts as the dispatching arm. Naming -a class a census row could dispatch is a defect. - **A kernel-unit cell - a model-less cell that dispatches one kernel class and asserts on its output - missing a compare against a CPU oracle that can witness the cell's property is a defect.** (A census row dispatches without asserting on output, so it is not one.) @@ -243,8 +228,8 @@ exact fixtures. **An embedding-parity cell that does not name its fixture, or does not log the measured maxdiff on green as well as red, is a defect.** -**A new gate, or a new or loosened tolerance bar, ships a control that reds it in the same -change.** A control is a run of the same gate that must RED - a poisoned input, a poisoned +**A kernel-unit cell that dispatches a kernel class no cell dispatched before, or a new or +loosened tolerance bar, ships a control that reds it in the same change.** A control is a run of the same gate that must RED - a poisoned input, a poisoned expectation, a disconnected mechanism, or a second independent lane; a gate's own reference is never its control. A bar nothing has ever exceeded is not known to discriminate, and a gate that reads state the same code path wrote can be a tautology - only the control proves either @@ -256,7 +241,7 @@ smallest GGUF that runs on the small tier (the file homes are `CLAUDE.md`'s "Mod no-arm tests" and "Out-of-folder test files" notes). A family whose vocab lacks the markers has no format to test. -**A kernel-unit gate that compares a GPU attention kernel against a wider-precision CPU oracle +**A kernel-unit gate whose kernel reads f16 operands and whose oracle is wider-precision feeds inputs that are exact in f16.** Otherwise the compare measures input rounding, and the bar has to be loosened until it no longer discriminates. diff --git a/modules/dasLLAMA/tests/test_kernel_coverage.das b/modules/dasLLAMA/tests/test_kernel_coverage.das index 55d1a10cf2..1a434534e7 100644 --- a/modules/dasLLAMA/tests/test_kernel_coverage.das +++ b/modules/dasLLAMA/tests/test_kernel_coverage.das @@ -83,7 +83,17 @@ let private CENSUS_NEVER_DISPATCHED <- [ "MetalKqGemvIq3xxs", "MetalKqMvB2Iq3xxs", "MetalKqMvB4Iq3xxs", "MetalKqMvB8Iq3xxs", "MetalKqMulMmIq3xxs", "MetalKqGemvIq2s", "MetalKqMvB2Iq2s", "MetalKqMvB4Iq2s", "MetalKqMvB8Iq2s", "MetalKqMulMmIq2s", "MetalKqGemvIq2xs", "MetalKqMvB2Iq2xs", "MetalKqMvB4Iq2xs", "MetalKqMvB8Iq2xs", "MetalKqMulMmIq2xs", - "MetalKqGemvIq2xxs", "MetalKqMvB2Iq2xxs", "MetalKqMvB4Iq2xxs", "MetalKqMvB8Iq2xxs", "MetalKqMulMmIq2xxs" + "MetalKqGemvIq2xxs", "MetalKqMvB2Iq2xxs", "MetalKqMvB4Iq2xxs", "MetalKqMvB8Iq2xxs", "MetalKqMulMmIq2xxs", + // the same nine formats' crown-gated forms (the metal-kq-race arc): the f4-slab GEMV twins + // and the tensor mul_mm T/TH stamps compile only behind a box crown and dispatch only on + // planes the zoo does not stock. Their coverage is the same kernel-unit gates plus the + // tensor-twins/iq2xxs matrix row and the mint's METAL_TWIN race rows + "MetalKqGemvIq3sF4", "MetalKqGemvIq3xxsF4", "MetalKqGemvIq2xxsF4", + "MetalKqMulMmIq4xsT", "MetalKqMulMmIq4xsTH", "MetalKqMulMmIq4nlT", "MetalKqMulMmIq4nlTH", + "MetalKqMulMmK3T", "MetalKqMulMmK3TH", "MetalKqMulMmK2T", "MetalKqMulMmK2TH", + "MetalKqMulMmIq3sT", "MetalKqMulMmIq3sTH", "MetalKqMulMmIq3xxsT", "MetalKqMulMmIq3xxsTH", + "MetalKqMulMmIq2sT", "MetalKqMulMmIq2sTH", "MetalKqMulMmIq2xsT", "MetalKqMulMmIq2xsTH", + "MetalKqMulMmIq2xxsT", "MetalKqMulMmIq2xxsTH" ] def private counting_prompt(start, last : int) : string { diff --git a/modules/dasLLAMA/tests/test_metal_gemm_kernels.das b/modules/dasLLAMA/tests/test_metal_gemm_kernels.das index ccaf9e4524..ee42d97375 100644 --- a/modules/dasLLAMA/tests/test_metal_gemm_kernels.das +++ b/modules/dasLLAMA/tests/test_metal_gemm_kernels.das @@ -60,7 +60,7 @@ def private kq_mulmm_gate(t : T?; dev, queue; fmt : int; tensor : bool; m, kdim, : (fmt == 25 ? MetalKqMulMmIq2xxs_metal_kq_mulmm_msl_tgmem : (fmt == 24 ? MetalKqMulMmIq2xs_metal_kq_mulmm_msl_tgmem : (fmt == 23 ? MetalKqMulMmIq2s_metal_kq_mulmm_msl_tgmem : (fmt == 45 ? MetalKqMulMmIq4nl_metal_kq_mulmm_msl_tgmem : (fmt == 2 ? MetalKqMulMmK2_metal_kq_mulmm_msl_tgmem : (fmt == 34 ? MetalKqMulMmIq3xxs_metal_kq_mulmm_msl_tgmem : (fmt == 33 ? MetalKqMulMmIq3s_metal_kq_mulmm_msl_tgmem : (fmt == 3 ? MetalKqMulMmK3_metal_kq_mulmm_msl_tgmem : (fmt == 44 ? MetalKqMulMmIq4xs_metal_kq_mulmm_msl_tgmem : (fmt == 4 ? MetalKqMulMmK4_metal_kq_mulmm_msl_tgmem : (fmt == 5 ? MetalKqMulMmK5_metal_kq_mulmm_msl_tgmem : MetalKqMulMmK6_metal_kq_mulmm_msl_tgmem)))))))))))) var pso = pipeline_from_source(dev, src, entry, fm, err) if (tensor && pso == null && !metal4_tensor_available(dev)) { - to_log(LOG_INFO, "{tag}: no Metal-4 tensor toolchain on this box - arm skipped\n") + t |> skip("{tag}: no Metal-4 tensor toolchain on this box") return } t |> success(pso != null, "{tag}: pipeline: {err}") @@ -147,6 +147,12 @@ def private kq_mulmm_gate(t : T?; dev, queue; fmt : int; tensor : bool; m, kdim, t |> success(ran, "{tag}: encode: {err}") if (ran) { t |> equal(buf_mismatch_env(by, want, env, 2e-4, tag), 0) + //! the control: the same compare must RED on a poisoned expectation (one element off by + //! an env-scaled-plus-absolute delta) - proves the bar can fail for this arm + var wantp := want + wantp[0] += env[0] * 0.1 + 1.0 + t |> success(buf_mismatch_env(by, wantp, env, 2e-4, "{tag} CONTROL(expected-red)") != 0, "{tag}: poison control reds") + delete wantp } metal_release(bks) metal_release(bkq) @@ -1006,7 +1012,7 @@ def private bf16_mulmm_gate(t : T?; dev, queue; tensor : bool; m, kdim, ndim : i : pipeline_from_source(dev, MetalBf16MulMmT_metal_bf16_mulmm_t_msl, MetalBf16MulMmT_metal_bf16_mulmm_t_msl_entry, MetalBf16MulMmT_metal_bf16_mulmm_t_msl_fastmath, err)) : pipeline_from_source(dev, metal_bf16_mulmm_msl, metal_bf16_mulmm_msl_entry, metal_bf16_mulmm_msl_fastmath, err))) if (tensor && pso == null && !metal4_tensor_available(dev)) { - to_log(LOG_INFO, "{tag}: no Metal-4 tensor toolchain on this box - arm skipped\n") + t |> skip("{tag}: no Metal-4 tensor toolchain on this box") return } // the simdgroup BASE asserts — production compiles it unconditionally, and this arm is its @@ -1097,7 +1103,7 @@ def private q8_gemm_b_gate(t : T?; dev, queue; tensor : bool; mp32, kdim, ndim, ? pipeline_from_source(dev, MetalQ8GemmBT_metal_q8_gemm_b_t_msl, MetalQ8GemmBT_metal_q8_gemm_b_t_msl_entry, MetalQ8GemmBT_metal_q8_gemm_b_t_msl_fastmath, err) : pipeline_from_source(dev, MetalQ8GemmB_metal_q8_gemm_b_msl, MetalQ8GemmB_metal_q8_gemm_b_msl_entry, MetalQ8GemmB_metal_q8_gemm_b_msl_fastmath, err)) if (tensor && pso == null && !metal4_tensor_available(dev)) { - to_log(LOG_INFO, "{tag}: no Metal-4 tensor toolchain on this box - arm skipped\n") + t |> skip("{tag}: no Metal-4 tensor toolchain on this box") return } t |> success(pso != null, "{tag}: pipeline: {err}") @@ -1168,7 +1174,7 @@ def private q8_gemm_sk_gate(t : T?; dev, queue; tensor : bool; kdim, ndim, kspli ? pipeline_from_source(dev, MetalQ8GemmBSkT_metal_q8_gemm_b_t_msl, MetalQ8GemmBSkT_metal_q8_gemm_b_t_msl_entry, MetalQ8GemmBSkT_metal_q8_gemm_b_t_msl_fastmath, err) : pipeline_from_source(dev, MetalQ8GemmBSk_metal_q8_gemm_b_msl, MetalQ8GemmBSk_metal_q8_gemm_b_msl_entry, MetalQ8GemmBSk_metal_q8_gemm_b_msl_fastmath, err)) if (tensor && pso == null && !metal4_tensor_available(dev)) { - to_log(LOG_INFO, "{tag}: no Metal-4 tensor toolchain on this box - arm skipped\n") + t |> skip("{tag}: no Metal-4 tensor toolchain on this box") return } t |> success(pso != null, "{tag}: pipeline: {err}") @@ -1266,7 +1272,7 @@ def private q8_gemm64_b_gate(t : T?; dev, queue; tensor : bool; mp32, kdim, ndim ? pipeline_from_source(dev, MetalQ8Gemm64BT_metal_q8_gemm_b_t_msl, MetalQ8Gemm64BT_metal_q8_gemm_b_t_msl_entry, MetalQ8Gemm64BT_metal_q8_gemm_b_t_msl_fastmath, err) : pipeline_from_source(dev, metal_q8_gemm64_b_msl, metal_q8_gemm64_b_msl_entry, metal_q8_gemm64_b_msl_fastmath, err)) if (tensor && pso == null && !metal4_tensor_available(dev)) { - to_log(LOG_INFO, "{tag}: no Metal-4 tensor toolchain on this box - arm skipped\n") + t |> skip("{tag}: no Metal-4 tensor toolchain on this box") return } t |> success(pso != null, "{tag}: pipeline: {err}") @@ -1609,7 +1615,7 @@ def private moe_mulmm_q8_gate(t : T?; dev, queue; tensor : bool; kdim, ndim : in : pipeline_from_source(dev, MetalMoeMulMmQ8T_metal_moe_mulmm_q8_t_msl, MetalMoeMulMmQ8T_metal_moe_mulmm_q8_t_msl_entry, MetalMoeMulMmQ8T_metal_moe_mulmm_q8_t_msl_fastmath, err)) : pipeline_from_source(dev, metal_moe_mulmm_q8_msl, metal_moe_mulmm_q8_msl_entry, metal_moe_mulmm_q8_msl_fastmath, err)) if (tensor && pso == null && !metal4_tensor_available(dev)) { - to_log(LOG_INFO, "{tag}: no Metal-4 tensor toolchain on this box - arm skipped\n") + t |> skip("{tag}: no Metal-4 tensor toolchain on this box") return } t |> success(pso != null, "{tag}: pipeline: {err}") @@ -1699,7 +1705,7 @@ def private moe_mulmm_mx4_gate(t : T?; dev, queue; tensor : bool; kdim, ndim : i : pipeline_from_source(dev, MetalMoeMulMmMx4T_metal_moe_mulmm_mx4_t_msl, MetalMoeMulMmMx4T_metal_moe_mulmm_mx4_t_msl_entry, MetalMoeMulMmMx4T_metal_moe_mulmm_mx4_t_msl_fastmath, err)) : pipeline_from_source(dev, metal_moe_mulmm_mx4_msl, metal_moe_mulmm_mx4_msl_entry, metal_moe_mulmm_mx4_msl_fastmath, err)) if (tensor && pso == null && !metal4_tensor_available(dev)) { - to_log(LOG_INFO, "{tag}: no Metal-4 tensor toolchain on this box - arm skipped\n") + t |> skip("{tag}: no Metal-4 tensor toolchain on this box") return } t |> success(pso != null, "{tag}: pipeline: {err}") @@ -1833,7 +1839,7 @@ def private attn_qk_mm_gate(t : T?; dev, queue; tensor : bool; heads, hs, kv_mul ? pipeline_from_source(dev, MetalAttnQKMmT_metal_attn_qk_mm_t_msl, MetalAttnQKMmT_metal_attn_qk_mm_t_msl_entry, MetalAttnQKMmT_metal_attn_qk_mm_t_msl_fastmath, err) : pipeline_from_source(dev, metal_attn_qk_mm_msl, metal_attn_qk_mm_msl_entry, metal_attn_qk_mm_msl_fastmath, err))) if (tensor && pso == null && !metal4_tensor_available(dev)) { - to_log(LOG_INFO, "{tag}: no Metal-4 tensor toolchain on this box - arm skipped\n") + t |> skip("{tag}: no Metal-4 tensor toolchain on this box") return } t |> success(pso != null, "{tag}: pipeline: {err}") @@ -1934,7 +1940,7 @@ def private attn_av_mm_gate(t : T?; dev, queue; tensor : bool; heads, hs, kv_mul ? pipeline_from_source(dev, MetalAttnAVMmP_metal_attn_av_mm_msl, MetalAttnAVMmP_metal_attn_av_mm_msl_entry, MetalAttnAVMmP_metal_attn_av_mm_msl_fastmath, err) : pipeline_from_source(dev, MetalAttnAVMm_metal_attn_av_mm_msl, MetalAttnAVMm_metal_attn_av_mm_msl_entry, MetalAttnAVMm_metal_attn_av_mm_msl_fastmath, err))) if (tensor && pso == null && !metal4_tensor_available(dev)) { - to_log(LOG_INFO, "{tag}: no Metal-4 tensor toolchain on this box - arm skipped\n") + t |> skip("{tag}: no Metal-4 tensor toolchain on this box") return } t |> success(pso != null, "{tag}: pipeline: {err}") diff --git a/modules/dasLLAMA/tests/test_metal_gemv_kernels.das b/modules/dasLLAMA/tests/test_metal_gemv_kernels.das index 02afd49769..02747b09c0 100644 --- a/modules/dasLLAMA/tests/test_metal_gemv_kernels.das +++ b/modules/dasLLAMA/tests/test_metal_gemv_kernels.das @@ -26,9 +26,9 @@ def private kq_gemv_gate(t : T?; dev, queue; vform : string; n, d : int) { let fmt = vform == "k4" ? 4 : (vform == "k6" ? 6 : (vform == "iq4xs" ? 44 : (vform == "k3" ? 3 : (vform == "iq3s" || vform == "iq3sf4" ? 33 : (vform == "iq3xxs" || vform == "iq3xxsf4" ? 34 : (vform == "iq4nl" ? 45 : (vform == "iq2s" ? 23 : (vform == "iq2xs" ? 24 : (vform == "iq2xxs" || vform == "iq2xxsf4" ? 25 : (vform == "k2" ? 2 : 5)))))))))) let tag = "kq_gemv_{vform} n={n} d={d}" var err : string - let src = vform == "iq2xxsf4" ? metal_kq_gemv_iq2xxs_f4_msl : (vform == "iq2xxs" ? metal_kq_gemv_iq2xxs_msl : (vform == "iq2xs" ? metal_kq_gemv_iq2xs_msl : (vform == "iq2s" ? metal_kq_gemv_iq2s_msl : (vform == "k2" ? metal_kq_gemv_k2_msl : (vform == "iq4nl" ? metal_kq_gemv_iq4nl_msl : (vform == "iq3xxsf4" ? metal_kq_gemv_iq3xxs_f4_msl : (vform == "iq3sf4" ? metal_kq_gemv_iq3s_f4_msl : (vform == "iq3xxs" ? metal_kq_gemv_iq3xxs_msl : (vform == "iq3s" ? metal_kq_gemv_iq3s_msl : (vform == "k3" ? metal_kq_gemv_k3_msl : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl : (vform == "k4" ? metal_kq_gemv_k4_msl : (vform == "k5" ? metal_kq_gemv_k5_msl : (vform == "k5c" ? metal_kq_gemv_k5c_msl : metal_kq_gemv_k6_msl)))))))))))))) - let entry = vform == "iq2xxsf4" ? metal_kq_gemv_iq2xxs_f4_msl_entry : (vform == "iq2xxs" ? metal_kq_gemv_iq2xxs_msl_entry : (vform == "iq2xs" ? metal_kq_gemv_iq2xs_msl_entry : (vform == "iq2s" ? metal_kq_gemv_iq2s_msl_entry : (vform == "k2" ? metal_kq_gemv_k2_msl_entry : (vform == "iq4nl" ? metal_kq_gemv_iq4nl_msl_entry : (vform == "iq3xxsf4" ? metal_kq_gemv_iq3xxs_f4_msl_entry : (vform == "iq3sf4" ? metal_kq_gemv_iq3s_f4_msl_entry : (vform == "iq3xxs" ? metal_kq_gemv_iq3xxs_msl_entry : (vform == "iq3s" ? metal_kq_gemv_iq3s_msl_entry : (vform == "k3" ? metal_kq_gemv_k3_msl_entry : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl_entry : (vform == "k4" ? metal_kq_gemv_k4_msl_entry : (vform == "k5" ? metal_kq_gemv_k5_msl_entry : (vform == "k5c" ? metal_kq_gemv_k5c_msl_entry : metal_kq_gemv_k6_msl_entry)))))))))))))) - let fm = vform == "iq2xxsf4" ? metal_kq_gemv_iq2xxs_f4_msl_fastmath : (vform == "iq2xxs" ? metal_kq_gemv_iq2xxs_msl_fastmath : (vform == "iq2xs" ? metal_kq_gemv_iq2xs_msl_fastmath : (vform == "iq2s" ? metal_kq_gemv_iq2s_msl_fastmath : (vform == "k2" ? metal_kq_gemv_k2_msl_fastmath : (vform == "iq4nl" ? metal_kq_gemv_iq4nl_msl_fastmath : (vform == "iq3xxsf4" ? metal_kq_gemv_iq3xxs_f4_msl_fastmath : (vform == "iq3sf4" ? metal_kq_gemv_iq3s_f4_msl_fastmath : (vform == "iq3xxs" ? metal_kq_gemv_iq3xxs_msl_fastmath : (vform == "iq3s" ? metal_kq_gemv_iq3s_msl_fastmath : (vform == "k3" ? metal_kq_gemv_k3_msl_fastmath : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl_fastmath : (vform == "k4" ? metal_kq_gemv_k4_msl_fastmath : (vform == "k5" ? metal_kq_gemv_k5_msl_fastmath : (vform == "k5c" ? metal_kq_gemv_k5c_msl_fastmath : metal_kq_gemv_k6_msl_fastmath)))))))))))))) + let src = vform == "iq2xxsf4" ? MetalKqGemvIq2xxsF4_metal_kq_gemv_iq2xxs_msl : (vform == "iq2xxs" ? MetalKqGemvIq2xxs_metal_kq_gemv_iq2xxs_msl : (vform == "iq2xs" ? metal_kq_gemv_iq2xs_msl : (vform == "iq2s" ? metal_kq_gemv_iq2s_msl : (vform == "k2" ? metal_kq_gemv_k2_msl : (vform == "iq4nl" ? metal_kq_gemv_iq4nl_msl : (vform == "iq3xxsf4" ? metal_kq_gemv_iq3xxs_f4_msl : (vform == "iq3sf4" ? metal_kq_gemv_iq3s_f4_msl : (vform == "iq3xxs" ? metal_kq_gemv_iq3xxs_msl : (vform == "iq3s" ? metal_kq_gemv_iq3s_msl : (vform == "k3" ? metal_kq_gemv_k3_msl : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl : (vform == "k4" ? metal_kq_gemv_k4_msl : (vform == "k5" ? metal_kq_gemv_k5_msl : (vform == "k5c" ? metal_kq_gemv_k5c_msl : metal_kq_gemv_k6_msl)))))))))))))) + let entry = vform == "iq2xxsf4" ? MetalKqGemvIq2xxsF4_metal_kq_gemv_iq2xxs_msl_entry : (vform == "iq2xxs" ? MetalKqGemvIq2xxs_metal_kq_gemv_iq2xxs_msl_entry : (vform == "iq2xs" ? metal_kq_gemv_iq2xs_msl_entry : (vform == "iq2s" ? metal_kq_gemv_iq2s_msl_entry : (vform == "k2" ? metal_kq_gemv_k2_msl_entry : (vform == "iq4nl" ? metal_kq_gemv_iq4nl_msl_entry : (vform == "iq3xxsf4" ? metal_kq_gemv_iq3xxs_f4_msl_entry : (vform == "iq3sf4" ? metal_kq_gemv_iq3s_f4_msl_entry : (vform == "iq3xxs" ? metal_kq_gemv_iq3xxs_msl_entry : (vform == "iq3s" ? metal_kq_gemv_iq3s_msl_entry : (vform == "k3" ? metal_kq_gemv_k3_msl_entry : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl_entry : (vform == "k4" ? metal_kq_gemv_k4_msl_entry : (vform == "k5" ? metal_kq_gemv_k5_msl_entry : (vform == "k5c" ? metal_kq_gemv_k5c_msl_entry : metal_kq_gemv_k6_msl_entry)))))))))))))) + let fm = vform == "iq2xxsf4" ? MetalKqGemvIq2xxsF4_metal_kq_gemv_iq2xxs_msl_fastmath : (vform == "iq2xxs" ? MetalKqGemvIq2xxs_metal_kq_gemv_iq2xxs_msl_fastmath : (vform == "iq2xs" ? metal_kq_gemv_iq2xs_msl_fastmath : (vform == "iq2s" ? metal_kq_gemv_iq2s_msl_fastmath : (vform == "k2" ? metal_kq_gemv_k2_msl_fastmath : (vform == "iq4nl" ? metal_kq_gemv_iq4nl_msl_fastmath : (vform == "iq3xxsf4" ? metal_kq_gemv_iq3xxs_f4_msl_fastmath : (vform == "iq3sf4" ? metal_kq_gemv_iq3s_f4_msl_fastmath : (vform == "iq3xxs" ? metal_kq_gemv_iq3xxs_msl_fastmath : (vform == "iq3s" ? metal_kq_gemv_iq3s_msl_fastmath : (vform == "k3" ? metal_kq_gemv_k3_msl_fastmath : (vform == "iq4xs" ? metal_kq_gemv_iq4xs_msl_fastmath : (vform == "k4" ? metal_kq_gemv_k4_msl_fastmath : (vform == "k5" ? metal_kq_gemv_k5_msl_fastmath : (vform == "k5c" ? metal_kq_gemv_k5c_msl_fastmath : metal_kq_gemv_k6_msl_fastmath)))))))))))))) var pso = pipeline_from_source(dev, src, entry, fm, err) t |> success(pso != null, "{tag}: pipeline: {err}") return if (pso == null) @@ -82,7 +82,7 @@ def private kq_gemv_gate(t : T?; dev, queue; vform : string; n, d : int) { metal_set_threadgroup_memory_length(enc, metal_kq_gemv_iq4nl_msl_tgmem, 0) // the codebook slab } if (vform == "iq2xxsf4") { - metal_set_threadgroup_memory_length(enc, metal_kq_gemv_iq2xxs_f4_msl_tgmem, 0) // the f4 magnitude slab + metal_set_threadgroup_memory_length(enc, MetalKqGemvIq2xxsF4_metal_kq_gemv_iq2xxs_msl_tgmem, 0) // the f4 magnitude slab } metal_set_buffer(enc, bks, fmt == 6 || fmt == 44 || fmt == 3 || fmt == 33 || fmt == 34 || fmt == 2 || fmt == 23 || fmt == 24 || fmt == 25 ? uint64(nsb * 16) : 0ul, 0) // the split-scale formats: the f16 d tail at nsb*16 metal_set_buffer(enc, bks, 0ul, 1) @@ -96,6 +96,12 @@ def private kq_gemv_gate(t : T?; dev, queue; vform : string; n, d : int) { t |> success(ran, "{tag}: encode: {err}") if (ran) { t |> equal(buf_mismatch_env(by, want, env, 2e-4, tag), 0) + //! the control: the same compare must RED on a poisoned expectation (one element off by + //! an env-scaled-plus-absolute delta) - proves the bar can fail for this arm + var wantp := want + wantp[0] += env[0] * 0.1 + 1.0 + t |> success(buf_mismatch_env(by, wantp, env, 2e-4, "{tag} CONTROL(expected-red)") != 0, "{tag}: poison control reds") + delete wantp } metal_release(bks) metal_release(bkq) diff --git a/modules/dasLLAMA/tests/test_metal_support_matrix.das b/modules/dasLLAMA/tests/test_metal_support_matrix.das index 0cce4ee59a..32d1537f0b 100644 --- a/modules/dasLLAMA/tests/test_metal_support_matrix.das +++ b/modules/dasLLAMA/tests/test_metal_support_matrix.das @@ -836,19 +836,27 @@ def private qwen35_row(t : T?; path, family, tag : string; expect_needs : uint; // crowns, recompile the PSOs, and assert the model still produces the CPU's tokens with twins in // the graph. A new tensor family joins CROWNED below. [unused_argument(t, path)] // off-Apple the gated body is empty -def private tensor_twin_row(t : T?; path, tag : string) { // nolint:LINT012,LINT019 — tag is read by the das_metal arm +def private tensor_twin_row(t : T?; path, tag : string; optional : bool = false) { // nolint:LINT012,LINT019 — tag is read by the das_metal arm static_if (typeinfo builtin_module_exists(das_metal)) { if (!family_on(t, "llama") || !arm_on(t, "tensor")) { return } // the carriers are provisioned (fetch_models.das), so absence is a RED, never a skip - - // a box that skips here ships every tensor twin unexercised + // a box that skips here ships every tensor twin unexercised. An `optional` carrier is a + // LOCAL requant no fetch can provision - absence skips loudly instead if (!stat(path).is_valid) { - t |> success(false, "tensor twin carrier missing: {path} - bin/daslang modules/dasLLAMA/performance/fetch_models.das -- --fetch") + if (optional) { + t |> skip("{tag}: local requant vehicle absent ({path}) - the iquant crown witness runs only where it is minted") + } else { + t |> success(false, "tensor twin carrier missing: {path} - bin/daslang modules/dasLLAMA/performance/fetch_models.das -- --fetch") + } return } let CROWNED = ("attn_avmm,attn_qkmm,gemm64b_q8,gemmb_q8,gemmb_sk_q8,kq_mulmm_k4," + - "kq_mulmm_k5,kq_mulmm_k6,moe_mulmm_mx4,moe_mulmm_q8,mulmm_bf16,mulmm_q8") + "kq_mulmm_k5,kq_mulmm_k6,moe_mulmm_mx4,moe_mulmm_q8,mulmm_bf16,mulmm_q8," + + "kq_mulmm_iq4xs,kq_mulmm_iq4nl,kq_mulmm_k3,kq_mulmm_iq3s,kq_mulmm_iq3xxs," + + "kq_mulmm_k2,kq_mulmm_iq2s,kq_mulmm_iq2xs,kq_mulmm_iq2xxs," + + "kq_gemv_iq3s_f4,kq_gemv_iq3xxs_f4,kq_gemv_iq2xxs_f4") var tr <- load_model_(path, QuantMode.q8) tr.config.seq_len = min(tr.config.seq_len, 512l) var trg <- blob_twin(t, path, 512l) @@ -1831,6 +1839,9 @@ def test_metal_family_matrix(t : T?) { // q8 drives the mulmm/attn twins, the K-quant row drives the kq mulmm twins tensor_twin_row(t, path_join(models_dir(), "Llama-3.2-1B-Instruct-Q8_0.gguf"), "tensor-twins/q8") tensor_twin_row(t, path_join(models_dir(), "Llama-3.2-1B-Instruct-Q5_K_M.gguf"), "tensor-twins/k5") + //! the iquant crown witness: the arc vehicle carries iq2xxs planes, so this row + //! proves the new crown names reach a served twin (mulmm + the f4 gemv crown) + tensor_twin_row(t, path_join(models_dir(), "Llama-3.2-1B-Instruct-IQ2_XXS-local.gguf"), "tensor-twins/iq2xxs", true) metal_decode_shutdown() metal_prefill_shutdown() t |> equal(metal_log_leaks_if_any(), 0l) diff --git a/modules/dasLLVM/REVIEW.md b/modules/dasLLVM/REVIEW.md index 030f00d7f8..89ba549483 100644 --- a/modules/dasLLVM/REVIEW.md +++ b/modules/dasLLVM/REVIEW.md @@ -11,11 +11,9 @@ `get_architecture_name()`, or `cpu_supports()` - runs the module-owned suite on a machine matching that condition.** -- **A diff whose new behavior runs only under a set target triple - a cross-compile, not the - host - names in its PR body the cross-compile (`write_exe`) for that target that exercised - the behavior.** This holds whether the diff adds that code path or finds it already there; - the suite runs on the host, so a target-triple branch is checked only by the artifact built - for that target. +- **A diff that adds or changes a branch on the target triple names in its PR body the + cross-compile (`write_exe`) for that target that exercised the behavior.** The suite runs on + the host, so a target-triple branch is checked only by the artifact built for that target. - **A diff that adds work to, or moves work within, what `run_jit` (`daslib/llvm_jit_run.das`) executes - its own body or any callee - also prints an @@ -29,12 +27,17 @@ the `[tune]` stamping - is not such a change: stamped arguments fold into the cache keys per function. -- **A diff that adds an environment or config input to the cache key folds it inside - `jit_env_salt` (`daslib/llvm_jit_run.das`), never directly into either cache key - the DLL +- **A diff that adds an environment or config input to a JIT cache key folds it inside + `jit_env_salt` (`daslib/llvm_jit_run.das`), never directly into either JIT key - the DLL key or the split-obj key (`ARCHITECTURE.md` sec.2)** - salt feeds both keys, and a config folded into one but not the other links stale objects. Inputs that vary per function set (AOT hashes) are key material, not salt. +- **A macro under `daslib/` that reads a file at compile time registers it with + `add_module_cache_dependency` before any early return, in the same change** + (`ARCHITECTURE.md` sec.5). An unpinned compile-time file read serves stale macro output + from the module cache until an unrelated source file changes - silently. + - **A change to a `[tune]`-family annotation is reviewed with `skills/tune.md`.** A change to the framework itself - `daslib/llvm_tune.das` or its tests - is reviewed with `skills/internal/llvm_tune_internals.md`. diff --git a/modules/dasLLVM/tests/cant_tuned_bad_fallback.das b/modules/dasLLVM/tests/cant_tuned_bad_fallback.das new file mode 100644 index 0000000000..32bcb3c37d --- /dev/null +++ b/modules/dasLLVM/tests/cant_tuned_bad_fallback.das @@ -0,0 +1,25 @@ +// the fallback arm of the stale-sidecar rescue keeps its own floor: a `fallback=` chain whose +// picked perm is outside the grid still refuses the compile ("tuned: fallback perm ... not in +// grid") - the sidecar rescue must never loop through a broken fallback (twin of +// cant_tuned_bad_pin.das; the sidecar arm is test_tuned's stale-sidecar cell) +options gen2 +expect 20800:1 +options _dasllama_internal = true + +require dasllama/dasllama_tune + +def template bad_fallback_dot_template(a, b : array) : float { + var s = 0f + for [tune = 1] (i in range(length(a))) { // nolint:PERF029 - the tuned kernel: perm hints stamp this index loop + s += a[i] * b[i] + } + return s +} + +[tuned(fallback = "no_such_fallback")] +def bad_fallback_dot(a, b : array) : float {} + +[export] +def main { + print("never compiles\n") +} diff --git a/modules/dasLLVM/tests/test_tuned.das b/modules/dasLLVM/tests/test_tuned.das index fd6c527272..1fb0371009 100644 --- a/modules/dasLLVM/tests/test_tuned.das +++ b/modules/dasLLVM/tests/test_tuned.das @@ -77,3 +77,59 @@ def test_tuned_stale_sidecar_perm(t : T?) { t |> success(find(outp, "STALE_CLIENT_OK") >= 0, "the client ran to completion (got: {outp})") remove(manifest) } + +def private spawn_capture(cmd : string; var outp : string&) : int { + outp = "" + var rc : int + unsafe { + rc = popen_timeout(cmd, 300.0) $(f) { + if (f != null) { + outp = fread(f) + } + } + } + return rc +} + +// The module-cache pin is the mechanism's ONLY production wiring (read_manifest -> +// add_module_cache_dependency, ARCHITECTURE.md sec.5): a [tuned] client compiled under +// -module-cache with a tune sidecar must REPARSE when the sidecar's content changes - +// served-from-cache stamps minted against the old sidecar are the silent failure this exists +// to prevent. +[test] +def test_tuned_sidecar_invalidates_module_cache(t : T?) { + let args <- get_command_line_arguments() + let bin = args[0] + let tmpRes = create_temp_file_result("tuned_mc_manifest_", ".json") + if (tmpRes is error) { + t |> failure("could not create a temp file: {tmpRes as error}") + return + } + let manifest = tmpRes as value + let mcRes = create_temp_file_result("tuned_mc_cache_", ".bin") + if (mcRes is error) { + t |> failure("could not create a temp cache file: {mcRes as error}") + return + } + let mc = mcRes as value + remove(mc) // the cache path must start absent so the cold run writes the stream head + fwrite(manifest, "\{\"kernels\": \{\}, \"provenance\": \{\"box\": \"{tune_box_identity()}\"\}\}") + let client = "{get_this_module_dir()}/tuned_stale_perm_client.das" + let env = (get_platform_name() == "windows" + ? "set DAS_TUNE_MANIFEST={manifest}&& " + : "DAS_TUNE_MANIFEST={manifest} ") + let cmd = "{env}\"{bin}\" -module-cache \"{mc}\" \"{client}\"" + var run_out = "" + var rc = spawn_capture(cmd, run_out) + t |> equal(rc, 0, "cold run compiles and runs") + t |> success(find(run_out, "ser: wrote") >= 0, "cold run wrote the module cache (got: {run_out})") + rc = spawn_capture(cmd, run_out) + t |> equal(rc, 0, "warm run compiles and runs") + t |> success(find(run_out, "deser: clean") >= 0, "unchanged sidecar serves from cache (got: {run_out})") + fwrite(manifest, "\{\"kernels\": \{\"stale_probe_dot\": \"u2\"\}, \"provenance\": \{\"box\": \"{tune_box_identity()}\"\}\}") + rc = spawn_capture(cmd, run_out) + t |> equal(rc, 0, "post-mint run compiles and runs") + t |> success(find(run_out, "ser: macro dependency changed") >= 0, "a re-minted sidecar invalidates the cached [tuned] stamps (got: {run_out})") + remove(manifest) + remove(mc) +} diff --git a/skills/review_md.md b/skills/review_md.md index 4cc0df38bf..cba87c90cd 100644 --- a/skills/review_md.md +++ b/skills/review_md.md @@ -19,7 +19,8 @@ block, verbatim except for the module name and the architecture-doc path: ``` A checklist whose folder has a follow-up ledger appends `` Planned work: ``. `` to -the same line. A checklist whose folder's architecture doc has split into companions writes +the same line - `` Planned work: `` (sec. ). `` when the ledger is one section +of a larger document, so the pointer still finds it. A checklist whose folder's architecture doc has split into companions writes the plural - `` Architecture docs: ``, ``. `` - listing every doc its rules cite. A checklist with routed companions - a shared concern file, subfolder checklists, a skill reviewing a construct family - carries their routing lines in or right after the opening, each keyed by KIND ("a `[test]` file, wherever the diff puts it, answers to diff --git a/src/ast/ast_parse.cpp b/src/ast/ast_parse.cpp index bf0377596c..be46298746 100644 --- a/src/ast/ast_parse.cpp +++ b/src/ast/ast_parse.cpp @@ -588,7 +588,7 @@ namespace das { // sanity bound on a record's macro-dependency count: a real program registers a handful // (one per consumed sidecar/config); anything past this reads as stream corruption static constexpr uint32_t SER_MAX_MACRO_DEPS = 4096u; - static constexpr uint64_t FNV64A_SEED = 14695981039346656037ul; // an EXISTING empty file must hash unlike an absent one (hash 0) + static constexpr uint64_t FNV64A_SEED = 14695981039346656037ul; // any nonzero constant serves: size (-1 absent vs 0 empty) already separates the states void statAndHashFileDependency ( const string & path, int64_t & size, uint64_t & hash ) { size = -1; diff --git a/src/builtin/REVIEW.md b/src/builtin/REVIEW.md index 6e33fb3ac5..dc67451b06 100644 --- a/src/builtin/REVIEW.md +++ b/src/builtin/REVIEW.md @@ -5,10 +5,9 @@ - **A diff that adds or changes an `addExtern...`/`addInterop` registration under this folder uses, for a plain-value bind, `addExternInline` or - `addExternInlineEx` when its module is an Inline module, and an `addExtern...` entry point + `addExternInlineEx` when the name its file passes to `Module(...)` is an Inline module, and an `addExtern...` entry point whose name does not contain `Inline` in every other module.** A bind is a C++ function - registered into a module with an `addExtern...` or `addInterop` entry point; annotation, - type, and structure registrations are not binds. The Inline modules are `$` + registered into a module with an `addExtern...` or `addInterop` entry point. The Inline modules are `$` (builtin), `math`, `strings` and `jit`. A plain-value bind returns nothing at all, or a value that is neither a reference nor a result the callee writes into the caller's result slot; it is not an interop bind (`addInterop`), and not a generic container or equality helper diff --git a/tests/module_cache/test_macro_dep_invalidate.das b/tests/module_cache/test_macro_dep_invalidate.das index 2d2a1f80cf..36728791e4 100644 --- a/tests/module_cache/test_macro_dep_invalidate.das +++ b/tests/module_cache/test_macro_dep_invalidate.das @@ -71,6 +71,26 @@ def test_macro_dep_invalidate(t : T?) { chOk = false } t |> success(chOk, "content change invalidated the cached record and the reparse passed") + // absent -> present, through a RELATIVE dep path: the case the architecture doc calls the + // one that matters most (a first mint has no sidecar), plus the builtin's cwd-absolutize + // arm - a relative registration must still validate from the reader's cwd + let depRel = "._mc_dep_probe_rel.txt" + remove(depRel) + // a FRESH cache: the phase-1 cache still validates its own (existing) dep and would serve clean + let cmd2 = "\"{das_exe()}\" -dasroot \"{root}\" -module-cache \"{tmp}/mc2.bin\" \"{root}/tests/module_cache/_fixtures/mc_dep_drv.das\"" + var absent : string + let absentRc = env_run(depRel, cmd2, absent) + t |> success(report_child(t, "absent", absentRc, absent, "ser: wrote"), "cold run with an ABSENT dep wrote the cache") + fwrite(depRel, "now it exists") + var appeared : string + let appearedRc = env_run(depRel, cmd2, appeared) + var apOk = report_child(t, "appeared", appearedRc, appeared, "ser: macro dependency changed") + if (find(appeared, "deser: clean") >= 0) { + t |> failure("the appearing dep did not invalidate the cached record") + apOk = false + } + t |> success(apOk, "a dep that APPEARS invalidates the record minted against its absence") + remove(depRel) var err : string rmdir_rec(tmp, err) } diff --git a/utils/mcp/REVIEW.md b/utils/mcp/REVIEW.md index 8ac4a30221..68578cea3e 100644 --- a/utils/mcp/REVIEW.md +++ b/utils/mcp/REVIEW.md @@ -14,3 +14,7 @@ adds it to the `install(FILES ...)` block that lists `utils/mcp/main.das` in `CM (repo root), in the same change.** `tools/` and `subtools/` are globbed; a top-level file left out of the list dies in the shipped SDK on `error[20605] missing prerequisite` while the in-tree server keeps working. + +**Weakening the kept-comment cases in `test_tools.das` is a defect** - they pin the +formatter's kept set (the leading header block, `//!` docs, `//fmt:` directives, `nolint:` +suppressions). From 246c52cf466591b30411607dd3571160a8a206ad Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 09:58:40 -0700 Subject: [PATCH 27/35] the dragon round: seven checklists take their verdicts, the grooming debt gets a ledger Five rule documents audited as documents, wave by wave. REVIEW_GPU_RACE.md - new this arc - takes every verdict: the routing trigger widens to kernel-side diffs (a diff that changes what an arm mirrors never touched an arm, so three rules could not fire from the route that existed), knockout and kargs defined in place, the selects-alone carve-out cut, the burn phase named for what it is, the A/B-lab port duty split from the empty-lab deletion, every dangling referent spelled. REVIEW_GPU.md's routing line widens in step - the duplicated trigger pair is what the sync gap cost. REVIEW_EXCHANGE.md's moved consent rule gets its path re-based (the ../ survived the move). performance/REVIEW.md sheds its two wrap artifacts, daslib/REVIEW.md its three plus the PERF023 doubled locator and the emit-gloss restatement. harness/REVIEW.md lands at its minimal form: the four-way ledger routing and the one rule the folder owns. Findings on rules this branch never touched go to plans/dasllama_review_grooming.md - one grooming sitting per document, plus the RECOMMEND LINT candidates each dragon named (the bind-number walk extension, the burn-phase counting cell, the consent-notice byte-identity check, the stamp-reach glob widening, three utils/lint census cells). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014mq9AKyoAwvD5T1cigTnRw --- daslib/REVIEW.md | 20 +++--- modules/dasLLAMA/REVIEW.md | 13 +--- modules/dasLLAMA/REVIEW_EXCHANGE.md | 2 +- modules/dasLLAMA/REVIEW_GPU.md | 83 ++++++------------------ modules/dasLLAMA/REVIEW_GPU_RACE.md | 66 +++++++++++++++++++ modules/dasLLAMA/benchmarks/REVIEW.md | 51 +++++++-------- modules/dasLLAMA/harness/REVIEW.md | 12 ++++ modules/dasLLAMA/performance/REVIEW.md | 7 +- modules/dasLLAMA/tests/REVIEW.md | 31 ++++----- modules/dasLLVM/REVIEW.md | 27 ++++---- plans/dasllama_review_grooming.md | 89 ++++++++++++++++++++++++++ 11 files changed, 253 insertions(+), 148 deletions(-) create mode 100644 modules/dasLLAMA/REVIEW_GPU_RACE.md create mode 100644 modules/dasLLAMA/harness/REVIEW.md create mode 100644 plans/dasllama_review_grooming.md diff --git a/daslib/REVIEW.md b/daslib/REVIEW.md index 2b66f2e2a4..c958b96f7e 100644 --- a/daslib/REVIEW.md +++ b/daslib/REVIEW.md @@ -20,9 +20,9 @@ the actionable line is an intermediate call site. inside a lint visitor method is a defect.** The guard's position decides which rules apply inside a lambda; the per-rule policy is in `ARCHITECTURE_LINT.md` sec. 1. -**Never move perf_lint's PERF023 `add_ptr_ref` splice-depth tracking (`perf_lint.das`) behind -the `in_closure` gate.** The rules block -is itself a closure, so a gated tracker never sees the splice. +**Never move PERF023's `add_ptr_ref` splice-depth tracking (`perf_lint.das`) behind the +`in_closure` gate.** The rules block is itself a closure, so a gated tracker never sees the +splice. **A lint warning that anchors anywhere but the code its fix rewrites is a defect.** A remedy that deletes a statement reports the statement; a remedy that edits one variable's @@ -39,9 +39,8 @@ survives into the sibling loop's exit path and unbalances its counter. decided by a C++-side definition records the pair in the architecture doc, in its module's section, naming both sides.** Nothing catches it when one side later moves alone. -**When a diff changes one side of a recorded daslib/C++ pair so the two no longer match, -it changes the other side and updates the pair's architecture-doc entry in the same -diff.** +**When a diff changes one side of a recorded daslib/C++ pair so the two no longer match, it +changes the other side and updates the pair's architecture-doc entry in the same diff.** **A diff that adds an architecture-doc entry adds it only where no name, shape, or test can carry the fact.** @@ -113,11 +112,10 @@ set without the cap is a silently missed finding; raising a cap without the over suggestion that does not compile. **A diff that adds or changes an emit entry point - a function that runs the emit visitor -(`CppAot` or any subclass of it - the visitors that write C++) and then -returns or writes the generated C++ - keeps the error check ahead of that return -or write.** The error check is the program's `macroException`/`failToCompile` state, read -directly or through `log_aot_emit_errors`; a codegen exception mid-visit leaves partial -C++. +(`CppAot` or any subclass of it) and then returns or writes the generated C++ - keeps the +error check ahead of that return or write.** The error check is the program's +`macroException`/`failToCompile` state, read directly or through `log_aot_emit_errors`; a +codegen exception mid-visit leaves partial C++. **Never gate a visitor override in `CppAot` or any subclass of it on `macroException`/`failToCompile` - the function that runs the visitor owns that check.** An diff --git a/modules/dasLLAMA/REVIEW.md b/modules/dasLLAMA/REVIEW.md index b565ca2ddb..415578528f 100644 --- a/modules/dasLLAMA/REVIEW.md +++ b/modules/dasLLAMA/REVIEW.md @@ -30,15 +30,8 @@ tune-boot path that reaches it, applies `REVIEW_EXCHANGE.md`.** **Every `dasllama/` change applies this folder's `tests/REVIEW.md`.** A `dasllama/` diff never opens that checklist on its own. -**A GPU kernel, driver, dispatch-class, or K/V-mirror change applies `REVIEW_GPU.md`.** - -**A `harness/` instrument that times or measures applies `../benchmarks/`'s instrument rules -(`benchmarks/REVIEW.md`) as if it lived there.** The harness folder carries no checklist of -its own, and a timing rig is the same instrument wherever it parks. - -**A diff that adds or moves a row in `harness/tune_kernels.das`'s bench list keeps -`dot_q8q8_laneq4x4` last.** That bench pins the repack backend for the rest of the process, so -a row after it is timed against the pinned backend instead of the backend it would select. +**A GPU kernel, driver, dispatch-class, or K/V-mirror change - and a GPU kernel A/B race, +knockout, or hand-binding arm, wherever the diff puts it - applies `REVIEW_GPU.md`.** **A change to the image rail - `dasllama/dasllama_image.das`, or, wherever the diff puts it, a `.dlim` mint (building a `.dlim` from a gguf), a `.dlim` load, an image identity, or a @@ -197,7 +190,7 @@ the file it checks - in `ARCHITECTURE_ENGINE.md`, `ARCHITECTURE_MEDIA.md`, or `ARCHITECTURE_GPU.md`.** The line names the check and the names it licenses. A licensed name is one that check does not flag. When the check licenses no names, the line says so. -**Checked-in prose this module owns - docs and comments, any language - that is not locating +**Checked-in prose this module owns - docs and comments, any language - that is not locating, patching, or reproducing work against the reference build describes an upstream mechanism in our own terms: no "lifted/ported verbatim from", and no upstream symbol, header, constant, or binary name - write "the reference exe" or "upstream" instead.** The reference build is the diff --git a/modules/dasLLAMA/REVIEW_EXCHANGE.md b/modules/dasLLAMA/REVIEW_EXCHANGE.md index d4399ffa44..08c6bc586c 100644 --- a/modules/dasLLAMA/REVIEW_EXCHANGE.md +++ b/modules/dasLLAMA/REVIEW_EXCHANGE.md @@ -35,7 +35,7 @@ page are the answer surfaces a supervised boot has. **A diff that changes the exchange consent notice wording updates every checked-in copy in the same change, and a diff that adds a copy names it here in the same change: -`EXCHANGE_CONSENT_NOTICE` (`../dasllama/dasllama_exchange.das`), `CONSENT_TITLE` / +`EXCHANGE_CONSENT_NOTICE` (`dasllama/dasllama_exchange.das`), `CONSENT_TITLE` / `CONSENT_TEXT` (`utils/watchdog/watchdog.py`, repo root), and the captured `utils/dasllama-server/tests/fixtures/exchange.json` (repo root).** The console prompt and the control page render the served constant, so they are not copies. diff --git a/modules/dasLLAMA/REVIEW_GPU.md b/modules/dasLLAMA/REVIEW_GPU.md index c52c4a6979..d0e695edb3 100644 --- a/modules/dasLLAMA/REVIEW_GPU.md +++ b/modules/dasLLAMA/REVIEW_GPU.md @@ -6,6 +6,11 @@ docs: `ARCHITECTURE_GPU.md`, `ARCHITECTURE_GPU_PREFILL.md`, `ARCHITECTURE_GPU_VU **Routed from `REVIEW.md`: a diff touching a GPU kernel, driver, dispatch class, or the K/V mirrors applies this list together with `REVIEW.md`.** +**A diff touching a GPU kernel A/B race, knockout, or hand-binding arm - or changing a +kernel class such an arm mirrors (binding numbers, kargs layout, threadgroup memory, staging +shape, grid or threadgroup geometry) - wherever the diff puts it - applies +`REVIEW_GPU_RACE.md` too.** + **A diff touching the tower driver (`dasllama/dasllama_metal_tower.das`), a kernel class or builder the tower dispatches, or the Metal ASR decoder applies `REVIEW_TOWER.md` too.** @@ -57,7 +62,7 @@ threadgroup-uniform value instead.** A per-thread exit leaves the threadgroup un complete the op. **An encoder that picks a kernel form whose loop carries no bounds or tail guard - stamped -without it, or monomorphized with none - shows that every address the form touches stays +without one, or generated from a template instance that has none - shows that every address the form touches stays inside its buffers' allocations.** One extent dividing evenly is not that showing. A padded chunk's walk can run past the live extent, and one poisoned read in a shared tile corrupts real rows; a deliberate tail over-read conforms only where the allocation carries the slack. @@ -78,8 +83,9 @@ wider-row site passes the full stride or dispatches the padded tile.** A split r them. **A scratch buffer a dispatch writes is never rebound for a new write before the reader of -its previous write is encoded; a flip set smaller than the chain's encode-overlap depth is a -defect.** One shared scratch serializes the whole chain through its write-after-read hazards. +its previous write is encoded - rotate through as many buffers as the chain has dispatches in +flight between a write and its read.** One shared scratch serializes the whole chain through +its write-after-read hazards. **A diff that adds dispatches to an encoder path to save bandwidth also gates that path on work size, in the same change.** The gate's threshold is measured at both ends of the size @@ -108,8 +114,8 @@ overridden method spliced flat at emission. `@template_gate` instead.** **A diff that forks a kernel class out of a shared template shows that the bodies no longer -differ on a single stamp axis.** The same diff names the axis that is gone, in the surviving -template's comment. +differ on the compile-time choice the template carried, and names that choice in the +surviving template's comment.** **A `[metal_dispatch]` / `[vk_dispatch]` field whose memory is load-once - a model plane, or an `upload_region` upload never written after arming - is a defect unless it carries @@ -123,14 +129,15 @@ defect; a per-encode field either omits `@role` or names the access its body per `[vk_dispatch]` declaration, or a new instance of a template carrying one - covers that class in `tests/test_kernel_coverage.das`, one of two ways.** Either a census row there dispatches the class, or the diff names it in that file's `CENSUS_NEVER_DISPATCHED` with the reason no -row can reach it - an unledgered class turns the census's blind-spot list into a lie. +row can reach it - a class in neither place leaves `CENSUS_NEVER_DISPATCHED` claiming coverage the census does not have. **A new kernel class declared in `dasllama/` carries `[metal_dispatch]` / `[vk_dispatch]` with every annotation that backend's generated builder reads - per-field `@binding` / `@role` / `@off` / `@default`.** A field carrying none of them is dropped from the bind list -with no error. (A source-text fixture a test spawns to compile is outside this rule.) The -`@workgroup`/`tgmem=` half is a lens compile refusal now - weakening that refusal, or its -gate `test_lens_tgmem_gate`, is a defect. +with no error. + +**Weakening the lens's `@workgroup`/`tgmem=` compile refusal, or its gate +`test_lens_tgmem_gate` (`tests/test_metal_misc_kernels.das`), is a defect.** **A kernel field carries `@span` only when every caller binds whole output rows.** A caller binding a column tile of a wider row passes the tile width as the kernel's n while its rows @@ -140,21 +147,9 @@ every row outside the tracked hazard range. **A NEW hand-written `enc_*` body is a defect unless it is a wrapper - a format or twin pick, a default-filling wrapper, or a composite over generated builders.** -**A hand-rolled bind list on a SERVED dispatch in `dasllama/` or `performance/` is a defect - -dispatch through the kernel's `enc_*` builder instead.** A race or knockout arm hand-binds by -construction and answers to the hand-binding-arm rules instead. - -**A hand-binding arm that binds a field at another field's declared number is a defect.** -A mis-numbered arm dispatches, reads the wrong buffer, and its timing crowns the wrong -kernel silently. - -**A hand-binding arm whose diff cannot be checked by `REVIEW.das`'s `check_race_bind_numbers` -- its pipeline source or threadgroup-memory size arrives as a function parameter instead of a -literal MSL global near the binds - states in the PR that its bind numbers were verified by -hand against the class declaration.** The machine check resolves the kernel class only from a -literal global (a `kn_tgmem` constant or an in-function `pipeline_from_source`); a -parameter-shaped arm is invisible to it, and an unverified invisible arm is the mis-numbered -crown's front door. +**A hand-rolled bind list on a dispatch that serves a user call - not a race or knockout +timing arm - in `dasllama/` or `performance/` is a defect: dispatch through the kernel's +`enc_*` builder instead.** Timing arms answer to `REVIEW_GPU_RACE.md`. **A value that reaches the kernel twice device-side - a scalar bound both as a uniform buffer and as a kargs field - is a defect.** A `params=` value that the `grid=`/`tg=` spec consumes @@ -178,29 +173,6 @@ device and queue from `dasllama/dasllama__common.das`'s init.** (`dasllama/`) other than the one that owns its kernel class** - it goes through that file's own init/release pair. -**Never put race code outside the file that owns the kernel family - the shared scaffolding -(`race_buf`, `race_envelope_ok`, `race_pair_ms`) belongs to -`dasllama/dasllama__common.das`.** Race code is the in-engine base-vs-twin check that -times both kernels on one queue and compares their outputs. - -**Race code sizes its operands at a real model shape - never at a small square slab.** A slab -small enough to sit in cache ranks the kernels by an effect production never sees, and the -race then picks the slower kernel. - -**A kernel A/B race arm whose verdict crowns or ships a kernel - wherever the race lives - -binds a DIFFERENT output buffer for consecutive dispatches of its chain, never one shared -output.** One shared output serializes the chain on its write-after-read hazard while the -served graph overlaps consecutive dispatches, so the race ranks the arms on a shape -production never runs. A comparison rig that deliberately matches a serialized reference -instrument's shape is the exception, and it says so where `ARCHITECTURE_MEASUREMENT.md` -sec.2.21 ledgers it - its verdicts never crown alone. - -**Weakening `race_pair_ms`'s burn phase is a defect, and a diff that adds a crowning race arm -OUTSIDE `race_pair_ms` burns GPU work on its own arms before its first timed round, in the -same change.** The arms alternate, so a first round on an idle clock bills the governor's -ramp to one side. The dispatches inside each timed encoder stay back to back - a sparse -encoder reads idle-clock times. - **A string-typed Metal decline reason is a defect - a Metal decline reason is an enum value in `dasllama/dasllama_metal_shapes.das`, one enum per driver.** @@ -268,23 +240,6 @@ cache. clear** - put it in a `*_ready` latch, or in a holder that function already clears in `dasllama/dasllama_vulkan_common.das`: `g_rd`, `g_gpu`, the weight arena. -**A diff that changes anything a hand-binding arm must mirror to dispatch a kernel - binding -numbers, kargs layout, threadgroup memory, grid or threadgroup geometry - fixes or deletes, -in the same change, every arm of a hand-binding lab that binds it** - a hand-binding lab is a -kernel A/B or knockout timing script, wherever it lives, that hand-lists its bindings -instead of dispatching through the `enc_*` builder. A lab left dispatching stale geometry -measures the wrong kernel silently. - -**A diff that ports an A/B lab's winning variant into a kernel deletes the ported arm in the -same change - its variant class and any variants-module code that exists only for it; when -the lab exists only for that decision, its driver and remaining arm go too.** An A/B lab is a -timing script whose output SELECTS between implementations of the same compute, wherever it -lives (`benchmarks/`, `harness/`); a decided arm that outlives its decision degrades into an -unmaintained duplicate of the kernel it seeded. An arm `ARCHITECTURE_GPU.md` sec.2.2b ledgers -as a retained reference is the sanctioned survivor - and a retained arm that reimplements a -shipped kernel is resynced or deleted in the same change that moves that kernel's staging -shape or geometry. - **Never read a `[spirv_decode]` callback's quant bytes by indexing `unpack8` of a 32-bit word with a runtime value - read them as 16-bit lanes instead: an `int16[N]` block member selected with `unpack8(w)[i & 1u]`, sub-fields pulled out by shift and mask.** The vendor driver's shader diff --git a/modules/dasLLAMA/REVIEW_GPU_RACE.md b/modules/dasLLAMA/REVIEW_GPU_RACE.md new file mode 100644 index 0000000000..181c124f42 --- /dev/null +++ b/modules/dasLLAMA/REVIEW_GPU_RACE.md @@ -0,0 +1,66 @@ +# dasLLAMA GPU Race / Lab / Hand-Binding Review Checklist + +**Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist.** Architecture +docs: `ARCHITECTURE_GPU.md`, `ARCHITECTURE_MEASUREMENT.md`. + +**Routed from `REVIEW_GPU.md`: a diff touching a GPU kernel A/B race, knockout, or +hand-binding arm - or changing a kernel class such an arm mirrors (binding numbers, kargs +layout, threadgroup memory, staging shape, grid or threadgroup geometry) - wherever the diff +puts it, applies this list together with `REVIEW_GPU.md`.** + +**A hand-binding arm - a race or knockout timing arm (a knockout attributes cost across +stages instead of selecting between implementations) that lists its bindings by number +instead of dispatching through the `enc_*` builder - that binds a field at another field's +declared number is a defect.** A mis-numbered arm dispatches, reads the wrong buffer, and its +timing selects the wrong kernel silently. + +**A hand-binding arm outside `dasllama/`, or one whose pipeline source or threadgroup-memory +size arrives as a function parameter rather than a literal global, states in the PR that its +bind numbers were verified by hand against the class declaration.** The `REVIEW.das` gate +`check_race_bind_numbers` cannot read those arms, and an arm nothing checked is where a +mis-numbered bind reaches the board. + +**A diff that changes anything a hand-binding arm must mirror to dispatch a kernel - binding +numbers, kargs (kernel-argument struct) layout, threadgroup memory, grid or threadgroup +geometry - fixes or deletes every such arm in the same change.** An arm left dispatching +stale geometry measures the wrong kernel silently. + +**Race code inside the engine (`dasllama/`) sits in the file that owns its kernel family and +never in another engine file; the shared scaffolding (`race_buf`, `race_envelope_ok`, +`race_pair_ms`) sits in `dasllama/dasllama__common.das`.** Race code is the base-vs-twin +check that times both kernels on one queue and compares their outputs. + +**Race code sizes its operands at a real model shape - never at a small square slab.** A slab +small enough to sit in cache ranks the kernels by an effect production never sees, and the +race then picks the slower kernel. + +**A kernel A/B race arm whose verdict alone selects the kernel the engine ships - wherever +the race lives - binds a DIFFERENT output buffer for consecutive dispatches of its chain, +never one shared output.** One shared output serializes the chain on its write-after-read +hazard while the served graph overlaps consecutive dispatches, so the race ranks the arms on +a shape production never runs. + +**Weakening the burn phase of `race_pair_ms` (`dasllama/dasllama_metal_common.das`) - the GPU +work it runs on both arms before the first timed round - is a defect.** + +**A race arm that selects the kernel the engine ships and does not go through `race_pair_ms` +burns GPU work on its own arms before its first timed round.** The arms alternate, so a first +round on a cold GPU clock charges the clock's ramp-up to whichever arm ran first. + +**A timed encoder in a race arm that selects the kernel the engine ships issues its +dispatches back to back.** An encoder that leaves gaps between its dispatches times an idle +clock. + +**A diff that ports an A/B lab's winning variant into a kernel deletes, in the same change, +every arm of that lab that `ARCHITECTURE_GPU.md` sec.2.2b does not ledger as a retained +reference, together with the variant class and any `*_variants.das` code that exists only for +that arm.** An A/B lab is a timing script whose output selects between implementations of the +same compute; a decided arm that outlives its decision degrades into an unmaintained +duplicate of the kernel it seeded. + +**A diff that leaves an A/B lab with no undecided arm deletes the lab's driver and its +remaining arm in the same change.** + +**A diff that moves a shipped kernel's staging shape or its grid or threadgroup geometry +resyncs or deletes, in the same change, every arm `ARCHITECTURE_GPU.md` sec.2.2b ledgers as a +retained reference that reimplements that kernel.** diff --git a/modules/dasLLAMA/benchmarks/REVIEW.md b/modules/dasLLAMA/benchmarks/REVIEW.md index cb2f791a71..bf58f4a8d1 100644 --- a/modules/dasLLAMA/benchmarks/REVIEW.md +++ b/modules/dasLLAMA/benchmarks/REVIEW.md @@ -10,14 +10,13 @@ everything else. pipeline the dasLLAMA engine selected also calls `tune_gate()` (`../performance/profile_common.das`) before that instrument's first timed rep.** A pipeline the instrument compiled itself - or a reference tool's own runtime - is not one the engine -selected. Without the gate the instrument measures fallback kernels silently. +selected. Without the gate the instrument measures fallback kernels silently. An instrument is a script whose output is a measured wall or rate. **A diff that adds or changes a race alternates its arms within one process - one timed round -per arm, best-of across rounds, GPU work burned before the first timed round.** A race is an -instrument that compares two implementations. Two separate runs measure the box's drift -between them as much as they measure the arms; `race_pair_ms` -(`../dasllama/dasllama_metal_common.das`) is the engine-side helper that carries all three -properties. +per arm, best-of across rounds.** A race is an instrument that compares two implementations. +Two separate runs measure how the machine changed between them as much as they measure the +arms; `race_pair_ms` (`../dasllama/dasllama_metal_common.das`) is the engine-side helper that +carries both properties, and `../REVIEW_GPU_RACE.md` binds the clock burn. **A diff that adds or changes a race arm offered as the reason to adopt a change proves the arm's output on its report line, by what the arm computes:** an arm producing no comparable @@ -25,7 +24,7 @@ output carries the literal token `timing-only`; an arm whose result is bit-ident baseline's prints the bit-exact compare over the sampled region - the set of output elements the run compares - on the report's "bit-exact vs ..." line; every other arm prints a bounded-difference compare (against the baseline arm or the CPU reference) plus the bound it -passed. Reduction order and fma contraction decide bit-identity, not declared precision. +passed. How the arm orders its sums, and whether its multiply-adds fuse, decide bit-identity - not the declared precision. **A diff that adds or changes a race also checks the race's baseline arm against a CPU reference.** The baseline arm is the arm running the implementation already in use. The @@ -38,40 +37,38 @@ is one whose arms attribute cost across stages instead of selecting between two implementations. The same header line also names what its arms attribute. Without that text a reader takes the sweep's arms for an adoption decision it never made. -**A new instrument that times a served turn - wherever it lives under this module - is a -defect: add a board cell to `../performance/gen_bench_records.das`, or a `lcpp_bench.das` -cell with its own `../PROFILE.md` section, instead.** A served turn is a whole -prefill-plus-decode run. A second instrument's numbers cannot be compared to any row the -board already carries. A tuner confirm that SPAWNS `lcpp_bench.das` and reads its standard -report line adds no instrument and conforms (`../ARCHITECTURE_MEASUREMENT.md` sec.2.21). +**A new instrument that puts its own clock around a served turn is a defect: add a board cell +to `../performance/gen_bench_records.das`, or a `lcpp_bench.das` cell with its own +`../PROFILE.md` section, instead.** A served turn is a whole prefill-plus-decode run. A +second instrument's numbers cannot be compared to any row the board already carries. **An out-of-process observer never measures what the benchmark process can measure about itself - that measurement goes inside the process instead.** An out-of-process observer is a script that measures a benchmark process from outside. -**A timing instrument this checklist governs never writes the wall time of a binary this -repository does not build - a third-party reference tool - into -`../performance/records/.json` or `../PERF_LEDGER.md`.** Board walls for such a tool -enter only through the reference cells of `../performance/gen_bench_records.das`, the cells -that time it on a board workload. +**Only the reference cells of `../performance/gen_bench_records.das` - the cells that time a +binary this repository does not build on a board workload - write such a binary's wall time +into `../performance/records/.json` or `../PERF_LEDGER.md`.** -**A file holding third-party walls outside those two - a reference leg's recovery file, a -pinned reference tsv an instrument reads back - is scratch: untracked, owned by exactly one -instrument, re-derivable from a documented command, and never an input to a board cell.** A +**A file holding a third-party wall outside `../performance/records/.json` and +`../PERF_LEDGER.md` - a reference leg's recovery file, a pinned reference tsv an instrument +reads back - is scratch: untracked, owned by exactly one instrument, re-derivable from a +command written where the owning instrument documents its flags, and never an input to a +board cell.** A tracked or shared copy of a third-party wall becomes a stale baseline nobody re-derives. -**A diff that adds or changes an instrument that prints a number formed by subtracting one -wall it also reports from another also prints both raw walls on that report line.** A plain -elapsed-time row (one clock pair, no attribution across stages) is not one. +**A diff that adds or changes an instrument that prints the difference of two walls also +prints both of those walls on that report line.** A plain elapsed-time row - one clock pair, +no attribution across stages - is not a difference. **A diff that changes what a board cell times ships before/after rows for each affected cell and corpus - or withdraws the affected rows and names the withdrawal and its reason in the PR body.** A board cell is a timed cell of the published results board: one `../performance/gen_bench_records.das` spawns, or a manual `lcpp_bench.das` cell with its own `../PROFILE.md` section. What a cell times changes when a change to its code, to its input -corpus, or to the pinned reference build - `DEFAULT_REF_SHA` in `setup_lcpp_ref.das`, the -targets it builds, a patch it applies, or a python reference leg's -`asr/requirements-*.txt` - moves the measured quantity. The re-mint or withdrawal lands in +corpus, or to the pinned reference build - anything that decides which reference binary or +reference environment the run measures, `DEFAULT_REF_SHA` in `setup_lcpp_ref.das` included - +moves the measured quantity. The re-mint or withdrawal lands in `../performance/records/.json`, the file the affected rows live in. **A diff that adds or changes a timing instrument makes it exit non-zero on a run that ends diff --git a/modules/dasLLAMA/harness/REVIEW.md b/modules/dasLLAMA/harness/REVIEW.md new file mode 100644 index 0000000000..178ba037ef --- /dev/null +++ b/modules/dasLLAMA/harness/REVIEW.md @@ -0,0 +1,12 @@ +# dasLLAMA harness Code Review Checklist + +**Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist.** Architecture +doc: `../ARCHITECTURE_MEASUREMENT.md`. Planned work: `../followup_metal.md` for anything about +the Metal backend, `../followup_vulkan.md` for anything about the Vulkan backend, +`../PERF_LEDGER.md` for any other performance followup, `../followup_general.md` for +everything else. + +**A diff that adds or moves a row in `tune_kernels.das`'s `benches` list - the name/function +pairs `main` declares up front - keeps `dot_q8q8_laneq4x4` last.** Running that bench pins one +matmul backend for the rest of the process, so a row after it is timed against the pinned +backend instead of the one it would have picked. diff --git a/modules/dasLLAMA/performance/REVIEW.md b/modules/dasLLAMA/performance/REVIEW.md index 3228fcfb5e..80e710ea2b 100644 --- a/modules/dasLLAMA/performance/REVIEW.md +++ b/modules/dasLLAMA/performance/REVIEW.md @@ -30,9 +30,9 @@ re-mint.** A reference row that carries no `sha` (the cli and python reference t pinned by its builder instead - the cli tools by that same ref pin's checkout, the python legs by `../benchmarks/asr/requirements-*.txt`. -**A diff that writes a records row, sidecar archive, or `defaults/` profile under this folder whose `provenance.dasllama_version` -differs from the `DASLLAMA_RELEASE` string (`../dasllama/dasllama_version.das`) is a defect - -re-mint.** For a sidecar with an `engine_sha`, read the value at that commit; a `defaults/` +**A diff that writes a records row, sidecar archive, or `defaults/` profile under this +folder whose `provenance.dasllama_version` differs from the `DASLLAMA_RELEASE` string +(`../dasllama/dasllama_version.das`) is a defect - re-mint.** For a sidecar with an `engine_sha`, read the value at that commit; a `defaults/` profile compares against the branch under review. **A diff that writes a row to `records/.json` mints that row from a board cell.** A board @@ -94,4 +94,3 @@ the row's canonical values** - the verify never reads the url of a file already **A diff that refreshes `last_known_good_sidecar.json` replaces it with one complete mint from the box its provenance names, at the current `DASLLAMA_RELEASE` - never a hand-edited copy.** - diff --git a/modules/dasLLAMA/tests/REVIEW.md b/modules/dasLLAMA/tests/REVIEW.md index 21d8cc2176..30526b968b 100644 --- a/modules/dasLLAMA/tests/REVIEW.md +++ b/modules/dasLLAMA/tests/REVIEW.md @@ -44,18 +44,16 @@ filter mechanics" section in the same change** - an arm the census does not name unreachable to whoever is choosing what to run. **Weakening a contract-pinning gate - dropping an assert, loosening a bound, or narrowing the -corpus or sweep it covers - is a defect.** The pinned set, each with what its gate holds: +corpus or sweep it covers - is a defect.** The pinned set, each with what it pins: `test_program_roots.das` (the `ROOT_DIRS` sweep, `options stack = 524288`, prefill intent); -`test_env_registry.das` (the `../ENVIRONMENT.md` knob contract); `test_model_specs.das` (the -model-set table, `../performance/model_specs.das`); the softcap/sink (`hass`)/span cells of -`test_metal_prefill_kernels.das` (what fails when `pf_p_weight` and `metal_attn_rowstat` -drift apart); `test_site_records.das` (the byte-compare of +`test_env_registry.das` (the `../ENVIRONMENT.md` knob contract); `test_model_specs.das` +(`../performance/model_specs.das`'s model-set table); `test_metal_prefill_kernels.das`'s +softcap, sink (`hass`) and span cells; `test_site_records.das` (the byte-compare of `site/files/dasllama/bench_records.json` (repo root) against a fresh `merge_site_records` -run); `test_exchange_schema.das` and `test_bench_records_schema.das` (the real +run); `test_exchange_schema.das` and `test_bench_records_schema.das` (the `write_bench_records` output, corpus sweeps included); `test_scheduler.das`'s media-stream -bypass check (a media stream attaches no cached hit at admit (`prefix_attach`) and donates no -pages at reap (`donate_stream`) - cache keys are token ids, and the KV past the splice does -not follow from them). +bypass check (no cached hit at `prefix_attach`, no donated pages at `donate_stream`). A gate +that pins a contract joins this list in the same change. **A test that silently vanishes on one platform is a defect, and so is a zero-assertion pass - a test passes or skips explicitly on every platform.** A cell that returns without asserting - @@ -80,10 +78,6 @@ carrier through that carrier's own loader.** Decoders load through `load_model_` (`../dasllama/dasllama_load.das`). Towers, embedders, and union carriers load through their family or carrier loaders. -**A function that gains a parameter, or a parameter that gains an accepted value, ships a -test for the new value.** The test feeds the new value and checks the result. It lives in -this folder, and it lands in the same change. "The model still runs" is not that test. - **A predicate whose value the BOX decides (a device capability, a policy default) and that therefore cannot differ between two runs on one machine is never tested through its own value; test it through the argv it gates or the mode it selects.** An argument-keyed pure @@ -97,8 +91,8 @@ change. A `[metal_dispatch]` declaration is not one of them. **A diff that changes a kernel's dispatch geometry - its grid divisor, threadgroup size, or threadgroup-memory length - updates every gate that hand-dispatches that kernel, in the same -change.** A hand-dispatched gate encodes the geometry itself, so a moved divisor leaves it -racing the wrong shape silently. +change.** A hand-dispatched gate encodes the geometry itself, so a moved divisor leaves the +gate dispatching the wrong shape with no error. **A diff that changes a kernel's kargs - the kernel-argument struct, or any buffer binding - updates every gate that hand-binds that kernel, in the same change.** A stale hand bind reads @@ -127,7 +121,7 @@ logits-tolerance form. Counting cells stay token-exact. **A kernel-unit cell - a model-less cell that dispatches one kernel class and asserts on its output - missing a compare against a CPU oracle that can witness the cell's property is a -defect.** (A census row dispatches without asserting on output, so it is not one.) +defect.** **A kernel-unit cell fills a GPU output buffer with a sentinel before every dispatch whose output it then reads.** An unprefilled output can pass by staying stale - the previous @@ -228,8 +222,9 @@ exact fixtures. **An embedding-parity cell that does not name its fixture, or does not log the measured maxdiff on green as well as red, is a defect.** -**A kernel-unit cell that dispatches a kernel class no cell dispatched before, or a new or -loosened tolerance bar, ships a control that reds it in the same change.** A control is a run of the same gate that must RED - a poisoned input, a poisoned +**A kernel-unit cell that dispatches a kernel class no cell dispatched before, and a cell +that adds or loosens a tolerance bar, each ship a control that reds them in the same +change.** A control is a run of the same gate that must RED - a poisoned input, a poisoned expectation, a disconnected mechanism, or a second independent lane; a gate's own reference is never its control. A bar nothing has ever exceeded is not known to discriminate, and a gate that reads state the same code path wrote can be a tautology - only the control proves either diff --git a/modules/dasLLVM/REVIEW.md b/modules/dasLLVM/REVIEW.md index 89ba549483..16fee954e1 100644 --- a/modules/dasLLVM/REVIEW.md +++ b/modules/dasLLVM/REVIEW.md @@ -11,7 +11,7 @@ `get_architecture_name()`, or `cpu_supports()` - runs the module-owned suite on a machine matching that condition.** -- **A diff that adds or changes a branch on the target triple names in its PR body the +- **A diff that adds or changes a branch on the target triple records in its PR body the cross-compile (`write_exe`) for that target that exercised the behavior.** The suite runs on the host, so a target-triple branch is checked only by the artifact built for that target. @@ -31,19 +31,20 @@ `jit_env_salt` (`daslib/llvm_jit_run.das`), never directly into either JIT key - the DLL key or the split-obj key (`ARCHITECTURE.md` sec.2)** - salt feeds both keys, and a config folded into one but not the other links stale objects. Inputs that vary per function set - (AOT hashes) are key material, not salt. + (AOT hashes) fold into the key directly, not through the salt. -- **A macro under `daslib/` that reads a file at compile time registers it with +- **A macro under this module's `daslib/` that reads a file at compile time registers it with `add_module_cache_dependency` before any early return, in the same change** (`ARCHITECTURE.md` sec.5). An unpinned compile-time file read serves stale macro output from the module cache until an unrelated source file changes - silently. -- **A change to a `[tune]`-family annotation is reviewed with `skills/tune.md`.** A change to - the framework itself - `daslib/llvm_tune.das` or its tests - is reviewed with - `skills/internal/llvm_tune_internals.md`. +- **A change to a `[tune]`-family annotation is reviewed with `skills/tune.md`.** -- **A diff that adds a new top-level section, or a new value shape inside one, to the tune - sidecar (`.tune.json`, written by `daslib/llvm_tune.das`) updates +- **A change to the tune framework - `daslib/llvm_tune.das` or its tests - is reviewed with + `skills/internal/llvm_tune_internals.md`.** + +- **A diff that adds a top-level section to the tune sidecar (`.tune.json`, written by + `daslib/llvm_tune.das`), or a new key or value type inside an existing section, updates `modules/dasLLAMA/dasllama/dasllama_exchange_schema.das` in the same change and keeps `modules/dasLLAMA/tests/test_exchange_schema.das` green** - the validator allow-lists sections, so a section it does not know fails every newly minted sidecar at submission, and @@ -56,11 +57,11 @@ annotation arguments - is a declaration, not an override. - **A diff that adds an override knob, or gives one a new effect, also logs at least one line - naming the knob where it takes effect.** An override that is set but changes nothing needs - no line; a diff that only exposes the knob defers the line to its consumer, in the same - change. + naming the knob where it takes effect.** A diff that only exposes the knob puts the line at + the consumer instead, in that same diff. - **Never read an environment variable by a computed name - `get_env_variable(expr)` / - `has_env_variable(expr)` outside `daslib/llvm_env.das`. Spell the name as a literal through - the declared forms, or declare the knob, instead** (the literal-name forms are + `has_env_variable(expr)` outside `daslib/llvm_env.das`. Spell the name as a literal in + `env_value_of("NAME")` / `env_is_set("NAME")`, or declare it as an `[EnvConfig]` field in + `daslib/llvm_env.das` and read the `g_env_*` field, instead** (the literal-name forms are scanner-enforced by `tests/llvm_env_registry.das`; weakening that test is a defect). diff --git a/plans/dasllama_review_grooming.md b/plans/dasllama_review_grooming.md new file mode 100644 index 0000000000..0d495b1c52 --- /dev/null +++ b/plans/dasllama_review_grooming.md @@ -0,0 +1,89 @@ +# dasLLAMA REVIEW.md grooming backlog (dragon findings, 2026-09-01) + +A full-document dragon pass over `modules/dasLLAMA/REVIEW.md` during the metal-kq-race PR +surfaced findings on rules that PR did not touch. The diff-scoped ones were fixed in that PR +(the harness routing duplicate deleted, the bench-list rule moved to `harness/REVIEW.md`, the +reference-build carve complement re-paired). The rest is a grooming pass of its own - one +sitting, then a fresh dragon: + +- SPLIT: the measured-number/servable-capability routing head; the `DASLLAMA_RELEASE` rule + (two triggers); the tokenizer `--tok` rule (the superlinear verdict is its own rule); the + override-announce rule (four criteria + a five-line definition - the longest in the file). +- WRONG DOCUMENT: the uniform/kargs dispatch-value ban -> `REVIEW_GPU.md`; the GPU-hook + CPU-form rule -> `REVIEW_GPU.md`. +- MERGE: the two consecutive sidecar-exchange routing rules. +- DUPLICATE: the routed-file meta-rule (restates `REVIEW_COMMON.md`); the restore-check + weakening clause (the general `REVIEW.das` weakening rule covers it). +- AMBIGUITY/undefined terms: "stocked model file", "a stated decision" (name the PR body), + "records a run" (name the PR body), "beside one that has its own sec.1 charter line", + "escaped file", "a parity or oracle rail" (define in place). +- REMOVE EXCEPTIONS: the STYLE037/038 ledger rule's dedup carve-out (move the boundary into + the trigger). +- STATEMENT: the facade-def TAUGHT rule (keep only the overload residue the + `check_tutorial_floor` gate cannot see). +- RECOMMEND LINT (gate candidates): bench-list-last as a `REVIEW.das` parse; + `[EnvConfig]`-struct <-> `env_markdown()` pairing; `requires=` names vs + `TUNE_KNOWN_FEATURES`; team-lane global reachability walk. + +## tests/REVIEW.md (same dragon pass) + +Diff-scoped fixes landed in the PR; the rest, one sitting: the suite-listing exception +absorbed into its trigger; the disarmed-file rule split; direct-dastest rule trimmed of its +statement tail; the -jit rule split from the compile-only-lane duty; the CMakeLists ban gains +its replacement clause; "re-lanes" defined; the box-decided-predicate and lane-pin carve +sentences dissolved into triggers; "silently vanishes"/"laundered"/"its grade" idioms +plained; the 70B batch-parity rule re-keyed on DASLLAMA_PARITY_FULL; the in-place-cell and +stocked-artifact ambiguities spelled; the establish-and-restore rule's ARCHITECTURE_GPU +sec.1.5 cross-cite and vulkan carve removed per the ledger handshake. STRUCTURAL: rename +tests/CLAUDE.md -> tests/ARCHITECTURE.md so the checklist's eight section cites stop being +rule-doc-to-rule-doc. RECOMMEND LINT (tests/REVIEW.das candidates): suite-listing glob vs +run.das lists; arm-name census vs CLAUDE.md's arm roster; the out-of-folder [test] ledger; +the CMakeLists registration ban. + +## REVIEW_EXCHANGE.md (same dragon pass) + +Diff-scoped fix landed (the moved consent rule's `../` path re-based). Backlog: drop the +cross-folder gate cite from the second-HTTP-path ban; the weakening rule reduces to the +test_exchange_client residue with "submission strip" defined in place; the tune-boot fallback +rule re-worded as its duty ("falls back to the local sidecar and the tune winners built into +the exe"); the consent-question trigger spelled to reading (b). RECOMMEND LINT: a REVIEW.das +byte-identity check over the three consent-notice copies; the strip/may_contact reachability +check over dasllama_exchange.das. + +## performance/REVIEW.md (same dragon pass) + +Diff-scoped fixes landed (the provenance head re-wrapped, the trailing blank dropped). +Backlog: the second-validator ban's trailing gate-cite sentence merges into a widened +"Weakening any of `REVIEW.das`'s checks is a defect" head, with the re-mint fix path split +out and "quiet, session-free box" spelled as `noise: ok` + `hardware.remote_desktop: off`; +`das` row defined at first use ("a row whose `engine` is `das`"); the no-`sha` reference-row +body re-worded as its duty; the board-cell rule's restating clause cut; the model-file +provenance rule compressed to the two-sentence form; the companion-artifact statement tail +reduced to the test_model_specs weakening residue; the `--tune` flag re-homed off +`fetch_models.das`. RECOMMEND LINT: widen make-pr `gate_stamp_reach`'s glob to every +checked-in `.json` under `*/performance/`; a `check_sidecar_archive` in performance/REVIEW.das +(filename sha12 == content hash; every row's `tune_sha` resolves). + +## REVIEW_GPU_RACE.md (same dragon pass, all verdicts APPLIED in the PR) + +RECOMMEND LINT residue only: extend `check_race_bind_numbers`'s walk beyond +`modules/dasLLAMA/dasllama` to `benchmarks/`, `harness/`, `performance/` (deletes the +"outside `dasllama/`" arm of the hand-verified rule); a `tests/` cell calling `race_pair_ms` +with counting blocks proving both arms ran before the first timed rep (reduces the +burn-weakening rule to the test residue). + +## daslib/REVIEW.md (same dragon pass) + +Diff-scoped fixes landed (the PERF023 doubled locator + all three wrap artifacts; the emit +entry-point gloss cut with its rewrap). Backlog, one sitting: the dedup-key rule names +`report_key`; the architecture-doc-entry rule DELETED (restates ARCHITECTURE_COMMON.md's +last rule); "collapse suggestion" defined inline; the residual-oracle criteria promoted to +the head; the ExprVar-string ban gains its replacement (`hash(expr.name)`); the Template +delete rule re-keyed as "Never add a `delete` for ... declared `inscope`"; the ast_verify +naming rule SPLIT (the post-infer placement duty its own rule); the `[|> name]` structure +macro rule's copula head promoted to duties. RECOMMEND LINT (utils/lint/REVIEW.das cells): +RULE_MODULES membership census over daslib emitters; the reverse census (fixture/alias/rst +id -> still-emitting module); message-literal starts with a registered id + colon. And one +style_lint candidate: raw string-loop char indexing a byte-class table without +`uint(uint8(ch))`. At 262/300 lines - when LINT027 fires, the split seams are the lint / +CppAot-emit / flatten_opt clusters. From e3f210cf01fbf72cb55a8767773957d401b2b47d Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 10:15:11 -0700 Subject: [PATCH 28/35] the race checklist takes its auditor's verdicts, and the twin racer stops double-burning The review-md auditor read the new REVIEW_GPU_RACE.md against the diff it binds and found the diff clean but the rules leaky. Five repairs: the mis-numbering criterion now carries the gate's real property (bound at a number the class does not declare for that field - the old wording missed an undeclared number outright); the crown rules' trigger is decidable (mints a runtime crown or a tune-sidecar row - the serialized per-format rig is sanctioned in ARCHITECTURE_MEASUREMENT.md sec.2.21 as the instrument's own probe shape); the port-deletion duty is keyed on the ported variant's own code, so a lab row that dispatches the shipped class's generated source stops reading as a deletion target; the invisible-arm PR attestation widens to kargs word layout (the hand-mirrored KqMvArgs words were exactly the unattested half); the scaffolding rule states the shared-by-two-sites property instead of enumerating three names. The harness benches-last trigger widens to runtime mutation of the list. One code fix the audit surfaced: race_gemv_f4_twin kept its hand-rolled 150ms base-only burn from before race_pair_ms grew a burn phase - the phase burns both arms, so the local loop was a slower, one-sided duplicate. Deleted; decode parity re-run green at this tree. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014mq9AKyoAwvD5T1cigTnRw --- modules/dasLLAMA/ARCHITECTURE_MEASUREMENT.md | 5 +- modules/dasLLAMA/REVIEW_GPU_RACE.md | 50 +++++++++---------- .../dasllama/dasllama_metal_kernels.das | 20 -------- modules/dasLLAMA/harness/REVIEW.md | 6 +-- plans/dasllama_review_grooming.md | 18 +++++++ 5 files changed, 50 insertions(+), 49 deletions(-) diff --git a/modules/dasLLAMA/ARCHITECTURE_MEASUREMENT.md b/modules/dasLLAMA/ARCHITECTURE_MEASUREMENT.md index 8be2087a2a..2a061e288c 100644 --- a/modules/dasLLAMA/ARCHITECTURE_MEASUREMENT.md +++ b/modules/dasLLAMA/ARCHITECTURE_MEASUREMENT.md @@ -152,4 +152,7 @@ hint. The per-format isolated rig is `benchmarks/matmul/bench_metal_kq_race.das`: synthetic planes, no model, no tuner, every arm gated against a CPU plane-dequant oracle before it is timed, and -`--burn-ms` (default 150) spent on GPU work before each cell's first timed round. +`--burn-ms` (default 150) spent on GPU work before each cell's first timed round. Its cells +chain every dispatch through ONE shared output buffer on purpose - the serialized regime is +the instrument's probe shape, imitating the reference tool it is compared against - and its +numbers reach the engine only through a human porting decision, never a minted crown. diff --git a/modules/dasLLAMA/REVIEW_GPU_RACE.md b/modules/dasLLAMA/REVIEW_GPU_RACE.md index 181c124f42..a2892bbba7 100644 --- a/modules/dasLLAMA/REVIEW_GPU_RACE.md +++ b/modules/dasLLAMA/REVIEW_GPU_RACE.md @@ -1,7 +1,7 @@ -# dasLLAMA GPU Race / Lab / Hand-Binding Review Checklist +# dasLLAMA GPU Race Code Review Checklist **Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist.** Architecture -docs: `ARCHITECTURE_GPU.md`, `ARCHITECTURE_MEASUREMENT.md`. +docs: `ARCHITECTURE_GPU.md`, `ARCHITECTURE_MEASUREMENT.md`. Planned work: `followup_metal.md`. **Routed from `REVIEW_GPU.md`: a diff touching a GPU kernel A/B race, knockout, or hand-binding arm - or changing a kernel class such an arm mirrors (binding numbers, kargs @@ -10,15 +10,15 @@ puts it, applies this list together with `REVIEW_GPU.md`.** **A hand-binding arm - a race or knockout timing arm (a knockout attributes cost across stages instead of selecting between implementations) that lists its bindings by number -instead of dispatching through the `enc_*` builder - that binds a field at another field's -declared number is a defect.** A mis-numbered arm dispatches, reads the wrong buffer, and its -timing selects the wrong kernel silently. +instead of dispatching through the `enc_*` builder - that binds a field at a number the +class does not declare for that field is a defect.** A mis-numbered arm dispatches, reads +the wrong buffer, and its timing selects the wrong kernel silently. **A hand-binding arm outside `dasllama/`, or one whose pipeline source or threadgroup-memory size arrives as a function parameter rather than a literal global, states in the PR that its -bind numbers were verified by hand against the class declaration.** The `REVIEW.das` gate -`check_race_bind_numbers` cannot read those arms, and an arm nothing checked is where a -mis-numbered bind reaches the board. +bind numbers and kargs word layout were verified by hand against the class declaration.** +The `REVIEW.das` gate `check_race_bind_numbers` cannot read those arms, and an arm nothing +checked is where a mis-numbered bind reaches the board. **A diff that changes anything a hand-binding arm must mirror to dispatch a kernel - binding numbers, kargs (kernel-argument struct) layout, threadgroup memory, grid or threadgroup @@ -26,37 +26,37 @@ geometry - fixes or deletes every such arm in the same change.** An arm left dis stale geometry measures the wrong kernel silently. **Race code inside the engine (`dasllama/`) sits in the file that owns its kernel family and -never in another engine file; the shared scaffolding (`race_buf`, `race_envelope_ok`, -`race_pair_ms`) sits in `dasllama/dasllama__common.das`.** Race code is the base-vs-twin -check that times both kernels on one queue and compares their outputs. +never in another engine file; every piece of scaffolding two race sites share sits in +`dasllama/dasllama__common.das`.** Race code is the base-vs-twin check that times both +kernels on one queue and compares their outputs. **Race code sizes its operands at a real model shape - never at a small square slab.** A slab small enough to sit in cache ranks the kernels by an effect production never sees, and the race then picks the slower kernel. -**A kernel A/B race arm whose verdict alone selects the kernel the engine ships - wherever -the race lives - binds a DIFFERENT output buffer for consecutive dispatches of its chain, -never one shared output.** One shared output serializes the chain on its write-after-read -hazard while the served graph overlaps consecutive dispatches, so the race ranks the arms on -a shape production never runs. +**A kernel A/B race arm that mints a runtime crown or a tune-sidecar row binds a DIFFERENT +output buffer for consecutive dispatches of its chain, never one shared output.** One shared +output serializes the chain on its write-after-read hazard while the served graph overlaps +consecutive dispatches, so the race ranks the arms on a shape production never runs. **Weakening the burn phase of `race_pair_ms` (`dasllama/dasllama_metal_common.das`) - the GPU work it runs on both arms before the first timed round - is a defect.** -**A race arm that selects the kernel the engine ships and does not go through `race_pair_ms` -burns GPU work on its own arms before its first timed round.** The arms alternate, so a first -round on a cold GPU clock charges the clock's ramp-up to whichever arm ran first. +**A race arm that mints a runtime crown or a tune-sidecar row and does not go through +`race_pair_ms` burns GPU work on its own arms before its first timed round.** The arms +alternate, so a first round on a cold GPU clock charges the clock's ramp-up to whichever arm +ran first. -**A timed encoder in a race arm that selects the kernel the engine ships issues its +**A timed encoder in a race arm that mints a runtime crown or a tune-sidecar row issues its dispatches back to back.** An encoder that leaves gaps between its dispatches times an idle clock. **A diff that ports an A/B lab's winning variant into a kernel deletes, in the same change, -every arm of that lab that `ARCHITECTURE_GPU.md` sec.2.2b does not ledger as a retained -reference, together with the variant class and any `*_variants.das` code that exists only for -that arm.** An A/B lab is a timing script whose output selects between implementations of the -same compute; a decided arm that outlives its decision degrades into an unmaintained -duplicate of the kernel it seeded. +that variant's class and any `*_variants.das` code that exists only for it; an arm that +survives the port dispatches the shipped kernel class's generated source, or is ledgered as +a retained reference in `ARCHITECTURE_GPU.md` sec.2.2b.** An A/B lab is a timing script +whose output selects between implementations of the same compute; a decided arm that +outlives its decision degrades into an unmaintained duplicate of the kernel it seeded. **A diff that leaves an A/B lab with no undecided arm deletes the lab's driver and its remaining arm in the same change.** diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das index 1d191f5fd1..9faaec9109 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_kernels.das @@ -4800,26 +4800,6 @@ def private race_gemv_f4_twin(dev : MetalDevice?; queue : MetalCommandQueue?; fa var bd = race_uniform_u32(dev, uint(ndim)) let doff = uint64(nblk * 16) let tg = uint3(64u, 1u, 1u) - var burn_ms = 0.0lf // clock: control — GPU burn budget, ramps the clock governor - var burn_err = "" - while (burn_ms < 150.0lf) { - var g = 0.0lf - let okb = with_compute_encoder_timed(queue, burn_err, g) $(enc) { - kn_pipeline(enc, base_pso) - kn_buffer(enc, bs, doff, 0) - kn_buffer(enc, bs, 0ul, 1) - kn_buffer(enc, bq, 0ul, 2) - kn_buffer(enc, bxa, 0ul, 3) - kn_buffer(enc, bn, 0ul, 5) - kn_buffer(enc, bd, 0ul, 6) - for (r in range(32)) { - kn_buffer(enc, (r & 1) == 0 ? by_base : by_base2, 0ul, 4) - kn_dispatch(enc, uint3(uint(ndim / base_gdiv), 1u, 1u), tg) - } - } - break if (!okb) - burn_ms += g - } race_pair_ms(queue, 5, res.base_ms, res.twin_ms, $(enc) { kn_pipeline(enc, base_pso) kn_buffer(enc, bs, doff, 0) diff --git a/modules/dasLLAMA/harness/REVIEW.md b/modules/dasLLAMA/harness/REVIEW.md index 178ba037ef..c387806242 100644 --- a/modules/dasLLAMA/harness/REVIEW.md +++ b/modules/dasLLAMA/harness/REVIEW.md @@ -7,6 +7,6 @@ the Metal backend, `../followup_vulkan.md` for anything about the Vulkan backend everything else. **A diff that adds or moves a row in `tune_kernels.das`'s `benches` list - the name/function -pairs `main` declares up front - keeps `dot_q8q8_laneq4x4` last.** Running that bench pins one -matmul backend for the rest of the process, so a row after it is timed against the pinned -backend instead of the one it would have picked. +pairs `main` declares up front, or any later mutation of that list - keeps `dot_q8q8_laneq4x4` +last.** Running that bench pins one matmul backend for the rest of the process, so a row after +it is timed against the pinned backend instead of the one it would have picked. diff --git a/plans/dasllama_review_grooming.md b/plans/dasllama_review_grooming.md index 0d495b1c52..58af4154cf 100644 --- a/plans/dasllama_review_grooming.md +++ b/plans/dasllama_review_grooming.md @@ -87,3 +87,21 @@ id -> still-emitting module); message-literal starts with a registered id + colo style_lint candidate: raw string-loop char indexing a byte-class table without `uint(uint8(ch))`. At 262/300 lines - when LINT027 fires, the split seams are the lint / CppAot-emit / flatten_opt clusters. + +## harness/REVIEW.md (review-md auditor, same round) + +Trigger repair landed in the PR (the benches-last rule now also catches runtime mutation of +the list). RECOMMEND LINT: a `harness/REVIEW.das` asserting the last `benches` row is +`dot_q8q8_laneq4x4`, retiring the prose rule per REVIEW_COMMON's automation clause. + +## REVIEW_GPU_RACE.md round 2 (review-md auditor, same round; verdicts APPLIED in the PR) + +Applied: mis-numbering criterion re-keyed to the gate's property (a number the class does +not declare for that field); scaffolding rule states the shared-by-two-sites property; the +crown rules' trigger re-keyed decidable ("mints a runtime crown or a tune-sidecar row" - +the serialized per-format rig is sanctioned in ARCHITECTURE_MEASUREMENT.md sec.2.21); the +port-deletion duty keyed on the variant's own code; the invisible-arm PR attestation widened +to kargs word layout; title + Planned-work pointer normalized. Code: race_gemv_f4_twin's +local 150ms burn deleted (race_pair_ms's burn phase covers both arms). Backlog: sec.2.2b is +titled "Tensor-GEMM shapes that measured out" while two rules cite it as the retained- +reference ledger - move that ledger to its own anchor or retitle. From 0380213cd6a2a250b7fa95cad1e548149e4079dc Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 10:20:07 -0700 Subject: [PATCH 29/35] add_module_cache_dependency gets its stdlib page - the ast module is C++-side, so the description is handmade Sphinx clean, zero warnings. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014mq9AKyoAwvD5T1cigTnRw --- ...nction-ast-add_module_cache_dependency-0x6bab73893c4ca81f.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 doc/source/stdlib/handmade/function-ast-add_module_cache_dependency-0x6bab73893c4ca81f.rst diff --git a/doc/source/stdlib/handmade/function-ast-add_module_cache_dependency-0x6bab73893c4ca81f.rst b/doc/source/stdlib/handmade/function-ast-add_module_cache_dependency-0x6bab73893c4ca81f.rst new file mode 100644 index 0000000000..a9806b5515 --- /dev/null +++ b/doc/source/stdlib/handmade/function-ast-add_module_cache_dependency-0x6bab73893c4ca81f.rst @@ -0,0 +1 @@ +Registers the file at ``path`` as a compile-time dependency of the module being compiled, so the module cache re-parses the module when that file's size or content hash changes. A macro that reads a file at compile time (a tune sidecar, a config, a generated table) calls this before acting on the contents; outside compilation the call is a no-op. From ee085913898459e0d728302cd9b4c25947f1af92 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 10:28:35 -0700 Subject: [PATCH 30/35] woodpecker: drive-relative Windows paths reach add_module_cache_dependency unabsolutized C:foo is drive-relative, not absolute - the second-character-colon test stored it raw, and a validating reader on a different per-drive cwd then stats a different file, or the same wrong one on both sides, and the dep never invalidates. Windows now resolves through normalizeFileName (GetFullPathName underneath, which owns the per-drive cwd rules); POSIX keeps the plain cwd-join. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014mq9AKyoAwvD5T1cigTnRw --- src/builtin/module_builtin_ast.cpp | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/builtin/module_builtin_ast.cpp b/src/builtin/module_builtin_ast.cpp index bcbc16a0c4..ecf072a08e 100644 --- a/src/builtin/module_builtin_ast.cpp +++ b/src/builtin/module_builtin_ast.cpp @@ -1,8 +1,7 @@ #include "daScript/misc/platform.h" -#if defined(_MSC_VER) - #include - #define das_dep_getcwd _getcwd +#if defined(_WIN32) + #include "daScript/misc/sysos.h" #else #include #define das_dep_getcwd getcwd @@ -277,13 +276,19 @@ namespace das { // path there would stat the wrong (or no) file string fullPath = path; #if !defined(DAS_NO_FILEIO) - bool isAbs = path[0]=='/' || path[0]=='\\' || (path[0] && path[1]==':'); - if ( !isAbs ) { +#if defined(_WIN32) + // a drive-relative spelling (C:foo) resolves against that drive's own cwd, which a + // hand-rolled cwd-join cannot reproduce - GetFullPathName (via normalizeFileName) can + string norm = normalizeFileName(path); + if ( !norm.empty() ) fullPath = norm; +#else + if ( path[0]!='/' ) { char cwd[4096]; if ( das_dep_getcwd(cwd, sizeof(cwd)) ) { fullPath = string(cwd) + "/" + path; } } +#endif #endif int64_t size = -1; uint64_t hash = 0; From f7e876e1a2c383c8e66505e72edf9ec0b444308d Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 10:31:38 -0700 Subject: [PATCH 31/35] woodpecker round 2: both burn loops get a round cap A device whose GPU timestamps read 0.0 for short dispatches would spin the ~150ms burn budget forever - race_pair_ms at every tensor race, the kq lab at every cell. The budget condition gains a 2048-round bound; on a healthy device the loop exits on budget long before the cap. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014mq9AKyoAwvD5T1cigTnRw --- modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das | 4 +++- modules/dasLLAMA/dasllama/dasllama_metal_common.das | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das b/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das index 8fac563e32..0b538f9f54 100644 --- a/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das +++ b/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das @@ -359,7 +359,9 @@ def private race_cell(queue, dev; f : KqRaceFmt; tier, shname : string; n, d : i } } var burned = 0.0lf - while (burned < double(args.burn_ms) * 1000.0lf) { + var burn_rounds = 0 //! round cap: a device whose timestamps read 0.0 must not spin the budget loop forever + while (burned < double(args.burn_ms) * 1000.0lf && burn_rounds < 2048) { + burn_rounds++ var did = false for (arm, lv, pso, by in arms, live, psos, bys) { continue if (!lv) diff --git a/modules/dasLLAMA/dasllama/dasllama_metal_common.das b/modules/dasLLAMA/dasllama/dasllama_metal_common.das index 62b2af6bd2..1e6b6305e1 100644 --- a/modules/dasLLAMA/dasllama/dasllama_metal_common.das +++ b/modules/dasLLAMA/dasllama/dasllama_metal_common.das @@ -2293,7 +2293,8 @@ def race_pair_ms(queue : MetalCommandQueue?; reps : int; var base_ms, twin_ms : var best_b = -1.0lf var best_t = -1.0lf var burn_ms = 0.0lf // clock: control — GPU burn budget, ramps the clock governor before round 1 - while (burn_ms < 150.0lf) { + var burn_rounds = 0 //! round cap: a device whose timestamps read 0.0 must not spin the budget loop forever + while (burn_ms < 150.0lf && burn_rounds < 2048) { var gb = 0.0lf let okb = with_compute_encoder_timed(queue, err, gb) $(enc) { invoke(base_blk, enc) @@ -2304,6 +2305,7 @@ def race_pair_ms(queue : MetalCommandQueue?; reps : int; var base_ms, twin_ms : } return if (!okb || !okt) burn_ms += gb + gt + burn_rounds++ } for (r in range(reps + 1)) { var g1 = 0.0lf From a773f150d6c09da67870937bc7e81bac9e9cc220 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 11:13:47 -0700 Subject: [PATCH 32/35] the race lab compiles on the metal-less lint lane - its typed helpers go generic The lab's ?das_metal requires were already conditional, but five fully-typed functions (the three arm builders, the oracle, main_apple) resolved unconditionally, so the linux lint mirror - where dasMetal's generated msl globals and _metal_kernel_common don't exist - failed the compile. Untyped-parameter generics instantiate only from call sites, and every call traces to main's das_metal static_if, so the metal-less lane never resolves them: the same shape bench_metal_kq_mm_lab already uses. Targeted lint gate green on all three rails; metal smoke cell unchanged. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014mq9AKyoAwvD5T1cigTnRw --- .../benchmarks/matmul/bench_metal_kq_race.das | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das b/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das index 0b538f9f54..68727cb9b1 100644 --- a/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das +++ b/modules/dasLLAMA/benchmarks/matmul/bench_metal_kq_race.das @@ -117,7 +117,7 @@ def private want(filter, name : string) : bool { return false } -def private gemv_arms(f : KqRaceFmt) : array { // candidate GEMV spellings register here, after prod +def private gemv_arms(f) : array { // candidate GEMV spellings register here, after prod; generic on purpose - instantiates only under main's das_metal guard, so the metal-less lint lane never resolves the msl globals var arms : array if (f.fmt == 4) { arms |> push(KqRaceArm(name = "prod", src = metal_kq_gemv_k4_msl, entry = metal_kq_gemv_k4_msl_entry, fastmath = metal_kq_gemv_k4_msl_fastmath, tgmem = 0ul, gdiv = f.gemv_gdiv)) @@ -150,7 +150,7 @@ def private gemv_arms(f : KqRaceFmt) : array { // candidate GEMV sp return <- arms } -def private mvb_arms(f : KqRaceFmt) : array { // the B2 batch-decode arm per grid format; candidate spellings register here, after prod +def private mvb_arms(f) : array { // the B2 batch-decode arm per grid format; candidate spellings register here, after prod; generic on purpose (see gemv_arms) var arms : array if (f.fmt == 33) { arms |> push(KqRaceArm(name = "prod", src = MetalKqMvB2Iq3s_metal_kq_mv_iq3s_msl, entry = MetalKqMvB2Iq3s_metal_kq_mv_iq3s_msl_entry, fastmath = MetalKqMvB2Iq3s_metal_kq_mv_iq3s_msl_fastmath, tgmem = 0ul, gdiv = 8)) @@ -166,7 +166,7 @@ def private mvb_arms(f : KqRaceFmt) : array { // the B2 batch-decod return <- arms } -def private mm_arms(f : KqRaceFmt) : array { // candidate mul_mm spellings register here, after prod +def private mm_arms(f) : array { // candidate mul_mm spellings register here, after prod; generic on purpose (see gemv_arms) var arms : array if (f.fmt == 4) { arms |> push(KqRaceArm(name = "prod", src = MetalKqMulMmK4_metal_kq_mulmm_msl, entry = MetalKqMulMmK4_metal_kq_mulmm_msl_entry, fastmath = MetalKqMulMmK4_metal_kq_mulmm_msl_fastmath, tgmem = MetalKqMulMmK4_metal_kq_mulmm_msl_tgmem, gdiv = 0)) @@ -273,8 +273,8 @@ def private time_arm_us(queue; pso; arm : KqRaceArm; bks, bkq, bx, by, b5, b6; s } //! the whole-output dot-envelope oracle the tests trust: want/env from kq_row_ref + double dot -def private build_oracle(fmt, n, d, nsb : int; kq, ks : array; xf : array; xstride, xrows : int; - var want : array; var env : array) { +def private build_oracle(fmt; n, d, nsb : int; kq, ks : array; xf : array; xstride, xrows : int; + var want : array; var env : array) { // fmt untyped on purpose (see gemv_arms) - kq_row_ref rides the ?das_metal require var wrow : array wrow |> resize(n) let nb = n / 256 @@ -446,7 +446,7 @@ def private race_cell(queue, dev; f : KqRaceFmt; tier, shname : string; n, d : i return rows } -def private main_apple(args : Args) { +def private main_apple(args) { // generic on purpose (see gemv_arms) var dev = metal_create_system_default_device() if (dev == null) { panic("no Metal device on this box") From b1616e7fa4c331f2a15fa6e39d1b7c9611a5958e Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 11:25:34 -0700 Subject: [PATCH 33/35] review round 1: the confirm rail's scratch moves to unique temp files Copilot caught the confirm rail writing its manifest and capture file at fixed names into das_root - an interrupted run strands debris in what can be an installed tree, and two concurrent tuner runs would mint a crown against each other's manifest. Both files now come from create_temp_file_result. The child-env spellings pick up the quoted house form (set "VAR=..."&& / single-quoted POSIX prefix) here and at the two test spawn sites. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014mq9AKyoAwvD5T1cigTnRw --- modules/dasLLAMA/harness/tune_kernels.das | 21 ++++++++++++++----- modules/dasLLVM/tests/test_tuned.das | 8 +++---- .../test_macro_dep_invalidate.das | 2 +- 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/modules/dasLLAMA/harness/tune_kernels.das b/modules/dasLLAMA/harness/tune_kernels.das index 09955cf1c2..f469b61b65 100644 --- a/modules/dasLLAMA/harness/tune_kernels.das +++ b/modules/dasLLAMA/harness/tune_kernels.das @@ -2396,15 +2396,26 @@ def private confirm_tg_run(model : string; wins : array; extra : string) if (!empty(extra)) { crowns = empty(crowns) ? extra : "{crowns},{extra}" } - let tmpm = "{get_das_root()}/_tune_confirm_decode_manifest.json" + let tmpmRes = create_temp_file_result("tune_confirm_manifest_", ".json") + if (tmpmRes is error) { + print("confirm: cannot create a temp manifest: {tmpmRes as error}\n") + return 0.0lf + } + let tmpm = tmpmRes as value if (!fwrite(tmpm, "\{\"kernels\": \{\}, \"runtime\": \{\"metal_tensor\": \"{crowns}\"\}, \"provenance\": \{\"box\": \"{tune_box_identity()}\"\}\}")) { - print("confirm: cannot write {tmpm} - is das_root writable?\n") + print("confirm: cannot write {tmpm}\n") return 0.0lf } let envp = (get_platform_name() == "windows" - ? "set DAS_TUNE_MANIFEST={tmpm}&& " - : "DAS_TUNE_MANIFEST={tmpm} ") - let tmpo = "{get_das_root()}/_tune_confirm_decode_out.txt" + ? "set \"DAS_TUNE_MANIFEST={tmpm}\"&& " + : "DAS_TUNE_MANIFEST='{tmpm}' ") + let tmpoRes = create_temp_file_result("tune_confirm_out_", ".txt") + if (tmpoRes is error) { + remove(tmpm) + print("confirm: cannot create a temp capture file: {tmpoRes as error}\n") + return 0.0lf + } + let tmpo = tmpoRes as value let cmd = "{envp}\"{get_das_exe()}\" -jit \"{get_das_root()}/modules/dasLLAMA/benchmarks/lcpp_bench.das\" -dasroot \"{get_das_root()}\" -- --for-debug-purposes -m \"{model}\" --ngl 99 -p 32 -n 128 -r 3 > \"{tmpo}\" 2>&1" unsafe(popen(cmd) $(f) { if (f != null) { diff --git a/modules/dasLLVM/tests/test_tuned.das b/modules/dasLLVM/tests/test_tuned.das index 1fb0371009..5f30cc117f 100644 --- a/modules/dasLLVM/tests/test_tuned.das +++ b/modules/dasLLVM/tests/test_tuned.das @@ -60,8 +60,8 @@ def test_tuned_stale_sidecar_perm(t : T?) { fwrite(manifest, "\{\"kernels\": \{\"stale_probe_dot\": \"no_such_perm\"\}, \"provenance\": \{\"box\": \"{tune_box_identity()}\"\}\}") let client = "{get_this_module_dir()}/tuned_stale_perm_client.das" let env = (get_platform_name() == "windows" - ? "set DAS_TUNE_MANIFEST={manifest}&& " - : "DAS_TUNE_MANIFEST={manifest} ") + ? "set \"DAS_TUNE_MANIFEST={manifest}\"&& " + : "DAS_TUNE_MANIFEST='{manifest}' ") let cmd = "{env}\"{bin}\" \"{client}\"" var outp = "" var rc : int @@ -116,8 +116,8 @@ def test_tuned_sidecar_invalidates_module_cache(t : T?) { fwrite(manifest, "\{\"kernels\": \{\}, \"provenance\": \{\"box\": \"{tune_box_identity()}\"\}\}") let client = "{get_this_module_dir()}/tuned_stale_perm_client.das" let env = (get_platform_name() == "windows" - ? "set DAS_TUNE_MANIFEST={manifest}&& " - : "DAS_TUNE_MANIFEST={manifest} ") + ? "set \"DAS_TUNE_MANIFEST={manifest}\"&& " + : "DAS_TUNE_MANIFEST='{manifest}' ") let cmd = "{env}\"{bin}\" -module-cache \"{mc}\" \"{client}\"" var run_out = "" var rc = spawn_capture(cmd, run_out) diff --git a/tests/module_cache/test_macro_dep_invalidate.das b/tests/module_cache/test_macro_dep_invalidate.das index 36728791e4..9371f961e6 100644 --- a/tests/module_cache/test_macro_dep_invalidate.das +++ b/tests/module_cache/test_macro_dep_invalidate.das @@ -16,7 +16,7 @@ def das_exe() : string { //! child env rides the command string (the portable set/prefix split popen_argv cannot carry) def env_run(dep, cmd : string; var output : string&) : int { let win = get_platform_name() == "windows" - let full = win ? "set MC_DEP_FILE={dep}&& {cmd}" : "MC_DEP_FILE={dep} {cmd}" + let full = win ? "set \"MC_DEP_FILE={dep}\"&& {cmd}" : "MC_DEP_FILE='{dep}' {cmd}" return unsafe(popen_timeout(full, 300.0, $(f) { if (f != null) { output = fread(f) From 3eefced6b081fb1466b9d9b4ffc0a0d17db35778 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 11:34:35 -0700 Subject: [PATCH 34/35] review round 2: the confirm spawn gets a timeout, the fail path cleans up, the seed spells ull A hung confirm child (driver stall, bad vehicle) no longer wedges an unattended mint - popen_timeout at a 30-minute budget, nonzero exit announced. The fwrite-fail path removes its temp manifest. FNV64A_SEED takes the ull suffix - the ul spelling was legal (a decimal literal past unsigned long promotes to unsigned long long) but reads as 32-bit on LLP64. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014mq9AKyoAwvD5T1cigTnRw --- modules/dasLLAMA/harness/tune_kernels.das | 8 ++++++-- src/ast/ast_parse.cpp | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/modules/dasLLAMA/harness/tune_kernels.das b/modules/dasLLAMA/harness/tune_kernels.das index f469b61b65..78e3275f30 100644 --- a/modules/dasLLAMA/harness/tune_kernels.das +++ b/modules/dasLLAMA/harness/tune_kernels.das @@ -2403,6 +2403,7 @@ def private confirm_tg_run(model : string; wins : array; extra : string) } let tmpm = tmpmRes as value if (!fwrite(tmpm, "\{\"kernels\": \{\}, \"runtime\": \{\"metal_tensor\": \"{crowns}\"\}, \"provenance\": \{\"box\": \"{tune_box_identity()}\"\}\}")) { + remove(tmpm) print("confirm: cannot write {tmpm}\n") return 0.0lf } @@ -2417,11 +2418,14 @@ def private confirm_tg_run(model : string; wins : array; extra : string) } let tmpo = tmpoRes as value let cmd = "{envp}\"{get_das_exe()}\" -jit \"{get_das_root()}/modules/dasLLAMA/benchmarks/lcpp_bench.das\" -dasroot \"{get_das_root()}\" -- --for-debug-purposes -m \"{model}\" --ngl 99 -p 32 -n 128 -r 3 > \"{tmpo}\" 2>&1" - unsafe(popen(cmd) $(f) { + let crc = unsafe(popen_timeout(cmd, 1800.0, $(f) { if (f != null) { fread(f) } - }) + })) + if (crc != 0) { + print("confirm arm ({empty(extra) ? "base" : extra}): child exited {crc}\n") + } var outp = "" if (fexist(tmpo)) { fopen(tmpo, "rb") $(f) { diff --git a/src/ast/ast_parse.cpp b/src/ast/ast_parse.cpp index be46298746..e2ef0f9259 100644 --- a/src/ast/ast_parse.cpp +++ b/src/ast/ast_parse.cpp @@ -588,7 +588,7 @@ namespace das { // sanity bound on a record's macro-dependency count: a real program registers a handful // (one per consumed sidecar/config); anything past this reads as stream corruption static constexpr uint32_t SER_MAX_MACRO_DEPS = 4096u; - static constexpr uint64_t FNV64A_SEED = 14695981039346656037ul; // any nonzero constant serves: size (-1 absent vs 0 empty) already separates the states + static constexpr uint64_t FNV64A_SEED = 14695981039346656037ull; // any nonzero constant serves: size (-1 absent vs 0 empty) already separates the states void statAndHashFileDependency ( const string & path, int64_t & size, uint64_t & hash ) { size = -1; From 065629c7c501c491e5a5ad12c903e10d338d0686 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 12:33:31 -0700 Subject: [PATCH 35/35] CI: ENVIRONMENT.md regenerated for the confirm_iq2xxs knob The serving-confirm vehicle override was declared in dasllama_env.das without re-running gen_env_doc; the env-registry lane rightly red on the stale doc. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014mq9AKyoAwvD5T1cigTnRw --- modules/dasLLAMA/ENVIRONMENT.md | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/dasLLAMA/ENVIRONMENT.md b/modules/dasLLAMA/ENVIRONMENT.md index 164d33d259..8cd5f1175a 100644 --- a/modules/dasLLAMA/ENVIRONMENT.md +++ b/modules/dasLLAMA/ENVIRONMENT.md @@ -168,6 +168,7 @@ Apple Accelerate / AMX float lane. `DASLLAMA_ACCEL` arms the whole group. |---|---|---|---| | `DASLLAMA_MODELS_DIR` | path | unset | Directory holding the .gguf models the probes, benches and tests load. dasllama-server's model catalog downloads here too when set. | | `DASLLAMA_CONFIRM_MODEL` | path | auto-resolved from the models dir | Model used by the tuner's confirm gate (FULL path, not a bare filename). Unset: the gate auto-resolves from the models dir - the preferred confirm carrier, else the largest present q8 gguf; the fallback pins only when the box has no q8 model at all. | +| `DASLLAMA_CONFIRM_IQ2XXS` | path | auto-resolved from the models dir | IQ2_XXS vehicle for the tuner's serving-crown confirm (FULL path). Unset: the confirm auto-resolves the first *IQ2_XXS*.gguf in the models dir; none found = no crown (the base kernel serves). | | `DASLLAMA_BATCH_CHUNKS` | text | unset | Override the batched-dispatch chunk count in the 1-core GEMM probe. | | `DASLLAMA_PROBE_MODEL` | text | Qwen3VL-8B-Instruct-Q8_0.gguf | Model filename (inside the models dir) the image-turn attribution probe loads. | | `DASLLAMA_BATCH_GRID_2D` | number | unset | Use the 2D batch grid in the parity probe. |