From e23d5648f791afc332ff204d43ba09b3d39e9a8f Mon Sep 17 00:00:00 2001 From: Hongsheng Liu Date: Sat, 3 Oct 2026 17:16:58 +0800 Subject: [PATCH] qwen3_5: compact GDN preparation without changing TF32 operands Signed-off-by: Hongsheng Liu --- benchmarks/gdn/README.md | 205 +++++++++++++++ .../20261003/e2e/baseline-measured-1.jsonl | 74 ++++++ .../20261003/e2e/baseline-measured-2.jsonl | 74 ++++++ .../20261003/e2e/candidate-measured-1.jsonl | 74 ++++++ .../20261003/e2e/candidate-measured-2.jsonl | 74 ++++++ .../gdn/artifacts/20261003/e2e/cleanup.json | 33 +++ .../gdn/artifacts/20261003/e2e/plan.json | 245 ++++++++++++++++++ .../gdn/artifacts/20261003/e2e/summary.json | 136 ++++++++++ .../artifacts/20261003/flashinfer/plan.json | 72 +++++ .../20261003/flashinfer/results.jsonl | 30 +++ .../kernel/analysis/recurrent-reference.log | 28 ++ .../20261003/kernel/analysis/results.jsonl | 28 ++ .../20261003/kernel/analysis/summary.json | 51 ++++ .../20261003/kernel/harness/compare.py | 136 ++++++++++ .../gdn/artifacts/20261003/kernel/plan.json | 90 +++++++ .../gdn/artifacts/20261003/manifest.json | 223 ++++++++++++++++ .../artifacts/20261003/rejected-variants.json | 146 +++++++++++ src/backends/cuda/qwen3_5/README.md | 4 +- src/backends/cuda/qwen3_5/gdn_prefill.cu | 204 +++++++++------ src/models/cua_s1/native/Cargo.toml | 4 + .../native/tests => tests/qwen3_5}/kernels.rs | 34 ++- 21 files changed, 1875 insertions(+), 90 deletions(-) create mode 100644 benchmarks/gdn/README.md create mode 100644 benchmarks/gdn/artifacts/20261003/e2e/baseline-measured-1.jsonl create mode 100644 benchmarks/gdn/artifacts/20261003/e2e/baseline-measured-2.jsonl create mode 100644 benchmarks/gdn/artifacts/20261003/e2e/candidate-measured-1.jsonl create mode 100644 benchmarks/gdn/artifacts/20261003/e2e/candidate-measured-2.jsonl create mode 100644 benchmarks/gdn/artifacts/20261003/e2e/cleanup.json create mode 100644 benchmarks/gdn/artifacts/20261003/e2e/plan.json create mode 100644 benchmarks/gdn/artifacts/20261003/e2e/summary.json create mode 100644 benchmarks/gdn/artifacts/20261003/flashinfer/plan.json create mode 100644 benchmarks/gdn/artifacts/20261003/flashinfer/results.jsonl create mode 100644 benchmarks/gdn/artifacts/20261003/kernel/analysis/recurrent-reference.log create mode 100644 benchmarks/gdn/artifacts/20261003/kernel/analysis/results.jsonl create mode 100644 benchmarks/gdn/artifacts/20261003/kernel/analysis/summary.json create mode 100644 benchmarks/gdn/artifacts/20261003/kernel/harness/compare.py create mode 100644 benchmarks/gdn/artifacts/20261003/kernel/plan.json create mode 100644 benchmarks/gdn/artifacts/20261003/manifest.json create mode 100644 benchmarks/gdn/artifacts/20261003/rejected-variants.json rename {src/models/cua_s1/native/tests => tests/qwen3_5}/kernels.rs (92%) diff --git a/benchmarks/gdn/README.md b/benchmarks/gdn/README.md new file mode 100644 index 0000000..c639d60 --- /dev/null +++ b/benchmarks/gdn/README.md @@ -0,0 +1,205 @@ +# Gated DeltaNet preparation on H200 + +Lossless TF32 operand packing and direct U/W stores reduce complete native GDN +call latency by about 13% at 936 and 3,399 tokens. The same change gives only a +0.42% mean HTTP latency reduction on the 74-request Open-Jev workload, below the +predeclared 2% end-to-end target. All compared model probabilities and decisions +are unchanged. These are separate kernel and serving results. + +## Implementation + +The preparation kernel previously kept normalized Q/K in float32 shared memory +and converted each loaded WMMA fragment to TF32. It now converts each component +once and stores the 19 meaningful TF32 bits in 16-bit and 3-bit planes, using +three bytes per component. This preserves the exponent range and TF32 operands. +Decayed Q/K still come from the original float32 normalization values. + +Explicit `mma.m16n8k4` keeps the four-term accumulation used by the original +WMMA path in the measured SM90 build. Using `m16n8k8` changed intermediate +rounding despite identical TF32 operands; that candidate was rejected. U/W +accumulators are packed directly to bfloat16 instead of passing through an 8 KiB +float32 shared staging area. Gate loads are prefetched in groups of eight while +retaining the sequential float32 sum. Dynamic shared memory falls from 92 KiB +to 72 KiB. The inverse stays float32, and the workspace, C ABI, state and output +kernels keep their existing contracts. + +## Fixed controls and run budget + +Measured on 2026-10-03, on one H200, SM90, 132 SMs. The driver reports this +device as `NVIDIA L20X`; H200 is the corrected hardware label used here. +All runs used scheduler-reserved GPU 2, UUID +`GPU-cbf66259-f4ab-0ede-1811-82037dde5924`, NUMA node 0 and CPUs 0–15. +The driver was 570.133.20 and native libraries were built with CUDA 13.0 `nvcc`. +No shared caches were dropped or clocks changed. + +The only native A/B variable was GDN preparation. The baseline source is main +at `58b8cbe9d4738f6c217f6fe584852dc9361b2c24`; its GDN source also matches PR #55. +The [archived plans and manifest](artifacts/20261003/manifest.json) freeze the +working patch, source/library hashes, commands, dependencies and original paths. +Builds and copies were completed before measurements. + +Standalone inputs use batch 1, 16 Q/K heads, 48 value heads and head dimension +128. Q/K/V/beta are bfloat16 and log gates are float32. Seed 20261003 generates +correlated Q/K with weak decay, using PyTorch 2.13.0+cu129. The token counts are +representative workload shapes; the tensors are synthetic inputs rather than +captured model activations. Both libraries use the same workspace and stream. + +Each shape/configuration has one excluded feasibility run and ten excluded +warmups, then exactly two measured passes of 100 calls. Pass 2 reverses arm +order. The complete call includes preparation, state and output kernels. Eager +CPU submission is included in wall time; CUDA event spans also include gaps +between submissions and must not be read as active kernel time. + +## Complete GDN call latency + +Wall time per call, in milliseconds: + +| Tokens | Baseline pass 1 / 2 | Candidate pass 1 / 2 | Reduction pass 1 / 2 | +| --- | --- | --- | --- | +| 107 | 0.051291 / 0.050976 | 0.050914 / 0.050957 | 0.74% / 0.04% | +| 936 | 0.247565 / 0.247337 | 0.214176 / 0.214178 | 13.49% / 13.41% | +| 3,399 | 0.800775 / 0.798935 | 0.696768 / 0.697262 | 12.99% / 12.73% | + +The predeclared kernel gate was at least 10% reduction at both longer shapes +in each pass, with no short-input regression. It passed. Short-input differences +are too small to support a meaningful speedup claim. Raw wall/event timings and +bitwise workspace comparisons are in [results.jsonl](artifacts/20261003/kernel/analysis/results.jsonl). + +Feasibility checks compare every initialized U/W/QD/KD/PB/decay element by its +bit representation; all changed counts are zero. Final outputs also match. +The recurrent float64 reference retains its original tolerance and now covers +tile/chunk boundaries, grouped heads, 936/3,399 tokens, weak decay, zero Q/K, +and amplitudes 1e-9 and 1e-18. All three GPU tests, including the existing +attention and Graph tests, passed; see the [test log](artifacts/20261003/kernel/analysis/recurrent-reference.log). +These checks establish parity for the sampled inputs on this device, rather +than a proof of identical results for all inputs or architectures. + +## Warm HTTP latency + +The serving comparison reuses the prepared merged BF16 Open-Jev-27B-v1.1 +checkpoint, temperature 2.5343690298472983, maximum length 16,384, one Noul +candidate and concurrency 1. The same 74 requests span 80–3,399 tokens. The +source JevBench revision is `f8ce71361165846101d02ebc83ad44e47ae44fc3`. + +Both arms use the same frozen Rust worker/frontend from +[PR #55](https://github.com/ThinkFlowLab/system1-omni/pull/55), worker revision +`202c0e163f868334a99d88407056ebe61dbb2dce`. ABI4 library sources are frozen at +`7b935723c6fa17145e3d866675202fb6f7ea1f5d` except for the candidate GDN +translation unit. The branch on main uses an ABI3 library for its Rust GPU +tests; that library was not substituted into the ABI4 Open-Jev worker. + +Model base revision is `1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0`, and checkpoint +revision is `28cf73067d5b337860bbef3c85b8b82ba8730956`. Prepared weights and +tokenizer were reused. Graph capture and shared-prefix caching were disabled. +No Nsight/CUPTI instrumentation was active. The HTTP timer includes receiving +and decoding the response body, and excludes request JSON serialization and +response JSON parsing. + +Each arm reuses one worker/frontend pair. Genuine readiness and the first +validated inference are followed by one excluded 74-request feasibility pass +and exactly two measured passes. Baseline precedes candidate. Worker readiness +took 11.63 s and 10.62 s respectively; these startup times are excluded. + +| Configuration | Pass 1 mean | Pass 2 mean | Aggregate mean | +| --- | --- | --- | --- | +| Baseline | 48.375 ms | 48.441 ms | 48.408 ms | +| Candidate | 48.105 ms | 48.305 ms | 48.205 ms | + +All 74 decisions, probabilities and token counts match the pinned native +reference in feasibility and both measured passes. The mean reduction is +0.42%; it misses the 2% gate despite both candidate passes being faster. +With two repetitions and this small difference, the result does not establish +a material end-to-end gain. The [summary](artifacts/20261003/e2e/summary.json) +and four adjacent JSONL files preserve every measured response and latency. +Task-owned processes exited and the GPU reservation was released. + +This is an incremental comparison against native Rust/CUDA. Raw HF Transformers +and open-jev-fast were not remeasured in this campaign, so prior HF speedups must +not be presented as a new matched comparison for this kernel change. + +## Ready kernels in vLLM + +At vLLM revision `5f30fc7031cae49bf51073fc953d419b08f8887c`, the +[Qwen GDN dispatcher](https://github.com/vllm-project/vllm/blob/5f30fc7031cae49bf51073fc953d419b08f8887c/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py) +selects FlashInfer on SM90 with `auto` or `flashinfer`. Its in-tree CuTe prefill +backend is opt-in and targets SM10x. The +[post-convolution preparation kernel](https://github.com/vllm-project/vllm/blob/5f30fc7031cae49bf51073fc953d419b08f8887c/vllm/third_party/flash_linear_attention/ops/fused_gdn_prefill_post_conv.py) +fuses layout preparation, Q/K normalization and gating. + +FlashInfer provides [SM90 fused and context-parallel prefill](https://github.com/flashinfer-ai/flashinfer/blob/7eb86aa0fdc1248fab43c89801de4ed450e35e77/flashinfer/gdn_prefill.py). +The fused CuTe kernel uses TMA, WGMMA and warp specialization; the context-parallel +path splits work across sequence chunks. Python/CuTe entry points need an AOT +export and C ABI adapter for Rust; a native integration was not implemented. + +The installed FlashInfer 0.6.16.post3 / CUTLASS DSL 4.6.2 comparison includes +required Q/K normalization, gate exponentiation and beta conversion. These +versions differ from the researched upstream revisions. Complete eager wall +time, in milliseconds: + +| Tokens | Native pass 1 / 2 | FlashInfer fused pass 1 / 2 | FlashInfer CP pass 1 / 2 | +| --- | --- | --- | --- | +| 107 | 0.0521 / 0.0518 | 0.4190 / 0.4379 | 0.8345 / 0.8161 | +| 936 | 0.2482 / 0.2481 | 0.4392 / 0.4305 | 0.8468 / 0.8274 | +| 3,399 | 0.8019 / 0.8001 | 0.4259 / 0.5080 | 0.8382 / 0.8566 | + +The fused path helps the longest shape, with substantial observed variation, +and is slower for the other two complete calls. CP is slower at these shapes. +This is a separate standalone experiment against the original native kernel; +it establishes neither full-model FlashInfer fidelity nor Rust integration +performance. [Raw results](artifacts/20261003/flashinfer/results.jsonl) include +numerical comparisons. No hardware-counter evidence attributes the differences +to a particular stall, occupancy or bandwidth cause. + +## Reproduction and rejected variants + +Build the current native library and register the GPU test binary without +initializing CUDA: + +```sh +NVCC=/usr/local/cuda/bin/nvcc src/backends/cuda/qwen3_5/build.sh /tmp/gdn-current 90 +cargo test --release --locked -p omni-cua-s1-native --test kernels --no-run +``` + +After checking the host scheduler, reserve an available exact device and run +the reference tests (GPU 2 is the recorded experiment device, not authorization +on another host): + +```sh +gpu run --gpu-ids 2 --timeout 10m --note "GDN reference validation" -- \ + numactl --membind=0 --physcpubind=0-15 \ + env CUA_S1_CUDA_LIB=/tmp/gdn-current/libqwen3_5_cuda.so \ + cargo test --release --locked -p omni-cua-s1-native --test kernels -- \ + --ignored --nocapture --test-threads=1 +``` + +The [frozen kernel runner](artifacts/20261003/kernel/harness/compare.py) records +the exact RNG, workspace comparisons and timing loop. For another run, copy it +into a fresh `/harness/` directory, create `/analysis/`, and write +`/plan.json` with `libraries.baseline`, `libraries.candidate`, +`validation_library`, `kernel_test_binary` and a SHA256 `hashes` map for the +frozen files. Build a baseline library from the recorded main revision. The +runner's device/affinity checks match the recorded controls; adapt and record +them when changing hosts. Run `compare.py --probe` before device work, then use +the scheduler command preserved in [plan.json](artifacts/20261003/kernel/plan.json). +The serving plan preserves all worker/model paths and hashes; reproducing that +comparison also requires the frozen PR #55 worker and prepared 74-request +manifest. Archived result directories must not be reused for new measurements. + +Earlier experiments remain in the local archives named in the manifest: + +- TF32 tensor-core off-diagonal inverse: 2–4% core gain, below the kernel gate. +- BF16 pair products: unit reference passed, but a probability near 0.5 changed + one model decision during feasibility; no measured candidate HTTP passes. +- RN-even FP16 pair products: probability drift exceeded 0.01 during feasibility. +- RNA-rounded FP16 pair products: one model decision changed during feasibility. +- Float32 staging with direct stores: exact comparisons passed, but 7–8% core + gain missed the kernel gate; no end-to-end comparison was run for that variant. +- Packed TF32 with eight-term MMA: intermediate bitwise comparisons failed; + timing stopped before measured passes. Restoring four-term MMA resolved the + compared differences and produced the final results above. + +No failed variant was promoted by relaxing the numerical gates or extending +its measured run budget. Hardware profiling counters were unavailable to this +account. SM80/SM89/SM90 compilation passed; current device execution and +performance evidence cover SM90. The four Rust CI commands also passed; their +exact commands and build outcomes are recorded in the manifest. diff --git a/benchmarks/gdn/artifacts/20261003/e2e/baseline-measured-1.jsonl b/benchmarks/gdn/artifacts/20261003/e2e/baseline-measured-1.jsonl new file mode 100644 index 0000000..fb20764 --- /dev/null +++ b/benchmarks/gdn/artifacts/20261003/e2e/baseline-measured-1.jsonl @@ -0,0 +1,74 @@ +{"id": "original-policy-01-0", "latency_s": 0.0205433, "probability_yes": 0.03047528794383215, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.03047528794383215}}, "usage": {"input_tokens": 110, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.020123011, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007167440045605, "client_end_monotonic_ns": 5007167460588905} +{"id": "original-policy-01-1", "latency_s": 0.020331928, "probability_yes": 0.03534008179644288, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.03534008179644288}}, "usage": {"input_tokens": 113, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019933891, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007167460684314, "client_end_monotonic_ns": 5007167481016242} +{"id": "original-policy-02-0", "latency_s": 0.020411296, "probability_yes": 0.9450513567824766, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.9450513567824766}}, "usage": {"input_tokens": 99, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019965028, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007167481114398, "client_end_monotonic_ns": 5007167501525694} +{"id": "original-policy-02-1", "latency_s": 0.020301888, "probability_yes": 0.9259408765258507, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.9259408765258507}}, "usage": {"input_tokens": 98, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019905567, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007167501634217, "client_end_monotonic_ns": 5007167521936105} +{"id": "original-policy-03-0", "latency_s": 0.020268914, "probability_yes": 0.8192057049596325, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.8192057049596325}}, "usage": {"input_tokens": 101, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019917914, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007167522028734, "client_end_monotonic_ns": 5007167542297648} +{"id": "original-policy-03-1", "latency_s": 0.020241111, "probability_yes": 0.7985696425158759, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.7985696425158759}}, "usage": {"input_tokens": 100, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019863566, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007167542386958, "client_end_monotonic_ns": 5007167562628069} +{"id": "original-policy-04-0", "latency_s": 0.019921615, "probability_yes": 0.022645581582854003, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.022645581582854003}}, "usage": {"input_tokens": 107, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019592816, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007167562716007, "client_end_monotonic_ns": 5007167582637622} +{"id": "original-policy-04-1", "latency_s": 0.020172696, "probability_yes": 0.024342206375083403, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.024342206375083403}}, "usage": {"input_tokens": 104, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019854751, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007167582718857, "client_end_monotonic_ns": 5007167602891553} +{"id": "original-policy-05-0", "latency_s": 0.020034196, "probability_yes": 0.04070117194736392, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.04070117194736392}}, "usage": {"input_tokens": 111, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019715318, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007167602971009, "client_end_monotonic_ns": 5007167623005205} +{"id": "original-policy-05-1", "latency_s": 0.02038789, "probability_yes": 0.05542359965397362, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.05542359965397362}}, "usage": {"input_tokens": 103, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019901763, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007167623091263, "client_end_monotonic_ns": 5007167643479153} +{"id": "original-policy-06-0", "latency_s": 0.020279466, "probability_yes": 0.9354533714741977, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.9354533714741977}}, "usage": {"input_tokens": 113, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019860315, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007167643588629, "client_end_monotonic_ns": 5007167663868095} +{"id": "original-policy-06-1", "latency_s": 0.020274808, "probability_yes": 0.26687647588636787, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.26687647588636787}}, "usage": {"input_tokens": 103, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019922043, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007167663957925, "client_end_monotonic_ns": 5007167684232733} +{"id": "original-adequacy-01-0", "latency_s": 0.019996752, "probability_yes": 0.7544465755976671, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.7544465755976671}}, "usage": {"input_tokens": 92, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019660717, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007167684315505, "client_end_monotonic_ns": 5007167704312257} +{"id": "original-adequacy-01-1", "latency_s": 0.02000046, "probability_yes": 0.7704937058511766, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.7704937058511766}}, "usage": {"input_tokens": 88, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019509267, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007167704384291, "client_end_monotonic_ns": 5007167724384751} +{"id": "original-adequacy-02-0", "latency_s": 0.019990632, "probability_yes": 0.09635610675757625, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.09635610675757625}}, "usage": {"input_tokens": 89, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019594945, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007167724498139, "client_end_monotonic_ns": 5007167744488771} +{"id": "original-adequacy-02-1", "latency_s": 0.02004171, "probability_yes": 0.08394373314289287, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.08394373314289287}}, "usage": {"input_tokens": 90, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019613459, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007167744573354, "client_end_monotonic_ns": 5007167764615064} +{"id": "original-adequacy-03-0", "latency_s": 0.020072605, "probability_yes": 0.17073801840382533, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.17073801840382533}}, "usage": {"input_tokens": 92, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019681136, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007167764711876, "client_end_monotonic_ns": 5007167784784481} +{"id": "original-adequacy-03-1", "latency_s": 0.019942362, "probability_yes": 0.22713849981308368, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.22713849981308368}}, "usage": {"input_tokens": 96, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019645477, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007167784866851, "client_end_monotonic_ns": 5007167804809213} +{"id": "original-adequacy-04-0", "latency_s": 0.019870143, "probability_yes": 0.7525499367436955, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.7525499367436955}}, "usage": {"input_tokens": 96, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.01959413, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007167804880371, "client_end_monotonic_ns": 5007167824750514} +{"id": "original-adequacy-04-1", "latency_s": 0.01978651, "probability_yes": 0.5988825971216104, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.5988825971216104}}, "usage": {"input_tokens": 93, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019532881, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007167824814461, "client_end_monotonic_ns": 5007167844600971} +{"id": "original-adequacy-05-0", "latency_s": 0.019565118, "probability_yes": 0.2599347544869691, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.2599347544869691}}, "usage": {"input_tokens": 85, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019313464, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007167844668103, "client_end_monotonic_ns": 5007167864233221} +{"id": "original-adequacy-05-1", "latency_s": 0.019592553, "probability_yes": 0.188362048820982, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.188362048820982}}, "usage": {"input_tokens": 86, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019313673, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007167864299646, "client_end_monotonic_ns": 5007167883892199} +{"id": "original-adequacy-06-0", "latency_s": 0.020131638, "probability_yes": 0.8080309304151047, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.8080309304151047}}, "usage": {"input_tokens": 101, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019786889, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007167883971557, "client_end_monotonic_ns": 5007167904103195} +{"id": "original-adequacy-06-1", "latency_s": 0.02037535, "probability_yes": 0.6635779695297708, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.6635779695297708}}, "usage": {"input_tokens": 103, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.0199291, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007167904191430, "client_end_monotonic_ns": 5007167924566780} +{"id": "easy-fact-00", "latency_s": 0.019906001, "probability_yes": 0.9674054242768856, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.9674054242768856}}, "usage": {"input_tokens": 90, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019552926, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007167924650786, "client_end_monotonic_ns": 5007167944556787} +{"id": "easy-fact-01", "latency_s": 0.019730625, "probability_yes": 0.03130854188970371, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.03130854188970371}}, "usage": {"input_tokens": 88, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019432046, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007167944638951, "client_end_monotonic_ns": 5007167964369576} +{"id": "easy-fact-02", "latency_s": 0.019893454, "probability_yes": 0.9728799832337615, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.9728799832337615}}, "usage": {"input_tokens": 96, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019618268, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007167964438353, "client_end_monotonic_ns": 5007167984331807} +{"id": "easy-fact-03", "latency_s": 0.01998413, "probability_yes": 0.013935738446912068, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.013935738446912068}}, "usage": {"input_tokens": 99, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019737067, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007167984396497, "client_end_monotonic_ns": 5007168004380627} +{"id": "easy-fact-04", "latency_s": 0.01967335, "probability_yes": 0.9522894788263, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.9522894788263}}, "usage": {"input_tokens": 88, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019383336, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007168004444884, "client_end_monotonic_ns": 5007168024118234} +{"id": "easy-fact-05", "latency_s": 0.01981784, "probability_yes": 0.029410065460985898, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.029410065460985898}}, "usage": {"input_tokens": 89, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019514399, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007168024205718, "client_end_monotonic_ns": 5007168044023558} +{"id": "easy-fact-06", "latency_s": 0.019877522, "probability_yes": 0.9706899759473617, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.9706899759473617}}, "usage": {"input_tokens": 109, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019606189, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007168044103754, "client_end_monotonic_ns": 5007168063981276} +{"id": "easy-fact-07", "latency_s": 0.019593477, "probability_yes": 0.02438033353329917, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.02438033353329917}}, "usage": {"input_tokens": 84, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.0193318, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007168064055884, "client_end_monotonic_ns": 5007168083649361} +{"id": "easy-fact-08", "latency_s": 0.019554232, "probability_yes": 0.9404350853099106, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.9404350853099106}}, "usage": {"input_tokens": 80, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019299168, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007168083716418, "client_end_monotonic_ns": 5007168103270650} +{"id": "easy-fact-09", "latency_s": 0.019675617, "probability_yes": 0.034599738070281126, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.034599738070281126}}, "usage": {"input_tokens": 80, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019378269, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007168103340369, "client_end_monotonic_ns": 5007168123015986} +{"id": "easy-fact-10", "latency_s": 0.019859841, "probability_yes": 0.6793076377503555, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.6793076377503555}}, "usage": {"input_tokens": 89, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019582506, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007168123082971, "client_end_monotonic_ns": 5007168142942812} +{"id": "easy-fact-11", "latency_s": 0.019859485, "probability_yes": 0.06041778591308143, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.06041778591308143}}, "usage": {"input_tokens": 90, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019588361, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007168143014803, "client_end_monotonic_ns": 5007168162874288} +{"id": "hard-opus-a-long_policy-11", "latency_s": 0.216586335, "probability_yes": 0.7098596537741193, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.7098596537741193}}, "usage": {"input_tokens": 2334, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.216144353, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007168163011282, "client_end_monotonic_ns": 5007168379597617} +{"id": "hard-opus-a-long_policy-13", "latency_s": 0.227743904, "probability_yes": 0.21768834160648515, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.21768834160648515}}, "usage": {"input_tokens": 2426, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.227098287, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007168379783445, "client_end_monotonic_ns": 5007168607527349} +{"id": "hard-opus-a-long_policy-19", "latency_s": 0.22885186, "probability_yes": 0.18473627906713375, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.18473627906713375}}, "usage": {"input_tokens": 2325, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.228193924, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007168607712783, "client_end_monotonic_ns": 5007168836564643} +{"id": "hard-opus-a-probability-03", "latency_s": 0.048422412, "probability_yes": 0.22782598391193778, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.22782598391193778}}, "usage": {"input_tokens": 429, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.047903208, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007168836716684, "client_end_monotonic_ns": 5007168885139096} +{"id": "hard-opus-a-probability-08", "latency_s": 0.043474886, "probability_yes": 0.19551009998444735, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.19551009998444735}}, "usage": {"input_tokens": 391, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.043134838, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007168885233380, "client_end_monotonic_ns": 5007168928708266} +{"id": "hard-opus-a-temporal_numeric-03", "latency_s": 0.049650673, "probability_yes": 0.1925798624930316, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.1925798624930316}}, "usage": {"input_tokens": 487, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.049305275, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007168928794491, "client_end_monotonic_ns": 5007168978445164} +{"id": "hard-opus-a-temporal_numeric-06", "latency_s": 0.056578214, "probability_yes": 0.054978919789340254, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.054978919789340254}}, "usage": {"input_tokens": 522, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.056216327, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007168978555739, "client_end_monotonic_ns": 5007169035133953} +{"id": "hard-opus-a-temporal_numeric-09", "latency_s": 0.053745447, "probability_yes": 0.49804298259995755, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.49804298259995755}}, "usage": {"input_tokens": 519, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.053376882, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007169035229482, "client_end_monotonic_ns": 5007169088974929} +{"id": "hard-opus-b-multi_hop-08", "latency_s": 0.219390449, "probability_yes": 0.23017195500915008, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.23017195500915008}}, "usage": {"input_tokens": 2312, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.218824054, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007169089123143, "client_end_monotonic_ns": 5007169308513592} +{"id": "hard-opus-b-probability-03", "latency_s": 0.11525784, "probability_yes": 0.1728043755923391, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.1728043755923391}}, "usage": {"input_tokens": 1213, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.114597248, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007169308658338, "client_end_monotonic_ns": 5007169423916178} +{"id": "hard-opus-b-tradeoff-06", "latency_s": 0.077427522, "probability_yes": 0.7575263998692227, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.7575263998692227}}, "usage": {"input_tokens": 790, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.076797301, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007169424052367, "client_end_monotonic_ns": 5007169501479889} +{"id": "hard-opus-b-tradeoff-08", "latency_s": 0.060552277, "probability_yes": 0.47298829137088333, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.47298829137088333}}, "usage": {"input_tokens": 601, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.060020517, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007169501614496, "client_end_monotonic_ns": 5007169562166773} +{"id": "hard-opus-c-long_policy-04", "latency_s": 0.319923132, "probability_yes": 0.5442802351228624, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.5442802351228624}}, "usage": {"input_tokens": 3312, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.31921673, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007169562387754, "client_end_monotonic_ns": 5007169882310886} +{"id": "hard-opus-c-long_policy-11", "latency_s": 0.320943951, "probability_yes": 0.5194427077546381, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.5194427077546381}}, "usage": {"input_tokens": 3399, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.320212652, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007169882548541, "client_end_monotonic_ns": 5007170203492492} +{"id": "hard-opus-c-temporal_numeric-02", "latency_s": 0.090533984, "probability_yes": 0.24180561207468904, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.24180561207468904}}, "usage": {"input_tokens": 936, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.089866353, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007170203652901, "client_end_monotonic_ns": 5007170294186885} +{"id": "hard-sol-a-adversarial-06", "latency_s": 0.028020786, "probability_yes": 0.018362853166429507, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.018362853166429507}}, "usage": {"input_tokens": 241, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.027520754, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007170294336340, "client_end_monotonic_ns": 5007170322357126} +{"id": "hard-sol-a-adversarial-08", "latency_s": 0.027956827, "probability_yes": 0.9375888015626979, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.9375888015626979}}, "usage": {"input_tokens": 218, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.02759912, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007170322442726, "client_end_monotonic_ns": 5007170350399553} +{"id": "hard-sol-a-trap-02", "latency_s": 0.032438416, "probability_yes": 0.03573969108794452, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.03573969108794452}}, "usage": {"input_tokens": 292, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.031883052, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007170350481665, "client_end_monotonic_ns": 5007170382920081} +{"id": "hard-sol-a-trap-04", "latency_s": 0.025488983, "probability_yes": 0.041796879885975066, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.041796879885975066}}, "usage": {"input_tokens": 198, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.025019539, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007170383063363, "client_end_monotonic_ns": 5007170408552346} +{"id": "hard-sol-a-trap-13", "latency_s": 0.025812971, "probability_yes": 0.030579705641278977, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.030579705641278977}}, "usage": {"input_tokens": 206, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.025453613, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007170408674501, "client_end_monotonic_ns": 5007170434487472} +{"id": "hard-sol-a-trap-15", "latency_s": 0.032604034, "probability_yes": 0.07570582831909627, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.07570582831909627}}, "usage": {"input_tokens": 308, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.032281262, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007170434574241, "client_end_monotonic_ns": 5007170467178275} +{"id": "hard-sol-b-judge_hard-01", "latency_s": 0.03208418, "probability_yes": 0.6743485180581948, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.6743485180581948}}, "usage": {"input_tokens": 288, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.031757398, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007170467255592, "client_end_monotonic_ns": 5007170499339772} +{"id": "hard-sol-b-judge_hard-02", "latency_s": 0.026587847, "probability_yes": 0.8525225556146606, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.8525225556146606}}, "usage": {"input_tokens": 212, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.026275318, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007170499422004, "client_end_monotonic_ns": 5007170526009851} +{"id": "hard-sol-b-judge_hard-05", "latency_s": 0.027725289, "probability_yes": 0.9192409990717809, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.9192409990717809}}, "usage": {"input_tokens": 231, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.027406701, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007170526102601, "client_end_monotonic_ns": 5007170553827890} +{"id": "hard-sol-b-judge_hard-08", "latency_s": 0.026302157, "probability_yes": 0.33265846433012053, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.33265846433012053}}, "usage": {"input_tokens": 214, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.026020503, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007170553907674, "client_end_monotonic_ns": 5007170580209831} +{"id": "hard-sol-b-judge_hard-10", "latency_s": 0.027933243, "probability_yes": 0.3554926261464918, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.3554926261464918}}, "usage": {"input_tokens": 233, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.027651451, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007170580278858, "client_end_monotonic_ns": 5007170608212101} +{"id": "hard-sol-b-judge_hard-14", "latency_s": 0.02794007, "probability_yes": 0.07750785419177868, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.07750785419177868}}, "usage": {"input_tokens": 250, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.027665738, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007170608283974, "client_end_monotonic_ns": 5007170636224044} +{"id": "hard-sol-b-judge_hard-15", "latency_s": 0.026633115, "probability_yes": 0.8936767977520365, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.8936767977520365}}, "usage": {"input_tokens": 206, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.026336331, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007170636300502, "client_end_monotonic_ns": 5007170662933617} +{"id": "hard-sol-b-judge_hard-18", "latency_s": 0.025148326, "probability_yes": 0.07374139181136073, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.07374139181136073}}, "usage": {"input_tokens": 173, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.024846549, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007170663018008, "client_end_monotonic_ns": 5007170688166334} +{"id": "hard-sol-c-judge_hard-03", "latency_s": 0.043644907, "probability_yes": 0.11480257968157337, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.11480257968157337}}, "usage": {"input_tokens": 395, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.043302327, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007170688246312, "client_end_monotonic_ns": 5007170731891219} +{"id": "hard-sol-c-judge_hard-05", "latency_s": 0.038794964, "probability_yes": 0.7398933422307209, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.7398933422307209}}, "usage": {"input_tokens": 357, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.038344196, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007170731974819, "client_end_monotonic_ns": 5007170770769783} +{"id": "hard-sol-c-judge_hard-07", "latency_s": 0.036876342, "probability_yes": 0.5950171562428843, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.5950171562428843}}, "usage": {"input_tokens": 321, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.036388112, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007170770884898, "client_end_monotonic_ns": 5007170807761240} +{"id": "hard-sol-c-judge_hard-08", "latency_s": 0.033052608, "probability_yes": 0.14899792588512642, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.14899792588512642}}, "usage": {"input_tokens": 315, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.032694407, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007170807886479, "client_end_monotonic_ns": 5007170840939087} +{"id": "hard-sol-c-judge_hard-09", "latency_s": 0.038909074, "probability_yes": 0.812841395018083, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.812841395018083}}, "usage": {"input_tokens": 346, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.038554022, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007170841030082, "client_end_monotonic_ns": 5007170879939156} +{"id": "hard-sol-c-judge_hard-10", "latency_s": 0.040432023, "probability_yes": 0.7456486341168127, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.7456486341168127}}, "usage": {"input_tokens": 375, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.040077634, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007170880034403, "client_end_monotonic_ns": 5007170920466426} +{"id": "hard-sol-c-judge_hard-11", "latency_s": 0.037100113, "probability_yes": 0.02054665416455969, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.02054665416455969}}, "usage": {"input_tokens": 346, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.036731007, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007170920550578, "client_end_monotonic_ns": 5007170957650691} +{"id": "hard-sol-c-judge_hard-13", "latency_s": 0.037118205, "probability_yes": 0.37867350884572815, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.37867350884572815}}, "usage": {"input_tokens": 336, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.036726547, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007170957735629, "client_end_monotonic_ns": 5007170994853834} +{"id": "hard-sol-c-judge_hard-15", "latency_s": 0.032116689, "probability_yes": 0.8561549717831729, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.8561549717831729}}, "usage": {"input_tokens": 290, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.031760816, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007170994940884, "client_end_monotonic_ns": 5007171027057573} diff --git a/benchmarks/gdn/artifacts/20261003/e2e/baseline-measured-2.jsonl b/benchmarks/gdn/artifacts/20261003/e2e/baseline-measured-2.jsonl new file mode 100644 index 0000000..9b6cd26 --- /dev/null +++ b/benchmarks/gdn/artifacts/20261003/e2e/baseline-measured-2.jsonl @@ -0,0 +1,74 @@ +{"id": "original-policy-01-0", "latency_s": 0.020458415, "probability_yes": 0.03047528794383215, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.03047528794383215}}, "usage": {"input_tokens": 110, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.020054809, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007171042548612, "client_end_monotonic_ns": 5007171063007027} +{"id": "original-policy-01-1", "latency_s": 0.020158814, "probability_yes": 0.03534008179644288, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.03534008179644288}}, "usage": {"input_tokens": 113, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019837215, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007171063091490, "client_end_monotonic_ns": 5007171083250304} +{"id": "original-policy-02-0", "latency_s": 0.020345022, "probability_yes": 0.9450513567824766, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.9450513567824766}}, "usage": {"input_tokens": 99, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019988085, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007171083330904, "client_end_monotonic_ns": 5007171103675926} +{"id": "original-policy-02-1", "latency_s": 0.020147824, "probability_yes": 0.9259408765258507, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.9259408765258507}}, "usage": {"input_tokens": 98, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019851157, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007171103743959, "client_end_monotonic_ns": 5007171123891783} +{"id": "original-policy-03-0", "latency_s": 0.020121646, "probability_yes": 0.8192057049596325, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.8192057049596325}}, "usage": {"input_tokens": 101, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.01986236, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007171123956564, "client_end_monotonic_ns": 5007171144078210} +{"id": "original-policy-03-1", "latency_s": 0.020044933, "probability_yes": 0.7985696425158759, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.7985696425158759}}, "usage": {"input_tokens": 100, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019775381, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007171144144190, "client_end_monotonic_ns": 5007171164189123} +{"id": "original-policy-04-0", "latency_s": 0.019855943, "probability_yes": 0.022645581582854003, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.022645581582854003}}, "usage": {"input_tokens": 107, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019568009, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007171164259621, "client_end_monotonic_ns": 5007171184115564} +{"id": "original-policy-04-1", "latency_s": 0.02008529, "probability_yes": 0.024342206375083403, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.024342206375083403}}, "usage": {"input_tokens": 104, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019816943, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007171184181610, "client_end_monotonic_ns": 5007171204266900} +{"id": "original-policy-05-0", "latency_s": 0.019985014, "probability_yes": 0.04070117194736392, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.04070117194736392}}, "usage": {"input_tokens": 111, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019705142, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007171204335862, "client_end_monotonic_ns": 5007171224320876} +{"id": "original-policy-05-1", "latency_s": 0.020108519, "probability_yes": 0.05542359965397362, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.05542359965397362}}, "usage": {"input_tokens": 103, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019856878, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007171224381237, "client_end_monotonic_ns": 5007171244489756} +{"id": "original-policy-06-0", "latency_s": 0.019941503, "probability_yes": 0.9354533714741977, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.9354533714741977}}, "usage": {"input_tokens": 113, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019706733, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007171244550670, "client_end_monotonic_ns": 5007171264492173} +{"id": "original-policy-06-1", "latency_s": 0.0200763, "probability_yes": 0.26687647588636787, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.26687647588636787}}, "usage": {"input_tokens": 103, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019817169, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007171264547492, "client_end_monotonic_ns": 5007171284623792} +{"id": "original-adequacy-01-0", "latency_s": 0.019909038, "probability_yes": 0.7544465755976671, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.7544465755976671}}, "usage": {"input_tokens": 92, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019606846, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007171284696785, "client_end_monotonic_ns": 5007171304605823} +{"id": "original-adequacy-01-1", "latency_s": 0.019797371, "probability_yes": 0.7704937058511766, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.7704937058511766}}, "usage": {"input_tokens": 88, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019438991, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007171304676298, "client_end_monotonic_ns": 5007171324473669} +{"id": "original-adequacy-02-0", "latency_s": 0.019875639, "probability_yes": 0.09635610675757625, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.09635610675757625}}, "usage": {"input_tokens": 89, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019547358, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007171324568402, "client_end_monotonic_ns": 5007171344444041} +{"id": "original-adequacy-02-1", "latency_s": 0.019863211, "probability_yes": 0.08394373314289287, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.08394373314289287}}, "usage": {"input_tokens": 90, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019549903, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007171344520603, "client_end_monotonic_ns": 5007171364383814} +{"id": "original-adequacy-03-0", "latency_s": 0.019891498, "probability_yes": 0.17073801840382533, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.17073801840382533}}, "usage": {"input_tokens": 92, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019595466, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007171364452359, "client_end_monotonic_ns": 5007171384343857} +{"id": "original-adequacy-03-1", "latency_s": 0.019898643, "probability_yes": 0.22713849981308368, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.22713849981308368}}, "usage": {"input_tokens": 96, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019634575, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007171384412069, "client_end_monotonic_ns": 5007171404310712} +{"id": "original-adequacy-04-0", "latency_s": 0.01988902, "probability_yes": 0.7525499367436955, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.7525499367436955}}, "usage": {"input_tokens": 96, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019611681, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007171404375396, "client_end_monotonic_ns": 5007171424264416} +{"id": "original-adequacy-04-1", "latency_s": 0.019858019, "probability_yes": 0.5988825971216104, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.5988825971216104}}, "usage": {"input_tokens": 93, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019575766, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007171424349158, "client_end_monotonic_ns": 5007171444207177} +{"id": "original-adequacy-05-0", "latency_s": 0.019618408, "probability_yes": 0.2599347544869691, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.2599347544869691}}, "usage": {"input_tokens": 85, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019346849, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007171444281362, "client_end_monotonic_ns": 5007171463899770} +{"id": "original-adequacy-05-1", "latency_s": 0.019714856, "probability_yes": 0.188362048820982, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.188362048820982}}, "usage": {"input_tokens": 86, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019338231, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007171463963127, "client_end_monotonic_ns": 5007171483677983} +{"id": "original-adequacy-06-0", "latency_s": 0.020272581, "probability_yes": 0.8080309304151047, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.8080309304151047}}, "usage": {"input_tokens": 101, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019836999, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007171483786145, "client_end_monotonic_ns": 5007171504058726} +{"id": "original-adequacy-06-1", "latency_s": 0.020209216, "probability_yes": 0.6635779695297708, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.6635779695297708}}, "usage": {"input_tokens": 103, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019838731, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007171504154060, "client_end_monotonic_ns": 5007171524363276} +{"id": "easy-fact-00", "latency_s": 0.019802176, "probability_yes": 0.9674054242768856, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.9674054242768856}}, "usage": {"input_tokens": 90, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019501086, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007171524444166, "client_end_monotonic_ns": 5007171544246342} +{"id": "easy-fact-01", "latency_s": 0.019650348, "probability_yes": 0.03130854188970371, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.03130854188970371}}, "usage": {"input_tokens": 88, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019346916, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007171544331868, "client_end_monotonic_ns": 5007171563982216} +{"id": "easy-fact-02", "latency_s": 0.019853977, "probability_yes": 0.9728799832337615, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.9728799832337615}}, "usage": {"input_tokens": 96, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019584917, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007171564052354, "client_end_monotonic_ns": 5007171583906331} +{"id": "easy-fact-03", "latency_s": 0.019965663, "probability_yes": 0.013935738446912068, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.013935738446912068}}, "usage": {"input_tokens": 99, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019708209, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007171583971698, "client_end_monotonic_ns": 5007171603937361} +{"id": "easy-fact-04", "latency_s": 0.019690001, "probability_yes": 0.9522894788263, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.9522894788263}}, "usage": {"input_tokens": 88, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019424986, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007171604004135, "client_end_monotonic_ns": 5007171623694136} +{"id": "easy-fact-05", "latency_s": 0.019758079, "probability_yes": 0.029410065460985898, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.029410065460985898}}, "usage": {"input_tokens": 89, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019479741, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007171623765083, "client_end_monotonic_ns": 5007171643523162} +{"id": "easy-fact-06", "latency_s": 0.020004665, "probability_yes": 0.9706899759473617, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.9706899759473617}}, "usage": {"input_tokens": 109, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019583842, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007171643608003, "client_end_monotonic_ns": 5007171663612668} +{"id": "easy-fact-07", "latency_s": 0.019574034, "probability_yes": 0.02438033353329917, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.02438033353329917}}, "usage": {"input_tokens": 84, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019296133, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007171663688193, "client_end_monotonic_ns": 5007171683262227} +{"id": "easy-fact-08", "latency_s": 0.019640315, "probability_yes": 0.9404350853099106, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.9404350853099106}}, "usage": {"input_tokens": 80, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019304669, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007171683332797, "client_end_monotonic_ns": 5007171702973112} +{"id": "easy-fact-09", "latency_s": 0.01954844, "probability_yes": 0.034599738070281126, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.034599738070281126}}, "usage": {"input_tokens": 80, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019236215, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007171703067839, "client_end_monotonic_ns": 5007171722616279} +{"id": "easy-fact-10", "latency_s": 0.019828233, "probability_yes": 0.6793076377503555, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.6793076377503555}}, "usage": {"input_tokens": 89, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.01952097, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007171722693625, "client_end_monotonic_ns": 5007171742521858} +{"id": "easy-fact-11", "latency_s": 0.019740625, "probability_yes": 0.06041778591308143, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.06041778591308143}}, "usage": {"input_tokens": 90, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.01946574, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007171742595409, "client_end_monotonic_ns": 5007171762336034} +{"id": "hard-opus-a-long_policy-11", "latency_s": 0.218167674, "probability_yes": 0.7098596537741193, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.7098596537741193}}, "usage": {"input_tokens": 2334, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.217646096, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007171762474818, "client_end_monotonic_ns": 5007171980642492} +{"id": "hard-opus-a-long_policy-13", "latency_s": 0.227704459, "probability_yes": 0.21768834160648515, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.21768834160648515}}, "usage": {"input_tokens": 2426, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.226935871, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007171980836941, "client_end_monotonic_ns": 5007172208541400} +{"id": "hard-opus-a-long_policy-19", "latency_s": 0.228296193, "probability_yes": 0.18473627906713375, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.18473627906713375}}, "usage": {"input_tokens": 2325, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.227522281, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007172208806867, "client_end_monotonic_ns": 5007172437103060} +{"id": "hard-opus-a-probability-03", "latency_s": 0.048737243, "probability_yes": 0.22782598391193778, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.22782598391193778}}, "usage": {"input_tokens": 429, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.048165675, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007172437264832, "client_end_monotonic_ns": 5007172486002075} +{"id": "hard-opus-a-probability-08", "latency_s": 0.043311509, "probability_yes": 0.19551009998444735, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.19551009998444735}}, "usage": {"input_tokens": 391, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.042788489, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007172486097916, "client_end_monotonic_ns": 5007172529409425} +{"id": "hard-opus-a-temporal_numeric-03", "latency_s": 0.049260075, "probability_yes": 0.1925798624930316, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.1925798624930316}}, "usage": {"input_tokens": 487, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.048821261, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007172529510230, "client_end_monotonic_ns": 5007172578770305} +{"id": "hard-opus-a-temporal_numeric-06", "latency_s": 0.057681389, "probability_yes": 0.054978919789340254, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.054978919789340254}}, "usage": {"input_tokens": 522, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.05721406, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007172578863009, "client_end_monotonic_ns": 5007172636544398} +{"id": "hard-opus-a-temporal_numeric-09", "latency_s": 0.054410933, "probability_yes": 0.49804298259995755, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.49804298259995755}}, "usage": {"input_tokens": 519, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.053844887, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007172636643852, "client_end_monotonic_ns": 5007172691054785} +{"id": "hard-opus-b-multi_hop-08", "latency_s": 0.21817646, "probability_yes": 0.23017195500915008, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.23017195500915008}}, "usage": {"input_tokens": 2312, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.217451779, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007172691248748, "client_end_monotonic_ns": 5007172909425208} +{"id": "hard-opus-b-probability-03", "latency_s": 0.114518669, "probability_yes": 0.1728043755923391, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.1728043755923391}}, "usage": {"input_tokens": 1213, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.113944736, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007172909568737, "client_end_monotonic_ns": 5007173024087406} +{"id": "hard-opus-b-tradeoff-06", "latency_s": 0.078509552, "probability_yes": 0.7575263998692227, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.7575263998692227}}, "usage": {"input_tokens": 790, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.078046157, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007173024207065, "client_end_monotonic_ns": 5007173102716617} +{"id": "hard-opus-b-tradeoff-08", "latency_s": 0.0601565, "probability_yes": 0.47298829137088333, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.47298829137088333}}, "usage": {"input_tokens": 601, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.059766131, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007173102826382, "client_end_monotonic_ns": 5007173162982882} +{"id": "hard-opus-c-long_policy-04", "latency_s": 0.315897746, "probability_yes": 0.5442802351228624, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.5442802351228624}}, "usage": {"input_tokens": 3312, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.315334162, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007173163170150, "client_end_monotonic_ns": 5007173479067896} +{"id": "hard-opus-c-long_policy-11", "latency_s": 0.323816472, "probability_yes": 0.5194427077546381, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.5194427077546381}}, "usage": {"input_tokens": 3399, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.323110495, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007173479288573, "client_end_monotonic_ns": 5007173803105045} +{"id": "hard-opus-c-temporal_numeric-02", "latency_s": 0.092227688, "probability_yes": 0.24180561207468904, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.24180561207468904}}, "usage": {"input_tokens": 936, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.091541793, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007173803251803, "client_end_monotonic_ns": 5007173895479491} +{"id": "hard-sol-a-adversarial-06", "latency_s": 0.027879022, "probability_yes": 0.018362853166429507, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.018362853166429507}}, "usage": {"input_tokens": 241, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.027335614, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007173895603236, "client_end_monotonic_ns": 5007173923482258} +{"id": "hard-sol-a-adversarial-08", "latency_s": 0.027569466, "probability_yes": 0.9375888015626979, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.9375888015626979}}, "usage": {"input_tokens": 218, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.027159317, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007173923588024, "client_end_monotonic_ns": 5007173951157490} +{"id": "hard-sol-a-trap-02", "latency_s": 0.032937056, "probability_yes": 0.03573969108794452, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.03573969108794452}}, "usage": {"input_tokens": 292, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.032463672, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007173951263764, "client_end_monotonic_ns": 5007173984200820} +{"id": "hard-sol-a-trap-04", "latency_s": 0.025827249, "probability_yes": 0.041796879885975066, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.041796879885975066}}, "usage": {"input_tokens": 198, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.02545228, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007173984316168, "client_end_monotonic_ns": 5007174010143417} +{"id": "hard-sol-a-trap-13", "latency_s": 0.026113619, "probability_yes": 0.030579705641278977, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.030579705641278977}}, "usage": {"input_tokens": 206, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.025747362, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007174010244008, "client_end_monotonic_ns": 5007174036357627} +{"id": "hard-sol-a-trap-15", "latency_s": 0.032627528, "probability_yes": 0.07570582831909627, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.07570582831909627}}, "usage": {"input_tokens": 308, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.032288672, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007174036452873, "client_end_monotonic_ns": 5007174069080401} +{"id": "hard-sol-b-judge_hard-01", "latency_s": 0.032263071, "probability_yes": 0.6743485180581948, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.6743485180581948}}, "usage": {"input_tokens": 288, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.031882217, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007174069170551, "client_end_monotonic_ns": 5007174101433622} +{"id": "hard-sol-b-judge_hard-02", "latency_s": 0.026752885, "probability_yes": 0.8525225556146606, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.8525225556146606}}, "usage": {"input_tokens": 212, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.026371818, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007174101535587, "client_end_monotonic_ns": 5007174128288472} +{"id": "hard-sol-b-judge_hard-05", "latency_s": 0.028507126, "probability_yes": 0.9192409990717809, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.9192409990717809}}, "usage": {"input_tokens": 231, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.027995975, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007174128399979, "client_end_monotonic_ns": 5007174156907105} +{"id": "hard-sol-b-judge_hard-08", "latency_s": 0.026753189, "probability_yes": 0.33265846433012053, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.33265846433012053}}, "usage": {"input_tokens": 214, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.026308411, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007174157023096, "client_end_monotonic_ns": 5007174183776285} +{"id": "hard-sol-b-judge_hard-10", "latency_s": 0.028388299, "probability_yes": 0.3554926261464918, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.3554926261464918}}, "usage": {"input_tokens": 233, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.028005831, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007174183874603, "client_end_monotonic_ns": 5007174212262902} +{"id": "hard-sol-b-judge_hard-14", "latency_s": 0.028105902, "probability_yes": 0.07750785419177868, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.07750785419177868}}, "usage": {"input_tokens": 250, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.027724285, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007174212356767, "client_end_monotonic_ns": 5007174240462669} +{"id": "hard-sol-b-judge_hard-15", "latency_s": 0.027283829, "probability_yes": 0.8936767977520365, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.8936767977520365}}, "usage": {"input_tokens": 206, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.026974406, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007174240564255, "client_end_monotonic_ns": 5007174267848084} +{"id": "hard-sol-b-judge_hard-18", "latency_s": 0.024898069, "probability_yes": 0.07374139181136073, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.07374139181136073}}, "usage": {"input_tokens": 173, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.024624818, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007174267934782, "client_end_monotonic_ns": 5007174292832851} +{"id": "hard-sol-c-judge_hard-03", "latency_s": 0.042847525, "probability_yes": 0.11480257968157337, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.11480257968157337}}, "usage": {"input_tokens": 395, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.042546171, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007174292919849, "client_end_monotonic_ns": 5007174335767374} +{"id": "hard-sol-c-judge_hard-05", "latency_s": 0.039532976, "probability_yes": 0.7398933422307209, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.7398933422307209}}, "usage": {"input_tokens": 357, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.039162083, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007174335858615, "client_end_monotonic_ns": 5007174375391591} +{"id": "hard-sol-c-judge_hard-07", "latency_s": 0.037695883, "probability_yes": 0.5950171562428843, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.5950171562428843}}, "usage": {"input_tokens": 321, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.03731455, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007174375478765, "client_end_monotonic_ns": 5007174413174648} +{"id": "hard-sol-c-judge_hard-08", "latency_s": 0.033323485, "probability_yes": 0.14899792588512642, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.14899792588512642}}, "usage": {"input_tokens": 315, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.032948048, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007174413280491, "client_end_monotonic_ns": 5007174446603976} +{"id": "hard-sol-c-judge_hard-09", "latency_s": 0.038428529, "probability_yes": 0.812841395018083, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.812841395018083}}, "usage": {"input_tokens": 346, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.038075135, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007174446707968, "client_end_monotonic_ns": 5007174485136497} +{"id": "hard-sol-c-judge_hard-10", "latency_s": 0.04057913, "probability_yes": 0.7456486341168127, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.7456486341168127}}, "usage": {"input_tokens": 375, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.040222904, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007174485237371, "client_end_monotonic_ns": 5007174525816501} +{"id": "hard-sol-c-judge_hard-11", "latency_s": 0.038643777, "probability_yes": 0.02054665416455969, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.02054665416455969}}, "usage": {"input_tokens": 346, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.038331379, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007174525903886, "client_end_monotonic_ns": 5007174564547663} +{"id": "hard-sol-c-judge_hard-13", "latency_s": 0.037169075, "probability_yes": 0.37867350884572815, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.37867350884572815}}, "usage": {"input_tokens": 336, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.036845298, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007174564633416, "client_end_monotonic_ns": 5007174601802491} +{"id": "hard-sol-c-judge_hard-15", "latency_s": 0.032458974, "probability_yes": 0.8561549717831729, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.8561549717831729}}, "usage": {"input_tokens": 290, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.032146843, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007174601891542, "client_end_monotonic_ns": 5007174634350516} diff --git a/benchmarks/gdn/artifacts/20261003/e2e/candidate-measured-1.jsonl b/benchmarks/gdn/artifacts/20261003/e2e/candidate-measured-1.jsonl new file mode 100644 index 0000000..4a4259b --- /dev/null +++ b/benchmarks/gdn/artifacts/20261003/e2e/candidate-measured-1.jsonl @@ -0,0 +1,74 @@ +{"id": "original-policy-01-0", "latency_s": 0.020488631, "probability_yes": 0.03047528794383215, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.03047528794383215}}, "usage": {"input_tokens": 110, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.020029621, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007189829021116, "client_end_monotonic_ns": 5007189849509747} +{"id": "original-policy-01-1", "latency_s": 0.020190618, "probability_yes": 0.03534008179644288, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.03534008179644288}}, "usage": {"input_tokens": 113, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019832162, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007189849618653, "client_end_monotonic_ns": 5007189869809271} +{"id": "original-policy-02-0", "latency_s": 0.020316195, "probability_yes": 0.9450513567824766, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.9450513567824766}}, "usage": {"input_tokens": 99, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019901851, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007189869889847, "client_end_monotonic_ns": 5007189890206042} +{"id": "original-policy-02-1", "latency_s": 0.020223447, "probability_yes": 0.9259408765258507, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.9259408765258507}}, "usage": {"input_tokens": 98, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.01982701, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007189890322606, "client_end_monotonic_ns": 5007189910546053} +{"id": "original-policy-03-0", "latency_s": 0.020192077, "probability_yes": 0.8192057049596325, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.8192057049596325}}, "usage": {"input_tokens": 101, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.01985999, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007189910635252, "client_end_monotonic_ns": 5007189930827329} +{"id": "original-policy-03-1", "latency_s": 0.020085592, "probability_yes": 0.7985696425158759, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.7985696425158759}}, "usage": {"input_tokens": 100, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019787036, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007189930910156, "client_end_monotonic_ns": 5007189950995748} +{"id": "original-policy-04-0", "latency_s": 0.019848269, "probability_yes": 0.022645581582854003, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.022645581582854003}}, "usage": {"input_tokens": 107, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.01954134, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007189951065143, "client_end_monotonic_ns": 5007189970913412} +{"id": "original-policy-04-1", "latency_s": 0.020122537, "probability_yes": 0.024342206375083403, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.024342206375083403}}, "usage": {"input_tokens": 104, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019819647, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007189970985533, "client_end_monotonic_ns": 5007189991108070} +{"id": "original-policy-05-0", "latency_s": 0.020012508, "probability_yes": 0.04070117194736392, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.04070117194736392}}, "usage": {"input_tokens": 111, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.01968749, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007189991182985, "client_end_monotonic_ns": 5007190011195493} +{"id": "original-policy-05-1", "latency_s": 0.020110243, "probability_yes": 0.05542359965397362, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.05542359965397362}}, "usage": {"input_tokens": 103, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019789655, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007190011278617, "client_end_monotonic_ns": 5007190031388860} +{"id": "original-policy-06-0", "latency_s": 0.019978891, "probability_yes": 0.9354533714741977, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.9354533714741977}}, "usage": {"input_tokens": 113, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019687948, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007190031466090, "client_end_monotonic_ns": 5007190051444981} +{"id": "original-policy-06-1", "latency_s": 0.020089535, "probability_yes": 0.26687647588636787, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.26687647588636787}}, "usage": {"input_tokens": 103, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019808136, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007190051513932, "client_end_monotonic_ns": 5007190071603467} +{"id": "original-adequacy-01-0", "latency_s": 0.019847453, "probability_yes": 0.7544465755976671, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.7544465755976671}}, "usage": {"input_tokens": 92, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019573188, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007190071679621, "client_end_monotonic_ns": 5007190091527074} +{"id": "original-adequacy-01-1", "latency_s": 0.019689221, "probability_yes": 0.7704937058511766, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.7704937058511766}}, "usage": {"input_tokens": 88, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019408344, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007190091594671, "client_end_monotonic_ns": 5007190111283892} +{"id": "original-adequacy-02-0", "latency_s": 0.019867399, "probability_yes": 0.09635610675757625, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.09635610675757625}}, "usage": {"input_tokens": 89, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019525785, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007190111353199, "client_end_monotonic_ns": 5007190131220598} +{"id": "original-adequacy-02-1", "latency_s": 0.019805229, "probability_yes": 0.08394373314289287, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.08394373314289287}}, "usage": {"input_tokens": 90, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019488039, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007190131289077, "client_end_monotonic_ns": 5007190151094306} +{"id": "original-adequacy-03-0", "latency_s": 0.01982934, "probability_yes": 0.17073801840382533, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.17073801840382533}}, "usage": {"input_tokens": 92, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019556394, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007190151154881, "client_end_monotonic_ns": 5007190170984221} +{"id": "original-adequacy-03-1", "latency_s": 0.019818913, "probability_yes": 0.22713849981308368, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.22713849981308368}}, "usage": {"input_tokens": 96, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019565176, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007190171050910, "client_end_monotonic_ns": 5007190190869823} +{"id": "original-adequacy-04-0", "latency_s": 0.019818895, "probability_yes": 0.7525499367436955, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.7525499367436955}}, "usage": {"input_tokens": 96, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019558733, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007190190931722, "client_end_monotonic_ns": 5007190210750617} +{"id": "original-adequacy-04-1", "latency_s": 0.019716309, "probability_yes": 0.5988825971216104, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.5988825971216104}}, "usage": {"input_tokens": 93, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019484547, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007190210809989, "client_end_monotonic_ns": 5007190230526298} +{"id": "original-adequacy-05-0", "latency_s": 0.019587196, "probability_yes": 0.2599347544869691, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.2599347544869691}}, "usage": {"input_tokens": 85, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019344956, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007190230592368, "client_end_monotonic_ns": 5007190250179564} +{"id": "original-adequacy-05-1", "latency_s": 0.019573336, "probability_yes": 0.188362048820982, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.188362048820982}}, "usage": {"input_tokens": 86, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.01931778, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007190250239088, "client_end_monotonic_ns": 5007190269812424} +{"id": "original-adequacy-06-0", "latency_s": 0.020049217, "probability_yes": 0.8080309304151047, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.8080309304151047}}, "usage": {"input_tokens": 101, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.01976714, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007190269879329, "client_end_monotonic_ns": 5007190289928546} +{"id": "original-adequacy-06-1", "latency_s": 0.020155039, "probability_yes": 0.6635779695297708, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.6635779695297708}}, "usage": {"input_tokens": 103, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019764663, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007190290005482, "client_end_monotonic_ns": 5007190310160521} +{"id": "easy-fact-00", "latency_s": 0.019894939, "probability_yes": 0.9674054242768856, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.9674054242768856}}, "usage": {"input_tokens": 90, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019556191, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007190310226347, "client_end_monotonic_ns": 5007190330121286} +{"id": "easy-fact-01", "latency_s": 0.019649805, "probability_yes": 0.03130854188970371, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.03130854188970371}}, "usage": {"input_tokens": 88, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019379992, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007190330189953, "client_end_monotonic_ns": 5007190349839758} +{"id": "easy-fact-02", "latency_s": 0.019879207, "probability_yes": 0.9728799832337615, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.9728799832337615}}, "usage": {"input_tokens": 96, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019617181, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007190349908220, "client_end_monotonic_ns": 5007190369787427} +{"id": "easy-fact-03", "latency_s": 0.020007443, "probability_yes": 0.013935738446912068, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.013935738446912068}}, "usage": {"input_tokens": 99, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019746291, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007190369854273, "client_end_monotonic_ns": 5007190389861716} +{"id": "easy-fact-04", "latency_s": 0.019744134, "probability_yes": 0.9522894788263, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.9522894788263}}, "usage": {"input_tokens": 88, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019462253, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007190389938526, "client_end_monotonic_ns": 5007190409682660} +{"id": "easy-fact-05", "latency_s": 0.019972084, "probability_yes": 0.029410065460985898, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.029410065460985898}}, "usage": {"input_tokens": 89, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019580872, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007190409757750, "client_end_monotonic_ns": 5007190429729834} +{"id": "easy-fact-06", "latency_s": 0.01989398, "probability_yes": 0.9706899759473617, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.9706899759473617}}, "usage": {"input_tokens": 109, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019546397, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007190429849401, "client_end_monotonic_ns": 5007190449743381} +{"id": "easy-fact-07", "latency_s": 0.019627783, "probability_yes": 0.02438033353329917, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.02438033353329917}}, "usage": {"input_tokens": 84, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019332537, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007190449827963, "client_end_monotonic_ns": 5007190469455746} +{"id": "easy-fact-08", "latency_s": 0.019566298, "probability_yes": 0.9404350853099106, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.9404350853099106}}, "usage": {"input_tokens": 80, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019304011, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007190469527560, "client_end_monotonic_ns": 5007190489093858} +{"id": "easy-fact-09", "latency_s": 0.019526828, "probability_yes": 0.034599738070281126, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.034599738070281126}}, "usage": {"input_tokens": 80, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019265761, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007190489162137, "client_end_monotonic_ns": 5007190508688965} +{"id": "easy-fact-10", "latency_s": 0.019693121, "probability_yes": 0.6793076377503555, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.6793076377503555}}, "usage": {"input_tokens": 89, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019454423, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007190508762485, "client_end_monotonic_ns": 5007190528455606} +{"id": "easy-fact-11", "latency_s": 0.019675295, "probability_yes": 0.06041778591308143, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.06041778591308143}}, "usage": {"input_tokens": 90, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019445211, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007190528527012, "client_end_monotonic_ns": 5007190548202307} +{"id": "hard-opus-a-long_policy-11", "latency_s": 0.213862995, "probability_yes": 0.7098596537741193, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.7098596537741193}}, "usage": {"input_tokens": 2334, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.213405635, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007190548344259, "client_end_monotonic_ns": 5007190762207254} +{"id": "hard-opus-a-long_policy-13", "latency_s": 0.225918293, "probability_yes": 0.21768834160648515, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.21768834160648515}}, "usage": {"input_tokens": 2426, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.225363496, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007190762392539, "client_end_monotonic_ns": 5007190988310832} +{"id": "hard-opus-a-long_policy-19", "latency_s": 0.226993519, "probability_yes": 0.18473627906713375, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.18473627906713375}}, "usage": {"input_tokens": 2325, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.226390073, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007190988471929, "client_end_monotonic_ns": 5007191215465448} +{"id": "hard-opus-a-probability-03", "latency_s": 0.046686735, "probability_yes": 0.22782598391193778, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.22782598391193778}}, "usage": {"input_tokens": 429, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.04619694, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007191215601786, "client_end_monotonic_ns": 5007191262288521} +{"id": "hard-opus-a-probability-08", "latency_s": 0.041814568, "probability_yes": 0.19551009998444735, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.19551009998444735}}, "usage": {"input_tokens": 391, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.041384742, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007191262381547, "client_end_monotonic_ns": 5007191304196115} +{"id": "hard-opus-a-temporal_numeric-03", "latency_s": 0.050126521, "probability_yes": 0.1925798624930316, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.1925798624930316}}, "usage": {"input_tokens": 487, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.049669805, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007191304317513, "client_end_monotonic_ns": 5007191354444034} +{"id": "hard-opus-a-temporal_numeric-06", "latency_s": 0.05938195, "probability_yes": 0.054978919789340254, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.054978919789340254}}, "usage": {"input_tokens": 522, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.058957579, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007191354548512, "client_end_monotonic_ns": 5007191413930462} +{"id": "hard-opus-a-temporal_numeric-09", "latency_s": 0.055016009, "probability_yes": 0.49804298259995755, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.49804298259995755}}, "usage": {"input_tokens": 519, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.054617544, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007191414030190, "client_end_monotonic_ns": 5007191469046199} +{"id": "hard-opus-b-multi_hop-08", "latency_s": 0.215921032, "probability_yes": 0.23017195500915008, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.23017195500915008}}, "usage": {"input_tokens": 2312, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.215369344, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007191469201853, "client_end_monotonic_ns": 5007191685122885} +{"id": "hard-opus-b-probability-03", "latency_s": 0.11400741, "probability_yes": 0.1728043755923391, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.1728043755923391}}, "usage": {"input_tokens": 1213, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.113471104, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007191685253156, "client_end_monotonic_ns": 5007191799260566} +{"id": "hard-opus-b-tradeoff-06", "latency_s": 0.077380201, "probability_yes": 0.7575263998692227, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.7575263998692227}}, "usage": {"input_tokens": 790, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.076875955, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007191799390124, "client_end_monotonic_ns": 5007191876770325} +{"id": "hard-opus-b-tradeoff-08", "latency_s": 0.059989996, "probability_yes": 0.47298829137088333, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.47298829137088333}}, "usage": {"input_tokens": 601, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.059518699, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007191876892375, "client_end_monotonic_ns": 5007191936882371} +{"id": "hard-opus-c-long_policy-04", "latency_s": 0.315673906, "probability_yes": 0.5442802351228624, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.5442802351228624}}, "usage": {"input_tokens": 3312, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.315054234, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007191937067215, "client_end_monotonic_ns": 5007192252741121} +{"id": "hard-opus-c-long_policy-11", "latency_s": 0.324642425, "probability_yes": 0.5194427077546381, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.5194427077546381}}, "usage": {"input_tokens": 3399, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.323927419, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007192252969487, "client_end_monotonic_ns": 5007192577611912} +{"id": "hard-opus-c-temporal_numeric-02", "latency_s": 0.09130927, "probability_yes": 0.24180561207468904, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.24180561207468904}}, "usage": {"input_tokens": 936, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.090628629, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007192577765784, "client_end_monotonic_ns": 5007192669075054} +{"id": "hard-sol-a-adversarial-06", "latency_s": 0.028250795, "probability_yes": 0.018362853166429507, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.018362853166429507}}, "usage": {"input_tokens": 241, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.027743634, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007192669212049, "client_end_monotonic_ns": 5007192697462844} +{"id": "hard-sol-a-adversarial-08", "latency_s": 0.028128444, "probability_yes": 0.9375888015626979, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.9375888015626979}}, "usage": {"input_tokens": 218, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.027737302, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007192697572800, "client_end_monotonic_ns": 5007192725701244} +{"id": "hard-sol-a-trap-02", "latency_s": 0.03312639, "probability_yes": 0.03573969108794452, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.03573969108794452}}, "usage": {"input_tokens": 292, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.032755299, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007192725799774, "client_end_monotonic_ns": 5007192758926164} +{"id": "hard-sol-a-trap-04", "latency_s": 0.02581685, "probability_yes": 0.041796879885975066, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.041796879885975066}}, "usage": {"input_tokens": 198, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.025420594, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007192759024673, "client_end_monotonic_ns": 5007192784841523} +{"id": "hard-sol-a-trap-13", "latency_s": 0.025861943, "probability_yes": 0.030579705641278977, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.030579705641278977}}, "usage": {"input_tokens": 206, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.025530838, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007192784922517, "client_end_monotonic_ns": 5007192810784460} +{"id": "hard-sol-a-trap-15", "latency_s": 0.032516479, "probability_yes": 0.07570582831909627, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.07570582831909627}}, "usage": {"input_tokens": 308, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.032223157, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007192810865303, "client_end_monotonic_ns": 5007192843381782} +{"id": "hard-sol-b-judge_hard-01", "latency_s": 0.031719378, "probability_yes": 0.6743485180581948, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.6743485180581948}}, "usage": {"input_tokens": 288, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.031432261, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007192843462542, "client_end_monotonic_ns": 5007192875181920} +{"id": "hard-sol-b-judge_hard-02", "latency_s": 0.026064619, "probability_yes": 0.8525225556146606, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.8525225556146606}}, "usage": {"input_tokens": 212, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.025779469, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007192875257618, "client_end_monotonic_ns": 5007192901322237} +{"id": "hard-sol-b-judge_hard-05", "latency_s": 0.02782718, "probability_yes": 0.9192409990717809, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.9192409990717809}}, "usage": {"input_tokens": 231, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.027567317, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007192901400441, "client_end_monotonic_ns": 5007192929227621} +{"id": "hard-sol-b-judge_hard-08", "latency_s": 0.026164987, "probability_yes": 0.33265846433012053, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.33265846433012053}}, "usage": {"input_tokens": 214, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.025913336, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007192929299840, "client_end_monotonic_ns": 5007192955464827} +{"id": "hard-sol-b-judge_hard-10", "latency_s": 0.027395205, "probability_yes": 0.3554926261464918, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.3554926261464918}}, "usage": {"input_tokens": 233, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.027095889, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007192955535478, "client_end_monotonic_ns": 5007192982930683} +{"id": "hard-sol-b-judge_hard-14", "latency_s": 0.027228437, "probability_yes": 0.07750785419177868, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.07750785419177868}}, "usage": {"input_tokens": 250, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.026950499, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007192983010012, "client_end_monotonic_ns": 5007193010238449} +{"id": "hard-sol-b-judge_hard-15", "latency_s": 0.026093138, "probability_yes": 0.8936767977520365, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.8936767977520365}}, "usage": {"input_tokens": 206, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.025797779, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007193010321226, "client_end_monotonic_ns": 5007193036414364} +{"id": "hard-sol-b-judge_hard-18", "latency_s": 0.023887688, "probability_yes": 0.07374139181136073, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.07374139181136073}}, "usage": {"input_tokens": 173, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.023602068, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007193036497912, "client_end_monotonic_ns": 5007193060385600} +{"id": "hard-sol-c-judge_hard-03", "latency_s": 0.040669623, "probability_yes": 0.11480257968157337, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.11480257968157337}}, "usage": {"input_tokens": 395, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.040294082, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007193060465331, "client_end_monotonic_ns": 5007193101134954} +{"id": "hard-sol-c-judge_hard-05", "latency_s": 0.038380082, "probability_yes": 0.7398933422307209, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.7398933422307209}}, "usage": {"input_tokens": 357, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.037980233, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007193101255965, "client_end_monotonic_ns": 5007193139636047} +{"id": "hard-sol-c-judge_hard-07", "latency_s": 0.03683791, "probability_yes": 0.5950171562428843, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.5950171562428843}}, "usage": {"input_tokens": 321, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.036471383, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007193139727487, "client_end_monotonic_ns": 5007193176565397} +{"id": "hard-sol-c-judge_hard-08", "latency_s": 0.033916803, "probability_yes": 0.14899792588512642, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.14899792588512642}}, "usage": {"input_tokens": 315, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.033565002, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007193176668364, "client_end_monotonic_ns": 5007193210585167} +{"id": "hard-sol-c-judge_hard-09", "latency_s": 0.037917548, "probability_yes": 0.812841395018083, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.812841395018083}}, "usage": {"input_tokens": 346, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.037606593, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007193210675202, "client_end_monotonic_ns": 5007193248592750} +{"id": "hard-sol-c-judge_hard-10", "latency_s": 0.039661431, "probability_yes": 0.7456486341168127, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.7456486341168127}}, "usage": {"input_tokens": 375, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.039360736, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007193248677300, "client_end_monotonic_ns": 5007193288338731} +{"id": "hard-sol-c-judge_hard-11", "latency_s": 0.037347603, "probability_yes": 0.02054665416455969, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.02054665416455969}}, "usage": {"input_tokens": 346, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.037009656, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007193288417161, "client_end_monotonic_ns": 5007193325764764} +{"id": "hard-sol-c-judge_hard-13", "latency_s": 0.037126906, "probability_yes": 0.37867350884572815, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.37867350884572815}}, "usage": {"input_tokens": 336, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.036773238, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007193325860132, "client_end_monotonic_ns": 5007193362987038} +{"id": "hard-sol-c-judge_hard-15", "latency_s": 0.032526282, "probability_yes": 0.8561549717831729, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.8561549717831729}}, "usage": {"input_tokens": 290, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.032179325, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007193363085433, "client_end_monotonic_ns": 5007193395611715} diff --git a/benchmarks/gdn/artifacts/20261003/e2e/candidate-measured-2.jsonl b/benchmarks/gdn/artifacts/20261003/e2e/candidate-measured-2.jsonl new file mode 100644 index 0000000..b321a2b --- /dev/null +++ b/benchmarks/gdn/artifacts/20261003/e2e/candidate-measured-2.jsonl @@ -0,0 +1,74 @@ +{"id": "original-policy-01-0", "latency_s": 0.02110884, "probability_yes": 0.03047528794383215, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.03047528794383215}}, "usage": {"input_tokens": 110, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.020564316, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007193413093878, "client_end_monotonic_ns": 5007193434202718} +{"id": "original-policy-01-1", "latency_s": 0.020608072, "probability_yes": 0.03534008179644288, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.03534008179644288}}, "usage": {"input_tokens": 113, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.020229652, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007193434321249, "client_end_monotonic_ns": 5007193454929321} +{"id": "original-policy-02-0", "latency_s": 0.021372746, "probability_yes": 0.9450513567824766, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.9450513567824766}}, "usage": {"input_tokens": 99, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.020863042, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007193455022462, "client_end_monotonic_ns": 5007193476395208} +{"id": "original-policy-02-1", "latency_s": 0.020300463, "probability_yes": 0.9259408765258507, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.9259408765258507}}, "usage": {"input_tokens": 98, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019945741, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007193476482652, "client_end_monotonic_ns": 5007193496783115} +{"id": "original-policy-03-0", "latency_s": 0.020239247, "probability_yes": 0.8192057049596325, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.8192057049596325}}, "usage": {"input_tokens": 101, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019880738, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007193496871744, "client_end_monotonic_ns": 5007193517110991} +{"id": "original-policy-03-1", "latency_s": 0.020336491, "probability_yes": 0.7985696425158759, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.7985696425158759}}, "usage": {"input_tokens": 100, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019830633, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007193517206506, "client_end_monotonic_ns": 5007193537542997} +{"id": "original-policy-04-0", "latency_s": 0.019891498, "probability_yes": 0.022645581582854003, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.022645581582854003}}, "usage": {"input_tokens": 107, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.01955486, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007193537626900, "client_end_monotonic_ns": 5007193557518398} +{"id": "original-policy-04-1", "latency_s": 0.020237059, "probability_yes": 0.024342206375083403, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.024342206375083403}}, "usage": {"input_tokens": 104, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019906148, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007193557606374, "client_end_monotonic_ns": 5007193577843433} +{"id": "original-policy-05-0", "latency_s": 0.020070689, "probability_yes": 0.04070117194736392, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.04070117194736392}}, "usage": {"input_tokens": 111, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019732096, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007193577926127, "client_end_monotonic_ns": 5007193597996816} +{"id": "original-policy-05-1", "latency_s": 0.020339783, "probability_yes": 0.05542359965397362, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.05542359965397362}}, "usage": {"input_tokens": 103, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.01998229, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007193598081240, "client_end_monotonic_ns": 5007193618421023} +{"id": "original-policy-06-0", "latency_s": 0.020122044, "probability_yes": 0.9354533714741977, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.9354533714741977}}, "usage": {"input_tokens": 113, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.01977702, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007193618512828, "client_end_monotonic_ns": 5007193638634872} +{"id": "original-policy-06-1", "latency_s": 0.020282716, "probability_yes": 0.26687647588636787, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.26687647588636787}}, "usage": {"input_tokens": 103, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019919678, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007193638716817, "client_end_monotonic_ns": 5007193658999533} +{"id": "original-adequacy-01-0", "latency_s": 0.020012957, "probability_yes": 0.7544465755976671, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.7544465755976671}}, "usage": {"input_tokens": 92, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019620074, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007193659080638, "client_end_monotonic_ns": 5007193679093595} +{"id": "original-adequacy-01-1", "latency_s": 0.019868988, "probability_yes": 0.7704937058511766, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.7704937058511766}}, "usage": {"input_tokens": 88, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019465609, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007193679173227, "client_end_monotonic_ns": 5007193699042215} +{"id": "original-adequacy-02-0", "latency_s": 0.019941031, "probability_yes": 0.09635610675757625, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.09635610675757625}}, "usage": {"input_tokens": 89, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.01954328, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007193699136867, "client_end_monotonic_ns": 5007193719077898} +{"id": "original-adequacy-02-1", "latency_s": 0.019863299, "probability_yes": 0.08394373314289287, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.08394373314289287}}, "usage": {"input_tokens": 90, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019517865, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007193719176023, "client_end_monotonic_ns": 5007193739039322} +{"id": "original-adequacy-03-0", "latency_s": 0.019922858, "probability_yes": 0.17073801840382533, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.17073801840382533}}, "usage": {"input_tokens": 92, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019578005, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007193739122131, "client_end_monotonic_ns": 5007193759044989} +{"id": "original-adequacy-03-1", "latency_s": 0.01996346, "probability_yes": 0.22713849981308368, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.22713849981308368}}, "usage": {"input_tokens": 96, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019598782, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007193759127386, "client_end_monotonic_ns": 5007193779090846} +{"id": "original-adequacy-04-0", "latency_s": 0.02004677, "probability_yes": 0.7525499367436955, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.7525499367436955}}, "usage": {"input_tokens": 96, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.01964561, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007193779179768, "client_end_monotonic_ns": 5007193799226538} +{"id": "original-adequacy-04-1", "latency_s": 0.019966763, "probability_yes": 0.5988825971216104, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.5988825971216104}}, "usage": {"input_tokens": 93, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.01958985, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007193799322693, "client_end_monotonic_ns": 5007193819289456} +{"id": "original-adequacy-05-0", "latency_s": 0.019731559, "probability_yes": 0.2599347544869691, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.2599347544869691}}, "usage": {"input_tokens": 85, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019395033, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007193819384929, "client_end_monotonic_ns": 5007193839116488} +{"id": "original-adequacy-05-1", "latency_s": 0.019736072, "probability_yes": 0.188362048820982, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.188362048820982}}, "usage": {"input_tokens": 86, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019382971, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007193839199178, "client_end_monotonic_ns": 5007193858935250} +{"id": "original-adequacy-06-0", "latency_s": 0.020199999, "probability_yes": 0.8080309304151047, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.8080309304151047}}, "usage": {"input_tokens": 101, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019871794, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007193859020118, "client_end_monotonic_ns": 5007193879220117} +{"id": "original-adequacy-06-1", "latency_s": 0.020185964, "probability_yes": 0.6635779695297708, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.6635779695297708}}, "usage": {"input_tokens": 103, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019857723, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007193879310388, "client_end_monotonic_ns": 5007193899496352} +{"id": "easy-fact-00", "latency_s": 0.019890338, "probability_yes": 0.9674054242768856, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.9674054242768856}}, "usage": {"input_tokens": 90, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.01957488, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007193899582086, "client_end_monotonic_ns": 5007193919472424} +{"id": "easy-fact-01", "latency_s": 0.019721907, "probability_yes": 0.03130854188970371, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.03130854188970371}}, "usage": {"input_tokens": 88, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019421599, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007193919563815, "client_end_monotonic_ns": 5007193939285722} +{"id": "easy-fact-02", "latency_s": 0.019963567, "probability_yes": 0.9728799832337615, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.9728799832337615}}, "usage": {"input_tokens": 96, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019612713, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007193939377777, "client_end_monotonic_ns": 5007193959341344} +{"id": "easy-fact-03", "latency_s": 0.020110031, "probability_yes": 0.013935738446912068, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.013935738446912068}}, "usage": {"input_tokens": 99, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019756488, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007193959426630, "client_end_monotonic_ns": 5007193979536661} +{"id": "easy-fact-04", "latency_s": 0.019755336, "probability_yes": 0.9522894788263, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.9522894788263}}, "usage": {"input_tokens": 88, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019384984, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007193979620546, "client_end_monotonic_ns": 5007193999375882} +{"id": "easy-fact-05", "latency_s": 0.019838846, "probability_yes": 0.029410065460985898, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.029410065460985898}}, "usage": {"input_tokens": 89, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.01949533, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007193999463261, "client_end_monotonic_ns": 5007194019302107} +{"id": "easy-fact-06", "latency_s": 0.019949204, "probability_yes": 0.9706899759473617, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.9706899759473617}}, "usage": {"input_tokens": 109, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019593533, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007194019394465, "client_end_monotonic_ns": 5007194039343669} +{"id": "easy-fact-07", "latency_s": 0.019670462, "probability_yes": 0.02438033353329917, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.02438033353329917}}, "usage": {"input_tokens": 84, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.01935688, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007194039427362, "client_end_monotonic_ns": 5007194059097824} +{"id": "easy-fact-08", "latency_s": 0.019638356, "probability_yes": 0.9404350853099106, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.9404350853099106}}, "usage": {"input_tokens": 80, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019319437, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007194059176533, "client_end_monotonic_ns": 5007194078814889} +{"id": "easy-fact-09", "latency_s": 0.019627577, "probability_yes": 0.034599738070281126, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.034599738070281126}}, "usage": {"input_tokens": 80, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019313218, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007194078902122, "client_end_monotonic_ns": 5007194098529699} +{"id": "easy-fact-10", "latency_s": 0.019867735, "probability_yes": 0.6793076377503555, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.6793076377503555}}, "usage": {"input_tokens": 89, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019535021, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007194098614423, "client_end_monotonic_ns": 5007194118482158} +{"id": "easy-fact-11", "latency_s": 0.019978326, "probability_yes": 0.06041778591308143, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.06041778591308143}}, "usage": {"input_tokens": 90, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.019528759, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007194118569480, "client_end_monotonic_ns": 5007194138547806} +{"id": "hard-opus-a-long_policy-11", "latency_s": 0.213479556, "probability_yes": 0.7098596537741193, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.7098596537741193}}, "usage": {"input_tokens": 2334, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.212873629, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007194138728413, "client_end_monotonic_ns": 5007194352207969} +{"id": "hard-opus-a-long_policy-13", "latency_s": 0.231014741, "probability_yes": 0.21768834160648515, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.21768834160648515}}, "usage": {"input_tokens": 2426, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.23034054, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007194352408721, "client_end_monotonic_ns": 5007194583423462} +{"id": "hard-opus-a-long_policy-19", "latency_s": 0.225791165, "probability_yes": 0.18473627906713375, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.18473627906713375}}, "usage": {"input_tokens": 2325, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.225083117, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007194583616609, "client_end_monotonic_ns": 5007194809407774} +{"id": "hard-opus-a-probability-03", "latency_s": 0.046791434, "probability_yes": 0.22782598391193778, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.22782598391193778}}, "usage": {"input_tokens": 429, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.046194315, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007194809561680, "client_end_monotonic_ns": 5007194856353114} +{"id": "hard-opus-a-probability-08", "latency_s": 0.041355917, "probability_yes": 0.19551009998444735, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.19551009998444735}}, "usage": {"input_tokens": 391, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.040969272, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007194856462351, "client_end_monotonic_ns": 5007194897818268} +{"id": "hard-opus-a-temporal_numeric-03", "latency_s": 0.049830509, "probability_yes": 0.1925798624930316, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.1925798624930316}}, "usage": {"input_tokens": 487, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.049420189, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007194897905389, "client_end_monotonic_ns": 5007194947735898} +{"id": "hard-opus-a-temporal_numeric-06", "latency_s": 0.056430421, "probability_yes": 0.054978919789340254, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.054978919789340254}}, "usage": {"input_tokens": 522, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.055952071, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007194947834297, "client_end_monotonic_ns": 5007195004264718} +{"id": "hard-opus-a-temporal_numeric-09", "latency_s": 0.052705006, "probability_yes": 0.49804298259995755, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.49804298259995755}}, "usage": {"input_tokens": 519, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.052221999, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007195004380132, "client_end_monotonic_ns": 5007195057085138} +{"id": "hard-opus-b-multi_hop-08", "latency_s": 0.214206521, "probability_yes": 0.23017195500915008, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.23017195500915008}}, "usage": {"input_tokens": 2312, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.213502079, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007195057231392, "client_end_monotonic_ns": 5007195271437913} +{"id": "hard-opus-b-probability-03", "latency_s": 0.114468215, "probability_yes": 0.1728043755923391, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.1728043755923391}}, "usage": {"input_tokens": 1213, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.11376452, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007195271593283, "client_end_monotonic_ns": 5007195386061498} +{"id": "hard-opus-b-tradeoff-06", "latency_s": 0.079101175, "probability_yes": 0.7575263998692227, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.7575263998692227}}, "usage": {"input_tokens": 790, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.078296905, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007195386206439, "client_end_monotonic_ns": 5007195465307614} +{"id": "hard-opus-b-tradeoff-08", "latency_s": 0.060853172, "probability_yes": 0.47298829137088333, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.47298829137088333}}, "usage": {"input_tokens": 601, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.060191014, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007195465449615, "client_end_monotonic_ns": 5007195526302787} +{"id": "hard-opus-c-long_policy-04", "latency_s": 0.31645678, "probability_yes": 0.5442802351228624, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.5442802351228624}}, "usage": {"input_tokens": 3312, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.31575317, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007195526507599, "client_end_monotonic_ns": 5007195842964379} +{"id": "hard-opus-c-long_policy-11", "latency_s": 0.328497239, "probability_yes": 0.5194427077546381, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.5194427077546381}}, "usage": {"input_tokens": 3399, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.327713226, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007195843198227, "client_end_monotonic_ns": 5007196171695466} +{"id": "hard-opus-c-temporal_numeric-02", "latency_s": 0.09080052, "probability_yes": 0.24180561207468904, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.24180561207468904}}, "usage": {"input_tokens": 936, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.090177681, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007196171854454, "client_end_monotonic_ns": 5007196262654974} +{"id": "hard-sol-a-adversarial-06", "latency_s": 0.028252864, "probability_yes": 0.018362853166429507, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.018362853166429507}}, "usage": {"input_tokens": 241, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.027743419, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007196262763969, "client_end_monotonic_ns": 5007196291016833} +{"id": "hard-sol-a-adversarial-08", "latency_s": 0.027773342, "probability_yes": 0.9375888015626979, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.9375888015626979}}, "usage": {"input_tokens": 218, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.027360893, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007196291115039, "client_end_monotonic_ns": 5007196318888381} +{"id": "hard-sol-a-trap-02", "latency_s": 0.033041074, "probability_yes": 0.03573969108794452, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.03573969108794452}}, "usage": {"input_tokens": 292, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.03268252, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007196318971610, "client_end_monotonic_ns": 5007196352012684} +{"id": "hard-sol-a-trap-04", "latency_s": 0.025938297, "probability_yes": 0.041796879885975066, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.041796879885975066}}, "usage": {"input_tokens": 198, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.025519458, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007196352092479, "client_end_monotonic_ns": 5007196378030776} +{"id": "hard-sol-a-trap-13", "latency_s": 0.026106558, "probability_yes": 0.030579705641278977, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.030579705641278977}}, "usage": {"input_tokens": 206, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.025684002, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007196378130006, "client_end_monotonic_ns": 5007196404236564} +{"id": "hard-sol-a-trap-15", "latency_s": 0.032898218, "probability_yes": 0.07570582831909627, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.07570582831909627}}, "usage": {"input_tokens": 308, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.032486581, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007196404363416, "client_end_monotonic_ns": 5007196437261634} +{"id": "hard-sol-b-judge_hard-01", "latency_s": 0.032077019, "probability_yes": 0.6743485180581948, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.6743485180581948}}, "usage": {"input_tokens": 288, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.031672137, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007196437358985, "client_end_monotonic_ns": 5007196469436004} +{"id": "hard-sol-b-judge_hard-02", "latency_s": 0.02632922, "probability_yes": 0.8525225556146606, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.8525225556146606}}, "usage": {"input_tokens": 212, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.025997692, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007196469524683, "client_end_monotonic_ns": 5007196495853903} +{"id": "hard-sol-b-judge_hard-05", "latency_s": 0.027530546, "probability_yes": 0.9192409990717809, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.9192409990717809}}, "usage": {"input_tokens": 231, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.027190999, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007196495938350, "client_end_monotonic_ns": 5007196523468896} +{"id": "hard-sol-b-judge_hard-08", "latency_s": 0.026039734, "probability_yes": 0.33265846433012053, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.33265846433012053}}, "usage": {"input_tokens": 214, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.025706851, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007196523544534, "client_end_monotonic_ns": 5007196549584268} +{"id": "hard-sol-b-judge_hard-10", "latency_s": 0.027905648, "probability_yes": 0.3554926261464918, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.3554926261464918}}, "usage": {"input_tokens": 233, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.027602565, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007196549657266, "client_end_monotonic_ns": 5007196577562914} +{"id": "hard-sol-b-judge_hard-14", "latency_s": 0.027848753, "probability_yes": 0.07750785419177868, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.07750785419177868}}, "usage": {"input_tokens": 250, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.027537647, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007196577640161, "client_end_monotonic_ns": 5007196605488914} +{"id": "hard-sol-b-judge_hard-15", "latency_s": 0.026956241, "probability_yes": 0.8936767977520365, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.8936767977520365}}, "usage": {"input_tokens": 206, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.026573585, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007196605562897, "client_end_monotonic_ns": 5007196632519138} +{"id": "hard-sol-b-judge_hard-18", "latency_s": 0.023819731, "probability_yes": 0.07374139181136073, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.07374139181136073}}, "usage": {"input_tokens": 173, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.023478274, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007196632597093, "client_end_monotonic_ns": 5007196656416824} +{"id": "hard-sol-c-judge_hard-03", "latency_s": 0.040330737, "probability_yes": 0.11480257968157337, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.11480257968157337}}, "usage": {"input_tokens": 395, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.040017528, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007196656493114, "client_end_monotonic_ns": 5007196696823851} +{"id": "hard-sol-c-judge_hard-05", "latency_s": 0.039127568, "probability_yes": 0.7398933422307209, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.7398933422307209}}, "usage": {"input_tokens": 357, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.038609273, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007196696894825, "client_end_monotonic_ns": 5007196736022393} +{"id": "hard-sol-c-judge_hard-07", "latency_s": 0.037233808, "probability_yes": 0.5950171562428843, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.5950171562428843}}, "usage": {"input_tokens": 321, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.036601131, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007196736152674, "client_end_monotonic_ns": 5007196773386482} +{"id": "hard-sol-c-judge_hard-08", "latency_s": 0.034030411, "probability_yes": 0.14899792588512642, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.14899792588512642}}, "usage": {"input_tokens": 315, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.033550306, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007196773523450, "client_end_monotonic_ns": 5007196807553861} +{"id": "hard-sol-c-judge_hard-09", "latency_s": 0.038080266, "probability_yes": 0.812841395018083, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.812841395018083}}, "usage": {"input_tokens": 346, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.037670414, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007196807659869, "client_end_monotonic_ns": 5007196845740135} +{"id": "hard-sol-c-judge_hard-10", "latency_s": 0.040012958, "probability_yes": 0.7456486341168127, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.7456486341168127}}, "usage": {"input_tokens": 375, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.03955988, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007196845847220, "client_end_monotonic_ns": 5007196885860178} +{"id": "hard-sol-c-judge_hard-11", "latency_s": 0.038055377, "probability_yes": 0.02054665416455969, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.02054665416455969}}, "usage": {"input_tokens": 346, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.037699945, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007196885956686, "client_end_monotonic_ns": 5007196924012063} +{"id": "hard-sol-c-judge_hard-13", "latency_s": 0.037478628, "probability_yes": 0.37867350884572815, "probability_delta": 0.0, "predicted": "no", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.37867350884572815}}, "usage": {"input_tokens": 336, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.03703268, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007196924099226, "client_end_monotonic_ns": 5007196961577854} +{"id": "hard-sol-c-judge_hard-15", "latency_s": 0.033563383, "probability_yes": 0.8561549717831729, "probability_delta": 0.0, "predicted": "yes", "raw": {"model": "Qwen/Qwen3.8-27B", "answers": {"decision": {"type": "noul", "noul": 0.8561549717831729}}, "usage": {"input_tokens": 290, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 2.5343690298472983, "candidate_sequences": 1, "inference_seconds": 0.033112629, "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "max_length": 16384, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "checks": {"finite_probability": true, "usage": true, "decision": true, "probability": true, "uninstrumented": true, "baseline_exact": true}, "client_start_monotonic_ns": 5007196961690063, "client_end_monotonic_ns": 5007196995253446} diff --git a/benchmarks/gdn/artifacts/20261003/e2e/cleanup.json b/benchmarks/gdn/artifacts/20261003/e2e/cleanup.json new file mode 100644 index 0000000..6dc945b --- /dev/null +++ b/benchmarks/gdn/artifacts/20261003/e2e/cleanup.json @@ -0,0 +1,33 @@ +{ + "all_task_processes_exited": true, + "processes": [ + { + "pid": 1790620, + "args": [ + "/home/hsliu2/tmp/system1-omni-open-jev-native/target/release/omni-open-jev-native" + ], + "returncode": -15 + }, + { + "pid": 1791785, + "args": [ + "/home/hsliu2/tmp/system1-omni-open-jev-native/target/release/omni-jev" + ], + "returncode": 0 + }, + { + "pid": 1793269, + "args": [ + "/home/hsliu2/tmp/system1-omni-open-jev-native/target/release/omni-open-jev-native" + ], + "returncode": -15 + }, + { + "pid": 1794598, + "args": [ + "/home/hsliu2/tmp/system1-omni-open-jev-native/target/release/omni-jev" + ], + "returncode": 0 + } + ] +} diff --git a/benchmarks/gdn/artifacts/20261003/e2e/plan.json b/benchmarks/gdn/artifacts/20261003/e2e/plan.json new file mode 100644 index 0000000..1958f3e --- /dev/null +++ b/benchmarks/gdn/artifacts/20261003/e2e/plan.json @@ -0,0 +1,245 @@ +{ + "hypothesis": "Lossless packed TF32 staging with the original four-term MMA accumulation reduces warm HTTP latency without changing model probabilities or decisions.", + "independent_variable": "Only GDN preparation implementation: original float32 staging/WMMA versus three-byte TF32 component planes and explicit m16n8k4 (matching baseline HMMA.1684 accumulation) TF32 MMA, one-time RNA conversion, vector-prefetched sequential gate sum, fused normalized/decayed QK stores, and direct packed BF16 U/W writes. Shared allocation drops from 92 KiB to 72 KiB; no accumulator or workspace dtype/shape/API changes. Baseline/candidate use otherwise identical frozen PR55 ABI4 libraries.", + "git_sha": "58b8cbe9d4738f6c217f6fe584852dc9361b2c24", + "working_tree_patch": "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-e2e-20261003-090732/harness/working-tree.patch", + "worker_git_sha": "202c0e163f868334a99d88407056ebe61dbb2dce", + "library_source_git_sha": "7b935723c6fa17145e3d866675202fb6f7ea1f5d", + "kernel_comparison": "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-20261003-090542", + "host": { + "hostname": "dedicated-developjob-8gpu2-a029z-64896bc8cf-8p2lw", + "uid": 2033, + "account": "hsliu2", + "home": "/home/hsliu2", + "worktree_directory": "/home/hsliu2/tmp", + "driver": "NVRM version: NVIDIA UNIX Open Kernel Module for x86_64 570.133.20 Release Build (dvs-builder@U22-I3-AF03-14-3) Sun Apr 13 04:34:43 UTC 2025" + }, + "fixed_controls": { + "gpu_ids": [ + 2 + ], + "gpu_uuid": "GPU-cbf66259-f4ab-0ede-1811-82037dde5924", + "cpu_affinity": "0-15", + "numa_memory_node": 0, + "dtype": "BF16", + "max_length": 16384, + "native_graph": "CUA_S1_GRAPH=0", + "server": "one worker/frontend pair per configuration reused for all requests", + "frontend": "same frozen PR55 Rust binary", + "http": "localhost; concurrency 1; client body serialization and response JSON parsing excluded from timer", + "cache": "reuse prepared weights and extensions; warm-up before listening; full 74-request feasibility pass before two measured passes; no cache drops or clock changes", + "timing_instrumentation": "Uninstrumented frozen Rust worker and frontend; no Nsight/CUPTI launcher. HTTP wall time includes response body receipt/UTF8 decode and excludes request JSON serialization/response JSON parsing.", + "hardware_name": "H200 (archived reported name NVIDIA L20X)", + "order": "baseline then candidate, one worker/frontend pair per arm, same 74-case order and full feasibility pass." + }, + "native": { + "worker": "/home/hsliu2/tmp/system1-omni-open-jev-native/target/release/omni-open-jev-native", + "model": "/home/hsliu2/tmp/system1-omni-open-jev-validation/jevbench-l20x-20261001/model-16k" + }, + "frontend": "/home/hsliu2/tmp/system1-omni-open-jev-native/target/release/omni-jev", + "libraries": { + "baseline": "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-20261003-090542/harness/build/baseline/libqwen3_5_cuda.so", + "candidate": "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-20261003-090542/harness/build/candidate/libqwen3_5_cuda.so" + }, + "model": { + "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", + "checkpoint_revision": "28cf73067d5b337860bbef3c85b8b82ba8730956", + "temperature": 2.5343690298472983, + "max_length": 16384, + "weights": "Prepared merged BF16 export reused; 11 shard paths/sizes/inodes/mtimes checked, no shared cache reset." + }, + "run_budget": { + "order": [ + "baseline", + "candidate" + ], + "feasibility_passes": 1, + "measured_passes": 2, + "requests_per_pass": 74, + "first_request": "Validate actual first inference after genuine worker readiness. First request and feasibility pass excluded from measured results." + }, + "acceptance": { + "latency": "At least 2% reduction in aggregate mean warm HTTP latency; each candidate measured-pass mean below matching baseline mean. Report both pass means and variability.", + "fidelity": "Finite probabilities; unchanged token usage and all 74 yes/no decisions; max probability delta <= 0.01 against pinned native reference. Baseline must reproduce reference exactly." + }, + "stop": "Stop on readiness failure, any request/output validation failure, 30 minute timeout, or completion of exactly the declared run budget. Preserve failed rows and clean up only task-owned processes.", + "preparation": "All libraries and binaries built before execution, model/requests already prepared, no downloads. Build timings preserved in the kernel comparison. Readiness and feasibility timings separate.", + "representatives": [ + { + "tag": "short", + "id": "original-policy-04-0", + "tokens": 107 + }, + { + "tag": "medium", + "id": "hard-opus-c-temporal_numeric-02", + "tokens": 936 + }, + { + "tag": "long", + "id": "hard-opus-c-long_policy-11", + "tokens": 3399 + } + ], + "model_files": { + "/home/hsliu2/tmp/system1-omni-open-jev-validation/jevbench-l20x-20261001/model-16k/config.json": { + "resolved": "/data/hsliu2/tmp/system1-omni-open-jev-validation/merged-pinned/config.json", + "size": 2739, + "mtime_ns": 1790775264020033494, + "inode": 2706300860 + }, + "/home/hsliu2/tmp/system1-omni-open-jev-validation/jevbench-l20x-20261001/model-16k/model-00009-of-00011.safetensors": { + "resolved": "/data/hsliu2/tmp/system1-omni-open-jev-validation/merged-pinned/model-00009-of-00011.safetensors", + "size": 4965246872, + "mtime_ns": 1790775290022046868, + "inode": 2711037874 + }, + "/home/hsliu2/tmp/system1-omni-open-jev-validation/jevbench-l20x-20261001/model-16k/model-00011-of-00011.safetensors": { + "resolved": "/data/hsliu2/tmp/system1-omni-open-jev-validation/merged-pinned/model-00011-of-00011.safetensors", + "size": 1689348640, + "mtime_ns": 1790775294085780002, + "inode": 2711300107 + }, + "/home/hsliu2/tmp/system1-omni-open-jev-validation/jevbench-l20x-20261001/model-16k/model-00004-of-00011.safetensors": { + "resolved": "/data/hsliu2/tmp/system1-omni-open-jev-validation/merged-pinned/model-00004-of-00011.safetensors", + "size": 4933767328, + "mtime_ns": 1790775275305772826, + "inode": 2707352863 + }, + "/home/hsliu2/tmp/system1-omni-open-jev-validation/jevbench-l20x-20261001/model-16k/tokenizer_config.json": { + "resolved": "/data/hsliu2/tmp/system1-omni-open-jev-validation/merged-pinned/tokenizer_config.json", + "size": 1123, + "mtime_ns": 1790775294124724606, + "inode": 2711564147 + }, + "/home/hsliu2/tmp/system1-omni-open-jev-validation/jevbench-l20x-20261001/model-16k/chat_template.jinja": { + "resolved": "/data/hsliu2/tmp/system1-omni-open-jev-validation/merged-pinned/chat_template.jinja", + "size": 8952, + "mtime_ns": 1790775294114896432, + "inode": 2711564146 + }, + "/home/hsliu2/tmp/system1-omni-open-jev-validation/jevbench-l20x-20261001/model-16k/model.safetensors.index.json": { + "resolved": "/data/hsliu2/tmp/system1-omni-open-jev-validation/merged-pinned/model.safetensors.index.json", + "size": 66001, + "mtime_ns": 1790775294102327783, + "inode": 2711564145 + }, + "/home/hsliu2/tmp/system1-omni-open-jev-validation/jevbench-l20x-20261001/model-16k/model-00007-of-00011.safetensors": { + "resolved": "/data/hsliu2/tmp/system1-omni-open-jev-validation/merged-pinned/model-00007-of-00011.safetensors", + "size": 4987283120, + "mtime_ns": 1790775284078534693, + "inode": 2711037870 + }, + "/home/hsliu2/tmp/system1-omni-open-jev-validation/jevbench-l20x-20261001/model-16k/model-00001-of-00011.safetensors": { + "resolved": "/data/hsliu2/tmp/system1-omni-open-jev-validation/merged-pinned/model-00001-of-00011.safetensors", + "size": 4989241624, + "mtime_ns": 1790775266917986837, + "inode": 2711564144 + }, + "/home/hsliu2/tmp/system1-omni-open-jev-validation/jevbench-l20x-20261001/model-16k/model-00008-of-00011.safetensors": { + "resolved": "/data/hsliu2/tmp/system1-omni-open-jev-validation/merged-pinned/model-00008-of-00011.safetensors", + "size": 4911731096, + "mtime_ns": 1790775287158207236, + "inode": 2711037880 + }, + "/home/hsliu2/tmp/system1-omni-open-jev-validation/jevbench-l20x-20261001/model-16k/model-00006-of-00011.safetensors": { + "resolved": "/data/hsliu2/tmp/system1-omni-open-jev-validation/merged-pinned/model-00006-of-00011.safetensors", + "size": 4911731080, + "mtime_ns": 1790775281095585148, + "inode": 2711037862 + }, + "/home/hsliu2/tmp/system1-omni-open-jev-validation/jevbench-l20x-20261001/model-16k/model-00002-of-00011.safetensors": { + "resolved": "/data/hsliu2/tmp/system1-omni-open-jev-validation/merged-pinned/model-00002-of-00011.safetensors", + "size": 4996683144, + "mtime_ns": 1790775269773373318, + "inode": 2706037244 + }, + "/home/hsliu2/tmp/system1-omni-open-jev-validation/jevbench-l20x-20261001/model-16k/model-00010-of-00011.safetensors": { + "resolved": "/data/hsliu2/tmp/system1-omni-open-jev-validation/merged-pinned/model-00010-of-00011.safetensors", + "size": 4933767328, + "mtime_ns": 1790775293029628030, + "inode": 2711037937 + }, + "/home/hsliu2/tmp/system1-omni-open-jev-validation/jevbench-l20x-20261001/model-16k/model-00003-of-00011.safetensors": { + "resolved": "/data/hsliu2/tmp/system1-omni-open-jev-validation/merged-pinned/model-00003-of-00011.safetensors", + "size": 4965246848, + "mtime_ns": 1790775272559435955, + "inode": 2707352813 + }, + "/home/hsliu2/tmp/system1-omni-open-jev-validation/jevbench-l20x-20261001/model-16k/tokenizer.json": { + "resolved": "/data/hsliu2/tmp/system1-omni-open-jev-validation/merged-pinned/tokenizer.json", + "size": 19989325, + "mtime_ns": 1790775294299071846, + "inode": 2711564148 + }, + "/home/hsliu2/tmp/system1-omni-open-jev-validation/jevbench-l20x-20261001/model-16k/open_jev_export.json": { + "resolved": "/data/hsliu2/tmp/system1-omni-open-jev-validation/jevbench-l20x-20261001/model-16k/open_jev_export.json", + "size": 135585, + "mtime_ns": 1790842306295001566, + "inode": 1980197732 + }, + "/home/hsliu2/tmp/system1-omni-open-jev-validation/jevbench-l20x-20261001/model-16k/model-00005-of-00011.safetensors": { + "resolved": "/data/hsliu2/tmp/system1-omni-open-jev-validation/merged-pinned/model-00005-of-00011.safetensors", + "size": 4965246880, + "mtime_ns": 1790775278128982885, + "inode": 2711037835 + } + }, + "hashes": { + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-e2e-20261003-090732/harness/requests.json": "0c756a7b0b4c1f1352225f2e01770b5b3a0646fe86d5cc6930ded9e292ef37df", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-e2e-20261003-090732/harness/lengths.json": "d40155fb353c1e3473c22abac16fb33ca582352268c3cabb58efc9b93446cc4f", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-e2e-20261003-090732/analysis/reference.jsonl": "f964c0e61ef3877aac130b795c46070de6e73d261272c33f1a2f45f4e6e66b83", + "/home/hsliu2/tmp/system1-omni-open-jev-native/target/release/omni-open-jev-native": "d91622696dbfbfb44a623b6a36fc939e78a913c3f9b4e62f501f7b093ee1df73", + "/home/hsliu2/tmp/system1-omni-open-jev-native/target/release/omni-jev": "0c40efdc61371b195267e1a22bb7824a76a15fcf15460c02fbba8dd369495086", + "/home/hsliu2/tmp/system1-omni-open-jev-validation/jevbench-l20x-20261001/model-16k/config.json": "193d46cdde2f62b603269f005044a9fdaf4ad6b4447997c359e16dcc7217edcd", + "/home/hsliu2/tmp/system1-omni-open-jev-validation/jevbench-l20x-20261001/model-16k/tokenizer_config.json": "bee8eba30f0eb4af73c0fe2cd06d0f89b657d7819941c438157ec42f7c80ea87", + "/home/hsliu2/tmp/system1-omni-open-jev-validation/jevbench-l20x-20261001/model-16k/chat_template.jinja": "c3cf9e34abf4f9e36c2d72165aa9c132d3e2a725b6c2586aaa3a8af9d7a81041", + "/home/hsliu2/tmp/system1-omni-open-jev-validation/jevbench-l20x-20261001/model-16k/model.safetensors.index.json": "5af6c022747934e687d7edf362de832a7fa2c82eedc36a8679495b40bb214b6f", + "/home/hsliu2/tmp/system1-omni-open-jev-validation/jevbench-l20x-20261001/model-16k/tokenizer.json": "06b9509352d2af50381ab2247e083b80d32d5c0aba91c272ca9ff729b6a0e523", + "/home/hsliu2/tmp/system1-omni-open-jev-validation/jevbench-l20x-20261001/model-16k/open_jev_export.json": "5fca82e74d77cbbcb83b03505130dad34a923731dc083ec7616cc73997688caa", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-e2e-20261003-090732/harness/collect.py": "13c8eeb1002c854ceccfe33b005d1535e9b3493250bce52b2c1268666c8e41d5", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-e2e-20261003-090732/harness/working-tree.patch": "8fe079d6580ddab0aef333f044a9a24dca18960cac17b04426b328ef8fc151bd", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-20261003-090542/harness/ops.h": "776376f82c8a01aacea4ec29178714c95079ec6454dc0466a483d073fee3e06c", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-20261003-090542/harness/working-tree.patch": "8fe079d6580ddab0aef333f044a9a24dca18960cac17b04426b328ef8fc151bd", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-20261003-090542/harness/gemm.cu": "a82b6d6012c6b13d5ec8dd8ce7107bd95e17f588c76dae1d314e9cff1585274a", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-20261003-090542/harness/common.cuh": "ea924f7c903b6d14b0e1f10c8c33a1a9ca23806df093628a178661a303d54d55", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-20261003-090542/harness/elementwise.cu": "b739f1bc2f08ae67399366063cc6c42bc0c5e65e98da23d8b425f9f9ea53ea59", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-20261003-090542/harness/norm.cu": "756c9d4023787cc3961ec72bcc96b3f715e705d5dfcb3bf77622879980df82ab", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-20261003-090542/harness/attention.cu": "3d6935a8a550314815ebe6575be03ce1a7e0b1d5bd611508a4bffbc24cff3132", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-20261003-090542/harness/compare.py": "00899a905c0ff9065c338d1cdad8e1f97e2959d629d40043ed8c7854b65101a3", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-20261003-090542/harness/gdn-packed-tf32-k4.cu": "9f2642a5f6e69cd22055419ecf1f799f1ccd811f7453763a41adf4f15b07a41d", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-20261003-090542/harness/gdn_prefill.cu": "1c9027b6f2d3677381d8d19611ecdf7e138718459cd61f341802cc2047a582be", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-20261003-090542/harness/runtime.cu": "3c19d2a1167b44d3a84ffe480605e14a72c775d2c2e51d0c594af9d0db6cb69e", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-20261003-090542/harness/mma.cuh": "64654f8b937585805daa72b692c243ae35d34b746e2cb6b0b55c255748495764", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-20261003-090542/harness/build/baseline/libqwen3_5_cuda.so": "8ea954d3b18d73d9fdb530fee0ddb898cf2151bb326bb67da471e9f43ca0c69e", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-20261003-090542/harness/build/candidate/libqwen3_5_cuda.so": "92cbe5485926910cd16ee911a536dddf0c74b2f6d23cb4cb6694f5e742907a48", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-20261003-090542/harness/build/validation/libqwen3_5_cuda.so": "adabeca799005b04706b1bd00b6a107b987e936642f9c345bdc9302d014e8179", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/target/release/deps/kernels-e1de1d464265a9b7": "77c9e80d7d549c14785a1de05fb2b1eced560da7ab9966446f668326254fa18a", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/src/backends/cuda/qwen3_5/gdn_prefill.cu": "9f2642a5f6e69cd22055419ecf1f799f1ccd811f7453763a41adf4f15b07a41d", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/tests/qwen3_5/kernels.rs": "2ccd7284f212152fcccfa31cab72c6bb2be6734440c3e877fe09cfc8e2077af4", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/src/models/cua_s1/native/Cargo.toml": "cf0050ee87319ab4d33cfc6ad8d111ba1b49ac113552fc7c9c42314c99f830e1" + }, + "command": [ + "gpu", + "run", + "--gpu-ids", + "2", + "--wait", + "10m", + "--timeout", + "30m", + "--note", + "Open-Jev GDN packed TF32 K4 HTTP A/B", + "--", + "numactl", + "--membind=0", + "--physcpubind=0-15", + "/home/hsliu2/tmp/venvs/open-jev-native/bin/python", + "-u", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-e2e-20261003-090732/harness/collect.py" + ], + "previous_candidate": { + "run": "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-rna-fp16-e2e-20261003-083539", + "result": "FP16 feasibility changed one decision and was stopped before measured candidate passes. The new TF32 K4 candidate passed strict workspace/output checks; acceptance thresholds are unchanged." + } +} diff --git a/benchmarks/gdn/artifacts/20261003/e2e/summary.json b/benchmarks/gdn/artifacts/20261003/e2e/summary.json new file mode 100644 index 0000000..b78475f --- /dev/null +++ b/benchmarks/gdn/artifacts/20261003/e2e/summary.json @@ -0,0 +1,136 @@ +{ + "completed": true, + "accepted_e2e_latency_gate": false, + "arms": { + "baseline": { + "passes": [ + { + "pass": 1, + "requests": 74, + "mean_ms": 48.37519689189189, + "p50_ms": 25.3186545, + "p95_ms": 227.743904, + "max_probability_delta": 0.0, + "raw": "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-e2e-20261003-090732/analysis/baseline-measured-1.jsonl" + }, + { + "pass": 2, + "requests": 74, + "mean_ms": 48.44110141891892, + "p50_ms": 25.362659, + "p95_ms": 227.70445899999999, + "max_probability_delta": 0.0, + "raw": "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-e2e-20261003-090732/analysis/baseline-measured-2.jsonl" + } + ], + "aggregate_mean_ms": 48.408149155405404, + "readiness": { + "worker": { + "process_to_readiness_seconds": 11.626411007717252, + "response": { + "status": "ready", + "model": "Qwen/Qwen3.8-27B" + } + }, + "frontend": { + "process_to_readiness_seconds": 0.005645955912768841, + "response": { + "status": "ready", + "model": "Qwen/Qwen3.8-27B" + } + } + }, + "first_inference": { + "finite_probability": true, + "usage": true, + "decision": true, + "probability": true, + "uninstrumented": true, + "baseline_exact": true + } + }, + "candidate": { + "passes": [ + { + "pass": 1, + "requests": 74, + "mean_ms": 48.10456159459459, + "p50_ms": 24.852269, + "p95_ms": 225.918293, + "max_probability_delta": 0.0, + "raw": "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-e2e-20261003-090732/analysis/candidate-measured-1.jsonl" + }, + { + "pass": 2, + "requests": 74, + "mean_ms": 48.305051418918914, + "p50_ms": 24.879013999999998, + "p95_ms": 225.79116499999998, + "max_probability_delta": 0.0, + "raw": "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-e2e-20261003-090732/analysis/candidate-measured-2.jsonl" + } + ], + "aggregate_mean_ms": 48.20480650675675, + "readiness": { + "worker": { + "process_to_readiness_seconds": 10.620182611979544, + "response": { + "status": "ready", + "model": "Qwen/Qwen3.8-27B" + } + }, + "frontend": { + "process_to_readiness_seconds": 0.006159857846796513, + "response": { + "status": "ready", + "model": "Qwen/Qwen3.8-27B" + } + } + }, + "first_inference": { + "finite_probability": true, + "usage": true, + "decision": true, + "probability": true, + "uninstrumented": true, + "baseline_exact": true + } + } + }, + "controls": "plan.json", + "cleanup": { + "all_task_processes_exited": true, + "processes": [ + { + "pid": 1790620, + "args": [ + "/home/hsliu2/tmp/system1-omni-open-jev-native/target/release/omni-open-jev-native" + ], + "returncode": -15 + }, + { + "pid": 1791785, + "args": [ + "/home/hsliu2/tmp/system1-omni-open-jev-native/target/release/omni-jev" + ], + "returncode": 0 + }, + { + "pid": 1793269, + "args": [ + "/home/hsliu2/tmp/system1-omni-open-jev-native/target/release/omni-open-jev-native" + ], + "returncode": -15 + }, + { + "pid": 1794598, + "args": [ + "/home/hsliu2/tmp/system1-omni-open-jev-native/target/release/omni-jev" + ], + "returncode": 0 + } + ] + }, + "aggregate_reduction_percent": 0.42005871365966563, + "conclusion": "Standalone GDN fidelity/latency gates passed. All 74 model probabilities and decisions unchanged, but aggregate E2E gain is below 2%; no material E2E improvement claim." +} diff --git a/benchmarks/gdn/artifacts/20261003/flashinfer/plan.json b/benchmarks/gdn/artifacts/20261003/flashinfer/plan.json new file mode 100644 index 0000000..ee0e397 --- /dev/null +++ b/benchmarks/gdn/artifacts/20261003/flashinfer/plan.json @@ -0,0 +1,72 @@ +{ + "hypothesis": "FlashInfer SM90 fused or context-parallel GDN reduces native core latency at representative Open-Jev shapes with acceptable numerical difference. Results select the next native implementation experiment, not an E2E gain claim.", + "independent_variable": "GDN implementation: unchanged native versus installed FlashInfer fused and CP, including required Q/K normalization and gate dtype/exp adapter work.", + "fixed_controls": { + "gpu_ids": [ + 2 + ], + "gpu_uuid": "GPU-cbf66259-f4ab-0ede-1811-82037dde5924", + "hardware": "H200 (driver name L20X)", + "numa_node": 0, + "cpu_affinity": "0-15", + "qk_heads": 16, + "v_heads": 48, + "head_dim": 128, + "dtype": "BF16 Q/K/V/beta; FP32 log g", + "seed": 20261003, + "tokens": [ + 107, + 936, + 3399 + ], + "inputs": "seeded synthetic correlated Q/K and weak-decay gates; actual workload shapes, not captured model activations", + "stream": "one default stream", + "cache": "reuse; no cache drops; eager submission in all arms" + }, + "run_budget": "Per shape/configuration: one excluded feasibility plus 10 excluded warmups, then exactly 2 measured passes of 50 calls; second pass reverses arm order. Failed feasibility arms receive no measured calls. No extra measured repetitions.", + "acceptance": { + "initial_kernel_fidelity": "finite output, max_abs <= 0.03 and relative_l2 <= 0.02 against native; final model PR gate is unchanged 74 decisions and max probability delta <= 0.01", + "candidate_latency": ">=10% GDN core reduction in both measured passes motivates integration; otherwise report inconclusive or no gain", + "later_e2e_gate": "new isolated A/B plan: >=2% mean warm HTTP improvement and both candidate passes faster, with unchanged decisions" + }, + "stop": "Stop native failure, stop each rejected candidate arm, timeout 30m, or finish declared run budget.", + "git_sha": "58b8cbe9d4738f6c217f6fe584852dc9361b2c24", + "working_tree": "", + "packages": { + "torch": "2.13.0+cu129", + "triton": "3.7.1", + "flashinfer-python": "0.6.16.post3", + "nvidia-cutlass-dsl": "4.6.2", + "cuda-python": "12.9.7" + }, + "files": { + "/home/hsliu2/tmp/system1-omni-gdn-optimization-20261003/src/backends/cuda/qwen3_5/gdn_prefill.cu": "1c9027b6f2d3677381d8d19611ecdf7e138718459cd61f341802cc2047a582be", + "/home/hsliu2/tmp/venvs/pr6930/lib/python3.12/site-packages/flashinfer/gdn_prefill.py": "a486f6d4a6a59681e754a19b8d52efd14b3bbf6c8835272fb2d846bb6b06debb", + "/home/hsliu2/tmp/venvs/pr6930/lib/python3.12/site-packages/flashinfer/gdn_kernels/delta_rule_dsl/delta_rule_sm90.py": "c72398f8f6e57ed2d708d364dd0299ce414e75a27d044ac83c6278c0954290a4", + "/home/hsliu2/tmp/venvs/open-jev-native/lib/python3.12/site-packages/fla/modules/l2norm.py": "30f7feebdbfa87b8e90c143ec62a7fa0781bfd736c856c0ec69452f0c778be3f", + "/home/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-ready-kernels-20261003-080129/harness/compare.py": "4fa800fa321f4e79350a3d151f211397b566cf03bd0982794e1e23dfa25a6662" + }, + "ncu": "Hardware counters unavailable; do not infer hardware stall/occupancy causes from timing.", + "command": [ + "gpu", + "run", + "--gpu-ids", + "2", + "--wait", + "10m", + "--timeout", + "30m", + "--note", + "Open-Jev GDN ready-kernel comparison", + "--", + "numactl", + "--membind=0", + "--physcpubind=0-15", + "/home/hsliu2/tmp/venvs/open-jev-native/bin/python", + "-u", + "/home/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-ready-kernels-20261003-080129/harness/compare.py" + ], + "cpu_probe_correction": "Before any GPU execution: the installed FlashInfer does not contain its newer normalize_qk helper; use the already-installed FLA l2norm for both operands, which vLLM also uses. Preserve initial plan/probe; no configuration measured.", + "previous_attempt": "/home/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-ready-kernels-20261003-075757", + "previous_attempt_result": "Import preparation failed before CUDA initialization or feasibility/measurement: cutlass DSL files were installed but the nested environment .pth was not processed. Session 54400 exited 1 and GPU 2 released at 0 MB." +} diff --git a/benchmarks/gdn/artifacts/20261003/flashinfer/results.jsonl b/benchmarks/gdn/artifacts/20261003/flashinfer/results.jsonl new file mode 100644 index 0000000..bd1b738 --- /dev/null +++ b/benchmarks/gdn/artifacts/20261003/flashinfer/results.jsonl @@ -0,0 +1,30 @@ +{"kind": "cpu_probe", "library": "/home/hsliu2/tmp/system1-omni-jev-single-candidate/profile/jev-single-candidate-silu-pack8-20261002/harness/build/optimized/libqwen3_5_cuda.so", "library_sha256": "e033315d4c67127809e62f41d991a149ac8ee0ce81827997af063815fd00d435", "shapes": [107, 936, 3399], "configs": ["native", "flashinfer_fused", "flashinfer_cp"], "device_execution": false} +{"kind": "hardware", "name": "NVIDIA L20X", "sm_count": 132, "capability": [9, 0], "uuid": "cbf66259-f4ab-0ede-1811-82037dde5924", "visibility": "2", "pid": 549291, "torch": "2.13.0+cu129"} +{"kind": "feasibility", "tokens": 107, "config": "native", "preparation_seconds": 0.20153669267892838, "max_abs": 0.0, "relative_l2": 0.0, "passed": true} +{"kind": "feasibility", "tokens": 107, "config": "flashinfer_fused", "preparation_seconds": 16.483370064757764, "max_abs": 0.00146484375, "relative_l2": 0.003793163690716028, "passed": true} +{"kind": "feasibility", "tokens": 107, "config": "flashinfer_cp", "preparation_seconds": 6.6084970608353615, "max_abs": 0.00146484375, "relative_l2": 0.0038435247261077166, "passed": true} +{"kind": "measurement", "tokens": 107, "config": "native", "pass": 1, "iterations": 50, "device_ms_per_call": 0.051557760238647464, "wall_ms_per_call": 0.052070280000000004, "max_abs": 0.0} +{"kind": "measurement", "tokens": 107, "config": "flashinfer_fused", "pass": 1, "iterations": 50, "device_ms_per_call": 0.41863361358642576, "wall_ms_per_call": 0.41896896, "max_abs": 0.00146484375} +{"kind": "measurement", "tokens": 107, "config": "flashinfer_cp", "pass": 1, "iterations": 50, "device_ms_per_call": 0.8341561889648438, "wall_ms_per_call": 0.83451756, "max_abs": 0.00146484375} +{"kind": "measurement", "tokens": 107, "config": "flashinfer_cp", "pass": 2, "iterations": 50, "device_ms_per_call": 0.815771484375, "wall_ms_per_call": 0.81613174, "max_abs": 0.00146484375} +{"kind": "measurement", "tokens": 107, "config": "flashinfer_fused", "pass": 2, "iterations": 50, "device_ms_per_call": 0.4374502563476563, "wall_ms_per_call": 0.4379203, "max_abs": 0.00146484375} +{"kind": "measurement", "tokens": 107, "config": "native", "pass": 2, "iterations": 50, "device_ms_per_call": 0.05144896030426025, "wall_ms_per_call": 0.05175709999999999, "max_abs": 0.0} +{"kind": "feasibility", "tokens": 936, "config": "native", "preparation_seconds": 0.0006458759307861328, "max_abs": 0.0, "relative_l2": 0.0, "passed": true} +{"kind": "feasibility", "tokens": 936, "config": "flashinfer_fused", "preparation_seconds": 0.0012157773599028587, "max_abs": 0.00146484375, "relative_l2": 0.004750747699290514, "passed": true} +{"kind": "feasibility", "tokens": 936, "config": "flashinfer_cp", "preparation_seconds": 0.0014576511457562447, "max_abs": 0.00146484375, "relative_l2": 0.004768683575093746, "passed": true} +{"kind": "measurement", "tokens": 936, "config": "native", "pass": 1, "iterations": 50, "device_ms_per_call": 0.24776895523071288, "wall_ms_per_call": 0.24821914, "max_abs": 0.0} +{"kind": "measurement", "tokens": 936, "config": "flashinfer_fused", "pass": 1, "iterations": 50, "device_ms_per_call": 0.43887359619140626, "wall_ms_per_call": 0.43923444000000006, "max_abs": 0.00146484375} +{"kind": "measurement", "tokens": 936, "config": "flashinfer_cp", "pass": 1, "iterations": 50, "device_ms_per_call": 0.8463705444335937, "wall_ms_per_call": 0.84678924, "max_abs": 0.00146484375} +{"kind": "measurement", "tokens": 936, "config": "flashinfer_cp", "pass": 2, "iterations": 50, "device_ms_per_call": 0.8269427490234375, "wall_ms_per_call": 0.8273754, "max_abs": 0.00146484375} +{"kind": "measurement", "tokens": 936, "config": "flashinfer_fused", "pass": 2, "iterations": 50, "device_ms_per_call": 0.43020416259765626, "wall_ms_per_call": 0.43054294, "max_abs": 0.00146484375} +{"kind": "measurement", "tokens": 936, "config": "native", "pass": 2, "iterations": 50, "device_ms_per_call": 0.24784320831298828, "wall_ms_per_call": 0.24814956, "max_abs": 0.0} +{"kind": "feasibility", "tokens": 3399, "config": "native", "preparation_seconds": 0.012499378994107246, "max_abs": 0.0, "relative_l2": 0.0, "passed": true} +{"kind": "feasibility", "tokens": 3399, "config": "flashinfer_fused", "preparation_seconds": 0.0016858763992786407, "max_abs": 0.00146484375, "relative_l2": 0.004838777706027031, "passed": true} +{"kind": "feasibility", "tokens": 3399, "config": "flashinfer_cp", "preparation_seconds": 0.0016576843336224556, "max_abs": 0.001953125, "relative_l2": 0.00486042769625783, "passed": true} +{"kind": "measurement", "tokens": 3399, "config": "native", "pass": 1, "iterations": 50, "device_ms_per_call": 0.8014905548095703, "wall_ms_per_call": 0.8019471, "max_abs": 0.0} +{"kind": "measurement", "tokens": 3399, "config": "flashinfer_fused", "pass": 1, "iterations": 50, "device_ms_per_call": 0.4256230545043945, "wall_ms_per_call": 0.42592638, "max_abs": 0.00146484375} +{"kind": "measurement", "tokens": 3399, "config": "flashinfer_cp", "pass": 1, "iterations": 50, "device_ms_per_call": 0.8378540802001954, "wall_ms_per_call": 0.8381907200000001, "max_abs": 0.001953125} +{"kind": "measurement", "tokens": 3399, "config": "flashinfer_cp", "pass": 2, "iterations": 50, "device_ms_per_call": 0.8562751770019531, "wall_ms_per_call": 0.85660518, "max_abs": 0.001953125} +{"kind": "measurement", "tokens": 3399, "config": "flashinfer_fused", "pass": 2, "iterations": 50, "device_ms_per_call": 0.5074425506591796, "wall_ms_per_call": 0.50796156, "max_abs": 0.00146484375} +{"kind": "measurement", "tokens": 3399, "config": "native", "pass": 2, "iterations": 50, "device_ms_per_call": 0.7997011566162109, "wall_ms_per_call": 0.8000645799999999, "max_abs": 0.0} +{"kind": "complete", "pid": 549291} diff --git a/benchmarks/gdn/artifacts/20261003/kernel/analysis/recurrent-reference.log b/benchmarks/gdn/artifacts/20261003/kernel/analysis/recurrent-reference.log new file mode 100644 index 0000000..088fa74 --- /dev/null +++ b/benchmarks/gdn/artifacts/20261003/kernel/analysis/recurrent-reference.log @@ -0,0 +1,28 @@ + +running 3 tests +test flash_attention_matches_float64_reference ... attention t = 1, amplitude 8: largest relative difference 0.00e0 +attention t = 63, amplitude 8: largest relative difference 3.46e-3 +attention t = 65, amplitude 0.5: largest relative difference 4.16e-3 +attention t = 139, amplitude 2: largest relative difference 4.85e-3 +attention t = 700, amplitude 8: largest relative difference 3.38e-3 +attention t = 2048, amplitude 0.5: largest relative difference 4.51e-3 +ok +test gated_delta_rule_matches_recurrent_reference ... gated delta t = 1, amplitude 1: largest difference 3.74e-4, largest |reference| 8.39e-2 +gated delta t = 16, amplitude 1: largest difference 4.87e-4, largest |reference| 8.39e-2 +gated delta t = 17, amplitude 1: largest difference 4.87e-4, largest |reference| 8.39e-2 +gated delta t = 32, amplitude 1: largest difference 5.20e-4, largest |reference| 8.41e-2 +gated delta t = 33, amplitude 1: largest difference 5.20e-4, largest |reference| 8.41e-2 +gated delta t = 63, amplitude 1: largest difference 5.60e-4, largest |reference| 8.53e-2 +gated delta t = 64, amplitude 1: largest difference 5.60e-4, largest |reference| 8.53e-2 +gated delta t = 65, amplitude 1: largest difference 5.80e-4, largest |reference| 8.53e-2 +gated delta t = 107, amplitude 1: largest difference 5.80e-4, largest |reference| 8.53e-2 +gated delta t = 150, amplitude 1: largest difference 5.80e-4, largest |reference| 8.55e-2 +gated delta t = 936, amplitude 1: largest difference 9.66e-4, largest |reference| 9.69e-2 +gated delta t = 3399, amplitude 1: largest difference 1.15e-3, largest |reference| 9.60e-2 +gated delta t = 65, amplitude 0.000000001: largest difference 2.26e-14, largest |reference| 4.05e-12 +gated delta t = 65, amplitude 0.000000000000000001: largest difference 2.43e-32, largest |reference| 4.05e-30 +gated delta t = 65, amplitude 0: largest difference 0.00e0, largest |reference| 0.00e0 +ok +test graph_replay_reads_updated_inputs_after_failed_capture ... ok + +test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 7.04s diff --git a/benchmarks/gdn/artifacts/20261003/kernel/analysis/results.jsonl b/benchmarks/gdn/artifacts/20261003/kernel/analysis/results.jsonl new file mode 100644 index 0000000..0818ef7 --- /dev/null +++ b/benchmarks/gdn/artifacts/20261003/kernel/analysis/results.jsonl @@ -0,0 +1,28 @@ +{"kind": "cpu_probe", "passed": true, "files": 19} +{"kind": "hardware", "uuid": "cbf66259-f4ab-0ede-1811-82037dde5924", "name": "NVIDIA L20X", "sm_count": 132, "pid": 1771200} +{"kind": "recurrent_reference", "command": ["/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/target/release/deps/kernels-e1de1d464265a9b7", "--ignored", "--nocapture", "--test-threads=1"], "exit_code": 0, "seconds": 7.3779330011457205} +{"kind": "workspace_fidelity", "tokens": 107, "config": "baseline", "differences": {"u": {"changed": 0, "max_abs": 0.0}, "w": {"changed": 0, "max_abs": 0.0}, "qd": {"changed": 0, "max_abs": 0.0}, "kd": {"changed": 0, "max_abs": 0.0}, "pb": {"changed": 0, "max_abs": 0.0}, "decay": {"changed": 0, "max_abs": 0.0}}} +{"kind": "feasibility", "tokens": 107, "config": "baseline", "passed": true, "max_abs": 0.0, "relative_l2": 0.0, "seconds": 0.0869095716625452} +{"kind": "workspace_fidelity", "tokens": 107, "config": "candidate", "differences": {"u": {"changed": 0, "max_abs": 0.0}, "w": {"changed": 0, "max_abs": 0.0}, "qd": {"changed": 0, "max_abs": 0.0}, "kd": {"changed": 0, "max_abs": 0.0}, "pb": {"changed": 0, "max_abs": 0.0}, "decay": {"changed": 0, "max_abs": 0.0}}} +{"kind": "feasibility", "tokens": 107, "config": "candidate", "passed": true, "max_abs": 0.0, "relative_l2": 0.0, "seconds": 0.004363702610135078} +{"kind": "measurement", "tokens": 107, "config": "baseline", "pass": 1, "iterations": 100, "device_span_ms_per_call": 0.05090112209320068, "wall_ms_per_call": 0.05129078, "max_abs": 0.0} +{"kind": "measurement", "tokens": 107, "config": "candidate", "pass": 1, "iterations": 100, "device_span_ms_per_call": 0.05078239917755127, "wall_ms_per_call": 0.050913560000000004, "max_abs": 0.0} +{"kind": "measurement", "tokens": 107, "config": "candidate", "pass": 2, "iterations": 100, "device_span_ms_per_call": 0.05083456039428711, "wall_ms_per_call": 0.0509565, "max_abs": 0.0} +{"kind": "measurement", "tokens": 107, "config": "baseline", "pass": 2, "iterations": 100, "device_span_ms_per_call": 0.05085087776184082, "wall_ms_per_call": 0.05097596, "max_abs": 0.0} +{"kind": "workspace_fidelity", "tokens": 936, "config": "baseline", "differences": {"u": {"changed": 0, "max_abs": 0.0}, "w": {"changed": 0, "max_abs": 0.0}, "qd": {"changed": 0, "max_abs": 0.0}, "kd": {"changed": 0, "max_abs": 0.0}, "pb": {"changed": 0, "max_abs": 0.0}, "decay": {"changed": 0, "max_abs": 0.0}}} +{"kind": "feasibility", "tokens": 936, "config": "baseline", "passed": true, "max_abs": 0.0, "relative_l2": 0.0, "seconds": 0.02221657894551754} +{"kind": "workspace_fidelity", "tokens": 936, "config": "candidate", "differences": {"u": {"changed": 0, "max_abs": 0.0}, "w": {"changed": 0, "max_abs": 0.0}, "qd": {"changed": 0, "max_abs": 0.0}, "kd": {"changed": 0, "max_abs": 0.0}, "pb": {"changed": 0, "max_abs": 0.0}, "decay": {"changed": 0, "max_abs": 0.0}}} +{"kind": "feasibility", "tokens": 936, "config": "candidate", "passed": true, "max_abs": 0.0, "relative_l2": 0.0, "seconds": 0.008761398494243622} +{"kind": "measurement", "tokens": 936, "config": "baseline", "pass": 1, "iterations": 100, "device_span_ms_per_call": 0.24733535766601564, "wall_ms_per_call": 0.24756526999999998, "max_abs": 0.0} +{"kind": "measurement", "tokens": 936, "config": "candidate", "pass": 1, "iterations": 100, "device_span_ms_per_call": 0.21404800415039063, "wall_ms_per_call": 0.21417622000000003, "max_abs": 0.0} +{"kind": "measurement", "tokens": 936, "config": "candidate", "pass": 2, "iterations": 100, "device_span_ms_per_call": 0.21405311584472655, "wall_ms_per_call": 0.21417843, "max_abs": 0.0} +{"kind": "measurement", "tokens": 936, "config": "baseline", "pass": 2, "iterations": 100, "device_span_ms_per_call": 0.24720319747924804, "wall_ms_per_call": 0.24733708, "max_abs": 0.0} +{"kind": "workspace_fidelity", "tokens": 3399, "config": "baseline", "differences": {"u": {"changed": 0, "max_abs": 0.0}, "w": {"changed": 0, "max_abs": 0.0}, "qd": {"changed": 0, "max_abs": 0.0}, "kd": {"changed": 0, "max_abs": 0.0}, "pb": {"changed": 0, "max_abs": 0.0}, "decay": {"changed": 0, "max_abs": 0.0}}} +{"kind": "feasibility", "tokens": 3399, "config": "baseline", "passed": true, "max_abs": 0.0, "relative_l2": 0.0, "seconds": 0.03598189912736416} +{"kind": "workspace_fidelity", "tokens": 3399, "config": "candidate", "differences": {"u": {"changed": 0, "max_abs": 0.0}, "w": {"changed": 0, "max_abs": 0.0}, "qd": {"changed": 0, "max_abs": 0.0}, "kd": {"changed": 0, "max_abs": 0.0}, "pb": {"changed": 0, "max_abs": 0.0}, "decay": {"changed": 0, "max_abs": 0.0}}} +{"kind": "feasibility", "tokens": 3399, "config": "candidate", "passed": true, "max_abs": 0.0, "relative_l2": 0.0, "seconds": 0.021518888883292675} +{"kind": "measurement", "tokens": 3399, "config": "baseline", "pass": 1, "iterations": 100, "device_span_ms_per_call": 0.8004367828369141, "wall_ms_per_call": 0.80077451, "max_abs": 0.0} +{"kind": "measurement", "tokens": 3399, "config": "candidate", "pass": 1, "iterations": 100, "device_span_ms_per_call": 0.6965641784667969, "wall_ms_per_call": 0.69676832, "max_abs": 0.0} +{"kind": "measurement", "tokens": 3399, "config": "candidate", "pass": 2, "iterations": 100, "device_span_ms_per_call": 0.6970108795166016, "wall_ms_per_call": 0.69726191, "max_abs": 0.0} +{"kind": "measurement", "tokens": 3399, "config": "baseline", "pass": 2, "iterations": 100, "device_span_ms_per_call": 0.7987455749511718, "wall_ms_per_call": 0.79893456, "max_abs": 0.0} +{"kind": "complete", "pid": 1771200} diff --git a/benchmarks/gdn/artifacts/20261003/kernel/analysis/summary.json b/benchmarks/gdn/artifacts/20261003/kernel/analysis/summary.json new file mode 100644 index 0000000..d0fe8ad --- /dev/null +++ b/benchmarks/gdn/artifacts/20261003/kernel/analysis/summary.json @@ -0,0 +1,51 @@ +{ + "status": "complete", + "accepted": true, + "shapes": [ + { + "tokens": 107, + "baseline_ms": [ + 0.05129078, + 0.05097596 + ], + "candidate_ms": [ + 0.050913560000000004, + 0.0509565 + ], + "reduction_percent": [ + 0.7354538184055626, + 0.03817485732490189 + ] + }, + { + "tokens": 936, + "baseline_ms": [ + 0.24756526999999998, + 0.24733708 + ], + "candidate_ms": [ + 0.21417622000000003, + 0.21417843 + ], + "reduction_percent": [ + 13.48696850733544, + 13.406259182812374 + ] + }, + { + "tokens": 3399, + "baseline_ms": [ + 0.80077451, + 0.79893456 + ], + "candidate_ms": [ + 0.69676832, + 0.69726191 + ], + "reduction_percent": [ + 12.98819938711585, + 12.726029776456272 + ] + } + ] +} diff --git a/benchmarks/gdn/artifacts/20261003/kernel/harness/compare.py b/benchmarks/gdn/artifacts/20261003/kernel/harness/compare.py new file mode 100644 index 0000000..bf25153 --- /dev/null +++ b/benchmarks/gdn/artifacts/20261003/kernel/harness/compare.py @@ -0,0 +1,136 @@ +"""Isolated native GDN inverse A/B. Run through the GPU scheduler.""" +import argparse +import ctypes +import hashlib +import json +import os +from pathlib import Path +import subprocess +import time + +ROOT = Path(__file__).resolve().parents[1] + + +def write(row): + with (ROOT / 'analysis/results.jsonl').open('a') as f: + f.write(json.dumps(row) + '\n') + print(json.dumps(row), flush=True) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--probe', action='store_true') + args = parser.parse_args() + plan = json.loads((ROOT / 'plan.json').read_text()) + for path, expected in plan['hashes'].items(): + assert hashlib.sha256(Path(path).read_bytes()).hexdigest() == expected, path + if args.probe: + write({'kind': 'cpu_probe', 'passed': True, 'files': len(plan['hashes'])}) + return + + import torch + + assert os.environ['CUDA_VISIBLE_DEVICES'] == '2' + assert sorted(os.sched_getaffinity(0)) == list(range(16)) + torch.set_num_threads(16) + prop = torch.cuda.get_device_properties(0) + assert str(prop.uuid) == 'cbf66259-f4ab-0ede-1811-82037dde5924' + write({'kind': 'hardware', 'uuid': str(prop.uuid), 'name': prop.name, + 'sm_count': prop.multi_processor_count, 'pid': os.getpid()}) + env = dict(os.environ, CUA_S1_CUDA_LIB=plan['validation_library']) + started = time.perf_counter() + command = [plan['kernel_test_binary'], '--ignored', '--nocapture', '--test-threads=1'] + with (ROOT / 'analysis/recurrent-reference.log').open('w') as log: + result = subprocess.run(command, env=env, stdout=log, stderr=subprocess.STDOUT, timeout=180) + write({'kind': 'recurrent_reference', 'command': command, 'exit_code': result.returncode, + 'seconds': time.perf_counter() - started}) + assert result.returncode == 0, (ROOT / 'analysis/recurrent-reference.log').read_text()[-2000:] + + libraries = {} + for name in ['baseline', 'candidate']: + lib = ctypes.CDLL(plan['libraries'][name]) + lib.cs1_gdn_workspace_floats.argtypes = [ctypes.c_int, ctypes.c_int] + lib.cs1_gdn_workspace_floats.restype = ctypes.c_size_t + lib.cs1_gdn_prefill.argtypes = [ctypes.c_void_p] * 7 + [ctypes.c_int] * 3 + [ctypes.c_float, ctypes.c_void_p] + lib.cs1_gdn_prefill.restype = ctypes.c_int + libraries[name] = lib + stream = torch.cuda.current_stream() + pointer = lambda x: ctypes.c_void_p(x.data_ptr()) + generator = torch.Generator(device='cpu').manual_seed(20261003) + for t in [107, 936, 3399]: + def random(shape): + return (torch.rand(shape, generator=generator) * 2 - 1).to(torch.bfloat16).cuda() + k = random((t, 16, 128)) + q = (0.8 * k.float() + 0.2 * random(k.shape).float()).to(torch.bfloat16) + v = random((t, 48, 128)) + g = (random((t, 48)).float() - 1) * 0.01 + beta = (random((t, 48)).float() * 0.5 + 0.5).to(torch.bfloat16) + outputs = {name: torch.empty_like(v) for name in libraries} + workspace = torch.empty(libraries['baseline'].cs1_gdn_workspace_floats(t, 48), + dtype=torch.float32, device='cuda') + + def forward(name): + status = libraries[name].cs1_gdn_prefill(pointer(q), pointer(k), pointer(v), pointer(g), + pointer(beta), pointer(outputs[name]), pointer(workspace), t, 48, 16, + 128 ** -0.5, ctypes.c_void_p(stream.cuda_stream)) + assert status == 0, status + return outputs[name] + + # Match the production workspace layout; compare only fully written fields. + fields, offset = {}, 0 + for field, size, dtype in [('u', 48 * ((t + 63) // 64) * 64 * 128 * 2, torch.bfloat16), + ('w', 48 * ((t + 63) // 64) * 64 * 128 * 2, torch.bfloat16), + ('qd', 48 * ((t + 63) // 64) * 64 * 128 * 2, torch.bfloat16), + ('kd', 48 * ((t + 63) // 64) * 64 * 128 * 2, torch.bfloat16), + ('p', 48 * ((t + 63) // 64) * 64 * 64 * 4, torch.float32), + ('pb', 48 * ((t + 63) // 64) * 64 * 64 * 2, torch.bfloat16), + ('decay', 48 * ((t + 63) // 64) * 4, torch.float32)]: + if field != 'p': + fields[field] = workspace[offset // 4:(offset + size) // 4].view(dtype) + offset = (offset + size + 255) // 256 * 256 + reference_fields = {} + baseline = None + for name in ['baseline', 'candidate']: + started = time.perf_counter() + output = forward(name) + torch.cuda.synchronize() + if baseline is None: + baseline = output.clone() + reference_fields = {field: part.clone() for field, part in fields.items()} + differences = {field: {'changed': int((part.view(torch.int16 if part.dtype == torch.bfloat16 else torch.int32) != reference_fields[field].view(torch.int16 if part.dtype == torch.bfloat16 else torch.int32)).count_nonzero()), + 'max_abs': float((part.float() - reference_fields[field].float()).abs().max())} + for field, part in fields.items()} + write({'kind': 'workspace_fidelity', 'tokens': t, 'config': name, 'differences': differences}) + assert all(differences[field]['changed'] == 0 for field in fields), differences + delta = output.float() - baseline.float() + max_abs = float(delta.abs().max().item()) + relative_l2 = float(torch.linalg.vector_norm(delta) / + torch.linalg.vector_norm(baseline.float()).clamp_min(1e-12)) + assert torch.isfinite(output).all().item() and max_abs == 0.0 and relative_l2 == 0.0 + for _ in range(10): + forward(name) + torch.cuda.synchronize() + write({'kind': 'feasibility', 'tokens': t, 'config': name, 'passed': True, + 'max_abs': max_abs, 'relative_l2': relative_l2, + 'seconds': time.perf_counter() - started}) + for run, order in [(1, ['baseline', 'candidate']), (2, ['candidate', 'baseline'])]: + for name in order: + torch.cuda.synchronize() + start, end = [torch.cuda.Event(enable_timing=True) for _ in range(2)] + before = time.perf_counter_ns() + start.record(stream) + for _ in range(100): + output = forward(name) + end.record(stream) + end.synchronize() + wall_ms = (time.perf_counter_ns() - before) / 1e6 / 100 + delta = output.float() - baseline.float() + write({'kind': 'measurement', 'tokens': t, 'config': name, 'pass': run, + 'iterations': 100, 'device_span_ms_per_call': start.elapsed_time(end) / 100, + 'wall_ms_per_call': wall_ms, 'max_abs': float(delta.abs().max().item())}) + del q, k, v, g, beta, workspace, outputs, baseline, fields, reference_fields + write({'kind': 'complete', 'pid': os.getpid()}) + + +if __name__ == '__main__': + main() diff --git a/benchmarks/gdn/artifacts/20261003/kernel/plan.json b/benchmarks/gdn/artifacts/20261003/kernel/plan.json new file mode 100644 index 0000000..8a77d38 --- /dev/null +++ b/benchmarks/gdn/artifacts/20261003/kernel/plan.json @@ -0,0 +1,90 @@ +{ + "hypothesis": "Lossless compact TF32 staging, direct tensor-core fragment stores, and vector gate prefetch with unchanged FP32 addition order reduce GDN preparation cost while preserving every compared output bit.", + "independent_variable": "Only GDN preparation implementation: original float32 staging/WMMA versus three-byte TF32 component planes and explicit m16n8k4 (matching baseline HMMA.1684 accumulation) TF32 MMA, one-time RNA conversion, vector-prefetched sequential gate sum, fused normalized/decayed QK stores, and direct packed BF16 U/W writes. Shared allocation drops from 92 KiB to 72 KiB; no accumulator or workspace dtype/shape/API changes.", + "fixed_controls": { + "gpu_ids": [ + 2 + ], + "gpu_uuid": "GPU-cbf66259-f4ab-0ede-1811-82037dde5924", + "hardware": "H200 (driver name NVIDIA L20X)", + "numa_node": 0, + "cpu_affinity": "0-15", + "qk_heads": 16, + "v_heads": 48, + "head_dim": 128, + "dtype": "BF16 Q/K/V/beta; FP32 log g", + "seed": 20261003, + "tokens": [ + 107, + 936, + 3399 + ], + "inputs": "Seeded synthetic correlated Q/K and weak-decay gates; representative workload shapes, not captured model activations. F64 reference test also exercises grouped heads and chunk/tile edges. New recurrent-reference checks include near-zero (amplitude 1e-9) and zero Q/K. Recurrent test includes Q/K amplitude 1e-18 to exercise the full exponent range.", + "cache": "reuse; no shared cache drops", + "stream": "same eager submission on default stream" + }, + "run_budget": "Run all 3 registered GPU tests once before timing. One excluded feasibility and 10 excluded warmups per shape/arm, then exactly two measured passes of 100 calls; reverse arm order for pass 2. No extra measured repetitions. Feasibility includes workspace diagnostics; no hardware stall/occupancy attribution from these timings.", + "acceptance": { + "fidelity": "All 3 existing GPU tests (GDN float64 reference tolerance unchanged) pass. All initialized compared workspace fields (u/w/qd/kd/pb/decay) and final GDN outputs match baseline bitwise in feasibility probes. Final model gate remains unchanged decisions and max probability drift <= 0.01.", + "latency": ">=10% lower complete GDN call latency at 936 and 3399 tokens in both measured passes, with no 107-token regression, motivates E2E. CUDA event span includes host submission bubbles.", + "later_e2e_gate": ">=2% lower mean warm HTTP latency, both candidate passes faster, same 74 decisions, max probability delta <= 0.01." + }, + "stop": "Stop immediately on validation failure, numerical rejection, 30 minute timeout, or completion of declared budget.", + "git_sha": "58b8cbe9d4738f6c217f6fe584852dc9361b2c24", + "working_tree_patch": "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-20261003-090542/harness/working-tree.patch", + "worker_source_sha": "7b935723c6fa17145e3d866675202fb6f7ea1f5d", + "libraries": { + "baseline": "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-20261003-090542/harness/build/baseline/libqwen3_5_cuda.so", + "candidate": "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-20261003-090542/harness/build/candidate/libqwen3_5_cuda.so" + }, + "validation_library": "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-20261003-090542/harness/build/validation/libqwen3_5_cuda.so", + "kernel_test_binary": "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/target/release/deps/kernels-e1de1d464265a9b7", + "compiler": "CUDA 13.0 nvcc; identical ABI4 sources/options except GDN translation unit; main ABI3 validation library for Rust tests. Baseline reused byte-for-byte from the previous frozen comparison; copy time and original compilation metadata recorded separately.", + "packages": { + "torch": "2.13.0+cu129" + }, + "hashes": { + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-20261003-090542/harness/ops.h": "776376f82c8a01aacea4ec29178714c95079ec6454dc0466a483d073fee3e06c", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-20261003-090542/harness/working-tree.patch": "8fe079d6580ddab0aef333f044a9a24dca18960cac17b04426b328ef8fc151bd", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-20261003-090542/harness/gemm.cu": "a82b6d6012c6b13d5ec8dd8ce7107bd95e17f588c76dae1d314e9cff1585274a", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-20261003-090542/harness/common.cuh": "ea924f7c903b6d14b0e1f10c8c33a1a9ca23806df093628a178661a303d54d55", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-20261003-090542/harness/elementwise.cu": "b739f1bc2f08ae67399366063cc6c42bc0c5e65e98da23d8b425f9f9ea53ea59", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-20261003-090542/harness/norm.cu": "756c9d4023787cc3961ec72bcc96b3f715e705d5dfcb3bf77622879980df82ab", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-20261003-090542/harness/attention.cu": "3d6935a8a550314815ebe6575be03ce1a7e0b1d5bd611508a4bffbc24cff3132", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-20261003-090542/harness/compare.py": "00899a905c0ff9065c338d1cdad8e1f97e2959d629d40043ed8c7854b65101a3", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-20261003-090542/harness/gdn-packed-tf32-k4.cu": "9f2642a5f6e69cd22055419ecf1f799f1ccd811f7453763a41adf4f15b07a41d", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-20261003-090542/harness/gdn_prefill.cu": "1c9027b6f2d3677381d8d19611ecdf7e138718459cd61f341802cc2047a582be", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-20261003-090542/harness/runtime.cu": "3c19d2a1167b44d3a84ffe480605e14a72c775d2c2e51d0c594af9d0db6cb69e", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-20261003-090542/harness/mma.cuh": "64654f8b937585805daa72b692c243ae35d34b746e2cb6b0b55c255748495764", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-20261003-090542/harness/build/baseline/libqwen3_5_cuda.so": "8ea954d3b18d73d9fdb530fee0ddb898cf2151bb326bb67da471e9f43ca0c69e", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-20261003-090542/harness/build/candidate/libqwen3_5_cuda.so": "92cbe5485926910cd16ee911a536dddf0c74b2f6d23cb4cb6694f5e742907a48", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-20261003-090542/harness/build/validation/libqwen3_5_cuda.so": "adabeca799005b04706b1bd00b6a107b987e936642f9c345bdc9302d014e8179", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/target/release/deps/kernels-e1de1d464265a9b7": "77c9e80d7d549c14785a1de05fb2b1eced560da7ab9966446f668326254fa18a", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/src/backends/cuda/qwen3_5/gdn_prefill.cu": "9f2642a5f6e69cd22055419ecf1f799f1ccd811f7453763a41adf4f15b07a41d", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/tests/qwen3_5/kernels.rs": "2ccd7284f212152fcccfa31cab72c6bb2be6734440c3e877fe09cfc8e2077af4", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/src/models/cua_s1/native/Cargo.toml": "cf0050ee87319ab4d33cfc6ad8d111ba1b49ac113552fc7c9c42314c99f830e1" + }, + "command": [ + "gpu", + "run", + "--gpu-ids", + "2", + "--wait", + "10m", + "--timeout", + "30m", + "--note", + "Open-Jev lossless packed TF32 K4 GDN comparison", + "--", + "numactl", + "--membind=0", + "--physcpubind=0-15", + "/home/hsliu2/tmp/venvs/open-jev-native/bin/python", + "-u", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-20261003-090542/harness/compare.py" + ], + "previous_candidate": { + "run": "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-20261003-085716", + "result": "K8 feasibility failed bitwise checks; no measured runs. This candidate restores the original K4 accumulation order." + } +} diff --git a/benchmarks/gdn/artifacts/20261003/manifest.json b/benchmarks/gdn/artifacts/20261003/manifest.json new file mode 100644 index 0000000..bb5b55b --- /dev/null +++ b/benchmarks/gdn/artifacts/20261003/manifest.json @@ -0,0 +1,223 @@ +{ + "date_utc": "2026-10-03", + "base_git_sha": "58b8cbe9d4738f6c217f6fe584852dc9361b2c24", + "source_patch_sha256": "8fe079d6580ddab0aef333f044a9a24dca18960cac17b04426b328ef8fc151bd", + "full_local_archives": [ + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-bf16-e2e-20261003-081649", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-bf16-pairs-20261003-081356", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-fp16-e2e-20261003-082620", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-fp16-pairs-20261003-082340", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-20261003-085716", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-20261003-090542", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-e2e-20261003-090732", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-ready-kernels-20261003-075757", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-ready-kernels-20261003-080129", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-rna-fp16-20261003-083233", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-rna-fp16-e2e-20261003-083539", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-tf32-direct-20261003-084132", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-tf32-inverse-20261003-080601" + ], + "files": { + "kernel/plan.json": { + "source": "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-20261003-090542/plan.json", + "sha256": "24f2d09b6690bbd2cafa484c228905c4fb1150b33462c703052d7e4831368607", + "bytes": 8303 + }, + "kernel/harness/compare.py": { + "source": "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-20261003-090542/harness/compare.py", + "sha256": "00899a905c0ff9065c338d1cdad8e1f97e2959d629d40043ed8c7854b65101a3", + "bytes": 7050 + }, + "kernel/analysis/results.jsonl": { + "source": "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-20261003-090542/analysis/results.jsonl", + "sha256": "d59965ba4b4b48ab97caefe43ccfde46d3f582cb9216e8581d3775b61c043692", + "bytes": 5500 + }, + "kernel/analysis/recurrent-reference.log": { + "source": "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-20261003-090542/analysis/recurrent-reference.log", + "sha256": "b4f598578e6153d809ed2f251a52627ac99e6430a879c883b0aaf6e862829c06", + "bytes": 2068, + "source_sha256": "82a1d94dbccd2ffc790d75699b3a6997d16b7ddbc257c9ed077a9b5e4f04ef67", + "normalization": "Trailing blank line removed in the display copy; original log remains unchanged in the local archive." + }, + "kernel/analysis/summary.json": { + "source": "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-20261003-090542/analysis/summary.json", + "sha256": "9197e1bd1201646e1e03c64791f93cfefd75037edfde85dfd570f7debd85cabe", + "bytes": 899 + }, + "e2e/plan.json": { + "source": "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-e2e-20261003-090732/plan.json", + "sha256": "3e785df1f60c6f8a403caeed564bce7f7f593a1011394096ae6eeddbcdcdad41", + "bytes": 17549 + }, + "e2e/summary.json": { + "source": "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-e2e-20261003-090732/analysis/summary.json", + "sha256": "d61f0cae30bf79e9263280d62d47d9442387583be6bc811ffb7e9ed5d35d9ce7", + "bytes": 4149 + }, + "e2e/cleanup.json": { + "source": "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-e2e-20261003-090732/analysis/cleanup.json", + "sha256": "14fa8b5c21cfc33c629b586d97139fd3453f39643f5a946aa8b0befb8b6d539f", + "bytes": 737 + }, + "flashinfer/plan.json": { + "source": "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-ready-kernels-20261003-080129/plan.json", + "sha256": "aff88d6ed4f19104102036cd9e49e329829d1824d6d825a571907f4b400dba5f", + "bytes": 4226 + }, + "flashinfer/results.jsonl": { + "source": "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-ready-kernels-20261003-080129/analysis/results.jsonl", + "sha256": "637b872461d54d382b8184e167a40c4c689a6fcc720653c706dc9d1851599a73", + "bytes": 5697 + }, + "e2e/baseline-measured-1.jsonl": { + "source": "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-e2e-20261003-090732/analysis/baseline-measured-1.jsonl", + "sha256": "ff0edc3cd0581b2a8374a6ed5cfb13e70c086bb027e3b04208115c62a12a1ff9", + "bytes": 62560 + }, + "e2e/baseline-measured-2.jsonl": { + "source": "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-e2e-20261003-090732/analysis/baseline-measured-2.jsonl", + "sha256": "5a60b08924461e9bfe50310c21832df29be5fed83825dc58502cf6c10c39e5d0", + "bytes": 62566 + }, + "e2e/candidate-measured-1.jsonl": { + "source": "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-e2e-20261003-090732/analysis/candidate-measured-1.jsonl", + "sha256": "aa11cb7a4ef76cd7551561cc50a09fa9885f11b89035f8457881376d7a024a2e", + "bytes": 62565 + }, + "e2e/candidate-measured-2.jsonl": { + "source": "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-e2e-20261003-090732/analysis/candidate-measured-2.jsonl", + "sha256": "37d517470e0a2eb26f4faf956da2339c87eb9ca7e2123b4d4efa6f086b28f44e", + "bytes": 62560 + }, + "rejected-variants.json": { + "source": "Projection of preserved failed feasibility records", + "sha256": "318c1ba7dda332ec346983b8c90eee826cd35bb3afaae489211e7486f599ed53", + "bytes": 4649 + } + }, + "build_and_check_records": [ + { + "command": [ + "/home/hsliu2/.cargo/bin/cargo", + "fmt", + "--all", + "--check" + ], + "exit_code": 0, + "seconds": 0.24558558594435453, + "log": "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-final-cpu-checks-20261003-090543/0.log" + }, + { + "command": [ + "/home/hsliu2/.cargo/bin/cargo", + "clippy", + "--workspace", + "--locked", + "--all-targets", + "--", + "-D", + "warnings" + ], + "exit_code": 0, + "seconds": 2.6444142553955317, + "log": "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-final-cpu-checks-20261003-090543/1.log" + }, + { + "command": [ + "/home/hsliu2/.cargo/bin/cargo", + "test", + "--workspace", + "--locked" + ], + "exit_code": 0, + "seconds": 6.1790889129042625, + "log": "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-final-cpu-checks-20261003-090543/2.log" + }, + { + "command": [ + "/home/hsliu2/.cargo/bin/cargo", + "build", + "--workspace", + "--release", + "--locked" + ], + "exit_code": 0, + "seconds": 1.5214574886485934, + "log": "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-final-cpu-checks-20261003-090543/3.log" + }, + { + "arch": "80", + "command": [ + "bash", + "src/backends/cuda/qwen3_5/build.sh", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-20261003-090542/analysis/architecture-builds/sm80", + "80" + ], + "exit_code": 0, + "seconds": 10.438040406443179 + }, + { + "arch": "89", + "command": [ + "bash", + "src/backends/cuda/qwen3_5/build.sh", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-20261003-090542/analysis/architecture-builds/sm89", + "89" + ], + "exit_code": 0, + "seconds": 10.547066879458725 + }, + { + "name": "baseline", + "reused_from": "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-20261003-085716/harness/build/baseline/libqwen3_5_cuda.so", + "copy_seconds": 0.01750018075108528, + "original_build": "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-fp16-pairs-20261003-082340/analysis/build.json" + }, + { + "name": "candidate", + "command": [ + "/usr/local/cuda/bin/nvcc", + "-O3", + "-std=c++17", + "-lineinfo", + "-gencode", + "arch=compute_90,code=[sm_90,compute_90]", + "-shared", + "-Xcompiler", + "-fPIC", + "-Xcompiler", + "-Wall,-Wextra", + "-I/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-20261003-090542/harness", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-20261003-090542/harness/gdn-packed-tf32-k4.cu", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-20261003-090542/harness/attention.cu", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-20261003-090542/harness/elementwise.cu", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-20261003-090542/harness/gemm.cu", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-20261003-090542/harness/norm.cu", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-20261003-090542/harness/runtime.cu", + "-L/usr/local/cuda/lib64", + "-lcublasLt", + "-Xlinker", + "-rpath", + "-Xlinker", + "/usr/local/cuda/lib64", + "-o", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-20261003-090542/harness/build/candidate/libqwen3_5_cuda.so" + ], + "seconds": 11.332285988144577, + "exit_code": 0 + }, + { + "name": "validation", + "command": [ + "bash", + "src/backends/cuda/qwen3_5/build.sh", + "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-packed-tf32-k4-20261003-090542/harness/build/validation", + "90" + ], + "seconds": 10.502563803456724, + "exit_code": 0 + } + ], + "source_dataset_revision": "f8ce71361165846101d02ebc83ad44e47ae44fc3" +} diff --git a/benchmarks/gdn/artifacts/20261003/rejected-variants.json b/benchmarks/gdn/artifacts/20261003/rejected-variants.json new file mode 100644 index 0000000..c956dad --- /dev/null +++ b/benchmarks/gdn/artifacts/20261003/rejected-variants.json @@ -0,0 +1,146 @@ +[ + { + "archive": "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-bf16-e2e-20261003-081649", + "completed_candidate_feasibility_requests": 44, + "failed_row": { + "id": "hard-opus-a-temporal_numeric-09", + "latency_s": 0.05343177, + "probability_yes": 0.5064797018921715, + "probability_delta": 0.008436719292213957, + "predicted": "yes", + "raw": { + "model": "Qwen/Qwen3.8-27B", + "answers": { + "decision": { + "type": "noul", + "noul": 0.5064797018921715 + } + }, + "usage": { + "input_tokens": 519, + "output_tokens": 0 + }, + "metadata": { + "method": "native_merged_lora_decision_head", + "temperature": 2.5343690298472983, + "candidate_sequences": 1, + "inference_seconds": 0.053131982, + "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", + "max_length": 16384, + "prefix_cache": { + "enabled": false, + "mode": "independent_candidates" + } + } + }, + "checks": { + "finite_probability": true, + "usage": true, + "decision": false, + "probability": true, + "uninstrumented": true, + "baseline_exact": true + }, + "client_start_monotonic_ns": 5004228895556697, + "client_end_monotonic_ns": 5004228948988467 + }, + "measured_candidate_passes": 0, + "raw_source_sha256": "eb896dce784577697683e54134bf0f59215eb80ecc4ef0a325cbfd616f5baefb" + }, + { + "archive": "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-fp16-e2e-20261003-082620", + "completed_candidate_feasibility_requests": 37, + "failed_row": { + "id": "hard-opus-a-long_policy-11", + "latency_s": 0.503507706, + "probability_yes": 0.6972432299494479, + "probability_delta": 0.01261642382467143, + "predicted": "yes", + "raw": { + "model": "Qwen/Qwen3.8-27B", + "answers": { + "decision": { + "type": "noul", + "noul": 0.6972432299494479 + } + }, + "usage": { + "input_tokens": 2334, + "output_tokens": 0 + }, + "metadata": { + "method": "native_merged_lora_decision_head", + "temperature": 2.5343690298472983, + "candidate_sequences": 1, + "inference_seconds": 0.5028902, + "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", + "max_length": 16384, + "prefix_cache": { + "enabled": false, + "mode": "independent_candidates" + } + } + }, + "checks": { + "finite_probability": true, + "usage": true, + "decision": true, + "probability": false, + "uninstrumented": true, + "baseline_exact": true + }, + "client_start_monotonic_ns": 5004742452824477, + "client_end_monotonic_ns": 5004742956332183 + }, + "measured_candidate_passes": 0, + "raw_source_sha256": "df354021ec29f8afc7d49e91e8d71f738f0a50732e46060336caa4fb264ee304" + }, + { + "archive": "/data/hsliu2/tmp/system1-omni-gdn-optimization-20261003/profile/gdn-rna-fp16-e2e-20261003-083539", + "completed_candidate_feasibility_requests": 44, + "failed_row": { + "id": "hard-opus-a-temporal_numeric-09", + "latency_s": 0.055783496, + "probability_yes": 0.5051902978382125, + "probability_delta": 0.007147315238254903, + "predicted": "yes", + "raw": { + "model": "Qwen/Qwen3.8-27B", + "answers": { + "decision": { + "type": "noul", + "noul": 0.5051902978382125 + } + }, + "usage": { + "input_tokens": 519, + "output_tokens": 0 + }, + "metadata": { + "method": "native_merged_lora_decision_head", + "temperature": 2.5343690298472983, + "candidate_sequences": 1, + "inference_seconds": 0.055335681, + "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", + "max_length": 16384, + "prefix_cache": { + "enabled": false, + "mode": "independent_candidates" + } + } + }, + "checks": { + "finite_probability": true, + "usage": true, + "decision": false, + "probability": true, + "uninstrumented": true, + "baseline_exact": true + }, + "client_start_monotonic_ns": 5005290022734274, + "client_end_monotonic_ns": 5005290078517770 + }, + "measured_candidate_passes": 0, + "raw_source_sha256": "b4693fb5acdb217c172aa4580d2e2b7e728711e2ec41720a25c5551b5dec946d" + } +] diff --git a/src/backends/cuda/qwen3_5/README.md b/src/backends/cuda/qwen3_5/README.md index b84f50b..e35c2be 100644 --- a/src/backends/cuda/qwen3_5/README.md +++ b/src/backends/cuda/qwen3_5/README.md @@ -6,4 +6,6 @@ CUDA kernels for a prefill-only Qwen3.5 forward pass, built into `libqwen3_5_cud src/backends/cuda/qwen3_5/build.sh [compute capability, default 89] ``` -The norm, elementwise and q/k preparation kernels round to bfloat16 where Transformers (`modeling_qwen3_5.py`) does. Attention (FlashAttention-2 style, on tensor cores) and the chunked gated delta rule keep some intermediate results in bfloat16, as FlashAttention and flash-linear-attention do. GEMMs go through cuBLASLt with its first heuristic choice. Tensor-core kernels need sm_80 or newer; only sm_89 has been run. +The norm, elementwise and q/k preparation kernels round to bfloat16 where Transformers (`modeling_qwen3_5.py`) does. Attention (FlashAttention-2 style, on tensor cores) and the chunked gated delta rule keep some intermediate results in bfloat16, as FlashAttention and flash-linear-attention do. GEMMs go through cuBLASLt with its first heuristic choice. Tensor-core kernels need sm_80 or newer. The native worker was previously run on sm_89; the current kernel tests and GDN measurements cover sm_90. Compilation passed for sm_80, sm_89 and sm_90; execution of the modified GDN kernel on sm_80/sm_89 remains unverified. + +Gated DeltaNet preparation stores converted TF32 operands in three-byte component planes, preserves the original four-term TF32 accumulation, and writes U/W fragments directly as bfloat16. Dynamic shared memory is 72 KiB per block. The [H200 comparison](../../../../benchmarks/gdn/README.md) records complete GDN call latency, numerical checks, and the small end-to-end change measured with the Open-Jev worker from PR #55. diff --git a/src/backends/cuda/qwen3_5/gdn_prefill.cu b/src/backends/cuda/qwen3_5/gdn_prefill.cu index 516e51b..676ce44 100644 --- a/src/backends/cuda/qwen3_5/gdn_prefill.cu +++ b/src/backends/cuda/qwen3_5/gdn_prefill.cu @@ -12,7 +12,7 @@ // 3. gdn_chunk_out, per (chunk, head): o = qd S + P v_new with mma.sync. // Transformers computes all of this in float32. Keeping the intermediate results in // bfloat16, as flash-linear-attention does, makes this kernel less precise than that -// path; tests/kernels.rs checks it against a float64 token-by-token reference. +// path; tests/qwen3_5/kernels.rs checks it against a float64 token-by-token reference. #include #include #include @@ -32,23 +32,32 @@ constexpr int C = 64; // chunk length constexpr int K = 128; // key head dim constexpr int V = 128; // value head dim constexpr int THREADS = 256; -// row strides: multiples of 16 bytes as WMMA needs, and not multiples of 32 floats -constexpr int KP = K + 4; // float +// Padded row strides for the shared matrices. +constexpr int KP = K + 4; // packed TF32 component planes constexpr int CP = C + 4; // float constexpr int HB = K + 8; // bfloat16 constexpr int TB = C + 8; // bfloat16 -using ATf32Row = wmma::fragment; -using BTf32Col = wmma::fragment; -using CTf32 = wmma::fragment; -using ABf16 = wmma::fragment; -using BBf16 = wmma::fragment; -using CBf16 = wmma::fragment; +// Store the 19 bits of converted TF32 operands in separate 16-bit and 3-bit +// planes, using three bytes per value. Preserve the full exponent range. +struct Tf32Row { + uint16_t lo[KP]; + uint8_t hi[KP]; +}; +static_assert(sizeof(Tf32Row) == KP * 3); -template -__device__ __forceinline__ void to_tf32(F& f) { -#pragma unroll - for (int t = 0; t < f.num_elements; t++) f.x[t] = wmma::__float_to_tf32(f.x[t]); +__device__ __forceinline__ uint32_t load_tf32(const Tf32Row* m, int row, int col) { + return (uint32_t(m[row].lo[col]) << 13) | (uint32_t(m[row].hi[col]) << 29); +} + +// Retain the four-term accumulation used by the original TF32 WMMA path. +// PTX m16n8k4 accumulators match cs1::mma16816. +__device__ __forceinline__ void mma1684_tf32(float (&d)[4], const uint32_t (&a)[2], uint32_t b) { + asm volatile( + "mma.sync.aligned.m16n8k4.row.col.f32.tf32.tf32.f32 {%0,%1,%2,%3}, {%4,%5}, {%6}, " + "{%0,%1,%2,%3};\n" + : "+f"(d[0]), "+f"(d[1]), "+f"(d[2]), "+f"(d[3]) + : "r"(a[0]), "r"(a[1]), "r"(b)); } struct Work { @@ -88,30 +97,31 @@ struct Layout { }; // kernel 1 shared memory, bytes -constexpr int R1 = 0; // kn float [C][KP]; later v as bf16 [C][HB] -constexpr int R2 = R1 + C * KP * 4; // qn float [C][KP]; later T float [C][CP] + scratch, then T as bf16 -constexpr int R3 = R2 + C * KP * 4; // A float [C][CP]; later k exp(cum) as bf16 [C][HB] -constexpr int R4 = R3 + C * CP * 4; // cum, beta, exp(cum), exp(cum_last - cum) -constexpr int R5 = R4 + 4 * C * 4; // per-warp 16x16 float staging of u and w -constexpr size_t SMEM1_BYTES = R5 + (THREADS / 32) * 256 * 4; constexpr int TBF = C * CP * 4 + 3 * 256 * 4; // offset of T as bf16 inside R2 +constexpr int R1 = 0; // kn packed TF32 [C]; later v as bf16 [C][HB] +constexpr int R2 = R1 + C * sizeof(Tf32Row); // qn packed TF32; later T + scratch, then T as bf16 +constexpr int R3 = R2 + TBF + C * TB * 2; // A float [C][CP]; later k exp(cum) as bf16 [C][HB] +constexpr int R4 = R3 + C * CP * 4; // cum, beta, exp(cum), exp(cum_last - cum), k norm inverse +constexpr size_t SMEM1_BYTES = R4 + 5 * C * 4; static_assert(C * HB * 2 <= C * CP * 4, "k exp(cum) as bf16 fits over A"); -static_assert(TBF + C * TB * 2 <= C * KP * 4, "T as bf16 fits in R2"); +static_assert(C * sizeof(Tf32Row) <= R3 - R2, "packed qn fits in R2"); +static_assert(C * HB * 2 <= R2 - R1, "v as bf16 fits over kn"); __global__ void __launch_bounds__(THREADS) gdn_chunk_prep( const __nv_bfloat16* __restrict__ q, const __nv_bfloat16* __restrict__ k, const __nv_bfloat16* __restrict__ v, const float* __restrict__ g, const __nv_bfloat16* __restrict__ beta, Work ws, int T, int H, int HK, float scale) { extern __shared__ __align__(128) unsigned char sm[]; - float* kn = reinterpret_cast(sm + R1); - float* qn = reinterpret_cast(sm + R2); - float* tm = qn; + Tf32Row* kn = reinterpret_cast(sm + R1); + Tf32Row* qn = reinterpret_cast(sm + R2); + float* tm = reinterpret_cast(sm + R2); float* sc = tm + C * CP; float* am = reinterpret_cast(sm + R3); float* cum = reinterpret_cast(sm + R4); float* bet = cum + C; float* ecum = bet + C; float* erem = ecum + C; + float* kinvs = erem + C; __nv_bfloat16* vb = reinterpret_cast<__nv_bfloat16*>(sm + R1); __nv_bfloat16* tb = reinterpret_cast<__nv_bfloat16*>(sm + R2 + TBF); __nv_bfloat16* kw = reinterpret_cast<__nv_bfloat16*>(sm + R3); @@ -120,7 +130,43 @@ __global__ void __launch_bounds__(THREADS) gdn_chunk_prep( const int hk = h / (H / HK); const int tid = threadIdx.x, lane = tid & 31, warp = tid >> 5; - // 1. load q and k rows of this chunk, L2-normalize (padding rows stay zero) + // 1. cumulative decays and learning rates (padding rows stay zero) + if (tid < C) { + const int t = c * C + tid; + cum[tid] = t < T ? g[(size_t)t * H + h] : 0.f; + bet[tid] = t < T ? __bfloat162float(beta[(size_t)t * H + h]) : 0.f; + } + __syncthreads(); + if (tid == 0) { + float s = 0.f; + for (int i = 0; i < C; i += 8) { + // Prefetch eight gates together, retaining the sequential FP32 sum. + const float4 a = *reinterpret_cast(cum + i); + const float4 b = *reinterpret_cast(cum + i + 4); + float4 p, q; + p.x = s + a.x; + p.y = p.x + a.y; + p.z = p.y + a.z; + p.w = p.z + a.w; + q.x = p.w + b.x; + q.y = q.x + b.y; + q.z = q.y + b.z; + q.w = q.z + b.w; + s = q.w; + *reinterpret_cast(cum + i) = p; + *reinterpret_cast(cum + i + 4) = q; + } + ws.decay[h * NC + c] = expf(s); + } + __syncthreads(); + if (tid < C) { + ecum[tid] = expf(cum[tid]); + erem[tid] = expf(cum[C - 1] - cum[tid]); + } + __syncthreads(); + + // 2. L2-normalize q/k in float32 and form decayed q/k before rounding to + // bfloat16. Convert the pair-product operands to TF32 once per row. for (int r = warp; r < C; r += THREADS / 32) { const int t = c * C + r; float kv[4], qv[4], ks = 0.f, qs = 0.f; @@ -140,42 +186,24 @@ __global__ void __launch_bounds__(THREADS) gdn_chunk_prep( ks = warp_sum(ks); qs = warp_sum(qs); const float kinv = rsqrtf(ks + 1e-6f), qinv = rsqrtf(qs + 1e-6f) * scale; + if (lane == 0) kinvs[r] = kinv; #pragma unroll for (int e = 0; e < 4; e++) { const int d = lane + 32 * e; - kn[r * KP + d] = kv[e] * kinv; - qn[r * KP + d] = qv[e] * qinv; + const float kx = kv[e] * kinv, qx = qv[e] * qinv; + const uint32_t kc = __float_as_uint(wmma::__float_to_tf32(kx)) >> 13; + const uint32_t qc = __float_as_uint(wmma::__float_to_tf32(qx)) >> 13; + kn[r].lo[d] = uint16_t(kc); + kn[r].hi[d] = uint8_t(kc >> 16); + qn[r].lo[d] = uint16_t(qc); + qn[r].hi[d] = uint8_t(qc >> 16); + const size_t row = ((size_t)h * NCC + c * C + r) * K + d; + ws.qd[row] = __float2bfloat16(qx * ecum[r]); + ws.kd[row] = __float2bfloat16(kx * erem[r]); } } - if (tid < C) { - const int t = c * C + tid; - cum[tid] = t < T ? g[(size_t)t * H + h] : 0.f; - bet[tid] = t < T ? __bfloat162float(beta[(size_t)t * H + h]) : 0.f; - } - __syncthreads(); - if (tid == 0) { - float s = 0.f; - for (int i = 0; i < C; i++) { - s += cum[i]; - cum[i] = s; - } - ws.decay[h * NC + c] = expf(s); - } - __syncthreads(); - if (tid < C) { - ecum[tid] = expf(cum[tid]); - erem[tid] = expf(cum[C - 1] - cum[tid]); - } __syncthreads(); - // 2. decayed q and k for the state and output kernels - for (int x = tid; x < C * K; x += THREADS) { - const int i = x / K, d = x % K; - const size_t row = ((size_t)h * NCC + c * C + i) * K + d; - ws.qd[row] = __float2bfloat16(qn[i * KP + d] * ecum[i]); - ws.kd[row] = __float2bfloat16(kn[i * KP + d] * erem[i]); - } - // 3. pair products on tensor cores: the 10 lower 16x16 tiles of k.k (into A) and of // q.k (into P, in global memory), then the masks and decays elementwise float* pout = ws.p + ((size_t)h * NC + c) * C * C; @@ -183,24 +211,31 @@ __global__ void __launch_bounds__(THREADS) gdn_chunk_prep( const int tile = e % 10; const int it = tile < 1 ? 0 : tile < 3 ? 1 : tile < 6 ? 2 : 3; const int jt = tile - it * (it + 1) / 2; - const float* a = (e < 10 ? kn : qn) + it * 16 * KP; - const float* b = kn + jt * 16 * KP; - CTf32 acc; - wmma::fill_fragment(acc, 0.f); + const Tf32Row* asrc = e < 10 ? kn : qn; + const int group = lane / 4, thread = lane % 4; + float acc[2][4] = {}; #pragma unroll 4 - for (int k0 = 0; k0 < K; k0 += 8) { - ATf32Row fa; - BTf32Col fb; - wmma::load_matrix_sync(fa, a + k0, KP); - wmma::load_matrix_sync(fb, b + k0, KP); - to_tf32(fa); - to_tf32(fb); - wmma::mma_sync(acc, fa, fb, acc); + for (int k0 = 0; k0 < K; k0 += 4) { + uint32_t a[2] = { + load_tf32(asrc, it * 16 + group, k0 + thread), + load_tf32(asrc, it * 16 + group + 8, k0 + thread)}; +#pragma unroll + for (int nt = 0; nt < 2; nt++) { + const int col = jt * 16 + group + nt * 8; + const uint32_t b = load_tf32(kn, col, k0 + thread); + mma1684_tf32(acc[nt], a, b); + } + } +#pragma unroll + for (int r = 0; r < 2; r++) { + const int row = it * 16 + group + r * 8; +#pragma unroll + for (int nt = 0; nt < 2; nt++) { + const int col = jt * 16 + 2 * thread + nt * 8; + float* dst = (e < 10 ? am + row * CP : pout + row * C) + col; + *reinterpret_cast(dst) = make_float2(acc[nt][2 * r], acc[nt][2 * r + 1]); + } } - if (e < 10) - wmma::store_matrix_sync(am + it * 16 * CP + jt * 16, acc, CP, wmma::mem_row_major); - else - wmma::store_matrix_sync(pout + it * 16 * C + jt * 16, acc, C, wmma::mem_row_major); } __syncthreads(); __nv_bfloat16* pb = ws.pb + ((size_t)h * NC + c) * C * C; @@ -255,7 +290,9 @@ __global__ void __launch_bounds__(THREADS) gdn_chunk_prep( // u = T (beta v) and w = T (beta exp(cum) k) on tensor cores, stored as bfloat16 for (int x = tid; x < C * K; x += THREADS) { const int j = x / K, d = x % K; - kw[j * HB + d] = __float2bfloat16(kn[j * KP + d] * ecum[j]); + const int t = c * C + j; + const float kx = t < T ? __bfloat162float(k[((size_t)t * HK + hk) * K + d]) : 0.f; + kw[j * HB + d] = __float2bfloat16((kx * kinvs[j]) * ecum[j]); } for (int x = tid; x < C * C; x += THREADS) { const int i = x / C, j = x % C; @@ -268,25 +305,26 @@ __global__ void __launch_bounds__(THREADS) gdn_chunk_prep( vb[j * HB + d] = t < T ? v[((size_t)t * H + h) * V + d] : __float2bfloat16(0.f); } __syncthreads(); - float* stage = reinterpret_cast(sm + R5) + warp * 256; for (int e = warp; e < 64; e += THREADS / 32) { const bool is_u = e < 32; const int it = (e % 32) / 8, dt = e % 8; const __nv_bfloat16* bsrc = is_u ? vb : kw; - CBf16 acc; - wmma::fill_fragment(acc, 0.f); + float acc[2][4] = {}; for (int kb = 0; kb <= it; kb++) { // T is lower triangular - ABf16 fa; - BBf16 fb; - wmma::load_matrix_sync(fa, tb + it * 16 * TB + kb * 16, TB); - wmma::load_matrix_sync(fb, bsrc + kb * 16 * HB + dt * 16, HB); - wmma::mma_sync(acc, fa, fb, acc); + uint32_t a[4], b[4]; + cs1::load_a(a, tb, TB, it * 16, kb * 16, lane); + cs1::load_b_kn(b, bsrc, HB, kb * 16, dt * 16, lane); + cs1::mma16816(acc[0], a, b[0], b[1]); + cs1::mma16816(acc[1], a, b[2], b[3]); + } +#pragma unroll + for (int r = 0; r < 2; r++) { + const int row = it * 16 + lane / 4 + r * 8; + bf16* dst = (is_u ? ws.u : ws.w) + ((size_t)h * NCC + c * C + row) * K + dt * 16 + 2 * (lane % 4); +#pragma unroll + for (int nt = 0; nt < 2; nt++) + *reinterpret_cast(dst + nt * 8) = cs1::pack_bf16(acc[nt][2 * r], acc[nt][2 * r + 1]); } - wmma::store_matrix_sync(stage, acc, 16, wmma::mem_row_major); - __syncwarp(); - bf16* dst = (is_u ? ws.u : ws.w) + ((size_t)h * NCC + c * C + it * 16) * K + dt * 16; - for (int x = lane; x < 256; x += 32) dst[(x / 16) * K + x % 16] = __float2bfloat16(stage[x]); - __syncwarp(); } } diff --git a/src/models/cua_s1/native/Cargo.toml b/src/models/cua_s1/native/Cargo.toml index 2e8caf6..1c4157d 100644 --- a/src/models/cua_s1/native/Cargo.toml +++ b/src/models/cua_s1/native/Cargo.toml @@ -5,6 +5,10 @@ edition = "2024" publish = false description = "Native /v1/systemone worker for Cua-S1 4B 0.2 (text adapter)" +[[test]] +name = "kernels" +path = "../../../../tests/qwen3_5/kernels.rs" + [[bin]] name = "omni-cua-s1-native" path = "src/main.rs" diff --git a/src/models/cua_s1/native/tests/kernels.rs b/tests/qwen3_5/kernels.rs similarity index 92% rename from src/models/cua_s1/native/tests/kernels.rs rename to tests/qwen3_5/kernels.rs index 692f1b5..20d2371 100644 --- a/src/models/cua_s1/native/tests/kernels.rs +++ b/tests/qwen3_5/kernels.rs @@ -269,20 +269,38 @@ fn gated_delta_reference( #[ignore = "needs a GPU and CUA_S1_CUDA_LIB"] fn gated_delta_rule_matches_recurrent_reference() { let st = setup(); - let (h, hk, d) = (4usize, 2usize, 128usize); - for t in [1usize, 64, 150] { + let d = 128usize; + // Tile/chunk boundaries, weak decay, grouped heads, and near-zero q/k. + // Keep the float64 recurrent reference and its tolerance. + for (t, h, hk, decay_scale, qk_amp) in [ + (1usize, 4usize, 2usize, 1.0f32, 1.0f32), + (16, 4, 2, 1.0, 1.0), + (17, 4, 2, 1.0, 1.0), + (32, 4, 2, 1.0, 1.0), + (33, 4, 2, 1.0, 1.0), + (63, 4, 2, 1.0, 1.0), + (64, 4, 2, 1.0, 1.0), + (65, 4, 2, 1.0, 1.0), + (107, 4, 2, 1.0, 1.0), + (150, 4, 2, 1.0, 1.0), + (936, 6, 2, 0.01, 1.0), + (3399, 3, 1, 0.01, 1.0), + (65, 4, 2, 0.01, 1e-9), + (65, 4, 2, 0.01, 1e-18), + (65, 4, 2, 0.01, 0.0), + ] { // q close to k, so that q.k and the outputs are of order one as in the model - let k = random(t * hk * d, 12, 1.0); + let k = random(t * hk * d, 12, qk_amp); let q: Vec = k .iter() - .zip(random(t * hk * d, 11, 1.0)) + .zip(random(t * hk * d, 11, qk_amp)) .map(|(k, n)| bf16::from_f32(0.8 * k.to_f32() + 0.2 * n.to_f32())) .collect(); let v = random(t * h * d, 13, 1.0); // log decays in (-2, 0) and learning rates in (0, 1), as sigmoid and -exp * softplus give let g: Vec = random(t * h, 14, 1.0) .iter() - .map(|x| x.to_f32() - 1.0) + .map(|x| (x.to_f32() - 1.0) * decay_scale) .collect(); let beta: Vec = random(t * h, 15, 0.5) .iter() @@ -322,6 +340,10 @@ fn gated_delta_rule_matches_recurrent_reference() { .unwrap(); } let got = from_device(&o, t * h * d, st); + assert!( + got.iter().all(|x| x.is_finite()), + "non-finite GDN at t = {t}" + ); let scale = want.iter().fold(0f64, |m, x| m.max(x.abs())); let worst = got .iter() @@ -329,7 +351,7 @@ fn gated_delta_rule_matches_recurrent_reference() { .map(|(a, b)| (*a as f64 - b).abs()) .fold(0f64, f64::max); eprintln!( - "gated delta t = {t}: largest difference {worst:.2e}, largest |reference| {scale:.2}" + "gated delta t = {t}, amplitude {qk_amp}: largest difference {worst:.2e}, largest |reference| {scale:.2e}" ); assert!(worst <= 2e-2 * scale, "t = {t}: {worst} vs scale {scale}"); }