topic14-docs

benchmarks/bench_const_merge.py

@@ -12,15 +12,22 @@
 from __future__ import annotations
 
 import argparse
+import os
 import statistics
+import sys
 import time
-from typing import Optional
 
-from scratchv.backend.const_merge import merge_constants
+BENCH_DIR = os.path.dirname(__file__)
+PROJ_DIR = os.path.dirname(BENCH_DIR)
+sys.path.insert(0, PROJ_DIR)
+
+from scratchv.backend._asm_parser import parse_asm
+from scratchv.backend.const_merge import merge_constants_detailed
 
 
 def _gen_synthetic_asm(num_instrs: int, seed: int = 42,
-                        lui_ratio: float = 0.3) -> str:
+                       lui_ratio: float = 0.3,
+                       redundant_lui_ratio: float = 0.1) -> str:
     """Generate synthetic assembly with lui+addi patterns.
 
     Parameters
@@ -31,16 +38,34 @@ def _gen_synthetic_asm(num_instrs: int, seed: int = 42,
         Random seed for reproducibility.
     lui_ratio:
         Fraction of instructions that form lui+addi pairs.
+    redundant_lui_ratio:
+        Fraction of generated groups that contain a redundant LUI pattern.
     """
+    if num_instrs < 0:
+        raise ValueError("num_instrs must be non-negative")
+    if not 0.0 <= lui_ratio <= 1.0:
+        raise ValueError("lui_ratio must be between 0 and 1")
+    if not 0.0 <= redundant_lui_ratio <= 1.0:
+        raise ValueError("redundant_lui_ratio must be between 0 and 1")
+    if lui_ratio + redundant_lui_ratio > 1.0:
+        raise ValueError("lui_ratio + redundant_lui_ratio must not exceed 1")
     import random
     random.seed(seed)
 
     lines = [".text", "synthetic_func:"]
     i = 0
     while i < num_instrs:
-        use_lui = random.random() < lui_ratio
+        choice = random.random()
 
-        if use_lui and i + 1 < num_instrs:
+        if choice < redundant_lui_ratio and i + 2 < num_instrs:
+            regs = ["t0", "t1", "t2", "s0", "s1", "a0", "a1"]
+            r = random.choice(regs)
+            imm_hi = random.choice([0x10000, 0x20000, 0x12345])
+            lines.append(f"  lui {r}, {hex(imm_hi)}")
+            lines.append(f"  add a4, a5, a6")
+            lines.append(f"  lui {r}, {hex(imm_hi)}")
+            i += 3
+        elif choice < redundant_lui_ratio + lui_ratio and i + 1 < num_instrs:
             regs = ["t0", "t1", "t2", "s0", "s1", "a0", "a1", "a2", "a3"]
             r = random.choice(regs)
             imm_hi = random.choice([0x10000, 0x20000, 0x12345, 0xABCDE, 0xFFFFF])
@@ -78,24 +103,39 @@ def _gen_synthetic_asm(num_instrs: int, seed: int = 42,
 
 def bench_merge(asm_text: str, repeats: int = 50) -> dict:
     """Benchmark the constant merge optimizer."""
+    if repeats < 1:
+        raise ValueError("repeats must be at least 1")
     times = []
     results = []
 
     for _ in range(repeats):
         t0 = time.perf_counter()
-        result, changes = merge_constants(asm_text)
+        result, stats = merge_constants_detailed(asm_text)
         t1 = time.perf_counter()
         times.append(t1 - t0)
-        results.append((result, changes))
-
-    changes_list = [r[1] for r in results]
-    input_lines = asm_text.count("\n")
-    output_lines = results[0][0].count("\n") if results else 0
+        results.append((result, stats))
+
+    changes_list = [r[1].total_changes for r in results]
+    first_stats = results[0][1]
+    parsed_input = parse_asm(asm_text)
+    parsed_output = parse_asm(results[0][0]) if results else []
+    input_instructions = sum(
+        line.opcode is not None and not line.is_directive
+        for line in parsed_input
+    )
+    output_instructions = sum(
+        line.opcode is not None and not line.is_directive
+        for line in parsed_output
+    )
 
     return {
-        "input_lines": input_lines,
-        "output_lines": output_lines,
-        "line_reduction": input_lines - output_lines,
+        "benchmark_type": "synthetic",
+        "input_instructions": input_instructions,
+        "output_instructions": output_instructions,
+        "instruction_reduction": input_instructions - output_instructions,
+        "candidate_pairs": first_stats.candidate_pairs,
+        "merged_pairs": first_stats.merged_pairs,
+        "redundant_lui_removed": first_stats.redundant_lui_removed,
         "changes_mean": statistics.mean(changes_list),
         "changes_stdev": statistics.stdev(changes_list) if len(changes_list) > 1 else 0,
         "repeats": repeats,
@@ -111,35 +151,46 @@ def main():
     parser = argparse.ArgumentParser(description="Constant Merge Benchmark")
     parser.add_argument("--repeats", type=int, default=50,
                         help="Number of repeat measurements")
+    parser.add_argument("--seed", type=int, default=42)
+    parser.add_argument("--pair-density", type=float, default=0.3)
+    parser.add_argument("--redundant-lui-density", type=float, default=0.1)
     args = parser.parse_args()
 
     sizes = [100, 500, 1000, 2000, 5000]
     print("=" * 80)
     print("RISC-V Constant Load Merge Optimizer Benchmark")
+    print("benchmark_type=synthetic")
     print("=" * 80)
 
     print(f"\n{'Size':>8} {'Mean(ms)':>10} {'Stdev(ms)':>10} "
-          f"{'Changes':>8} {'InpLines':>10} {'OutLines':>10} {'Reduc':>8}")
+          f"{'Pairs':>8} {'RedLUI':>8} {'InpInst':>10} {'OutInst':>10}")
     print("-" * 80)
 
     for size in sizes:
-        asm = _gen_synthetic_asm(size, lui_ratio=0.3)
+        asm = _gen_synthetic_asm(
+            size, seed=args.seed, lui_ratio=args.pair_density,
+            redundant_lui_ratio=args.redundant_lui_density,
+        )
         stats = bench_merge(asm, repeats=args.repeats)
         print(f"{size:>8} {stats['mean_s'] * 1000:>10.3f} "
               f"{stats['stdev_s'] * 1000:>10.3f} "
-              f"{stats['changes_mean']:>8.1f} "
-              f"{stats['input_lines']:>10} {stats['output_lines']:>10} "
-              f"{stats['line_reduction']:>8}")
+              f"{stats['merged_pairs']:>8} "
+              f"{stats['redundant_lui_removed']:>8} "
+              f"{stats['input_instructions']:>10} "
+              f"{stats['output_instructions']:>10}")
 
     # Test different lui densities
     print(f"\nLUI Density Impact (2000 instructions):")
     print("-" * 60)
     for ratio in [0.0, 0.1, 0.3, 0.5]:
-        asm = _gen_synthetic_asm(2000, lui_ratio=ratio)
+        asm = _gen_synthetic_asm(
+            2000, seed=args.seed, lui_ratio=ratio,
+            redundant_lui_ratio=args.redundant_lui_density,
+        )
         stats = bench_merge(asm, repeats=args.repeats)
         print(f"  ratio={ratio:.1f}  {stats['mean_s'] * 1000:.3f} ms  "
               f"changes: {stats['changes_mean']:.1f}  "
-              f"reduction: {stats['line_reduction']}")
+              f"reduction: {stats['instruction_reduction']}")
 
 
 if __name__ == "__main__":

benchmarks/run_benchmark.py

@@ -36,6 +36,7 @@
 from scratchv.frontend.dsl_parser import DSLParser
 from scratchv.ir.builder import IRBuilder
 from scratchv.ir.types import Program
+from scratchv.backend._asm_parser import ParsedAsmLine, parse_asm
 
 
 # ---------------------------------------------------------------------------
@@ -55,6 +56,18 @@ class BenchResult:
     ir_opt_inst_count: int = 0
     codegen_time_s: float = 0.0
     asm_line_count: int = 0
+    lui_count_before: int = 0
+    candidate_pairs: int = 0
+    merged_pairs: int = 0
+    redundant_lui_removed: int = 0
+    asm_instructions_before: int = 0
+    asm_instructions_after: int = 0
+    machine_instructions_before: Optional[int] = None
+    machine_instructions_after: Optional[int] = None
+    code_size_before: Optional[int] = None
+    code_size_after: Optional[int] = None
+    const_merge_time_ms: float = 0.0
+    output_equal: Optional[bool] = None
     total_time_s: float = 0.0
     verified: bool = False
     error: Optional[str] = None
@@ -70,6 +83,18 @@ def _count_ir(program: Program) -> tuple[int, int]:
     return inst, bb
 
 
+def _count_asm_instructions(asm_text: str) -> int:
+    """Count assembly instructions, excluding labels, blanks and directives."""
+    return _count_parsed_asm_instructions(parse_asm(asm_text))
+
+
+def _count_parsed_asm_instructions(lines: list[ParsedAsmLine]) -> int:
+    return sum(
+        1 for line in lines
+        if line.opcode is not None and not line.is_directive
+    )
+
+
 def _parse_onnx(path: str) -> Program:
     parser = ONNXParser()
     return parser.parse(path)
@@ -173,6 +198,23 @@ def run_benchmark(model_name: str, model_path: str, *,
             asm_str, result.codegen_time_s = _codegen_llvm(program)
         else:
             asm_str, result.codegen_time_s = _codegen_riscv(program)
+            from scratchv.backend.const_merge import merge_constants_detailed
+            parsed_before = parse_asm(asm_str)
+            result.lui_count_before = sum(
+                line.opcode == "lui" for line in parsed_before
+            )
+            result.asm_instructions_before = _count_parsed_asm_instructions(
+                parsed_before,
+            )
+            t0 = time.perf_counter()
+            asm_after, merge_stats = merge_constants_detailed(asm_str)
+            result.const_merge_time_ms = (time.perf_counter() - t0) * 1000
+            result.candidate_pairs = merge_stats.candidate_pairs
+            result.merged_pairs = merge_stats.merged_pairs
+            result.redundant_lui_removed = merge_stats.redundant_lui_removed
+            result.asm_instructions_after = _count_parsed_asm_instructions(
+                parse_asm(asm_after),
+            )
         result.asm_line_count = len(asm_str.splitlines())
 
         # 4. Verify

benchmarks/test_benchmark.py

@@ -23,6 +23,7 @@
 
 from benchmarks.generate_models import ensure_all_models
 from benchmarks.run_benchmark import run_benchmark
+from benchmarks.bench_const_merge import _gen_synthetic_asm, bench_merge
 
 
 # ---------------------------------------------------------------------------
@@ -38,6 +39,23 @@ def benchmark_models() -> dict[str, str]:
 BACKEND_PARAMS = ["riscv"]
 
 
+@pytest.mark.parametrize(
+    "pair_density,redundant_density",
+    [(-0.1, 0.1), (0.1, -0.1), (1.1, 0.0), (0.6, 0.5)],
+)
+def test_synthetic_density_validation(pair_density, redundant_density):
+    with pytest.raises(ValueError):
+        _gen_synthetic_asm(
+            10, lui_ratio=pair_density,
+            redundant_lui_ratio=redundant_density,
+        )
+
+
+def test_synthetic_repeats_validation():
+    with pytest.raises(ValueError, match="repeats"):
+        bench_merge("  nop\n", repeats=0)
+
+
 def _model_id(name: str) -> str:
     return name
 
@@ -151,6 +169,15 @@ def test_perf_pipeline(model_name: str, benchmark_models: dict[str, str]):
 
     assert result.error is None, f"Benchmark failed: {result.error}"
     assert result.ir_inst_count > 0
+    assert isinstance(result.asm_instructions_before, int)
+    assert isinstance(result.asm_instructions_after, int)
+    reduction = (
+        result.asm_instructions_before - result.asm_instructions_after
+    )
+    tracked_changes = result.merged_pairs + result.redundant_lui_removed
+    assert (
+        reduction == tracked_changes
+    ), f"instruction reduction {reduction} != tracked changes {tracked_changes}"
 
     print(f"\n    {model_name}:")
     print(f"      parse:  {result.parse_time_s:.4f}s")