feat(evaluation): add reproducible accessible leaderboard charts

docs/2026-08-09_leaderboard-charts.md

@@ -0,0 +1,80 @@
+# Leaderboard chart generation
+
+AssetOpsBench can generate publication-ready leaderboard charts directly from
+the structured evaluation report. Chart values are derived from individual
+`ScenarioResult` records in `EvalReport.results`; they are not hard-coded or
+scraped from README images or papers.
+
+## Install
+
+Matplotlib is optional so evaluation users who do not create charts do not
+need to install it:
+
+```bash
+uv sync --dev --group visualization
+```
+
+You can also select the group for one command with `uv run --group
+visualization ...`.
+
+## Generate charts
+
+```bash
+uv run --group visualization evaluate \
+  --trajectories traces/trajectories \
+  --scenarios groundtruth/*.json \
+  --scorer-default llm_judge \
+  --judge-model litellm_proxy/azure/gpt-5.4 \
+  --reports-dir reports \
+  --charts
+```
+
+Chart rendering runs after evaluation from the completed report and does not
+make an additional LLM call. Output is deterministic by runner name:
+
+```text
+reports/
+├── _aggregate.json
+└── charts/
+    ├── leaderboard-<runner>.svg
+    └── leaderboard-<runner>.png
+```
+
+SVG is the preferred README and publication format. PNG is provided for quick
+preview. Runner names that are not filesystem-safe receive a stable sanitized
+name and hash suffix.
+
+## Aggregation rules
+
+The compact leaderboard includes task completion, data retrieval accuracy,
+and generalized result verification. For each runner, model, and criterion,
+the success percentage is the number of `true` outcomes divided by the number
+of results containing an actual Boolean value for that criterion.
+
+- Only results produced by `llm_judge` are included.
+- Missing, null, numeric, and string values are excluded from the denominator.
+- Models and runners are aggregated independently.
+- The optional raw `hallucinations` criterion is inverted and presented as
+  “Hallucination-free,” so its positive meaning is explicit.
+
+The historical README leaderboard predates this report pipeline and has no
+checked-in `EvalReport` that can reproduce its published values. It remains
+unchanged; future leaderboard figures can be generated reproducibly with this
+command.
+
+## Accessibility measures
+
+The renderer follows Carbon's official
+[categorical palette](https://carbondesignsystem.com/data-visualization/color-palettes/)
+in its documented order because models are discrete categories. Each model
+also receives a hatch pattern and dark outline, and its style is kept
+consistent across every runner in the same report. Charts use a zero baseline,
+direct percentage labels, a percent scale, readable legends, and `N/A` gaps
+instead of treating missing criteria as failures. SVG text remains selectable,
+and each SVG includes image semantics plus a description containing the plotted
+models, percentages, and Boolean counts. PNG metadata carries the same data.
+
+These measures improve color-vision, grayscale, and reduced-size readability;
+they are not a claim of complete accessibility conformance. When publishing a
+chart, provide meaningful surrounding text or alternative text and retain the
+canonical JSON report for readers who need the underlying values.

docs/evaluation.md

@@ -193,9 +193,15 @@ uv run evaluate \
   [--reports-dir DIR]            # default: reports/
   [--scorer-default NAME]        # default: llm_judge
   [--judge-model MODEL_ID]       # required when llm_judge runs
+  [--charts]                     # optional SVG/PNG leaderboards
   [-v]
 ```
 
+`--charts` derives one leaderboard per runner from the completed
+`EvalReport` and writes it under `<reports-dir>/charts`. See
+[Leaderboard chart generation](2026-08-09_leaderboard-charts.md) for the
+optional dependency, aggregation rules, outputs, and accessibility measures.
+
 
 ## Available scorers in this branch
 

pyproject.toml

@@ -78,6 +78,10 @@ otel = [
     "opentelemetry-exporter-otlp-proto-http>=1.27.0",
     "opentelemetry-instrumentation-httpx>=0.48b0",
 ]
+# Optional static SVG/PNG leaderboard generation for evaluation reports.
+visualization = [
+    "matplotlib>=3.10",
+]
 
 [tool.uv]
 package = true

src/evaluation/cli.py

@@ -58,8 +58,7 @@ def _build_parser() -> argparse.ArgumentParser:
         "--scorer-default",
         dest="scorer_default",
         default="llm_judge",
-        help="Scorer name when scenario.scoring_method is unset. "
-        "Default: llm_judge.",
+        help="Scorer name when scenario.scoring_method is unset. Default: llm_judge.",
     )
     p.add_argument(
         "--judge-model",
@@ -68,6 +67,14 @@ def _build_parser() -> argparse.ArgumentParser:
         "litellm_proxy/anthropic/claude-opus-4-5). "
         "Required when any scenario routes to llm_judge.",
     )
+    p.add_argument(
+        "--charts",
+        action="store_true",
+        help=(
+            "Generate report-derived SVG and PNG leaderboards under "
+            "<reports-dir>/charts. Requires the visualization dependency group."
+        ),
+    )
     p.add_argument(
         "-v",
         "--verbose",
@@ -114,6 +121,17 @@ def main(argv: list[str] | None = None) -> int:
         format="%(asctime)s %(levelname)s %(name)s: %(message)s",
     )
 
+    if args.charts:
+        from .visualization import (
+            VisualizationDependencyError,
+            require_visualization_dependency,
+        )
+
+        try:
+            require_visualization_dependency()
+        except VisualizationDependencyError as exc:
+            parser.error(str(exc))
+
     try:
         scenario_ids = _resolve_scenario_ids(args.scenario_ids)
     except (FileNotFoundError, ValueError) as exc:
@@ -131,8 +149,21 @@ def main(argv: list[str] | None = None) -> int:
     )
 
     out_dir = write_reports_dir(report, args.reports_dir)
+    chart_paths: tuple[Path, ...] = ()
+    if args.charts:
+        from .visualization import render_leaderboards
+
+        chart_paths = render_leaderboards(report, out_dir / "charts")
     print(render_summary(report))
     print(f"\nAggregate report written: {out_dir}/_aggregate.json")
+    if args.charts:
+        if chart_paths:
+            print(f"Leaderboard charts written: {out_dir}/charts")
+        else:
+            print(
+                "No leaderboard charts generated: the report has no applicable "
+                "Boolean llm_judge criterion results."
+            )
     return 0
 
 

src/evaluation/tests/test_cli.py

@@ -2,7 +2,15 @@
 
 from __future__ import annotations
 
+from pathlib import Path
+
+import pytest
+
+from evaluation import cli
 from evaluation.cli import _build_parser, _resolve_scenario_ids
+from evaluation.models import EvalReport
+from evaluation.report import build_report
+from evaluation.visualization import VisualizationDependencyError
 
 
 def test_cli_accepts_optional_scenario_selector() -> None:
@@ -20,6 +28,103 @@ def test_cli_accepts_optional_scenario_selector() -> None:
     assert args.scenario_ids == "fcc+fmsr_all"
 
 
+def test_cli_charts_are_opt_in() -> None:
+    base_args = [
+        "--trajectories",
+        "trajectories",
+        "--scenarios",
+        "scenarios",
+    ]
+
+    assert _build_parser().parse_args(base_args).charts is False
+    assert _build_parser().parse_args([*base_args, "--charts"]).charts is True
+
+
+@pytest.mark.parametrize("charts", [False, True])
+def test_cli_chart_generation_is_opt_in(
+    monkeypatch: pytest.MonkeyPatch, tmp_path: Path, charts: bool
+) -> None:
+    calls: list[Path] = []
+
+    class StubEvaluator:
+        def __init__(self, **kwargs: object) -> None:
+            pass
+
+        def evaluate(self, **kwargs: object) -> EvalReport:
+            return build_report([])
+
+    def write_reports_dir(report: EvalReport, path: Path) -> Path:
+        return path
+
+    def record_render(report: EvalReport, path: Path) -> tuple[Path, ...]:
+        calls.append(path)
+        return ()
+
+    def accept_scorer(name: str) -> None:
+        pass
+
+    def dependency_available() -> None:
+        pass
+
+    monkeypatch.setattr(cli, "Evaluator", StubEvaluator)
+    monkeypatch.setattr(cli, "_validate_scorer_default", accept_scorer)
+    monkeypatch.setattr(cli, "write_reports_dir", write_reports_dir)
+    monkeypatch.setattr(
+        "evaluation.visualization.require_visualization_dependency",
+        dependency_available,
+    )
+    monkeypatch.setattr(
+        "evaluation.visualization.render_leaderboards",
+        record_render,
+    )
+
+    args = [
+        "--trajectories",
+        "trajectories",
+        "--scenarios",
+        "scenarios",
+        "--reports-dir",
+        str(tmp_path),
+    ]
+    if charts:
+        args.append("--charts")
+
+    result = cli.main(args)
+
+    assert result == 0
+    assert calls == ([tmp_path / "charts"] if charts else [])
+
+
+def test_cli_fails_before_evaluation_when_chart_dependency_is_missing(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    def dependency_error() -> None:
+        raise VisualizationDependencyError("install charts")
+
+    monkeypatch.setattr(
+        "evaluation.visualization.require_visualization_dependency",
+        dependency_error,
+    )
+    monkeypatch.setattr(
+        cli,
+        "Evaluator",
+        lambda **kwargs: pytest.fail("evaluation should not run"),
+    )
+
+    with pytest.raises(SystemExit) as exc_info:
+        cli.main(
+            [
+                "--trajectories",
+                "trajectories",
+                "--scenarios",
+                "scenarios",
+                "--charts",
+            ]
+        )
+
+    assert exc_info.value.code == 2
+
+
 def test_resolve_scenario_ids_loads_all_yaml_categories() -> None:
     selected = _resolve_scenario_ids("fcc+fmsr_all")