Expand evaluation cost reporting

src/evaluation/metrics.py

@@ -11,11 +11,17 @@
 # only for the optional ``est_cost_usd`` rollup; consumers should treat
 # it as an estimate, not a billing source of truth.
 _PRICE_PER_1M: dict[str, tuple[float, float]] = {
+    "claude-opus-5": (5.0, 25.0),
+    # Introductory public price through 2026-08-31; update after it expires.
+    "claude-sonnet-5": (2.0, 10.0),
     "claude-opus-4-5": (15.0, 75.0),
     "claude-opus-4-1": (15.0, 75.0),
     "claude-sonnet-4-6": (3.0, 15.0),
     "claude-haiku-4-5": (1.0, 5.0),
     "gpt-5": (10.0, 30.0),
+    "gpt-5.6-sol": (5.0, 30.0),
+    "gemini-3.6-flash": (1.5, 7.5),
+    "minimax-m3": (0.3, 1.2),
     "gpt-4.1": (3.0, 12.0),
     "gpt-4o": (2.5, 10.0),
     "llama-4-maverick": (0.27, 0.85),
@@ -146,14 +152,24 @@ def _from_plan_execute(steps: list[Any], model: str) -> OpsMetrics:
 
 
 def _estimate_cost(model: str, tokens_in: int, tokens_out: int) -> float | None:
+    components = _estimate_cost_components(model, tokens_in, tokens_out)
+    if components is None:
+        return None
+    input_cost, output_cost = components
+    return round(input_cost + output_cost, 6)
+
+
+def _estimate_cost_components(
+    model: str, tokens_in: int, tokens_out: int
+) -> tuple[float, float] | None:
     if not model or (tokens_in == 0 and tokens_out == 0):
         return None
     key = _normalize_model(model)
     rate = _PRICE_PER_1M.get(key)
     if rate is None:
         return None
     in_rate, out_rate = rate
-    return round((tokens_in * in_rate + tokens_out * out_rate) / 1_000_000, 6)
+    return tokens_in * in_rate / 1_000_000, tokens_out * out_rate / 1_000_000
 
 
 def _normalize_model(model: str) -> str:
@@ -171,15 +187,37 @@ def aggregate_ops(results: list[ScenarioResult]) -> AggregateOps:
         return AggregateOps()
 
     durations = [r.ops.duration_ms for r in results if r.ops.duration_ms is not None]
-    costs = [r.ops.est_cost_usd for r in results if r.ops.est_cost_usd is not None]
+    input_costs: list[float] = []
+    output_costs: list[float] = []
+    fallback_costs: list[float] = []
+    for result in results:
+        components = _estimate_cost_components(
+            result.model, result.ops.tokens_in, result.ops.tokens_out
+        )
+        if components is not None:
+            input_cost, output_cost = components
+            input_costs.append(input_cost)
+            output_costs.append(output_cost)
+        elif result.ops.est_cost_usd is not None:
+            fallback_costs.append(result.ops.est_cost_usd)
+
+    estimated_costs = input_costs + output_costs + fallback_costs
 
     return AggregateOps(
         tokens_in_total=sum(r.ops.tokens_in for r in results),
         tokens_out_total=sum(r.ops.tokens_out for r in results),
+        est_input_cost_usd_total=(
+            round(sum(input_costs), 6) if input_costs else None
+        ),
+        est_output_cost_usd_total=(
+            round(sum(output_costs), 6) if output_costs else None
+        ),
         duration_ms_p50=_percentile(durations, 50),
         duration_ms_p95=_percentile(durations, 95),
         tool_calls_total=sum(r.ops.tool_call_count for r in results),
-        est_cost_usd_total=round(sum(costs), 6) if costs else None,
+        est_cost_usd_total=(
+            round(sum(estimated_costs), 6) if estimated_costs else None
+        ),
     )
 
 

src/evaluation/models.py

@@ -97,6 +97,8 @@ class ScenarioResult(BaseModel):
 class AggregateOps(BaseModel):
     tokens_in_total: int = 0
     tokens_out_total: int = 0
+    est_input_cost_usd_total: float | None = None
+    est_output_cost_usd_total: float | None = None
     duration_ms_p50: float | None = None
     duration_ms_p95: float | None = None
     tool_calls_total: int = 0

src/evaluation/report.py

@@ -68,6 +68,8 @@ def _aggregate_score_summary(results: list[ScenarioResult]) -> dict[str, Any]:
     extra_keys_total = 0
     detail_entries_total = 0
     scored_results = 0
+    passed = sum(1 for result in results if result.score.passed)
+    total = len(results)
 
     for result in results:
         # Top-level score field, if present
@@ -98,7 +100,24 @@ def _aggregate_score_summary(results: list[ScenarioResult]) -> dict[str, Any]:
         if isinstance(per_key_details, list):
             detail_entries_total += len(per_key_details)
 
+    ops = aggregate_ops(results).model_dump()
+    estimated_cost = ops["est_cost_usd_total"]
+
     return {
+        "total": total,
+        "passed": passed,
+        "pass_rate": round(passed / total, 4) if total else 0.0,
+        "ops": ops,
+        "est_cost_per_scenario_usd": (
+            round(estimated_cost / total, 6)
+            if estimated_cost is not None and total
+            else None
+        ),
+        "est_cost_per_pass_usd": (
+            round(estimated_cost / passed, 6)
+            if estimated_cost is not None and passed
+            else None
+        ),
         "scored_results": scored_results,
         "score_avg": _avg(score_values["score"]),
         "score_min": round(min(score_values["score"]), 4) if score_values["score"] else None,
@@ -268,6 +287,10 @@ def _append_score_summary(
         "matched_keys_avg": "matched_keys_avg",
         "exact_value_matches_avg": "exact_value_matches_avg",
     }
+    lines.append(
+        f"{indent}passed: {summary.get('passed', 0)}/{summary.get('total', 0)} "
+        f"({summary.get('pass_rate', 0):.1%})"
+    )
     for metric, label in metric_labels.items():
         value = summary.get(metric)
         if value is not None:

src/evaluation/tests/test_metrics.py

@@ -133,6 +133,26 @@ def test_cost_only_when_some_present(self):
         agg = aggregate_ops(results)
         assert agg.est_cost_usd_total == 0.03
 
+    def test_cost_components_use_model_token_rates(self):
+        results = [
+            ScenarioResult(
+                scenario_id="1",
+                scenario_type="structured",
+                runner="stirrup-agent",
+                model="litellm_proxy/azure/gpt-5.6-sol",
+                question="q",
+                answer="a",
+                score=ScorerResult(scorer="static_json", passed=True),
+                ops=OpsMetrics(tokens_in=1_000_000, tokens_out=100_000),
+            )
+        ]
+
+        agg = aggregate_ops(results)
+
+        assert agg.est_input_cost_usd_total == 5.0
+        assert agg.est_output_cost_usd_total == 3.0
+        assert agg.est_cost_usd_total == 8.0
+
 
 class TestNormalizeModel:
     def test_strips_provider_prefix(self):

src/evaluation/tests/test_report.py

@@ -50,6 +50,17 @@ def test_build_report_totals_and_breakdown():
     assert report.by_scenario_type["iot"].passed == 1
     assert report.by_scenario_type["tsfm"].pass_rate == 1.0
     assert report.ops.tokens_in_total == 38
+    summary_ops = report.score_summary["plan-execute_watsonx/ibm/granite"]["ops"]
+    assert summary_ops == {
+        "tokens_in_total": 38,
+        "tokens_out_total": 19,
+        "est_input_cost_usd_total": None,
+        "est_output_cost_usd_total": None,
+        "duration_ms_p50": None,
+        "duration_ms_p95": None,
+        "tool_calls_total": 0,
+        "est_cost_usd_total": None,
+    }
 
 
 def test_build_report_handles_empty():
@@ -82,6 +93,11 @@ def test_write_reports_dir_writes_only_aggregate(tmp_path: Path):
     agg = json.loads((out_dir / "_aggregate.json").read_text())
     assert agg["totals"]["scenarios"] == 2
     assert set(agg["score_summary"]) == {"plan-execute_watsonx/ibm/granite"}
+    score_summary = agg["score_summary"]["plan-execute_watsonx/ibm/granite"]
+    assert score_summary["total"] == 2
+    assert score_summary["passed"] == 1
+    assert score_summary["pass_rate"] == 0.5
+    assert score_summary["ops"]["tokens_in_total"] == 0
     assert len(agg["results"]) == 2
 
 
@@ -93,6 +109,7 @@ def test_render_summary_includes_headlines():
     text = render_summary(build_report(results))
     assert "Pass rate" in text
     assert "plan-execute_watsonx/ibm/granite" in text
+    assert "passed: 1/2 (50.0%)" in text
     assert "iot" in text
     assert "tokens_in_total" in text
 
@@ -145,6 +162,9 @@ def test_build_report_includes_score_summary():
 
     assert report.score_summary is not None
     summary = report.score_summary["direct-llm-agent_tokenrouter/MiniMax-M3"]
+    assert summary["total"] == 1
+    assert summary["passed"] == 0
+    assert summary["pass_rate"] == 0.0
     assert summary["partial_exact_match_accuracy_avg"] == 0.0
     assert summary["strict_exact_match_accuracy_avg"] == 0.0
     assert summary["missing_keys_total"] == 0