From bfa5456781a525b1a163d158b6ffc51fde558e50 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 18 Aug 2026 20:39:17 +0000 Subject: [PATCH] feat: expand evaluation cost reporting Signed-off-by: root --- src/evaluation/metrics.py | 44 ++++++++++++++++++++++++++-- src/evaluation/models.py | 2 ++ src/evaluation/report.py | 23 +++++++++++++++ src/evaluation/tests/test_metrics.py | 20 +++++++++++++ src/evaluation/tests/test_report.py | 20 +++++++++++++ 5 files changed, 106 insertions(+), 3 deletions(-) diff --git a/src/evaluation/metrics.py b/src/evaluation/metrics.py index 9450c3100..665026c25 100644 --- a/src/evaluation/metrics.py +++ b/src/evaluation/metrics.py @@ -11,11 +11,17 @@ # only for the optional ``est_cost_usd`` rollup; consumers should treat # it as an estimate, not a billing source of truth. _PRICE_PER_1M: dict[str, tuple[float, float]] = { + "claude-opus-5": (5.0, 25.0), + # Introductory public price through 2026-08-31; update after it expires. + "claude-sonnet-5": (2.0, 10.0), "claude-opus-4-5": (15.0, 75.0), "claude-opus-4-1": (15.0, 75.0), "claude-sonnet-4-6": (3.0, 15.0), "claude-haiku-4-5": (1.0, 5.0), "gpt-5": (10.0, 30.0), + "gpt-5.6-sol": (5.0, 30.0), + "gemini-3.6-flash": (1.5, 7.5), + "minimax-m3": (0.3, 1.2), "gpt-4.1": (3.0, 12.0), "gpt-4o": (2.5, 10.0), "llama-4-maverick": (0.27, 0.85), @@ -146,6 +152,16 @@ def _from_plan_execute(steps: list[Any], model: str) -> OpsMetrics: def _estimate_cost(model: str, tokens_in: int, tokens_out: int) -> float | None: + components = _estimate_cost_components(model, tokens_in, tokens_out) + if components is None: + return None + input_cost, output_cost = components + return round(input_cost + output_cost, 6) + + +def _estimate_cost_components( + model: str, tokens_in: int, tokens_out: int +) -> tuple[float, float] | None: if not model or (tokens_in == 0 and tokens_out == 0): return None key = _normalize_model(model) @@ -153,7 +169,7 @@ def _estimate_cost(model: str, tokens_in: int, tokens_out: int) -> float | None: if rate is None: return None in_rate, out_rate = rate - return round((tokens_in * in_rate + tokens_out * out_rate) / 1_000_000, 6) + return tokens_in * in_rate / 1_000_000, tokens_out * out_rate / 1_000_000 def _normalize_model(model: str) -> str: @@ -171,15 +187,37 @@ def aggregate_ops(results: list[ScenarioResult]) -> AggregateOps: return AggregateOps() durations = [r.ops.duration_ms for r in results if r.ops.duration_ms is not None] - costs = [r.ops.est_cost_usd for r in results if r.ops.est_cost_usd is not None] + input_costs: list[float] = [] + output_costs: list[float] = [] + fallback_costs: list[float] = [] + for result in results: + components = _estimate_cost_components( + result.model, result.ops.tokens_in, result.ops.tokens_out + ) + if components is not None: + input_cost, output_cost = components + input_costs.append(input_cost) + output_costs.append(output_cost) + elif result.ops.est_cost_usd is not None: + fallback_costs.append(result.ops.est_cost_usd) + + estimated_costs = input_costs + output_costs + fallback_costs return AggregateOps( tokens_in_total=sum(r.ops.tokens_in for r in results), tokens_out_total=sum(r.ops.tokens_out for r in results), + est_input_cost_usd_total=( + round(sum(input_costs), 6) if input_costs else None + ), + est_output_cost_usd_total=( + round(sum(output_costs), 6) if output_costs else None + ), duration_ms_p50=_percentile(durations, 50), duration_ms_p95=_percentile(durations, 95), tool_calls_total=sum(r.ops.tool_call_count for r in results), - est_cost_usd_total=round(sum(costs), 6) if costs else None, + est_cost_usd_total=( + round(sum(estimated_costs), 6) if estimated_costs else None + ), ) diff --git a/src/evaluation/models.py b/src/evaluation/models.py index 7042f4c9e..fcfd3356f 100644 --- a/src/evaluation/models.py +++ b/src/evaluation/models.py @@ -97,6 +97,8 @@ class ScenarioResult(BaseModel): class AggregateOps(BaseModel): tokens_in_total: int = 0 tokens_out_total: int = 0 + est_input_cost_usd_total: float | None = None + est_output_cost_usd_total: float | None = None duration_ms_p50: float | None = None duration_ms_p95: float | None = None tool_calls_total: int = 0 diff --git a/src/evaluation/report.py b/src/evaluation/report.py index 7a80884b1..0be6769d9 100644 --- a/src/evaluation/report.py +++ b/src/evaluation/report.py @@ -68,6 +68,8 @@ def _aggregate_score_summary(results: list[ScenarioResult]) -> dict[str, Any]: extra_keys_total = 0 detail_entries_total = 0 scored_results = 0 + passed = sum(1 for result in results if result.score.passed) + total = len(results) for result in results: # Top-level score field, if present @@ -98,7 +100,24 @@ def _aggregate_score_summary(results: list[ScenarioResult]) -> dict[str, Any]: if isinstance(per_key_details, list): detail_entries_total += len(per_key_details) + ops = aggregate_ops(results).model_dump() + estimated_cost = ops["est_cost_usd_total"] + return { + "total": total, + "passed": passed, + "pass_rate": round(passed / total, 4) if total else 0.0, + "ops": ops, + "est_cost_per_scenario_usd": ( + round(estimated_cost / total, 6) + if estimated_cost is not None and total + else None + ), + "est_cost_per_pass_usd": ( + round(estimated_cost / passed, 6) + if estimated_cost is not None and passed + else None + ), "scored_results": scored_results, "score_avg": _avg(score_values["score"]), "score_min": round(min(score_values["score"]), 4) if score_values["score"] else None, @@ -268,6 +287,10 @@ def _append_score_summary( "matched_keys_avg": "matched_keys_avg", "exact_value_matches_avg": "exact_value_matches_avg", } + lines.append( + f"{indent}passed: {summary.get('passed', 0)}/{summary.get('total', 0)} " + f"({summary.get('pass_rate', 0):.1%})" + ) for metric, label in metric_labels.items(): value = summary.get(metric) if value is not None: diff --git a/src/evaluation/tests/test_metrics.py b/src/evaluation/tests/test_metrics.py index d1bce4838..96807a053 100644 --- a/src/evaluation/tests/test_metrics.py +++ b/src/evaluation/tests/test_metrics.py @@ -133,6 +133,26 @@ def test_cost_only_when_some_present(self): agg = aggregate_ops(results) assert agg.est_cost_usd_total == 0.03 + def test_cost_components_use_model_token_rates(self): + results = [ + ScenarioResult( + scenario_id="1", + scenario_type="structured", + runner="stirrup-agent", + model="litellm_proxy/azure/gpt-5.6-sol", + question="q", + answer="a", + score=ScorerResult(scorer="static_json", passed=True), + ops=OpsMetrics(tokens_in=1_000_000, tokens_out=100_000), + ) + ] + + agg = aggregate_ops(results) + + assert agg.est_input_cost_usd_total == 5.0 + assert agg.est_output_cost_usd_total == 3.0 + assert agg.est_cost_usd_total == 8.0 + class TestNormalizeModel: def test_strips_provider_prefix(self): diff --git a/src/evaluation/tests/test_report.py b/src/evaluation/tests/test_report.py index 2ad58a27f..e558f1ae3 100644 --- a/src/evaluation/tests/test_report.py +++ b/src/evaluation/tests/test_report.py @@ -50,6 +50,17 @@ def test_build_report_totals_and_breakdown(): assert report.by_scenario_type["iot"].passed == 1 assert report.by_scenario_type["tsfm"].pass_rate == 1.0 assert report.ops.tokens_in_total == 38 + summary_ops = report.score_summary["plan-execute_watsonx/ibm/granite"]["ops"] + assert summary_ops == { + "tokens_in_total": 38, + "tokens_out_total": 19, + "est_input_cost_usd_total": None, + "est_output_cost_usd_total": None, + "duration_ms_p50": None, + "duration_ms_p95": None, + "tool_calls_total": 0, + "est_cost_usd_total": None, + } def test_build_report_handles_empty(): @@ -82,6 +93,11 @@ def test_write_reports_dir_writes_only_aggregate(tmp_path: Path): agg = json.loads((out_dir / "_aggregate.json").read_text()) assert agg["totals"]["scenarios"] == 2 assert set(agg["score_summary"]) == {"plan-execute_watsonx/ibm/granite"} + score_summary = agg["score_summary"]["plan-execute_watsonx/ibm/granite"] + assert score_summary["total"] == 2 + assert score_summary["passed"] == 1 + assert score_summary["pass_rate"] == 0.5 + assert score_summary["ops"]["tokens_in_total"] == 0 assert len(agg["results"]) == 2 @@ -93,6 +109,7 @@ def test_render_summary_includes_headlines(): text = render_summary(build_report(results)) assert "Pass rate" in text assert "plan-execute_watsonx/ibm/granite" in text + assert "passed: 1/2 (50.0%)" in text assert "iot" in text assert "tokens_in_total" in text @@ -145,6 +162,9 @@ def test_build_report_includes_score_summary(): assert report.score_summary is not None summary = report.score_summary["direct-llm-agent_tokenrouter/MiniMax-M3"] + assert summary["total"] == 1 + assert summary["passed"] == 0 + assert summary["pass_rate"] == 0.0 assert summary["partial_exact_match_accuracy_avg"] == 0.0 assert summary["strict_exact_match_accuracy_avg"] == 0.0 assert summary["missing_keys_total"] == 0