diff --git a/cyberai/cli/bench.py b/cyberai/cli/bench.py index 06ed75e..f0c0365 100644 --- a/cyberai/cli/bench.py +++ b/cyberai/cli/bench.py @@ -166,10 +166,12 @@ def _model_participation(report) -> tuple[int | None, str | None]: a value. """ results = list(report.results) - if not results: - return None, None proven = [r for r in results if r.details.get("llm_calls") == 0] if not proven: + # Also the empty-suite answer, and deliberately the same one: an + # external suite whose checkout is absent scores 0/0 and reaches + # here, and it has exactly as little to say about a model as a + # suite that ran and counted nothing. return None, None if len(proven) == len(results): reasons = {str(r.details.get("llm_zero_reason")) for r in proven} diff --git a/tests/unit/test_bench_cli.py b/tests/unit/test_bench_cli.py index cf2df21..09ff399 100644 --- a/tests/unit/test_bench_cli.py +++ b/tests/unit/test_bench_cli.py @@ -490,3 +490,21 @@ def test_the_placeholder_engine_publishes_no_model_row(tmp_path): text = out.read_text() assert "| llm calls |" not in text assert "| llm zero reason |" not in text + + +def test_a_suite_with_no_tasks_writes_a_card_and_claims_nothing(monkeypatch, tmp_path): + """An external suite whose checkout is absent scores 0/0 and still + writes a card. Nothing ran, so nothing can be said about a model -- + and the rollup must answer that rather than divide by an empty run.""" + monkeypatch.setenv("CVEBENCH_DIR", str(tmp_path / "nowhere")) + out = tmp_path / "sc.md" + + result = CliRunner().invoke( + bench, ["run", "--suite", "cve-bench", "--engine", "agent", "--scorecard", str(out)] + ) + + assert result.exit_code == 0 + assert "pass@1: 0/0" in result.output + text = out.read_text() + assert "| llm calls |" not in text + assert "| llm zero reason |" not in text