From 221553cc7ee2c87da9739006a7778e730fa11709 Mon Sep 17 00:00:00 2001
From: Evgeny Kiriyak <224408464+evkir@users.noreply.github.com>
Date: Wed, 26 Aug 2026 07:06:26 +0200
Subject: [PATCH] fix(cli): one guard for the two ways a run can say nothing
 about a model

Codecov found a line the patch never executed, and it was a duplicate:
an empty result list falls through the emptiness check and the
no-proof check to the same answer, so the first guard could not be
reached except through the second.

Removed, and the reachable path it stood for is now tested instead. An
external suite whose checkout is absent scores 0/0 and still writes a
card -- nothing ran, so the rollup has to answer that rather than
divide by an empty run. Dropping the surviving guard raises a KeyError
on that path, which is what the test catches.
---
 cyberai/cli/bench.py         |  6 ++++--
 tests/unit/test_bench_cli.py | 18 ++++++++++++++++++
 2 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/cyberai/cli/bench.py b/cyberai/cli/bench.py
index 06ed75e..f0c0365 100644
--- a/cyberai/cli/bench.py
+++ b/cyberai/cli/bench.py
@@ -166,10 +166,12 @@ def _model_participation(report) -> tuple[int | None, str | None]:
     a value.
     """
     results = list(report.results)
-    if not results:
-        return None, None
     proven = [r for r in results if r.details.get("llm_calls") == 0]
     if not proven:
+        # Also the empty-suite answer, and deliberately the same one: an
+        # external suite whose checkout is absent scores 0/0 and reaches
+        # here, and it has exactly as little to say about a model as a
+        # suite that ran and counted nothing.
         return None, None
     if len(proven) == len(results):
         reasons = {str(r.details.get("llm_zero_reason")) for r in proven}
diff --git a/tests/unit/test_bench_cli.py b/tests/unit/test_bench_cli.py
index cf2df21..09ff399 100644
--- a/tests/unit/test_bench_cli.py
+++ b/tests/unit/test_bench_cli.py
@@ -490,3 +490,21 @@ def test_the_placeholder_engine_publishes_no_model_row(tmp_path):
     text = out.read_text()
     assert "| llm calls |" not in text
     assert "| llm zero reason |" not in text
+
+
+def test_a_suite_with_no_tasks_writes_a_card_and_claims_nothing(monkeypatch, tmp_path):
+    """An external suite whose checkout is absent scores 0/0 and still
+    writes a card. Nothing ran, so nothing can be said about a model --
+    and the rollup must answer that rather than divide by an empty run."""
+    monkeypatch.setenv("CVEBENCH_DIR", str(tmp_path / "nowhere"))
+    out = tmp_path / "sc.md"
+
+    result = CliRunner().invoke(
+        bench, ["run", "--suite", "cve-bench", "--engine", "agent", "--scorecard", str(out)]
+    )
+
+    assert result.exit_code == 0
+    assert "pass@1: 0/0" in result.output
+    text = out.read_text()
+    assert "| llm calls |" not in text
+    assert "| llm zero reason |" not in text