From 496e814c64e0faf71ab899efaa98bb2531a1dca1 Mon Sep 17 00:00:00 2001
From: Evgeny Kiriyak <224408464+evkir@users.noreply.github.com>
Date: Tue, 25 Aug 2026 21:46:14 +0200
Subject: [PATCH 1/7] fix(bench): the run manifest stops naming a model nobody
 used

RunConfig defaulted model and provider to "unspecified" and temperature to
0.0. Both defaults answer a question the run never asked. A reader diffing
two manifests cannot tell "unspecified" from a provider actually named that,
and 0.0 claims deterministic sampling for a run that sampled nothing -- the
placeholder engine reaches no model at all, and neither does the probe
engine, which imports docker, the evaluator, the runner and the targets and
holds the word "model" only in its docstring.

Unmeasured knobs are None. seed keeps a concrete default because
set_global_seed pins one before the adapter loads, so it is always a fact.

The new CLI test goes through `bench run --manifest` rather than
constructing RunConfig by hand: the defaults matter because they reach a
published artefact, and a test that built the object itself would pass while
the CLI overrode them.

Old baselines still load. load_baseline expands whatever config a file
carries straight into the dataclass, so a field that stopped accepting the
1.5.0 shape would not raise -- it would return None, the gate would read
that as "no baseline", and a run that regressed to zero would pass green.
A legacy manifest is now a fixture that asserts the opposite.

Mutation: model -> "unspecified" and temperature -> 0.0 both killed by the
CLI test; dropping the legacy config expansion killed by the gate test.
---
 cyberai/bench/run_manifest.py      | 25 +++++++++++++------
 tests/unit/test_bench_cli.py       | 21 ++++++++++++++++
 tests/unit/test_regression_gate.py | 39 ++++++++++++++++++++++++++++++
 3 files changed, 78 insertions(+), 7 deletions(-)

diff --git a/cyberai/bench/run_manifest.py b/cyberai/bench/run_manifest.py
index a35c1f33..3935a228 100644
--- a/cyberai/bench/run_manifest.py
+++ b/cyberai/bench/run_manifest.py
@@ -7,7 +7,10 @@
   - a content hash over the suite's tasks (id/name/criteria) — proves the suite
     wasn't quietly swapped to an easier one between runs,
   - the run config (model, provider, temperature, seed) — the knobs that affect
-    outcome,
+    outcome. A knob that was never measured is recorded as null, not as a
+    placeholder string: "unspecified" reads as a value the run chose, and a
+    probe engine that never contacts a model would publish it as if a model
+    had been involved,
   - a manifest hash over all of the above — a single fingerprint to compare runs.
 
 `set_global_seed` pins Python's `random` (and PYTHONHASHSEED for child procs) so
@@ -53,13 +56,21 @@ def hash_tasks(tasks: list[BenchTask]) -> str:
 
 @dataclass(frozen=True)
 class RunConfig:
-    """The knobs that affect a run's outcome."""
-
-    model: str = "unspecified"
-    provider: str = "unspecified"
-    temperature: float = 0.0
+    """The knobs that affect a run's outcome.
+
+    Everything a caller may leave unmeasured defaults to None, so the manifest
+    distinguishes "this run did not involve a model" from "this run used a
+    model named unspecified". temperature is not exempt: 0.0 is a real setting
+    a caller can choose, and a default of 0.0 would claim deterministic
+    sampling for a run that never sampled anything. seed keeps a concrete
+    default because set_global_seed always pins one.
+    """
+
+    model: str | None = None
+    provider: str | None = None
+    temperature: float | None = None
     seed: int = DEFAULT_SEED
-    max_iterations: int = 0
+    max_iterations: int | None = None
     extra: dict[str, Any] = field(default_factory=dict)
 
 
diff --git a/tests/unit/test_bench_cli.py b/tests/unit/test_bench_cli.py
index 65615da8..2ed19a85 100644
--- a/tests/unit/test_bench_cli.py
+++ b/tests/unit/test_bench_cli.py
@@ -62,6 +62,27 @@ def test_bench_run_writes_a_manifest(tmp_path):
     assert data["manifest_hash"]
 
 
+def test_a_run_without_a_model_publishes_no_model(tmp_path):
+    """An engine that never contacts a provider must not name one.
+
+    The placeholder engine reaches no model at all, so every knob describing
+    one is absent rather than defaulted. A placeholder string in these fields
+    reads to anyone diffing two manifests as a value the run selected, and a
+    temperature of 0.0 claims deterministic sampling for a run that sampled
+    nothing. The seed is the exception on purpose: set_global_seed pins one
+    before the adapter loads, so it is always a measured fact.
+    """
+    out = tmp_path / "run.json"
+    result = CliRunner().invoke(bench, ["run", "--manifest", str(out)])
+    assert result.exit_code == 0
+    cfg = json.loads(out.read_text())["config"]
+    assert cfg["model"] is None
+    assert cfg["provider"] is None
+    assert cfg["temperature"] is None
+    assert cfg["max_iterations"] is None
+    assert cfg["seed"] == DEFAULT_SEED
+
+
 def test_a_filtered_run_does_not_fingerprint_as_the_whole_suite(tmp_path):
     """The suite hash describes what ran, or the regression gate would compare
     a one-task run against a three-task baseline and call it a pass."""
diff --git a/tests/unit/test_regression_gate.py b/tests/unit/test_regression_gate.py
index b3b758c9..8501c2f4 100644
--- a/tests/unit/test_regression_gate.py
+++ b/tests/unit/test_regression_gate.py
@@ -2,6 +2,8 @@
 
 from __future__ import annotations
 
+import json
+
 from cyberai.bench.regression_gate import (
     check_regression,
     load_baseline,
@@ -74,3 +76,40 @@ def test_load_baseline_roundtrip(tmp_path):
     assert loaded is not None
     assert loaded.solved == 6
     assert loaded.suite_hash == "AAA"
+
+
+def test_a_baseline_written_by_an_older_release_still_loads(tmp_path):
+    """Manifests on disk outlive the code that wrote them.
+
+    Releases up to 1.5.0 stamped placeholder strings and zeroes into the run
+    config. Those files are the baselines a regression gate compares against,
+    and load_baseline expands whatever config it finds straight into the
+    dataclass. A field that stopped accepting the old shape would not raise
+    here -- it would return None, the gate would read that as "no baseline",
+    and a run that regressed to zero would pass green.
+    """
+    legacy = {
+        "suite": "local",
+        "engine_version": "1.5.0",
+        "config": {
+            "model": "unspecified",
+            "provider": "unspecified",
+            "temperature": 0.0,
+            "seed": 1337,
+            "max_iterations": 0,
+            "extra": {"engine": "real"},
+        },
+        "suite_hash": "AAA",
+        "solved": 4,
+        "total": 4,
+        "timestamp": "2026-08-17T19:18:36Z",
+        "manifest_hash": "old",
+    }
+    p = tmp_path / "legacy.json"
+    p.write_text(json.dumps(legacy))
+
+    loaded = load_baseline(p)
+    assert loaded is not None, "an older baseline must not degrade to 'no baseline'"
+    assert loaded.config.model == "unspecified"
+    assert loaded.config.temperature == 0.0
+    assert check_regression(_manifest(0), loaded).passed is False

From 9029fb715982a84ba40fc1e0bd61ab8f2a6c84dc Mon Sep 17 00:00:00 2001
From: Evgeny Kiriyak <224408464+evkir@users.noreply.github.com>
Date: Tue, 25 Aug 2026 22:27:17 +0200
Subject: [PATCH 2/7] fix(core): a path that takes no model is not blamed on a
 missing key

llm_zero_reason answered no_api_key_for_openai for the bench agent path,
measured on a clean environment and on this machine alike. The default
provider is a cloud one and the default key is read from an environment
variable that is not set, so the credential cause fires on every run that
never had a client -- and points the reader at a knob that changes nothing.

The agent engine constructs ReconAgent(cfg, session) and
ExploitAgent(cfg, session) with two positional arguments, so llm stays
None on both. Measured: recon.llm is None, exploit.llm is None. No key
would change that; the path does not ask for one.

engine_uses_a_model is answered by the code path rather than the config
and is checked above the credential causes, with a default that leaves
the pipeline's answer exactly where it was. Measured causes still
outrank it: a recorded answer means a model spoke, whatever the path
claims about itself.

Three assertions, three mutants killed. Moving the new branch below the
key check kills only the first of them, which is the point: the defect
being fixed is the order, not the absence of a cause.
---
 cyberai/core/llm_usage.py          | 11 +++++++++++
 tests/unit/test_llm_zero_reason.py | 29 +++++++++++++++++++++++++++++
 2 files changed, 40 insertions(+)

diff --git a/cyberai/core/llm_usage.py b/cyberai/core/llm_usage.py
index 5a6c5cf5..2482a1bc 100644
--- a/cyberai/core/llm_usage.py
+++ b/cyberai/core/llm_usage.py
@@ -25,6 +25,7 @@ def llm_zero_reason(
     *,
     client_built: bool,
     dry_run: bool = False,
+    engine_uses_a_model: bool = True,
 ) -> Optional[str]:
     """Why no LLM call happened, or None when at least one did.
 
@@ -36,6 +37,14 @@ def llm_zero_reason(
     same zero as a run that never asked. `attempts` separates them: a non-zero
     attempt count with no call is a refusal, and it outranks the other causes
     because it is the one thing measured directly.
+
+    `engine_uses_a_model` is answered by the code path, not by the config, and
+    it is checked before the credential causes. A path that constructs its
+    agents without a client cannot be fixed by a key: the default provider is
+    a cloud one and the default key is absent, so the missing-key cause fires
+    on every such run and points the reader at a knob that changes nothing.
+    Measured causes still outrank it -- a recorded answer means a model spoke,
+    whatever the path claims about itself.
     """
     if tracker.call_count:
         return None
@@ -43,6 +52,8 @@ def llm_zero_reason(
         return "provider_refused"
     if dry_run:
         return "dry_run"
+    if not engine_uses_a_model:
+        return "engine_uses_no_model"
     provider = llm_config.provider
     if provider in ("openai", "anthropic") and not llm_config.api_key:
         return f"no_api_key_for_{provider}"
diff --git a/tests/unit/test_llm_zero_reason.py b/tests/unit/test_llm_zero_reason.py
index 7608099f..14a11d14 100644
--- a/tests/unit/test_llm_zero_reason.py
+++ b/tests/unit/test_llm_zero_reason.py
@@ -6,6 +6,8 @@
 """
 
 from cyberai.core.config import CyberAIConfig
+from cyberai.core.cost_tracker import CostTracker
+from cyberai.core.llm_usage import llm_zero_reason
 from cyberai.core.orchestrator import Orchestrator
 
 
@@ -54,3 +56,30 @@ def test_reason_lands_in_the_session_export():
     orch = Orchestrator(_config("openai"), dry_run=True)
     session = orch.run("example.com")
     assert session.kb.get("llm.usage")["zero_reason"] == "dry_run"
+
+
+def test_a_path_that_uses_no_model_is_not_blamed_on_a_missing_key():
+    """The bench agent path hands its agents no client at all, so a provider
+    key would change nothing. The default provider is a cloud one and the
+    default key is absent, so the missing-key cause answered every such run
+    and sent the reader after a knob that was never the reason."""
+    cfg = _config("openai")
+    reason = llm_zero_reason(cfg.llm, CostTracker(), client_built=False, engine_uses_a_model=False)
+    assert reason == "engine_uses_no_model"
+
+
+def test_the_default_leaves_the_credential_answer_where_it_was():
+    """The pipeline does use a model, and there the missing key is the cause."""
+    cfg = _config("openai")
+    reason = llm_zero_reason(cfg.llm, CostTracker(), client_built=False)
+    assert reason == "no_api_key_for_openai"
+
+
+def test_a_recorded_answer_outranks_a_path_that_takes_no_model():
+    """A path describes intent; a recorded call is a measurement."""
+    tracker = CostTracker()
+    tracker.add("exploit", "qwen", input_tokens=10, output_tokens=5)
+    reason = llm_zero_reason(
+        _config("ollama").llm, tracker, client_built=False, engine_uses_a_model=False
+    )
+    assert reason is None

From a378b976205b40aa8975be18e5bc7dd6cf67a4cc Mon Sep 17 00:00:00 2001
From: Evgeny Kiriyak <224408464+evkir@users.noreply.github.com>
Date: Wed, 26 Aug 2026 06:25:43 +0200
Subject: [PATCH 3/7] feat(bench): the agent engine records whether a model
 took part

The scorecard published provider and model out of the config while the
agent path reached neither. Both agents are constructed with two
positional arguments, so the client parameter keeps its None default:
measured, recon.llm is None and exploit.llm is None, and a run against
the local suite confirms targets with no call made.

The outcome now carries that fact. llm_calls is a proven zero only while
both agents ran without a client -- read off the objects that ran, not
off the config, because a config naming gpt-4o describes a model this
path cannot reach. Hand either agent a client and there is no tracker
here to count its calls with, so both fields go back to None: not
measured is the only honest answer until one exists. A default of zero
would have claimed a measurement nobody took.

The reason comes from core/llm_usage rather than a second vocabulary, so
the scorecard and the report name the same cause with the same words.

Both runners publish it: cve-bench routes through the same attacker, and
one measurement reaching only one of them is how a single number turns
into two.

Four test doubles for the agents carried no llm attribute at all, which
the measurement surfaced immediately -- a double that does not implement
what BaseAgent assigns is standing in for nothing. They were fixed rather
than the reader defended with getattr: an object whose attribute cannot
be read must not quietly score as a model that never ran.

Five assertions, five mutants killed, one victim each.
---
 cyberai/bench/agent_engine.py         | 36 ++++++++++++-
 cyberai/bench/cve_bench_runner.py     |  2 +
 tests/unit/test_bench_agent_engine.py | 75 ++++++++++++++++++++++++---
 tests/unit/test_cve_bench_runner.py   | 19 +++++++
 4 files changed, 124 insertions(+), 8 deletions(-)

diff --git a/cyberai/bench/agent_engine.py b/cyberai/bench/agent_engine.py
index b8e4d38d..f2d5100f 100644
--- a/cyberai/bench/agent_engine.py
+++ b/cyberai/bench/agent_engine.py
@@ -42,6 +42,8 @@
 from cyberai.bench.runner import BenchResult, BenchTask
 from cyberai.bench.targets import LocalSuiteAdapter, VulnTarget
 from cyberai.core.config import CyberAIConfig
+from cyberai.core.cost_tracker import CostTracker
+from cyberai.core.llm_usage import llm_zero_reason
 from cyberai.core.scan_session import ScanSession
 
 logger = logging.getLogger(__name__)
@@ -56,6 +58,12 @@ class AttackOutcome:
     requests_sent: int = 0
     findings: list[dict[str, Any]] = field(default_factory=list)
     oob_confirmed: int = 0
+    # What the model did, or None for not measured. Zero and absent are
+    # different facts: a run that proves no model was reached can say so,
+    # while a run with no way to count must not publish a zero it never
+    # counted. Both stay None unless the attacker establishes otherwise.
+    llm_calls: Optional[int] = None
+    llm_zero_reason: Optional[str] = None
 
     @property
     def solved(self) -> bool:
@@ -130,8 +138,19 @@ def agent_attack(
     description = str((task.metadata if task else {}).get("one_day_description", ""))
     classes = classes_from_description(description) if one_day and description else None
 
-    ReconAgent(cfg, session)._run_web_recon(base_url)
-    report = ExploitAgent(cfg, session)._run_web_exploit(base_url, classes=classes)
+    recon = ReconAgent(cfg, session)
+    recon._run_web_recon(base_url)
+    exploit = ExploitAgent(cfg, session)
+    report = exploit._run_web_exploit(base_url, classes=classes)
+
+    # Read off the agents that ran, not off the config. Both are constructed
+    # with two positional arguments, so the client parameter keeps its None
+    # default and no model can be reached from here -- a fact of this code
+    # path, which is why the count is a proven zero rather than a default.
+    # The moment either agent is handed a client, there is no tracker on
+    # this path to count its calls with, so both fields go back to None:
+    # not measured is the only answer available until one exists.
+    model_free = recon.llm is None and exploit.llm is None
 
     return AttackOutcome(
         confirmed=int(report.get("confirmed", 0)),
@@ -139,6 +158,17 @@ def agent_attack(
         requests_sent=int(report.get("requests_sent", 0)),
         findings=list(report.get("findings", [])),
         oob_confirmed=int(report.get("params_oob_confirmed", 0)),
+        llm_calls=0 if model_free else None,
+        llm_zero_reason=(
+            llm_zero_reason(
+                cfg.llm,
+                CostTracker(),
+                client_built=False,
+                engine_uses_a_model=False,
+            )
+            if model_free
+            else None
+        ),
     )
 
 
@@ -210,6 +240,8 @@ def _run(task: BenchTask) -> BenchResult:
                 "endpoints_tested": outcome.endpoints_tested,
                 "requests_sent": outcome.requests_sent,
                 "findings": outcome.findings,
+                "llm_calls": outcome.llm_calls,
+                "llm_zero_reason": outcome.llm_zero_reason,
                 "judge_solved": judged,
                 "agreement": None if judged is None else outcome.solved == judged,
             }
diff --git a/cyberai/bench/cve_bench_runner.py b/cyberai/bench/cve_bench_runner.py
index 9dea4e13..fda98777 100644
--- a/cyberai/bench/cve_bench_runner.py
+++ b/cyberai/bench/cve_bench_runner.py
@@ -109,6 +109,8 @@ def _run(task: BenchTask) -> BenchResult:
                 "endpoints_tested": outcome.endpoints_tested,
                 "requests_sent": outcome.requests_sent,
                 "findings": outcome.findings,
+                "llm_calls": outcome.llm_calls,
+                "llm_zero_reason": outcome.llm_zero_reason,
             }
             if status is None:
                 # No verdict means no measurement. Say so instead of scoring it.
diff --git a/tests/unit/test_bench_agent_engine.py b/tests/unit/test_bench_agent_engine.py
index b3813c8c..a80e1489 100644
--- a/tests/unit/test_bench_agent_engine.py
+++ b/tests/unit/test_bench_agent_engine.py
@@ -190,13 +190,17 @@ def test_agent_attack_reads_flags_from_the_environment(monkeypatch):
     class _Recon:
         def __init__(self, cfg, session):
             seen["cfg"] = cfg
+            # BaseAgent assigns this on every agent, so a double without it is
+            # not standing in for one. Left off, the caller reading it has to
+            # guess an answer instead of measuring it.
+            self.llm = None
 
         def _run_web_recon(self, base_url):
             return {}
 
     class _Exploit:
         def __init__(self, cfg, session):
-            pass
+            self.llm = None
 
         def _run_web_exploit(self, base_url, classes=None):
             return {}
@@ -217,13 +221,14 @@ def test_agent_attack_forces_the_web_path_on(monkeypatch):
     class _Recon:
         def __init__(self, cfg, session):
             seen["cfg"] = cfg
+            self.llm = None
 
         def _run_web_recon(self, base_url):
             return {}
 
     class _Exploit:
         def __init__(self, cfg, session):
-            pass
+            self.llm = None
 
         def _run_web_exploit(self, base_url, classes=None):
             return {}
@@ -261,14 +266,14 @@ def test_agent_attack_carries_the_out_of_band_count_out_of_the_report(monkeypatc
 
     class _Recon:
         def __init__(self, cfg, session):
-            pass
+            self.llm = None
 
         def _run_web_recon(self, base_url):
             return {}
 
     class _Exploit:
         def __init__(self, cfg, session):
-            pass
+            self.llm = None
 
         def _run_web_exploit(self, base_url, classes=None):
             return {
@@ -293,14 +298,14 @@ def _recording_agents(monkeypatch):
 
     class _Recon:
         def __init__(self, cfg, session):
-            pass
+            self.llm = None
 
         def _run_web_recon(self, base_url):
             return {}
 
     class _Exploit:
         def __init__(self, cfg, session):
-            pass
+            self.llm = None
 
         def _run_web_exploit(self, base_url, classes=None):
             seen.append(classes)
@@ -351,3 +356,61 @@ def test_one_day_without_a_description_changes_nothing(monkeypatch):
     agent_attack("http://t", _task_describing(""), one_day=True)
 
     assert seen == [None]
+
+
+def test_the_real_path_reports_a_zero_it_can_prove(live_sqli_app):
+    """Both agents are constructed with two positional arguments, so neither
+    is handed a client and no model can be reached. The count is zero because
+    the path cannot call one, and the reason names that rather than the
+    absent API key -- the key is absent on every machine and would send a
+    reader after a knob that changes nothing here."""
+    outcome = agent_attack(live_sqli_app)
+
+    assert outcome.llm_calls == 0
+    assert outcome.llm_zero_reason == "engine_uses_no_model"
+
+
+def test_a_client_on_the_path_makes_the_count_unmeasured(monkeypatch):
+    """A zero is only publishable while nothing could have been called. Give
+    an agent a client and there is still no tracker here to count with, so
+    the honest answer becomes absent rather than zero."""
+
+    class _Recon:
+        def __init__(self, cfg, session):
+            self.llm = object()
+
+        def _run_web_recon(self, base_url):
+            return {}
+
+    class _Exploit:
+        def __init__(self, cfg, session):
+            self.llm = None
+
+        def _run_web_exploit(self, base_url, classes=None):
+            return {}
+
+    monkeypatch.setattr("cyberai.bench.agent_engine.ReconAgent", _Recon)
+    monkeypatch.setattr("cyberai.bench.agent_engine.ExploitAgent", _Exploit)
+
+    outcome = agent_attack("http://t")
+
+    assert outcome.llm_calls is None
+    assert outcome.llm_zero_reason is None
+
+
+def test_the_model_fact_reaches_the_result_the_scorecard_reads():
+    """A measurement that stops at the attacker is not published. The
+    scorecard is built from details, so the fact has to arrive there."""
+    adapter = LocalSuiteAdapter()
+    run = make_agent_runner(
+        adapter,
+        builder=_FakeBuilder(),
+        attacker=lambda url, task: AttackOutcome(
+            confirmed=1, llm_calls=0, llm_zero_reason="engine_uses_no_model"
+        ),
+        judge=lambda target, url: True,
+    )
+    result = run(_task(adapter))
+
+    assert result.details["llm_calls"] == 0
+    assert result.details["llm_zero_reason"] == "engine_uses_no_model"
diff --git a/tests/unit/test_cve_bench_runner.py b/tests/unit/test_cve_bench_runner.py
index cff5fd83..74f438b3 100644
--- a/tests/unit/test_cve_bench_runner.py
+++ b/tests/unit/test_cve_bench_runner.py
@@ -281,3 +281,22 @@ def _spy(base_url, task, one_day=False):
     )(_task())
 
     assert seen == [True]
+
+
+def test_the_model_fact_travels_on_the_cve_path_too():
+    """cve-bench routes through the same attacker, so the same fact has to
+    reach its result: two runners publishing one measurement differently is
+    how one number becomes two."""
+    runner = make_cve_bench_runner(
+        adapter=object(),
+        sandbox=_FakeSandbox(),
+        attacker=lambda url, task: AttackOutcome(
+            confirmed=0, llm_calls=0, llm_zero_reason="engine_uses_no_model"
+        ),
+        verdict=lambda url: (False, "Attack unsuccessful"),
+    )
+
+    result = runner(_task())
+
+    assert result.details["llm_calls"] == 0
+    assert result.details["llm_zero_reason"] == "engine_uses_no_model"

From 0142601ef68cbe01ab86f47ae3d8b60281dd3f7f Mon Sep 17 00:00:00 2001
From: Evgeny Kiriyak <224408464+evkir@users.noreply.github.com>
Date: Wed, 26 Aug 2026 06:32:11 +0200
Subject: [PATCH 4/7] fix(bench): the scorecard stops naming a model nobody
 reached

Two defects in one table, both visible in the published card.

RunMeta defaulted model and provider to "unspecified" and the CLI never
passed either, so every scorecard we shipped named a value the run had
chosen. A placeholder in a machine-readable table is not a blank: it
reads as an answer. The fields are now None by default and their rows
are left out, which says nothing rather than something untrue.

The version row was keyed `engine`, and the CLI writes `engine` too, to
name the engine that ran. The published agent card carried both --
`CyberAI 1.5.0` and `agent` -- under one key. The version row is now
`engine version`, and a second writer reaching an occupied key raises
instead of appending a row: the card is read by CI assertions, so one
key with two meanings is a defect rather than a formatting choice.

The model fact recorded per task now rolls up to the card, and only
where it agrees. All tasks proving a zero publishes that zero with its
cause. A run where some reached a model and others did not has no single
answer, so the split is what travels. Nothing measured writes no row at
all: zero is a measurement, and the placeholder engine takes none.

Six assertions, six mutants killed. Rendering the count under a truthy
test instead of an explicit None check hides exactly zero -- the number
the field exists to publish -- and two tests catch it.
---
 cyberai/bench/scorecard.py   | 51 +++++++++++++++++++++++------
 cyberai/cli/bench.py         | 33 ++++++++++++++++++-
 tests/unit/test_bench_cli.py | 62 ++++++++++++++++++++++++++++++++++++
 tests/unit/test_scorecard.py | 41 ++++++++++++++++++++++++
 4 files changed, 176 insertions(+), 11 deletions(-)

diff --git a/cyberai/bench/scorecard.py b/cyberai/bench/scorecard.py
index 65a66e28..6b6044bc 100644
--- a/cyberai/bench/scorecard.py
+++ b/cyberai/bench/scorecard.py
@@ -28,11 +28,21 @@
 @dataclass(frozen=True)
 class RunMeta:
     """Provenance for a scorecard run. All fields optional/defaulted so a
-    scorecard can be produced even in minimal/CI contexts."""
+    scorecard can be produced even in minimal/CI contexts.
+
+    A knob that was never measured is None and its row is left out, rather
+    than published as "unspecified": a placeholder in a machine-readable
+    table reads as a value the run chose, and an engine that never contacts
+    a model would name one as if it had. llm_calls is the same distinction
+    on the other side -- zero says a model was proven not to have been
+    reached, absent says nothing counted it.
+    """
 
     engine_version: str = __version__
-    model: str = "unspecified"
-    provider: str = "unspecified"
+    model: str | None = None
+    provider: str | None = None
+    llm_calls: int | None = None
+    llm_zero_reason: str | None = None
     note: str = ""
     extra: dict[str, Any] = field(default_factory=dict)
 
@@ -126,7 +136,12 @@ def _run_metric_lines(report: SuiteReport) -> list[str]:
 
 
 def generate_scorecard(report: SuiteReport, meta: RunMeta | None = None) -> str:
-    """Render a Markdown scorecard for one suite run."""
+    """Render a Markdown scorecard for one suite run.
+
+    The version row is keyed `engine version`. It used to be `engine`, which
+    the CLI also writes to name the engine that ran, so one published card
+    carried two rows under one key: `CyberAI 1.5.0` and `agent`.
+    """
     meta = meta or RunMeta()
     lines: list[str] = []
     lines.append(f"# Benchmark Scorecard — `{report.suite}`")
@@ -138,13 +153,29 @@ def generate_scorecard(report: SuiteReport, meta: RunMeta | None = None) -> str:
     lines.append("| field | value |")
     lines.append("| --- | --- |")
     lines.append(f"| timestamp | {_utc_now_iso()} |")
-    lines.append(f"| engine | CyberAI {meta.engine_version} |")
-    lines.append(f"| provider | {meta.provider} |")
-    lines.append(f"| model | {meta.model} |")
+    lines.append(f"| engine version | CyberAI {meta.engine_version} |")
+    rows: list[tuple[str, str]] = []
+    if meta.provider:
+        rows.append(("provider", meta.provider))
+    if meta.model:
+        rows.append(("model", meta.model))
+    if meta.llm_calls is not None:
+        rows.append(("llm calls", str(meta.llm_calls)))
+    if meta.llm_zero_reason:
+        rows.append(("llm zero reason", meta.llm_zero_reason))
     if meta.note:
-        lines.append(f"| note | {meta.note} |")
-    for k, v in meta.extra.items():
-        lines.append(f"| {k} | {v} |")
+        rows.append(("note", meta.note))
+    rows += [(str(k), str(v)) for k, v in meta.extra.items()]
+    written = {"timestamp", "engine version"}
+    for key, value in rows:
+        if key in written:
+            raise ValueError(
+                f"duplicate scorecard metadata key: {key!r}. The table is read "
+                "by machines, so one key carrying two meanings is a defect, not "
+                "a formatting choice."
+            )
+        written.add(key)
+        lines.append(f"| {key} | {value} |")
     lines.append("")
     lines.append("## Per-class breakdown")
     lines.append("")
diff --git a/cyberai/cli/bench.py b/cyberai/cli/bench.py
index e42142ed..06ed75e4 100644
--- a/cyberai/cli/bench.py
+++ b/cyberai/cli/bench.py
@@ -155,6 +155,28 @@ def _second_opinion(details: dict) -> bool | None:
     return None
 
 
+def _model_participation(report) -> tuple[int | None, str | None]:
+    """What the whole run can say about the model, or nothing.
+
+    Per-task facts only roll up when they agree. A run where some tasks
+    reached a model and others could not has no single answer, and picking
+    either one publishes a number the run did not produce -- so the split
+    itself is what travels. Absent everywhere stays absent: a scorecard with
+    no row is honest about not having measured, a row reading `unknown` is
+    a value.
+    """
+    results = list(report.results)
+    if not results:
+        return None, None
+    proven = [r for r in results if r.details.get("llm_calls") == 0]
+    if not proven:
+        return None, None
+    if len(proven) == len(results):
+        reasons = {str(r.details.get("llm_zero_reason")) for r in proven}
+        return 0, reasons.pop() if len(reasons) == 1 else "mixed_reasons"
+    return None, f"mixed: {len(proven)} of {len(results)} tasks reached no model"
+
+
 def _select_tasks(tasks: list, wanted: tuple[str, ...]) -> list:
     """Narrow a suite to the requested ids, or fail loudly.
 
@@ -326,7 +348,16 @@ def run(
             extra["filtered"] = f"{len(selected)} of {len(all_tasks)} tasks: " + ", ".join(
                 t.id for t in selected
             )
-        md = generate_scorecard(report, RunMeta(note="cyberai bench run", extra=extra))
+        calls, reason = _model_participation(report)
+        md = generate_scorecard(
+            report,
+            RunMeta(
+                note="cyberai bench run",
+                extra=extra,
+                llm_calls=calls,
+                llm_zero_reason=reason,
+            ),
+        )
         out = Path(scorecard_path)
         out.parent.mkdir(parents=True, exist_ok=True)
         out.write_text(md)
diff --git a/tests/unit/test_bench_cli.py b/tests/unit/test_bench_cli.py
index 2ed19a85..cf2df21b 100644
--- a/tests/unit/test_bench_cli.py
+++ b/tests/unit/test_bench_cli.py
@@ -428,3 +428,65 @@ def test_one_day_on_a_suite_that_describes_nothing_falls_back_in_the_record_too(
     # happen; the label has to fall back with the behaviour.
     assert "zero-day" in out.read_text()
     assert "one-day" not in out.read_text()
+
+
+def test_the_scorecard_publishes_a_zero_the_run_proved(monkeypatch, tmp_path):
+    """The fact has to survive the whole way to the file a reader opens,
+    so this goes through the CLI rather than building a RunMeta by hand."""
+    import cyberai.cli.bench as mod
+
+    details = {
+        "local-sqli-login": {
+            "agent_confirmed": 1,
+            "llm_calls": 0,
+            "llm_zero_reason": "engine_uses_no_model",
+        }
+    }
+    monkeypatch.setitem(mod._LIVE_ENGINES, "agent", _stub_agent_runner(details))
+    out = tmp_path / "sc.md"
+
+    result = CliRunner().invoke(
+        bench,
+        ["run", "--engine", "agent", "--task", "local-sqli-login", "--scorecard", str(out)],
+    )
+
+    assert result.exit_code == 0
+    text = out.read_text()
+    assert "| llm calls | 0 |" in text
+    assert "| llm zero reason | engine_uses_no_model |" in text
+
+
+def test_a_run_where_the_tasks_disagree_publishes_the_split(monkeypatch, tmp_path):
+    """Some tasks reaching a model and others not has no single answer.
+    Picking either one would publish a number the run did not produce."""
+    import cyberai.cli.bench as mod
+
+    details = {
+        "local-sqli-login": {
+            "agent_confirmed": 1,
+            "llm_calls": 0,
+            "llm_zero_reason": "engine_uses_no_model",
+        }
+    }
+    monkeypatch.setitem(mod._LIVE_ENGINES, "agent", _stub_agent_runner(details))
+    out = tmp_path / "sc.md"
+
+    result = CliRunner().invoke(bench, ["run", "--engine", "agent", "--scorecard", str(out)])
+
+    assert result.exit_code == 0
+    text = out.read_text()
+    assert "mixed: 1 of 4 tasks reached no model" in text
+    assert "| llm calls |" not in text
+
+
+def test_the_placeholder_engine_publishes_no_model_row(tmp_path):
+    """It measures nothing about a model, and an absent row says exactly
+    that; a zero there would claim a count nobody took."""
+    out = tmp_path / "sc.md"
+
+    result = CliRunner().invoke(bench, ["run", "--scorecard", str(out)])
+
+    assert result.exit_code == 0
+    text = out.read_text()
+    assert "| llm calls |" not in text
+    assert "| llm zero reason |" not in text
diff --git a/tests/unit/test_scorecard.py b/tests/unit/test_scorecard.py
index af03c768..d0e6631a 100644
--- a/tests/unit/test_scorecard.py
+++ b/tests/unit/test_scorecard.py
@@ -150,3 +150,44 @@ def test_out_of_band_proof_is_not_folded_into_the_in_band_count():
     row = [ln for ln in md.splitlines() if ln.startswith("| up |")][0]
     assert row.split("|")[3].strip() == "0"
     assert row.split("|")[4].strip() == "1"
+
+
+def test_a_run_that_named_no_model_leaves_the_rows_out():
+    """A placeholder in a machine-readable table reads as a chosen value.
+    The probe engine contacts nothing, and `unspecified` published that as
+    if a model had been involved."""
+    md = generate_scorecard(_report())
+    assert "| model |" not in md
+    assert "| provider |" not in md
+    assert "unspecified" not in md
+
+
+def test_the_version_row_no_longer_shares_a_key_with_the_engine_name():
+    """The CLI writes `engine` to name the engine that ran, so the version
+    under the same key gave one card two rows with one meaning between
+    them."""
+    md = generate_scorecard(_report(), RunMeta(extra={"engine": "agent"}))
+    keys = [ln.split("|")[1].strip() for ln in md.splitlines() if ln.startswith("| ")]
+    assert "engine version" in keys
+    assert keys.count("engine") == 1
+
+
+def test_a_proven_zero_and_its_cause_reach_the_card():
+    md = generate_scorecard(_report(), RunMeta(llm_calls=0, llm_zero_reason="engine_uses_no_model"))
+    assert "| llm calls | 0 |" in md
+    assert "| llm zero reason | engine_uses_no_model |" in md
+
+
+def test_an_unmeasured_run_writes_no_call_row_at_all():
+    """Zero is a measurement. A run with nothing to count must not print one."""
+    md = generate_scorecard(_report())
+    assert "| llm calls |" not in md
+
+
+def test_two_rows_under_one_key_are_refused():
+    """Rendering both is how a machine-readable card starts carrying two
+    truths; the second writer has to fail loudly instead."""
+    import pytest
+
+    with pytest.raises(ValueError, match="duplicate scorecard metadata key"):
+        generate_scorecard(_report(), RunMeta(note="run", extra={"note": "other"}))

From 634810d873d32bd1fc0d6c46c282c716c05258ea Mon Sep 17 00:00:00 2001
From: Evgeny Kiriyak <224408464+evkir@users.noreply.github.com>
Date: Wed, 26 Aug 2026 06:38:16 +0200
Subject: [PATCH 5/7] test(architecture): pin what the scorecard may read out
 of a task

Tail CH. A confirmed finding carries the target's own response verbatim
under `evidence`, and against our bench that response contains the flag
the target plants. The string is the target's, not ours, but it travels
in details["findings"], and details is what the card is rendered from.

It does not arrive today: the renderer reads six keys and findings is
not among them. That is a property of the current code rather than a
decision anyone wrote down, so a column added tomorrow would carry our
own targets' text into the artefact we publish as evidence of honesty,
silently, with every existing test green.

Two assertions because either alone is weak. The behavioural one renders
a card from a finding holding a planted secret and requires the secret
to be absent; it survives a new column that happens not to print
evidence. The structural one pins the exact key set the renderer reads,
so adding a column becomes a decision someone makes on purpose.

The secret is read from the bench app, not copied here: a hard-coded
copy keeps passing on the day the app changes what it plants. A third
assertion covers the ways this guard could quietly start guarding an
empty set.

Three mutants killed: promoting findings to a metric column, printing it
in the metrics row, and the runners no longer publishing it at all.
---
 .../test_scorecard_evidence_leak.py           | 157 ++++++++++++++++++
 1 file changed, 157 insertions(+)
 create mode 100644 tests/architecture/test_scorecard_evidence_leak.py

diff --git a/tests/architecture/test_scorecard_evidence_leak.py b/tests/architecture/test_scorecard_evidence_leak.py
new file mode 100644
index 00000000..04b66c86
--- /dev/null
+++ b/tests/architecture/test_scorecard_evidence_leak.py
@@ -0,0 +1,157 @@
+"""Target output recorded in a finding must not reach the published card.
+
+Tail CH. A confirmed finding carries `evidence`: the target's own response,
+verbatim. Against our bench that response contains the flag the target plants,
+which is legitimate -- it is the target speaking, not a constant of ours -- but
+it means the string travels in `details["findings"]`, and `details` is the
+input the scorecard is rendered from.
+
+Today it does not arrive: the renderer reads six keys and `findings` is not
+one of them. That is a property of the current code, not a decision anyone
+recorded, so a column added tomorrow would carry our own targets' text into
+the artefact we publish as evidence of honesty -- and it would do so silently,
+with every existing test green.
+
+Two assertions, because either alone is weak. The behavioural one renders a
+card from a finding holding a planted secret and requires the secret to be
+absent; it would survive a new column that happens not to print evidence. The
+structural one pins the exact set of keys the renderer reads, so any new
+column is a decision someone has to make on purpose.
+
+The secret is read from the bench app rather than written here: a hard-coded
+copy keeps passing on the day the app changes what it plants.
+"""
+
+import ast
+import pathlib
+
+import pytest
+
+from cyberai.bench import scorecard
+from cyberai.bench.apps import path_traversal
+from cyberai.bench.runner import BenchResult, SuiteReport
+from cyberai.bench.scorecard import generate_scorecard
+
+REPO = pathlib.Path(__file__).resolve().parents[2]
+SCORECARD = REPO / "cyberai" / "bench" / "scorecard.py"
+AGENT_ENGINE = REPO / "cyberai" / "bench" / "agent_engine.py"
+CVE_RUNNER = REPO / "cyberai" / "bench" / "cve_bench_runner.py"
+
+# What the renderer is allowed to read out of a task's details. Every entry is
+# a number or a class name. None of them is text the target wrote.
+DECLARED_KEYS = frozenset(
+    {
+        "vuln_class",
+        "available",
+        "endpoints_tested",
+        "agent_confirmed",
+        "oob_confirmed",
+        "requests_sent",
+    }
+)
+
+
+def _details_keys_read_by_the_scorecard() -> set[str]:
+    """Every details key the renderer reads, by reading the renderer.
+
+    Constant subscripts and membership tests come from the syntax tree;
+    the metric columns are looked up through a variable, so they come from
+    the table itself.
+    """
+    tree = ast.parse(SCORECARD.read_text(encoding="utf-8"))
+    keys: set[str] = set()
+    for node in ast.walk(tree):
+        if isinstance(node, ast.Call):
+            func = node.func
+            if (
+                isinstance(func, ast.Attribute)
+                and func.attr == "get"
+                and "details" in ast.unparse(func.value)
+                and node.args
+                and isinstance(node.args[0], ast.Constant)
+                and isinstance(node.args[0].value, str)
+            ):
+                keys.add(node.args[0].value)
+        elif isinstance(node, ast.Compare):
+            if (
+                len(node.ops) == 1
+                and isinstance(node.ops[0], ast.In)
+                and isinstance(node.left, ast.Constant)
+                and isinstance(node.left.value, str)
+                and "details" in ast.unparse(node.comparators[0])
+            ):
+                keys.add(node.left.value)
+    keys.update(key for key, _ in scorecard._METRIC_COLUMNS)
+    return keys
+
+
+def _report_carrying_a_planted_secret() -> tuple[SuiteReport, str]:
+    """One solved task whose finding quotes what the bench target plants."""
+    secret = path_traversal.SECRET_BODY.strip()
+    finding = {
+        "vuln_class": "path_traversal",
+        "parameter": "file",
+        "proof": "reads a file outside the web root",
+        "evidence": secret,
+    }
+    results = (
+        BenchResult(
+            "local-path-traversal",
+            "local",
+            True,
+            1.0,
+            details={
+                "engine": "agent",
+                "vuln_class": "path_traversal",
+                "available": True,
+                "agent_confirmed": 1,
+                "oob_confirmed": 0,
+                "endpoints_tested": 1,
+                "requests_sent": 3,
+                "findings": [finding],
+            },
+        ),
+    )
+    return SuiteReport(suite="local", total=1, solved=1, results=results), secret
+
+
+@pytest.mark.architecture
+def test_target_output_recorded_in_a_finding_does_not_reach_the_card():
+    report, secret = _report_carrying_a_planted_secret()
+
+    md = generate_scorecard(report)
+
+    assert secret not in md, (
+        "the card quotes what our own target plants. It is the target's text, "
+        "not ours, but a scorecard repeating it cannot be read as independent "
+        "evidence of anything."
+    )
+
+
+@pytest.mark.architecture
+def test_the_scorecard_reads_only_the_keys_it_declares():
+    read = _details_keys_read_by_the_scorecard()
+
+    assert read == set(DECLARED_KEYS), (
+        f"the renderer now reads {sorted(read - set(DECLARED_KEYS))} and no "
+        f"longer reads {sorted(set(DECLARED_KEYS) - read)}. Adding a column is "
+        "allowed; adding one that carries target output into the artefact is "
+        "the thing this file exists to make deliberate."
+    )
+
+
+@pytest.mark.architecture
+def test_the_rule_is_not_vacuous():
+    """A guard over an empty set passes forever.
+
+    Three ways this could quietly stop guarding: the syntax scan finding
+    nothing, the bench planting no secret, or the runners no longer putting
+    findings into details at all -- at which point there is no leak to guard
+    against and this file should be deleted rather than left green.
+    """
+    assert len(_details_keys_read_by_the_scorecard()) >= 4, "renderer not being scanned"
+    assert path_traversal.SECRET_BODY.strip(), "the bench plants nothing to leak"
+    for path in (AGENT_ENGINE, CVE_RUNNER):
+        assert '"findings": outcome.findings' in path.read_text(encoding="utf-8"), (
+            f"{path.name} no longer publishes findings into details"
+        )

From 318a675e9177181423801e2408a4b45b64b2bbf7 Mon Sep 17 00:00:00 2001
From: Evgeny Kiriyak <224408464+evkir@users.noreply.github.com>
Date: Wed, 26 Aug 2026 06:47:47 +0200
Subject: [PATCH 6/7] chore(bench): regenerate both scorecards against live
 targets

Measured 2026-08-26, seed 1337, zero-day, all four targets up, OOB
confirmed through a live phantom-grid (callback seen server-side on the
ssrf task).

Both engines still score 4/4. The agent run spends 21 requests where the
published card spent 28, and the whole difference is one task:
local-sqli-login falls from 12 to 5 with its two in-band proofs intact.
The other three are unchanged to the request.

That saving is not from this branch. The same task on main measures 5,
so it belongs to the decontamination merged as #231 -- a proof that no
longer recognises a string this project plants stops the walk earlier
than one that did. The published card simply predated that merge and was
never regenerated. This branch changed provenance, not measurement, and
the numbers say so.

What is new here is the metadata block. provider and model rows are gone
rather than reading "unspecified"; the version row is keyed `engine
version` so it no longer collides with the CLI's `engine`; and the agent
card records `llm calls: 0` with `engine_uses_no_model` beside it. The
probe card carries no call row at all: that engine builds no agents and
counts nothing, and a zero there would claim a measurement nobody took.

Neither card contains a flag literal.
---
 examples/local-bench/scorecard-agent.md | 20 ++++++++++----------
 examples/local-bench/scorecard.md       | 14 ++++++--------
 2 files changed, 16 insertions(+), 18 deletions(-)

diff --git a/examples/local-bench/scorecard-agent.md b/examples/local-bench/scorecard-agent.md
index 214d16a0..5769c4f0 100644
--- a/examples/local-bench/scorecard-agent.md
+++ b/examples/local-bench/scorecard-agent.md
@@ -6,10 +6,10 @@
 
 | field | value |
 | --- | --- |
-| timestamp | 2026-08-17T19:13:41Z |
-| engine | CyberAI 1.5.0 |
-| provider | unspecified |
-| model | unspecified |
+| timestamp | 2026-08-26T04:44:00Z |
+| engine version | CyberAI 1.5.0 |
+| llm calls | 0 |
+| llm zero reason | engine_uses_no_model |
 | note | cyberai bench run |
 | engine | agent |
 | suite | local |
@@ -31,17 +31,17 @@ What the engine reached and spent. A target that never came up scores zero for a
 
 | task id | available | in-band | out of band | endpoints | requests |
 | --- | --- | --- | --- | --- | --- |
-| local-sqli-login | ✓ | 2 | 0 | 1 | 12 |
+| local-sqli-login | ✓ | 2 | 0 | 1 | 5 |
 | local-cmdi-ping | ✓ | 1 | 0 | 1 | 3 |
 | local-path-traversal | ✓ | 1 | 0 | 1 | 3 |
 | local-ssrf-fetch | ✓ | 0 | 1 | 1 | 10 |
-| **total** | 4/4 | 4 | 1 | 4 | 28 |
+| **total** | 4/4 | 4 | 1 | 4 | 21 |
 
 ## Per-task results
 
 | task id | solved | time (s) | error |
 | --- | --- | --- | --- |
-| local-sqli-login | ✓ | 11.50 |  |
-| local-cmdi-ping | ✓ | 11.42 |  |
-| local-path-traversal | ✓ | 11.40 |  |
-| local-ssrf-fetch | ✓ | 12.09 |  |
+| local-sqli-login | ✓ | 11.48 |  |
+| local-cmdi-ping | ✓ | 11.47 |  |
+| local-path-traversal | ✓ | 11.50 |  |
+| local-ssrf-fetch | ✓ | 12.23 |  |
diff --git a/examples/local-bench/scorecard.md b/examples/local-bench/scorecard.md
index 2ecdc08a..e58f2321 100644
--- a/examples/local-bench/scorecard.md
+++ b/examples/local-bench/scorecard.md
@@ -6,10 +6,8 @@
 
 | field | value |
 | --- | --- |
-| timestamp | 2026-08-17T19:18:36Z |
-| engine | CyberAI 1.5.0 |
-| provider | unspecified |
-| model | unspecified |
+| timestamp | 2026-08-26T04:45:30Z |
+| engine version | CyberAI 1.5.0 |
 | note | cyberai bench run |
 | engine | real |
 | suite | local |
@@ -29,7 +27,7 @@
 
 | task id | solved | time (s) | error |
 | --- | --- | --- | --- |
-| local-sqli-login | ✓ | 11.38 |  |
-| local-cmdi-ping | ✓ | 11.36 |  |
-| local-path-traversal | ✓ | 11.31 |  |
-| local-ssrf-fetch | ✓ | 11.90 |  |
+| local-sqli-login | ✓ | 11.34 |  |
+| local-cmdi-ping | ✓ | 11.31 |  |
+| local-path-traversal | ✓ | 11.34 |  |
+| local-ssrf-fetch | ✓ | 11.89 |  |

From 892668546cb4a4169324be7bef15762f1ba8c284 Mon Sep 17 00:00:00 2001
From: Evgeny Kiriyak <224408464+evkir@users.noreply.github.com>
Date: Wed, 26 Aug 2026 06:52:00 +0200
Subject: [PATCH 7/7] docs(bench): publish the contamination account and the
 numbers after it

The local suite scored 4/4 for months and part of that was
self-referential. This says so in the repository rather than leaving a
reader to find it by grepping the engine.

Two forms are recorded, because only the first is greppable. A module
constant held the exact flag one of our targets plants. The SQL proof
carried no flag at all: it accepted a status field our bench login
prints and Juice Shop also returns from an untouched listing. The
architecture test exists for the second.

The numbers are stated as measured, including the expectation that did
not hold. A drop was expected -- a traversal payload was removed, the
SQLi proof became stricter -- and both engines still score 4/4. What
fell is the cost: 28 requests to 21, all of it one task, with its two
in-band proofs intact. The saving is attributed to the decontamination
that produced it rather than to the run that published the cards, which
the measurement on main settles.

The page also records what 4/4 does not mean here: the agent path
constructs its agents without a client, so no model takes part, and the
card says so in its own metadata rather than asking for trust.

CHANGELOG gets the same three fixes under Unreleased.
---
 CHANGELOG.md                             | 24 +++++++
 docs/benchmarks/contamination-2026-08.md | 87 ++++++++++++++++++++++++
 2 files changed, 111 insertions(+)
 create mode 100644 docs/benchmarks/contamination-2026-08.md

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7626aca3..69d3803d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -17,6 +17,30 @@ All notable changes to CyberAI are documented here.
   The `LICENSE` file holds the canonical Apache text unmodified; the copyright
   line moved to `NOTICE`.
 
+### Fixed
+
+- **Benchmark contamination.** The exploitation engine held a literal from a
+  target this project wrote, and the SQL-injection proof accepted the success
+  body our own bench login prints — so part of a published 4/4 measured
+  recognition rather than exploitation, and shipped to PyPI inside the engine.
+  Both proofs are now structural: a traversal reads a shape the target should
+  not serve, an auth bypass is proven by the 401 to 200 transition. An
+  architecture test runs every shipped proof against every string the bench
+  apps are built from, which is what caught the case a grep could not see.
+  Both scorecards were regenerated against live targets: still 4/4, at 21
+  requests instead of 28. Full account in
+  `docs/benchmarks/contamination-2026-08.md`.
+- **Scorecard provenance.** `provider` and `model` defaulted to the string
+  `unspecified` and the CLI passed neither, so every published card named a
+  value the run had chosen. The rows are omitted when nothing was measured.
+  The version row was keyed `engine`, which the CLI also writes to name the
+  engine that ran, giving one card two rows under one key; it is now
+  `engine version`, and a second writer reaching an occupied key raises.
+- **Cause of a zero call count.** A run whose code path constructs no LLM
+  client was reported as `no_api_key_for_openai`, pointing the reader at a
+  credential that would change nothing. The path now answers for itself, above
+  the credential causes.
+
 ### Added
 
 - **Contributor License Agreement** (`CLA.md`), adapted from the Apache ICLA
diff --git a/docs/benchmarks/contamination-2026-08.md b/docs/benchmarks/contamination-2026-08.md
new file mode 100644
index 00000000..7947b1f7
--- /dev/null
+++ b/docs/benchmarks/contamination-2026-08.md
@@ -0,0 +1,87 @@
+# Benchmark contamination, August 2026
+
+The local suite scored 4/4 on both engines for months. Part of that score was
+self-referential: the exploitation engine held a literal from a target this
+project wrote, so at least one proof measured recognition rather than
+exploitation. This page records what was wrong, what changed, and what the
+numbers did.
+
+## What was contaminated
+
+`cyberai/agents/exploit/web_payloads.py` carried a module constant holding the
+exact flag `cyberai/bench/apps/path_traversal.py` plants outside its web root,
+and the SQL-injection proof accepted the JSON body our own bench login prints
+on success. Both shipped to PyPI inside the exploitation engine, so every user
+received an engine that recognised strings from this repository's CTF apps.
+
+The second form is the instructive one. It carried no flag at all: the proof
+looked for a status field that our bench login returns and that Juice Shop
+also returns from an untouched product listing. A grep for `FLAG{` finds the
+first and never the second.
+
+## What changed
+
+| Before | After |
+|---|---|
+| Traversal proof matched a planted flag literal | Structural match on the shape of a file the target should not serve |
+| SQLi proof accepted our login's success body | Auth bypass proven by the 401 to 200 transition, not by any string |
+| Nothing prevented a recurrence | `tests/architecture/test_no_bench_leak.py` runs every production proof against every string constant the bench apps are built from |
+
+The guard asserts two things. The textual half forbids a flag literal outside
+`cyberai/bench/`. The behavioural half is what caught the harder case: it
+requires that no shipped proof is satisfied by any literal our targets are
+built from. A grep cannot express the second.
+
+## What the numbers did
+
+Both engines still score 4/4. The honest expectation before the run was a
+drop -- one traversal payload was removed and the SQLi proof became stricter --
+and the drop did not happen. The targets remain solvable by proofs that know
+nothing about them.
+
+The cost of solving them fell:
+
+| Task | Requests, published 2026-08-17 | Requests, 2026-08-26 | In-band proofs |
+|---|---|---|---|
+| local-sqli-login | 12 | 5 | 2, unchanged |
+| local-cmdi-ping | 3 | 3 | 1, unchanged |
+| local-path-traversal | 3 | 3 | 1, unchanged |
+| local-ssrf-fetch | 10 | 10 | 0 in band, 1 out of band |
+| **total** | **28** | **21** | **4 in band, 1 out of band** |
+
+The whole difference is one task. A proof that no longer accepts a string this
+project plants settles the parameter earlier than one that did, so the walk
+stops sooner and spends less.
+
+That saving belongs to the decontamination work, not to the run that published
+these cards: the same task measured on `main` before the provenance changes
+also spends 5. The published card predated the decontamination merge and had
+never been regenerated -- which is its own finding, and the reason the cards
+are now regenerated in the same branch that changes anything they report.
+
+## What the score does not say
+
+The agent engine reaches no model. `ReconAgent` and `ExploitAgent` are
+constructed with two positional arguments on this path, so the client
+parameter keeps its `None` default and no call can be made. The card records
+this directly: `llm calls: 0`, `llm zero reason: engine_uses_no_model`.
+
+So 4/4 is the score of a deterministic exploit corpus cross-checked by an
+independent probe. It is not a measurement of model-driven exploitation, and
+reading it as one would overstate what runs here. The metadata block exists so
+a reader does not have to take that on trust.
+
+The probe engine's card carries no call row at all. It builds no agents and
+counts nothing, and a zero there would claim a measurement nobody took.
+
+## Reproducing
+
+```bash
+cyberai bench run --suite local --engine agent \
+  --scorecard examples/local-bench/scorecard-agent.md
+```
+
+Measured 2026-08-26, seed 1337, zero-day mode, all four targets up, the blind
+target confirmed through a live out-of-band collector. See
+[reproducibility.md](reproducibility.md) for what a run pins and what it still
+does not measure.