diff --git a/CHANGELOG.md b/CHANGELOG.md index 7626aca3..69d3803d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,30 @@ All notable changes to CyberAI are documented here. The `LICENSE` file holds the canonical Apache text unmodified; the copyright line moved to `NOTICE`. +### Fixed + +- **Benchmark contamination.** The exploitation engine held a literal from a + target this project wrote, and the SQL-injection proof accepted the success + body our own bench login prints — so part of a published 4/4 measured + recognition rather than exploitation, and shipped to PyPI inside the engine. + Both proofs are now structural: a traversal reads a shape the target should + not serve, an auth bypass is proven by the 401 to 200 transition. An + architecture test runs every shipped proof against every string the bench + apps are built from, which is what caught the case a grep could not see. + Both scorecards were regenerated against live targets: still 4/4, at 21 + requests instead of 28. Full account in + `docs/benchmarks/contamination-2026-08.md`. +- **Scorecard provenance.** `provider` and `model` defaulted to the string + `unspecified` and the CLI passed neither, so every published card named a + value the run had chosen. The rows are omitted when nothing was measured. + The version row was keyed `engine`, which the CLI also writes to name the + engine that ran, giving one card two rows under one key; it is now + `engine version`, and a second writer reaching an occupied key raises. +- **Cause of a zero call count.** A run whose code path constructs no LLM + client was reported as `no_api_key_for_openai`, pointing the reader at a + credential that would change nothing. The path now answers for itself, above + the credential causes. + ### Added - **Contributor License Agreement** (`CLA.md`), adapted from the Apache ICLA diff --git a/cyberai/bench/agent_engine.py b/cyberai/bench/agent_engine.py index b8e4d38d..f2d5100f 100644 --- a/cyberai/bench/agent_engine.py +++ b/cyberai/bench/agent_engine.py @@ -42,6 +42,8 @@ from cyberai.bench.runner import BenchResult, BenchTask from cyberai.bench.targets import LocalSuiteAdapter, VulnTarget from cyberai.core.config import CyberAIConfig +from cyberai.core.cost_tracker import CostTracker +from cyberai.core.llm_usage import llm_zero_reason from cyberai.core.scan_session import ScanSession logger = logging.getLogger(__name__) @@ -56,6 +58,12 @@ class AttackOutcome: requests_sent: int = 0 findings: list[dict[str, Any]] = field(default_factory=list) oob_confirmed: int = 0 + # What the model did, or None for not measured. Zero and absent are + # different facts: a run that proves no model was reached can say so, + # while a run with no way to count must not publish a zero it never + # counted. Both stay None unless the attacker establishes otherwise. + llm_calls: Optional[int] = None + llm_zero_reason: Optional[str] = None @property def solved(self) -> bool: @@ -130,8 +138,19 @@ def agent_attack( description = str((task.metadata if task else {}).get("one_day_description", "")) classes = classes_from_description(description) if one_day and description else None - ReconAgent(cfg, session)._run_web_recon(base_url) - report = ExploitAgent(cfg, session)._run_web_exploit(base_url, classes=classes) + recon = ReconAgent(cfg, session) + recon._run_web_recon(base_url) + exploit = ExploitAgent(cfg, session) + report = exploit._run_web_exploit(base_url, classes=classes) + + # Read off the agents that ran, not off the config. Both are constructed + # with two positional arguments, so the client parameter keeps its None + # default and no model can be reached from here -- a fact of this code + # path, which is why the count is a proven zero rather than a default. + # The moment either agent is handed a client, there is no tracker on + # this path to count its calls with, so both fields go back to None: + # not measured is the only answer available until one exists. + model_free = recon.llm is None and exploit.llm is None return AttackOutcome( confirmed=int(report.get("confirmed", 0)), @@ -139,6 +158,17 @@ def agent_attack( requests_sent=int(report.get("requests_sent", 0)), findings=list(report.get("findings", [])), oob_confirmed=int(report.get("params_oob_confirmed", 0)), + llm_calls=0 if model_free else None, + llm_zero_reason=( + llm_zero_reason( + cfg.llm, + CostTracker(), + client_built=False, + engine_uses_a_model=False, + ) + if model_free + else None + ), ) @@ -210,6 +240,8 @@ def _run(task: BenchTask) -> BenchResult: "endpoints_tested": outcome.endpoints_tested, "requests_sent": outcome.requests_sent, "findings": outcome.findings, + "llm_calls": outcome.llm_calls, + "llm_zero_reason": outcome.llm_zero_reason, "judge_solved": judged, "agreement": None if judged is None else outcome.solved == judged, } diff --git a/cyberai/bench/cve_bench_runner.py b/cyberai/bench/cve_bench_runner.py index 9dea4e13..fda98777 100644 --- a/cyberai/bench/cve_bench_runner.py +++ b/cyberai/bench/cve_bench_runner.py @@ -109,6 +109,8 @@ def _run(task: BenchTask) -> BenchResult: "endpoints_tested": outcome.endpoints_tested, "requests_sent": outcome.requests_sent, "findings": outcome.findings, + "llm_calls": outcome.llm_calls, + "llm_zero_reason": outcome.llm_zero_reason, } if status is None: # No verdict means no measurement. Say so instead of scoring it. diff --git a/cyberai/bench/run_manifest.py b/cyberai/bench/run_manifest.py index a35c1f33..3935a228 100644 --- a/cyberai/bench/run_manifest.py +++ b/cyberai/bench/run_manifest.py @@ -7,7 +7,10 @@ - a content hash over the suite's tasks (id/name/criteria) — proves the suite wasn't quietly swapped to an easier one between runs, - the run config (model, provider, temperature, seed) — the knobs that affect - outcome, + outcome. A knob that was never measured is recorded as null, not as a + placeholder string: "unspecified" reads as a value the run chose, and a + probe engine that never contacts a model would publish it as if a model + had been involved, - a manifest hash over all of the above — a single fingerprint to compare runs. `set_global_seed` pins Python's `random` (and PYTHONHASHSEED for child procs) so @@ -53,13 +56,21 @@ def hash_tasks(tasks: list[BenchTask]) -> str: @dataclass(frozen=True) class RunConfig: - """The knobs that affect a run's outcome.""" - - model: str = "unspecified" - provider: str = "unspecified" - temperature: float = 0.0 + """The knobs that affect a run's outcome. + + Everything a caller may leave unmeasured defaults to None, so the manifest + distinguishes "this run did not involve a model" from "this run used a + model named unspecified". temperature is not exempt: 0.0 is a real setting + a caller can choose, and a default of 0.0 would claim deterministic + sampling for a run that never sampled anything. seed keeps a concrete + default because set_global_seed always pins one. + """ + + model: str | None = None + provider: str | None = None + temperature: float | None = None seed: int = DEFAULT_SEED - max_iterations: int = 0 + max_iterations: int | None = None extra: dict[str, Any] = field(default_factory=dict) diff --git a/cyberai/bench/scorecard.py b/cyberai/bench/scorecard.py index 65a66e28..6b6044bc 100644 --- a/cyberai/bench/scorecard.py +++ b/cyberai/bench/scorecard.py @@ -28,11 +28,21 @@ @dataclass(frozen=True) class RunMeta: """Provenance for a scorecard run. All fields optional/defaulted so a - scorecard can be produced even in minimal/CI contexts.""" + scorecard can be produced even in minimal/CI contexts. + + A knob that was never measured is None and its row is left out, rather + than published as "unspecified": a placeholder in a machine-readable + table reads as a value the run chose, and an engine that never contacts + a model would name one as if it had. llm_calls is the same distinction + on the other side -- zero says a model was proven not to have been + reached, absent says nothing counted it. + """ engine_version: str = __version__ - model: str = "unspecified" - provider: str = "unspecified" + model: str | None = None + provider: str | None = None + llm_calls: int | None = None + llm_zero_reason: str | None = None note: str = "" extra: dict[str, Any] = field(default_factory=dict) @@ -126,7 +136,12 @@ def _run_metric_lines(report: SuiteReport) -> list[str]: def generate_scorecard(report: SuiteReport, meta: RunMeta | None = None) -> str: - """Render a Markdown scorecard for one suite run.""" + """Render a Markdown scorecard for one suite run. + + The version row is keyed `engine version`. It used to be `engine`, which + the CLI also writes to name the engine that ran, so one published card + carried two rows under one key: `CyberAI 1.5.0` and `agent`. + """ meta = meta or RunMeta() lines: list[str] = [] lines.append(f"# Benchmark Scorecard — `{report.suite}`") @@ -138,13 +153,29 @@ def generate_scorecard(report: SuiteReport, meta: RunMeta | None = None) -> str: lines.append("| field | value |") lines.append("| --- | --- |") lines.append(f"| timestamp | {_utc_now_iso()} |") - lines.append(f"| engine | CyberAI {meta.engine_version} |") - lines.append(f"| provider | {meta.provider} |") - lines.append(f"| model | {meta.model} |") + lines.append(f"| engine version | CyberAI {meta.engine_version} |") + rows: list[tuple[str, str]] = [] + if meta.provider: + rows.append(("provider", meta.provider)) + if meta.model: + rows.append(("model", meta.model)) + if meta.llm_calls is not None: + rows.append(("llm calls", str(meta.llm_calls))) + if meta.llm_zero_reason: + rows.append(("llm zero reason", meta.llm_zero_reason)) if meta.note: - lines.append(f"| note | {meta.note} |") - for k, v in meta.extra.items(): - lines.append(f"| {k} | {v} |") + rows.append(("note", meta.note)) + rows += [(str(k), str(v)) for k, v in meta.extra.items()] + written = {"timestamp", "engine version"} + for key, value in rows: + if key in written: + raise ValueError( + f"duplicate scorecard metadata key: {key!r}. The table is read " + "by machines, so one key carrying two meanings is a defect, not " + "a formatting choice." + ) + written.add(key) + lines.append(f"| {key} | {value} |") lines.append("") lines.append("## Per-class breakdown") lines.append("") diff --git a/cyberai/cli/bench.py b/cyberai/cli/bench.py index e42142ed..06ed75e4 100644 --- a/cyberai/cli/bench.py +++ b/cyberai/cli/bench.py @@ -155,6 +155,28 @@ def _second_opinion(details: dict) -> bool | None: return None +def _model_participation(report) -> tuple[int | None, str | None]: + """What the whole run can say about the model, or nothing. + + Per-task facts only roll up when they agree. A run where some tasks + reached a model and others could not has no single answer, and picking + either one publishes a number the run did not produce -- so the split + itself is what travels. Absent everywhere stays absent: a scorecard with + no row is honest about not having measured, a row reading `unknown` is + a value. + """ + results = list(report.results) + if not results: + return None, None + proven = [r for r in results if r.details.get("llm_calls") == 0] + if not proven: + return None, None + if len(proven) == len(results): + reasons = {str(r.details.get("llm_zero_reason")) for r in proven} + return 0, reasons.pop() if len(reasons) == 1 else "mixed_reasons" + return None, f"mixed: {len(proven)} of {len(results)} tasks reached no model" + + def _select_tasks(tasks: list, wanted: tuple[str, ...]) -> list: """Narrow a suite to the requested ids, or fail loudly. @@ -326,7 +348,16 @@ def run( extra["filtered"] = f"{len(selected)} of {len(all_tasks)} tasks: " + ", ".join( t.id for t in selected ) - md = generate_scorecard(report, RunMeta(note="cyberai bench run", extra=extra)) + calls, reason = _model_participation(report) + md = generate_scorecard( + report, + RunMeta( + note="cyberai bench run", + extra=extra, + llm_calls=calls, + llm_zero_reason=reason, + ), + ) out = Path(scorecard_path) out.parent.mkdir(parents=True, exist_ok=True) out.write_text(md) diff --git a/cyberai/core/llm_usage.py b/cyberai/core/llm_usage.py index 5a6c5cf5..2482a1bc 100644 --- a/cyberai/core/llm_usage.py +++ b/cyberai/core/llm_usage.py @@ -25,6 +25,7 @@ def llm_zero_reason( *, client_built: bool, dry_run: bool = False, + engine_uses_a_model: bool = True, ) -> Optional[str]: """Why no LLM call happened, or None when at least one did. @@ -36,6 +37,14 @@ def llm_zero_reason( same zero as a run that never asked. `attempts` separates them: a non-zero attempt count with no call is a refusal, and it outranks the other causes because it is the one thing measured directly. + + `engine_uses_a_model` is answered by the code path, not by the config, and + it is checked before the credential causes. A path that constructs its + agents without a client cannot be fixed by a key: the default provider is + a cloud one and the default key is absent, so the missing-key cause fires + on every such run and points the reader at a knob that changes nothing. + Measured causes still outrank it -- a recorded answer means a model spoke, + whatever the path claims about itself. """ if tracker.call_count: return None @@ -43,6 +52,8 @@ def llm_zero_reason( return "provider_refused" if dry_run: return "dry_run" + if not engine_uses_a_model: + return "engine_uses_no_model" provider = llm_config.provider if provider in ("openai", "anthropic") and not llm_config.api_key: return f"no_api_key_for_{provider}" diff --git a/docs/benchmarks/contamination-2026-08.md b/docs/benchmarks/contamination-2026-08.md new file mode 100644 index 00000000..7947b1f7 --- /dev/null +++ b/docs/benchmarks/contamination-2026-08.md @@ -0,0 +1,87 @@ +# Benchmark contamination, August 2026 + +The local suite scored 4/4 on both engines for months. Part of that score was +self-referential: the exploitation engine held a literal from a target this +project wrote, so at least one proof measured recognition rather than +exploitation. This page records what was wrong, what changed, and what the +numbers did. + +## What was contaminated + +`cyberai/agents/exploit/web_payloads.py` carried a module constant holding the +exact flag `cyberai/bench/apps/path_traversal.py` plants outside its web root, +and the SQL-injection proof accepted the JSON body our own bench login prints +on success. Both shipped to PyPI inside the exploitation engine, so every user +received an engine that recognised strings from this repository's CTF apps. + +The second form is the instructive one. It carried no flag at all: the proof +looked for a status field that our bench login returns and that Juice Shop +also returns from an untouched product listing. A grep for `FLAG{` finds the +first and never the second. + +## What changed + +| Before | After | +|---|---| +| Traversal proof matched a planted flag literal | Structural match on the shape of a file the target should not serve | +| SQLi proof accepted our login's success body | Auth bypass proven by the 401 to 200 transition, not by any string | +| Nothing prevented a recurrence | `tests/architecture/test_no_bench_leak.py` runs every production proof against every string constant the bench apps are built from | + +The guard asserts two things. The textual half forbids a flag literal outside +`cyberai/bench/`. The behavioural half is what caught the harder case: it +requires that no shipped proof is satisfied by any literal our targets are +built from. A grep cannot express the second. + +## What the numbers did + +Both engines still score 4/4. The honest expectation before the run was a +drop -- one traversal payload was removed and the SQLi proof became stricter -- +and the drop did not happen. The targets remain solvable by proofs that know +nothing about them. + +The cost of solving them fell: + +| Task | Requests, published 2026-08-17 | Requests, 2026-08-26 | In-band proofs | +|---|---|---|---| +| local-sqli-login | 12 | 5 | 2, unchanged | +| local-cmdi-ping | 3 | 3 | 1, unchanged | +| local-path-traversal | 3 | 3 | 1, unchanged | +| local-ssrf-fetch | 10 | 10 | 0 in band, 1 out of band | +| **total** | **28** | **21** | **4 in band, 1 out of band** | + +The whole difference is one task. A proof that no longer accepts a string this +project plants settles the parameter earlier than one that did, so the walk +stops sooner and spends less. + +That saving belongs to the decontamination work, not to the run that published +these cards: the same task measured on `main` before the provenance changes +also spends 5. The published card predated the decontamination merge and had +never been regenerated -- which is its own finding, and the reason the cards +are now regenerated in the same branch that changes anything they report. + +## What the score does not say + +The agent engine reaches no model. `ReconAgent` and `ExploitAgent` are +constructed with two positional arguments on this path, so the client +parameter keeps its `None` default and no call can be made. The card records +this directly: `llm calls: 0`, `llm zero reason: engine_uses_no_model`. + +So 4/4 is the score of a deterministic exploit corpus cross-checked by an +independent probe. It is not a measurement of model-driven exploitation, and +reading it as one would overstate what runs here. The metadata block exists so +a reader does not have to take that on trust. + +The probe engine's card carries no call row at all. It builds no agents and +counts nothing, and a zero there would claim a measurement nobody took. + +## Reproducing + +```bash +cyberai bench run --suite local --engine agent \ + --scorecard examples/local-bench/scorecard-agent.md +``` + +Measured 2026-08-26, seed 1337, zero-day mode, all four targets up, the blind +target confirmed through a live out-of-band collector. See +[reproducibility.md](reproducibility.md) for what a run pins and what it still +does not measure. diff --git a/examples/local-bench/scorecard-agent.md b/examples/local-bench/scorecard-agent.md index 214d16a0..5769c4f0 100644 --- a/examples/local-bench/scorecard-agent.md +++ b/examples/local-bench/scorecard-agent.md @@ -6,10 +6,10 @@ | field | value | | --- | --- | -| timestamp | 2026-08-17T19:13:41Z | -| engine | CyberAI 1.5.0 | -| provider | unspecified | -| model | unspecified | +| timestamp | 2026-08-26T04:44:00Z | +| engine version | CyberAI 1.5.0 | +| llm calls | 0 | +| llm zero reason | engine_uses_no_model | | note | cyberai bench run | | engine | agent | | suite | local | @@ -31,17 +31,17 @@ What the engine reached and spent. A target that never came up scores zero for a | task id | available | in-band | out of band | endpoints | requests | | --- | --- | --- | --- | --- | --- | -| local-sqli-login | ✓ | 2 | 0 | 1 | 12 | +| local-sqli-login | ✓ | 2 | 0 | 1 | 5 | | local-cmdi-ping | ✓ | 1 | 0 | 1 | 3 | | local-path-traversal | ✓ | 1 | 0 | 1 | 3 | | local-ssrf-fetch | ✓ | 0 | 1 | 1 | 10 | -| **total** | 4/4 | 4 | 1 | 4 | 28 | +| **total** | 4/4 | 4 | 1 | 4 | 21 | ## Per-task results | task id | solved | time (s) | error | | --- | --- | --- | --- | -| local-sqli-login | ✓ | 11.50 | | -| local-cmdi-ping | ✓ | 11.42 | | -| local-path-traversal | ✓ | 11.40 | | -| local-ssrf-fetch | ✓ | 12.09 | | +| local-sqli-login | ✓ | 11.48 | | +| local-cmdi-ping | ✓ | 11.47 | | +| local-path-traversal | ✓ | 11.50 | | +| local-ssrf-fetch | ✓ | 12.23 | | diff --git a/examples/local-bench/scorecard.md b/examples/local-bench/scorecard.md index 2ecdc08a..e58f2321 100644 --- a/examples/local-bench/scorecard.md +++ b/examples/local-bench/scorecard.md @@ -6,10 +6,8 @@ | field | value | | --- | --- | -| timestamp | 2026-08-17T19:18:36Z | -| engine | CyberAI 1.5.0 | -| provider | unspecified | -| model | unspecified | +| timestamp | 2026-08-26T04:45:30Z | +| engine version | CyberAI 1.5.0 | | note | cyberai bench run | | engine | real | | suite | local | @@ -29,7 +27,7 @@ | task id | solved | time (s) | error | | --- | --- | --- | --- | -| local-sqli-login | ✓ | 11.38 | | -| local-cmdi-ping | ✓ | 11.36 | | -| local-path-traversal | ✓ | 11.31 | | -| local-ssrf-fetch | ✓ | 11.90 | | +| local-sqli-login | ✓ | 11.34 | | +| local-cmdi-ping | ✓ | 11.31 | | +| local-path-traversal | ✓ | 11.34 | | +| local-ssrf-fetch | ✓ | 11.89 | | diff --git a/tests/architecture/test_scorecard_evidence_leak.py b/tests/architecture/test_scorecard_evidence_leak.py new file mode 100644 index 00000000..04b66c86 --- /dev/null +++ b/tests/architecture/test_scorecard_evidence_leak.py @@ -0,0 +1,157 @@ +"""Target output recorded in a finding must not reach the published card. + +Tail CH. A confirmed finding carries `evidence`: the target's own response, +verbatim. Against our bench that response contains the flag the target plants, +which is legitimate -- it is the target speaking, not a constant of ours -- but +it means the string travels in `details["findings"]`, and `details` is the +input the scorecard is rendered from. + +Today it does not arrive: the renderer reads six keys and `findings` is not +one of them. That is a property of the current code, not a decision anyone +recorded, so a column added tomorrow would carry our own targets' text into +the artefact we publish as evidence of honesty -- and it would do so silently, +with every existing test green. + +Two assertions, because either alone is weak. The behavioural one renders a +card from a finding holding a planted secret and requires the secret to be +absent; it would survive a new column that happens not to print evidence. The +structural one pins the exact set of keys the renderer reads, so any new +column is a decision someone has to make on purpose. + +The secret is read from the bench app rather than written here: a hard-coded +copy keeps passing on the day the app changes what it plants. +""" + +import ast +import pathlib + +import pytest + +from cyberai.bench import scorecard +from cyberai.bench.apps import path_traversal +from cyberai.bench.runner import BenchResult, SuiteReport +from cyberai.bench.scorecard import generate_scorecard + +REPO = pathlib.Path(__file__).resolve().parents[2] +SCORECARD = REPO / "cyberai" / "bench" / "scorecard.py" +AGENT_ENGINE = REPO / "cyberai" / "bench" / "agent_engine.py" +CVE_RUNNER = REPO / "cyberai" / "bench" / "cve_bench_runner.py" + +# What the renderer is allowed to read out of a task's details. Every entry is +# a number or a class name. None of them is text the target wrote. +DECLARED_KEYS = frozenset( + { + "vuln_class", + "available", + "endpoints_tested", + "agent_confirmed", + "oob_confirmed", + "requests_sent", + } +) + + +def _details_keys_read_by_the_scorecard() -> set[str]: + """Every details key the renderer reads, by reading the renderer. + + Constant subscripts and membership tests come from the syntax tree; + the metric columns are looked up through a variable, so they come from + the table itself. + """ + tree = ast.parse(SCORECARD.read_text(encoding="utf-8")) + keys: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Call): + func = node.func + if ( + isinstance(func, ast.Attribute) + and func.attr == "get" + and "details" in ast.unparse(func.value) + and node.args + and isinstance(node.args[0], ast.Constant) + and isinstance(node.args[0].value, str) + ): + keys.add(node.args[0].value) + elif isinstance(node, ast.Compare): + if ( + len(node.ops) == 1 + and isinstance(node.ops[0], ast.In) + and isinstance(node.left, ast.Constant) + and isinstance(node.left.value, str) + and "details" in ast.unparse(node.comparators[0]) + ): + keys.add(node.left.value) + keys.update(key for key, _ in scorecard._METRIC_COLUMNS) + return keys + + +def _report_carrying_a_planted_secret() -> tuple[SuiteReport, str]: + """One solved task whose finding quotes what the bench target plants.""" + secret = path_traversal.SECRET_BODY.strip() + finding = { + "vuln_class": "path_traversal", + "parameter": "file", + "proof": "reads a file outside the web root", + "evidence": secret, + } + results = ( + BenchResult( + "local-path-traversal", + "local", + True, + 1.0, + details={ + "engine": "agent", + "vuln_class": "path_traversal", + "available": True, + "agent_confirmed": 1, + "oob_confirmed": 0, + "endpoints_tested": 1, + "requests_sent": 3, + "findings": [finding], + }, + ), + ) + return SuiteReport(suite="local", total=1, solved=1, results=results), secret + + +@pytest.mark.architecture +def test_target_output_recorded_in_a_finding_does_not_reach_the_card(): + report, secret = _report_carrying_a_planted_secret() + + md = generate_scorecard(report) + + assert secret not in md, ( + "the card quotes what our own target plants. It is the target's text, " + "not ours, but a scorecard repeating it cannot be read as independent " + "evidence of anything." + ) + + +@pytest.mark.architecture +def test_the_scorecard_reads_only_the_keys_it_declares(): + read = _details_keys_read_by_the_scorecard() + + assert read == set(DECLARED_KEYS), ( + f"the renderer now reads {sorted(read - set(DECLARED_KEYS))} and no " + f"longer reads {sorted(set(DECLARED_KEYS) - read)}. Adding a column is " + "allowed; adding one that carries target output into the artefact is " + "the thing this file exists to make deliberate." + ) + + +@pytest.mark.architecture +def test_the_rule_is_not_vacuous(): + """A guard over an empty set passes forever. + + Three ways this could quietly stop guarding: the syntax scan finding + nothing, the bench planting no secret, or the runners no longer putting + findings into details at all -- at which point there is no leak to guard + against and this file should be deleted rather than left green. + """ + assert len(_details_keys_read_by_the_scorecard()) >= 4, "renderer not being scanned" + assert path_traversal.SECRET_BODY.strip(), "the bench plants nothing to leak" + for path in (AGENT_ENGINE, CVE_RUNNER): + assert '"findings": outcome.findings' in path.read_text(encoding="utf-8"), ( + f"{path.name} no longer publishes findings into details" + ) diff --git a/tests/unit/test_bench_agent_engine.py b/tests/unit/test_bench_agent_engine.py index b3813c8c..a80e1489 100644 --- a/tests/unit/test_bench_agent_engine.py +++ b/tests/unit/test_bench_agent_engine.py @@ -190,13 +190,17 @@ def test_agent_attack_reads_flags_from_the_environment(monkeypatch): class _Recon: def __init__(self, cfg, session): seen["cfg"] = cfg + # BaseAgent assigns this on every agent, so a double without it is + # not standing in for one. Left off, the caller reading it has to + # guess an answer instead of measuring it. + self.llm = None def _run_web_recon(self, base_url): return {} class _Exploit: def __init__(self, cfg, session): - pass + self.llm = None def _run_web_exploit(self, base_url, classes=None): return {} @@ -217,13 +221,14 @@ def test_agent_attack_forces_the_web_path_on(monkeypatch): class _Recon: def __init__(self, cfg, session): seen["cfg"] = cfg + self.llm = None def _run_web_recon(self, base_url): return {} class _Exploit: def __init__(self, cfg, session): - pass + self.llm = None def _run_web_exploit(self, base_url, classes=None): return {} @@ -261,14 +266,14 @@ def test_agent_attack_carries_the_out_of_band_count_out_of_the_report(monkeypatc class _Recon: def __init__(self, cfg, session): - pass + self.llm = None def _run_web_recon(self, base_url): return {} class _Exploit: def __init__(self, cfg, session): - pass + self.llm = None def _run_web_exploit(self, base_url, classes=None): return { @@ -293,14 +298,14 @@ def _recording_agents(monkeypatch): class _Recon: def __init__(self, cfg, session): - pass + self.llm = None def _run_web_recon(self, base_url): return {} class _Exploit: def __init__(self, cfg, session): - pass + self.llm = None def _run_web_exploit(self, base_url, classes=None): seen.append(classes) @@ -351,3 +356,61 @@ def test_one_day_without_a_description_changes_nothing(monkeypatch): agent_attack("http://t", _task_describing(""), one_day=True) assert seen == [None] + + +def test_the_real_path_reports_a_zero_it_can_prove(live_sqli_app): + """Both agents are constructed with two positional arguments, so neither + is handed a client and no model can be reached. The count is zero because + the path cannot call one, and the reason names that rather than the + absent API key -- the key is absent on every machine and would send a + reader after a knob that changes nothing here.""" + outcome = agent_attack(live_sqli_app) + + assert outcome.llm_calls == 0 + assert outcome.llm_zero_reason == "engine_uses_no_model" + + +def test_a_client_on_the_path_makes_the_count_unmeasured(monkeypatch): + """A zero is only publishable while nothing could have been called. Give + an agent a client and there is still no tracker here to count with, so + the honest answer becomes absent rather than zero.""" + + class _Recon: + def __init__(self, cfg, session): + self.llm = object() + + def _run_web_recon(self, base_url): + return {} + + class _Exploit: + def __init__(self, cfg, session): + self.llm = None + + def _run_web_exploit(self, base_url, classes=None): + return {} + + monkeypatch.setattr("cyberai.bench.agent_engine.ReconAgent", _Recon) + monkeypatch.setattr("cyberai.bench.agent_engine.ExploitAgent", _Exploit) + + outcome = agent_attack("http://t") + + assert outcome.llm_calls is None + assert outcome.llm_zero_reason is None + + +def test_the_model_fact_reaches_the_result_the_scorecard_reads(): + """A measurement that stops at the attacker is not published. The + scorecard is built from details, so the fact has to arrive there.""" + adapter = LocalSuiteAdapter() + run = make_agent_runner( + adapter, + builder=_FakeBuilder(), + attacker=lambda url, task: AttackOutcome( + confirmed=1, llm_calls=0, llm_zero_reason="engine_uses_no_model" + ), + judge=lambda target, url: True, + ) + result = run(_task(adapter)) + + assert result.details["llm_calls"] == 0 + assert result.details["llm_zero_reason"] == "engine_uses_no_model" diff --git a/tests/unit/test_bench_cli.py b/tests/unit/test_bench_cli.py index 65615da8..cf2df21b 100644 --- a/tests/unit/test_bench_cli.py +++ b/tests/unit/test_bench_cli.py @@ -62,6 +62,27 @@ def test_bench_run_writes_a_manifest(tmp_path): assert data["manifest_hash"] +def test_a_run_without_a_model_publishes_no_model(tmp_path): + """An engine that never contacts a provider must not name one. + + The placeholder engine reaches no model at all, so every knob describing + one is absent rather than defaulted. A placeholder string in these fields + reads to anyone diffing two manifests as a value the run selected, and a + temperature of 0.0 claims deterministic sampling for a run that sampled + nothing. The seed is the exception on purpose: set_global_seed pins one + before the adapter loads, so it is always a measured fact. + """ + out = tmp_path / "run.json" + result = CliRunner().invoke(bench, ["run", "--manifest", str(out)]) + assert result.exit_code == 0 + cfg = json.loads(out.read_text())["config"] + assert cfg["model"] is None + assert cfg["provider"] is None + assert cfg["temperature"] is None + assert cfg["max_iterations"] is None + assert cfg["seed"] == DEFAULT_SEED + + def test_a_filtered_run_does_not_fingerprint_as_the_whole_suite(tmp_path): """The suite hash describes what ran, or the regression gate would compare a one-task run against a three-task baseline and call it a pass.""" @@ -407,3 +428,65 @@ def test_one_day_on_a_suite_that_describes_nothing_falls_back_in_the_record_too( # happen; the label has to fall back with the behaviour. assert "zero-day" in out.read_text() assert "one-day" not in out.read_text() + + +def test_the_scorecard_publishes_a_zero_the_run_proved(monkeypatch, tmp_path): + """The fact has to survive the whole way to the file a reader opens, + so this goes through the CLI rather than building a RunMeta by hand.""" + import cyberai.cli.bench as mod + + details = { + "local-sqli-login": { + "agent_confirmed": 1, + "llm_calls": 0, + "llm_zero_reason": "engine_uses_no_model", + } + } + monkeypatch.setitem(mod._LIVE_ENGINES, "agent", _stub_agent_runner(details)) + out = tmp_path / "sc.md" + + result = CliRunner().invoke( + bench, + ["run", "--engine", "agent", "--task", "local-sqli-login", "--scorecard", str(out)], + ) + + assert result.exit_code == 0 + text = out.read_text() + assert "| llm calls | 0 |" in text + assert "| llm zero reason | engine_uses_no_model |" in text + + +def test_a_run_where_the_tasks_disagree_publishes_the_split(monkeypatch, tmp_path): + """Some tasks reaching a model and others not has no single answer. + Picking either one would publish a number the run did not produce.""" + import cyberai.cli.bench as mod + + details = { + "local-sqli-login": { + "agent_confirmed": 1, + "llm_calls": 0, + "llm_zero_reason": "engine_uses_no_model", + } + } + monkeypatch.setitem(mod._LIVE_ENGINES, "agent", _stub_agent_runner(details)) + out = tmp_path / "sc.md" + + result = CliRunner().invoke(bench, ["run", "--engine", "agent", "--scorecard", str(out)]) + + assert result.exit_code == 0 + text = out.read_text() + assert "mixed: 1 of 4 tasks reached no model" in text + assert "| llm calls |" not in text + + +def test_the_placeholder_engine_publishes_no_model_row(tmp_path): + """It measures nothing about a model, and an absent row says exactly + that; a zero there would claim a count nobody took.""" + out = tmp_path / "sc.md" + + result = CliRunner().invoke(bench, ["run", "--scorecard", str(out)]) + + assert result.exit_code == 0 + text = out.read_text() + assert "| llm calls |" not in text + assert "| llm zero reason |" not in text diff --git a/tests/unit/test_cve_bench_runner.py b/tests/unit/test_cve_bench_runner.py index cff5fd83..74f438b3 100644 --- a/tests/unit/test_cve_bench_runner.py +++ b/tests/unit/test_cve_bench_runner.py @@ -281,3 +281,22 @@ def _spy(base_url, task, one_day=False): )(_task()) assert seen == [True] + + +def test_the_model_fact_travels_on_the_cve_path_too(): + """cve-bench routes through the same attacker, so the same fact has to + reach its result: two runners publishing one measurement differently is + how one number becomes two.""" + runner = make_cve_bench_runner( + adapter=object(), + sandbox=_FakeSandbox(), + attacker=lambda url, task: AttackOutcome( + confirmed=0, llm_calls=0, llm_zero_reason="engine_uses_no_model" + ), + verdict=lambda url: (False, "Attack unsuccessful"), + ) + + result = runner(_task()) + + assert result.details["llm_calls"] == 0 + assert result.details["llm_zero_reason"] == "engine_uses_no_model" diff --git a/tests/unit/test_llm_zero_reason.py b/tests/unit/test_llm_zero_reason.py index 7608099f..14a11d14 100644 --- a/tests/unit/test_llm_zero_reason.py +++ b/tests/unit/test_llm_zero_reason.py @@ -6,6 +6,8 @@ """ from cyberai.core.config import CyberAIConfig +from cyberai.core.cost_tracker import CostTracker +from cyberai.core.llm_usage import llm_zero_reason from cyberai.core.orchestrator import Orchestrator @@ -54,3 +56,30 @@ def test_reason_lands_in_the_session_export(): orch = Orchestrator(_config("openai"), dry_run=True) session = orch.run("example.com") assert session.kb.get("llm.usage")["zero_reason"] == "dry_run" + + +def test_a_path_that_uses_no_model_is_not_blamed_on_a_missing_key(): + """The bench agent path hands its agents no client at all, so a provider + key would change nothing. The default provider is a cloud one and the + default key is absent, so the missing-key cause answered every such run + and sent the reader after a knob that was never the reason.""" + cfg = _config("openai") + reason = llm_zero_reason(cfg.llm, CostTracker(), client_built=False, engine_uses_a_model=False) + assert reason == "engine_uses_no_model" + + +def test_the_default_leaves_the_credential_answer_where_it_was(): + """The pipeline does use a model, and there the missing key is the cause.""" + cfg = _config("openai") + reason = llm_zero_reason(cfg.llm, CostTracker(), client_built=False) + assert reason == "no_api_key_for_openai" + + +def test_a_recorded_answer_outranks_a_path_that_takes_no_model(): + """A path describes intent; a recorded call is a measurement.""" + tracker = CostTracker() + tracker.add("exploit", "qwen", input_tokens=10, output_tokens=5) + reason = llm_zero_reason( + _config("ollama").llm, tracker, client_built=False, engine_uses_a_model=False + ) + assert reason is None diff --git a/tests/unit/test_regression_gate.py b/tests/unit/test_regression_gate.py index b3b758c9..8501c2f4 100644 --- a/tests/unit/test_regression_gate.py +++ b/tests/unit/test_regression_gate.py @@ -2,6 +2,8 @@ from __future__ import annotations +import json + from cyberai.bench.regression_gate import ( check_regression, load_baseline, @@ -74,3 +76,40 @@ def test_load_baseline_roundtrip(tmp_path): assert loaded is not None assert loaded.solved == 6 assert loaded.suite_hash == "AAA" + + +def test_a_baseline_written_by_an_older_release_still_loads(tmp_path): + """Manifests on disk outlive the code that wrote them. + + Releases up to 1.5.0 stamped placeholder strings and zeroes into the run + config. Those files are the baselines a regression gate compares against, + and load_baseline expands whatever config it finds straight into the + dataclass. A field that stopped accepting the old shape would not raise + here -- it would return None, the gate would read that as "no baseline", + and a run that regressed to zero would pass green. + """ + legacy = { + "suite": "local", + "engine_version": "1.5.0", + "config": { + "model": "unspecified", + "provider": "unspecified", + "temperature": 0.0, + "seed": 1337, + "max_iterations": 0, + "extra": {"engine": "real"}, + }, + "suite_hash": "AAA", + "solved": 4, + "total": 4, + "timestamp": "2026-08-17T19:18:36Z", + "manifest_hash": "old", + } + p = tmp_path / "legacy.json" + p.write_text(json.dumps(legacy)) + + loaded = load_baseline(p) + assert loaded is not None, "an older baseline must not degrade to 'no baseline'" + assert loaded.config.model == "unspecified" + assert loaded.config.temperature == 0.0 + assert check_regression(_manifest(0), loaded).passed is False diff --git a/tests/unit/test_scorecard.py b/tests/unit/test_scorecard.py index af03c768..d0e6631a 100644 --- a/tests/unit/test_scorecard.py +++ b/tests/unit/test_scorecard.py @@ -150,3 +150,44 @@ def test_out_of_band_proof_is_not_folded_into_the_in_band_count(): row = [ln for ln in md.splitlines() if ln.startswith("| up |")][0] assert row.split("|")[3].strip() == "0" assert row.split("|")[4].strip() == "1" + + +def test_a_run_that_named_no_model_leaves_the_rows_out(): + """A placeholder in a machine-readable table reads as a chosen value. + The probe engine contacts nothing, and `unspecified` published that as + if a model had been involved.""" + md = generate_scorecard(_report()) + assert "| model |" not in md + assert "| provider |" not in md + assert "unspecified" not in md + + +def test_the_version_row_no_longer_shares_a_key_with_the_engine_name(): + """The CLI writes `engine` to name the engine that ran, so the version + under the same key gave one card two rows with one meaning between + them.""" + md = generate_scorecard(_report(), RunMeta(extra={"engine": "agent"})) + keys = [ln.split("|")[1].strip() for ln in md.splitlines() if ln.startswith("| ")] + assert "engine version" in keys + assert keys.count("engine") == 1 + + +def test_a_proven_zero_and_its_cause_reach_the_card(): + md = generate_scorecard(_report(), RunMeta(llm_calls=0, llm_zero_reason="engine_uses_no_model")) + assert "| llm calls | 0 |" in md + assert "| llm zero reason | engine_uses_no_model |" in md + + +def test_an_unmeasured_run_writes_no_call_row_at_all(): + """Zero is a measurement. A run with nothing to count must not print one.""" + md = generate_scorecard(_report()) + assert "| llm calls |" not in md + + +def test_two_rows_under_one_key_are_refused(): + """Rendering both is how a machine-readable card starts carrying two + truths; the second writer has to fail loudly instead.""" + import pytest + + with pytest.raises(ValueError, match="duplicate scorecard metadata key"): + generate_scorecard(_report(), RunMeta(note="run", extra={"note": "other"}))