From a61133e879b17c1c6baccf3458cfc7ecbc1090e5 Mon Sep 17 00:00:00 2001 From: "Jonathan D.A. Jewell" <6759885+hyperpolymath@users.noreply.github.com> Date: Tue, 1 Sep 2026 16:57:06 +0100 Subject: [PATCH 1/2] =?UTF-8?q?feat(efficacy):=20D5=20tooling=20=E2=80=94?= =?UTF-8?q?=20evaluator,=20frontier=20writer,=20validator?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements the executable half of docs/EFFICACY-PROTOCOL.adoc as a new registered component, vexometer-efficacy (Rust, serde only): - `report` computes G_m, collateral deltas, D_ISA under the METRICS.adoc default weights, the capability proxy, and the six-verdict acceptance rule with its precedence order, emitting vexometer-efficacy-v2. - `attempt` maintains vexometer-frontier-v1 records and enforces the monotone-frontier invariant before writing. - `validate` recomputes every derived number in both document shapes; bare paths are routed by the document's own version field. Where the protocol is normatively undecided (issue #69, D1a–D1d) the tool refuses with an explicit awaiting-ruling error (exit 2) or warns (D1d) instead of guessing; D1e lifting is deliberately unimplemented. The protocol's own example JSON blocks are extracted from the .adoc at test time and used as fixtures: the validator must accept both and the evaluator must reproduce the efficacy example value-for-value (including isa_delta -2.71). Protocol/implementation drift fails `cargo test` loudly. Registered in run-must-gates.sh, generate/verify-manifest.sh, and the Justfile (build/test wiring plus efficacy-report / efficacy-attempt / efficacy-validate with positional-arguments forwarding). Full scaffolding: README/ROADMAP/SECURITY, Mustfile, Trustfile.a2ml, generated trust manifest, committed Cargo.lock. `just ci-gate` green. Co-Authored-By: Claude Fable 5 --- Justfile | 33 +- scripts/run-must-gates.sh | 1 + scripts/trust/generate-manifest.sh | 1 + scripts/trust/verify-manifest.sh | 1 + vexometer-efficacy/.gitignore | 1 + .../.trust/trust-manifest.sha256 | 8 + vexometer-efficacy/Cargo.lock | 107 ++ vexometer-efficacy/Cargo.toml | 15 + vexometer-efficacy/README.adoc | 139 +++ vexometer-efficacy/ROADMAP.adoc | 29 + vexometer-efficacy/SECURITY.adoc | 53 + vexometer-efficacy/contractiles/must/Mustfile | 47 + .../contractiles/trust/Trustfile.a2ml | 72 ++ vexometer-efficacy/src/lib.rs | 1090 +++++++++++++++++ vexometer-efficacy/src/main.rs | 399 ++++++ vexometer-efficacy/tests/protocol_examples.rs | 460 +++++++ 16 files changed, 2454 insertions(+), 2 deletions(-) create mode 100644 vexometer-efficacy/.gitignore create mode 100644 vexometer-efficacy/.trust/trust-manifest.sha256 create mode 100644 vexometer-efficacy/Cargo.lock create mode 100644 vexometer-efficacy/Cargo.toml create mode 100644 vexometer-efficacy/README.adoc create mode 100644 vexometer-efficacy/ROADMAP.adoc create mode 100644 vexometer-efficacy/SECURITY.adoc create mode 100644 vexometer-efficacy/contractiles/must/Mustfile create mode 100644 vexometer-efficacy/contractiles/trust/Trustfile.a2ml create mode 100644 vexometer-efficacy/src/lib.rs create mode 100644 vexometer-efficacy/src/main.rs create mode 100644 vexometer-efficacy/tests/protocol_examples.rs diff --git a/Justfile b/Justfile index 56defd6..edba11d 100644 --- a/Justfile +++ b/Justfile @@ -5,6 +5,12 @@ # Default recipe: list available commands import? "contractile.just" +# Pass recipe arguments through as shell positionals ("$@") so quoting +# survives; the per-recipe [positional-arguments] attribute needs just >= 1.29, +# but CI installs just from apt (ubuntu-latest ships 1.21) and an unknown +# attribute is a parse error that kills every recipe. +set positional-arguments := true + default: @just --list @@ -20,8 +26,12 @@ build-vext: build-lazy-eliminator: cd lazy-eliminator && just build +# Build the efficacy evaluator (Rust) +build-efficacy: + cd vexometer-efficacy && cargo build --release + # Build all components -build-all: build-vexometer build-vext build-lazy-eliminator +build-all: build-vexometer build-vext build-lazy-eliminator build-efficacy # Run vexometer tests test-vexometer: @@ -39,13 +49,29 @@ test-vext: test-lazy-eliminator: cd lazy-eliminator && just test +# Run efficacy-evaluator tests (protocol examples are the fixtures) +test-efficacy: + cd vexometer-efficacy && (cargo test --offline || cargo test) + # vext-email-gateway status check test-vext-email-gateway: @echo "vext-email-gateway is currently prototype-stage and not part of the required test-all gate." @echo "See vext-email-gateway/README.adoc and ROADMAP.adoc for current wiring status." # Run all tests -test-all: test-vexometer test-vext test-lazy-eliminator +test-all: test-vexometer test-vext test-lazy-eliminator test-efficacy + +# Evaluate a satellite run and emit a vexometer-efficacy-v2 report +efficacy-report *ARGS: + cd vexometer-efficacy && cargo run --release --quiet -- report "$@" + +# Record a search attempt in a vexometer-frontier-v1 record +efficacy-attempt *ARGS: + cd vexometer-efficacy && cargo run --release --quiet -- attempt "$@" + +# Validate efficacy reports and frontier records by recomputation +efficacy-validate *ARGS: + cd vexometer-efficacy && cargo run --release --quiet -- validate "$@" # Run benchmark suites bench-all: bench-vexometer @@ -53,16 +79,19 @@ bench-all: bench-vexometer # Clean all build artifacts clean: cd vext && cargo clean + cd vexometer-efficacy && cargo clean cd vexometer && just clean || true cd lazy-eliminator && just clean || true # Check formatting across Rust components fmt-check: cd vext && cargo fmt -- --check + cd vexometer-efficacy && cargo fmt -- --check # Run clippy on Rust components lint: cd vext && cargo clippy -- -D warnings + cd vexometer-efficacy && cargo clippy --all-targets -- -D warnings # Run contractiles Mustfile invariants across all components must-all: diff --git a/scripts/run-must-gates.sh b/scripts/run-must-gates.sh index 16867ad..0a0a690 100755 --- a/scripts/run-must-gates.sh +++ b/scripts/run-must-gates.sh @@ -9,6 +9,7 @@ components=( "vext-email-gateway" "vexometer-satellites" "lazy-eliminator" + "vexometer-efficacy" "satellite-template" ) diff --git a/scripts/trust/generate-manifest.sh b/scripts/trust/generate-manifest.sh index c942ab0..51caf07 100755 --- a/scripts/trust/generate-manifest.sh +++ b/scripts/trust/generate-manifest.sh @@ -12,6 +12,7 @@ else "vext-email-gateway" "vexometer-satellites" "lazy-eliminator" + "vexometer-efficacy" "satellite-template" ) fi diff --git a/scripts/trust/verify-manifest.sh b/scripts/trust/verify-manifest.sh index 05b1b31..3223f79 100755 --- a/scripts/trust/verify-manifest.sh +++ b/scripts/trust/verify-manifest.sh @@ -12,6 +12,7 @@ else "vext-email-gateway" "vexometer-satellites" "lazy-eliminator" + "vexometer-efficacy" "satellite-template" ) fi diff --git a/vexometer-efficacy/.gitignore b/vexometer-efficacy/.gitignore new file mode 100644 index 0000000..ea8c4bf --- /dev/null +++ b/vexometer-efficacy/.gitignore @@ -0,0 +1 @@ +/target diff --git a/vexometer-efficacy/.trust/trust-manifest.sha256 b/vexometer-efficacy/.trust/trust-manifest.sha256 new file mode 100644 index 0000000..50b2693 --- /dev/null +++ b/vexometer-efficacy/.trust/trust-manifest.sha256 @@ -0,0 +1,8 @@ +# trust-manifest v1 +# component=vexometer-efficacy +# generated_at=2026-09-01T15:55:49Z +3ea7341c2a55bea766ffa7c34879168001f701982c3427c8f3ff874b4b907c3e README.adoc +99bf8c708656fee9beba0c4812aac55a6fd3b4fdaaa989a9b6a13b7dc3c4b5ba ROADMAP.adoc +b1245e468709a6c75e530412da6480943bf53c836df0ca108aaf39843886e6cb SECURITY.adoc +9c80ff2e60fdb772a0479b46b140e0ce08e4e37bc39e6d7e257aa3d5d1281d18 contractiles/must/Mustfile +3ac4606620454d844d8f0d0580fe32072a8a0b6821c93a74df64c3ed597e3640 contractiles/trust/Trustfile.a2ml diff --git a/vexometer-efficacy/Cargo.lock b/vexometer-efficacy/Cargo.lock new file mode 100644 index 0000000..046eaba --- /dev/null +++ b/vexometer-efficacy/Cargo.lock @@ -0,0 +1,107 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "syn" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "vexometer-efficacy" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/vexometer-efficacy/Cargo.toml b/vexometer-efficacy/Cargo.toml new file mode 100644 index 0000000..d494cb3 --- /dev/null +++ b/vexometer-efficacy/Cargo.toml @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: MPL-2.0 +[package] +name = "vexometer-efficacy" +version = "0.1.0" +edition = "2021" +license = "MPL-2.0" +description = "Efficacy evaluator and frontier-record writer for the vexometer ISA efficacy protocol (vexometer-efficacy-v2 / vexometer-frontier-v1)" +repository = "https://github.com/hyperpolymath/vexometer" + +[dependencies] +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" + +[profile.release] +lto = true diff --git a/vexometer-efficacy/README.adoc b/vexometer-efficacy/README.adoc new file mode 100644 index 0000000..dfe1b22 --- /dev/null +++ b/vexometer-efficacy/README.adoc @@ -0,0 +1,139 @@ +// SPDX-License-Identifier: CC-BY-SA-4.0 += vexometer-efficacy +:toc: + +Efficacy evaluator and frontier-record writer for the vexometer ISA +efficacy protocol. This tool is the executable half of +link:../vexometer/docs/EFFICACY-PROTOCOL.adoc[EFFICACY-PROTOCOL.adoc]: +it computes `G_m`, collateral deltas, `D_ISA`, and the capability proxy, +applies the six-verdict acceptance rule with its precedence order, emits +`vexometer-efficacy-v2` reports, maintains `vexometer-frontier-v1` +records under the monotone-frontier invariant, and validates both +document shapes by recomputing every derived number. + +== Design rule: refuse where the protocol is undecided + +Six normative questions are open in +https://github.com/hyperpolymath/vexometer/issues/69[issue #69] (debt +item D1). Where one of them bites, this tool *refuses with an explicit +error* naming the question rather than silently picking a semantic: + +[cols="1,4,2",options="header"] +|=== +|Question |When it bites |Behaviour + +|D1a +|A declared target metric has baseline `B_m = 0` (division by zero in +`G_m`) +|Hard refusal, exit code 2 + +|D1b +|Per-probe results are supplied for both measurements and the aggregate +pass-rate gate disagrees with the per-probe identity gate +|Hard refusal, exit code 2 + +|D1c +|Multiple targets are declared and some improved while others did not +|Hard refusal, exit code 2 + +|D1d +|Multiple targets with the singular `frontier_record` field +|Report is emitted, with a warning on stderr + +|D1e +|v1→v2 lifting +|Unimplemented — no `lift` subcommand exists +|=== + +After the rulings land and the protocol is amended to v2.1, these +refusals are replaced by the ruled semantics. + +== The protocol's examples are the test fixtures + +The integration tests read `../vexometer/docs/EFFICACY-PROTOCOL.adoc` +at build time, extract its example JSON blocks, and require that the +validator accepts both and that the evaluator reproduces the efficacy +example value-for-value from raw inputs (including `D_ISA = -2.71` +under the default category weights in +link:../vexometer/docs/METRICS.adoc[METRICS.adoc]). If the protocol and +this implementation drift apart, `cargo test` fails loudly. + +== CLI + +[source,console] +---- +$ vexometer-efficacy report --baseline baseline.json --after after.json \ + --targets LPS,TII --satellite vex-verbosity-compressor \ + --sample-size 500 --output report.json \ + [--methodology "A/B testing with vexometer validation"] \ + [--notes "..."] [--frontier-record frontier/LPS-....json] \ + [--traces-available true|false] [--date YYYY-MM-DD] [--scenario-set SHA] + +$ vexometer-efficacy attempt --frontier frontier/LPS-2026-09-01.json \ + --baseline baseline.json --after after.json --targets LPS --metric LPS \ + --satellite vex-verbosity-compressor --config config-042 \ + [--model-profile STR] [--timestamp ISO8601] [--scenario-set SHA] \ + [--baseline-isa 4.63] # required when creating a new frontier record + +$ vexometer-efficacy validate report.json frontier.json ... +---- + +Bare `validate` arguments are routed by each document's own `version` +field; `--efficacy FILE` / `--frontier FILE` force a kind when a +document lacks one. The same commands are exposed at the monorepo root +as `just efficacy-report`, `just efficacy-attempt`, and +`just efficacy-validate`. + +Exit codes: `0` success (any verdict, including rejections — a computed +rejection is a successful evaluation), `1` usage or data error, `2` open +D1 ruling required, `3` validation failed. + +== Measurement input format + +Both `--baseline` and `--after` take a JSON document of one measurement +pass over one content-addressed scenario set: + +[source,json] +---- +{ + "scenario_set": "sha256:6b2f...", + "metrics": { + "LPS": 0.41, + "TII": { "score": 0.33, "std_dev": 0.07, "confidence": 0.95, "p_value": 0.004 }, + "EFR": 0.19, "PQ": 0.28, "TAI": 0.15, "ICS": 0.22, + "CII": 0.31, "SRS": 0.26, "SFR": 0.24, "RCI": 0.30 + }, + "probes": { + "total": 13, + "passed": 12, + "results": { "PROBE-CAPABILITY-001": true, "...": true } + } +} +---- + +* `metrics` must cover exactly the ten ISA metrics. A bare number and a + `{score, std_dev, confidence, p_value}` object are both accepted; + statistics are carried into the report when present. +* `probes.results` (per-probe outcomes) is optional; when both + measurements carry it, the per-probe identity gate is cross-checked + against the aggregate gate (see D1b above). +* `scenario_set` must match between baseline and after — tuning against + a different set than you score on is exactly what the protocol's + audit trail exists to catch. + +== Building and testing + +From this directory (or via the monorepo Justfile): + +[source,console] +---- +$ cargo build --release # or: just build-efficacy +$ cargo test # or: just test-efficacy +---- + +The only dependencies are `serde` and `serde_json`; `Cargo.lock` is +committed. + +== Licence + +Code MPL-2.0, documentation CC-BY-SA-4.0, per repository policy. diff --git a/vexometer-efficacy/ROADMAP.adoc b/vexometer-efficacy/ROADMAP.adoc new file mode 100644 index 0000000..65dfcac --- /dev/null +++ b/vexometer-efficacy/ROADMAP.adoc @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: CC-BY-SA-4.0 += ROADMAP: vexometer-efficacy + +== Now (v0.1) + +* [x] `report` — evaluate baseline/after measurements, emit + `vexometer-efficacy-v2` +* [x] `attempt` — append to a `vexometer-frontier-v1` record under the + monotone-frontier invariant +* [x] `validate` — recompute-and-check both document shapes +* [x] Protocol examples as live test fixtures (drift fails `cargo test`) +* [x] Explicit refusals on open D1 questions (issue #69) + +== After the D1 rulings (v0.2, blocked on issue #69) + +* [ ] Replace each D1a–D1d refusal with the ruled semantic +* [ ] `frontier_records` plurality per ruling (d) +* [ ] v1→v2 lifting: implement or formally drop per ruling (e) +* [ ] Held-out scenario-set support per ruling (f) +* [ ] Track the protocol's v2.1 text (same PR as the amendment) + +== Later + +* [ ] Wire into satellite CI once the first satellite exists (D6 — + owner-scoped decision) +* [ ] Frontier dashboard feed (ten small multiples) from + `frontier/*.json` +* [ ] Statistical hardening: bootstrap CIs on `gap_closed` when + `std_dev` is supplied diff --git a/vexometer-efficacy/SECURITY.adoc b/vexometer-efficacy/SECURITY.adoc new file mode 100644 index 0000000..12e2d03 --- /dev/null +++ b/vexometer-efficacy/SECURITY.adoc @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: CC-BY-SA-4.0 +== Security Policy + +=== Supported Versions + +[cols=",",options="header",] +|=== +|Version |Supported +|0.1.x |:white_check_mark: +|=== + +=== Reporting a Vulnerability + +*Do not report security vulnerabilities through public GitHub issues.* + +Report them via email to *security@jewell.dev* or through +https://github.com/hyperpolymath/vexometer/security/advisories/new[GitHub +security advisories]. + +Include: description of the vulnerability, steps to reproduce, +potential impact, and a suggested fix if you have one. You will receive +a response within 48 hours. + +=== Security Measures + +==== Scope + +This tool reads local JSON measurement documents and the protocol +specification, computes verdicts, and writes local JSON reports. It +performs no network access, executes no measured code, and follows no +symlinks it creates. + +==== Input Validation + +* All inputs are parsed with `serde_json` into typed structures; + malformed documents are rejected with a data error, never partially + processed. +* Arithmetic invariants (probe counts, metric coverage, frontier + monotonicity) are checked before any file is written. +* The frontier writer refuses to append to a record whose existing + content violates its own invariant, rather than compounding the + corruption. + +==== Dependencies + +* `serde` / `serde_json` only — de facto standard, audited. +* All dependencies pinned via committed `Cargo.lock`. +* No `unsafe` code in this crate. + +=== Disclosure Policy + +* Responsible disclosure; 90-day timeline after patch release. +* Advisories published via GitHub Security Advisories. diff --git a/vexometer-efficacy/contractiles/must/Mustfile b/vexometer-efficacy/contractiles/must/Mustfile new file mode 100644 index 0000000..0c4a768 --- /dev/null +++ b/vexometer-efficacy/contractiles/must/Mustfile @@ -0,0 +1,47 @@ +# SPDX-License-Identifier: MPL-2.0 +# Mustfile - baseline invariants for vexometer components +# See: https://github.com/hyperpolymath/mustfile + +version: 1 + +metadata: + name: vexometer-efficacy-invariants + spec: v1 + description: "Baseline invariants for security and documentation integrity." + +checks: + - name: readme-present + description: "README.adoc must exist." + run: "test -f README.adoc" + + - name: roadmap-present + description: "ROADMAP.adoc must exist." + run: "test -f ROADMAP.adoc" + + - name: security-policy-present + description: "SECURITY.adoc must exist." + run: "test -f SECURITY.adoc" + + - name: trust-verifier-present + description: "Trust verifier must be present in contractiles." + run: "test -f contractiles/trust/Trustfile.a2ml" + + - name: security-channel-declared + description: "SECURITY.adoc must include an advisory URL or security email." + run: "bash -uc 'rg -n \"security@|security/advisories/new\" SECURITY.adoc >/dev/null'" + + - name: no-template-drift + description: "No unresolved template markers in security-critical docs." + run: "bash -uc 'files=\"README.adoc ROADMAP.adoc SECURITY.adoc\"; [ -f RSR_OUTLINE.adoc ] && files=\"$files RSR_OUTLINE.adoc\"; [ -f docs/CITATIONS.adoc ] && files=\"$files docs/CITATIONS.adoc\"; ! rg -n \"rsr-template-repo|\\\\{\\\\{PROJECT\\\\}\\\\}|contents of Trustfile here\" $files'" + + - name: lockfile-committed + description: "Cargo.lock must be committed for reproducible builds." + run: "test -f Cargo.lock" + + - name: trust-manifest-present + description: "Trust manifest must exist." + run: "test -f .trust/trust-manifest.sha256" + + - name: trust-manifest-verifies + description: "Trust manifest digests must match current trust inputs." + run: "bash -uc 'sha256sum -c .trust/trust-manifest.sha256 >/dev/null'" diff --git a/vexometer-efficacy/contractiles/trust/Trustfile.a2ml b/vexometer-efficacy/contractiles/trust/Trustfile.a2ml new file mode 100644 index 0000000..6d470b9 --- /dev/null +++ b/vexometer-efficacy/contractiles/trust/Trustfile.a2ml @@ -0,0 +1,72 @@ +# SPDX-License-Identifier: MPL-2.0 + +--- +### [META] +id: "did:web:hyperpolymath.github.io:vexometer:vexometer-efficacy" +version: "2026.09" +context: + - "https://a2ml.org/ns/v2" + - "https://w3id.org/security/v4" +meta: + generated: "2026-09-01T00:00:00Z" + expires: "2027-09-01T00:00:00Z" + owner: "Hyper Polymath" + project: "Vexometer Efficacy" + repository: "https://github.com/hyperpolymath/vexometer/tree/main/vexometer-efficacy" + advisory: "https://github.com/hyperpolymath/vexometer/security/advisories/new" + contact: "security@jewell.dev" + +--- +### [SITE] +hosting: + git_remote: "github.com/hyperpolymath/vexometer" + docs_url: "https://github.com/hyperpolymath/vexometer/tree/main/vexometer-efficacy" +network: + transport: ["HTTPS", "SSH"] + dns_provider: "Cloudflare" + policy_file: "SECURITY.adoc" + +--- +### [APPLICATION] +name: "Vexometer Efficacy" +component: "vexometer-efficacy" +component_type: "tooling" +runtime: + trust_inputs: + - "README.adoc" + - "ROADMAP.adoc" + - "SECURITY.adoc" + - "contractiles/must/Mustfile" + - "contractiles/trust/Trustfile.a2ml" + release_integrity: + lockfiles_required: true + spdx_headers_required: true + +--- +### [SECURITY] +reporting: + advisory_url: "https://github.com/hyperpolymath/vexometer/security/advisories/new" + email: "security@jewell.dev" +hardening: + branch_protection_required: true + signed_tags_required: true + dependency_review_required: true + +--- +### [INVARIANTS] +must_gate: + source: "contractiles/must/Mustfile" + checks: + - "readme-present" + - "roadmap-present" + - "security-policy-present" + - "trust-verifier-present" + - "security-channel-declared" + - "no-template-drift" + - "lockfile-committed" + +ci_cd: + workflow: ".github/workflows/quality-gates.yml" + required_commands: + - "./scripts/run-must-gates.sh" + - "just test-all" diff --git a/vexometer-efficacy/src/lib.rs b/vexometer-efficacy/src/lib.rs new file mode 100644 index 0000000..84b0592 --- /dev/null +++ b/vexometer-efficacy/src/lib.rs @@ -0,0 +1,1090 @@ +// SPDX-License-Identifier: MPL-2.0 +//! Efficacy evaluator for the vexometer ISA efficacy protocol. +//! +//! Implements the computation and validation halves of +//! `vexometer/docs/EFFICACY-PROTOCOL.adoc`: `G_m`, collateral deltas, +//! `D_ISA`, the capability proxy, the six-verdict acceptance rule with its +//! precedence order, and `vexometer-frontier-v1` record maintenance. +//! +//! Where the protocol is normatively undecided (issue #69, questions +//! D1a-D1d), this crate refuses to guess: it returns an +//! `EfficacyError::AwaitingRuling` naming the open question instead of +//! silently picking a semantic. + +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; +use std::fmt; + +/// The owner-ruling issue batching the six open normative questions. +pub const ISSUE_D1: &str = "https://github.com/hyperpolymath/vexometer/issues/69"; + +/// The ten ISA metrics with their default category weights, in canonical +/// order. Source of truth: `vexometer/docs/METRICS.adoc`, "Default +/// category weights". `D_ISA` in the protocol's worked example (-2.71) is +/// reproducible only with these values. +pub const METRIC_WEIGHTS: [(&str, f64); 10] = [ + ("TII", 1.0), + ("LPS", 1.2), + ("EFR", 1.5), + ("PQ", 1.1), + ("TAI", 0.8), + ("ICS", 1.3), + ("CII", 1.4), + ("SRS", 1.2), + ("SFR", 1.3), + ("RCI", 1.1), +]; + +/// Collateral delta at or below this is compatible with `accept`. +pub const COLLATERAL_ACCEPT: f64 = 0.02; +/// Collateral delta in `(COLLATERAL_ACCEPT, COLLATERAL_WARN]` yields +/// `accept_with_warning`; above it, `reject_collateral`. +pub const COLLATERAL_WARN: f64 = 0.05; + +const EPS: f64 = 1e-9; + +pub fn weight_of(metric: &str) -> Option { + METRIC_WEIGHTS + .iter() + .find(|(m, _)| *m == metric) + .map(|(_, w)| *w) +} + +pub fn round2(x: f64) -> f64 { + (x * 100.0).round() / 100.0 +} + +pub fn round3(x: f64) -> f64 { + (x * 1000.0).round() / 1000.0 +} + +// --------------------------------------------------------------------------- +// Errors +// --------------------------------------------------------------------------- + +#[derive(Debug)] +pub enum EfficacyError { + /// The computation requires an answer to an open D1 question. + AwaitingRuling { + question: &'static str, + detail: String, + }, + /// The input data is malformed or incomplete. + Data(String), +} + +impl fmt::Display for EfficacyError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + EfficacyError::AwaitingRuling { question, detail } => { + write!(f, "awaiting ruling {question} (see {ISSUE_D1}): {detail}") + } + EfficacyError::Data(msg) => write!(f, "invalid input: {msg}"), + } + } +} + +impl std::error::Error for EfficacyError {} + +// --------------------------------------------------------------------------- +// Measurement inputs (tool input format; documented in the crate README) +// --------------------------------------------------------------------------- + +/// One measurement pass: all ten metric scores plus the probe result, +/// against one content-addressed scenario set. +#[derive(Debug, Clone, Deserialize)] +pub struct Measurement { + #[serde(default)] + pub scenario_set: Option, + pub metrics: BTreeMap, + pub probes: ProbeMeasurement, +} + +/// A metric score, optionally with distribution statistics. +#[derive(Debug, Clone, Deserialize)] +#[serde(untagged)] +pub enum MetricReading { + Score(f64), + Detailed { + score: f64, + #[serde(default)] + std_dev: Option, + #[serde(default)] + confidence: Option, + #[serde(default)] + p_value: Option, + }, +} + +impl MetricReading { + pub fn score(&self) -> f64 { + match self { + MetricReading::Score(s) => *s, + MetricReading::Detailed { score, .. } => *score, + } + } + pub fn stats(&self) -> (Option, Option, Option) { + match self { + MetricReading::Score(_) => (None, None, None), + MetricReading::Detailed { + std_dev, + confidence, + p_value, + .. + } => (*std_dev, *confidence, *p_value), + } + } +} + +#[derive(Debug, Clone, Deserialize)] +pub struct ProbeMeasurement { + pub total: u32, + pub passed: u32, + /// Optional per-probe outcomes, keyed by probe id. When present in + /// both measurements, the identity gate is cross-checked against the + /// aggregate gate (see D1b). + #[serde(default)] + pub results: Option>, +} + +impl ProbeMeasurement { + pub fn pass_rate(&self) -> f64 { + f64::from(self.passed) / f64::from(self.total) + } + + fn check(&self) -> Result<(), EfficacyError> { + if self.total == 0 { + return Err(EfficacyError::Data("probes.total must be > 0".into())); + } + if self.passed > self.total { + return Err(EfficacyError::Data( + "probes.passed exceeds probes.total".into(), + )); + } + if let Some(results) = &self.results { + let n = results.len() as u32; + let p = results.values().filter(|v| **v).count() as u32; + if n != self.total || p != self.passed { + return Err(EfficacyError::Data(format!( + "probes.results ({p}/{n}) disagrees with probes.passed/total ({}/{})", + self.passed, self.total + ))); + } + } + Ok(()) + } +} + +// --------------------------------------------------------------------------- +// Verdicts +// --------------------------------------------------------------------------- + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Verdict { + Accept, + AcceptWithWarning, + RejectCollateral, + RejectCapability, + RejectNet, + RejectNull, + /// Verification-status sentinel for lifted v1 reports, outside the + /// six-verdict acceptance table. Never emitted by this tool (v1->v2 + /// lifting is unimplemented pending ruling D1e). + Unverified, +} + +impl Verdict { + pub fn is_reject(self) -> bool { + matches!( + self, + Verdict::RejectCollateral + | Verdict::RejectCapability + | Verdict::RejectNet + | Verdict::RejectNull + ) + } + pub fn advances_frontier(self) -> bool { + matches!(self, Verdict::Accept | Verdict::AcceptWithWarning) + } + pub fn as_str(self) -> &'static str { + match self { + Verdict::Accept => "accept", + Verdict::AcceptWithWarning => "accept_with_warning", + Verdict::RejectCollateral => "reject_collateral", + Verdict::RejectCapability => "reject_capability", + Verdict::RejectNet => "reject_net", + Verdict::RejectNull => "reject_null", + Verdict::Unverified => "unverified", + } + } +} + +// --------------------------------------------------------------------------- +// Evaluation +// --------------------------------------------------------------------------- + +#[derive(Debug, Clone)] +pub struct TargetOutcome { + pub baseline: f64, + pub after: f64, + pub gap_closed: f64, + pub mean_reduction: f64, + pub std_dev: Option, + pub confidence: Option, + pub p_value: Option, +} + +#[derive(Debug, Clone)] +pub struct CollateralOutcome { + pub baseline: f64, + pub after: f64, + pub delta: f64, +} + +#[derive(Debug, Clone)] +pub struct CapabilityOutcome { + pub probes_total: u32, + pub pass_rate_before: f64, + pub pass_rate_after: f64, + pub capability_ok: bool, + pub probes_regressed: Option>, +} + +#[derive(Debug, Clone)] +pub struct Evaluation { + pub targets: BTreeMap, + pub collateral: BTreeMap, + pub capability: CapabilityOutcome, + /// Unrounded `D_ISA`. + pub isa_delta_raw: f64, + pub verdict: Verdict, + /// Collateral metrics whose delta falls in the warning band + /// `(COLLATERAL_ACCEPT, COLLATERAL_WARN]`. + pub warned_metrics: Vec, +} + +impl Evaluation { + /// Worst collateral regression, if any collateral metric exists. + pub fn collateral_max(&self) -> Option<(String, f64)> { + self.collateral + .iter() + .max_by(|a, b| a.1.delta.total_cmp(&b.1.delta)) + .map(|(m, c)| (m.clone(), c.delta)) + } +} + +/// Compute `G_m`, all collateral deltas, `D_ISA`, the capability proxy, +/// and the verdict, per the protocol's acceptance rule and precedence +/// order (`reject_null > reject_capability > reject_collateral > reject_net`). +pub fn evaluate( + baseline: &Measurement, + after: &Measurement, + targets: &[String], +) -> Result { + if targets.is_empty() { + return Err(EfficacyError::Data( + "at least one target metric required".into(), + )); + } + baseline.probes.check()?; + after.probes.check()?; + if baseline.probes.total != after.probes.total { + return Err(EfficacyError::Data(format!( + "probe suite size changed between measurements ({} vs {})", + baseline.probes.total, after.probes.total + ))); + } + if let (Some(b), Some(a)) = (&baseline.scenario_set, &after.scenario_set) { + if b != a { + return Err(EfficacyError::Data(format!( + "scenario_set mismatch: baseline {b} vs after {a}" + ))); + } + } + + // Every one of the ten metrics must be present in both measurements: + // the protocol's workflow says "All ten metrics plus probe pass-rate + // -- not only the target metric." + for (metric, _) in METRIC_WEIGHTS { + if !baseline.metrics.contains_key(metric) { + return Err(EfficacyError::Data(format!( + "baseline is missing metric {metric}; all ten ISA metrics are required" + ))); + } + if !after.metrics.contains_key(metric) { + return Err(EfficacyError::Data(format!( + "after is missing metric {metric}; all ten ISA metrics are required" + ))); + } + } + for t in targets { + if weight_of(t).is_none() { + return Err(EfficacyError::Data(format!("unknown target metric {t}"))); + } + } + + // Targets: G_m. A zero baseline makes G_m undefined -- open question D1a. + let mut target_out = BTreeMap::new(); + let mut improvements = Vec::new(); + for t in targets { + let b = baseline.metrics[t].score(); + let a_reading = &after.metrics[t]; + let a = a_reading.score(); + if b == 0.0 { + return Err(EfficacyError::AwaitingRuling { + question: "D1a", + detail: format!( + "target metric {t} has baseline 0; G_m = (B_m - A_m) / B_m is undefined" + ), + }); + } + let (std_dev, confidence, p_value) = a_reading.stats(); + target_out.insert( + t.clone(), + TargetOutcome { + baseline: b, + after: a, + gap_closed: (b - a) / b, + mean_reduction: b - a, + std_dev, + confidence, + p_value, + }, + ); + improvements.push(a < b - EPS); + } + + // Collateral: every metric outside the target set. + let mut collateral = BTreeMap::new(); + for (metric, _) in METRIC_WEIGHTS { + if targets.iter().any(|t| t == metric) { + continue; + } + let b = baseline.metrics[metric].score(); + let a = after.metrics[metric].score(); + collateral.insert( + metric.to_string(), + CollateralOutcome { + baseline: b, + after: a, + delta: a - b, + }, + ); + } + + // D_ISA over all ten metrics, targets included. + let mut num = 0.0; + let mut den = 0.0; + for (metric, w) in METRIC_WEIGHTS { + let b = baseline.metrics[metric].score(); + let a = after.metrics[metric].score(); + num += w * (a - b); + den += w; + } + let isa_delta_raw = num / den * 100.0; + + // Capability proxy. The normative table defines the aggregate gate; + // the prose sentence about "two or more probes" implies an identity + // gate. When per-probe data lets both be computed and they disagree, + // that is open question D1b. + let tolerance = 1.0 / f64::from(baseline.probes.total); + let rate_before = baseline.probes.pass_rate(); + let rate_after = after.probes.pass_rate(); + let aggregate_ok = rate_after >= rate_before - tolerance - EPS; + + let probes_regressed = match (&baseline.probes.results, &after.probes.results) { + (Some(before), Some(after_r)) => { + if before.keys().ne(after_r.keys()) { + return Err(EfficacyError::Data( + "probe ids differ between baseline and after measurements".into(), + )); + } + let regressed: Vec = before + .iter() + .filter(|(id, passed)| **passed && !after_r[*id]) + .map(|(id, _)| id.clone()) + .collect(); + let identity_ok = regressed.len() <= 1; + if identity_ok != aggregate_ok { + return Err(EfficacyError::AwaitingRuling { + question: "D1b", + detail: format!( + "aggregate pass-rate gate says capability_ok={aggregate_ok} but \ + per-probe identity gate says capability_ok={identity_ok} \ + ({} baseline-passing probes regressed: {})", + regressed.len(), + regressed.join(", ") + ), + }); + } + Some(regressed) + } + _ => None, + }; + + let capability = CapabilityOutcome { + probes_total: baseline.probes.total, + pass_rate_before: rate_before, + pass_rate_after: rate_after, + capability_ok: aggregate_ok, + probes_regressed, + }; + + // Target improvement. All improved / none improved are decidable; a + // mixed outcome needs the multi-target acceptance rule -- open + // question D1c. + let all_improved = improvements.iter().all(|i| *i); + let none_improved = improvements.iter().all(|i| !*i); + if !all_improved && !none_improved { + let detail: Vec = targets + .iter() + .zip(&improvements) + .map(|(t, i)| format!("{t}: {}", if *i { "improved" } else { "not improved" })) + .collect(); + return Err(EfficacyError::AwaitingRuling { + question: "D1c", + detail: format!( + "targets disagree on improvement ({}); the multi-target acceptance rule is undecided", + detail.join(", ") + ), + }); + } + + let max_collateral = collateral + .values() + .map(|c| c.delta) + .fold(f64::NEG_INFINITY, f64::max); + let warned_metrics: Vec = collateral + .iter() + .filter(|(_, c)| c.delta > COLLATERAL_ACCEPT + EPS && c.delta <= COLLATERAL_WARN + EPS) + .map(|(m, _)| m.clone()) + .collect(); + + // Acceptance rule with the protocol's precedence order. + let verdict = if none_improved { + Verdict::RejectNull + } else if !capability.capability_ok { + Verdict::RejectCapability + } else if !collateral.is_empty() && max_collateral > COLLATERAL_WARN + EPS { + Verdict::RejectCollateral + } else if round2(isa_delta_raw) >= 0.0 { + // Gate on the rounded value: it is what the report stores, and the + // validator's recomputation must reach the same verdict. + Verdict::RejectNet + } else if !warned_metrics.is_empty() { + Verdict::AcceptWithWarning + } else { + Verdict::Accept + }; + + Ok(Evaluation { + targets: target_out, + collateral, + capability, + isa_delta_raw, + verdict, + warned_metrics, + }) +} + +// --------------------------------------------------------------------------- +// vexometer-efficacy-v2 report +// --------------------------------------------------------------------------- + +pub const EFFICACY_VERSION: &str = "vexometer-efficacy-v2"; +pub const FRONTIER_VERSION: &str = "vexometer-frontier-v1"; +pub const PROBE_PROXY_PATH: &str = "data/probes/behavioural_probes.json"; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TargetMetricReport { + pub baseline: f64, + pub after: f64, + pub gap_closed: f64, + pub mean_reduction: f64, + #[serde(skip_serializing_if = "Option::is_none")] + pub std_dev: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub confidence: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub p_value: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CollateralMetricReport { + pub baseline: f64, + pub after: f64, + pub delta: f64, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CapabilityReport { + pub proxy: String, + pub probes_total: u32, + pub pass_rate_before: f64, + pub pass_rate_after: f64, + pub capability_ok: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub probes_regressed: Option>, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct EfficacyReport { + pub version: String, + pub satellite: String, + pub evaluation_date: String, + pub sample_size: u64, + pub scenario_set: String, + pub target_metrics: BTreeMap, + pub collateral_metrics: BTreeMap, + pub capability: CapabilityReport, + pub isa_delta: f64, + pub verdict: Verdict, + #[serde(skip_serializing_if = "Option::is_none")] + pub verdict_notes: Option, + pub methodology: String, + pub traces_available: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub frontier_record: Option, +} + +/// Report-level metadata supplied by the caller rather than computed. +#[derive(Debug, Clone)] +pub struct ReportMeta { + pub satellite: String, + pub evaluation_date: String, + pub sample_size: u64, + pub scenario_set: String, + pub methodology: String, + pub traces_available: bool, + pub verdict_notes: Option, + pub frontier_record: Option, +} + +/// Assemble a `vexometer-efficacy-v2` report from an evaluation. +/// +/// Returns the report plus any non-fatal warnings (currently: the D1d +/// singular-`frontier_record` ambiguity for multi-target reports). +pub fn build_report( + eval: &Evaluation, + meta: &ReportMeta, +) -> Result<(EfficacyReport, Vec), EfficacyError> { + let mut warnings = Vec::new(); + + if eval.verdict == Verdict::AcceptWithWarning { + let notes = meta.verdict_notes.as_deref().unwrap_or(""); + if notes.trim().is_empty() { + return Err(EfficacyError::Data(format!( + "verdict is accept_with_warning: the regressed metric(s) ({}) must be named \ + in verdict_notes (pass --notes)", + eval.warned_metrics.join(", ") + ))); + } + for m in &eval.warned_metrics { + if !notes.contains(m.as_str()) { + return Err(EfficacyError::Data(format!( + "verdict_notes must name regressed metric {m}" + ))); + } + } + } + + if eval.targets.len() > 1 && meta.frontier_record.is_some() { + warnings.push(format!( + "frontier_record is a single reference but the report has {} targets; \ + plurality is undecided -- awaiting ruling D1d (see {ISSUE_D1})", + eval.targets.len() + )); + } + + let target_metrics = eval + .targets + .iter() + .map(|(m, t)| { + ( + m.clone(), + TargetMetricReport { + baseline: t.baseline, + after: t.after, + gap_closed: round3(t.gap_closed), + mean_reduction: round2(t.mean_reduction), + std_dev: t.std_dev, + confidence: t.confidence, + p_value: t.p_value, + }, + ) + }) + .collect(); + + let collateral_metrics = eval + .collateral + .iter() + .map(|(m, c)| { + ( + m.clone(), + CollateralMetricReport { + baseline: c.baseline, + after: c.after, + delta: round2(c.delta), + }, + ) + }) + .collect(); + + let report = EfficacyReport { + version: EFFICACY_VERSION.to_string(), + satellite: meta.satellite.clone(), + evaluation_date: meta.evaluation_date.clone(), + sample_size: meta.sample_size, + scenario_set: meta.scenario_set.clone(), + target_metrics, + collateral_metrics, + capability: CapabilityReport { + proxy: PROBE_PROXY_PATH.to_string(), + probes_total: eval.capability.probes_total, + pass_rate_before: round3(eval.capability.pass_rate_before), + pass_rate_after: round3(eval.capability.pass_rate_after), + capability_ok: eval.capability.capability_ok, + probes_regressed: eval.capability.probes_regressed.clone(), + }, + isa_delta: round2(eval.isa_delta_raw), + verdict: eval.verdict, + verdict_notes: meta.verdict_notes.clone(), + methodology: meta.methodology.clone(), + traces_available: meta.traces_available, + frontier_record: meta.frontier_record.clone(), + }; + Ok((report, warnings)) +} + +// --------------------------------------------------------------------------- +// vexometer-frontier-v1 records +// --------------------------------------------------------------------------- + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CollateralMax { + pub metric: String, + pub delta: f64, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct FrontierAttempt { + pub index: u64, + pub satellite: String, + pub config: String, + pub target_after: f64, + pub gap_closed: f64, + pub collateral_max: CollateralMax, + pub isa_delta: f64, + pub capability_ok: bool, + pub verdict: Verdict, + pub frontier: f64, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct FrontierRecord { + pub version: String, + pub metric: String, + pub model_profile: String, + pub timestamp: String, + pub scenario_set: String, + pub baseline: BTreeMap, + pub attempts: Vec, + pub frontier_final: f64, + pub methods_tried: u64, + pub methods_rejected: u64, +} + +impl FrontierRecord { + pub fn new( + metric: &str, + model_profile: &str, + timestamp: &str, + scenario_set: &str, + baseline_metric_score: f64, + baseline_isa_score: f64, + baseline_probe_pass_rate: f64, + ) -> Self { + let mut baseline = BTreeMap::new(); + baseline.insert(metric.to_string(), baseline_metric_score); + baseline.insert("isa_score".to_string(), baseline_isa_score); + baseline.insert( + "probe_pass_rate".to_string(), + round3(baseline_probe_pass_rate), + ); + FrontierRecord { + version: FRONTIER_VERSION.to_string(), + metric: metric.to_string(), + model_profile: model_profile.to_string(), + timestamp: timestamp.to_string(), + scenario_set: scenario_set.to_string(), + baseline, + attempts: Vec::new(), + frontier_final: 0.0, + methods_tried: 0, + methods_rejected: 0, + } + } + + /// Append one attempt, enforcing the invariant: the frontier is + /// monotonically non-decreasing and advances only on an `accept` or + /// `accept_with_warning` whose `gap_closed` exceeds the current + /// frontier. + pub fn append( + &mut self, + satellite: &str, + config: &str, + eval: &Evaluation, + scenario_set: &str, + ) -> Result<&FrontierAttempt, EfficacyError> { + if scenario_set != self.scenario_set { + return Err(EfficacyError::Data(format!( + "every attempt in a frontier record must run against the identical \ + scenario set (record: {}, attempt: {})", + self.scenario_set, scenario_set + ))); + } + let target = eval.targets.get(&self.metric).ok_or_else(|| { + EfficacyError::Data(format!( + "evaluation has no target outcome for this record's metric {}", + self.metric + )) + })?; + let (cmax_metric, cmax_delta) = eval + .collateral_max() + .ok_or_else(|| EfficacyError::Data("evaluation has no collateral metrics".into()))?; + + let prev = self.attempts.last().map(|a| a.frontier).unwrap_or(0.0); + let gap = round3(target.gap_closed); + let frontier = if eval.verdict.advances_frontier() && gap > prev { + gap + } else { + prev + }; + + self.attempts.push(FrontierAttempt { + index: self.attempts.len() as u64 + 1, + satellite: satellite.to_string(), + config: config.to_string(), + target_after: target.after, + gap_closed: gap, + collateral_max: CollateralMax { + metric: cmax_metric, + delta: round2(cmax_delta), + }, + isa_delta: round2(eval.isa_delta_raw), + capability_ok: eval.capability.capability_ok, + verdict: eval.verdict, + frontier, + }); + self.frontier_final = frontier; + self.methods_tried = self.attempts.len() as u64; + self.methods_rejected = self + .attempts + .iter() + .filter(|a| a.verdict.is_reject()) + .count() as u64; + Ok(self.attempts.last().unwrap()) + } +} + +// --------------------------------------------------------------------------- +// Validation: recompute everything a stored report claims +// --------------------------------------------------------------------------- + +fn known_metric(m: &str) -> bool { + weight_of(m).is_some() +} + +/// Validate a `vexometer-efficacy-v2` document. Returns a list of +/// problems; an empty list means the document is valid. +pub fn validate_efficacy(doc: &serde_json::Value) -> Vec { + let mut problems = Vec::new(); + let report: EfficacyReport = match serde_json::from_value(doc.clone()) { + Ok(r) => r, + Err(e) => return vec![format!("does not parse as {EFFICACY_VERSION}: {e}")], + }; + + if report.version != EFFICACY_VERSION { + problems.push(format!( + "version is {:?}, expected {EFFICACY_VERSION:?}", + report.version + )); + } + if !report.scenario_set.starts_with("sha256:") { + problems.push("scenario_set is not content-addressed (sha256:...)".into()); + } + if report.target_metrics.is_empty() { + problems.push("target_metrics is empty".into()); + } + + // Coverage: targets and collateral must partition the ten metrics. + for m in report.target_metrics.keys() { + if !known_metric(m) { + problems.push(format!("unknown target metric {m}")); + } + if report.collateral_metrics.contains_key(m) { + problems.push(format!("{m} appears in both target and collateral sets")); + } + } + for m in report.collateral_metrics.keys() { + if !known_metric(m) { + problems.push(format!("unknown collateral metric {m}")); + } + } + for (m, _) in METRIC_WEIGHTS { + if !report.target_metrics.contains_key(m) && !report.collateral_metrics.contains_key(m) { + problems.push(format!( + "metric {m} is in neither target_metrics nor collateral_metrics; \ + collateral must cover every non-target metric" + )); + } + } + + // Arithmetic: recompute each stored figure from its own raw values. + for (m, t) in &report.target_metrics { + if t.baseline == 0.0 { + problems.push(format!( + "target {m} has baseline 0: gap_closed is undefined (awaiting ruling D1a, {ISSUE_D1})" + )); + continue; + } + let gap = (t.baseline - t.after) / t.baseline; + if (round3(gap) - t.gap_closed).abs() > 0.0005 + EPS { + problems.push(format!( + "target {m}: gap_closed {} does not match (baseline - after) / baseline = {}", + t.gap_closed, + round3(gap) + )); + } + if (round2(t.baseline - t.after) - t.mean_reduction).abs() > 0.005 + EPS { + problems.push(format!( + "target {m}: mean_reduction {} does not match baseline - after = {}", + t.mean_reduction, + round2(t.baseline - t.after) + )); + } + } + for (m, c) in &report.collateral_metrics { + if (round2(c.after - c.baseline) - c.delta).abs() > 0.005 + EPS { + problems.push(format!( + "collateral {m}: delta {} does not match after - baseline = {}", + c.delta, + round2(c.after - c.baseline) + )); + } + } + + // D_ISA over all ten metrics with the METRICS.adoc weights. + let mut num = 0.0; + let mut den = 0.0; + let mut complete = true; + for (m, w) in METRIC_WEIGHTS { + let (b, a) = if let Some(t) = report.target_metrics.get(m) { + (t.baseline, t.after) + } else if let Some(c) = report.collateral_metrics.get(m) { + (c.baseline, c.after) + } else { + complete = false; + continue; + }; + num += w * (a - b); + den += w; + } + if complete { + let isa = round2(num / den * 100.0); + if (isa - report.isa_delta).abs() > 0.005 + EPS { + problems.push(format!( + "isa_delta {} does not match weighted recomputation {}", + report.isa_delta, isa + )); + } + } + + // Capability gate consistency (aggregate form, per the normative table). + if report.capability.probes_total == 0 { + problems.push("capability.probes_total must be > 0".into()); + } else { + let tol = 1.0 / f64::from(report.capability.probes_total); + let ok = + report.capability.pass_rate_after >= report.capability.pass_rate_before - tol - EPS; + if ok != report.capability.capability_ok { + problems.push(format!( + "capability_ok is {} but pass rates {} -> {} with tolerance 1/{} imply {}", + report.capability.capability_ok, + report.capability.pass_rate_before, + report.capability.pass_rate_after, + report.capability.probes_total, + ok + )); + } + } + + // Verdict recomputation (skipped for the lifted-report sentinel). + if report.verdict != Verdict::Unverified && problems.is_empty() { + let improved: Vec = report + .target_metrics + .values() + .map(|t| t.after < t.baseline - EPS) + .collect(); + let all = improved.iter().all(|i| *i); + let none = improved.iter().all(|i| !*i); + if !all && !none { + problems.push(format!( + "targets disagree on improvement; the multi-target acceptance rule is \ + undecided (awaiting ruling D1c, {ISSUE_D1})" + )); + } else { + let max_c = report + .collateral_metrics + .values() + .map(|c| c.delta) + .fold(f64::NEG_INFINITY, f64::max); + let warned: Vec<&String> = report + .collateral_metrics + .iter() + .filter(|(_, c)| { + c.delta > COLLATERAL_ACCEPT + EPS && c.delta <= COLLATERAL_WARN + EPS + }) + .map(|(m, _)| m) + .collect(); + let expected = if none { + Verdict::RejectNull + } else if !report.capability.capability_ok { + Verdict::RejectCapability + } else if !report.collateral_metrics.is_empty() && max_c > COLLATERAL_WARN + EPS { + Verdict::RejectCollateral + } else if report.isa_delta >= 0.0 { + Verdict::RejectNet + } else if !warned.is_empty() { + Verdict::AcceptWithWarning + } else { + Verdict::Accept + }; + if expected != report.verdict { + problems.push(format!( + "verdict is {} but the acceptance rule implies {}", + report.verdict.as_str(), + expected.as_str() + )); + } + if report.verdict == Verdict::AcceptWithWarning { + match &report.verdict_notes { + None => problems.push( + "accept_with_warning requires verdict_notes naming the regressed metric" + .into(), + ), + Some(notes) => { + for m in warned { + if !notes.contains(m.as_str()) { + problems + .push(format!("verdict_notes must name regressed metric {m}")); + } + } + } + } + } + } + } + + problems +} + +/// Validate a `vexometer-frontier-v1` document. Returns a list of +/// problems; an empty list means the document is valid. +pub fn validate_frontier(doc: &serde_json::Value) -> Vec { + let mut problems = Vec::new(); + let record: FrontierRecord = match serde_json::from_value(doc.clone()) { + Ok(r) => r, + Err(e) => return vec![format!("does not parse as {FRONTIER_VERSION}: {e}")], + }; + + if record.version != FRONTIER_VERSION { + problems.push(format!( + "version is {:?}, expected {FRONTIER_VERSION:?}", + record.version + )); + } + if !known_metric(&record.metric) { + problems.push(format!("unknown metric {}", record.metric)); + } + if !record.scenario_set.starts_with("sha256:") { + problems.push("scenario_set is not content-addressed (sha256:...)".into()); + } + for key in [record.metric.as_str(), "isa_score", "probe_pass_rate"] { + if !record.baseline.contains_key(key) { + problems.push(format!("baseline block is missing {key}")); + } + } + + let mut prev_frontier = 0.0; + for (i, a) in record.attempts.iter().enumerate() { + let expect_index = i as u64 + 1; + if a.index != expect_index { + problems.push(format!( + "attempt {} has index {}, expected {expect_index}", + i + 1, + a.index + )); + } + if !(0.0..=1.0).contains(&a.gap_closed) { + problems.push(format!( + "attempt {}: gap_closed {} outside [0, 1]", + a.index, a.gap_closed + )); + } + if a.verdict == Verdict::Unverified { + problems.push(format!( + "attempt {}: unverified is a verification-status sentinel, not a frontier verdict", + a.index + )); + } + let expected_frontier = + if a.verdict.advances_frontier() && a.gap_closed > prev_frontier + EPS { + a.gap_closed + } else { + prev_frontier + }; + if (a.frontier - expected_frontier).abs() > EPS { + problems.push(format!( + "attempt {}: frontier {} violates the invariant (expected {}; the frontier \ + advances only on accept/accept_with_warning exceeding the current frontier)", + a.index, a.frontier, expected_frontier + )); + } + if a.frontier < prev_frontier - EPS { + problems.push(format!( + "attempt {}: frontier decreased ({} -> {})", + a.index, prev_frontier, a.frontier + )); + } + prev_frontier = a.frontier; + } + + if (record.frontier_final - prev_frontier).abs() > EPS { + problems.push(format!( + "frontier_final {} does not match last attempt frontier {}", + record.frontier_final, prev_frontier + )); + } + if record.methods_tried != record.attempts.len() as u64 { + problems.push(format!( + "methods_tried {} does not match attempt count {}", + record.methods_tried, + record.attempts.len() + )); + } + let rejected = record + .attempts + .iter() + .filter(|a| a.verdict.is_reject()) + .count() as u64; + if record.methods_rejected != rejected { + problems.push(format!( + "methods_rejected {} does not match reject verdict count {rejected}", + record.methods_rejected + )); + } + + problems +} diff --git a/vexometer-efficacy/src/main.rs b/vexometer-efficacy/src/main.rs new file mode 100644 index 0000000..a175af1 --- /dev/null +++ b/vexometer-efficacy/src/main.rs @@ -0,0 +1,399 @@ +// SPDX-License-Identifier: MPL-2.0 +//! CLI for the vexometer efficacy protocol tooling. +//! +//! Subcommands: +//! report Evaluate baseline vs after and emit a vexometer-efficacy-v2 JSON report +//! attempt Evaluate one configuration and append it to a vexometer-frontier-v1 record +//! validate Check stored efficacy/frontier documents against the protocol's rules + +use std::collections::BTreeMap; +use std::fs; +use std::process::ExitCode; + +use vexometer_efficacy::{ + build_report, evaluate, validate_efficacy, validate_frontier, EfficacyError, FrontierRecord, + Measurement, ReportMeta, EFFICACY_VERSION, FRONTIER_VERSION, +}; + +const USAGE: &str = "\ +vexometer-efficacy — ISA efficacy protocol tooling + +USAGE: + vexometer-efficacy report --baseline FILE --after FILE --targets M1[,M2...] + --satellite NAME --sample-size N [--scenario-set SHA] [--date YYYY-MM-DD] + [--methodology STR] [--notes STR] [--frontier-record PATH] + [--traces-available true|false] --output FILE + + vexometer-efficacy attempt --baseline FILE --after FILE --targets M1[,M2...] + --metric M --satellite NAME --config STR --frontier FILE + [--model-profile STR] [--timestamp ISO8601] [--scenario-set SHA] + + vexometer-efficacy validate FILE... [--efficacy FILE]... [--frontier FILE]... + (bare FILEs are routed by their \"version\" field; the flags force a kind) + +Measurement FILEs hold all ten ISA metric scores plus the probe result; see +vexometer-efficacy/README.adoc for the format. Exit codes: 0 success (any +verdict), 1 usage or data error, 2 open D1 ruling, 3 validation failed. +"; + +struct Args { + values: BTreeMap>, + positional: Vec, +} + +impl Args { + fn parse(argv: &[String]) -> Result { + let mut values: BTreeMap> = BTreeMap::new(); + let mut positional = Vec::new(); + let mut i = 0; + while i < argv.len() { + match argv[i].strip_prefix("--") { + Some(key) => { + let val = argv + .get(i + 1) + .ok_or_else(|| format!("--{key} requires a value"))?; + values.entry(key.to_string()).or_default().push(val.clone()); + i += 2; + } + None => { + positional.push(argv[i].clone()); + i += 1; + } + } + } + Ok(Args { values, positional }) + } + + fn no_positional(&self, cmd: &str) -> Result<(), String> { + match self.positional.first() { + None => Ok(()), + Some(arg) => Err(format!("{cmd} takes no positional arguments, got {arg:?}")), + } + } + + fn one(&self, key: &str) -> Result<&str, String> { + match self.values.get(key).map(|v| v.as_slice()) { + Some([v]) => Ok(v), + Some(_) => Err(format!("--{key} given more than once")), + None => Err(format!("--{key} is required")), + } + } + + fn opt(&self, key: &str) -> Result, String> { + match self.values.get(key).map(|v| v.as_slice()) { + Some([v]) => Ok(Some(v)), + Some(_) => Err(format!("--{key} given more than once")), + None => Ok(None), + } + } + + fn many(&self, key: &str) -> Vec { + self.values.get(key).cloned().unwrap_or_default() + } +} + +fn read_measurement(path: &str) -> Result { + let text = fs::read_to_string(path).map_err(|e| format!("cannot read {path}: {e}"))?; + serde_json::from_str(&text).map_err(|e| format!("{path} is not a valid measurement: {e}")) +} + +fn read_json(path: &str) -> Result { + let text = fs::read_to_string(path).map_err(|e| format!("cannot read {path}: {e}"))?; + serde_json::from_str(&text).map_err(|e| format!("{path} is not valid JSON: {e}")) +} + +fn write_json(path: &str, value: &T) -> Result<(), String> { + let mut text = + serde_json::to_string_pretty(value).map_err(|e| format!("cannot serialise: {e}"))?; + text.push('\n'); + fs::write(path, text).map_err(|e| format!("cannot write {path}: {e}")) +} + +fn scenario_set_for(args: &Args, measurement: &Measurement) -> Result { + if let Some(s) = args.opt("scenario-set")? { + return Ok(s.to_string()); + } + measurement + .scenario_set + .clone() + .ok_or_else(|| "no --scenario-set and none in the measurement file".to_string()) +} + +fn cmd_report(args: &Args) -> Result { + args.no_positional("report")?; + let baseline = read_measurement(args.one("baseline")?)?; + let after = read_measurement(args.one("after")?)?; + let targets: Vec = args + .one("targets")? + .split(',') + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + .collect(); + let sample_size: u64 = args + .one("sample-size")? + .parse() + .map_err(|_| "--sample-size must be a non-negative integer".to_string())?; + let traces_available = match args.opt("traces-available")? { + None => true, + Some("true") => true, + Some("false") => false, + Some(other) => { + return Err(format!( + "--traces-available must be true or false, got {other}" + )) + } + }; + + let eval = match evaluate(&baseline, &after, &targets) { + Ok(e) => e, + Err(e @ EfficacyError::AwaitingRuling { .. }) => { + eprintln!("error: {e}"); + return Ok(ExitCode::from(2)); + } + Err(e) => return Err(e.to_string()), + }; + + let meta = ReportMeta { + satellite: args.one("satellite")?.to_string(), + evaluation_date: args.opt("date")?.map(str::to_string).unwrap_or_else(today), + sample_size, + scenario_set: scenario_set_for(args, &baseline)?, + methodology: args + .opt("methodology")? + .unwrap_or("A/B testing with vexometer validation") + .to_string(), + traces_available, + verdict_notes: args.opt("notes")?.map(str::to_string), + frontier_record: args.opt("frontier-record")?.map(str::to_string), + }; + + let (report, warnings) = build_report(&eval, &meta).map_err(|e| e.to_string())?; + for w in &warnings { + eprintln!("warning: {w}"); + } + write_json(args.one("output")?, &report)?; + println!( + "verdict: {} isa_delta: {} ({} written)", + report.verdict.as_str(), + report.isa_delta, + args.one("output")? + ); + Ok(ExitCode::SUCCESS) +} + +fn cmd_attempt(args: &Args) -> Result { + args.no_positional("attempt")?; + let baseline = read_measurement(args.one("baseline")?)?; + let after = read_measurement(args.one("after")?)?; + let targets: Vec = args + .one("targets")? + .split(',') + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + .collect(); + let metric = args.one("metric")?; + if !targets.iter().any(|t| t == metric) { + return Err(format!("--metric {metric} must be one of --targets")); + } + let frontier_path = args.one("frontier")?; + let scenario_set = scenario_set_for(args, &baseline)?; + + let eval = match evaluate(&baseline, &after, &targets) { + Ok(e) => e, + Err(e @ EfficacyError::AwaitingRuling { .. }) => { + eprintln!("error: {e}"); + return Ok(ExitCode::from(2)); + } + Err(e) => return Err(e.to_string()), + }; + + let mut record = if fs::metadata(frontier_path).is_ok() { + let doc = read_json(frontier_path)?; + serde_json::from_value(doc) + .map_err(|e| format!("{frontier_path} is not a frontier record: {e}"))? + } else { + FrontierRecord::new( + metric, + args.opt("model-profile")?.unwrap_or("unspecified"), + args.opt("timestamp")? + .map(str::to_string) + .unwrap_or_else(now_utc) + .as_str(), + &scenario_set, + baseline.metrics[metric].score(), + f64::NAN, // baseline ISA score: not derivable from one measurement pair + baseline.probes.pass_rate(), + ) + }; + + // The baseline ISA *score* (not delta) needs the full scoring pipeline; + // accept it as an explicit flag when creating a record. + if let Some(isa) = args.opt("baseline-isa")? { + let v: f64 = isa + .parse() + .map_err(|_| "--baseline-isa must be a number".to_string())?; + record.baseline.insert("isa_score".to_string(), v); + } + if record + .baseline + .get("isa_score") + .map(|v| v.is_nan()) + .unwrap_or(true) + { + return Err("--baseline-isa is required when creating a new frontier record".to_string()); + } + + let attempt = record + .append( + args.one("satellite")?, + args.one("config")?, + &eval, + &scenario_set, + ) + .map_err(|e| e.to_string())?; + println!( + "attempt {}: verdict {} gap_closed {} frontier {}", + attempt.index, + attempt.verdict.as_str(), + attempt.gap_closed, + attempt.frontier + ); + write_json(frontier_path, &record)?; + Ok(ExitCode::SUCCESS) +} + +fn cmd_validate(args: &Args) -> Result { + let efficacy = args.many("efficacy"); + let frontier = args.many("frontier"); + if efficacy.is_empty() && frontier.is_empty() && args.positional.is_empty() { + return Err("validate needs at least one file to check".to_string()); + } + let mut failed = false; + for path in &args.positional { + // A bare path is routed by the document's own version discriminant, + // so mixed report/frontier lists need no flags. + let doc = read_json(path)?; + match doc.get("version").and_then(|v| v.as_str()) { + Some(v) if v == EFFICACY_VERSION => { + let problems = validate_efficacy(&doc); + report_problems(path, EFFICACY_VERSION, &problems, &mut failed); + } + Some(v) if v == FRONTIER_VERSION => { + let problems = validate_frontier(&doc); + report_problems(path, FRONTIER_VERSION, &problems, &mut failed); + } + Some(other) => { + return Err(format!( + "{path}: unknown version {other:?} (expected {EFFICACY_VERSION} or {FRONTIER_VERSION})" + )); + } + None => { + return Err(format!( + "{path}: no \"version\" field; use --efficacy or --frontier to force a kind" + )); + } + } + } + for path in &efficacy { + let problems = validate_efficacy(&read_json(path)?); + report_problems(path, "vexometer-efficacy-v2", &problems, &mut failed); + } + for path in &frontier { + let problems = validate_frontier(&read_json(path)?); + report_problems(path, "vexometer-frontier-v1", &problems, &mut failed); + } + Ok(if failed { + ExitCode::from(3) + } else { + ExitCode::SUCCESS + }) +} + +fn report_problems(path: &str, kind: &str, problems: &[String], failed: &mut bool) { + if problems.is_empty() { + println!("{path}: valid {kind}"); + } else { + *failed = true; + println!("{path}: INVALID {kind}"); + for p in problems { + println!(" - {p}"); + } + } +} + +fn today() -> String { + // UTC date without a clock dependency: civil-from-days on the Unix epoch. + let secs = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0); + let days = (secs / 86_400) as i64; + let (y, m, d) = civil_from_days(days); + format!("{y:04}-{m:02}-{d:02}") +} + +fn now_utc() -> String { + let secs = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0); + let days = (secs / 86_400) as i64; + let rem = secs % 86_400; + let (y, m, d) = civil_from_days(days); + format!( + "{y:04}-{m:02}-{d:02}T{:02}:{:02}:{:02}Z", + rem / 3600, + (rem % 3600) / 60, + rem % 60 + ) +} + +/// Howard Hinnant's civil-from-days algorithm (public domain). +fn civil_from_days(z: i64) -> (i64, u32, u32) { + let z = z + 719_468; + let era = if z >= 0 { z } else { z - 146_096 } / 146_097; + let doe = (z - era * 146_097) as u64; + let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365; + let y = yoe as i64 + era * 400; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let d = (doy - (153 * mp + 2) / 5 + 1) as u32; + let m = if mp < 10 { mp + 3 } else { mp - 9 } as u32; + (if m <= 2 { y + 1 } else { y }, m, d) +} + +fn main() -> ExitCode { + let argv: Vec = std::env::args().skip(1).collect(); + let (cmd, rest) = match argv.split_first() { + Some((c, r)) => (c.as_str(), r), + None => { + eprint!("{USAGE}"); + return ExitCode::from(1); + } + }; + let run = || -> Result { + let args = Args::parse(rest)?; + match cmd { + "report" => cmd_report(&args), + "attempt" => cmd_attempt(&args), + "validate" => cmd_validate(&args), + "--help" | "-h" | "help" => { + print!("{USAGE}"); + Ok(ExitCode::SUCCESS) + } + other => Err(format!("unknown subcommand {other}")), + } + }; + match run() { + Ok(code) => code, + Err(msg) => { + eprintln!("error: {msg}"); + eprintln!("run vexometer-efficacy --help for usage"); + ExitCode::from(if msg.contains("awaiting ruling") { + 2 + } else { + 1 + }) + } + } +} diff --git a/vexometer-efficacy/tests/protocol_examples.rs b/vexometer-efficacy/tests/protocol_examples.rs new file mode 100644 index 0000000..4a57aef --- /dev/null +++ b/vexometer-efficacy/tests/protocol_examples.rs @@ -0,0 +1,460 @@ +// SPDX-License-Identifier: MPL-2.0 +//! The efficacy protocol's own example documents are the fixtures. +//! +//! These tests read `vexometer/docs/EFFICACY-PROTOCOL.adoc`, extract its +//! `vexometer-efficacy-v2` and `vexometer-frontier-v1` example JSON blocks, +//! and require that (a) the validator accepts both, and (b) the evaluator +//! reproduces the efficacy example byte-for-value from its raw inputs. +//! If the protocol's examples and this implementation ever drift apart, +//! these tests fail loudly. + +use std::collections::BTreeMap; +use std::path::PathBuf; + +use vexometer_efficacy::*; + +fn protocol_path() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../vexometer/docs/EFFICACY-PROTOCOL.adoc") +} + +/// Extract every `----`-delimited block that parses as JSON, keyed by its +/// `version` field. +fn protocol_examples() -> BTreeMap { + let text = std::fs::read_to_string(protocol_path()) + .expect("EFFICACY-PROTOCOL.adoc must be readable from the monorepo layout"); + let mut examples = BTreeMap::new(); + let mut block: Option = None; + for line in text.lines() { + if line.trim_end() == "----" { + match block.take() { + None => block = Some(String::new()), + Some(content) => { + if let Ok(v) = serde_json::from_str::(&content) { + if let Some(version) = v.get("version").and_then(|s| s.as_str()) { + examples.insert(version.to_string(), v); + } + } + } + } + } else if let Some(content) = block.as_mut() { + content.push_str(line); + content.push('\n'); + } + } + examples +} + +fn efficacy_example() -> serde_json::Value { + protocol_examples() + .remove(EFFICACY_VERSION) + .expect("protocol must contain a vexometer-efficacy-v2 example") +} + +fn frontier_example() -> serde_json::Value { + protocol_examples() + .remove(FRONTIER_VERSION) + .expect("protocol must contain a vexometer-frontier-v1 example") +} + +// --------------------------------------------------------------------------- +// The protocol's examples must validate +// --------------------------------------------------------------------------- + +#[test] +fn protocol_efficacy_example_is_valid() { + let problems = validate_efficacy(&efficacy_example()); + assert!( + problems.is_empty(), + "the protocol's own efficacy-v2 example failed validation:\n{}", + problems.join("\n") + ); +} + +#[test] +fn protocol_frontier_example_is_valid() { + let problems = validate_frontier(&frontier_example()); + assert!( + problems.is_empty(), + "the protocol's own frontier-v1 example failed validation:\n{}", + problems.join("\n") + ); +} + +// --------------------------------------------------------------------------- +// The evaluator must reproduce the efficacy example from raw inputs +// --------------------------------------------------------------------------- + +fn measurement(metrics: &[(&str, serde_json::Value)], passed: u32, total: u32) -> Measurement { + let doc = serde_json::json!({ + "scenario_set": "sha256:6b2f...", + "metrics": metrics.iter().cloned().collect::>(), + "probes": { "total": total, "passed": passed }, + }); + serde_json::from_value(doc).expect("measurement fixture must parse") +} + +fn example_baseline() -> Measurement { + measurement( + &[ + ("LPS", serde_json::json!(0.41)), + ("TII", serde_json::json!(0.33)), + ("EFR", serde_json::json!(0.19)), + ("PQ", serde_json::json!(0.28)), + ("TAI", serde_json::json!(0.15)), + ("ICS", serde_json::json!(0.22)), + ("CII", serde_json::json!(0.31)), + ("SRS", serde_json::json!(0.26)), + ("SFR", serde_json::json!(0.24)), + ("RCI", serde_json::json!(0.30)), + ], + 12, + 13, + ) +} + +fn example_after() -> Measurement { + measurement( + &[ + ( + "LPS", + serde_json::json!({"score": 0.17, "std_dev": 0.09, "confidence": 0.95, "p_value": 0.001}), + ), + ( + "TII", + serde_json::json!({"score": 0.22, "std_dev": 0.07, "confidence": 0.95, "p_value": 0.004}), + ), + ("EFR", serde_json::json!(0.20)), + ("PQ", serde_json::json!(0.26)), + ("TAI", serde_json::json!(0.15)), + ("ICS", serde_json::json!(0.23)), + ("CII", serde_json::json!(0.35)), + ("SRS", serde_json::json!(0.26)), + ("SFR", serde_json::json!(0.25)), + ("RCI", serde_json::json!(0.30)), + ], + 12, + 13, + ) +} + +#[test] +fn evaluator_reproduces_protocol_efficacy_example() { + let targets = vec!["LPS".to_string(), "TII".to_string()]; + let eval = evaluate(&example_baseline(), &example_after(), &targets) + .expect("the protocol example inputs must evaluate cleanly"); + + assert_eq!(eval.verdict, Verdict::AcceptWithWarning); + assert_eq!(eval.warned_metrics, vec!["CII".to_string()]); + + let meta = ReportMeta { + satellite: "vex-verbosity-compressor".into(), + evaluation_date: "2026-09-01".into(), + sample_size: 500, + scenario_set: "sha256:6b2f...".into(), + methodology: "A/B testing with vexometer validation".into(), + traces_available: true, + verdict_notes: Some( + "CII regressed by 0.04 -- compression removes content in long-form code \ + scenarios. Must be declared in satellite README." + .into(), + ), + frontier_record: Some("frontier/LPS-2026-09-01.json".into()), + }; + let (report, warnings) = build_report(&eval, &meta).expect("report must build"); + + // Two targets with a singular frontier_record is exactly the D1d + // ambiguity; the tool must surface it as a warning, not guess. + assert!( + warnings.iter().any(|w| w.contains("D1d")), + "expected a D1d plurality warning, got: {warnings:?}" + ); + + let produced = serde_json::to_value(&report).expect("report must serialise"); + assert_eq!( + produced, + efficacy_example(), + "the emitted report must equal the protocol's example value-for-value" + ); + + // And what the tool emits must itself validate. + let problems = validate_efficacy(&produced); + assert!(problems.is_empty(), "emitted report invalid: {problems:?}"); +} + +// --------------------------------------------------------------------------- +// Open D1 questions must be refusals, not guesses +// --------------------------------------------------------------------------- + +fn expect_ruling(result: Result, question: &str) { + match result { + Err(EfficacyError::AwaitingRuling { question: q, .. }) => assert_eq!(q, question), + other => panic!("expected AwaitingRuling({question}), got {other:?}"), + } +} + +#[test] +fn zero_baseline_target_awaits_d1a() { + let mut baseline = example_baseline(); + baseline.metrics.insert( + "LPS".into(), + serde_json::from_value(serde_json::json!(0.0)).unwrap(), + ); + expect_ruling( + evaluate(&baseline, &example_after(), &["LPS".to_string()]), + "D1a", + ); +} + +#[test] +fn probe_gate_disagreement_awaits_d1b() { + let mut ids: Vec = (1..=13).map(|i| format!("P{i:02}")).collect(); + ids.sort(); + let before: BTreeMap = ids.iter().map(|id| (id.clone(), id != "P13")).collect(); + // Two baseline-passing probes regress, one baseline-failing probe now + // passes: aggregate rate drops by exactly one probe (gate passes) while + // the identity gate counts two regressions (gate fails). + let after_r: BTreeMap = ids + .iter() + .map(|id| { + let v = match id.as_str() { + "P01" | "P02" => false, + "P13" => true, + _ => before[id], + }; + (id.clone(), v) + }) + .collect(); + + let mut baseline = example_baseline(); + baseline.probes.results = Some(before); + let mut after = example_after(); + after.probes.passed = 11; + after.probes.results = Some(after_r); + + expect_ruling( + evaluate(&baseline, &after, &["LPS".to_string(), "TII".to_string()]), + "D1b", + ); +} + +#[test] +fn mixed_target_improvement_awaits_d1c() { + let mut after = example_after(); + // TII regresses while LPS improves. + after.metrics.insert( + "TII".into(), + serde_json::from_value(serde_json::json!(0.34)).unwrap(), + ); + expect_ruling( + evaluate( + &example_baseline(), + &after, + &["LPS".to_string(), "TII".to_string()], + ), + "D1c", + ); +} + +// --------------------------------------------------------------------------- +// Verdict precedence +// --------------------------------------------------------------------------- + +#[test] +fn reject_null_outranks_all_other_rejects() { + let baseline = example_baseline(); + let mut after = example_after(); + // Target worse, capability destroyed, collateral blown, D_ISA positive. + after.metrics.insert( + "LPS".into(), + serde_json::from_value(serde_json::json!(0.60)).unwrap(), + ); + after.metrics.insert( + "CII".into(), + serde_json::from_value(serde_json::json!(0.90)).unwrap(), + ); + after.probes.passed = 5; + let eval = evaluate(&baseline, &after, &["LPS".to_string()]).unwrap(); + assert_eq!(eval.verdict, Verdict::RejectNull); +} + +#[test] +fn reject_capability_outranks_collateral_and_net() { + let baseline = example_baseline(); + let mut after = example_after(); + after.metrics.insert( + "CII".into(), + serde_json::from_value(serde_json::json!(0.90)).unwrap(), + ); + after.probes.passed = 5; + let eval = evaluate(&baseline, &after, &["LPS".to_string(), "TII".to_string()]).unwrap(); + assert_eq!(eval.verdict, Verdict::RejectCapability); +} + +#[test] +fn clean_improvement_accepts() { + let baseline = example_baseline(); + let mut after = example_after(); + // Remove the CII warning-band regression. + after.metrics.insert( + "CII".into(), + serde_json::from_value(serde_json::json!(0.31)).unwrap(), + ); + let eval = evaluate(&baseline, &after, &["LPS".to_string(), "TII".to_string()]).unwrap(); + assert_eq!(eval.verdict, Verdict::Accept); + assert!(eval.warned_metrics.is_empty()); +} + +// --------------------------------------------------------------------------- +// Frontier invariants +// --------------------------------------------------------------------------- + +fn frontier_eval(after_lps: f64, capability_pass: u32) -> Evaluation { + let baseline = example_baseline(); + let mut after = example_after(); + after.metrics.insert( + "LPS".into(), + serde_json::from_value(serde_json::json!(after_lps)).unwrap(), + ); + // Keep CII clean so verdicts differ only via capability. + after.metrics.insert( + "CII".into(), + serde_json::from_value(serde_json::json!(0.31)).unwrap(), + ); + after.probes.passed = capability_pass; + evaluate(&baseline, &after, &["LPS".to_string(), "TII".to_string()]).unwrap() +} + +#[test] +fn rejected_attempt_with_higher_gap_does_not_advance_frontier() { + let mut record = FrontierRecord::new( + "LPS", + "claude-opus-5", + "2026-09-01T10:30:00Z", + "sha256:6b2f...", + 0.41, + 46.2, + 12.0 / 13.0, + ); + + // Attempt 1: modest accepted improvement. + let a1 = frontier_eval(0.34, 12); + assert!(a1.verdict.advances_frontier()); + record + .append( + "vex-verbosity-compressor", + "strip_filler=true", + &a1, + "sha256:6b2f...", + ) + .unwrap(); + let f1 = record.attempts[0].frontier; + assert!(f1 > 0.0); + + // Attempt 2: much larger gap, but capability-rejected. + let a2 = frontier_eval(0.05, 10); + assert_eq!(a2.verdict, Verdict::RejectCapability); + record + .append( + "vex-verbosity-compressor", + "aggressive=true", + &a2, + "sha256:6b2f...", + ) + .unwrap(); + assert_eq!( + record.attempts[1].frontier, f1, + "a rejected attempt must not advance the frontier" + ); + + // Attempt 3: accepted and better than attempt 1. + let a3 = frontier_eval(0.17, 12); + assert!(a3.verdict.advances_frontier()); + record + .append( + "vex-verbosity-compressor", + "preserve_code=true", + &a3, + "sha256:6b2f...", + ) + .unwrap(); + assert!(record.attempts[2].frontier > f1); + + assert_eq!(record.methods_tried, 3); + assert_eq!(record.methods_rejected, 1); + assert_eq!(record.frontier_final, record.attempts[2].frontier); + + // The record the writer produces must satisfy the validator. + let doc = serde_json::to_value(&record).unwrap(); + let problems = validate_frontier(&doc); + assert!(problems.is_empty(), "written record invalid: {problems:?}"); +} + +#[test] +fn scenario_set_mismatch_is_rejected() { + let mut record = FrontierRecord::new( + "LPS", + "claude-opus-5", + "2026-09-01T10:30:00Z", + "sha256:aaaa", + 0.41, + 46.2, + 12.0 / 13.0, + ); + let a1 = frontier_eval(0.34, 12); + let err = record + .append("vex-verbosity-compressor", "cfg", &a1, "sha256:bbbb") + .unwrap_err(); + assert!(err.to_string().contains("identical")); +} + +// --------------------------------------------------------------------------- +// The validator must reject corrupted documents +// --------------------------------------------------------------------------- + +#[test] +fn validator_catches_frontier_advanced_by_reject() { + let mut doc = frontier_example(); + // Corrupt attempt 2 (reject_capability) to advance the frontier. + doc["attempts"][1]["frontier"] = serde_json::json!(0.780); + let problems = validate_frontier(&doc); + assert!( + problems.iter().any(|p| p.contains("invariant")), + "expected an invariant violation, got: {problems:?}" + ); +} + +#[test] +fn validator_catches_wrong_verdict() { + let mut doc = efficacy_example(); + doc["verdict"] = serde_json::json!("accept"); + let problems = validate_efficacy(&doc); + assert!( + problems.iter().any(|p| p.contains("acceptance rule")), + "expected a verdict mismatch, got: {problems:?}" + ); +} + +#[test] +fn validator_catches_bad_gap_arithmetic() { + let mut doc = efficacy_example(); + doc["target_metrics"]["LPS"]["gap_closed"] = serde_json::json!(0.9); + let problems = validate_efficacy(&doc); + assert!( + problems.iter().any(|p| p.contains("gap_closed")), + "expected a gap_closed mismatch, got: {problems:?}" + ); +} + +#[test] +fn validator_catches_missing_collateral_coverage() { + let mut doc = efficacy_example(); + doc["collateral_metrics"] + .as_object_mut() + .unwrap() + .remove("RCI"); + let problems = validate_efficacy(&doc); + assert!( + problems.iter().any(|p| p.contains("RCI")), + "expected missing-coverage problem, got: {problems:?}" + ); +} From 424a8b27b8d1e4e8f530cdb6c1e718bf9780605e Mon Sep 17 00:00:00 2001 From: "Jonathan D.A. Jewell" <6759885+hyperpolymath@users.noreply.github.com> Date: Tue, 1 Sep 2026 17:07:35 +0100 Subject: [PATCH 2/2] fix(efficacy): replace bare unwrap after push with total error path Resolves the Hypatia code-scanning alert (code_safety/unwrap_without_check, CWE-754) raised on PR #70: the frontier writer returned the just-pushed attempt via .last().unwrap(). Logically infallible, but now expressed as a total ok_or_else in the same style as the surrounding error paths. Co-Authored-By: Claude Fable 5 --- vexometer-efficacy/src/lib.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/vexometer-efficacy/src/lib.rs b/vexometer-efficacy/src/lib.rs index 84b0592..0b55122 100644 --- a/vexometer-efficacy/src/lib.rs +++ b/vexometer-efficacy/src/lib.rs @@ -784,7 +784,9 @@ impl FrontierRecord { .iter() .filter(|a| a.verdict.is_reject()) .count() as u64; - Ok(self.attempts.last().unwrap()) + self.attempts + .last() + .ok_or_else(|| EfficacyError::Data("internal: attempts empty after push".into())) } }