diff --git a/Justfile b/Justfile index edba11d..75dc75c 100644 --- a/Justfile +++ b/Justfile @@ -69,6 +69,10 @@ efficacy-report *ARGS: efficacy-attempt *ARGS: cd vexometer-efficacy && cargo run --release --quiet -- attempt "$@" +# Mechanically lift a vexometer-efficacy-v1 report to v2.1 shape (ruling e2) +efficacy-lift *ARGS: + cd vexometer-efficacy && cargo run --release --quiet -- lift "$@" + # Validate efficacy reports and frontier records by recomputation efficacy-validate *ARGS: cd vexometer-efficacy && cargo run --release --quiet -- validate "$@" diff --git a/lazy-eliminator/.trust/trust-manifest.sha256 b/lazy-eliminator/.trust/trust-manifest.sha256 index 94678d2..f5be2b9 100644 --- a/lazy-eliminator/.trust/trust-manifest.sha256 +++ b/lazy-eliminator/.trust/trust-manifest.sha256 @@ -1,6 +1,6 @@ # trust-manifest v1 # component=lazy-eliminator -# generated_at=2026-09-01T14:22:53Z +# generated_at=2026-09-01T23:29:45Z 339d25795fa89149354d4101c533492f0e5bbe39fc953ac02be17a225f2fd270 README.adoc 6772e621da4e50257728886f568bd652b8e9c58a02aec325112e57def73da46a ROADMAP.adoc 504199ed09a9acbd183fe9c37a8330ec254f33f5612e0f36f2f5e7be170a57ff SECURITY.adoc diff --git a/satellite-template/.trust/trust-manifest.sha256 b/satellite-template/.trust/trust-manifest.sha256 index 27fe34e..4b81e38 100644 --- a/satellite-template/.trust/trust-manifest.sha256 +++ b/satellite-template/.trust/trust-manifest.sha256 @@ -1,6 +1,6 @@ # trust-manifest v1 # component=satellite-template -# generated_at=2026-09-01T14:22:53Z +# generated_at=2026-09-01T23:29:45Z e90437cd512f3ac6824b42e733394bc69dfa4bcab046d779842bc876e11f870e README.adoc 5a62f5611eecfafa43d931b4d7e0917fa1b0f0275fdfd74c61415ab635e72805 ROADMAP.adoc 38ccfdc1a04c12616acfb030522358383702184480f52507a5f72fccccbe76b9 SECURITY.adoc diff --git a/vexometer-efficacy/.trust/trust-manifest.sha256 b/vexometer-efficacy/.trust/trust-manifest.sha256 index 50b2693..a8ed9bc 100644 --- a/vexometer-efficacy/.trust/trust-manifest.sha256 +++ b/vexometer-efficacy/.trust/trust-manifest.sha256 @@ -1,8 +1,8 @@ # trust-manifest v1 # component=vexometer-efficacy -# generated_at=2026-09-01T15:55:49Z -3ea7341c2a55bea766ffa7c34879168001f701982c3427c8f3ff874b4b907c3e README.adoc -99bf8c708656fee9beba0c4812aac55a6fd3b4fdaaa989a9b6a13b7dc3c4b5ba ROADMAP.adoc +# generated_at=2026-09-01T23:29:45Z +df1c5e511fc5e2e7cbcd094fea57728ce193ee267a267037f4043ae5edbf3279 README.adoc +272d7670bb46837c46db61d519d849c60314fecaaf3cd807ee9fdd18ec76ba35 ROADMAP.adoc b1245e468709a6c75e530412da6480943bf53c836df0ca108aaf39843886e6cb SECURITY.adoc 9c80ff2e60fdb772a0479b46b140e0ce08e4e37bc39e6d7e257aa3d5d1281d18 contractiles/must/Mustfile 3ac4606620454d844d8f0d0580fe32072a8a0b6821c93a74df64c3ed597e3640 contractiles/trust/Trustfile.a2ml diff --git a/vexometer-efficacy/README.adoc b/vexometer-efficacy/README.adoc index dfe1b22..026476d 100644 --- a/vexometer-efficacy/README.adoc +++ b/vexometer-efficacy/README.adoc @@ -11,52 +11,65 @@ applies the six-verdict acceptance rule with its precedence order, emits records under the monotone-frontier invariant, and validates both document shapes by recomputing every derived number. -== Design rule: refuse where the protocol is undecided +== The ruled semantics (protocol v2.1) -Six normative questions are open in -https://github.com/hyperpolymath/vexometer/issues/69[issue #69] (debt -item D1). Where one of them bites, this tool *refuses with an explicit -error* naming the question rather than silently picking a semantic: +The six normative questions this tool originally refused to guess at +(https://github.com/hyperpolymath/vexometer/issues/69[issue #69], +D1a-D1f) were ruled `a1, b1, c1, d1, e2, f1` on 2026-09-01 and are +implemented here: -[cols="1,4,2",options="header"] +[cols="1,4,3",options="header"] |=== -|Question |When it bites |Behaviour - -|D1a -|A declared target metric has baseline `B_m = 0` (division by zero in -`G_m`) -|Hard refusal, exit code 2 - -|D1b -|Per-probe results are supplied for both measurements and the aggregate -pass-rate gate disagrees with the per-probe identity gate -|Hard refusal, exit code 2 - -|D1c -|Multiple targets are declared and some improved while others did not -|Hard refusal, exit code 2 - -|D1d -|Multiple targets with the singular `frontier_record` field -|Report is emitted, with a warning on stderr - -|D1e +|Ruling |When it bites |Behaviour + +|a1 +|A declared target metric has baseline `B_m = 0` +|`G_m := 0`, so the target cannot improve and the verdict is +`reject_null`; the report lists the metric in a diagnosability warning. +Zero-baseline *collateral* metrics stay fully protected. + +|b1 +|Per-probe results exist in both measurements +|The per-probe identity gate is normative: at most one baseline-passing +probe may fail after; newly-passing probes buy nothing back. The +aggregate pass-rate gate is the degraded fallback when per-probe results +are absent. + +|c1 +|Multiple targets are declared +|All-targets rule: every declared target must improve, or the verdict is +`reject_null`. + +|d1 +|Frontier references +|Plural `frontier_records`, one per-metric record per target, in target +order; a length mismatch is a hard error and the pre-ruling singular +`frontier_record` key fails validation. + +|e2 |v1→v2 lifting -|Unimplemented — no `lift` subcommand exists +|The `lift` subcommand: v1 fields carried verbatim, missing v2 evidence +as explicit `null`, `lifted_from` marker, `verdict: "unverified"` +(reserved for lifted reports). + +|f1 +|Scenario-set provenance +|`validate --scenario-registry FILE` checks every scored `scenario_set` +against the held-out partition registry at +`../vexometer/data/scenario_sets/registry.json`. |=== -After the rulings land and the protocol is amended to v2.1, these -refusals are replaced by the ruled semantics. - == The protocol's examples are the test fixtures The integration tests read `../vexometer/docs/EFFICACY-PROTOCOL.adoc` at build time, extract its example JSON blocks, and require that the -validator accepts both and that the evaluator reproduces the efficacy +validator accepts them all, that the evaluator reproduces the efficacy example value-for-value from raw inputs (including `D_ISA = -2.71` under the default category weights in -link:../vexometer/docs/METRICS.adoc[METRICS.adoc]). If the protocol and -this implementation drift apart, `cargo test` fails loudly. +link:../vexometer/docs/METRICS.adoc[METRICS.adoc]), and that `lift` +reproduces the protocol's lifted example from its v1 example. If the +protocol and this implementation drift apart, `cargo test` fails +loudly. == CLI @@ -66,7 +79,8 @@ $ vexometer-efficacy report --baseline baseline.json --after after.json \ --targets LPS,TII --satellite vex-verbosity-compressor \ --sample-size 500 --output report.json \ [--methodology "A/B testing with vexometer validation"] \ - [--notes "..."] [--frontier-record frontier/LPS-....json] \ + [--notes "..."] \ + [--frontier-records frontier/LPS-....json]... \ [--traces-available true|false] [--date YYYY-MM-DD] [--scenario-set SHA] $ vexometer-efficacy attempt --frontier frontier/LPS-2026-09-01.json \ @@ -75,18 +89,24 @@ $ vexometer-efficacy attempt --frontier frontier/LPS-2026-09-01.json \ [--model-profile STR] [--timestamp ISO8601] [--scenario-set SHA] \ [--baseline-isa 4.63] # required when creating a new frontier record -$ vexometer-efficacy validate report.json frontier.json ... +$ vexometer-efficacy lift --input v1-report.json --output lifted.json + +$ vexometer-efficacy validate report.json frontier.json ... \ + [--scenario-registry ../vexometer/data/scenario_sets/registry.json] ---- -Bare `validate` arguments are routed by each document's own `version` -field; `--efficacy FILE` / `--frontier FILE` force a kind when a -document lacks one. The same commands are exposed at the monorepo root -as `just efficacy-report`, `just efficacy-attempt`, and -`just efficacy-validate`. +Pass `--frontier-records` once per target metric, in target order +(ruling d1). Bare `validate` arguments are routed by each document's own +`version` field; `--efficacy FILE` / `--frontier FILE` force a kind when +a document lacks one, and `--scenario-registry` enforces ruling f1 +against every scored scenario set. The same commands are exposed at the +monorepo root as `just efficacy-report`, `just efficacy-attempt`, +`just efficacy-lift`, and `just efficacy-validate`. Exit codes: `0` success (any verdict, including rejections — a computed -rejection is a successful evaluation), `1` usage or data error, `2` open -D1 ruling required, `3` validation failed. +rejection is a successful evaluation), `1` usage or data error, `3` +validation failed. (Exit code `2`, the pre-ruling "open D1 question" +refusal, is retired.) == Measurement input format @@ -115,8 +135,9 @@ pass over one content-addressed scenario set: `{score, std_dev, confidence, p_value}` object are both accepted; statistics are carried into the report when present. * `probes.results` (per-probe outcomes) is optional; when both - measurements carry it, the per-probe identity gate is cross-checked - against the aggregate gate (see D1b above). + measurements carry it, the per-probe identity gate is normative + (ruling b1) and the report records any regressed probes in + `capability.probes_regressed`. * `scenario_set` must match between baseline and after — tuning against a different set than you score on is exactly what the protocol's audit trail exists to catch. diff --git a/vexometer-efficacy/ROADMAP.adoc b/vexometer-efficacy/ROADMAP.adoc index 65dfcac..b6564fe 100644 --- a/vexometer-efficacy/ROADMAP.adoc +++ b/vexometer-efficacy/ROADMAP.adoc @@ -11,13 +11,15 @@ * [x] Protocol examples as live test fixtures (drift fails `cargo test`) * [x] Explicit refusals on open D1 questions (issue #69) -== After the D1 rulings (v0.2, blocked on issue #69) +== The D1 rulings (v0.2, shipped — issue #69 ruled `a1,b1,c1,d1,e2,f1`) -* [ ] Replace each D1a–D1d refusal with the ruled semantic -* [ ] `frontier_records` plurality per ruling (d) -* [ ] v1→v2 lifting: implement or formally drop per ruling (e) -* [ ] Held-out scenario-set support per ruling (f) -* [ ] Track the protocol's v2.1 text (same PR as the amendment) +* [x] Replace each D1a–D1d refusal with the ruled semantic (a1 zero + baseline, b1 per-probe identity gate, c1 all-targets rule) +* [x] `frontier_records` plurality per ruling d1 +* [x] v1→v2 lifting: `lift` subcommand per ruling e2 +* [x] Held-out scenario-set registry + `--scenario-registry` per + ruling f1 +* [x] Track the protocol's v2.1 text (same PR as the amendment) == Later diff --git a/vexometer-efficacy/src/lib.rs b/vexometer-efficacy/src/lib.rs index 0b55122..9fc3e86 100644 --- a/vexometer-efficacy/src/lib.rs +++ b/vexometer-efficacy/src/lib.rs @@ -2,20 +2,22 @@ //! Efficacy evaluator for the vexometer ISA efficacy protocol. //! //! Implements the computation and validation halves of -//! `vexometer/docs/EFFICACY-PROTOCOL.adoc`: `G_m`, collateral deltas, -//! `D_ISA`, the capability proxy, the six-verdict acceptance rule with its -//! precedence order, and `vexometer-frontier-v1` record maintenance. +//! `vexometer/docs/EFFICACY-PROTOCOL.adoc` (v2.1): `G_m`, collateral +//! deltas, `D_ISA`, the capability proxy, the six-verdict acceptance rule +//! with its precedence order, `vexometer-frontier-v1` record maintenance, +//! and mechanical v1 -> v2 report lifting. //! -//! Where the protocol is normatively undecided (issue #69, questions -//! D1a-D1d), this crate refuses to guess: it returns an -//! `EfficacyError::AwaitingRuling` naming the open question instead of -//! silently picking a semantic. +//! The six normative questions this crate once refused to guess at +//! (issue #69, D1a-D1f) were ruled a1, b1, c1, d1, e2, f1; the ruled +//! semantics are implemented here and the protocol text is amended to +//! v2.1 in the same change. use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; use std::fmt; -/// The owner-ruling issue batching the six open normative questions. +/// The owner-ruling issue that batched the six normative questions +/// (D1a-D1f), ruled a1, b1, c1, d1, e2, f1 on 2026-09-01. pub const ISSUE_D1: &str = "https://github.com/hyperpolymath/vexometer/issues/69"; /// The ten ISA metrics with their default category weights, in canonical @@ -64,11 +66,6 @@ pub fn round3(x: f64) -> f64 { #[derive(Debug)] pub enum EfficacyError { - /// The computation requires an answer to an open D1 question. - AwaitingRuling { - question: &'static str, - detail: String, - }, /// The input data is malformed or incomplete. Data(String), } @@ -76,9 +73,6 @@ pub enum EfficacyError { impl fmt::Display for EfficacyError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - EfficacyError::AwaitingRuling { question, detail } => { - write!(f, "awaiting ruling {question} (see {ISSUE_D1}): {detail}") - } EfficacyError::Data(msg) => write!(f, "invalid input: {msg}"), } } @@ -141,8 +135,9 @@ pub struct ProbeMeasurement { pub total: u32, pub passed: u32, /// Optional per-probe outcomes, keyed by probe id. When present in - /// both measurements, the identity gate is cross-checked against the - /// aggregate gate (see D1b). + /// both measurements, the per-probe identity gate is normative + /// (ruling b1); without it the aggregate pass-rate gate is the + /// degraded fallback. #[serde(default)] pub results: Option>, } @@ -189,8 +184,9 @@ pub enum Verdict { RejectNet, RejectNull, /// Verification-status sentinel for lifted v1 reports, outside the - /// six-verdict acceptance table. Never emitted by this tool (v1->v2 - /// lifting is unimplemented pending ruling D1e). + /// six-verdict acceptance table. Emitted only by [`lift_v1`] + /// (ruling e2); a report without the `lifted_from` marker must never + /// carry it. Unverified, } @@ -262,6 +258,10 @@ pub struct Evaluation { /// Collateral metrics whose delta falls in the warning band /// `(COLLATERAL_ACCEPT, COLLATERAL_WARN]`. pub warned_metrics: Vec, + /// Targets declared with a zero baseline. Ineligible under ruling + /// a1: `G_m` is defined as 0 there, improvement is impossible at the + /// floor, and the all-targets rule forces `reject_null`. + pub zero_baseline_targets: Vec, } impl Evaluation { @@ -324,35 +324,37 @@ pub fn evaluate( } } - // Targets: G_m. A zero baseline makes G_m undefined -- open question D1a. + // Targets: G_m. A zero baseline leaves no gap to close: G_m is + // defined as 0 there and improvement is impossible at the floor, so + // a zero-baseline metric is ineligible as a target (ruling a1) and + // the all-targets rule below yields reject_null. let mut target_out = BTreeMap::new(); let mut improvements = Vec::new(); + let mut zero_baseline_targets = Vec::new(); for t in targets { let b = baseline.metrics[t].score(); let a_reading = &after.metrics[t]; let a = a_reading.score(); - if b == 0.0 { - return Err(EfficacyError::AwaitingRuling { - question: "D1a", - detail: format!( - "target metric {t} has baseline 0; G_m = (B_m - A_m) / B_m is undefined" - ), - }); - } + let gap_closed = if b == 0.0 { + zero_baseline_targets.push(t.clone()); + 0.0 + } else { + (b - a) / b + }; let (std_dev, confidence, p_value) = a_reading.stats(); target_out.insert( t.clone(), TargetOutcome { baseline: b, after: a, - gap_closed: (b - a) / b, + gap_closed, mean_reduction: b - a, std_dev, confidence, p_value, }, ); - improvements.push(a < b - EPS); + improvements.push(b != 0.0 && a < b - EPS); } // Collateral: every metric outside the target set. @@ -384,16 +386,19 @@ pub fn evaluate( } let isa_delta_raw = num / den * 100.0; - // Capability proxy. The normative table defines the aggregate gate; - // the prose sentence about "two or more probes" implies an identity - // gate. When per-probe data lets both be computed and they disagree, - // that is open question D1b. + // Capability proxy. When per-probe results exist for both + // measurements, the identity gate is normative (ruling b1): at most + // one baseline-passing probe may fail after the intervention, and a + // newly-passing probe cannot buy back a regression. The aggregate + // pass-rate gate (P_after >= P_before - 1/N) is the degraded + // fallback when per-probe results are absent. let tolerance = 1.0 / f64::from(baseline.probes.total); let rate_before = baseline.probes.pass_rate(); let rate_after = after.probes.pass_rate(); let aggregate_ok = rate_after >= rate_before - tolerance - EPS; - let probes_regressed = match (&baseline.probes.results, &after.probes.results) { + let (capability_ok, probes_regressed) = match (&baseline.probes.results, &after.probes.results) + { (Some(before), Some(after_r)) => { if before.keys().ne(after_r.keys()) { return Err(EfficacyError::Data( @@ -405,51 +410,23 @@ pub fn evaluate( .filter(|(id, passed)| **passed && !after_r[*id]) .map(|(id, _)| id.clone()) .collect(); - let identity_ok = regressed.len() <= 1; - if identity_ok != aggregate_ok { - return Err(EfficacyError::AwaitingRuling { - question: "D1b", - detail: format!( - "aggregate pass-rate gate says capability_ok={aggregate_ok} but \ - per-probe identity gate says capability_ok={identity_ok} \ - ({} baseline-passing probes regressed: {})", - regressed.len(), - regressed.join(", ") - ), - }); - } - Some(regressed) + (regressed.len() <= 1, Some(regressed)) } - _ => None, + _ => (aggregate_ok, None), }; let capability = CapabilityOutcome { probes_total: baseline.probes.total, pass_rate_before: rate_before, pass_rate_after: rate_after, - capability_ok: aggregate_ok, + capability_ok, probes_regressed, }; - // Target improvement. All improved / none improved are decidable; a - // mixed outcome needs the multi-target acceptance rule -- open - // question D1c. + // Target improvement, under the all-targets rule (ruling c1): every + // declared target must improve, or the attempt is a null result. A + // partial win is a moved irritation surface, not a shrunk one. let all_improved = improvements.iter().all(|i| *i); - let none_improved = improvements.iter().all(|i| !*i); - if !all_improved && !none_improved { - let detail: Vec = targets - .iter() - .zip(&improvements) - .map(|(t, i)| format!("{t}: {}", if *i { "improved" } else { "not improved" })) - .collect(); - return Err(EfficacyError::AwaitingRuling { - question: "D1c", - detail: format!( - "targets disagree on improvement ({}); the multi-target acceptance rule is undecided", - detail.join(", ") - ), - }); - } let max_collateral = collateral .values() @@ -462,7 +439,7 @@ pub fn evaluate( .collect(); // Acceptance rule with the protocol's precedence order. - let verdict = if none_improved { + let verdict = if !all_improved { Verdict::RejectNull } else if !capability.capability_ok { Verdict::RejectCapability @@ -485,6 +462,7 @@ pub fn evaluate( isa_delta_raw, verdict, warned_metrics, + zero_baseline_targets, }) } @@ -544,8 +522,9 @@ pub struct EfficacyReport { pub verdict_notes: Option, pub methodology: String, pub traces_available: bool, + /// One frontier-record reference per target metric (ruling d1). #[serde(skip_serializing_if = "Option::is_none")] - pub frontier_record: Option, + pub frontier_records: Option>, } /// Report-level metadata supplied by the caller rather than computed. @@ -558,13 +537,13 @@ pub struct ReportMeta { pub methodology: String, pub traces_available: bool, pub verdict_notes: Option, - pub frontier_record: Option, + pub frontier_records: Option>, } /// Assemble a `vexometer-efficacy-v2` report from an evaluation. /// -/// Returns the report plus any non-fatal warnings (currently: the D1d -/// singular-`frontier_record` ambiguity for multi-target reports). +/// Returns the report plus any non-fatal warnings (currently: notes +/// naming zero-baseline targets, which are ineligible under ruling a1). pub fn build_report( eval: &Evaluation, meta: &ReportMeta, @@ -589,14 +568,24 @@ pub fn build_report( } } - if eval.targets.len() > 1 && meta.frontier_record.is_some() { + for m in &eval.zero_baseline_targets { warnings.push(format!( - "frontier_record is a single reference but the report has {} targets; \ - plurality is undecided -- awaiting ruling D1d (see {ISSUE_D1})", - eval.targets.len() + "target {m} has baseline 0 and is ineligible as an efficacy target \ + (ruling a1): G_m is defined as 0 and the verdict is reject_null" )); } + if let Some(records) = &meta.frontier_records { + if records.len() != eval.targets.len() { + return Err(EfficacyError::Data(format!( + "frontier_records must carry one per-metric record per target \ + (ruling d1): {} target(s) but {} record(s)", + eval.targets.len(), + records.len() + ))); + } + } + let target_metrics = eval .targets .iter() @@ -652,11 +641,182 @@ pub fn build_report( verdict_notes: meta.verdict_notes.clone(), methodology: meta.methodology.clone(), traces_available: meta.traces_available, - frontier_record: meta.frontier_record.clone(), + frontier_records: meta.frontier_records.clone(), }; Ok((report, warnings)) } +// --------------------------------------------------------------------------- +// v1 -> v2 lifting (ruling e2) +// --------------------------------------------------------------------------- + +pub const EFFICACY_V1_VERSION: &str = "vexometer-efficacy-v1"; + +/// Mechanically lift a `vexometer-efficacy-v1` report to v2.1 shape +/// (ruling e2). Every v1 field is carried verbatim; every required v2 +/// field whose evidence does not exist in v1 becomes an explicit `null` +/// -- nothing is synthesised. The result carries +/// `"lifted_from": "vexometer-efficacy-v1"` and +/// `"verdict": "unverified"`, and satisfies the lifted branch of +/// [`validate_efficacy`]. +pub fn lift_v1(doc: &serde_json::Value) -> Result { + let obj = doc + .as_object() + .ok_or_else(|| EfficacyError::Data("a v1 report must be a JSON object".into()))?; + match obj.get("version").and_then(|v| v.as_str()) { + Some(EFFICACY_V1_VERSION) => {} + Some(other) => { + return Err(EfficacyError::Data(format!( + "lift takes a {EFFICACY_V1_VERSION} report, got version {other:?}" + ))) + } + None => { + return Err(EfficacyError::Data( + "lift takes a v1 report with a \"version\" field".into(), + )) + } + } + + // The lift domain is exactly the protocol's v1 mapping table. A key + // outside it either collides with a v2 slot the lift must control + // (verdict, capability, ...) or would be dropped silently -- both + // break the carried-verbatim promise, so refuse instead. + const V1_FIELDS: [&str; 7] = [ + "version", + "metrics", + "satellite", + "evaluation_date", + "sample_size", + "methodology", + "traces_available", + ]; + for key in obj.keys() { + if !V1_FIELDS.contains(&key.as_str()) { + return Err(EfficacyError::Data(format!( + "v1 field {key:?} has no v2 mapping: the lift carries v1 fields verbatim and refuses what it cannot carry (ruling e2)" + ))); + } + } + + let metrics = match obj.get("metrics") { + Some(serde_json::Value::Object(m)) => m, + _ => { + return Err(EfficacyError::Data( + "v1 report has no \"metrics\" object to lift".into(), + )) + } + }; + let mut target_metrics = serde_json::Map::new(); + for (name, reading) in metrics { + let serde_json::Value::Object(fields) = reading else { + return Err(EfficacyError::Data(format!( + "v1 metric {name} is not an object" + ))); + }; + // v2 evidence keys cannot pre-exist in a v1 report; letting one + // through would either clobber the explicit null or ship a value + // the lift did not verify, and the failure would only surface at + // a later validate run. + for reserved in ["baseline", "after", "gap_closed"] { + if fields.contains_key(reserved) { + return Err(EfficacyError::Data(format!( + "v1 metric {name} already carries {reserved:?}: v2 evidence cannot pre-exist in a v1 report (ruling e2)" + ))); + } + } + let mut lifted = serde_json::Map::new(); + // The v2 fields with no v1 evidence: explicit null, never + // synthesised. The v1 sub-fields (mean_reduction, std_dev, + // confidence, p_value) then carry over verbatim. + lifted.insert("baseline".into(), serde_json::Value::Null); + lifted.insert("after".into(), serde_json::Value::Null); + lifted.insert("gap_closed".into(), serde_json::Value::Null); + for (k, v) in fields { + lifted.insert(k.clone(), v.clone()); + } + target_metrics.insert(name.clone(), serde_json::Value::Object(lifted)); + } + + let carried = |key: &str| obj.get(key).cloned().unwrap_or(serde_json::Value::Null); + let mut out = serde_json::Map::new(); + out.insert("version".into(), serde_json::json!(EFFICACY_VERSION)); + out.insert("lifted_from".into(), serde_json::json!(EFFICACY_V1_VERSION)); + out.insert("satellite".into(), carried("satellite")); + out.insert("evaluation_date".into(), carried("evaluation_date")); + out.insert("sample_size".into(), carried("sample_size")); + out.insert("scenario_set".into(), serde_json::Value::Null); + out.insert( + "target_metrics".into(), + serde_json::Value::Object(target_metrics), + ); + out.insert("collateral_metrics".into(), serde_json::Value::Null); + out.insert("capability".into(), serde_json::Value::Null); + out.insert("isa_delta".into(), serde_json::Value::Null); + out.insert("verdict".into(), serde_json::json!("unverified")); + out.insert("methodology".into(), carried("methodology")); + out.insert("traces_available".into(), carried("traces_available")); + Ok(serde_json::Value::Object(out)) +} + +// --------------------------------------------------------------------------- +// Held-out scenario partition registry (ruling f1) +// --------------------------------------------------------------------------- + +pub const SCENARIO_REGISTRY_VERSION: &str = "vexometer-scenario-registry-v1"; + +#[derive(Debug, Clone, Deserialize)] +struct ScenarioPartition { + name: String, + tuning_set: String, + held_out_set: String, +} + +#[derive(Debug, Clone, Deserialize)] +struct ScenarioRegistry { + version: String, + partitions: Vec, +} + +/// Check a scored `scenario_set` against the held-out partition registry +/// (`vexometer/data/scenario_sets/registry.json`, ruling f1). Scoring +/// must use a registered held-out hash: a tuning hash is always a +/// violation, and once any partition is registered an unrecognised hash +/// is too. An empty registry (no corpus yet) enforces nothing. +pub fn check_scenario_registry(registry: &serde_json::Value, scenario_set: &str) -> Vec { + let reg: ScenarioRegistry = match serde_json::from_value(registry.clone()) { + Ok(r) => r, + Err(e) => { + return vec![format!( + "does not parse as {SCENARIO_REGISTRY_VERSION}: {e}" + )] + } + }; + let mut problems = Vec::new(); + if reg.version != SCENARIO_REGISTRY_VERSION { + problems.push(format!( + "registry version is {:?}, expected {SCENARIO_REGISTRY_VERSION:?}", + reg.version + )); + } + if let Some(p) = reg.partitions.iter().find(|p| p.tuning_set == scenario_set) { + problems.push(format!( + "scenario_set {scenario_set} is the TUNING partition of {:?}; scoring must \ + use its held-out partition {} (ruling f1)", + p.name, p.held_out_set + )); + } else if !reg.partitions.is_empty() + && !reg + .partitions + .iter() + .any(|p| p.held_out_set == scenario_set) + { + problems.push(format!( + "scenario_set {scenario_set} matches no registered held-out partition (ruling f1)" + )); + } + problems +} + // --------------------------------------------------------------------------- // vexometer-frontier-v1 records // --------------------------------------------------------------------------- @@ -800,7 +960,15 @@ fn known_metric(m: &str) -> bool { /// Validate a `vexometer-efficacy-v2` document. Returns a list of /// problems; an empty list means the document is valid. +/// +/// A document carrying the `lifted_from` marker is routed to the lifted +/// branch (ruling e2), which checks the lift contract instead of +/// recomputing evidence that does not exist. pub fn validate_efficacy(doc: &serde_json::Value) -> Vec { + if doc.get("lifted_from").is_some() { + return validate_lifted(doc); + } + let mut problems = Vec::new(); let report: EfficacyReport = match serde_json::from_value(doc.clone()) { Ok(r) => r, @@ -819,6 +987,31 @@ pub fn validate_efficacy(doc: &serde_json::Value) -> Vec { if report.target_metrics.is_empty() { problems.push("target_metrics is empty".into()); } + if doc.get("frontier_record").is_some() { + problems.push( + "frontier_record is the pre-ruling singular field; v2.1 uses \ + frontier_records, one per target metric (ruling d1)" + .into(), + ); + } + if let Some(records) = &report.frontier_records { + if records.len() != report.target_metrics.len() { + problems.push(format!( + "frontier_records has {} entr{} for {} target metric(s); ruling d1 \ + requires one per-metric record per target", + records.len(), + if records.len() == 1 { "y" } else { "ies" }, + report.target_metrics.len() + )); + } + } + if report.verdict == Verdict::Unverified { + problems.push( + "verdict \"unverified\" is reserved for lifted v1 reports carrying the \ + lifted_from marker (ruling e2)" + .into(), + ); + } // Coverage: targets and collateral must partition the ten metrics. for m in report.target_metrics.keys() { @@ -846,18 +1039,23 @@ pub fn validate_efficacy(doc: &serde_json::Value) -> Vec { // Arithmetic: recompute each stored figure from its own raw values. for (m, t) in &report.target_metrics { if t.baseline == 0.0 { - problems.push(format!( - "target {m} has baseline 0: gap_closed is undefined (awaiting ruling D1a, {ISSUE_D1})" - )); - continue; - } - let gap = (t.baseline - t.after) / t.baseline; - if (round3(gap) - t.gap_closed).abs() > 0.0005 + EPS { - problems.push(format!( - "target {m}: gap_closed {} does not match (baseline - after) / baseline = {}", - t.gap_closed, - round3(gap) - )); + // Ruling a1: G_m is defined as 0 when the baseline is 0. + if t.gap_closed != 0.0 { + problems.push(format!( + "target {m}: baseline is 0, so gap_closed is 0 by definition \ + (ruling a1), not {}", + t.gap_closed + )); + } + } else { + let gap = (t.baseline - t.after) / t.baseline; + if (round3(gap) - t.gap_closed).abs() > 0.0005 + EPS { + problems.push(format!( + "target {m}: gap_closed {} does not match (baseline - after) / baseline = {}", + t.gap_closed, + round3(gap) + )); + } } if (round2(t.baseline - t.after) - t.mean_reduction).abs() > 0.005 + EPS { problems.push(format!( @@ -903,85 +1101,78 @@ pub fn validate_efficacy(doc: &serde_json::Value) -> Vec { } } - // Capability gate consistency (aggregate form, per the normative table). + // Capability gate consistency. The per-probe identity gate is + // normative when probes_regressed is recorded (ruling b1); the + // aggregate pass-rate gate is the degraded fallback. if report.capability.probes_total == 0 { problems.push("capability.probes_total must be > 0".into()); } else { - let tol = 1.0 / f64::from(report.capability.probes_total); - let ok = - report.capability.pass_rate_after >= report.capability.pass_rate_before - tol - EPS; + let (gate, ok) = match &report.capability.probes_regressed { + Some(regressed) => ("per-probe identity", regressed.len() <= 1), + None => { + let tol = 1.0 / f64::from(report.capability.probes_total); + let ok = report.capability.pass_rate_after + >= report.capability.pass_rate_before - tol - EPS; + ("aggregate pass-rate", ok) + } + }; if ok != report.capability.capability_ok { problems.push(format!( - "capability_ok is {} but pass rates {} -> {} with tolerance 1/{} imply {}", - report.capability.capability_ok, - report.capability.pass_rate_before, - report.capability.pass_rate_after, - report.capability.probes_total, - ok + "capability_ok is {} but the {gate} gate implies {ok}", + report.capability.capability_ok )); } } - // Verdict recomputation (skipped for the lifted-report sentinel). - if report.verdict != Verdict::Unverified && problems.is_empty() { - let improved: Vec = report + // Verdict recomputation (only once the figures themselves check out). + if problems.is_empty() { + // All-targets rule (ruling c1): every declared target must + // improve, or the verdict is reject_null. A zero-baseline target + // (ruling a1) cannot improve and forces it. + let all_improved = report .target_metrics .values() - .map(|t| t.after < t.baseline - EPS) + .all(|t| t.baseline != 0.0 && t.after < t.baseline - EPS); + let max_c = report + .collateral_metrics + .values() + .map(|c| c.delta) + .fold(f64::NEG_INFINITY, f64::max); + let warned: Vec<&String> = report + .collateral_metrics + .iter() + .filter(|(_, c)| c.delta > COLLATERAL_ACCEPT + EPS && c.delta <= COLLATERAL_WARN + EPS) + .map(|(m, _)| m) .collect(); - let all = improved.iter().all(|i| *i); - let none = improved.iter().all(|i| !*i); - if !all && !none { + let expected = if !all_improved { + Verdict::RejectNull + } else if !report.capability.capability_ok { + Verdict::RejectCapability + } else if !report.collateral_metrics.is_empty() && max_c > COLLATERAL_WARN + EPS { + Verdict::RejectCollateral + } else if report.isa_delta >= 0.0 { + Verdict::RejectNet + } else if !warned.is_empty() { + Verdict::AcceptWithWarning + } else { + Verdict::Accept + }; + if expected != report.verdict { problems.push(format!( - "targets disagree on improvement; the multi-target acceptance rule is \ - undecided (awaiting ruling D1c, {ISSUE_D1})" + "verdict is {} but the acceptance rule implies {}", + report.verdict.as_str(), + expected.as_str() )); - } else { - let max_c = report - .collateral_metrics - .values() - .map(|c| c.delta) - .fold(f64::NEG_INFINITY, f64::max); - let warned: Vec<&String> = report - .collateral_metrics - .iter() - .filter(|(_, c)| { - c.delta > COLLATERAL_ACCEPT + EPS && c.delta <= COLLATERAL_WARN + EPS - }) - .map(|(m, _)| m) - .collect(); - let expected = if none { - Verdict::RejectNull - } else if !report.capability.capability_ok { - Verdict::RejectCapability - } else if !report.collateral_metrics.is_empty() && max_c > COLLATERAL_WARN + EPS { - Verdict::RejectCollateral - } else if report.isa_delta >= 0.0 { - Verdict::RejectNet - } else if !warned.is_empty() { - Verdict::AcceptWithWarning - } else { - Verdict::Accept - }; - if expected != report.verdict { - problems.push(format!( - "verdict is {} but the acceptance rule implies {}", - report.verdict.as_str(), - expected.as_str() - )); - } - if report.verdict == Verdict::AcceptWithWarning { - match &report.verdict_notes { - None => problems.push( - "accept_with_warning requires verdict_notes naming the regressed metric" - .into(), - ), - Some(notes) => { - for m in warned { - if !notes.contains(m.as_str()) { - problems - .push(format!("verdict_notes must name regressed metric {m}")); - } + } + if report.verdict == Verdict::AcceptWithWarning { + match &report.verdict_notes { + None => problems.push( + "accept_with_warning requires verdict_notes naming the regressed metric".into(), + ), + Some(notes) => { + for m in warned { + if !notes.contains(m.as_str()) { + problems.push(format!("verdict_notes must name regressed metric {m}")); } } } @@ -992,6 +1183,74 @@ pub fn validate_efficacy(doc: &serde_json::Value) -> Vec { problems } +/// Validate a lifted (v1-origin) efficacy document against the lift +/// contract of ruling e2: the `lifted_from` marker, the `unverified` +/// verdict, and an explicit `null` for every v2 field whose evidence +/// does not exist in v1. +fn validate_lifted(doc: &serde_json::Value) -> Vec { + let Some(obj) = doc.as_object() else { + return vec!["lifted report must be a JSON object".into()]; + }; + let mut problems = Vec::new(); + if obj.get("lifted_from").and_then(|v| v.as_str()) != Some(EFFICACY_V1_VERSION) { + problems.push(format!("lifted_from must be {EFFICACY_V1_VERSION:?}")); + } + if obj.get("version").and_then(|v| v.as_str()) != Some(EFFICACY_VERSION) { + problems.push(format!("version must be {EFFICACY_VERSION:?}")); + } + if obj.get("verdict").and_then(|v| v.as_str()) != Some("unverified") { + problems.push( + "a lifted report's verdict must be \"unverified\": the collateral and \ + capability evidence needed for a real verdict does not exist in v1" + .into(), + ); + } + for key in [ + "scenario_set", + "collateral_metrics", + "capability", + "isa_delta", + ] { + match obj.get(key) { + Some(serde_json::Value::Null) => {} + Some(_) => problems.push(format!( + "{key} must be an explicit null in a lifted report: v1 carries no such \ + evidence and nothing may be synthesised (ruling e2)" + )), + None => problems.push(format!( + "{key} must be present as an explicit null in a lifted report (ruling e2)" + )), + } + } + match obj.get("target_metrics") { + Some(serde_json::Value::Object(metrics)) if !metrics.is_empty() => { + for (m, reading) in metrics { + if !known_metric(m) { + problems.push(format!("unknown target metric {m}")); + } + let Some(fields) = reading.as_object() else { + problems.push(format!("target {m} is not an object")); + continue; + }; + for key in ["baseline", "after", "gap_closed"] { + if !matches!(fields.get(key), Some(serde_json::Value::Null)) { + problems.push(format!( + "target {m}: {key} must be an explicit null in a lifted \ + report (ruling e2)" + )); + } + } + } + } + _ => problems.push("target_metrics must be a non-empty object".into()), + } + if obj.get("frontier_record").is_some() || obj.get("frontier_records").is_some() { + problems + .push("a lifted report cannot reference a frontier record: none existed in v1".into()); + } + problems +} + /// Validate a `vexometer-frontier-v1` document. Returns a list of /// problems; an empty list means the document is valid. pub fn validate_frontier(doc: &serde_json::Value) -> Vec { diff --git a/vexometer-efficacy/src/main.rs b/vexometer-efficacy/src/main.rs index a175af1..1ecee2b 100644 --- a/vexometer-efficacy/src/main.rs +++ b/vexometer-efficacy/src/main.rs @@ -4,6 +4,7 @@ //! Subcommands: //! report Evaluate baseline vs after and emit a vexometer-efficacy-v2 JSON report //! attempt Evaluate one configuration and append it to a vexometer-frontier-v1 record +//! lift Mechanically lift a v1 efficacy report to the v2.1 shape (ruling e2) //! validate Check stored efficacy/frontier documents against the protocol's rules use std::collections::BTreeMap; @@ -11,8 +12,8 @@ use std::fs; use std::process::ExitCode; use vexometer_efficacy::{ - build_report, evaluate, validate_efficacy, validate_frontier, EfficacyError, FrontierRecord, - Measurement, ReportMeta, EFFICACY_VERSION, FRONTIER_VERSION, + build_report, check_scenario_registry, evaluate, lift_v1, validate_efficacy, validate_frontier, + FrontierRecord, Measurement, ReportMeta, EFFICACY_VERSION, FRONTIER_VERSION, }; const USAGE: &str = "\ @@ -21,19 +22,25 @@ vexometer-efficacy — ISA efficacy protocol tooling USAGE: vexometer-efficacy report --baseline FILE --after FILE --targets M1[,M2...] --satellite NAME --sample-size N [--scenario-set SHA] [--date YYYY-MM-DD] - [--methodology STR] [--notes STR] [--frontier-record PATH] + [--methodology STR] [--notes STR] [--frontier-records PATH]... [--traces-available true|false] --output FILE vexometer-efficacy attempt --baseline FILE --after FILE --targets M1[,M2...] --metric M --satellite NAME --config STR --frontier FILE [--model-profile STR] [--timestamp ISO8601] [--scenario-set SHA] + vexometer-efficacy lift --input FILE --output FILE + (vexometer-efficacy-v1 in, lifted v2 report out, verdict unverified; + ruling e2) + vexometer-efficacy validate FILE... [--efficacy FILE]... [--frontier FILE]... + [--scenario-registry FILE] (bare FILEs are routed by their \"version\" field; the flags force a kind) Measurement FILEs hold all ten ISA metric scores plus the probe result; see -vexometer-efficacy/README.adoc for the format. Exit codes: 0 success (any -verdict), 1 usage or data error, 2 open D1 ruling, 3 validation failed. +vexometer-efficacy/README.adoc for the format. Pass --frontier-records once +per target metric, in target order (ruling d1). Exit codes: 0 success (any +verdict), 1 usage or data error, 3 validation failed. "; struct Args { @@ -144,15 +151,16 @@ fn cmd_report(args: &Args) -> Result { } }; - let eval = match evaluate(&baseline, &after, &targets) { - Ok(e) => e, - Err(e @ EfficacyError::AwaitingRuling { .. }) => { - eprintln!("error: {e}"); - return Ok(ExitCode::from(2)); - } - Err(e) => return Err(e.to_string()), - }; + let eval = evaluate(&baseline, &after, &targets).map_err(|e| e.to_string())?; + if args.opt("frontier-record")?.is_some() { + return Err( + "--frontier-record was renamed --frontier-records: pass it once per \ + target metric, in target order (ruling d1)" + .to_string(), + ); + } + let frontier_records = args.many("frontier-records"); let meta = ReportMeta { satellite: args.one("satellite")?.to_string(), evaluation_date: args.opt("date")?.map(str::to_string).unwrap_or_else(today), @@ -164,7 +172,11 @@ fn cmd_report(args: &Args) -> Result { .to_string(), traces_available, verdict_notes: args.opt("notes")?.map(str::to_string), - frontier_record: args.opt("frontier-record")?.map(str::to_string), + frontier_records: if frontier_records.is_empty() { + None + } else { + Some(frontier_records) + }, }; let (report, warnings) = build_report(&eval, &meta).map_err(|e| e.to_string())?; @@ -198,14 +210,7 @@ fn cmd_attempt(args: &Args) -> Result { let frontier_path = args.one("frontier")?; let scenario_set = scenario_set_for(args, &baseline)?; - let eval = match evaluate(&baseline, &after, &targets) { - Ok(e) => e, - Err(e @ EfficacyError::AwaitingRuling { .. }) => { - eprintln!("error: {e}"); - return Ok(ExitCode::from(2)); - } - Err(e) => return Err(e.to_string()), - }; + let eval = evaluate(&baseline, &after, &targets).map_err(|e| e.to_string())?; let mut record = if fs::metadata(frontier_path).is_ok() { let doc = read_json(frontier_path)?; @@ -262,12 +267,27 @@ fn cmd_attempt(args: &Args) -> Result { Ok(ExitCode::SUCCESS) } +fn cmd_lift(args: &Args) -> Result { + args.no_positional("lift")?; + let input = args.one("input")?; + let doc = read_json(input)?; + let lifted = lift_v1(&doc).map_err(|e| e.to_string())?; + let output = args.one("output")?; + write_json(output, &lifted)?; + println!("lifted {input} -> {output} (verdict unverified; ruling e2)"); + Ok(ExitCode::SUCCESS) +} + fn cmd_validate(args: &Args) -> Result { let efficacy = args.many("efficacy"); let frontier = args.many("frontier"); if efficacy.is_empty() && frontier.is_empty() && args.positional.is_empty() { return Err("validate needs at least one file to check".to_string()); } + let registry = match args.opt("scenario-registry")? { + Some(path) => Some(read_json(path)?), + None => None, + }; let mut failed = false; for path in &args.positional { // A bare path is routed by the document's own version discriminant, @@ -275,11 +295,13 @@ fn cmd_validate(args: &Args) -> Result { let doc = read_json(path)?; match doc.get("version").and_then(|v| v.as_str()) { Some(v) if v == EFFICACY_VERSION => { - let problems = validate_efficacy(&doc); + let mut problems = validate_efficacy(&doc); + problems.extend(registry_problems(registry.as_ref(), &doc)); report_problems(path, EFFICACY_VERSION, &problems, &mut failed); } Some(v) if v == FRONTIER_VERSION => { - let problems = validate_frontier(&doc); + let mut problems = validate_frontier(&doc); + problems.extend(registry_problems(registry.as_ref(), &doc)); report_problems(path, FRONTIER_VERSION, &problems, &mut failed); } Some(other) => { @@ -295,11 +317,15 @@ fn cmd_validate(args: &Args) -> Result { } } for path in &efficacy { - let problems = validate_efficacy(&read_json(path)?); + let doc = read_json(path)?; + let mut problems = validate_efficacy(&doc); + problems.extend(registry_problems(registry.as_ref(), &doc)); report_problems(path, "vexometer-efficacy-v2", &problems, &mut failed); } for path in &frontier { - let problems = validate_frontier(&read_json(path)?); + let doc = read_json(path)?; + let mut problems = validate_frontier(&doc); + problems.extend(registry_problems(registry.as_ref(), &doc)); report_problems(path, "vexometer-frontier-v1", &problems, &mut failed); } Ok(if failed { @@ -309,6 +335,31 @@ fn cmd_validate(args: &Args) -> Result { }) } +/// Check every scenario-set hash a document scores against the held-out +/// registry (ruling f1). Efficacy reports carry one top-level hash; frontier +/// records carry one per attempt as well. +fn registry_problems(registry: Option<&serde_json::Value>, doc: &serde_json::Value) -> Vec { + let Some(registry) = registry else { + return Vec::new(); + }; + let mut sets: Vec<&str> = Vec::new(); + if let Some(s) = doc.get("scenario_set").and_then(|v| v.as_str()) { + sets.push(s); + } + if let Some(attempts) = doc.get("attempts").and_then(|v| v.as_array()) { + sets.extend( + attempts + .iter() + .filter_map(|a| a.get("scenario_set").and_then(|v| v.as_str())), + ); + } + sets.sort_unstable(); + sets.dedup(); + sets.iter() + .flat_map(|s| check_scenario_registry(registry, s)) + .collect() +} + fn report_problems(path: &str, kind: &str, problems: &[String], failed: &mut bool) { if problems.is_empty() { println!("{path}: valid {kind}"); @@ -376,6 +427,7 @@ fn main() -> ExitCode { match cmd { "report" => cmd_report(&args), "attempt" => cmd_attempt(&args), + "lift" => cmd_lift(&args), "validate" => cmd_validate(&args), "--help" | "-h" | "help" => { print!("{USAGE}"); @@ -389,11 +441,7 @@ fn main() -> ExitCode { Err(msg) => { eprintln!("error: {msg}"); eprintln!("run vexometer-efficacy --help for usage"); - ExitCode::from(if msg.contains("awaiting ruling") { - 2 - } else { - 1 - }) + ExitCode::from(1) } } } diff --git a/vexometer-efficacy/tests/protocol_examples.rs b/vexometer-efficacy/tests/protocol_examples.rs index 4a57aef..49ef7d2 100644 --- a/vexometer-efficacy/tests/protocol_examples.rs +++ b/vexometer-efficacy/tests/protocol_examples.rs @@ -2,9 +2,10 @@ //! The efficacy protocol's own example documents are the fixtures. //! //! These tests read `vexometer/docs/EFFICACY-PROTOCOL.adoc`, extract its -//! `vexometer-efficacy-v2` and `vexometer-frontier-v1` example JSON blocks, -//! and require that (a) the validator accepts both, and (b) the evaluator -//! reproduces the efficacy example byte-for-value from its raw inputs. +//! example JSON blocks (efficacy-v2, lifted-v2, efficacy-v1, frontier-v1), +//! and require that (a) the validator accepts them, (b) the evaluator +//! reproduces the efficacy example byte-for-value from its raw inputs, and +//! (c) the v1 lift reproduces the lifted example from the v1 example. //! If the protocol's examples and this implementation ever drift apart, //! these tests fail loudly. @@ -17,12 +18,14 @@ fn protocol_path() -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../vexometer/docs/EFFICACY-PROTOCOL.adoc") } -/// Extract every `----`-delimited block that parses as JSON, keyed by its -/// `version` field. -fn protocol_examples() -> BTreeMap { +/// Extract every `----`-delimited block that parses as JSON and carries a +/// `version` field. Returned in document order; several blocks can share a +/// version (the main v2 example and the lifted one), so selection happens +/// in the helpers below, not by version key. +fn protocol_examples() -> Vec { let text = std::fs::read_to_string(protocol_path()) .expect("EFFICACY-PROTOCOL.adoc must be readable from the monorepo layout"); - let mut examples = BTreeMap::new(); + let mut examples = Vec::new(); let mut block: Option = None; for line in text.lines() { if line.trim_end() == "----" { @@ -30,8 +33,8 @@ fn protocol_examples() -> BTreeMap { None => block = Some(String::new()), Some(content) => { if let Ok(v) = serde_json::from_str::(&content) { - if let Some(version) = v.get("version").and_then(|s| s.as_str()) { - examples.insert(version.to_string(), v); + if v.get("version").and_then(|s| s.as_str()).is_some() { + examples.push(v); } } } @@ -44,16 +47,54 @@ fn protocol_examples() -> BTreeMap { examples } +fn only(matching: Vec, what: &str) -> serde_json::Value { + assert_eq!( + matching.len(), + 1, + "protocol must contain exactly one {what} example, found {}", + matching.len() + ); + matching.into_iter().next().unwrap() +} + +/// The main v2 example: version v2, no `lifted_from` marker. fn efficacy_example() -> serde_json::Value { - protocol_examples() - .remove(EFFICACY_VERSION) - .expect("protocol must contain a vexometer-efficacy-v2 example") + let matching = protocol_examples() + .into_iter() + .filter(|v| { + v.get("version").and_then(|s| s.as_str()) == Some(EFFICACY_VERSION) + && v.get("lifted_from").is_none() + }) + .collect(); + only(matching, "native vexometer-efficacy-v2") +} + +/// The lifted example: version v2 with the `lifted_from` marker (ruling e2). +fn lifted_example() -> serde_json::Value { + let matching = protocol_examples() + .into_iter() + .filter(|v| { + v.get("version").and_then(|s| s.as_str()) == Some(EFFICACY_VERSION) + && v.get("lifted_from").is_some() + }) + .collect(); + only(matching, "lifted vexometer-efficacy-v2") +} + +fn v1_example() -> serde_json::Value { + let matching = protocol_examples() + .into_iter() + .filter(|v| v.get("version").and_then(|s| s.as_str()) == Some(EFFICACY_V1_VERSION)) + .collect(); + only(matching, "vexometer-efficacy-v1") } fn frontier_example() -> serde_json::Value { - protocol_examples() - .remove(FRONTIER_VERSION) - .expect("protocol must contain a vexometer-frontier-v1 example") + let matching = protocol_examples() + .into_iter() + .filter(|v| v.get("version").and_then(|s| s.as_str()) == Some(FRONTIER_VERSION)) + .collect(); + only(matching, "vexometer-frontier-v1") } // --------------------------------------------------------------------------- @@ -70,6 +111,16 @@ fn protocol_efficacy_example_is_valid() { ); } +#[test] +fn protocol_lifted_example_is_valid() { + let problems = validate_efficacy(&lifted_example()); + assert!( + problems.is_empty(), + "the protocol's own lifted example failed validation:\n{}", + problems.join("\n") + ); +} + #[test] fn protocol_frontier_example_is_valid() { let problems = validate_frontier(&frontier_example()); @@ -137,16 +188,8 @@ fn example_after() -> Measurement { ) } -#[test] -fn evaluator_reproduces_protocol_efficacy_example() { - let targets = vec!["LPS".to_string(), "TII".to_string()]; - let eval = evaluate(&example_baseline(), &example_after(), &targets) - .expect("the protocol example inputs must evaluate cleanly"); - - assert_eq!(eval.verdict, Verdict::AcceptWithWarning); - assert_eq!(eval.warned_metrics, vec!["CII".to_string()]); - - let meta = ReportMeta { +fn example_meta() -> ReportMeta { + ReportMeta { satellite: "vex-verbosity-compressor".into(), evaluation_date: "2026-09-01".into(), sample_size: 500, @@ -158,15 +201,26 @@ fn evaluator_reproduces_protocol_efficacy_example() { scenarios. Must be declared in satellite README." .into(), ), - frontier_record: Some("frontier/LPS-2026-09-01.json".into()), - }; - let (report, warnings) = build_report(&eval, &meta).expect("report must build"); + frontier_records: Some(vec![ + "frontier/LPS-2026-09-01.json".into(), + "frontier/TII-2026-09-01.json".into(), + ]), + } +} + +#[test] +fn evaluator_reproduces_protocol_efficacy_example() { + let targets = vec!["LPS".to_string(), "TII".to_string()]; + let eval = evaluate(&example_baseline(), &example_after(), &targets) + .expect("the protocol example inputs must evaluate cleanly"); - // Two targets with a singular frontier_record is exactly the D1d - // ambiguity; the tool must surface it as a warning, not guess. + assert_eq!(eval.verdict, Verdict::AcceptWithWarning); + assert_eq!(eval.warned_metrics, vec!["CII".to_string()]); + + let (report, warnings) = build_report(&eval, &example_meta()).expect("report must build"); assert!( - warnings.iter().any(|w| w.contains("D1d")), - "expected a D1d plurality warning, got: {warnings:?}" + warnings.is_empty(), + "a well-formed two-target report must emit no warnings, got: {warnings:?}" ); let produced = serde_json::to_value(&report).expect("report must serialise"); @@ -182,37 +236,46 @@ fn evaluator_reproduces_protocol_efficacy_example() { } // --------------------------------------------------------------------------- -// Open D1 questions must be refusals, not guesses +// Ruling a1: zero-baseline targets are ineligible, not errors // --------------------------------------------------------------------------- -fn expect_ruling(result: Result, question: &str) { - match result { - Err(EfficacyError::AwaitingRuling { question: q, .. }) => assert_eq!(q, question), - other => panic!("expected AwaitingRuling({question}), got {other:?}"), - } -} - #[test] -fn zero_baseline_target_awaits_d1a() { +fn zero_baseline_target_is_reject_null() { let mut baseline = example_baseline(); baseline.metrics.insert( "LPS".into(), serde_json::from_value(serde_json::json!(0.0)).unwrap(), ); - expect_ruling( - evaluate(&baseline, &example_after(), &["LPS".to_string()]), - "D1a", + let eval = evaluate(&baseline, &example_after(), &["LPS".to_string()]) + .expect("a zero-baseline target must evaluate, not error (ruling a1)"); + assert_eq!(eval.verdict, Verdict::RejectNull); + assert_eq!(eval.zero_baseline_targets, vec!["LPS".to_string()]); + assert_eq!(eval.targets["LPS"].gap_closed, 0.0); + + // The report surfaces the design error as a diagnosable warning. + let mut meta = example_meta(); + meta.verdict_notes = None; + meta.frontier_records = None; + let (report, warnings) = build_report(&eval, &meta).expect("report must build"); + assert_eq!(report.verdict, Verdict::RejectNull); + assert!( + warnings.iter().any(|w| w.contains("ruling a1")), + "expected a ruling-a1 ineligibility warning, got: {warnings:?}" ); } +// --------------------------------------------------------------------------- +// Ruling b1: the per-probe identity gate is normative +// --------------------------------------------------------------------------- + #[test] -fn probe_gate_disagreement_awaits_d1b() { +fn probe_identity_gate_outvotes_aggregate_rate() { let mut ids: Vec = (1..=13).map(|i| format!("P{i:02}")).collect(); ids.sort(); let before: BTreeMap = ids.iter().map(|id| (id.clone(), id != "P13")).collect(); // Two baseline-passing probes regress, one baseline-failing probe now - // passes: aggregate rate drops by exactly one probe (gate passes) while - // the identity gate counts two regressions (gate fails). + // passes: aggregate rate drops by exactly one probe (the fallback gate + // would pass) while the identity gate counts two regressions and fails. let after_r: BTreeMap = ids .iter() .map(|id| { @@ -231,27 +294,214 @@ fn probe_gate_disagreement_awaits_d1b() { after.probes.passed = 11; after.probes.results = Some(after_r); - expect_ruling( - evaluate(&baseline, &after, &["LPS".to_string(), "TII".to_string()]), - "D1b", + let eval = evaluate(&baseline, &after, &["LPS".to_string(), "TII".to_string()]) + .expect("per-probe disagreement must evaluate, not error (ruling b1)"); + assert!(!eval.capability.capability_ok); + assert_eq!( + eval.capability.probes_regressed, + Some(vec!["P01".to_string(), "P02".to_string()]) ); + // Both targets improved, so the capability gate decides the verdict. + assert_eq!(eval.verdict, Verdict::RejectCapability); } +// --------------------------------------------------------------------------- +// Ruling c1: every declared target must improve +// --------------------------------------------------------------------------- + #[test] -fn mixed_target_improvement_awaits_d1c() { +fn mixed_target_improvement_is_reject_null() { let mut after = example_after(); // TII regresses while LPS improves. after.metrics.insert( "TII".into(), serde_json::from_value(serde_json::json!(0.34)).unwrap(), ); - expect_ruling( - evaluate( - &example_baseline(), - &after, - &["LPS".to_string(), "TII".to_string()], - ), - "D1c", + let eval = evaluate( + &example_baseline(), + &after, + &["LPS".to_string(), "TII".to_string()], + ) + .expect("mixed improvement must evaluate, not error (ruling c1)"); + assert_eq!(eval.verdict, Verdict::RejectNull); +} + +// --------------------------------------------------------------------------- +// Ruling d1: one frontier record per target metric +// --------------------------------------------------------------------------- + +#[test] +fn frontier_records_length_mismatch_is_a_hard_error() { + let targets = vec!["LPS".to_string(), "TII".to_string()]; + let eval = evaluate(&example_baseline(), &example_after(), &targets).unwrap(); + let mut meta = example_meta(); + meta.frontier_records = Some(vec!["frontier/LPS-2026-09-01.json".into()]); + let err = build_report(&eval, &meta).unwrap_err(); + assert!( + err.to_string().contains("ruling d1"), + "expected a ruling-d1 error, got: {err}" + ); +} + +#[test] +fn validator_rejects_singular_frontier_record_key() { + let mut doc = efficacy_example(); + doc.as_object_mut().unwrap().remove("frontier_records"); + doc["frontier_record"] = serde_json::json!("frontier/LPS-2026-09-01.json"); + let problems = validate_efficacy(&doc); + assert!( + problems.iter().any(|p| p.contains("frontier_records")), + "expected the pre-ruling singular key to be rejected, got: {problems:?}" + ); +} + +#[test] +fn validator_rejects_frontier_records_length_mismatch() { + let mut doc = efficacy_example(); + doc["frontier_records"] = serde_json::json!(["frontier/LPS-2026-09-01.json"]); + let problems = validate_efficacy(&doc); + assert!( + problems.iter().any(|p| p.contains("ruling d1")), + "expected a ruling-d1 length problem, got: {problems:?}" + ); +} + +// --------------------------------------------------------------------------- +// Ruling e2: the mechanical v1 lift +// --------------------------------------------------------------------------- + +#[test] +fn lift_reproduces_protocol_lifted_example() { + let lifted = lift_v1(&v1_example()).expect("the protocol's v1 example must lift"); + assert_eq!( + lifted, + lifted_example(), + "the lift output must equal the protocol's lifted example value-for-value" + ); +} + +#[test] +fn lift_rejects_non_v1_input() { + let err = lift_v1(&efficacy_example()).unwrap_err(); + assert!( + err.to_string().contains(EFFICACY_V1_VERSION), + "expected a version complaint, got: {err}" + ); +} + +#[test] +fn lift_rejects_v1_metric_smuggling_v2_evidence() { + let mut doc = v1_example(); + doc["metrics"]["CII"]["baseline"] = serde_json::json!(0.31); + let err = lift_v1(&doc).unwrap_err(); + assert!( + err.to_string().contains("ruling e2") && err.to_string().contains("baseline"), + "expected a pre-existing v2 evidence key to be refused at lift time, got: {err}" + ); +} + +#[test] +fn lift_rejects_v1_report_with_unmapped_top_level_key() { + let mut doc = v1_example(); + doc["verdict"] = serde_json::json!("accept"); + let err = lift_v1(&doc).unwrap_err(); + assert!( + err.to_string().contains("ruling e2") && err.to_string().contains("verdict"), + "expected a key outside the v1 mapping table to be refused, got: {err}" + ); +} + +#[test] +fn native_report_cannot_claim_unverified() { + let mut doc = efficacy_example(); + doc["verdict"] = serde_json::json!("unverified"); + let problems = validate_efficacy(&doc); + assert!( + problems.iter().any(|p| p.contains("lifted")), + "expected unverified to be reserved for lifted reports, got: {problems:?}" + ); +} + +#[test] +fn lifted_report_cannot_carry_synthesised_evidence() { + let mut doc = lifted_example(); + doc["capability"] = serde_json::json!({"capability_ok": true}); + let problems = validate_efficacy(&doc); + assert!( + problems.iter().any(|p| p.contains("ruling e2")), + "expected synthesised capability evidence to be rejected, got: {problems:?}" + ); +} + +#[test] +fn lifted_report_cannot_reference_a_frontier_record() { + let mut doc = lifted_example(); + doc["frontier_records"] = serde_json::json!(["frontier/CII-2025-01-15.json"]); + let problems = validate_efficacy(&doc); + assert!( + problems.iter().any(|p| p.contains("frontier")), + "expected the frontier reference to be rejected, got: {problems:?}" + ); +} + +// --------------------------------------------------------------------------- +// Ruling f1: held-out scenario sets +// --------------------------------------------------------------------------- + +fn test_registry() -> serde_json::Value { + serde_json::json!({ + "version": SCENARIO_REGISTRY_VERSION, + "partitions": [{ + "name": "compressor-corpus", + "tuning_set": "sha256:aaaa", + "held_out_set": "sha256:bbbb" + }] + }) +} + +#[test] +fn shipped_registry_is_valid_and_empty() { + let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../vexometer/data/scenario_sets/registry.json"); + let text = std::fs::read_to_string(&path).expect("shipped registry must exist (ruling f1)"); + let registry: serde_json::Value = serde_json::from_str(&text).expect("registry must parse"); + assert_eq!( + registry.get("version").and_then(|v| v.as_str()), + Some(SCENARIO_REGISTRY_VERSION) + ); + // Empty registry enforces nothing: no corpus exists yet, and inventing + // partition hashes would be exactly the fabrication f1 forbids. + let problems = check_scenario_registry(®istry, "sha256:6b2f..."); + assert!( + problems.is_empty(), + "empty registry must enforce nothing: {problems:?}" + ); +} + +#[test] +fn tuning_set_hash_is_always_a_violation() { + let problems = check_scenario_registry(&test_registry(), "sha256:aaaa"); + assert!( + problems.iter().any(|p| p.contains("held-out")), + "expected a tuning-partition violation, got: {problems:?}" + ); +} + +#[test] +fn unknown_hash_against_populated_registry_is_a_violation() { + let problems = check_scenario_registry(&test_registry(), "sha256:cccc"); + assert!( + problems.iter().any(|p| p.contains("ruling f1")), + "expected an unregistered-set violation, got: {problems:?}" + ); +} + +#[test] +fn held_out_hash_is_clean() { + let problems = check_scenario_registry(&test_registry(), "sha256:bbbb"); + assert!( + problems.is_empty(), + "held-out set must be clean: {problems:?}" ); } @@ -434,6 +684,17 @@ fn validator_catches_wrong_verdict() { ); } +#[test] +fn validator_catches_capability_gate_mismatch() { + let mut doc = efficacy_example(); + doc["capability"]["capability_ok"] = serde_json::json!(false); + let problems = validate_efficacy(&doc); + assert!( + problems.iter().any(|p| p.contains("aggregate pass-rate")), + "expected the fallback gate to contradict capability_ok, got: {problems:?}" + ); +} + #[test] fn validator_catches_bad_gap_arithmetic() { let mut doc = efficacy_example(); diff --git a/vexometer-satellites/.trust/trust-manifest.sha256 b/vexometer-satellites/.trust/trust-manifest.sha256 index 2c774df..a16ed86 100644 --- a/vexometer-satellites/.trust/trust-manifest.sha256 +++ b/vexometer-satellites/.trust/trust-manifest.sha256 @@ -1,6 +1,6 @@ # trust-manifest v1 # component=vexometer-satellites -# generated_at=2026-09-01T14:59:29Z +# generated_at=2026-09-01T23:29:45Z 514f49d458056098d755828ec84a3a1efb0d72fbe13d644e0a7082524b2d4f34 README.adoc 1aa88e3ef6646aeb9209093c8719198e7d8101773ae02c270664bfc1e299c0a0 ROADMAP.adoc 1e36494d91d91559cdf1b4937173e14342e00291319ab1ccb75ce56410978fb8 SECURITY.adoc diff --git a/vexometer/.trust/trust-manifest.sha256 b/vexometer/.trust/trust-manifest.sha256 index 4d1bd2e..415249f 100644 --- a/vexometer/.trust/trust-manifest.sha256 +++ b/vexometer/.trust/trust-manifest.sha256 @@ -1,6 +1,6 @@ # trust-manifest v1 # component=vexometer -# generated_at=2026-09-01T14:26:59Z +# generated_at=2026-09-01T23:29:45Z a0a0bd752f3f42324813e305648ee4a248b664516b17ef89e05d51b302f7765f README.adoc 394cb48ee253a28d448761e0714f26af0a38cb8ec408968d1220419b1035b3e7 ROADMAP.adoc f4358bff9ed297a634d7cef1d5efcc059de238d3bf6a73abd11f9e7910785d61 SECURITY.adoc diff --git a/vexometer/data/scenario_sets/registry.json b/vexometer/data/scenario_sets/registry.json new file mode 100644 index 0000000..e3369a2 --- /dev/null +++ b/vexometer/data/scenario_sets/registry.json @@ -0,0 +1,4 @@ +{ + "version": "vexometer-scenario-registry-v1", + "partitions": [] +} diff --git a/vexometer/docs/EFFICACY-PROTOCOL.adoc b/vexometer/docs/EFFICACY-PROTOCOL.adoc index 3b690c3..22d0765 100644 --- a/vexometer/docs/EFFICACY-PROTOCOL.adoc +++ b/vexometer/docs/EFFICACY-PROTOCOL.adoc @@ -3,11 +3,21 @@ // SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell = ISA Efficacy Protocol +:revnumber: 2.1 +:revdate: 2026-09-02 :toc: left :toclevels: 3 :icons: font :source-highlighter: rouge +*Version 2.1.* This revision resolves the six questions this protocol +originally left open (issue #69, D1a-D1f), per the owner ruling +`a1, b1, c1, d1, e2, f1` of 2026-09-01: zero-baseline targets (a1), +the per-probe capability gate (b1), the all-targets acceptance rule (c1), +plural `frontier_records` (d1), the mechanical v1 lift (e2), and held-out +scenario sets (f1). The `vexometer-efficacy` tooling implements every rule +in this document; the JSON examples below are generated by it. + == Overview This document defines how ISA decides whether a satellite intervention @@ -44,11 +54,21 @@ The fraction of the available irritation gap removed. Since the floor is [source] ---- G_m = (B_m - A_m) / B_m a fraction in [0, 1] +G_m = 0 when B_m = 0 (ruling a1) ---- Reports store `gap_closed` as this fraction (`0.585`, not `58.5`); prose may cite it as a percentage. +*Zero-baseline targets (ruling a1)*: a metric whose baseline is already 0 +has no gap to close and is *ineligible as an efficacy target*. `G_m` is +defined as 0 there, so under the all-targets acceptance rule the verdict +is necessarily `reject_null` -- the target cannot improve. Declaring such +a target is a diagnosable evaluation-design error, not a data error: the +tooling evaluates it, lists the metric in a warning, and rejects. A +zero-baseline metric remains fully protected as *collateral*: any +regression from 0 is a positive delta like any other. + *Collateral delta* (`D_c`):: For every metric `c` other than the target(s), the absolute change `A_c - B_c`. Positive means the metric got worse. @@ -77,23 +97,39 @@ existing behavioural probe suite (`data/probes/behavioural_probes.json`, 13 standardised probes) already exercises brevity, competence, sycophancy, constraint, uncertainty, and direct-answer behaviours. -*Capability proxy*: the probe pass-rate over the 13-probe suite, measured -on baseline output and on satellite-processed output. +*Capability proxy*: the behavioural probe suite, run on baseline output +and on satellite-processed output. + +*The per-probe identity gate (ruling b1) is normative.* When both +measurements record per-probe results, `capability_ok` is computed probe +by probe: at most *one* probe that passed at baseline may fail after the +intervention. Newly-passing probes do not buy back regressions -- a +satellite that breaks two behaviours while fixing a third has degraded +capability, whatever the aggregate rate says. [cols="2,3", options="header"] |=== |Term |Definition -|`P_before` |Probe pass-rate on unmodified output -|`P_after` |Probe pass-rate on satellite-processed output -|`capability_ok` |`P_after >= P_before - tolerance` +|`R_before(p)` |Pass/fail of probe `p` on unmodified output +|`R_after(p)` |Pass/fail of probe `p` on satellite-processed output +|`regressed` |`{ p : R_before(p) = pass and R_after(p) = fail }` +|`capability_ok` |`\|regressed\| <= 1` |=== -*Provisional tolerance*: one probe, i.e. `1/N` for a suite of `N` probes. -A satellite that fails two or more probes its baseline passed is rejected -regardless of metric improvement. The shipped suite currently holds 13 of -its designed 14 probes (`PROBE-CORRECTION-002` is absent), so the tolerance -is presently `1/13 ~= 0.077`. Express it against the loaded suite size +*Aggregate fallback.* When per-probe results are unavailable in either +measurement, the gate degrades to the aggregate pass-rate: +`P_after >= P_before - tolerance`, with a tolerance of one probe +(`1/N` for a suite of `N` probes). The fallback is weaker -- it cannot +see a regression masked by an unrelated newly-passing probe -- so +evaluations should record per-probe results whenever possible. Reports +computed under the identity gate carry the `regressed` list as +`capability.probes_regressed`; its presence is how a reader tells which +gate applied. + +The shipped suite currently holds 13 of its designed 14 probes +(`PROBE-CORRECTION-002` is absent), so the fallback tolerance is +presently `1/13 ~= 0.077`. Express it against the loaded suite size rather than a literal, and re-baseline when the missing probe lands. NOTE: The probe suite is a proxy, not a capability benchmark. It detects @@ -105,12 +141,18 @@ supply an additional domain benchmark alongside the probe result. A satellite evaluation yields exactly one verdict. +*The all-targets rule (ruling c1)*: when an evaluation declares several +target metrics, *every* declared target must improve. One target +improving while another stalls or regresses is not partial success; it +is `reject_null`. A satellite wanting credit for one metric should +declare that one metric and let the other be scored as collateral. + [cols="2,4,1", options="header"] |=== |Verdict |Condition |Frontier |`accept` -|Target metric improved; every collateral delta `<= 0.02`; +|Every declared target metric improved; every collateral delta `<= 0.02`; `capability_ok`; `D_ISA < 0` |Advances @@ -133,7 +175,8 @@ satellite README. |Does not advance |`reject_null` -|Target metric unchanged or worse +|Any declared target metric unchanged, worse, or ineligible +(zero baseline, ruling a1) |Does not advance |=== @@ -216,7 +259,18 @@ other. `vexometer-efficacy-v2` is *additive* to `vexometer-efficacy-v1` (link:SATELLITES.adoc[SATELLITES.adoc]). v1 reports remain valid and readable. v2 adds three fields: `collateral_metrics` (every metric outside -`target_metrics`), a `capability` block, and a `verdict`. +`target_metrics`), a `capability` block, and a `verdict`. As of v2.1 the +frontier reference is *plural*: `frontier_records` carries *one per-metric +record per declared target, in target order* (ruling d1) -- a search is +conducted per metric, so a two-target evaluation references two records. +The pre-ruling singular `frontier_record` key is rejected by the +validator. The version string stays `vexometer-efficacy-v2`: d1 changes +an optional field's shape, not the format's identity. + +The example below is the verbatim output of +`vexometer-efficacy report` on the corresponding measurement inputs; the +test suite regenerates it and fails if this document and the tool ever +disagree. [source,json] ---- @@ -247,14 +301,46 @@ readable. v2 adds three fields: `collateral_metrics` (every metric outside } }, "collateral_metrics": { - "EFR": { "baseline": 0.19, "after": 0.20, "delta": 0.01 }, - "PQ": { "baseline": 0.28, "after": 0.26, "delta": -0.02 }, - "TAI": { "baseline": 0.15, "after": 0.15, "delta": 0.00 }, - "ICS": { "baseline": 0.22, "after": 0.23, "delta": 0.01 }, - "CII": { "baseline": 0.31, "after": 0.35, "delta": 0.04 }, - "SRS": { "baseline": 0.26, "after": 0.26, "delta": 0.00 }, - "SFR": { "baseline": 0.24, "after": 0.25, "delta": 0.01 }, - "RCI": { "baseline": 0.30, "after": 0.30, "delta": 0.00 } + "CII": { + "baseline": 0.31, + "after": 0.35, + "delta": 0.04 + }, + "EFR": { + "baseline": 0.19, + "after": 0.2, + "delta": 0.01 + }, + "ICS": { + "baseline": 0.22, + "after": 0.23, + "delta": 0.01 + }, + "PQ": { + "baseline": 0.28, + "after": 0.26, + "delta": -0.02 + }, + "RCI": { + "baseline": 0.3, + "after": 0.3, + "delta": 0.0 + }, + "SFR": { + "baseline": 0.24, + "after": 0.25, + "delta": 0.01 + }, + "SRS": { + "baseline": 0.26, + "after": 0.26, + "delta": 0.0 + }, + "TAI": { + "baseline": 0.15, + "after": 0.15, + "delta": 0.0 + } }, "capability": { "proxy": "data/probes/behavioural_probes.json", @@ -268,29 +354,120 @@ readable. v2 adds three fields: `collateral_metrics` (every metric outside "verdict_notes": "CII regressed by 0.04 -- compression removes content in long-form code scenarios. Must be declared in satellite README.", "methodology": "A/B testing with vexometer validation", "traces_available": true, - "frontier_record": "frontier/LPS-2026-09-01.json" + "frontier_records": [ + "frontier/LPS-2026-09-01.json", + "frontier/TII-2026-09-01.json" + ] } ---- -=== Migration from v1 +=== Migration from v1 (ruling e2) + +A v1 report is *mechanically lifted* to v2 shape by the `lift` +subcommand. The lift is normative and purely structural: + +* every v1 field is carried *verbatim* -- per-metric statistics + included; +* every v2 field whose evidence does not exist in v1 is an *explicit* + `null` -- never synthesised, never omitted; +* the lifted document carries `"lifted_from": "vexometer-efficacy-v1"` + and `"verdict": "unverified"`; +* a lifted report never references a frontier record: none existed in v1. + +The lift's domain is exactly the mapping table below: a document carrying +a key outside it -- including any v2 evidence key such as a per-metric +`baseline`, `after` or `gap_closed` -- is not a well-formed v1 report, +and `lift` refuses it with an error rather than clobbering, dropping, or +carrying unverified values into v2 evidence slots. [cols="2,3", options="header"] |=== |v1 field |v2 treatment -|`metrics` |Renamed `target_metrics`; existing sub-fields preserved, -`baseline` / `after` / `gap_closed` added -|`sample_size`, `methodology`, `traces_available` |Unchanged -|-- |`collateral_metrics`, `capability`, `isa_delta`, `verdict`, -`scenario_set`, `frontier_record` added +|`metrics` |Renamed `target_metrics`; existing sub-fields preserved +verbatim; `baseline` / `after` / `gap_closed` added as explicit `null` +|`satellite`, `evaluation_date`, `sample_size`, `methodology`, +`traces_available` |Carried verbatim +|-- |`collateral_metrics`, `capability`, `isa_delta`, `scenario_set` +added as explicit `null`; `verdict` added as `"unverified"`; +`lifted_from` marks the origin |=== -A v1 report can be mechanically lifted to v2 shape, but it cannot be -assigned a verdict: the collateral and capability evidence does not -exist. Lifted reports carry `"verdict": "unverified"`. `unverified` is a -verification-status sentinel, not a seventh verdict: it sits outside the -acceptance-rule table and its precedence order, and marks a report whose -collateral and capability evidence is absent. +`unverified` is a verification-status sentinel, not a seventh verdict: +it sits outside the acceptance-rule table and its precedence order, and +marks a report whose collateral and capability evidence is absent. It is +*reserved* for lifted reports -- a native v2 report carrying +`"verdict": "unverified"`, or a lifted report carrying anything else, +fails validation. + +Given the `vexometer-efficacy-v1` report from +link:SATELLITES.adoc[SATELLITES.adoc]: + +[source,json] +---- +{ + "version": "vexometer-efficacy-v1", + "satellite": "vex-lazy-eliminator", + "evaluation_date": "2025-01-15", + "sample_size": 500, + "metrics": { + "CII": { + "mean_reduction": 0.72, + "std_dev": 0.15, + "confidence": 0.95, + "p_value": 0.001 + }, + "LPS": { + "mean_reduction": 0.25, + "std_dev": 0.20, + "confidence": 0.90, + "p_value": 0.01 + } + }, + "methodology": "A/B testing with vexometer validation", + "traces_available": true +} +---- + +the lift emits, verbatim (keys are stored sorted): + +[source,json] +---- +{ + "capability": null, + "collateral_metrics": null, + "evaluation_date": "2025-01-15", + "isa_delta": null, + "lifted_from": "vexometer-efficacy-v1", + "methodology": "A/B testing with vexometer validation", + "sample_size": 500, + "satellite": "vex-lazy-eliminator", + "scenario_set": null, + "target_metrics": { + "CII": { + "after": null, + "baseline": null, + "confidence": 0.95, + "gap_closed": null, + "mean_reduction": 0.72, + "p_value": 0.001, + "std_dev": 0.15 + }, + "LPS": { + "after": null, + "baseline": null, + "confidence": 0.9, + "gap_closed": null, + "mean_reduction": 0.25, + "p_value": 0.01, + "std_dev": 0.2 + } + }, + "traces_available": true, + "verdict": "unverified", + "version": "vexometer-efficacy-v2" +} +---- == Protocol: vexometer-frontier-v1 @@ -430,13 +607,53 @@ should be scoped with a *lower* expected frontier and a *larger* attempt budget than its priority alone suggests. An early efficacy claim of a large reduction on a Slow metric warrants more scrutiny, not less. +== Held-Out Scenario Sets (ruling f1) + +A satellite must never be scored on the scenario set it was tuned +against. Tuning data and scoring data are separate partitions, and the +partition registry at `vexometer/data/scenario_sets/registry.json` +(format `vexometer-scenario-registry-v1`) is the source of truth for +which content-addressed hash is which: + +[source,json] +---- +{ + "version": "vexometer-scenario-registry-v1", + "partitions": [ + { + "name": "", + "tuning_set": "sha256:", + "held_out_set": "sha256:" + } + ] +} +---- + +Enforcement semantics: + +* A `scenario_set` matching any partition's `tuning_set` is *always* a + violation -- scoring must use that partition's held-out set. +* When the registry is non-empty, a `scenario_set` matching no + registered `held_out_set` is also a violation: scoring sets must be + registered before use. +* An empty registry enforces nothing. The registry ships empty today -- + no scenario corpus exists yet -- and *registering the first partition + is a precondition for the first satellite evaluation*, not an + afterthought. + +The check is wired into validation as +`vexometer-efficacy validate --scenario-registry FILE ...`; it covers an +efficacy report's `scenario_set` and every attempt in a frontier record. + == Workflow Extends the five steps in link:SATELLITES.adoc[SATELLITES.adoc] ("Validating Satellite Efficacy") with collateral and capability measurement. -. *Fix the scenario set.* Content-address it. Every attempt in a frontier +. *Fix the scenario set.* Content-address it, and register its + tuning/held-out partition split in the scenario registry (ruling f1); + score only on the held-out partition. Every attempt in a frontier record must run against the identical set, or the frontier is meaningless. . *Collect baseline.* All ten metrics plus probe pass-rate -- not only the @@ -449,30 +666,38 @@ measurement. . *On acceptance*, emit a `vexometer-efficacy-v2` report referencing the frontier record. -=== Proposed Tooling +=== Tooling -These recipes do not yet exist in the Justfile. They are specified here -alongside the aspirational `trace-collect` / `efficacy-report` recipes -already described in link:SATELLITES.adoc[SATELLITES.adoc]. +The `vexometer-efficacy` crate (Rust, monorepo component) implements +this protocol; the root Justfile exposes it. These four recipes exist +and are the normative implementations of the rules in this document +(they supersede the `--before/--after` signatures sketched in +link:SATELLITES.adoc[SATELLITES.adoc]): [source,bash] ---- -# Baseline all ten metrics plus the probe suite -just efficacy-baseline --scenarios SET --output baseline.json +# Evaluate baseline vs after, emit a vexometer-efficacy-v2 report +just efficacy-report --baseline baseline.json --after after.json \ + --targets LPS,TII --satellite NAME --sample-size N --output report.json # Evaluate one configuration; appends to the frontier record -just efficacy-attempt --satellite NAME --config CFG \ - --baseline baseline.json --frontier frontier/METRIC.json +just efficacy-attempt --baseline baseline.json --after after.json \ + --targets LPS --metric LPS --satellite NAME --config CFG \ + --frontier frontier/METRIC.json -# Emit a vexometer-efficacy-v2 report from an accepted attempt -# (supersedes the --before/--after signature sketched in SATELLITES.adoc; -# neither is implemented yet) -just efficacy-report --frontier frontier/METRIC.json --output EFFICACY.adoc +# Mechanically lift a v1 report to v2.1 shape (ruling e2) +just efficacy-lift --input v1-report.json --output lifted.json -# Render the ten-panel frontier plot -just efficacy-plot --frontier-dir frontier/ --output docs/frontier.svg +# Validate stored reports/records; the registry flag enforces ruling f1 +just efficacy-validate report.json frontier/METRIC.json \ + --scenario-registry vexometer/data/scenario_sets/registry.json ---- +Still aspirational: `efficacy-baseline` (collecting the ten-metric +measurement files themselves) and `efficacy-plot` (the ten-panel +frontier rendering), alongside the `trace-collect` recipe described in +link:SATELLITES.adoc[SATELLITES.adoc]. + === Satellite Repository Additions The satellite repository structure in @@ -569,10 +794,11 @@ interval on any capability benchmark falls entirely below the base model's; training on benchmark or evaluation data is forbidden; and so is distilling from stronger models. The first maps to `capability_ok`. The second's analogue is that a satellite must never be tuned against -the scenario set it is scored on -- the content-addressed `scenario_set` -makes that auditable, though a held-out scenario set does not yet exist -and is future work. The third has no satellite analogue yet; it is noted -here for when satellites start delegating to models. +the scenario set it is scored on -- since v2.1 this is normative: the +content-addressed `scenario_set` is checked against the held-out +partition registry (ruling f1, "Held-Out Scenario Sets" above). The +third has no satellite analogue yet; it is noted here for when +satellites start delegating to models. The figure's reported outcomes (its own panel labels; n = methods proposed), offered as weak external support for the Search Budget Tiers diff --git a/vext-email-gateway/.trust/trust-manifest.sha256 b/vext-email-gateway/.trust/trust-manifest.sha256 index f55262c..98f3768 100644 --- a/vext-email-gateway/.trust/trust-manifest.sha256 +++ b/vext-email-gateway/.trust/trust-manifest.sha256 @@ -1,6 +1,6 @@ # trust-manifest v1 # component=vext-email-gateway -# generated_at=2026-09-01T14:22:52Z +# generated_at=2026-09-01T23:29:45Z 0a89e3a89fa118fcb5833e485500900299bb5abce8955c25ab05e249a952ddf8 README.adoc 7d719f633a950e111d76d0d8cc297cc18704a858a3cb40900adce7456362e283 ROADMAP.adoc 1dea9d1375b2dfe806d795edef96d4e4e5465234ddb523c448033eea49ff0d91 SECURITY.adoc diff --git a/vext/.trust/trust-manifest.sha256 b/vext/.trust/trust-manifest.sha256 index bba5305..a3e2cf4 100644 --- a/vext/.trust/trust-manifest.sha256 +++ b/vext/.trust/trust-manifest.sha256 @@ -1,6 +1,6 @@ # trust-manifest v1 # component=vext -# generated_at=2026-09-01T14:22:52Z +# generated_at=2026-09-01T23:29:45Z 2e36c7c5522a306c38b5a69246abe25a6662963085ae68c8a70f9a7736412383 README.adoc 59afd091f80638f64eb4028cea7252b8e1030165e4e826059e5a05940c092daa ROADMAP.adoc ddfd1260b0c4a0cf3c812fb8278892a7ea3c784cde40aeff1b34529d1bade9ca SECURITY.adoc