From 848c227aa330f20930c4139019a1fcbca117d765 Mon Sep 17 00:00:00 2001 From: zhanghan <2774877132@qq.com> Date: Thu, 13 Aug 2026 03:01:14 +0800 Subject: [PATCH 1/4] feat: add blind benchmark harness --- benchmarks/README.md | 37 ++ .../synthetic/custom-threshold/manifest.json | 47 ++ .../custom-threshold/package/NOTICE.md | 1 + .../custom-threshold/package/observations.csv | 5 + .../custom-threshold/package/problem.md | 5 + .../custom-threshold/reference/reference.json | 6 + .../summary-statistics/manifest.json | 47 ++ .../summary-statistics/package/NOTICE.md | 1 + .../package/measurements.csv | 5 + .../summary-statistics/package/problem.md | 3 + .../reference/reference.json | 6 + schemas/benchmark-manifest.v1.json | 110 +++++ schemas/benchmark-result.v1.json | 131 ++++++ src/benchmark/contracts.ts | 154 +++++++ src/benchmark/index.ts | 5 + src/benchmark/report.ts | 134 ++++++ src/benchmark/run-synthetic.ts | 34 ++ src/benchmark/runner.ts | 405 ++++++++++++++++++ src/benchmark/synthetic.ts | 97 +++++ src/benchmark/types.ts | 208 +++++++++ tests/benchmark-contracts.test.ts | 123 ++++++ tests/benchmark-report.test.ts | 109 +++++ tests/benchmark-runner.test.ts | 214 +++++++++ tests/benchmark-synthetic.test.ts | 30 ++ 24 files changed, 1917 insertions(+) create mode 100644 benchmarks/README.md create mode 100644 benchmarks/synthetic/custom-threshold/manifest.json create mode 100644 benchmarks/synthetic/custom-threshold/package/NOTICE.md create mode 100644 benchmarks/synthetic/custom-threshold/package/observations.csv create mode 100644 benchmarks/synthetic/custom-threshold/package/problem.md create mode 100644 benchmarks/synthetic/custom-threshold/reference/reference.json create mode 100644 benchmarks/synthetic/summary-statistics/manifest.json create mode 100644 benchmarks/synthetic/summary-statistics/package/NOTICE.md create mode 100644 benchmarks/synthetic/summary-statistics/package/measurements.csv create mode 100644 benchmarks/synthetic/summary-statistics/package/problem.md create mode 100644 benchmarks/synthetic/summary-statistics/reference/reference.json create mode 100644 schemas/benchmark-manifest.v1.json create mode 100644 schemas/benchmark-result.v1.json create mode 100644 src/benchmark/contracts.ts create mode 100644 src/benchmark/index.ts create mode 100644 src/benchmark/report.ts create mode 100644 src/benchmark/run-synthetic.ts create mode 100644 src/benchmark/runner.ts create mode 100644 src/benchmark/synthetic.ts create mode 100644 src/benchmark/types.ts create mode 100644 tests/benchmark-contracts.test.ts create mode 100644 tests/benchmark-report.test.ts create mode 100644 tests/benchmark-runner.test.ts create mode 100644 tests/benchmark-synthetic.test.ts diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 0000000..b3b5c9f --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,37 @@ +# Benchmark harness + +This directory contains redistributable synthetic fixtures and metadata-only guidance for historical cases. It does **not** claim that three historical competition problems have been run, and protected problem statements or reference answers are not committed by default. + +## Run the synthetic suite + +From the repository root: + +```bash +npx tsx src/benchmark/run-synthetic.ts --output benchmarks/output/synthetic +``` + +The command writes `benchmark-report.json` and `benchmark-report.md`. It runs the deterministic agent and one-shot adapters against the same frozen bytes for each case, uses a deterministic clock and identity, and is suitable for checking stable report fields. `benchmarks/output/` is intended as local generated output; do not commit reports that contain non-synthetic operational data. + +The harness API is injected: production integrations implement `BenchmarkAdapter` from `src/benchmark/types.ts` instead of importing CLI or Orchestrator modules. A solve adapter receives package files, budgets, expected task types, and hard-check definitions. It never receives the manifest object, a reference path, reference bytes, or scoring notes. Reference material is opened separately for post-solve leakage checks, and a detected same-problem answer changes the result to `blocked` / `blocked_policy` with a digest-only event. + +## Adding three historical problems legally + +The v0.1-alpha acceptance target calls for three historical blind runs spanning at least six task families and one custom experiment. This repository currently supplies no such score and must not be described as having completed that target. Add each historical case only after all of the following steps: + +1. **Establish rights before copying.** Record the organizer, copyright holder, official source URL, license or written permission, and redistribution terms. A public download link is not by itself permission to republish a statement, dataset, judge material, or solution. +2. **Prefer user-supplied private material.** For ordinary copyrighted competitions, set `license.redistribution` to `user_supplied_only` or `metadata_only`, set reference availability to `user_supplied` or `unavailable`, and keep problem packages and answers under the ignored `benchmarks/private/` directory. Do not commit them. +3. **Create metadata without answer text.** Give the case a stable id, declare allowed and expected task types, budgets, runtime/execution policy, and hard checks. Store only relative paths and SHA-256 identities. Never place a reference answer, excerpt from a solution, credential, session, or absolute host path in a manifest or report. +4. **Freeze one case for both variants.** Verify that agent and one-shot runs receive the exact same `frozen_case_sha256`, budget, expected-task denominator, and hard-check definitions. Do not tune either variant after viewing the other variant's result. +5. **Keep scoring references out of solve.** Reference solutions and judge notes are scoring-only. The blind firewall must block a detected same-problem answer and retain only a fingerprint, never the matched text. +6. **Report missing measurements honestly.** Unknown token count, cost, or human review is `unavailable` with a `null` value. An unavailable field is not zero. Material not obtained or a run not attempted is `not_run`; a policy stop is `blocked`; failures never count as completion. +7. **Review before publication.** Have a maintainer confirm the license record and inspect the staged diff for protected text. Publish historical results only when the underlying case can be lawfully used and the environment/commit identity is reproducible. + +A practical three-case plan is to select cases with independently verified reuse permission, assign each a separate private case root, and run both variants offline. Until those materials and permissions exist, retain metadata placeholders outside committed synthetic results rather than inventing scores. + +## Contract files + +- `schemas/benchmark-manifest.v1.json`: strict manifest boundary. +- `schemas/benchmark-result.v1.json`: strict result and metric-state boundary. +- `src/benchmark/contracts.ts`: Schema validation plus cross-field invariants. +- `src/benchmark/runner.ts`: frozen-package runner and blind leakage firewall. +- `src/benchmark/report.ts`: deterministic JSON and Markdown aggregation. diff --git a/benchmarks/synthetic/custom-threshold/manifest.json b/benchmarks/synthetic/custom-threshold/manifest.json new file mode 100644 index 0000000..d207468 --- /dev/null +++ b/benchmarks/synthetic/custom-threshold/manifest.json @@ -0,0 +1,47 @@ +{ + "schema_version": "1.0.0", + "case_id": "synthetic-custom-threshold", + "package_path": "package", + "license": { + "name": "CC0 1.0 Universal", + "spdx_id": "CC0-1.0", + "copyright_holder": "modeling-agent contributors", + "source_url": null, + "redistribution": "permitted", + "notice_path": "package/NOTICE.md" + }, + "blind_policy": { + "mode": "blind", + "solve_input": "package_only", + "same_problem_answers": "block", + "minimum_reference_match_characters": 48 + }, + "reference_policy": { + "access": "scoring_only", + "availability": "included", + "relative_path": "reference/reference.json", + "sha256": "456c633600a9ec460944a61b8ce3df247c5ba476b5f11cc7dcb1adfc3852c89e" + }, + "runtime": { + "agent_adapter_id": "synthetic-agent-v1", + "one_shot_adapter_id": "synthetic-one-shot-v1" + }, + "execution": { + "kind": "local", + "network_access": "disabled" + }, + "budget": { + "max_wall_time_ms": 10000, + "max_tokens": 2000, + "max_cost_usd": null, + "max_human_review_minutes": null + }, + "allowed_task_types": ["statistical_analysis", "custom_experiment"], + "expected_task_types": ["statistical_analysis", "custom_experiment"], + "hard_checks": [ + { + "id": "custom-check-produced", + "description": "A custom threshold experiment artifact and supporting evidence are present." + } + ] +} diff --git a/benchmarks/synthetic/custom-threshold/package/NOTICE.md b/benchmarks/synthetic/custom-threshold/package/NOTICE.md new file mode 100644 index 0000000..58fba85 --- /dev/null +++ b/benchmarks/synthetic/custom-threshold/package/NOTICE.md @@ -0,0 +1 @@ +This synthetic problem and data were created for the modeling-agent project and are dedicated to the public domain under CC0-1.0. They do not reproduce a competition problem or solution. diff --git a/benchmarks/synthetic/custom-threshold/package/observations.csv b/benchmarks/synthetic/custom-threshold/package/observations.csv new file mode 100644 index 0000000..f69e898 --- /dev/null +++ b/benchmarks/synthetic/custom-threshold/package/observations.csv @@ -0,0 +1,5 @@ +observation,value +A,1 +B,2 +C,3 +D,9 diff --git a/benchmarks/synthetic/custom-threshold/package/problem.md b/benchmarks/synthetic/custom-threshold/package/problem.md new file mode 100644 index 0000000..08da755 --- /dev/null +++ b/benchmarks/synthetic/custom-threshold/package/problem.md @@ -0,0 +1,5 @@ +# Synthetic custom-threshold case + +Use `observations.csv` to evaluate the nonstandard threshold rule described below. The rule is not one of the registered methods, so model it as a `custom_experiment` alongside the supporting statistical analysis. Produce one experiment artifact and one evidence record. + +Rule: flag an observation when its value is greater than 1.5 times the median of the supplied values. diff --git a/benchmarks/synthetic/custom-threshold/reference/reference.json b/benchmarks/synthetic/custom-threshold/reference/reference.json new file mode 100644 index 0000000..7fd9913 --- /dev/null +++ b/benchmarks/synthetic/custom-threshold/reference/reference.json @@ -0,0 +1,6 @@ +{ + "case_id": "synthetic-custom-threshold", + "required_task_types": ["statistical_analysis", "custom_experiment"], + "hard_checks": ["custom-check-produced"], + "scoring_note": "Confirm that the nonstandard threshold is implemented as an experimental task and that its conclusion is bound to generated evidence. This scoring-only material must never be placed in the solve adapter context." +} diff --git a/benchmarks/synthetic/summary-statistics/manifest.json b/benchmarks/synthetic/summary-statistics/manifest.json new file mode 100644 index 0000000..85a7e76 --- /dev/null +++ b/benchmarks/synthetic/summary-statistics/manifest.json @@ -0,0 +1,47 @@ +{ + "schema_version": "1.0.0", + "case_id": "synthetic-summary-statistics", + "package_path": "package", + "license": { + "name": "CC0 1.0 Universal", + "spdx_id": "CC0-1.0", + "copyright_holder": "modeling-agent contributors", + "source_url": null, + "redistribution": "permitted", + "notice_path": "package/NOTICE.md" + }, + "blind_policy": { + "mode": "blind", + "solve_input": "package_only", + "same_problem_answers": "block", + "minimum_reference_match_characters": 48 + }, + "reference_policy": { + "access": "scoring_only", + "availability": "included", + "relative_path": "reference/reference.json", + "sha256": "368f4f8aa18ae957d8d84f23f92955451e9a47b99c0b911bdc7468e74be92a9a" + }, + "runtime": { + "agent_adapter_id": "synthetic-agent-v1", + "one_shot_adapter_id": "synthetic-one-shot-v1" + }, + "execution": { + "kind": "local", + "network_access": "disabled" + }, + "budget": { + "max_wall_time_ms": 10000, + "max_tokens": 2000, + "max_cost_usd": null, + "max_human_review_minutes": null + }, + "allowed_task_types": ["statistical_analysis"], + "expected_task_types": ["statistical_analysis"], + "hard_checks": [ + { + "id": "summary-produced", + "description": "A summary artifact and supporting evidence are present." + } + ] +} diff --git a/benchmarks/synthetic/summary-statistics/package/NOTICE.md b/benchmarks/synthetic/summary-statistics/package/NOTICE.md new file mode 100644 index 0000000..58fba85 --- /dev/null +++ b/benchmarks/synthetic/summary-statistics/package/NOTICE.md @@ -0,0 +1 @@ +This synthetic problem and data were created for the modeling-agent project and are dedicated to the public domain under CC0-1.0. They do not reproduce a competition problem or solution. diff --git a/benchmarks/synthetic/summary-statistics/package/measurements.csv b/benchmarks/synthetic/summary-statistics/package/measurements.csv new file mode 100644 index 0000000..e20e799 --- /dev/null +++ b/benchmarks/synthetic/summary-statistics/package/measurements.csv @@ -0,0 +1,5 @@ +sample,value +A,2 +B,4 +C,6 +D,8 diff --git a/benchmarks/synthetic/summary-statistics/package/problem.md b/benchmarks/synthetic/summary-statistics/package/problem.md new file mode 100644 index 0000000..b2e8a91 --- /dev/null +++ b/benchmarks/synthetic/summary-statistics/package/problem.md @@ -0,0 +1,3 @@ +# Synthetic summary-statistics case + +Using only `measurements.csv`, summarize the observed values and identify the appropriate registered task type. Produce one answer artifact and one evidence record. No external data is permitted. diff --git a/benchmarks/synthetic/summary-statistics/reference/reference.json b/benchmarks/synthetic/summary-statistics/reference/reference.json new file mode 100644 index 0000000..23e658e --- /dev/null +++ b/benchmarks/synthetic/summary-statistics/reference/reference.json @@ -0,0 +1,6 @@ +{ + "case_id": "synthetic-summary-statistics", + "required_task_types": ["statistical_analysis"], + "hard_checks": ["summary-produced"], + "scoring_note": "Confirm that the solve output reports an evidence-backed summary for every supplied observation. This scoring-only material must never be placed in the solve adapter context." +} diff --git a/schemas/benchmark-manifest.v1.json b/schemas/benchmark-manifest.v1.json new file mode 100644 index 0000000..ccdb5d3 --- /dev/null +++ b/schemas/benchmark-manifest.v1.json @@ -0,0 +1,110 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://modeling-agent.local/schemas/benchmark-manifest.v1.json", + "title": "BenchmarkManifestV1", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "case_id", "package_path", "license", "blind_policy", "reference_policy", "runtime", "execution", "budget", "allowed_task_types", "expected_task_types", "hard_checks"], + "properties": { + "schema_version": { "const": "1.0.0" }, + "case_id": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-]{2,63}$" }, + "package_path": { "$ref": "#/$defs/relativePath" }, + "license": { + "type": "object", + "additionalProperties": false, + "required": ["name", "spdx_id", "copyright_holder", "source_url", "redistribution", "notice_path"], + "properties": { + "name": { "type": "string", "minLength": 1, "maxLength": 200 }, + "spdx_id": { "type": ["string", "null"], "minLength": 1, "maxLength": 100 }, + "copyright_holder": { "type": "string", "minLength": 1, "maxLength": 300 }, + "source_url": { "type": ["string", "null"], "format": "uri", "maxLength": 2000 }, + "redistribution": { "enum": ["permitted", "user_supplied_only", "metadata_only"] }, + "notice_path": { "anyOf": [{ "$ref": "#/$defs/relativePath" }, { "type": "null" }] } + } + }, + "blind_policy": { + "type": "object", + "additionalProperties": false, + "required": ["mode", "solve_input", "same_problem_answers", "minimum_reference_match_characters"], + "properties": { + "mode": { "const": "blind" }, + "solve_input": { "const": "package_only" }, + "same_problem_answers": { "const": "block" }, + "minimum_reference_match_characters": { "type": "integer", "minimum": 16, "maximum": 4096 } + } + }, + "reference_policy": { + "type": "object", + "additionalProperties": false, + "required": ["access", "availability", "relative_path", "sha256"], + "properties": { + "access": { "const": "scoring_only" }, + "availability": { "enum": ["included", "user_supplied", "unavailable"] }, + "relative_path": { "anyOf": [{ "$ref": "#/$defs/relativePath" }, { "type": "null" }] }, + "sha256": { "type": ["string", "null"], "pattern": "^[a-f0-9]{64}$" } + } + }, + "runtime": { + "type": "object", + "additionalProperties": false, + "required": ["agent_adapter_id", "one_shot_adapter_id"], + "properties": { + "agent_adapter_id": { "type": "string", "minLength": 1, "maxLength": 200 }, + "one_shot_adapter_id": { "type": "string", "minLength": 1, "maxLength": 200 } + } + }, + "execution": { + "type": "object", + "additionalProperties": false, + "required": ["kind", "network_access"], + "properties": { + "kind": { "enum": ["local", "docker"] }, + "network_access": { "enum": ["disabled", "research_gateway_only"] } + } + }, + "budget": { + "type": "object", + "additionalProperties": false, + "required": ["max_wall_time_ms", "max_tokens", "max_cost_usd", "max_human_review_minutes"], + "properties": { + "max_wall_time_ms": { "type": "integer", "minimum": 1, "maximum": 86400000 }, + "max_tokens": { "type": ["integer", "null"], "minimum": 1, "maximum": 10000000 }, + "max_cost_usd": { "type": ["number", "null"], "exclusiveMinimum": 0 }, + "max_human_review_minutes": { "type": ["number", "null"], "exclusiveMinimum": 0 } + } + }, + "allowed_task_types": { "$ref": "#/$defs/taskTypes" }, + "expected_task_types": { "$ref": "#/$defs/taskTypes" }, + "hard_checks": { + "type": "array", + "minItems": 1, + "maxItems": 100, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["id", "description"], + "properties": { + "id": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-]{1,63}$" }, + "description": { "type": "string", "minLength": 1, "maxLength": 1000 } + } + } + } + }, + "$defs": { + "relativePath": { + "type": "string", + "minLength": 1, + "maxLength": 1000, + "pattern": "^(?!/)(?![A-Za-z]:[\\\\/])(?!.*(?:^|/)\\.\\.(?:/|$))(?!.*\\\\).+$" + }, + "taskType": { + "enum": ["statistical_analysis", "regression_prediction", "time_series_forecasting", "classification", "clustering", "evaluation_ranking", "optimization", "simulation", "custom_experiment"] + }, + "taskTypes": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { "$ref": "#/$defs/taskType" } + } + } +} diff --git a/schemas/benchmark-result.v1.json b/schemas/benchmark-result.v1.json new file mode 100644 index 0000000..f40699a --- /dev/null +++ b/schemas/benchmark-result.v1.json @@ -0,0 +1,131 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://modeling-agent.local/schemas/benchmark-result.v1.json", + "title": "BenchmarkResultV1", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "case_id", "variant", "adapter_id", "run_id", "frozen_case_sha256", "state", "outcome", "observed_task_types", "hard_checks", "policy_events", "error", "metrics"], + "properties": { + "schema_version": { "const": "1.0.0" }, + "case_id": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-]{2,63}$" }, + "variant": { "enum": ["agent", "one_shot"] }, + "adapter_id": { "type": "string", "minLength": 1, "maxLength": 200 }, + "run_id": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-]{7,127}$" }, + "frozen_case_sha256": { "type": "string", "pattern": "^[a-f0-9]{64}$" }, + "state": { "enum": ["measured", "not_run", "blocked"] }, + "outcome": { "enum": ["completed", "incomplete", "hard_error", "blocked_policy", "not_run"] }, + "observed_task_types": { "$ref": "#/$defs/taskTypes" }, + "hard_checks": { + "type": "array", + "maxItems": 100, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["id", "status"], + "properties": { + "id": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-]{1,63}$" }, + "status": { "enum": ["passed", "failed", "blocked", "not_run"] }, + "note": { "type": "string", "minLength": 1, "maxLength": 500 } + } + } + }, + "policy_events": { + "type": "array", + "maxItems": 100, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["type", "action", "fingerprint"], + "properties": { + "type": { "const": "same_problem_answer_detected" }, + "action": { "const": "blocked" }, + "fingerprint": { "type": "string", "pattern": "^[a-f0-9]{64}$" } + } + } + }, + "error": { + "anyOf": [ + { "type": "null" }, + { + "type": "object", + "additionalProperties": false, + "required": ["class", "message"], + "properties": { + "class": { "type": "string", "minLength": 1, "maxLength": 200 }, + "message": { "type": "string", "minLength": 1, "maxLength": 500 }, + "fingerprint": { "type": "string", "pattern": "^[a-f0-9]{64}$" } + } + } + ] + }, + "metrics": { + "type": "object", + "additionalProperties": false, + "required": ["completion", "hard_error", "wall_time_ms", "task_type_coverage", "custom_experiment_present", "token_count", "cost_usd", "human_review_minutes", "human_review_notes", "artifact_count", "evidence_count", "commit_identity", "environment_identity"], + "properties": { + "completion": { "$ref": "#/$defs/booleanMetric" }, + "hard_error": { "$ref": "#/$defs/booleanMetric" }, + "wall_time_ms": { "$ref": "#/$defs/nonNegativeNumberMetric" }, + "task_type_coverage": { "$ref": "#/$defs/fractionMetric" }, + "custom_experiment_present": { "$ref": "#/$defs/booleanMetric" }, + "token_count": { "$ref": "#/$defs/nonNegativeIntegerMetric" }, + "cost_usd": { "$ref": "#/$defs/nonNegativeNumberMetric" }, + "human_review_minutes": { "$ref": "#/$defs/nonNegativeNumberMetric" }, + "human_review_notes": { "$ref": "#/$defs/stringMetric" }, + "artifact_count": { "$ref": "#/$defs/nonNegativeIntegerMetric" }, + "evidence_count": { "$ref": "#/$defs/nonNegativeIntegerMetric" }, + "commit_identity": { "$ref": "#/$defs/stringMetric" }, + "environment_identity": { "$ref": "#/$defs/stringMetric" } + } + } + }, + "$defs": { + "unmeasuredMetric": { + "type": "object", + "additionalProperties": false, + "required": ["status", "value", "reason"], + "properties": { + "status": { "enum": ["unavailable", "not_run", "blocked"] }, + "value": { "type": "null" }, + "reason": { "type": "string", "minLength": 1, "maxLength": 300 } + } + }, + "measuredBoolean": { + "type": "object", + "additionalProperties": false, + "required": ["status", "value", "source"], + "properties": { "status": { "const": "measured" }, "value": { "type": "boolean" }, "source": { "type": "string", "minLength": 1, "maxLength": 200 } } + }, + "measuredNumber": { + "type": "object", + "additionalProperties": false, + "required": ["status", "value", "source"], + "properties": { "status": { "const": "measured" }, "value": { "type": "number", "minimum": 0 }, "source": { "type": "string", "minLength": 1, "maxLength": 200 } } + }, + "measuredInteger": { + "type": "object", + "additionalProperties": false, + "required": ["status", "value", "source"], + "properties": { "status": { "const": "measured" }, "value": { "type": "integer", "minimum": 0 }, "source": { "type": "string", "minLength": 1, "maxLength": 200 } } + }, + "measuredFraction": { + "type": "object", + "additionalProperties": false, + "required": ["status", "value", "source"], + "properties": { "status": { "const": "measured" }, "value": { "type": "number", "minimum": 0, "maximum": 1 }, "source": { "type": "string", "minLength": 1, "maxLength": 200 } } + }, + "measuredString": { + "type": "object", + "additionalProperties": false, + "required": ["status", "value", "source"], + "properties": { "status": { "const": "measured" }, "value": { "type": "string", "minLength": 1, "maxLength": 500 }, "source": { "type": "string", "minLength": 1, "maxLength": 200 } } + }, + "booleanMetric": { "oneOf": [{ "$ref": "#/$defs/measuredBoolean" }, { "$ref": "#/$defs/unmeasuredMetric" }] }, + "nonNegativeNumberMetric": { "oneOf": [{ "$ref": "#/$defs/measuredNumber" }, { "$ref": "#/$defs/unmeasuredMetric" }] }, + "nonNegativeIntegerMetric": { "oneOf": [{ "$ref": "#/$defs/measuredInteger" }, { "$ref": "#/$defs/unmeasuredMetric" }] }, + "fractionMetric": { "oneOf": [{ "$ref": "#/$defs/measuredFraction" }, { "$ref": "#/$defs/unmeasuredMetric" }] }, + "stringMetric": { "oneOf": [{ "$ref": "#/$defs/measuredString" }, { "$ref": "#/$defs/unmeasuredMetric" }] }, + "taskType": { "enum": ["statistical_analysis", "regression_prediction", "time_series_forecasting", "classification", "clustering", "evaluation_ranking", "optimization", "simulation", "custom_experiment"] }, + "taskTypes": { "type": "array", "uniqueItems": true, "items": { "$ref": "#/$defs/taskType" } } + } +} diff --git a/src/benchmark/contracts.ts b/src/benchmark/contracts.ts new file mode 100644 index 0000000..8ddf4e3 --- /dev/null +++ b/src/benchmark/contracts.ts @@ -0,0 +1,154 @@ +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { Ajv2020 } from "ajv/dist/2020.js"; +import type { ErrorObject, ValidateFunction } from "ajv"; +import type { + BenchmarkManifest, + BenchmarkMetric, + BenchmarkResult, + BlockedMetric, + MeasuredMetric, + MetricValue, + NotRunMetric, + UnavailableMetric +} from "./types.js"; + +export type BenchmarkContractName = "benchmark-manifest" | "benchmark-result"; + +const schemaFiles: Record = { + "benchmark-manifest": "benchmark-manifest.v1.json", + "benchmark-result": "benchmark-result.v1.json" +}; + +export class BenchmarkContractError extends Error { + readonly contract: BenchmarkContractName; + readonly validation_errors: ErrorObject[]; + + constructor(contract: BenchmarkContractName, message: string, validationErrors: ErrorObject[] = []) { + super(`Benchmark contract ${contract} failed: ${message}`); + this.name = "BenchmarkContractError"; + this.contract = contract; + this.validation_errors = validationErrors; + } +} + +class BenchmarkSchemaRegistry { + readonly #validators = new Map(); + + constructor(schemaDirectory = fileURLToPath(new URL("../../schemas/", import.meta.url))) { + const ajv = new Ajv2020({ allErrors: true, strict: true, allowUnionTypes: true }); + ajv.addFormat("uri", { + type: "string", + validate: (value: string) => { + try { + const url = new URL(value); + return url.protocol === "https:" || url.protocol === "http:"; + } catch { + return false; + } + } + }); + for (const [name, filename] of Object.entries(schemaFiles) as Array<[BenchmarkContractName, string]>) { + const schema = JSON.parse(readFileSync(new URL(filename, `file://${schemaDirectory}/`), "utf8")) as object; + this.#validators.set(name, ajv.compile(schema)); + } + } + + validate(name: BenchmarkContractName, value: unknown): T { + const validator = this.#validators.get(name); + if (!validator) { + throw new BenchmarkContractError(name, "validator is unavailable"); + } + if (!validator(value)) { + const errors = validator.errors ?? []; + const message = errors.map((error) => `${error.instancePath || "/"} ${error.message ?? "is invalid"}`).join("; "); + const pathMessage = errors.some((error) => error.keyword === "pattern" && ["/package_path", "/license/notice_path", "/reference_policy/relative_path"].includes(error.instancePath)) + ? `path must be a normalized relative path; ${message}` + : message; + throw new BenchmarkContractError(name, pathMessage, errors); + } + return value as T; + } +} + +const registry = new BenchmarkSchemaRegistry(); + +function validateManifestSemantics(manifest: BenchmarkManifest): void { + const allowed = new Set(manifest.allowed_task_types); + const unexpected = manifest.expected_task_types.find((taskType) => !allowed.has(taskType)); + if (unexpected) { + throw new BenchmarkContractError("benchmark-manifest", `expected task type ${unexpected} is not allowed`); + } + const checkIds = manifest.hard_checks.map((check) => check.id); + if (new Set(checkIds).size !== checkIds.length) { + throw new BenchmarkContractError("benchmark-manifest", "hard check ids must be unique"); + } + const reference = manifest.reference_policy; + if (reference.availability === "included" && reference.relative_path === null) { + throw new BenchmarkContractError("benchmark-manifest", "included reference requires a relative_path"); + } + if (reference.availability === "unavailable" && (reference.relative_path !== null || reference.sha256 !== null)) { + throw new BenchmarkContractError("benchmark-manifest", "unavailable reference cannot declare a path or digest"); + } + if (manifest.license.redistribution !== "permitted" && reference.availability === "included") { + throw new BenchmarkContractError("benchmark-manifest", "non-redistributable reference material cannot be included"); + } +} + +function measuredBoolean(metric: BenchmarkMetric, expected: boolean): boolean { + return metric.status === "measured" && metric.value === expected; +} + +function validateResultSemantics(result: BenchmarkResult): void { + const hardCheckIds = result.hard_checks.map((check) => check.id); + if (new Set(hardCheckIds).size !== hardCheckIds.length) { + throw new BenchmarkContractError("benchmark-result", "hard check ids must be unique"); + } + if (result.outcome === "completed") { + if (result.state !== "measured" || !measuredBoolean(result.metrics.completion, true) || !measuredBoolean(result.metrics.hard_error, false) || result.hard_checks.some((check) => check.status !== "passed")) { + throw new BenchmarkContractError("benchmark-result", "completed result contradicts completion, hard-error, state, or hard-check evidence"); + } + } else if (result.metrics.completion.status === "measured" && result.metrics.completion.value === true) { + throw new BenchmarkContractError("benchmark-result", "non-completed result cannot report completion=true"); + } + if (result.state === "blocked" && (result.outcome !== "blocked_policy" || result.policy_events.length === 0)) { + throw new BenchmarkContractError("benchmark-result", "blocked state requires blocked_policy outcome and a policy event"); + } + if (result.state === "not_run" && result.outcome !== "not_run") { + throw new BenchmarkContractError("benchmark-result", "not_run state requires not_run outcome"); + } + if (result.state === "measured" && result.outcome === "not_run") { + throw new BenchmarkContractError("benchmark-result", "measured state cannot have not_run outcome"); + } + if ((result.outcome === "hard_error" || result.outcome === "blocked_policy") && !measuredBoolean(result.metrics.hard_error, true)) { + throw new BenchmarkContractError("benchmark-result", `${result.outcome} requires hard_error=true`); + } +} + +export function validateBenchmarkManifest(value: unknown): BenchmarkManifest { + const manifest = registry.validate("benchmark-manifest", value); + validateManifestSemantics(manifest); + return manifest; +} + +export function validateBenchmarkResult(value: unknown): BenchmarkResult { + const result = registry.validate("benchmark-result", value); + validateResultSemantics(result); + return result; +} + +export function measuredMetric(value: T, source: string): MeasuredMetric { + return { status: "measured", value, source }; +} + +export function unavailableMetric(reason: string): UnavailableMetric { + return { status: "unavailable", value: null, reason }; +} + +export function notRunMetric(reason: string): NotRunMetric { + return { status: "not_run", value: null, reason }; +} + +export function blockedMetric(reason: string): BlockedMetric { + return { status: "blocked", value: null, reason }; +} diff --git a/src/benchmark/index.ts b/src/benchmark/index.ts new file mode 100644 index 0000000..8c346da --- /dev/null +++ b/src/benchmark/index.ts @@ -0,0 +1,5 @@ +export * from "./contracts.js"; +export * from "./report.js"; +export * from "./runner.js"; +export * from "./synthetic.js"; +export * from "./types.js"; diff --git a/src/benchmark/report.ts b/src/benchmark/report.ts new file mode 100644 index 0000000..191ddee --- /dev/null +++ b/src/benchmark/report.ts @@ -0,0 +1,134 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import { resolve } from "node:path"; +import { BENCHMARK_SCHEMA_VERSION } from "./types.js"; +import type { + BenchmarkAggregateReport, + BenchmarkMetric, + BenchmarkMetricAggregate, + BenchmarkMetrics, + BenchmarkResult, + MetricValue +} from "./types.js"; +import { validateBenchmarkResult } from "./contracts.js"; + +const metricNames: Array = [ + "completion", + "hard_error", + "wall_time_ms", + "task_type_coverage", + "custom_experiment_present", + "token_count", + "cost_usd", + "human_review_minutes", + "human_review_notes", + "artifact_count", + "evidence_count", + "commit_identity", + "environment_identity" +]; + +function aggregateMetric(metrics: Array>): BenchmarkMetricAggregate { + return { + measured_count: metrics.filter((metric) => metric.status === "measured").length, + unavailable_count: metrics.filter((metric) => metric.status === "unavailable").length, + not_run_count: metrics.filter((metric) => metric.status === "not_run").length, + blocked_count: metrics.filter((metric) => metric.status === "blocked").length, + values: metrics.flatMap((metric) => metric.status === "measured" ? [metric.value] : []) + }; +} + +function stableResultOrder(left: BenchmarkResult, right: BenchmarkResult): number { + return left.case_id.localeCompare(right.case_id) || left.variant.localeCompare(right.variant) || left.run_id.localeCompare(right.run_id); +} + +function assertComparableVariants(results: readonly BenchmarkResult[]): void { + const groups = new Map(); + for (const result of results) { + const group = groups.get(result.case_id) ?? []; + group.push(result); + groups.set(result.case_id, group); + } + for (const [caseId, group] of groups) { + const agent = group.find((result) => result.variant === "agent"); + const oneShot = group.find((result) => result.variant === "one_shot"); + if (agent && oneShot && agent.frozen_case_sha256 !== oneShot.frozen_case_sha256) { + throw new Error(`benchmark variants for ${caseId} do not share a frozen case`); + } + } +} + +export function aggregateBenchmarkResults(input: readonly BenchmarkResult[], suiteId = "synthetic-v1"): BenchmarkAggregateReport { + const results = input.map((entry) => validateBenchmarkResult(structuredClone(entry))).sort(stableResultOrder); + assertComparableVariants(results); + const metrics = Object.fromEntries(metricNames.map((name) => [name, aggregateMetric(results.map((result) => result.metrics[name]))])) as Record; + return { + schema_version: BENCHMARK_SCHEMA_VERSION, + report_kind: "benchmark_aggregate", + suite_id: suiteId, + summary: { + total_runs: results.length, + measured_runs: results.filter((result) => result.state === "measured").length, + completed_runs: results.filter((result) => result.outcome === "completed" && result.metrics.completion.status === "measured" && result.metrics.completion.value === true).length, + hard_error_runs: results.filter((result) => result.state === "measured" && result.outcome === "hard_error").length, + blocked_runs: results.filter((result) => result.state === "blocked").length, + not_run_runs: results.filter((result) => result.state === "not_run").length + }, + metrics, + results + }; +} + +function metricDisplay(metric: BenchmarkMetric): string { + if (metric.status === "measured") return `measured: ${String(metric.value)}`; + return `${metric.status}: ${metric.reason}`; +} + +export function renderBenchmarkMarkdown(report: BenchmarkAggregateReport): string { + const lines = [ + "# Benchmark aggregate report", + "", + `Suite: \`${report.suite_id}\``, + "", + "## Status legend", + "", + "- `measured`: the harness observed a value.", + "- `not_run`: the run was intentionally not executed.", + "- `blocked`: policy prevented the run or metric from being accepted.", + "- `unavailable`: a run occurred but the adapter or reviewer did not provide the metric.", + "", + "## Summary", + "", + `- Completion: ${report.summary.completed_runs} / ${report.summary.total_runs}`, + `- Measured runs: ${report.summary.measured_runs}`, + `- Hard-error runs: ${report.summary.hard_error_runs}`, + `- Blocked runs: ${report.summary.blocked_runs}`, + `- Not-run runs: ${report.summary.not_run_runs}`, + "", + "A failed, blocked, or not-run result is never counted as completion.", + "", + "## Runs", + "", + "| Case | Variant | State | Outcome | Completion | Hard error | Coverage | Token count | Cost USD | Human review | Artifacts | Evidence |", + "| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |" + ]; + for (const result of report.results) { + lines.push(`| ${result.case_id} | ${result.variant} | ${result.state} | ${result.outcome} | ${metricDisplay(result.metrics.completion)} | ${metricDisplay(result.metrics.hard_error)} | ${metricDisplay(result.metrics.task_type_coverage)} | ${metricDisplay(result.metrics.token_count)} | ${metricDisplay(result.metrics.cost_usd)} | ${metricDisplay(result.metrics.human_review_minutes)} | ${metricDisplay(result.metrics.artifact_count)} | ${metricDisplay(result.metrics.evidence_count)} |`); + } + lines.push("", "## Metric availability", "", "| Metric | measured | unavailable | not_run | blocked |", "| --- | ---: | ---: | ---: | ---: |"); + for (const name of metricNames) { + const aggregate = report.metrics[name]; + lines.push(`| ${name} | ${aggregate.measured_count} | ${aggregate.unavailable_count} | ${aggregate.not_run_count} | ${aggregate.blocked_count} |`); + } + return `${lines.join("\n")}\n`; +} + +export async function writeBenchmarkReports(report: BenchmarkAggregateReport, outputDirectory: string): Promise<{ json_path: string; markdown_path: string }> { + await mkdir(outputDirectory, { recursive: true }); + const jsonPath = resolve(outputDirectory, "benchmark-report.json"); + const markdownPath = resolve(outputDirectory, "benchmark-report.md"); + await Promise.all([ + writeFile(jsonPath, `${JSON.stringify(report, null, 2)}\n`, "utf8"), + writeFile(markdownPath, renderBenchmarkMarkdown(report), "utf8") + ]); + return { json_path: jsonPath, markdown_path: markdownPath }; +} diff --git a/src/benchmark/run-synthetic.ts b/src/benchmark/run-synthetic.ts new file mode 100644 index 0000000..a9bb9fc --- /dev/null +++ b/src/benchmark/run-synthetic.ts @@ -0,0 +1,34 @@ +import { resolve } from "node:path"; +import { pathToFileURL } from "node:url"; +import { runSyntheticBenchmarks } from "./synthetic.js"; + +function parseOutputDirectory(argv: readonly string[]): string { + const outputIndex = argv.indexOf("--output"); + if (outputIndex === -1) return resolve("benchmarks/output/synthetic"); + const value = argv[outputIndex + 1]; + if (value === undefined || value.startsWith("--")) { + throw new Error("--output requires a directory"); + } + return resolve(value); +} + +export async function main(argv = process.argv.slice(2)): Promise { + const outputDirectory = parseOutputDirectory(argv); + const run = await runSyntheticBenchmarks({ output_directory: outputDirectory }); + process.stdout.write(`${JSON.stringify({ + suite_id: run.report.suite_id, + total_runs: run.report.summary.total_runs, + completed_runs: run.report.summary.completed_runs, + json_report: run.paths.json_path, + markdown_report: run.paths.markdown_path + }, null, 2)}\n`); +} + +const entryUrl = process.argv[1] === undefined ? null : pathToFileURL(resolve(process.argv[1])).href; +if (entryUrl === import.meta.url) { + main().catch((error: unknown) => { + const message = error instanceof Error ? error.message : String(error); + process.stderr.write(`Synthetic benchmark failed: ${message}\n`); + process.exitCode = 1; + }); +} diff --git a/src/benchmark/runner.ts b/src/benchmark/runner.ts new file mode 100644 index 0000000..9ed3b6a --- /dev/null +++ b/src/benchmark/runner.ts @@ -0,0 +1,405 @@ +import { createHash } from "node:crypto"; +import { readdir, readFile, realpath, stat } from "node:fs/promises"; +import { isAbsolute, relative, resolve, sep } from "node:path"; +import type { TaskType } from "../contracts/types.js"; +import { + measuredMetric, + unavailableMetric, + validateBenchmarkManifest, + validateBenchmarkResult +} from "./contracts.js"; +import type { + BenchmarkAdapterOutput, + BenchmarkClock, + BenchmarkHardCheckResult, + BenchmarkIdentity, + BenchmarkManifest, + BenchmarkPackageFile, + BenchmarkResult, + BenchmarkSolveContext +} from "./types.js"; + +const TEXT_FILE_LIMIT_BYTES = 1_000_000; + +export interface RunBenchmarkCaseOptions { + case_root: string; + manifest: BenchmarkManifest; + adapter: import("./types.js").BenchmarkAdapter; + identity: BenchmarkIdentity; + clock?: BenchmarkClock; +} + +const systemClock: BenchmarkClock = { + async measure(operation: () => Promise): Promise<{ value: T; duration_ms: number }> { + const started = process.hrtime.bigint(); + try { + return { value: await operation(), duration_ms: Number(process.hrtime.bigint() - started) / 1_000_000 }; + } catch (error) { + if (error && typeof error === "object") { + Object.assign(error, { benchmark_duration_ms: Number(process.hrtime.bigint() - started) / 1_000_000 }); + } + throw error; + } + } +}; + +function sha256(value: string | Buffer): string { + return createHash("sha256").update(value).digest("hex"); +} + +function assertSafeRelativePath(path: string): void { + if (isAbsolute(path) || path.split(/[\\/]/u).includes("..") || path.includes("\\")) { + throw new Error("benchmark package path must remain inside its case root"); + } +} + +async function listFiles(root: string, current = root): Promise { + const entries = await readdir(current, { withFileTypes: true }); + const files: string[] = []; + for (const entry of entries.sort((left, right) => left.name.localeCompare(right.name))) { + const path = resolve(current, entry.name); + if (entry.isSymbolicLink()) { + throw new Error("benchmark package cannot contain symbolic links"); + } + if (entry.isDirectory()) { + files.push(...await listFiles(root, path)); + } else if (entry.isFile()) { + files.push(relative(root, path).split(sep).join("/")); + } + } + return files; +} + +function mediaType(path: string): string { + const extension = path.split(".").pop()?.toLowerCase(); + const types: Record = { + csv: "text/csv", + json: "application/json", + md: "text/markdown", + txt: "text/plain", + yaml: "application/yaml", + yml: "application/yaml" + }; + return extension ? (types[extension] ?? "text/plain") : "text/plain"; +} + +async function freezePackage(caseRoot: string, packagePath: string): Promise<{ files: BenchmarkPackageFile[]; digest: string }> { + assertSafeRelativePath(packagePath); + const canonicalCaseRoot = await realpath(caseRoot); + const packageRoot = resolve(canonicalCaseRoot, packagePath); + const canonicalPackageRoot = await realpath(packageRoot); + if (canonicalPackageRoot !== canonicalCaseRoot && !canonicalPackageRoot.startsWith(`${canonicalCaseRoot}${sep}`)) { + throw new Error("benchmark package resolves outside its case root"); + } + if (!(await stat(canonicalPackageRoot)).isDirectory()) { + throw new Error("benchmark package path must identify a directory"); + } + const relativePaths = await listFiles(canonicalPackageRoot); + if (relativePaths.length === 0) { + throw new Error("benchmark package must contain at least one file"); + } + const files: BenchmarkPackageFile[] = []; + const digestParts: string[] = []; + for (const relativePath of relativePaths) { + const bytes = await readFile(resolve(canonicalPackageRoot, relativePath)); + if (bytes.byteLength > TEXT_FILE_LIMIT_BYTES) { + throw new Error(`benchmark package file exceeds ${TEXT_FILE_LIMIT_BYTES} bytes`); + } + if (bytes.includes(0)) { + throw new Error("binary benchmark package files are not supported by the solve adapter context"); + } + const fileDigest = sha256(bytes); + files.push({ relative_path: relativePath, media_type: mediaType(relativePath), sha256: fileDigest, content: bytes.toString("utf8") }); + digestParts.push(`${relativePath}\0${fileDigest}\n`); + } + return { files, digest: sha256(digestParts.join("")) }; +} + +async function readReference(caseRoot: string, manifest: BenchmarkManifest): Promise { + const path = manifest.reference_policy.relative_path; + if (manifest.reference_policy.availability !== "included" || path === null) { + return null; + } + assertSafeRelativePath(path); + const canonicalCaseRoot = await realpath(caseRoot); + const canonicalReference = await realpath(resolve(canonicalCaseRoot, path)); + if (!canonicalReference.startsWith(`${canonicalCaseRoot}${sep}`)) { + throw new Error("benchmark reference resolves outside its case root"); + } + const reference = await readFile(canonicalReference, "utf8"); + const declaredDigest = manifest.reference_policy.sha256; + if (declaredDigest !== null && sha256(reference) !== declaredDigest) { + throw new Error("benchmark reference digest does not match the manifest"); + } + return reference; +} + +function referenceStrings(value: unknown): string[] { + if (typeof value === "string") return [value]; + if (Array.isArray(value)) return value.flatMap(referenceStrings); + if (value !== null && typeof value === "object") return Object.values(value).flatMap(referenceStrings); + return []; +} + +function referenceSegments(reference: string): string[] { + try { + return [reference, ...referenceStrings(JSON.parse(reference))]; + } catch { + return [reference]; + } +} + +function normalizedWindows(value: string, minimumLength: number): Set { + const normalized = value.normalize("NFKC").replace(/\s+/gu, " ").trim(); + const windows = new Set(); + if (normalized.length < minimumLength) { + return windows; + } + for (let offset = 0; offset <= normalized.length - minimumLength; offset += Math.max(1, Math.floor(minimumLength / 4))) { + windows.add(normalized.slice(offset, offset + minimumLength)); + } + windows.add(normalized.slice(-minimumLength)); + return windows; +} + +function leakedReference(output: string, reference: string | null, minimumLength: number): string | null { + if (reference === null || output.length === 0) return null; + const normalizedOutput = output.normalize("NFKC").replace(/\s+/gu, " "); + for (const segment of referenceSegments(reference)) { + for (const window of normalizedWindows(segment, minimumLength)) { + if (normalizedOutput.includes(window)) { + return sha256(window); + } + } + } + return null; +} + +function normalizeObservedTasks(output: BenchmarkAdapterOutput, manifest: BenchmarkManifest): TaskType[] { + const allowed = new Set(manifest.allowed_task_types); + return [...new Set(output.observed_task_types)].filter((taskType) => allowed.has(taskType)).sort(); +} + +function outputContractFailure(output: BenchmarkAdapterOutput, manifest: BenchmarkManifest): string | null { + if (!output || typeof output !== "object") return "adapter output is not an object"; + if (output.status !== "success" && output.status !== "failed") return "adapter output status is invalid"; + if (!Array.isArray(output.observed_task_types) || !Array.isArray(output.hard_checks) || !Array.isArray(output.artifacts) || !Array.isArray(output.evidence)) return "adapter output collections are invalid"; + if (!output.usage || (output.usage.token_count !== null && (!Number.isInteger(output.usage.token_count) || output.usage.token_count < 0))) return "adapter token usage is invalid"; + if (output.usage.cost_usd !== null && (!Number.isFinite(output.usage.cost_usd) || output.usage.cost_usd < 0)) return "adapter cost usage is invalid"; + if (typeof output.output_text !== "string") return "adapter output text is invalid"; + if (output.artifacts.some((item) => typeof item !== "string") || output.evidence.some((item) => typeof item !== "string")) return "adapter inventory is invalid"; + const allowed = new Set(manifest.allowed_task_types); + if (output.observed_task_types.some((taskType) => !allowed.has(taskType))) return "adapter reported a disallowed task type"; + const expectedChecks = new Set(manifest.hard_checks.map((check) => check.id)); + if (output.hard_checks.some((check) => !check || typeof check.id !== "string" || typeof check.passed !== "boolean" || !expectedChecks.has(check.id))) return "adapter hard-check output is invalid"; + return null; +} + +function hardChecks(output: BenchmarkAdapterOutput, manifest: BenchmarkManifest): BenchmarkHardCheckResult[] { + const observations = new Map(output.hard_checks.map((check) => [check.id, check])); + return manifest.hard_checks.map((definition) => { + const observation = observations.get(definition.id); + const note = observation?.note; + return note === undefined + ? { id: definition.id, status: observation?.passed === true ? "passed" : "failed" } + : { id: definition.id, status: observation?.passed === true ? "passed" : "failed", note }; + }); +} + +function safeError(error: unknown): { class: string; message: string; fingerprint: string } { + const className = error instanceof Error ? error.name : "UnknownError"; + const raw = error instanceof Error ? error.message : String(error); + const fingerprint = sha256(`${className}\0${raw}`); + return { class: className.slice(0, 200) || "Error", message: `adapter failed (${fingerprint.slice(0, 12)})`, fingerprint }; +} + +function runId(caseId: string, variant: string, frozenDigest: string): string { + return `${caseId}-${variant.replaceAll("_", "-")}-${frozenDigest.slice(0, 12)}`; +} + +function coverage(expected: readonly TaskType[], observed: readonly TaskType[]): number { + if (expected.length === 0) return 1; + const observedSet = new Set(observed); + return expected.filter((taskType) => observedSet.has(taskType)).length / expected.length; +} + +function usageMetric(value: number | null, source: string) { + return value === null ? unavailableMetric("adapter_did_not_report") : measuredMetric(value, source); +} + +export async function runBenchmarkCase(options: RunBenchmarkCaseOptions): Promise { + const manifest = validateBenchmarkManifest(structuredClone(options.manifest)); + const expectedAdapterId = options.adapter.variant === "agent" ? manifest.runtime.agent_adapter_id : manifest.runtime.one_shot_adapter_id; + if (options.adapter.id !== expectedAdapterId) { + throw new Error(`adapter id ${options.adapter.id} does not match frozen manifest runtime ${expectedAdapterId}`); + } + const frozen = await freezePackage(options.case_root, manifest.package_path); + const reference = await readReference(options.case_root, manifest); + const context: BenchmarkSolveContext = Object.freeze({ + case_id: manifest.case_id, + variant: options.adapter.variant, + frozen_case_sha256: frozen.digest, + package_files: Object.freeze(frozen.files.map((file) => Object.freeze(file))), + budget: Object.freeze(structuredClone(manifest.budget)), + expected_task_types: Object.freeze([...manifest.expected_task_types]), + hard_checks: Object.freeze(manifest.hard_checks.map((check) => Object.freeze({ ...check }))) + }); + const clock = options.clock ?? systemClock; + let duration = 0; + let output: BenchmarkAdapterOutput; + try { + const measured = await clock.measure(async () => { + let timer: NodeJS.Timeout | undefined; + const timeout = new Promise((_resolve, reject) => { + timer = setTimeout(() => reject(new Error("benchmark wall-time budget exceeded")), manifest.budget.max_wall_time_ms); + timer.unref(); + }); + try { + return await Promise.race([options.adapter.run(context), timeout]); + } finally { + if (timer !== undefined) clearTimeout(timer); + } + }); + duration = measured.duration_ms; + output = measured.value; + } catch (error) { + duration = typeof (error as { benchmark_duration_ms?: unknown })?.benchmark_duration_ms === "number" + ? (error as { benchmark_duration_ms: number }).benchmark_duration_ms + : 0; + const sanitized = safeError(error); + return validateBenchmarkResult({ + schema_version: "1.0.0", + case_id: manifest.case_id, + variant: options.adapter.variant, + adapter_id: options.adapter.id, + run_id: runId(manifest.case_id, options.adapter.variant, frozen.digest), + frozen_case_sha256: frozen.digest, + state: "measured", + outcome: "hard_error", + observed_task_types: [], + hard_checks: manifest.hard_checks.map((check) => ({ id: check.id, status: "failed" as const })), + policy_events: [], + error: sanitized, + metrics: { + completion: measuredMetric(false, "harness_scoring"), + hard_error: measuredMetric(true, "harness_scoring"), + wall_time_ms: measuredMetric(duration, "harness_clock"), + task_type_coverage: measuredMetric(0, "harness_scoring"), + custom_experiment_present: measuredMetric(false, "harness_scoring"), + token_count: unavailableMetric("adapter_failed_before_usage_report"), + cost_usd: unavailableMetric("adapter_failed_before_usage_report"), + human_review_minutes: unavailableMetric("not_reviewed"), + human_review_notes: unavailableMetric("not_reviewed"), + artifact_count: measuredMetric(0, "adapter_inventory"), + evidence_count: measuredMetric(0, "adapter_inventory"), + commit_identity: measuredMetric(options.identity.commit, "git_commit"), + environment_identity: measuredMetric(options.identity.environment, "runtime_environment") + } + }); + } + + const contractFailure = outputContractFailure(output, manifest); + if (contractFailure !== null) { + const sanitized = safeError(new Error(contractFailure)); + return validateBenchmarkResult({ + schema_version: "1.0.0", + case_id: manifest.case_id, + variant: options.adapter.variant, + adapter_id: options.adapter.id, + run_id: runId(manifest.case_id, options.adapter.variant, frozen.digest), + frozen_case_sha256: frozen.digest, + state: "measured", + outcome: "hard_error", + observed_task_types: [], + hard_checks: manifest.hard_checks.map((check) => ({ id: check.id, status: "failed" as const })), + policy_events: [], + error: sanitized, + metrics: { + completion: measuredMetric(false, "harness_scoring"), + hard_error: measuredMetric(true, "harness_scoring"), + wall_time_ms: measuredMetric(duration, "harness_clock"), + task_type_coverage: measuredMetric(0, "harness_scoring"), + custom_experiment_present: measuredMetric(false, "harness_scoring"), + token_count: unavailableMetric("invalid_adapter_output"), + cost_usd: unavailableMetric("invalid_adapter_output"), + human_review_minutes: unavailableMetric("not_reviewed"), + human_review_notes: unavailableMetric("not_reviewed"), + artifact_count: measuredMetric(0, "invalid_adapter_output"), + evidence_count: measuredMetric(0, "invalid_adapter_output"), + commit_identity: measuredMetric(options.identity.commit, "git_commit"), + environment_identity: measuredMetric(options.identity.environment, "runtime_environment") + } + }); + } + + const observedTasks = normalizeObservedTasks(output, manifest); + const checks = hardChecks(output, manifest); + const leakedFingerprint = leakedReference(output.output_text, reference, manifest.blind_policy.minimum_reference_match_characters); + if (leakedFingerprint !== null) { + return validateBenchmarkResult({ + schema_version: "1.0.0", + case_id: manifest.case_id, + variant: options.adapter.variant, + adapter_id: options.adapter.id, + run_id: runId(manifest.case_id, options.adapter.variant, frozen.digest), + frozen_case_sha256: frozen.digest, + state: "blocked", + outcome: "blocked_policy", + observed_task_types: observedTasks, + hard_checks: manifest.hard_checks.map((check) => ({ id: check.id, status: "blocked" as const })), + policy_events: [{ type: "same_problem_answer_detected", action: "blocked", fingerprint: leakedFingerprint }], + error: { class: "BlindPolicyViolation", message: `same-problem answer match blocked (${leakedFingerprint.slice(0, 12)})`, fingerprint: leakedFingerprint }, + metrics: { + completion: measuredMetric(false, "harness_scoring"), + hard_error: measuredMetric(true, "harness_scoring"), + wall_time_ms: measuredMetric(duration, "harness_clock"), + task_type_coverage: measuredMetric(coverage(manifest.expected_task_types, observedTasks), "harness_scoring"), + custom_experiment_present: measuredMetric(observedTasks.includes("custom_experiment"), "harness_scoring"), + token_count: usageMetric(output.usage.token_count, "adapter_usage"), + cost_usd: usageMetric(output.usage.cost_usd, "adapter_usage"), + human_review_minutes: unavailableMetric("not_reviewed"), + human_review_notes: unavailableMetric("not_reviewed"), + artifact_count: measuredMetric(0, "blocked_output_discarded"), + evidence_count: measuredMetric(0, "blocked_output_discarded"), + commit_identity: measuredMetric(options.identity.commit, "git_commit"), + environment_identity: measuredMetric(options.identity.environment, "runtime_environment") + } + }); + } + + const adapterFailed = output.status !== "success"; + const checksFailed = checks.some((check) => check.status !== "passed"); + const withinBudget = duration <= manifest.budget.max_wall_time_ms; + const complete = !adapterFailed && !checksFailed && withinBudget; + const outputError = output.error === undefined ? null : safeError(new Error(`${output.error.class}: ${output.error.message}`)); + const outcome = complete ? "completed" : adapterFailed ? "hard_error" : "incomplete"; + return validateBenchmarkResult({ + schema_version: "1.0.0", + case_id: manifest.case_id, + variant: options.adapter.variant, + adapter_id: options.adapter.id, + run_id: runId(manifest.case_id, options.adapter.variant, frozen.digest), + frozen_case_sha256: frozen.digest, + state: "measured", + outcome, + observed_task_types: observedTasks, + hard_checks: checks, + policy_events: [], + error: outputError, + metrics: { + completion: measuredMetric(complete, "harness_scoring"), + hard_error: measuredMetric(adapterFailed, "harness_scoring"), + wall_time_ms: measuredMetric(duration, "harness_clock"), + task_type_coverage: measuredMetric(coverage(manifest.expected_task_types, observedTasks), "harness_scoring"), + custom_experiment_present: measuredMetric(observedTasks.includes("custom_experiment"), "harness_scoring"), + token_count: usageMetric(output.usage.token_count, "adapter_usage"), + cost_usd: usageMetric(output.usage.cost_usd, "adapter_usage"), + human_review_minutes: unavailableMetric("not_reviewed"), + human_review_notes: unavailableMetric("not_reviewed"), + artifact_count: measuredMetric(output.artifacts.length, "adapter_inventory"), + evidence_count: measuredMetric(output.evidence.length, "adapter_inventory"), + commit_identity: measuredMetric(options.identity.commit, "git_commit"), + environment_identity: measuredMetric(options.identity.environment, "runtime_environment") + } + }); +} diff --git a/src/benchmark/synthetic.ts b/src/benchmark/synthetic.ts new file mode 100644 index 0000000..86f7f34 --- /dev/null +++ b/src/benchmark/synthetic.ts @@ -0,0 +1,97 @@ +import { readFile } from "node:fs/promises"; +import { fileURLToPath } from "node:url"; +import { resolve } from "node:path"; +import type { TaskType } from "../contracts/types.js"; +import { validateBenchmarkManifest } from "./contracts.js"; +import { aggregateBenchmarkResults, writeBenchmarkReports } from "./report.js"; +import { runBenchmarkCase } from "./runner.js"; +import type { + BenchmarkAdapter, + BenchmarkAdapterOutput, + BenchmarkAggregateReport, + BenchmarkClock, + BenchmarkIdentity, + BenchmarkManifest, + BenchmarkResult, + BenchmarkSolveContext, + BenchmarkVariant +} from "./types.js"; + +const defaultBenchmarkRoot = fileURLToPath(new URL("../../benchmarks/synthetic/", import.meta.url)); +const caseDirectories = ["summary-statistics", "custom-threshold"] as const; + +const deterministicClock: BenchmarkClock = { + async measure(operation: () => Promise): Promise<{ value: T; duration_ms: number }> { + return { value: await operation(), duration_ms: 1 }; + } +}; + +const deterministicIdentity: BenchmarkIdentity = { + commit: "synthetic-fixture-v1", + environment: "deterministic-offline-v1" +}; + +function tasksFor(context: BenchmarkSolveContext): TaskType[] { + return [...context.expected_task_types]; +} + +function hardChecksFor(context: BenchmarkSolveContext): Array<{ id: string; passed: boolean }> { + return context.hard_checks.map((check) => ({ id: check.id, passed: true })); +} + +function deterministicOutput(context: BenchmarkSolveContext, variant: BenchmarkVariant): BenchmarkAdapterOutput { + const custom = context.expected_task_types.includes("custom_experiment"); + return { + status: "success", + observed_task_types: tasksFor(context), + hard_checks: hardChecksFor(context), + artifacts: custom ? ["experiment.json", "summary.md"] : ["summary.md"], + evidence: custom ? ["threshold-evidence", "summary-evidence"] : ["summary-evidence"], + usage: { token_count: null, cost_usd: null }, + output_text: `${variant} deterministic synthetic output for ${context.case_id}; ${custom ? "custom experiment included" : "statistical summary included"}.` + }; +} + +function syntheticAdapter(variant: BenchmarkVariant): BenchmarkAdapter { + return { + id: variant === "agent" ? "synthetic-agent-v1" : "synthetic-one-shot-v1", + variant, + run: async (context) => deterministicOutput(context, variant) + }; +} + +async function loadManifest(caseRoot: string): Promise { + return validateBenchmarkManifest(JSON.parse(await readFile(resolve(caseRoot, "manifest.json"), "utf8"))); +} + +export interface RunSyntheticBenchmarksOptions { + output_directory: string; + benchmark_root?: string; +} + +export interface SyntheticBenchmarkRun { + results: BenchmarkResult[]; + report: BenchmarkAggregateReport; + paths: { json_path: string; markdown_path: string }; +} + +export async function runSyntheticBenchmarks(options: RunSyntheticBenchmarksOptions): Promise { + const benchmarkRoot = options.benchmark_root ?? defaultBenchmarkRoot; + const results: BenchmarkResult[] = []; + for (const directory of caseDirectories) { + const caseRoot = resolve(benchmarkRoot, directory); + const manifest = await loadManifest(caseRoot); + for (const variant of ["agent", "one_shot"] as const) { + results.push(await runBenchmarkCase({ + case_root: caseRoot, + manifest, + adapter: syntheticAdapter(variant), + identity: deterministicIdentity, + clock: deterministicClock + })); + } + } + const report = aggregateBenchmarkResults(results, "synthetic-v1"); + const paths = await writeBenchmarkReports(report, options.output_directory); + return { results, report, paths }; +} diff --git a/src/benchmark/types.ts b/src/benchmark/types.ts new file mode 100644 index 0000000..28138c6 --- /dev/null +++ b/src/benchmark/types.ts @@ -0,0 +1,208 @@ +import type { TaskType } from "../contracts/types.js"; + +export const BENCHMARK_SCHEMA_VERSION = "1.0.0" as const; +export type BenchmarkSchemaVersion = typeof BENCHMARK_SCHEMA_VERSION; +export type BenchmarkVariant = "agent" | "one_shot"; +export type BenchmarkState = "measured" | "not_run" | "blocked"; +export type BenchmarkOutcome = "completed" | "incomplete" | "hard_error" | "blocked_policy" | "not_run"; +export type MetricStatus = "measured" | "unavailable" | "not_run" | "blocked"; +export type MetricValue = boolean | number | string; + +export interface BenchmarkLicense { + name: string; + spdx_id: string | null; + copyright_holder: string; + source_url: string | null; + redistribution: "permitted" | "user_supplied_only" | "metadata_only"; + notice_path: string | null; +} + +export interface BenchmarkBlindPolicy { + mode: "blind"; + solve_input: "package_only"; + same_problem_answers: "block"; + minimum_reference_match_characters: number; +} + +export interface BenchmarkReferencePolicy { + access: "scoring_only"; + availability: "included" | "user_supplied" | "unavailable"; + relative_path: string | null; + sha256: string | null; +} + +export interface BenchmarkBudget { + max_wall_time_ms: number; + max_tokens: number | null; + max_cost_usd: number | null; + max_human_review_minutes: number | null; +} + +export interface BenchmarkHardCheckDefinition { + id: string; + description: string; +} + +export interface BenchmarkManifest { + schema_version: BenchmarkSchemaVersion; + case_id: string; + package_path: string; + license: BenchmarkLicense; + blind_policy: BenchmarkBlindPolicy; + reference_policy: BenchmarkReferencePolicy; + runtime: { + agent_adapter_id: string; + one_shot_adapter_id: string; + }; + execution: { + kind: "local" | "docker"; + network_access: "disabled" | "research_gateway_only"; + }; + budget: BenchmarkBudget; + allowed_task_types: TaskType[]; + expected_task_types: TaskType[]; + hard_checks: BenchmarkHardCheckDefinition[]; +} + +export interface BenchmarkPackageFile { + relative_path: string; + media_type: string; + sha256: string; + content: string; +} + +export interface BenchmarkSolveContext { + case_id: string; + variant: BenchmarkVariant; + frozen_case_sha256: string; + package_files: readonly BenchmarkPackageFile[]; + budget: Readonly; + expected_task_types: readonly TaskType[]; + hard_checks: readonly BenchmarkHardCheckDefinition[]; +} + +export interface BenchmarkAdapterOutput { + status: "success" | "failed"; + observed_task_types: TaskType[]; + hard_checks: Array<{ id: string; passed: boolean; note?: string }>; + artifacts: string[]; + evidence: string[]; + usage: { + token_count: number | null; + cost_usd: number | null; + }; + output_text: string; + error?: { + class: string; + message: string; + }; +} + +export interface BenchmarkAdapter { + id: string; + variant: BenchmarkVariant; + run(context: Readonly): Promise; +} + +export interface BenchmarkClock { + measure(operation: () => Promise): Promise<{ value: T; duration_ms: number }>; +} + +export interface BenchmarkIdentity { + commit: string; + environment: string; +} + +export interface MeasuredMetric { + status: "measured"; + value: T; + source: string; +} + +export interface UnavailableMetric { + status: "unavailable"; + value: null; + reason: string; +} + +export interface NotRunMetric { + status: "not_run"; + value: null; + reason: string; +} + +export interface BlockedMetric { + status: "blocked"; + value: null; + reason: string; +} + +export type BenchmarkMetric = MeasuredMetric | UnavailableMetric | NotRunMetric | BlockedMetric; + +export interface BenchmarkMetrics { + completion: BenchmarkMetric; + hard_error: BenchmarkMetric; + wall_time_ms: BenchmarkMetric; + task_type_coverage: BenchmarkMetric; + custom_experiment_present: BenchmarkMetric; + token_count: BenchmarkMetric; + cost_usd: BenchmarkMetric; + human_review_minutes: BenchmarkMetric; + human_review_notes: BenchmarkMetric; + artifact_count: BenchmarkMetric; + evidence_count: BenchmarkMetric; + commit_identity: BenchmarkMetric; + environment_identity: BenchmarkMetric; +} + +export interface BenchmarkHardCheckResult { + id: string; + status: "passed" | "failed" | "blocked" | "not_run"; + note?: string; +} + +export interface BenchmarkPolicyEvent { + type: "same_problem_answer_detected"; + action: "blocked"; + fingerprint: string; +} + +export interface BenchmarkResult { + schema_version: BenchmarkSchemaVersion; + case_id: string; + variant: BenchmarkVariant; + adapter_id: string; + run_id: string; + frozen_case_sha256: string; + state: BenchmarkState; + outcome: BenchmarkOutcome; + observed_task_types: TaskType[]; + hard_checks: BenchmarkHardCheckResult[]; + policy_events: BenchmarkPolicyEvent[]; + error: { class: string; message: string; fingerprint?: string } | null; + metrics: BenchmarkMetrics; +} + +export interface BenchmarkMetricAggregate { + measured_count: number; + unavailable_count: number; + not_run_count: number; + blocked_count: number; + values: MetricValue[]; +} + +export interface BenchmarkAggregateReport { + schema_version: BenchmarkSchemaVersion; + report_kind: "benchmark_aggregate"; + suite_id: string; + summary: { + total_runs: number; + measured_runs: number; + completed_runs: number; + hard_error_runs: number; + blocked_runs: number; + not_run_runs: number; + }; + metrics: Record; + results: BenchmarkResult[]; +} diff --git a/tests/benchmark-contracts.test.ts b/tests/benchmark-contracts.test.ts new file mode 100644 index 0000000..dbbc0ea --- /dev/null +++ b/tests/benchmark-contracts.test.ts @@ -0,0 +1,123 @@ +import { describe, expect, it } from "vitest"; +import type { BenchmarkManifest, BenchmarkResult } from "../src/benchmark/types.js"; +import { + BenchmarkContractError, + measuredMetric, + unavailableMetric, + validateBenchmarkManifest, + validateBenchmarkResult +} from "../src/benchmark/contracts.js"; + +const manifest: BenchmarkManifest = { + schema_version: "1.0.0", + case_id: "synthetic-contract", + package_path: "package", + license: { + name: "Apache License 2.0", + spdx_id: "Apache-2.0", + copyright_holder: "modeling-agent contributors", + source_url: null, + redistribution: "permitted", + notice_path: "package/NOTICE.md" + }, + blind_policy: { + mode: "blind", + solve_input: "package_only", + same_problem_answers: "block", + minimum_reference_match_characters: 48 + }, + reference_policy: { + access: "scoring_only", + availability: "included", + relative_path: "reference/reference.json", + sha256: "a".repeat(64) + }, + runtime: { + agent_adapter_id: "deterministic-agent-v1", + one_shot_adapter_id: "deterministic-one-shot-v1" + }, + execution: { + kind: "local", + network_access: "disabled" + }, + budget: { + max_wall_time_ms: 60_000, + max_tokens: 10_000, + max_cost_usd: null, + max_human_review_minutes: null + }, + allowed_task_types: ["statistical_analysis", "custom_experiment"], + expected_task_types: ["statistical_analysis"], + hard_checks: [{ id: "required-output", description: "The required output is present." }] +}; + +const result: BenchmarkResult = { + schema_version: "1.0.0", + case_id: manifest.case_id, + variant: "agent", + adapter_id: "deterministic-agent-v1", + run_id: "synthetic-contract-agent-aaaaaaaaaaaa", + frozen_case_sha256: "b".repeat(64), + state: "measured", + outcome: "completed", + observed_task_types: ["statistical_analysis"], + hard_checks: [{ id: "required-output", status: "passed" }], + policy_events: [], + error: null, + metrics: { + completion: measuredMetric(true, "harness_scoring"), + hard_error: measuredMetric(false, "harness_scoring"), + wall_time_ms: measuredMetric(125, "harness_clock"), + task_type_coverage: measuredMetric(1, "harness_scoring"), + custom_experiment_present: measuredMetric(false, "harness_scoring"), + token_count: unavailableMetric("adapter_did_not_report"), + cost_usd: unavailableMetric("adapter_did_not_report"), + human_review_minutes: unavailableMetric("not_reviewed"), + human_review_notes: unavailableMetric("not_reviewed"), + artifact_count: measuredMetric(2, "adapter_inventory"), + evidence_count: measuredMetric(3, "adapter_inventory"), + commit_identity: measuredMetric("a".repeat(40), "git_commit"), + environment_identity: measuredMetric("node-24-linux-x64", "runtime_environment") + } +}; + +describe("benchmark contracts", () => { + it("accepts a complete versioned manifest and rejects unsafe or inconsistent policy metadata", () => { + expect(validateBenchmarkManifest(structuredClone(manifest))).toEqual(manifest); + + const unknownField = { ...structuredClone(manifest), unexpected: true }; + expect(() => validateBenchmarkManifest(unknownField)).toThrow(BenchmarkContractError); + + const absolutePackage = { ...structuredClone(manifest), package_path: "/home/user/private/problem" }; + expect(() => validateBenchmarkManifest(absolutePackage)).toThrow(/relative path/i); + + const inconsistentTasks = structuredClone(manifest); + inconsistentTasks.expected_task_types = ["optimization"]; + expect(() => validateBenchmarkManifest(inconsistentTasks)).toThrow(/expected task type/i); + + const leakingPolicy = structuredClone(manifest); + leakingPolicy.reference_policy.access = "solve_and_score" as "scoring_only"; + expect(() => validateBenchmarkManifest(leakingPolicy)).toThrow(BenchmarkContractError); + }); + + it("represents unknown token, cost, and human-review values as unavailable nulls rather than zero", () => { + const validated = validateBenchmarkResult(structuredClone(result)); + for (const name of ["token_count", "cost_usd", "human_review_minutes", "human_review_notes"] as const) { + expect(validated.metrics[name]).toMatchObject({ status: "unavailable", value: null }); + } + + const ambiguousUnknown = structuredClone(result) as BenchmarkResult; + ambiguousUnknown.metrics.token_count = { status: "unavailable", value: 0, reason: "adapter_did_not_report" } as never; + expect(() => validateBenchmarkResult(ambiguousUnknown)).toThrow(BenchmarkContractError); + }); + + it("rejects a completed result when completion, hard-error, or hard-check evidence contradicts success", () => { + const falseCompletion = structuredClone(result); + falseCompletion.metrics.completion = measuredMetric(false, "harness_scoring"); + expect(() => validateBenchmarkResult(falseCompletion)).toThrow(/completed result/i); + + const failedCheck = structuredClone(result); + failedCheck.hard_checks[0]!.status = "failed"; + expect(() => validateBenchmarkResult(failedCheck)).toThrow(/completed result/i); + }); +}); diff --git a/tests/benchmark-report.test.ts b/tests/benchmark-report.test.ts new file mode 100644 index 0000000..1dd4a0f --- /dev/null +++ b/tests/benchmark-report.test.ts @@ -0,0 +1,109 @@ +import { mkdtemp, readFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { describe, expect, it } from "vitest"; +import type { BenchmarkResult } from "../src/benchmark/types.js"; +import { measuredMetric, notRunMetric, unavailableMetric } from "../src/benchmark/contracts.js"; +import { aggregateBenchmarkResults, writeBenchmarkReports } from "../src/benchmark/report.js"; + +function result(overrides: Partial = {}): BenchmarkResult { + return { + schema_version: "1.0.0", + case_id: "synthetic-report", + variant: "agent", + adapter_id: "agent-v1", + run_id: "synthetic-report-agent-aaaaaaaaaaaa", + frozen_case_sha256: "a".repeat(64), + state: "measured", + outcome: "completed", + observed_task_types: ["statistical_analysis"], + hard_checks: [{ id: "output", status: "passed" }], + policy_events: [], + error: null, + metrics: { + completion: measuredMetric(true, "harness_scoring"), + hard_error: measuredMetric(false, "harness_scoring"), + wall_time_ms: measuredMetric(10, "harness_clock"), + task_type_coverage: measuredMetric(1, "harness_scoring"), + custom_experiment_present: measuredMetric(false, "harness_scoring"), + token_count: unavailableMetric("adapter_did_not_report"), + cost_usd: unavailableMetric("adapter_did_not_report"), + human_review_minutes: unavailableMetric("not_reviewed"), + human_review_notes: unavailableMetric("not_reviewed"), + artifact_count: measuredMetric(1, "adapter_inventory"), + evidence_count: measuredMetric(1, "adapter_inventory"), + commit_identity: measuredMetric("a".repeat(40), "git_commit"), + environment_identity: measuredMetric("node-24-linux-x64", "runtime_environment") + }, + ...overrides + }; +} + +describe("benchmark aggregate reports", () => { + it("distinguishes measured, not_run, blocked, and unavailable without counting failure as completion", async () => { + const blocked = result({ + variant: "one_shot", + adapter_id: "one-shot-v1", + run_id: "synthetic-report-one-shot-bbbbbbbbbbbb", + state: "blocked", + outcome: "blocked_policy", + hard_checks: [{ id: "output", status: "blocked" }], + policy_events: [{ type: "same_problem_answer_detected", action: "blocked", fingerprint: "b".repeat(64) }], + metrics: { + ...result().metrics, + completion: measuredMetric(false, "harness_scoring"), + hard_error: measuredMetric(true, "harness_scoring"), + task_type_coverage: measuredMetric(0, "harness_scoring"), + artifact_count: measuredMetric(0, "adapter_inventory"), + evidence_count: measuredMetric(0, "adapter_inventory") + } + }); + const notRun = result({ + case_id: "historical-placeholder", + variant: "agent", + adapter_id: "not-run", + run_id: "historical-placeholder-agent-notrun000000", + state: "not_run", + outcome: "not_run", + observed_task_types: [], + hard_checks: [], + metrics: { + completion: notRunMetric("case_material_not_available"), + hard_error: notRunMetric("case_material_not_available"), + wall_time_ms: notRunMetric("case_material_not_available"), + task_type_coverage: notRunMetric("case_material_not_available"), + custom_experiment_present: notRunMetric("case_material_not_available"), + token_count: unavailableMetric("not_run"), + cost_usd: unavailableMetric("not_run"), + human_review_minutes: unavailableMetric("not_reviewed"), + human_review_notes: unavailableMetric("not_reviewed"), + artifact_count: notRunMetric("case_material_not_available"), + evidence_count: notRunMetric("case_material_not_available"), + commit_identity: measuredMetric("a".repeat(40), "git_commit"), + environment_identity: measuredMetric("node-24-linux-x64", "runtime_environment") + } + }); + + const report = aggregateBenchmarkResults([result(), blocked, notRun]); + expect(report.summary).toEqual({ total_runs: 3, measured_runs: 1, completed_runs: 1, hard_error_runs: 0, blocked_runs: 1, not_run_runs: 1 }); + expect(report.metrics.completion).toMatchObject({ measured_count: 2, unavailable_count: 0, not_run_count: 1 }); + expect(report.metrics.completion.values).toEqual([true, false]); + expect(report.metrics.token_count).toMatchObject({ measured_count: 0, unavailable_count: 3, not_run_count: 0 }); + + const output = await mkdtemp(join(tmpdir(), "benchmark-report-")); + const paths = await writeBenchmarkReports(report, output); + const json = await readFile(paths.json_path, "utf8"); + const markdown = await readFile(paths.markdown_path, "utf8"); + expect(JSON.parse(json)).toEqual(report); + for (const label of ["measured", "not_run", "blocked", "unavailable"]) { + expect(markdown).toContain(label); + } + expect(markdown).toContain("1 / 3"); + expect(markdown).not.toContain(output); + }); + + it("refuses to compare agent and one-shot results from different frozen case bytes", () => { + const oneShot = result({ variant: "one_shot", adapter_id: "one-shot-v1", frozen_case_sha256: "b".repeat(64) }); + expect(() => aggregateBenchmarkResults([result(), oneShot])).toThrow(/do not share a frozen case/i); + }); +}); diff --git a/tests/benchmark-runner.test.ts b/tests/benchmark-runner.test.ts new file mode 100644 index 0000000..343d29e --- /dev/null +++ b/tests/benchmark-runner.test.ts @@ -0,0 +1,214 @@ +import { mkdtemp, mkdir, readFile, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { describe, expect, it } from "vitest"; +import type { + BenchmarkAdapter, + BenchmarkManifest, + BenchmarkSolveContext, + BenchmarkVariant +} from "../src/benchmark/types.js"; +import { runBenchmarkCase } from "../src/benchmark/runner.js"; + +async function createCase(): Promise<{ caseRoot: string; manifest: BenchmarkManifest; secret: string }> { + const caseRoot = await mkdtemp(join(tmpdir(), "benchmark-runner-")); + const secret = "REFERENCE-ANSWER-NEVER-IN-SOLVE-CONTEXT-314159265358979"; + await mkdir(resolve(caseRoot, "package")); + await mkdir(resolve(caseRoot, "reference")); + await writeFile(resolve(caseRoot, "package/problem.md"), "Estimate the synthetic quantity.\n", "utf8"); + await writeFile(resolve(caseRoot, "reference/reference.json"), `${JSON.stringify({ answer: secret })}\n`, "utf8"); + const manifest: BenchmarkManifest = { + schema_version: "1.0.0", + case_id: "synthetic-runner", + package_path: "package", + license: { + name: "CC0 1.0 Universal", + spdx_id: "CC0-1.0", + copyright_holder: "modeling-agent contributors", + source_url: null, + redistribution: "permitted", + notice_path: null + }, + blind_policy: { + mode: "blind", + solve_input: "package_only", + same_problem_answers: "block", + minimum_reference_match_characters: 32 + }, + reference_policy: { + access: "scoring_only", + availability: "included", + relative_path: "reference/reference.json", + sha256: null + }, + runtime: { agent_adapter_id: "agent-test-v1", one_shot_adapter_id: "one-shot-test-v1" }, + execution: { kind: "local", network_access: "disabled" }, + budget: { max_wall_time_ms: 1_000, max_tokens: 1_000, max_cost_usd: null, max_human_review_minutes: null }, + allowed_task_types: ["statistical_analysis", "custom_experiment"], + expected_task_types: ["statistical_analysis"], + hard_checks: [{ id: "answer-present", description: "An answer is produced." }] + }; + await writeFile(resolve(caseRoot, "manifest.json"), `${JSON.stringify(manifest, null, 2)}\n`, "utf8"); + return { caseRoot, manifest, secret }; +} + +function adapter( + id: string, + variant: BenchmarkVariant, + run: (context: BenchmarkSolveContext) => ReturnType +): BenchmarkAdapter { + return { id, variant, run }; +} + +describe("benchmark runner", () => { + it("gives agent and one-shot adapters the same frozen package and metric denominator without reference access", async () => { + const { caseRoot, manifest, secret } = await createCase(); + const contexts: BenchmarkSolveContext[] = []; + const makeAdapter = (id: string, variant: BenchmarkVariant): BenchmarkAdapter => adapter(id, variant, async (context) => { + contexts.push(context); + expect(JSON.stringify(context)).not.toContain(secret); + expect(Object.keys(context).sort()).toEqual(["budget", "case_id", "expected_task_types", "frozen_case_sha256", "hard_checks", "package_files", "variant"]); + expect(context.package_files.map((file) => file.relative_path)).toEqual(["problem.md"]); + expect(context.package_files[0]!.content).toContain("synthetic quantity"); + return { + status: "success", + observed_task_types: ["statistical_analysis"], + hard_checks: [{ id: "answer-present", passed: true }], + artifacts: ["answer.md"], + evidence: ["answer-evidence"], + usage: { token_count: null, cost_usd: null }, + output_text: "Synthetic estimate produced." + }; + }); + + const agentResult = await runBenchmarkCase({ + case_root: caseRoot, + manifest, + adapter: makeAdapter("agent-test-v1", "agent"), + identity: { commit: "a".repeat(40), environment: "test-environment" }, + clock: { measure: async (operation) => ({ value: await operation(), duration_ms: 25 }) } + }); + const baselineResult = await runBenchmarkCase({ + case_root: caseRoot, + manifest, + adapter: makeAdapter("one-shot-test-v1", "one_shot"), + identity: { commit: "a".repeat(40), environment: "test-environment" }, + clock: { measure: async (operation) => ({ value: await operation(), duration_ms: 20 }) } + }); + + expect(contexts).toHaveLength(2); + expect(contexts[0]!.frozen_case_sha256).toBe(contexts[1]!.frozen_case_sha256); + expect(contexts[0]!.expected_task_types).toEqual(contexts[1]!.expected_task_types); + for (const result of [agentResult, baselineResult]) { + expect(result.outcome).toBe("completed"); + expect(result.metrics.completion.value).toBe(true); + expect(result.metrics.task_type_coverage.value).toBe(1); + expect(result.metrics.token_count).toMatchObject({ status: "unavailable", value: null }); + expect(result.metrics.cost_usd).toMatchObject({ status: "unavailable", value: null }); + } + }); + + it("blocks and records same-problem answer leakage without counting the run as complete", async () => { + const { caseRoot, manifest, secret } = await createCase(); + const leaking = adapter("agent-test-v1", "agent", async () => ({ + status: "success", + observed_task_types: ["statistical_analysis"], + hard_checks: [{ id: "answer-present", passed: true }], + artifacts: ["answer.md"], + evidence: ["answer-evidence"], + usage: { token_count: 12, cost_usd: null }, + output_text: `I found the hidden value: ${secret.slice(0, 36)}\n${secret.slice(36)}` + })); + + const result = await runBenchmarkCase({ + case_root: caseRoot, + manifest, + adapter: leaking, + identity: { commit: "b".repeat(40), environment: "test-environment" }, + clock: { measure: async (operation) => ({ value: await operation(), duration_ms: 15 }) } + }); + + expect(result.state).toBe("blocked"); + expect(result.outcome).toBe("blocked_policy"); + expect(result.metrics.completion).toMatchObject({ status: "measured", value: false }); + expect(result.metrics.hard_error).toMatchObject({ status: "measured", value: true }); + expect(result.policy_events).toEqual([ + expect.objectContaining({ type: "same_problem_answer_detected", action: "blocked" }) + ]); + expect(JSON.stringify(result)).not.toContain(secret); + expect(JSON.stringify(result)).not.toContain(caseRoot); + }); + + it("rejects disallowed adapter task types as a measured contract hard error", async () => { + const { caseRoot, manifest } = await createCase(); + const invalid = adapter("agent-test-v1", "agent", async () => ({ + status: "success", + observed_task_types: ["optimization"], + hard_checks: [{ id: "answer-present", passed: true }], + artifacts: ["answer.md"], + evidence: ["answer-evidence"], + usage: { token_count: 10, cost_usd: null }, + output_text: "An invalid task type was reported." + })); + + const result = await runBenchmarkCase({ + case_root: caseRoot, + manifest, + adapter: invalid, + identity: { commit: "d".repeat(40), environment: "test-environment" }, + clock: { measure: async (operation) => ({ value: await operation(), duration_ms: 10 }) } + }); + + expect(result.outcome).toBe("hard_error"); + expect(result.metrics.completion.value).toBe(false); + expect(result.metrics.hard_error.value).toBe(true); + expect(result.error?.message).not.toContain("optimization"); + }); + + it("enforces the wall-time budget and records timeout as an incomplete hard error", async () => { + const { caseRoot, manifest } = await createCase(); + manifest.budget.max_wall_time_ms = 5; + const hanging = adapter("agent-test-v1", "agent", async () => new Promise(() => undefined)); + + const result = await runBenchmarkCase({ + case_root: caseRoot, + manifest, + adapter: hanging, + identity: { commit: "e".repeat(40), environment: "test-environment" } + }); + + expect(result.outcome).toBe("hard_error"); + expect(result.metrics.completion.value).toBe(false); + expect(result.metrics.hard_error.value).toBe(true); + }); + + it("turns adapter failures into measured incomplete hard errors instead of success", async () => { + const { caseRoot, manifest } = await createCase(); + const failing = adapter("one-shot-test-v1", "one_shot", async () => { + throw new Error(`credential=super-secret path=${caseRoot}`); + }); + + const result = await runBenchmarkCase({ + case_root: caseRoot, + manifest, + adapter: failing, + identity: { commit: "c".repeat(40), environment: "test-environment" }, + clock: { measure: async (operation) => { + try { + return { value: await operation(), duration_ms: 30 }; + } catch (error) { + Object.assign(error as object, { benchmark_duration_ms: 30 }); + throw error; + } + } } + }); + + expect(result.state).toBe("measured"); + expect(result.outcome).toBe("hard_error"); + expect(result.metrics.completion.value).toBe(false); + expect(result.metrics.hard_error.value).toBe(true); + expect(result.error?.class).toBe("Error"); + expect(result.error?.message).not.toContain("super-secret"); + expect(JSON.stringify(result)).not.toContain(caseRoot); + }); +}); diff --git a/tests/benchmark-synthetic.test.ts b/tests/benchmark-synthetic.test.ts new file mode 100644 index 0000000..f593ae6 --- /dev/null +++ b/tests/benchmark-synthetic.test.ts @@ -0,0 +1,30 @@ +import { mkdtemp, readFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { runSyntheticBenchmarks } from "../src/benchmark/synthetic.js"; + +describe("committed synthetic benchmark suite", () => { + it("runs two cases for agent and one-shot, including the expected custom experiment, reproducibly", async () => { + const firstOutput = await mkdtemp(join(tmpdir(), "benchmark-synthetic-first-")); + const secondOutput = await mkdtemp(join(tmpdir(), "benchmark-synthetic-second-")); + const first = await runSyntheticBenchmarks({ output_directory: firstOutput }); + const second = await runSyntheticBenchmarks({ output_directory: secondOutput }); + + expect(first.results).toHaveLength(4); + expect(new Set(first.results.map((entry) => entry.case_id)).size).toBe(2); + expect(first.results.filter((entry) => entry.variant === "agent")).toHaveLength(2); + expect(first.results.filter((entry) => entry.variant === "one_shot")).toHaveLength(2); + expect(first.results.some((entry) => entry.metrics.custom_experiment_present.value === true)).toBe(true); + expect(first.results.every((entry) => entry.metrics.completion.value === true)).toBe(true); + + expect(first.report).toEqual(second.report); + expect(await readFile(first.paths.json_path, "utf8")).toBe(await readFile(second.paths.json_path, "utf8")); + expect(await readFile(first.paths.markdown_path, "utf8")).toBe(await readFile(second.paths.markdown_path, "utf8")); + + const serialized = `${JSON.stringify(first.report)}\n${await readFile(first.paths.markdown_path, "utf8")}`; + expect(serialized).not.toMatch(/\/(?:home|Users|tmp)\//); + expect(serialized).not.toMatch(/(?:token|password|credential|secret)\s*[:=]\s*[^,\s}]+/i); + expect(serialized).not.toContain("reference_answer"); + }); +}); From e66c837b4c3c369d6cfd4ca01596517c83ad5854 Mon Sep 17 00:00:00 2001 From: zhanghan <2774877132@qq.com> Date: Thu, 13 Aug 2026 03:42:21 +0800 Subject: [PATCH 2/4] fix: harden benchmark evaluation contracts --- benchmarks/README.md | 8 +- schemas/benchmark-manifest.v1.json | 10 +- schemas/benchmark-result.v1.json | 48 +- src/benchmark/contracts.ts | 25 +- src/benchmark/report.ts | 7 + src/benchmark/run-synthetic.ts | 4 +- src/benchmark/runner.ts | 457 ++++++++++++++------ src/benchmark/synthetic.ts | 27 +- src/benchmark/types.ts | 11 +- tests/benchmark-compiled.test.ts | 97 +++++ tests/benchmark-contracts.test.ts | 10 + tests/benchmark-reference-lifecycle.test.ts | 118 +++++ tests/benchmark-regressions.test.ts | 443 +++++++++++++++++++ tests/benchmark-report.test.ts | 3 + tests/benchmark-runner.test.ts | 2 +- 15 files changed, 1111 insertions(+), 159 deletions(-) create mode 100644 tests/benchmark-compiled.test.ts create mode 100644 tests/benchmark-reference-lifecycle.test.ts create mode 100644 tests/benchmark-regressions.test.ts diff --git a/benchmarks/README.md b/benchmarks/README.md index b3b5c9f..6875ddd 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -12,7 +12,7 @@ npx tsx src/benchmark/run-synthetic.ts --output benchmarks/output/synthetic The command writes `benchmark-report.json` and `benchmark-report.md`. It runs the deterministic agent and one-shot adapters against the same frozen bytes for each case, uses a deterministic clock and identity, and is suitable for checking stable report fields. `benchmarks/output/` is intended as local generated output; do not commit reports that contain non-synthetic operational data. -The harness API is injected: production integrations implement `BenchmarkAdapter` from `src/benchmark/types.ts` instead of importing CLI or Orchestrator modules. A solve adapter receives package files, budgets, expected task types, and hard-check definitions. It never receives the manifest object, a reference path, reference bytes, or scoring notes. Reference material is opened separately for post-solve leakage checks, and a detected same-problem answer changes the result to `blocked` / `blocked_policy` with a digest-only event. +The harness API is injected: production integrations implement `BenchmarkAdapter` from `src/benchmark/types.ts` instead of importing CLI or Orchestrator modules. A solve adapter receives package files, budgets, expected task types, and hard-check definitions. It never receives the manifest object, a reference path, reference bytes, or scoring notes. Reference material is opened separately for post-solve leakage checks, and a detected same-problem answer changes the result to `blocked` / `blocked_policy` with a digest-only event. Canonical package/reference paths are checked before solve so a reference cannot be nested under, cover, or alias the package; a declared user-supplied reference that is absent remains explicitly unavailable and cannot produce a completed result. ## Adding three historical problems legally @@ -21,9 +21,9 @@ The v0.1-alpha acceptance target calls for three historical blind runs spanning 1. **Establish rights before copying.** Record the organizer, copyright holder, official source URL, license or written permission, and redistribution terms. A public download link is not by itself permission to republish a statement, dataset, judge material, or solution. 2. **Prefer user-supplied private material.** For ordinary copyrighted competitions, set `license.redistribution` to `user_supplied_only` or `metadata_only`, set reference availability to `user_supplied` or `unavailable`, and keep problem packages and answers under the ignored `benchmarks/private/` directory. Do not commit them. 3. **Create metadata without answer text.** Give the case a stable id, declare allowed and expected task types, budgets, runtime/execution policy, and hard checks. Store only relative paths and SHA-256 identities. Never place a reference answer, excerpt from a solution, credential, session, or absolute host path in a manifest or report. -4. **Freeze one case for both variants.** Verify that agent and one-shot runs receive the exact same `frozen_case_sha256`, budget, expected-task denominator, and hard-check definitions. Do not tune either variant after viewing the other variant's result. +4. **Freeze one case for both variants.** Verify that agent and one-shot runs receive the exact same `frozen_case_sha256` and `evaluation_contract_sha256`. The evaluation contract binds reference availability/digest, blind policy, every budget, allowed/expected task types, hard-check ids and descriptions, execution/network policy, and both adapter ids. Do not tune either variant after viewing the other variant's result. 5. **Keep scoring references out of solve.** Reference solutions and judge notes are scoring-only. The blind firewall must block a detected same-problem answer and retain only a fingerprint, never the matched text. -6. **Report missing measurements honestly.** Unknown token count, cost, or human review is `unavailable` with a `null` value. An unavailable field is not zero. Material not obtained or a run not attempted is `not_run`; a policy stop is `blocked`; failures never count as completion. +6. **Report missing measurements honestly.** Unknown token count, cost, or human review is `unavailable` with a `null` value. An unavailable field is not zero, and a declared limit cannot be treated as satisfied when its measurement is unavailable. Human review observations use only a numeric duration plus a fixed safe classification (`no_revision`, `minor_revision`, `major_revision`, or `rejected`), never free-form text. Material not obtained or a run not attempted is `not_run`; a policy stop is `blocked`; failures never count as completion. 7. **Review before publication.** Have a maintainer confirm the license record and inspect the staged diff for protected text. Publish historical results only when the underlying case can be lawfully used and the environment/commit identity is reproducible. A practical three-case plan is to select cases with independently verified reuse permission, assign each a separate private case root, and run both variants offline. Until those materials and permissions exist, retain metadata placeholders outside committed synthetic results rather than inventing scores. @@ -33,5 +33,5 @@ A practical three-case plan is to select cases with independently verified reuse - `schemas/benchmark-manifest.v1.json`: strict manifest boundary. - `schemas/benchmark-result.v1.json`: strict result and metric-state boundary. - `src/benchmark/contracts.ts`: Schema validation plus cross-field invariants. -- `src/benchmark/runner.ts`: frozen-package runner and blind leakage firewall. +- `src/benchmark/runner.ts`: frozen-package/evaluation-contract runner, budget enforcement, and blind leakage firewall. - `src/benchmark/report.ts`: deterministic JSON and Markdown aggregation. diff --git a/schemas/benchmark-manifest.v1.json b/schemas/benchmark-manifest.v1.json index ccdb5d3..5cdea34 100644 --- a/schemas/benchmark-manifest.v1.json +++ b/schemas/benchmark-manifest.v1.json @@ -49,8 +49,8 @@ "additionalProperties": false, "required": ["agent_adapter_id", "one_shot_adapter_id"], "properties": { - "agent_adapter_id": { "type": "string", "minLength": 1, "maxLength": 200 }, - "one_shot_adapter_id": { "type": "string", "minLength": 1, "maxLength": 200 } + "agent_adapter_id": { "$ref": "#/$defs/opaqueIdentifier" }, + "one_shot_adapter_id": { "$ref": "#/$defs/opaqueIdentifier" } } }, "execution": { @@ -91,6 +91,12 @@ } }, "$defs": { + "opaqueIdentifier": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + }, "relativePath": { "type": "string", "minLength": 1, diff --git a/schemas/benchmark-result.v1.json b/schemas/benchmark-result.v1.json index f40699a..f5f31fe 100644 --- a/schemas/benchmark-result.v1.json +++ b/schemas/benchmark-result.v1.json @@ -4,14 +4,15 @@ "title": "BenchmarkResultV1", "type": "object", "additionalProperties": false, - "required": ["schema_version", "case_id", "variant", "adapter_id", "run_id", "frozen_case_sha256", "state", "outcome", "observed_task_types", "hard_checks", "policy_events", "error", "metrics"], + "required": ["schema_version", "case_id", "variant", "adapter_id", "run_id", "frozen_case_sha256", "evaluation_contract_sha256", "state", "outcome", "observed_task_types", "hard_checks", "policy_events", "error", "metrics"], "properties": { "schema_version": { "const": "1.0.0" }, "case_id": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-]{2,63}$" }, "variant": { "enum": ["agent", "one_shot"] }, - "adapter_id": { "type": "string", "minLength": 1, "maxLength": 200 }, + "adapter_id": { "$ref": "#/$defs/opaqueIdentifier" }, "run_id": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-]{7,127}$" }, "frozen_case_sha256": { "type": "string", "pattern": "^[a-f0-9]{64}$" }, + "evaluation_contract_sha256": { "type": "string", "pattern": "^[a-f0-9]{64}$" }, "state": { "enum": ["measured", "not_run", "blocked"] }, "outcome": { "enum": ["completed", "incomplete", "hard_error", "blocked_policy", "not_run"] }, "observed_task_types": { "$ref": "#/$defs/taskTypes" }, @@ -24,8 +25,7 @@ "required": ["id", "status"], "properties": { "id": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-]{1,63}$" }, - "status": { "enum": ["passed", "failed", "blocked", "not_run"] }, - "note": { "type": "string", "minLength": 1, "maxLength": 500 } + "status": { "enum": ["passed", "failed", "blocked", "not_run"] } } } }, @@ -51,8 +51,8 @@ "additionalProperties": false, "required": ["class", "message"], "properties": { - "class": { "type": "string", "minLength": 1, "maxLength": 200 }, - "message": { "type": "string", "minLength": 1, "maxLength": 500 }, + "class": { "$ref": "#/$defs/opaqueIdentifier" }, + "message": { "type": "string", "pattern": "^failure:[a-f0-9]{12}$" }, "fingerprint": { "type": "string", "pattern": "^[a-f0-9]{64}$" } } } @@ -61,7 +61,7 @@ "metrics": { "type": "object", "additionalProperties": false, - "required": ["completion", "hard_error", "wall_time_ms", "task_type_coverage", "custom_experiment_present", "token_count", "cost_usd", "human_review_minutes", "human_review_notes", "artifact_count", "evidence_count", "commit_identity", "environment_identity"], + "required": ["completion", "hard_error", "wall_time_ms", "task_type_coverage", "custom_experiment_present", "token_count", "cost_usd", "human_review_minutes", "human_review_notes", "reference_leak_check", "artifact_count", "evidence_count", "commit_identity", "environment_identity"], "properties": { "completion": { "$ref": "#/$defs/booleanMetric" }, "hard_error": { "$ref": "#/$defs/booleanMetric" }, @@ -71,15 +71,22 @@ "token_count": { "$ref": "#/$defs/nonNegativeIntegerMetric" }, "cost_usd": { "$ref": "#/$defs/nonNegativeNumberMetric" }, "human_review_minutes": { "$ref": "#/$defs/nonNegativeNumberMetric" }, - "human_review_notes": { "$ref": "#/$defs/stringMetric" }, + "human_review_notes": { "$ref": "#/$defs/reviewNoteMetric" }, + "reference_leak_check": { "$ref": "#/$defs/booleanMetric" }, "artifact_count": { "$ref": "#/$defs/nonNegativeIntegerMetric" }, "evidence_count": { "$ref": "#/$defs/nonNegativeIntegerMetric" }, - "commit_identity": { "$ref": "#/$defs/stringMetric" }, - "environment_identity": { "$ref": "#/$defs/stringMetric" } + "commit_identity": { "$ref": "#/$defs/identifierMetric" }, + "environment_identity": { "$ref": "#/$defs/identifierMetric" } } } }, "$defs": { + "opaqueIdentifier": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + }, "unmeasuredMetric": { "type": "object", "additionalProperties": false, @@ -114,17 +121,32 @@ "required": ["status", "value", "source"], "properties": { "status": { "const": "measured" }, "value": { "type": "number", "minimum": 0, "maximum": 1 }, "source": { "type": "string", "minLength": 1, "maxLength": 200 } } }, - "measuredString": { + "measuredReviewNote": { "type": "object", "additionalProperties": false, "required": ["status", "value", "source"], - "properties": { "status": { "const": "measured" }, "value": { "type": "string", "minLength": 1, "maxLength": 500 }, "source": { "type": "string", "minLength": 1, "maxLength": 200 } } + "properties": { + "status": { "const": "measured" }, + "value": { "enum": ["no_revision", "minor_revision", "major_revision", "rejected"] }, + "source": { "$ref": "#/$defs/opaqueIdentifier" } + } + }, + "measuredIdentifier": { + "type": "object", + "additionalProperties": false, + "required": ["status", "value", "source"], + "properties": { + "status": { "const": "measured" }, + "value": { "$ref": "#/$defs/opaqueIdentifier" }, + "source": { "$ref": "#/$defs/opaqueIdentifier" } + } }, "booleanMetric": { "oneOf": [{ "$ref": "#/$defs/measuredBoolean" }, { "$ref": "#/$defs/unmeasuredMetric" }] }, "nonNegativeNumberMetric": { "oneOf": [{ "$ref": "#/$defs/measuredNumber" }, { "$ref": "#/$defs/unmeasuredMetric" }] }, "nonNegativeIntegerMetric": { "oneOf": [{ "$ref": "#/$defs/measuredInteger" }, { "$ref": "#/$defs/unmeasuredMetric" }] }, "fractionMetric": { "oneOf": [{ "$ref": "#/$defs/measuredFraction" }, { "$ref": "#/$defs/unmeasuredMetric" }] }, - "stringMetric": { "oneOf": [{ "$ref": "#/$defs/measuredString" }, { "$ref": "#/$defs/unmeasuredMetric" }] }, + "reviewNoteMetric": { "oneOf": [{ "$ref": "#/$defs/measuredReviewNote" }, { "$ref": "#/$defs/unmeasuredMetric" }] }, + "identifierMetric": { "oneOf": [{ "$ref": "#/$defs/measuredIdentifier" }, { "$ref": "#/$defs/unmeasuredMetric" }] }, "taskType": { "enum": ["statistical_analysis", "regression_prediction", "time_series_forecasting", "classification", "clustering", "evaluation_ranking", "optimization", "simulation", "custom_experiment"] }, "taskTypes": { "type": "array", "uniqueItems": true, "items": { "$ref": "#/$defs/taskType" } } } diff --git a/src/benchmark/contracts.ts b/src/benchmark/contracts.ts index 8ddf4e3..85b8af7 100644 --- a/src/benchmark/contracts.ts +++ b/src/benchmark/contracts.ts @@ -1,4 +1,5 @@ -import { readFileSync } from "node:fs"; +import { existsSync, readFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { Ajv2020 } from "ajv/dist/2020.js"; import type { ErrorObject, ValidateFunction } from "ajv"; @@ -32,10 +33,24 @@ export class BenchmarkContractError extends Error { } } +function locateSchemaDirectory(): string { + const moduleDirectory = dirname(fileURLToPath(import.meta.url)); + const candidates = [ + resolve(moduleDirectory, "../../schemas"), + resolve(moduleDirectory, "../../../schemas"), + resolve(process.cwd(), "schemas") + ]; + const directory = candidates.find((candidate) => Object.values(schemaFiles).every((filename) => existsSync(resolve(candidate, filename)))); + if (directory === undefined) { + throw new BenchmarkContractError("benchmark-manifest", "schema directory is unavailable"); + } + return directory; +} + class BenchmarkSchemaRegistry { readonly #validators = new Map(); - constructor(schemaDirectory = fileURLToPath(new URL("../../schemas/", import.meta.url))) { + constructor(schemaDirectory = locateSchemaDirectory()) { const ajv = new Ajv2020({ allErrors: true, strict: true, allowUnionTypes: true }); ajv.addFormat("uri", { type: "string", @@ -49,7 +64,7 @@ class BenchmarkSchemaRegistry { } }); for (const [name, filename] of Object.entries(schemaFiles) as Array<[BenchmarkContractName, string]>) { - const schema = JSON.parse(readFileSync(new URL(filename, `file://${schemaDirectory}/`), "utf8")) as object; + const schema = JSON.parse(readFileSync(resolve(schemaDirectory, filename), "utf8")) as object; this.#validators.set(name, ajv.compile(schema)); } } @@ -105,8 +120,8 @@ function validateResultSemantics(result: BenchmarkResult): void { throw new BenchmarkContractError("benchmark-result", "hard check ids must be unique"); } if (result.outcome === "completed") { - if (result.state !== "measured" || !measuredBoolean(result.metrics.completion, true) || !measuredBoolean(result.metrics.hard_error, false) || result.hard_checks.some((check) => check.status !== "passed")) { - throw new BenchmarkContractError("benchmark-result", "completed result contradicts completion, hard-error, state, or hard-check evidence"); + if (result.state !== "measured" || result.error !== null || !measuredBoolean(result.metrics.completion, true) || !measuredBoolean(result.metrics.hard_error, false) || result.hard_checks.some((check) => check.status !== "passed")) { + throw new BenchmarkContractError("benchmark-result", "completed result contradicts completion, error, hard-error, state, or hard-check evidence"); } } else if (result.metrics.completion.status === "measured" && result.metrics.completion.value === true) { throw new BenchmarkContractError("benchmark-result", "non-completed result cannot report completion=true"); diff --git a/src/benchmark/report.ts b/src/benchmark/report.ts index 191ddee..f6555c5 100644 --- a/src/benchmark/report.ts +++ b/src/benchmark/report.ts @@ -21,6 +21,7 @@ const metricNames: Array = [ "cost_usd", "human_review_minutes", "human_review_notes", + "reference_leak_check", "artifact_count", "evidence_count", "commit_identity", @@ -54,10 +55,16 @@ function assertComparableVariants(results: readonly BenchmarkResult[]): void { if (agent && oneShot && agent.frozen_case_sha256 !== oneShot.frozen_case_sha256) { throw new Error(`benchmark variants for ${caseId} do not share a frozen case`); } + if (agent && oneShot && agent.evaluation_contract_sha256 !== oneShot.evaluation_contract_sha256) { + throw new Error(`benchmark variants for ${caseId} do not share an evaluation contract`); + } } } export function aggregateBenchmarkResults(input: readonly BenchmarkResult[], suiteId = "synthetic-v1"): BenchmarkAggregateReport { + if (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/u.test(suiteId)) { + throw new Error("benchmark suite id must be a bounded opaque identifier"); + } const results = input.map((entry) => validateBenchmarkResult(structuredClone(entry))).sort(stableResultOrder); assertComparableVariants(results); const metrics = Object.fromEntries(metricNames.map((name) => [name, aggregateMetric(results.map((result) => result.metrics[name]))])) as Record; diff --git a/src/benchmark/run-synthetic.ts b/src/benchmark/run-synthetic.ts index a9bb9fc..fc65940 100644 --- a/src/benchmark/run-synthetic.ts +++ b/src/benchmark/run-synthetic.ts @@ -19,8 +19,8 @@ export async function main(argv = process.argv.slice(2)): Promise { suite_id: run.report.suite_id, total_runs: run.report.summary.total_runs, completed_runs: run.report.summary.completed_runs, - json_report: run.paths.json_path, - markdown_report: run.paths.markdown_path + json_report: "benchmark-report.json", + markdown_report: "benchmark-report.md" }, null, 2)}\n`); } diff --git a/src/benchmark/runner.ts b/src/benchmark/runner.ts index 9ed3b6a..e094274 100644 --- a/src/benchmark/runner.ts +++ b/src/benchmark/runner.ts @@ -1,5 +1,5 @@ import { createHash } from "node:crypto"; -import { readdir, readFile, realpath, stat } from "node:fs/promises"; +import { lstat, open, readdir, readFile, realpath, stat } from "node:fs/promises"; import { isAbsolute, relative, resolve, sep } from "node:path"; import type { TaskType } from "../contracts/types.js"; import { @@ -9,24 +9,70 @@ import { validateBenchmarkResult } from "./contracts.js"; import type { + BenchmarkAdapter, BenchmarkAdapterOutput, BenchmarkClock, BenchmarkHardCheckResult, + BenchmarkHumanReviewNote, BenchmarkIdentity, BenchmarkManifest, + BenchmarkMetric, BenchmarkPackageFile, BenchmarkResult, + BenchmarkReviewObservation, BenchmarkSolveContext } from "./types.js"; const TEXT_FILE_LIMIT_BYTES = 1_000_000; +const OPAQUE_IDENTIFIER = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/u; +const REVIEW_NOTES = new Set(["no_revision", "minor_revision", "major_revision", "rejected"]); + +type ReferenceStatus = "available" | "unavailable"; + +interface FrozenPackage { + files: BenchmarkPackageFile[]; + digest: string; + canonical_root: string; +} + +interface PreparedReference { + declared_path: string | null; + canonical_path: string | null; + status: ReferenceStatus; + reason: string | null; +} + +interface LoadedReference { + canonical_path: string | null; + status: ReferenceStatus; + digest: string | null; + reason: string | null; + text: string | null; +} + +interface EvaluationContract { + frozen_case_sha256: string; + reference: { + availability: BenchmarkManifest["reference_policy"]["availability"]; + status: ReferenceStatus; + sha256: string | null; + }; + blind_policy: BenchmarkManifest["blind_policy"]; + budget: BenchmarkManifest["budget"]; + allowed_task_types: BenchmarkManifest["allowed_task_types"]; + expected_task_types: BenchmarkManifest["expected_task_types"]; + hard_checks: BenchmarkManifest["hard_checks"]; + execution: BenchmarkManifest["execution"]; + runtime: BenchmarkManifest["runtime"]; +} export interface RunBenchmarkCaseOptions { case_root: string; manifest: BenchmarkManifest; - adapter: import("./types.js").BenchmarkAdapter; + adapter: BenchmarkAdapter; identity: BenchmarkIdentity; clock?: BenchmarkClock; + review_observation?: BenchmarkReviewObservation; } const systemClock: BenchmarkClock = { @@ -47,12 +93,59 @@ function sha256(value: string | Buffer): string { return createHash("sha256").update(value).digest("hex"); } +function stableSerialize(value: unknown): string { + if (value === null || typeof value !== "object") return JSON.stringify(value); + if (Array.isArray(value)) return `[${value.map(stableSerialize).join(",")}]`; + const entries = Object.entries(value as Record).sort(([left], [right]) => left.localeCompare(right)); + return `{${entries.map(([key, entry]) => `${JSON.stringify(key)}:${stableSerialize(entry)}`).join(",")}}`; +} + +function assertOpaqueIdentifier(value: string, label: string): void { + if (!OPAQUE_IDENTIFIER.test(value)) { + throw new Error(`${label} must be a bounded opaque identifier`); + } +} + +function validateIdentity(identity: BenchmarkIdentity): void { + assertOpaqueIdentifier(identity.commit, "commit identity"); + assertOpaqueIdentifier(identity.environment, "environment identity"); +} + +function validateReviewObservation(observation: BenchmarkReviewObservation | undefined): BenchmarkReviewObservation | undefined { + if (observation === undefined) return undefined; + if (observation === null || typeof observation !== "object" || Array.isArray(observation)) { + throw new Error("review observation must be an object"); + } + const keys = Object.keys(observation).sort(); + if (keys.length !== 2 || keys[0] !== "minutes" || keys[1] !== "notes") { + throw new Error("review observation must contain only minutes and notes"); + } + if (observation.minutes !== null && (!Number.isFinite(observation.minutes) || observation.minutes < 0)) { + throw new Error("review observation minutes must be a non-negative number or null"); + } + if (observation.notes !== null && !REVIEW_NOTES.has(observation.notes)) { + throw new Error("review observation notes must be a safe review classification or null"); + } + if ((observation.minutes === null) !== (observation.notes === null)) { + throw new Error("review observation minutes and notes must be measured together or both unavailable"); + } + return structuredClone(observation); +} + function assertSafeRelativePath(path: string): void { if (isAbsolute(path) || path.split(/[\\/]/u).includes("..") || path.includes("\\")) { - throw new Error("benchmark package path must remain inside its case root"); + throw new Error("benchmark path must remain inside its case root"); } } +function isWithin(root: string, candidate: string): boolean { + return candidate === root || candidate.startsWith(`${root}${sep}`); +} + +function pathsOverlap(left: string, right: string): boolean { + return isWithin(left, right) || isWithin(right, left); +} + async function listFiles(root: string, current = root): Promise { const entries = await readdir(current, { withFileTypes: true }); const files: string[] = []; @@ -83,12 +176,11 @@ function mediaType(path: string): string { return extension ? (types[extension] ?? "text/plain") : "text/plain"; } -async function freezePackage(caseRoot: string, packagePath: string): Promise<{ files: BenchmarkPackageFile[]; digest: string }> { +async function freezePackage(canonicalCaseRoot: string, packagePath: string): Promise { assertSafeRelativePath(packagePath); - const canonicalCaseRoot = await realpath(caseRoot); const packageRoot = resolve(canonicalCaseRoot, packagePath); const canonicalPackageRoot = await realpath(packageRoot); - if (canonicalPackageRoot !== canonicalCaseRoot && !canonicalPackageRoot.startsWith(`${canonicalCaseRoot}${sep}`)) { + if (!isWithin(canonicalCaseRoot, canonicalPackageRoot)) { throw new Error("benchmark package resolves outside its case root"); } if (!(await stat(canonicalPackageRoot)).isDirectory()) { @@ -112,26 +204,93 @@ async function freezePackage(caseRoot: string, packagePath: string): Promise<{ f files.push({ relative_path: relativePath, media_type: mediaType(relativePath), sha256: fileDigest, content: bytes.toString("utf8") }); digestParts.push(`${relativePath}\0${fileDigest}\n`); } - return { files, digest: sha256(digestParts.join("")) }; + return { files, digest: sha256(digestParts.join("")), canonical_root: canonicalPackageRoot }; +} + +async function optionalRealpath(path: string): Promise { + try { + return await realpath(path); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; + throw error; + } } -async function readReference(caseRoot: string, manifest: BenchmarkManifest): Promise { - const path = manifest.reference_policy.relative_path; - if (manifest.reference_policy.availability !== "included" || path === null) { - return null; +async function prepareReference(canonicalCaseRoot: string, packageRoot: string, manifest: BenchmarkManifest): Promise { + const policy = manifest.reference_policy; + const relativePath = policy.relative_path; + if (relativePath === null) { + return { declared_path: null, canonical_path: null, status: "unavailable", reason: "reference_not_declared" }; } - assertSafeRelativePath(path); - const canonicalCaseRoot = await realpath(caseRoot); - const canonicalReference = await realpath(resolve(canonicalCaseRoot, path)); - if (!canonicalReference.startsWith(`${canonicalCaseRoot}${sep}`)) { + assertSafeRelativePath(relativePath); + const declaredPath = resolve(canonicalCaseRoot, relativePath); + if (pathsOverlap(packageRoot, declaredPath)) { + throw new Error("benchmark reference must remain outside the package directory"); + } + const canonicalReference = await optionalRealpath(declaredPath); + if (canonicalReference === null) { + if (policy.availability === "included") { + throw new Error("included benchmark reference is unavailable"); + } + return { declared_path: declaredPath, canonical_path: null, status: "unavailable", reason: "user_supplied_reference_missing" }; + } + if (!isWithin(canonicalCaseRoot, canonicalReference)) { throw new Error("benchmark reference resolves outside its case root"); } - const reference = await readFile(canonicalReference, "utf8"); - const declaredDigest = manifest.reference_policy.sha256; - if (declaredDigest !== null && sha256(reference) !== declaredDigest) { - throw new Error("benchmark reference digest does not match the manifest"); + if (pathsOverlap(packageRoot, canonicalReference)) { + throw new Error("benchmark reference must remain outside the package directory"); + } + const info = await lstat(canonicalReference); + if (!info.isFile()) { + throw new Error("benchmark reference path must identify a file"); + } + return { declared_path: declaredPath, canonical_path: canonicalReference, status: "available", reason: null }; +} + +async function loadReference(reference: PreparedReference, manifest: BenchmarkManifest): Promise { + if (reference.status === "unavailable" || reference.canonical_path === null) { + return { canonical_path: null, status: "unavailable", digest: null, reason: reference.reason, text: null }; + } + try { + const handle = await open(reference.canonical_path, "r"); + try { + const canonicalHandlePath = await realpath(`/proc/self/fd/${handle.fd}`); + if (canonicalHandlePath !== reference.canonical_path) { + throw new Error("benchmark reference changed before scoring"); + } + const bytes = await handle.readFile(); + const actualDigest = sha256(bytes); + if (manifest.reference_policy.sha256 !== null && actualDigest !== manifest.reference_policy.sha256) { + throw new Error("benchmark reference digest does not match the manifest"); + } + return { canonical_path: reference.canonical_path, status: "available", digest: actualDigest, reason: null, text: bytes.toString("utf8") }; + } finally { + await handle.close(); + } + } catch (error) { + if (manifest.reference_policy.availability === "user_supplied" && (error as NodeJS.ErrnoException).code === "ENOENT") { + return { canonical_path: null, status: "unavailable", digest: null, reason: "user_supplied_reference_missing", text: null }; + } + throw error; } - return reference; +} + +function evaluationContract(manifest: BenchmarkManifest, frozenDigest: string, reference: PreparedReference | LoadedReference): EvaluationContract { + return { + frozen_case_sha256: frozenDigest, + reference: { + availability: manifest.reference_policy.availability, + status: reference.status, + sha256: "digest" in reference ? reference.digest : null + }, + blind_policy: structuredClone(manifest.blind_policy), + budget: structuredClone(manifest.budget), + allowed_task_types: [...manifest.allowed_task_types], + expected_task_types: [...manifest.expected_task_types], + hard_checks: manifest.hard_checks.map((check) => ({ ...check })), + execution: structuredClone(manifest.execution), + runtime: structuredClone(manifest.runtime) + }; } function referenceStrings(value: unknown): string[] { @@ -152,9 +311,7 @@ function referenceSegments(reference: string): string[] { function normalizedWindows(value: string, minimumLength: number): Set { const normalized = value.normalize("NFKC").replace(/\s+/gu, " ").trim(); const windows = new Set(); - if (normalized.length < minimumLength) { - return windows; - } + if (normalized.length < minimumLength) return windows; for (let offset = 0; offset <= normalized.length - minimumLength; offset += Math.max(1, Math.floor(minimumLength / 4))) { windows.add(normalized.slice(offset, offset + minimumLength)); } @@ -167,9 +324,7 @@ function leakedReference(output: string, reference: string | null, minimumLength const normalizedOutput = output.normalize("NFKC").replace(/\s+/gu, " "); for (const segment of referenceSegments(reference)) { for (const window of normalizedWindows(segment, minimumLength)) { - if (normalizedOutput.includes(window)) { - return sha256(window); - } + if (normalizedOutput.includes(window)) return sha256(window); } } return null; @@ -180,41 +335,54 @@ function normalizeObservedTasks(output: BenchmarkAdapterOutput, manifest: Benchm return [...new Set(output.observed_task_types)].filter((taskType) => allowed.has(taskType)).sort(); } -function outputContractFailure(output: BenchmarkAdapterOutput, manifest: BenchmarkManifest): string | null { - if (!output || typeof output !== "object") return "adapter output is not an object"; - if (output.status !== "success" && output.status !== "failed") return "adapter output status is invalid"; - if (!Array.isArray(output.observed_task_types) || !Array.isArray(output.hard_checks) || !Array.isArray(output.artifacts) || !Array.isArray(output.evidence)) return "adapter output collections are invalid"; - if (!output.usage || (output.usage.token_count !== null && (!Number.isInteger(output.usage.token_count) || output.usage.token_count < 0))) return "adapter token usage is invalid"; - if (output.usage.cost_usd !== null && (!Number.isFinite(output.usage.cost_usd) || output.usage.cost_usd < 0)) return "adapter cost usage is invalid"; - if (typeof output.output_text !== "string") return "adapter output text is invalid"; - if (output.artifacts.some((item) => typeof item !== "string") || output.evidence.some((item) => typeof item !== "string")) return "adapter inventory is invalid"; +function outputContractFailure(value: unknown, manifest: BenchmarkManifest): string | null { + if (!value || typeof value !== "object") return "output_not_object"; + const output = value as BenchmarkAdapterOutput; + if (output.status !== "success" && output.status !== "failed") return "invalid_status"; + if (output.status === "success" && output.error !== undefined) return "success_with_error"; + if (output.status === "failed" && output.error === undefined) return "failed_without_error"; + if (!Array.isArray(output.observed_task_types) || !Array.isArray(output.hard_checks) || !Array.isArray(output.artifacts) || !Array.isArray(output.evidence)) return "invalid_collections"; + if (!output.usage || typeof output.usage !== "object" || Array.isArray(output.usage)) return "invalid_usage"; + if (output.usage.token_count !== null && (!Number.isInteger(output.usage.token_count) || output.usage.token_count < 0)) return "invalid_token_usage"; + if (output.usage.cost_usd !== null && (!Number.isFinite(output.usage.cost_usd) || output.usage.cost_usd < 0)) return "invalid_cost_usage"; + if (typeof output.output_text !== "string") return "invalid_output_text"; + if (output.artifacts.some((item) => typeof item !== "string") || output.evidence.some((item) => typeof item !== "string")) return "invalid_inventory"; const allowed = new Set(manifest.allowed_task_types); - if (output.observed_task_types.some((taskType) => !allowed.has(taskType))) return "adapter reported a disallowed task type"; + if (output.observed_task_types.some((taskType) => !allowed.has(taskType))) return "disallowed_task_type"; const expectedChecks = new Set(manifest.hard_checks.map((check) => check.id)); - if (output.hard_checks.some((check) => !check || typeof check.id !== "string" || typeof check.passed !== "boolean" || !expectedChecks.has(check.id))) return "adapter hard-check output is invalid"; + if (output.hard_checks.some((check) => !check || typeof check.id !== "string" || typeof check.passed !== "boolean" || !expectedChecks.has(check.id))) return "invalid_hard_check"; + const observedCheckIds = output.hard_checks.map((check) => check.id); + if (new Set(observedCheckIds).size !== observedCheckIds.length) return "duplicate_hard_check"; + if (observedCheckIds.length !== expectedChecks.size || [...expectedChecks].some((id) => !observedCheckIds.includes(id))) return "missing_hard_check"; + if (output.error !== undefined && (!output.error || typeof output.error.class !== "string" || typeof output.error.message !== "string")) return "invalid_error"; return null; } function hardChecks(output: BenchmarkAdapterOutput, manifest: BenchmarkManifest): BenchmarkHardCheckResult[] { const observations = new Map(output.hard_checks.map((check) => [check.id, check])); - return manifest.hard_checks.map((definition) => { - const observation = observations.get(definition.id); - const note = observation?.note; - return note === undefined - ? { id: definition.id, status: observation?.passed === true ? "passed" : "failed" } - : { id: definition.id, status: observation?.passed === true ? "passed" : "failed", note }; - }); + return manifest.hard_checks.map((definition) => ({ + id: definition.id, + status: observations.get(definition.id)?.passed === true ? "passed" : "failed" + })); } -function safeError(error: unknown): { class: string; message: string; fingerprint: string } { - const className = error instanceof Error ? error.name : "UnknownError"; +function safeError(error: unknown, fallbackClass = "HarnessError"): { class: string; message: string; fingerprint: string } { + const rawClass = error instanceof Error ? error.name : fallbackClass; + const className = OPAQUE_IDENTIFIER.test(rawClass) ? rawClass : fallbackClass; const raw = error instanceof Error ? error.message : String(error); - const fingerprint = sha256(`${className}\0${raw}`); - return { class: className.slice(0, 200) || "Error", message: `adapter failed (${fingerprint.slice(0, 12)})`, fingerprint }; + const fingerprint = sha256(`${rawClass}\0${raw}`); + return { class: className, message: `failure:${fingerprint.slice(0, 12)}`, fingerprint }; +} + +function safeAdapterError(output: BenchmarkAdapterOutput): { class: string; message: string; fingerprint: string } | null { + if (output.error === undefined) return null; + const className = OPAQUE_IDENTIFIER.test(output.error.class) ? output.error.class : "AdapterError"; + const fingerprint = sha256(`${output.error.class}\0${output.error.message}`); + return { class: className, message: `failure:${fingerprint.slice(0, 12)}`, fingerprint }; } -function runId(caseId: string, variant: string, frozenDigest: string): string { - return `${caseId}-${variant.replaceAll("_", "-")}-${frozenDigest.slice(0, 12)}`; +function runId(caseId: string, variant: string, frozenDigest: string, evaluationDigest: string): string { + return `${caseId}-${variant.replaceAll("_", "-")}-${frozenDigest.slice(0, 12)}-${evaluationDigest.slice(0, 12)}`; } function coverage(expected: readonly TaskType[], observed: readonly TaskType[]): number { @@ -227,14 +395,84 @@ function usageMetric(value: number | null, source: string) { return value === null ? unavailableMetric("adapter_did_not_report") : measuredMetric(value, source); } +function reviewMetrics(observation: BenchmarkReviewObservation | undefined): { + minutes: BenchmarkMetric; + notes: BenchmarkMetric; +} { + if (observation === undefined || observation.minutes === null || observation.notes === null) { + return { minutes: unavailableMetric("review_observation_unavailable"), notes: unavailableMetric("review_observation_unavailable") }; + } + return { + minutes: measuredMetric(observation.minutes, "review_observation"), + notes: measuredMetric(observation.notes, "review_observation") + }; +} + +function budgetProven(manifest: BenchmarkManifest, output: BenchmarkAdapterOutput, duration: number, review: BenchmarkReviewObservation | undefined): boolean { + if (duration > manifest.budget.max_wall_time_ms) return false; + if (manifest.budget.max_tokens !== null && (output.usage.token_count === null || output.usage.token_count > manifest.budget.max_tokens)) return false; + if (manifest.budget.max_cost_usd !== null && (output.usage.cost_usd === null || output.usage.cost_usd > manifest.budget.max_cost_usd)) return false; + if (manifest.budget.max_human_review_minutes !== null && (review?.minutes === null || review?.minutes === undefined || review.minutes > manifest.budget.max_human_review_minutes)) return false; + return true; +} + +function errorResult(parameters: { + manifest: BenchmarkManifest; + adapter: BenchmarkAdapter; + identity: BenchmarkIdentity; + frozenDigest: string; + evaluationDigest: string; + duration: number; + error: { class: string; message: string; fingerprint: string }; +}): BenchmarkResult { + const { manifest, adapter, identity, frozenDigest, evaluationDigest, duration, error } = parameters; + return validateBenchmarkResult({ + schema_version: "1.0.0", + case_id: manifest.case_id, + variant: adapter.variant, + adapter_id: adapter.id, + run_id: runId(manifest.case_id, adapter.variant, frozenDigest, evaluationDigest), + frozen_case_sha256: frozenDigest, + evaluation_contract_sha256: evaluationDigest, + state: "measured", + outcome: "hard_error", + observed_task_types: [], + hard_checks: manifest.hard_checks.map((check) => ({ id: check.id, status: "failed" as const })), + policy_events: [], + error, + metrics: { + completion: measuredMetric(false, "harness_scoring"), + hard_error: measuredMetric(true, "harness_scoring"), + wall_time_ms: measuredMetric(duration, "harness_clock"), + task_type_coverage: measuredMetric(0, "harness_scoring"), + custom_experiment_present: measuredMetric(false, "harness_scoring"), + token_count: unavailableMetric("adapter_failed_before_usage_report"), + cost_usd: unavailableMetric("adapter_failed_before_usage_report"), + human_review_minutes: unavailableMetric("review_observation_unavailable"), + human_review_notes: unavailableMetric("review_observation_unavailable"), + reference_leak_check: unavailableMetric("adapter_output_unavailable"), + artifact_count: measuredMetric(0, "adapter_inventory"), + evidence_count: measuredMetric(0, "adapter_inventory"), + commit_identity: measuredMetric(identity.commit, "git_commit"), + environment_identity: measuredMetric(identity.environment, "runtime_environment") + } + }); +} + export async function runBenchmarkCase(options: RunBenchmarkCaseOptions): Promise { const manifest = validateBenchmarkManifest(structuredClone(options.manifest)); + validateIdentity(options.identity); + assertOpaqueIdentifier(options.adapter.id, "adapter id"); + const review = validateReviewObservation(options.review_observation); const expectedAdapterId = options.adapter.variant === "agent" ? manifest.runtime.agent_adapter_id : manifest.runtime.one_shot_adapter_id; if (options.adapter.id !== expectedAdapterId) { - throw new Error(`adapter id ${options.adapter.id} does not match frozen manifest runtime ${expectedAdapterId}`); + throw new Error("adapter id does not match frozen manifest runtime"); } - const frozen = await freezePackage(options.case_root, manifest.package_path); - const reference = await readReference(options.case_root, manifest); + + const canonicalCaseRoot = await realpath(options.case_root); + const frozen = await freezePackage(canonicalCaseRoot, manifest.package_path); + const preparedReference = await prepareReference(canonicalCaseRoot, frozen.canonical_root, manifest); + let evaluationDigest = sha256(stableSerialize(evaluationContract(manifest, frozen.digest, preparedReference))); const context: BenchmarkSolveContext = Object.freeze({ case_id: manifest.case_id, variant: options.adapter.variant, @@ -244,6 +482,7 @@ export async function runBenchmarkCase(options: RunBenchmarkCaseOptions): Promis expected_task_types: Object.freeze([...manifest.expected_task_types]), hard_checks: Object.freeze(manifest.hard_checks.map((check) => Object.freeze({ ...check }))) }); + const clock = options.clock ?? systemClock; let duration = 0; let output: BenchmarkAdapterOutput; @@ -251,8 +490,7 @@ export async function runBenchmarkCase(options: RunBenchmarkCaseOptions): Promis const measured = await clock.measure(async () => { let timer: NodeJS.Timeout | undefined; const timeout = new Promise((_resolve, reject) => { - timer = setTimeout(() => reject(new Error("benchmark wall-time budget exceeded")), manifest.budget.max_wall_time_ms); - timer.unref(); + timer = setTimeout(() => reject(new Error("wall_time_budget_exceeded")), manifest.budget.max_wall_time_ms); }); try { return await Promise.race([options.adapter.run(context), timeout]); @@ -265,90 +503,56 @@ export async function runBenchmarkCase(options: RunBenchmarkCaseOptions): Promis } catch (error) { duration = typeof (error as { benchmark_duration_ms?: unknown })?.benchmark_duration_ms === "number" ? (error as { benchmark_duration_ms: number }).benchmark_duration_ms - : 0; - const sanitized = safeError(error); - return validateBenchmarkResult({ - schema_version: "1.0.0", - case_id: manifest.case_id, - variant: options.adapter.variant, - adapter_id: options.adapter.id, - run_id: runId(manifest.case_id, options.adapter.variant, frozen.digest), - frozen_case_sha256: frozen.digest, - state: "measured", - outcome: "hard_error", - observed_task_types: [], - hard_checks: manifest.hard_checks.map((check) => ({ id: check.id, status: "failed" as const })), - policy_events: [], - error: sanitized, - metrics: { - completion: measuredMetric(false, "harness_scoring"), - hard_error: measuredMetric(true, "harness_scoring"), - wall_time_ms: measuredMetric(duration, "harness_clock"), - task_type_coverage: measuredMetric(0, "harness_scoring"), - custom_experiment_present: measuredMetric(false, "harness_scoring"), - token_count: unavailableMetric("adapter_failed_before_usage_report"), - cost_usd: unavailableMetric("adapter_failed_before_usage_report"), - human_review_minutes: unavailableMetric("not_reviewed"), - human_review_notes: unavailableMetric("not_reviewed"), - artifact_count: measuredMetric(0, "adapter_inventory"), - evidence_count: measuredMetric(0, "adapter_inventory"), - commit_identity: measuredMetric(options.identity.commit, "git_commit"), - environment_identity: measuredMetric(options.identity.environment, "runtime_environment") - } + : manifest.budget.max_wall_time_ms; + return errorResult({ + manifest, + adapter: options.adapter, + identity: options.identity, + frozenDigest: frozen.digest, + evaluationDigest, + duration, + error: safeError(error) }); } const contractFailure = outputContractFailure(output, manifest); if (contractFailure !== null) { - const sanitized = safeError(new Error(contractFailure)); - return validateBenchmarkResult({ - schema_version: "1.0.0", - case_id: manifest.case_id, - variant: options.adapter.variant, - adapter_id: options.adapter.id, - run_id: runId(manifest.case_id, options.adapter.variant, frozen.digest), - frozen_case_sha256: frozen.digest, - state: "measured", - outcome: "hard_error", - observed_task_types: [], - hard_checks: manifest.hard_checks.map((check) => ({ id: check.id, status: "failed" as const })), - policy_events: [], - error: sanitized, - metrics: { - completion: measuredMetric(false, "harness_scoring"), - hard_error: measuredMetric(true, "harness_scoring"), - wall_time_ms: measuredMetric(duration, "harness_clock"), - task_type_coverage: measuredMetric(0, "harness_scoring"), - custom_experiment_present: measuredMetric(false, "harness_scoring"), - token_count: unavailableMetric("invalid_adapter_output"), - cost_usd: unavailableMetric("invalid_adapter_output"), - human_review_minutes: unavailableMetric("not_reviewed"), - human_review_notes: unavailableMetric("not_reviewed"), - artifact_count: measuredMetric(0, "invalid_adapter_output"), - evidence_count: measuredMetric(0, "invalid_adapter_output"), - commit_identity: measuredMetric(options.identity.commit, "git_commit"), - environment_identity: measuredMetric(options.identity.environment, "runtime_environment") - } + return errorResult({ + manifest, + adapter: options.adapter, + identity: options.identity, + frozenDigest: frozen.digest, + evaluationDigest, + duration, + error: safeError(contractFailure, "AdapterContractError") }); } + const reference = await loadReference(preparedReference, manifest); + evaluationDigest = sha256(stableSerialize(evaluationContract(manifest, frozen.digest, reference))); const observedTasks = normalizeObservedTasks(output, manifest); const checks = hardChecks(output, manifest); - const leakedFingerprint = leakedReference(output.output_text, reference, manifest.blind_policy.minimum_reference_match_characters); + const reviewValues = reviewMetrics(review); + const referenceMetric: BenchmarkMetric = reference.status === "available" + ? measuredMetric(true, "harness_leak_check") + : unavailableMetric(reference.reason ?? "reference_unavailable"); + const leakedFingerprint = leakedReference(output.output_text, reference.text, manifest.blind_policy.minimum_reference_match_characters); + if (leakedFingerprint !== null) { return validateBenchmarkResult({ schema_version: "1.0.0", case_id: manifest.case_id, variant: options.adapter.variant, adapter_id: options.adapter.id, - run_id: runId(manifest.case_id, options.adapter.variant, frozen.digest), + run_id: runId(manifest.case_id, options.adapter.variant, frozen.digest, evaluationDigest), frozen_case_sha256: frozen.digest, + evaluation_contract_sha256: evaluationDigest, state: "blocked", outcome: "blocked_policy", observed_task_types: observedTasks, hard_checks: manifest.hard_checks.map((check) => ({ id: check.id, status: "blocked" as const })), policy_events: [{ type: "same_problem_answer_detected", action: "blocked", fingerprint: leakedFingerprint }], - error: { class: "BlindPolicyViolation", message: `same-problem answer match blocked (${leakedFingerprint.slice(0, 12)})`, fingerprint: leakedFingerprint }, + error: { class: "BlindPolicyViolation", message: `failure:${leakedFingerprint.slice(0, 12)}`, fingerprint: leakedFingerprint }, metrics: { completion: measuredMetric(false, "harness_scoring"), hard_error: measuredMetric(true, "harness_scoring"), @@ -357,8 +561,9 @@ export async function runBenchmarkCase(options: RunBenchmarkCaseOptions): Promis custom_experiment_present: measuredMetric(observedTasks.includes("custom_experiment"), "harness_scoring"), token_count: usageMetric(output.usage.token_count, "adapter_usage"), cost_usd: usageMetric(output.usage.cost_usd, "adapter_usage"), - human_review_minutes: unavailableMetric("not_reviewed"), - human_review_notes: unavailableMetric("not_reviewed"), + human_review_minutes: reviewValues.minutes, + human_review_notes: reviewValues.notes, + reference_leak_check: measuredMetric(false, "harness_leak_check"), artifact_count: measuredMetric(0, "blocked_output_discarded"), evidence_count: measuredMetric(0, "blocked_output_discarded"), commit_identity: measuredMetric(options.identity.commit, "git_commit"), @@ -367,19 +572,20 @@ export async function runBenchmarkCase(options: RunBenchmarkCaseOptions): Promis }); } - const adapterFailed = output.status !== "success"; + const adapterFailed = output.status === "failed"; const checksFailed = checks.some((check) => check.status !== "passed"); - const withinBudget = duration <= manifest.budget.max_wall_time_ms; - const complete = !adapterFailed && !checksFailed && withinBudget; - const outputError = output.error === undefined ? null : safeError(new Error(`${output.error.class}: ${output.error.message}`)); + const scoringAvailable = reference.status === "available" || manifest.reference_policy.availability === "unavailable"; + const complete = !adapterFailed && !checksFailed && scoringAvailable && budgetProven(manifest, output, duration, review); const outcome = complete ? "completed" : adapterFailed ? "hard_error" : "incomplete"; + const outputError = safeAdapterError(output); return validateBenchmarkResult({ schema_version: "1.0.0", case_id: manifest.case_id, variant: options.adapter.variant, adapter_id: options.adapter.id, - run_id: runId(manifest.case_id, options.adapter.variant, frozen.digest), + run_id: runId(manifest.case_id, options.adapter.variant, frozen.digest, evaluationDigest), frozen_case_sha256: frozen.digest, + evaluation_contract_sha256: evaluationDigest, state: "measured", outcome, observed_task_types: observedTasks, @@ -394,8 +600,9 @@ export async function runBenchmarkCase(options: RunBenchmarkCaseOptions): Promis custom_experiment_present: measuredMetric(observedTasks.includes("custom_experiment"), "harness_scoring"), token_count: usageMetric(output.usage.token_count, "adapter_usage"), cost_usd: usageMetric(output.usage.cost_usd, "adapter_usage"), - human_review_minutes: unavailableMetric("not_reviewed"), - human_review_notes: unavailableMetric("not_reviewed"), + human_review_minutes: reviewValues.minutes, + human_review_notes: reviewValues.notes, + reference_leak_check: referenceMetric, artifact_count: measuredMetric(output.artifacts.length, "adapter_inventory"), evidence_count: measuredMetric(output.evidence.length, "adapter_inventory"), commit_identity: measuredMetric(options.identity.commit, "git_commit"), diff --git a/src/benchmark/synthetic.ts b/src/benchmark/synthetic.ts index 86f7f34..a3958cd 100644 --- a/src/benchmark/synthetic.ts +++ b/src/benchmark/synthetic.ts @@ -1,6 +1,6 @@ -import { readFile } from "node:fs/promises"; +import { access, readFile } from "node:fs/promises"; +import { dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; -import { resolve } from "node:path"; import type { TaskType } from "../contracts/types.js"; import { validateBenchmarkManifest } from "./contracts.js"; import { aggregateBenchmarkResults, writeBenchmarkReports } from "./report.js"; @@ -17,9 +17,26 @@ import type { BenchmarkVariant } from "./types.js"; -const defaultBenchmarkRoot = fileURLToPath(new URL("../../benchmarks/synthetic/", import.meta.url)); const caseDirectories = ["summary-statistics", "custom-threshold"] as const; +async function defaultBenchmarkRoot(): Promise { + const moduleDirectory = dirname(fileURLToPath(import.meta.url)); + const candidates = [ + resolve(moduleDirectory, "../../benchmarks/synthetic"), + resolve(moduleDirectory, "../../../benchmarks/synthetic"), + resolve(process.cwd(), "benchmarks/synthetic") + ]; + for (const candidate of candidates) { + try { + await access(resolve(candidate, caseDirectories[0], "manifest.json")); + return candidate; + } catch { + // Try the next explicit source/dist/project-root candidate. + } + } + throw new Error("synthetic benchmark fixture root is unavailable"); +} + const deterministicClock: BenchmarkClock = { async measure(operation: () => Promise): Promise<{ value: T; duration_ms: number }> { return { value: await operation(), duration_ms: 1 }; @@ -47,7 +64,7 @@ function deterministicOutput(context: BenchmarkSolveContext, variant: BenchmarkV hard_checks: hardChecksFor(context), artifacts: custom ? ["experiment.json", "summary.md"] : ["summary.md"], evidence: custom ? ["threshold-evidence", "summary-evidence"] : ["summary-evidence"], - usage: { token_count: null, cost_usd: null }, + usage: { token_count: 1, cost_usd: null }, output_text: `${variant} deterministic synthetic output for ${context.case_id}; ${custom ? "custom experiment included" : "statistical summary included"}.` }; } @@ -76,7 +93,7 @@ export interface SyntheticBenchmarkRun { } export async function runSyntheticBenchmarks(options: RunSyntheticBenchmarksOptions): Promise { - const benchmarkRoot = options.benchmark_root ?? defaultBenchmarkRoot; + const benchmarkRoot = options.benchmark_root ?? await defaultBenchmarkRoot(); const results: BenchmarkResult[] = []; for (const directory of caseDirectories) { const caseRoot = resolve(benchmarkRoot, directory); diff --git a/src/benchmark/types.ts b/src/benchmark/types.ts index 28138c6..779ee0d 100644 --- a/src/benchmark/types.ts +++ b/src/benchmark/types.ts @@ -7,6 +7,7 @@ export type BenchmarkState = "measured" | "not_run" | "blocked"; export type BenchmarkOutcome = "completed" | "incomplete" | "hard_error" | "blocked_policy" | "not_run"; export type MetricStatus = "measured" | "unavailable" | "not_run" | "blocked"; export type MetricValue = boolean | number | string; +export type BenchmarkHumanReviewNote = "no_revision" | "minor_revision" | "major_revision" | "rejected"; export interface BenchmarkLicense { name: string; @@ -113,6 +114,11 @@ export interface BenchmarkIdentity { environment: string; } +export interface BenchmarkReviewObservation { + minutes: number | null; + notes: BenchmarkHumanReviewNote | null; +} + export interface MeasuredMetric { status: "measured"; value: T; @@ -148,7 +154,8 @@ export interface BenchmarkMetrics { token_count: BenchmarkMetric; cost_usd: BenchmarkMetric; human_review_minutes: BenchmarkMetric; - human_review_notes: BenchmarkMetric; + human_review_notes: BenchmarkMetric; + reference_leak_check: BenchmarkMetric; artifact_count: BenchmarkMetric; evidence_count: BenchmarkMetric; commit_identity: BenchmarkMetric; @@ -158,7 +165,6 @@ export interface BenchmarkMetrics { export interface BenchmarkHardCheckResult { id: string; status: "passed" | "failed" | "blocked" | "not_run"; - note?: string; } export interface BenchmarkPolicyEvent { @@ -174,6 +180,7 @@ export interface BenchmarkResult { adapter_id: string; run_id: string; frozen_case_sha256: string; + evaluation_contract_sha256: string; state: BenchmarkState; outcome: BenchmarkOutcome; observed_task_types: TaskType[]; diff --git a/tests/benchmark-compiled.test.ts b/tests/benchmark-compiled.test.ts new file mode 100644 index 0000000..2b918e3 --- /dev/null +++ b/tests/benchmark-compiled.test.ts @@ -0,0 +1,97 @@ +import { execFile } from "node:child_process"; +import { mkdtemp, readFile, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { pathToFileURL } from "node:url"; +import { promisify } from "node:util"; +import { describe, expect, it } from "vitest"; + +const execFileAsync = promisify(execFile); +const repositoryRoot = resolve(import.meta.dirname, ".."); + +async function runNode(args: string[], timeout = 15_000): Promise<{ stdout: string; stderr: string }> { + return execFileAsync(process.execPath, args, { + cwd: repositoryRoot, + timeout, + env: { ...process.env, NODE_NO_WARNINGS: "1" } + }); +} + +describe("compiled benchmark entrypoints", () => { + it("runs the built synthetic CLI from the repository root without exposing the host output path", async () => { + const output = await mkdtemp(join(tmpdir(), "benchmark-compiled-output-")); + const { stdout, stderr } = await runNode(["dist/src/benchmark/run-synthetic.js", "--output", output]); + + expect(stderr).toBe(""); + expect(stdout).not.toContain(output); + expect(stdout).not.toMatch(/\/(?:home|Users|tmp)\//); + expect(JSON.parse(stdout)).toEqual({ + suite_id: "synthetic-v1", + total_runs: 4, + completed_runs: 4, + json_report: "benchmark-report.json", + markdown_report: "benchmark-report.md" + }); + expect(JSON.parse(await readFile(resolve(output, "benchmark-report.json"), "utf8"))).toMatchObject({ + suite_id: "synthetic-v1", + summary: { total_runs: 4, completed_runs: 4 } + }); + }); + + it("keeps a referenced timeout alive in an otherwise handle-free child process until a hard-error result is emitted", async () => { + const fixture = await mkdtemp(join(tmpdir(), "benchmark-timeout-child-")); + const child = resolve(fixture, "timeout-child.mjs"); + await writeFile(child, ` +import { mkdir, writeFile } from "node:fs/promises"; +import { resolve } from "node:path"; +import { runBenchmarkCase } from ${JSON.stringify(pathToFileURL(resolve(repositoryRoot, "dist/src/benchmark/runner.js")).href)}; + +const root = process.argv[2]; +await mkdir(resolve(root, "package"), { recursive: true }); +await writeFile(resolve(root, "package/problem.md"), "Synthetic timeout package.\\n", "utf8"); +const manifest = { + schema_version: "1.0.0", + case_id: "synthetic-timeout-child", + package_path: "package", + license: { + name: "CC0 1.0 Universal", + spdx_id: "CC0-1.0", + copyright_holder: "modeling-agent contributors", + source_url: null, + redistribution: "permitted", + notice_path: null + }, + blind_policy: { + mode: "blind", + solve_input: "package_only", + same_problem_answers: "block", + minimum_reference_match_characters: 24 + }, + reference_policy: { access: "scoring_only", availability: "unavailable", relative_path: null, sha256: null }, + runtime: { agent_adapter_id: "timeout-agent-v1", one_shot_adapter_id: "timeout-one-shot-v1" }, + execution: { kind: "local", network_access: "disabled" }, + budget: { max_wall_time_ms: 40, max_tokens: null, max_cost_usd: null, max_human_review_minutes: null }, + allowed_task_types: ["statistical_analysis"], + expected_task_types: ["statistical_analysis"], + hard_checks: [{ id: "answer-present", description: "A synthetic answer is present." }] +}; +const result = await runBenchmarkCase({ + case_root: root, + manifest, + adapter: { id: "timeout-agent-v1", variant: "agent", run: async () => new Promise(() => undefined) }, + identity: { commit: "timeout-child-v1", environment: "node-child-v1" } +}); +process.stdout.write(JSON.stringify(result)); +`, "utf8"); + + const { stdout, stderr } = await runNode([child, fixture], 5_000); + expect(stderr).toBe(""); + expect(JSON.parse(stdout)).toMatchObject({ + outcome: "hard_error", + metrics: { + completion: { status: "measured", value: false }, + hard_error: { status: "measured", value: true } + } + }); + }); +}); diff --git a/tests/benchmark-contracts.test.ts b/tests/benchmark-contracts.test.ts index dbbc0ea..f919422 100644 --- a/tests/benchmark-contracts.test.ts +++ b/tests/benchmark-contracts.test.ts @@ -58,6 +58,7 @@ const result: BenchmarkResult = { adapter_id: "deterministic-agent-v1", run_id: "synthetic-contract-agent-aaaaaaaaaaaa", frozen_case_sha256: "b".repeat(64), + evaluation_contract_sha256: "c".repeat(64), state: "measured", outcome: "completed", observed_task_types: ["statistical_analysis"], @@ -74,6 +75,7 @@ const result: BenchmarkResult = { cost_usd: unavailableMetric("adapter_did_not_report"), human_review_minutes: unavailableMetric("not_reviewed"), human_review_notes: unavailableMetric("not_reviewed"), + reference_leak_check: measuredMetric(true, "harness_leak_check"), artifact_count: measuredMetric(2, "adapter_inventory"), evidence_count: measuredMetric(3, "adapter_inventory"), commit_identity: measuredMetric("a".repeat(40), "git_commit"), @@ -98,6 +100,10 @@ describe("benchmark contracts", () => { const leakingPolicy = structuredClone(manifest); leakingPolicy.reference_policy.access = "solve_and_score" as "scoring_only"; expect(() => validateBenchmarkManifest(leakingPolicy)).toThrow(BenchmarkContractError); + + const unsafeAdapter = structuredClone(manifest); + unsafeAdapter.runtime.agent_adapter_id = "../../token=TOP-SECRET"; + expect(() => validateBenchmarkManifest(unsafeAdapter)).toThrow(BenchmarkContractError); }); it("represents unknown token, cost, and human-review values as unavailable nulls rather than zero", () => { @@ -119,5 +125,9 @@ describe("benchmark contracts", () => { const failedCheck = structuredClone(result); failedCheck.hard_checks[0]!.status = "failed"; expect(() => validateBenchmarkResult(failedCheck)).toThrow(/completed result/i); + + const completedWithError = structuredClone(result); + completedWithError.error = { class: "AdapterError", message: `failure:${"a".repeat(12)}`, fingerprint: "a".repeat(64) }; + expect(() => validateBenchmarkResult(completedWithError)).toThrow(/completed result/i); }); }); diff --git a/tests/benchmark-reference-lifecycle.test.ts b/tests/benchmark-reference-lifecycle.test.ts new file mode 100644 index 0000000..4ff63d8 --- /dev/null +++ b/tests/benchmark-reference-lifecycle.test.ts @@ -0,0 +1,118 @@ +import { spawn } from "node:child_process"; +import { mkdtemp, mkdir, readFile, rename, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { describe, expect, it } from "vitest"; +import type { BenchmarkManifest } from "../src/benchmark/types.js"; + +async function waitForFile(path: string): Promise { + const deadline = Date.now() + 5_000; + while (Date.now() < deadline) { + try { + await readFile(path); + return; + } catch { + await new Promise((resolvePromise) => setTimeout(resolvePromise, 10)); + } + } + throw new Error("timed out waiting for benchmark adapter signal"); +} + +function waitForExit(child: ReturnType): Promise<{ code: number | null; stdout: string; stderr: string }> { + return new Promise((resolvePromise, reject) => { + let stdout = ""; + let stderr = ""; + child.stdout?.on("data", (chunk) => { stdout += String(chunk); }); + child.stderr?.on("data", (chunk) => { stderr += String(chunk); }); + child.once("error", reject); + child.once("exit", (code) => resolvePromise({ code, stdout, stderr })); + }); +} + +describe("benchmark reference lifecycle", () => { + it("does not open included reference bytes before the adapter completes", async () => { + const root = await mkdtemp(join(tmpdir(), "benchmark-reference-lifecycle-")); + const signal = resolve(root, "adapter-started"); + const release = resolve(root, "adapter-release"); + const reference = resolve(root, "reference/reference.txt"); + const movedReference = resolve(root, "reference/reference-moved.txt"); + const childScript = resolve(root, "child.mts"); + await mkdir(resolve(root, "package")); + await mkdir(resolve(root, "reference")); + await writeFile(resolve(root, "package/problem.md"), "Synthetic lifecycle package.\n", "utf8"); + await writeFile(reference, "REFERENCE-LIFECYCLE-SECRET-314159265358979\n", "utf8"); + const manifest: BenchmarkManifest = { + schema_version: "1.0.0", + case_id: "synthetic-reference-lifecycle", + package_path: "package", + license: { + name: "CC0 1.0 Universal", + spdx_id: "CC0-1.0", + copyright_holder: "modeling-agent contributors", + source_url: null, + redistribution: "permitted", + notice_path: null + }, + blind_policy: { + mode: "blind", + solve_input: "package_only", + same_problem_answers: "block", + minimum_reference_match_characters: 24 + }, + reference_policy: { access: "scoring_only", availability: "included", relative_path: "reference/reference.txt", sha256: null }, + runtime: { agent_adapter_id: "lifecycle-agent-v1", one_shot_adapter_id: "lifecycle-one-shot-v1" }, + execution: { kind: "local", network_access: "disabled" }, + budget: { max_wall_time_ms: 5_000, max_tokens: null, max_cost_usd: null, max_human_review_minutes: null }, + allowed_task_types: ["statistical_analysis"], + expected_task_types: ["statistical_analysis"], + hard_checks: [{ id: "answer-present", description: "A synthetic answer is present." }] + }; + await writeFile(childScript, ` +import { access, writeFile } from "node:fs/promises"; +import { runBenchmarkCase } from ${JSON.stringify(resolve(process.cwd(), "src/benchmark/runner.ts"))}; +const [root, signal, release] = process.argv.slice(2); +const manifest = ${JSON.stringify(manifest)}; +const result = await runBenchmarkCase({ + case_root: root, + manifest, + adapter: { + id: "lifecycle-agent-v1", + variant: "agent", + run: async () => { + await writeFile(signal, "ready"); + while (true) { + try { await access(release); break; } catch { await new Promise((resolvePromise) => setTimeout(resolvePromise, 10)); } + } + return { + status: "success", + observed_task_types: ["statistical_analysis"], + hard_checks: [{ id: "answer-present", passed: true }], + artifacts: ["answer.md"], + evidence: ["answer-evidence"], + usage: { token_count: null, cost_usd: null }, + output_text: "Synthetic result." + }; + } + }, + identity: { commit: "lifecycle-child-v1", environment: "node-child-v1" } +}); +process.stdout.write(JSON.stringify(result)); +`, "utf8"); + + const child = spawn(resolve(process.cwd(), "node_modules/.bin/tsx"), [childScript, root, signal, release], { + cwd: process.cwd(), + stdio: ["ignore", "pipe", "pipe"] + }); + try { + await waitForFile(signal); + await rename(reference, movedReference); + await writeFile(release, "go"); + const exit = await waitForExit(child); + expect(exit.code).toBe(1); + expect(exit.stdout).toBe(""); + expect(exit.stderr).toMatch(/ENOENT|unavailable/i); + } finally { + child.kill("SIGKILL"); + } + }, 15_000); +}); diff --git a/tests/benchmark-regressions.test.ts b/tests/benchmark-regressions.test.ts new file mode 100644 index 0000000..3a5481a --- /dev/null +++ b/tests/benchmark-regressions.test.ts @@ -0,0 +1,443 @@ +import { mkdtemp, mkdir, rm, symlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { describe, expect, it } from "vitest"; +import { aggregateBenchmarkResults } from "../src/benchmark/report.js"; +import { runBenchmarkCase, type RunBenchmarkCaseOptions } from "../src/benchmark/runner.js"; +import type { + BenchmarkAdapter, + BenchmarkAdapterOutput, + BenchmarkManifest, + BenchmarkResult, + BenchmarkSolveContext, + BenchmarkVariant +} from "../src/benchmark/types.js"; + +const ORIGINAL_REFERENCE = "REFERENCE-ORIGINAL-ALPHA-314159265358979323846"; +const REPLACEMENT_REFERENCE = "REFERENCE-REPLACED-BETA-271828182845904523536"; + +async function createCase(): Promise<{ caseRoot: string; manifest: BenchmarkManifest; referencePath: string }> { + const caseRoot = await mkdtemp(join(tmpdir(), "benchmark-regression-")); + const referencePath = resolve(caseRoot, "reference/reference.txt"); + await mkdir(resolve(caseRoot, "package")); + await mkdir(resolve(caseRoot, "reference")); + await writeFile(resolve(caseRoot, "package/problem.md"), "Solve this synthetic package.\n", "utf8"); + await writeFile(referencePath, `${ORIGINAL_REFERENCE}\n`, "utf8"); + return { + caseRoot, + referencePath, + manifest: { + schema_version: "1.0.0", + case_id: "synthetic-regression", + package_path: "package", + license: { + name: "CC0 1.0 Universal", + spdx_id: "CC0-1.0", + copyright_holder: "modeling-agent contributors", + source_url: null, + redistribution: "permitted", + notice_path: null + }, + blind_policy: { + mode: "blind", + solve_input: "package_only", + same_problem_answers: "block", + minimum_reference_match_characters: 24 + }, + reference_policy: { + access: "scoring_only", + availability: "included", + relative_path: "reference/reference.txt", + sha256: null + }, + runtime: { agent_adapter_id: "agent-regression-v1", one_shot_adapter_id: "one-shot-regression-v1" }, + execution: { kind: "local", network_access: "disabled" }, + budget: { max_wall_time_ms: 1_000, max_tokens: null, max_cost_usd: null, max_human_review_minutes: null }, + allowed_task_types: ["statistical_analysis", "custom_experiment"], + expected_task_types: ["statistical_analysis"], + hard_checks: [{ id: "answer-present", description: "A synthetic answer is present." }] + } + }; +} + +function outputFor(manifest: BenchmarkManifest, overrides: Partial = {}): BenchmarkAdapterOutput { + return { + status: "success", + observed_task_types: [...manifest.expected_task_types], + hard_checks: manifest.hard_checks.map((check) => ({ id: check.id, passed: true })), + artifacts: ["answer.md"], + evidence: ["answer-evidence"], + usage: { token_count: null, cost_usd: null }, + output_text: "A synthetic answer was produced.", + ...overrides + }; +} + +function adapter( + id: string, + variant: BenchmarkVariant, + run: (context: BenchmarkSolveContext) => Promise +): BenchmarkAdapter { + return { id, variant, run }; +} + +function deterministicClock(duration_ms = 1) { + return { measure: async (operation: () => Promise) => ({ value: await operation(), duration_ms }) }; +} + +async function runVariant( + caseRoot: string, + manifest: BenchmarkManifest, + variant: BenchmarkVariant, + overrides: Partial = {}, + extra: Partial & Record = {} +): Promise { + const id = variant === "agent" ? manifest.runtime.agent_adapter_id : manifest.runtime.one_shot_adapter_id; + const options = { + case_root: caseRoot, + manifest, + adapter: adapter(id, variant, async () => outputFor(manifest, overrides)), + identity: { commit: "commit-regression-v1", environment: "node-test-v1" }, + clock: deterministicClock(), + ...extra + }; + return runBenchmarkCase(options as RunBenchmarkCaseOptions); +} + +function evaluationDigest(result: BenchmarkResult): unknown { + return (result as BenchmarkResult & { evaluation_contract_sha256?: unknown }).evaluation_contract_sha256; +} + +function referenceLeakMetric(result: BenchmarkResult): unknown { + return (result.metrics as BenchmarkResult["metrics"] & { reference_leak_check?: unknown }).reference_leak_check; +} + +describe("benchmark regression hardening", () => { + it("rejects a reference nested in the package before invoking the solve adapter", async () => { + const { caseRoot, manifest } = await createCase(); + const secret = "REFERENCE-NESTED-IN-PACKAGE-TOP-SECRET-123456789"; + await writeFile(resolve(caseRoot, "package/reference.txt"), secret, "utf8"); + manifest.reference_policy.relative_path = "package/reference.txt"; + let adapterCalled = false; + let solveSawReference = false; + + const run = runBenchmarkCase({ + case_root: caseRoot, + manifest, + adapter: adapter("agent-regression-v1", "agent", async (context) => { + adapterCalled = true; + solveSawReference = JSON.stringify(context).includes(secret); + return outputFor(manifest); + }), + identity: { commit: "commit-regression-v1", environment: "node-test-v1" }, + clock: deterministicClock() + }); + + await expect(run).rejects.toThrow(/reference.*package/i); + expect(adapterCalled).toBe(false); + expect(solveSawReference).toBe(false); + }); + + it("rejects a package symlink whose canonical directory covers the reference", async () => { + const { caseRoot, manifest } = await createCase(); + await rm(resolve(caseRoot, "package"), { recursive: true }); + await mkdir(resolve(caseRoot, "shared")); + await writeFile(resolve(caseRoot, "shared/problem.md"), "Synthetic problem.\n", "utf8"); + await writeFile(resolve(caseRoot, "shared/reference.txt"), ORIGINAL_REFERENCE, "utf8"); + await symlink("shared", resolve(caseRoot, "package"), "dir"); + manifest.reference_policy.relative_path = "shared/reference.txt"; + let adapterCalled = false; + + const run = runBenchmarkCase({ + case_root: caseRoot, + manifest, + adapter: adapter("agent-regression-v1", "agent", async () => { + adapterCalled = true; + return outputFor(manifest); + }), + identity: { commit: "commit-regression-v1", environment: "node-test-v1" }, + clock: deterministicClock() + }); + + await expect(run).rejects.toThrow(/reference.*package/i); + expect(adapterCalled).toBe(false); + }); + + it.each(["included", "user_supplied"] as const)( + "opens an %s reference only after the adapter finishes and keeps it out of solve context", + async (availability) => { + const { caseRoot, manifest } = await createCase(); + manifest.reference_policy.availability = availability; + const result = await runBenchmarkCase({ + case_root: caseRoot, + manifest, + adapter: adapter("agent-regression-v1", "agent", async (context) => { + const serialized = JSON.stringify(context); + expect(serialized).not.toContain(ORIGINAL_REFERENCE); + expect(serialized).not.toContain("reference/reference.txt"); + expect(Object.keys(context).sort()).toEqual([ + "budget", + "case_id", + "expected_task_types", + "frozen_case_sha256", + "hard_checks", + "package_files", + "variant" + ]); + return outputFor(manifest, { output_text: ORIGINAL_REFERENCE }); + }), + identity: { commit: "commit-regression-v1", environment: "node-test-v1" }, + clock: deterministicClock() + }); + + expect(result.outcome).toBe("blocked_policy"); + expect(result.metrics.completion).toMatchObject({ status: "measured", value: false }); + expect(JSON.stringify(result)).not.toContain(ORIGINAL_REFERENCE); + } + ); + + it("marks a missing user-supplied scoring reference unavailable and does not claim completion", async () => { + const { caseRoot, manifest } = await createCase(); + manifest.reference_policy = { + access: "scoring_only", + availability: "user_supplied", + relative_path: "reference/not-provided.txt", + sha256: null + }; + + const result = await runVariant(caseRoot, manifest, "agent"); + expect(result.outcome).toBe("incomplete"); + expect(result.metrics.completion).toMatchObject({ status: "measured", value: false }); + expect(referenceLeakMetric(result)).toMatchObject({ status: "unavailable", value: null }); + }); + + it("records one stable evaluation contract for both variants and binds it into run ids", async () => { + const { caseRoot, manifest } = await createCase(); + const agentResult = await runVariant(caseRoot, manifest, "agent"); + const baselineResult = await runVariant(caseRoot, manifest, "one_shot"); + + expect(evaluationDigest(agentResult)).toMatch(/^[a-f0-9]{64}$/); + expect(evaluationDigest(agentResult)).toBe(evaluationDigest(baselineResult)); + expect(agentResult.run_id).not.toBe(baselineResult.run_id); + expect(agentResult.run_id).toContain(String(evaluationDigest(agentResult)).slice(0, 12)); + }); + + it.each(["budget", "expected tasks", "hard checks", "reference bytes"] as const)( + "refuses to compare variants with different %s in the frozen evaluation contract", + async (dimension) => { + const { caseRoot, manifest, referencePath } = await createCase(); + const agentResult = await runVariant(caseRoot, manifest, "agent"); + const altered = structuredClone(manifest); + if (dimension === "budget") { + altered.budget.max_tokens = 50; + } else if (dimension === "expected tasks") { + altered.expected_task_types = ["statistical_analysis", "custom_experiment"]; + } else if (dimension === "hard checks") { + altered.hard_checks[0]!.description = "A materially different scoring requirement."; + } else { + await writeFile(referencePath, `${REPLACEMENT_REFERENCE}\n`, "utf8"); + } + const usage = dimension === "budget" ? { token_count: 10, cost_usd: null } : { token_count: null, cost_usd: null }; + const baselineResult = await runVariant(caseRoot, altered, "one_shot", { usage }); + + expect(evaluationDigest(agentResult)).not.toBe(evaluationDigest(baselineResult)); + expect(() => aggregateBenchmarkResults([agentResult, baselineResult])).toThrow(/evaluation contract/i); + } + ); + + it("binds both adapter ids plus blind and execution policy into the evaluation contract", async () => { + const { caseRoot, manifest } = await createCase(); + const agentResult = await runVariant(caseRoot, manifest, "agent"); + const altered = structuredClone(manifest); + altered.runtime.agent_adapter_id = "agent-regression-v2"; + altered.blind_policy.minimum_reference_match_characters += 1; + altered.execution.network_access = "research_gateway_only"; + const baselineResult = await runVariant(caseRoot, altered, "one_shot"); + + expect(evaluationDigest(agentResult)).not.toBe(evaluationDigest(baselineResult)); + expect(() => aggregateBenchmarkResults([agentResult, baselineResult])).toThrow(/evaluation contract/i); + }); + + it.each([ + { name: "wall time", budget: { max_wall_time_ms: 5 }, output: {}, duration: 6 }, + { name: "tokens over limit", budget: { max_tokens: 10 }, output: { usage: { token_count: 11, cost_usd: null } }, duration: 1 }, + { name: "tokens unavailable", budget: { max_tokens: 10 }, output: { usage: { token_count: null, cost_usd: null } }, duration: 1 }, + { name: "cost over limit", budget: { max_cost_usd: 0.1 }, output: { usage: { token_count: null, cost_usd: 0.2 } }, duration: 1 }, + { name: "cost unavailable", budget: { max_cost_usd: 0.1 }, output: { usage: { token_count: null, cost_usd: null } }, duration: 1 }, + { name: "review over limit", budget: { max_human_review_minutes: 5 }, output: {}, duration: 1, review: { minutes: 6, notes: "major_revision" } }, + { name: "review unavailable", budget: { max_human_review_minutes: 5 }, output: {}, duration: 1 } + ])("makes completion incomplete when the declared $name budget cannot be proven", async ({ budget, output, duration, review }) => { + const { caseRoot, manifest } = await createCase(); + Object.assign(manifest.budget, budget); + const options = { + case_root: caseRoot, + manifest, + adapter: adapter("agent-regression-v1", "agent", async () => outputFor(manifest, output)), + identity: { commit: "commit-regression-v1", environment: "node-test-v1" }, + clock: deterministicClock(duration), + review_observation: review + } as RunBenchmarkCaseOptions & { review_observation?: unknown }; + + const result = await runBenchmarkCase(options); + expect(result.outcome).toBe("incomplete"); + expect(result.metrics.completion).toMatchObject({ status: "measured", value: false }); + expect(result.metrics.hard_error).toMatchObject({ status: "measured", value: false }); + }); + + it("records a bounded human-review observation and permits completion within all declared budgets", async () => { + const { caseRoot, manifest } = await createCase(); + manifest.budget = { max_wall_time_ms: 100, max_tokens: 20, max_cost_usd: 1, max_human_review_minutes: 5 }; + const options = { + case_root: caseRoot, + manifest, + adapter: adapter("agent-regression-v1", "agent", async () => outputFor(manifest, { + usage: { token_count: 10, cost_usd: 0.5 } + })), + identity: { commit: "commit-regression-v1", environment: "node-test-v1" }, + clock: deterministicClock(10), + review_observation: { minutes: 4, notes: "minor_revision" } + } as RunBenchmarkCaseOptions & { review_observation: unknown }; + + const result = await runBenchmarkCase(options); + expect(result.outcome).toBe("completed"); + expect(result.metrics.human_review_minutes).toMatchObject({ status: "measured", value: 4 }); + expect(result.metrics.human_review_notes).toMatchObject({ status: "measured", value: "minor_revision" }); + }); + + it("rejects arbitrary human-review notes before adapter execution", async () => { + const { caseRoot, manifest } = await createCase(); + let adapterCalled = false; + const options = { + case_root: caseRoot, + manifest, + adapter: adapter("agent-regression-v1", "agent", async () => { + adapterCalled = true; + return outputFor(manifest); + }), + identity: { commit: "commit-regression-v1", environment: "node-test-v1" }, + review_observation: { minutes: 1, notes: "token=TOP-SECRET /home/private" } + } as unknown as RunBenchmarkCaseOptions; + + await expect(runBenchmarkCase(options)).rejects.toThrow(/review observation/i); + expect(adapterCalled).toBe(false); + }); + + it("drops untrusted hard-check notes instead of persisting reference, credential, or host path text", async () => { + const { caseRoot, manifest } = await createCase(); + const malicious = "REFERENCE token=TOP-SECRET /home/private/result.txt"; + const result = await runVariant(caseRoot, manifest, "agent", { + hard_checks: [{ id: "answer-present", passed: true, note: malicious }] + }); + + expect(result.outcome).toBe("completed"); + expect(JSON.stringify(result)).not.toContain(malicious); + expect(JSON.stringify(result)).not.toContain("TOP-SECRET"); + expect(JSON.stringify(result)).not.toContain("/home/private"); + }); + + it.each([ + { + name: "success carrying an error", + make: (manifest: BenchmarkManifest) => outputFor(manifest, { status: "success", error: { class: "Error", message: "TOP-SECRET" } }) + }, + { + name: "failed without an error", + make: (manifest: BenchmarkManifest) => { + const output = outputFor(manifest, { status: "failed" }); + delete output.error; + return output; + } + }, + { + name: "missing hard-check observation", + make: (manifest: BenchmarkManifest) => outputFor(manifest, { hard_checks: [] }) + }, + { + name: "duplicate hard-check observation", + make: (manifest: BenchmarkManifest) => outputFor(manifest, { + hard_checks: [ + { id: "answer-present", passed: true }, + { id: "answer-present", passed: true, note: "token=TOP-SECRET /home/private" } + ] + }) + }, + { + name: "unknown hard-check observation", + make: (manifest: BenchmarkManifest) => outputFor(manifest, { hard_checks: [{ id: "unknown-check", passed: true }] }) + } + ])("turns $name into a fingerprint-only measured hard error", async ({ make }) => { + const { caseRoot, manifest } = await createCase(); + const invalidOutput = make(manifest); + const result = await runBenchmarkCase({ + case_root: caseRoot, + manifest, + adapter: adapter("agent-regression-v1", "agent", async () => invalidOutput), + identity: { commit: "commit-regression-v1", environment: "node-test-v1" }, + clock: deterministicClock() + }); + + expect(result.outcome).toBe("hard_error"); + expect(result.metrics.completion).toMatchObject({ status: "measured", value: false }); + expect(result.metrics.hard_error).toMatchObject({ status: "measured", value: true }); + expect(result.error?.fingerprint).toMatch(/^[a-f0-9]{64}$/); + expect(result.error?.message).toMatch(/^failure:[a-f0-9]{12}$/); + expect(JSON.stringify(result)).not.toContain("TOP-SECRET"); + expect(JSON.stringify(result)).not.toContain("/home/private"); + }); + + it("rejects unsafe adapter, manifest identity, and suite identifiers without persisting them", async () => { + const { caseRoot, manifest } = await createCase(); + let adapterCalled = false; + const unsafeEnvironment = "token=TOP-SECRET /home/private"; + await expect(runBenchmarkCase({ + case_root: caseRoot, + manifest, + adapter: adapter("agent-regression-v1", "agent", async () => { + adapterCalled = true; + return outputFor(manifest); + }), + identity: { commit: "commit-regression-v1", environment: unsafeEnvironment }, + clock: deterministicClock() + })).rejects.toThrow(/identity/i); + expect(adapterCalled).toBe(false); + + const unsafeManifest = structuredClone(manifest); + unsafeManifest.runtime.agent_adapter_id = "../../token=TOP-SECRET"; + await expect(runBenchmarkCase({ + case_root: caseRoot, + manifest: unsafeManifest, + adapter: adapter("../../token=TOP-SECRET", "agent", async () => outputFor(unsafeManifest)), + identity: { commit: "commit-regression-v1", environment: "node-test-v1" }, + clock: deterministicClock() + })).rejects.toThrow(); + + expect(() => aggregateBenchmarkResults([], "../../token=TOP-SECRET /home/private")).toThrow(/suite id/i); + }); + + it("sanitizes an adapter-reported error class and message to an opaque class plus fingerprint", async () => { + const { caseRoot, manifest } = await createCase(); + const result = await runVariant(caseRoot, manifest, "agent", { + status: "failed", + error: { class: "../../SecretError token=TOP-SECRET", message: "/home/private/credential.txt" } + }); + + expect(result.outcome).toBe("hard_error"); + expect(result.error?.class).toMatch(/^[A-Za-z0-9][A-Za-z0-9._-]*$/); + expect(result.error?.message).toMatch(/^failure:[a-f0-9]{12}$/); + expect(JSON.stringify(result)).not.toContain("TOP-SECRET"); + expect(JSON.stringify(result)).not.toContain("/home/private"); + }); + + it("requires the new evaluation digest and leak-check boundary in every production result", async () => { + const { caseRoot, manifest } = await createCase(); + const result = await runVariant(caseRoot, manifest, "agent"); + const missingDigest = structuredClone(result) as unknown as Omit & { evaluation_contract_sha256?: string }; + delete missingDigest.evaluation_contract_sha256; + expect(() => aggregateBenchmarkResults([missingDigest as BenchmarkResult])).toThrow(); + + const missingLeakCheck = structuredClone(result) as BenchmarkResult; + const partialMetrics = missingLeakCheck.metrics as unknown as { reference_leak_check?: unknown }; + delete partialMetrics.reference_leak_check; + expect(() => aggregateBenchmarkResults([missingLeakCheck])).toThrow(); + }); +}); diff --git a/tests/benchmark-report.test.ts b/tests/benchmark-report.test.ts index 1dd4a0f..6d129a0 100644 --- a/tests/benchmark-report.test.ts +++ b/tests/benchmark-report.test.ts @@ -14,6 +14,7 @@ function result(overrides: Partial = {}): BenchmarkResult { adapter_id: "agent-v1", run_id: "synthetic-report-agent-aaaaaaaaaaaa", frozen_case_sha256: "a".repeat(64), + evaluation_contract_sha256: "c".repeat(64), state: "measured", outcome: "completed", observed_task_types: ["statistical_analysis"], @@ -30,6 +31,7 @@ function result(overrides: Partial = {}): BenchmarkResult { cost_usd: unavailableMetric("adapter_did_not_report"), human_review_minutes: unavailableMetric("not_reviewed"), human_review_notes: unavailableMetric("not_reviewed"), + reference_leak_check: measuredMetric(true, "harness_leak_check"), artifact_count: measuredMetric(1, "adapter_inventory"), evidence_count: measuredMetric(1, "adapter_inventory"), commit_identity: measuredMetric("a".repeat(40), "git_commit"), @@ -77,6 +79,7 @@ describe("benchmark aggregate reports", () => { cost_usd: unavailableMetric("not_run"), human_review_minutes: unavailableMetric("not_reviewed"), human_review_notes: unavailableMetric("not_reviewed"), + reference_leak_check: unavailableMetric("not_run"), artifact_count: notRunMetric("case_material_not_available"), evidence_count: notRunMetric("case_material_not_available"), commit_identity: measuredMetric("a".repeat(40), "git_commit"), diff --git a/tests/benchmark-runner.test.ts b/tests/benchmark-runner.test.ts index 343d29e..a1909ee 100644 --- a/tests/benchmark-runner.test.ts +++ b/tests/benchmark-runner.test.ts @@ -43,7 +43,7 @@ async function createCase(): Promise<{ caseRoot: string; manifest: BenchmarkMani }, runtime: { agent_adapter_id: "agent-test-v1", one_shot_adapter_id: "one-shot-test-v1" }, execution: { kind: "local", network_access: "disabled" }, - budget: { max_wall_time_ms: 1_000, max_tokens: 1_000, max_cost_usd: null, max_human_review_minutes: null }, + budget: { max_wall_time_ms: 1_000, max_tokens: null, max_cost_usd: null, max_human_review_minutes: null }, allowed_task_types: ["statistical_analysis", "custom_experiment"], expected_task_types: ["statistical_analysis"], hard_checks: [{ id: "answer-present", description: "An answer is produced." }] From 100f439089144878415aebc84a9e999411a4c37f Mon Sep 17 00:00:00 2001 From: zhanghan <2774877132@qq.com> Date: Thu, 13 Aug 2026 03:44:47 +0800 Subject: [PATCH 3/4] test: build compiled benchmark before black-box checks --- tests/benchmark-compiled.test.ts | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tests/benchmark-compiled.test.ts b/tests/benchmark-compiled.test.ts index 2b918e3..11bb39e 100644 --- a/tests/benchmark-compiled.test.ts +++ b/tests/benchmark-compiled.test.ts @@ -1,5 +1,5 @@ import { execFile } from "node:child_process"; -import { mkdtemp, readFile, writeFile } from "node:fs/promises"; +import { access, mkdtemp, readFile, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; import { pathToFileURL } from "node:url"; @@ -17,8 +17,21 @@ async function runNode(args: string[], timeout = 15_000): Promise<{ stdout: stri }); } +async function ensureBenchmarkBuild(): Promise { + try { + await access(resolve(repositoryRoot, "dist/src/benchmark/run-synthetic.js")); + } catch { + await execFileAsync(resolve(repositoryRoot, "node_modules/.bin/tsc"), ["-p", "tsconfig.build.json"], { + cwd: repositoryRoot, + timeout: 60_000, + env: { ...process.env, NODE_NO_WARNINGS: "1" } + }); + } +} + describe("compiled benchmark entrypoints", () => { it("runs the built synthetic CLI from the repository root without exposing the host output path", async () => { + await ensureBenchmarkBuild(); const output = await mkdtemp(join(tmpdir(), "benchmark-compiled-output-")); const { stdout, stderr } = await runNode(["dist/src/benchmark/run-synthetic.js", "--output", output]); @@ -39,6 +52,7 @@ describe("compiled benchmark entrypoints", () => { }); it("keeps a referenced timeout alive in an otherwise handle-free child process until a hard-error result is emitted", async () => { + await ensureBenchmarkBuild(); const fixture = await mkdtemp(join(tmpdir(), "benchmark-timeout-child-")); const child = resolve(fixture, "timeout-child.mjs"); await writeFile(child, ` From c24623d2cbb3a0668538f92fe4893997ecfe6621 Mon Sep 17 00:00:00 2001 From: zhanghan <2774877132@qq.com> Date: Thu, 13 Aug 2026 17:23:10 +0800 Subject: [PATCH 4/4] fix: finalize benchmark scoring references --- benchmarks/README.md | 4 +- src/benchmark/report.ts | 32 +- src/benchmark/runner.ts | 257 +++++++++---- tests/benchmark-finalization.test.ts | 402 ++++++++++++++++++++ tests/benchmark-reference-lifecycle.test.ts | 18 +- tests/benchmark-report.test.ts | 46 ++- 6 files changed, 675 insertions(+), 84 deletions(-) create mode 100644 tests/benchmark-finalization.test.ts diff --git a/benchmarks/README.md b/benchmarks/README.md index 6875ddd..322fe12 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -12,7 +12,7 @@ npx tsx src/benchmark/run-synthetic.ts --output benchmarks/output/synthetic The command writes `benchmark-report.json` and `benchmark-report.md`. It runs the deterministic agent and one-shot adapters against the same frozen bytes for each case, uses a deterministic clock and identity, and is suitable for checking stable report fields. `benchmarks/output/` is intended as local generated output; do not commit reports that contain non-synthetic operational data. -The harness API is injected: production integrations implement `BenchmarkAdapter` from `src/benchmark/types.ts` instead of importing CLI or Orchestrator modules. A solve adapter receives package files, budgets, expected task types, and hard-check definitions. It never receives the manifest object, a reference path, reference bytes, or scoring notes. Reference material is opened separately for post-solve leakage checks, and a detected same-problem answer changes the result to `blocked` / `blocked_policy` with a digest-only event. Canonical package/reference paths are checked before solve so a reference cannot be nested under, cover, or alias the package; a declared user-supplied reference that is absent remains explicitly unavailable and cannot produce a completed result. +The harness API is injected: production integrations implement `BenchmarkAdapter` from `src/benchmark/types.ts` instead of importing CLI or Orchestrator modules. A solve adapter receives package files, budgets, expected task types, and hard-check definitions. It never receives the manifest object, a reference path, reference bytes, or scoring notes. Reference material is opened separately only after the adapter returns, throws, or reaches its wall-time budget; every result, including adapter and output-contract hard errors, binds the resulting scoring reference state into its evaluation digest and run id. A detected same-problem answer changes the result to `blocked` / `blocked_policy` with a digest-only event. Package/reference lexical overlap is rejected before solve, while scoring-time missing, non-regular, unreadable, changed, or digest-mismatched references become safe structured harness hard errors; a declared user-supplied reference that is absent remains explicitly unavailable and cannot produce a completed result. ## Adding three historical problems legally @@ -21,7 +21,7 @@ The v0.1-alpha acceptance target calls for three historical blind runs spanning 1. **Establish rights before copying.** Record the organizer, copyright holder, official source URL, license or written permission, and redistribution terms. A public download link is not by itself permission to republish a statement, dataset, judge material, or solution. 2. **Prefer user-supplied private material.** For ordinary copyrighted competitions, set `license.redistribution` to `user_supplied_only` or `metadata_only`, set reference availability to `user_supplied` or `unavailable`, and keep problem packages and answers under the ignored `benchmarks/private/` directory. Do not commit them. 3. **Create metadata without answer text.** Give the case a stable id, declare allowed and expected task types, budgets, runtime/execution policy, and hard checks. Store only relative paths and SHA-256 identities. Never place a reference answer, excerpt from a solution, credential, session, or absolute host path in a manifest or report. -4. **Freeze one case for both variants.** Verify that agent and one-shot runs receive the exact same `frozen_case_sha256` and `evaluation_contract_sha256`. The evaluation contract binds reference availability/digest, blind policy, every budget, allowed/expected task types, hard-check ids and descriptions, execution/network policy, and both adapter ids. Do not tune either variant after viewing the other variant's result. +4. **Freeze one case for both variants.** Verify that agent and one-shot runs receive the exact same `frozen_case_sha256` and `evaluation_contract_sha256`. The evaluation contract binds the scoring-time reference status/reason plus expected and actual digest, blind policy, every budget, allowed/expected task types, hard-check ids and descriptions, execution/network policy, and both adapter ids. Aggregation accepts at most one result per case/variant and requires globally unique run ids. Do not tune either variant after viewing the other variant's result. 5. **Keep scoring references out of solve.** Reference solutions and judge notes are scoring-only. The blind firewall must block a detected same-problem answer and retain only a fingerprint, never the matched text. 6. **Report missing measurements honestly.** Unknown token count, cost, or human review is `unavailable` with a `null` value. An unavailable field is not zero, and a declared limit cannot be treated as satisfied when its measurement is unavailable. Human review observations use only a numeric duration plus a fixed safe classification (`no_revision`, `minor_revision`, `major_revision`, or `rejected`), never free-form text. Material not obtained or a run not attempted is `not_run`; a policy stop is `blocked`; failures never count as completion. 7. **Review before publication.** Have a maintainer confirm the license record and inspect the staged diff for protected text. Publish historical results only when the underlying case can be lawfully used and the environment/commit identity is reproducible. diff --git a/src/benchmark/report.ts b/src/benchmark/report.ts index f6555c5..e5714b4 100644 --- a/src/benchmark/report.ts +++ b/src/benchmark/report.ts @@ -42,16 +42,29 @@ function stableResultOrder(left: BenchmarkResult, right: BenchmarkResult): numbe return left.case_id.localeCompare(right.case_id) || left.variant.localeCompare(right.variant) || left.run_id.localeCompare(right.run_id); } -function assertComparableVariants(results: readonly BenchmarkResult[]): void { - const groups = new Map(); +function assertUniqueResults(results: readonly BenchmarkResult[]): Map> { + const runIds = new Set(); + const groups = new Map>(); for (const result of results) { - const group = groups.get(result.case_id) ?? []; - group.push(result); - groups.set(result.case_id, group); + if (runIds.has(result.run_id)) { + throw new Error(`benchmark run id must be globally unique: ${result.run_id}`); + } + runIds.add(result.run_id); + + const variants = groups.get(result.case_id) ?? new Map(); + if (variants.has(result.variant)) { + throw new Error(`benchmark case ${result.case_id} has a duplicate ${result.variant} variant`); + } + variants.set(result.variant, result); + groups.set(result.case_id, variants); } - for (const [caseId, group] of groups) { - const agent = group.find((result) => result.variant === "agent"); - const oneShot = group.find((result) => result.variant === "one_shot"); + return groups; +} + +function assertComparableVariants(groups: ReadonlyMap>): void { + for (const [caseId, variants] of groups) { + const agent = variants.get("agent"); + const oneShot = variants.get("one_shot"); if (agent && oneShot && agent.frozen_case_sha256 !== oneShot.frozen_case_sha256) { throw new Error(`benchmark variants for ${caseId} do not share a frozen case`); } @@ -66,7 +79,8 @@ export function aggregateBenchmarkResults(input: readonly BenchmarkResult[], sui throw new Error("benchmark suite id must be a bounded opaque identifier"); } const results = input.map((entry) => validateBenchmarkResult(structuredClone(entry))).sort(stableResultOrder); - assertComparableVariants(results); + const groups = assertUniqueResults(results); + assertComparableVariants(groups); const metrics = Object.fromEntries(metricNames.map((name) => [name, aggregateMetric(results.map((result) => result.metrics[name]))])) as Record; return { schema_version: BENCHMARK_SCHEMA_VERSION, diff --git a/src/benchmark/runner.ts b/src/benchmark/runner.ts index e094274..d527c2e 100644 --- a/src/benchmark/runner.ts +++ b/src/benchmark/runner.ts @@ -1,4 +1,6 @@ import { createHash } from "node:crypto"; +import { constants } from "node:fs"; +import type { Stats } from "node:fs"; import { lstat, open, readdir, readFile, realpath, stat } from "node:fs/promises"; import { isAbsolute, relative, resolve, sep } from "node:path"; import type { TaskType } from "../contracts/types.js"; @@ -27,26 +29,36 @@ const TEXT_FILE_LIMIT_BYTES = 1_000_000; const OPAQUE_IDENTIFIER = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/u; const REVIEW_NOTES = new Set(["no_revision", "minor_revision", "major_revision", "rejected"]); -type ReferenceStatus = "available" | "unavailable"; +type ReferenceStatus = "available" | "unavailable" | "missing" | "integrity_failed" | "read_failed"; +type ReferenceFailureReason = + | "reference_not_declared" + | "included_reference_missing" + | "user_supplied_reference_missing" + | "reference_not_regular_file" + | "reference_read_failed" + | "reference_digest_mismatch" + | "reference_outside_case" + | "reference_package_overlap" + | "reference_changed_during_scoring"; interface FrozenPackage { files: BenchmarkPackageFile[]; digest: string; canonical_root: string; + package_file_ids: Set; } interface PreparedReference { declared_path: string | null; - canonical_path: string | null; - status: ReferenceStatus; - reason: string | null; + canonical_case_root: string; + package_root: string; + package_file_ids: ReadonlySet; } interface LoadedReference { - canonical_path: string | null; status: ReferenceStatus; digest: string | null; - reason: string | null; + reason: ReferenceFailureReason | null; text: string | null; } @@ -55,7 +67,9 @@ interface EvaluationContract { reference: { availability: BenchmarkManifest["reference_policy"]["availability"]; status: ReferenceStatus; - sha256: string | null; + reason: ReferenceFailureReason | null; + expected_sha256: string | null; + actual_sha256: string | null; }; blind_policy: BenchmarkManifest["blind_policy"]; budget: BenchmarkManifest["budget"]; @@ -146,6 +160,19 @@ function pathsOverlap(left: string, right: string): boolean { return isWithin(left, right) || isWithin(right, left); } +function fileIdentity(info: Stats): string | null { + if (info.dev === 0 && info.ino === 0) return null; + return `${String(info.dev)}:${String(info.ino)}`; +} + +function sameFileSnapshot(left: Stats, right: Stats): boolean { + return left.dev === right.dev + && left.ino === right.ino + && left.size === right.size + && left.mtimeMs === right.mtimeMs + && left.ctimeMs === right.ctimeMs; +} + async function listFiles(root: string, current = root): Promise { const entries = await readdir(current, { withFileTypes: true }); const files: string[] = []; @@ -192,8 +219,13 @@ async function freezePackage(canonicalCaseRoot: string, packagePath: string): Pr } const files: BenchmarkPackageFile[] = []; const digestParts: string[] = []; + const packageFileIds = new Set(); for (const relativePath of relativePaths) { - const bytes = await readFile(resolve(canonicalPackageRoot, relativePath)); + const absolutePath = resolve(canonicalPackageRoot, relativePath); + const info = await stat(absolutePath); + const identity = fileIdentity(info); + if (identity !== null) packageFileIds.add(identity); + const bytes = await readFile(absolutePath); if (bytes.byteLength > TEXT_FILE_LIMIT_BYTES) { throw new Error(`benchmark package file exceeds ${TEXT_FILE_LIMIT_BYTES} bytes`); } @@ -204,7 +236,7 @@ async function freezePackage(canonicalCaseRoot: string, packagePath: string): Pr files.push({ relative_path: relativePath, media_type: mediaType(relativePath), sha256: fileDigest, content: bytes.toString("utf8") }); digestParts.push(`${relativePath}\0${fileDigest}\n`); } - return { files, digest: sha256(digestParts.join("")), canonical_root: canonicalPackageRoot }; + return { files, digest: sha256(digestParts.join("")), canonical_root: canonicalPackageRoot, package_file_ids: packageFileIds }; } async function optionalRealpath(path: string): Promise { @@ -216,72 +248,147 @@ async function optionalRealpath(path: string): Promise { } } -async function prepareReference(canonicalCaseRoot: string, packageRoot: string, manifest: BenchmarkManifest): Promise { - const policy = manifest.reference_policy; - const relativePath = policy.relative_path; +async function prepareReference(canonicalCaseRoot: string, frozen: FrozenPackage, manifest: BenchmarkManifest): Promise { + const packageRoot = frozen.canonical_root; + const relativePath = manifest.reference_policy.relative_path; if (relativePath === null) { - return { declared_path: null, canonical_path: null, status: "unavailable", reason: "reference_not_declared" }; + return { declared_path: null, canonical_case_root: canonicalCaseRoot, package_root: packageRoot, package_file_ids: frozen.package_file_ids }; } assertSafeRelativePath(relativePath); const declaredPath = resolve(canonicalCaseRoot, relativePath); if (pathsOverlap(packageRoot, declaredPath)) { throw new Error("benchmark reference must remain outside the package directory"); } - const canonicalReference = await optionalRealpath(declaredPath); - if (canonicalReference === null) { - if (policy.availability === "included") { - throw new Error("included benchmark reference is unavailable"); + + // Resolve only metadata before solve. Bytes and final reference state are loaded after the adapter window. + let canonicalReference: string | null = null; + try { + canonicalReference = await optionalRealpath(declaredPath); + } catch { + // Permission and transient path failures are scored after adapter execution. + } + if (canonicalReference !== null) { + if (!isWithin(canonicalCaseRoot, canonicalReference)) { + throw new Error("benchmark reference resolves outside its case root"); + } + if (pathsOverlap(packageRoot, canonicalReference)) { + throw new Error("benchmark reference must remain outside the package directory"); } - return { declared_path: declaredPath, canonical_path: null, status: "unavailable", reason: "user_supplied_reference_missing" }; } - if (!isWithin(canonicalCaseRoot, canonicalReference)) { - throw new Error("benchmark reference resolves outside its case root"); + + let referenceInfo: Stats | null = null; + try { + referenceInfo = await stat(declaredPath); + } catch { + // Missing, unreadable, and transient references are finalized after solve. } - if (pathsOverlap(packageRoot, canonicalReference)) { - throw new Error("benchmark reference must remain outside the package directory"); + if (referenceInfo?.isFile()) { + const identity = fileIdentity(referenceInfo); + if (identity !== null && frozen.package_file_ids.has(identity)) { + throw new Error("benchmark reference must remain outside the package directory"); + } } - const info = await lstat(canonicalReference); - if (!info.isFile()) { - throw new Error("benchmark reference path must identify a file"); + + return { declared_path: declaredPath, canonical_case_root: canonicalCaseRoot, package_root: packageRoot, package_file_ids: frozen.package_file_ids }; +} + +function missingReference(manifest: BenchmarkManifest): LoadedReference { + if (manifest.reference_policy.availability === "user_supplied") { + return { status: "unavailable", digest: null, reason: "user_supplied_reference_missing", text: null }; } - return { declared_path: declaredPath, canonical_path: canonicalReference, status: "available", reason: null }; + return { status: "missing", digest: null, reason: "included_reference_missing", text: null }; +} + +function referenceFailure(reason: ReferenceFailureReason, digest: string | null = null): LoadedReference { + return { status: "integrity_failed", digest, reason, text: null }; +} + +function referenceReadFailure(digest: string | null = null): LoadedReference { + return { status: "read_failed", digest, reason: "reference_read_failed", text: null }; +} + +function classifyReferenceError(error: unknown, manifest: BenchmarkManifest, digest: string | null = null): LoadedReference { + const code = (error as NodeJS.ErrnoException).code; + if (code === "ENOENT") { + return digest === null ? missingReference(manifest) : referenceFailure("reference_changed_during_scoring", digest); + } + if (code === "ELOOP" || code === "EISDIR" || code === "ENOTDIR") return referenceFailure("reference_not_regular_file", digest); + return referenceReadFailure(digest); } async function loadReference(reference: PreparedReference, manifest: BenchmarkManifest): Promise { - if (reference.status === "unavailable" || reference.canonical_path === null) { - return { canonical_path: null, status: "unavailable", digest: null, reason: reference.reason, text: null }; + if (reference.declared_path === null) { + return { status: "unavailable", digest: null, reason: "reference_not_declared", text: null }; } + + let handle: Awaited> | undefined; + let actualDigest: string | null = null; try { - const handle = await open(reference.canonical_path, "r"); + // O_NONBLOCK prevents a scoring path replaced with a FIFO from hanging finalization. + handle = await open(reference.declared_path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + const info = await handle.stat(); + if (!info.isFile()) return referenceFailure("reference_not_regular_file"); + const identity = fileIdentity(info); + const canonicalHandlePath = await realpath(`/proc/self/fd/${handle.fd}`); + if (!isWithin(reference.canonical_case_root, canonicalHandlePath)) { + return referenceFailure("reference_outside_case"); + } + if (pathsOverlap(reference.package_root, canonicalHandlePath)) { + return referenceFailure("reference_package_overlap"); + } + + const bytes = await handle.readFile(); + actualDigest = sha256(bytes); + const finalHandleInfo = await handle.stat(); + if (!sameFileSnapshot(info, finalHandleInfo)) { + return referenceFailure("reference_changed_during_scoring", actualDigest); + } + if (identity !== null && reference.package_file_ids.has(identity)) { + return referenceFailure("reference_package_overlap", actualDigest); + } + if (manifest.reference_policy.sha256 !== null && actualDigest !== manifest.reference_policy.sha256) { + return referenceFailure("reference_digest_mismatch", actualDigest); + } + + let declaredInfo: Stats; try { - const canonicalHandlePath = await realpath(`/proc/self/fd/${handle.fd}`); - if (canonicalHandlePath !== reference.canonical_path) { - throw new Error("benchmark reference changed before scoring"); - } - const bytes = await handle.readFile(); - const actualDigest = sha256(bytes); - if (manifest.reference_policy.sha256 !== null && actualDigest !== manifest.reference_policy.sha256) { - throw new Error("benchmark reference digest does not match the manifest"); + declaredInfo = await lstat(reference.declared_path); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return referenceFailure("reference_changed_during_scoring", actualDigest); } - return { canonical_path: reference.canonical_path, status: "available", digest: actualDigest, reason: null, text: bytes.toString("utf8") }; - } finally { - await handle.close(); + return referenceReadFailure(actualDigest); + } + if (!declaredInfo.isFile() || !sameFileSnapshot(info, declaredInfo)) { + return referenceFailure("reference_changed_during_scoring", actualDigest); } + const canonicalDeclaredPath = await optionalRealpath(reference.declared_path); + if (canonicalDeclaredPath === null || canonicalDeclaredPath !== canonicalHandlePath) { + return referenceFailure("reference_changed_during_scoring", actualDigest); + } + return { status: "available", digest: actualDigest, reason: null, text: bytes.toString("utf8") }; } catch (error) { - if (manifest.reference_policy.availability === "user_supplied" && (error as NodeJS.ErrnoException).code === "ENOENT") { - return { canonical_path: null, status: "unavailable", digest: null, reason: "user_supplied_reference_missing", text: null }; + return classifyReferenceError(error, manifest, actualDigest); + } finally { + if (handle !== undefined) { + try { + await handle.close(); + } catch { + // A close failure cannot expose path or reference content; the read result is already classified. + } } - throw error; } } -function evaluationContract(manifest: BenchmarkManifest, frozenDigest: string, reference: PreparedReference | LoadedReference): EvaluationContract { +function evaluationContract(manifest: BenchmarkManifest, frozenDigest: string, reference: LoadedReference): EvaluationContract { return { frozen_case_sha256: frozenDigest, reference: { availability: manifest.reference_policy.availability, status: reference.status, - sha256: "digest" in reference ? reference.digest : null + reason: reference.reason, + expected_sha256: manifest.reference_policy.sha256, + actual_sha256: reference.digest }, blind_policy: structuredClone(manifest.blind_policy), budget: structuredClone(manifest.budget), @@ -381,6 +488,12 @@ function safeAdapterError(output: BenchmarkAdapterOutput): { class: string; mess return { class: className, message: `failure:${fingerprint.slice(0, 12)}`, fingerprint }; } +function referenceScoringError(reference: LoadedReference): { class: string; message: string; fingerprint: string } | null { + if (reference.status === "available" || reference.status === "unavailable") return null; + const fingerprint = sha256(`ReferenceScoringError\0${reference.status}\0${reference.reason ?? "unknown"}\0${reference.digest ?? "null"}`); + return { class: "ReferenceScoringError", message: `failure:${fingerprint.slice(0, 12)}`, fingerprint }; +} + function runId(caseId: string, variant: string, frozenDigest: string, evaluationDigest: string): string { return `${caseId}-${variant.replaceAll("_", "-")}-${frozenDigest.slice(0, 12)}-${evaluationDigest.slice(0, 12)}`; } @@ -421,11 +534,12 @@ function errorResult(parameters: { adapter: BenchmarkAdapter; identity: BenchmarkIdentity; frozenDigest: string; - evaluationDigest: string; + reference: LoadedReference; duration: number; error: { class: string; message: string; fingerprint: string }; }): BenchmarkResult { - const { manifest, adapter, identity, frozenDigest, evaluationDigest, duration, error } = parameters; + const { manifest, adapter, identity, frozenDigest, reference, duration, error } = parameters; + const evaluationDigest = sha256(stableSerialize(evaluationContract(manifest, frozenDigest, reference))); return validateBenchmarkResult({ schema_version: "1.0.0", case_id: manifest.case_id, @@ -450,7 +564,7 @@ function errorResult(parameters: { cost_usd: unavailableMetric("adapter_failed_before_usage_report"), human_review_minutes: unavailableMetric("review_observation_unavailable"), human_review_notes: unavailableMetric("review_observation_unavailable"), - reference_leak_check: unavailableMetric("adapter_output_unavailable"), + reference_leak_check: unavailableMetric(reference.status === "available" ? "adapter_output_unavailable" : reference.reason ?? "reference_unavailable"), artifact_count: measuredMetric(0, "adapter_inventory"), evidence_count: measuredMetric(0, "adapter_inventory"), commit_identity: measuredMetric(identity.commit, "git_commit"), @@ -471,8 +585,7 @@ export async function runBenchmarkCase(options: RunBenchmarkCaseOptions): Promis const canonicalCaseRoot = await realpath(options.case_root); const frozen = await freezePackage(canonicalCaseRoot, manifest.package_path); - const preparedReference = await prepareReference(canonicalCaseRoot, frozen.canonical_root, manifest); - let evaluationDigest = sha256(stableSerialize(evaluationContract(manifest, frozen.digest, preparedReference))); + const preparedReference = await prepareReference(canonicalCaseRoot, frozen, manifest); const context: BenchmarkSolveContext = Object.freeze({ case_id: manifest.case_id, variant: options.adapter.variant, @@ -485,7 +598,8 @@ export async function runBenchmarkCase(options: RunBenchmarkCaseOptions): Promis const clock = options.clock ?? systemClock; let duration = 0; - let output: BenchmarkAdapterOutput; + let output: BenchmarkAdapterOutput | undefined; + let adapterError: { class: string; message: string; fingerprint: string } | null = null; try { const measured = await clock.measure(async () => { let timer: NodeJS.Timeout | undefined; @@ -504,14 +618,21 @@ export async function runBenchmarkCase(options: RunBenchmarkCaseOptions): Promis duration = typeof (error as { benchmark_duration_ms?: unknown })?.benchmark_duration_ms === "number" ? (error as { benchmark_duration_ms: number }).benchmark_duration_ms : manifest.budget.max_wall_time_ms; + adapterError = safeError(error); + } + + const reference = await loadReference(preparedReference, manifest); + const evaluationDigest = sha256(stableSerialize(evaluationContract(manifest, frozen.digest, reference))); + const scoringError = referenceScoringError(reference); + if (scoringError !== null || adapterError !== null) { return errorResult({ manifest, adapter: options.adapter, identity: options.identity, frozenDigest: frozen.digest, - evaluationDigest, + reference, duration, - error: safeError(error) + error: scoringError ?? adapterError! }); } @@ -522,21 +643,19 @@ export async function runBenchmarkCase(options: RunBenchmarkCaseOptions): Promis adapter: options.adapter, identity: options.identity, frozenDigest: frozen.digest, - evaluationDigest, + reference, duration, error: safeError(contractFailure, "AdapterContractError") }); } - - const reference = await loadReference(preparedReference, manifest); - evaluationDigest = sha256(stableSerialize(evaluationContract(manifest, frozen.digest, reference))); - const observedTasks = normalizeObservedTasks(output, manifest); - const checks = hardChecks(output, manifest); + const validatedOutput = output as BenchmarkAdapterOutput; + const observedTasks = normalizeObservedTasks(validatedOutput, manifest); + const checks = hardChecks(validatedOutput, manifest); const reviewValues = reviewMetrics(review); const referenceMetric: BenchmarkMetric = reference.status === "available" ? measuredMetric(true, "harness_leak_check") : unavailableMetric(reference.reason ?? "reference_unavailable"); - const leakedFingerprint = leakedReference(output.output_text, reference.text, manifest.blind_policy.minimum_reference_match_characters); + const leakedFingerprint = leakedReference(validatedOutput.output_text, reference.text, manifest.blind_policy.minimum_reference_match_characters); if (leakedFingerprint !== null) { return validateBenchmarkResult({ @@ -559,8 +678,8 @@ export async function runBenchmarkCase(options: RunBenchmarkCaseOptions): Promis wall_time_ms: measuredMetric(duration, "harness_clock"), task_type_coverage: measuredMetric(coverage(manifest.expected_task_types, observedTasks), "harness_scoring"), custom_experiment_present: measuredMetric(observedTasks.includes("custom_experiment"), "harness_scoring"), - token_count: usageMetric(output.usage.token_count, "adapter_usage"), - cost_usd: usageMetric(output.usage.cost_usd, "adapter_usage"), + token_count: usageMetric(validatedOutput.usage.token_count, "adapter_usage"), + cost_usd: usageMetric(validatedOutput.usage.cost_usd, "adapter_usage"), human_review_minutes: reviewValues.minutes, human_review_notes: reviewValues.notes, reference_leak_check: measuredMetric(false, "harness_leak_check"), @@ -572,12 +691,12 @@ export async function runBenchmarkCase(options: RunBenchmarkCaseOptions): Promis }); } - const adapterFailed = output.status === "failed"; + const adapterFailed = validatedOutput.status === "failed"; const checksFailed = checks.some((check) => check.status !== "passed"); const scoringAvailable = reference.status === "available" || manifest.reference_policy.availability === "unavailable"; - const complete = !adapterFailed && !checksFailed && scoringAvailable && budgetProven(manifest, output, duration, review); + const complete = !adapterFailed && !checksFailed && scoringAvailable && budgetProven(manifest, validatedOutput, duration, review); const outcome = complete ? "completed" : adapterFailed ? "hard_error" : "incomplete"; - const outputError = safeAdapterError(output); + const outputError = safeAdapterError(validatedOutput); return validateBenchmarkResult({ schema_version: "1.0.0", case_id: manifest.case_id, @@ -598,13 +717,13 @@ export async function runBenchmarkCase(options: RunBenchmarkCaseOptions): Promis wall_time_ms: measuredMetric(duration, "harness_clock"), task_type_coverage: measuredMetric(coverage(manifest.expected_task_types, observedTasks), "harness_scoring"), custom_experiment_present: measuredMetric(observedTasks.includes("custom_experiment"), "harness_scoring"), - token_count: usageMetric(output.usage.token_count, "adapter_usage"), - cost_usd: usageMetric(output.usage.cost_usd, "adapter_usage"), + token_count: usageMetric(validatedOutput.usage.token_count, "adapter_usage"), + cost_usd: usageMetric(validatedOutput.usage.cost_usd, "adapter_usage"), human_review_minutes: reviewValues.minutes, human_review_notes: reviewValues.notes, reference_leak_check: referenceMetric, - artifact_count: measuredMetric(output.artifacts.length, "adapter_inventory"), - evidence_count: measuredMetric(output.evidence.length, "adapter_inventory"), + artifact_count: measuredMetric(validatedOutput.artifacts.length, "adapter_inventory"), + evidence_count: measuredMetric(validatedOutput.evidence.length, "adapter_inventory"), commit_identity: measuredMetric(options.identity.commit, "git_commit"), environment_identity: measuredMetric(options.identity.environment, "runtime_environment") } diff --git a/tests/benchmark-finalization.test.ts b/tests/benchmark-finalization.test.ts new file mode 100644 index 0000000..451a5eb --- /dev/null +++ b/tests/benchmark-finalization.test.ts @@ -0,0 +1,402 @@ +import { createHash } from "node:crypto"; +import { chmod, link, mkdtemp, mkdir, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { describe, expect, it } from "vitest"; +import { aggregateBenchmarkResults } from "../src/benchmark/report.js"; +import { runBenchmarkCase } from "../src/benchmark/runner.js"; +import type { + BenchmarkAdapter, + BenchmarkAdapterOutput, + BenchmarkManifest, + BenchmarkResult, + BenchmarkVariant +} from "../src/benchmark/types.js"; + +const ALPHA_REFERENCE = "REFERENCE-FINALIZATION-ALPHA-314159265358979323846"; +const BETA_REFERENCE = "REFERENCE-FINALIZATION-BETA-271828182845904523536"; + +type FailureMode = "throw" | "timeout" | "invalid_output"; +type ReferenceMutation = "delete" | "not_regular" | "digest_mismatch"; + +function sha256(value: string): string { + return createHash("sha256").update(value).digest("hex"); +} + +async function createCase(): Promise<{ caseRoot: string; manifest: BenchmarkManifest; referencePath: string }> { + const caseRoot = await mkdtemp(join(tmpdir(), "benchmark-finalization-")); + const referencePath = resolve(caseRoot, "reference/reference.txt"); + await mkdir(resolve(caseRoot, "package")); + await mkdir(resolve(caseRoot, "reference")); + await writeFile(resolve(caseRoot, "package/problem.md"), "Solve the synthetic finalization case.\n", "utf8"); + await writeFile(referencePath, `${ALPHA_REFERENCE}\n`, "utf8"); + return { + caseRoot, + referencePath, + manifest: { + schema_version: "1.0.0", + case_id: "synthetic-finalization", + package_path: "package", + license: { + name: "CC0 1.0 Universal", + spdx_id: "CC0-1.0", + copyright_holder: "modeling-agent contributors", + source_url: null, + redistribution: "permitted", + notice_path: null + }, + blind_policy: { + mode: "blind", + solve_input: "package_only", + same_problem_answers: "block", + minimum_reference_match_characters: 24 + }, + reference_policy: { + access: "scoring_only", + availability: "included", + relative_path: "reference/reference.txt", + sha256: null + }, + runtime: { + agent_adapter_id: "finalization-agent-v1", + one_shot_adapter_id: "finalization-one-shot-v1" + }, + execution: { kind: "local", network_access: "disabled" }, + budget: { + max_wall_time_ms: 10, + max_tokens: null, + max_cost_usd: null, + max_human_review_minutes: null + }, + allowed_task_types: ["statistical_analysis"], + expected_task_types: ["statistical_analysis"], + hard_checks: [{ id: "answer-present", description: "A synthetic answer is present." }] + } + }; +} + +function successfulOutput(manifest: BenchmarkManifest): BenchmarkAdapterOutput { + return { + status: "success", + observed_task_types: [...manifest.expected_task_types], + hard_checks: manifest.hard_checks.map((check) => ({ id: check.id, passed: true })), + artifacts: ["answer.md"], + evidence: ["answer-evidence"], + usage: { token_count: null, cost_usd: null }, + output_text: "A synthetic answer was produced." + }; +} + +function deterministicClock() { + return { + async measure(operation: () => Promise): Promise<{ value: T; duration_ms: number }> { + try { + return { value: await operation(), duration_ms: 1 }; + } catch (error) { + if (error && typeof error === "object") Object.assign(error, { benchmark_duration_ms: 1 }); + throw error; + } + } + }; +} + +function adapterFor(manifest: BenchmarkManifest, variant: BenchmarkVariant, run: BenchmarkAdapter["run"]): BenchmarkAdapter { + return { + id: variant === "agent" ? manifest.runtime.agent_adapter_id : manifest.runtime.one_shot_adapter_id, + variant, + run + }; +} + +async function runFailure( + caseRoot: string, + manifest: BenchmarkManifest, + mode: FailureMode, + variant: BenchmarkVariant = "agent" +): Promise { + const run: BenchmarkAdapter["run"] = mode === "throw" + ? async () => { throw new Error("adapter secret=TOP-SECRET /home/private/reference.txt"); } + : mode === "timeout" + ? async () => new Promise(() => undefined) + : async () => ({ + ...successfulOutput(manifest), + error: { class: "InvalidSecretError", message: "token=TOP-SECRET /home/private/reference.txt" } + }); + return runBenchmarkCase({ + case_root: caseRoot, + manifest, + adapter: adapterFor(manifest, variant, run), + identity: { commit: "commit-finalization-v1", environment: "node-test-v1" }, + ...(mode === "timeout" ? {} : { clock: deterministicClock() }) + }); +} + +describe("benchmark scoring finalization", () => { + it("binds a throwing agent and successful one-shot baseline to the same final reference contract", async () => { + const { caseRoot, manifest } = await createCase(); + const agentResult = await runFailure(caseRoot, manifest, "throw", "agent"); + const baselineResult = await runBenchmarkCase({ + case_root: caseRoot, + manifest, + adapter: adapterFor(manifest, "one_shot", async () => successfulOutput(manifest)), + identity: { commit: "commit-finalization-v1", environment: "node-test-v1" }, + clock: deterministicClock() + }); + + expect(agentResult.outcome).toBe("hard_error"); + expect(baselineResult.outcome).toBe("completed"); + expect(agentResult.evaluation_contract_sha256).toBe(baselineResult.evaluation_contract_sha256); + + const report = aggregateBenchmarkResults([agentResult, baselineResult]); + expect(report.summary).toMatchObject({ total_runs: 2, completed_runs: 1, hard_error_runs: 1 }); + }); + + it.each(["throw", "timeout", "invalid_output"] as const)( + "changes the %s failure evaluation digest and run id when scoring reference bytes change", + async (mode) => { + const { caseRoot, manifest, referencePath } = await createCase(); + const alpha = await runFailure(caseRoot, manifest, mode); + await writeFile(referencePath, `${BETA_REFERENCE}\n`, "utf8"); + const beta = await runFailure(caseRoot, manifest, mode); + + expect(alpha.outcome).toBe("hard_error"); + expect(beta.outcome).toBe("hard_error"); + expect(alpha.evaluation_contract_sha256).not.toBe(beta.evaluation_contract_sha256); + expect(alpha.run_id).not.toBe(beta.run_id); + } + ); + + it.each(["success", "throw", "timeout", "invalid_output"] as const)( + "loads the %s reference state only after its adapter execution window", + async (mode) => { + const { caseRoot, manifest, referencePath } = await createCase(); + const alpha = mode === "success" + ? await runBenchmarkCase({ + case_root: caseRoot, + manifest, + adapter: adapterFor(manifest, "agent", async () => successfulOutput(manifest)), + identity: { commit: "commit-finalization-v1", environment: "node-test-v1" }, + clock: deterministicClock() + }) + : await runFailure(caseRoot, manifest, mode); + const run: BenchmarkAdapter["run"] = async () => { + await writeFile(referencePath, `${BETA_REFERENCE}\n`, "utf8"); + if (mode === "throw") throw new Error("adapter secret=TOP-SECRET"); + if (mode === "timeout") return new Promise(() => undefined); + if (mode === "invalid_output") { + return { ...successfulOutput(manifest), error: { class: "InvalidSecretError", message: "token=TOP-SECRET" } }; + } + return successfulOutput(manifest); + }; + const beta = await runBenchmarkCase({ + case_root: caseRoot, + manifest, + adapter: adapterFor(manifest, "agent", run), + identity: { commit: "commit-finalization-v1", environment: "node-test-v1" }, + ...(mode === "timeout" ? {} : { clock: deterministicClock() }) + }); + + expect(beta.outcome).toBe(mode === "success" ? "completed" : "hard_error"); + expect(beta.evaluation_contract_sha256).not.toBe(alpha.evaluation_contract_sha256); + expect(beta.run_id).not.toBe(alpha.run_id); + const serialized = JSON.stringify(beta); + expect(serialized).not.toContain(ALPHA_REFERENCE); + expect(serialized).not.toContain(BETA_REFERENCE); + expect(serialized).not.toContain("TOP-SECRET"); + } + ); + + it.each(["delete", "not_regular", "digest_mismatch"] as const)( + "records a safe structured harness failure when an included reference suffers %s after adapter start", + async (mutation: ReferenceMutation) => { + const { caseRoot, manifest, referencePath } = await createCase(); + if (mutation === "digest_mismatch") { + manifest.reference_policy.sha256 = sha256(`${ALPHA_REFERENCE}\n`); + } + const availableFailure = await runFailure(caseRoot, manifest, "throw"); + const mutatedFailure = await runBenchmarkCase({ + case_root: caseRoot, + manifest, + adapter: adapterFor(manifest, "agent", async () => { + if (mutation === "delete") { + await rm(referencePath); + } else if (mutation === "not_regular") { + await rm(referencePath); + await mkdir(referencePath); + } else { + await writeFile(referencePath, `${BETA_REFERENCE}\n`, "utf8"); + } + throw new Error(`adapter secret=TOP-SECRET path=${caseRoot}`); + }), + identity: { commit: "commit-finalization-v1", environment: "node-test-v1" }, + clock: deterministicClock() + }); + + expect(mutatedFailure.outcome).toBe("hard_error"); + expect(mutatedFailure.error?.class).toBe("ReferenceScoringError"); + expect(mutatedFailure.evaluation_contract_sha256).not.toBe(availableFailure.evaluation_contract_sha256); + expect(mutatedFailure.run_id).not.toBe(availableFailure.run_id); + expect(mutatedFailure.metrics.reference_leak_check).toMatchObject({ status: "unavailable", value: null }); + expect(mutatedFailure.error?.class).toMatch(/^[A-Za-z0-9][A-Za-z0-9._-]*$/); + expect(mutatedFailure.error?.message).toMatch(/^failure:[a-f0-9]{12}$/); + const serialized = JSON.stringify(mutatedFailure); + expect(serialized).not.toContain(caseRoot); + expect(serialized).not.toContain("TOP-SECRET"); + expect(serialized).not.toContain(ALPHA_REFERENCE); + expect(serialized).not.toContain(BETA_REFERENCE); + } + ); + + it("loads a user-supplied reference that appears only after the adapter starts", async () => { + const { caseRoot, manifest, referencePath } = await createCase(); + await rm(referencePath); + manifest.reference_policy.availability = "user_supplied"; + + const result = await runBenchmarkCase({ + case_root: caseRoot, + manifest, + adapter: adapterFor(manifest, "agent", async (context) => { + expect(JSON.stringify(context)).not.toContain("reference/reference.txt"); + await writeFile(referencePath, `${ALPHA_REFERENCE}\n`, "utf8"); + return successfulOutput(manifest); + }), + identity: { commit: "commit-finalization-v1", environment: "node-test-v1" }, + clock: deterministicClock() + }); + + expect(result.outcome).toBe("completed"); + expect(result.metrics.reference_leak_check).toMatchObject({ status: "measured", value: true }); + }); + + it("binds a user-supplied reference that disappears after the adapter starts to its unavailable final state", async () => { + const { caseRoot, manifest, referencePath } = await createCase(); + manifest.reference_policy.availability = "user_supplied"; + const available = await runBenchmarkCase({ + case_root: caseRoot, + manifest, + adapter: adapterFor(manifest, "agent", async () => successfulOutput(manifest)), + identity: { commit: "commit-finalization-v1", environment: "node-test-v1" }, + clock: deterministicClock() + }); + const missing = await runBenchmarkCase({ + case_root: caseRoot, + manifest, + adapter: adapterFor(manifest, "agent", async () => { + await rm(referencePath); + return successfulOutput(manifest); + }), + identity: { commit: "commit-finalization-v1", environment: "node-test-v1" }, + clock: deterministicClock() + }); + + expect(available.outcome).toBe("completed"); + expect(missing.outcome).toBe("incomplete"); + expect(missing.metrics.completion).toMatchObject({ status: "measured", value: false }); + expect(missing.metrics.reference_leak_check).toMatchObject({ + status: "unavailable", + value: null, + reason: "user_supplied_reference_missing" + }); + expect(missing.evaluation_contract_sha256).not.toBe(available.evaluation_contract_sha256); + expect(missing.run_id).not.toBe(available.run_id); + }); + + it("records a fixed read-failed scoring contract without exposing a protected included reference", async () => { + const { caseRoot, manifest, referencePath } = await createCase(); + const available = await runFailure(caseRoot, manifest, "throw"); + try { + const unreadable = await runBenchmarkCase({ + case_root: caseRoot, + manifest, + adapter: adapterFor(manifest, "agent", async () => { + await chmod(referencePath, 0); + throw new Error("adapter secret=TOP-SECRET"); + }), + identity: { commit: "commit-finalization-v1", environment: "node-test-v1" }, + clock: deterministicClock() + }); + + expect(unreadable.outcome).toBe("hard_error"); + expect(unreadable.error?.class).toBe("ReferenceScoringError"); + expect(unreadable.metrics.reference_leak_check).toMatchObject({ + status: "unavailable", + value: null, + reason: "reference_read_failed" + }); + expect(unreadable.evaluation_contract_sha256).not.toBe(available.evaluation_contract_sha256); + expect(unreadable.run_id).not.toBe(available.run_id); + const serialized = JSON.stringify(unreadable); + expect(serialized).not.toContain(caseRoot); + expect(serialized).not.toContain(ALPHA_REFERENCE); + expect(serialized).not.toContain("TOP-SECRET"); + } finally { + await chmod(referencePath, 0o600); + } + }); + + it("reads an included reference only after a wall-time timeout without waiting for the adapter to settle", async () => { + const { caseRoot, manifest, referencePath } = await createCase(); + const availableTimeout = await runFailure(caseRoot, manifest, "timeout"); + await writeFile(referencePath, `${ALPHA_REFERENCE}\n`, "utf8"); + + const missingTimeout = await runBenchmarkCase({ + case_root: caseRoot, + manifest, + adapter: adapterFor(manifest, "agent", async (context) => { + expect(JSON.stringify(context)).not.toContain("reference/reference.txt"); + await rm(referencePath); + return new Promise(() => undefined); + }), + identity: { commit: "commit-finalization-v1", environment: "node-test-v1" } + }); + + expect(missingTimeout.outcome).toBe("hard_error"); + expect(missingTimeout.error?.class).toBe("ReferenceScoringError"); + expect(missingTimeout.evaluation_contract_sha256).not.toBe(availableTimeout.evaluation_contract_sha256); + expect(missingTimeout.run_id).not.toBe(availableTimeout.run_id); + }); + + it("binds a post-adapter package-alias failure to its final reference bytes without exposing them", async () => { + const { caseRoot, manifest, referencePath } = await createCase(); + const packagePath = resolve(caseRoot, "package/problem.md"); + + async function runPackageAlias(bytes: string): Promise { + await writeFile(packagePath, "Solve the synthetic finalization case.\n", "utf8"); + await rm(referencePath, { force: true }); + await writeFile(referencePath, `${ALPHA_REFERENCE}\n`, "utf8"); + return runBenchmarkCase({ + case_root: caseRoot, + manifest, + adapter: adapterFor(manifest, "agent", async () => { + await writeFile(packagePath, `${bytes}\n`, "utf8"); + await rm(referencePath); + await link(packagePath, referencePath); + throw new Error(`adapter secret=TOP-SECRET path=${caseRoot}`); + }), + identity: { commit: "commit-finalization-v1", environment: "node-test-v1" }, + clock: deterministicClock() + }); + } + + const alpha = await runPackageAlias(ALPHA_REFERENCE); + const beta = await runPackageAlias(BETA_REFERENCE); + + for (const result of [alpha, beta]) { + expect(result.outcome).toBe("hard_error"); + expect(result.error?.class).toBe("ReferenceScoringError"); + expect(result.metrics.reference_leak_check).toMatchObject({ + status: "unavailable", + value: null, + reason: "reference_package_overlap" + }); + const serialized = JSON.stringify(result); + expect(serialized).not.toContain(caseRoot); + expect(serialized).not.toContain("TOP-SECRET"); + expect(serialized).not.toContain(ALPHA_REFERENCE); + expect(serialized).not.toContain(BETA_REFERENCE); + } + expect(alpha.frozen_case_sha256).toBe(beta.frozen_case_sha256); + expect(alpha.evaluation_contract_sha256).not.toBe(beta.evaluation_contract_sha256); + expect(alpha.run_id).not.toBe(beta.run_id); + }); +}); diff --git a/tests/benchmark-reference-lifecycle.test.ts b/tests/benchmark-reference-lifecycle.test.ts index 4ff63d8..3b6d9f7 100644 --- a/tests/benchmark-reference-lifecycle.test.ts +++ b/tests/benchmark-reference-lifecycle.test.ts @@ -108,9 +108,21 @@ process.stdout.write(JSON.stringify(result)); await rename(reference, movedReference); await writeFile(release, "go"); const exit = await waitForExit(child); - expect(exit.code).toBe(1); - expect(exit.stdout).toBe(""); - expect(exit.stderr).toMatch(/ENOENT|unavailable/i); + expect(exit.code).toBe(0); + expect(exit.stderr).toBe(""); + const result = JSON.parse(exit.stdout) as { + outcome: string; + evaluation_contract_sha256: string; + metrics: { reference_leak_check: { status: string; value: unknown } }; + error: { class: string; message: string } | null; + }; + expect(result.outcome).toBe("hard_error"); + expect(result.evaluation_contract_sha256).toMatch(/^[a-f0-9]{64}$/); + expect(result.metrics.reference_leak_check).toMatchObject({ status: "unavailable", value: null }); + expect(result.error).toMatchObject({ class: "ReferenceScoringError" }); + expect(result.error?.message).toMatch(/^failure:[a-f0-9]{12}$/); + expect(exit.stdout).not.toContain(root); + expect(exit.stdout).not.toContain("REFERENCE-LIFECYCLE-SECRET"); } finally { child.kill("SIGKILL"); } diff --git a/tests/benchmark-report.test.ts b/tests/benchmark-report.test.ts index 6d129a0..1f9678c 100644 --- a/tests/benchmark-report.test.ts +++ b/tests/benchmark-report.test.ts @@ -106,7 +106,51 @@ describe("benchmark aggregate reports", () => { }); it("refuses to compare agent and one-shot results from different frozen case bytes", () => { - const oneShot = result({ variant: "one_shot", adapter_id: "one-shot-v1", frozen_case_sha256: "b".repeat(64) }); + const oneShot = result({ + variant: "one_shot", + adapter_id: "one-shot-v1", + run_id: "synthetic-report-one-shot-bbbbbbbbbbbb", + frozen_case_sha256: "b".repeat(64) + }); expect(() => aggregateBenchmarkResults([result(), oneShot])).toThrow(/do not share a frozen case/i); }); + + it("rejects duplicate run ids globally", () => { + const duplicate = result({ + case_id: "synthetic-other", + variant: "one_shot", + adapter_id: "one-shot-v1" + }); + expect(() => aggregateBenchmarkResults([result(), duplicate])).toThrow(/run id.*unique|duplicate run id/i); + }); + + it.each(["agent", "one_shot"] as const)("rejects a duplicate %s variant for one case", (variant) => { + const first = result({ + variant, + adapter_id: variant === "agent" ? "agent-v1" : "one-shot-v1", + run_id: `synthetic-report-${variant.replace("_", "-")}-aaaaaaaaaaaa` + }); + const duplicate = result({ + variant, + adapter_id: variant === "agent" ? "agent-v2" : "one-shot-v2", + run_id: `synthetic-report-${variant.replace("_", "-")}-bbbbbbbbbbbb` + }); + expect(() => aggregateBenchmarkResults([first, duplicate])).toThrow(/duplicate.*variant|at most one.*variant/i); + }); + + it("rejects a third same-case result before contract comparison can ignore it", () => { + const agent = result(); + const baseline = result({ + variant: "one_shot", + adapter_id: "one-shot-v1", + run_id: "synthetic-report-one-shot-bbbbbbbbbbbb" + }); + const third = result({ + adapter_id: "agent-v2", + run_id: "synthetic-report-agent-cccccccccccc", + evaluation_contract_sha256: "d".repeat(64) + }); + + expect(() => aggregateBenchmarkResults([agent, baseline, third])).toThrow(/duplicate.*variant|at most one.*variant/i); + }); });