From 7deede4fed75447cbd3fe2712f5d2475de508e0a Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Thu, 30 Jul 2026 17:46:44 -0600 Subject: [PATCH] fix(benchmark): survive release version bumps without orphaning evidence The dependency-lock digest covers package.json, so the 0.138.0 version bump invalidated the published CodeTraceBench evidence pin and blocked the release. Split the pin: the current constant tracks the live lockfiles, the evidence keeps its creation-time digest and package version, and a new test recomputes the lock digest with the version stamped back to prove the two differ by the version field alone. A real dependency change still fails that proof and forces a new benchmark run or explicit retirement. --- CHANGELOG.md | 5 +++ src/analyst/benchmark-implementation.ts | 11 +++++++ .../benchmark-reference-result.test.ts | 33 +++++++++++++++++-- src/analyst/index.ts | 2 ++ 4 files changed, 48 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 85b53a62..68c7480f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,11 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval- ## [0.138.0] - 2026-07-30 - exact analyst runs with sealed receipts +### Fixed + +- A release version bump no longer invalidates the published analyst benchmark evidence. + The dependency-lock pin now tracks the current lockfiles, the evidence keeps its own creation-time digest and package version, and a test proves the two locks differ by the version stamp alone — a real dependency change still forces a new benchmark run or explicit retirement of the evidence. + ### Added - `AnalystRegistry.runExact()` requires ordered analyst ids and an explicit value for every run-policy field. diff --git a/src/analyst/benchmark-implementation.ts b/src/analyst/benchmark-implementation.ts index 175eafa0..feef9215 100644 --- a/src/analyst/benchmark-implementation.ts +++ b/src/analyst/benchmark-implementation.ts @@ -8,6 +8,17 @@ export const ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES = Object.freeze([ ]) export const ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256 = + 'd06e3e3f171cc6bd5cf36b14325f507f34b1c6a9d5129ce8a85bf3c1681f8223' + +/** The published benchmark evidence was produced at this package version. + * A release changes package.json's version field, which is part of the lock + * manifest but cannot change benchmark behavior, so the evidence stays bound + * to the digest at its creation. A test proves the current lock differs from + * the evidence lock by the version stamp alone; any real dependency change + * still forces a new benchmark run or explicit retirement of the evidence. */ +export const ANALYST_BENCHMARK_EVIDENCE_PACKAGE_VERSION = '0.137.0' + +export const ANALYST_BENCHMARK_EVIDENCE_DEPENDENCY_LOCK_SHA256 = '1e03f2daed356d60316aabefb407ec1e437ac94d408d61eea4ae096e9c6fbb5b' export const ANALYST_BENCHMARK_IMPLEMENTATION_FILES = Object.freeze([ diff --git a/src/analyst/benchmark-reference-result.test.ts b/src/analyst/benchmark-reference-result.test.ts index 1ddc7a25..91065691 100644 --- a/src/analyst/benchmark-reference-result.test.ts +++ b/src/analyst/benchmark-reference-result.test.ts @@ -4,7 +4,8 @@ import { fileURLToPath } from 'node:url' import { describe, expect, it } from 'vitest' import { readAnalystBenchmarkArtifact } from './benchmark-command-result' import { - ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256, + ANALYST_BENCHMARK_EVIDENCE_DEPENDENCY_LOCK_SHA256, + ANALYST_BENCHMARK_EVIDENCE_PACKAGE_VERSION, ANALYST_BENCHMARK_IMPLEMENTATION_SHA256, } from './benchmark-implementation' @@ -87,7 +88,7 @@ describe('CodeTraceBench GLM-5.2 reference result', () => { selectedCases: 32, protocolSha256: '166e399c9a93c9806b007273bf0b54078709389c52c6a33e3cbce0f554dab302', implementationSha256: ANALYST_BENCHMARK_IMPLEMENTATION_SHA256, - dependencyLockSha256: ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256, + dependencyLockSha256: ANALYST_BENCHMARK_EVIDENCE_DEPENDENCY_LOCK_SHA256, observations: 128, verificationAvailability: { cases: 32, @@ -215,6 +216,32 @@ describe('CodeTraceBench GLM-5.2 reference result', () => { }) }) + it('proves the current dependency lock differs from the evidence lock by version stamp only', async () => { + const packagePath = fileURLToPath(new URL('../../package.json', import.meta.url)) + const lockPath = fileURLToPath(new URL('../../pnpm-lock.yaml', import.meta.url)) + const packageSource = await readFile(packagePath, 'utf8') + const currentVersion = JSON.parse(packageSource).version as string + const stampedSource = packageSource.replace( + `"version": "${currentVersion}"`, + `"version": "${ANALYST_BENCHMARK_EVIDENCE_PACKAGE_VERSION}"`, + ) + const normalize = (value: string) => value.replace(/\r\n?/g, '\n') + const fileSha256 = (value: string) => + createHash('sha256').update(normalize(value), 'utf8').digest('hex') + const recomputed = createHash('sha256') + .update( + JSON.stringify({ + domain: 'agent-eval/public-analyst-benchmark/dependency-lock', + files: [ + { path: 'package.json', sha256: fileSha256(stampedSource) }, + { path: 'pnpm-lock.yaml', sha256: fileSha256(await readFile(lockPath, 'utf8')) }, + ], + }), + ) + .digest('hex') + expect(recomputed).toBe(ANALYST_BENCHMARK_EVIDENCE_DEPENDENCY_LOCK_SHA256) + }) + it('binds the verification receipt to all three published results', async () => { const receipt = JSON.parse(await readFile(VERIFICATION_RECEIPT, 'utf8')) const digest = async (path: string) => @@ -248,7 +275,7 @@ describe('CodeTraceBench GLM-5.2 reference result', () => { }, implementation: { sourceSha256: ANALYST_BENCHMARK_IMPLEMENTATION_SHA256, - dependencyLockSha256: ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256, + dependencyLockSha256: ANALYST_BENCHMARK_EVIDENCE_DEPENDENCY_LOCK_SHA256, }, failureProof: { parallelStart: { diff --git a/src/analyst/index.ts b/src/analyst/index.ts index 8e69ba55..7af0d9c7 100644 --- a/src/analyst/index.ts +++ b/src/analyst/index.ts @@ -114,6 +114,8 @@ export { ANALYST_BENCHMARK_DEPENDENCY_LOCK_DIGEST_ALGORITHM, ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES, ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256, + ANALYST_BENCHMARK_EVIDENCE_DEPENDENCY_LOCK_SHA256, + ANALYST_BENCHMARK_EVIDENCE_PACKAGE_VERSION, ANALYST_BENCHMARK_IMPLEMENTATION_DIGEST_ALGORITHM, ANALYST_BENCHMARK_IMPLEMENTATION_FILES, ANALYST_BENCHMARK_IMPLEMENTATION_SHA256,