From 1e43c19597e9256b66b73dbf1ecc73c70849a22b Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Thu, 30 Jul 2026 23:52:01 -0600 Subject: [PATCH] fix(cost): restore caller failure reasons on reconciled receipts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 0.138.0 narrowed CostReceipt.error to the ledger's own 'paid-call-failed' and reconcile silently dropped a caller's { error } option — a crash orphan settled as a successful zero-dollar call, and ledgers persisted before 0.138.0 with custom reasons failed schema parse. reconcile now accepts { error } (implying failure), buildReceipt records the caller reason with 'paid-call-failed' as the default, and the receipt schema accepts any non-empty reason. Release 0.139.0 also carries the DSPy RLM analyst engine (#495) that merged without a release. --- CHANGELOG.md | 10 ++++ clients/python/pyproject.toml | 2 +- clients/python/src/agent_eval_rpc/__init__.py | 2 +- clients/python/uv.lock | 2 +- package.json | 2 +- src/analyst/benchmark-implementation.ts | 4 +- src/cost-ledger.test.ts | 50 +++++++++++++++++++ src/cost-ledger.ts | 21 +++++--- 8 files changed, 81 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 12fd1e0f..55bb7b5b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,16 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval- --- +## [0.139.0] - 2026-07-31 - recursive RLM trace analysts and caller failure reasons + +### Added + +- Trace analysts run official DSPy RLMs; the Ax stack is retired (#495). + The published CodeTraceBench evidence remains bound to the retired direct runner via the evidence digests; a fresh certified run must replace it before any accuracy number is attributed to the new engine. +- `CostLedger.reconcile` accepts a caller-supplied failure reason: `reconcile(callId, observed, { error })` settles a failed receipt carrying that reason, and supplying a reason implies failure. + 0.138.0 had narrowed `CostReceipt.error` to the ledger's own `'paid-call-failed'`, silently discarding caller reasons — a crash orphan settled as a successful $0 call. + The receipt schema accepts any non-empty reason again, so ledgers persisted before 0.138.0 parse. + ## [0.138.0] - 2026-07-30 - exact analyst runs with sealed receipts ### Fixed diff --git a/clients/python/pyproject.toml b/clients/python/pyproject.toml index 9c59fbf0..243d1c62 100644 --- a/clients/python/pyproject.toml +++ b/clients/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "agent-eval-rpc" -version = "0.138.0" +version = "0.139.0" description = "Python RPC client, official optimizer bridge, and DSPy metric adapter for @tangle-network/agent-eval." readme = "README.md" requires-python = ">=3.10" diff --git a/clients/python/src/agent_eval_rpc/__init__.py b/clients/python/src/agent_eval_rpc/__init__.py index 976adb75..eb91c5c1 100644 --- a/clients/python/src/agent_eval_rpc/__init__.py +++ b/clients/python/src/agent_eval_rpc/__init__.py @@ -53,7 +53,7 @@ try: __version__ = version("agent-eval-rpc") except PackageNotFoundError: - __version__ = "0.138.0" + __version__ = "0.139.0" __all__ = [ "Client", diff --git a/clients/python/uv.lock b/clients/python/uv.lock index 7dc22618..8d6f5d14 100644 --- a/clients/python/uv.lock +++ b/clients/python/uv.lock @@ -34,7 +34,7 @@ conflicts = [[ [[package]] name = "agent-eval-rpc" -version = "0.138.0" +version = "0.139.0" source = { editable = "." } dependencies = [ { name = "filelock" }, diff --git a/package.json b/package.json index afcf6779..9ee401d1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-eval", - "version": "0.138.0", + "version": "0.139.0", "description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.", "homepage": "https://github.com/tangle-network/agent-eval#readme", "repository": { diff --git a/src/analyst/benchmark-implementation.ts b/src/analyst/benchmark-implementation.ts index c4bd7cf6..6fb24f24 100644 --- a/src/analyst/benchmark-implementation.ts +++ b/src/analyst/benchmark-implementation.ts @@ -10,7 +10,7 @@ export const ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES = Object.freeze([ ]) export const ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256 = - '779ff0146f56c14a059aac9276f4b673ee70515afe95959a62a9f2d376c688dc' + '6f3dc59755ff2a26fba4c2c1ad670f436cdd183faf252a5b9c2d50e6b190b460' /** The published benchmark evidence was produced at this package version, by * the retired one-shot direct runner, before trace analysts moved to the @@ -111,7 +111,7 @@ export const ANALYST_BENCHMARK_IMPLEMENTATION_FILES = Object.freeze([ ]) export const ANALYST_BENCHMARK_IMPLEMENTATION_SHA256 = - '8fc6d925934c4563a2851d2b3986211aa0ee145f3fdda0e1bcb447774d84b519' + '5e47fc9d9f49c468d3ef06c5d552925e6a026f9e22a75056a9c8c5a2879744c0' export function analystBenchmarkImplementationDigest() { return ANALYST_BENCHMARK_IMPLEMENTATION_SHA256 diff --git a/src/cost-ledger.test.ts b/src/cost-ledger.test.ts index 6c89e5f6..95297569 100644 --- a/src/cost-ledger.test.ts +++ b/src/cost-ledger.test.ts @@ -1075,6 +1075,56 @@ describe('CostLedger', () => { }) }) + it('records a caller-supplied failure reason on a reconciled receipt', async () => { + const pending = { + status: 'pending', + callId: 'orphan-1', + channel: 'agent', + phase: 'search', + actor: 'worker', + model: 'gpt-4o', + maximumCostUsd: 0.5, + timestamp: 1, + } + const { persistence, state } = memoryPersistence(eventFor(pending)) + const ledger = new CostLedger({ costCeilingUsd: 1, persistence }) + const receipt = ledger.reconcile( + 'orphan-1', + { model: 'gpt-4o', inputTokens: 0, outputTokens: 0, actualCostUsd: 0 }, + { error: 'process-crash-orphan' }, + ) + + expect(receipt).toMatchObject({ + status: 'settled', + error: 'process-crash-orphan', + costUsd: 0, + }) + expect(persistedRecords(state.events).at(-1)).toMatchObject({ + error: 'process-crash-orphan', + }) + }) + + it('keeps the ledger default reason when reconcile marks failure without one', async () => { + const pending = { + status: 'pending', + callId: 'orphan-2', + channel: 'agent', + phase: 'search', + actor: 'worker', + model: 'gpt-4o', + maximumCostUsd: 0.5, + timestamp: 1, + } + const { persistence } = memoryPersistence(eventFor(pending)) + const ledger = new CostLedger({ costCeilingUsd: 1, persistence }) + const receipt = ledger.reconcile( + 'orphan-2', + { model: 'gpt-4o', inputTokens: 0, outputTokens: 0, actualCostUsd: 0 }, + { failed: true }, + ) + expect(receipt).toMatchObject({ status: 'settled', error: 'paid-call-failed' }) + }) + it('never replays an unresolved provider call', async () => { const pending = { status: 'pending', diff --git a/src/cost-ledger.ts b/src/cost-ledger.ts index aa7a367a..296988ca 100644 --- a/src/cost-ledger.ts +++ b/src/cost-ledger.ts @@ -57,7 +57,9 @@ export interface CostReceipt extends CostCallBase, CostUsage { actualCostUsd?: number /** Known non-provider amount supplied by an external executor. */ estimatedCostUsd?: number - error?: 'paid-call-failed' + /** Why the call failed. `'paid-call-failed'` is the ledger's own value for + * recovery settles; a reconciling caller may record its own reason. */ + error?: string } export type CostLedgerRecord = PendingCostCall | CostReceipt @@ -461,7 +463,7 @@ export class CostLedger { reconcile( callId: string, observed: CostReceiptInput, - options: { failed?: boolean } = {}, + options: { failed?: boolean; error?: string } = {}, ): CostReceipt { const pending = [...this.records.values()].find( (record): record is PendingCostCall => @@ -472,7 +474,12 @@ export class CostLedger { throw new CostCallConflictError(`CostLedger: call '${callId}' is still active`) } this.ensureCostLimitPersisted(callId) - return this.commitReceipt(pending, observed, options.failed === true) + return this.commitReceipt( + pending, + observed, + options.failed === true || options.error !== undefined, + options.error, + ) } list(filter?: CostLedgerFilter): CostReceipt[] { @@ -751,8 +758,9 @@ export class CostLedger { pending: PendingCostCall, observed: CostReceiptInput, failed = false, + error?: string, ): CostReceipt { - const receipt = buildReceipt(pending, observed, failed) + const receipt = buildReceipt(pending, observed, failed, error) if (this.records.get(pending.callId)?.status !== 'pending') { throw new CostCallConflictError(`CostLedger: call '${pending.callId}' is not pending`) } @@ -933,6 +941,7 @@ function buildReceipt( pending: PendingCostCall, observed: CostReceiptInput, failed = false, + error?: string, ): CostReceipt { assertUsage(observed) assertString(observed.model, 'receipt.model') @@ -1018,7 +1027,7 @@ function buildReceipt( ...(hasActual ? { actualCostUsd: observed.actualCostUsd } : {}), ...(hasExternalEstimate ? { estimatedCostUsd: observed.estimatedCostUsd } : {}), ...(pending.maximumCostUsd === undefined ? {} : { maximumCostUsd: pending.maximumCostUsd }), - ...(failed ? { error: 'paid-call-failed' as const } : {}), + ...(failed ? { error: error ?? 'paid-call-failed' } : {}), ...(pending.tags ? { tags: { ...pending.tags } } : {}), timestamp: pending.timestamp, }, @@ -1062,7 +1071,7 @@ const CostReceiptBaseShape = { pricing: CostPricingSchema.optional(), actualCostUsd: NonNegative.optional(), estimatedCostUsd: NonNegative.optional(), - error: z.literal('paid-call-failed').optional(), + error: NonEmptyString.optional(), } const CostReceiptSchema = z .strictObject({