From 70a104a38ca466a14c1baaf771b40a27c9f3ab93 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Wed, 29 Jul 2026 18:08:39 -0600 Subject: [PATCH 1/2] feat(memory): measure continual learning against stateless runs --- CHANGELOG.md | 16 +- README.md | 30 ++ docs/architecture.md | 10 +- src/memory/experiment.ts | 18 + src/memory/experiment/cases.ts | 4 +- src/memory/experiment/cell.ts | 194 +++++-- src/memory/experiment/comparison-ref.ts | 85 +++ src/memory/experiment/execution-context.ts | 4 + src/memory/experiment/learning-evidence.ts | 178 ++++++ src/memory/experiment/learning-metrics.ts | 444 +++++++++++++++ src/memory/experiment/learning-pairs.ts | 424 +++++++++++++++ src/memory/experiment/learning.ts | 111 ++++ src/memory/experiment/metrics.ts | 9 +- src/memory/experiment/probe-evaluation.ts | 53 ++ src/memory/experiment/recovery.ts | 240 +++++---- src/memory/experiment/run.ts | 88 ++- src/memory/experiment/runtime.ts | 18 +- src/memory/experiment/types.ts | 192 ++++++- src/memory/experiment/validation.ts | 108 +++- src/memory/improvement/evaluation.ts | 52 +- src/memory/lifecycle.ts | 96 +++- .../experiment-learning-validation.test.ts | 55 ++ tests/memory/experiment-learning.test.ts | 506 ++++++++++++++++++ tests/memory/experiment-parallel.test.ts | 149 ++++++ tests/memory/experiment-privacy.test.ts | 39 ++ .../memory/experiment-recovery-safety.test.ts | 302 +++++++++++ tests/memory/experiment-recovery.test.ts | 72 ++- tests/memory/experiment-safety.test.ts | 48 ++ tests/support/memory-learning.ts | 143 +++++ 29 files changed, 3458 insertions(+), 230 deletions(-) create mode 100644 src/memory/experiment/comparison-ref.ts create mode 100644 src/memory/experiment/learning-evidence.ts create mode 100644 src/memory/experiment/learning-metrics.ts create mode 100644 src/memory/experiment/learning-pairs.ts create mode 100644 src/memory/experiment/learning.ts create mode 100644 src/memory/experiment/probe-evaluation.ts create mode 100644 tests/memory/experiment-learning-validation.test.ts create mode 100644 tests/memory/experiment-learning.test.ts create mode 100644 tests/memory/experiment-parallel.test.ts create mode 100644 tests/memory/experiment-recovery-safety.test.ts create mode 100644 tests/support/memory-learning.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 47c97f5..a74b9b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,19 @@ # Changelog +## Unreleased + +### Added + +- Added `runAgentMemoryLearningExperiment` for matched stateful-versus-stateless memory measurement under one shared cost limit. +- Added exact paired gain, explicitly labeled transfer probes, and repeated-probe forgetting reports. +- Limited learning gain to post-first-step probes, averaged repetitions within independent sequences, and added per-candidate intervals. +- Added recorded arm order for counterbalanced runs and exact sequence references for safe crash recovery. +- Added abort and resume support, content-addressed comparison and probe evidence, and exact cell artifact hashes. + +### Changed + +- Memory experiment artifacts and cache identities now record `memoryMode` and a full `comparisonRef`; non-equivalent arms fail comparison. + ## 7.0.0 ### Breaking Changes @@ -49,7 +63,7 @@ - Load `proper-lockfile` with a dynamic import inside the functions that take a lock, instead of at module scope. It pulls in `graceful-fs`, which patches Node's `fs` at import time (`fs.close = ...`). - workerd exposes those as getter-only accessors, so the assignment threw while Cloudflare validated an uploaded Worker (`Cannot set property close of # which has only a getter [code: 10021]`), rejecting the whole Worker — including consumers that never take a lock. + workerd exposes those as getter-only accessors, so the assignment threw while Cloudflare validated an uploaded Worker (`Cannot set property close of # which has only a getter [code: 10021]`), rejecting the whole Worker, including consumers that never take a lock. `verify:package` now fails on any static import of a module that patches a Node builtin, because `wrangler deploy --dry-run` bundles without executing and cannot see this class of failure. ## 6.1.8 diff --git a/README.md b/README.md index 9c1f630..c305874 100644 --- a/README.md +++ b/README.md @@ -239,6 +239,36 @@ Positive external work without a receipt is recorded as incomplete cost accounti Use `0` only for a free local path. Paid memory improvement defaults to a zero-dollar total limit; set `maxTotalCostUsd` and `maximumEvaluationCostUsd` before enabling paid work. +Use `runAgentMemoryLearningExperiment` to measure whether retained memory helps across ordered steps: + +```ts +import { runAgentMemoryLearningExperiment } from '@tangle-network/agent-knowledge/memory' + +const result = await runAgentMemoryLearningExperiment({ + experimentId: 'support-memory', + runDir: 'support-memory', + candidates: [memoryCandidate], + sequences, + seed: 42, + reps: 5, + armOrder: 'stateful-first', + costCeiling: 10, +}) + +console.log(result.comparison.gain) +``` + +The function runs matched stateful and stateless arms with the same immutable candidate, tasks, executor, policy, seed, and repetitions. +The stateless arm clears declared scopes between steps; adapters must support scoped `clear`. +Gain excludes first-step probes and averages candidates and repetitions within each independent sequence. +Use `transferKey` on later probes for transfer and repeat one `retentionKey` across steps for forgetting. +Unmarked probes are not assigned those meanings. + +Both arms share one cost limit and must have identical comparison references. +Run independent experiments with opposite `armOrder` values when provider behavior may drift. +Each saved probe includes the exact scoring input and content hash, so protect the run directory like the memory data itself. +Pass `signal` to cancel; rerun the same options and directory to resume completed work and cost records. + ## Run benchmarks `@tangle-network/agent-knowledge/benchmarks` provides common case and report types for: diff --git a/docs/architecture.md b/docs/architecture.md index 8d51181..14533d8 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -38,17 +38,18 @@ When that string is the canonical `/.agent-knowledge` directory, both form | --- | --- | | `.agent-knowledge/index.json` | the built knowledge index (`writeKnowledgeIndex` writes it through this store) | | `.agent-knowledge/events.json` | the knowledge event log, including one `research.iteration` per research-loop round | -| `.agent-knowledge/claim-ledgers/.json` | one research run's claim ledger — corroboration counts, contradiction edges, open deep questions | +| `.agent-knowledge/claim-ledgers/.json` | one research run's claim ledger: corroboration counts, contradiction edges, open deep questions | | `.agent-knowledge/sources.json` | the immutable source registry | | `.agent-knowledge/mutation.lock.durable`, `mutation-epoch.json`, `file-transactions/` | the cross-process mutation lock and its crash-recovery state | The root is also the directory `withKnowledgeMutation` locks, so every record above is written under one lock and one epoch. There is exactly one writer per file: a second index writer alongside this one is a defect, not a variation. -A claim ledger is the one record several writers legitimately share — a resumed run beside a live one, or several workers researching one goal in parallel. +A claim ledger is the one record several writers legitimately share, such as a resumed run beside a live one or several workers researching one goal in parallel. They reach it through `mergeClaimLedger(id, merge)`, which holds the mutation lock across the read, the merge, and the write, so no writer can build its record from a value another writer has already replaced. `putClaimLedger` writes the whole record and is correct only for a single writer. -The combining rule is `mergeClaimLedgers`: support and contradiction edges union, `contested` and `addressed` latch on, `firstSeenRound` moves earlier, and every collection is sorted — so the merge is commutative, associative, and idempotent, and the bytes on disk depend on the evidence rather than on scheduling. +The combining rule is `mergeClaimLedgers`: support and contradiction edges union, `contested` and `addressed` latch on, `firstSeenRound` moves earlier, and every collection is sorted. +The merge is commutative, associative, and idempotent, so the bytes on disk depend on the evidence rather than on scheduling. Ledgers for two different goals refuse to merge (`ClaimLedgerGoalConflictError`) rather than pooling unrelated evidence into one corroboration count. The live driver exposes the published Set-based `TrackedClaim`; the ledger stores a separate `ResearchClaimRecord` with sorted arrays so JSON serialization cannot erase those sets. Source verification snapshots the proposal and persists a `ResearchClaimEvidence` observation containing the expected registry id, original URI, and full content hash; that observation cannot affect claim support or completion by itself. @@ -57,7 +58,8 @@ The ledger materializes only observations whose complete source identity matches Unversioned URI-only ledgers cannot prove which bytes produced their observations; reads and writes fail with `ClaimLedgerMigrationRequiredError` and preserve the original file for an explicit archive-and-reverify migration. Before synchronous question generation, the persistent driver records `preparedRounds`; a resume reconstructs and checkpoints any prepared round whose questions were interrupted, and the loop publishes its `research.iteration` event only after that checkpoint succeeds. -Every write in this layer goes through `durable-fs` (`writeFileDurable`, `writeJsonDurableWithinRoot`) — temp file, fsync, atomic rename, fsync parent, through `O_NOFOLLOW` descriptors anchored via `/proc/self/fd` so a directory swapped for a symlink mid-write cannot redirect it outside the root. +Every write in this layer goes through `durable-fs` (`writeFileDurable`, `writeJsonDurableWithinRoot`): temp file, fsync, atomic rename, and parent fsync. +`O_NOFOLLOW` descriptors anchored through `/proc/self/fd` prevent a directory swapped for a symlink during a write from redirecting it outside the root. These are exported from the package entrypoint; consumers that keep their own journals should use them rather than reimplement them. ## Runtime Loop diff --git a/src/memory/experiment.ts b/src/memory/experiment.ts index 07961b4..1f01b85 100644 --- a/src/memory/experiment.ts +++ b/src/memory/experiment.ts @@ -3,9 +3,14 @@ export { buildAgentMemorySequenceScenarios, buildAgentMemorySequencesFromBenchmarkCases, } from './experiment/cases' +export { + compareAgentMemoryLearning, + runAgentMemoryLearningExperiment, +} from './experiment/learning' export { runAgentMemoryExperiment } from './experiment/run' export type { AgentMemoryAttemptEvent, + AgentMemoryEvidenceRef, AgentMemoryExecutionContext, AgentMemoryExecutionCostMeter, AgentMemoryExecutionCostReceipt, @@ -13,15 +18,28 @@ export type { AgentMemoryExecutionPaidCallResult, AgentMemoryExecutionStep, AgentMemoryExperimentCandidate, + AgentMemoryExperimentCandidateRef, + AgentMemoryExperimentComparisonRef, AgentMemoryExperimentRankingRow, AgentMemoryExperimentRunLease, + AgentMemoryForgettingComparison, + AgentMemoryLearningArmOrder, + AgentMemoryLearningCandidateSummary, + AgentMemoryLearningCellComparison, + AgentMemoryLearningComparison, + AgentMemoryMode, AgentMemorySequence, AgentMemorySequenceArtifact, AgentMemorySequenceProbe, AgentMemorySequenceProbeResult, AgentMemorySequenceScenario, AgentMemorySequenceStep, + AgentMemoryTransferCellComparison, + AgentMemoryTransferStepSummary, BuildAgentMemorySequencesFromBenchmarkCasesOptions, + CompareAgentMemoryLearningOptions, RunAgentMemoryExperimentOptions, RunAgentMemoryExperimentResult, + RunAgentMemoryLearningExperimentOptions, + RunAgentMemoryLearningExperimentResult, } from './experiment/types' diff --git a/src/memory/experiment/cases.ts b/src/memory/experiment/cases.ts index 75169b8..fca04cb 100644 --- a/src/memory/experiment/cases.ts +++ b/src/memory/experiment/cases.ts @@ -32,7 +32,7 @@ export function buildAgentMemorySequencesFromBenchmarkCases( scope: compactScope( options.eventScope?.({ event, case: testCase, eventIndex }) ?? { agentId: memoryAgentId, - sessionId: testCase.id, + sessionId: 'benchmark-session', }, ), writes: [ @@ -56,7 +56,7 @@ export function buildAgentMemorySequencesFromBenchmarkCases( scope: compactScope( options.probeScope?.(testCase) ?? { agentId: memoryAgentId, - sessionId: testCase.id, + sessionId: 'benchmark-session', }, ), probes: [ diff --git a/src/memory/experiment/cell.ts b/src/memory/experiment/cell.ts index f560efe..e605e07 100644 --- a/src/memory/experiment/cell.ts +++ b/src/memory/experiment/cell.ts @@ -1,13 +1,22 @@ import { randomUUID } from 'node:crypto' -import type { CampaignStorage, DispatchContext } from '@tangle-network/agent-eval/campaign' -import { scoreMemoryBenchmarkArtifact } from '../../benchmarks/index' +import { + type CampaignStorage, + canonicalDigest, + type DispatchContext, +} from '@tangle-network/agent-eval/campaign' import { stableId } from '../../ids' import { type AgentMemoryBranch, createAgentMemoryBranch } from '../branch' -import { resolveMemoryCleanupTimeoutMs, runBoundedMemoryLifecycle } from '../lifecycle' +import { + createBoundedMemoryAdapter, + MEMORY_OPERATION_CANCELLATION_TIMEOUT_MS, + resolveMemoryCleanupTimeoutMs, + runBoundedMemoryLifecycle, +} from '../lifecycle' import type { AgentMemoryAdapter } from '../types' import { createAgentMemoryCostRecorder } from './cost' import { createAgentMemoryExecutionContext } from './execution-context' import { countDimensions, mean, meanDimensions } from './metrics' +import { type AgentMemoryProbeScoringEvidence, scoreAgentMemoryProbe } from './probe-evaluation' import { appendMemoryAttemptEvent, memoryAttemptCostCallId, memoryAttemptEvent } from './recovery' import { AgentMemoryCleanupError, @@ -20,6 +29,8 @@ import { import type { AgentMemoryExecutionStep, AgentMemoryExperimentCandidate, + AgentMemoryExperimentComparisonRef, + AgentMemoryMode, AgentMemorySequence, AgentMemorySequenceArtifact, AgentMemorySequenceProbe, @@ -31,6 +42,10 @@ import type { } from './types' import { stringArray, uniqueStrings } from './validation' +export class AgentMemoryAdapterCapabilityError extends Error { + override readonly name = 'AgentMemoryAdapterCapabilityError' +} + export async function runSequenceCell(input: { options: RunAgentMemoryExperimentOptions candidate: AgentMemoryExperimentCandidate @@ -40,9 +55,21 @@ export async function runSequenceCell(input: { storage: CampaignStorage attemptLogPath: string lease: OwnedMemoryExperimentRunLease + memoryMode: AgentMemoryMode + comparisonRef: AgentMemoryExperimentComparisonRef }): Promise { - const { options, candidate, scenario, context, runIdentity, storage, attemptLogPath, lease } = - input + const { + options, + candidate, + scenario, + context, + runIdentity, + storage, + attemptLogPath, + lease, + memoryMode, + comparisonRef, + } = input const cleanupBranches = options.cleanupBranches ?? true const costUsd = candidate.externalCostUsdPerSequence ?? 0 if (!Number.isFinite(costUsd) || costUsd < 0) { @@ -59,7 +86,6 @@ export async function runSequenceCell(input: { options.cleanupTimeoutMs, `${candidate.id}: memory sequence`, ) - context.signal.throwIfAborted() await lease.assertOwned() const startedAt = Date.now() @@ -78,6 +104,7 @@ export async function runSequenceCell(input: { cleanupBranches, now: options.now, }) + await lease.assertOwned() appendMemoryAttemptEvent(storage, attemptLogPath, attempt) let externalCallAttempted = false @@ -111,18 +138,30 @@ export async function runSequenceCell(input: { let finalClearStarted = false let finalClearCompleted = false try { - const created = await candidate.createAdapter({ - branchId, - purpose: 'execute', + const created = await createBoundedMemoryAdapter({ + operation: `${candidate.id}: execution adapter creation`, + timeoutMs: cleanupTimeoutMs, signal: context.signal, - maximumCostUsd: costUsd, - markExternalCall: () => { - externalCallAttempted = true - }, - recordExternalCost: (actualCostUsd) => { - externalCallAttempted = true - costRecorder.record(actualCostUsd) - }, + create: (signal) => + candidate.createAdapter({ + branchId, + purpose: 'execute', + signal, + maximumCostUsd: costUsd, + markExternalCall: () => { + externalCallAttempted = true + }, + recordExternalCost: (actualCostUsd) => { + externalCallAttempted = true + costRecorder.record(actualCostUsd) + }, + }), + dispose: candidate.disposeAdapter + ? async (lateAdapter) => { + externalCallAttempted = true + await candidate.disposeAdapter?.(lateAdapter) + } + : undefined, }) if (!created) throw new Error(`${candidate.id}: createAdapter returned no execution adapter`) rawAdapter = created @@ -131,6 +170,11 @@ export async function runSequenceCell(input: { }) await lease.assertOwned() context.signal.throwIfAborted() + if (memoryMode === 'stateless' && !adapter.clear) { + throw new AgentMemoryAdapterCapabilityError( + `${candidate.id}: stateless memoryMode requires scoped clear support`, + ) + } if (cleanupBranches && !adapter.clear) { throw new Error( `${candidate.id}: cleanupBranches requires an adapter with scoped clear support`, @@ -142,41 +186,81 @@ export async function runSequenceCell(input: { lifetime: 'attempt', policy: candidate.policy, allowedWriteScopes: sequenceCleanupScopes(scenario.sequence), - baseScope: memoryExperimentBaseScope(options, candidate, scenario.sequenceId), + baseScope: memoryExperimentBaseScope(candidate), }) + const runProviderOperation = (operation: string, run: () => Promise): Promise => + runBoundedMemoryLifecycle({ + operation, + timeoutMs: cleanupTimeoutMs, + resource: adapter, + signal: context.signal, + cancellationTimeoutMs: MEMORY_OPERATION_CANCELLATION_TIMEOUT_MS, + run, + }) const probes: AgentMemorySequenceProbeResult[] = [] for (const [ordinal, step] of scenario.sequence.steps.entries()) { context.signal.throwIfAborted() await lease.assertOwned() - await writeStep(memory, step) + if (memoryMode === 'stateless' && ordinal > 0) { + await runBoundedMemoryLifecycle({ + operation: `${candidate.id}: stateless reset before step ${step.id}`, + timeoutMs: cleanupTimeoutMs, + resource: adapter, + run: () => clearSequenceScopes(memory!, scenario.sequence), + }) + context.signal.throwIfAborted() + await lease.assertOwned() + } + await runProviderOperation(`${candidate.id}: writes for step ${step.id}`, () => + writeStep(memory!, step), + ) await lease.assertOwned() if (options.executeStep) { const execution = createAgentMemoryExecutionContext(context, scenario.sequence) try { - await options.executeStep({ - memory, - candidateId: candidate.id, - step: executionStep(step, ordinal), - context: execution.context, - }) + await runProviderOperation(`${candidate.id}: execution for step ${step.id}`, () => + options.executeStep!({ + memory: memory!, + candidateId: candidate.id, + step: executionStep(step, ordinal), + context: execution.context, + }), + ) + } catch (error) { + execution.abort() + throw error } finally { execution.dispose() } } context.signal.throwIfAborted() await lease.assertOwned() - const stepProbes = await probeStep(memory, scenario.sequence, step) + const stepProbes = await runProviderOperation( + `${candidate.id}: probes for step ${step.id}`, + () => probeStep(memory!, scenario.sequence, step, ordinal, context), + ) await lease.assertOwned() probes.push(...stepProbes) } - const snapshot = await memory.snapshot() + const snapshot = await runProviderOperation(`${candidate.id}: branch snapshot`, () => + memory!.snapshot(), + ) await lease.assertOwned() - await options.onBranchSnapshot?.({ - candidateId: candidate.id, - sequenceId: scenario.sequenceId, - cellId: context.cellId, - snapshot, - }) + if (options.onBranchSnapshot) { + await runBoundedMemoryLifecycle({ + operation: `${candidate.id}: branch snapshot callback`, + timeoutMs: cleanupTimeoutMs, + signal: context.signal, + cancellationTimeoutMs: MEMORY_OPERATION_CANCELLATION_TIMEOUT_MS, + run: () => + options.onBranchSnapshot!({ + candidateId: candidate.id, + sequenceId: scenario.sequenceId, + cellId: context.cellId, + snapshot, + }), + }) + } await lease.assertOwned() const dimensions = meanDimensions(probes.map((probe) => probe.dimensions)) const dimensionSampleCounts = countDimensions( @@ -185,6 +269,8 @@ export async function runSequenceCell(input: { const artifact: AgentMemorySequenceArtifact = { candidateId: candidate.id, sequenceId: scenario.sequenceId, + memoryMode, + comparisonRef, score: mean(probes.map((probe) => probe.score)), passed: probes.length > 0 && probes.every((probe) => probe.passed), dimensions, @@ -277,6 +363,14 @@ export async function runSequenceCell(input: { new Error(`${candidate.id}: adapter creation failed before cleanup could be confirmed`), ) } + if (cleanupOwned && cleanupErrors.length === 0) { + try { + await lease.assertOwned() + } catch (error) { + cleanupOwned = false + ownershipError = error + } + } if (cleanupOwned && cleanupErrors.length === 0) appendCleanedAttempt(primaryError) if (!cleanupOwned && cleanupErrors.length === 0) { if (primaryError) throw primaryError @@ -348,11 +442,13 @@ async function probeStep( memory: AgentMemoryBranch, sequence: AgentMemorySequence, step: AgentMemorySequenceStep, + stepOrdinal: number, + dispatch: DispatchContext, ): Promise { const run = async (probe: AgentMemorySequenceProbe): Promise => { const scope = mergeScopes(step.scope, probe.scope) const context = await memory.getContext(probe.query, { scope, limit: probe.limit }) - const artifact = { + const scoringEvidence: AgentMemoryProbeScoringEvidence = { answer: context.text, rememberedFacts: context.hits.map((hit) => hit.text), citedEventIds: uniqueStrings([ @@ -365,30 +461,18 @@ async function probeStep( ...context.hits.flatMap((hit) => stringArray(hit.metadata?.actorIds)), ]), } - const evaluation = scoreMemoryBenchmarkArtifact( - { - id: `${sequence.id}:${step.id}:${probe.id}`, - family: sequence.family, - taskKind: probe.taskKind ?? 'memory-recall', - split: sequence.split, - events: [], - prompt: probe.query, - requiredFacts: - probe.requiredFacts && probe.requiredFacts.length > 0 - ? probe.requiredFacts - : probe.referenceAnswer - ? [{ id: `${probe.id}:reference`, anyOf: [probe.referenceAnswer] }] - : undefined, - forbiddenFacts: probe.forbiddenFacts, - expectedEventIds: probe.expectedEventIds, - expectedActorIds: probe.expectedActorIds, - referenceAnswer: probe.referenceAnswer, - }, - artifact, + const evidenceRef = canonicalDigest(scoringEvidence) + const evidencePath = await dispatch.artifacts.writeJson( + `memory-evidence/${evidenceRef.slice(7)}.json`, + scoringEvidence, ) + const evaluation = scoreAgentMemoryProbe(sequence, step, probe, scoringEvidence) return { id: probe.id, stepId: step.id, + stepOrdinal, + ...(probe.retentionKey !== undefined ? { retentionKey: probe.retentionKey } : {}), + ...(probe.transferKey !== undefined ? { transferKey: probe.transferKey } : {}), query: probe.query, score: evaluation.score, passed: evaluation.passed, @@ -396,6 +480,8 @@ async function probeStep( applicableDimensions: evaluation.applicableDimensions ?? Object.keys(evaluation.dimensions), notes: evaluation.notes, hitIds: context.hits.map((hit) => hit.id), + evidenceRef, + evidencePath, } } if (step.parallelProbes === false) { diff --git a/src/memory/experiment/comparison-ref.ts b/src/memory/experiment/comparison-ref.ts new file mode 100644 index 0000000..c00c78f --- /dev/null +++ b/src/memory/experiment/comparison-ref.ts @@ -0,0 +1,85 @@ +import { canonicalJson } from '@tangle-network/agent-eval' +import { sha256 } from '../../ids' +import { DEFAULT_MEMORY_RECOVERY_RETRIES_PER_ATTEMPT } from '../attempt-log' +import { DEFAULT_MEMORY_CLEANUP_TIMEOUT_MS } from '../lifecycle' +import type { + AgentMemoryExperimentCandidate, + AgentMemoryExperimentComparisonRef, + AgentMemoryMode, + RunAgentMemoryExperimentOptions, +} from './types' + +export const MEMORY_EXPERIMENT_IMPLEMENTATION_REF = `sha256:${sha256( + canonicalJson({ + stateful: 'retain declared sequence scopes across ordered steps', + stateless: 'clear every declared sequence scope before each step after the first', + isolation: 'provider scopes receive an opaque branch id but no experiment or sequence labels', + recovery: 'unfinished attempts bind cleanup to the exact sequence sha256', + gain: 'post-first-step stateful minus stateless reward', + inference: 'average candidates and repetitions within each sequence before paired bootstrap', + preTreatment: 'report first-step score balance separately from gain', + transfer: 'only probes with explicit transferKey after the first step', + forgetting: 'signed prior peak minus final score for exact repeated retentionKey probes', + evidence: 'full sha256 design, manifest, split, artifact, and probe scoring-input references', + }), +)}` + +const COMPARISON_REF_PATTERN = /^sha256:[a-f0-9]{64}$/ + +export function resolveAgentMemoryMode(value: unknown): AgentMemoryMode { + if (value === undefined) return 'stateful' + if (value !== 'stateful' && value !== 'stateless') { + throw new Error("memory experiment memoryMode must be 'stateful' or 'stateless'") + } + return value +} + +export function memoryExperimentComparisonRef( + options: RunAgentMemoryExperimentOptions, + resolvedRunDir: string, +): AgentMemoryExperimentComparisonRef { + const material = canonicalJson({ + implementationRef: MEMORY_EXPERIMENT_IMPLEMENTATION_REF, + experimentId: options.experimentId, + experimentRunId: options.experimentRunId ?? resolvedRunDir, + sequences: options.sequences, + candidates: options.candidates.map(candidateConditions), + recoveryCandidates: (options.recoveryCandidates ?? []).map(candidateConditions), + executeStepRef: options.executeStepRef ?? 'fixtures', + cleanupBranches: options.cleanupBranches ?? true, + seed: options.seed ?? 42, + reps: options.reps ?? 1, + resumable: options.resumable ?? true, + costCeilingUsd: options.costCeiling ?? options.costLedger?.costCeilingUsd ?? 0, + costPhase: options.costPhase ?? null, + maxConcurrency: options.maxConcurrency ?? 2, + dispatchTimeoutMs: options.dispatchTimeoutMs || null, + cleanupTimeoutMs: options.cleanupTimeoutMs ?? DEFAULT_MEMORY_CLEANUP_TIMEOUT_MS, + maxRecoveryAttempts: options.maxRecoveryAttempts ?? 1_000, + maxRecoveryRetriesPerAttempt: + options.maxRecoveryRetriesPerAttempt ?? DEFAULT_MEMORY_RECOVERY_RETRIES_PER_ATTEMPT, + }) + return `sha256:${sha256(material)}` +} + +export function assertAgentMemoryExperimentComparisonRef( + value: unknown, + label: string, +): asserts value is AgentMemoryExperimentComparisonRef { + if (typeof value !== 'string' || !COMPARISON_REF_PATTERN.test(value)) { + throw new Error(`${label} must be a full lowercase sha256 reference`) + } +} + +function candidateConditions(candidate: AgentMemoryExperimentCandidate) { + return { + id: candidate.id, + label: candidate.label ?? null, + ref: candidate.ref, + policy: candidate.policy ?? null, + baseScope: candidate.baseScope ?? null, + externalCostUsdPerSequence: candidate.externalCostUsdPerSequence ?? 0, + externalRecoveryCostUsdPerAttempt: candidate.externalRecoveryCostUsdPerAttempt ?? 0, + externalCostAccounting: candidate.externalCostAccounting ?? 'exact', + } +} diff --git a/src/memory/experiment/execution-context.ts b/src/memory/experiment/execution-context.ts index 06f5f69..68984ca 100644 --- a/src/memory/experiment/execution-context.ts +++ b/src/memory/experiment/execution-context.ts @@ -11,6 +11,7 @@ import type { export interface OwnedAgentMemoryExecutionContext { context: AgentMemoryExecutionContext + abort(): void dispose(): void } @@ -36,6 +37,7 @@ export function createAgentMemoryExecutionContext( }) return { context: Object.freeze({ signal: signal.signal, cost }), + abort: signal.abort, dispose: signal.dispose, } } @@ -93,6 +95,7 @@ async function withRedactedAbortSignal( function relayAbortWithoutReason(source: AbortSignal): { signal: AbortSignal + abort(): void dispose(): void } { const controller = new AbortController() @@ -101,6 +104,7 @@ function relayAbortWithoutReason(source: AbortSignal): { else source.addEventListener('abort', abort, { once: true }) return { signal: controller.signal, + abort, dispose() { source.removeEventListener('abort', abort) }, diff --git a/src/memory/experiment/learning-evidence.ts b/src/memory/experiment/learning-evidence.ts new file mode 100644 index 0000000..ff6a612 --- /dev/null +++ b/src/memory/experiment/learning-evidence.ts @@ -0,0 +1,178 @@ +import { canonicalJson } from '@tangle-network/agent-eval' +import { type CampaignStorage, canonicalDigest } from '@tangle-network/agent-eval/campaign' +import { countDimensions, mean, meanDimensions } from './metrics' +import { type AgentMemoryProbeScoringEvidence, scoreAgentMemoryProbe } from './probe-evaluation' +import type { + AgentMemorySequence, + AgentMemorySequenceArtifact, + AgentMemorySequenceProbe, + AgentMemorySequenceProbeResult, + AgentMemorySequenceStep, + RunAgentMemoryExperimentResult, +} from './types' + +interface ProbeDefinition { + step: AgentMemorySequenceStep + stepOrdinal: number + probe: AgentMemorySequenceProbe +} + +export function assertStoredAgentMemoryEvidence( + storage: CampaignStorage, + result: RunAgentMemoryExperimentResult, + sequences: readonly AgentMemorySequence[], +): void { + const sequenceById = new Map(sequences.map((sequence) => [sequence.id, sequence])) + for (const cell of result.campaign.cells) { + if (cell.error || !cell.artifact) { + throw new Error( + `memory learning cell '${cell.cellId}' has no valid artifact${cell.error ? `: ${cell.error}` : ''}`, + ) + } + const sequence = sequenceById.get(cell.artifact.sequenceId) + if (!sequence) { + throw new Error( + `memory learning cell '${cell.cellId}' references unknown sequence '${cell.artifact.sequenceId}'`, + ) + } + assertStoredCellEvidence(storage, cell.cellId, cell.artifact, sequence) + } +} + +function assertStoredCellEvidence( + storage: CampaignStorage, + cellId: string, + artifact: AgentMemorySequenceArtifact, + sequence: AgentMemorySequence, +): void { + const definitions = indexProbeDefinitions(sequence) + if (artifact.probes.length !== definitions.size) { + throw new Error( + `memory learning cell '${cellId}' has ${artifact.probes.length} probes; expected ${definitions.size}`, + ) + } + const seen = new Set() + for (const probe of artifact.probes) { + const key = probeKey(probe.stepOrdinal, probe.stepId, probe.id) + const definition = definitions.get(key) + if (!definition || seen.has(key)) { + throw new Error(`memory learning cell '${cellId}' has unexpected probe '${key}'`) + } + seen.add(key) + assertStoredProbeEvidence(storage, cellId, probe, definition, sequence) + } + const expectedScore = mean(artifact.probes.map((probe) => probe.score)) + const expectedPassed = + artifact.probes.length > 0 && artifact.probes.every((probe) => probe.passed) + const expectedDimensions = meanDimensions(artifact.probes.map((probe) => probe.dimensions)) + const expectedCounts = countDimensions(artifact.probes.map((probe) => probe.applicableDimensions)) + if ( + artifact.score !== expectedScore || + artifact.passed !== expectedPassed || + canonicalJson(artifact.dimensions) !== canonicalJson(expectedDimensions) || + canonicalJson(artifact.dimensionSampleCounts) !== canonicalJson(expectedCounts) + ) { + throw new Error(`memory learning cell '${cellId}' aggregate does not match its probes`) + } +} + +function assertStoredProbeEvidence( + storage: CampaignStorage, + cellId: string, + observed: AgentMemorySequenceProbeResult, + definition: ProbeDefinition, + sequence: AgentMemorySequence, +): void { + const { probe, step } = definition + if ( + observed.query !== probe.query || + observed.retentionKey !== probe.retentionKey || + observed.transferKey !== probe.transferKey + ) { + throw new Error(`memory learning cell '${cellId}' probe '${observed.id}' changed definition`) + } + const raw = storage.read(observed.evidencePath) + if (raw === undefined) { + throw new Error(`memory learning probe evidence is missing at '${observed.evidencePath}'`) + } + let parsed: unknown + try { + parsed = JSON.parse(raw) + } catch (error) { + throw new Error(`memory learning probe evidence is malformed at '${observed.evidencePath}'`, { + cause: error, + }) + } + if (canonicalDigest(parsed) !== observed.evidenceRef) { + throw new Error( + `memory learning probe evidence hash does not match at '${observed.evidencePath}'`, + ) + } + const evidence = parseProbeEvidence(parsed, observed.evidencePath) + const expected = scoreAgentMemoryProbe(sequence, step, probe, evidence) + const observedEvaluation = { + score: observed.score, + passed: observed.passed, + dimensions: observed.dimensions, + applicableDimensions: observed.applicableDimensions, + notes: observed.notes, + } + const expectedEvaluation = { + score: expected.score, + passed: expected.passed, + dimensions: expected.dimensions, + applicableDimensions: expected.applicableDimensions ?? Object.keys(expected.dimensions), + notes: expected.notes, + } + if (canonicalJson(observedEvaluation) !== canonicalJson(expectedEvaluation)) { + throw new Error( + `memory learning probe score does not match saved evidence at '${observed.evidencePath}'`, + ) + } + if (canonicalJson(observed.hitIds) !== canonicalJson(evidence.usedMemoryIds)) { + throw new Error( + `memory learning probe hits do not match saved evidence at '${observed.evidencePath}'`, + ) + } +} + +function indexProbeDefinitions(sequence: AgentMemorySequence): Map { + const definitions = new Map() + for (const [stepOrdinal, step] of sequence.steps.entries()) { + for (const probe of step.probes ?? []) { + definitions.set(probeKey(stepOrdinal, step.id, probe.id), { step, stepOrdinal, probe }) + } + } + return definitions +} + +function probeKey(stepOrdinal: number, stepId: string, probeId: string): string { + return JSON.stringify([stepOrdinal, stepId, probeId]) +} + +function parseProbeEvidence(value: unknown, path: string): AgentMemoryProbeScoringEvidence { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new Error(`memory learning probe evidence has invalid shape at '${path}'`) + } + const record = value as Record + if ( + typeof record.answer !== 'string' || + !isStringArray(record.rememberedFacts) || + !isStringArray(record.citedEventIds) || + !isStringArray(record.usedMemoryIds) || + !isStringArray(record.actorIds) + ) { + throw new Error(`memory learning probe evidence has invalid shape at '${path}'`) + } + return { + answer: record.answer, + rememberedFacts: record.rememberedFacts, + citedEventIds: record.citedEventIds, + usedMemoryIds: record.usedMemoryIds, + actorIds: record.actorIds, + } +} + +function isStringArray(value: unknown): value is string[] { + return Array.isArray(value) && value.every((item) => typeof item === 'string') +} diff --git a/src/memory/experiment/learning-metrics.ts b/src/memory/experiment/learning-metrics.ts new file mode 100644 index 0000000..9823aca --- /dev/null +++ b/src/memory/experiment/learning-metrics.ts @@ -0,0 +1,444 @@ +import { pairedBootstrap } from '@tangle-network/agent-eval' +import type { PairedMemoryLearningCell, PairedMemoryLearningProbe } from './learning-pairs' +import { mean } from './metrics' +import type { + AgentMemoryEvidenceRef, + AgentMemoryForgettingComparison, + AgentMemoryLearningCandidateSummary, + AgentMemoryLearningCellComparison, + AgentMemoryLearningComparison, + AgentMemoryTransferCellComparison, + AgentMemoryTransferStepSummary, + RunAgentMemoryExperimentResult, +} from './types' + +export function measureAgentMemoryLearning( + paired: readonly PairedMemoryLearningCell[], + stateful: RunAgentMemoryExperimentResult, + stateless: RunAgentMemoryExperimentResult, +): AgentMemoryLearningComparison { + const cells = paired.map(toCellComparison) + const transferCells = paired.flatMap(toTransferCells) + const forgettingProbes = paired.flatMap(toForgettingComparisons) + const gainUnits = collapseLearningCells(cells) + const forgetting = summarizeForgetting(forgettingProbes, stateful.campaign.seed) + const preTreatment = measurePreTreatment(paired, stateful.campaign.seed) + const first = paired[0] + if (!first) throw new Error('cannot compare memory learning: no paired cells') + + return { + comparisonRef: stateful.comparisonRef, + evidence: { + splitRef: evidenceRef(stateful.campaign.splitDigest, 'task split'), + statefulManifestRef: evidenceRef( + stateful.campaign.manifestHash, + 'stateful campaign manifest', + ), + statelessManifestRef: evidenceRef( + stateless.campaign.manifestHash, + 'stateless campaign manifest', + ), + statefulRunDir: stateful.campaign.runDir, + statelessRunDir: stateless.campaign.runDir, + candidateRefs: stateful.candidateRefs.map((candidate) => ({ ...candidate })), + executionRef: stateful.executionRef, + }, + cells, + preTreatment, + gain: pairedBootstrap( + gainUnits.map((unit) => unit.statelessReward), + gainUnits.map((unit) => unit.statefulReward), + { statistic: 'mean', seed: stateful.campaign.seed }, + ), + gainByCandidate: summarizeCandidateGain(cells, stateful.campaign.seed), + transfer: { + definition: 'explicit-transfer-probes', + cells: transferCells, + byStep: summarizeTransfer(transferCells, stateful.campaign.seed), + }, + forgetting: { + definition: 'prior-peak-minus-final', + probes: forgettingProbes, + ...forgetting, + }, + } +} + +function measurePreTreatment( + paired: readonly PairedMemoryLearningCell[], + seed: number, +): AgentMemoryLearningComparison['preTreatment'] { + const cells = paired.flatMap((pair) => { + const probes = pair.probes.filter((probe) => probe.stateful.stepOrdinal === 0) + if (probes.length === 0) return [] + return [ + { + sequenceId: pair.stateful.artifact.sequenceId, + statefulReward: mean(probes.map((probe) => probe.stateful.score)), + statelessReward: mean(probes.map((probe) => probe.stateless.score)), + exact: probes.every((probe) => probe.stateful.score === probe.stateless.score), + }, + ] + }) + const bySequence = new Map< + string, + { + statefulRewards: number[] + statelessRewards: number[] + exact: boolean + } + >() + for (const cell of cells) { + const unit = bySequence.get(cell.sequenceId) ?? { + statefulRewards: [], + statelessRewards: [], + exact: true, + } + unit.statefulRewards.push(cell.statefulReward) + unit.statelessRewards.push(cell.statelessReward) + unit.exact &&= cell.exact + bySequence.set(cell.sequenceId, unit) + } + const units = [...bySequence.values()].map((unit) => ({ + statefulReward: mean(unit.statefulRewards), + statelessReward: mean(unit.statelessRewards), + exact: unit.exact, + })) + return { + definition: 'first-step-probes', + cells: cells.length, + n: units.length, + exactMatchRate: units.length === 0 ? null : mean(units.map((unit) => (unit.exact ? 1 : 0))), + difference: + units.length === 0 + ? null + : pairedBootstrap( + units.map((unit) => unit.statelessReward), + units.map((unit) => unit.statefulReward), + { statistic: 'mean', seed }, + ), + } +} + +function toCellComparison(pair: PairedMemoryLearningCell): AgentMemoryLearningCellComparison { + const probes = pair.probes.filter((probe) => probe.stateful.stepOrdinal > 0) + if (probes.length === 0) { + throw new Error( + `cannot compare memory learning: cell ${pair.stateful.cell.cellId} has no probe after the first step`, + ) + } + const statefulReward = mean(probes.map((probe) => probe.stateful.score)) + const statelessReward = mean(probes.map((probe) => probe.stateless.score)) + return { + cellId: pair.stateful.cell.cellId, + candidateId: pair.stateful.artifact.candidateId, + sequenceId: pair.stateful.artifact.sequenceId, + rep: pair.stateful.cell.rep, + seed: pair.stateful.cell.seed, + statefulArtifactRef: pair.stateful.artifactRef, + statelessArtifactRef: pair.stateless.artifactRef, + probeCount: probes.length, + statefulReward, + statelessReward, + gain: difference(statefulReward, statelessReward), + } +} + +function toTransferCells(pair: PairedMemoryLearningCell): AgentMemoryTransferCellComparison[] { + const groups = new Map< + string, + { stepOrdinal: number; transferKey: string; probes: PairedMemoryLearningProbe[] } + >() + for (const probe of pair.probes) { + const transferKey = probe.stateful.transferKey + if (transferKey === undefined) continue + const key = JSON.stringify([probe.stateful.stepOrdinal, transferKey]) + const group = groups.get(key) ?? { + stepOrdinal: probe.stateful.stepOrdinal, + transferKey, + probes: [], + } + group.probes.push(probe) + groups.set(key, group) + } + return [...groups.values()] + .map(({ stepOrdinal, transferKey, probes }) => { + const stepIds = sortedUnique(probes.map((probe) => probe.stateful.stepId)) + if (stepIds.length !== 1) { + throw new Error( + `cannot compare memory learning: step ordinal ${stepOrdinal} maps to multiple step ids in ${pair.stateful.cell.cellId}`, + ) + } + const statefulReward = mean(probes.map((probe) => probe.stateful.score)) + const statelessReward = mean(probes.map((probe) => probe.stateless.score)) + return { + cellId: pair.stateful.cell.cellId, + candidateId: pair.stateful.artifact.candidateId, + sequenceId: pair.stateful.artifact.sequenceId, + rep: pair.stateful.cell.rep, + seed: pair.stateful.cell.seed, + stepId: stepIds[0]!, + stepOrdinal, + transferKey, + probeCount: probes.length, + statefulArtifactRef: pair.stateful.artifactRef, + statelessArtifactRef: pair.stateless.artifactRef, + statefulReward, + statelessReward, + gain: difference(statefulReward, statelessReward), + } + }) + .sort( + (left, right) => + left.stepOrdinal - right.stepOrdinal || left.transferKey.localeCompare(right.transferKey), + ) +} + +function toForgettingComparisons( + pair: PairedMemoryLearningCell, +): AgentMemoryForgettingComparison[] { + const byRetentionKey = new Map() + for (const probe of pair.probes) { + const retentionKey = probe.stateful.retentionKey + if (retentionKey === undefined) continue + const probes = byRetentionKey.get(retentionKey) ?? [] + probes.push(probe) + byRetentionKey.set(retentionKey, probes) + } + return [...byRetentionKey.entries()] + .map(([retentionKey, observations]) => { + observations.sort((left, right) => left.stateful.stepOrdinal - right.stateful.stepOrdinal) + if (observations.length < 2) { + throw new Error( + `cannot compare memory learning: retention key ${retentionKey} in ${pair.stateful.cell.cellId} has fewer than two observations`, + ) + } + const ordinals = observations.map((probe) => probe.stateful.stepOrdinal) + if (new Set(ordinals).size !== ordinals.length) { + throw new Error( + `cannot compare memory learning: retention key ${retentionKey} in ${pair.stateful.cell.cellId} repeats within one step`, + ) + } + const final = observations[observations.length - 1]! + const prior = observations.slice(0, -1) + const statefulPeak = maxProbe(prior, (probe) => probe.stateful.score) + const statelessPeak = maxProbe(prior, (probe) => probe.stateless.score) + const statefulForgetting = difference(statefulPeak.reward, final.stateful.score) + const statelessForgetting = difference(statelessPeak.reward, final.stateless.score) + return { + cellId: pair.stateful.cell.cellId, + candidateId: pair.stateful.artifact.candidateId, + sequenceId: pair.stateful.artifact.sequenceId, + rep: pair.stateful.cell.rep, + seed: pair.stateful.cell.seed, + retentionKey, + observations: observations.length, + firstStepOrdinal: observations[0]!.stateful.stepOrdinal, + finalStepOrdinal: final.stateful.stepOrdinal, + statefulArtifactRef: pair.stateful.artifactRef, + statelessArtifactRef: pair.stateless.artifactRef, + statefulPriorPeakReward: statefulPeak.reward, + statefulPriorPeakStepOrdinal: statefulPeak.stepOrdinal, + statefulFinalReward: final.stateful.score, + statefulForgetting, + statelessPriorPeakReward: statelessPeak.reward, + statelessPriorPeakStepOrdinal: statelessPeak.stepOrdinal, + statelessFinalReward: final.stateless.score, + statelessForgetting, + excessForgetting: difference(statefulForgetting, statelessForgetting), + } + }) + .sort( + (left, right) => + left.retentionKey.localeCompare(right.retentionKey) || + left.sequenceId.localeCompare(right.sequenceId) || + left.rep - right.rep || + left.seed - right.seed, + ) +} + +function summarizeCandidateGain( + cells: readonly AgentMemoryLearningCellComparison[], + seed: number, +): AgentMemoryLearningCandidateSummary[] { + return sortedUnique(cells.map((cell) => cell.candidateId)).map((candidateId) => { + const units = collapseLearningCells(cells.filter((cell) => cell.candidateId === candidateId)) + return { + candidateId, + cells: cells.filter((cell) => cell.candidateId === candidateId).length, + gain: pairedBootstrap( + units.map((unit) => unit.statelessReward), + units.map((unit) => unit.statefulReward), + { statistic: 'mean', seed }, + ), + } + }) +} + +function summarizeTransfer( + cells: readonly AgentMemoryTransferCellComparison[], + seed: number, +): AgentMemoryTransferStepSummary[] { + const groups = new Map< + string, + { + candidateId: string + transferKey: string + stepOrdinal: number + rewardsBySequence: Map + } + >() + for (const cell of cells) { + const key = JSON.stringify([cell.candidateId, cell.transferKey, cell.stepOrdinal]) + const group = groups.get(key) ?? { + candidateId: cell.candidateId, + transferKey: cell.transferKey, + stepOrdinal: cell.stepOrdinal, + rewardsBySequence: new Map< + string, + { + stateful: number[] + stateless: number[] + } + >(), + } + const rewards = group.rewardsBySequence.get(cell.sequenceId) ?? { + stateful: [], + stateless: [], + } + rewards.stateful.push(cell.statefulReward) + rewards.stateless.push(cell.statelessReward) + group.rewardsBySequence.set(cell.sequenceId, rewards) + groups.set(key, group) + } + return [...groups.values()] + .sort( + (left, right) => + left.candidateId.localeCompare(right.candidateId) || + left.transferKey.localeCompare(right.transferKey) || + left.stepOrdinal - right.stepOrdinal, + ) + .map(({ candidateId, transferKey, stepOrdinal, rewardsBySequence }) => { + const sequenceRewards = [...rewardsBySequence.entries()] + .sort(([left], [right]) => left.localeCompare(right)) + .map(([, rewards]) => ({ + stateful: mean(rewards.stateful), + stateless: mean(rewards.stateless), + })) + return { + candidateId, + transferKey, + stepOrdinal, + gain: pairedBootstrap( + sequenceRewards.map((reward) => reward.stateless), + sequenceRewards.map((reward) => reward.stateful), + { statistic: 'mean', seed }, + ), + } + }) +} + +function collapseLearningCells( + cells: readonly AgentMemoryLearningCellComparison[], +): Array<{ sequenceId: string; statefulReward: number; statelessReward: number }> { + const bySequence = new Map() + for (const cell of cells) { + const unit = bySequence.get(cell.sequenceId) ?? { + statefulRewards: [], + statelessRewards: [], + } + unit.statefulRewards.push(cell.statefulReward) + unit.statelessRewards.push(cell.statelessReward) + bySequence.set(cell.sequenceId, unit) + } + return [...bySequence.entries()] + .sort(([left], [right]) => left.localeCompare(right)) + .map(([sequenceId, unit]) => ({ + sequenceId, + statefulReward: mean(unit.statefulRewards), + statelessReward: mean(unit.statelessRewards), + })) +} + +function summarizeForgetting( + probes: readonly AgentMemoryForgettingComparison[], + seed: number, +): { + n: number + meanStatefulForgetting: number | null + meanStatelessForgetting: number | null + meanExcessForgetting: number | null + excess: ReturnType | null +} { + const byTarget = new Map< + string, + { + stateful: number[] + stateless: number[] + excess: number[] + } + >() + for (const probe of probes) { + const key = JSON.stringify([probe.sequenceId, probe.retentionKey]) + const unit = byTarget.get(key) ?? { stateful: [], stateless: [], excess: [] } + unit.stateful.push(probe.statefulForgetting) + unit.stateless.push(probe.statelessForgetting) + unit.excess.push(probe.excessForgetting) + byTarget.set(key, unit) + } + const units = [...byTarget.values()].map((unit) => ({ + stateful: mean(unit.stateful), + stateless: mean(unit.stateless), + excess: mean(unit.excess), + })) + return { + n: units.length, + meanStatefulForgetting: optionalMean(units.map((unit) => unit.stateful)), + meanStatelessForgetting: optionalMean(units.map((unit) => unit.stateless)), + meanExcessForgetting: optionalMean(units.map((unit) => unit.excess)), + excess: + units.length === 0 + ? null + : pairedBootstrap( + units.map((unit) => unit.stateless), + units.map((unit) => unit.stateful), + { statistic: 'mean', seed }, + ), + } +} + +function maxProbe( + probes: readonly PairedMemoryLearningProbe[], + reward: (probe: PairedMemoryLearningProbe) => number, +): { reward: number; stepOrdinal: number } { + let best = probes[0]! + for (const probe of probes.slice(1)) { + if (reward(probe) > reward(best)) best = probe + } + return { reward: reward(best), stepOrdinal: best.stateful.stepOrdinal } +} + +function optionalMean(values: readonly number[]): number | null { + return values.length === 0 ? null : mean(values) +} + +function difference(left: number, right: number): number { + const value = left - right + return Object.is(value, -0) ? 0 : value +} + +function sortedUnique(values: readonly string[]): string[] { + return [...new Set(values)].sort() +} + +function evidenceRef(value: unknown, label: string): AgentMemoryEvidenceRef { + if (typeof value === 'string' && /^sha256:[a-f0-9]{64}$/.test(value)) { + return value as AgentMemoryEvidenceRef + } + if (typeof value !== 'string' || !/^[a-f0-9]{64}$/.test(value)) { + throw new Error( + `cannot compare memory learning: ${label} must be a full lowercase sha256 digest; received ${String(value)}`, + ) + } + return `sha256:${value}` +} diff --git a/src/memory/experiment/learning-pairs.ts b/src/memory/experiment/learning-pairs.ts new file mode 100644 index 0000000..645a81c --- /dev/null +++ b/src/memory/experiment/learning-pairs.ts @@ -0,0 +1,424 @@ +import { + assertCampaignSplitIdentity, + type CampaignCellResult, + canonicalDigest, +} from '@tangle-network/agent-eval/campaign' +import { stableId } from '../../ids' +import { assertImmutableRef } from '../../immutable-ref' +import { assertAgentMemoryExperimentComparisonRef } from './comparison-ref' +import { mean } from './metrics' +import type { + AgentMemoryEvidenceRef, + AgentMemorySequenceArtifact, + AgentMemorySequenceProbeResult, + CompareAgentMemoryLearningOptions, + RunAgentMemoryExperimentResult, +} from './types' + +export interface IndexedMemoryLearningCell { + cell: CampaignCellResult + artifact: AgentMemorySequenceArtifact + artifactRef: AgentMemoryEvidenceRef +} + +export interface PairedMemoryLearningProbe { + stateful: AgentMemorySequenceProbeResult + stateless: AgentMemorySequenceProbeResult +} + +export interface PairedMemoryLearningCell { + stateful: IndexedMemoryLearningCell + stateless: IndexedMemoryLearningCell + probes: PairedMemoryLearningProbe[] +} + +export function pairAgentMemoryLearningRuns( + options: CompareAgentMemoryLearningOptions, +): PairedMemoryLearningCell[] { + assertComparableDesign(options.stateful, options.stateless) + const statefulCells = indexCompleteCells('stateful', options.stateful) + const statelessCells = indexCompleteCells('stateless', options.stateless) + assertSameKeys(statefulCells, statelessCells, 'experiment cell') + + return [...statefulCells.entries()] + .map(([key, stateful]): PairedMemoryLearningCell => { + const stateless = statelessCells.get(key) + if (!stateless) { + throw new Error(`cannot compare memory learning: missing stateless cell ${key}`) + } + if (stateful.cell.cellId !== stateless.cell.cellId) { + throw new Error( + `cannot compare memory learning: cell identity differs for ${key} (${stateful.cell.cellId} vs ${stateless.cell.cellId})`, + ) + } + return { stateful, stateless, probes: pairProbes(stateful, stateless) } + }) + .sort(comparePairedCells) +} + +function assertComparableDesign( + stateful: RunAgentMemoryExperimentResult, + stateless: RunAgentMemoryExperimentResult, +): void { + if (stateful.memoryMode !== 'stateful') { + throw new Error( + `cannot compare memory learning: stateful arm is labeled ${String(stateful.memoryMode)}`, + ) + } + if (stateless.memoryMode !== 'stateless') { + throw new Error( + `cannot compare memory learning: stateless arm is labeled ${String(stateless.memoryMode)}`, + ) + } + assertAgentMemoryExperimentComparisonRef( + stateful.comparisonRef, + 'cannot compare memory learning: stateful comparisonRef', + ) + assertAgentMemoryExperimentComparisonRef( + stateless.comparisonRef, + 'cannot compare memory learning: stateless comparisonRef', + ) + if (stateful.comparisonRef !== stateless.comparisonRef) { + throw new Error( + `cannot compare memory learning: experimental conditions differ (${stateful.comparisonRef} vs ${stateless.comparisonRef})`, + ) + } + if (stateful.campaign.manifestHash === stateless.campaign.manifestHash) { + throw new Error( + 'cannot compare memory learning: campaign cache identity does not distinguish memoryMode', + ) + } + assertRunSplitIdentity('stateful', stateful) + assertRunSplitIdentity('stateless', stateless) + if (stateful.campaign.splitDigest !== stateless.campaign.splitDigest) { + throw new Error('cannot compare memory learning: stateful and stateless task identities differ') + } + if (stateful.campaign.reps !== stateless.campaign.reps) { + throw new Error( + `cannot compare memory learning: repetitions differ (${stateful.campaign.reps} vs ${stateless.campaign.reps})`, + ) + } + if (stateful.campaign.seed !== stateless.campaign.seed) { + throw new Error( + `cannot compare memory learning: campaign seeds differ (${stateful.campaign.seed} vs ${stateless.campaign.seed})`, + ) + } + if ( + JSON.stringify(stateful.candidateRefs) !== JSON.stringify(stateless.candidateRefs) || + stateful.candidateRefs.length === 0 + ) { + throw new Error('cannot compare memory learning: candidate references differ between arms') + } + const candidateIds = new Set() + for (const candidate of stateful.candidateRefs) { + assertNonEmpty(candidate.id, 'candidate reference id') + if (candidateIds.has(candidate.id)) { + throw new Error( + `cannot compare memory learning: duplicate candidate reference ${candidate.id}`, + ) + } + candidateIds.add(candidate.id) + try { + assertImmutableRef(candidate.ref, `${candidate.id}: candidate reference`) + } catch (error) { + throw new Error( + `cannot compare memory learning: ${error instanceof Error ? error.message : String(error)}`, + ) + } + } + if (stateful.executionRef !== stateless.executionRef) { + throw new Error('cannot compare memory learning: executor references differ between arms') + } + if (stateful.executionRef !== 'fixtures') { + try { + assertImmutableRef(stateful.executionRef, 'memory learning executor reference') + } catch (error) { + throw new Error( + `cannot compare memory learning: ${error instanceof Error ? error.message : String(error)}`, + ) + } + } +} + +function assertRunSplitIdentity( + arm: 'stateful' | 'stateless', + result: RunAgentMemoryExperimentResult, +): void { + try { + assertCampaignSplitIdentity( + result.campaign.scenarios, + result.campaign.reps, + result.campaign.splitDigest, + ) + } catch (error) { + throw new Error( + `cannot compare memory learning: ${arm} run has invalid task identity: ${error instanceof Error ? error.message : String(error)}`, + ) + } +} + +function indexCompleteCells( + arm: 'stateful' | 'stateless', + result: RunAgentMemoryExperimentResult, +): Map { + const expectedCellIds = new Set( + result.campaign.scenarios.flatMap((scenario) => + Array.from({ length: result.campaign.reps }, (_, rep) => `${scenario.id}:${rep}`), + ), + ) + const byIdentity = new Map() + const seenCellIds = new Set() + for (const cell of result.campaign.cells) { + if (!expectedCellIds.has(cell.cellId)) { + throw new Error( + `cannot compare memory learning: ${arm} run has unexpected cell ${cell.cellId}`, + ) + } + if (seenCellIds.has(cell.cellId)) { + throw new Error(`cannot compare memory learning: ${arm} run duplicates cell ${cell.cellId}`) + } + seenCellIds.add(cell.cellId) + if (cell.cellId !== `${cell.scenarioId}:${cell.rep}`) { + throw new Error( + `cannot compare memory learning: ${arm} cell ${cell.cellId} does not match its scenario and repetition`, + ) + } + if (cell.manifestHash !== result.campaign.manifestHash) { + throw new Error( + `cannot compare memory learning: ${arm} cell ${cell.cellId} manifest differs from its campaign`, + ) + } + if (cell.error) { + throw new Error( + `cannot compare memory learning: ${arm} cell ${cell.cellId} failed: ${cell.error}`, + ) + } + const artifact = cell.artifact as AgentMemorySequenceArtifact | null | undefined + if (!artifact) { + throw new Error(`cannot compare memory learning: ${arm} cell ${cell.cellId} has no artifact`) + } + assertArtifactIdentity(arm, result, cell, artifact) + if (!Number.isSafeInteger(cell.rep) || cell.rep < 0 || cell.rep >= result.campaign.reps) { + throw new Error(`cannot compare memory learning: ${arm} cell ${cell.cellId} has invalid rep`) + } + if (!Number.isSafeInteger(cell.seed)) { + throw new Error(`cannot compare memory learning: ${arm} cell ${cell.cellId} has invalid seed`) + } + assertReward(artifact.score, `${arm} cell ${cell.cellId} reward`) + if (artifact.probes.length === 0) { + throw new Error(`cannot compare memory learning: ${arm} cell ${cell.cellId} has no probes`) + } + for (const probe of artifact.probes) assertProbe(arm, cell.cellId, probe) + const probeMean = mean(artifact.probes.map((probe) => probe.score)) + if (Math.abs(probeMean - artifact.score) > 1e-12) { + throw new Error( + `cannot compare memory learning: ${arm} cell ${cell.cellId} reward does not equal its probe mean`, + ) + } + const identity = cellIdentity(artifact, cell.rep, cell.seed) + if (byIdentity.has(identity)) { + throw new Error(`cannot compare memory learning: ${arm} run duplicates identity ${identity}`) + } + byIdentity.set(identity, { cell, artifact, artifactRef: canonicalDigest(artifact) }) + } + const missing = [...expectedCellIds].filter((cellId) => !seenCellIds.has(cellId)) + if (missing.length > 0) { + throw new Error( + `cannot compare memory learning: ${arm} run is missing cells ${missing.sort().join(', ')}`, + ) + } + return byIdentity +} + +function assertArtifactIdentity( + arm: 'stateful' | 'stateless', + result: RunAgentMemoryExperimentResult, + cell: CampaignCellResult, + artifact: AgentMemorySequenceArtifact, +): void { + assertNonEmpty(artifact.candidateId, `${arm} cell ${cell.cellId} candidateId`) + assertNonEmpty(artifact.sequenceId, `${arm} cell ${cell.cellId} sequenceId`) + if (!result.candidateRefs.some((candidate) => candidate.id === artifact.candidateId)) { + throw new Error( + `cannot compare memory learning: ${arm} cell ${cell.cellId} uses an undeclared candidate`, + ) + } + if (artifact.memoryMode !== arm) { + throw new Error( + `cannot compare memory learning: ${arm} cell ${cell.cellId} artifact is labeled ${String(artifact.memoryMode)}`, + ) + } + if (artifact.comparisonRef !== result.comparisonRef) { + throw new Error( + `cannot compare memory learning: ${arm} cell ${cell.cellId} comparisonRef differs from its run`, + ) + } + if (!/^sha256:[a-f0-9]{64}$/.test(artifact.branchDigest)) { + throw new Error( + `cannot compare memory learning: ${arm} cell ${cell.cellId} has invalid branch evidence`, + ) + } + const expectedScenarioId = `${stableId('candidate', artifact.candidateId)}:${artifact.sequenceId}` + if (cell.scenarioId !== expectedScenarioId) { + throw new Error( + `cannot compare memory learning: ${arm} cell ${cell.cellId} artifact identity does not match its scenario`, + ) + } +} + +function pairProbes( + stateful: IndexedMemoryLearningCell, + stateless: IndexedMemoryLearningCell, +): PairedMemoryLearningProbe[] { + const statefulProbes = indexProbes('stateful', stateful) + const statelessProbes = indexProbes('stateless', stateless) + assertSameKeys(statefulProbes, statelessProbes, `probe in cell ${stateful.cell.cellId}`) + return [...statefulProbes.entries()] + .map(([key, statefulProbe]): PairedMemoryLearningProbe => { + const statelessProbe = statelessProbes.get(key) + if (!statelessProbe) { + throw new Error( + `cannot compare memory learning: missing stateless probe ${key} in ${stateful.cell.cellId}`, + ) + } + if (statefulProbe.query !== statelessProbe.query) { + throw new Error( + `cannot compare memory learning: probe query differs for ${key} in ${stateful.cell.cellId}`, + ) + } + if (statefulProbe.retentionKey !== statelessProbe.retentionKey) { + throw new Error( + `cannot compare memory learning: retention identity differs for ${key} in ${stateful.cell.cellId}`, + ) + } + if (statefulProbe.transferKey !== statelessProbe.transferKey) { + throw new Error( + `cannot compare memory learning: transfer identity differs for ${key} in ${stateful.cell.cellId}`, + ) + } + if ( + sortedUnique(statefulProbe.applicableDimensions).join('\0') !== + sortedUnique(statelessProbe.applicableDimensions).join('\0') + ) { + throw new Error( + `cannot compare memory learning: measured dimensions differ for ${key} in ${stateful.cell.cellId}`, + ) + } + return { stateful: statefulProbe, stateless: statelessProbe } + }) + .sort((left, right) => + left.stateful.stepOrdinal !== right.stateful.stepOrdinal + ? left.stateful.stepOrdinal - right.stateful.stepOrdinal + : left.stateful.id.localeCompare(right.stateful.id), + ) +} + +function indexProbes( + arm: 'stateful' | 'stateless', + indexed: IndexedMemoryLearningCell, +): Map { + const probes = new Map() + for (const probe of indexed.artifact.probes) { + const key = probeIdentity(probe) + if (probes.has(key)) { + throw new Error( + `cannot compare memory learning: ${arm} cell ${indexed.cell.cellId} duplicates probe ${key}`, + ) + } + probes.set(key, probe) + } + return probes +} + +function assertProbe( + arm: 'stateful' | 'stateless', + cellId: string, + probe: AgentMemorySequenceProbeResult, +): void { + assertNonEmpty(probe.id, `${arm} cell ${cellId} probe id`) + assertNonEmpty(probe.stepId, `${arm} cell ${cellId} probe ${probe.id} stepId`) + assertNonEmpty(probe.query, `${arm} cell ${cellId} probe ${probe.id} query`) + if (!Number.isSafeInteger(probe.stepOrdinal) || probe.stepOrdinal < 0) { + throw new Error( + `cannot compare memory learning: ${arm} cell ${cellId} probe ${probe.id} has invalid stepOrdinal`, + ) + } + if (probe.retentionKey !== undefined) { + assertNonEmpty(probe.retentionKey, `${arm} cell ${cellId} probe ${probe.id} retentionKey`) + if (probe.retentionKey !== probe.retentionKey.trim()) { + throw new Error( + `cannot compare memory learning: ${arm} cell ${cellId} probe ${probe.id} retentionKey has surrounding whitespace`, + ) + } + } + if (probe.transferKey !== undefined) { + assertNonEmpty(probe.transferKey, `${arm} cell ${cellId} probe ${probe.id} transferKey`) + if (probe.transferKey !== probe.transferKey.trim()) { + throw new Error( + `cannot compare memory learning: ${arm} cell ${cellId} probe ${probe.id} transferKey has surrounding whitespace`, + ) + } + if (probe.retentionKey !== undefined || probe.stepOrdinal === 0) { + throw new Error( + `cannot compare memory learning: ${arm} cell ${cellId} probe ${probe.id} has invalid transfer placement`, + ) + } + } + if (!/^sha256:[a-f0-9]{64}$/.test(probe.evidenceRef)) { + throw new Error( + `cannot compare memory learning: ${arm} cell ${cellId} probe ${probe.id} has invalid evidence`, + ) + } + const expectedEvidenceSuffix = `/memory-evidence/${probe.evidenceRef.slice(7)}.json` + if (!probe.evidencePath.endsWith(expectedEvidenceSuffix)) { + throw new Error( + `cannot compare memory learning: ${arm} cell ${cellId} probe ${probe.id} evidence is missing`, + ) + } + assertReward(probe.score, `${arm} cell ${cellId} probe ${probe.id} reward`) +} + +function assertReward(value: number, label: string): void { + if (!Number.isFinite(value) || value < 0 || value > 1) { + throw new Error(`cannot compare memory learning: ${label} must be finite and between 0 and 1`) + } +} + +function assertNonEmpty(value: string, label: string): void { + if (typeof value !== 'string' || value.trim().length === 0) { + throw new Error(`cannot compare memory learning: ${label} must be a non-empty string`) + } +} + +function assertSameKeys(left: Map, right: Map, label: string): void { + const missingRight = [...left.keys()].filter((key) => !right.has(key)) + const missingLeft = [...right.keys()].filter((key) => !left.has(key)) + if (missingRight.length === 0 && missingLeft.length === 0) return + throw new Error( + `cannot compare memory learning: unmatched ${label} identities; stateful-only=${missingRight.sort().join(', ') || 'none'}; stateless-only=${missingLeft.sort().join(', ') || 'none'}`, + ) +} + +function cellIdentity(artifact: AgentMemorySequenceArtifact, rep: number, seed: number): string { + return JSON.stringify([artifact.candidateId, artifact.sequenceId, rep, seed]) +} + +function probeIdentity(probe: AgentMemorySequenceProbeResult): string { + return JSON.stringify([probe.stepOrdinal, probe.stepId, probe.id]) +} + +function comparePairedCells( + left: PairedMemoryLearningCell, + right: PairedMemoryLearningCell, +): number { + return ( + left.stateful.artifact.candidateId.localeCompare(right.stateful.artifact.candidateId) || + left.stateful.artifact.sequenceId.localeCompare(right.stateful.artifact.sequenceId) || + left.stateful.cell.rep - right.stateful.cell.rep || + left.stateful.cell.seed - right.stateful.cell.seed + ) +} + +function sortedUnique(values: readonly string[]): string[] { + return [...new Set(values)].sort() +} diff --git a/src/memory/experiment/learning.ts b/src/memory/experiment/learning.ts new file mode 100644 index 0000000..b90c58e --- /dev/null +++ b/src/memory/experiment/learning.ts @@ -0,0 +1,111 @@ +import { join } from 'node:path' +import { + canonicalDigest, + createRunCostLedger, + fsCampaignStorage, + resolveRunDir, +} from '@tangle-network/agent-eval/campaign' +import { assertImmutableRef } from '../../immutable-ref' +import { assertStoredAgentMemoryEvidence } from './learning-evidence' +import { measureAgentMemoryLearning } from './learning-metrics' +import { pairAgentMemoryLearningRuns } from './learning-pairs' +import { normalizeUsd } from './metrics' +import { runAgentMemoryExperiment } from './run' +import type { + AgentMemoryLearningComparison, + CompareAgentMemoryLearningOptions, + RunAgentMemoryLearningExperimentOptions, + RunAgentMemoryLearningExperimentResult, +} from './types' +import { assertMemoryLearningSequences, assertNonEmptyString } from './validation' + +/** Compare exact matched stateful and stateless memory runs. */ +export function compareAgentMemoryLearning( + options: CompareAgentMemoryLearningOptions, +): AgentMemoryLearningComparison { + const paired = pairAgentMemoryLearningRuns(options) + return measureAgentMemoryLearning(paired, options.stateful, options.stateless) +} + +/** Run a matched stateful-versus-stateless memory experiment under one cost limit. */ +export async function runAgentMemoryLearningExperiment( + options: RunAgentMemoryLearningExperimentOptions, +): Promise { + assertNonEmptyString(options.runDir, 'memory learning experiment runDir') + options.signal?.throwIfAborted() + assertMemoryLearningSequences(options.sequences) + for (const candidate of [...options.candidates, ...(options.recoveryCandidates ?? [])]) { + assertImmutableRef(candidate.ref, `${candidate.id}: memory learning candidate ref`) + } + if (options.executeStep) { + assertImmutableRef(options.executeStepRef, 'memory learning experiment executeStepRef') + } + const storage = options.storage ?? fsCampaignStorage() + const runDir = resolveRunDir(options.runDir, options.repo) + storage.ensureDir(runDir) + const costCeiling = options.costCeiling ?? options.costLedger?.costCeilingUsd ?? 0 + const costLedger = + options.costLedger ?? createRunCostLedger({ storage, runDir, costCeilingUsd: costCeiling }) + if (costLedger.costCeilingUsd !== costCeiling) { + throw new Error( + 'memory learning experiment costCeiling must match the shared cost ledger ceiling', + ) + } + const armOrder = options.armOrder ?? 'stateful-first' + if (armOrder !== 'stateful-first' && armOrder !== 'stateless-first') { + throw new Error( + "memory learning experiment armOrder must be 'stateful-first' or 'stateless-first'", + ) + } + const { armOrder: _armOrder, ...experimentOptions } = options + const shared = { + ...experimentOptions, + experimentRunId: options.experimentRunId ?? runDir, + costCeiling, + costLedger, + } + const runArm = async (memoryMode: 'stateful' | 'stateless') => { + const result = await runAgentMemoryExperiment({ + ...shared, + memoryMode, + runDir: armRunDir(runDir, memoryMode), + }) + assertStoredAgentMemoryEvidence(storage, result, options.sequences) + return result + } + const firstMode = armOrder === 'stateful-first' ? 'stateful' : 'stateless' + const secondMode = firstMode === 'stateful' ? 'stateless' : 'stateful' + const first = await runArm(firstMode) + const second = await runArm(secondMode) + const stateful = firstMode === 'stateful' ? first : second + const stateless = firstMode === 'stateless' ? first : second + const comparison = compareAgentMemoryLearning({ stateful, stateless }) + const ledgerSummary = costLedger.summary() + const cost = { + experimentUsd: normalizeUsd(stateful.totalCostUsd + stateless.totalCostUsd), + ledgerUsd: normalizeUsd(ledgerSummary.totalCostUsd), + ceilingUsd: costLedger.costCeilingUsd, + accountingComplete: ledgerSummary.accountingComplete, + } + const report = { armOrder, comparison, cost } + const evidenceRef = canonicalDigest(report) + const comparisonDir = armRunDir(runDir, 'comparisons') + storage.ensureDir(comparisonDir) + const comparisonPath = armRunDir(comparisonDir, `${evidenceRef.slice(7)}.json`) + storage.write(comparisonPath, `${JSON.stringify({ evidenceRef, ...report }, null, 2)}\n`) + return { + armOrder, + stateful, + stateless, + comparison, + evidenceRef, + comparisonPath, + cost, + } +} + +function armRunDir(runDir: string, child: string): string { + return runDir.startsWith('mem://') + ? `${runDir.replace(/\/+$/, '')}/${child}` + : join(runDir, child) +} diff --git a/src/memory/experiment/metrics.ts b/src/memory/experiment/metrics.ts index ebf3496..35c9bd0 100644 --- a/src/memory/experiment/metrics.ts +++ b/src/memory/experiment/metrics.ts @@ -1,4 +1,5 @@ import type { CampaignResult, CostLedgerHandle } from '@tangle-network/agent-eval/campaign' +import { stableId } from '../../ids' import type { AgentMemoryExperimentCandidate, AgentMemoryExperimentRankingRow, @@ -59,6 +60,12 @@ export function memoryExperimentCostByCandidate( scenarios.map((scenario) => [scenario.id, scenario.candidateId]), ) const candidateIds = new Set(candidateIdsInput) + const sequenceIds = new Set(scenarios.map((scenario) => scenario.sequenceId)) + for (const candidateId of candidateIds) { + for (const sequenceId of sequenceIds) { + candidateByScenario.set(`${stableId('candidate', candidateId)}:${sequenceId}`, candidateId) + } + } const totals = new Map() for (const receipt of costLedger.list()) { if (receipt.tags?.runDir !== runDir) continue @@ -85,7 +92,7 @@ export function renderAgentMemoryExperimentRanking( '# Agent Memory Experiment', '', `- total cost: $${format(totalCostUsd)}`, - `- retired-candidate recovery cost: $${format(unrankedRecoveryCostUsd)}`, + `- retired-candidate cost: $${format(unrankedRecoveryCostUsd)}`, '', '| rank | candidate | sequences | cells | probes | failed | score | pass rate | cost | duration ms |', '| ---: | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |', diff --git a/src/memory/experiment/probe-evaluation.ts b/src/memory/experiment/probe-evaluation.ts new file mode 100644 index 0000000..4585058 --- /dev/null +++ b/src/memory/experiment/probe-evaluation.ts @@ -0,0 +1,53 @@ +import { + type KnowledgeBenchmarkArtifact, + type KnowledgeBenchmarkEvaluation, + type KnowledgeMemoryBenchmarkCase, + scoreMemoryBenchmarkArtifact, +} from '../../benchmarks/index' +import type { + AgentMemorySequence, + AgentMemorySequenceProbe, + AgentMemorySequenceStep, +} from './types' + +export interface AgentMemoryProbeScoringEvidence extends KnowledgeBenchmarkArtifact { + answer: string + rememberedFacts: readonly string[] + citedEventIds: readonly string[] + usedMemoryIds: readonly string[] + actorIds: readonly string[] +} + +export function scoreAgentMemoryProbe( + sequence: AgentMemorySequence, + step: AgentMemorySequenceStep, + probe: AgentMemorySequenceProbe, + evidence: AgentMemoryProbeScoringEvidence, +): KnowledgeBenchmarkEvaluation { + return scoreMemoryBenchmarkArtifact(memoryProbeBenchmarkCase(sequence, step, probe), evidence) +} + +function memoryProbeBenchmarkCase( + sequence: AgentMemorySequence, + step: AgentMemorySequenceStep, + probe: AgentMemorySequenceProbe, +): KnowledgeMemoryBenchmarkCase { + return { + id: `${sequence.id}:${step.id}:${probe.id}`, + family: sequence.family, + taskKind: probe.taskKind ?? 'memory-recall', + split: sequence.split, + events: [], + prompt: probe.query, + requiredFacts: + probe.requiredFacts && probe.requiredFacts.length > 0 + ? probe.requiredFacts + : probe.referenceAnswer + ? [{ id: `${probe.id}:reference`, anyOf: [probe.referenceAnswer] }] + : undefined, + forbiddenFacts: probe.forbiddenFacts, + expectedEventIds: probe.expectedEventIds, + expectedActorIds: probe.expectedActorIds, + referenceAnswer: probe.referenceAnswer, + } +} diff --git a/src/memory/experiment/recovery.ts b/src/memory/experiment/recovery.ts index b64ed06..9370281 100644 --- a/src/memory/experiment/recovery.ts +++ b/src/memory/experiment/recovery.ts @@ -1,5 +1,9 @@ import { canonicalJson } from '@tangle-network/agent-eval' -import type { CampaignStorage, CostLedgerHandle } from '@tangle-network/agent-eval/campaign' +import { + type CampaignStorage, + type CostLedgerHandle, + canonicalDigest, +} from '@tangle-network/agent-eval/campaign' import { stableId } from '../../ids' import { appendAttemptJournalEvent, @@ -11,9 +15,9 @@ import { } from '../attempt-log' import { type AgentMemoryBranch, createAgentMemoryBranch } from '../branch' import { + createBoundedMemoryAdapter, createMemoryExecutionPool, memoryRecoveryDelayMs, - releaseMemoryAdapterCreatedAfterAbort, resolveMemoryCleanupTimeoutMs, runBoundedMemoryLifecycle, sleepForMemoryRecovery, @@ -49,6 +53,7 @@ export async function recoverAbandonedMemoryAttempts(input: { recoveryLogPath: string maxRecoveryRetriesPerAttempt: number }): Promise { + input.options.signal?.throwIfAborted() let attempts = readActiveMemoryAttempts(input.storage, input.attemptLogPath) if (attempts.length > input.maxRecoveryAttempts) { throw new Error( @@ -63,11 +68,13 @@ export async function recoverAbandonedMemoryAttempts(input: { ) } assertMemoryAttemptCandidateMatches(attempt, candidate) - if (!input.sequenceById.has(attempt.sequenceId)) { + const sequence = input.sequenceById.get(attempt.sequenceId) + if (!sequence) { throw new Error( `cannot recover memory branch '${attempt.branchId}': sequence '${attempt.sequenceId}' is missing`, ) } + assertMemoryAttemptSequenceMatches(attempt, sequence) if ((input.options.cleanupBranches ?? true) !== attempt.cleanupBranches) { throw new Error(`cannot recover memory branch '${attempt.branchId}': cleanupBranches changed`) } @@ -83,6 +90,7 @@ export async function recoverAbandonedMemoryAttempts(input: { executionCostUsd > 0 && !hasSettledPaidCall(input.costLedger, memoryAttemptCostCallId(attempt, 'execute', 0)) ) { + await input.lease.assertOwned() appendMemoryAttemptEvent(input.storage, input.attemptLogPath, { ...attempt, status: 'cleaned', @@ -92,92 +100,100 @@ export async function recoverAbandonedMemoryAttempts(input: { } } attempts = readActiveMemoryAttempts(input.storage, input.attemptLogPath) - const recoveryGenerations = reserveRecoveryAttempts({ - storage: input.storage, - path: input.recoveryLogPath, - attemptIds: attempts.map((attempt) => attempt.branchId), - maxRetriesPerAttempt: input.maxRecoveryRetriesPerAttempt, - label: 'memory recovery attempt log', - now: input.options.now, - }) + const recoveryGenerations = new Map() + for (const attempt of attempts.sort((left, right) => + left.branchId.localeCompare(right.branchId), + )) { + await input.lease.assertOwned() + const reserved = reserveRecoveryAttempts({ + storage: input.storage, + path: input.recoveryLogPath, + attemptIds: [attempt.branchId], + maxRetriesPerAttempt: input.maxRecoveryRetriesPerAttempt, + label: 'memory recovery attempt log', + now: input.options.now, + }) + recoveryGenerations.set(attempt.branchId, reserved.get(attempt.branchId)!) + } const pool = createMemoryExecutionPool(input.maxConcurrency) const settled = await Promise.allSettled( - attempts - .sort((left, right) => left.branchId.localeCompare(right.branchId)) - .map((attempt) => - pool.run(async () => { + attempts.map((attempt) => + pool.run(async () => { + input.options.signal?.throwIfAborted() + await input.lease.assertOwned() + const candidate = input.candidateById.get(attempt.candidateId) + if (!candidate) { + throw new Error( + `cannot recover memory branch '${attempt.branchId}': candidate '${attempt.candidateId}' is missing; pass it in recoveryCandidates`, + ) + } + assertMemoryAttemptCandidateMatches(attempt, candidate) + const sequence = input.sequenceById.get(attempt.sequenceId) + if (!sequence) { + throw new Error( + `cannot recover memory branch '${attempt.branchId}': sequence '${attempt.sequenceId}' is missing`, + ) + } + assertMemoryAttemptSequenceMatches(attempt, sequence) + if ((input.options.cleanupBranches ?? true) !== attempt.cleanupBranches) { + throw new Error( + `cannot recover memory branch '${attempt.branchId}': cleanupBranches changed`, + ) + } + const recoveryCostUsd = candidate.externalRecoveryCostUsdPerAttempt ?? 0 + const recoveryGeneration = recoveryGenerations.get(attempt.branchId) + if (recoveryGeneration === undefined) { + throw new Error(`missing recovery generation for memory branch '${attempt.branchId}'`) + } + let externalRecoveryAttempted = false + const costRecorder = createAgentMemoryCostRecorder({ + candidateRef: candidate.ref, + maximumCostUsd: recoveryCostUsd, + operation: `${candidate.id}: memory recovery`, + }) + const recover = async (): Promise => { + await recoverMemoryAttempt({ + options: input.options, + candidate, + sequence, + attempt, + lease: input.lease, + onExternalCall: () => { + externalRecoveryAttempted = true + }, + recordExternalCost: (actualCostUsd) => { + externalRecoveryAttempted = true + costRecorder.record(actualCostUsd) + }, + }) await input.lease.assertOwned() - const candidate = input.candidateById.get(attempt.candidateId) - if (!candidate) { - throw new Error( - `cannot recover memory branch '${attempt.branchId}': candidate '${attempt.candidateId}' is missing; pass it in recoveryCandidates`, - ) - } - assertMemoryAttemptCandidateMatches(attempt, candidate) - const sequence = input.sequenceById.get(attempt.sequenceId) - if (!sequence) { - throw new Error( - `cannot recover memory branch '${attempt.branchId}': sequence '${attempt.sequenceId}' is missing`, - ) - } - if ((input.options.cleanupBranches ?? true) !== attempt.cleanupBranches) { - throw new Error( - `cannot recover memory branch '${attempt.branchId}': cleanupBranches changed`, - ) - } - const recoveryCostUsd = candidate.externalRecoveryCostUsdPerAttempt ?? 0 - const recoveryGeneration = recoveryGenerations.get(attempt.branchId) - if (recoveryGeneration === undefined) { - throw new Error(`missing recovery generation for memory branch '${attempt.branchId}'`) - } - let externalRecoveryAttempted = false - const costRecorder = createAgentMemoryCostRecorder({ - candidateRef: candidate.ref, - maximumCostUsd: recoveryCostUsd, - operation: `${candidate.id}: memory recovery`, + appendMemoryAttemptEvent(input.storage, input.attemptLogPath, { + ...attempt, + status: 'cleaned', + recovery: true, + recordedAt: (input.options.now ?? (() => new Date()))().toISOString(), }) - const recover = async (): Promise => { - await recoverMemoryAttempt({ - options: input.options, - candidate, - sequence, - attempt, - lease: input.lease, - onExternalCall: () => { - externalRecoveryAttempted = true - }, - recordExternalCost: (actualCostUsd) => { - externalRecoveryAttempted = true - costRecorder.record(actualCostUsd) - }, - }) - appendMemoryAttemptEvent(input.storage, input.attemptLogPath, { - ...attempt, - status: 'cleaned', - recovery: true, - recordedAt: (input.options.now ?? (() => new Date()))().toISOString(), - }) - } - if (recoveryCostUsd === 0) { - await recover() - } else { - const tags = memoryRecoveryCostTags(input.runDir, candidate.id, attempt.branchId) - const paid = await input.costLedger.runPaidCall({ - callId: memoryAttemptCostCallId(attempt, 'recovery', recoveryGeneration), - channel: 'driver', - phase: `${input.options.costPhase ?? 'memory.experiment'}.recovery`, - actor: `agent-knowledge:memory-recovery:${candidate.id}`, - model: candidate.ref, - tags, - maximumCharge: { externallyEnforcedMaximumUsd: recoveryCostUsd }, - execute: recover, - receipt: () => costRecorder.receipt(externalRecoveryAttempted), - receiptFromError: () => costRecorder.receipt(externalRecoveryAttempted), - }) - if (!paid.succeeded) throw paid.error - } - }), - ), + } + if (recoveryCostUsd === 0) { + await recover() + } else { + const tags = memoryRecoveryCostTags(input.runDir, candidate.id, attempt.branchId) + const paid = await input.costLedger.runPaidCall({ + callId: memoryAttemptCostCallId(attempt, 'recovery', recoveryGeneration), + channel: 'driver', + phase: `${input.options.costPhase ?? 'memory.experiment'}.recovery`, + actor: `agent-knowledge:memory-recovery:${candidate.id}`, + model: candidate.ref, + tags, + maximumCharge: { externallyEnforcedMaximumUsd: recoveryCostUsd }, + execute: recover, + receipt: () => costRecorder.receipt(externalRecoveryAttempted), + receiptFromError: () => costRecorder.receipt(externalRecoveryAttempted), + }) + if (!paid.succeeded) throw paid.error + } + }), + ), ) const failures = settled.flatMap((result) => result.status === 'rejected' ? [result.reason] : [], @@ -208,20 +224,20 @@ async function recoverMemoryAttempt(input: { let memory: AgentMemoryBranch | undefined let primaryError: unknown try { - const abortController = new AbortController() - const creation = Promise.resolve().then(() => - candidate.createAdapter({ - branchId: attempt.branchId, - purpose: 'recovery', - signal: abortController.signal, - maximumCostUsd: candidate.externalRecoveryCostUsdPerAttempt ?? 0, - markExternalCall: onExternalCall, - recordExternalCost, - }), - ) - releaseMemoryAdapterCreatedAfterAbort({ - creation, - signal: abortController.signal, + options.signal?.throwIfAborted() + const recovered = await createBoundedMemoryAdapter({ + operation: `${candidate.id}: recovery adapter creation`, + timeoutMs: cleanupTimeoutMs, + signal: options.signal, + create: (signal) => + candidate.createAdapter({ + branchId: attempt.branchId, + purpose: 'recovery', + signal, + maximumCostUsd: candidate.externalRecoveryCostUsdPerAttempt ?? 0, + markExternalCall: onExternalCall, + recordExternalCost, + }), dispose: candidate.disposeAdapter ? async (created) => { onExternalCall() @@ -229,12 +245,7 @@ async function recoverMemoryAttempt(input: { } : undefined, }) - const recovered = await runBoundedMemoryLifecycle({ - operation: `${candidate.id}: recovery adapter creation`, - timeoutMs: cleanupTimeoutMs, - abortController, - run: () => creation, - }) + options.signal?.throwIfAborted() await lease.assertOwned() if (recovered === null) return rawAdapter = recovered @@ -245,6 +256,7 @@ async function recoverMemoryAttempt(input: { () => lease.assertOwned(), cleanupTimeoutMs, `${candidate.id}: abandoned branch recovery visibility wait`, + options.signal, ) if (cleanupBranches) { if (!adapter.clear) { @@ -258,7 +270,7 @@ async function recoverMemoryAttempt(input: { lifetime: 'attempt', policy: candidate.policy, allowedWriteScopes: sequenceCleanupScopes(sequence), - baseScope: memoryExperimentBaseScope(options, candidate, sequence.id), + baseScope: memoryExperimentBaseScope(candidate), }) await runBoundedMemoryLifecycle({ operation: `${candidate.id}: abandoned branch cleanup`, @@ -344,6 +356,7 @@ export function memoryAttemptCostCallId( externalCostUsdPerSequence: attempt.externalCostUsdPerSequence, externalRecoveryCostUsdPerAttempt: attempt.externalRecoveryCostUsdPerAttempt, sequenceId: attempt.sequenceId, + sequenceRef: attempt.sequenceRef, rep: attempt.rep, seed: attempt.seed, }), @@ -380,6 +393,7 @@ export function memoryAttemptEvent(input: { candidateId: input.candidate.id, candidateRef: input.candidate.ref, sequenceId: input.sequence.id, + sequenceRef: canonicalDigest(input.sequence), rep: input.rep, seed: input.seed, cleanupBranches: input.cleanupBranches ?? true, @@ -435,6 +449,8 @@ function parseMemoryAttemptEvent( event.candidateRef.length > 0 && typeof event.sequenceId === 'string' && event.sequenceId.length > 0 && + typeof event.sequenceRef === 'string' && + /^sha256:[a-f0-9]{64}$/.test(event.sequenceRef) && Number.isSafeInteger(event.rep) && Number.isSafeInteger(event.seed) && typeof event.cleanupBranches === 'boolean' && @@ -457,6 +473,7 @@ function sameMemoryAttempt(left: AgentMemoryAttemptEvent, right: AgentMemoryAtte left.candidateId === right.candidateId && left.candidateRef === right.candidateRef && left.sequenceId === right.sequenceId && + left.sequenceRef === right.sequenceRef && left.rep === right.rep && left.seed === right.seed && left.cleanupBranches === right.cleanupBranches && @@ -465,6 +482,17 @@ function sameMemoryAttempt(left: AgentMemoryAttemptEvent, right: AgentMemoryAtte ) } +function assertMemoryAttemptSequenceMatches( + attempt: AgentMemoryAttemptEvent, + sequence: AgentMemorySequence, +): void { + if (canonicalDigest(sequence) !== attempt.sequenceRef) { + throw new Error( + `cannot recover memory branch '${attempt.branchId}': sequence '${attempt.sequenceId}' changed; restore the recorded sequence before cleanup`, + ) + } +} + function assertMemoryAttemptCandidateMatches( attempt: AgentMemoryAttemptEvent, candidate: AgentMemoryExperimentCandidate, diff --git a/src/memory/experiment/run.ts b/src/memory/experiment/run.ts index 4e57a2f..f3e2c93 100644 --- a/src/memory/experiment/run.ts +++ b/src/memory/experiment/run.ts @@ -10,10 +10,19 @@ import { } from '@tangle-network/agent-eval/campaign' import { stableId } from '../../ids' import { DEFAULT_MEMORY_RECOVERY_RETRIES_PER_ATTEMPT } from '../attempt-log' -import { createMemoryExecutionPool, resolveMemoryCleanupTimeoutMs } from '../lifecycle' +import { + createMemoryExecutionPool, + MEMORY_CAMPAIGN_DISPATCH_SHUTDOWN_TIMEOUT_MS, + resolveMemoryCleanupTimeoutMs, +} from '../lifecycle' import { acquireAgentMemoryRunLease } from '../run-control' import { agentMemorySequenceJudge, buildAgentMemorySequenceScenarios } from './cases' -import { runSequenceCell } from './cell' +import { AgentMemoryAdapterCapabilityError, runSequenceCell } from './cell' +import { + MEMORY_EXPERIMENT_IMPLEMENTATION_REF, + memoryExperimentComparisonRef, + resolveAgentMemoryMode, +} from './comparison-ref' import { memoryExperimentCostByCandidate, normalizeUsd, @@ -23,6 +32,8 @@ import { import { recoverAbandonedMemoryAttempts } from './recovery' import { AgentMemoryCleanupError } from './runtime' import type { + AgentMemoryExperimentComparisonRef, + AgentMemoryMode, AgentMemorySequenceArtifact, AgentMemorySequenceScenario, OwnedMemoryExperimentRunLease, @@ -31,17 +42,17 @@ import type { } from './types' import { assertMemorySequences, assertNonEmptyString, assertUnique } from './validation' -const MEMORY_EXPERIMENT_IMPLEMENTATION_REF = 'agent-knowledge:memory-experiment:v6' - /** Runs ordered, branch-isolated memory histories across candidate systems. */ export async function runAgentMemoryExperiment( options: RunAgentMemoryExperimentOptions, ): Promise { + options.signal?.throwIfAborted() assertNonEmptyString(options.experimentId, 'memory experiment experimentId') assertNonEmptyString(options.runDir, 'memory experiment runDir') if (options.experimentRunId !== undefined) { assertNonEmptyString(options.experimentRunId, 'memory experiment experimentRunId') } + resolveAgentMemoryMode(options.memoryMode) if (options.sequences.length === 0) throw new Error('memory experiment requires sequences') if (options.candidates.length === 0) throw new Error('memory experiment requires candidates') if (options.executeStep && !options.executeStepRef) { @@ -156,11 +167,14 @@ async function runOwnedAgentMemoryExperiment( runDir: string, lease: OwnedMemoryExperimentRunLease, ): Promise { + const memoryMode = resolveAgentMemoryMode(options.memoryMode) + const comparisonRef = memoryExperimentComparisonRef(options, runDir) const runIdentity = stableId( 'memory_run', canonicalJson({ experimentId: options.experimentId, experimentRunId: options.experimentRunId ?? runDir, + memoryMode, }), ) const maxConcurrency = options.maxConcurrency ?? 2 @@ -207,6 +221,7 @@ async function runOwnedAgentMemoryExperiment( maxRecoveryRetriesPerAttempt: options.maxRecoveryRetriesPerAttempt ?? DEFAULT_MEMORY_RECOVERY_RETRIES_PER_ATTEMPT, }) + options.signal?.throwIfAborted() await lease.assertOwned() let campaign: CampaignResult | undefined let campaignError: unknown @@ -214,6 +229,7 @@ async function runOwnedAgentMemoryExperiment( try { campaign = await runCampaign({ scenarios, + signal: options.signal, dispatch: (scenario, context) => { const candidate = candidateById.get(scenario.candidateId) if (!candidate) throw new Error(`unknown memory candidate ${scenario.candidateId}`) @@ -227,12 +243,14 @@ async function runOwnedAgentMemoryExperiment( storage, attemptLogPath, lease, + memoryMode, + comparisonRef, }), ) dispatchedExecutions.push(operation) return operation }, - dispatchRef: memoryExperimentDispatchRef(options), + dispatchRef: memoryExperimentDispatchRef(memoryMode, comparisonRef), judges: [agentMemorySequenceJudge()], runDir, storage, @@ -244,6 +262,7 @@ async function runOwnedAgentMemoryExperiment( costPhase: options.costPhase, maxConcurrency, dispatchTimeoutMs: options.dispatchTimeoutMs, + dispatchShutdownTimeoutMs: MEMORY_CAMPAIGN_DISPATCH_SHUTDOWN_TIMEOUT_MS, expectUsage: 'off', now: options.now, }) @@ -257,16 +276,32 @@ async function runOwnedAgentMemoryExperiment( ? [settled.reason] : [], ) - if (campaignError && cleanupFailures.length > 0) { + const capabilityFailures = settledExecutions.flatMap((settled) => + settled.status === 'rejected' && settled.reason instanceof AgentMemoryAdapterCapabilityError + ? [settled.reason] + : [], + ) + const terminalFailures = [...cleanupFailures, ...capabilityFailures] + if (campaignError && terminalFailures.length > 0) { throw new AggregateError( - [campaignError, ...cleanupFailures], - 'memory experiment failed and provider cleanup also failed', + [campaignError, ...terminalFailures], + 'memory experiment failed with terminal dispatch errors', ) } if (campaignError) throw campaignError + if (cleanupFailures.length > 0 && capabilityFailures.length > 0) { + throw new AggregateError( + terminalFailures, + 'memory experiment adapter requirements and provider cleanup failed', + ) + } if (cleanupFailures.length > 0) { throw new AggregateError(cleanupFailures, 'memory experiment cleanup failed after dispatch') } + if (capabilityFailures.length === 1) throw capabilityFailures[0] + if (capabilityFailures.length > 1) { + throw new AggregateError(capabilityFailures, 'memory experiment adapter requirements failed') + } if (!campaign) throw new Error('memory experiment produced no campaign result') await lease.assertOwned() const costByCandidate = memoryExperimentCostByCandidate( @@ -290,7 +325,16 @@ async function runOwnedAgentMemoryExperiment( storage.write( rankingJsonPath, `${JSON.stringify( - { experimentId: options.experimentId, totalCostUsd, unrankedRecoveryCostUsd, rows }, + { + experimentId: options.experimentId, + memoryMode, + comparisonRef, + candidateRefs: options.candidates.map(({ id, ref }) => ({ id, ref })), + executionRef: options.executeStepRef ?? 'fixtures', + totalCostUsd, + unrankedRecoveryCostUsd, + rows, + }, null, 2, )}\n`, @@ -300,6 +344,10 @@ async function runOwnedAgentMemoryExperiment( renderAgentMemoryExperimentRanking(rows, totalCostUsd, unrankedRecoveryCostUsd), ) return { + memoryMode, + comparisonRef, + candidateRefs: options.candidates.map(({ id, ref }) => ({ id, ref })), + executionRef: options.executeStepRef ?? 'fixtures', campaign, rows, totalCostUsd, @@ -312,26 +360,16 @@ async function runOwnedAgentMemoryExperiment( } } -function memoryExperimentDispatchRef(options: RunAgentMemoryExperimentOptions): string { +function memoryExperimentDispatchRef( + memoryMode: AgentMemoryMode, + comparisonRef: AgentMemoryExperimentComparisonRef, +): string { return stableId( 'memory_experiment', canonicalJson({ implementationRef: MEMORY_EXPERIMENT_IMPLEMENTATION_REF, - experimentId: options.experimentId, - experimentRunId: options.experimentRunId ?? null, - executeStepRef: options.executeStepRef ?? 'fixtures', - cleanupBranches: options.cleanupBranches ?? true, - candidates: options.candidates - .map((candidate) => ({ - id: candidate.id, - ref: candidate.ref, - policy: candidate.policy ?? null, - baseScope: candidate.baseScope ?? null, - externalCostUsdPerSequence: candidate.externalCostUsdPerSequence ?? 0, - externalRecoveryCostUsdPerAttempt: candidate.externalRecoveryCostUsdPerAttempt ?? 0, - externalCostAccounting: candidate.externalCostAccounting ?? 'exact', - })) - .sort((a, b) => a.id.localeCompare(b.id)), + memoryMode, + comparisonRef, }), ) } diff --git a/src/memory/experiment/runtime.ts b/src/memory/experiment/runtime.ts index 81aea45..f9f4c0c 100644 --- a/src/memory/experiment/runtime.ts +++ b/src/memory/experiment/runtime.ts @@ -1,10 +1,6 @@ import type { AgentMemoryBranch } from '../branch' import type { AgentMemoryAdapter, AgentMemoryScope } from '../types' -import type { - AgentMemoryExperimentCandidate, - AgentMemorySequence, - RunAgentMemoryExperimentOptions, -} from './types' +import type { AgentMemoryExperimentCandidate, AgentMemorySequence } from './types' import { normalizeCleanupScope } from './validation' export class AgentMemoryCleanupError extends AggregateError { @@ -69,17 +65,9 @@ export function mergeScopes(base?: AgentMemoryScope, extra?: AgentMemoryScope): } export function memoryExperimentBaseScope( - options: Pick, - candidate: Pick, - sequenceId: string, + candidate: Pick, ): AgentMemoryScope { - return mergeScopes(candidate.baseScope, { - tags: { - memoryExperimentId: options.experimentId, - memoryCandidateId: candidate.id, - memorySequenceId: sequenceId, - }, - }) + return mergeScopes(candidate.baseScope) } export function sequenceCleanupScopes(sequence: AgentMemorySequence): AgentMemoryScope[] { diff --git a/src/memory/experiment/types.ts b/src/memory/experiment/types.ts index 99a2898..dc9ac3f 100644 --- a/src/memory/experiment/types.ts +++ b/src/memory/experiment/types.ts @@ -1,4 +1,9 @@ -import type { CostChannel, CostReceipt, RunPaidCallInput } from '@tangle-network/agent-eval' +import type { + CostChannel, + CostReceipt, + PairedBootstrapResult, + RunPaidCallInput, +} from '@tangle-network/agent-eval' import type { CampaignResult, CampaignStorage, @@ -26,8 +31,22 @@ import type { } from '../run-control' import type { AgentMemoryAdapter, AgentMemoryScope, AgentMemoryWriteInput } from '../types' +export type AgentMemoryMode = 'stateful' | 'stateless' +export type AgentMemoryLearningArmOrder = 'stateful-first' | 'stateless-first' +export type AgentMemoryExperimentComparisonRef = `sha256:${string}` +export type AgentMemoryEvidenceRef = `sha256:${string}` + +export interface AgentMemoryExperimentCandidateRef { + id: string + ref: string +} + export interface AgentMemorySequenceProbe { id: string + /** Stable identity shared by exact repeated measurements of one retention target. */ + retentionKey?: string + /** Explicitly marks a later-step probe as measuring transfer from prior steps. */ + transferKey?: string query: string scope?: AgentMemoryScope limit?: number @@ -144,6 +163,9 @@ export interface AgentMemoryExperimentCandidate { export interface AgentMemorySequenceProbeResult { id: string stepId: string + stepOrdinal: number + retentionKey?: string + transferKey?: string query: string score: number passed: boolean @@ -151,11 +173,16 @@ export interface AgentMemorySequenceProbeResult { applicableDimensions: readonly string[] notes: string hitIds: readonly string[] + /** Exact scoring input, saved under the cell artifact directory. */ + evidenceRef: AgentMemoryEvidenceRef + evidencePath: string } export interface AgentMemorySequenceArtifact { candidateId: string sequenceId: string + memoryMode: AgentMemoryMode + comparisonRef: AgentMemoryExperimentComparisonRef score: number passed: boolean dimensions: Record @@ -189,10 +216,138 @@ export interface AgentMemoryExperimentRankingRow { dimensions: Record } +export interface AgentMemoryLearningCellComparison { + cellId: string + candidateId: string + sequenceId: string + rep: number + seed: number + statefulArtifactRef: AgentMemoryEvidenceRef + statelessArtifactRef: AgentMemoryEvidenceRef + /** Probes after the first step used for this contrast. */ + probeCount: number + statefulReward: number + statelessReward: number + gain: number +} + +export interface AgentMemoryLearningCandidateSummary { + candidateId: string + cells: number + /** One value per independent sequence after averaging repetitions. */ + gain: PairedBootstrapResult +} + +export interface AgentMemoryTransferCellComparison { + cellId: string + candidateId: string + sequenceId: string + rep: number + seed: number + stepId: string + stepOrdinal: number + transferKey: string + probeCount: number + statefulArtifactRef: AgentMemoryEvidenceRef + statelessArtifactRef: AgentMemoryEvidenceRef + statefulReward: number + statelessReward: number + gain: number +} + +export interface AgentMemoryTransferStepSummary { + candidateId: string + transferKey: string + stepOrdinal: number + /** Stateful minus stateless transfer reward, one pair per independent sequence. */ + gain: PairedBootstrapResult +} + +export interface AgentMemoryForgettingComparison { + cellId: string + candidateId: string + sequenceId: string + rep: number + seed: number + retentionKey: string + observations: number + firstStepOrdinal: number + finalStepOrdinal: number + statefulArtifactRef: AgentMemoryEvidenceRef + statelessArtifactRef: AgentMemoryEvidenceRef + statefulPriorPeakReward: number + statefulPriorPeakStepOrdinal: number + statefulFinalReward: number + /** Prior peak minus final reward. Negative means the final result improved. */ + statefulForgetting: number + statelessPriorPeakReward: number + statelessPriorPeakStepOrdinal: number + statelessFinalReward: number + /** Prior peak minus final reward. Negative means the final result improved. */ + statelessForgetting: number + /** Stateful forgetting minus stateless forgetting. Negative favors stateful memory. */ + excessForgetting: number +} + +export interface AgentMemoryLearningComparison { + comparisonRef: AgentMemoryExperimentComparisonRef + evidence: { + splitRef: AgentMemoryEvidenceRef + statefulManifestRef: AgentMemoryEvidenceRef + statelessManifestRef: AgentMemoryEvidenceRef + statefulRunDir: string + statelessRunDir: string + candidateRefs: readonly AgentMemoryExperimentCandidateRef[] + executionRef: string + } + cells: readonly AgentMemoryLearningCellComparison[] + preTreatment: { + definition: 'first-step-probes' + /** Raw matched cells with at least one first-step probe. */ + cells: number + /** Independent sequences after averaging candidates and repetitions. */ + n: number + /** Fraction of independent sequences whose first-step scores matched exactly in every cell. */ + exactMatchRate: number | null + /** Stateful minus stateless first-step reward. Null when no first-step probes exist. */ + difference: PairedBootstrapResult | null + } + /** + * Paired stateful minus stateless reward on probes after the first step. + * Repetitions and candidates are averaged within each sequence before bootstrapping. + */ + gain: PairedBootstrapResult + gainByCandidate: readonly AgentMemoryLearningCandidateSummary[] + /** Later-step probes explicitly labeled with transferKey. No task meaning is inferred. */ + transfer: { + definition: 'explicit-transfer-probes' + cells: readonly AgentMemoryTransferCellComparison[] + byStep: readonly AgentMemoryTransferStepSummary[] + } + forgetting: { + /** Signed prior peak minus final reward. Negative values mean improvement. */ + definition: 'prior-peak-minus-final' + probes: readonly AgentMemoryForgettingComparison[] + n: number + meanStatefulForgetting: number | null + meanStatelessForgetting: number | null + meanExcessForgetting: number | null + /** Stateful minus stateless forgetting, one pair per independent retention target. */ + excess: PairedBootstrapResult | null + } +} + +export interface CompareAgentMemoryLearningOptions { + stateful: RunAgentMemoryExperimentResult + stateless: RunAgentMemoryExperimentResult +} + export interface RunAgentMemoryExperimentOptions { experimentId: string /** Stable external branch namespace; distributed workers must use the same value. */ experimentRunId?: string + /** Stateful by default. Stateless clears every declared sequence scope between steps. */ + memoryMode?: AgentMemoryMode sequences: readonly AgentMemorySequence[] candidates: readonly AgentMemoryExperimentCandidate[] /** Retired candidates retained only so interrupted branches can be cleaned on resume. */ @@ -235,6 +390,8 @@ export interface RunAgentMemoryExperimentOptions { controllerMode?: AgentMemoryControllerMode /** Required for distributed controllers that share custom storage. */ acquireRunLease?: AgentMemoryAcquireRunLease + /** Cancels active cells; completed cells remain resumable. Cleanup still drains safely. */ + signal?: AbortSignal } export type AgentMemoryExperimentRunLease = AgentMemoryRunLease @@ -245,6 +402,7 @@ export interface AgentMemoryAttemptEvent { candidateId: string candidateRef: string sequenceId: string + sequenceRef: AgentMemoryEvidenceRef rep: number seed: number cleanupBranches: boolean @@ -255,10 +413,15 @@ export interface AgentMemoryAttemptEvent { } export interface RunAgentMemoryExperimentResult { + memoryMode: AgentMemoryMode + comparisonRef: AgentMemoryExperimentComparisonRef + candidateRefs: readonly AgentMemoryExperimentCandidateRef[] + /** Immutable executor identity, or 'fixtures' when only declared writes are used. */ + executionRef: string campaign: CampaignResult rows: readonly AgentMemoryExperimentRankingRow[] totalCostUsd: number - /** Recovery spend for retired candidates, excluded from ranking rows but included in totalCostUsd. */ + /** Spend attributed to retired recovery candidates, excluded from ranking rows. */ unrankedRecoveryCostUsd: number leaderCandidateId?: string rankingJsonPath: string @@ -267,4 +430,29 @@ export interface RunAgentMemoryExperimentResult { recoveryLogPath: string } +export type RunAgentMemoryLearningExperimentOptions = Omit< + RunAgentMemoryExperimentOptions, + 'memoryMode' +> & { + /** Counterbalance this across independent runs when provider behavior may drift over time. */ + armOrder?: AgentMemoryLearningArmOrder +} + +export interface RunAgentMemoryLearningExperimentResult { + armOrder: AgentMemoryLearningArmOrder + stateful: RunAgentMemoryExperimentResult + stateless: RunAgentMemoryExperimentResult + comparison: AgentMemoryLearningComparison + evidenceRef: AgentMemoryEvidenceRef + comparisonPath: string + cost: { + /** Receipts attributed only to these two arms, including interrupted attempts. */ + experimentUsd: number + /** All receipts in the shared ledger, including an owning workflow's other work. */ + ledgerUsd: number + ceilingUsd: number + accountingComplete: boolean + } +} + export type OwnedMemoryExperimentRunLease = OwnedAgentMemoryRunLease diff --git a/src/memory/experiment/validation.ts b/src/memory/experiment/validation.ts index 6da7924..7694822 100644 --- a/src/memory/experiment/validation.ts +++ b/src/memory/experiment/validation.ts @@ -1,6 +1,7 @@ +import { canonicalJson } from '@tangle-network/agent-eval' import type { KnowledgeMemoryFactMatcher } from '../../benchmarks/index' import type { AgentMemoryScope } from '../types' -import type { AgentMemorySequence } from './types' +import type { AgentMemorySequence, AgentMemorySequenceProbe } from './types' export function assertMemorySequences(sequences: readonly AgentMemorySequence[]): void { for (const sequence of sequences) { @@ -13,7 +14,11 @@ export function assertMemorySequences(sequences: readonly AgentMemorySequence[]) `step in sequence ${sequence.id}`, ) let probeCount = 0 - for (const step of sequence.steps) { + const retentionMeasurements = new Map< + string, + { definition: string; stepOrdinals: Set } + >() + for (const [stepOrdinal, step] of sequence.steps.entries()) { for (const write of step.writes ?? []) { assertNonEmptyString(write.text, `memory experiment write in ${sequence.id}/${step.id}`) if (write.id !== undefined) { @@ -51,6 +56,53 @@ export function assertMemorySequences(sequences: readonly AgentMemorySequence[]) `memory experiment reference answer ${sequence.id}/${step.id}/${probe.id}`, ) } + if (probe.retentionKey !== undefined) { + assertNonEmptyString( + probe.retentionKey, + `memory experiment retention key ${sequence.id}/${step.id}/${probe.id}`, + ) + if (probe.retentionKey !== probe.retentionKey.trim()) { + throw new Error( + `memory experiment retention key ${sequence.id}/${step.id}/${probe.id} must not have surrounding whitespace`, + ) + } + const definition = retentionMeasurementDefinition(step.scope, probe) + const prior = retentionMeasurements.get(probe.retentionKey) + if (prior?.stepOrdinals.has(stepOrdinal)) { + throw new Error( + `memory experiment retention key ${sequence.id}/${probe.retentionKey} may appear only once per step`, + ) + } + if (prior && prior.definition !== definition) { + throw new Error( + `memory experiment retention key ${sequence.id}/${probe.retentionKey} must repeat the exact same measurement`, + ) + } + const measurement = prior ?? { definition, stepOrdinals: new Set() } + measurement.stepOrdinals.add(stepOrdinal) + retentionMeasurements.set(probe.retentionKey, measurement) + } + if (probe.transferKey !== undefined) { + assertNonEmptyString( + probe.transferKey, + `memory experiment transfer key ${sequence.id}/${step.id}/${probe.id}`, + ) + if (probe.transferKey !== probe.transferKey.trim()) { + throw new Error( + `memory experiment transfer key ${sequence.id}/${step.id}/${probe.id} must not have surrounding whitespace`, + ) + } + if (probe.retentionKey !== undefined) { + throw new Error( + `memory experiment probe ${sequence.id}/${step.id}/${probe.id} cannot be both transfer and retention`, + ) + } + if (stepOrdinal === 0) { + throw new Error( + `memory experiment transfer probe ${sequence.id}/${step.id}/${probe.id} must run after the first step`, + ) + } + } const hasTarget = (probe.requiredFacts?.length ?? 0) > 0 || (probe.forbiddenFacts?.length ?? 0) > 0 || @@ -67,9 +119,61 @@ export function assertMemorySequences(sequences: readonly AgentMemorySequence[]) if (probeCount === 0) { throw new Error(`memory experiment sequence ${sequence.id} has no probes`) } + for (const [retentionKey, measurement] of retentionMeasurements) { + if (measurement.stepOrdinals.size < 2) { + throw new Error( + `memory experiment retention key ${sequence.id}/${retentionKey} must appear in at least two distinct steps`, + ) + } + } } } +export function assertMemoryLearningSequences(sequences: readonly AgentMemorySequence[]): void { + for (const sequence of sequences) { + if (sequence.steps.length < 2) { + throw new Error(`memory learning sequence ${sequence.id} requires at least two ordered steps`) + } + if (!sequence.steps.slice(1).some((step) => (step.probes?.length ?? 0) > 0)) { + throw new Error( + `memory learning sequence ${sequence.id} requires a probe after the first step`, + ) + } + } +} + +function retentionMeasurementDefinition( + stepScope: AgentMemoryScope | undefined, + probe: AgentMemorySequenceProbe, +): string { + return canonicalJson({ + query: probe.query, + scope: compactScope({ + ...(stepScope ?? {}), + ...(probe.scope ?? {}), + tags: { ...(stepScope?.tags ?? {}), ...(probe.scope?.tags ?? {}) }, + }), + limit: probe.limit ?? null, + taskKind: probe.taskKind ?? 'memory-recall', + requiredFacts: normalizeFactMatchers(probe.requiredFacts ?? []), + forbiddenFacts: normalizeFactMatchers(probe.forbiddenFacts ?? []), + expectedEventIds: [...(probe.expectedEventIds ?? [])], + expectedActorIds: [...(probe.expectedActorIds ?? [])], + referenceAnswer: probe.referenceAnswer ?? null, + }) +} + +function normalizeFactMatchers(matchers: readonly KnowledgeMemoryFactMatcher[]) { + return matchers.map((matcher) => ({ + id: matcher.id, + anyOf: [...matcher.anyOf], + weight: matcher.weight ?? 1, + sourceEventIds: [...(matcher.sourceEventIds ?? [])], + validAt: matcher.validAt ?? null, + obsolete: matcher.obsolete ?? false, + })) +} + export function normalizeCleanupScope(scope: AgentMemoryScope): AgentMemoryScope { const normalized = compactScope(scope) if (normalized.tags && Object.keys(normalized.tags).length === 0) { diff --git a/src/memory/improvement/evaluation.ts b/src/memory/improvement/evaluation.ts index 3803a46..338cb85 100644 --- a/src/memory/improvement/evaluation.ts +++ b/src/memory/improvement/evaluation.ts @@ -14,6 +14,7 @@ import { type AgentMemorySequenceArtifact, runAgentMemoryExperiment, } from '../experiment' +import { assertAgentMemoryExperimentComparisonRef } from '../experiment/comparison-ref' import { experimentOptions } from './candidate' import { memorySequenceFingerprint, parseMemoryConfig, serializeMemoryConfig } from './identity' import type { @@ -121,6 +122,7 @@ export async function evaluateMemoryCandidate(input: maxConcurrency: 1, costCeiling: evaluationCostLimit, costPhase: `memory.config.${input.surfaceHash}`, + signal: input.signal, }) const cell = experiment.campaign.cells[0] if (!cell || cell.error || cell.artifact.candidateId !== input.candidate.id) { @@ -366,9 +368,18 @@ function parseMemoryArtifact( throw new Error(`memory config artifact '${path}' has no artifact`) } const artifact = value as Partial + try { + assertAgentMemoryExperimentComparisonRef( + artifact.comparisonRef, + `memory config artifact '${path}' comparisonRef`, + ) + } catch { + throw new Error(`memory config artifact '${path}' is malformed`) + } if ( artifact.candidateId !== candidateId || artifact.sequenceId !== sequenceId || + artifact.memoryMode !== 'stateful' || typeof artifact.score !== 'number' || !Number.isFinite(artifact.score) || artifact.score < 0 || @@ -377,8 +388,9 @@ function parseMemoryArtifact( !isFiniteNumberRecord(artifact.dimensions) || !isNonnegativeIntegerRecord(artifact.dimensionSampleCounts) || !Array.isArray(artifact.probes) || + !artifact.probes.every(isMemoryProbeResult) || typeof artifact.branchDigest !== 'string' || - !artifact.branchDigest || + !/^sha256:[a-f0-9]{64}$/.test(artifact.branchDigest) || typeof artifact.journalEntries !== 'number' || !Number.isSafeInteger(artifact.journalEntries) || artifact.journalEntries < 0 || @@ -391,6 +403,44 @@ function parseMemoryArtifact( return artifact as AgentMemorySequenceArtifact } +function isMemoryProbeResult(value: unknown): boolean { + if (!value || typeof value !== 'object' || Array.isArray(value)) return false + const probe = value as Record + if ( + typeof probe.id !== 'string' || + !probe.id || + typeof probe.stepId !== 'string' || + !probe.stepId || + !Number.isSafeInteger(probe.stepOrdinal) || + (probe.stepOrdinal as number) < 0 || + typeof probe.query !== 'string' || + !probe.query || + typeof probe.score !== 'number' || + !Number.isFinite(probe.score) || + probe.score < 0 || + probe.score > 1 || + typeof probe.passed !== 'boolean' || + !isFiniteNumberRecord(probe.dimensions) || + !Array.isArray(probe.applicableDimensions) || + !probe.applicableDimensions.every((entry) => typeof entry === 'string' && entry.length > 0) || + typeof probe.notes !== 'string' || + !Array.isArray(probe.hitIds) || + !probe.hitIds.every((entry) => typeof entry === 'string') || + typeof probe.evidenceRef !== 'string' || + !/^sha256:[a-f0-9]{64}$/.test(probe.evidenceRef) || + typeof probe.evidencePath !== 'string' || + !probe.evidencePath.endsWith(`/memory-evidence/${probe.evidenceRef.slice(7)}.json`) + ) { + return false + } + return ( + (probe.retentionKey === undefined || + (typeof probe.retentionKey === 'string' && probe.retentionKey.length > 0)) && + (probe.transferKey === undefined || + (typeof probe.transferKey === 'string' && probe.transferKey.length > 0)) + ) +} + function isFiniteNumberRecord(value: unknown): value is Record { if (!value || typeof value !== 'object' || Array.isArray(value)) return false return Object.values(value).every((entry) => typeof entry === 'number' && Number.isFinite(entry)) diff --git a/src/memory/lifecycle.ts b/src/memory/lifecycle.ts index 1a8ab5e..f7d0a02 100644 --- a/src/memory/lifecycle.ts +++ b/src/memory/lifecycle.ts @@ -1,6 +1,8 @@ import type { AgentMemoryAdapter } from './types' export const DEFAULT_MEMORY_CLEANUP_TIMEOUT_MS = 180_000 +export const MEMORY_OPERATION_CANCELLATION_TIMEOUT_MS = 4_000 +export const MEMORY_CAMPAIGN_DISPATCH_SHUTDOWN_TIMEOUT_MS = 5_000 export class AgentMemoryLifecycleTimeoutError extends Error { constructor( @@ -46,6 +48,30 @@ export function releaseMemoryAdapterCreatedAfterAbort(input: { ) } +export async function createBoundedMemoryAdapter(input: { + operation: string + timeoutMs: number + signal?: AbortSignal + create(signal: AbortSignal): AgentMemoryAdapter | null | Promise + dispose?: (adapter: AgentMemoryAdapter) => Promise +}): Promise { + input.signal?.throwIfAborted() + const abortController = new AbortController() + const creation = Promise.resolve().then(() => input.create(abortController.signal)) + releaseMemoryAdapterCreatedAfterAbort({ + creation, + signal: abortController.signal, + dispose: input.dispose, + }) + return runBoundedMemoryLifecycle({ + operation: input.operation, + timeoutMs: input.timeoutMs, + signal: input.signal, + abortController, + run: () => creation, + }) +} + export function resolveMemoryCleanupTimeoutMs(value: number | undefined, label: string): number { const timeoutMs = value ?? DEFAULT_MEMORY_CLEANUP_TIMEOUT_MS if (!Number.isSafeInteger(timeoutMs) || timeoutMs <= 0) { @@ -61,12 +87,25 @@ export async function runBoundedMemoryLifecycle(input: { resource?: object /** Cooperatively cancel provider work before reporting a timeout. */ abortController?: AbortController + /** Stop waiting when the owning operation is cancelled. */ + signal?: AbortSignal + /** Let active work settle after cancellation before marking its resource unsafe. */ + cancellationTimeoutMs?: number run(): Promise | T }): Promise { + input.signal?.throwIfAborted() + if ( + input.cancellationTimeoutMs !== undefined && + (!Number.isSafeInteger(input.cancellationTimeoutMs) || input.cancellationTimeoutMs <= 0) + ) { + throw new Error(`${input.operation} cancellationTimeoutMs must be a positive safe integer`) + } const priorTimeout = input.resource ? timedOutResources.get(input.resource) : undefined if (priorTimeout) throw new AgentMemoryLifecycleUnsafeError(input.operation, priorTimeout) let timeout: ReturnType | undefined + let cancellationTimeout: ReturnType | undefined let timeoutError: AgentMemoryLifecycleTimeoutError | undefined + let relayAbort: (() => void) | undefined const work = Promise.resolve().then(input.run) if (input.resource) { const resource = input.resource @@ -91,13 +130,44 @@ export async function runBoundedMemoryLifecycle(input: { reject(timeoutError) }, input.timeoutMs) }) + const signal = input.signal + const cancellation = signal + ? new Promise((_, reject) => { + relayAbort = () => { + const error = memoryLifecycleAbortError(input.operation) + input.abortController?.abort(error) + if (input.cancellationTimeoutMs === undefined) { + reject(signal.reason ?? error) + return + } + cancellationTimeout = setTimeout(() => { + timeoutError = new AgentMemoryLifecycleTimeoutError( + `${input.operation} cancellation`, + input.cancellationTimeoutMs!, + ) + if (input.resource) timedOutResources.set(input.resource, timeoutError) + reject(timeoutError) + }, input.cancellationTimeoutMs) + } + if (signal.aborted) relayAbort() + else signal.addEventListener('abort', relayAbort, { once: true }) + }) + : undefined try { - return await Promise.race([work, deadline]) + return await Promise.race([work, deadline, ...(cancellation ? [cancellation] : [])]) } finally { if (timeout) clearTimeout(timeout) + if (cancellationTimeout) clearTimeout(cancellationTimeout) + if (relayAbort) signal?.removeEventListener('abort', relayAbort) } } +function memoryLifecycleAbortError(operation: string): Error { + const error = new Error(`${operation} aborted`) + error.name = 'AbortError' + return error +} + export function memoryRecoveryDelayMs(adapter: AgentMemoryAdapter): number { const isolation = adapter.branchIsolation if (isolation?.mode !== 'scoped' || isolation.processExitSafe !== false) return 0 @@ -115,16 +185,38 @@ export async function sleepForMemoryRecovery( assertOwned: () => Promise, timeoutMs = delayMs, operation = 'memory recovery visibility wait', + signal?: AbortSignal, ): Promise { if (delayMs <= 0) return if (!Number.isSafeInteger(timeoutMs) || timeoutMs <= 0) { throw new Error(`${operation} timeout must be a positive safe integer`) } - await new Promise((resolve) => setTimeout(resolve, Math.min(delayMs, timeoutMs))) + signal?.throwIfAborted() + await waitForMemoryRecoveryDelay(Math.min(delayMs, timeoutMs), signal, operation) await assertOwned() if (delayMs > timeoutMs) throw new AgentMemoryLifecycleTimeoutError(operation, timeoutMs) } +async function waitForMemoryRecoveryDelay( + delayMs: number, + signal: AbortSignal | undefined, + operation: string, +): Promise { + await new Promise((resolve, reject) => { + let timeout: ReturnType | undefined + const abort = () => { + if (timeout) clearTimeout(timeout) + reject(signal?.reason ?? memoryLifecycleAbortError(operation)) + } + timeout = setTimeout(() => { + signal?.removeEventListener('abort', abort) + resolve() + }, delayMs) + if (signal?.aborted) abort() + else signal?.addEventListener('abort', abort, { once: true }) + }) +} + export function createMemoryExecutionPool(limit: number): { run(operation: () => Promise): Promise } { diff --git a/tests/memory/experiment-learning-validation.test.ts b/tests/memory/experiment-learning-validation.test.ts new file mode 100644 index 0000000..a382c22 --- /dev/null +++ b/tests/memory/experiment-learning-validation.test.ts @@ -0,0 +1,55 @@ +import { describe, expect, it } from 'vitest' +import { controlProbe, retentionProbe, runInvalidSequence } from '../support/memory-learning' + +describe('memory learning measurement validation', () => { + it('requires a retention key to repeat across distinct steps', async () => { + await expect(runInvalidSequence([{ id: 'only', probes: [retentionProbe()] }])).rejects.toThrow( + 'must appear in at least two distinct steps', + ) + }) + + it('requires repeated retention probes to be the exact same measurement', async () => { + await expect( + runInvalidSequence([ + { id: 'first', probes: [retentionProbe()] }, + { + id: 'second', + probes: [{ ...retentionProbe(), query: 'A different query' }], + }, + ]), + ).rejects.toThrow('must repeat the exact same measurement') + }) + + it('allows at most one observation of a retention target per step', async () => { + await expect( + runInvalidSequence([ + { + id: 'first', + probes: [retentionProbe(), { ...retentionProbe(), id: 'duplicate' }], + }, + { id: 'second', probes: [retentionProbe()] }, + ]), + ).rejects.toThrow('may appear only once per step') + }) + + it('requires transfer probes to be explicit, later, and distinct from retention', async () => { + await expect( + runInvalidSequence([ + { + id: 'first', + probes: [{ ...controlProbe(), transferKey: 'transfer' }], + }, + ]), + ).rejects.toThrow('must run after the first step') + + await expect( + runInvalidSequence([ + { id: 'first', probes: [retentionProbe()] }, + { + id: 'second', + probes: [{ ...retentionProbe(), transferKey: 'transfer' }], + }, + ]), + ).rejects.toThrow('cannot be both transfer and retention') + }) +}) diff --git a/tests/memory/experiment-learning.test.ts b/tests/memory/experiment-learning.test.ts new file mode 100644 index 0000000..85fde99 --- /dev/null +++ b/tests/memory/experiment-learning.test.ts @@ -0,0 +1,506 @@ +import { + canonicalDigest, + createRunCostLedger, + inMemoryCampaignStorage, +} from '@tangle-network/agent-eval/campaign' +import { beforeAll, describe, expect, it } from 'vitest' +import { + compareAgentMemoryLearning, + type RunAgentMemoryLearningExperimentResult, + runAgentMemoryExperiment, + runAgentMemoryLearningExperiment, +} from '../../src/memory/index' +import { + continualSequences, + immutableRef, + memoryCandidate, + memoryCandidateWithoutClear, + runDirectArm, + twoStepControlSequence, +} from '../support/memory-learning' + +describe('agent memory controlled learning experiment', () => { + let result: RunAgentMemoryLearningExperimentResult + let sharedCostUsd = 0 + let executorCalls = 0 + let comparisonJson = '' + let storage: ReturnType + const observedBranchScopes = new Set() + const observedProviderTags: Record[] = [] + + beforeAll(async () => { + storage = inMemoryCampaignStorage() + const costLedger = createRunCostLedger({ + storage, + runDir: '/runs/controlled-memory-learning', + costCeilingUsd: 0.1, + }) + const candidate = memoryCandidate('memory:v1', 0.01, (scope) => { + const branchId = scope.tags?.memoryBranchId + if (branchId) observedBranchScopes.add(branchId) + observedProviderTags.push({ ...(scope.tags ?? {}) }) + }) + result = await runAgentMemoryLearningExperiment({ + experimentId: 'controlled-memory-learning', + sequences: continualSequences(), + candidates: [candidate], + runDir: '/runs/controlled-memory-learning', + storage, + controllerMode: 'process-local', + costCeiling: 0.1, + costLedger, + seed: 17, + reps: 2, + executeStepRef: immutableRef('test:shared-executor'), + async executeStep() { + executorCalls += 1 + }, + }) + sharedCostUsd = costLedger.summary().totalCostUsd + comparisonJson = storage.read(result.comparisonPath) ?? '' + }) + + it('changes only memory persistence and shares identity, execution, and spend', () => { + expect(result.stateful.memoryMode).toBe('stateful') + expect(result.stateless.memoryMode).toBe('stateless') + expect(result.armOrder).toBe('stateful-first') + expect(result.stateful.comparisonRef).toBe(result.stateless.comparisonRef) + expect(result.stateful.campaign.manifestHash).not.toBe(result.stateless.campaign.manifestHash) + expect( + result.stateful.campaign.cells.every((cell) => cell.artifact.memoryMode === 'stateful'), + ).toBe(true) + expect( + result.stateless.campaign.cells.every((cell) => cell.artifact.memoryMode === 'stateless'), + ).toBe(true) + expect(sharedCostUsd).toBeCloseTo(0.08, 12) + expect(result.stateful.totalCostUsd + result.stateless.totalCostUsd).toBeCloseTo(0.08, 12) + expect(executorCalls).toBe(16) + expect(observedBranchScopes.size).toBe(8) + for (const tags of observedProviderTags) { + expect(tags).not.toHaveProperty('memoryExperimentId') + expect(tags).not.toHaveProperty('memoryCandidateId') + expect(tags).not.toHaveProperty('memorySequenceId') + } + }) + + it('defaults a single-arm memory experiment to stateful mode', async () => { + const single = await runAgentMemoryExperiment({ + experimentId: 'default-stateful-memory', + sequences: [twoStepControlSequence()], + candidates: [memoryCandidate('memory:default-stateful')], + runDir: '/runs/default-stateful-memory', + storage: inMemoryCampaignStorage(), + controllerMode: 'process-local', + }) + + expect(single.memoryMode).toBe('stateful') + expect(single.campaign.cells[0]!.artifact.memoryMode).toBe('stateful') + }) + + it('records and executes a counterbalanced stateless-first arm order', async () => { + const storage = inMemoryCampaignStorage() + const append = storage.append!.bind(storage) + const startedPaths: string[] = [] + storage.append = (path, value, expectedBytes) => { + if (path.endsWith('/memory-attempts.jsonl') && value.includes('"status":"started"')) { + startedPaths.push(path) + } + return append(path, value, expectedBytes) + } + + const counterbalanced = await runAgentMemoryLearningExperiment({ + experimentId: 'counterbalanced-memory-learning', + runDir: '/runs/counterbalanced-memory-learning', + armOrder: 'stateless-first', + sequences: [twoStepControlSequence()], + candidates: [memoryCandidate('memory:counterbalanced')], + storage, + controllerMode: 'process-local', + }) + + expect(counterbalanced.armOrder).toBe('stateless-first') + expect(startedPaths[0]).toContain('/stateless/') + expect(startedPaths.at(-1)).toContain('/stateful/') + expect(JSON.parse(storage.read(counterbalanced.comparisonPath)!)).toMatchObject({ + armOrder: 'stateless-first', + }) + }) + + it('reports paired gain, explicit transfer, and explicit forgetting', () => { + const { comparison } = result + + expect(comparison.comparisonRef).toBe(result.stateful.comparisonRef) + expect(comparison.cells).toHaveLength(4) + expect(comparison.gain).toMatchObject({ + n: 2, + mean: 0.375, + median: 0.375, + confidence: 0.95, + resamples: 2_000, + gateEligible: false, + }) + expect(comparison.gainByCandidate).toHaveLength(1) + expect(comparison.gainByCandidate[0]).toMatchObject({ + candidateId: 'memory', + cells: 4, + gain: { n: 2, mean: 0.375, gateEligible: false }, + }) + expect(comparison.preTreatment).toMatchObject({ + definition: 'first-step-probes', + cells: 4, + n: 2, + exactMatchRate: 1, + difference: { n: 2, mean: 0 }, + }) + expect(comparison.cells.find((cell) => cell.sequenceId === 'retention')).toMatchObject({ + candidateId: 'memory', + statefulReward: 1, + statelessReward: 0.25, + gain: 0.75, + }) + + expect(comparison.transfer.definition).toBe('explicit-transfer-probes') + expect(comparison.transfer.cells).toHaveLength(2) + expect(comparison.transfer.cells.every((cell) => cell.probeCount === 1)).toBe(true) + expect(comparison.transfer.byStep).toMatchObject([ + { + candidateId: 'memory', + transferKey: 'cross-capital-recall', + stepOrdinal: 1, + gain: { n: 1, mean: 0.5, gateEligible: false }, + }, + ]) + + expect(comparison.forgetting.probes).toHaveLength(2) + expect(comparison.forgetting.probes[0]).toMatchObject({ + retentionKey: 'capital-of-france', + observations: 2, + firstStepOrdinal: 0, + finalStepOrdinal: 1, + statefulPriorPeakReward: 1, + statefulFinalReward: 1, + statefulForgetting: 0, + statelessPriorPeakReward: 1, + statelessFinalReward: 0, + statelessForgetting: 1, + excessForgetting: -1, + }) + expect(comparison.forgetting).toMatchObject({ + n: 1, + meanStatefulForgetting: 0, + meanStatelessForgetting: 1, + meanExcessForgetting: -1, + excess: { n: 1, mean: -1, gateEligible: false }, + }) + }) + + it('binds the report to exact tasks, manifests, candidates, and artifacts', () => { + expect(result.evidenceRef).toMatch(/^sha256:[a-f0-9]{64}$/) + expect(result.comparisonPath).toContain(result.evidenceRef.slice(7)) + expect(result.cost).toEqual({ + experimentUsd: 0.08, + ledgerUsd: 0.08, + ceilingUsd: 0.1, + accountingComplete: true, + }) + expect(result.comparison.evidence).toMatchObject({ + splitRef: expect.stringMatching(/^sha256:[a-f0-9]{64}$/), + statefulManifestRef: expect.stringMatching(/^sha256:[a-f0-9]{64}$/), + statelessManifestRef: expect.stringMatching(/^sha256:[a-f0-9]{64}$/), + candidateRefs: [{ id: 'memory', ref: immutableRef('memory:v1') }], + executionRef: immutableRef('test:shared-executor'), + }) + expect(result.comparison.evidence.statefulManifestRef).not.toBe( + result.comparison.evidence.statelessManifestRef, + ) + expect( + result.comparison.cells.every( + (cell) => + /^sha256:[a-f0-9]{64}$/.test(cell.statefulArtifactRef) && + /^sha256:[a-f0-9]{64}$/.test(cell.statelessArtifactRef) && + cell.statefulArtifactRef !== cell.statelessArtifactRef, + ), + ).toBe(true) + for (const arm of [result.stateful, result.stateless]) { + for (const cell of arm.campaign.cells) { + for (const probe of cell.artifact.probes) { + expect(Object.values(arm.campaign.artifactsByPath)).toContain(probe.evidencePath) + const raw = storage.read(probe.evidencePath) + expect(raw).toBeTruthy() + expect(canonicalDigest(JSON.parse(raw!))).toBe(probe.evidenceRef) + } + } + } + + const stored = JSON.parse(comparisonJson) + expect(stored).toEqual({ + evidenceRef: result.evidenceRef, + armOrder: 'stateful-first', + comparison: result.comparison, + cost: result.cost, + }) + expect( + canonicalDigest({ + armOrder: stored.armOrder, + comparison: stored.comparison, + cost: stored.cost, + }), + ).toBe(stored.evidenceRef) + }) + + it('does not infer forgetting from repeated probe ids or prompt text', () => { + expect( + result.comparison.forgetting.probes.some((probe) => probe.sequenceId === 'control'), + ).toBe(false) + }) + + it('reports first-step arm drift instead of hiding it in post-step gain', () => { + const drifted = structuredClone(result.stateless) + const artifact = drifted.campaign.cells.find( + (cell) => cell.artifact.sequenceId === 'retention', + )!.artifact + const firstProbe = artifact.probes.find((probe) => probe.stepOrdinal === 0)! + firstProbe.score = 0 + artifact.score = + artifact.probes.reduce((sum, probe) => sum + probe.score, 0) / artifact.probes.length + + const comparison = compareAgentMemoryLearning({ + stateful: result.stateful, + stateless: drifted, + }) + expect(comparison.preTreatment).toMatchObject({ + n: 2, + exactMatchRate: 0.5, + difference: { mean: 0.25 }, + }) + }) + + it('rejects swapped labels, incomplete runs, and mismatched probes', () => { + expect(() => + compareAgentMemoryLearning({ stateful: result.stateless, stateless: result.stateful }), + ).toThrow('stateful arm is labeled stateless') + + const missingCell = structuredClone(result.stateless) + missingCell.campaign.cells.pop() + expect(() => + compareAgentMemoryLearning({ stateful: result.stateful, stateless: missingCell }), + ).toThrow('stateless run is missing cells') + + const mismatchedProbe = structuredClone(result.stateless) + const artifact = mismatchedProbe.campaign.cells[0]!.artifact + artifact.probes = artifact.probes.map((probe, index) => + index === 0 ? { ...probe, id: 'different-probe' } : probe, + ) + expect(() => + compareAgentMemoryLearning({ stateful: result.stateful, stateless: mismatchedProbe }), + ).toThrow('unmatched probe in cell') + }) + + it('rejects damaged task identity and failed cells before reporting gain', () => { + const damagedIdentity = structuredClone(result.stateless) + damagedIdentity.campaign.splitDigest = `sha256:${'0'.repeat(64)}` + expect(() => + compareAgentMemoryLearning({ stateful: result.stateful, stateless: damagedIdentity }), + ).toThrow('stateless run has invalid task identity') + + const failed = structuredClone(result.stateless) + failed.campaign.cells[0]!.error = 'provider failed' + expect(() => + compareAgentMemoryLearning({ stateful: result.stateful, stateless: failed }), + ).toThrow('stateless cell') + + const damagedEvidence = structuredClone(result.stateless) + damagedEvidence.campaign.cells[0]!.artifact.branchDigest = 'sha256:not-evidence' + expect(() => + compareAgentMemoryLearning({ stateful: result.stateful, stateless: damagedEvidence }), + ).toThrow('invalid branch evidence') + }) + + it('requires scoped clear support for the stateless arm', async () => { + await expect( + runAgentMemoryExperiment({ + experimentId: 'missing-stateless-clear', + memoryMode: 'stateless', + sequences: [twoStepControlSequence()], + candidates: [memoryCandidateWithoutClear()], + runDir: '/runs/missing-stateless-clear', + storage: inMemoryCampaignStorage(), + controllerMode: 'process-local', + cleanupBranches: false, + }), + ).rejects.toThrow('stateless memoryMode requires scoped clear support') + }) + + it('rejects arms whose candidate implementation references differ', async () => { + const experimentRunId = 'unequal-memory-arms' + const [stateful, stateless] = await Promise.all([ + runDirectArm('stateful', 'memory:a', experimentRunId), + runDirectArm('stateless', 'memory:b', experimentRunId), + ]) + + expect(() => compareAgentMemoryLearning({ stateful, stateless })).toThrow( + 'experimental conditions differ', + ) + }) + + it('rejects a mutable executor reference in separately run arms', async () => { + const storage = inMemoryCampaignStorage() + const common = { + experimentId: 'mutable-executor-arms', + experimentRunId: 'mutable-executor-arms', + sequences: [twoStepControlSequence()], + candidates: [memoryCandidate('memory:mutable-executor')], + storage, + controllerMode: 'process-local' as const, + executeStepRef: 'executor:v1', + async executeStep() {}, + } + const [stateful, stateless] = await Promise.all([ + runAgentMemoryExperiment({ + ...common, + memoryMode: 'stateful', + runDir: '/runs/mutable-executor-arms/stateful', + }), + runAgentMemoryExperiment({ + ...common, + memoryMode: 'stateless', + runDir: '/runs/mutable-executor-arms/stateless', + }), + ]) + + expect(() => compareAgentMemoryLearning({ stateful, stateless })).toThrow( + 'executor reference must be lowercase sha256', + ) + }) + + it('aborts active work and resumes without replaying completed cells', async () => { + const storage = inMemoryCampaignStorage() + const runDir = '/runs/resumable-memory-learning' + const costLedger = createRunCostLedger({ storage, runDir, costCeilingUsd: 1 }) + const abortController = new AbortController() + let executeCalls = 0 + const base = { + experimentId: 'resumable-memory-learning', + experimentRunId: 'resumable-memory-learning-run', + sequences: continualSequences(), + candidates: [memoryCandidate('memory:resumable', 0.01)], + runDir, + storage, + controllerMode: 'process-local' as const, + costCeiling: 1, + costLedger, + seed: 31, + reps: 1, + maxConcurrency: 1, + executeStepRef: immutableRef('test:resumable-executor'), + async executeStep() { + executeCalls += 1 + if (executeCalls === 5) abortController.abort(new Error('stop after stateful arm')) + }, + } + + await expect( + runAgentMemoryLearningExperiment({ ...base, signal: abortController.signal }), + ).rejects.toThrow() + + const resumed = await runAgentMemoryLearningExperiment({ + ...base, + signal: new AbortController().signal, + }) + + expect(resumed.stateful.campaign.cells.every((cell) => cell.cached)).toBe(true) + expect(resumed.stateless.campaign.cells.every((cell) => !cell.error)).toBe(true) + expect(resumed.comparison.cells).toHaveLength(2) + expect(executeCalls).toBe(9) + expect(costLedger.summary()).toMatchObject({ + totalCalls: 5, + totalCostUsd: 0.05, + accountingComplete: true, + unresolvedCalls: 0, + }) + expect(resumed.cost).toEqual({ + experimentUsd: 0.05, + ledgerUsd: 0.05, + ceilingUsd: 1, + accountingComplete: true, + }) + + const evidencePath = resumed.stateful.campaign.cells[0]!.artifact.probes[0]!.evidencePath + storage.write(evidencePath, '{}\n') + await expect( + runAgentMemoryLearningExperiment({ + ...base, + signal: new AbortController().signal, + }), + ).rejects.toThrow('probe evidence hash does not match') + }) + + it('checks cached stateful evidence before starting the stateless arm', async () => { + const storage = inMemoryCampaignStorage() + const runDir = '/runs/corrupt-stateful-memory-learning' + let executeCalls = 0 + const base = { + experimentId: 'corrupt-stateful-memory-learning', + experimentRunId: runDir, + sequences: continualSequences(), + candidates: [memoryCandidate('memory:corrupt-stateful')], + storage, + controllerMode: 'process-local' as const, + seed: 41, + reps: 1, + executeStepRef: immutableRef('test:corrupt-stateful-executor'), + async executeStep() { + executeCalls += 1 + }, + } + const stateful = await runAgentMemoryExperiment({ + ...base, + memoryMode: 'stateful', + runDir: `${runDir}/stateful`, + }) + expect(executeCalls).toBe(4) + const evidencePath = stateful.campaign.cells[0]!.artifact.probes[0]!.evidencePath + storage.write(evidencePath, '{}\n') + + await expect( + runAgentMemoryLearningExperiment({ + ...base, + runDir, + }), + ).rejects.toThrow('probe evidence hash does not match') + expect(executeCalls).toBe(4) + }) + + it('recomputes cached probe scores from their saved retrieval evidence', async () => { + const storage = inMemoryCampaignStorage() + const runDir = '/runs/tampered-memory-learning-score' + const base = { + experimentId: 'tampered-memory-learning-score', + experimentRunId: runDir, + sequences: [twoStepControlSequence()], + candidates: [memoryCandidate('memory:tampered-score')], + runDir, + storage, + controllerMode: 'process-local' as const, + } + const first = await runAgentMemoryLearningExperiment(base) + const cell = first.stateful.campaign.cells[0]! + const cachePath = `${first.stateful.campaign.runDir}/${cell.cellId.replace( + /[^a-zA-Z0-9_-]/g, + '_', + )}/cached-result.json` + const cached = JSON.parse(storage.read(cachePath)!) + cached.artifact.probes[1].score = 0 + cached.artifact.score = + cached.artifact.probes.reduce( + (sum: number, probe: { score: number }) => sum + probe.score, + 0, + ) / cached.artifact.probes.length + cached.artifact.passed = false + storage.write(cachePath, JSON.stringify(cached)) + + await expect(runAgentMemoryLearningExperiment(base)).rejects.toThrow( + 'probe score does not match saved evidence', + ) + }) +}) diff --git a/tests/memory/experiment-parallel.test.ts b/tests/memory/experiment-parallel.test.ts new file mode 100644 index 0000000..689bb80 --- /dev/null +++ b/tests/memory/experiment-parallel.test.ts @@ -0,0 +1,149 @@ +import { inMemoryCampaignStorage } from '@tangle-network/agent-eval/campaign' +import { describe, expect, it } from 'vitest' +import type { + AgentMemoryAdapter, + AgentMemoryBranchSnapshot, + AgentMemoryScope, +} from '../../src/memory/index' +import { createScopedTestAdapter, runAgentMemoryExperiment } from '../support/memory' + +describe('agent memory experiment parallel operations', () => { + it('runs independent writes concurrently and keeps journal order deterministic', async () => { + let activeWrites = 0 + let maxActiveWrites = 0 + let snapshot: AgentMemoryBranchSnapshot | undefined + const adapter = createScopedTestAdapter('parallel-writes', async (_scope, text) => { + activeWrites += 1 + maxActiveWrites = Math.max(maxActiveWrites, activeWrites) + await new Promise((resolve) => setTimeout(resolve, text.includes('first') ? 10 : 1)) + activeWrites -= 1 + }) + + const result = await runAgentMemoryExperiment({ + experimentId: 'parallel-write-order', + sequences: [ + { + id: 'history', + family: 'first-party', + steps: [ + { + id: 'write', + parallelWrites: true, + parallelProbes: false, + writes: [ + { + id: 'first', + kind: 'fact', + text: 'first fact', + scope: { agentId: 'first-agent' }, + }, + { + id: 'second', + kind: 'fact', + text: 'second fact', + scope: { agentId: 'second-agent' }, + }, + ], + probes: [ + { + id: 'first', + query: 'first', + scope: { agentId: 'first-agent' }, + referenceAnswer: 'first fact', + }, + { + id: 'second', + query: 'second', + scope: { agentId: 'second-agent' }, + referenceAnswer: 'second fact', + }, + ], + }, + ], + }, + ], + candidates: [ + { + id: 'memory', + ref: 'memory:parallel', + createAdapter: () => adapter, + }, + ], + runDir: '/runs/parallel-write-order', + storage: inMemoryCampaignStorage(), + controllerMode: 'process-local', + onBranchSnapshot(input) { + snapshot = input.snapshot + }, + }) + + expect(result.rows[0]).toMatchObject({ cellsFailed: 0, scoreMean: 1 }) + expect(maxActiveWrites).toBe(2) + expect(snapshot?.journal.map((entry) => entry.sequence)).toEqual([0, 1]) + expect(snapshot?.journal.map((entry) => entry.input.id)).toEqual(['first', 'second']) + }) + + it('drains successful sibling writes before clearing a partially failed step', async () => { + let adapter: AgentMemoryAdapter | undefined + const touchedScopes: AgentMemoryScope[] = [] + adapter = createScopedTestAdapter('parallel-write-failure', async (scope, text) => { + touchedScopes.push(structuredClone(scope)) + if (text.includes('fail')) throw new Error('simulated write failure') + await new Promise((resolve) => setTimeout(resolve, 10)) + }) + + const result = await runAgentMemoryExperiment({ + experimentId: 'parallel-write-failure', + sequences: [ + { + id: 'history', + family: 'first-party', + steps: [ + { + id: 'write', + parallelWrites: true, + writes: [ + { + id: 'survivor', + kind: 'fact', + text: 'successful sibling', + scope: { agentId: 'survivor' }, + }, + { + id: 'failure', + kind: 'fact', + text: 'fail this write', + scope: { agentId: 'failure' }, + }, + ], + probes: [ + { + id: 'unused', + query: 'sibling', + scope: { agentId: 'survivor' }, + referenceAnswer: 'successful sibling', + }, + ], + }, + ], + }, + ], + candidates: [ + { + id: 'memory', + ref: 'memory:parallel-failure', + createAdapter: () => adapter!, + }, + ], + runDir: '/runs/parallel-write-failure', + storage: inMemoryCampaignStorage(), + controllerMode: 'process-local', + }) + + expect(result.rows[0]).toMatchObject({ cellsFailed: 1 }) + expect(touchedScopes).toHaveLength(2) + for (const scope of touchedScopes) { + await expect(adapter.search('sibling', { scope })).resolves.toEqual([]) + } + }) +}) diff --git a/tests/memory/experiment-privacy.test.ts b/tests/memory/experiment-privacy.test.ts index 82a5a62..2035d49 100644 --- a/tests/memory/experiment-privacy.test.ts +++ b/tests/memory/experiment-privacy.test.ts @@ -1,8 +1,47 @@ import { inMemoryCampaignStorage } from '@tangle-network/agent-eval/campaign' import { describe, expect, it } from 'vitest' +import { buildAgentMemorySequencesFromBenchmarkCases } from '../../src/memory/index' import { createScopedTestAdapter, runAgentMemoryExperiment } from '../support/memory' describe('agent memory experiment privacy', () => { + it('does not expose raw benchmark case ids in default provider scopes', async () => { + const caseId = 'FINAL_BENCHMARK_CASE_SECRET' + const scopes: unknown[] = [] + const sequences = buildAgentMemorySequencesFromBenchmarkCases([ + { + id: caseId, + family: 'first-party', + taskKind: 'memory-recall', + events: [{ id: 'fact', actorId: 'user', text: 'The answer is green.' }], + prompt: 'What is the answer?', + referenceAnswer: 'answer is green', + }, + ]) + + await runAgentMemoryExperiment({ + experimentId: 'redacted-benchmark-case', + sequences, + candidates: [ + { + id: 'memory', + ref: 'memory:redacted-case', + createAdapter: () => + createScopedTestAdapter('redacted-case', async (scope) => { + scopes.push(structuredClone(scope)) + }), + }, + ], + runDir: '/runs/redacted-benchmark-case', + storage: inMemoryCampaignStorage(), + }) + + expect(scopes).not.toHaveLength(0) + expect(JSON.stringify(scopes)).not.toContain(caseId) + expect(scopes).toEqual( + expect.arrayContaining([expect.objectContaining({ sessionId: 'benchmark-session' })]), + ) + }) + it('does not expose random seeds or repetitions to candidate adapters', async () => { const adapterInputs: unknown[] = [] await runAgentMemoryExperiment({ diff --git a/tests/memory/experiment-recovery-safety.test.ts b/tests/memory/experiment-recovery-safety.test.ts new file mode 100644 index 0000000..001d957 --- /dev/null +++ b/tests/memory/experiment-recovery-safety.test.ts @@ -0,0 +1,302 @@ +import { canonicalDigest, inMemoryCampaignStorage } from '@tangle-network/agent-eval/campaign' +import { describe, expect, it } from 'vitest' +import type { AgentMemoryAdapter } from '../../src/memory/index' +import { createScopedTestAdapter, runAgentMemoryExperiment } from '../support/memory' + +describe('agent memory experiment recovery safety', () => { + it('returns with recoverable state when a provider write never settles', async () => { + const storage = inMemoryCampaignStorage() + const runDir = '/runs/hung-provider-write' + let writeStarted!: () => void + const started = new Promise((resolve) => { + writeStarted = resolve + }) + const adapter: AgentMemoryAdapter = { + id: 'hung-provider', + branchIsolation: { mode: 'scoped' }, + async search() { + return [] + }, + async getContext(query) { + return { query, text: '', hits: [], sourceRecords: [] } + }, + async write() { + writeStarted() + return new Promise(() => undefined) + }, + async clear() {}, + } + const run = runAgentMemoryExperiment({ + experimentId: 'hung-provider-write', + sequences: [ + { + id: 'history', + family: 'first-party', + steps: [ + { + id: 'write', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact', text: 'fact' }], + probes: [{ id: 'recall', query: 'fact', referenceAnswer: 'fact' }], + }, + ], + }, + ], + candidates: [{ id: 'memory', ref: 'memory:v1', createAdapter: () => adapter }], + runDir, + storage, + dispatchTimeoutMs: 5, + cleanupTimeoutMs: 20, + }) + + await started + let timeout: ReturnType | undefined + const outcome = await Promise.race([ + run.then( + () => ({ status: 'resolved' as const }), + (error: unknown) => ({ status: 'rejected' as const, error }), + ), + new Promise<{ status: 'hung' }>((resolve) => { + timeout = setTimeout(() => resolve({ status: 'hung' }), 500) + }), + ]).finally(() => { + if (timeout) clearTimeout(timeout) + }) + + expect(outcome.status).toBe('rejected') + if (outcome.status !== 'rejected') return + expect(outcome.error).toBeInstanceOf(Error) + expect((outcome.error as Error).message).toContain('cleanup failed after dispatch') + const attempts = storage.read(`${runDir}/memory-attempts.jsonl`)!.trim().split('\n') + expect(attempts).toHaveLength(1) + expect(JSON.parse(attempts[0]!)).toMatchObject({ status: 'started', recovery: false }) + }) + + it('cancels adapter creation and disposes an execution adapter that arrives late', async () => { + const storage = inMemoryCampaignStorage() + const controller = new AbortController() + let resolveCreation!: (adapter: AgentMemoryAdapter) => void + const creation = new Promise((resolve) => { + resolveCreation = resolve + }) + let reportStarted!: () => void + const started = new Promise((resolve) => { + reportStarted = resolve + }) + let reportDisposed!: () => void + const disposed = new Promise((resolve) => { + reportDisposed = resolve + }) + let providerSignal: AbortSignal | undefined + let closeCalls = 0 + let disposeCalls = 0 + const lateAdapter = createScopedTestAdapter('late-execution-provider') + lateAdapter.close = async () => { + closeCalls += 1 + } + const run = runAgentMemoryExperiment({ + experimentId: 'late-execution-adapter', + sequences: [ + { + id: 'history', + family: 'first-party', + steps: [ + { + id: 'probe', + scope: { agentId: 'worker' }, + probes: [{ id: 'recall', query: 'fact', referenceAnswer: 'fact' }], + }, + ], + }, + ], + candidates: [ + { + id: 'memory', + ref: 'memory:v1', + createAdapter({ signal }) { + providerSignal = signal + reportStarted() + return creation + }, + async disposeAdapter(adapter) { + expect(adapter).toBe(lateAdapter) + disposeCalls += 1 + reportDisposed() + }, + }, + ], + runDir: '/runs/late-execution-adapter', + storage, + signal: controller.signal, + }) + + await started + controller.abort(new Error('caller cancellation')) + await expect(run).rejects.toThrow() + expect(providerSignal?.aborted).toBe(true) + expect((providerSignal!.reason as Error).message).toBe( + 'memory: execution adapter creation aborted', + ) + + resolveCreation(lateAdapter) + await disposed + expect({ closeCalls, disposeCalls }).toEqual({ closeCalls: 1, disposeCalls: 1 }) + }) + + it('cancels an abandoned-write visibility wait without clearing the branch', async () => { + const storage = inMemoryCampaignStorage() + const runDir = '/runs/cancelled-recovery-delay' + const sequence = { + id: 'history', + family: 'first-party' as const, + steps: [ + { + id: 'probe', + scope: { agentId: 'worker' }, + probes: [{ id: 'recall', query: 'fact', referenceAnswer: 'fact' }], + }, + ], + } + storage.write( + `${runDir}/memory-attempts.jsonl`, + `${JSON.stringify({ + status: 'started', + branchId: 'unfinished-branch', + candidateId: 'memory', + candidateRef: 'memory:v1', + sequenceId: sequence.id, + sequenceRef: canonicalDigest(sequence), + rep: 0, + seed: 1, + cleanupBranches: true, + externalCostUsdPerSequence: 0, + externalRecoveryCostUsdPerAttempt: 0, + recordedAt: '2026-01-01T00:00:00.000Z', + recovery: false, + })}\n`, + ) + const controller = new AbortController() + let recoveryStarted!: () => void + const started = new Promise((resolve) => { + recoveryStarted = resolve + }) + let clearCalls = 0 + let closeCalls = 0 + const run = runAgentMemoryExperiment({ + experimentId: 'cancelled-recovery-delay', + sequences: [sequence], + candidates: [ + { + id: 'memory', + ref: 'memory:v1', + createAdapter({ purpose }) { + if (purpose === 'recovery') recoveryStarted() + return { + id: 'delayed-provider', + branchIsolation: { + mode: 'scoped', + processExitSafe: false, + recoveryDelayMs: 1_000, + }, + async search() { + return [] + }, + async getContext(query) { + return { query, text: '', hits: [], sourceRecords: [] } + }, + async write(input) { + return { + accepted: true, + id: 'write', + uri: 'memory://delayed-provider/write', + kind: input.kind, + } + }, + async clear() { + clearCalls += 1 + }, + async close() { + closeCalls += 1 + }, + } + }, + }, + ], + runDir, + storage, + signal: controller.signal, + }) + + await started + controller.abort(new Error('caller cancellation')) + await expect(run).rejects.toThrow('recovery failed') + expect({ clearCalls, closeCalls }).toEqual({ clearCalls: 0, closeCalls: 1 }) + expect(storage.read(`${runDir}/memory-attempts.jsonl`)!.trim().split('\n')).toHaveLength(1) + }) + + it('refuses to clean an unfinished branch with changed sequence scopes', async () => { + const storage = inMemoryCampaignStorage() + const runDir = '/runs/changed-recovery-sequence' + const recordedSequence = { + id: 'history', + family: 'first-party' as const, + steps: [ + { + id: 'probe', + scope: { agentId: 'original-worker' }, + probes: [{ id: 'recall', query: 'fact', referenceAnswer: 'fact' }], + }, + ], + } + storage.write( + `${runDir}/memory-attempts.jsonl`, + `${JSON.stringify({ + status: 'started', + branchId: 'unfinished-branch', + candidateId: 'memory', + candidateRef: 'memory:v1', + sequenceId: recordedSequence.id, + sequenceRef: canonicalDigest(recordedSequence), + rep: 0, + seed: 1, + cleanupBranches: true, + externalCostUsdPerSequence: 0, + externalRecoveryCostUsdPerAttempt: 0, + recordedAt: '2026-01-01T00:00:00.000Z', + recovery: false, + })}\n`, + ) + let adapterCreates = 0 + + await expect( + runAgentMemoryExperiment({ + experimentId: 'changed-recovery-sequence', + sequences: [ + { + ...recordedSequence, + steps: [ + { + ...recordedSequence.steps[0], + scope: { agentId: 'different-worker' }, + }, + ], + }, + ], + candidates: [ + { + id: 'memory', + ref: 'memory:v1', + createAdapter() { + adapterCreates += 1 + return createScopedTestAdapter('memory') + }, + }, + ], + runDir, + storage, + }), + ).rejects.toThrow("sequence 'history' changed") + + expect(adapterCreates).toBe(0) + }) +}) diff --git a/tests/memory/experiment-recovery.test.ts b/tests/memory/experiment-recovery.test.ts index 309db17..4e7dfd8 100644 --- a/tests/memory/experiment-recovery.test.ts +++ b/tests/memory/experiment-recovery.test.ts @@ -1,6 +1,14 @@ -import { createRunCostLedger, inMemoryCampaignStorage } from '@tangle-network/agent-eval/campaign' +import { + canonicalDigest, + createRunCostLedger, + inMemoryCampaignStorage, +} from '@tangle-network/agent-eval/campaign' import { describe, expect, it } from 'vitest' -import type { AgentMemoryAdapter, AgentMemoryHit } from '../../src/memory/index' +import { + type AgentMemoryAdapter, + type AgentMemoryHit, + buildAgentMemorySequenceScenarios, +} from '../../src/memory/index' import { createScopedTestAdapter, hitText, runAgentMemoryExperiment } from '../support/memory' describe('agent memory experiment recovery', () => { @@ -117,6 +125,18 @@ describe('agent memory experiment recovery', () => { it('closes and disposes a recovery adapter that arrives after its factory timeout', async () => { const storage = inMemoryCampaignStorage() const runDir = '/runs/late-recovery-adapter' + const sequence = { + id: 'history', + family: 'first-party' as const, + steps: [ + { + id: 'remember', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact' as const, text: 'fact' }], + probes: [{ id: 'recall', query: 'fact', referenceAnswer: 'fact' }], + }, + ], + } storage.write( `${runDir}/memory-attempts.jsonl`, `${JSON.stringify({ @@ -125,6 +145,7 @@ describe('agent memory experiment recovery', () => { candidateId: 'memory', candidateRef: 'memory:v1', sequenceId: 'history', + sequenceRef: canonicalDigest(sequence), rep: 0, seed: 42, cleanupBranches: true, @@ -134,18 +155,6 @@ describe('agent memory experiment recovery', () => { recovery: false, })}\n`, ) - const sequence = { - id: 'history', - family: 'first-party' as const, - steps: [ - { - id: 'remember', - scope: { agentId: 'worker' }, - writes: [{ kind: 'fact' as const, text: 'fact' }], - probes: [{ id: 'recall', query: 'fact', referenceAnswer: 'fact' }], - }, - ], - } let resolveCreation!: (adapter: AgentMemoryAdapter) => void const creation = new Promise((resolve) => { resolveCreation = resolve @@ -436,6 +445,7 @@ describe('agent memory experiment recovery', () => { candidateId: 'retired', candidateRef: 'retired:v1', sequenceId: sequence.id, + sequenceRef: canonicalDigest(sequence), rep: 0, seed: 1, cleanupBranches: true, @@ -445,6 +455,31 @@ describe('agent memory experiment recovery', () => { recovery: false, })}\n`, ) + const costLedger = createRunCostLedger({ storage, runDir, costCeilingUsd: 1 }) + const retiredScenarioId = buildAgentMemorySequenceScenarios([sequence], [{ id: 'retired' }])[0]! + .id + await costLedger.runPaidCall({ + callId: 'retired-interrupted-execution', + channel: 'agent', + phase: 'memory.experiment', + actor: 'retired-worker', + model: 'retired-worker', + tags: { + runDir, + scenarioId: retiredScenarioId, + cellId: `${retiredScenarioId}:0`, + rep: '0', + runAttemptId: 'retired-attempt', + }, + maximumCharge: { externallyEnforcedMaximumUsd: 0.2 }, + execute: async () => undefined, + receipt: () => ({ + model: 'retired-worker', + inputTokens: 0, + outputTokens: 0, + actualCostUsd: 0.2, + }), + }) const purposes: string[] = [] let retiredClears = 0 const result = await runAgentMemoryExperiment({ @@ -479,6 +514,7 @@ describe('agent memory experiment recovery', () => { ], runDir, storage, + costLedger, resumable: false, costCeiling: 1, }) @@ -486,7 +522,9 @@ describe('agent memory experiment recovery', () => { expect(result.rows).toHaveLength(1) expect(result.rows[0]?.candidateId).toBe('active') expect(result.rows[0]?.totalCostUsd).toBe(0) - expect(result).toMatchObject({ totalCostUsd: 0.1, unrankedRecoveryCostUsd: 0.1 }) + expect(result).toMatchObject({ totalCostUsd: 0.3, unrankedRecoveryCostUsd: 0.3 }) + expect(costLedger.summary().totalCostUsd).toBeCloseTo(0.3, 12) + expect(costLedger.summary().accountingComplete).toBe(true) expect(purposes).toEqual(['retired:recovery', 'active:execute']) expect(retiredClears).toBeGreaterThan(0) }) @@ -507,6 +545,7 @@ describe('agent memory experiment recovery', () => { candidateId: 'memory', candidateRef: 'memory:v1', sequenceId: sequence.id, + sequenceRef: canonicalDigest(sequence), rep: 0, seed: 1, cleanupBranches: true, @@ -567,6 +606,7 @@ describe('agent memory experiment recovery', () => { candidateId: 'memory', candidateRef: 'memory:v1', sequenceId: sequence.id, + sequenceRef: canonicalDigest(sequence), rep: 0, seed: 1, cleanupBranches: true, @@ -581,6 +621,7 @@ describe('agent memory experiment recovery', () => { candidateId: 'memory', candidateRef: 'memory:v1', sequenceId: sequence.id, + sequenceRef: canonicalDigest(sequence), rep: 0, seed: 2, cleanupBranches: true, @@ -633,6 +674,7 @@ describe('agent memory experiment recovery', () => { candidateId: 'memory', candidateRef: 'memory:v1', sequenceId: sequence.id, + sequenceRef: canonicalDigest(sequence), rep: 0, seed: 1, cleanupBranches: true, diff --git a/tests/memory/experiment-safety.test.ts b/tests/memory/experiment-safety.test.ts index 1283ec1..d19c0fe 100644 --- a/tests/memory/experiment-safety.test.ts +++ b/tests/memory/experiment-safety.test.ts @@ -190,6 +190,54 @@ describe('agent memory experiment safety', () => { expect({ purposes: purposes.length, searches, clears, closes }).toEqual(callsBeforeCachedRun) }) + it('does not mark an attempt clean after ownership expires during adapter close', async () => { + const storage = inMemoryCampaignStorage() + const runDir = '/runs/ownership-lost-during-close' + let owned = true + const adapter = createScopedTestAdapter('ownership-lost-during-close') + adapter.close = async () => { + owned = false + } + + await expect( + runAgentMemoryExperiment({ + experimentId: 'ownership-lost-during-close', + sequences: [ + { + id: 'history', + family: 'first-party', + steps: [ + { + id: 'probe', + scope: { agentId: 'worker' }, + probes: [{ id: 'recall', query: 'fact', referenceAnswer: 'fact' }], + }, + ], + }, + ], + candidates: [ + { + id: 'memory', + ref: 'memory:v1', + createAdapter: () => adapter, + }, + ], + runDir, + storage, + acquireRunLease: async () => ({ + assertOwned() { + if (!owned) throw new Error('controller ownership expired during close') + }, + release() {}, + }), + }), + ).rejects.toThrow('controller ownership expired during close') + + const attempts = storage.read(`${runDir}/memory-attempts.jsonl`)!.trim().split('\n') + expect(attempts).toHaveLength(1) + expect(JSON.parse(attempts[0]!)).toMatchObject({ status: 'started' }) + }) + it('preserves both the run failure and controller release failure', async () => { const error = await runAgentMemoryExperiment({ experimentId: 'run-and-release-failure', diff --git a/tests/support/memory-learning.ts b/tests/support/memory-learning.ts new file mode 100644 index 0000000..28252bc --- /dev/null +++ b/tests/support/memory-learning.ts @@ -0,0 +1,143 @@ +import { createHash } from 'node:crypto' +import { inMemoryCampaignStorage } from '@tangle-network/agent-eval/campaign' +import { + type AgentMemoryExperimentCandidate, + type AgentMemoryMode, + type AgentMemoryScope, + type AgentMemorySequence, + runAgentMemoryExperiment, +} from '../../src/memory/index' +import { createScopedTestAdapter } from './memory' + +export function continualSequences(): AgentMemorySequence[] { + return [retentionSequence(), twoStepControlSequence()] +} + +export function retentionSequence(): AgentMemorySequence { + return { + id: 'retention', + family: 'first-party', + steps: [ + { + id: 'learn', + scope: { agentId: 'worker' }, + writes: [{ id: 'france', kind: 'fact', text: 'The capital of France is Paris.' }], + probes: [retentionProbe()], + }, + { + id: 'interference', + scope: { agentId: 'worker' }, + writes: [{ id: 'germany', kind: 'fact', text: 'The capital of Germany is Berlin.' }], + probes: [ + retentionProbe(), + { + id: 'both-capitals', + transferKey: 'cross-capital-recall', + query: 'Recall both learned capitals.', + requiredFacts: [ + { id: 'france', anyOf: ['capital of France is Paris'] }, + { id: 'germany', anyOf: ['capital of Germany is Berlin'] }, + ], + }, + ], + }, + ], + } +} + +export function twoStepControlSequence(): AgentMemorySequence { + return { + id: 'control', + family: 'first-party', + steps: [ + { + id: 'first', + scope: { agentId: 'worker' }, + writes: [{ id: 'green-1', kind: 'fact', text: 'The control value is green.' }], + probes: [controlProbe()], + }, + { + id: 'second', + scope: { agentId: 'worker' }, + writes: [{ id: 'green-2', kind: 'fact', text: 'The control value is green.' }], + probes: [controlProbe()], + }, + ], + } +} + +export function retentionProbe() { + return { + id: 'capital', + retentionKey: 'capital-of-france', + query: 'What is the capital of France?', + referenceAnswer: 'capital of France is Paris', + } +} + +export function controlProbe() { + return { + id: 'same-local-id', + query: 'What is the control value?', + referenceAnswer: 'control value is green', + } +} + +export function memoryCandidate( + ref: string, + costUsd = 0, + onWriteScope?: (scope: AgentMemoryScope) => void, +): AgentMemoryExperimentCandidate { + return { + id: 'memory', + ref: immutableRef(ref), + externalCostUsdPerSequence: costUsd, + externalRecoveryCostUsdPerAttempt: 0, + externalCostAccounting: 'exact', + createAdapter({ branchId, recordExternalCost }) { + if (costUsd > 0) recordExternalCost(costUsd) + return createScopedTestAdapter(`${ref}:${branchId}`, async (scope) => onWriteScope?.(scope)) + }, + } +} + +export function immutableRef(value: string): string { + return `sha256:${createHash('sha256').update(value).digest('hex')}` +} + +export function memoryCandidateWithoutClear(): AgentMemoryExperimentCandidate { + return { + id: 'memory', + ref: 'memory:no-clear', + createAdapter({ branchId }) { + const { clear: _clear, ...adapter } = createScopedTestAdapter(branchId) + return adapter + }, + async disposeAdapter() {}, + } +} + +export function runDirectArm(mode: AgentMemoryMode, ref: string, experimentRunId: string) { + return runAgentMemoryExperiment({ + experimentId: 'unequal-memory-arms', + experimentRunId, + memoryMode: mode, + sequences: [twoStepControlSequence()], + candidates: [memoryCandidate(ref)], + runDir: `/runs/unequal-memory-arms/${mode}`, + storage: inMemoryCampaignStorage(), + controllerMode: 'process-local', + seed: 23, + }) +} + +export function runInvalidSequence(steps: AgentMemorySequence['steps']) { + return runAgentMemoryExperiment({ + experimentId: 'invalid-retention-measurement', + sequences: [{ id: 'invalid', family: 'first-party', steps }], + candidates: [memoryCandidate('memory:validation')], + runDir: '/runs/invalid-retention-measurement', + storage: inMemoryCampaignStorage(), + controllerMode: 'process-local', + }) +} From 077017c22171931ca1096f53d8686236e0124258 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Wed, 29 Jul 2026 18:36:26 -0600 Subject: [PATCH 2/2] refactor(memory): reuse paired-arm comparison --- src/memory/experiment/learning-metrics.ts | 110 +++++++++++++--------- src/memory/experiment/learning-pairs.ts | 49 ++++++++-- 2 files changed, 109 insertions(+), 50 deletions(-) diff --git a/src/memory/experiment/learning-metrics.ts b/src/memory/experiment/learning-metrics.ts index 9823aca..94cc67d 100644 --- a/src/memory/experiment/learning-metrics.ts +++ b/src/memory/experiment/learning-metrics.ts @@ -1,4 +1,4 @@ -import { pairedBootstrap } from '@tangle-network/agent-eval' +import { comparePairedArms, type PairedBootstrapResult } from '@tangle-network/agent-eval' import type { PairedMemoryLearningCell, PairedMemoryLearningProbe } from './learning-pairs' import { mean } from './metrics' import type { @@ -45,11 +45,7 @@ export function measureAgentMemoryLearning( }, cells, preTreatment, - gain: pairedBootstrap( - gainUnits.map((unit) => unit.statelessReward), - gainUnits.map((unit) => unit.statefulReward), - { statistic: 'mean', seed: stateful.campaign.seed }, - ), + gain: comparePairedRewards(gainUnits, stateful.campaign.seed, 'overall gain'), gainByCandidate: summarizeCandidateGain(cells, stateful.campaign.seed), transfer: { definition: 'explicit-transfer-probes', @@ -99,7 +95,8 @@ function measurePreTreatment( unit.exact &&= cell.exact bySequence.set(cell.sequenceId, unit) } - const units = [...bySequence.values()].map((unit) => ({ + const units = [...bySequence.entries()].map(([sequenceId, unit]) => ({ + pairKey: sequenceId, statefulReward: mean(unit.statefulRewards), statelessReward: mean(unit.statelessRewards), exact: unit.exact, @@ -110,13 +107,7 @@ function measurePreTreatment( n: units.length, exactMatchRate: units.length === 0 ? null : mean(units.map((unit) => (unit.exact ? 1 : 0))), difference: - units.length === 0 - ? null - : pairedBootstrap( - units.map((unit) => unit.statelessReward), - units.map((unit) => unit.statefulReward), - { statistic: 'mean', seed }, - ), + units.length === 0 ? null : comparePairedRewards(units, seed, 'pre-treatment difference'), } } @@ -266,11 +257,7 @@ function summarizeCandidateGain( return { candidateId, cells: cells.filter((cell) => cell.candidateId === candidateId).length, - gain: pairedBootstrap( - units.map((unit) => unit.statelessReward), - units.map((unit) => unit.statefulReward), - { statistic: 'mean', seed }, - ), + gain: comparePairedRewards(units, seed, `candidate ${candidateId} gain`), } }) } @@ -321,18 +308,19 @@ function summarizeTransfer( .map(({ candidateId, transferKey, stepOrdinal, rewardsBySequence }) => { const sequenceRewards = [...rewardsBySequence.entries()] .sort(([left], [right]) => left.localeCompare(right)) - .map(([, rewards]) => ({ - stateful: mean(rewards.stateful), - stateless: mean(rewards.stateless), + .map(([sequenceId, rewards]) => ({ + pairKey: sequenceId, + statefulReward: mean(rewards.stateful), + statelessReward: mean(rewards.stateless), })) return { candidateId, transferKey, stepOrdinal, - gain: pairedBootstrap( - sequenceRewards.map((reward) => reward.stateless), - sequenceRewards.map((reward) => reward.stateful), - { statistic: 'mean', seed }, + gain: comparePairedRewards( + sequenceRewards, + seed, + `transfer ${candidateId}:${transferKey}:${stepOrdinal}`, ), } }) @@ -340,7 +328,7 @@ function summarizeTransfer( function collapseLearningCells( cells: readonly AgentMemoryLearningCellComparison[], -): Array<{ sequenceId: string; statefulReward: number; statelessReward: number }> { +): Array<{ pairKey: string; statefulReward: number; statelessReward: number }> { const bySequence = new Map() for (const cell of cells) { const unit = bySequence.get(cell.sequenceId) ?? { @@ -354,7 +342,7 @@ function collapseLearningCells( return [...bySequence.entries()] .sort(([left], [right]) => left.localeCompare(right)) .map(([sequenceId, unit]) => ({ - sequenceId, + pairKey: sequenceId, statefulReward: mean(unit.statefulRewards), statelessReward: mean(unit.statelessRewards), })) @@ -368,7 +356,7 @@ function summarizeForgetting( meanStatefulForgetting: number | null meanStatelessForgetting: number | null meanExcessForgetting: number | null - excess: ReturnType | null + excess: PairedBootstrapResult | null } { const byTarget = new Map< string, @@ -386,25 +374,63 @@ function summarizeForgetting( unit.excess.push(probe.excessForgetting) byTarget.set(key, unit) } - const units = [...byTarget.values()].map((unit) => ({ - stateful: mean(unit.stateful), - stateless: mean(unit.stateless), + const units = [...byTarget.entries()].map(([pairKey, unit]) => ({ + pairKey, + statefulReward: mean(unit.stateful), + statelessReward: mean(unit.stateless), excess: mean(unit.excess), })) return { n: units.length, - meanStatefulForgetting: optionalMean(units.map((unit) => unit.stateful)), - meanStatelessForgetting: optionalMean(units.map((unit) => unit.stateless)), + meanStatefulForgetting: optionalMean(units.map((unit) => unit.statefulReward)), + meanStatelessForgetting: optionalMean(units.map((unit) => unit.statelessReward)), meanExcessForgetting: optionalMean(units.map((unit) => unit.excess)), - excess: - units.length === 0 - ? null - : pairedBootstrap( - units.map((unit) => unit.stateless), - units.map((unit) => unit.stateful), - { statistic: 'mean', seed }, - ), + excess: units.length === 0 ? null : comparePairedRewards(units, seed, 'excess forgetting'), + } +} + +function comparePairedRewards( + units: readonly { + pairKey: string + statefulReward: number + statelessReward: number + }[], + seed: number, + label: string, +): PairedBootstrapResult { + const comparison = comparePairedArms( + units.flatMap((unit) => [ + { + pairKey: unit.pairKey, + arm: 'stateless', + metrics: { reward: unit.statelessReward }, + }, + { + pairKey: unit.pairKey, + arm: 'stateful', + metrics: { reward: unit.statefulReward }, + }, + ]), + { + baselineArm: 'stateless', + treatmentArm: 'stateful', + metricNames: ['reward'], + bootstrap: { statistic: 'mean', seed }, + }, + ) + const metric = comparison.metricDeltas[0] + if ( + comparison.nPairs !== units.length || + comparison.nUnpairedBaseline !== 0 || + comparison.nUnpairedTreatment !== 0 || + !metric || + metric.n !== units.length || + metric.nMissing !== 0 || + !metric.bootstrapCi + ) { + throw new Error(`cannot compare memory learning: incomplete paired ${label}`) } + return metric.bootstrapCi } function maxProbe( diff --git a/src/memory/experiment/learning-pairs.ts b/src/memory/experiment/learning-pairs.ts index 645a81c..fce043f 100644 --- a/src/memory/experiment/learning-pairs.ts +++ b/src/memory/experiment/learning-pairs.ts @@ -1,3 +1,4 @@ +import { type PairedArmRow, pairArms } from '@tangle-network/agent-eval' import { assertCampaignSplitIdentity, type CampaignCellResult, @@ -32,23 +33,36 @@ export interface PairedMemoryLearningCell { probes: PairedMemoryLearningProbe[] } +interface MemoryLearningArmRow extends PairedArmRow { + indexed: IndexedMemoryLearningCell +} + export function pairAgentMemoryLearningRuns( options: CompareAgentMemoryLearningOptions, ): PairedMemoryLearningCell[] { assertComparableDesign(options.stateful, options.stateless) const statefulCells = indexCompleteCells('stateful', options.stateful) const statelessCells = indexCompleteCells('stateless', options.stateless) - assertSameKeys(statefulCells, statelessCells, 'experiment cell') + const matched = pairArms( + [ + ...memoryLearningArmRows('stateful', statefulCells.values()), + ...memoryLearningArmRows('stateless', statelessCells.values()), + ], + { baselineArm: 'stateless', treatmentArm: 'stateful' }, + ) + if (matched.unpairedBaseline.length > 0 || matched.unpairedTreatment.length > 0) { + throw new Error( + `cannot compare memory learning: unmatched experiment cell identities; stateful-only=${unpairedCellIds(matched.unpairedTreatment).join(', ') || 'none'}; stateless-only=${unpairedCellIds(matched.unpairedBaseline).join(', ') || 'none'}`, + ) + } - return [...statefulCells.entries()] - .map(([key, stateful]): PairedMemoryLearningCell => { - const stateless = statelessCells.get(key) - if (!stateless) { - throw new Error(`cannot compare memory learning: missing stateless cell ${key}`) - } + return matched.pairs + .map((pair): PairedMemoryLearningCell => { + const stateful = (pair.treatment as MemoryLearningArmRow).indexed + const stateless = (pair.baseline as MemoryLearningArmRow).indexed if (stateful.cell.cellId !== stateless.cell.cellId) { throw new Error( - `cannot compare memory learning: cell identity differs for ${key} (${stateful.cell.cellId} vs ${stateless.cell.cellId})`, + `cannot compare memory learning: cell identity differs for ${pair.pairKey} (${stateful.cell.cellId} vs ${stateless.cell.cellId})`, ) } return { stateful, stateless, probes: pairProbes(stateful, stateless) } @@ -56,6 +70,25 @@ export function pairAgentMemoryLearningRuns( .sort(comparePairedCells) } +function memoryLearningArmRows( + arm: 'stateful' | 'stateless', + cells: Iterable, +): MemoryLearningArmRow[] { + return [...cells].map((indexed) => ({ + arm, + pairKey: JSON.stringify([indexed.artifact.candidateId, indexed.artifact.sequenceId]), + repKey: JSON.stringify([indexed.cell.rep, indexed.cell.seed]), + indexed, + })) +} + +function unpairedCellIds(rows: readonly PairedArmRow[]): string[] { + return rows + .map((row) => (row as MemoryLearningArmRow).indexed) + .map((indexed) => cellIdentity(indexed.artifact, indexed.cell.rep, indexed.cell.seed)) + .sort() +} + function assertComparableDesign( stateful: RunAgentMemoryExperimentResult, stateless: RunAgentMemoryExperimentResult,