diff --git a/.evolve/skill-runs.jsonl b/.evolve/skill-runs.jsonl index 77255c63..c5bba81b 100644 --- a/.evolve/skill-runs.jsonl +++ b/.evolve/skill-runs.jsonl @@ -40,3 +40,7 @@ {"skill":"/agent-eval","ts":"2026-07-30T11:07:02Z","project":"agent-eval-analyst-proof-20260730","target":"crash-safe resumable public analyst benchmark command","operatorPrompt":"","durationMin":null,"verdict":"PASS","dispatchedTo":"/verify","operatorOverride":null,"transcriptPath":null,"traceDir":null} {"skill":"/verify","ts":"2026-07-30T11:07:07Z","project":"agent-eval-analyst-proof-20260730","target":"benchmark command recovery, artifacts, and AgentRx metrics","operatorPrompt":"","durationMin":null,"verdict":"PASS","dispatchedTo":"/stop","operatorOverride":null,"transcriptPath":null,"traceDir":null} {"skill":"/semgrep","ts":"2026-07-30T15:47:18Z","project":"agent-eval-analyst-proof-20260730","target":"agent-eval public analyst benchmark, 664 files, 492 rules","operatorPrompt":"","durationMin":null,"verdict":"PASS","dispatchedTo":"/stop","operatorOverride":null,"transcriptPath":null,"traceDir":null} +{"skill":"/review-to-green","ts":"2026-07-30T19:04:25Z","project":"agent-eval-exact-analyst-run-20260730","target":"exact AnalystRegistry audit findings","operatorPrompt":"","durationMin":null,"verdict":"DONE","dispatchedTo":"/ship","operatorOverride":null,"transcriptPath":null,"traceDir":null} +{"skill":"/review-to-green","ts":"2026-07-30T20:07:14Z","project":"agent-eval-exact-analyst-run-20260730","target":"exact AnalystRegistry six audit findings","operatorPrompt":"","durationMin":null,"verdict":"PASS","dispatchedTo":"/simplify","operatorOverride":null,"transcriptPath":null,"traceDir":null} +{"skill":"/simplify","ts":"2026-07-30T20:07:14Z","project":"agent-eval-exact-analyst-run-20260730","target":"exact AnalystRegistry execution policy","operatorPrompt":"","durationMin":null,"verdict":"PASS","dispatchedTo":"/verify","operatorOverride":null,"transcriptPath":null,"traceDir":null} +{"skill":"/report","ts":"2026-07-30T20:07:31Z","project":"agent-eval-exact-analyst-run-20260730","target":"exact AnalystRegistry replacement status","operatorPrompt":"","durationMin":null,"verdict":"PASS","dispatchedTo":"/stop","operatorOverride":null,"transcriptPath":null,"traceDir":null} diff --git a/CHANGELOG.md b/CHANGELOG.md index bf6a7907..4b63ee36 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,14 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval- ### Added +- `AnalystRegistry.runExact()` requires ordered analyst ids and an explicit value for every run-policy field. + It never inherits registry insertion order or the constructor's default budget. + Disabled budgets, timeouts, cancellation, cost attribution, tags, and prior findings use explicit `null` values. + Exact runs bind analyst, ledger, hook, chat, and policy identities with non-secret configuration digests. + They use the registry's shared serial execution path and persist canonical equal or weighted allocations instead of adding a second scheduler or allocator. + Exact receipts explicitly distinguish complete execution from a failed ordered prefix. + `ExactAnalystRunExecutionError.result` is always a canonical immutable failed receipt with valid completed work and accounting. +- `defineTraceAnalyst()` accepts canonical `executionConfig` and returns an exact-capable custom analyst when it is supplied, while preserving its existing minimal form. - **Breaking:** `AnalystBenchmarkCase` now requires `clusterId` and `labelState`. A case must identify its independent source unit and state whether labels prove an issue, prove no issue, or leave the outcome unknown. The benchmark no longer guesses either field from an empty issue list. diff --git a/README.md b/README.md index a0dce60e..18ad4f45 100644 --- a/README.md +++ b/README.md @@ -359,6 +359,38 @@ Use `analyzeRuns()` for `RunRecord[]`. For traces, run a registry of built-in or custom analysts, measure it on labeled issues and exact span locations, then turn only reviewed findings into eval data. For a public quality check, convert CodeTraceBench with `traces import-codetracebench`, then run `agent-eval analyst-benchmark` against the pinned labels and a real model. +Use `AnalystRegistry.runExact()` when the caller, rather than registry defaults, must own every execution choice. +The ordered `analystIds` array is the execution order, and `null` explicitly disables optional budget, timeout, cancellation, cost, tag, or prior-finding channels. +Exact runs are serial; callers that need recursive or concurrent scheduling compose them through their runtime rather than adding a second scheduler here. + +```ts +const result = await registry.runExact('analysis-1', inputs, { + analystIds: ['failure-mode', 'improvement'], + budget: { kind: 'equal', totalUsd: 2 }, + totalTimeoutMs: 30_000, + signal: null, + costLedger: null, + costLedgerIdentity: null, + costPhase: null, + tags: null, + priorFindings: null, + chainFindings: true, + missingInputMode: 'abort', + applyRegistryHooks: false, + useRegistryChat: false, +}) +``` + +Custom analysts passed to `runExact()` declare canonical `executionConfig`. +The same `defineTraceAnalyst()` helper returns an exact-capable analyst when that field is present. +Built-in analysts already declare it. +Trace analysts selected by `runExact()` also require `aiIdentity`, using the same non-secret `id`, `version`, and canonical `config` shape as cost ledgers, registry hooks, and registry chat clients. +Exact lifecycle hooks receive frozen snapshots for observation; they cannot rewrite the planned context. +Persisted results store configuration digests, not raw configuration. +The persisted plan records the exact equal or weighted allocation for every routed analyst, and archival validates summaries against that same plan. +Every exact receipt says whether it is `complete` or `failed`; a complete receipt must cover the full plan, while a failed receipt may contain only the executed prefix. +Any failure after an exact run starts rejects with `ExactAnalystRunExecutionError`; its immutable failed receipt preserves valid completed summaries, findings, usage, and cost. + See [concepts](./docs/concepts.md), [customer paths](./docs/customer-journeys.md), and [trace analysis](./docs/trace-analysis.md). ## Entry Points diff --git a/scripts/verify-package-exports.mjs b/scripts/verify-package-exports.mjs index b45a14fc..8a16163b 100644 --- a/scripts/verify-package-exports.mjs +++ b/scripts/verify-package-exports.mjs @@ -154,6 +154,7 @@ try { import { CostLedger, CONTROL_INTEGRITY_ANALYST as ROOT_CONTROL_INTEGRITY_ANALYST, + AnalystRegistry as RootAnalystRegistry, analyzeSupervisorRunIntegrity as ROOT_ANALYZE_SUPERVISOR_RUN_INTEGRITY, analystFindingDigest, analystRunDigest, @@ -167,6 +168,8 @@ try { type ChatClient, type CostLedgerHandle as RootCostLedgerHandle, type ExecutorConfig, + type ExactExecutionComponentIdentity as RootExactExecutionComponentIdentity, + type ExactRegistryRunOpts as RootExactRegistryRunOpts, type JudgeFn, type LlmJudgeOptions as RootLlmJudgeOptions, type LlmClientOptions, @@ -180,6 +183,7 @@ try { import { CONTROL_INTEGRITY_ANALYST, ControlIntegrityAnalyst, + AnalystRegistry as AnalystSubpathRegistry, RawAnalystFindingSchema, agentRxBenchmarkCase, agentRxPredictionsToFindings, @@ -188,6 +192,8 @@ try { emitControlIntegrityFindings, traceStoreEvidenceResolver, type AnalystBenchmarkCase, + type ExactExecutionComponentIdentity as AnalystExactExecutionComponentIdentity, + type ExactRegistryRunOpts as AnalystExactRegistryRunOpts, type RawAnalystFinding, } from '@tangle-network/agent-eval/analyst' import { @@ -360,6 +366,31 @@ try { const removedDatasetRecordInput: Parameters[0] = removedRecordInputs const canonicalChat = null as unknown as ChatClient const canonicalJudge = null as unknown as JudgeFn + const exactComponentIdentity: RootExactExecutionComponentIdentity = { + id: 'package-check', + version: '1.0.0', + config: { mode: 'compile-check' }, + } + const analystExactComponentIdentity: AnalystExactExecutionComponentIdentity = + exactComponentIdentity + const exactRunOptions: RootExactRegistryRunOpts = { + analystIds: ['package-check'], + budget: null, + totalTimeoutMs: null, + signal: null, + costLedger: null, + costLedgerIdentity: null, + costPhase: null, + tags: null, + priorFindings: null, + chainFindings: false, + missingInputMode: 'abort', + applyRegistryHooks: false, + useRegistryChat: false, + } + const analystExactRunOptions: AnalystExactRegistryRunOpts = exactRunOptions + const rootExactRegistry = new RootAnalystRegistry() + const analystExactRegistry = new AnalystSubpathRegistry() const controlIntegrityAnalyst: ControlIntegrityAnalyst = CONTROL_INTEGRITY_ANALYST const controlIntegrityInput = undefined as SupervisorRunSources | SupervisorRunTree | undefined const controlIntegrityReport = undefined as SupervisorRunIntegrityReport | undefined @@ -548,6 +579,12 @@ try { removedDatasetRecordInput, canonicalChat, canonicalJudge, + exactComponentIdentity, + analystExactComponentIdentity, + exactRunOptions, + analystExactRunOptions, + rootExactRegistry, + analystExactRegistry, controlIntegrityAnalyst, controlIntegrityInput, controlIntegrityReport, diff --git a/src/analyst/analyst.test.ts b/src/analyst/analyst.test.ts index 09ece7d0..ee460cd3 100644 --- a/src/analyst/analyst.test.ts +++ b/src/analyst/analyst.test.ts @@ -362,6 +362,7 @@ describe('AnalystHooks', () => { it('does not reclassify a successful analyst when its after hook fails', async () => { let onErrorCalls = 0 + let onCompleteCalls = 0 const reg = new AnalystRegistry({ hooks: { onAfterAnalyze: () => { @@ -371,6 +372,9 @@ describe('AnalystHooks', () => { onErrorCalls += 1 return [] }, + onComplete: () => { + onCompleteCalls += 1 + }, }, }) reg.register(ok('a')) @@ -378,6 +382,7 @@ describe('AnalystHooks', () => { await expect(reg.run('run-1', { runRecord: record })).rejects.toThrow('telemetry failed') expect(onErrorCalls).toBe(0) + expect(onCompleteCalls).toBe(0) }) it('onError can convert a thrown analyst into findings', async () => { @@ -860,6 +865,97 @@ describe('AnalystRegistry.runStream', () => { 'run-completed', ]) }) + + it('does not admit the next analyst after a stream consumer stops', async () => { + const started: string[] = [] + const analyst = (id: string): Analyst => ({ + id, + description: id, + inputKind: 'custom', + cost: { kind: 'deterministic' }, + version: '1', + async analyze() { + started.push(id) + return [] + }, + }) + const registry = new AnalystRegistry() + registry.register(analyst('a')) + registry.register(analyst('b')) + + for await (const event of registry.runStream('consumer-stop', { + custom: { a: 1, b: 1 }, + })) { + if (event.type === 'analyst-completed') break + } + + expect(started).toEqual(['a']) + }) +}) + +describe('legacy lifecycle context', () => { + it('preserves a hook-selected signal and deadline for the analyst', async () => { + const selected = new AbortController() + let observedSignal: AbortSignal | undefined + let observedDeadline: number | undefined + const registry = new AnalystRegistry({ + hooks: { + onBeforeAnalyze({ ctx }) { + ctx.signal = selected.signal + ctx.deadlineMs = 123 + }, + }, + }) + registry.register({ + id: 'a', + description: 'a', + inputKind: 'custom', + cost: { kind: 'deterministic' }, + version: '1', + async analyze(_input, ctx) { + observedSignal = ctx.signal + observedDeadline = ctx.deadlineMs + return [] + }, + }) + + await registry.run('hook-context', { custom: { a: 1 } }) + + expect(observedSignal).toBe(selected.signal) + expect(observedDeadline).toBe(123) + }) + + it('keeps the run timeout authoritative when a hook selects another signal', async () => { + const selected = new AbortController() + let completed = false + let observedSignal: AbortSignal | undefined + const registry = new AnalystRegistry({ + hooks: { + onBeforeAnalyze({ ctx }) { + ctx.signal = selected.signal + }, + }, + }) + registry.register({ + id: 'a', + description: 'a', + inputKind: 'custom', + cost: { kind: 'deterministic' }, + version: '1', + async analyze(_input, ctx) { + observedSignal = ctx.signal + await new Promise((resolve) => setTimeout(resolve, 40)) + completed = true + return [] + }, + }) + + const result = await registry.run('hook-timeout', { custom: { a: 1 } }, { timeoutMs: 5 }) + + expect(observedSignal).toBe(selected.signal) + expect(result.per_analyst[0]?.status).toBe('failed') + expect(completed).toBe(false) + }) }) describe('RegistryRunOpts.priorFindings forwarding', () => { diff --git a/src/analyst/behavioral-analyst.ts b/src/analyst/behavioral-analyst.ts index b41a513e..9b347b42 100644 --- a/src/analyst/behavioral-analyst.ts +++ b/src/analyst/behavioral-analyst.ts @@ -10,7 +10,8 @@ import { type SuboptimalCode, } from '../trace-analyst/behavioral-metrics' import type { TraceAnalysisStore } from '../trace-analyst/store' -import { type Analyst, type AnalystFinding, makeFinding } from './types' +import type { ExactCapableAnalyst } from './exact-types' +import { type AnalystFinding, makeFinding } from './types' const RECOMMENDED_ACTION: Record = { 'monotonic-input-growth': @@ -134,7 +135,7 @@ export function deriveEfficiencyFindings( /** The deterministic behavioral/efficiency analyst (no LLM, any-model). */ export function behavioralAnalyst( options: BehavioralAnalystOptions = {}, -): Analyst { +): ExactCapableAnalyst { const maxTraces = positiveInteger(options.maxTraces ?? DEFAULT_MAX_TRACES, 'maxTraces') const maxEvidenceRefsPerFinding = positiveInteger( options.maxEvidenceRefsPerFinding ?? DEFAULT_MAX_EVIDENCE_REFS, @@ -147,6 +148,11 @@ export function behavioralAnalyst( inputKind: 'trace-store', cost: { kind: 'deterministic' }, version: '2.0.0', + executionConfig: { + kind: 'behavioral-efficiency', + max_traces: maxTraces, + max_evidence_refs_per_finding: maxEvidenceRefsPerFinding, + }, async analyze(store, context) { const analyzedTraceIds = await listTraceIds(store, maxTraces, context.signal) const findingsById = new Map< diff --git a/src/analyst/default-registry.test.ts b/src/analyst/default-registry.test.ts index 75b7d8e0..6e0b5a8a 100644 --- a/src/analyst/default-registry.test.ts +++ b/src/analyst/default-registry.test.ts @@ -356,4 +356,14 @@ describe('buildDefaultAnalystRegistry', () => { /maxEvidenceRefsPerFinding/, ) }) + + it('binds effective deterministic limits for exact-run provenance', () => { + expect( + behavioralAnalyst({ maxTraces: 25, maxEvidenceRefsPerFinding: 3 }).executionConfig, + ).toEqual({ + kind: 'behavioral-efficiency', + max_traces: 25, + max_evidence_refs_per_finding: 3, + }) + }) }) diff --git a/src/analyst/default-registry.ts b/src/analyst/default-registry.ts index e59de0ed..bb8b1d95 100644 --- a/src/analyst/default-registry.ts +++ b/src/analyst/default-registry.ts @@ -11,6 +11,7 @@ import type { AxAIService } from '@ax-llm/ax' import { type BehavioralAnalystOptions, behavioralAnalyst } from './behavioral-analyst' +import type { ExactExecutionComponentIdentity } from './exact-types' import { createTraceAnalystKind, type TraceAnalystKindSpec } from './kind-factory' import { DEFAULT_TRACE_ANALYST_KINDS } from './kinds' import { AnalystRegistry, type AnalystRegistryOptions } from './registry' @@ -18,6 +19,8 @@ import { AnalystRegistry, type AnalystRegistryOptions } from './registry' export interface DefaultAnalystRegistryOptions { /** Ax service for the agentic RLM kinds. Omit → only the deterministic analyst. */ ai?: AxAIService + /** Required when agentic kinds will be selected by `runExact`. */ + aiIdentity?: ExactExecutionComponentIdentity /** Required unless `ai` was created by `createAnalystAi`. */ model?: string /** Which agentic kinds to register when `ai` is present. Default = the shipped suite. */ @@ -40,7 +43,13 @@ export function buildDefaultAnalystRegistry( if (opts.ai) { const kinds = opts.kinds ?? DEFAULT_TRACE_ANALYST_KINDS for (const spec of kinds) { - registry.register(createTraceAnalystKind(spec, { ai: opts.ai, model: opts.model })) + registry.register( + createTraceAnalystKind(spec, { + ai: opts.ai, + model: opts.model, + aiIdentity: opts.aiIdentity, + }), + ) } } return registry diff --git a/src/analyst/define.test.ts b/src/analyst/define.test.ts index b0705255..fdfda843 100644 --- a/src/analyst/define.test.ts +++ b/src/analyst/define.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from 'vitest' import { defineTraceAnalyst } from './define' +import type { ExactCapableAnalyst } from './exact-types' describe('defineTraceAnalyst', () => { it('fills the fixed trace-store fields and preserves the declared cost', () => { @@ -46,4 +47,40 @@ describe('defineTraceAnalyst', () => { defineTraceAnalyst(missingCost) }).toThrow(/cost must be declared/) }) + + it('creates an exact-capable analyst when execution configuration is declared', () => { + const analyst: ExactCapableAnalyst = defineTraceAnalyst({ + id: 'tool-fidelity', + description: 'Compare native and normalized tool calls.', + version: '2.0.0', + cost: { kind: 'deterministic' }, + executionConfig: { + kind: 'tool-fidelity', + schemaVersion: '1', + }, + async analyze() { + return [] + }, + }) + + expect(analyst.executionConfig).toEqual({ + kind: 'tool-fidelity', + schemaVersion: '1', + }) + }) + + it('rejects a non-object exact execution configuration', () => { + expect(() => + defineTraceAnalyst({ + id: 'tool-fidelity', + description: 'Compare native and normalized tool calls.', + cost: { kind: 'deterministic' }, + // @ts-expect-error JavaScript callers can supply a non-object value. + executionConfig: [], + async analyze() { + return [] + }, + }), + ).toThrow(/executionConfig must be an object/) + }) }) diff --git a/src/analyst/define.ts b/src/analyst/define.ts index 7aaab949..664c6c39 100644 --- a/src/analyst/define.ts +++ b/src/analyst/define.ts @@ -1,4 +1,5 @@ import type { TraceAnalysisStore } from '../trace-analyst/store' +import type { ExactCapableAnalyst } from './exact-types' import type { Analyst, AnalystCost, AnalystFinding } from './types' export interface DefineTraceAnalystOptions { @@ -9,10 +10,19 @@ export interface DefineTraceAnalystOptions { analyze: Analyst['analyze'] } +export interface DefineExactTraceAnalystOptions extends DefineTraceAnalystOptions { + /** Canonical JSON for behavior knobs not already bound by `version`. */ + executionConfig: Readonly> +} + /** Define a custom trace analyst without repeating fixed registry fields. */ export function defineTraceAnalyst( - options: DefineTraceAnalystOptions, -): Analyst { + options: DefineExactTraceAnalystOptions, +): ExactCapableAnalyst +export function defineTraceAnalyst(options: DefineTraceAnalystOptions): Analyst +export function defineTraceAnalyst( + options: DefineTraceAnalystOptions | DefineExactTraceAnalystOptions, +): Analyst | ExactCapableAnalyst { if (!options.id.trim()) throw new TypeError('defineTraceAnalyst: id must not be empty') if (!options.description.trim()) { throw new TypeError('defineTraceAnalyst: description must not be empty') @@ -20,6 +30,14 @@ export function defineTraceAnalyst( if (options.cost === undefined) { throw new TypeError('defineTraceAnalyst: cost must be declared') } + if ( + 'executionConfig' in options && + (!options.executionConfig || + typeof options.executionConfig !== 'object' || + Array.isArray(options.executionConfig)) + ) { + throw new TypeError('defineTraceAnalyst: executionConfig must be an object') + } return { id: options.id, description: options.description, @@ -27,6 +45,7 @@ export function defineTraceAnalyst( inputKind: 'trace-store', cost: options.cost, analyze: options.analyze, + ...('executionConfig' in options ? { executionConfig: options.executionConfig } : {}), } } diff --git a/src/analyst/exact-types.ts b/src/analyst/exact-types.ts new file mode 100644 index 00000000..183f3d78 --- /dev/null +++ b/src/analyst/exact-types.ts @@ -0,0 +1,298 @@ +import { z } from 'zod' +import { canonicalString, hashCanonical } from '../ledger-core/canonical' +import { deepFreezeCanonicalJson } from '../ledger-core/deep-freeze' +import type { Analyst, AnalystRunEvent, AnalystRunResult, AnalystRunSummary } from './types' + +/** Analyst metadata required before the exact registry path will execute it. */ +export interface ExactCapableAnalyst extends Analyst { + /** Canonical JSON for every effective behavior knob not already bound by `version`. */ + readonly executionConfig: Readonly> +} + +type DeepReadonly = T extends readonly (infer Item)[] + ? readonly DeepReadonly[] + : T extends object + ? { readonly [Key in keyof T]: DeepReadonly } + : T + +export type ExactAnalystSnapshot = DeepReadonly> +export type ExactAnalystBudgetSnapshot = DeepReadonly> + +/** Caller-provided identity for a component used by an exact run. */ +export interface ExactExecutionComponentIdentity { + id: string + version: string + config: Readonly> +} + +/** Persisted identity; configuration is bound by digest and never stored raw. */ +export interface ExactExecutionComponentSnapshot { + id: string + version: string + config_digest: string +} + +/** Canonical identity for any live component admitted to an exact run. */ +export function snapshotExactExecutionComponentIdentity( + value: ExactExecutionComponentIdentity, + context: string, +): ExactExecutionComponentSnapshot { + let detached: unknown + try { + detached = JSON.parse(canonicalString(value)) as unknown + } catch (cause) { + throw new TypeError(`${context} must have a canonical JSON representation`, { cause }) + } + const parsed = componentIdentitySchema.safeParse(detached) + if (!parsed.success) { + throw new TypeError(`${context} requires non-empty id/version and object config`) + } + return deepFreezeCanonicalJson({ + id: parsed.data.id, + version: parsed.data.version, + config_digest: hashCanonical(parsed.data.config), + }) +} + +export type ExactAnalystRunPolicySnapshot = DeepReadonly> +export type ExactAnalystExecutionPlanSnapshot = DeepReadonly< + z.infer +> + +export type ExactAnalystRunSummary = AnalystRunSummary & { + /** Effective caller-owned budget; null means uncapped. Skipped work has no allocation. */ + allocated_budget_usd?: number | null +} + +export type ExactAnalystRunCompletion = + | { status: 'complete' } + | { + status: 'failed' + error: { class: string; message: string } + } + +export interface ExactAnalystRunResult extends AnalystRunResult { + per_analyst: ExactAnalystRunSummary[] + execution_plan: ExactAnalystExecutionPlanSnapshot + completion: ExactAnalystRunCompletion +} + +/** Exact events are detached immutable snapshots of the shared serial execution stream. */ +export type ExactAnalystRunEvent = + | (Extract & { + execution_plan: ExactAnalystExecutionPlanSnapshot + }) + | (Omit, 'summary'> & { + summary: ExactAnalystRunSummary + }) + | Extract + | (Omit, 'summary'> & { + summary: ExactAnalystRunSummary + }) + | (Omit, 'result'> & { + result: ExactAnalystRunResult + }) + +const nonEmptyString = z.string().min(1) +const digest = z.string().regex(/^sha256:[a-f0-9]{64}$/) +const finiteNonnegative = z.number().finite().nonnegative() +const nonnegativeSafeInteger = z.number().int().min(0).max(Number.MAX_SAFE_INTEGER) +const positiveTimeout = z.number().int().positive().max(2_147_483_647) + +const componentSnapshotSchema = z.strictObject({ + id: nonEmptyString, + version: nonEmptyString, + config_digest: digest, +}) +const componentIdentitySchema = z.strictObject({ + id: nonEmptyString, + version: nonEmptyString, + config: z.record(z.string(), z.unknown()), +}) + +const deterministicCostSchema = z.strictObject({ + kind: z.literal('deterministic'), + est_usd_per_run: finiteNonnegative.optional(), + models: z.array(nonEmptyString).optional(), +}) + +const llmCostSchema = z.strictObject({ + kind: z.literal('llm'), + est_usd_per_run: finiteNonnegative.optional(), + models: z.array(nonEmptyString).optional(), + settlement_timeout_ms: nonnegativeSafeInteger.optional(), +}) + +const requirementsSchema = z + .strictObject({ + min_shots: nonnegativeSafeInteger.optional(), + capabilities: z.array(nonEmptyString).optional(), + }) + .nullable() + +const analystSnapshotSchema = z.strictObject({ + id: nonEmptyString, + version: nonEmptyString, + input_kind: z.enum(['trace-store', 'artifact-dir', 'run-record', 'judge-input', 'custom']), + cost: z.discriminatedUnion('kind', [deterministicCostSchema, llmCostSchema]), + requirements: requirementsSchema, + execution_config_digest: digest, +}) + +const allocationsSchema = z.record(nonEmptyString, z.union([finiteNonnegative, z.null()])) +const weightsSchema = z.record(nonEmptyString, finiteNonnegative) +const budgetSnapshotSchema = z.discriminatedUnion('kind', [ + z.strictObject({ kind: z.literal('none') }), + z.strictObject({ + kind: z.literal('equal'), + total_usd: finiteNonnegative, + allocations_usd: allocationsSchema, + }), + z.strictObject({ + kind: z.literal('weighted'), + total_usd: finiteNonnegative, + weights: weightsSchema, + allocations_usd: allocationsSchema, + }), +]) + +const priorFindingsSchema = z.discriminatedUnion('kind', [ + z.strictObject({ kind: z.literal('none') }), + z.strictObject({ + kind: z.literal('ordered'), + count: nonnegativeSafeInteger, + digest, + }), + z.strictObject({ + kind: z.literal('by_analyst'), + keys: z.array(nonEmptyString), + count: nonnegativeSafeInteger, + digest, + }), +]) + +const exactRunPolicySchema = z.strictObject({ + budget: budgetSnapshotSchema, + total_timeout_ms: positiveTimeout.nullable(), + signal_provided: z.boolean(), + cost_ledger: componentSnapshotSchema.nullable(), + cost_phase: nonEmptyString.nullable(), + tags: z.record(z.string(), z.string()).nullable(), + prior_findings: priorFindingsSchema, + chain_findings: z.boolean(), + missing_input_mode: z.enum(['skip', 'abort']), + registry_hooks: componentSnapshotSchema.nullable(), + registry_chat: componentSnapshotSchema.nullable(), +}) + +const exactExecutionPlanSchema = z + .strictObject({ + schema_version: z.literal('1.0.0'), + analysts: z.array(analystSnapshotSchema).min(1), + policy: exactRunPolicySchema, + digest, + }) + .superRefine((plan, context) => { + const issue = (path: PropertyKey[], message: string): void => + context.addIssue({ code: 'custom', path, message }) + const analystIds = plan.analysts.map((analyst) => analyst.id) + if (new Set(analystIds).size !== analystIds.length) { + issue(['analysts'], 'analyst ids must be unique') + } + if (plan.policy.cost_ledger === null && plan.policy.cost_phase !== null) { + issue(['policy', 'cost_phase'], 'cost phase requires a cost ledger') + } + if ( + plan.policy.prior_findings.kind === 'by_analyst' && + plan.policy.prior_findings.keys.some( + (key, index, keys) => index > 0 && key <= keys[index - 1]!, + ) + ) { + issue(['policy', 'prior_findings', 'keys'], 'keys must be sorted and unique') + } + + const budget = plan.policy.budget + if (budget.kind === 'none') return + const allocationIds = Object.keys(budget.allocations_usd).sort() + const selectedIds = [...analystIds].sort() + if ( + allocationIds.length !== selectedIds.length || + allocationIds.some((id, index) => id !== selectedIds[index]) + ) { + issue( + ['policy', 'budget', 'allocations_usd'], + 'allocations must name every analyst and no others', + ) + return + } + const runnableIds = analystIds.filter((id) => budget.allocations_usd[id] !== null) + const epsilon = Math.max(1, budget.total_usd) * Number.EPSILON * 8 + if (runnableIds.length === 0) return + + if (budget.kind === 'weighted') { + const weightIds = Object.keys(budget.weights).sort() + if ( + weightIds.length !== selectedIds.length || + weightIds.some((id, index) => id !== selectedIds[index]) + ) { + issue(['policy', 'budget', 'weights'], 'weights must name every analyst and no others') + return + } + const totalWeight = runnableIds.reduce((sum, id) => sum + (budget.weights[id] ?? 0), 0) + if (totalWeight === 0) { + issue(['policy', 'budget', 'weights'], 'runnable analysts must have positive total weight') + return + } + for (const id of runnableIds) { + const expected = (budget.total_usd * (budget.weights[id] ?? 0)) / totalWeight + if (Math.abs((budget.allocations_usd[id] ?? 0) - expected) > epsilon) { + issue( + ['policy', 'budget', 'allocations_usd', id], + 'allocation does not match the weighted policy', + ) + } + } + return + } + + const expected = budget.total_usd / runnableIds.length + for (const id of runnableIds) { + if (Math.abs((budget.allocations_usd[id] ?? 0) - expected) > epsilon) { + issue( + ['policy', 'budget', 'allocations_usd', id], + 'allocation does not match the equal policy', + ) + } + } + }) + +/** + * Canonicalize and validate the one exact-plan representation shared by execution and archival. + * Unknown fields fail at every level; the returned graph is detached and deeply frozen. + */ +export function snapshotExactExecutionPlan( + value: unknown, + context = 'exact analyst execution plan', +): ExactAnalystExecutionPlanSnapshot { + let detached: unknown + try { + detached = JSON.parse(canonicalString(value)) as unknown + } catch (cause) { + throw new TypeError(`${context} must have a canonical JSON representation`, { cause }) + } + const parsed = exactExecutionPlanSchema.safeParse(detached) + if (!parsed.success) { + const issue = parsed.error.issues[0] + const path = issue?.path.length ? ` ${issue.path.join('.')}` : '' + throw new TypeError(`${context}${path}: ${issue?.message ?? 'is invalid'}`) + } + const expectedDigest = hashCanonical({ + schema_version: parsed.data.schema_version, + analysts: parsed.data.analysts, + policy: parsed.data.policy, + }) + if (parsed.data.digest !== expectedDigest) { + throw new TypeError(`${context} digest does not match its content`) + } + return deepFreezeCanonicalJson(parsed.data as ExactAnalystExecutionPlanSnapshot) +} diff --git a/src/analyst/index.ts b/src/analyst/index.ts index f7347b43..8e69ba55 100644 --- a/src/analyst/index.ts +++ b/src/analyst/index.ts @@ -186,8 +186,25 @@ export { buildDefaultAnalystRegistry, type DefaultAnalystRegistryOptions, } from './default-registry' -export type { DefineTraceAnalystOptions, TraceAnalystAnalyze } from './define' +export type { + DefineExactTraceAnalystOptions, + DefineTraceAnalystOptions, + TraceAnalystAnalyze, +} from './define' export { defineTraceAnalyst } from './define' +export type { + ExactAnalystBudgetSnapshot, + ExactAnalystExecutionPlanSnapshot, + ExactAnalystRunCompletion, + ExactAnalystRunEvent, + ExactAnalystRunPolicySnapshot, + ExactAnalystRunResult, + ExactAnalystRunSummary, + ExactAnalystSnapshot, + ExactCapableAnalyst, + ExactExecutionComponentIdentity, + ExactExecutionComponentSnapshot, +} from './exact-types' export type { RawAnalystEvidence, RawAnalystFinding, @@ -247,9 +264,15 @@ export type { AnalystHooks, AnalystRegistryOptions, BudgetPolicy, + ExactAnalystBudgetPolicy, + ExactRegistryRunOpts, RegistryRunOpts, } from './registry' -export { AnalystRegistry } from './registry' +export { + AnalystRegistry, + assertExactRegistryRunOpts, + ExactAnalystRunExecutionError, +} from './registry' export { type StructureFindingsOptions, type StructureFindingsResult, diff --git a/src/analyst/kind-factory.ts b/src/analyst/kind-factory.ts index bd146640..4a5aaa22 100644 --- a/src/analyst/kind-factory.ts +++ b/src/analyst/kind-factory.ts @@ -23,10 +23,16 @@ import type { AxAIService, AxFunction } from '@ax-llm/ax' import { CostLedger } from '../cost-ledger' +import { hashCanonical } from '../ledger-core/canonical' import { runTraceAnalysisLoop } from '../trace-analyst/loop' import type { TraceAnalysisStore } from '../trace-analyst/store' import { meterAxChatService } from './ax-cost-service' import { resolveAnalystModel } from './ax-service' +import { + type ExactCapableAnalyst, + type ExactExecutionComponentIdentity, + snapshotExactExecutionComponentIdentity, +} from './exact-types' import { evidenceRefsFromRawFinding, parseRawFinding, @@ -35,7 +41,7 @@ import { } from './finding-signature' import { KIND_EXPECTED_SUBJECTS, parseFindingSubject } from './finding-subject' import { structureFindings } from './structure-findings' -import type { Analyst, AnalystContext, AnalystCost, AnalystFinding } from './types' +import type { AnalystContext, AnalystCost, AnalystFinding } from './types' import { makeFinding } from './types' import { settleUsageReceiptFromCostLedger, validateUsageSettlementTimeout } from './usage-receipt' @@ -86,6 +92,8 @@ export interface TraceAnalystKindSpec { export interface CreateTraceAnalystKindOpts { /** AxAIService bound at registration time. */ ai: AxAIService + /** Required when this analyst is used by `runExact`; binds the effective AI service. */ + aiIdentity?: ExactExecutionComponentIdentity /** Required unless `ai` was created by {@link createAnalystAi}. */ model?: string /** Override the spec's `version` (e.g. when an optimizer has fitted a new prompt). */ @@ -113,7 +121,7 @@ export interface CreateTraceAnalystKindOpts { export function createTraceAnalystKind( spec: TraceAnalystKindSpec, opts: CreateTraceAnalystKindOpts, -): Analyst { +): ExactCapableAnalyst { rejectRemovedKindOptions(spec) const version = opts.versionSuffix ? `${spec.version}+${opts.versionSuffix}` : spec.version const model = resolveAnalystModel(opts.ai, opts.model) @@ -122,14 +130,47 @@ export function createTraceAnalystKind( throw new TypeError('minimumEvidenceCitations must be a positive integer') } const settlementTimeoutMs = validateUsageSettlementTimeout(opts.settlementTimeoutMs) + const maxOutputTokens = spec.maxOutputTokens ?? 4096 + const aiIdentity = + opts.aiIdentity === undefined + ? null + : snapshotExactExecutionComponentIdentity( + opts.aiIdentity, + 'createTraceAnalystKind aiIdentity', + ) return { id: spec.id, description: spec.description, inputKind: 'trace-store', cost: { ...spec.cost, settlement_timeout_ms: settlementTimeoutMs }, version, + executionConfig: { + kind: 'trace-analyst', + model, + ai_identity: aiIdentity, + actor_description_digest: hashCanonical(spec.actorDescription.trim()), + max_subqueries: spec.subqueries?.maxCalls ?? 0, + max_parallel_subqueries: spec.subqueries?.maxParallel ?? 2, + max_turns: spec.maxTurns ?? 12, + max_runtime_chars: spec.maxRuntimeChars ?? 6000, + max_output_tokens: maxOutputTokens, + minimum_evidence_citations: minimumEvidenceCitations, + require_structured_findings: spec.requireStructuredFindings ?? false, + prepare_context: spec.prepareContext === undefined ? 'disabled' : 'version-bound', + post_process: spec.postProcess === undefined ? 'disabled' : 'version-bound', + recovery: + opts.recovery === undefined + ? null + : { + base_url: opts.recovery.baseUrl, + model: opts.recovery.model ?? model, + api_key_provided: opts.recovery.apiKey !== undefined, + fetch_implementation: + opts.recovery.fetchImpl === undefined ? 'global' : 'version-bound', + }, + settlement_timeout_ms: settlementTimeoutMs, + }, async analyze(store, ctx) { - const maxOutputTokens = spec.maxOutputTokens ?? 4096 const costLedger = ctx.costLedger ?? new CostLedger(ctx.budgetUsd) const costTags = { ...(ctx.tags ?? {}), diff --git a/src/analyst/kinds/control-integrity.test.ts b/src/analyst/kinds/control-integrity.test.ts index fd872c0d..eaac1215 100644 --- a/src/analyst/kinds/control-integrity.test.ts +++ b/src/analyst/kinds/control-integrity.test.ts @@ -15,6 +15,10 @@ describe('control-integrity analyst adapter', () => { } const registry = new AnalystRegistry() registry.register(CONTROL_INTEGRITY_ANALYST) + expect(CONTROL_INTEGRITY_ANALYST.executionConfig).toEqual({ + kind: 'control-integrity', + produced_at_source: 'tags.producedAt-or-system-clock', + }) const result = await registry.run( 'run-1', diff --git a/src/analyst/kinds/control-integrity.ts b/src/analyst/kinds/control-integrity.ts index cad67aeb..635c25cd 100644 --- a/src/analyst/kinds/control-integrity.ts +++ b/src/analyst/kinds/control-integrity.ts @@ -3,13 +3,8 @@ import { type SupervisorRunIntegrityEvidence, } from '../../supervisor-run/integrity' import type { SupervisorRunSources, SupervisorRunTree } from '../../supervisor-run/types' -import { - type Analyst, - type AnalystContext, - type AnalystFinding, - type EvidenceRef, - makeFinding, -} from '../types' +import type { ExactCapableAnalyst } from '../exact-types' +import { type AnalystContext, type AnalystFinding, type EvidenceRef, makeFinding } from '../types' const ANALYST_ID = 'control-integrity' @@ -59,13 +54,19 @@ export function emitControlIntegrityFindings( } /** Deterministic Analyst adapter for `SupervisorRunSources | SupervisorRunTree`. */ -export class ControlIntegrityAnalyst implements Analyst { +export class ControlIntegrityAnalyst + implements ExactCapableAnalyst +{ readonly id = ANALYST_ID readonly description = 'Deterministic supervisor-run integrity checks with explicit unavailable evidence.' readonly inputKind = 'custom' as const readonly cost = { kind: 'deterministic' as const, est_usd_per_run: 0 } readonly version = '2.0.0' + readonly executionConfig = { + kind: 'control-integrity', + produced_at_source: 'tags.producedAt-or-system-clock', + } as const async analyze( input: SupervisorRunSources | SupervisorRunTree, diff --git a/src/analyst/kinds/kinds.test.ts b/src/analyst/kinds/kinds.test.ts index 1d17f650..8b6abfe5 100644 --- a/src/analyst/kinds/kinds.test.ts +++ b/src/analyst/kinds/kinds.test.ts @@ -199,6 +199,30 @@ describe('tool-groups filter the analyst tool surface narrowly', () => { }) describe('createTraceAnalystKind wires the spec into the Analyst contract', () => { + it('binds versioned AI service identity into exact execution configuration', () => { + const spec: TraceAnalystKindSpec = { + id: 'identity-kind', + description: 'identity', + area: 'identity', + version: '1', + actorDescription: 'inspect identity', + buildTools: () => [], + cost: { kind: 'llm' }, + } + const first = createTraceAnalystKind(spec, { + ai: stubAi(), + model: 'test-model', + aiIdentity: { id: 'provider', version: '1', config: { baseUrl: 'https://one.test' } }, + }) + const second = createTraceAnalystKind(spec, { + ai: stubAi(), + model: 'test-model', + aiIdentity: { id: 'provider', version: '2', config: { baseUrl: 'https://two.test' } }, + }) + + expect(first.executionConfig.ai_identity).not.toEqual(second.executionConfig.ai_identity) + }) + it('returns a registry-ready Analyst that delegates to the kind id + version', () => { const spec: TraceAnalystKindSpec = { id: 'test-kind', @@ -214,6 +238,12 @@ describe('createTraceAnalystKind wires the spec into the Analyst contract', () = expect(analyst.version).toBe('0.0.1') expect(analyst.inputKind).toBe('trace-store') expect(analyst.cost.kind).toBe('llm') + expect(analyst.executionConfig).toMatchObject({ + kind: 'trace-analyst', + model: 'test-model', + max_turns: 12, + max_output_tokens: 4096, + }) }) it('versionSuffix appends to the kind version (used by optimizer pipelines)', () => { diff --git a/src/analyst/kinds/skill-usage.test.ts b/src/analyst/kinds/skill-usage.test.ts index 13da9424..ae690f06 100644 --- a/src/analyst/kinds/skill-usage.test.ts +++ b/src/analyst/kinds/skill-usage.test.ts @@ -175,6 +175,10 @@ describe('emitSkillUsageFindings', () => { }) it('the analyst wires emission through analyze() with the canonical envelope', async () => { + expect(SKILL_USAGE_ANALYST.executionConfig).toMatchObject({ + kind: 'skill-usage', + produced_at_source: 'tags.producedAt-or-system-clock', + }) const fs = await SKILL_USAGE_ANALYST.analyze(report([rec({ name: 'dead' })]), { runId: 'r', correlationId: 'c', diff --git a/src/analyst/kinds/skill-usage.ts b/src/analyst/kinds/skill-usage.ts index f4c39e62..a26a9a97 100644 --- a/src/analyst/kinds/skill-usage.ts +++ b/src/analyst/kinds/skill-usage.ts @@ -21,7 +21,8 @@ import { type Dirent, existsSync, readdirSync, readFileSync, statSync } from 'node:fs' import { join } from 'node:path' -import type { Analyst, AnalystContext, AnalystFinding, AnalystSeverity } from '../types' +import type { ExactCapableAnalyst } from '../exact-types' +import type { AnalystContext, AnalystFinding, AnalystSeverity } from '../types' import { computeFindingId } from '../types' // ── Input model ────────────────────────────────────────────────────── @@ -384,13 +385,18 @@ export function emitSkillUsageFindings( // ── The Analyst ────────────────────────────────────────────────────── -export class SkillUsageAnalyst implements Analyst { +export class SkillUsageAnalyst implements ExactCapableAnalyst { readonly id = ANALYST_ID readonly description = 'Deterministic multi-signal skill-usage analysis: flags dead skills, measurement-invisible (orchestrated) usage, discovery gaps, public-repo leaks, bloat, missing evals, and missing run-logging.' readonly inputKind = 'custom' as const readonly cost = { kind: 'deterministic' as const, est_usd_per_run: 0 } readonly version = '1.0.0' + readonly executionConfig = { + kind: 'skill-usage', + bloat_line_threshold: BLOAT_LINE_THRESHOLD, + produced_at_source: 'tags.producedAt-or-system-clock', + } as const async analyze(input: SkillUsageReport, ctx: AnalystContext): Promise { const producedAt = ctx.tags?.producedAt ?? new Date().toISOString() diff --git a/src/analyst/registry.exact.test.ts b/src/analyst/registry.exact.test.ts new file mode 100644 index 00000000..6cf6cede --- /dev/null +++ b/src/analyst/registry.exact.test.ts @@ -0,0 +1,451 @@ +import { describe, expect, it } from 'vitest' +import { snapshotAnalystRun, snapshotExactAnalystRunReceipt } from '../feedback-trajectory-review' +import type { ExactAnalystRunResult, ExactCapableAnalyst } from './exact-types' +import { + AnalystRegistry, + ExactAnalystRunExecutionError, + type ExactRegistryRunOpts, +} from './registry' +import type { AnalystContext, AnalystFinding, AnalystRunInputs, AnalystUsageReceipt } from './types' + +function exactOptions(overrides: Partial = {}): ExactRegistryRunOpts { + return { + analystIds: ['a'], + budget: null, + totalTimeoutMs: null, + signal: null, + costLedger: null, + costLedgerIdentity: null, + costPhase: null, + tags: null, + priorFindings: null, + chainFindings: false, + missingInputMode: 'skip', + applyRegistryHooks: false, + useRegistryChat: false, + ...overrides, + } +} + +function recordingAnalyst( + id: string, + calls: Array<{ id: string; context: AnalystContext }>, + analyze?: (context: AnalystContext) => Promise, +): ExactCapableAnalyst { + return { + id, + description: id, + inputKind: 'custom', + cost: { kind: 'deterministic' }, + version: '1', + executionConfig: { kind: 'recording', id }, + async analyze(_input, context): Promise { + calls.push({ id, context }) + return analyze?.(context) ?? [] + }, + } +} + +function twoAnalystRegistry(): { + registry: AnalystRegistry + calls: Array<{ id: string; context: AnalystContext }> +} { + const calls: Array<{ id: string; context: AnalystContext }> = [] + const registry = new AnalystRegistry() + registry.register(recordingAnalyst('a', calls)) + registry.register(recordingAnalyst('b', calls)) + return { registry, calls } +} + +describe('AnalystRegistry exact run policy', () => { + it.each([ + ['undefined', undefined], + ['empty', {}], + ['partial', { analystIds: ['a'] }], + ['empty analyst ids', exactOptions({ analystIds: [] })], + ['missing budget', { ...exactOptions(), budget: undefined }], + ])('refuses %s policy before any analyst executes', async (_label, policy) => { + const calls: Array<{ id: string; context: AnalystContext }> = [] + const registry = new AnalystRegistry() + registry.register(recordingAnalyst('a', calls)) + + await expect( + registry.runExact('invalid', { custom: { a: 1 } }, policy as ExactRegistryRunOpts), + ).rejects.toThrow(/ExactRegistryRunOpts/) + expect(calls).toEqual([]) + }) + + it('rejects an invalid run id before any analyst executes', async () => { + const calls: Array<{ id: string; context: AnalystContext }> = [] + const registry = new AnalystRegistry() + registry.register(recordingAnalyst('a', calls)) + + await expect(registry.runExact('', { custom: { a: 1 } }, exactOptions())).rejects.toThrow( + /runId must be a non-empty string/, + ) + expect(calls).toEqual([]) + }) + + it('runs exactly the declared analysts in order without inheriting registry defaults', async () => { + const calls: Array<{ id: string; context: AnalystContext }> = [] + const registry = new AnalystRegistry({ defaultBudget: { totalUsd: 99 } }) + registry.register(recordingAnalyst('a', calls)) + registry.register(recordingAnalyst('b', calls)) + registry.register(recordingAnalyst('c', calls)) + + const result = await registry.runExact( + 'ordered', + { custom: { a: 1, b: 2, c: 3 } }, + exactOptions({ analystIds: ['c', 'a'] }), + ) + + expect(calls.map((call) => call.id)).toEqual(['c', 'a']) + expect(calls.map((call) => call.context.budgetUsd)).toEqual([undefined, undefined]) + expect(result.per_analyst.map((summary) => summary.analyst_id)).toEqual(['c', 'a']) + expect(result.execution_plan.policy.budget).toEqual({ kind: 'none' }) + expect(() => snapshotAnalystRun(result)).not.toThrow() + }) + + it('rejects mutation of plan-bound context by onBeforeAnalyze', async () => { + const calls: Array<{ id: string; context: AnalystContext }> = [] + const registry = new AnalystRegistry({ + hooks: { + onBeforeAnalyze({ ctx }) { + ctx.budgetUsd = 99 + }, + }, + hooksIdentity: { id: 'hooks', version: '1', config: {} }, + }) + registry.register(recordingAnalyst('a', calls)) + + const error = await registry + .runExact('frozen-context', { custom: { a: 1 } }, exactOptions({ applyRegistryHooks: true })) + .catch((cause: unknown) => cause) + + expect(error).toBeInstanceOf(ExactAnalystRunExecutionError) + expect(calls).toEqual([]) + expect((error as ExactAnalystRunExecutionError).result.completion.status).toBe('failed') + }) + + it('records and executes the same equal allocations from one canonical plan', async () => { + const { registry, calls } = twoAnalystRegistry() + + const result = await registry.runExact( + 'equal', + { custom: { a: 1, b: 2 } }, + exactOptions({ + analystIds: ['a', 'b'], + budget: { kind: 'equal', totalUsd: 2 }, + }), + ) + + expect(calls.map((call) => call.context.budgetUsd)).toEqual([1, 1]) + expect(result.execution_plan.policy.budget).toEqual({ + kind: 'equal', + total_usd: 2, + allocations_usd: { a: 1, b: 1 }, + }) + expect(result.per_analyst.map((summary) => summary.allocated_budget_usd)).toEqual([1, 1]) + }) + + it('detaches and freezes every streamed exact record from terminal accounting', async () => { + const calls: Array<{ id: string; context: AnalystContext }> = [] + const registry = new AnalystRegistry() + registry.register(recordingAnalyst('a', calls)) + const stream = registry.runExactStream('immutable', { custom: { a: 1 } }, exactOptions()) + + const runStarted = await stream.next() + const analystStarted = await stream.next() + const analystCompleted = await stream.next() + expect(runStarted.value?.type).toBe('run-started') + expect(analystStarted.value?.type).toBe('analyst-started') + expect(analystCompleted.value?.type).toBe('analyst-completed') + if (analystCompleted.value?.type !== 'analyst-completed') { + throw new Error('expected analyst-completed event') + } + expect(Object.isFrozen(analystCompleted.value)).toBe(true) + expect(Object.isFrozen(analystCompleted.value.summary.usage.cost)).toBe(true) + expect(Reflect.set(analystCompleted.value.summary.usage.cost, 'usd', 99)).toBe(false) + + const completed = await stream.next() + if (completed.value?.type !== 'run-completed') { + throw new Error('expected run-completed event') + } + expect(completed.value.result.total_cost_usd).toBe(0) + expect(completed.value.result.per_analyst[0]?.usage.cost).toEqual({ + kind: 'observed', + usd: 0, + }) + expect(completed.value.result.per_analyst[0]).not.toBe(analystCompleted.value.summary) + expect(Object.isFrozen(completed.value.result)).toBe(true) + expect(Object.isFrozen(completed.value.result.per_analyst[0])).toBe(true) + }) + + it.each([ + { + label: 'equal', + budget: { kind: 'equal', totalUsd: 2 } as const, + forged: [2, 2], + }, + { + label: 'weighted', + budget: { kind: 'weighted', totalUsd: 4, weights: { a: 1, b: 3 } } as const, + forged: [4, 4], + }, + ])( + 'rejects $label archived allocations that exceed the canonical plan', + async ({ budget, forged }) => { + const { registry } = twoAnalystRegistry() + const result = await registry.runExact( + 'archive-budget', + { custom: { a: 1, b: 2 } }, + exactOptions({ analystIds: ['a', 'b'], budget }), + ) + const tampered = structuredClone(result) + tampered.per_analyst[0]!.allocated_budget_usd = forged[0] + tampered.per_analyst[1]!.allocated_budget_usd = forged[1] + + expect(() => snapshotAnalystRun(tampered)).toThrow(/allocation|budget/) + }, + ) + + it('rejects a registered analyst whose live id changed before exact normalization', async () => { + const calls: Array<{ id: string; context: AnalystContext }> = [] + const analyst = recordingAnalyst('a', calls) + const registry = new AnalystRegistry() + registry.register(analyst) + ;(analyst as { id: string }).id = 'b' + + await expect( + registry.runExact('changed-id', { custom: { a: 1 } }, exactOptions()), + ).rejects.toThrow(/changed id/) + expect(calls).toEqual([]) + }) + + it('routes each selected input once and executes the captured value', async () => { + const calls: Array<{ id: string; context: AnalystContext }> = [] + const registry = new AnalystRegistry() + registry.register(recordingAnalyst('a', calls)) + let customReads = 0 + let valueReads = 0 + const inputs = { + get custom(): Record | undefined { + customReads += 1 + if (customReads > 1) return undefined + return { + get a(): unknown { + valueReads += 1 + return 1 + }, + } + }, + } + + const result = await registry.runExact( + 'single-route', + inputs, + exactOptions({ missingInputMode: 'abort' }), + ) + + expect(customReads).toBe(1) + expect(valueReads).toBe(1) + expect(calls).toHaveLength(1) + expect(result.per_analyst[0]?.status).toBe('ok') + }) + + it('captures each shared input channel once for every selected analyst', async () => { + const seen: unknown[] = [] + const registry = new AnalystRegistry() + for (const id of ['a', 'b']) { + const analyst: ExactCapableAnalyst = { + id, + description: id, + inputKind: 'trace-store', + cost: { kind: 'deterministic' }, + version: '1', + executionConfig: { kind: 'shared-input', id }, + async analyze(input) { + seen.push(input) + return [] + }, + } + registry.register(analyst) + } + const first = { source: 'first' } + const second = { source: 'second' } + let traceStoreReads = 0 + const inputs = { + get traceStore() { + traceStoreReads += 1 + return traceStoreReads === 1 ? first : second + }, + } as unknown as AnalystRunInputs + + await registry.runExact('shared-input', inputs, exactOptions({ analystIds: ['a', 'b'] })) + + expect(traceStoreReads).toBe(1) + expect(seen).toEqual([first, first]) + }) + + it('rejects unknown nested budget fields before execution', async () => { + const calls: Array<{ id: string; context: AnalystContext }> = [] + const registry = new AnalystRegistry() + registry.register(recordingAnalyst('a', calls)) + const budget = { + kind: 'equal', + totalUsd: 1, + weights: { a: 999 }, + } as unknown as ExactRegistryRunOpts['budget'] + + await expect( + registry.runExact('strict-budget', { custom: { a: 1 } }, exactOptions({ budget })), + ).rejects.toThrow(/exactly kind, totalUsd/) + expect(calls).toEqual([]) + }) + + it('never returns success when cancellation prevents declared hooks from completing', async () => { + const calls: Array<{ id: string; context: AnalystContext }> = [] + let after = 0 + let complete = 0 + const registry = new AnalystRegistry({ + hooks: { + onAfterAnalyze() { + after += 1 + }, + onComplete() { + complete += 1 + }, + }, + hooksIdentity: { id: 'hooks', version: '1', config: {} }, + }) + registry.register( + recordingAnalyst('a', calls, async () => new Promise(() => {})), + ) + + let caught: unknown + try { + await registry.runExact( + 'cancelled-hooks', + { custom: { a: 1 } }, + exactOptions({ + totalTimeoutMs: 10, + applyRegistryHooks: true, + }), + ) + } catch (error) { + caught = error + } + + expect(caught).toBeInstanceOf(ExactAnalystRunExecutionError) + const exactError = caught as ExactAnalystRunExecutionError + expect(exactError.result.execution_plan.policy.registry_hooks).not.toBeNull() + expect(exactError.result.per_analyst).toHaveLength(1) + expect(after).toBe(0) + expect(complete).toBe(0) + }) + + it('round-trips weighted output through the archive validator', async () => { + const { registry } = twoAnalystRegistry() + const result = await registry.runExact( + 'round-trip', + { custom: { a: 1, b: 2 } }, + exactOptions({ + analystIds: ['a', 'b'], + budget: { kind: 'weighted', totalUsd: 4, weights: { a: 1, b: 3 } }, + }), + ) + + const archived = snapshotAnalystRun(result) as ExactAnalystRunResult + expect(archived.execution_plan.policy.budget).toEqual({ + kind: 'weighted', + total_usd: 4, + weights: { a: 1, b: 3 }, + allocations_usd: { a: 1, b: 3 }, + }) + expect(archived).toEqual(result) + }) + + it('rejects an incomplete successful exact receipt', async () => { + const { registry } = twoAnalystRegistry() + const result = await registry.runExact( + 'truncated-success', + { custom: { a: 1, b: 2 } }, + exactOptions({ analystIds: ['a', 'b'] }), + ) + const truncated = structuredClone(result) + truncated.per_analyst.pop() + + expect(() => snapshotExactAnalystRunReceipt(truncated)).toThrow( + /complete receipt must contain every execution_plan analyst/, + ) + }) + + it('snapshots usage before validating it', async () => { + let callsReads = 0 + const registry = new AnalystRegistry() + registry.register( + recordingAnalyst('a', [], async (context) => { + const receipt = { + get calls() { + callsReads += 1 + return callsReads === 1 ? 0 : Number.MAX_SAFE_INTEGER + 1 + }, + tokens: { input: 0, output: 0 }, + cost: { kind: 'observed', usd: 0 }, + } as unknown as AnalystUsageReceipt + context.recordUsage?.(receipt) + return [] + }), + ) + + const result = await registry.runExact('read-once-usage', { custom: { a: 1 } }, exactOptions()) + + expect(callsReads).toBe(1) + expect(result.per_analyst[0]?.usage.calls).toBe(0) + }) + + it('always attaches a valid frozen failed receipt to exact execution errors', async () => { + const registry = new AnalystRegistry() + registry.register( + recordingAnalyst('a', [], async (context) => { + const receipt: AnalystUsageReceipt = { + calls: Number.MAX_SAFE_INTEGER, + tokens: { input: 0, output: 0 }, + cost: { kind: 'observed', usd: 0 }, + } + context.recordUsage?.(receipt) + context.recordUsage?.(receipt) + return [] + }), + ) + + const error = await registry + .runExact('overflow', { custom: { a: 1 } }, exactOptions()) + .catch((cause: unknown) => cause) + + expect(error).toBeInstanceOf(ExactAnalystRunExecutionError) + const result = (error as ExactAnalystRunExecutionError).result + expect(result.completion).toMatchObject({ + status: 'failed', + error: { class: 'RangeError' }, + }) + expect(result.per_analyst).toEqual([]) + expect(Object.isFrozen(result)).toBe(true) + expect(Object.isFrozen(result.completion)).toBe(true) + expect(Reflect.set(result, 'total_cost_usd', 7)).toBe(false) + expect(() => snapshotExactAnalystRunReceipt(result)).not.toThrow() + }) + + it('rejects archived usage whose reasoning tokens exceed output tokens', async () => { + const calls: Array<{ id: string; context: AnalystContext }> = [] + const registry = new AnalystRegistry() + registry.register(recordingAnalyst('a', calls)) + const result = await registry.runExact('invalid-usage', { custom: { a: 1 } }, exactOptions()) + const tampered = structuredClone(result) + const tokens = tampered.per_analyst[0]?.usage.tokens + if (!tokens) throw new Error('expected deterministic token receipt') + tokens.reasoning = tokens.output + 1 + + expect(() => snapshotAnalystRun(tampered)).toThrow(/reasoning must not exceed tokens.output/) + }) +}) diff --git a/src/analyst/registry.ts b/src/analyst/registry.ts index b58d1df1..1532551d 100644 --- a/src/analyst/registry.ts +++ b/src/analyst/registry.ts @@ -17,10 +17,26 @@ */ import { randomUUID } from 'node:crypto' +import { z } from 'zod' import { combineAbortSignals } from '../abort-signal' import type { CostLedgerHandle } from '../cost-ledger' +import { + snapshotAnalystFindings, + snapshotExactAnalystRunReceipt, +} from '../feedback-trajectory-review' +import { canonicalString, hashCanonical } from '../ledger-core/canonical' +import { deepFreezeCanonicalJson } from '../ledger-core/deep-freeze' import type { RunCostProvenance, RunTokenUsage } from '../run-record' import type { ChatClient } from './chat-client' +import type { + ExactAnalystExecutionPlanSnapshot, + ExactAnalystRunEvent, + ExactAnalystRunResult, + ExactCapableAnalyst, + ExactExecutionComponentIdentity, + ExactExecutionComponentSnapshot, +} from './exact-types' +import { snapshotExactExecutionComponentIdentity, snapshotExactExecutionPlan } from './exact-types' import type { Analyst, AnalystContext, @@ -36,7 +52,7 @@ import { assertValidAnalystUsageReceipt, validateUsageSettlementTimeout } from ' // ── Hook + policy surfaces ───────────────────────────────────────── export interface AnalystHooks { - /** Before analyze() — last chance to mutate ctx (e.g. inject tags, override budget). */ + /** Legacy runs may mutate ctx; exact runs provide a frozen observational context. */ onBeforeAnalyze?(args: { analyst: Analyst ctx: AnalystContext @@ -89,8 +105,12 @@ export interface AnalystRegistryOptions { log?: (msg: string, fields?: Record) => void /** Hooks invoked around analyze() — observability + customization seam. */ hooks?: AnalystHooks + /** Required identity/config when `runExact` applies `hooks`. */ + hooksIdentity?: ExactExecutionComponentIdentity /** Default budget when run() doesn't override. */ defaultBudget?: BudgetPolicy + /** Required identity/config when `runExact` provides `chat`. */ + chatIdentity?: ExactExecutionComponentIdentity } export interface RegistryRunOpts { @@ -127,6 +147,94 @@ export interface RegistryRunOpts { chainFindings?: boolean } +/** A caller-selected allocation rule for an exact analyst run. */ +export type ExactAnalystBudgetPolicy = + | { + readonly kind: 'equal' + readonly totalUsd: number + } + | { + readonly kind: 'weighted' + readonly totalUsd: number + /** One explicit weight for every selected analyst id. */ + readonly weights: Readonly> + } + +/** + * Complete per-run analyst policy. + * + * Every field is required. `null` explicitly disables an optional resource or context channel. + * `analystIds` is execution order; it is not filtered through registry insertion order. + * Missing-input behavior is explicit so callers cannot accidentally inherit it by omission. + * Exact runs are serial; more elaborate scheduling belongs in the caller's runtime. + */ +export interface ExactRegistryRunOpts { + readonly analystIds: readonly string[] + readonly budget: ExactAnalystBudgetPolicy | null + readonly totalTimeoutMs: number | null + readonly signal: AbortSignal | null + readonly costLedger: CostLedgerHandle | null + readonly costLedgerIdentity: ExactExecutionComponentIdentity | null + readonly costPhase: string | null + readonly tags: Readonly> | null + readonly priorFindings: + | ReadonlyArray + | Readonly>> + | null + readonly chainFindings: boolean + readonly missingInputMode: 'skip' | 'abort' + readonly applyRegistryHooks: boolean + readonly useRegistryChat: boolean +} + +type NormalizedBudgetPlan = + | { readonly kind: 'none' } + | { readonly kind: 'dynamic'; readonly policy: BudgetPolicy } + +interface PreparedAnalyst { + readonly analyst: Analyst + readonly input: + | { readonly kind: 'present'; readonly value: unknown } + | { readonly kind: 'missing' } +} + +interface AnalystExecutionPlan { + readonly runId: string + readonly prepared: readonly PreparedAnalyst[] + readonly budget: NormalizedBudgetPlan + readonly totalTimeoutMs: number | null + readonly signal: AbortSignal | null + readonly costLedger: CostLedgerHandle | null + readonly costPhase: string | null + readonly tags: Readonly> | null + readonly priorFindings: + | ReadonlyArray + | Readonly>> + | null + readonly chainFindings: boolean + readonly hooks: AnalystHooks + readonly chat: ChatClient | undefined + readonly log: (msg: string, fields?: Record) => void + readonly executionSnapshot: ExactAnalystExecutionPlanSnapshot | undefined +} + +type InternalAnalystRunEvent = AnalystRunEvent + +/** A post-start exact-run failure; completed work remains attached for accounting and review. */ +export class ExactAnalystRunExecutionError extends Error { + readonly name: string = 'ExactAnalystRunExecutionError' + readonly result: ExactAnalystRunResult + + constructor(message: string, result: ExactAnalystRunResult, options?: ErrorOptions) { + super(message, options) + const snapshot = snapshotExactAnalystRunReceipt(result, 'ExactAnalystRunExecutionError result') + if (snapshot.completion.status !== 'failed') { + throw new TypeError('ExactAnalystRunExecutionError result must be a failed receipt') + } + this.result = snapshot + } +} + export class AnalystRegistry { private readonly analysts = new Map() private readonly options: AnalystRegistryOptions @@ -136,22 +244,25 @@ export class AnalystRegistry { } register(analyst: Analyst): void { - if (!analyst.id) throw new Error('AnalystRegistry.register: analyst.id is required') - if (this.analysts.has(analyst.id)) { - throw new Error(`AnalystRegistry.register: duplicate analyst id "${analyst.id}"`) + const id = analyst.id + const version = analyst.version + const cost = analyst.cost + if (!id) throw new Error('AnalystRegistry.register: analyst.id is required') + if (this.analysts.has(id)) { + throw new Error(`AnalystRegistry.register: duplicate analyst id "${id}"`) } - if (!analyst.version) { - throw new Error(`AnalystRegistry.register: analyst "${analyst.id}" must declare a version`) + if (!version) { + throw new Error(`AnalystRegistry.register: analyst "${id}" must declare a version`) } - if (analyst.cost.kind === 'deterministic' && analyst.cost.settlement_timeout_ms !== undefined) { + if (cost.kind === 'deterministic' && cost.settlement_timeout_ms !== undefined) { throw new TypeError( - `AnalystRegistry.register: deterministic analyst "${analyst.id}" cannot declare settlement_timeout_ms`, + `AnalystRegistry.register: deterministic analyst "${id}" cannot declare settlement_timeout_ms`, ) } - if (analyst.cost.settlement_timeout_ms !== undefined) { - validateUsageSettlementTimeout(analyst.cost.settlement_timeout_ms) + if (cost.settlement_timeout_ms !== undefined) { + validateUsageSettlementTimeout(cost.settlement_timeout_ms) } - this.analysts.set(analyst.id, analyst) + this.analysts.set(id, analyst) } list(): ReadonlyArray<{ @@ -181,6 +292,31 @@ export class AnalystRegistry { throw new Error('AnalystRegistry.run: stream completed without run-completed event') } + /** Run exactly the ordered analysts and complete policy supplied by the caller. */ + async runExact( + runId: string, + inputs: AnalystRunInputs, + runOpts: ExactRegistryRunOpts, + ): Promise { + for await (const ev of this.runExactStream(runId, inputs, runOpts)) { + if (ev.type === 'run-completed') return ev.result + } + throw new Error('AnalystRegistry.runExact: stream completed without run-completed event') + } + + /** Streaming counterpart to {@link runExact}. */ + async *runExactStream( + runId: string, + inputs: AnalystRunInputs, + runOpts: ExactRegistryRunOpts, + ): AsyncGenerator { + for await (const event of this.executePlanStream( + this.normalizeExactPlan(runId, inputs, runOpts), + )) { + yield event as ExactAnalystRunEvent + } + } + /** * Streaming counterpart to `run()`. Emits `AnalystRunEvent` values * in real time — `run-started`, then per-analyst `skipped` / @@ -197,57 +333,198 @@ export class AnalystRegistry { inputs: AnalystRunInputs, runOpts: RegistryRunOpts = {}, ): AsyncGenerator { - const correlationId = `ar_${randomUUID().slice(0, 12)}` - const log = this.options.log ?? (() => {}) - const hooks = this.options.hooks ?? {} - const startedAt = new Date().toISOString() - const started = Date.now() - const timeoutMs = validateTimeout(runOpts.timeoutMs) - const deadlineMs = timeoutMs === undefined ? undefined : started + timeoutMs - const timeoutSignal = timeoutMs === undefined ? undefined : AbortSignal.timeout(timeoutMs) - const runSignal = combineAbortSignals(runOpts.signal, timeoutSignal) + yield* this.executePlanStream(this.normalizeLegacyPlan(runId, inputs, runOpts)) + } - const selected = this.selectAnalysts(runOpts) + private normalizeLegacyPlan( + runId: string, + inputs: AnalystRunInputs, + runOpts: RegistryRunOpts, + ): AnalystExecutionPlan { + const timeoutMs = validateTimeout(runOpts.timeoutMs) ?? null const budget = runOpts.budget ?? this.options.defaultBudget validateBudgetPolicy(budget) + const selected = this.selectAnalysts(runOpts) + return { + runId, + prepared: selected.map((analyst) => ({ + analyst, + input: this.routeInput(analyst, inputs), + })), + budget: budget ? { kind: 'dynamic', policy: budget } : { kind: 'none' }, + totalTimeoutMs: timeoutMs, + signal: runOpts.signal ?? null, + costLedger: runOpts.costLedger ?? null, + costPhase: runOpts.costPhase ?? null, + tags: runOpts.tags ?? null, + priorFindings: runOpts.priorFindings ?? null, + chainFindings: runOpts.chainFindings ?? false, + hooks: this.options.hooks ?? {}, + chat: this.options.chat, + log: this.options.log ?? (() => {}), + executionSnapshot: undefined, + } + } - yield { - type: 'run-started', - run_id: runId, - correlation_id: correlationId, - started_at: startedAt, - analyst_ids: selected.map((a) => a.id), + private normalizeExactPlan( + runId: string, + inputs: AnalystRunInputs, + runOpts: ExactRegistryRunOpts, + ): AnalystExecutionPlan { + const exactRunId = snapshotExactRunId(runId) + const exact = snapshotExactRegistryRunOpts(runOpts) + const selected = normalizeExactAnalysts(this.selectExactAnalysts(exact.analystIds)) + const registryChat = this.options.chat + const registryChatIdentity = this.options.chatIdentity + const registryHooks = this.options.hooks + const registryHooksIdentity = this.options.hooksIdentity + if (exact.useRegistryChat && registryChat === undefined) { + throw new TypeError( + 'ExactRegistryRunOpts.useRegistryChat is true but the registry has no chat client', + ) + } + if (exact.applyRegistryHooks && !hasRegistryHooks(registryHooks)) { + throw new TypeError( + 'ExactRegistryRunOpts.applyRegistryHooks is true but the registry has no lifecycle hooks', + ) } + const inputSnapshot = snapshotAnalystRunInputChannels(inputs) + const prepared = selected.map((analyst) => ({ + analyst, + input: this.routeInput(analyst, inputSnapshot), + })) + if (exact.missingInputMode === 'abort') { + const missing = prepared.find((candidate) => candidate.input.kind === 'missing')?.analyst + if (missing) { + throw new TypeError( + `ExactRegistryRunOpts.missingInputMode abort preflight found no "${missing.inputKind}" input for "${missing.id}"`, + ) + } + } + const hooksIdentity = + exact.applyRegistryHooks && registryHooks + ? requireExactComponentIdentity(registryHooksIdentity, 'registry hooks') + : null + const chatIdentity = exact.useRegistryChat + ? requireExactComponentIdentity(registryChatIdentity, 'registry chat') + : null + const costLedgerIdentity = + exact.costLedger === null + ? null + : requireExactComponentIdentity(exact.costLedgerIdentity ?? undefined, 'cost ledger') + const allocations = exactFixedBudgets( + exact.budget, + prepared + .filter( + ( + candidate, + ): candidate is { + analyst: ExactCapableAnalyst + input: { kind: 'present'; value: unknown } + } => candidate.input.kind === 'present', + ) + .map((candidate) => candidate.analyst), + selected, + ) + const executionSnapshot = exactExecutionSnapshot( + selected, + exact, + allocations, + costLedgerIdentity, + hooksIdentity, + chatIdentity, + ) + return { + runId: exactRunId, + prepared, + budget: { kind: 'none' }, + totalTimeoutMs: exact.totalTimeoutMs, + signal: exact.signal, + costLedger: exact.costLedger, + costPhase: exact.costPhase, + tags: exact.tags, + priorFindings: exact.priorFindings, + chainFindings: exact.chainFindings, + hooks: exact.applyRegistryHooks && registryHooks ? snapshotHooks(registryHooks) : {}, + chat: exact.useRegistryChat && registryChat ? snapshotChat(registryChat) : undefined, + // Exact runs expose live events and versioned hooks. An inherited logger is intentionally + // excluded because an anonymous callback can throw, block, or mutate shared state. + log: () => {}, + executionSnapshot, + } + } - const summaries: AnalystRunSummary[] = [] - const allFindings: AnalystFinding[] = [] - let totalCost = 0 - let remainingUsd = budget?.totalUsd + private async *executePlanStream( + plan: AnalystExecutionPlan, + ): AsyncGenerator { + const exact = plan.executionSnapshot !== undefined + if (exact && plan.signal?.aborted) throw abortReason(plan.signal) - // Budget is split only across analysts that actually run. Analysts skipped - // for missing input never spend, so counting them would under-budget the - // ones that do. routeInput is pure, so the pre-count is safe. - const runnableAnalysts = selected.filter((a) => this.routeInput(a, inputs).kind !== 'missing') - const runnableCount = runnableAnalysts.length - const weights = budget?.weights + const correlationId = `ar_${randomUUID().slice(0, 12)}` + const log = plan.log + const startedAt = new Date().toISOString() + const started = Date.now() + const timeoutSignal = + plan.totalTimeoutMs === null ? undefined : AbortSignal.timeout(plan.totalTimeoutMs) + const runSignal = combineAbortSignals(plan.signal ?? undefined, timeoutSignal) + const deadlineMs = plan.totalTimeoutMs === null ? undefined : started + plan.totalTimeoutMs + const runnable = plan.prepared + .filter( + ( + candidate, + ): candidate is { + analyst: Analyst + input: { kind: 'present'; value: unknown } + } => candidate.input.kind === 'present', + ) + .map((candidate) => candidate.analyst) + let remainingUsd = plan.budget.kind === 'dynamic' ? plan.budget.policy.totalUsd : undefined + const weights = plan.budget.kind === 'dynamic' ? plan.budget.policy.weights : undefined const totalWeight = - weights && budget?.totalUsd != null && !budget.allocate && runnableCount > 0 - ? runnableAnalysts.reduce((sum, analyst) => sum + analystWeight(weights, analyst.id), 0) + weights && + plan.budget.kind === 'dynamic' && + plan.budget.policy.totalUsd != null && + !plan.budget.policy.allocate && + runnable.length > 0 + ? runnable.reduce((sum, analyst) => sum + analystWeight(weights, analyst.id), 0) : undefined if (totalWeight === 0) { throw new Error('BudgetPolicy.weights must allocate positive weight to a runnable analyst') } + const upstreamFindings: AnalystFinding[] = [] + + yield snapshotExecutionEvent( + { + type: 'run-started', + run_id: plan.runId, + correlation_id: correlationId, + started_at: startedAt, + analyst_ids: plan.prepared.map(({ analyst }) => analyst.id), + ...(plan.executionSnapshot === undefined ? {} : { execution_plan: plan.executionSnapshot }), + }, + exact, + ) + + const executions: AnalystExecution[] = [] + let executionFailure: unknown - for (const analyst of selected) { + for (const { analyst, input } of plan.prepared) { const t0 = Date.now() if (runSignal?.aborted) { const summary = abortedBeforeStartSummary(analyst, runSignal) - summaries.push(summary) - log(`[analyst] skip ${analyst.id} — run aborted`, { runId, reason: summary.reason }) - yield { type: 'analyst-skipped', summary } + executions.push({ summary, findings: [], budgetDebitUsd: 0 }) + log(`[analyst] skip ${analyst.id} — run aborted`, { + runId: plan.runId, + reason: summary.reason, + }) + yield snapshotExecutionEvent({ type: 'analyst-skipped', summary }, exact) + if (exact) { + executionFailure = abortReason(runSignal) + break + } continue } - const input = this.routeInput(analyst, inputs) + if (input.kind === 'missing') { const summary: AnalystRunSummary = { analyst_id: analyst.id, @@ -257,177 +534,351 @@ export class AnalystRegistry { latency_ms: 0, usage: zeroUsage(), } - summaries.push(summary) - log(`[analyst] skip ${analyst.id} — missing input`, { runId, kind: analyst.inputKind }) - await waitForHook( - hooks.onAfterAnalyze - ? () => hooks.onAfterAnalyze?.({ analyst, summary, findings: [], runId }) - : undefined, - runSignal, - ) - yield { type: 'analyst-skipped', summary } + const execution = { summary, findings: [], budgetDebitUsd: 0 } satisfies AnalystExecution + executions.push(execution) + log(`[analyst] skip ${analyst.id} — missing input`, { + runId: plan.runId, + kind: analyst.inputKind, + }) + const hookValues = snapshotAfterHookValues(summary, [], exact) + try { + await waitForHook( + plan.hooks.onAfterAnalyze + ? () => + plan.hooks.onAfterAnalyze?.({ + analyst, + summary: hookValues.summary, + findings: hookValues.findings, + runId: plan.runId, + }) + : undefined, + runSignal, + ) + } catch (error) { + if (!exact) throw error + executionFailure = error + } + yield snapshotExecutionEvent({ type: 'analyst-skipped', summary }, exact) + if (executionFailure !== undefined) break continue } - const perBudget = allocateBudget(budget, { - analyst, - remainingUsd, - runningCount: runnableCount, - totalWeight, - }) + const allocatedUsd = + plan.executionSnapshot === undefined + ? allocateBudget(plan.budget.kind === 'dynamic' ? plan.budget.policy : undefined, { + analyst, + remainingUsd, + runningCount: runnable.length, + totalWeight, + }) + : exactPlannedAllocation(plan.executionSnapshot, analyst.id) + const budgetCeilingUsd = plan.executionSnapshot === undefined ? remainingUsd : allocatedUsd const usageReceipts: AnalystUsageReceipt[] = [] - + const contextTags = plan.tags === null ? undefined : { ...plan.tags } + const priorFindings = selectPriorFindings(plan.priorFindings ?? undefined, analyst.id) + const chainedFindings = + plan.chainFindings && upstreamFindings.length > 0 ? [...upstreamFindings] : undefined const ctx: AnalystContext = { - runId, + runId: plan.runId, correlationId, deadlineMs, - budgetUsd: perBudget, - costLedger: runOpts.costLedger, - costPhase: runOpts.costPhase, - chat: this.options.chat, - tags: runOpts.tags, - log: (msg, fields) => log(`[${analyst.id}] ${msg}`, { runId, correlationId, ...fields }), + budgetUsd: allocatedUsd, + costLedger: plan.costLedger ?? undefined, + costPhase: plan.costPhase ?? undefined, + chat: plan.chat, + tags: contextTags, + log: (message, fields) => + log(`[${analyst.id}] ${message}`, { + runId: plan.runId, + correlationId, + ...fields, + }), signal: runSignal, - priorFindings: selectPriorFindings(runOpts.priorFindings, analyst.id), - upstreamFindings: - runOpts.chainFindings && allFindings.length > 0 ? [...allFindings] : undefined, + priorFindings, + upstreamFindings: chainedFindings, recordUsage: (receipt) => { - assertValidAnalystUsageReceipt(receipt) - usageReceipts.push(receipt) + if (!exact) { + assertValidAnalystUsageReceipt(receipt) + usageReceipts.push(receipt) + return + } + usageReceipts.push( + snapshotUsageReceiptOnce( + receipt, + `AnalystRegistry.runExact analyst "${analyst.id}" usage`, + ), + ) }, } + if (exact) { + if (contextTags) deepFreezeCanonicalJson(contextTags) + if (priorFindings) deepFreezeCanonicalJson(priorFindings) + if (chainedFindings) deepFreezeCanonicalJson(chainedFindings) + Object.freeze(ctx) + } - await waitForHook( - hooks.onBeforeAnalyze ? () => hooks.onBeforeAnalyze?.({ analyst, ctx, runId }) : undefined, - runSignal, - ) + try { + await waitForHook( + plan.hooks.onBeforeAnalyze + ? () => plan.hooks.onBeforeAnalyze?.({ analyst, ctx, runId: plan.runId }) + : undefined, + runSignal, + ) + } catch (error) { + if (!exact) throw error + executionFailure = error + break + } if (runSignal?.aborted) { const summary = abortedBeforeStartSummary(analyst, runSignal, Date.now() - t0) - summaries.push(summary) - log(`[analyst] skip ${analyst.id} — run aborted`, { runId, reason: summary.reason }) - yield { type: 'analyst-skipped', summary } + executions.push({ summary, findings: [], budgetDebitUsd: 0 }) + yield snapshotExecutionEvent({ type: 'analyst-skipped', summary }, exact) + if (exact) { + executionFailure = abortReason(runSignal) + break + } continue } - const effectiveBudget = validateEffectiveBudget(ctx.budgetUsd, remainingUsd, analyst.id) - yield { - type: 'analyst-started', - analyst_id: analyst.id, - started_at: new Date(t0).toISOString(), + let effectiveBudget: number | undefined + try { + effectiveBudget = validateEffectiveBudget(ctx.budgetUsd, budgetCeilingUsd, analyst.id) + } catch (error) { + if (!exact) throw error + executionFailure = error + break } + const analystContext: AnalystContext = exact ? ctx : { ...ctx } + const executionSignal = exact ? ctx.signal : runSignal + yield snapshotExecutionEvent( + { + type: 'analyst-started', + analyst_id: analyst.id, + started_at: new Date(t0).toISOString(), + }, + exact, + ) let findings: AnalystFinding[] let summary: AnalystRunSummary + let lifecycleFailure: unknown + let analysisFailure: Error | undefined try { if (runSignal?.aborted) throw abortReason(runSignal) - findings = await waitForOperation( - (analyst as Analyst).analyze(input.value, ctx), - runSignal, + const analyzed = await waitForOperation( + (analyst as Analyst).analyze(input.value, analystContext), + executionSignal, analystAbortGraceMs(analyst), ) - const latency = Date.now() - t0 - const usage = resolveUsage(analyst, usageReceipts) - const cost = knownCostUsd(usage) - totalCost += cost - if (typeof remainingUsd === 'number') { - remainingUsd = Math.max(0, remainingUsd - budgetDebit(usage, effectiveBudget)) + findings = snapshotExecutionFindings( + analyzed, + exact, + `AnalystRegistry.runExact analyst "${analyst.id}" findings`, + ) + } catch (error) { + const cause = error instanceof Error ? error : new Error(String(error)) + analysisFailure = cause + let hookFindings: AnalystFinding[] = [] + if (!executionSignal?.aborted) { + try { + const rawHookFindings = + (await waitForHook( + plan.hooks.onError + ? () => + plan.hooks.onError?.({ + analyst, + error: cause, + runId: plan.runId, + }) + : undefined, + executionSignal, + )) ?? [] + hookFindings = snapshotExecutionFindings( + rawHookFindings, + exact, + `AnalystRegistry.runExact analyst "${analyst.id}" onError findings`, + ) + } catch (error) { + lifecycleFailure = error + } } - allFindings.push(...findings) + findings = hookFindings + } + + let usage: AnalystUsageReceipt + try { + usage = resolveUsage(analyst, usageReceipts, exact) + } catch (error) { + if (!exact) throw error + executionFailure = error + break + } + if (analysisFailure === undefined) { summary = { analyst_id: analyst.id, status: 'ok', findings_count: findings.length, - latency_ms: latency, + latency_ms: Date.now() - t0, usage, + ...(exact ? { allocated_budget_usd: effectiveBudget ?? null } : {}), } - summaries.push(summary) log(`[analyst] ok ${analyst.id}`, { - runId, + runId: plan.runId, findings: findings.length, - latency_ms: latency, - cost_usd: cost, + latency_ms: summary.latency_ms, + cost_usd: knownCostUsd(usage), cost_kind: usage.cost.kind, input_tokens: usage.tokens?.input ?? null, output_tokens: usage.tokens?.output ?? null, }) - if (effectiveBudget !== undefined && usage.cost.kind === 'uncaptured') { - log(`[analyst] WARN ${analyst.id} — USD cost uncaptured; budget not reconciled`, { - runId, - budget_usd: effectiveBudget, - cost_captured: false, - }) - } - } catch (err) { - const latency = Date.now() - t0 - const e = err instanceof Error ? err : new Error(String(err)) - // Hook gets first chance to convert the error into findings. - const hookFindings = runSignal?.aborted - ? [] - : ((await hooks.onError?.({ analyst, error: e, runId })) ?? []) - if (hookFindings.length) allFindings.push(...hookFindings) - const usage = resolveUsage(analyst, usageReceipts) - const cost = knownCostUsd(usage) - totalCost += cost - if (typeof remainingUsd === 'number') { - remainingUsd = Math.max(0, remainingUsd - budgetDebit(usage, effectiveBudget)) - } - const summary: AnalystRunSummary = { + } else { + const errorClass = analysisFailure.constructor.name || 'Error' + const errorMessage = + exact && analysisFailure.message.length === 0 + ? 'Analyst failed without an error message' + : analysisFailure.message + summary = { analyst_id: analyst.id, status: 'failed', - findings_count: hookFindings.length, - latency_ms: latency, + findings_count: findings.length, + latency_ms: Date.now() - t0, usage, - error: { class: e.constructor.name, message: e.message }, + ...(exact ? { allocated_budget_usd: effectiveBudget ?? null } : {}), + error: { class: errorClass, message: errorMessage }, } - summaries.push(summary) log(`[analyst] FAIL ${analyst.id}`, { - runId, - error_class: e.constructor.name, - error: e.message, - cost_usd: cost, + runId: plan.runId, + error_class: errorClass, + error: errorMessage, + cost_usd: knownCostUsd(usage), cost_kind: usage.cost.kind, }) - if (effectiveBudget !== undefined && usage.cost.kind === 'uncaptured') { - log(`[analyst] WARN ${analyst.id} — USD cost uncaptured; budget not reconciled`, { - runId, - budget_usd: effectiveBudget, - cost_captured: false, - }) + } + logUncapturedBudgetWarning({ + analyst, + runId: plan.runId, + budgetUsd: effectiveBudget, + usage, + log, + }) + + const execution = { + summary, + findings, + budgetDebitUsd: budgetDebit(summary.usage, effectiveBudget), + } satisfies AnalystExecution + if (exact) { + try { + executionCost([...executions, execution], true) + } catch (error) { + executionFailure = error + break } + } + executions.push(execution) + if (plan.budget.kind === 'dynamic' && remainingUsd !== undefined) { + remainingUsd = Math.max(0, remainingUsd - execution.budgetDebitUsd) + } + if (plan.chainFindings) upstreamFindings.push(...findings) + if (lifecycleFailure !== undefined) { + if (!exact) throw lifecycleFailure + executionFailure = lifecycleFailure + break + } + + const hookValues = snapshotAfterHookValues(summary, findings, exact) + try { await waitForHook( - hooks.onAfterAnalyze - ? () => hooks.onAfterAnalyze?.({ analyst, summary, findings: hookFindings, runId }) + plan.hooks.onAfterAnalyze + ? () => + plan.hooks.onAfterAnalyze?.({ + analyst, + summary: hookValues.summary, + findings: hookValues.findings, + runId: plan.runId, + }) : undefined, - runSignal, + executionSignal, ) - yield { type: 'analyst-completed', summary, findings: hookFindings } - continue + } catch (error) { + if (!exact) throw error + executionFailure = error + break + } + yield snapshotExecutionEvent({ type: 'analyst-completed', summary, findings }, exact) + if (exact && runSignal?.aborted) { + executionFailure = abortReason(runSignal) + break } - await waitForHook( - hooks.onAfterAnalyze - ? () => hooks.onAfterAnalyze?.({ analyst, summary, findings, runId }) - : undefined, - runSignal, - ) - yield { type: 'analyst-completed', summary, findings } } - const result: AnalystRunResult = { - run_id: runId, + const summaries = executions.map(({ summary }) => summary) + const findings = executions.flatMap((execution) => execution.findings) + const cost = executionCost(executions, exact) + const baseResult: AnalystRunResult = { + run_id: plan.runId, correlation_id: correlationId, started_at: startedAt, ended_at: new Date().toISOString(), - findings: allFindings, + findings, per_analyst: summaries, - total_cost_usd: totalCost, - total_cost_provenance: aggregateCostProvenance( - summaries.map((summary) => summary.usage?.cost ?? { kind: 'uncaptured', usd: null }), - ), - } - await waitForHook( - hooks.onComplete ? () => hooks.onComplete?.({ result }) : undefined, - runSignal, + total_cost_usd: cost.known, + total_cost_provenance: cost.provenance, + } + if (plan.executionSnapshot === undefined) { + await waitForHook( + plan.hooks.onComplete ? () => plan.hooks.onComplete?.({ result: baseResult }) : undefined, + runSignal, + ) + yield { type: 'run-completed', result: baseResult } + return + } + + let completeResult: ExactAnalystRunResult | undefined + if (executionFailure === undefined) { + try { + completeResult = snapshotExactAnalystRunReceipt( + { + ...baseResult, + execution_plan: plan.executionSnapshot, + completion: { status: 'complete' }, + }, + 'AnalystRegistry.runExact result', + ) + await waitForHook( + plan.hooks.onComplete + ? () => plan.hooks.onComplete?.({ result: completeResult! }) + : undefined, + runSignal, + ) + } catch (error) { + executionFailure = error + } + } + if (runSignal?.aborted) executionFailure ??= abortReason(runSignal) + if (executionFailure === undefined && completeResult) { + yield snapshotExecutionEvent({ type: 'run-completed', result: completeResult }, true) + return + } + + const cause = + executionFailure instanceof Error ? executionFailure : new Error(String(executionFailure)) + const errorClass = cause.constructor.name || 'Error' + const errorMessage = + cause.message.trim().length === 0 + ? 'Exact analyst run failed without a message' + : cause.message + throw new ExactAnalystRunExecutionError( + `exact analyst run failed after starting: ${errorMessage}; partial result is attached`, + { + ...baseResult, + execution_plan: plan.executionSnapshot, + completion: { + status: 'failed', + error: { class: errorClass, message: errorMessage }, + }, + }, + { cause }, ) - yield { type: 'run-completed', result } } private selectAnalysts(opts: RegistryRunOpts): Analyst[] { @@ -443,31 +894,531 @@ export class AnalystRegistry { return candidates } + private selectExactAnalysts( + ids: readonly string[], + ): ReadonlyArray<{ readonly registeredId: string; readonly analyst: Analyst }> { + return ids.map((id) => { + const analyst = this.analysts.get(id) + if (!analyst) throw new Error(`ExactRegistryRunOpts.analystIds names unknown analyst "${id}"`) + return { registeredId: id, analyst } + }) + } + private routeInput( analyst: Analyst, inputs: AnalystRunInputs, ): { kind: 'present'; value: unknown } | { kind: 'missing' } { switch (analyst.inputKind) { - case 'trace-store': - return inputs.traceStore - ? { kind: 'present', value: inputs.traceStore } - : { kind: 'missing' } - case 'artifact-dir': - return inputs.artifactDir - ? { kind: 'present', value: inputs.artifactDir } - : { kind: 'missing' } - case 'run-record': - return inputs.runRecord ? { kind: 'present', value: inputs.runRecord } : { kind: 'missing' } - case 'judge-input': - return inputs.judgeInput - ? { kind: 'present', value: inputs.judgeInput } - : { kind: 'missing' } + case 'trace-store': { + const value = inputs.traceStore + return value ? { kind: 'present', value } : { kind: 'missing' } + } + case 'artifact-dir': { + const value = inputs.artifactDir + return value ? { kind: 'present', value } : { kind: 'missing' } + } + case 'run-record': { + const value = inputs.runRecord + return value ? { kind: 'present', value } : { kind: 'missing' } + } + case 'judge-input': { + const value = inputs.judgeInput + return value ? { kind: 'present', value } : { kind: 'missing' } + } case 'custom': { - const v = inputs.custom?.[analyst.id] - return v !== undefined ? { kind: 'present', value: v } : { kind: 'missing' } + const custom = inputs.custom + const value = custom?.[analyst.id] + return value !== undefined ? { kind: 'present', value } : { kind: 'missing' } + } + } + } +} + +const exactRunFields = [ + 'analystIds', + 'budget', + 'totalTimeoutMs', + 'signal', + 'costLedger', + 'costLedgerIdentity', + 'costPhase', + 'tags', + 'priorFindings', + 'chainFindings', + 'missingInputMode', + 'applyRegistryHooks', + 'useRegistryChat', +] as const + +const exactNonEmptyString = z.string().min(1) +const exactFiniteNonnegative = z.number().finite().nonnegative() +const exactBudgetSchema = z.discriminatedUnion('kind', [ + z.strictObject({ + kind: z.literal('equal'), + totalUsd: exactFiniteNonnegative, + }), + z.strictObject({ + kind: z.literal('weighted'), + totalUsd: exactFiniteNonnegative, + weights: z.record(exactNonEmptyString, exactFiniteNonnegative), + }), +]) +const exactRunDataSchema = z + .strictObject({ + analystIds: z.array(exactNonEmptyString).min(1), + budget: exactBudgetSchema.nullable(), + totalTimeoutMs: z.number().int().positive().max(2_147_483_647).nullable(), + costLedgerIdentity: z.unknown().nullable(), + costPhase: exactNonEmptyString.nullable(), + tags: z.record(z.string(), z.string()).nullable(), + chainFindings: z.boolean(), + missingInputMode: z.enum(['skip', 'abort']), + applyRegistryHooks: z.boolean(), + useRegistryChat: z.boolean(), + }) + .superRefine((policy, context) => { + const issue = (path: PropertyKey[], message: string): void => + context.addIssue({ code: 'custom', path, message }) + if (new Set(policy.analystIds).size !== policy.analystIds.length) { + issue(['analystIds'], 'must not contain duplicates') + } + if ( + policy.budget?.kind === 'weighted' && + Object.values(policy.budget.weights).every((weight) => weight === 0) + ) { + issue(['budget', 'weights'], 'must allocate positive weight to at least one analyst') + } + if (policy.budget?.kind === 'weighted') { + const selected = [...policy.analystIds].sort() + const weighted = Object.keys(policy.budget.weights).sort() + if ( + selected.length !== weighted.length || + selected.some((id, index) => id !== weighted[index]) + ) { + issue(['budget', 'weights'], 'must name every selected analyst and no others') } } + }) + +/** Validate the canonical exact-run policy before any analyst can start. */ +export function assertExactRegistryRunOpts(value: unknown): asserts value is ExactRegistryRunOpts { + void snapshotExactRegistryRunOpts(value) +} + +function snapshotExactRunId(value: unknown): string { + if (typeof value !== 'string' || value.length === 0) { + throw new TypeError('AnalystRegistry.runExact: runId must be a non-empty string') + } + return canonicalJsonSnapshot(value, 'AnalystRegistry.runExact runId') +} + +function snapshotAnalystRunInputChannels(inputs: AnalystRunInputs): AnalystRunInputs { + if (!inputs || typeof inputs !== 'object' || Array.isArray(inputs)) { + throw new TypeError('AnalystRegistry.runExact: inputs must be an object') + } + const traceStore = inputs.traceStore + const artifactDir = inputs.artifactDir + const runRecord = inputs.runRecord + const judgeInput = inputs.judgeInput + const custom = inputs.custom + return Object.freeze({ + traceStore, + artifactDir, + runRecord, + judgeInput, + custom, + }) +} + +/** + * Read the untrusted caller object once, then validate and execute only this frozen snapshot. + * Functions and resource handles retain identity; all data fields are copied canonically. + */ +function snapshotExactRegistryRunOpts(value: unknown): ExactRegistryRunOpts { + const captured = readOwnFields(value, exactRunFields, 'ExactRegistryRunOpts') + const missing = exactRunFields.find((field) => !Object.hasOwn(captured, field)) + if (missing) { + throw new TypeError(`ExactRegistryRunOpts.${missing} must be supplied explicitly`) + } + const { signal, costLedger, priorFindings, ...rawData } = captured + const data = canonicalJsonSnapshot(rawData, 'ExactRegistryRunOpts') + const parsed = exactRunDataSchema.safeParse(data) + if (!parsed.success) { + const issue = parsed.error.issues[0] + if (issue?.code === 'unrecognized_keys' && issue.path.join('.') === 'budget') { + const required = + isPlainRecord(data.budget) && data.budget.kind === 'weighted' + ? 'kind, totalUsd, weights' + : 'kind, totalUsd' + throw new TypeError(`ExactRegistryRunOpts.budget must contain exactly ${required}`) + } + const path = issue?.path.length ? `.${issue.path.join('.')}` : '' + throw new TypeError(`ExactRegistryRunOpts${path}: ${issue?.message ?? 'is invalid'}`) + } + if ( + signal !== null && + (!signal || + typeof signal !== 'object' || + typeof (signal as AbortSignal).addEventListener !== 'function') + ) { + throw new TypeError('ExactRegistryRunOpts.signal must be an AbortSignal or null') + } + if (costLedger !== null && (!costLedger || typeof costLedger !== 'object')) { + throw new TypeError('ExactRegistryRunOpts.costLedger must be a CostLedgerHandle or null') + } + if (costLedger === null && parsed.data.costLedgerIdentity !== null) { + throw new TypeError('ExactRegistryRunOpts.costLedgerIdentity must be null without costLedger') + } + if (costLedger !== null && parsed.data.costLedgerIdentity === null) { + throw new TypeError('ExactRegistryRunOpts.costLedgerIdentity is required with costLedger') + } + if (costLedger === null && parsed.data.costPhase !== null) { + throw new TypeError('ExactRegistryRunOpts.costPhase requires a non-null costLedger') + } + return Object.freeze({ + ...deepFreezeCanonicalJson(parsed.data), + signal: signal as AbortSignal | null, + costLedger: costLedger as CostLedgerHandle | null, + priorFindings: snapshotExactPriorFindings(priorFindings), + }) as ExactRegistryRunOpts +} + +function snapshotExactPriorFindings(value: unknown): ExactRegistryRunOpts['priorFindings'] { + if (value === null) return null + if (Array.isArray(value)) { + return snapshotAnalystFindings(value, 'ExactRegistryRunOpts.priorFindings') + } + if (!isPlainRecord(value)) { + throw new TypeError( + 'ExactRegistryRunOpts.priorFindings must be an array, a findings record, or null', + ) + } + const result: Record> = {} + for (const [key, findings] of Object.entries(value)) { + if (!Array.isArray(findings)) { + throw new TypeError(`ExactRegistryRunOpts.priorFindings.${key} must be an array`) + } + result[key] = snapshotAnalystFindings(findings, `ExactRegistryRunOpts.priorFindings.${key}`) + } + return deepFreezeCanonicalJson(result) +} + +function normalizeExactAnalysts( + selections: ReadonlyArray<{ readonly registeredId: string; readonly analyst: Analyst }>, +): ExactCapableAnalyst[] { + return selections.map(({ registeredId, analyst }) => { + const exactAnalyst = analyst as Analyst & { + readonly executionConfig?: Readonly> + } + const id = analyst.id + const description = analyst.description + const inputKind = analyst.inputKind + const rawCostValue = analyst.cost + const requiresValue = analyst.requires + const version = analyst.version + const executionConfigValue = exactAnalyst.executionConfig + const analyzeValue = analyst.analyze + if (id !== registeredId) { + throw new TypeError( + `AnalystRegistry.runExact: registered analyst "${registeredId}" changed id to "${id}"`, + ) + } + if (executionConfigValue === undefined) { + throw new TypeError(`AnalystRegistry.runExact: analyst "${id}" must declare executionConfig`) + } + const executionConfig = canonicalJsonSnapshot( + executionConfigValue, + `AnalystRegistry.runExact analyst "${id}" executionConfig`, + ) + if (!isPlainRecord(executionConfig)) { + throw new TypeError( + `AnalystRegistry.runExact analyst "${id}" executionConfig must be an object`, + ) + } + const rawCost = canonicalJsonSnapshot( + rawCostValue, + `AnalystRegistry.runExact analyst "${id}" cost`, + ) + const cost = + rawCost.kind === 'llm' + ? Object.freeze({ + ...rawCost, + settlement_timeout_ms: validateUsageSettlementTimeout(rawCost.settlement_timeout_ms), + }) + : rawCost + const requires = + requiresValue === undefined + ? undefined + : canonicalJsonSnapshot( + requiresValue, + `AnalystRegistry.runExact analyst "${id}" requirements`, + ) + const analyze = analyzeValue.bind(analyst) + return Object.freeze({ + id, + description, + inputKind, + cost, + ...(requires === undefined ? {} : { requires }), + version, + executionConfig, + analyze, + }) satisfies ExactCapableAnalyst + }) +} + +function hasRegistryHooks(hooks: AnalystHooks | undefined): hooks is AnalystHooks { + return Boolean( + hooks && (hooks.onBeforeAnalyze || hooks.onAfterAnalyze || hooks.onError || hooks.onComplete), + ) +} + +function snapshotHooks(hooks: AnalystHooks): AnalystHooks { + const onBeforeAnalyze = hooks.onBeforeAnalyze + const onAfterAnalyze = hooks.onAfterAnalyze + const onError = hooks.onError + const onComplete = hooks.onComplete + return Object.freeze({ + ...(onBeforeAnalyze === undefined ? {} : { onBeforeAnalyze: onBeforeAnalyze.bind(hooks) }), + ...(onAfterAnalyze === undefined ? {} : { onAfterAnalyze: onAfterAnalyze.bind(hooks) }), + ...(onError === undefined ? {} : { onError: onError.bind(hooks) }), + ...(onComplete === undefined ? {} : { onComplete: onComplete.bind(hooks) }), + }) +} + +function snapshotChat(chat: ChatClient): ChatClient { + const transport = chat.transport + const defaultModel = chat.defaultModel + const maximumAttempts = chat.maximumAttempts + const call = chat.chat + return Object.freeze({ + transport, + ...(defaultModel === undefined ? {} : { defaultModel }), + ...(maximumAttempts === undefined ? {} : { maximumAttempts }), + chat: call.bind(chat), + }) +} + +function requireExactComponentIdentity( + value: ExactExecutionComponentIdentity | undefined, + label: string, +): ExactExecutionComponentSnapshot { + if (value === undefined) { + throw new TypeError(`AnalystRegistry.runExact: ${label} requires a versioned identity`) + } + return snapshotExactExecutionComponentIdentity( + value, + `AnalystRegistry.runExact ${label} identity`, + ) +} + +function exactExecutionSnapshot( + analysts: readonly ExactCapableAnalyst[], + opts: ExactRegistryRunOpts, + allocations: Readonly>, + costLedger: ExactExecutionComponentSnapshot | null, + hooks: ExactExecutionComponentSnapshot | null, + chat: ExactExecutionComponentSnapshot | null, +): ExactAnalystExecutionPlanSnapshot { + const priorFindings = exactPriorFindingsSnapshot(opts.priorFindings) + const budget = + opts.budget === null + ? ({ kind: 'none' } as const) + : opts.budget.kind === 'equal' + ? ({ + kind: 'equal', + total_usd: opts.budget.totalUsd, + allocations_usd: { ...allocations }, + } as const) + : ({ + kind: 'weighted', + total_usd: opts.budget.totalUsd, + weights: { ...opts.budget.weights }, + allocations_usd: { ...allocations }, + } as const) + const material = { + schema_version: '1.0.0' as const, + analysts: analysts.map((analyst) => ({ + id: analyst.id, + version: analyst.version, + input_kind: analyst.inputKind, + cost: analyst.cost, + requirements: analyst.requires ?? null, + execution_config_digest: hashCanonical(analyst.executionConfig), + })), + policy: { + budget, + total_timeout_ms: opts.totalTimeoutMs, + signal_provided: opts.signal !== null, + cost_ledger: costLedger, + cost_phase: opts.costPhase, + tags: opts.tags === null ? null : { ...opts.tags }, + prior_findings: priorFindings, + chain_findings: opts.chainFindings, + missing_input_mode: opts.missingInputMode, + registry_hooks: hooks, + registry_chat: chat, + }, + } + return snapshotExactExecutionPlan( + { ...material, digest: hashCanonical(material) }, + 'AnalystRegistry.runExact execution plan', + ) +} + +function exactPriorFindingsSnapshot( + findings: ExactRegistryRunOpts['priorFindings'], +): ExactAnalystExecutionPlanSnapshot['policy']['prior_findings'] { + if (findings === null) return { kind: 'none' } + if (Array.isArray(findings)) { + return { + kind: 'ordered', + count: findings.length, + digest: hashCanonical(findings), + } + } + const record = findings as Readonly>> + const keys = Object.keys(record).sort() + return { + kind: 'by_analyst', + keys, + count: keys.reduce((sum, key) => sum + (record[key]?.length ?? 0), 0), + digest: hashCanonical(record), + } +} + +function canonicalJsonSnapshot(value: T, label: string): T { + let snapshot: T + try { + snapshot = JSON.parse(canonicalString(value)) as T + } catch (cause) { + throw new TypeError(`${label} must be canonical JSON`, { cause }) } + return deepFreezeCanonicalJson(snapshot) +} + +function snapshotUsageReceiptOnce( + receipt: AnalystUsageReceipt, + context: string, +): AnalystUsageReceipt { + const data = readOwnFields(receipt, ['calls', 'tokens', 'cost', 'knownCostUsd'], context) + data.tokens = + data.tokens === null + ? null + : readOwnFields( + data.tokens, + ['input', 'output', 'reasoning', 'cached', 'cacheWrite'], + `${context} tokens`, + ) + data.cost = readOwnFields(data.cost, ['kind', 'usd'], `${context} cost`) + const snapshot = canonicalJsonSnapshot(data as unknown as AnalystUsageReceipt, context) + assertValidAnalystUsageReceipt(snapshot, context) + return snapshot +} + +function readOwnFields( + value: unknown, + fields: readonly string[], + context: string, +): Record { + if (!isPlainRecord(value)) throw new TypeError(`${context} must be a plain object`) + const unexpected = Object.keys(value).filter((key) => !fields.includes(key)) + if (unexpected.length > 0) { + throw new TypeError(`${context} contains unknown fields: ${unexpected.sort().join(', ')}`) + } + return Object.fromEntries( + fields.flatMap((field) => (Object.hasOwn(value, field) ? [[field, value[field]]] : [])), + ) +} + +function isPlainRecord(value: unknown): value is Record { + if (!value || typeof value !== 'object' || Array.isArray(value)) return false + const prototype = Object.getPrototypeOf(value) + return prototype === Object.prototype || prototype === null +} + +function exactFixedBudgets( + exact: ExactAnalystBudgetPolicy | null, + runnable: readonly Analyst[], + selected: readonly Analyst[], +): Readonly> { + if (exact === null) return {} + const allocations: Record = Object.fromEntries( + selected.map((analyst) => [analyst.id, null]), + ) + if (runnable.length === 0) return deepFreezeCanonicalJson(allocations) + if (exact.kind === 'equal') { + const each = exact.totalUsd / runnable.length + for (const analyst of runnable) allocations[analyst.id] = each + return deepFreezeCanonicalJson(allocations) + } + const totalWeight = runnable.reduce((sum, analyst) => sum + exact.weights[analyst.id]!, 0) + if (totalWeight === 0) { + throw new Error( + 'ExactRegistryRunOpts weighted budget must allocate positive weight to a runnable analyst', + ) + } + for (const analyst of runnable) { + allocations[analyst.id] = (exact.totalUsd * exact.weights[analyst.id]!) / totalWeight + } + return deepFreezeCanonicalJson(allocations) +} + +interface AnalystExecution { + readonly summary: AnalystRunSummary + readonly findings: AnalystFinding[] + readonly budgetDebitUsd: number +} + +function snapshotExecutionFindings( + findings: AnalystFinding[], + exact: boolean, + context: string, +): AnalystFinding[] { + return exact ? deepFreezeCanonicalJson(snapshotAnalystFindings(findings, context)) : findings +} + +function snapshotAfterHookValues( + summary: AnalystRunSummary, + findings: AnalystFinding[], + exact: boolean, +): { summary: AnalystRunSummary; findings: AnalystFinding[] } { + if (!exact) return { summary, findings } + return { + summary: canonicalJsonSnapshot(summary, 'AnalystRegistry.runExact onAfterAnalyze summary'), + findings: deepFreezeCanonicalJson( + snapshotAnalystFindings(findings, 'AnalystRegistry.runExact onAfterAnalyze findings'), + ), + } +} + +function exactPlannedAllocation( + plan: ExactAnalystExecutionPlanSnapshot, + analystId: string, +): number | undefined { + const budget = plan.policy.budget + if (budget.kind === 'none') return undefined + const allocated = budget.allocations_usd[analystId] + return allocated === null ? undefined : allocated +} + +function snapshotExecutionEvent(event: T, exact: boolean): T { + return exact ? canonicalJsonSnapshot(event, 'AnalystRegistry.runExact event') : event +} + +function logUncapturedBudgetWarning(args: { + analyst: Analyst + runId: string + budgetUsd: number | undefined + usage: AnalystUsageReceipt + log: (message: string, fields?: Record) => void +}): void { + if (args.budgetUsd === undefined || args.usage.cost.kind !== 'uncaptured') return + args.log(`[analyst] WARN ${args.analyst.id} — USD cost uncaptured; budget not reconciled`, { + runId: args.runId, + budget_usd: args.budgetUsd, + cost_captured: false, + }) } function validateTimeout(timeoutMs: number | undefined): number | undefined { @@ -664,42 +1615,57 @@ function zeroUsage(): AnalystUsageReceipt { function resolveUsage( analyst: Analyst, receipts: ReadonlyArray, + exact = false, ): AnalystUsageReceipt { - if (receipts.length > 0) return mergeUsageReceipts(receipts) + if (receipts.length > 0) return mergeUsageReceipts(receipts, exact) if (analyst.cost.kind === 'deterministic') return zeroUsage() return { calls: null, tokens: null, cost: { kind: 'uncaptured', usd: null } } } -function mergeUsageReceipts(receipts: ReadonlyArray): AnalystUsageReceipt { +function mergeUsageReceipts( + receipts: ReadonlyArray, + exact = false, +): AnalystUsageReceipt { const calls = receipts.every((receipt) => receipt.calls !== null) - ? receipts.reduce((sum, receipt) => sum + (receipt.calls ?? 0), 0) + ? usageSum( + receipts.map((receipt) => receipt.calls ?? 0), + exact, + 'calls', + true, + ) : null const tokens = receipts.every((receipt) => receipt.tokens !== null) - ? receipts.reduce( - (sum, receipt) => ({ - input: sum.input + (receipt.tokens?.input ?? 0), - output: sum.output + (receipt.tokens?.output ?? 0), - ...(sum.reasoning !== undefined || receipt.tokens?.reasoning !== undefined - ? { reasoning: (sum.reasoning ?? 0) + (receipt.tokens?.reasoning ?? 0) } - : {}), - ...(sum.cached !== undefined || receipt.tokens?.cached !== undefined - ? { cached: (sum.cached ?? 0) + (receipt.tokens?.cached ?? 0) } - : {}), - ...(sum.cacheWrite !== undefined || receipt.tokens?.cacheWrite !== undefined - ? { cacheWrite: (sum.cacheWrite ?? 0) + (receipt.tokens?.cacheWrite ?? 0) } - : {}), - }), - { input: 0, output: 0 }, - ) + ? (Object.fromEntries( + (['input', 'output', 'reasoning', 'cached', 'cacheWrite'] as const).flatMap((field) => + field === 'input' || + field === 'output' || + receipts.some((receipt) => receipt.tokens?.[field] !== undefined) + ? [ + [ + field, + usageSum( + receipts.map((receipt) => receipt.tokens?.[field] ?? 0), + exact, + `tokens.${field}`, + true, + ), + ], + ] + : [], + ), + ) as unknown as RunTokenUsage) : null - const cost = aggregateCostProvenance(receipts.map((receipt) => receipt.cost)) + const cost = aggregateCostProvenance( + receipts.map((receipt) => receipt.cost), + exact, + ) return { calls, tokens, cost, ...(cost.kind === 'uncaptured' ? { - knownCostUsd: receipts.reduce((sum, receipt) => sum + knownCostUsd(receipt), 0), + knownCostUsd: usageSum(receipts.map(knownCostUsd), exact, 'known cost'), } : {}), } @@ -716,16 +1682,52 @@ function budgetDebit(receipt: AnalystUsageReceipt, allocatedUsd: number | undefi : known } -function aggregateCostProvenance(costs: ReadonlyArray): RunCostProvenance { +function aggregateCostProvenance( + costs: ReadonlyArray, + exact = false, +): RunCostProvenance { if (costs.some((cost) => cost.kind === 'uncaptured')) { return { kind: 'uncaptured', usd: null } } - const usd = costs.reduce((sum, cost) => sum + (cost.usd ?? 0), 0) + const usd = usageSum( + costs.map((cost) => cost.usd ?? 0), + exact, + 'captured cost', + ) return costs.some((cost) => cost.kind === 'estimated') ? { kind: 'estimated', usd } : { kind: 'observed', usd } } +function executionCost( + executions: ReadonlyArray, + exact: boolean, +): { known: number; provenance: RunCostProvenance } { + const usages = executions.map((execution) => execution.summary.usage) + return { + known: usageSum(usages.map(knownCostUsd), exact, 'run known cost'), + provenance: aggregateCostProvenance( + usages.map((usage) => usage.cost), + exact, + ), + } +} + +function usageSum( + values: readonly number[], + exact: boolean, + field: string, + integer = false, +): number { + const sum = values.reduce((total, value) => total + value, 0) + if (exact && (integer ? !Number.isSafeInteger(sum) : !Number.isFinite(sum))) { + throw new RangeError( + `exact analyst usage ${field} aggregate ${integer ? 'exceeds a safe integer' : 'is not finite'}`, + ) + } + return sum +} + /** * Resolve the `priorFindings` slice an analyst sees. * diff --git a/src/feedback-trajectory-review.ts b/src/feedback-trajectory-review.ts index 2606e8bc..db17b72c 100644 --- a/src/feedback-trajectory-review.ts +++ b/src/feedback-trajectory-review.ts @@ -1,6 +1,14 @@ -import type { AnalystFinding, AnalystRunResult, EvidenceRef } from './analyst/types' +import { type ExactAnalystRunResult, snapshotExactExecutionPlan } from './analyst/exact-types' +import type { + AnalystFinding, + AnalystRunResult, + AnalystUsageReceipt, + EvidenceRef, +} from './analyst/types' +import { assertValidAnalystUsageReceipt } from './analyst/usage-receipt' import type { FeedbackLabelSource, FeedbackTrajectory } from './feedback-trajectory' import { canonicalString, hashCanonical } from './ledger-core/canonical' +import { deepFreezeCanonicalJson } from './ledger-core/deep-freeze' export type AnalystReviewSource = Exclude export type AnalystFindingDigest = `sha256:${string}` @@ -69,6 +77,25 @@ export function analystRunDigest(run: AnalystRunResult): AnalystRunDigest { } export function snapshotAnalystRun(value: unknown, context = 'analyst run'): AnalystRunResult { + const snapshot = snapshotAnalystRunRecord(value, context) + if (snapshot.execution_plan !== undefined) { + return sealExactAnalystRunReceipt(snapshot, context) + } + if (snapshot.completion !== undefined) { + throw new TypeError(`${context} completion is valid only for an exact run`) + } + return snapshot as unknown as AnalystRunResult +} + +/** Canonicalize, validate, and deeply freeze one complete or failed exact-run receipt. */ +export function snapshotExactAnalystRunReceipt( + value: unknown, + context = 'exact analyst run receipt', +): ExactAnalystRunResult { + return sealExactAnalystRunReceipt(snapshotAnalystRunRecord(value, context), context) +} + +function snapshotAnalystRunRecord(value: unknown, context: string): Record { let snapshot: unknown try { snapshot = JSON.parse(canonicalString(value)) as unknown @@ -87,6 +114,8 @@ export function snapshotAnalystRun(value: unknown, context = 'analyst run'): Ana 'per_analyst', 'total_cost_usd', 'total_cost_provenance', + 'execution_plan', + 'completion', ], context, ) @@ -99,8 +128,7 @@ export function snapshotAnalystRun(value: unknown, context = 'analyst run'): Ana throw new TypeError(`${context} per_analyst must be an array`) } for (const [index, summary] of snapshot.per_analyst.entries()) { - if (!isRecord(summary)) throw new TypeError(`${context} per_analyst ${index} must be an object`) - requiredString(summary.analyst_id, `${context} per_analyst ${index} analyst_id`) + assertAnalystRunSummary(summary, `${context} per_analyst ${index}`) } if ( typeof snapshot.total_cost_usd !== 'number' || @@ -109,10 +137,216 @@ export function snapshotAnalystRun(value: unknown, context = 'analyst run'): Ana ) { throw new TypeError(`${context} total_cost_usd must be a finite non-negative number`) } - if (snapshot.total_cost_provenance !== undefined && !isRecord(snapshot.total_cost_provenance)) { - throw new TypeError(`${context} total_cost_provenance must be an object`) + if (snapshot.total_cost_provenance !== undefined) { + assertCostProvenance(snapshot.total_cost_provenance, `${context} total_cost_provenance`) } - return snapshot as unknown as AnalystRunResult + return snapshot +} + +function assertAnalystRunSummary(value: unknown, context: string): void { + if (!isRecord(value)) throw new TypeError(`${context} must be an object`) + assertOnlyKeys( + value, + [ + 'analyst_id', + 'status', + 'reason', + 'findings_count', + 'latency_ms', + 'usage', + 'allocated_budget_usd', + 'error', + ], + context, + ) + requiredString(value.analyst_id, `${context} analyst_id`) + if (value.status !== 'ok' && value.status !== 'skipped' && value.status !== 'failed') { + throw new TypeError(`${context} status is invalid`) + } + if (value.reason !== undefined) requiredString(value.reason, `${context} reason`) + if (value.status === 'skipped' && value.reason === undefined) { + throw new TypeError(`${context} skipped summary requires reason`) + } + nonnegativeSafeInteger(value.findings_count, `${context} findings_count`) + finiteNonnegative(value.latency_ms, `${context} latency_ms`) + assertAnalystUsageReceipt(value.usage, `${context} usage`) + if (value.allocated_budget_usd !== undefined && value.allocated_budget_usd !== null) { + finiteNonnegative(value.allocated_budget_usd, `${context} allocated_budget_usd`) + } + if (value.error !== undefined) { + if (value.status !== 'failed' || !isRecord(value.error)) { + throw new TypeError(`${context} error is valid only for failed summaries`) + } + assertOnlyKeys(value.error, ['class', 'message'], `${context} error`) + requiredString(value.error.class, `${context} error class`) + requiredString(value.error.message, `${context} error message`) + } else if (value.status === 'failed') { + throw new TypeError(`${context} failed summary requires error`) + } +} + +function assertAnalystUsageReceipt(value: unknown, context: string): void { + if (!isRecord(value)) throw new TypeError(`${context} must be an object`) + assertOnlyKeys(value, ['calls', 'tokens', 'cost', 'knownCostUsd'], context) + for (const field of ['calls', 'tokens', 'cost'] as const) { + if (!Object.hasOwn(value, field)) { + throw new TypeError(`${context} ${field} is required`) + } + } + if (value.tokens !== null) { + if (!isRecord(value.tokens)) throw new TypeError(`${context} tokens must be an object or null`) + assertOnlyKeys( + value.tokens, + ['input', 'output', 'reasoning', 'cached', 'cacheWrite'], + `${context} tokens`, + ) + } + assertCostProvenance(value.cost, `${context} cost`) + assertValidAnalystUsageReceipt(value as unknown as AnalystUsageReceipt, context) +} + +function assertCostProvenance(value: unknown, context: string): void { + if (!isRecord(value)) throw new TypeError(`${context} must be an object`) + assertOnlyKeys(value, ['kind', 'usd'], context) + if (value.kind === 'uncaptured') { + if (value.usd !== null) throw new TypeError(`${context} uncaptured usd must be null`) + return + } + if (value.kind !== 'observed' && value.kind !== 'estimated') { + throw new TypeError(`${context} kind is invalid`) + } + finiteNonnegative(value.usd, `${context} usd`) +} + +function sealExactAnalystRunReceipt( + run: Record, + context: string, +): ExactAnalystRunResult { + if (run.execution_plan === undefined) { + throw new TypeError(`${context} exact run requires execution_plan`) + } + const plan = snapshotExactExecutionPlan(run.execution_plan, `${context} execution_plan`) + const completion = snapshotExactRunCompletion(run.completion, `${context} completion`) + run.execution_plan = plan + run.completion = completion + const summaries = run.per_analyst as Array<{ + analyst_id: string + status: 'ok' | 'skipped' | 'failed' + findings_count: number + usage: { cost: { kind: string; usd: number | null }; knownCostUsd?: number } + allocated_budget_usd?: number | null + }> + const findings = run.findings as AnalystFinding[] + const planned = plan.analysts.map((analyst) => analyst.id) + const completed = summaries.map((summary) => summary.analyst_id) + const ordered = completed.every((analystId, index) => analystId === planned[index]) + if (!ordered || (completion.status === 'complete' && completed.length !== planned.length)) { + throw new TypeError( + completion.status === 'complete' + ? `${context} complete receipt must contain every execution_plan analyst in exact order` + : `${context} failed receipt per_analyst must be an execution_plan prefix`, + ) + } + const completedIds = new Set(completed) + for (const finding of findings) { + if (!completedIds.has(finding.analyst_id)) { + throw new TypeError(`${context} finding names an analyst absent from per_analyst`) + } + } + for (const summary of summaries) { + const actual = findings.filter((finding) => finding.analyst_id === summary.analyst_id).length + if (summary.findings_count !== actual) { + throw new TypeError( + `${context} findings_count does not match findings for "${summary.analyst_id}"`, + ) + } + const hasAllocation = Object.hasOwn(summary, 'allocated_budget_usd') + if (summary.status === 'skipped') { + if (hasAllocation) { + throw new TypeError( + `${context} skipped summary "${summary.analyst_id}" cannot report an allocated budget`, + ) + } + continue + } + const allocation = summary.allocated_budget_usd + const validAllocation = + hasAllocation && + (plan.policy.budget.kind === 'none' + ? allocation === null + : typeof allocation === 'number' && + plan.policy.budget.allocations_usd[summary.analyst_id] !== null && + plan.policy.budget.allocations_usd[summary.analyst_id] !== undefined && + allocation <= plan.policy.budget.allocations_usd[summary.analyst_id]!) + if (!validAllocation) { + throw new TypeError( + `${context} summary "${summary.analyst_id}" allocation does not match its execution plan`, + ) + } + } + let knownCost = 0 + for (const summary of summaries) { + const amount = + summary.usage.cost.kind === 'uncaptured' + ? (summary.usage.knownCostUsd ?? 0) + : (summary.usage.cost.usd ?? 0) + knownCost = finiteNonnegative(knownCost + amount, `${context} aggregate known cost`) + } + if (run.total_cost_usd !== knownCost) { + throw new TypeError(`${context} total_cost_usd does not match per_analyst usage`) + } + if (run.total_cost_provenance === undefined) { + throw new TypeError(`${context} exact run requires total_cost_provenance`) + } + const costs = summaries.map((summary) => summary.usage.cost) + const expectedProvenance = costs.some((cost) => cost.kind === 'uncaptured') + ? { kind: 'uncaptured', usd: null } + : { + kind: costs.some((cost) => cost.kind === 'estimated') ? 'estimated' : 'observed', + usd: costs.reduce( + (sum, cost) => + finiteNonnegative(sum + (cost.usd ?? 0), `${context} aggregate captured cost`), + 0, + ), + } + if (hashCanonical(run.total_cost_provenance) !== hashCanonical(expectedProvenance)) { + throw new TypeError(`${context} total_cost_provenance does not match per_analyst usage`) + } + return deepFreezeCanonicalJson(run as unknown as ExactAnalystRunResult) +} + +function snapshotExactRunCompletion( + value: unknown, + context: string, +): ExactAnalystRunResult['completion'] { + if (!isRecord(value)) throw new TypeError(`${context} must be an object`) + if (value.status === 'complete') { + assertOnlyKeys(value, ['status'], context) + return value as ExactAnalystRunResult['completion'] + } + if (value.status !== 'failed') { + throw new TypeError(`${context} status must be complete or failed`) + } + assertOnlyKeys(value, ['status', 'error'], context) + if (!isRecord(value.error)) throw new TypeError(`${context} failed receipt requires error`) + assertOnlyKeys(value.error, ['class', 'message'], `${context} error`) + requiredString(value.error.class, `${context} error class`) + requiredString(value.error.message, `${context} error message`) + return value as unknown as ExactAnalystRunResult['completion'] +} + +function finiteNonnegative(value: unknown, context: string): number { + if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) { + throw new TypeError(`${context} must be a non-negative finite number`) + } + return value +} + +function nonnegativeSafeInteger(value: unknown, context: string): number { + if (!Number.isSafeInteger(value) || (value as number) < 0) { + throw new TypeError(`${context} must be a non-negative safe integer`) + } + return value as number } export function snapshotAnalystFindings( @@ -210,6 +444,12 @@ export function readAnalystReview(trajectory: FeedbackTrajectory): StoredAnalyst ...(analysis.costProvenance === undefined ? {} : { total_cost_provenance: analysis.costProvenance }), + ...(artifact.executionPlan === undefined + ? {} + : { + execution_plan: artifact.executionPlan, + completion: artifact.completion, + }), }, `analyst trajectory "${trajectory.id}" archived run`, ) diff --git a/src/feedback-trajectory.ts b/src/feedback-trajectory.ts index a80a779d..a8093ba5 100644 --- a/src/feedback-trajectory.ts +++ b/src/feedback-trajectory.ts @@ -436,6 +436,8 @@ export function analystRunToFeedbackTrajectory( correlationId: archivedRun.correlation_id, analystIds, findings, + ...('execution_plan' in archivedRun ? { executionPlan: archivedRun.execution_plan } : {}), + ...('completion' in archivedRun ? { completion: archivedRun.completion } : {}), }, createdAt, metadata: { diff --git a/src/index.ts b/src/index.ts index 5a43885a..12044b8c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -66,6 +66,19 @@ export { buildDefaultAnalystRegistry, type DefaultAnalystRegistryOptions, } from './analyst/default-registry' +export type { + ExactAnalystBudgetSnapshot, + ExactAnalystExecutionPlanSnapshot, + ExactAnalystRunCompletion, + ExactAnalystRunEvent, + ExactAnalystRunPolicySnapshot, + ExactAnalystRunResult, + ExactAnalystRunSummary, + ExactAnalystSnapshot, + ExactCapableAnalyst, + ExactExecutionComponentIdentity, + ExactExecutionComponentSnapshot, +} from './analyst/exact-types' export type { RawAnalystEvidence, RawAnalystFinding, @@ -103,7 +116,11 @@ export { type AnalystHooks, AnalystRegistry, type AnalystRegistryOptions, + assertExactRegistryRunOpts, type BudgetPolicy, + type ExactAnalystBudgetPolicy, + ExactAnalystRunExecutionError, + type ExactRegistryRunOpts, type RegistryRunOpts, } from './analyst/registry' export { diff --git a/src/ledger-core/deep-freeze.ts b/src/ledger-core/deep-freeze.ts new file mode 100644 index 00000000..0bf2c976 --- /dev/null +++ b/src/ledger-core/deep-freeze.ts @@ -0,0 +1,11 @@ +/** Freeze a detached canonical-JSON graph. Canonicalization has already ruled out cycles. + * + * Lives outside canonical.ts so the analyst-benchmark implementation digest, + * which covers canonical.ts, stays bound to the published benchmark evidence. */ +export function deepFreezeCanonicalJson(value: T): T { + if (value && typeof value === 'object' && !Object.isFrozen(value)) { + Object.freeze(value) + for (const nested of Object.values(value)) deepFreezeCanonicalJson(nested) + } + return value +}