From 52041f3bafebb349ee74521e0b2296ed610ede0f Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Thu, 30 Jul 2026 11:37:28 -0600 Subject: [PATCH 1/2] fix(supervisor-run): read Runtime file run contexts --- .evolve/skill-runs.jsonl | 2 + CHANGELOG.md | 3 + README.md | 2 +- docs/trace-analysis.md | 13 +- src/index.ts | 3 + src/supervisor-run/analyze.ts | 47 +- src/supervisor-run/fixtures.ts | 2 + src/supervisor-run/index.test.ts | 16 +- src/supervisor-run/index.ts | 5 + src/supervisor-run/loops-reader.ts | 36 +- src/supervisor-run/render.ts | 7 +- src/supervisor-run/rollout-nodes.ts | 28 +- src/supervisor-run/runtime-reader.test.ts | 522 ++++++++++++++++++++++ src/supervisor-run/runtime-reader.ts | 475 ++++++++++++++++++++ src/supervisor-run/source-facts.ts | 69 ++- src/supervisor-run/types.ts | 14 + 16 files changed, 1193 insertions(+), 51 deletions(-) create mode 100644 src/supervisor-run/runtime-reader.test.ts create mode 100644 src/supervisor-run/runtime-reader.ts diff --git a/.evolve/skill-runs.jsonl b/.evolve/skill-runs.jsonl index 77255c63..92fc060e 100644 --- a/.evolve/skill-runs.jsonl +++ b/.evolve/skill-runs.jsonl @@ -40,3 +40,5 @@ {"skill":"/agent-eval","ts":"2026-07-30T11:07:02Z","project":"agent-eval-analyst-proof-20260730","target":"crash-safe resumable public analyst benchmark command","operatorPrompt":"","durationMin":null,"verdict":"PASS","dispatchedTo":"/verify","operatorOverride":null,"transcriptPath":null,"traceDir":null} {"skill":"/verify","ts":"2026-07-30T11:07:07Z","project":"agent-eval-analyst-proof-20260730","target":"benchmark command recovery, artifacts, and AgentRx metrics","operatorPrompt":"","durationMin":null,"verdict":"PASS","dispatchedTo":"/stop","operatorOverride":null,"transcriptPath":null,"traceDir":null} {"skill":"/semgrep","ts":"2026-07-30T15:47:18Z","project":"agent-eval-analyst-proof-20260730","target":"agent-eval public analyst benchmark, 664 files, 492 rules","operatorPrompt":"","durationMin":null,"verdict":"PASS","dispatchedTo":"/stop","operatorOverride":null,"transcriptPath":null,"traceDir":null} +{"skill":"/agent-eval","ts":"2026-07-30T17:36:56Z","project":"agent-eval-runtime-run-reader","target":"Runtime FileRunContext supervisor-run reader","operatorPrompt":"","durationMin":null,"verdict":"PASS","dispatchedTo":"/verify","operatorOverride":null,"transcriptPath":null,"traceDir":null} +{"skill":"/verify","ts":"2026-07-30T17:36:56Z","project":"agent-eval-runtime-run-reader","target":"Runtime FileRunContext reader, real traces CLI path, and package","operatorPrompt":"","durationMin":null,"verdict":"PASS","dispatchedTo":"/stop","operatorOverride":null,"transcriptPath":null,"traceDir":null} diff --git a/CHANGELOG.md b/CHANGELOG.md index bf6a7907..867d19b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,9 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval- ### Fixed +- Supervisor-run analysis now reads agent-runtime `createFileRunContext(dir)` directories directly. + It composes nested journal envelopes by exact node identity, preserves profile digests, runtime tags, failures, and metered spend, and leaves unrecorded roles, terminal times, policy, transcripts, verification, and judge evidence unavailable. + A direct run path produces one report, while a parent directory containing multiple direct runs produces a rollup. - Canonical `trace:///span/` evidence is classified as span evidence instead of artifact evidence. - Public model output selects positive integer assistant step ids. The runner builds canonical trace URIs and exact action excerpts from those spans, and rejects missing, non-assistant, or empty steps. diff --git a/README.md b/README.md index a0dce60e..4380aa46 100644 --- a/README.md +++ b/README.md @@ -372,7 +372,7 @@ See [concepts](./docs/concepts.md), [customer paths](./docs/customer-journeys.md | `@tangle-network/agent-eval/reporting` | Statistical comparisons and report rendering. | | `@tangle-network/agent-eval/analyst` | Built-in and custom trace analysts, labeled comparison, costs, and reports. | | `@tangle-network/agent-eval/traces` | Store, replay, and inspect structured traces. | -| `@tangle-network/agent-eval/supervisor-run` | Read and analyze recursive supervisor runs without collapsing missing measurements to zero. | +| `@tangle-network/agent-eval/supervisor-run` | Read loops or agent-runtime recursive run directories without collapsing missing measurements to zero. | | `@tangle-network/agent-eval/benchmarks` | Benchmark adapters and retrieval metrics. | | `@tangle-network/agent-eval/rl` | Export rewards, preferences, and training rows. | | `@tangle-network/agent-eval/wire` | HTTP and RPC schemas for other languages. | diff --git a/docs/trace-analysis.md b/docs/trace-analysis.md index d5940953..805c57fe 100644 --- a/docs/trace-analysis.md +++ b/docs/trace-analysis.md @@ -121,6 +121,14 @@ for (const c of overview.error_clusters) { `CONTROL_INTEGRITY_ANALYST` checks the existing `SupervisorRunSources` or `SupervisorRunTree` directly. It does not define another run format. +`analyzeSupervisorRun(runDir)` detects both the loops layout and an agent-runtime `createFileRunContext(dir)` layout. +Runtime directories contain `spawn-journal.jsonl` plus optional `result.json` and `trajectory.json`. +The Runtime reader unwraps `{ kind: 'begin' | 'event', root, event }` storage envelopes and feeds the normalized events into the same source parser and analyzer used by every other reader. +Nested tree root markers are joined to their parent spawn by exact node id. +Profile digests, runtime tags, terminal reasons, structured verdicts, and metered spend are retained exactly when recorded. +The reader does not derive invocation roles, outcome quality, completion time, provider policy, worker transcripts, or verification results from labels or artifacts. +Those fields remain unavailable when their source is absent. + Register it as a custom-input analyst and pass the existing value under its stable id: ```ts @@ -129,10 +137,10 @@ import { CONTROL_INTEGRITY_ANALYST, } from '@tangle-network/agent-eval/analyst' import { - readLoopsSupervisorRun, + readRuntimeSupervisorRun, } from '@tangle-network/agent-eval/supervisor-run' -const sources = await readLoopsSupervisorRun(runDir) +const sources = await readRuntimeSupervisorRun(runDir) const registry = new AnalystRegistry() registry.register(CONTROL_INTEGRITY_ANALYST) @@ -142,6 +150,7 @@ const result = await registry.run('run-123', { ``` Pass `SupervisorRunSources` when it is available. +Use `readLoopsSupervisorRun()` when the input is explicitly the loops layout. A `SupervisorRunTree` does not retain raw journal multiplicity or worker request and acknowledgement rows, so tree input explicitly reports those checks as unavailable. The deterministic pass can prove only facts represented by these two existing surfaces. diff --git a/src/index.ts b/src/index.ts index 5a43885a..ac09bfdf 100644 --- a/src/index.ts +++ b/src/index.ts @@ -557,12 +557,15 @@ export { analyzeSupervisorRunIntegrity, analyzeSupervisorRunSources, claudeCodeSupervisorRunReader, + isRuntimeSupervisorRunDir, isUnavailable, type Measured, readClaudeCodeSupervisorRun, + readRuntimeSupervisorRun, renderSupervisorRunHeadline, renderSupervisorRunMarkdown, rollupSupervisorRuns, + runtimeSupervisorRunReader, type SourceLimits, SUPERVISOR_RUN_INTEGRITY_SCHEMA, SUPERVISOR_RUN_SCHEMA, diff --git a/src/supervisor-run/analyze.ts b/src/supervisor-run/analyze.ts index 8e4ed6a1..b0274612 100644 --- a/src/supervisor-run/analyze.ts +++ b/src/supervisor-run/analyze.ts @@ -77,6 +77,8 @@ export function analyzeSupervisorRunSources( const result = parseJson(src.result) const judge = parseJson(src.judge) const { rootId, workerSpawns, workerCloses, startedAt, completedAt } = tree + const rootSpawn = + rootId === null ? null : (tree.spawns.find((spawn) => spawn.id === rootId) ?? null) const spawnById = new Map(workerSpawns.map((spawn) => [spawn.id, spawn])) const spawnsByLabel = new Map() for (const spawn of workerSpawns) { @@ -410,9 +412,13 @@ export function analyzeSupervisorRunSources( } // ── economics ────────────────────────────────────────────────────────── - const journalWorkerIn = workerCloses.reduce((a, c) => a + c.spend.tokens.input, 0) - const journalWorkerOut = workerCloses.reduce((a, c) => a + c.spend.tokens.output, 0) - const journalWorkerUsd = workerCloses.reduce((a, c) => a + c.spend.usd, 0) + const rootChildIds = new Set( + workerSpawns.filter((spawn) => spawn.parent === rootId).map((spawn) => spawn.id), + ) + const rootChildCloses = workerCloses.filter((close) => rootChildIds.has(close.id)) + const journalWorkerIn = rootChildCloses.reduce((a, c) => a + c.spend.tokens.input, 0) + const journalWorkerOut = rootChildCloses.reduce((a, c) => a + c.spend.tokens.output, 0) + const journalWorkerUsd = rootChildCloses.reduce((a, c) => a + c.spend.usd, 0) const workerUsdById = new Map() for (const c of workerCloses) { workerUsdById.set(c.id, (workerUsdById.get(c.id) ?? 0) + c.spend.usd) @@ -466,13 +472,29 @@ export function analyzeSupervisorRunSources( const close = spawn === null ? null : (closeById.get(spawn.id) ?? null) const passed = close?.valid ?? f?.passed ?? null const matchingRoles = new Set(matchingSpawns.map((candidate) => candidate.role)) + const matchingRuntimes = new Set(matchingSpawns.map((candidate) => candidate.runtime)) + const matchingProfiles = new Set(matchingSpawns.map((candidate) => candidate.profileDigest)) + const journalWallMs = + spawn?.at !== null && + spawn?.at !== undefined && + close?.at !== null && + close?.at !== undefined && + close.at >= spawn.at + ? close.at - spawn.at + : null return { workerId: w.workerId ?? null, worker: w.label, role: matchingRoles.size === 1 ? (matchingSpawns[0]?.role ?? null) : null, - wallMs: f?.started != null && f.finishedAt != null ? f.finishedAt - f.started : null, - tokensIn: w.tokensIn ?? null, - tokensOut: w.tokensOut ?? null, + runtime: matchingRuntimes.size === 1 ? (matchingSpawns[0]?.runtime ?? null) : null, + profileDigest: + matchingProfiles.size === 1 ? (matchingSpawns[0]?.profileDigest ?? null) : null, + status: close?.status ?? null, + failure: close?.reason ?? null, + infra: close?.infra ?? null, + wallMs: f?.started != null && f.finishedAt != null ? f.finishedAt - f.started : journalWallMs, + tokensIn: w.tokensIn ?? (close?.hasSpend ? close.spend.tokens.input : null), + tokensOut: w.tokensOut ?? (close?.hasSpend ? close.spend.tokens.output : null), usd: usdLimit !== null ? null @@ -535,9 +557,10 @@ export function analyzeSupervisorRunSources( src.brainLog === null ? gap( 'brain.brainTruncations', - src.supRunDir === null - ? 'no supervisor run dir under /.loops/supervisor' - : 'brain.jsonl absent — loops predates the brain-call tap, so truncation cannot be ruled out', + src.brainLogMissingReason ?? + (src.supRunDir === null + ? 'no supervisor run dir under /.loops/supervisor' + : 'brain.jsonl absent — loops predates the brain-call tap, so truncation cannot be ruled out'), ) : brainCalls.filter((c) => c.finish_reason === 'length').length, workers: { @@ -573,7 +596,7 @@ export function analyzeSupervisorRunSources( usdLimit !== null ? usdLimit : stateUsd !== null - ? `state.json result.spentUsd${journalWorkerUsd === 0 ? ' — brain-priced only; worker CLI inference is unpriced (see worker token counts)' : ''}` + ? `state.json result.spentUsd${rootChildCloses.length > 0 && journalWorkerUsd === 0 ? ' — brain-priced only; worker CLI inference is unpriced (see worker token counts)' : ''}` : haveJournal ? 'journal metered + settled usd' : journalMissing, @@ -663,6 +686,10 @@ export function analyzeSupervisorRunSources( instanceId: src.instanceId, arm: src.arm, supervisorId: rootId !== null ? rootId : unavailable(journalMissing), + supervisorProfileDigest: + rootSpawn?.profileDigest !== null && rootSpawn?.profileDigest !== undefined + ? rootSpawn.profileDigest + : gap('supervisorProfileDigest', 'root spawned event has no profile digest'), generatedAt: new Date(now()).toISOString(), orchestration, decision, diff --git a/src/supervisor-run/fixtures.ts b/src/supervisor-run/fixtures.ts index e648132c..b61fcb2b 100644 --- a/src/supervisor-run/fixtures.ts +++ b/src/supervisor-run/fixtures.ts @@ -26,6 +26,7 @@ export function fixtureJournal(plan: JournalPlan): string { kind: 'spawned', id: root, label: 'root', + role: 'supervisor', budget: {}, runtime: 'inline', seq: 0, @@ -42,6 +43,7 @@ export function fixtureJournal(plan: JournalPlan): string { id, parent: root, label, + role: 'worker', budget: {}, runtime: 'inline', seq: i, diff --git a/src/supervisor-run/index.test.ts b/src/supervisor-run/index.test.ts index 35ff6d03..d244beb6 100644 --- a/src/supervisor-run/index.test.ts +++ b/src/supervisor-run/index.test.ts @@ -1,7 +1,15 @@ import { describe, expect, it } from 'vitest' -import type { SupervisorRunNodeRole as RootSupervisorRunNodeRole } from '../index' import { + type SupervisorRunNodeRole as RootSupervisorRunNodeRole, + isRuntimeSupervisorRunDir as rootIsRuntimeSupervisorRunDir, + readRuntimeSupervisorRun as rootReadRuntimeSupervisorRun, + runtimeSupervisorRunReader as rootRuntimeSupervisorRunReader, +} from '../index' +import { + isRuntimeSupervisorRunDir, NO_SOURCE_LIMITS, + readRuntimeSupervisorRun, + runtimeSupervisorRunReader, type SourceLimits, type SupervisorRunNodeRole, type WorkerLogSource, @@ -23,5 +31,11 @@ describe('supervisor-run public contract', () => { expect(rootRole).toBe('supervisor') expect(worker.workerId).toBe('worker-1') expect(limits).toMatchObject({ managerTokens: null, workerTokens: null }) + expect(typeof readRuntimeSupervisorRun).toBe('function') + expect(typeof runtimeSupervisorRunReader).toBe('function') + expect(typeof isRuntimeSupervisorRunDir).toBe('function') + expect(rootReadRuntimeSupervisorRun).toBe(readRuntimeSupervisorRun) + expect(rootRuntimeSupervisorRunReader).toBe(runtimeSupervisorRunReader) + expect(rootIsRuntimeSupervisorRunDir).toBe(isRuntimeSupervisorRunDir) }) }) diff --git a/src/supervisor-run/index.ts b/src/supervisor-run/index.ts index 1dc663ab..b018680a 100644 --- a/src/supervisor-run/index.ts +++ b/src/supervisor-run/index.ts @@ -62,6 +62,11 @@ export { renderSupervisorRunMarkdown, } from './render' export { type SupervisorRolloutOptions, supervisorRunRolloutLines } from './rollout-nodes' +export { + isRuntimeSupervisorRunDir, + readRuntimeSupervisorRun, + runtimeSupervisorRunReader, +} from './runtime-reader' export { type DecisionMetrics, type EconomicsMetrics, diff --git a/src/supervisor-run/loops-reader.ts b/src/supervisor-run/loops-reader.ts index 49268b76..c42878f1 100644 --- a/src/supervisor-run/loops-reader.ts +++ b/src/supervisor-run/loops-reader.ts @@ -25,6 +25,7 @@ import { renderSupervisorRunHeadline, renderSupervisorRunMarkdown, } from './render' +import { isRuntimeSupervisorRunDir, readRuntimeSupervisorRun } from './runtime-reader' import { NO_SOURCE_LIMITS, type SupervisorRunReader, @@ -270,7 +271,11 @@ export async function analyzeSupervisorRun( opts: LoopsReaderOptions = {}, ): Promise { if (typeof input === 'string') { - return analyzeSupervisorRunSources(await readLoopsSupervisorRun(input, opts)) + return analyzeSupervisorRunSources( + (await isRuntimeSupervisorRunDir(input)) + ? await readRuntimeSupervisorRun(input) + : await readLoopsSupervisorRun(input, opts), + ) } if (isReader(input)) return analyzeSupervisorRunSources(await input.read()) return analyzeSupervisorRunSources(input) @@ -303,7 +308,9 @@ export async function writeSupervisorRunReport( runDir: string, opts: WriteSupervisorRunOptions = {}, ): Promise { - const sources = await readLoopsSupervisorRun(runDir, opts) + const sources = (await isRuntimeSupervisorRunDir(runDir)) + ? await readRuntimeSupervisorRun(runDir) + : await readLoopsSupervisorRun(runDir, opts) const report = analyzeSupervisorRunSources(sources) const md = renderSupervisorRunMarkdown(report) const dest = opts.reportDir ?? runDir @@ -389,8 +396,19 @@ export async function reportSupervisorRound( return rollup } -/** Every `<...>/runs//` directory under `root`. */ +/** + * Every loops or Runtime supervisor run below `root`. + * + * When `root` itself is one run, return no children so callers can distinguish + * a single report from a parent-directory rollup. + */ export async function findSupervisorRunDirs(root: string): Promise { + if ( + (await isRuntimeSupervisorRunDir(root)) || + (await findSupervisorRunDirIn(join(root, 'ws'))) !== null + ) { + return [] + } const found: string[] = [] const walk = async (dir: string, depth: number): Promise => { if (depth > 8) return @@ -399,13 +417,11 @@ export async function findSupervisorRunDirs(root: string): Promise { if (!e.isDirectory()) continue if (e.name === 'node_modules' || e.name === '.git') continue const full = join(dir, e.name) - if (e.name === 'runs') { - const iids = await readdir(full, { withFileTypes: true }).catch(() => []) - for (const iid of iids) { - if (!iid.isDirectory()) continue - const arms = await readdir(join(full, iid.name), { withFileTypes: true }).catch(() => []) - for (const arm of arms) if (arm.isDirectory()) found.push(join(full, iid.name, arm.name)) - } + if ( + (await isRuntimeSupervisorRunDir(full)) || + (await findSupervisorRunDirIn(join(full, 'ws'))) !== null + ) { + found.push(full) continue } await walk(full, depth + 1) diff --git a/src/supervisor-run/render.ts b/src/supervisor-run/render.ts index ca2bd672..4bc9f8bf 100644 --- a/src/supervisor-run/render.ts +++ b/src/supervisor-run/render.ts @@ -62,6 +62,7 @@ export function renderSupervisorRunMarkdown(r: SupervisorRunReport): string { out.push('') out.push(`- Run: \`${r.runRef}\``) out.push(`- Supervisor: \`${showMeasured(r.supervisorId)}\``) + out.push(`- Supervisor profile: \`${showMeasured(r.supervisorProfileDigest)}\``) out.push(`- Generated: ${r.generatedAt}`) out.push('') @@ -161,12 +162,12 @@ export function renderSupervisorRunMarkdown(r: SupervisorRunReport): string { out.push('') if (!isUnavailable(e.perWorker) && e.perWorker.length > 0) { out.push( - '| Worker id | Label | Role | Wall | Tokens in | Tokens out | Patch bytes | Verify passed | Score |', + '| Worker id | Label | Role | Runtime | Profile digest | Status | Failure | Infra | Wall | Tokens in | Tokens out | Patch bytes | Verify passed | Score |', ) - out.push('|---|---|---|---:|---:|---:|---:|---|---:|') + out.push('|---|---|---|---|---|---|---|---|---:|---:|---:|---:|---|---:|') for (const w of e.perWorker) { out.push( - `| ${w.workerId === null ? 'unavailable — legacy label join' : `\`${w.workerId}\``} | \`${w.worker}\` | ${w.role ?? 'unavailable — no journal join'} | ${w.wallMs === null ? 'unavailable — no start/finish pair' : fmtMs(w.wallMs)} | ${w.tokensIn ?? 'unavailable — store does not attribute tokens per worker'} | ${w.tokensOut ?? 'unavailable — store does not attribute tokens per worker'} | ${w.patchBytes ?? 'unavailable — no worker patch file'} | ${w.passed === null ? 'unavailable — no verdict' : String(w.passed)} | ${w.score ?? 'unavailable — no numeric score'} |`, + `| ${w.workerId === null ? 'unavailable — legacy label join' : `\`${w.workerId}\``} | \`${w.worker}\` | ${w.role ?? 'unavailable — source recorded no role'} | ${w.runtime ?? 'unavailable — source recorded no runtime'} | ${w.profileDigest === null ? 'unavailable — source recorded no profile digest' : `\`${w.profileDigest}\``} | ${w.status ?? 'unavailable — no terminal event'} | ${w.failure ?? 'none recorded'} | ${w.infra ?? 'unavailable'} | ${w.wallMs === null ? 'unavailable — no spawn/finish pair' : fmtMs(w.wallMs)} | ${w.tokensIn ?? 'unavailable — store does not attribute tokens per worker'} | ${w.tokensOut ?? 'unavailable — store does not attribute tokens per worker'} | ${w.patchBytes ?? 'unavailable — no worker patch file'} | ${w.passed === null ? 'unavailable — no verdict' : String(w.passed)} | ${w.score ?? 'unavailable — no numeric score'} |`, ) } out.push('') diff --git a/src/supervisor-run/rollout-nodes.ts b/src/supervisor-run/rollout-nodes.ts index 2c08193e..0fbd1149 100644 --- a/src/supervisor-run/rollout-nodes.ts +++ b/src/supervisor-run/rollout-nodes.ts @@ -176,6 +176,7 @@ export function supervisorRunRolloutLinesFromFacts( model: opts.supervisorModel ?? null, provider: null, profile_commit: null, + agent_profile_cell_id: rootSpawn?.profileDigest ?? null, sampling: null, }, outcome: { @@ -203,10 +204,15 @@ export function supervisorRunRolloutLinesFromFacts( completed_at: tree.completedAt, workers_spawned: tree.workerSpawns.length, brain_metered_events: tree.brain.meteredCount, + runtime: rootSpawn?.runtime ?? null, + profile_digest: rootSpawn?.profileDigest ?? null, }, is_completed: state?.status === 'completed', is_truncated: false, - error: null, + error: + result?.kind === 'interrupted' && typeof result.reason === 'string' + ? result.reason + : null, realness_gated: false, }, cost: { @@ -261,6 +267,14 @@ export function supervisorRunRolloutLinesFromFacts( // Stable ids are authoritative. Labels are a compatibility join for stores // that predate workerId and are only safe when the store keeps them unique. for (const spawn of tree.workerSpawns) { + if (spawn.role === null) { + gaps.push({ + code: 'node-role-unavailable', + message: `child ${JSON.stringify(spawn.label)} has no explicit invocation role`, + nodeId: spawn.id, + }) + continue + } const close = closeById.get(spawn.id) ?? null const workerSource = sourceById.get(spawn.id) ?? fallbackSourceByLabel.get(spawn.label) ?? null const facts = @@ -289,6 +303,7 @@ export function supervisorRunRolloutLinesFromFacts( model: isSupervisor ? (opts.supervisorModel ?? null) : (opts.workerModel ?? null), provider: null, profile_commit: null, + agent_profile_cell_id: spawn.profileDigest, sampling: null, }, outcome: { @@ -319,10 +334,17 @@ export function supervisorRunRolloutLinesFromFacts( steers_queued: facts?.steersQueued ?? null, steers_delivered: facts?.steersDelivered ?? null, questions: facts?.questions ?? null, + runtime: spawn.runtime, + profile_digest: spawn.profileDigest, }, - is_completed: close?.kind === 'settled', + is_completed: close?.kind === 'settled' && close.status === 'done', is_truncated: close?.kind === 'cancelled', - error: close?.kind === 'cancelled' ? (close.verdict ?? 'cancelled') : null, + error: + close?.kind === 'cancelled' + ? (close.reason ?? 'cancelled') + : close?.status === 'down' + ? (close.reason ?? 'child down') + : null, realness_gated: false, }, // `close.spend` is only a measurement when the close event carried one; diff --git a/src/supervisor-run/runtime-reader.test.ts b/src/supervisor-run/runtime-reader.test.ts new file mode 100644 index 00000000..4c7313e1 --- /dev/null +++ b/src/supervisor-run/runtime-reader.test.ts @@ -0,0 +1,522 @@ +import { mkdir, mkdtemp, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { describe, expect, it } from 'vitest' +import { analyzeSupervisorRun, findSupervisorRunDirs } from './loops-reader' +import { supervisorRunRolloutLines } from './rollout-nodes' +import { readRuntimeSupervisorRun } from './runtime-reader' +import { isUnavailable } from './types' + +const T0 = Date.parse('2026-07-30T00:00:00.000Z') +const at = (seconds: number): string => new Date(T0 + seconds * 1_000).toISOString() +const ROOT_PROFILE = `sha256:${'a'.repeat(64)}` +const CHILD_PROFILE = `sha256:${'b'.repeat(64)}` +const LEAF_PROFILE = `sha256:${'c'.repeat(64)}` + +function begin(root: string, seconds: number): Record { + return { kind: 'begin', root, at: at(seconds) } +} + +function event(root: string, value: Record): Record { + return { kind: 'event', root, event: value } +} + +async function writeJournal( + runDir: string, + rows: readonly Record[], +): Promise { + await mkdir(runDir, { recursive: true }) + await writeFile( + join(runDir, 'spawn-journal.jsonl'), + `${rows.map((row) => JSON.stringify(row)).join('\n')}\n`, + ) +} + +async function nestedRuntimeRun(): Promise { + const parent = await mkdtemp(join(tmpdir(), 'runtime-supervisor-run-')) + const runDir = join(parent, 'run-nested') + await writeJournal(runDir, [ + begin('run-root', 0), + event('run-root', { + kind: 'spawned', + id: 'run-root', + label: 'research-leader', + profileDigest: ROOT_PROFILE, + budget: { maxIterations: 3, maxTokens: 1_000 }, + seq: 0, + at: at(0), + }), + event('run-root', { + kind: 'metered', + id: 'run-root', + spend: { + iterations: 1, + tokens: { input: 10, output: 2 }, + usd: 0.01, + ms: 500, + }, + seq: 0, + at: at(0.5), + }), + event('run-root', { + kind: 'spawned', + id: 'run-root:s0', + parent: 'run-root', + label: 'nested-researcher', + profileDigest: CHILD_PROFILE, + runtime: 'driver', + budget: { maxIterations: 2, maxTokens: 500 }, + seq: 0, + at: at(1), + }), + begin('run-root:s0', 1), + event('run-root:s0', { + kind: 'spawned', + id: 'run-root:s0', + label: 'nested-researcher', + profileDigest: CHILD_PROFILE, + budget: { maxIterations: 2, maxTokens: 500 }, + seq: 0, + at: at(1), + }), + event('run-root:s0', { + kind: 'metered', + id: 'run-root:s0', + spend: { + iterations: 1, + tokens: { input: 20, output: 4 }, + usd: 0.02, + ms: 1_000, + }, + seq: 0, + at: at(1.5), + }), + event('run-root:s0', { + kind: 'spawned', + id: 'run-root:s0:s0', + parent: 'run-root:s0', + label: 'leaf', + profileDigest: LEAF_PROFILE, + runtime: 'cli', + budget: { maxIterations: 1, maxTokens: 100 }, + seq: 0, + at: at(2), + }), + event('run-root:s0', { + kind: 'settled', + id: 'run-root:s0:s0', + status: 'done', + verdict: { valid: true, score: 0.9 }, + spent: { + iterations: 1, + tokens: { input: 5, output: 1 }, + usd: 0.005, + ms: 1_000, + }, + seq: 0, + at: at(3), + }), + event('run-root', { + kind: 'settled', + id: 'run-root:s0', + status: 'down', + reason: 'nested budget exhausted', + infra: false, + spent: { + iterations: 2, + tokens: { input: 25, output: 5 }, + usd: 0.025, + ms: 3_000, + }, + seq: 0, + at: at(4), + }), + ]) + await writeFile( + join(runDir, 'result.json'), + JSON.stringify({ + kind: 'completed', + artifact: { summary: 'synthetic' }, + artifactRef: `sha256:${'d'.repeat(64)}`, + tree: { root: 'run-root', nodes: [] }, + spentTotal: { + iterations: 3, + tokens: { input: 35, output: 7 }, + usd: 0.035, + ms: 3_500, + }, + }), + ) + await writeFile( + join(runDir, 'trajectory.json'), + JSON.stringify({ + root: 'run-root', + nodes: [ + { + id: 'run-root', + children: ['run-root:s0'], + label: 'research-leader', + status: 'pending', + ownSpend: {}, + rolledUpSpend: {}, + }, + ], + total: {}, + statusCounts: { done: 0, failed: 0, cancelled: 0, pending: 1, waiting: 0 }, + }), + ) + return runDir +} + +describe('Runtime FileRunContext supervisor reader', () => { + it('flattens nested tree envelopes and reuses the existing analyzer without double-counting', async () => { + const runDir = await nestedRuntimeRun() + const source = await readRuntimeSupervisorRun(runDir) + const report = await analyzeSupervisorRun(runDir) + + expect(source.journal).not.toContain('"kind":"event"') + expect(report.instanceId).toBe('run-root') + expect(report.supervisorId).toBe('run-root') + expect(report.supervisorProfileDigest).toBe(ROOT_PROFILE) + expect(report.orchestration.workersSpawned).toBe(2) + expect(report.orchestration.workersSettled).toBe(2) + expect(report.orchestration.delegationDepth).toBe(2) + expect(report.economics.brain.tokensIn).toBe(10) + expect(report.economics.brain.tokensOut).toBe(2) + expect(report.economics.workers.tokensIn).toBe(25) + expect(report.economics.workers.tokensOut).toBe(5) + expect(report.economics.totalUsd).toBe(0.035) + expect(report.outcome.supStatus).toBe('completed') + expect(isUnavailable(report.orchestration.supervisorWallMs)).toBe(true) + + if (isUnavailable(report.economics.perWorker)) { + throw new Error(report.economics.perWorker.unavailable) + } + expect(report.economics.perWorker).toEqual([ + expect.objectContaining({ + workerId: 'run-root:s0', + role: null, + runtime: 'driver', + profileDigest: CHILD_PROFILE, + status: 'down', + failure: 'nested budget exhausted', + infra: false, + wallMs: 3_000, + tokensIn: 25, + tokensOut: 5, + }), + expect.objectContaining({ + workerId: 'run-root:s0:s0', + role: null, + runtime: 'cli', + profileDigest: LEAF_PROFILE, + status: 'done', + wallMs: 1_000, + tokensIn: 5, + tokensOut: 1, + passed: true, + score: 0.9, + }), + ]) + + const tree = supervisorRunRolloutLines(source, { capturedAt: at(10) }) + expect(tree.nodes).toHaveLength(1) + expect(tree.nodes[0]?.policy.agent_profile_cell_id).toBe(ROOT_PROFILE) + expect(tree.gaps.filter((gap) => gap.code === 'node-role-unavailable')).toHaveLength(2) + }) + + it('leaves root outcome and terminal wall time unavailable without terminal evidence', async () => { + const parent = await mkdtemp(join(tmpdir(), 'runtime-supervisor-run-')) + const runDir = join(parent, 'unfinished') + await writeJournal(runDir, [ + begin('unfinished-root', 0), + event('unfinished-root', { + kind: 'spawned', + id: 'unfinished-root', + label: 'root', + profileDigest: ROOT_PROFILE, + budget: { maxIterations: 1, maxTokens: 100 }, + seq: 0, + at: at(0), + }), + event('unfinished-root', { + kind: 'metered', + id: 'unfinished-root', + spend: { + iterations: 1, + tokens: { input: 8, output: 1 }, + usd: 0.004, + ms: 400, + }, + seq: 0, + at: at(1), + }), + ]) + + const report = await analyzeSupervisorRun(runDir) + expect(isUnavailable(report.outcome.supStatus)).toBe(true) + expect(isUnavailable(report.orchestration.supervisorWallMs)).toBe(true) + expect(report.economics.brain.tokensIn).toBe(8) + expect(report.economics.totalUsd).toBe(0.004) + expect(report.orchestration.workersSpawned).toBe(0) + expect(await findSupervisorRunDirs(runDir)).toEqual([]) + }) + + it('records Runtime interruption as execution status and reason, never as quality success', async () => { + const parent = await mkdtemp(join(tmpdir(), 'runtime-supervisor-run-')) + const runDir = join(parent, 'interrupted') + await writeJournal(runDir, [ + begin('interrupted-root', 0), + event('interrupted-root', { + kind: 'spawned', + id: 'interrupted-root', + label: 'root', + profileDigest: ROOT_PROFILE, + budget: { maxIterations: 1, maxTokens: 100 }, + seq: 0, + at: at(0), + }), + ]) + await writeFile( + join(runDir, 'result.json'), + JSON.stringify({ + kind: 'interrupted', + reason: 'budget-exhausted', + tree: { root: 'interrupted-root', nodes: [] }, + downCount: 0, + spentTotal: { + iterations: 0, + tokens: { input: 0, output: 0 }, + usd: 0, + ms: 0, + }, + }), + ) + + const source = await readRuntimeSupervisorRun(runDir) + const report = await analyzeSupervisorRun(runDir) + const tree = supervisorRunRolloutLines(source, { capturedAt: at(10) }) + expect(report.outcome.supStatus).toBe('interrupted') + expect(isUnavailable(report.outcome.supVerdict)).toBe(true) + expect(isUnavailable(report.outcome.verifyPass)).toBe(true) + expect(isUnavailable(report.economics.brain.tokensIn)).toBe(true) + expect(isUnavailable(report.economics.brain.usd)).toBe(true) + expect(isUnavailable(report.economics.totalUsd)).toBe(true) + expect(tree.nodes[0]?.outcome.is_completed).toBe(false) + expect(tree.nodes[0]?.outcome.error).toBe('budget-exhausted') + }) + + it('keeps explicitly unknown Runtime prices unavailable while retaining measured tokens', async () => { + const parent = await mkdtemp(join(tmpdir(), 'runtime-supervisor-run-')) + const runDir = join(parent, 'unknown-usd') + await writeJournal(runDir, [ + begin('unknown-usd-root', 0), + event('unknown-usd-root', { + kind: 'spawned', + id: 'unknown-usd-root', + label: 'root', + profileDigest: ROOT_PROFILE, + budget: { maxIterations: 1, maxTokens: 100 }, + seq: 0, + at: at(0), + }), + event('unknown-usd-root', { + kind: 'metered', + id: 'unknown-usd-root', + spend: { + iterations: 1, + tokens: { input: 8, output: 1 }, + usd: 0, + usdKnown: false, + ms: 400, + }, + seq: 0, + at: at(1), + }), + ]) + + const report = await analyzeSupervisorRun(runDir) + expect(report.economics.brain.tokensIn).toBe(8) + expect(report.economics.brain.tokensOut).toBe(1) + expect(isUnavailable(report.economics.brain.usd)).toBe(true) + expect(isUnavailable(report.economics.totalUsd)).toBe(true) + }) + + it('does not turn an unsettled child into zero worker usage or zero total spend', async () => { + const parent = await mkdtemp(join(tmpdir(), 'runtime-supervisor-run-')) + const runDir = join(parent, 'unsettled-child') + await writeJournal(runDir, [ + begin('unsettled-root', 0), + event('unsettled-root', { + kind: 'spawned', + id: 'unsettled-root', + label: 'root', + profileDigest: ROOT_PROFILE, + budget: { maxIterations: 1, maxTokens: 100 }, + seq: 0, + at: at(0), + }), + event('unsettled-root', { + kind: 'metered', + id: 'unsettled-root', + spend: { + iterations: 1, + tokens: { input: 8, output: 1 }, + usd: 0.004, + ms: 400, + }, + seq: 0, + at: at(1), + }), + event('unsettled-root', { + kind: 'spawned', + id: 'unsettled-child', + parent: 'unsettled-root', + label: 'child', + profileDigest: CHILD_PROFILE, + runtime: 'cli', + budget: { maxIterations: 1, maxTokens: 100 }, + seq: 0, + at: at(2), + }), + ]) + + const report = await analyzeSupervisorRun(runDir) + expect(isUnavailable(report.economics.workers.tokensIn)).toBe(true) + expect(isUnavailable(report.economics.workers.tokensOut)).toBe(true) + expect(isUnavailable(report.economics.totalUsd)).toBe(true) + if (isUnavailable(report.economics.perWorker)) { + throw new Error(report.economics.perWorker.unavailable) + } + expect(report.economics.perWorker[0]?.tokensIn).toBeNull() + expect(report.economics.perWorker[0]?.tokensOut).toBeNull() + expect(report.economics.perWorker[0]?.usd).toBeNull() + }) + + it('finds multiple direct Runtime run directories below a parent without finding their blobs', async () => { + const root = await mkdtemp(join(tmpdir(), 'runtime-supervisor-parent-')) + for (const id of ['run-a', 'run-b']) { + await writeJournal(join(root, id), [ + begin(id, 0), + event(id, { + kind: 'spawned', + id, + label: id, + profileDigest: ROOT_PROFILE, + budget: { maxIterations: 1, maxTokens: 1 }, + seq: 0, + at: at(0), + }), + ]) + await mkdir(join(root, id, 'blobs'), { recursive: true }) + await writeFile(join(root, id, 'blobs', 'artifact.json'), '{}') + } + + expect(await findSupervisorRunDirs(root)).toEqual([join(root, 'run-a'), join(root, 'run-b')]) + }) + + it('refuses a nested tree whose exact profile identity disagrees with its parent spawn', async () => { + const root = await mkdtemp(join(tmpdir(), 'runtime-supervisor-run-')) + const runDir = join(root, 'mismatch') + await writeJournal(runDir, [ + begin('root', 0), + event('root', { + kind: 'spawned', + id: 'root', + label: 'root', + profileDigest: ROOT_PROFILE, + budget: {}, + seq: 0, + at: at(0), + }), + event('root', { + kind: 'spawned', + id: 'child', + parent: 'root', + label: 'child', + profileDigest: CHILD_PROFILE, + runtime: 'driver', + budget: {}, + seq: 0, + at: at(1), + }), + begin('child', 1), + event('child', { + kind: 'spawned', + id: 'child', + label: 'child', + profileDigest: LEAF_PROFILE, + budget: {}, + seq: 0, + at: at(1), + }), + ]) + + await expect(readRuntimeSupervisorRun(runDir)).rejects.toThrow( + /disagrees with its parent profile digest/, + ) + }) + + it('refuses a Runtime result attached to a different journal root', async () => { + const root = await mkdtemp(join(tmpdir(), 'runtime-supervisor-run-')) + const runDir = join(root, 'result-mismatch') + await writeJournal(runDir, [ + begin('journal-root', 0), + event('journal-root', { + kind: 'spawned', + id: 'journal-root', + label: 'root', + profileDigest: ROOT_PROFILE, + budget: {}, + seq: 0, + at: at(0), + }), + ]) + await writeFile( + join(runDir, 'result.json'), + JSON.stringify({ + kind: 'completed', + tree: { root: 'other-root', nodes: [] }, + spentTotal: { + iterations: 0, + tokens: { input: 0, output: 0 }, + usd: 0, + ms: 0, + }, + }), + ) + + await expect(readRuntimeSupervisorRun(runDir)).rejects.toThrow(/does not match journal root/) + }) + + it('refuses one journal file containing two independent top-level runs', async () => { + const root = await mkdtemp(join(tmpdir(), 'runtime-supervisor-run-')) + const runDir = join(root, 'two-roots') + await writeJournal(runDir, [ + begin('root-a', 0), + event('root-a', { + kind: 'spawned', + id: 'root-a', + label: 'root-a', + profileDigest: ROOT_PROFILE, + budget: {}, + seq: 0, + at: at(0), + }), + begin('root-b', 1), + event('root-b', { + kind: 'spawned', + id: 'root-b', + label: 'root-b', + profileDigest: CHILD_PROFILE, + budget: {}, + seq: 0, + at: at(1), + }), + ]) + + await expect(readRuntimeSupervisorRun(runDir)).rejects.toThrow( + /expected one top-level tree, found 2/, + ) + }) +}) diff --git a/src/supervisor-run/runtime-reader.ts b/src/supervisor-run/runtime-reader.ts new file mode 100644 index 00000000..295b8790 --- /dev/null +++ b/src/supervisor-run/runtime-reader.ts @@ -0,0 +1,475 @@ +/** + * Reader for agent-runtime's file-backed supervision context. + * + * Runtime stores multiple recursive trees in one `spawn-journal.jsonl`. + * Each line is an envelope whose `root` identifies the local tree. A nested + * driver is represented twice: once as a child spawn in its parent's tree and + * once as the parentless root marker of its own tree. This reader removes only + * that duplicate root marker and passes the remaining events to the existing + * supervisor-run analyzer. + */ + +import { readFile, stat } from 'node:fs/promises' +import { join } from 'node:path' +import type { + SourceLimits, + SupervisorRunReader, + SupervisorRunSources, + WorkerLogSource, +} from './types' + +const JOURNAL_FILE = 'spawn-journal.jsonl' +const RESULT_FILE = 'result.json' +const TRAJECTORY_FILE = 'trajectory.json' + +interface BeginRecord { + readonly root: string + readonly at: string + readonly line: number +} + +interface EventRecord { + readonly root: string + readonly event: Record + readonly line: number +} + +interface NormalizedRuntimeJournal { + readonly root: string + readonly startedAt: string + readonly journal: string + readonly events: readonly Record[] +} + +async function readMaybe(path: string): Promise { + return readFile(path, 'utf8').catch((error: unknown) => { + if ( + typeof error === 'object' && + error !== null && + 'code' in error && + (error as { code: unknown }).code === 'ENOENT' + ) { + return null + } + throw error + }) +} + +function record(value: unknown): Record | null { + return typeof value === 'object' && value !== null && !Array.isArray(value) + ? (value as Record) + : null +} + +function nonEmptyString(value: unknown): string | null { + return typeof value === 'string' && value.length > 0 ? value : null +} + +function formatError(path: string, line: number, detail: string): Error { + return new Error(`${path}:${line}: invalid Runtime spawn journal: ${detail}`) +} + +function parseEnvelopeJournal(text: string, path: string): NormalizedRuntimeJournal { + const begins: BeginRecord[] = [] + const events: EventRecord[] = [] + const begun = new Map() + + for (const [index, sourceLine] of text.split('\n').entries()) { + const line = index + 1 + const trimmed = sourceLine.trim() + if (trimmed.length === 0) continue + let parsed: unknown + try { + parsed = JSON.parse(trimmed) + } catch { + throw formatError(path, line, 'line is not JSON') + } + const envelope = record(parsed) + if (envelope === null) throw formatError(path, line, 'line is not an object') + const kind = nonEmptyString(envelope.kind) + const root = nonEmptyString(envelope.root) + if (root === null) throw formatError(path, line, 'root must be a non-empty string') + + if (kind === 'begin') { + const at = nonEmptyString(envelope.at) + if (at === null || !Number.isFinite(Date.parse(at))) { + throw formatError(path, line, 'begin.at must be an ISO timestamp') + } + if (begun.has(root)) throw formatError(path, line, `tree ${JSON.stringify(root)} began twice`) + const begin = { root, at, line } + begun.set(root, begin) + begins.push(begin) + continue + } + + if (kind !== 'event') { + throw formatError(path, line, "kind must be 'begin' or 'event'") + } + if (!begun.has(root)) { + throw formatError(path, line, `event for tree ${JSON.stringify(root)} precedes begin`) + } + const event = record(envelope.event) + if (event === null) throw formatError(path, line, 'event must be an object') + if (nonEmptyString(event.kind) === null) { + throw formatError(path, line, 'event.kind must be a non-empty string') + } + events.push({ root, event: { ...event }, line }) + } + + if (begins.length === 0) throw formatError(path, 1, 'no begin record') + + const spawnedById = new Map() + for (const entry of events) { + if (entry.event.kind !== 'spawned') continue + const id = nonEmptyString(entry.event.id) + if (id === null) continue + const matches = spawnedById.get(id) ?? [] + matches.push(entry) + spawnedById.set(id, matches) + } + + const nestedRoots = new Set() + for (const begin of begins) { + const parentSpawns = (spawnedById.get(begin.root) ?? []).filter( + (entry) => entry.root !== begin.root && nonEmptyString(entry.event.parent) !== null, + ) + if (parentSpawns.length > 1) { + throw formatError( + path, + begin.line, + `tree ${JSON.stringify(begin.root)} has ${parentSpawns.length} parent spawns`, + ) + } + if (parentSpawns.length === 1) nestedRoots.add(begin.root) + } + + const topRoots = begins.filter((begin) => !nestedRoots.has(begin.root)) + if (topRoots.length !== 1) { + throw formatError( + path, + topRoots[0]?.line ?? 1, + `expected one top-level tree, found ${topRoots.length}`, + ) + } + const top = topRoots[0] as BeginRecord + + for (const nestedRoot of nestedRoots) { + const marker = events.filter( + (entry) => + entry.root === nestedRoot && + entry.event.kind === 'spawned' && + entry.event.id === nestedRoot && + entry.event.parent === undefined, + ) + if (marker.length !== 1) { + throw formatError( + path, + begun.get(nestedRoot)?.line ?? 1, + `nested tree ${JSON.stringify(nestedRoot)} must contain one root marker`, + ) + } + const parentSpawn = (spawnedById.get(nestedRoot) ?? []).find( + (entry) => entry.root !== nestedRoot && nonEmptyString(entry.event.parent) !== null, + ) + if (parentSpawn === undefined) { + throw formatError( + path, + begun.get(nestedRoot)?.line ?? 1, + `nested tree ${JSON.stringify(nestedRoot)} has no parent spawn`, + ) + } + const markerDigest = nonEmptyString(marker[0]?.event.profileDigest) + const parentDigest = nonEmptyString(parentSpawn.event.profileDigest) + if (markerDigest !== null && parentDigest !== null && markerDigest !== parentDigest) { + throw formatError( + path, + marker[0]?.line ?? 1, + `nested tree ${JSON.stringify(nestedRoot)} disagrees with its parent profile digest`, + ) + } + if (parentDigest === null && markerDigest !== null) { + parentSpawn.event.profileDigest = markerDigest + } + } + + const normalized = events + .filter( + (entry) => + !( + nestedRoots.has(entry.root) && + entry.event.kind === 'spawned' && + entry.event.id === entry.root && + entry.event.parent === undefined + ), + ) + .map((entry) => entry.event) + + const rootMarkers = normalized.filter( + (event) => event.kind === 'spawned' && event.id === top.root && event.parent === undefined, + ) + if (rootMarkers.length !== 1) { + throw formatError( + path, + top.line, + `top-level tree ${JSON.stringify(top.root)} must contain one root marker`, + ) + } + + return { + root: top.root, + startedAt: top.at, + journal: `${normalized.map((event) => JSON.stringify(event)).join('\n')}\n`, + events: normalized, + } +} + +function parseOptionalRecord(text: string | null, path: string): Record | null { + if (text === null) return null + let parsed: unknown + try { + parsed = JSON.parse(text) + } catch { + throw new Error(`${path}: invalid JSON`) + } + const value = record(parsed) + if (value === null) throw new Error(`${path}: expected a JSON object`) + return value +} + +function spendRecord(value: unknown): Record | null { + const spend = record(value) + if (spend === null) return null + const tokens = record(spend.tokens) + if ( + tokens === null || + typeof tokens.input !== 'number' || + !Number.isFinite(tokens.input) || + tokens.input < 0 || + typeof tokens.output !== 'number' || + !Number.isFinite(tokens.output) || + tokens.output < 0 || + typeof spend.usd !== 'number' || + !Number.isFinite(spend.usd) || + spend.usd < 0 || + (spend.usdKnown !== undefined && typeof spend.usdKnown !== 'boolean') + ) { + return null + } + return spend +} + +function sourceLimits( + root: string, + events: readonly Record[], + workerIds: ReadonlySet, +): SourceLimits { + const rootMeters = events.filter((event) => event.kind === 'metered' && event.id === root) + const invalidRootMeters = rootMeters.filter((event) => spendRecord(event.spend) === null) + const rootMeterReason = + rootMeters.length === 0 + ? 'Runtime journal has no root metered event' + : invalidRootMeters.length > 0 + ? `${invalidRootMeters.length} root metered event(s) lack complete spend` + : null + const closes = events.filter( + (event) => + workerIds.has(nonEmptyString(event.id) ?? '') && + (event.kind === 'settled' || event.kind === 'cancelled'), + ) + const settledById = new Map[]>() + for (const event of closes) { + const id = nonEmptyString(event.id) + if (id === null) continue + const matches = settledById.get(id) ?? [] + matches.push(event) + settledById.set(id, matches) + } + const incompleteWorkers = [...workerIds].filter((id) => { + const terminal = settledById.get(id) + return ( + terminal?.length !== 1 || + terminal[0]?.kind !== 'settled' || + spendRecord(terminal[0]?.spent) === null + ) + }) + const spendRows = [ + ...rootMeters.map((event) => spendRecord(event.spend)), + ...closes.filter((event) => event.kind === 'settled').map((event) => spendRecord(event.spent)), + ].filter((spend): spend is Record => spend !== null) + const unknownUsd = spendRows.some((spend) => spend.usdKnown === false) + const missingVerdicts = [...workerIds].filter((id) => { + const terminal = settledById.get(id)?.[0] + if (terminal?.kind !== 'settled') return true + const verdict = record(terminal.verdict) + return typeof verdict?.valid !== 'boolean' + }) + + return { + managerTokens: rootMeterReason, + workerTokens: + incompleteWorkers.length === 0 + ? null + : `${incompleteWorkers.length}/${workerIds.size} child invocation(s) lack one settled spend record`, + spendUsd: + rootMeterReason !== null + ? rootMeterReason + : unknownUsd + ? 'Runtime recorded usdKnown:false for at least one spend' + : incompleteWorkers.length > 0 + ? 'at least one child invocation lacks a settled spend record' + : null, + workerVerdicts: + missingVerdicts.length === 0 + ? null + : `${missingVerdicts.length}/${workerIds.size} child invocation(s) lack a structured validity verdict`, + deliverables: + workerIds.size === 0 + ? null + : 'Runtime FileRunContext does not retain per-child delivered patches', + } +} + +function runtimeState( + root: string, + startedAt: string, + result: Record | null, + trajectory: Record | null, + resultPath: string, + trajectoryPath: string, +): string { + const resultKind = + result?.kind === 'completed' || result?.kind === 'interrupted' ? result.kind : null + if (resultKind !== null && result !== null) { + const resultRoot = nonEmptyString(record(result.tree)?.root) + if (resultRoot === null) { + throw new Error(`${resultPath}: Runtime ${resultKind} result has no tree.root`) + } + if (resultRoot !== root) { + throw new Error( + `${resultPath}: root ${JSON.stringify(resultRoot)} does not match journal root ${JSON.stringify(root)}`, + ) + } + } + + if (trajectory !== null && nonEmptyString(trajectory.root) === null) { + throw new Error(`${trajectoryPath}: Runtime trajectory has no root`) + } + if (typeof trajectory?.root === 'string' && trajectory.root !== root) { + throw new Error( + `${trajectoryPath}: root ${JSON.stringify(trajectory.root)} does not match journal root ${JSON.stringify(root)}`, + ) + } + if (trajectory !== null && !Array.isArray(trajectory.nodes)) { + throw new Error(`${trajectoryPath}: Runtime trajectory nodes must be an array`) + } + + let status: string | null = null + if (resultKind === 'completed') status = 'completed' + else if (resultKind === 'interrupted') status = 'interrupted' + else if (Array.isArray(trajectory?.nodes)) { + const rootNodes = trajectory.nodes + .map((node) => record(node)) + .filter((node) => node?.id === root) + if (rootNodes.length > 1) { + throw new Error(`${trajectoryPath}: Runtime trajectory contains duplicate root nodes`) + } + status = nonEmptyString(rootNodes[0]?.status) + } + + return JSON.stringify({ + id: root, + startedAt, + ...(status === null ? {} : { status }), + }) +} + +/** + * Read one agent-runtime `createFileRunContext(dir)` directory. + * + * The reader translates storage envelopes only. It does not assign research + * roles, interpret artifacts, or turn process completion into a quality + * verdict. + */ +export async function readRuntimeSupervisorRun(runDir: string): Promise { + const journalPath = join(runDir, JOURNAL_FILE) + const rawJournal = await readFile(journalPath, 'utf8') + const normalized = parseEnvelopeJournal(rawJournal, journalPath) + const resultText = await readMaybe(join(runDir, RESULT_FILE)) + const trajectoryText = await readMaybe(join(runDir, TRAJECTORY_FILE)) + const result = parseOptionalRecord(resultText, join(runDir, RESULT_FILE)) + const trajectory = parseOptionalRecord(trajectoryText, join(runDir, TRAJECTORY_FILE)) + const resultPath = join(runDir, RESULT_FILE) + const trajectoryPath = join(runDir, TRAJECTORY_FILE) + + const spawns = normalized.events.filter( + (event) => event.kind === 'spawned' && nonEmptyString(event.id) !== null, + ) + const childSpawns = spawns.filter((event) => event.id !== normalized.root) + const workerIds = new Set( + childSpawns.map((event) => nonEmptyString(event.id)).filter((id): id is string => id !== null), + ) + const workers: WorkerLogSource[] = childSpawns.map((event) => ({ + workerId: nonEmptyString(event.id) as string, + label: nonEmptyString(event.label) ?? String(event.id), + events: null, + inbox: null, + patchBytes: null, + transcriptRef: null, + patchPath: null, + })) + + return { + runRef: runDir, + instanceId: normalized.root, + arm: null, + supRunDir: runDir, + journal: normalized.journal, + brainLog: null, + brainLogMissingReason: + 'Runtime FileRunContext records spend but not model completion finish reasons', + state: runtimeState( + normalized.root, + normalized.startedAt, + result, + trajectory, + resultPath, + trajectoryPath, + ), + progress: null, + workers, + workersMissingReason: null, + result: resultText, + judge: null, + judgeSource: null, + patch: null, + driverLog: null, + harnessWorkerTokens: null, + harnessMissingReason: 'Runtime FileRunContext has no external worker-token join', + limits: sourceLimits(normalized.root, normalized.events, workerIds), + rootTranscriptRef: null, + traceCommand: 'unavailable — Runtime FileRunContext records no provider-session trace identity', + } +} + +/** The agent-runtime file-backed layout as a `SupervisorRunReader`. */ +export function runtimeSupervisorRunReader(runDir: string): SupervisorRunReader { + return { runRef: runDir, read: () => readRuntimeSupervisorRun(runDir) } +} + +/** True when a directory contains Runtime's canonical file-backed journal. */ +export async function isRuntimeSupervisorRunDir(runDir: string): Promise { + return stat(join(runDir, JOURNAL_FILE)) + .then((entry) => entry.isFile()) + .catch((error: unknown) => { + if ( + typeof error === 'object' && + error !== null && + 'code' in error && + ((error as { code: unknown }).code === 'ENOENT' || + (error as { code: unknown }).code === 'ENOTDIR') + ) { + return false + } + throw error + }) +} diff --git a/src/supervisor-run/source-facts.ts b/src/supervisor-run/source-facts.ts index 92328927..d506e15a 100644 --- a/src/supervisor-run/source-facts.ts +++ b/src/supervisor-run/source-facts.ts @@ -10,9 +10,12 @@ interface JournalEvent { parent?: unknown label?: unknown role?: unknown + profileDigest?: unknown + runtime?: unknown status?: unknown verdict?: unknown reason?: unknown + infra?: unknown seq?: unknown at?: unknown spend?: unknown @@ -116,7 +119,12 @@ export interface SpawnRow { id: string parent: string | null label: string - role: SupervisorRunNodeRole + /** Null when the source did not explicitly record a role. */ + role: SupervisorRunNodeRole | null + /** Exact canonical AgentProfile digest, when the source recorded one. */ + profileDigest: string | null + /** Exact execution runtime tag, when the source recorded one. */ + runtime: string | null at: number | null /** False when identity, parentage, label, or role was malformed. */ valid: boolean @@ -135,6 +143,10 @@ export interface CloseRow { score: number | null /** The verdict exactly as the journal carried it. */ rawVerdict: unknown | null + /** Terminal failure detail exactly as recorded. */ + reason: string | null + /** Whether Runtime classified the failure as infrastructure-caused. */ + infra: boolean | null at: number | null spend: SpendLike /** False when the close event carried no spend object — not "spent nothing". */ @@ -260,6 +272,7 @@ export function parseSupervisorTree(src: SupervisorRunSources): SupervisorTreeFa let brainUsd = 0 let meteredCount = 0 let rootId: string | null = null + const meteredRows: Array<{ id: string; spend: SpendLike }> = [] for (const [sourceRow, ev] of (events as JournalEvent[]).entries()) { const kind = typeof ev.kind === 'string' ? ev.kind : '' const id = typeof ev.id === 'string' ? ev.id : '' @@ -280,12 +293,17 @@ export function parseSupervisorTree(src: SupervisorRunSources): SupervisorTreeFa if (ev.role !== undefined && ev.role !== 'supervisor' && ev.role !== 'worker') { invalidFields.push('role') } - const role: SupervisorRunNodeRole = - ev.role === 'supervisor' || ev.role === 'worker' - ? ev.role - : parent === null - ? 'supervisor' - : 'worker' + if ( + ev.profileDigest !== undefined && + !(typeof ev.profileDigest === 'string' && ev.profileDigest.length > 0) + ) { + invalidFields.push('profileDigest') + } + if (ev.runtime !== undefined && !(typeof ev.runtime === 'string' && ev.runtime.length > 0)) { + invalidFields.push('runtime') + } + const role: SupervisorRunNodeRole | null = + ev.role === 'supervisor' || ev.role === 'worker' ? ev.role : null const valid = invalidFields.length === 0 if (valid && parent === null && rootId === null) rootId = id spawns.push({ @@ -294,6 +312,11 @@ export function parseSupervisorTree(src: SupervisorRunSources): SupervisorTreeFa parent, label, role, + profileDigest: + typeof ev.profileDigest === 'string' && ev.profileDigest.length > 0 + ? ev.profileDigest + : null, + runtime: typeof ev.runtime === 'string' && ev.runtime.length > 0 ? ev.runtime : null, at: ms(ev.at), valid, invalidFields, @@ -308,6 +331,8 @@ export function parseSupervisorTree(src: SupervisorRunSources): SupervisorTreeFa valid: verdict.valid, score: verdict.score, rawVerdict: verdict.raw, + reason: typeof ev.reason === 'string' ? ev.reason : null, + infra: typeof ev.infra === 'boolean' ? ev.infra : null, at: ms(ev.at), spend: readSpend(ev.spent), hasSpend: asRecord(ev.spent).tokens !== undefined, @@ -321,6 +346,8 @@ export function parseSupervisorTree(src: SupervisorRunSources): SupervisorTreeFa valid: null, score: null, rawVerdict: typeof ev.reason === 'string' ? ev.reason : null, + reason: typeof ev.reason === 'string' ? ev.reason : null, + infra: null, at: ms(ev.at), spend: { tokens: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, hasCache: false }, @@ -329,17 +356,21 @@ export function parseSupervisorTree(src: SupervisorRunSources): SupervisorTreeFa hasSpend: false, }) } else if (kind === 'metered') { - const s = readSpend(ev.spend) - brainIn += s.tokens.input - brainOut += s.tokens.output - brainCacheRead += s.tokens.cacheRead - brainCacheWrite += s.tokens.cacheWrite - brainHasCache = brainHasCache || s.tokens.hasCache - brainUsd += s.usd - meteredCount += 1 + meteredRows.push({ id, spend: readSpend(ev.spend) }) } } + for (const row of meteredRows) { + if (rootId === null || row.id !== rootId) continue + brainIn += row.spend.tokens.input + brainOut += row.spend.tokens.output + brainCacheRead += row.spend.tokens.cacheRead + brainCacheWrite += row.spend.tokens.cacheWrite + brainHasCache = brainHasCache || row.spend.tokens.hasCache + brainUsd += row.spend.usd + meteredCount += 1 + } + const workerSpawns = spawns.filter((s) => s.valid && s.id !== rootId) const workerIds = new Set(workerSpawns.map((s) => s.id)) const workerCloses = closes.filter((c) => workerIds.has(c.id)) @@ -460,12 +491,8 @@ export function parseSupervisorTree(src: SupervisorRunSources): SupervisorTreeFa } const startedAt = ms(state?.startedAt) ?? spawns[0]?.at ?? null - const completedAt = - ms(state?.completedAt) ?? - [...spawns.map((s) => s.at), ...closes.map((c) => c.at)].reduce( - (acc, t) => (t === null ? acc : acc === null ? t : Math.max(acc, t)), - null, - ) + const rootClose = rootId === null ? null : closes.find((close) => close.id === rootId) + const completedAt = ms(state?.completedAt) ?? rootClose?.at ?? null return { rootId, diff --git a/src/supervisor-run/types.ts b/src/supervisor-run/types.ts index 4705b28a..fd06ff8c 100644 --- a/src/supervisor-run/types.ts +++ b/src/supervisor-run/types.ts @@ -147,6 +147,8 @@ export interface SupervisorRunSources { readonly journal: string | null /** Per-brain-call tap (JSONL): finish_reason, completion tokens, requested max tokens. */ readonly brainLog: string | null + /** Source-specific reason `brainLog` is absent. */ + readonly brainLogMissingReason?: string /** Supervisor state document (JSON). */ readonly state: string | null /** Supervisor progress stream (JSONL). */ @@ -297,6 +299,16 @@ export interface PerWorkerRow { readonly worker: string /** Explicit journal role after the worker source was joined to its spawn. */ readonly role: SupervisorRunNodeRole | null + /** Exact execution runtime tag from the spawn event. */ + readonly runtime: string | null + /** Exact canonical AgentProfile digest from the spawn event. */ + readonly profileDigest: string | null + /** Terminal lifecycle status exactly as recorded. */ + readonly status: string | null + /** Terminal failure detail exactly as recorded. */ + readonly failure: string | null + /** Runtime infrastructure classification, when recorded. */ + readonly infra: boolean | null readonly wallMs: number | null /** `null` = this store does not attribute tokens per worker (NOT "zero tokens"). */ readonly tokensIn: number | null @@ -371,6 +383,7 @@ export interface SupervisorRunReport { readonly instanceId: string | null readonly arm: string | null readonly supervisorId: Measured + readonly supervisorProfileDigest: Measured readonly generatedAt: string readonly orchestration: OrchestrationMetrics readonly decision: DecisionMetrics @@ -430,6 +443,7 @@ export type SupervisorRunTreeGapCode = | 'root-spawn-unavailable' | 'root-reward-unavailable' | 'child-reward-unavailable' + | 'node-role-unavailable' | 'node-schema-invalid' export interface SupervisorRunTreeGap { From 1e4061557599231c8bac76fd8b0318aec87aa621 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Thu, 30 Jul 2026 11:43:35 -0600 Subject: [PATCH 2/2] perf(supervisor-run): index nested Runtime roots --- src/supervisor-run/runtime-reader.test.ts | 63 +++++++++++++++++++++++ src/supervisor-run/runtime-reader.ts | 42 ++++++++------- 2 files changed, 85 insertions(+), 20 deletions(-) diff --git a/src/supervisor-run/runtime-reader.test.ts b/src/supervisor-run/runtime-reader.test.ts index 4c7313e1..8d00a6c8 100644 --- a/src/supervisor-run/runtime-reader.test.ts +++ b/src/supervisor-run/runtime-reader.test.ts @@ -415,6 +415,69 @@ describe('Runtime FileRunContext supervisor reader', () => { expect(await findSupervisorRunDirs(root)).toEqual([join(root, 'run-a'), join(root, 'run-b')]) }) + it('composes 4,000 nested Runtime trees without scanning the journal once per tree', { + timeout: 10_000, + }, async () => { + const root = await mkdtemp(join(tmpdir(), 'runtime-supervisor-run-')) + const runDir = join(root, 'large-nested') + const rows: Record[] = [ + begin('node-0', 0), + event('node-0', { + kind: 'spawned', + id: 'node-0', + label: 'node-0', + profileDigest: ROOT_PROFILE, + budget: {}, + seq: 0, + at: at(0), + }), + event('node-0', { + kind: 'metered', + id: 'node-0', + spend: { + iterations: 1, + tokens: { input: 1, output: 1 }, + usd: 0, + ms: 1, + }, + seq: 0, + at: at(0), + }), + ] + for (let index = 1; index < 4_000; index += 1) { + const parent = `node-${index - 1}` + const id = `node-${index}` + rows.push( + event(parent, { + kind: 'spawned', + id, + parent, + label: id, + profileDigest: CHILD_PROFILE, + runtime: 'driver', + budget: {}, + seq: 0, + at: at(index), + }), + begin(id, index), + event(id, { + kind: 'spawned', + id, + label: id, + profileDigest: CHILD_PROFILE, + budget: {}, + seq: 0, + at: at(index), + }), + ) + } + await writeJournal(runDir, rows) + + const source = await readRuntimeSupervisorRun(runDir) + expect(source.workers).toHaveLength(3_999) + expect(source.journal?.split('\n').filter(Boolean)).toHaveLength(4_001) + }) + it('refuses a nested tree whose exact profile identity disagrees with its parent spawn', async () => { const root = await mkdtemp(join(tmpdir(), 'runtime-supervisor-run-')) const runDir = join(root, 'mismatch') diff --git a/src/supervisor-run/runtime-reader.ts b/src/supervisor-run/runtime-reader.ts index 295b8790..a390f3b7 100644 --- a/src/supervisor-run/runtime-reader.ts +++ b/src/supervisor-run/runtime-reader.ts @@ -118,20 +118,29 @@ function parseEnvelopeJournal(text: string, path: string): NormalizedRuntimeJour if (begins.length === 0) throw formatError(path, 1, 'no begin record') - const spawnedById = new Map() + const parentSpawnsById = new Map() + const rootMarkersByTree = new Map() for (const entry of events) { if (entry.event.kind !== 'spawned') continue const id = nonEmptyString(entry.event.id) if (id === null) continue - const matches = spawnedById.get(id) ?? [] - matches.push(entry) - spawnedById.set(id, matches) + if (nonEmptyString(entry.event.parent) !== null) { + const matches = parentSpawnsById.get(id) ?? [] + matches.push(entry) + parentSpawnsById.set(id, matches) + } + if (entry.root === id && entry.event.parent === undefined) { + const markers = rootMarkersByTree.get(entry.root) ?? [] + markers.push(entry) + rootMarkersByTree.set(entry.root, markers) + } } const nestedRoots = new Set() + const nestedParentSpawns = new Map() for (const begin of begins) { - const parentSpawns = (spawnedById.get(begin.root) ?? []).filter( - (entry) => entry.root !== begin.root && nonEmptyString(entry.event.parent) !== null, + const parentSpawns = (parentSpawnsById.get(begin.root) ?? []).filter( + (entry) => entry.root !== begin.root, ) if (parentSpawns.length > 1) { throw formatError( @@ -140,7 +149,10 @@ function parseEnvelopeJournal(text: string, path: string): NormalizedRuntimeJour `tree ${JSON.stringify(begin.root)} has ${parentSpawns.length} parent spawns`, ) } - if (parentSpawns.length === 1) nestedRoots.add(begin.root) + if (parentSpawns.length === 1) { + nestedRoots.add(begin.root) + nestedParentSpawns.set(begin.root, parentSpawns[0] as EventRecord) + } } const topRoots = begins.filter((begin) => !nestedRoots.has(begin.root)) @@ -154,13 +166,7 @@ function parseEnvelopeJournal(text: string, path: string): NormalizedRuntimeJour const top = topRoots[0] as BeginRecord for (const nestedRoot of nestedRoots) { - const marker = events.filter( - (entry) => - entry.root === nestedRoot && - entry.event.kind === 'spawned' && - entry.event.id === nestedRoot && - entry.event.parent === undefined, - ) + const marker = rootMarkersByTree.get(nestedRoot) ?? [] if (marker.length !== 1) { throw formatError( path, @@ -168,9 +174,7 @@ function parseEnvelopeJournal(text: string, path: string): NormalizedRuntimeJour `nested tree ${JSON.stringify(nestedRoot)} must contain one root marker`, ) } - const parentSpawn = (spawnedById.get(nestedRoot) ?? []).find( - (entry) => entry.root !== nestedRoot && nonEmptyString(entry.event.parent) !== null, - ) + const parentSpawn = nestedParentSpawns.get(nestedRoot) if (parentSpawn === undefined) { throw formatError( path, @@ -204,9 +208,7 @@ function parseEnvelopeJournal(text: string, path: string): NormalizedRuntimeJour ) .map((entry) => entry.event) - const rootMarkers = normalized.filter( - (event) => event.kind === 'spawned' && event.id === top.root && event.parent === undefined, - ) + const rootMarkers = rootMarkersByTree.get(top.root) ?? [] if (rootMarkers.length !== 1) { throw formatError( path,