From 0f7760e226837dc9fe6254fd617e6196623e0a8b Mon Sep 17 00:00:00 2001 From: Lu Nelson Date: Wed, 26 Aug 2026 15:53:13 +0200 Subject: [PATCH 01/10] Measure condition 5 turns by purpose Record Flue wall-clock timings in run artifacts so latency evidence no longer relies on token-volume inference. Co-authored-by: Cursor --- .../test/baseline-harness.test.ts | 53 ++++++-- .../fixtures/baseline-harness-interviewer.ts | 34 ++++- apps/brunch-agent/test/turn-timing.test.ts | 115 ++++++++++++++++ .../baseline/harness-run.ts | 47 ++++++- .../baseline/protocol.md | 11 +- .../baseline/turn-timing.ts | 123 ++++++++++++++++++ 6 files changed, 366 insertions(+), 17 deletions(-) create mode 100644 apps/brunch-agent/test/turn-timing.test.ts create mode 100644 libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/turn-timing.ts diff --git a/apps/brunch-agent/test/baseline-harness.test.ts b/apps/brunch-agent/test/baseline-harness.test.ts index 7eed3d23721..ebb5240eaee 100644 --- a/apps/brunch-agent/test/baseline-harness.test.ts +++ b/apps/brunch-agent/test/baseline-harness.test.ts @@ -13,6 +13,7 @@ import { import { runNodeScript } from "./run-node-script"; import type { HarnessRunRecord } from "../../../libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/harness-run.ts"; +import type { TurnTimingRecord } from "../../../libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/turn-timing.ts"; const isRecord = (value: unknown): value is Record => typeof value === "object" && value !== null; @@ -68,6 +69,7 @@ test("condition 5 drives the shipped elicitor through the binding and reads the BRUNCH_BASELINE_ANTHROPIC_MODULE: expertStub, BRUNCH_BASELINE_INTERVIEWER_PROVIDER_MODULE: interviewer, BASELINE_STUB_REPLIES_PATH: expertRepliesPath, + BASELINE_STUB_REFUSE_FIRST_SWEEP: "1", BRUNCH_SDCPN_MODEL: "claude-haiku-4-5", }, ); @@ -75,7 +77,34 @@ test("condition 5 drives the shipped elicitor through the binding and reads the const run = JSON.parse( await readFile(join(outputDirectory, "condition-5.raw.json"), "utf8"), - ) as HarnessRunRecord; + ) as HarnessRunRecord & { + readonly timings?: readonly TurnTimingRecord[]; + }; + const timingRecords = ( + await readFile(join(outputDirectory, "condition-5.timings.jsonl"), "utf8") + ) + .trim() + .split("\n") + .map((line) => JSON.parse(line) as TurnTimingRecord); + expect(timingRecords.length).toBeGreaterThan(0); + expect( + timingRecords.every( + (record) => + record.interviewerTurn >= 1 && + record.durationMs >= 0 && + ["interview", "sweep", "repair"].includes(record.purpose), + ), + ).toBe(true); + expect( + new Set(timingRecords.map((record) => record.purpose)), + JSON.stringify(timingRecords, null, 2), + ).toEqual(new Set(["interview", "sweep", "repair"])); + expect(timingRecords).toHaveLength(run.usage.interviewer.calls); + expect(run.timings).toEqual(timingRecords); + expect(run.turns.flatMap((turn) => turn.timings)).toEqual(timingRecords); + expect( + timingRecords.filter((record) => record.purpose === "repair"), + ).toHaveLength(2); // Turn 1 asks; the expert's reply is bound to that ask on the next dispatch. expect(run.turns[0]?.pendingQuestion).toBe(FIRST_QUESTION); @@ -86,14 +115,17 @@ test("condition 5 drives the shipped elicitor through the binding and reads the ), ).toBe(true); - // Turn 2 sweeps the settled range: one capture applied, completion reported - // by the harness, and the second question left pending. - const sweep = run.turns[1]?.sweeps[0]; - expect(sweep?.status).toBe("applied"); - expect(sweep?.appliedCaptureIds).toHaveLength(1); - expect(sweep?.completion).toMatchObject({ complete: false }); - expect(run.turns[1]?.pendingQuestion).toBe(SECOND_QUESTION); - expect(run.turns[1]?.completion).toMatchObject({ + // The first sweep is refused on its deliberately bad quote, then the + // repair continuation re-emits it with the verbatim expert evidence. + const sweeps = run.turns.flatMap((turn) => turn.sweeps); + expect(sweeps[0]?.status).toBe("refused"); + const appliedSweep = sweeps.find((sweep) => sweep.status === "applied"); + expect(appliedSweep?.appliedCaptureIds).toHaveLength(1); + expect(appliedSweep?.completion).toMatchObject({ complete: false }); + expect( + run.turns.some((turn) => turn.pendingQuestion === SECOND_QUESTION), + ).toBe(true); + expect(run.turns.at(-1)?.completion).toMatchObject({ captures: 1, complete: false, }); @@ -167,6 +199,9 @@ test("condition 5 drives the shipped elicitor through the binding and reads the expect(model).toContain("### objective (1)"); expect(model).toContain("Complete: **no**"); expect(transcript).toContain("Stop reason: stalled"); + expect(transcript).toContain("**Interviewer — turn 1** | interview "); + expect(transcript).toContain("| sweep "); + expect(transcript).toMatch(/\| repair \d+ ms/u); expect(transcript).toContain("> harness — sweep applied; applied 1"); expect(transcript).toContain(FIRST_QUESTION); expect(system).toContain("brunch_ask"); diff --git a/apps/brunch-agent/test/fixtures/baseline-harness-interviewer.ts b/apps/brunch-agent/test/fixtures/baseline-harness-interviewer.ts index 7dea6d0beeb..1dedd167013 100644 --- a/apps/brunch-agent/test/fixtures/baseline-harness-interviewer.ts +++ b/apps/brunch-agent/test/fixtures/baseline-harness-interviewer.ts @@ -55,6 +55,11 @@ const objectiveProposal: SlotAssertedProposalInput = { }, }; +const refusedObjectiveProposal: SlotAssertedProposalInput = { + ...objectiveProposal, + evidence: [{ excerpt: "This quote is deliberately absent." }], +}; + const countToolCalls = (context: Context, name: string): number => { let count = 0; for (const message of context.messages) { @@ -66,19 +71,46 @@ const countToolCalls = (context: Context, name: string): number => { return count; }; +const latestUserText = (context: Context): string | undefined => { + const latestMessage = context.messages.at(-1); + if (latestMessage?.role !== "user") return undefined; + return typeof latestMessage.content === "string" + ? latestMessage.content + : latestMessage.content + .filter((part) => part.type === "text") + .map((part) => part.text) + .join("\n"); +}; + const faux = fauxProvider({ provider: "anthropic", models: [{ id: SDCPN_MODEL_ID }], }); +let extractionCalls = 0; faux.setResponses( Array.from({ length: 64 }, () => (context: Context) => { if (context.tools?.some((tool) => tool.name === "finish")) { + extractionCalls += 1; return fauxAssistantMessage( - [fauxToolCall("finish", { proposals: [objectiveProposal] })], + [ + fauxToolCall("finish", { + proposals: [ + process.env["BASELINE_STUB_REFUSE_FIRST_SWEEP"] === "1" && + extractionCalls === 1 + ? refusedObjectiveProposal + : objectiveProposal, + ], + }), + ], { stopReason: "toolUse" }, ); } + if (latestUserText(context)?.startsWith(" & Pick, +): FlueObservation => event as FlueObservation; + +const request = (latestUserMessage = "Continue."): ModelRequest => ({ + providerId: "faux", + providerName: "faux", + requestedModel: "faux-model", + api: "faux", + input: { + messages: [{ role: "user", content: latestUserMessage }], + }, +}); + +const completedTurn = ( + turnId: string, + operationId: string, + purpose: "agent" | "compaction", +): FlueObservation => + observation({ + type: "turn", + turnId, + operationId, + purpose, + durationMs: 17, + request: { + providerId: "faux", + providerName: "faux", + requestedModel: "faux-model", + api: "faux", + }, + response: {}, + isError: false, + }); + +test("attributes compaction and nested extraction to the active harness purpose", () => { + const recorder = createTurnTimingRecorder(); + recorder.startInterviewerTurn(1); + recorder.observe( + observation({ + type: "operation_start", + operationId: "outer", + operationKind: "prompt", + }), + ); + recorder.observe( + observation({ + type: "turn_request", + turnId: "interview", + operationId: "outer", + purpose: "agent", + request: request(), + }), + ); + recorder.observe(completedTurn("interview", "outer", "agent")); + recorder.observe( + observation({ + type: "turn_request", + turnId: "repair", + operationId: "outer", + purpose: "agent", + request: request('\nRetry.\n'), + }), + ); + recorder.observe(completedTurn("repair", "outer", "agent")); + recorder.observe( + observation({ + type: "turn_request", + turnId: "repair-compaction", + operationId: "outer", + purpose: "compaction", + request: request(), + }), + ); + recorder.observe(completedTurn("repair-compaction", "outer", "compaction")); + recorder.observe( + observation({ + type: "operation_start", + operationId: "nested", + operationKind: "prompt", + }), + ); + recorder.observe( + observation({ + type: "turn_request", + turnId: "repair-extraction", + operationId: "nested", + purpose: "agent", + request: request("Extract repaired proposals."), + }), + ); + recorder.observe(completedTurn("repair-extraction", "nested", "agent")); + + expect( + Object.fromEntries( + recorder + .all() + .map((timing) => [timing.flueTurnId, timing.purpose] as const), + ), + ).toEqual>({ + interview: "interview", + repair: "repair", + "repair-compaction": "repair", + "repair-extraction": "repair", + }); +}); diff --git a/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/harness-run.ts b/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/harness-run.ts index 9e7a9036227..18a4eb48049 100644 --- a/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/harness-run.ts +++ b/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/harness-run.ts @@ -34,6 +34,7 @@ * condition-5-model.md the capture store folded into the elicited model, with the completion report * condition-5-captures.json the capture-store snapshot verbatim * condition-5-system.md the interviewer's instructions, reconstructed with the binding's own functions + * condition-5.timings.jsonl each observed Flue model call, tagged by interviewer-turn purpose */ import { cp, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; @@ -72,6 +73,12 @@ import { import { sdcpn, sdcpnDefinition } from "@hashintel/brunch-agent-plugin-sdcpn"; import { repertoire } from "@hashintel/brunch-agent-repertoire"; +import { + createTurnTimingRecorder, + type TurnTimingPurpose, + type TurnTimingRecord, +} from "./turn-timing.ts"; + import type Anthropic from "@anthropic-ai/sdk"; import type { Provider } from "@earendil-works/pi-ai"; @@ -198,6 +205,8 @@ export interface HarnessTurnRecord { readonly settlement?: "failed" | "aborted"; /** The one question left open for the expert, if any. */ readonly pendingQuestion?: string; + /** Flue model-call timings observed while this interviewer turn ran. */ + readonly timings: readonly TurnTimingRecord[]; /** The harness's read-time completion over the capture store after this turn. */ readonly completion: HarnessCompletionRecord; /** What the expert was then sent: their reply, or a stimulus. */ @@ -216,6 +225,7 @@ export interface HarnessRunRecord { readonly conversationId: string; readonly stopReason: string; readonly turns: readonly HarnessTurnRecord[]; + readonly timings: readonly TurnTimingRecord[]; readonly usage: { readonly interviewer: Usage; readonly expert: Usage }; readonly history: FlueConversationSnapshot; readonly store: CaptureStoreSnapshot; @@ -334,7 +344,10 @@ const sweepRecordOf = (output: unknown): HarnessSweepRecord => { /** Everything the interviewer did between two of our dispatches. */ function readTurn( messages: readonly FlueConversationMessage[], -): Omit { +): Omit< + HarnessTurnRecord, + "turn" | "completion" | "pendingQuestion" | "timings" +> { const text: string[] = []; const asks: HarnessTurnRecord["asks"][number][] = []; const sweeps: HarnessSweepRecord[] = []; @@ -523,6 +536,19 @@ function renderModel( const formatUsage = (usage: Usage): string => `${usage.input} in (+${usage.cacheWrite} cache write, +${usage.cacheRead} cache read) / ${usage.output} out across ${usage.calls} calls`; +const formatPurposeTiming = ( + timings: readonly TurnTimingRecord[], + purpose: TurnTimingPurpose, +): string => { + const matching = timings.filter((timing) => timing.purpose === purpose); + if (matching.length === 0) return "—"; + const durationMs = matching.reduce( + (total, timing) => total + timing.durationMs, + 0, + ); + return `${durationMs} ms (${matching.length} call${matching.length === 1 ? "" : "s"})`; +}; + function renderTranscript( run: HarnessRunRecord, openingMessage: string, @@ -552,7 +578,12 @@ function renderTranscript( openingMessage, ]; const body = run.turns.map((turn) => { - const parts: string[] = ["---", "", "**Interviewer**:", ""]; + const parts: string[] = [ + "---", + "", + `**Interviewer — turn ${turn.turn}** | interview ${formatPurposeTiming(turn.timings, "interview")} | sweep ${formatPurposeTiming(turn.timings, "sweep")} | repair ${formatPurposeTiming(turn.timings, "repair")}`, + "", + ]; if (turn.text.length === 0 && turn.asks.length === 0) { parts.push("_(no visible text this turn)_"); } @@ -675,7 +706,9 @@ const interviewerUsage: Usage = { cacheWrite: 0, calls: 0, }; +const turnTimingRecorder = createTurnTimingRecorder(); const stopObserving = observe((event) => { + turnTimingRecorder.observe(event); if (event.type !== "turn") return; interviewerUsage.calls += 1; const usage = event.response.usage; @@ -727,6 +760,7 @@ async function writeArtifacts(): Promise { conversationId, stopReason, turns, + timings: turnTimingRecorder.all(), usage: { interviewer: interviewerUsage, expert: expertUsage }, history, store: storeSnapshot, @@ -744,6 +778,13 @@ async function writeArtifacts(): Promise { `${stem}.md`, renderTranscript(run, openingMessage, sweepTally), ); + await writeFile( + `${stem}.timings.jsonl`, + `${turnTimingRecorder + .all() + .map((timing) => JSON.stringify(timing)) + .join("\n")}\n`, + ); await writeFile(`${stem}-model.md`, renderModel(model, report, record)); await writeFile( `${stem}-captures.json`, @@ -780,6 +821,7 @@ try { while (turns.length < HARD_STOP_AT) { const turnNumber = turns.length + 1; console.error(`turn ${turnNumber} (interviewer)`); + turnTimingRecorder.startInterviewerTurn(turnNumber); await dispatch(outgoing, initial); initial = false; @@ -802,6 +844,7 @@ try { turn: turnNumber, ...observed, ...(pendingQuestion === undefined ? {} : { pendingQuestion }), + timings: turnTimingRecorder.forInterviewerTurn(turnNumber), completion, }; turns.push(turn); diff --git a/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/protocol.md b/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/protocol.md index d0a87e061b2..6a3b5048d64 100644 --- a/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/protocol.md +++ b/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/protocol.md @@ -100,8 +100,8 @@ Condition 4 otherwise uses conditions 1–2's mechanics: the legacy impatience p budget, and delivery classifier. Condition 5 runs from the application package, which owns the agent composition: `turbo run baseline:harness --filter '@apps/brunch-agent'` (builds the workspace first; writes `condition-5.md`, `condition-5.raw.json`, `condition-5-model.md`, -`condition-5-captures.json`, and `condition-5-system.md`). `run.ts` accepts only `1`, `2`, and `4`; -condition 3 has no entry point. Production transcripts land in +`condition-5-captures.json`, `condition-5-system.md`, and `condition-5.timings.jsonl`). `run.ts` +accepts only `1`, `2`, and `4`; condition 3 has no entry point. Production transcripts land in `docs/evidence/evaluations/process-model-elicitation/baseline/transcripts/`. Tests set `BRUNCH_BASELINE_TEST_OUTPUT_DIR` to an isolated directory and never write committed evidence; the condition-5 test additionally swaps both models for stand-ins @@ -128,9 +128,10 @@ condition 3 has no entry point. Production transcripts land in completion, and how it uses the completion cue and the settlement nudge. 6. **Excavation checks**: did the interviewer surface the _(tacit)_ facts, correct the _(believes)_ errors, and record the _(doesn't know)_ absences as absences? -7. **Turn cost (condition 5 only)**: tokens per turn by purpose (interview, sweep, repair) and, - once the runner records Flue's `turn` event `durationMs`, wall-clock per purpose and time to - the visible question. The first run recorded tokens and the run window only; see the +7. **Turn cost (condition 5 only)**: Flue's `turn` event `durationMs` per model call, grouped by + interviewer turn and tagged as interview, sweep, or repair from harness signal order. The raw + record, transcript turn headers, and JSONL timing artifact carry the measurements. The first run + recorded tokens and the run window only; see the [turn latency assessment](../../../../docs/evidence/evaluations/process-model-elicitation/baseline/condition-5-turn-latency.md). ## Threats to validity (acknowledged) diff --git a/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/turn-timing.ts b/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/turn-timing.ts new file mode 100644 index 00000000000..70936d379bb --- /dev/null +++ b/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/turn-timing.ts @@ -0,0 +1,123 @@ +import type { FlueObservation, ModelRequest } from "@flue/runtime"; + +export const TURN_TIMING_PURPOSES = ["interview", "sweep", "repair"] as const; + +export type TurnTimingPurpose = (typeof TURN_TIMING_PURPOSES)[number]; + +export interface TurnTimingRecord { + readonly interviewerTurn: number; + readonly flueTurnId: string; + readonly purpose: TurnTimingPurpose; + readonly durationMs: number; +} + +export interface TurnTimingRecorder { + startInterviewerTurn(interviewerTurn: number): void; + observe(event: FlueObservation): void; + forInterviewerTurn(interviewerTurn: number): readonly TurnTimingRecord[]; + all(): readonly TurnTimingRecord[]; +} + +const signalPurpose = ( + request: ModelRequest, +): TurnTimingPurpose | undefined => { + const latestMessage = request.input.messages.at(-1); + if (latestMessage?.role !== "user") return undefined; + const content = + typeof latestMessage.content === "string" + ? latestMessage.content + : latestMessage.content + .filter((part) => part.type === "text") + .map((part) => part.text) + .join("\n"); + if (content.startsWith(" { + let currentInterviewerTurn: number | undefined; + const activePromptOperationIds: string[] = []; + const nestedPromptOperationIds = new Set(); + const purposeByOperation = new Map(); + const purposeByFlueTurn = new Map(); + const records: TurnTimingRecord[] = []; + + return { + startInterviewerTurn(interviewerTurn) { + currentInterviewerTurn = interviewerTurn; + }, + observe(event) { + if ( + event.type === "operation_start" && + event.operationKind === "prompt" + ) { + const parentOperationId = activePromptOperationIds.at(-1); + if (parentOperationId !== undefined) { + nestedPromptOperationIds.add(event.operationId); + purposeByOperation.set( + event.operationId, + purposeByOperation.get(parentOperationId) === "repair" + ? "repair" + : "sweep", + ); + } + activePromptOperationIds.push(event.operationId); + return; + } + if (event.type === "operation") { + const activeIndex = activePromptOperationIds.lastIndexOf( + event.operationId, + ); + if (activeIndex !== -1) activePromptOperationIds.splice(activeIndex, 1); + nestedPromptOperationIds.delete(event.operationId); + purposeByOperation.delete(event.operationId); + return; + } + if (event.type === "turn_request") { + const operationPurpose = + event.operationId === undefined + ? undefined + : purposeByOperation.get(event.operationId); + const purpose = + event.operationId !== undefined && + nestedPromptOperationIds.has(event.operationId) + ? (operationPurpose ?? "sweep") + : event.purpose === "agent" + ? (signalPurpose(event.request) ?? "interview") + : (operationPurpose ?? "interview"); + purposeByFlueTurn.set(event.turnId, purpose); + if ( + event.operationId !== undefined && + !nestedPromptOperationIds.has(event.operationId) + ) { + purposeByOperation.set(event.operationId, purpose); + } + return; + } + if (event.type !== "turn" || currentInterviewerTurn === undefined) { + return; + } + records.push({ + interviewerTurn: currentInterviewerTurn, + flueTurnId: event.turnId, + purpose: + purposeByFlueTurn.get(event.turnId) ?? + (event.operationId === undefined + ? undefined + : purposeByOperation.get(event.operationId)) ?? + "interview", + durationMs: event.durationMs, + }); + purposeByFlueTurn.delete(event.turnId); + }, + forInterviewerTurn(interviewerTurn) { + return records.filter( + (record) => record.interviewerTurn === interviewerTurn, + ); + }, + all() { + return records; + }, + }; +}; From e5f1292c5f9728cda5a17724821dde691524320e Mon Sep 17 00:00:00 2001 From: Lu Nelson Date: Wed, 26 Aug 2026 16:02:14 +0200 Subject: [PATCH 02/10] Emit privacy-safe server telemetry spans Co-authored-by: Cursor --- apps/brunch-agent/package.json | 4 ++ apps/brunch-agent/src/app.ts | 4 ++ .../test/walking-skeleton.integration.ts | 39 ++++++++++++++++++- .../test/walking-skeleton.test.ts | 2 + yarn.lock | 14 +++++++ 5 files changed, 62 insertions(+), 1 deletion(-) diff --git a/apps/brunch-agent/package.json b/apps/brunch-agent/package.json index 637dc659233..f81915b9b43 100644 --- a/apps/brunch-agent/package.json +++ b/apps/brunch-agent/package.json @@ -16,6 +16,7 @@ "test:unit": "vitest run --config vitest.config.ts" }, "dependencies": { + "@flue/opentelemetry": "2.0.3", "@flue/react": "2.0.3", "@flue/runtime": "2.0.3", "@flue/sdk": "2.0.3", @@ -24,6 +25,7 @@ "@hashintel/brunch-agent-plugin-gherkin": "workspace:*", "@hashintel/brunch-agent-plugin-sdcpn": "workspace:*", "@hashintel/brunch-agent-transport-aisdk": "workspace:*", + "@opentelemetry/api": "1.9.1", "hono": "4.13.2", "react": "19.2.6", "react-dom": "19.2.6", @@ -32,6 +34,8 @@ "devDependencies": { "@earendil-works/pi-ai": "0.83.0", "@flue/vite": "2.0.3", + "@opentelemetry/sdk-trace-base": "2.9.0", + "@opentelemetry/sdk-trace-node": "2.9.0", "@types/node": "22.18.13", "@types/react": "19.2.14", "@types/react-dom": "19.2.3", diff --git a/apps/brunch-agent/src/app.ts b/apps/brunch-agent/src/app.ts index 43f0315289f..308dbe9389c 100644 --- a/apps/brunch-agent/src/app.ts +++ b/apps/brunch-agent/src/app.ts @@ -9,6 +9,8 @@ import { readFile } from "node:fs/promises"; +import { createOpenTelemetryInstrumentation } from "@flue/opentelemetry"; +import { instrument } from "@flue/runtime"; import { createAgentRouter } from "@flue/runtime/routing"; import { Hono } from "hono"; @@ -22,6 +24,8 @@ import { SDCPN_AGENT_ROUTE, } from "./routes.ts"; +instrument(createOpenTelemetryInstrumentation({ content: false })); + const app = new Hono(); // One route per target agent. The gallery grows an entry per plugin; gherkin diff --git a/apps/brunch-agent/test/walking-skeleton.integration.ts b/apps/brunch-agent/test/walking-skeleton.integration.ts index 607ca8872dc..39ee6cbe5d0 100644 --- a/apps/brunch-agent/test/walking-skeleton.integration.ts +++ b/apps/brunch-agent/test/walking-skeleton.integration.ts @@ -12,7 +12,13 @@ import { type Context, } from "@earendil-works/pi-ai"; import { start } from "@flue/runtime/node"; +import { CONTENT_ATTR } from "@flue/runtime/telemetry"; import { createFlueClient } from "@flue/sdk"; +import { + InMemorySpanExporter, + SimpleSpanProcessor, +} from "@opentelemetry/sdk-trace-base"; +import { NodeTracerProvider } from "@opentelemetry/sdk-trace-node"; import { toolName } from "@hashintel/brunch-agent"; import { @@ -25,18 +31,28 @@ import { GHERKIN_MODEL_ID, GherkinElicitor, } from "../src/agents/gherkin-elicitor.ts"; -import app from "../src/app.ts"; import { GHERKIN_AGENT_ROUTE } from "../src/routes.ts"; import { targetDocumentPath } from "../src/target-document-path.ts"; import type { StatementNotedProposalInput } from "@hashintel/brunch-agent-plugin-gherkin"; +const spanExporter = new InMemorySpanExporter(); +const traceProvider = new NodeTracerProvider({ + spanProcessors: [new SimpleSpanProcessor(spanExporter)], +}); +traceProvider.register(); +const { default: app } = await import("../src/app.ts"); + const ask = toolName("ask"); const sweep = toolName("sweep"); const omittedQuote = "A shopper completes checkout."; const newlyCapturedQuote = "Payment is authorized before fulfillment."; const repairedQuote = "Refunds require approval."; const missingQuote = "This quote is not in the conversation."; +const exceptionContentAttributes = [ + "exception.message", + "exception.stacktrace", +] as const; const statementNoted = (quote: string): StatementNotedProposalInput => ({ evidence: [{ excerpt: quote }], epistemicStatus: "explicit", @@ -275,6 +291,8 @@ try { }); const serializedReplyContext = replyContext === undefined ? undefined : JSON.stringify(replyContext); + await traceProvider.forceFlush(); + const spans = spanExporter.getFinishedSpans(); process.stdout.write( `WALKING_SKELETON_RESULT ${JSON.stringify({ @@ -299,6 +317,24 @@ try { noInstructionWake: !JSON.stringify(history.messages) .toLowerCase() .includes("instructions updated"), + openTelemetrySpans: + spans.some((span) => span.name.startsWith("invoke_agent ")) && + spans.some((span) => span.name.startsWith("chat ")) && + spans.some((span) => span.name.startsWith("execute_tool ")), + openTelemetryContentSuppressed: spans.every((span) => + [ + Object.values(CONTENT_ATTR).every( + (attributeName) => span.attributes[attributeName] === undefined, + ), + span.status.message === undefined, + span.events.every((event) => + exceptionContentAttributes.every( + (attributeName) => + event.attributes?.[attributeName] === undefined, + ), + ), + ].every(Boolean), + ), pendingAskSuppressedSettlement: !firstHistory.messages.some( (message) => message.signal?.tagName === "settlement-check", ), @@ -333,5 +369,6 @@ try { } finally { delete process.env.BRUNCH_DEV_TARGET_DOCUMENT_DIR; await flue.stop(); + await traceProvider.shutdown(); await rm(targetDirectory, { recursive: true }); } diff --git a/apps/brunch-agent/test/walking-skeleton.test.ts b/apps/brunch-agent/test/walking-skeleton.test.ts index c40650f227c..85d39257c26 100644 --- a/apps/brunch-agent/test/walking-skeleton.test.ts +++ b/apps/brunch-agent/test/walking-skeleton.test.ts @@ -29,6 +29,8 @@ test("the dev app suspends for free-text replies without instruction wakes", asy durableOutput: true, markdownFloor: true, noInstructionWake: true, + openTelemetryContentSuppressed: true, + openTelemetrySpans: true, pendingAskSuppressedSettlement: true, quoteAbsentFromPreviousArchive: true, refusalStopReopenedRange: true, diff --git a/yarn.lock b/yarn.lock index 253a1baae03..f9071161c17 100644 --- a/yarn.lock +++ b/yarn.lock @@ -435,6 +435,7 @@ __metadata: resolution: "@apps/brunch-agent@workspace:apps/brunch-agent" dependencies: "@earendil-works/pi-ai": "npm:0.83.0" + "@flue/opentelemetry": "npm:2.0.3" "@flue/react": "npm:2.0.3" "@flue/runtime": "npm:2.0.3" "@flue/sdk": "npm:2.0.3" @@ -444,6 +445,9 @@ __metadata: "@hashintel/brunch-agent-plugin-gherkin": "workspace:*" "@hashintel/brunch-agent-plugin-sdcpn": "workspace:*" "@hashintel/brunch-agent-transport-aisdk": "workspace:*" + "@opentelemetry/api": "npm:1.9.1" + "@opentelemetry/sdk-trace-base": "npm:2.9.0" + "@opentelemetry/sdk-trace-node": "npm:2.9.0" "@types/node": "npm:22.18.13" "@types/react": "npm:19.2.14" "@types/react-dom": "npm:19.2.3" @@ -6508,6 +6512,16 @@ __metadata: languageName: node linkType: hard +"@flue/opentelemetry@npm:2.0.3": + version: 2.0.3 + resolution: "@flue/opentelemetry@npm:2.0.3" + peerDependencies: + "@flue/runtime": ^2.0.3 + "@opentelemetry/api": ^1.9.0 + checksum: 10c0/3eb38f7af8c46019f74417599fc099e02d8675635e8d23f1e49c07bc26a644ad4f14be8a80b38b86dedf795db5bdb3d0045430c4b12ca7e92b51d5608baa75e7 + languageName: node + linkType: hard + "@flue/react@npm:2.0.3": version: 2.0.3 resolution: "@flue/react@npm:2.0.3" From 4455b5cd074bf17e4723722cba9d9c363057fb64 Mon Sep 17 00:00:00 2001 From: Lu Nelson Date: Wed, 26 Aug 2026 16:08:37 +0200 Subject: [PATCH 03/10] Preserve run-specific timing evidence Co-authored-by: Cursor --- apps/brunch-agent/test/baseline-harness.test.ts | 12 ++++++------ apps/brunch-agent/turbo.json | 1 + .../baseline/harness-run.ts | 13 +++++++++---- .../process-model-elicitation/baseline/protocol.md | 4 +++- .../baseline/turn-timing.ts | 10 +++++----- 5 files changed, 24 insertions(+), 16 deletions(-) diff --git a/apps/brunch-agent/test/baseline-harness.test.ts b/apps/brunch-agent/test/baseline-harness.test.ts index ebb5240eaee..40f803775f3 100644 --- a/apps/brunch-agent/test/baseline-harness.test.ts +++ b/apps/brunch-agent/test/baseline-harness.test.ts @@ -89,21 +89,21 @@ test("condition 5 drives the shipped elicitor through the binding and reads the expect(timingRecords.length).toBeGreaterThan(0); expect( timingRecords.every( - (record) => - record.interviewerTurn >= 1 && - record.durationMs >= 0 && - ["interview", "sweep", "repair"].includes(record.purpose), + (timingRecord) => + timingRecord.interviewerTurn >= 1 && + timingRecord.durationMs >= 0 && + ["interview", "sweep", "repair"].includes(timingRecord.purpose), ), ).toBe(true); expect( - new Set(timingRecords.map((record) => record.purpose)), + new Set(timingRecords.map((timingRecord) => timingRecord.purpose)), JSON.stringify(timingRecords, null, 2), ).toEqual(new Set(["interview", "sweep", "repair"])); expect(timingRecords).toHaveLength(run.usage.interviewer.calls); expect(run.timings).toEqual(timingRecords); expect(run.turns.flatMap((turn) => turn.timings)).toEqual(timingRecords); expect( - timingRecords.filter((record) => record.purpose === "repair"), + timingRecords.filter((timingRecord) => timingRecord.purpose === "repair"), ).toHaveLength(2); // Turn 1 asks; the expert's reply is bound to that ask on the next dispatch. diff --git a/apps/brunch-agent/turbo.json b/apps/brunch-agent/turbo.json index c1f0446a71c..7d978d9ad3a 100644 --- a/apps/brunch-agent/turbo.json +++ b/apps/brunch-agent/turbo.json @@ -9,6 +9,7 @@ "BRUNCH_SDCPN_MODEL", "BRUNCH_BASELINE_ANTHROPIC_MODULE", "BRUNCH_BASELINE_INTERVIEWER_PROVIDER_MODULE", + "BRUNCH_BASELINE_OUTPUT_DIR", "BRUNCH_BASELINE_TEST_OUTPUT_DIR" ] }, diff --git a/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/harness-run.ts b/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/harness-run.ts index 18a4eb48049..5755bf89b91 100644 --- a/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/harness-run.ts +++ b/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/harness-run.ts @@ -25,6 +25,7 @@ * BRUNCH_SDCPN_MODEL interviewer model id; this runner defaults it to claude-opus-5 * BRUNCH_BASELINE_ANTHROPIC_MODULE test-only stand-in for the expert's Anthropic client * BRUNCH_BASELINE_INTERVIEWER_PROVIDER_MODULE test-only pi provider module (default export) for the interviewer + * BRUNCH_BASELINE_OUTPUT_DIR optional run-specific production output directory * BRUNCH_BASELINE_TEST_OUTPUT_DIR test-only output directory; requires both stand-ins * * Artifacts (beside the other conditions' transcripts unless the test directory is set): @@ -104,6 +105,7 @@ const EXPERT_MAX_TOKENS = 1_500; // Environment and stand-ins. // --------------------------------------------------------------------------- +const outputDirectory = process.env["BRUNCH_BASELINE_OUTPUT_DIR"]; const testOutputDirectory = process.env["BRUNCH_BASELINE_TEST_OUTPUT_DIR"]; const expertClientModule = process.env["BRUNCH_BASELINE_ANTHROPIC_MODULE"]; const interviewerProviderModule = @@ -141,6 +143,7 @@ const caseDir = fileURLToPath( ); const transcriptDir = testOutputDirectory ?? + outputDirectory ?? fileURLToPath( new URL( "../../../../docs/evidence/evaluations/process-model-elicitation/baseline/transcripts/", @@ -540,13 +543,15 @@ const formatPurposeTiming = ( timings: readonly TurnTimingRecord[], purpose: TurnTimingPurpose, ): string => { - const matching = timings.filter((timing) => timing.purpose === purpose); - if (matching.length === 0) return "—"; - const durationMs = matching.reduce( + const matchingTimings = timings.filter( + (timing) => timing.purpose === purpose, + ); + if (matchingTimings.length === 0) return "—"; + const durationMs = matchingTimings.reduce( (total, timing) => total + timing.durationMs, 0, ); - return `${durationMs} ms (${matching.length} call${matching.length === 1 ? "" : "s"})`; + return `${durationMs} ms (${matchingTimings.length} call${matchingTimings.length === 1 ? "" : "s"})`; }; function renderTranscript( diff --git a/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/protocol.md b/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/protocol.md index 6a3b5048d64..08b5a4c4b0d 100644 --- a/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/protocol.md +++ b/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/protocol.md @@ -102,7 +102,9 @@ agent composition: `turbo run baseline:harness --filter '@apps/brunch-agent'` (b first; writes `condition-5.md`, `condition-5.raw.json`, `condition-5-model.md`, `condition-5-captures.json`, `condition-5-system.md`, and `condition-5.timings.jsonl`). `run.ts` accepts only `1`, `2`, and `4`; condition 3 has no entry point. Production transcripts land in -`docs/evidence/evaluations/process-model-elicitation/baseline/transcripts/`. Tests set +`docs/evidence/evaluations/process-model-elicitation/baseline/transcripts/`; set +`BRUNCH_BASELINE_OUTPUT_DIR` to preserve a run under a separate production evidence directory. +Tests set `BRUNCH_BASELINE_TEST_OUTPUT_DIR` to an isolated directory and never write committed evidence; the condition-5 test additionally swaps both models for stand-ins (`BRUNCH_BASELINE_ANTHROPIC_MODULE`, `BRUNCH_BASELINE_INTERVIEWER_PROVIDER_MODULE`). diff --git a/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/turn-timing.ts b/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/turn-timing.ts index 70936d379bb..b1979a67599 100644 --- a/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/turn-timing.ts +++ b/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/turn-timing.ts @@ -41,7 +41,7 @@ export const createTurnTimingRecorder = (): TurnTimingRecorder => { const nestedPromptOperationIds = new Set(); const purposeByOperation = new Map(); const purposeByFlueTurn = new Map(); - const records: TurnTimingRecord[] = []; + const timingRecords: TurnTimingRecord[] = []; return { startInterviewerTurn(interviewerTurn) { @@ -98,7 +98,7 @@ export const createTurnTimingRecorder = (): TurnTimingRecorder => { if (event.type !== "turn" || currentInterviewerTurn === undefined) { return; } - records.push({ + timingRecords.push({ interviewerTurn: currentInterviewerTurn, flueTurnId: event.turnId, purpose: @@ -112,12 +112,12 @@ export const createTurnTimingRecorder = (): TurnTimingRecorder => { purposeByFlueTurn.delete(event.turnId); }, forInterviewerTurn(interviewerTurn) { - return records.filter( - (record) => record.interviewerTurn === interviewerTurn, + return timingRecords.filter( + (timingRecord) => timingRecord.interviewerTurn === interviewerTurn, ); }, all() { - return records; + return timingRecords; }, }; }; From d6573829c08b13b67222df00fdfc32d1f8c6a5bc Mon Sep 17 00:00:00 2001 From: Lu Nelson Date: Wed, 26 Aug 2026 17:17:07 +0200 Subject: [PATCH 04/10] Fix repair timing attribution Co-authored-by: Cursor --- apps/brunch-agent/test/turn-timing.test.ts | 113 ++++++++++++++++-- .../baseline/turn-timing.ts | 35 +++++- 2 files changed, 133 insertions(+), 15 deletions(-) diff --git a/apps/brunch-agent/test/turn-timing.test.ts b/apps/brunch-agent/test/turn-timing.test.ts index 65680d23def..d7c5563043d 100644 --- a/apps/brunch-agent/test/turn-timing.test.ts +++ b/apps/brunch-agent/test/turn-timing.test.ts @@ -23,7 +23,7 @@ const request = (latestUserMessage = "Continue."): ModelRequest => ({ const completedTurn = ( turnId: string, - operationId: string, + operationId: string | undefined, purpose: "agent" | "compaction", ): FlueObservation => observation({ @@ -42,7 +42,7 @@ const completedTurn = ( isError: false, }); -test("attributes compaction and nested extraction to the active harness purpose", () => { +test("attributes harness signals and unscoped compaction to the active purpose", () => { const recorder = createTurnTimingRecorder(); recorder.startInterviewerTurn(1); recorder.observe( @@ -62,6 +62,27 @@ test("attributes compaction and nested extraction to the active harness purpose" }), ); recorder.observe(completedTurn("interview", "outer", "agent")); + recorder.observe( + observation({ + type: "turn_request", + turnId: "sweep", + operationId: "outer", + purpose: "agent", + request: request( + '\nSweep.\n', + ), + }), + ); + recorder.observe(completedTurn("sweep", "outer", "agent")); + recorder.observe( + observation({ + type: "turn_request", + turnId: "sweep-compaction", + purpose: "compaction", + request: request(), + }), + ); + recorder.observe(completedTurn("sweep-compaction", undefined, "compaction")); recorder.observe( observation({ type: "turn_request", @@ -72,33 +93,105 @@ test("attributes compaction and nested extraction to the active harness purpose" }), ); recorder.observe(completedTurn("repair", "outer", "agent")); + + expect( + Object.fromEntries( + recorder + .all() + .map((timing) => [timing.flueTurnId, timing.purpose] as const), + ), + ).toEqual>({ + interview: "interview", + sweep: "sweep", + "sweep-compaction": "sweep", + repair: "repair", + }); +}); + +test("attributes an inline retry after a refused sweep as repair", () => { + const recorder = createTurnTimingRecorder(); + recorder.startInterviewerTurn(1); + recorder.observe( + observation({ + type: "operation_start", + operationId: "outer", + operationKind: "prompt", + }), + ); recorder.observe( observation({ type: "turn_request", - turnId: "repair-compaction", + turnId: "interview", operationId: "outer", - purpose: "compaction", + purpose: "agent", request: request(), }), ); - recorder.observe(completedTurn("repair-compaction", "outer", "compaction")); recorder.observe( observation({ type: "operation_start", - operationId: "nested", + operationId: "initial-extraction", + operationKind: "prompt", + }), + ); + recorder.observe( + observation({ + type: "turn_request", + turnId: "sweep-extraction", + operationId: "initial-extraction", + purpose: "agent", + request: request("Extract proposals."), + }), + ); + recorder.observe( + completedTurn("sweep-extraction", "initial-extraction", "agent"), + ); + recorder.observe( + observation({ + type: "operation", + operationId: "initial-extraction", + operationKind: "prompt", + }), + ); + recorder.observe( + observation({ + type: "tool", + toolName: "brunch_sweep", + toolCallId: "refused-sweep", + isError: false, + result: { status: "refused" }, + durationMs: 1, + }), + ); + recorder.observe( + observation({ + type: "operation_start", + operationId: "repair-extraction", operationKind: "prompt", }), ); + recorder.observe( + observation({ + type: "turn_request", + turnId: "repair-compaction", + purpose: "compaction", + request: request(), + }), + ); + recorder.observe(completedTurn("repair-compaction", undefined, "compaction")); recorder.observe( observation({ type: "turn_request", turnId: "repair-extraction", - operationId: "nested", + operationId: "repair-extraction", purpose: "agent", request: request("Extract repaired proposals."), }), ); - recorder.observe(completedTurn("repair-extraction", "nested", "agent")); + recorder.observe( + completedTurn("repair-extraction", "repair-extraction", "agent"), + ); + recorder.observe(completedTurn("interview", "outer", "agent")); expect( Object.fromEntries( @@ -107,9 +200,9 @@ test("attributes compaction and nested extraction to the active harness purpose" .map((timing) => [timing.flueTurnId, timing.purpose] as const), ), ).toEqual>({ - interview: "interview", - repair: "repair", + "sweep-extraction": "sweep", "repair-compaction": "repair", "repair-extraction": "repair", + interview: "interview", }); }); diff --git a/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/turn-timing.ts b/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/turn-timing.ts index b1979a67599..549cb15e244 100644 --- a/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/turn-timing.ts +++ b/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/turn-timing.ts @@ -35,10 +35,22 @@ const signalPurpose = ( return undefined; }; +const toolResultStatus = (event: FlueObservation): string | undefined => { + if (event.type !== "tool") return undefined; + const result = event.effectiveResult ?? event.result; + if (typeof result !== "object" || result === null) return undefined; + const output = "output" in result ? result.output : result; + if (typeof output !== "object" || output === null) return undefined; + return "status" in output && typeof output.status === "string" + ? output.status + : undefined; +}; + export const createTurnTimingRecorder = (): TurnTimingRecorder => { let currentInterviewerTurn: number | undefined; const activePromptOperationIds: string[] = []; const nestedPromptOperationIds = new Set(); + const repairingPromptOperationIds = new Set(); const purposeByOperation = new Map(); const purposeByFlueTurn = new Map(); const timingRecords: TurnTimingRecord[] = []; @@ -57,7 +69,8 @@ export const createTurnTimingRecorder = (): TurnTimingRecorder => { nestedPromptOperationIds.add(event.operationId); purposeByOperation.set( event.operationId, - purposeByOperation.get(parentOperationId) === "repair" + repairingPromptOperationIds.has(parentOperationId) || + purposeByOperation.get(parentOperationId) === "repair" ? "repair" : "sweep", ); @@ -71,17 +84,29 @@ export const createTurnTimingRecorder = (): TurnTimingRecorder => { ); if (activeIndex !== -1) activePromptOperationIds.splice(activeIndex, 1); nestedPromptOperationIds.delete(event.operationId); + repairingPromptOperationIds.delete(event.operationId); purposeByOperation.delete(event.operationId); return; } + if (event.type === "tool" && event.toolName === "brunch_sweep") { + const activeOperationId = activePromptOperationIds.at(-1); + if (activeOperationId === undefined) return; + if (toolResultStatus(event) === "refused") { + repairingPromptOperationIds.add(activeOperationId); + } else { + repairingPromptOperationIds.delete(activeOperationId); + } + return; + } if (event.type === "turn_request") { + const operationId = + event.operationId ?? activePromptOperationIds.at(-1); const operationPurpose = - event.operationId === undefined + operationId === undefined ? undefined - : purposeByOperation.get(event.operationId); + : purposeByOperation.get(operationId); const purpose = - event.operationId !== undefined && - nestedPromptOperationIds.has(event.operationId) + operationId !== undefined && nestedPromptOperationIds.has(operationId) ? (operationPurpose ?? "sweep") : event.purpose === "agent" ? (signalPurpose(event.request) ?? "interview") From c2040164f0ef337a2ac08a0a6e2c0330e0a4df94 Mon Sep 17 00:00:00 2001 From: Lu Nelson Date: Wed, 26 Aug 2026 17:56:17 +0200 Subject: [PATCH 05/10] Attribute compaction to harness purpose Co-authored-by: Cursor --- apps/brunch-agent/test/turn-timing.test.ts | 45 ++++++++++++------- .../baseline/turn-timing.ts | 11 ++++- 2 files changed, 38 insertions(+), 18 deletions(-) diff --git a/apps/brunch-agent/test/turn-timing.test.ts b/apps/brunch-agent/test/turn-timing.test.ts index d7c5563043d..a90b6533e9d 100644 --- a/apps/brunch-agent/test/turn-timing.test.ts +++ b/apps/brunch-agent/test/turn-timing.test.ts @@ -42,7 +42,7 @@ const completedTurn = ( isError: false, }); -test("attributes harness signals and unscoped compaction to the active purpose", () => { +test("attributes post-sweep compaction to the completed harness purpose", () => { const recorder = createTurnTimingRecorder(); recorder.startInterviewerTurn(1); recorder.observe( @@ -62,37 +62,49 @@ test("attributes harness signals and unscoped compaction to the active purpose", }), ); recorder.observe(completedTurn("interview", "outer", "agent")); + recorder.observe( + observation({ + type: "operation_start", + operationId: "sweep-extraction", + operationKind: "prompt", + }), + ); recorder.observe( observation({ type: "turn_request", turnId: "sweep", - operationId: "outer", + operationId: "sweep-extraction", purpose: "agent", - request: request( - '\nSweep.\n', - ), + request: request("Extract proposals."), }), ); - recorder.observe(completedTurn("sweep", "outer", "agent")); + recorder.observe(completedTurn("sweep", "sweep-extraction", "agent")); recorder.observe( observation({ - type: "turn_request", - turnId: "sweep-compaction", - purpose: "compaction", - request: request(), + type: "operation", + operationId: "sweep-extraction", + operationKind: "prompt", + }), + ); + recorder.observe( + observation({ + type: "tool", + toolName: "brunch_sweep", + toolCallId: "applied-sweep", + isError: false, + result: { status: "applied" }, + durationMs: 1, }), ); - recorder.observe(completedTurn("sweep-compaction", undefined, "compaction")); recorder.observe( observation({ type: "turn_request", - turnId: "repair", - operationId: "outer", - purpose: "agent", - request: request('\nRetry.\n'), + turnId: "sweep-compaction", + purpose: "compaction", + request: request(), }), ); - recorder.observe(completedTurn("repair", "outer", "agent")); + recorder.observe(completedTurn("sweep-compaction", undefined, "compaction")); expect( Object.fromEntries( @@ -104,7 +116,6 @@ test("attributes harness signals and unscoped compaction to the active purpose", interview: "interview", sweep: "sweep", "sweep-compaction": "sweep", - repair: "repair", }); }); diff --git a/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/turn-timing.ts b/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/turn-timing.ts index 549cb15e244..feae8b34735 100644 --- a/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/turn-timing.ts +++ b/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/turn-timing.ts @@ -91,9 +91,18 @@ export const createTurnTimingRecorder = (): TurnTimingRecorder => { if (event.type === "tool" && event.toolName === "brunch_sweep") { const activeOperationId = activePromptOperationIds.at(-1); if (activeOperationId === undefined) return; - if (toolResultStatus(event) === "refused") { + const status = toolResultStatus(event); + if (status === undefined) return; + if (status === "refused") { repairingPromptOperationIds.add(activeOperationId); + purposeByOperation.set(activeOperationId, "repair"); } else { + purposeByOperation.set( + activeOperationId, + repairingPromptOperationIds.has(activeOperationId) + ? "repair" + : "sweep", + ); repairingPromptOperationIds.delete(activeOperationId); } return; From 962d8bbf237bef09e240b0349a88e708f193c908 Mon Sep 17 00:00:00 2001 From: Lu Nelson Date: Wed, 26 Aug 2026 18:38:19 +0200 Subject: [PATCH 06/10] Attribute compact operations to harness purpose Co-authored-by: Cursor --- apps/brunch-agent/test/turn-timing.test.ts | 18 ++++++++++++++++-- .../baseline/turn-timing.ts | 5 ++++- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/apps/brunch-agent/test/turn-timing.test.ts b/apps/brunch-agent/test/turn-timing.test.ts index a90b6533e9d..1c6e32462fb 100644 --- a/apps/brunch-agent/test/turn-timing.test.ts +++ b/apps/brunch-agent/test/turn-timing.test.ts @@ -100,11 +100,18 @@ test("attributes post-sweep compaction to the completed harness purpose", () => observation({ type: "turn_request", turnId: "sweep-compaction", + operationId: "sweep-compaction-operation", purpose: "compaction", request: request(), }), ); - recorder.observe(completedTurn("sweep-compaction", undefined, "compaction")); + recorder.observe( + completedTurn( + "sweep-compaction", + "sweep-compaction-operation", + "compaction", + ), + ); expect( Object.fromEntries( @@ -185,11 +192,18 @@ test("attributes an inline retry after a refused sweep as repair", () => { observation({ type: "turn_request", turnId: "repair-compaction", + operationId: "repair-compaction-operation", purpose: "compaction", request: request(), }), ); - recorder.observe(completedTurn("repair-compaction", undefined, "compaction")); + recorder.observe( + completedTurn( + "repair-compaction", + "repair-compaction-operation", + "compaction", + ), + ); recorder.observe( observation({ type: "turn_request", diff --git a/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/turn-timing.ts b/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/turn-timing.ts index feae8b34735..6cab46adaa0 100644 --- a/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/turn-timing.ts +++ b/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/turn-timing.ts @@ -109,7 +109,10 @@ export const createTurnTimingRecorder = (): TurnTimingRecorder => { } if (event.type === "turn_request") { const operationId = - event.operationId ?? activePromptOperationIds.at(-1); + event.operationId !== undefined && + purposeByOperation.has(event.operationId) + ? event.operationId + : activePromptOperationIds.at(-1); const operationPurpose = operationId === undefined ? undefined From 3a779beb8023f9b55710c4503646215add2ea58b Mon Sep 17 00:00:00 2001 From: Lu Nelson Date: Wed, 26 Aug 2026 18:39:26 +0200 Subject: [PATCH 07/10] Attribute model continuations to harness purpose Co-authored-by: Cursor --- apps/brunch-agent/test/baseline-harness.test.ts | 10 +++++++++- .../process-model-elicitation/baseline/turn-timing.ts | 4 +++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/apps/brunch-agent/test/baseline-harness.test.ts b/apps/brunch-agent/test/baseline-harness.test.ts index 40f803775f3..39df398a0dc 100644 --- a/apps/brunch-agent/test/baseline-harness.test.ts +++ b/apps/brunch-agent/test/baseline-harness.test.ts @@ -102,9 +102,17 @@ test("condition 5 drives the shipped elicitor through the binding and reads the expect(timingRecords).toHaveLength(run.usage.interviewer.calls); expect(run.timings).toEqual(timingRecords); expect(run.turns.flatMap((turn) => turn.timings)).toEqual(timingRecords); + expect( + run.turns.every( + (turn) => + turn.timings.filter( + (timingRecord) => timingRecord.purpose === "interview", + ).length === 1, + ), + ).toBe(true); expect( timingRecords.filter((timingRecord) => timingRecord.purpose === "repair"), - ).toHaveLength(2); + ).toHaveLength(3); // Turn 1 asks; the expert's reply is bound to that ask on the next dispatch. expect(run.turns[0]?.pendingQuestion).toBe(FIRST_QUESTION); diff --git a/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/turn-timing.ts b/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/turn-timing.ts index 6cab46adaa0..92a18a778d3 100644 --- a/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/turn-timing.ts +++ b/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/turn-timing.ts @@ -121,7 +121,9 @@ export const createTurnTimingRecorder = (): TurnTimingRecorder => { operationId !== undefined && nestedPromptOperationIds.has(operationId) ? (operationPurpose ?? "sweep") : event.purpose === "agent" - ? (signalPurpose(event.request) ?? "interview") + ? (signalPurpose(event.request) ?? + operationPurpose ?? + "interview") : (operationPurpose ?? "interview"); purposeByFlueTurn.set(event.turnId, purpose); if ( From 254017767b9c62a9133aa33b88dab6c707dd1397 Mon Sep 17 00:00:00 2001 From: Lu Nelson Date: Wed, 26 Aug 2026 18:53:02 +0200 Subject: [PATCH 08/10] Make baseline timing runs kill-safe Allow short cross-check runs and persist each completed turn's timing records immediately so interrupted runs retain evidence. Co-authored-by: Cursor --- .../test/baseline-harness.test.ts | 74 +++++++++++++++++++ apps/brunch-agent/turbo.json | 1 + .../baseline/harness-run.ts | 45 +++++++---- 3 files changed, 107 insertions(+), 13 deletions(-) diff --git a/apps/brunch-agent/test/baseline-harness.test.ts b/apps/brunch-agent/test/baseline-harness.test.ts index 39df398a0dc..26ed5196483 100644 --- a/apps/brunch-agent/test/baseline-harness.test.ts +++ b/apps/brunch-agent/test/baseline-harness.test.ts @@ -45,6 +45,80 @@ afterEach(async () => { ); }); +test("stops at the configured turn and persists first-turn timings before exit", async () => { + const outputDirectory = await mkdtemp( + join(tmpdir(), "brunch-baseline-c5-short-test-"), + ); + temporaryDirectories.push(outputDirectory); + const expertRepliesPath = join(outputDirectory, "expert-replies.json"); + await writeFile( + expertRepliesPath, + JSON.stringify([ + { text: EXPERT_OBJECTIVE_QUOTE }, + { text: "Better is fewer late promises, then fewer changeovers." }, + { text: "Alright. Anything else?" }, + { text: "Then I'll get back to the floor." }, + { text: "Cheers." }, + ]), + ); + + let runCompleted = false; + const runPromise = runNodeScript(runner, join(testDirectory, "../../.."), { + BRUNCH_BASELINE_HARD_STOP: "2", + BRUNCH_BASELINE_TEST_OUTPUT_DIR: outputDirectory, + BRUNCH_BASELINE_ANTHROPIC_MODULE: expertStub, + BRUNCH_BASELINE_INTERVIEWER_PROVIDER_MODULE: interviewer, + BASELINE_STUB_REPLIES_PATH: expertRepliesPath, + BRUNCH_SDCPN_MODEL: "claude-haiku-4-5", + }).finally(() => { + runCompleted = true; + }); + + const timingsPath = join(outputDirectory, "condition-5.timings.jsonl"); + let timingsAfterFirstTurn: string | undefined; + await expect + .poll( + async () => { + try { + const timings = await readFile(timingsPath, "utf8"); + if (timings.includes('"interviewerTurn":1')) { + timingsAfterFirstTurn = timings; + } + } catch (error) { + if ( + !(error instanceof Error) || + !("code" in error) || + error.code !== "ENOENT" + ) { + throw error; + } + } + return timingsAfterFirstTurn; + }, + { interval: 1, timeout: 1_000 }, + ) + .toBeDefined(); + + expect(timingsAfterFirstTurn).toContain('"interviewerTurn":1'); + expect( + timingsAfterFirstTurn + ?.trim() + .split("\n") + .every( + (line) => (JSON.parse(line) as TurnTimingRecord).interviewerTurn === 1, + ), + ).toBe(true); + expect(runCompleted).toBe(false); + + const { exitCode, stderr } = await runPromise; + expect(exitCode, stderr).toBe(0); + const run = JSON.parse( + await readFile(join(outputDirectory, "condition-5.raw.json"), "utf8"), + ) as HarnessRunRecord; + expect(run.turns).toHaveLength(2); + expect(run.stopReason).toBe("hard-stop"); +}); + test("condition 5 drives the shipped elicitor through the binding and reads the harness's facts back", async () => { const outputDirectory = await mkdtemp( join(tmpdir(), "brunch-baseline-c5-test-"), diff --git a/apps/brunch-agent/turbo.json b/apps/brunch-agent/turbo.json index 7d978d9ad3a..3e79f0e62b2 100644 --- a/apps/brunch-agent/turbo.json +++ b/apps/brunch-agent/turbo.json @@ -8,6 +8,7 @@ "ANTHROPIC_API_KEY", "BRUNCH_SDCPN_MODEL", "BRUNCH_BASELINE_ANTHROPIC_MODULE", + "BRUNCH_BASELINE_HARD_STOP", "BRUNCH_BASELINE_INTERVIEWER_PROVIDER_MODULE", "BRUNCH_BASELINE_OUTPUT_DIR", "BRUNCH_BASELINE_TEST_OUTPUT_DIR" diff --git a/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/harness-run.ts b/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/harness-run.ts index 5755bf89b91..fe8f1c5525e 100644 --- a/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/harness-run.ts +++ b/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/harness-run.ts @@ -25,6 +25,7 @@ * BRUNCH_SDCPN_MODEL interviewer model id; this runner defaults it to claude-opus-5 * BRUNCH_BASELINE_ANTHROPIC_MODULE test-only stand-in for the expert's Anthropic client * BRUNCH_BASELINE_INTERVIEWER_PROVIDER_MODULE test-only pi provider module (default export) for the interviewer + * BRUNCH_BASELINE_HARD_STOP positive interviewer-turn limit; defaults to 24 * BRUNCH_BASELINE_OUTPUT_DIR optional run-specific production output directory * BRUNCH_BASELINE_TEST_OUTPUT_DIR test-only output directory; requires both stand-ins * @@ -38,7 +39,15 @@ * condition-5.timings.jsonl each observed Flue model call, tagged by interviewer-turn purpose */ -import { cp, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { + appendFile, + cp, + mkdir, + mkdtemp, + readFile, + rm, + writeFile, +} from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; @@ -91,7 +100,7 @@ const CONDITION = "5"; const EXPERT_MODEL = "claude-sonnet-5"; const DEFAULT_INTERVIEWER_MODEL = "claude-opus-5"; const FORCE_WRAP_AT = 20; -const HARD_STOP_AT = 24; +const DEFAULT_HARD_STOP_AT = 24; const IMPATIENCE_AT = 8; const IMPATIENCE_LINE = "(Sorry — I've just seen the time, I have the floor huddle in ten minutes. How much more do you need?)"; @@ -111,6 +120,15 @@ const expertClientModule = process.env["BRUNCH_BASELINE_ANTHROPIC_MODULE"]; const interviewerProviderModule = process.env["BRUNCH_BASELINE_INTERVIEWER_PROVIDER_MODULE"]; const apiKey = process.env["ANTHROPIC_API_KEY"]; +const configuredHardStop = process.env["BRUNCH_BASELINE_HARD_STOP"]; +const hardStopAt = + configuredHardStop === undefined + ? DEFAULT_HARD_STOP_AT + : Number(configuredHardStop); + +if (!Number.isSafeInteger(hardStopAt) || hardStopAt <= 0) { + throw new Error("BRUNCH_BASELINE_HARD_STOP must be a positive integer"); +} if (testOutputDirectory && !(expertClientModule && interviewerProviderModule)) { console.error( @@ -150,6 +168,9 @@ const transcriptDir = import.meta.url, ), ); +await mkdir(transcriptDir, { recursive: true }); +const timingsPath = join(transcriptDir, `condition-${CONDITION}.timings.jsonl`); +await writeFile(timingsPath, ""); // --------------------------------------------------------------------------- // Records. @@ -566,7 +587,7 @@ function renderTranscript( `- Run started: ${run.startedAt}`, `- Interviewer: ${run.interviewerModel} as the shipped SDCPN elicitor in the Flue runtime — binding-flue's ask, settlement nudge, sweep, fold, and completion (instructions reconstructed in condition-5-system.md)`, `- Simulated expert: ${run.expertModel} + situation-pack.md`, - `- Interviewer turns: ${run.turns.length} (impatience probe at ${IMPATIENCE_AT}, forced wrap at ${FORCE_WRAP_AT}, hard stop ${HARD_STOP_AT})`, + `- Interviewer turns: ${run.turns.length} (impatience probe at ${IMPATIENCE_AT}, forced wrap at ${FORCE_WRAP_AT}, hard stop ${hardStopAt})`, `- Stop reason: ${run.stopReason}`, last === undefined ? "- Harness at close: no turn completed" @@ -783,13 +804,6 @@ async function writeArtifacts(): Promise { `${stem}.md`, renderTranscript(run, openingMessage, sweepTally), ); - await writeFile( - `${stem}.timings.jsonl`, - `${turnTimingRecorder - .all() - .map((timing) => JSON.stringify(timing)) - .join("\n")}\n`, - ); await writeFile(`${stem}-model.md`, renderModel(model, report, record)); await writeFile( `${stem}-captures.json`, @@ -823,7 +837,7 @@ try { ); let outgoing = openingMessage; let initial = true; - while (turns.length < HARD_STOP_AT) { + while (turns.length < hardStopAt) { const turnNumber = turns.length + 1; console.error(`turn ${turnNumber} (interviewer)`); turnTimingRecorder.startInterviewerTurn(turnNumber); @@ -845,14 +859,19 @@ try { !ask.rejected && `affordance_${ask.toolCallId}` === pendingId, )?.question; const { record: completion } = readCompletion(await store.read()); + const turnTimings = turnTimingRecorder.forInterviewerTurn(turnNumber); const turn: HarnessTurnRecord = { turn: turnNumber, ...observed, ...(pendingQuestion === undefined ? {} : { pendingQuestion }), - timings: turnTimingRecorder.forInterviewerTurn(turnNumber), + timings: turnTimings, completion, }; turns.push(turn); + await appendFile( + timingsPath, + `${turnTimings.map((timing) => JSON.stringify(timing)).join("\n")}\n`, + ); console.error( ` harness: ${completion.captures} captures, complete ${yesNo(completion.complete)}, ${completion.unsatisfied} unsatisfied; sweeps ${observed.sweeps.map((sweep) => sweep.status).join(",") || "none"}; ask ${pendingQuestion === undefined ? "none" : "pending"}`, ); @@ -878,7 +897,7 @@ try { } else { turnsWithoutAsk = 0; } - if (turns.length >= HARD_STOP_AT) break; + if (turns.length >= hardStopAt) break; // What the expert sees: the interviewer's visible text and its question. const visible = [ From 75ec95a079b4293e4e398de91910b0c00da30714 Mon Sep 17 00:00:00 2001 From: Lu Nelson Date: Wed, 26 Aug 2026 18:58:55 +0200 Subject: [PATCH 09/10] Reconcile baseline timing assertions after re-braid Select the applied sweep explicitly now that timing coverage exercises a refused repair path, and allow the incremental-write probe enough time under the full suite. Co-authored-by: Cursor --- apps/brunch-agent/test/baseline-harness.test.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/apps/brunch-agent/test/baseline-harness.test.ts b/apps/brunch-agent/test/baseline-harness.test.ts index 26ed5196483..958ae38c432 100644 --- a/apps/brunch-agent/test/baseline-harness.test.ts +++ b/apps/brunch-agent/test/baseline-harness.test.ts @@ -95,7 +95,7 @@ test("stops at the configured turn and persists first-turn timings before exit", } return timingsAfterFirstTurn; }, - { interval: 1, timeout: 1_000 }, + { interval: 1, timeout: 5_000 }, ) .toBeDefined(); @@ -217,7 +217,9 @@ test("condition 5 drives the shipped elicitor through the binding and reads the (part) => part.type === "dynamic-tool" && part.toolName === "brunch_sweep" && - part.state === "output-available", + part.state === "output-available" && + isRecord(part.output) && + part.output.status === "applied", ); if ( appliedSweepPart?.type !== "dynamic-tool" || From 26fcca8fcd3e52280a7dea03ccce70187e9ade5c Mon Sep 17 00:00:00 2001 From: Lu Nelson Date: Thu, 27 Aug 2026 10:35:40 +0200 Subject: [PATCH 10/10] Review the turn-timing test as a substrate type-import The purpose-splitter unit test imports Flue observation types, so the hermetic entry-point inventory has to name it or the architecture gate fails. Co-authored-by: Cursor --- .../packages/core/test/architecture/boundaries.test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libs/@hashintel/brunch-agent/packages/core/test/architecture/boundaries.test.ts b/libs/@hashintel/brunch-agent/packages/core/test/architecture/boundaries.test.ts index 68e34b86a02..2e2183a7689 100644 --- a/libs/@hashintel/brunch-agent/packages/core/test/architecture/boundaries.test.ts +++ b/libs/@hashintel/brunch-agent/packages/core/test/architecture/boundaries.test.ts @@ -473,6 +473,8 @@ describe("the HASH smoke is runnable without a model key or a network (spec §12 "Boots the real Gherkin elicitor on Flue's node runtime with pi-ai's faux provider and drives the committed application route over app.fetch through a full ask suspend/return/resume cycle plus a refused duplicate — no provider key, no socket, no external checkout mutation.", "apps/brunch-agent/test/petrinaut-chat.integration.ts": "Boots the real Gherkin elicitor on Flue's node runtime with pi-ai's faux provider, drives the committed application AI SDK route over app.fetch, and proves live reasoning/text plus inspection events without a provider key, socket, or external checkout mutation.", + "apps/brunch-agent/test/turn-timing.test.ts": + "Types recorded Flue observations and model requests so the condition-5 purpose splitter can be unit-tested; the import is type-only — no provider key, no socket, no model call, no runtime boot.", "apps/brunch-agent/test/walking-skeleton.integration.ts": "Boots the dev app on Flue's node runtime with pi-ai's faux provider and drives it over app.fetch — no provider key, no socket, no model call. Run as a child process by walking-skeleton.test.ts, which is what makes the node runtime drivable from this suite at all.", };