diff --git a/.changeset/quiet-voice-guide.md b/.changeset/quiet-voice-guide.md new file mode 100644 index 00000000000..d75050aa262 --- /dev/null +++ b/.changeset/quiet-voice-guide.md @@ -0,0 +1,5 @@ +--- +"@hashintel/petrinaut": patch +--- + +Update the AI assistant guide to explain separate Brunch-authored spoken takeaways, complete on-screen reports, and explicit full-response reading. diff --git a/apps/brunch-agent/src/agents/chat-agent/agent.ts b/apps/brunch-agent/src/agents/chat-agent/agent.ts index bcf191f3b07..f4a3635d845 100644 --- a/apps/brunch-agent/src/agents/chat-agent/agent.ts +++ b/apps/brunch-agent/src/agents/chat-agent/agent.ts @@ -17,6 +17,7 @@ import { import { useBrunchAgent } from "@hashintel/brunch-agent/flue"; import { ping } from "./tools/ping.ts"; +import { useVoiceResponse } from "./voice-response.ts"; export const CHAT_MODEL_ID = process.env["BRUNCH_CHAT_MODEL"] || "claude-haiku-4-5"; @@ -38,12 +39,7 @@ export function ChatAgent() { "responseMode" in context && context.responseMode === "voice" ) { - useInstruction(`Voice response style for this delivery only: -Respond conversationally and concisely. Put the necessary question or conclusion first. -Avoid unnecessary preambles and repetition; preserve consequential qualifications. -For a short clarification, prefer one or two spoken sentences, with any consequential qualification, rather than an unsolicited report or a repeated summary. Expand only when the question requires it. -When a detailed report is needed, keep it complete in the visible canonical response; the application offers to read long responses on request. -These are presentation instructions only. Retain all domain, evidence, workpiece, and tool obligations.`); + useVoiceResponse(); } useInstruction( diff --git a/apps/brunch-agent/src/agents/chat-agent/voice-response.ts b/apps/brunch-agent/src/agents/chat-agent/voice-response.ts new file mode 100644 index 00000000000..bf8e03a8f36 --- /dev/null +++ b/apps/brunch-agent/src/agents/chat-agent/voice-response.ts @@ -0,0 +1,51 @@ +import { + defineTool, + useDataWriter, + useInstruction, + useTool, +} from "@flue/runtime"; +import * as v from "valibot"; + +import { + BRUNCH_VOICE_DATA_NAME, + BRUNCH_VOICE_TOOL_NAME, + BrunchVoiceDataSchema, + BrunchVoiceInputSchema, +} from "@hashintel/brunch-agent/voice-response"; + +/** App-owned delivery instructions and output, mounted only for Voice deliveries. */ +export const useVoiceResponse = (): void => { + const writeSpeech = useDataWriter(BRUNCH_VOICE_DATA_NAME, { + schema: BrunchVoiceDataSchema, + }); + useTool( + defineTool({ + name: BRUNCH_VOICE_TOOL_NAME, + description: + "Author the spoken answer or takeaway for this Voice reply after gathering its tool evidence. This records text, not playback or completion. Then deliver the full visible response in ordinary assistant prose. If further substantive tools are needed, replace the speech after their results.", + input: BrunchVoiceInputSchema, + output: v.object({ title: v.string(), detail: v.string() }), + run({ data, toolCallId }) { + writeSpeech({ speech: data.speech, toolCallId }); + return { + output: { + title: "Brunch-authored speech (not playback confirmation)", + detail: data.speech, + }, + }; + }, + }), + ); + useInstruction(`Voice response style for this delivery only: +Author both the spoken content and the complete visible canonical response. Realtime reads your spoken content verbatim; it does not select, summarize, or add meaning. +For a short answer, give a brief useful answer, then ask a follow-up only when it materially advances the person's modelling goal. Do not force a follow-up every turn. Clarify first when ambiguity would materially change the answer. +For a long analysis, author a brief substantive takeaway for speech while keeping the complete report and required recoverable workpiece in ordinary visible prose. Preserve consequential qualifications in the takeaway, not only on screen. Avoid unnecessary preambles and repetition. +Voice delivery order (including clarification-only replies): +1. Finish gathering the evidence and substantive tool results needed for this reply. +2. Call brunch_set_voice_response with the exact spoken answer, takeaway, or clarification question. A clarification-only reply still needs authored speech; use the exact question text when speaking a question. +3. If asking a direct question, call brunch_mark_question with its exact text immediately before presenting it in ordinary assistant prose. The speech tool does not replace question marking. +4. Deliver the full visible response, including the exact marked question text, then finish. Do not skip speech authoring when the visible response is only a question. +If another substantive tool is needed after speech authoring, replace the spoken content after that tool's result before final delivery. Do not claim a change or successful check before its evidence exists. +The application waits for the whole reply, including browser-tool continuations, before playback. Read full response reads the complete visible prose on request. Recording speech is not evidence of playback, user agreement, or completed modelling. +These are presentation instructions only. Retain all domain, evidence, workpiece, and tool obligations.`); +}; diff --git a/apps/brunch-agent/test/architecture/boundaries.integration.ts b/apps/brunch-agent/test/architecture/boundaries.integration.ts index 4e4433d26a4..49e0176c0bd 100644 --- a/apps/brunch-agent/test/architecture/boundaries.integration.ts +++ b/apps/brunch-agent/test/architecture/boundaries.integration.ts @@ -387,6 +387,7 @@ describe("core auxiliary subpaths stay in their assigned lanes", () => { "./flue", "./question-marker", "./storage", + "./voice-response", "./workpiece", ]); }); @@ -449,6 +450,8 @@ describe("the HASH smoke is runnable without a model key or a network (spec §12 "Constructs Flue's content-free OpenTelemetry instrumentation with an injected exporter setup to prove disposal order; it registers no global instrumentation, opens no socket, and makes no provider call.", "apps/brunch-agent/test/voice-context.test.ts": "Boots the production ChatAgent with a faux provider to compare effective system prompts across typed, Voice, and browser-result deliveries — no provider key, socket, or network model call.", + "apps/brunch-agent/test/voice-response.test.ts": + "Boots the production ChatAgent with a faux provider through the HTTP router and AI SDK transport, then restarts its temporary SQLite runtime to prove speech and visible response durability — no provider key, socket, or network model call.", "apps/brunch-agent/test/workpiece.test.ts": "Types Flue's public conversation snapshot so the substrate-neutral workpiece selector and app-owned SHA-256 projection can be unit-tested against in-memory messages — no provider key, no socket, no model call, no runtime boot.", "libs/@hashintel/brunch-agent/packages/transport-aisdk/test/chat-transport.test.ts": diff --git a/apps/brunch-agent/test/voice-context.test.ts b/apps/brunch-agent/test/voice-context.test.ts index 5d973f527e4..f80f2df2307 100644 --- a/apps/brunch-agent/test/voice-context.test.ts +++ b/apps/brunch-agent/test/voice-context.test.ts @@ -11,6 +11,7 @@ import { ChatAgent, CHAT_MODEL_ID } from "../src/agents/chat-agent/agent"; test("ChatAgent scopes its fixed Voice instructions to the current delivery", async () => { const prompts: string[] = []; + const tools: string[][] = []; const provider = fauxProvider({ provider: "anthropic", models: [{ id: CHAT_MODEL_ID }], @@ -18,6 +19,7 @@ test("ChatAgent scopes its fixed Voice instructions to the current delivery", as provider.setResponses( Array.from({ length: 5 }, () => (context) => { prompts.push(context.systemPrompt ?? ""); + tools.push(context.tools?.map((tool) => tool.name) ?? []); return fauxAssistantMessage([fauxText("Canonical answer.")]); }), ); @@ -66,10 +68,26 @@ test("ChatAgent scopes its fixed Voice instructions to the current delivery", as expect(prompts[1]).toContain("Voice response style"); expect(prompts[1]).toContain("consequential qualifications"); expect(prompts[1]).toContain("visible canonical response"); + // Assert presence without printing the effective prompt on failure. + expect( + prompts[1]?.includes( + "Voice delivery order (including clarification-only replies):", + ), + ).toBe(true); + expect( + /1\. Finish gathering[^\n]*\n2\. Call brunch_set_voice_response[^\n]*\n3\. If asking a direct question, call brunch_mark_question[^\n]*\n4\. Deliver the full visible response/u.test( + prompts[1] ?? "", + ), + ).toBe(true); expect(prompts[2]).toBe(prompts[1]); expect(prompts[3]).toBe(prompts[0]); expect(prompts[4]).toBe(prompts[0]); expect(prompts.join("\n")).not.toContain("UNTRUSTED_CONTEXT"); + expect(tools[0]).not.toContain("brunch_set_voice_response"); + expect(tools[1]).toContain("brunch_set_voice_response"); + expect(tools[2]).toEqual(tools[1]); + expect(tools[3]).toEqual(tools[0]); + expect(tools[4]).toEqual(tools[0]); } finally { await runtime.stop(); } diff --git a/apps/brunch-agent/test/voice-response.test.ts b/apps/brunch-agent/test/voice-response.test.ts new file mode 100644 index 00000000000..acd2f71513e --- /dev/null +++ b/apps/brunch-agent/test/voice-response.test.ts @@ -0,0 +1,139 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { + fauxAssistantMessage, + fauxProvider, + fauxText, + fauxToolCall, +} from "@earendil-works/pi-ai"; +import { sqlite, start } from "@flue/runtime/node"; +import { createAgentRouter } from "@flue/runtime/routing"; +import { createFlueClient } from "@flue/sdk"; +import { expect, test } from "vitest"; + +import { + createFlueChatTransport, + snapshotToUiMessages, +} from "@hashintel/brunch-agent-transport-aisdk"; +import { BRUNCH_VOICE_TOOL_NAME } from "@hashintel/brunch-agent/voice-response"; + +import { ChatAgent, CHAT_MODEL_ID } from "../src/agents/chat-agent/agent"; + +import type { UIMessageChunk } from "ai"; + +test("Voice speech crosses the real runtime and transport and survives restart beside the full response", async () => { + const speech = " Capacity is not established. Which limit matters? "; + const report = + "# Full analysis\n\nCapacity is not established.\n\nWhich limit matters?\n\n```runbook-ir\n# Workpiece\nTiming remains unknown.\n```"; + const provider = fauxProvider({ + provider: "anthropic", + models: [{ id: CHAT_MODEL_ID }], + }); + provider.setResponses([ + fauxAssistantMessage( + [fauxToolCall(BRUNCH_VOICE_TOOL_NAME, { speech }, { id: "speech-1" })], + { stopReason: "toolUse" }, + ), + fauxAssistantMessage( + [ + fauxToolCall( + "brunch_mark_question", + { question: "Which limit matters?" }, + { id: "question-1" }, + ), + ], + { stopReason: "toolUse" }, + ), + fauxAssistantMessage([fauxText(report)]), + ]); + const directory = await mkdtemp(join(tmpdir(), "brunch-voice-response-")); + const boot = () => + start({ + agents: [{ agent: ChatAgent, name: ChatAgent.agentName }], + providers: [provider.provider], + db: sqlite(join(directory, "conversation.db")), + }); + let runtime = await boot(); + try { + const router = createAgentRouter(ChatAgent); + const client = createFlueClient({ + url: "http://local.test/voice-response", + fetch: async (input, options) => + router.fetch( + input instanceof Request ? input : new Request(input, options), + ), + }); + const transport = createFlueChatTransport({ + client, + clientToolNames: new Set(), + }); + const stream = await transport.sendMessages({ + chatId: "voice-response", + trigger: "submit-message", + messageId: undefined, + messages: [ + { + id: "voice-user", + role: "user", + metadata: { source: "voice" }, + parts: [{ type: "text", text: "Explain the limit." }], + }, + ], + abortSignal: undefined, + }); + const chunks: UIMessageChunk[] = []; + for await (const chunk of stream) chunks.push(chunk); + expect(chunks).toContainEqual({ + type: "data-brunch-voice-response", + data: { speech, toolCallId: "speech-1" }, + }); + expect(chunks).toContainEqual({ + type: "tool-output-available", + toolCallId: "speech-1", + output: { + title: "Brunch-authored speech (not playback confirmation)", + detail: speech, + }, + providerExecuted: true, + }); + expect( + chunks + .filter((chunk) => chunk.type === "text-delta") + .map((chunk) => chunk.delta) + .join(""), + ).toBe(report); + expect(chunks.at(-1)).toEqual({ type: "finish", finishReason: "stop" }); + + const before = snapshotToUiMessages(await client.history(), { + clientToolNames: new Set(), + hiddenToolNames: new Set(["brunch_mark_question"]), + }); + const assistant = before.find((message) => message.role === "assistant"); + expect(assistant?.parts).toContainEqual({ + type: "data-brunch-voice-response", + data: { speech, toolCallId: "speech-1" }, + }); + expect(assistant?.parts).toContainEqual({ + type: "text", + text: report, + state: "done", + }); + expect(assistant?.parts).toContainEqual({ + type: "data-brunch-question", + data: { question: "Which limit matters?", toolCallId: "question-1" }, + }); + await runtime.stop(); + runtime = await boot(); + expect( + snapshotToUiMessages(await client.history(), { + clientToolNames: new Set(), + hiddenToolNames: new Set(["brunch_mark_question"]), + }), + ).toEqual(before); + } finally { + await runtime.stop(); + await rm(directory, { recursive: true, force: true }); + } +}); diff --git a/apps/petrinaut-website/README.md b/apps/petrinaut-website/README.md index f05dc0515cc..1f6058bd0c0 100644 --- a/apps/petrinaut-website/README.md +++ b/apps/petrinaut-website/README.md @@ -124,6 +124,19 @@ bubble is replaced by the finalized message or pending-question tool output, which retains a waveform indicator without duplicating the answer. Provisional transcription and Realtime audio are not persisted as chat history. +For Voice replies, Brunch authors separate spoken content and complete visible +prose. Short replies should give a useful brief answer; long analyses should +give a substantive spoken takeaway while keeping the complete report on screen. +The **Brunch-authored speech (not playback confirmation)** tool result retains +the exact authored speech alongside its response, including after reopening. +It records authorship, not whether the audio was heard. Automatic playback waits +for the whole correlated reply, including browser-tool continuations. Later +substantive tools invalidate an earlier speech draft unless Brunch replaces it. +Missing, unusable, or uncorrelated speech produces only the fixed reading notice, +never an application-generated summary or automatic full-report reading. That +notice is not successful substantive Voice delivery. Content usefulness, +speech/report fidelity, and audible delay require separate human assessment. + The text composer remains available. Sending typed text ends Voice mode first, then submits the draft exactly once through the same conversation; a failed handoff restores the draft. Closing the assistant pauses capture and speech @@ -134,7 +147,7 @@ acknowledgements and response terminal event, and only then opens the microphone for fresh capture. Its playback menu offers **Repeat question** and **Read full response**. Full-response replay becomes available once the matching response and audio output have both finished, enqueues all exact retained -canonical segments in order, and is disabled during capture, submission, +visible prose segments in order—not just the spoken takeaway—and is disabled during capture, submission, cancellation, pause, and errors. **Repeat question** has the same safety gates and replays only exact question text carrying Brunch's non-interactive marker; if the marker is missing, malformed, or does not match finalized prose, the diff --git a/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.test.ts index 50807c4ba97..1d6fc0fadaf 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.test.ts @@ -2,6 +2,7 @@ import { describe, expect, test } from "vitest"; import { hashCanonicalSpeechText, + selectAuthoredVoiceSpeech, selectCanonicalSpeech, selectCanonicalSpeechSegments, } from "./canonical-speech"; @@ -11,6 +12,149 @@ import type { PetrinautAiMessage } from "@hashintel/petrinaut/ui"; const select = (messages: PetrinautAiMessage[]) => selectCanonicalSpeechSegments(messages); +const speechParts = ( + speech = " The limit remains unknown. ", + toolCallId = "speech-1", +): PetrinautAiMessage["parts"] => [ + { + type: "dynamic-tool", + toolName: "brunch_set_voice_response", + toolCallId, + state: "output-available", + input: { speech }, + output: { title: "Authored speech", detail: speech }, + }, + { type: "data-brunch-voice-response", data: { speech, toolCallId } }, +]; +const fullReport = { + type: "text", + text: "# Full report\n\nThe limit remains unknown. Further validation is required.", + state: "done", +} as const; + +describe("Brunch-authored speech selection", () => { + test("keeps exact speech separate from displayed prose, with stable response identity on reload", () => { + const messages: PetrinautAiMessage[] = [ + { + id: "reply-1", + role: "assistant", + parts: [...speechParts(), fullReport], + }, + { + id: "reply-2", + role: "assistant", + parts: [...speechParts(), fullReport], + }, + ]; + const selected = selectAuthoredVoiceSpeech(messages); + expect(selected.map(({ text }) => text)).toEqual([ + " The limit remains unknown. ", + " The limit remains unknown. ", + ]); + expect(selected[0]?.id).not.toBe(selected[1]?.id); + expect(selectAuthoredVoiceSpeech(structuredClone(messages))).toEqual( + selected, + ); + expect(select(messages).map(({ text }) => text)).toEqual([ + fullReport.text, + fullReport.text, + ]); + }); + + test.each([ + { name: "missing speech", parts: [fullReport] }, + { name: "blank speech", parts: [...speechParts(" "), fullReport] }, + { name: "missing report", parts: speechParts() }, + { name: "only preceding prose", parts: [fullReport, ...speechParts()] }, + { + name: "unfinished report", + parts: [...speechParts(), { ...fullReport, state: "streaming" }], + }, + { + name: "unmatched tool identity", + parts: [ + ...speechParts(), + { + type: "data-brunch-voice-response", + data: { speech: "Wrong identity", toolCallId: "other" }, + }, + fullReport, + ], + }, + { + name: "later substantive tool", + parts: [ + ...speechParts(), + { + type: "dynamic-tool", + toolName: "updateModel", + toolCallId: "update-1", + state: "output-available", + input: {}, + output: {}, + }, + fullReport, + ], + }, + ] satisfies Array<{ name: string; parts: PetrinautAiMessage["parts"] }>)( + "withholds $name rather than deriving a summary", + ({ parts }) => { + expect( + selectAuthoredVoiceSpeech([{ id: "reply", role: "assistant", parts }]), + ).toEqual([]); + }, + ); + + test("replaces an obsolete draft after tools and preserves marked question replay", () => { + const question = "Which limit matters?"; + const messages: PetrinautAiMessage[] = [ + { + id: "reply", + role: "assistant", + parts: [ + ...speechParts("An obsolete claim."), + { + type: "dynamic-tool", + toolName: "readModel", + toolCallId: "read-1", + state: "output-available", + input: {}, + output: {}, + }, + ...speechParts(`The limit is unknown. ${question}`, "speech-2"), + { + type: "dynamic-tool", + toolName: "brunch_mark_question", + toolCallId: "question-1", + state: "output-available", + input: { question }, + output: {}, + }, + { + type: "data-brunch-question", + data: { question, toolCallId: "question-1" }, + }, + { ...fullReport, text: `${fullReport.text}\n\n${question}` }, + ], + }, + ]; + expect(selectAuthoredVoiceSpeech(messages).map(({ text }) => text)).toEqual( + [`The limit is unknown. ${question}`], + ); + expect(selectCanonicalSpeech(messages).questionSegment?.text).toBe( + question, + ); + expect( + selectAuthoredVoiceSpeech( + messages.map((message) => ({ + ...message, + metadata: { stopped: true }, + })), + ), + ).toEqual([]); + }); +}); + describe("canonical speech selection", () => { test("selects only finalized assistant text without changing it", () => { const messages = [ diff --git a/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.ts b/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.ts index fd466e1448d..b3a7bb0c020 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.ts @@ -1,7 +1,15 @@ +import { getToolName, isToolUIPart } from "ai"; + import { BRUNCH_QUESTION_DATA_NAME, + BRUNCH_QUESTION_TOOL_NAME, parseBrunchQuestionData, } from "@hashintel/brunch-agent/question-marker"; +import { + BRUNCH_VOICE_DATA_NAME, + BRUNCH_VOICE_TOOL_NAME, + parseBrunchVoiceData, +} from "@hashintel/brunch-agent/voice-response"; import { hashCanonicalSpeechText } from "../../../canonical-speech-fingerprint"; @@ -15,7 +23,7 @@ export interface CanonicalSpeechSegment { readonly id: string; readonly messageId: string; readonly partId: string; - readonly source: "assistant-question" | "assistant-text"; + readonly source: "assistant-question" | "assistant-text" | "assistant-voice"; /** * Every Flue submission that wrote to this segment's message: the one that * started it plus any client-tool continuation projected back onto it. @@ -115,3 +123,50 @@ export const selectCanonicalSpeech = ( export const selectCanonicalSpeechSegments = ( messages: PetrinautAiMessage[], ): CanonicalSpeechSegment[] => selectCanonicalSpeech(messages).segments; + +/** Select authored speech, never derive a summary from displayed prose. */ +export const selectAuthoredVoiceSpeech = ( + messages: PetrinautAiMessage[], +): CanonicalSpeechSegment[] => + messages.flatMap((message) => { + if (message.role !== "assistant" || message.metadata?.stopped) return []; + let candidate: CanonicalSpeechSegment | undefined; + let hasFollowingProse = false; + for (const part of message.parts) { + if ( + isToolUIPart(part) && + getToolName(part) !== BRUNCH_QUESTION_TOOL_NAME + ) { + // A later tool may change the evidence, or replace an earlier speech draft. + candidate = undefined; + hasFollowingProse = false; + } + if (part.type === `data-${BRUNCH_VOICE_DATA_NAME}`) { + const speech = parseBrunchVoiceData(part.data); + const recorded = + speech && + message.parts.some( + (tool) => + isToolUIPart(tool) && + getToolName(tool) === BRUNCH_VOICE_TOOL_NAME && + tool.toolCallId === speech.toolCallId && + tool.state === "output-available", + ); + candidate = + speech && recorded + ? createSegment( + message.id, + `voice:${speech.toolCallId}`, + "assistant-voice", + speech.speech, + ) + : undefined; + hasFollowingProse = false; + } + if (part.type === "text") { + if (part.state === "streaming") return []; + if (candidate && part.text.trim()) hasFollowingProse = true; + } + } + return candidate && hasFollowingProse ? [candidate] : []; + }); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.test.ts index 0c1d76c5371..48051cd1917 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.test.ts @@ -28,6 +28,16 @@ const segment = ( text, }); +const authoredSpeech = ( + response: CanonicalSpeechSegment, +): CanonicalSpeechSegment => ({ + ...response, + id: `${response.id}-voice`, + partId: `${response.partId}-voice`, + source: "assistant-voice", + text: "A distinct Brunch-authored takeaway.", +}); + const transcriptKey = ( connectionEpoch: number, itemId = "user-item-1", @@ -67,7 +77,7 @@ const createHarness = () => { let listener: ((event: OpenAIRealtimeSessionEvent) => void) | undefined; const session = { offerFullResponse: vi.fn(), - speakCanonical: vi.fn(), + speakCanonical: vi.fn<(segments: CanonicalSpeechSegment[]) => void>(), subscribe: vi.fn((next: (event: OpenAIRealtimeSessionEvent) => void) => { listener = next; return () => { @@ -234,8 +244,8 @@ describe("RealtimeBrunchBridge", () => { status: "ready", }); - expect(harness.session.speakCanonical).toHaveBeenCalledOnce(); - expect(harness.session.speakCanonical).toHaveBeenCalledWith([laterSegment]); + expect(harness.session.speakCanonical).not.toHaveBeenCalled(); + expect(harness.session.offerFullResponse).not.toHaveBeenCalled(); }); test("submits only a completed transcript through the user admission target", async () => { @@ -615,12 +625,15 @@ describe("RealtimeBrunchBridge", () => { harness.bridge.updateChat({ canAcceptInterviewAnswer: true, canonicalSegments: [unrelated, correlated], + voiceSegments: [authoredSpeech(unrelated), authoredSpeech(correlated)], questionSegment: correlatedQuestion, status: "ready", }); const deliveryId = createRealtimeSubmissionId(transcriptKey(7)); - expect(harness.session.speakCanonical).toHaveBeenCalledWith([correlated]); + expect(harness.session.speakCanonical).toHaveBeenCalledWith([ + authoredSpeech(correlated), + ]); expect(harness.events.map(({ type }) => type)).toEqual([ "submission-started", "submission-admitted", @@ -637,7 +650,7 @@ describe("RealtimeBrunchBridge", () => { }); }); - test("speaks a completed canonical segment while chat remains streaming and settles separately", async () => { + test("withholds authored speech until the whole correlated reply completes", async () => { const harness = createHarness(); startReady(harness, 7); harness.emit(completedTranscript(7)); @@ -646,12 +659,13 @@ describe("RealtimeBrunchBridge", () => { ); const correlated = segment( "correlated", - "Speak this committed response.", + "The complete visible response.", "submission-voice-1", ); harness.bridge.updateChat({ canAcceptInterviewAnswer: false, canonicalSegments: [correlated], + voiceSegments: [authoredSpeech(correlated)], status: "streaming", }); @@ -660,7 +674,7 @@ describe("RealtimeBrunchBridge", () => { completedResponseMessage(correlated.messageId, "submission-voice-1", 1), ); - expect(harness.session.speakCanonical).toHaveBeenCalledWith([correlated]); + expect(harness.session.speakCanonical).not.toHaveBeenCalled(); expect(harness.events.map(({ type }) => type)).not.toContain( "submission-settled", ); @@ -671,10 +685,14 @@ describe("RealtimeBrunchBridge", () => { harness.bridge.updateChat({ canAcceptInterviewAnswer: true, canonicalSegments: [correlated], + voiceSegments: [authoredSpeech(correlated)], status: "ready", }); expect(harness.session.speakCanonical).toHaveBeenCalledOnce(); + expect(harness.session.speakCanonical).toHaveBeenCalledWith([ + authoredSpeech(correlated), + ]); expect(harness.events.slice(-2).map(({ type }) => type)).toEqual([ "submission-settled", "canonical-response-ready", @@ -727,10 +745,10 @@ describe("RealtimeBrunchBridge", () => { harness.bridge.notifyResponseMessageCompleted( completedResponseMessage(messageId, "submission-voice-1", 3), ); - expect(harness.session.speakCanonical).toHaveBeenCalledWith([laterText]); + expect(harness.session.speakCanonical).not.toHaveBeenCalled(); }); - test("speaks later continuation segments once and in canonical order", async () => { + test("waits through browser continuations and delivers only the final authored speech once", async () => { const harness = createHarness(); startReady(harness, 7); harness.emit(completedTranscript(7)); @@ -772,9 +790,10 @@ describe("RealtimeBrunchBridge", () => { harness.bridge.updateChat({ canAcceptInterviewAnswer: false, canonicalSegments: [first, second, third], + voiceSegments: [authoredSpeech(third)], status: "streaming", }); - expect(harness.session.speakCanonical).toHaveBeenCalledTimes(1); + expect(harness.session.speakCanonical).not.toHaveBeenCalled(); harness.bridge.notifyResponseMessageCompleted( completedResponseMessage(first.messageId, "submission-continuation", 2), @@ -788,17 +807,29 @@ describe("RealtimeBrunchBridge", () => { harness.bridge.updateChat({ canAcceptInterviewAnswer: false, canonicalSegments: [first, second, third, fourth], + voiceSegments: [authoredSpeech(fourth)], status: "streaming", }); harness.bridge.notifyResponseMessageCompleted( completedResponseMessage(first.messageId, "submission-continuation", 3), ); + expect(harness.session.speakCanonical).not.toHaveBeenCalled(); + for (let i = 0; i < 2; i++) { + harness.bridge.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [first, second, third, fourth], + voiceSegments: [authoredSpeech(fourth)], + status: "ready", + }); + } expect(harness.session.speakCanonical.mock.calls).toEqual([ - [[first]], - [[second, third]], - [[fourth]], + [[authoredSpeech(fourth)]], ]); + expect(harness.events.at(-1)).toMatchObject({ + type: "canonical-response-ready", + segments: [first, second, third, fourth], + }); }); test("does not start speech cancelled while its correlated response is pending", async () => { @@ -966,6 +997,7 @@ describe("RealtimeBrunchBridge", () => { harness.bridge.updateChat({ canAcceptInterviewAnswer: true, canonicalSegments: [response], + voiceSegments: [authoredSpeech(response)], status: "ready", }); expect(harness.session.speakCanonical).not.toHaveBeenCalled(); @@ -978,9 +1010,136 @@ describe("RealtimeBrunchBridge", () => { harness.bridge.updateChat({ canAcceptInterviewAnswer: true, canonicalSegments: [response], + voiceSegments: [authoredSpeech(response)], status: "ready", }); - expect(harness.session.speakCanonical).toHaveBeenCalledWith([response]); + expect(harness.session.speakCanonical).toHaveBeenCalledWith([ + authoredSpeech(response), + ]); + }); + + test.each([ + "failed", + "aborted", + "cancelled", + "wrong-submission", + "missing", + ] as const)( + "does not release substantive speech for a %s continuation", + async (outcome) => { + const harness = createHarness(); + startReady(harness); + harness.emit(completedTranscript(3)); + await vi.waitFor(() => + expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce(), + ); + const report = segment( + "final", + "Complete report. ".repeat(100), + "submission-voice-1", + ); + const speech = authoredSpeech(report); + harness.bridge.updateChat({ + canAcceptInterviewAnswer: false, + canonicalSegments: [report], + voiceSegments: [speech], + status: "streaming", + }); + harness.bridge.notifyResponseMessageStarted({ + messageId: report.messageId, + submissionId: "continuation", + position: { batch: 2, index: 0 }, + }); + if (outcome === "cancelled") harness.bridge.cancelPendingSpeech(); + for (let i = 0; i < 2; i++) { + harness.bridge.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [report], + status: "ready", + voiceSegments: + outcome === "missing" + ? [] + : [ + { + ...speech, + submissionIds: [ + outcome === "wrong-submission" + ? "unrelated" + : "submission-voice-1", + ], + }, + ], + settlements: + outcome === "failed" || outcome === "aborted" + ? [ + { submissionId: "submission-voice-1", outcome: "completed" }, + { submissionId: "continuation", outcome }, + ] + : [], + }); + } + expect(harness.session.speakCanonical).not.toHaveBeenCalled(); + expect(harness.session.offerFullResponse).toHaveBeenCalledTimes( + outcome === "missing" || outcome === "wrong-submission" ? 1 : 0, + ); + }, + ); + + test("delivers identical takeaways on distinct Voice replies, but never typed replies or reload", async () => { + const harness = createHarness(); + startReady(harness); + const reports: CanonicalSpeechSegment[] = []; + for (let i = 0; i < 2; i++) { + const submissionId = `submission-${i}`; + harness.submitInterviewAnswer.mockImplementationOnce(async (input) => { + input.onAdmission(submissionId); + return { kind: "message", messageId: input.id, submissionId }; + }); + harness.emit(completedTranscript(3, "Explain.", `input-${i}`)); + await vi.waitFor(() => + expect(harness.submitInterviewAnswer).toHaveBeenCalledTimes(i + 1), + ); + const report = segment( + `reply-${i}`, + "Complete report. ".repeat(100), + submissionId, + ); + reports.push(report); + harness.bridge.updateChat({ + canAcceptInterviewAnswer: false, + canonicalSegments: reports, + status: "streaming", + }); + harness.bridge.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: reports, + voiceSegments: [authoredSpeech(report)], + status: "ready", + }); + } + expect( + harness.session.speakCanonical.mock.calls.map(([segments]) => + segments.map(({ text }) => text), + ), + ).toEqual([ + ["A distinct Brunch-authored takeaway."], + ["A distinct Brunch-authored takeaway."], + ]); + reports.push(segment("typed", "Typed report.", "typed-submission")); + harness.bridge.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: reports, + status: "ready", + }); + harness.bridge.stop(); + harness.bridge.start(4); + harness.bridge.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: reports, + status: "ready", + }); + expect(harness.session.speakCanonical).toHaveBeenCalledTimes(2); + expect(harness.session.offerFullResponse).not.toHaveBeenCalled(); }); }); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.ts b/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.ts index 898c24dd922..8da98ef3233 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.ts @@ -25,6 +25,7 @@ export type VoiceSubmissionSettlement = Pick< interface ChatUpdate { readonly canAcceptInterviewAnswer: boolean; readonly canonicalSegments: CanonicalSpeechSegment[]; + readonly voiceSegments?: readonly CanonicalSpeechSegment[]; readonly questionSegment?: CanonicalSpeechSegment; /** Local logical termination when the panel withheld a continuation. */ readonly stopped?: boolean; @@ -70,14 +71,10 @@ interface RealtimeBrunchBridgeDependencies { ) => Promise; } -interface CompletedResponseMessage extends FlueChatResponseMessageCompletedEvent { - consumed: boolean; -} - interface ActiveSubmission { readonly abortController: AbortController; readonly baselineSegmentIds: ReadonlySet; - readonly completedResponseMessages: CompletedResponseMessage[]; + readonly responseSubmissionIds: Set; readonly deliveryId: string; correlated: boolean; firstTextEmitted: boolean; @@ -181,13 +178,6 @@ const transcriptKeyId = (key: OpenAIRealtimeTranscriptKey): string => const normalizeTranscript = (transcript: string): string => transcript.trim().replace(/\s+/gu, " "); -const positionPrecedes = ( - first: FlueChatResponseMessageCompletedEvent["position"], - second: FlueChatResponseMessageStartedEvent["position"], -): boolean => - first.batch < second.batch || - (first.batch === second.batch && first.index < second.index); - const admissionErrorCode = ( failure: FlueChatAdmissionFailure, ): RealtimeAdmissionErrorCode => { @@ -214,7 +204,6 @@ export class RealtimeBrunchBridge { readonly #submitInterviewAnswer: ( input: SubmitInterviewAnswerInput, ) => Promise; - readonly #seenSegmentIds = new Set(); #activeEpoch: number | null = null; #activeSubmission: ActiveSubmission | null = null; #chat: ChatUpdate = { @@ -255,40 +244,15 @@ export class RealtimeBrunchBridge { public notifyResponseMessageCompleted( event: FlueChatResponseMessageCompletedEvent, ): void { - const active = this.#activeSubmission; - if ( - active === null || - active.completedResponseMessages.some( - ({ position }) => - position.batch === event.position.batch && - position.index === event.position.index, - ) - ) { - return; - } - active.completedResponseMessages.push({ - ...event, - consumed: false, - }); - this.#completeCorrelatedSubmission(); + // A completed step is not permission to speak: the panel owns the whole + // reply's status, including queued browser-tool continuations. + this.#activeSubmission?.responseSubmissionIds.add(event.submissionId); } public notifyResponseMessageStarted( event: FlueChatResponseMessageStartedEvent, ): void { - const active = this.#activeSubmission; - if (active === null) { - return; - } - for (const completion of active.completedResponseMessages) { - if ( - !completion.consumed && - completion.messageId === event.messageId && - positionPrecedes(completion.position, event.position) - ) { - completion.consumed = true; - } - } + this.#activeSubmission?.responseSubmissionIds.add(event.submissionId); } public start(connectionEpoch: number): void { @@ -302,10 +266,6 @@ export class RealtimeBrunchBridge { this.#activeOutputResponseIds.clear(); this.#outputCancellationPending = false; this.#pendingSpeechRequestIds.clear(); - this.#seenSegmentIds.clear(); - for (const segment of this.#chat.canonicalSegments) { - this.#seenSegmentIds.add(segment.id); - } } public stop(): void { @@ -340,30 +300,8 @@ export class RealtimeBrunchBridge { this.#completeCorrelatedSubmission(); return; } - if (this.#outputCancellationPending || update.stopped) { - for (const segment of update.canonicalSegments) { - this.#seenSegmentIds.add(segment.id); - } - return; - } - if (update.status !== "ready") { - return; - } - - const newSegments = update.canonicalSegments.filter( - ({ id }) => !this.#seenSegmentIds.has(id), - ); - if (newSegments.length === 0) { - return; - } - try { - this.#session.speakCanonical(newSegments); - for (const segment of newSegments) { - this.#seenSegmentIds.add(segment.id); - } - } catch { - this.#fail(INVALID_BRIDGE_EVENT); - } + // History, typed replies, and late chunks have no active Voice admission. + // Never turn their displayed prose into automatic speech. } #emit(event: RealtimeBrunchBridgeEvent): void { @@ -493,7 +431,7 @@ export class RealtimeBrunchBridge { baselineSegmentIds: new Set( this.#chat.canonicalSegments.map(({ id }) => id), ), - completedResponseMessages: [], + responseSubmissionIds: new Set(), correlated: false, deliveryId, firstTextEmitted: false, @@ -507,6 +445,7 @@ export class RealtimeBrunchBridge { #ownsOutputTurn(): boolean { return ( + this.#outputCancellationPending || this.#activeOutputResponseIds.size > 0 || this.#pendingSpeechRequestIds.size > 0 ); @@ -595,9 +534,6 @@ export class RealtimeBrunchBridge { if (this.#chat.stopped && this.#chat.status === "ready") { // Cancellation can finish before this step commits its final prose. // Retire it now so a later render cannot restart the withheld speech. - for (const segment of this.#chat.canonicalSegments) { - this.#seenSegmentIds.add(segment.id); - } const settlement = this.#chat.settlements?.find( ({ submissionId }) => submissionId === active.submissionId, ); @@ -632,69 +568,22 @@ export class RealtimeBrunchBridge { type: "canonical-text-ready", }); } - const stoppedSettlement = - active.submissionId === null - ? undefined - : this.#chat.settlements?.find( - ({ submissionId }) => submissionId === active.submissionId, - ); - if (stoppedSettlement && stoppedSettlement.outcome !== "completed") { - if (this.#chat.status === "ready") { - this.#completeStoppedSubmission(active); - } - return; - } - const completionMatchesSegment = ( - completion: CompletedResponseMessage, - segment: CanonicalSpeechSegment, - ): boolean => - completion.messageId === segment.messageId && - (segment.submissionIds?.includes(completion.submissionId) ?? false); - const pendingCompletions = active.completedResponseMessages.filter( - ({ consumed }) => !consumed, - ); - const eligibleCompletions = pendingCompletions.filter((completion) => - responseSegments.some( - (segment) => - !this.#seenSegmentIds.has(segment.id) && - completionMatchesSegment(completion, segment), - ), - ); - const completedSegments = responseSegments.filter( - (segment) => - !this.#seenSegmentIds.has(segment.id) && - eligibleCompletions.some((completion) => - completionMatchesSegment(completion, segment), - ), - ); - // FE-1630 experimental delivery budget, not a canonical-text truncation. - // Count the whole visible response, including earlier completed steps. - const responseText = responseSegments.map(({ text }) => text).join("\n"); - const requiresExplicitReading = - responseText.trim().split(/\s+/u).length > 120 || - responseText.length > 1_200 || - responseText.includes("```"); - if (!active.speechCancelled && !requiresExplicitReading) { - if (completedSegments.length > 0) { - try { - this.#session.speakCanonical(completedSegments); - for (const segment of completedSegments) { - this.#seenSegmentIds.add(segment.id); - } - } catch { - this.#fail(INVALID_BRIDGE_EVENT); - return; - } + for (const segment of responseSegments) { + for (const submissionId of segment.submissionIds ?? []) { + active.responseSubmissionIds.add(submissionId); } } - for (const completion of eligibleCompletions) { - completion.consumed = true; - } + // The panel's derived status stays busy through automatic browser tools, + // even when the SDK has finished an individual submission. if (this.#chat.status !== "ready") { return; } + if (this.#completeStoppedSubmission(active)) return; if (responseSegments.length === 0) { - this.#completeStoppedSubmission(active); + this.#fail( + "The reply completed without visible content. Use the composer to retry.", + "interview-response", + ); return; } @@ -703,28 +592,27 @@ export class RealtimeBrunchBridge { type: "submission-settled", }); if (!active.speechCancelled) { - const unscheduledSegments = responseSegments.filter( - ({ id }) => !this.#seenSegmentIds.has(id), + // The final message must supply its own speech, after any substantive + // tools. Never fall back to an earlier draft or the full visible report. + const finalMessageId = responseSegments.at(-1)?.messageId; + const speech = this.#chat.voiceSegments?.findLast( + (segment) => + segment.source === "assistant-voice" && + segment.messageId === finalMessageId && + active.submissionId !== null && + (segment.submissionIds?.includes(active.submissionId) ?? false), ); - if (requiresExplicitReading) { - try { + try { + if (speech) { + this.#session.speakCanonical([speech]); + } else { this.#session.offerFullResponse(); - } catch { - this.#fail(INVALID_BRIDGE_EVENT); - return; - } - } else if (unscheduledSegments.length > 0) { - try { - this.#session.speakCanonical(unscheduledSegments); - } catch { - this.#fail(INVALID_BRIDGE_EVENT); - return; } + } catch { + this.#fail(INVALID_BRIDGE_EVENT); + return; } } - for (const segment of responseSegments) { - this.#seenSegmentIds.add(segment.id); - } const questionSegment = this.#chat.questionSegment; const correlatedQuestion = questionSegment && @@ -745,19 +633,16 @@ export class RealtimeBrunchBridge { }); } - /** - * A turn that settled short of a reply leaves no canonical text behind. Only - * Flue's settlement index distinguishes it from a turn still in progress or - * a completed step whose client-tool follow-up the panel is about to send, - * so wait for that record and never treat silence alone as a stop. - */ - #completeStoppedSubmission(active: ActiveSubmission): void { - if (active.submissionId === null) return; + /** A failed continuation invalidates the reply even if its first step succeeded. */ + #completeStoppedSubmission(active: ActiveSubmission): boolean { const settlement = this.#chat.settlements?.find( - ({ submissionId }) => submissionId === active.submissionId, + ({ submissionId, outcome }) => + outcome !== "completed" && + (submissionId === active.submissionId || + active.responseSubmissionIds.has(submissionId)), ); if (settlement === undefined || settlement.outcome === "completed") { - return; + return false; } this.#emit({ deliveryId: active.deliveryId, @@ -769,5 +654,6 @@ export class RealtimeBrunchBridge { outcome: settlement.outcome, type: "submission-stopped", }); + return true; } } diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-browser-tools.integration.test.tsx b/apps/petrinaut-website/src/main/app/voice-interview/voice-browser-tools.integration.test.tsx index b2b6123b247..157944c2539 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-browser-tools.integration.test.tsx +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-browser-tools.integration.test.tsx @@ -10,7 +10,10 @@ import { BrunchPanelConversationTracker, createBrunchPanelTransport, } from "../local-storage-demo/brunch-panel-transport"; -import { selectCanonicalSpeech } from "./canonical-speech"; +import { + selectAuthoredVoiceSpeech, + selectCanonicalSpeech, +} from "./canonical-speech"; import { RealtimeBrunchBridge } from "./realtime-brunch-bridge"; import { submitVoiceInputWithAdmission } from "./voice-interview-control"; @@ -116,6 +119,37 @@ test.each([ turnId: messageId, position: position(), }); + const speech = continuation + ? "The requested guide was read." + : "An obsolete draft before checking."; + const speechToolCallId = `speech-${submissionId}`; + await options?.onEvent?.({ + type: "tool-input", + conversationId: "test", + messageId, + toolCallId: speechToolCallId, + toolName: "brunch_set_voice_response", + input: { speech }, + position: position(), + }); + await options?.onEvent?.({ + type: "data-part", + conversationId: "test", + messageId, + name: "brunch-voice-response", + data: { speech, toolCallId: speechToolCallId }, + position: position(), + }); + await options?.onEvent?.({ + type: "tool-output", + conversationId: "test", + toolCallId: speechToolCallId, + output: { + title: "Brunch-authored speech (not playback confirmation)", + detail: speech, + }, + position: position(), + }); if (preamble || continuation) await options?.onEvent?.({ type: "message-delta", @@ -202,6 +236,12 @@ test.each([ submissionIds: tracker.submissionsForResponse(segment.messageId), }), ), + voiceSegments: selectAuthoredVoiceSpeech(current.messages).map( + (segment) => ({ + ...segment, + submissionIds: tracker.submissionsForResponse(segment.messageId), + }), + ), }); }; const handle = createJsonDocHandle({ @@ -277,6 +317,7 @@ test.each([ await waitFor(() => expect(finishContinuation).toBeDefined()); expect(send).toHaveBeenCalledTimes(2); expect(context?.status).not.toBe("ready"); + expect(speakCanonical).not.toHaveBeenCalled(); expect( events.some((event) => event.type === "canonical-response-ready"), ).toBe(false); @@ -298,10 +339,16 @@ test.each([ speakCanonical.mock.calls .flatMap(([segments]) => segments) .map((segment) => segment.text), - ).toEqual( - preamble - ? ["Checking the guide.", "The guide is available."] - : ["The guide is available."], - ); + ).toEqual(["The requested guide was read."]); + expect(speakCanonical).toHaveBeenCalledOnce(); + expect(events.at(-1)).toMatchObject({ + type: "canonical-response-ready", + segments: preamble + ? [ + expect.objectContaining({ text: "Checking the guide." }), + expect.objectContaining({ text: "The guide is available." }), + ] + : [expect.objectContaining({ text: "The guide is available." })], + }); }, ); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx index 7094a09f9aa..50c338c867b 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx @@ -15,7 +15,10 @@ import { Button, Checkbox } from "@hashintel/ds-components"; import { css } from "@hashintel/ds-helpers/css"; import { reportVoiceDiagnostic } from "../../../voice-diagnostics"; -import { selectCanonicalSpeech } from "./canonical-speech"; +import { + selectAuthoredVoiceSpeech, + selectCanonicalSpeech, +} from "./canonical-speech"; import { OpenAIRealtimeSession } from "./openai-realtime-session"; import { RealtimeBrunchBridge, @@ -563,6 +566,9 @@ const AvailableVoiceInterviewControl = ({ store.controller.updateChat({ canAcceptInterviewAnswer: context.canAcceptVoiceInput, canonicalSegments: canonicalSpeech.segments.map(correlateSegment), + voiceSegments: selectAuthoredVoiceSpeech(context.messages).map( + correlateSegment, + ), ...(canonicalSpeech.questionSegment ? { questionSegment: correlateSegment(canonicalSpeech.questionSegment) } : {}), diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts index 6dd1679ccb0..06ac350c236 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts @@ -10,7 +10,10 @@ import { BrunchPanelConversationTracker, createBrunchPanelTransport, } from "../local-storage-demo/brunch-panel-transport"; -import { selectCanonicalSpeech } from "./canonical-speech"; +import { + selectAuthoredVoiceSpeech, + selectCanonicalSpeech, +} from "./canonical-speech"; import { OpenAIRealtimeSession } from "./openai-realtime-session"; import { RealtimeBrunchBridge } from "./realtime-brunch-bridge"; import { submitVoiceInputWithAdmission } from "./voice-interview-control"; @@ -28,6 +31,7 @@ const providerAnswer = "v=0\r\na=private-provider-sdp\r\n"; const spokenAnswer = "The supervisor approves it."; const canonicalReply = "Thanks. I have recorded that."; const canonicalQuestion = "Who is informed next?"; +const authoredReply = `The supervisor approves it. ${canonicalQuestion}`; const requestIds = [ "00000000-0000-4000-8000-000000000011", "00000000-0000-4000-8000-000000000012", @@ -102,6 +106,18 @@ const responseMessages = [ { id: "next-question-message", parts: [ + { + type: "dynamic-tool", + toolName: "brunch_set_voice_response", + toolCallId: "speech-next-question", + state: "output-available", + input: { speech: authoredReply }, + output: { title: "Authored speech", detail: authoredReply }, + }, + { + type: "data-brunch-voice-response", + data: { speech: authoredReply, toolCallId: "speech-next-question" }, + }, { data: { question: canonicalQuestion, @@ -386,6 +402,9 @@ describe("controlled voice preview", () => { questionSegment: initialSelection.questionSegment, status: "ready", }); + // History never autoplays. Explicitly request reading to exercise an + // already-started microphone item overlapping application-owned output. + session.speakCanonical(initialSegments); dataChannel.receive({ content_index: 0, item_id: "pre-output-item", @@ -517,6 +536,8 @@ describe("controlled voice preview", () => { controller.updateChat({ canAcceptInterviewAnswer: true, canonicalSegments: correlatedSegments, + voiceSegments: + selectAuthoredVoiceSpeech(responseMessages).map(correlateResponse), questionSegment: responseSelection.questionSegment ? correlateResponse(responseSelection.questionSegment) : undefined, @@ -534,7 +555,7 @@ describe("controlled voice preview", () => { content: [ { text: JSON.stringify({ - response_text: [canonicalReply, canonicalQuestion], + response_text: [authoredReply], }), type: "input_text", }, diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts index c242a88c624..14e0a1c10e8 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts @@ -89,6 +89,39 @@ const markedQuestion = ( }); describe("VoiceTurnController", () => { + test.each(["full", "question"] as const)( + "restores explicit %s reading after reload without autoplay", + async (reading) => { + const harness = createHarness(); + const report = question( + "restored", + "Full report, with qualifications. Which limit matters?", + ); + const marked = markedQuestion("restored", "Which limit matters?"); + harness.controller.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [report], + questionSegment: marked, + voiceSegments: [ + { ...report, source: "assistant-voice", text: "A brief takeaway." }, + ], + status: "ready", + }); + await harness.controller.start(); + expect(harness.controller.getSnapshot()).toMatchObject({ + canReadFullResponse: true, + canRepeatQuestion: true, + }); + expect(harness.session.speakCanonical).not.toHaveBeenCalled(); + expect(harness.submitText).not.toHaveBeenCalled(); + if (reading === "full") harness.controller.readFullResponse(); + else harness.controller.repeatQuestion(); + expect(harness.session.speakCanonical).toHaveBeenCalledWith([ + reading === "full" ? report : marked, + ]); + }, + ); + test("records the content-free Voice lifecycle once in causal order", async () => { const harness = createHarness(); await harness.controller.start(); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts index 67a50a17e5c..5b577a99dcc 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts @@ -99,6 +99,7 @@ interface VoiceTurnControllerDependencies { interface ChatUpdate { readonly canAcceptInterviewAnswer: boolean; readonly canonicalSegments: CanonicalSpeechSegment[]; + readonly voiceSegments?: readonly CanonicalSpeechSegment[]; readonly questionSegment?: CanonicalSpeechSegment; readonly settlements?: readonly VoiceSubmissionSettlement[]; readonly stopped?: boolean; @@ -544,6 +545,30 @@ export class VoiceTurnController { public updateChat(update: ChatUpdate): void { const question = update.questionSegment; + if ( + update.status === "ready" && + this.#pendingSubmissionSettlement === null + ) { + // Hydration restores explicit reading/replay, never automatic playback. + const latestMessageId = update.canonicalSegments.at(-1)?.messageId; + const responseSegments = update.canonicalSegments.filter( + (segment) => segment.messageId === latestMessageId, + ); + const responseQuestion = + question?.messageId === latestMessageId ? (question ?? null) : null; + if ( + responseSegments.length !== this.#lastResponseSegments.length || + responseSegments.some( + (segment, index) => + segment.id !== this.#lastResponseSegments[index]?.id, + ) || + responseQuestion?.id !== this.#lastResponseQuestion?.id + ) { + this.#lastResponseSegments = responseSegments; + this.#lastResponseQuestion = responseQuestion; + this.#update({}); + } + } if (question && question.id !== this.#currentQuestionId) { this.#currentQuestionId = question.id; this.#update({ currentQuestion: question.text }); diff --git a/libs/@hashintel/brunch-agent/MISSION.md b/libs/@hashintel/brunch-agent/MISSION.md index 086ab0d5705..7182ee2af33 100644 --- a/libs/@hashintel/brunch-agent/MISSION.md +++ b/libs/@hashintel/brunch-agent/MISSION.md @@ -1,159 +1,155 @@ -# Brunch remote browser-origin policy +# Separate Brunch-authored Voice speech and display ## Status -**Live as of 2026-09-08** for -[SRE-1042](https://linear.app/hash/issue/SRE-1042/configure-petrinauts-deployment-variables-for-the-brunch-agent-chat) -on `t/sre-1042-allow-wildcard-origins-for-brunch-previews`, cut from `main` after -[FE-1626](https://github.com/hashintel/hash/pull/9583) established the exact-origin allow-list for -`/agents/*`. This file is the branch's sole execution authority. +**Live; scope and implementation authorized by the owner.** This is the sole mission on local +`voice/separate-brunch-speech`, stacked on [#9585](https://github.com/hashintel/hash/pull/9585) +at [0902dddb](https://github.com/hashintel/hash/commit/0902dddbbfd53ac98499c44a7302339bca135563). +Authority was committed separately before product changes. The owner has authorized committing, +pushing, and opening this follow-up as a draft stacked PR. Issue creation, paid provider activity, +manual deployment changes, and mission acceptance remain unauthorized. -Exact origins alone do not fit the deployment: every Petrinaut preview has its own -`https://petrinaut-git-.stage.hash.ai` origin, so the allow-list additionally accepts a -wildcard for exactly one leading host label. CORS governs whether a conforming browser exposes a cross-origin response -to client code; it does not authenticate or restrict non-browser callers, authorize a -conversation, or make public exposure safe by itself. +The accepted scope and timing decision are in the +[owner conversation](https://ampcode.com/threads/T-01a085b2-4d2a-73ca-bf56-95ec31430d52). +The prior planning conversation contains the pasted meeting transcript; the Notion proposal and +Slack discussion remain unreviewed. The inherited CORS contract is preserved without adjudicating +its acceptance in [its historical record](docs/mission-archive/sre-1042-browser-origin-policy.md). +No provisional future draft is consumed. ## Imperative -Let a deployed Petrinaut website use the Brunch `/agents/*` Flue routes from an explicitly trusted -browser origin while causing browsers to withhold cross-origin access from unlisted origins. Do -this now because the deployed website and Brunch service are separate origins and -[SRE-1042](https://linear.app/hash/issue/SRE-1042/configure-petrinauts-deployment-variables-for-the-brunch-agent-chat) -cannot point the browser at the deployed Brunch route until preflight and response headers work. +Determine whether Brunch can give a useful brief spoken answer or takeaway alongside complete +on-screen content while retaining domain authority. Judge usefulness, fidelity, and delay +separately. This tests separate Brunch-authored outputs under whole-correlated-reply completion +gating, not the best possible latency of a relay. + +Visible advance: a Voice clarification gives a useful answer rather than only a reading notice; +a long analysis gives a substantive takeaway while preserving the full report on screen. +Demo: open the prepared crew-reservation fixture, ask the two comparison inputs below, inspect +the spoken content and report, request full reading, repeat a marked question, interrupt, Stop, +and reopen. The local panel is the initial proof boundary; no deployed claim follows from it. ## Throughline -```text -Petrinaut browser at one configured exact origin -→ OPTIONS /agents// with requested method and headers -→ route-scoped Hono CORS middleware before ownership middleware -→ 204 preflight carrying the matching origin, GET/POST/OPTIONS, and Flue request headers -→ browser FlueClient GET/POST with x-brunch-principal + x-brunch-conversation -→ existing agentOwnershipGuard and createAgentRouter -→ response exposes the Flue/Durable Streams headers the browser SDK reads -``` - -`BRUNCH_CORS_ALLOWED_ORIGINS` is read once at startup as a comma-separated list of HTTP(S) -origins, each either exact or with a wildcard as the whole leading host label in front of a domain -with at least two labels (`https://*.stage.hash.ai`). A wildcard matches exactly one label, like a -wildcard TLS certificate. Parsing trims whitespace, normalizes an optional trailing slash through -`URL.origin`, and deduplicates values. Credentials, non-root paths, queries, fragments, wildcards in -any other position, opaque origins, and non-HTTP(S) schemes are startup configuration errors. Missing or blank configuration means an -empty allowlist: same-origin and non-browser callers continue through the existing route, but -browser code at another origin receives no CORS grant. See the -[Brunch application README](../../../apps/brunch-agent/README.md#production-container) for -operator configuration details. - -The middleware applies only to `/agents/*` and runs before `agentOwnershipGuard`, so a valid -preflight does not need conversation headers. It permits `GET`, `POST`, and `OPTIONS`; permits -`Content-Type`, `x-brunch-principal`, and `x-brunch-conversation`; does not permit credentials; and -uses a 600-second preflight cache. It exposes the non-safelisted response headers read by the -installed Flue 2.0.3 and Durable Streams 0.2.6 clients: - -- `flue-error-ref` -- `Stream-Next-Offset` -- `Stream-Cursor` -- `Stream-Up-To-Date` -- `Stream-Closed` -- `stream-sse-data-encoding` - -Hono's maintained CORS middleware owns header emission, `Vary` handling, and the `OPTIONS` response. -Non-browser callers can still send requests and receive ordinary HTTP responses because CORS is -enforced by browsers, not by the service as caller authentication. A response to an unlisted -browser origin carries no `Access-Control-Allow-Origin`, so the browser withholds that response -from client code. +Existing Voice admission and delivery-scoped context → Brunch tools and browser continuations → +Brunch-authored spoken and displayed outputs → completion of the whole correlated reply → +application-selected verbatim Realtime playback. + +- Short answers give a brief useful answer. Ask a follow-up only when it materially advances the + modelling goal; clarify first when ambiguity would materially change the answer. +- Long analyses have a substantive spoken takeaway and complete visible report/workpiece. +- Speech remains inspectable and associated with its originating response after reload. Authored + speech is not proof that it was heard. Read full response selects the complete displayed text. +- Direct questions retain the existing exact marker tool, exact visible prose, and accessible + replay; a spoken question preserves that wording. +- Automatic speech waits for the whole correlated reply, including browser-tool continuations. + An earlier completed message/submission does not suffice. The gate does not wait for the next + user answer. Failed/aborted replies do not release pending automatic speech. +- Missing, invalid, or uncorrelated speech never triggers an invented summary or automatic full + report reading. A delivery notice is not successful substantive delivery. + +### Ownership and permitted changes + +App-owned ChatAgent Voice instructions own output separation. Core SYSTEM.md, its question +semantics, and SDCPN prompts/skills remain unchanged, including full recoverable-workpiece and +prepared-fixture obligations. Realtime remains a delivery-only renderer with no domain tools, +independent questions, conclusions, or summaries. Typed effective instructions remain unchanged. + +The owner approved a bounded app-only wording experiment after live diagnosis found speech +authoring omitted despite the overlay and tool being present: explicitly order evidence gathering +→ speech authoring → optional question marking → visible delivery, including clarification-only +replies. The owner-reported retry after backend restart still produced no speech-tool call or +speech data; the wording experiment has not resolved fallback. Further mechanism changes or +additional provider calls require a new bounded decision; do not stack speculative prompt edits. + +Existing structured data writers/transport are a candidate, not a preselected schema. First pin +live completion, persisted history, response identity, continuation folding, and replay. Use the +existing conversation route/store. Stop if a new store or broader runtime redesign is required. + +Expected owners: ChatAgent and its tests; core's shared data contract if required without changing +universal prompts; AI SDK streaming/history/correlation; website Voice selection, bridge, +controller, session/policy, and minimal response-associated inspection UI. Update relevant user +documentation if exposed behavior requires it. No unrelated prompt or infrastructure cleanup. ## Proof -This mission establishes the application-side CORS contract required by the deployed browser -transport. It does **not** establish authentication, authorization, rate limiting, infrastructure -configuration, a deployed endpoint, or end-to-end remote verification. - -1. **Configuration is exact and fail-closed.** Missing and blank configuration produce no allowed - origins; whitespace, trailing slashes, duplicates, and multiple exact origins normalize - deterministically; malformed or broader-than-origin entries fail with the offending variable - named. Oracle: focused unit cases in `apps/brunch-agent/test/cors.test.ts`. -2. **Allowed browser traffic receives the complete grant.** An allowed origin receives its exact - value on an `/agents/*` response. Its preflight receives 204 before ownership, the three allowed - methods, the three allowed request headers, the six exposed response headers, no credentials - grant, and the required `Vary` values. Oracle: in-process Hono requests in - `apps/brunch-agent/test/cors.test.ts`. -3. **Rejected origins receive no grant.** An unlisted origin's preflight and ordinary response omit - `Access-Control-Allow-Origin`; an allowed origin does not make another origin pass. Oracle: - focused negative cases in `apps/brunch-agent/test/cors.test.ts`. -4. **The policy cannot widen unrelated routes.** `/health`, `/`, and `/assets/*` carry no Brunch - CORS grant. Existing ownership checks still return 401/403 for actual agent requests with - missing or mismatched identity. Oracle: CORS route-scope tests plus the existing - `apps/brunch-agent/test/agent-ownership.test.ts`. -5. **The shipped artifact and operator contract agree.** Brunch's README documents the variable, - exact-origin configuration, empty-list behavior, and the fact that CORS governs browser access - rather than authenticating or restricting non-browser callers. Oracle: - `yarn workspace @apps/brunch-agent test:unit`, - `yarn workspace @apps/brunch-agent lint:tsc`, - `yarn workspace @apps/brunch-agent lint:eslint`, and - `yarn workspace @apps/brunch-agent build`. +1. **Context isolation:** real-runtime effective prompt tests in + `apps/brunch-agent/test/voice-context.test.ts` and transport admissions distinguish + typed → Voice → browser continuation → typed. Typed instructions and tool availability remain + unchanged. Unknown preferences do not enable Voice behavior. +2. **Routing and durability:** targeted transport `ui-stream.test.ts`, `transcript.test.ts`, and + `chat-transport.test.ts`, plus website `canonical-speech.test.ts`, + `realtime-brunch-bridge.test.ts`, `voice-turn-controller.test.ts`, + `openai-realtime-session.test.ts`, and browser-tool integration tests. Exercise speech data + before report completion, earlier completions followed by continuations, identical text on + distinct replies, failed/aborted continuations, missing/invalid speech, full-report selection, + exact question replay, interruption versus durable Stop, and reload without autoplay, + duplicate admission, or duplicate content. Test actual outputs, not only absence of crashes. +3. **Content quality:** human inspection of audible speech against the request, full report, + fixture and tool evidence. A simple clarification must be useful without gratuitous follow-up; + a consequential gap must ask a relevant marked question; a long takeaway must not contradict + the report or omit qualifications that change its meaning. A browser-tool continuation must + report success, rejection, and no-op truthfully. Deterministic checks cannot accept this leaf. +4. **Comparable demonstration:** repeat “What does reserving a dispatch crew mean here?” and + “Give me a detailed analysis of this model, including assumptions, possible bottlenecks, + missing constraints, and what still needs validation. Do not change the model.” Use + `crew-reservation-v1` and record model/configuration differences from #9585. Inspect a + synchronized audible browser recording plus representative UI/accessibility states. + Paid execution is blocked until an explicit bounded owner authorization; no campaign. +5. **Latency:** record end of user speech, completed transcription, whole-reply completion, + speech request, and first substantive audible answer separately. Synchronized audio/human + inspection is the first-audible oracle; provider buffer events and notices are not answers. + Report no answer when none is heard. No invented word or latency acceptance threshold. +6. **Repository:** affected workspace `test:unit`, `lint:tsc`, `lint:eslint`, `build`, changed-file + Oxfmt and `git diff --check`. Render and inspect the affected UI. Report blocked/failed checks + honestly. Local and mocked checks do not establish deployed end-to-end behavior. + +The #9585 evidence records 192 → 151 clarification words and only a notice spoken afterward; +the long report stayed complete and opt-in reading worked in the recorded run. These were +individual synthetic-input real-provider diagnostics, not a statistical campaign or human +acceptance. They do not prove core caused verbosity or exhaust relay prompt alternatives. ## Constraints -- Use Hono's built-in CORS middleware; do not create a parallel HTTP server or hand-maintain generic - CORS response logic. -- Keep one Flue product route and the existing ownership guard. CORS must not add, proxy, rename, or - reinterpret an agent route. -- The origin list is explicit: exact origins or one-label wildcards, matched by scheme, host and - port. Do not hard-code Petrinaut domains, reflect arbitrary `Origin` values, or silently skip - malformed entries. -- Keep credentials disabled. The current browser client uses explicit ownership headers, not - cookies, and those headers are not authentication. -- Answer preflight before ownership while preserving ownership enforcement on every non-preflight - agent request. -- Read configuration once at startup. Dynamic policy storage or hot reload is not earned by this - deployment. -- Preserve local same-origin proxying when the variable is unset. -- No implementation begins until this authority cut is committed separately. Material changes to - this contract require owner review and another focused authority commit. - -### Expected touched paths - -```text -~ libs/@hashintel/brunch-agent/MISSION.md branch authority -~ apps/brunch-agent/src/http/cors.ts exact and one-label wildcard origins, Hono middleware -~ apps/brunch-agent/src/app.ts mount CORS before ownership on /agents/* -+ apps/brunch-agent/test/cors.test.ts parser, allowed, rejected, preflight, route-scope tests -~ apps/brunch-agent/README.md deployment variable and security boundary -~ apps/brunch-agent/turbo.json pass the variable into the local dev task -``` +- Brunch owns domain meaning, questions, conclusions, tools, and workpiece state. +- Preserve tool-result truthfulness, submission correlation, direct-question semantics, + interruption versus durable Stop, and reload without autoplay/duplication. +- Preserve typed effective instructions and behavior; no conversation-wide Voice switch. +- No Realtime reasoning/delegation, Brunch-as-client-tool, broad core redesign, new conversation + store, workpiece/provenance redesign, unrelated infrastructure work, or paid campaign. +- The inherited #9585 delivery-context patch remains a maintained local Flue 2.0.3 exception, + not an upstream-supported API; do not broaden that exception silently. +- No external writes, push, PR/issue creation, paid demonstration, or deployment without approval. ## Fog-line -- Infrastructure repository access is unavailable in this worktree, so this branch can prove only - the application contract. Runtime deployment configuration must supply the chosen origins before - remote verification. -- A one-label wildcard admits every host directly under the configured domain, not only Petrinaut - previews. Narrow the deployed pattern or return to exact origins if that breadth becomes a - problem in practice. -- The allowed and exposed headers are pinned to the installed Flue and Durable Streams clients. - Re-evaluate them from client source when either dependency changes. +The existing structured data writer is now selected for the local implementation: runtime +restart, transport, real-panel continuation, and replay checks establish the tested routing +contract. [Local verification](docs/evidence/implementations/separate-voice-speech/verification.md) +records the evidence and its limits; no real-provider or deployed acceptance follows. +Model adherence, useful brevity, speech/report consistency, actual audio fidelity, and tolerable +delay remain experimental. Completion gating avoids speculative delivery, not semantic errors. +Historical preview configuration and backend-deployment verification remain unresolved; any +remote claim requires a new real deployed witness. Human acceptance and paid ceilings are +owner-held. No separate Linear issue is linked, and no Linear integration is available in this +orb. Draft publication uses the repository's descriptive-title contribution workflow; Linear +writes still require explicit approval. ## Stop or reorient -Stop if the real browser client emits a request method or non-safelisted request header outside the -pinned contract, reads another non-safelisted response header, or needs cookie credentials. Bring -that evidence back to the contract before broadening the grant. - -Stop if middleware ordering bypasses ownership for a non-`OPTIONS` request, if an invalid -configuration widens access or is ignored, if an unlisted origin receives -`Access-Control-Allow-Origin`, or if `/health`, `/`, or `/assets/*` inherit the policy. +Stop if typed behavior inherits Voice, speech loses response identity, cancellation allows later +autoplay, replay duplicates content, workpieces are incomplete, or claims exceed tool evidence. +Reorient if outputs repeatedly contradict, substantive speech is absent, or the small routing +change requires broader mechanisms. Do not weaken the oracle or repair content in Realtime. -Do not represent a green CORS test as permission for unauthenticated public exposure. Authentication, -per-conversation authorization, rate/spend controls, and the infrastructure ingress boundary remain -separate release gates. +Verdicts update this relay variant only. Content success with unacceptable delay leaves earlier +delivery unresolved for a follow-up; it does not select another architecture. Prepare evidence +and stop for owner acceptance rather than declaring naturalness or mission closure. ## Deferred -- SRE-1013 owns injection of the allowlist into the Brunch runtime deployment. SRE-1042 owns - `VITE_BRUNCH_CHAT_ENDPOINT`, Voice deployment variables, and the deployed browser verification - after this application contract lands. -- FE-1615 and FE-1616 retain authentication and rate-limit work. CORS does not discharge either. -- A same-origin Petrinaut proxy stays deferred; the one-label wildcard covers the preview - deployments the exact list could not. +[MISSION.next.md](MISSION.next.md) retains the existing future spine and inherited limitations. +Its CORS transition pointer preserves deployment, authentication, and rate-limit owners. +Earlier delivery re-enters only if measured delay is unacceptable despite content success; +its safety and benefit need a separate scope and audible oracle. Alternative architecture +selection remains owner-held, not an automatic consequence of any experimental failure. diff --git a/libs/@hashintel/brunch-agent/MISSION.next.md b/libs/@hashintel/brunch-agent/MISSION.next.md index 928bed27037..dcf491917b9 100644 --- a/libs/@hashintel/brunch-agent/MISSION.next.md +++ b/libs/@hashintel/brunch-agent/MISSION.next.md @@ -4,6 +4,15 @@ This spine and its four linked drafts form one future-planning record. Keep each consequential meaning in one authoritative planning home: shared contracts and unallocated concerns live here; mission-specific detail lives in its draft. A spine pointer is not a second contract. Material omitted from a future cut returns to this record at full fidelity, and the consumed draft is removed. +The owner-authorized separate-speech follow-up now owns this branch's `MISSION.md`; no future +draft was consumed. The inherited CORS contract remains historical, not newly accepted, in +[sre-1042-browser-origin-policy.md](docs/mission-archive/sre-1042-browser-origin-policy.md). +Its Deferred section preserves SRE-1013 deployment configuration, SRE-1042 deployed verification, +FE-1615 authentication, FE-1616 rate limits, and the deferred same-origin proxy. None is closed +by local Voice evidence. Earlier speech delivery re-enters after this experiment only if content +succeeds but measured delay is unacceptable; safety and audible benefit require a separately +approved follow-up. Alternative architecture selection remains owner-held. + The record was recut on 2026-09-04 around provenance by lineage with declared basis; the [2026-09-04 migration disposition](#2026-09-04-provenance-replanning-migration-disposition) maps every prior planning item to its surviving home. ## Current authority and accepted spine diff --git a/libs/@hashintel/brunch-agent/docs/evidence/implementations/separate-voice-speech/verification.md b/libs/@hashintel/brunch-agent/docs/evidence/implementations/separate-voice-speech/verification.md new file mode 100644 index 00000000000..0f989164cde --- /dev/null +++ b/libs/@hashintel/brunch-agent/docs/evidence/implementations/separate-voice-speech/verification.md @@ -0,0 +1,254 @@ +# Separate Brunch-authored speech: local implementation evidence + +2026-09-09. Implementation evidence, not mission acceptance or a content-quality verdict. +The execution authority is [MISSION.md](../../../../MISSION.md). The parent experiment +[#9585](https://github.com/hashintel/hash/pull/9585) remains open at +[0902dddb](https://github.com/hashintel/hash/commit/0902dddbbfd53ac98499c44a7302339bca135563). +The meeting transcript was available in the prior planning conversation. The Notion proposal +and Slack discussion remain unreviewed. + +## Implemented boundary + +The Voice-only ChatAgent contribution registers `brunch_set_voice_response`. It accepts +nonblank Brunch-authored speech, writes schema-validated `brunch-voice-response` data with +the tool-call identity, and returns that exact speech in the existing visible tool-result +card. This uses the existing Flue store and AI SDK data/history projection; it adds no store +or transport protocol. Ordinary assistant prose remains the full visible response. + +The website selects speech only with a matching successful authoring tool and subsequent +finalized visible prose. A later substantive tool invalidates an earlier draft; Brunch must +author another one after the tool result. The question marker does not invalidate speech. +The bridge releases the final message's correlated speech only when the panel's derived +whole-reply status is ready, including all automatic browser-tool continuations. An individual +message completion is not permission to play. Failed, aborted, or locally withheld replies +do not release pending speech. Playback interruption remains distinct from durable Stop. + +Missing, unusable, or uncorrelated speech falls back only to the existing fixed reading +notice when a complete visible reply exists. The notice is not substantive delivery. Neither +the application nor Realtime summarizes a report. Read full response selects visible prose; +Repeat question selects the exact visible, marked question. Reload restores explicit reading +without admitting a user turn or automatically playing saved speech. + +Prompt changes are confined to the app-owned Voice overlay and Voice-only authoring tool. +Core SYSTEM.md, core Flue question instructions, SDCPN append prompt, modelling skill, +prepared-fixture instructions, and Realtime policy are unchanged. Realtime still receives +only application-selected text, with no domain tools or autonomous response permission. + +## Deterministic oracles + +- `apps/brunch-agent/test/voice-context.test.ts`: real runtime effective instructions and tool + availability across typed → Voice → Voice continuation → typed → unknown preference. + Typed prompts and tools are unchanged across that sequence. +- `apps/brunch-agent/test/voice-response.test.ts`: production ChatAgent, faux provider, HTTP + router, FlueClient, AI SDK transport, temporary SQLite database, runtime shutdown/restart. + Exact speech data, full report/workpiece prose, and exact question marker survive together. + This is model-independent routing/durability evidence, not model adherence evidence. +- `canonical-speech.test.ts`: separate exact speech/report selection, stable identities, + identical speech on different replies, blank/missing/unmatched/obsolete speech, provisional + prose, later tools, replacement drafts, stopped messages, and unchanged question replay. +- `realtime-brunch-bridge.test.ts`: no early playback at message completion; one final + correlated takeaway after continuations; no fabricated summary or automatic long-report + fallback; no replay for typed/history updates; failed/aborted continuation and cancellation. +- `voice-browser-tools.integration.test.tsx`: actual Petrinaut panel, tracker, transport, and + browser `readPetrinautDoc` execution with scripted Flue events. The panel stays busy across + continuation admission and produces one final authored speech request, never the obsolete + pre-tool draft or the report. Invalid browser input and Stop withhold speech. +- `voice-preview.integration.test.ts`: actual session request construction with fake WebRTC + and transport; the distinct authored takeaway reaches `response_text` unchanged, with no + tools. Existing capture/echo, cancellation and replay checks remain active. +- `voice-turn-controller.test.ts`: full-prose and exact-question replay after reload, no + autoplay or resubmission, plus existing half-duplex/interruption/Stop state checks. + +## Executed checks + +Commands run from the repository root after building the required local dependencies: + +```sh +yarn workspace @apps/brunch-agent test:unit +# 28 files, 206 tests passed +NODE_OPTIONS=--no-experimental-webstorage yarn workspace @apps/petrinaut-website test:unit +# 41 files, 384 tests passed +yarn workspace @hashintel/brunch-agent test:unit +# 11 files, 96 tests passed +yarn workspace @hashintel/brunch-agent-transport-aisdk test:unit +# 4 files, 49 tests passed +``` + +The targeted website Voice suite also passed 193 tests without that Node option. The full +website run initially failed seven unrelated local-storage tests under Node 26.5.1 because +its experimental global storage displaced jsdom storage. Disabling Node's experimental +storage restores jsdom and passes all 384 tests without changing tests or application code. + +Build, `lint:tsc`, and `lint:eslint` passed for the Brunch application, Brunch core, and website. +Brunch application lint retains 14 warnings in unchanged code; website retains its existing +set-state-in-effect warning; neither has errors. The Petrinaut library build and targeted +user-guide-content test passed. Changed TypeScript/JSON formatting and `git diff --check` +passed. No whole-monorepo build claim is made. + +Environment preparation initially exposed missing generated design-system tokens, library +builds, website example JSON, and optimizer-client types. Their existing codegen/build commands +were run without tracked infrastructure changes. The unrelated full `hash-backend-utils` +build still reports missing graph-workspace dependencies; it emitted the OpenTelemetry +subpath needed by Brunch, whose own build and full test suite subsequently passed. + +## Rendered inspection + +Chromium rendered the actual website/Petrinaut panel at 1280 × 900, device scale 2. A seeded +local UI fixture displayed a distinct speech card and complete report. The inspected capture +is attached in the [implementation thread](https://ampcode.com/threads/T-01a085b2-4d2a-73ca-bf56-95ec31430d52) +as `.amp/in/artifacts/separate-voice-speech.png`. The card includes its entire qualification, +“The release rule still needs validation,” without clipping, and explicitly disclaims playback +confirmation. Its detail is visible without expansion. After reload and reopening the panel, +DOM checks found both speech and report and exactly one user message. This seeded browser +check is not the prepared real-provider demonstration or remote Flue deployment evidence. + +## Local live fallback diagnosis + +On 2026-09-09, the owner reported fallback playback and no speech card despite a browser +Voice delivery. The initial context/response tests passed after `yarn install --immutable`, +but live fallback persisted. The backend on port 4321 started at 17:56:11 Europe/Tirane, +after the installed patched runtime files were updated at 17:54:41. Read-only inspection +of its open SQLite store confirmed that the 17:57:10 submission retained Voice context and +registered the speech tool, but wrote no speech data. + +The owner then manually initiated one coordinated Voice request in the existing conversation. +A temporary development-only `observe()` subscriber inspected `turn_request`, logging only +presence flags, timestamps, and correlation IDs; no credentials, prompts, arguments, or +tool-result content were logged. The owner supplied the live output in the +[local diagnosis thread](https://ampcode.com/threads/T-01a086e5-f3c7-717e-b27f-f2574eca4053). +For submission `sub_ik_4cc5adbc94e5b1b4ffc62800f4cde6d0`: + +- Both model requests, `turn_01M23ERQ1DMXVJTRY8W31BW89F` and + `turn_01M23ERVN4DSSVDCQAA16KQ6YH`, contained the Voice instruction marker and + `brunch_set_voice_response` tool (`voiceInstructions: true`, `speechTool: true`). +- Read-only canonical stream inspection found exactly one tool call: + `brunch_mark_question`, which succeeded. The only data write was `brunch-question`. + There was no speech-tool call and no `brunch-voice-response` data write. +- The second model turn stopped normally; the submission completed at + `2026-09-09T16:07:18.753Z`. Duplicate observer lines had identical turn IDs; canonical + records contain two model turns, not four. + +This locates the observed failure at speech authoring: the model did not call the available +speech tool despite receiving the Voice overlay. There was no authored speech for transport +or UI selection to recover. It does not establish why the model omitted the tool, general +adherence rates, audible quality, or that all UI selection paths are correct. The fixed +reading-notice branch remains the intended response to missing speech, not successful +substantive delivery. Concurrent OpenTelemetry export errors targeted the unavailable local +collector at `::1:4317`; the recorded model work nevertheless completed. + +The temporary probe was removed after diagnosis. Context and response tests passed locally +(two files, two tests); probe typechecking, changed-file lint, formatting, and diff checks +passed. No automated provider request, storage reset, prompt change, completion-gate change, +Realtime change, or architecture change was made. Further paid execution and any material +reorientation remain owner-held; this diagnostic is not experiment acceptance. + +### Instruction-adherence follow-up + +Read-only inspection identified the live provider/model as +`anthropic/claude-haiku-4-5`. The observed conversation contains fixture initialization and +two Voice submissions, with no skill activations or browser-tool continuations. In the +instrumented submission, 22 words of visible prose preceded the question-marker call; +the next model turn emitted only the exact marked question (16 words), then stopped. +This is a question-delivery sequence with speech omitted, not a failed speech-tool execution. + +Inspection of the core, SDCPN, prepared-fixture, and Voice instructions found no explicit +prohibition on the speech call. Core's question marker says to call it immediately before +presenting the question; the Voice overlay separately requires speech authoring before final +visible delivery. These obligations can coexist, but their combined order is not spelled out. +At diagnosis, the response test scripted question marker → speech authoring → prose, so its +success proved routing for that sequence, not that the live model would choose it. + +Flue 2.0.3 [prompt composition](https://github.com/withastro/flue/blob/ac610378741d879a9d12d3f927ff9634e0b4f7ae/packages/runtime/src/hooks/render.ts#L215-L225) +joins the returned core prompt and `useInstruction` contributions in call order. Its +[provider boundary](https://github.com/withastro/flue/blob/ac610378741d879a9d12d3f927ff9634e0b4f7ae/packages/runtime/src/session.ts#L823-L836) +passes the observed context directly to `pi-ai`; this excludes post-observation filtering +inside Flue, not uninspected provider-adapter normalization. No active skill in this +conversation supplies a competing instruction. + +The next bounded hypothesis is that explicitly composing the delivery order in the app-owned +Voice overlay—finish domain/tool work, author speech, mark any direct question, deliver visible +prose—improves adherence, including clarification-only replies. This was proposed as a wording +experiment, not an established cause or fix. No further provider request or prompt edit was +made during the read-only follow-up. + +### Approved wording experiment, fallback persists after restart + +The owner approved the narrow experiment in the local diagnosis thread. The app-owned Voice +overlay now explicitly orders evidence gathering → speech authoring → optional question marking +→ complete visible delivery, and states that clarification-only replies also require speech. +No core/SDCPN prompt, tool implementation, typed instruction, Realtime policy, conversation +storage, or completion gate changed. + +The effective-prompt test first failed on the missing instruction, then passed after the edit. +It checks the ordered instruction and retains typed/Voice/continuation isolation checks; +boolean assertions avoid logging the effective prompt on failure. The runtime/transport test +now scripts speech authoring before question marking and verifies exact speech, question, and +full report persistence across restart. Existing UI tests confirm that a later question marker +does not invalidate authored speech and that playback still requires correlated completion. + +Verification: app build passed; all 28 app test files / 206 tests passed; canonical-speech and +realtime-brunch-bridge tests passed (two files / 52 tests); app typecheck, changed-TypeScript +lint, and Oxfmt checks passed. The first full app test run had three failures from a stale +emitted server bundle (old store, CORS, and fixture schema); rebuilding the app resolved them +without source changes. The agent sent no provider request. These checks establish prompt +placement and routing, not improved model adherence or audible delivery. + +A final read-only store check found another Voice submission, +`sub_ik_c7d46e277bf1cebd80139d99f3ff4648`, admitted at 18:15:01 Europe/Tirane during +verification. It completed with one Haiku turn and no tool calls or data writes. Its origin +has not been confirmed with the owner, and no live request probe captured the revised overlay +for it; do not count it as a verified trial of the wording experiment yet. + +The owner subsequently followed the restart/retest instructions and reported no card and the +same reading notice. The backend listening on port 4321 started at 18:21:37 Europe/Tirane, +after the overlay edit at 18:14:38. The latest Voice submission, +`sub_ik_2f131d148d5a75fc433972e63d67b0e5`, was admitted at 18:22:33.982 and retained +Voice context. Its resource snapshot registered the speech tool. Haiku completed one turn +normally at 18:22:41.269 with zero tool calls and zero data writes. The observer was no longer +installed, so this retry does not provide a direct capture of the revised model request. + +The reported fallback persists after the wording experiment and restart; there is still no +speech for transport or UI selection to recover. Do not infer general model adherence rates +or add another speculative prompt edit. A bounded app-level missing-speech check with a +Brunch-authored repair is a candidate for investigation, not an approved mechanism: provider +cost, continuation/cancellation behavior, and Flue support must be settled before implementation. +No repair call or additional provider request was initiated by the agent. + +Flue 2.0.3 exposes a supported `useAgentFinish` hook with successful/failed tool-call history +and `append()` for a correction signal within the same response/submission. Its built-in limit +is 32 finish continuations, which is not an acceptable implicit retry budget here. Any proposal +must establish an application-level one-repair bound and preserve cancellation and continuation +correlation before implementation; no runtime patch or forced provider tool choice is selected. + +## Outstanding acceptance gates + +Initial deterministic/seeded verification involved no paid provider activity, campaign, +deployment, external write, push, or PR creation. The owner subsequently authorized committing +and publishing a draft stacked PR; that authorization does not authorize a paid trial or +accept the experiment. Beyond the manually initiated diagnostics above, the full audible +comparison demonstration awaits bounded paid-run authorization; human review remains +outstanding. Repeat the two historical inputs on `crew-reservation-v1` once that run is authorized: + +1. “What does reserving a dispatch crew mean here?” +2. “Give me a detailed analysis of this model, including assumptions, possible bottlenecks, + missing constraints, and what still needs validation. Do not change the model.” + +Also witness a consequential modelling gap, evidence-backed browser continuation (including +rejection/no-op truthfulness), mixed typed/Voice use, interruption, Stop, and reload. Record +model/configuration differences from #9585. A scripted correct answer cannot judge whether +the model naturally authors useful or faithful content. + +Judge three axes separately: + +- **Usefulness:** a brief useful answer without gratuitous follow-up; a relevant marked + question when the modelling gap matters; a substantive takeaway for a long report. +- **Fidelity:** speech agrees with full prose and tool evidence, preserves consequential + qualifications and exact spoken questions, while the visible workpiece remains complete. +- **Delay:** independently record user speech end, final transcription, whole-reply completion, + speech request, and first substantive audible answer. Audio-buffer events, acknowledgements, + and reading notices are not first-audible answers. No audible delay was measured here. + +No word-count or latency acceptance threshold has been invented. Content success with +unacceptable delay leaves earlier delivery as an unresolved follow-up. Failure updates only +this relay variant's assessment; it does not exhaust relay alternatives, implicate core +SYSTEM.md without evidence, or select delegation or Brunch-as-client-tool. diff --git a/libs/@hashintel/brunch-agent/docs/mission-archive/sre-1042-browser-origin-policy.md b/libs/@hashintel/brunch-agent/docs/mission-archive/sre-1042-browser-origin-policy.md new file mode 100644 index 00000000000..265fbeda3c6 --- /dev/null +++ b/libs/@hashintel/brunch-agent/docs/mission-archive/sre-1042-browser-origin-policy.md @@ -0,0 +1,163 @@ +# Brunch remote browser-origin policy + +## Status + +**Historical inherited contract, preserved on the owner-authorized Voice branch transition.** +This archival does not adjudicate acceptance or close its outstanding deployment gates. +The following describes its original scope as of 2026-09-08, not this branch's execution authority. + +Originally live for +[SRE-1042](https://linear.app/hash/issue/SRE-1042/configure-petrinauts-deployment-variables-for-the-brunch-agent-chat) +on `t/sre-1042-allow-wildcard-origins-for-brunch-previews`, cut from `main` after +[FE-1626](https://github.com/hashintel/hash/pull/9583) established the exact-origin allow-list for +`/agents/*`. + +Exact origins alone do not fit the deployment: every Petrinaut preview has its own +`https://petrinaut-git-.stage.hash.ai` origin, so the allow-list additionally accepts a +wildcard for exactly one leading host label. CORS governs whether a conforming browser exposes a cross-origin response +to client code; it does not authenticate or restrict non-browser callers, authorize a +conversation, or make public exposure safe by itself. + +## Imperative + +Let a deployed Petrinaut website use the Brunch `/agents/*` Flue routes from an explicitly trusted +browser origin while causing browsers to withhold cross-origin access from unlisted origins. Do +this now because the deployed website and Brunch service are separate origins and +[SRE-1042](https://linear.app/hash/issue/SRE-1042/configure-petrinauts-deployment-variables-for-the-brunch-agent-chat) +cannot point the browser at the deployed Brunch route until preflight and response headers work. + +## Throughline + +```text +Petrinaut browser at one configured exact origin +→ OPTIONS /agents// with requested method and headers +→ route-scoped Hono CORS middleware before ownership middleware +→ 204 preflight carrying the matching origin, GET/POST/OPTIONS, and Flue request headers +→ browser FlueClient GET/POST with x-brunch-principal + x-brunch-conversation +→ existing agentOwnershipGuard and createAgentRouter +→ response exposes the Flue/Durable Streams headers the browser SDK reads +``` + +`BRUNCH_CORS_ALLOWED_ORIGINS` is read once at startup as a comma-separated list of HTTP(S) +origins, each either exact or with a wildcard as the whole leading host label in front of a domain +with at least two labels (`https://*.stage.hash.ai`). A wildcard matches exactly one label, like a +wildcard TLS certificate. Parsing trims whitespace, normalizes an optional trailing slash through +`URL.origin`, and deduplicates values. Credentials, non-root paths, queries, fragments, wildcards in +any other position, opaque origins, and non-HTTP(S) schemes are startup configuration errors. Missing or blank configuration means an +empty allowlist: same-origin and non-browser callers continue through the existing route, but +browser code at another origin receives no CORS grant. See the +[Brunch application README](../../../../../apps/brunch-agent/README.md#production-container) for +operator configuration details. + +The middleware applies only to `/agents/*` and runs before `agentOwnershipGuard`, so a valid +preflight does not need conversation headers. It permits `GET`, `POST`, and `OPTIONS`; permits +`Content-Type`, `x-brunch-principal`, and `x-brunch-conversation`; does not permit credentials; and +uses a 600-second preflight cache. It exposes the non-safelisted response headers read by the +installed Flue 2.0.3 and Durable Streams 0.2.6 clients: + +- `flue-error-ref` +- `Stream-Next-Offset` +- `Stream-Cursor` +- `Stream-Up-To-Date` +- `Stream-Closed` +- `stream-sse-data-encoding` + +Hono's maintained CORS middleware owns header emission, `Vary` handling, and the `OPTIONS` response. +Non-browser callers can still send requests and receive ordinary HTTP responses because CORS is +enforced by browsers, not by the service as caller authentication. A response to an unlisted +browser origin carries no `Access-Control-Allow-Origin`, so the browser withholds that response +from client code. + +## Proof + +This mission establishes the application-side CORS contract required by the deployed browser +transport. It does **not** establish authentication, authorization, rate limiting, infrastructure +configuration, a deployed endpoint, or end-to-end remote verification. + +1. **Configuration is exact and fail-closed.** Missing and blank configuration produce no allowed + origins; whitespace, trailing slashes, duplicates, and multiple exact origins normalize + deterministically; malformed or broader-than-origin entries fail with the offending variable + named. Oracle: focused unit cases in `apps/brunch-agent/test/cors.test.ts`. +2. **Allowed browser traffic receives the complete grant.** An allowed origin receives its exact + value on an `/agents/*` response. Its preflight receives 204 before ownership, the three allowed + methods, the three allowed request headers, the six exposed response headers, no credentials + grant, and the required `Vary` values. Oracle: in-process Hono requests in + `apps/brunch-agent/test/cors.test.ts`. +3. **Rejected origins receive no grant.** An unlisted origin's preflight and ordinary response omit + `Access-Control-Allow-Origin`; an allowed origin does not make another origin pass. Oracle: + focused negative cases in `apps/brunch-agent/test/cors.test.ts`. +4. **The policy cannot widen unrelated routes.** `/health`, `/`, and `/assets/*` carry no Brunch + CORS grant. Existing ownership checks still return 401/403 for actual agent requests with + missing or mismatched identity. Oracle: CORS route-scope tests plus the existing + `apps/brunch-agent/test/agent-ownership.test.ts`. +5. **The shipped artifact and operator contract agree.** Brunch's README documents the variable, + exact-origin configuration, empty-list behavior, and the fact that CORS governs browser access + rather than authenticating or restricting non-browser callers. Oracle: + `yarn workspace @apps/brunch-agent test:unit`, + `yarn workspace @apps/brunch-agent lint:tsc`, + `yarn workspace @apps/brunch-agent lint:eslint`, and + `yarn workspace @apps/brunch-agent build`. + +## Constraints + +- Use Hono's built-in CORS middleware; do not create a parallel HTTP server or hand-maintain generic + CORS response logic. +- Keep one Flue product route and the existing ownership guard. CORS must not add, proxy, rename, or + reinterpret an agent route. +- The origin list is explicit: exact origins or one-label wildcards, matched by scheme, host and + port. Do not hard-code Petrinaut domains, reflect arbitrary `Origin` values, or silently skip + malformed entries. +- Keep credentials disabled. The current browser client uses explicit ownership headers, not + cookies, and those headers are not authentication. +- Answer preflight before ownership while preserving ownership enforcement on every non-preflight + agent request. +- Read configuration once at startup. Dynamic policy storage or hot reload is not earned by this + deployment. +- Preserve local same-origin proxying when the variable is unset. +- No implementation begins until this authority cut is committed separately. Material changes to + this contract require owner review and another focused authority commit. + +### Expected touched paths + +```text +~ libs/@hashintel/brunch-agent/MISSION.md branch authority +~ apps/brunch-agent/src/http/cors.ts exact and one-label wildcard origins, Hono middleware +~ apps/brunch-agent/src/app.ts mount CORS before ownership on /agents/* ++ apps/brunch-agent/test/cors.test.ts parser, allowed, rejected, preflight, route-scope tests +~ apps/brunch-agent/README.md deployment variable and security boundary +~ apps/brunch-agent/turbo.json pass the variable into the local dev task +``` + +## Fog-line + +- Infrastructure repository access is unavailable in this worktree, so this branch can prove only + the application contract. Runtime deployment configuration must supply the chosen origins before + remote verification. +- A one-label wildcard admits every host directly under the configured domain, not only Petrinaut + previews. Narrow the deployed pattern or return to exact origins if that breadth becomes a + problem in practice. +- The allowed and exposed headers are pinned to the installed Flue and Durable Streams clients. + Re-evaluate them from client source when either dependency changes. + +## Stop or reorient + +Stop if the real browser client emits a request method or non-safelisted request header outside the +pinned contract, reads another non-safelisted response header, or needs cookie credentials. Bring +that evidence back to the contract before broadening the grant. + +Stop if middleware ordering bypasses ownership for a non-`OPTIONS` request, if an invalid +configuration widens access or is ignored, if an unlisted origin receives +`Access-Control-Allow-Origin`, or if `/health`, `/`, or `/assets/*` inherit the policy. + +Do not represent a green CORS test as permission for unauthenticated public exposure. Authentication, +per-conversation authorization, rate/spend controls, and the infrastructure ingress boundary remain +separate release gates. + +## Deferred + +- SRE-1013 owns injection of the allowlist into the Brunch runtime deployment. SRE-1042 owns + `VITE_BRUNCH_CHAT_ENDPOINT`, Voice deployment variables, and the deployed browser verification + after this application contract lands. +- FE-1615 and FE-1616 retain authentication and rate-limit work. CORS does not discharge either. +- A same-origin Petrinaut proxy stays deferred; the one-label wildcard covers the preview + deployments the exact list could not. diff --git a/libs/@hashintel/brunch-agent/packages/core/package.json b/libs/@hashintel/brunch-agent/packages/core/package.json index 70593a8eb94..d64c4e9250d 100644 --- a/libs/@hashintel/brunch-agent/packages/core/package.json +++ b/libs/@hashintel/brunch-agent/packages/core/package.json @@ -26,6 +26,10 @@ "types": "./src/storage.ts", "import": "./dist/storage.js" }, + "./voice-response": { + "types": "./src/voice-response.ts", + "import": "./dist/voice-response.js" + }, "./workpiece": { "types": "./src/workpiece.ts", "import": "./dist/workpiece.js" diff --git a/libs/@hashintel/brunch-agent/packages/core/src/voice-response.ts b/libs/@hashintel/brunch-agent/packages/core/src/voice-response.ts new file mode 100644 index 00000000000..cea89939f6d --- /dev/null +++ b/libs/@hashintel/brunch-agent/packages/core/src/voice-response.ts @@ -0,0 +1,24 @@ +import * as v from "valibot"; + +export const BRUNCH_VOICE_TOOL_NAME = "brunch_set_voice_response"; +export const BRUNCH_VOICE_DATA_NAME = "brunch-voice-response"; + +const nonBlankString = v.pipe( + v.string(), + v.check((text) => /\S/u.test(text), "Expected non-blank text."), +); + +export const BrunchVoiceInputSchema = v.object({ speech: nonBlankString }); +export const BrunchVoiceDataSchema = v.object({ + speech: nonBlankString, + toolCallId: nonBlankString, +}); + +export type BrunchVoiceData = v.InferOutput; + +export const parseBrunchVoiceData = ( + data: unknown, +): BrunchVoiceData | undefined => { + const parsed = v.safeParse(BrunchVoiceDataSchema, data); + return parsed.success ? parsed.output : undefined; +}; diff --git a/libs/@hashintel/brunch-agent/packages/core/vite.config.ts b/libs/@hashintel/brunch-agent/packages/core/vite.config.ts index d3583e3c839..7c674c1c723 100644 --- a/libs/@hashintel/brunch-agent/packages/core/vite.config.ts +++ b/libs/@hashintel/brunch-agent/packages/core/vite.config.ts @@ -17,6 +17,9 @@ export default defineConfig({ new URL("src/question-marker.ts", import.meta.url), ), storage: fileURLToPath(new URL("src/storage.ts", import.meta.url)), + "voice-response": fileURLToPath( + new URL("src/voice-response.ts", import.meta.url), + ), workpiece: fileURLToPath(new URL("src/workpiece.ts", import.meta.url)), }, fileName: (_format, entryName) => `${entryName}.js`, diff --git a/libs/@hashintel/petrinaut/docs/ai-assistant.md b/libs/@hashintel/petrinaut/docs/ai-assistant.md index a7209260ae4..1955bea14ff 100644 --- a/libs/@hashintel/petrinaut/docs/ai-assistant.md +++ b/libs/@hashintel/petrinaut/docs/ai-assistant.md @@ -85,7 +85,7 @@ Every session control lives in the dock: **Collapse voice session** / **Expand v **Voice playback options** on the left, and the available handoff, microphone, recovery, and end actions on the right. **Read full response** becomes available after the matching response and speech have both finished -and replays every exact retained canonical segment in order. **Repeat question** uses the same +and reads the complete visible response, not just its spoken takeaway. **Repeat question** uses the same availability gates and replays only exact question text explicitly marked by Brunch. It stays disabled when that marker is missing or does not match finalized assistant text rather than guessing that the final segment is a question. @@ -96,7 +96,15 @@ after a failure. Nothing is added to the canvas toolbar. Sending non-empty typed composer or first-run prompt ends Voice mode before it sends the message once through the same conversation; repeated send actions are ignored while that short handoff completes. -The interviewer uses a warm, calm, curious, and professionally neutral voice and treats you as the authority on your system. Brunch still chooses every question and interview decision; OpenAI only transcribes your completed input and delivers Brunch's words. The question and finalized response shown in the Petrinaut conversation are authoritative. The speech request receives that exact Brunch text in part order; synthesized audio is generated from it but is not a verbatim recording. Interrupting audio does not undo the visible response or change the interview's saved history. +Brunch authors both the spoken answer or takeaway and the complete on-screen response. Voice playback +waits for the whole reply, including browser tools. Long reports stay on screen while a brief +substantive takeaway is spoken; use **Read full response** to hear the report. Read the +**Brunch-authored speech (not playback confirmation)** tool result to inspect the exact authored +speech. This saved text records what Brunch asked to say, not what you actually heard. If usable +speech is missing, you hear a reading notice instead of a generated summary. Reopening the +conversation does not automatically play saved speech. + +The interviewer uses a warm, calm, curious, and professionally neutral voice and treats you as the authority on your system. Brunch still chooses every question and interview decision; OpenAI only transcribes your completed input and delivers Brunch's words. Realtime receives the selected Brunch-authored text verbatim; synthesized audio is generated from it but is not a verbatim recording. Interrupting audio does not undo the visible response or change the interview's saved history. Closing the AI panel pauses microphone capture and active speech, then hides the dock until you reopen the panel. The same mounted session stays paused; choose **Resume voice mode** when you are