diff --git a/.changeset/flue-voice-safety.md b/.changeset/flue-voice-safety.md new file mode 100644 index 00000000000..4c6a8d48b4a --- /dev/null +++ b/.changeset/flue-voice-safety.md @@ -0,0 +1,8 @@ +--- +"@hashintel/petrinaut": patch +--- + +Add half-duplex Voice handoff, exact full-response replay, recoverable transcript notices, and multi-origin Voice attribution for client-tool results. +Voice transcripts now appear live with a dock-only collapse mode, while assistant and tool failures retain and display their complete details in persistent, copyable notifications and inline tool cards. +Voice selected from the first-run prompt now opens with its consent and microphone card above a compact setup dock, while Voice started from the composer retains the full panel. Collapse controls retain ghost-button styling in every state. +Ending Voice from the compact dock closes the AI panel instead of reopening its text composer. diff --git a/apps/brunch-agent/petrinaut-local.vite.config.ts b/apps/brunch-agent/petrinaut-local.vite.config.ts index 78036a815ae..ea37867c9c5 100644 --- a/apps/brunch-agent/petrinaut-local.vite.config.ts +++ b/apps/brunch-agent/petrinaut-local.vite.config.ts @@ -1,10 +1,10 @@ /** * Local FE-1436 panel launcher. * - * Loads the real hash Petrinaut website config, removes only the website's - * stock `/api/chat` dev handler, and proxies Brunch's mounted Flue route to - * the committed application server. The real panel, wrappers, and editor - * stay untouched; hash's tracked checkout stays clean. + * Loads the real hash Petrinaut website config, including its stock API + * handlers, and proxies Brunch's mounted Flue route to the committed + * application server. The real panel, wrappers, and editor stay untouched; + * hash's tracked checkout stays clean. */ import { join, resolve } from "node:path"; @@ -13,7 +13,7 @@ import { defineConfig, loadConfigFromFile, mergeConfig, - type PluginOption, + type UserConfig, } from "vite"; import { @@ -21,21 +21,20 @@ import { petrinautLocalServer, } from "./src/http/local-origins.ts"; -const withoutIncumbentChatHandler = ( - plugins: readonly PluginOption[], -): PluginOption[] => - plugins.filter((plugin) => { - if ( - plugin === false || - plugin === null || - plugin === undefined || - Array.isArray(plugin) || - typeof plugin !== "object" || - !("name" in plugin) - ) { - return true; - } - return plugin.name !== "petrinaut-api-dev"; +interface PetrinautPanelConfigOptions { + readonly chatOrigin: string; + readonly loadedConfig: UserConfig; + readonly root: string; +} + +export const mergePetrinautPanelConfig = ({ + chatOrigin, + loadedConfig, + root, +}: PetrinautPanelConfigOptions): UserConfig => + mergeConfig(loadedConfig, { + root, + server: petrinautLocalServer(chatOrigin), }); export default defineConfig(async (environment) => { @@ -58,14 +57,9 @@ export default defineConfig(async (environment) => { throw new Error(`Could not load Petrinaut's Vite config from ${root}.`); const chatOrigin = process.env.BRUNCH_CHAT_ORIGIN ?? defaultChatOrigin; - return mergeConfig( - { - ...loaded.config, - plugins: withoutIncumbentChatHandler(loaded.config.plugins ?? []), - }, - { - root, - server: petrinautLocalServer(chatOrigin), - }, - ); + return mergePetrinautPanelConfig({ + chatOrigin, + loadedConfig: loaded.config, + root, + }); }); diff --git a/apps/brunch-agent/test/local-dev-origins.test.ts b/apps/brunch-agent/test/local-dev-origins.test.ts index ddf0d734ec3..8a20c937cae 100644 --- a/apps/brunch-agent/test/local-dev-origins.test.ts +++ b/apps/brunch-agent/test/local-dev-origins.test.ts @@ -2,6 +2,7 @@ import { readFileSync } from "node:fs"; import { expect, test } from "vitest"; +import { mergePetrinautPanelConfig } from "../petrinaut-local.vite.config.ts"; import { defaultChatOrigin, localChatListen, @@ -21,7 +22,7 @@ test("one documented root command starts the Brunch server and Petrinaut panel", }; expect(rootPackage.scripts["dev:brunch"]).toBe( - "CARGO_TERM_PROGRESS_WHEN=never turbo run build --filter '@apps/petrinaut-website^...' && npm-run-all --parallel dev:brunch:server dev:brunch:panel", + "CARGO_TERM_PROGRESS_WHEN=never turbo run build --filter '@apps/brunch-agent^...' --filter '@apps/petrinaut-website^...' && npm-run-all --parallel dev:brunch:server dev:brunch:panel", ); expect(rootPackage.scripts["dev:brunch:server"]).toBe( "yarn workspace @apps/brunch-agent dev", @@ -64,3 +65,19 @@ test("petrinaut:dev proxies the mounted Flue conversation route", () => { 'VITE_BRUNCH_CHAT_ENDPOINT ??= "/agents/chat"', ); }); + +test("petrinaut:dev retains the website API handlers needed by Voice", () => { + const config = mergePetrinautPanelConfig({ + chatOrigin: defaultChatOrigin, + loadedConfig: { + plugins: [{ name: "petrinaut-api-dev" }], + }, + root: "/test/petrinaut-website", + }); + + expect(config.plugins).toEqual( + expect.arrayContaining([ + expect.objectContaining({ name: "petrinaut-api-dev" }), + ]), + ); +}); diff --git a/apps/brunch-agent/test/petrinaut-chat-result.ts b/apps/brunch-agent/test/petrinaut-chat-result.ts index c7cc142689c..b1a70cb413c 100644 --- a/apps/brunch-agent/test/petrinaut-chat-result.ts +++ b/apps/brunch-agent/test/petrinaut-chat-result.ts @@ -21,6 +21,10 @@ export interface PetrinautChatResult { readonly resumedStatus: number; readonly resumedText: string; readonly resumedFinish: UIMessageChunk | undefined; + readonly questionMarkerLive: unknown; + readonly questionMarkerHistory: unknown; + readonly questionToolVisibleLive: boolean; + readonly questionToolVisibleHistory: boolean; readonly historyUserEntryCount: number; readonly historyClientToolResultCount: number; readonly historyGetStatus: number; @@ -51,5 +55,7 @@ export interface PetrinautChatResult { export interface PetrinautResumeResult { readonly historyGetStatus: number; readonly historyUserText: string; + readonly questionMarkerHistory: unknown; + readonly questionToolVisibleHistory: boolean; readonly transcript: string; } diff --git a/apps/brunch-agent/test/petrinaut-chat.integration.ts b/apps/brunch-agent/test/petrinaut-chat.integration.ts index 20750ca68a4..253fc8ff4d3 100644 --- a/apps/brunch-agent/test/petrinaut-chat.integration.ts +++ b/apps/brunch-agent/test/petrinaut-chat.integration.ts @@ -18,6 +18,10 @@ import { snapshotToUiMessages, } from "@hashintel/brunch-agent-transport-aisdk"; import { ELICITATION_SKILL_NAME } from "@hashintel/brunch-agent/flue"; +import { + BRUNCH_QUESTION_DATA_NAME, + BRUNCH_QUESTION_TOOL_NAME, +} from "@hashintel/brunch-agent/question-marker"; import { PING_TOOL_NAME } from "../src/agents/chat-agent/tools/ping.ts"; import { applyCaptureSweep } from "../src/capture/apply-sweep.ts"; @@ -43,6 +47,7 @@ const ACTIVATE_SKILL_TOOL_NAME = "activate_skill"; const CHAT_MODEL_ID = "claude-haiku-4-5"; const RUNBOOK_SKILL_NAME = "sdcpn-modelling"; const READ_SKILL_RESOURCE_TOOL_NAME = "read_skill_resource"; +const question = "Which documentation page should we inspect next?"; const principalKey = "principal-mission-1"; const conversationId = "conversation-mission-1"; @@ -79,6 +84,35 @@ const userTextFromHistory = ( .map((part) => part.text) .join(""); +const questionMarkerFromHistory = ( + messages: ReturnType, +): unknown => { + const marker = messages + .flatMap((message) => message.parts) + .find( + (part) => + part.type === `data-${BRUNCH_QUESTION_DATA_NAME}` && "data" in part, + ); + return marker !== undefined && "data" in marker ? marker.data : undefined; +}; + +const questionMarkerFromChunks = ( + chunks: readonly UIMessageChunk[], +): unknown => { + const marker = chunks.find( + (chunk) => + chunk.type === `data-${BRUNCH_QUESTION_DATA_NAME}` && "data" in chunk, + ); + return marker !== undefined && "data" in marker ? marker.data : undefined; +}; + +const questionToolVisibleInHistory = ( + messages: ReturnType, +): boolean => + messages + .flatMap((message) => message.parts) + .some((part) => part.type === `tool-${BRUNCH_QUESTION_TOOL_NAME}`); + const faux = fauxProvider({ provider: "anthropic", models: [{ id: CHAT_MODEL_ID, reasoning: true }], @@ -98,19 +132,24 @@ try { const panelTransport = createFlueChatTransport({ client: historyClient, clientToolNames, + hiddenToolNames: new Set([BRUNCH_QUESTION_TOOL_NAME]), }); const projectHistory = ( snapshot: Awaited>, ) => snapshotToUiMessages(snapshot, { clientToolNames, + hiddenToolNames: new Set([BRUNCH_QUESTION_TOOL_NAME]), }); if (process.env.BRUNCH_RESUME_PHASE === "1") { const snapshot = await historyClient.history(); + const historyMessages = projectHistory(snapshot); const result: PetrinautResumeResult = { historyGetStatus: 200, - historyUserText: userTextFromHistory(projectHistory(snapshot)), + historyUserText: userTextFromHistory(historyMessages), + questionMarkerHistory: questionMarkerFromHistory(historyMessages), + questionToolVisibleHistory: questionToolVisibleInHistory(historyMessages), transcript: formatFlueTranscript(snapshot), }; process.stdout.write(`PETRINAUT_RESUME_RESULT ${JSON.stringify(result)}\n`); @@ -196,9 +235,19 @@ try { ], { stopReason: "toolUse" }, ), + fauxAssistantMessage( + [ + fauxToolCall( + BRUNCH_QUESTION_TOOL_NAME, + { question }, + { id: "tool-question-1" }, + ), + ], + { stopReason: "toolUse" }, + ), fauxAssistantMessage([ fauxText( - "The guide says the assistant can read its own documentation pages.", + `The guide says the assistant can read its own documentation pages. ${question}`, ), ]), fauxAssistantMessage([ @@ -400,6 +449,14 @@ try { .map((chunk) => chunk.delta) .join(""), resumedFinish: resumedChunks.at(-1), + questionMarkerLive: questionMarkerFromChunks(resumedChunks), + questionMarkerHistory: questionMarkerFromHistory(historyMessages), + questionToolVisibleLive: resumedChunks.some( + (chunk) => + chunk.type === "tool-input-available" && + chunk.toolName === BRUNCH_QUESTION_TOOL_NAME, + ), + questionToolVisibleHistory: questionToolVisibleInHistory(historyMessages), historyUserEntryCount: userEntryIds.length, historyClientToolResultCount: clientToolResultCount, historyGetStatus: 200, diff --git a/apps/brunch-agent/test/petrinaut-chat.test.ts b/apps/brunch-agent/test/petrinaut-chat.test.ts index daf97db27ea..2ee108a9ac0 100644 --- a/apps/brunch-agent/test/petrinaut-chat.test.ts +++ b/apps/brunch-agent/test/petrinaut-chat.test.ts @@ -72,6 +72,16 @@ test("the browser transport streams the mounted Flue agent through server and cl type: "finish", finishReason: "stop", }); + expect(result.questionMarkerLive).toEqual({ + question: "Which documentation page should we inspect next?", + toolCallId: "tool-question-1", + }); + expect(result.questionToolVisibleLive).toBe(false); + expect(result.questionMarkerHistory).toEqual({ + question: "Which documentation page should we inspect next?", + toolCallId: "tool-question-1", + }); + expect(result.questionToolVisibleHistory).toBe(false); expect(result.historyUserEntryCount).toBe(1); expect(result.historyClientToolResultCount).toBe(1); @@ -107,6 +117,7 @@ test("the browser transport streams the mounted Flue agent through server and cl expect(result.interviewerToolNames).toContain("read_skill_resource"); expect(result.interviewerToolNames).toContain("ping"); expect(result.interviewerToolNames).toContain("readPetrinautDoc"); + expect(result.interviewerToolNames).toContain("brunch_mark_question"); expect(result.interviewerToolNames).not.toContain("brunch_ask"); expect(result.interviewerToolNames).not.toContain("sweep"); expect(result.interviewerToolNames).not.toContain("brunch_sweep"); @@ -151,10 +162,16 @@ test("the browser transport streams the mounted Flue agent through server and cl expect(resumeResult.historyUserText).toContain( "Run the FE-1435 transport probe.", ); + expect(resumeResult.questionMarkerHistory).toEqual({ + question: "Which documentation page should we inspect next?", + toolCallId: "tool-question-1", + }); + expect(resumeResult.questionToolVisibleHistory).toBe(false); expect(resumeResult.transcript).toContain("tool ping"); expect(resumeResult.transcript).toContain("tool readPetrinautDoc"); expect(resumeResult.transcript).toContain("tool activate_skill"); expect(resumeResult.transcript).toContain("tool read_skill_resource"); + expect(resumeResult.transcript).toContain("tool brunch_mark_question"); } finally { await rm(dbDirectory, { recursive: true, force: true }); } diff --git a/apps/petrinaut-website/README.md b/apps/petrinaut-website/README.md index 8e31217aa9e..1f6919f3a32 100644 --- a/apps/petrinaut-website/README.md +++ b/apps/petrinaut-website/README.md @@ -89,7 +89,20 @@ disclosure before requesting microphone access. The disclosure also provides a microphone check and is remembered in browser storage only after Voice mode starts. -When Brunch is selected, typed and finalized spoken turns both enter the same mounted Flue conversation route. **Stop** requests a durable Brunch abort before the panel cancels its local response stream. Closing or speaking over Voice playback only stops local media; it does not alter canonical conversation history. Reopening the same net restores its observed Flue conversation without resubmitting a turn or replaying settled audio. +When Brunch is selected, typed turns and completed Voice transcripts both enter +the same mounted Flue conversation route. Each logical turn carries a stable +delivery key so a replayed request converges on the existing admission instead +of creating another turn. If admission cannot be confirmed, the UI reports the +ambiguity and does not retry automatically. **Stop** requests a durable Brunch +abort before the panel cancels its local response stream. Local playback +cancellation remains separate and does not alter canonical history. Canonical +Flue history is the source used when the same net is reopened. Automated +coverage guards a locally submitted turn from an older hydration snapshot and +does not resubmit turns or replay settled audio. The real hard-reload witness is +still pending, so reload parity is not yet claimed for this preview. +Voice-origin client-tool results retain their markers in Flue history. Direct +spoken user turns remain canonical text, but Flue 2.0.3 does not yet expose the +caller delivery metadata needed to restore their Voice chip after reopening. An active session stays at the end of the transcript. Its compact divider shows a waveform and **Connecting**, **Listening**, **Speaking**, **Paused**, or a @@ -103,24 +116,43 @@ The text composer remains available. Sending typed text ends Voice mode first, then submits the draft exactly once through the same conversation; a failed handoff restores the draft. Closing the assistant pauses capture and speech before hiding it. Reopening preserves the mounted session in **Paused** state. -**Pause** and **End voice mode** live under **Voice mode actions**, while -**Resume** or **Reconnect** appears as the primary action when applicable. +The dock exposes **Your turn** while canonical audio owns the turn. That action +clears pending input and output, waits for the provider's matching +acknowledgements and response terminal event, and only then opens the +microphone for fresh capture. Its playback menu offers **Repeat question** and +**Read full response**. Full-response replay becomes available once the matching +response and audio output have both finished, enqueues all exact retained +canonical segments in order, and is disabled during capture, submission, +cancellation, pause, and errors. **Repeat question** has the same safety gates +and replays only exact question text carrying Brunch's non-interactive marker; +if the marker is missing, malformed, or does not match finalized prose, the +action stays disabled rather than guessing from the final segment. The browser sends its SDP offer to this app; the server initializes a trusted `gpt-realtime-2` audio-input/audio-output session through OpenAI's unified -Realtime call endpoint. The provider key, model, instructions, tools, language, -and vocabulary policy stay server-side. The session uses semantic VAD with low -eagerness so natural thinking pauses are less likely to end an answer early. - -Realtime is the disposable media plane: it carries continuous microphone and remote audio, detects complete turns, and handles barge-in. Brunch remains the control plane and sole authority for questions, captures, state, completion, and durable history. The browser bridge accepts only the configured `continue_interview` function, validates and serializes its arguments, rejects duplicate or stale calls, and submits the answer through Petrinaut's shared composer path with pending-question correlation. +Realtime call endpoint. The provider key, model, instructions, language, and +vocabulary policy stay server-side. Realtime exposes no tools, uses +`tool_choice: "none"`, and configures semantic VAD to detect an input boundary +without creating a model response. + +Realtime is the disposable media plane: it carries microphone and remote audio, +detects complete turns, and transcribes input. Brunch remains the control plane +and sole authority for questions, captures, state, completion, and durable +history. The bridge accepts only +`conversation.item.input_audio_transcription.completed` as an answer, ignores +model function arguments, and submits the normalized transcript through +Petrinaut's shared composer path. Connection epoch, item id, and content index +form its stable identity. Duplicate, empty, failed, unavailable, and over-limit +transcripts never submit; recoverable failures leave a not-heard or too-long +notice in the dock. Provisional transcription remains display-only. The bridge waits for the correlated Brunch turn before returning canonical -speech segments to Realtime. It then requests audio with tools disabled and -instructs Realtime to speak only those segments. Generated audio is not a -verbatim record: canonical Brunch text remains visible and authoritative. The -microphone stays active while the interviewer speaks and while Brunch is -working. Speaking over assistant audio interrupts playback automatically; -WebRTC truncates provider-side unheard audio without changing Brunch history. +speech segments to Realtime. It instructs Realtime to speak only those +segments. Generated audio is not a verbatim recording: canonical Brunch text +remains visible and authoritative. Voice is half-duplex: the physical +microphone is closed while the interviewer speaks, while Brunch is working, and +through cancellation. Audio captured before a **Your turn** handoff is +discarded and cannot become a later answer. The local Brunch preview reaches the mounted route through its same-origin, protocol-preserving proxy; this does not establish remote authentication or public ingress. Denying microphone permission leaves the text composer available and submits nothing to Brunch. When Voice mode cannot continue, the inline recovery state distinguishes microphone, connection, and other Voice failures, explains the next action, and offers **Reconnect** where appropriate. Sanitized error codes and diagnostic references remain collapsed under **Technical details**. diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-panel-transport.test.ts b/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-panel-transport.test.ts index 6df757d8614..d616fa9fe5b 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-panel-transport.test.ts +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-panel-transport.test.ts @@ -1,3 +1,4 @@ +import { FlueApiError } from "@flue/sdk"; import { expect, test, vi } from "vitest"; import { @@ -24,12 +25,18 @@ test("delegates one typed message to the supplied Flue conversation", async () = turnId: "turn-1", position: { batch: 1, index: 0 }, }); + await options?.onEvent?.({ + type: "message-completed", + conversationId: "conversation-stable", + messageId: "assistant-1", + position: { batch: 1, index: 1 }, + }); await options?.onEvent?.({ type: "submission-settled", conversationId: "conversation-stable", submissionId: admission.submissionId, outcome: "completed", - position: { batch: 1, index: 1 }, + position: { batch: 1, index: 2 }, }); }); const client = { @@ -42,6 +49,10 @@ test("delegates one typed message to the supplied Flue conversation", async () = { kind: "user", messageId: "user-1" }, admissionListener, ); + const responseCompletedListener = vi.fn(); + tracker.subscribeToResponseMessageCompleted(responseCompletedListener); + const responseStartedListener = vi.fn(); + tracker.subscribeToResponseMessageStarted(responseStartedListener); const onAdmission = vi.fn(); const transport = createBrunchPanelTransport( Promise.resolve(client), @@ -71,6 +82,7 @@ test("delegates one typed message to the supplied Flue conversation", async () = expect(send).toHaveBeenCalledOnce(); expect(send).toHaveBeenCalledWith({ + idempotencyKey: "ai-sdk:user-1", message: { kind: "user", body: "Typed tracer." }, signal: undefined, }); @@ -78,6 +90,18 @@ test("delegates one typed message to the supplied Flue conversation", async () = expect(tracker.submissionsForResponse("assistant-1")).toEqual([ "submission-1", ]); + expect(responseStartedListener).toHaveBeenCalledOnce(); + expect(responseStartedListener).toHaveBeenCalledWith({ + messageId: "assistant-1", + position: { batch: 1, index: 0 }, + submissionId: "submission-1", + }); + expect(responseCompletedListener).toHaveBeenCalledOnce(); + expect(responseCompletedListener).toHaveBeenCalledWith({ + messageId: "assistant-1", + position: { batch: 1, index: 1 }, + submissionId: "submission-1", + }); expect(onAdmission).toHaveBeenCalledOnce(); expect(onAdmission).toHaveBeenCalledWith(admission); }); @@ -139,15 +163,66 @@ test("matches client-tool admissions once and supports unsubscribe", () => { test("records every submission that wrote a resumed assistant message", () => { const tracker = new BrunchPanelConversationTracker(); - tracker.recordResponse("assistant-1", "submission-1"); - tracker.recordResponse("assistant-1", "submission-continuation"); - tracker.recordResponse("assistant-1", "submission-continuation"); + const responseStartedListener = vi.fn(); + tracker.subscribeToResponseMessageStarted(responseStartedListener); + tracker.recordResponse({ + messageId: "assistant-1", + position: { batch: 1, index: 0 }, + submissionId: "submission-1", + }); + tracker.recordResponse({ + messageId: "assistant-1", + position: { batch: 2, index: 0 }, + submissionId: "submission-continuation", + }); + tracker.recordResponse({ + messageId: "assistant-1", + position: { batch: 2, index: 0 }, + submissionId: "submission-continuation", + }); expect(tracker.submissionsForResponse("assistant-1")).toEqual([ "submission-1", "submission-continuation", ]); expect(tracker.submissionsForResponse("assistant-2")).toBeUndefined(); + expect(responseStartedListener.mock.calls).toEqual([ + [ + { + messageId: "assistant-1", + position: { batch: 1, index: 0 }, + submissionId: "submission-1", + }, + ], + [ + { + messageId: "assistant-1", + position: { batch: 2, index: 0 }, + submissionId: "submission-continuation", + }, + ], + [ + { + messageId: "assistant-1", + position: { batch: 2, index: 0 }, + submissionId: "submission-continuation", + }, + ], + ]); +}); + +test("publishes Stop immediately and supports unsubscribe", () => { + const tracker = new BrunchPanelConversationTracker(); + const listener = vi.fn(); + const unsubscribedListener = vi.fn(); + tracker.subscribeToStopRequested(listener); + const unsubscribe = tracker.subscribeToStopRequested(unsubscribedListener); + unsubscribe(); + + tracker.recordStopRequested(); + + expect(listener).toHaveBeenCalledOnce(); + expect(unsubscribedListener).not.toHaveBeenCalled(); }); test("settles in-flight submissions before a durable abort can target them", async () => { @@ -175,3 +250,46 @@ test("settles in-flight submissions before a durable abort can target them", asy await expect(rejected).rejects.toThrow("rejected admission"); await expect(tracker.settleInFlightSubmissions()).resolves.toBeUndefined(); }); + +test("publishes a typed admission failure for the exact panel input", async () => { + const send = vi.fn(async () => { + throw new FlueApiError(500, ""); + }); + const tracker = new BrunchPanelConversationTracker(); + const failureListener = vi.fn(); + tracker.subscribeToAdmissionFailure( + { kind: "user", messageId: "voice-realtime:1:item-1:0" }, + failureListener, + ); + const transport = createBrunchPanelTransport( + Promise.resolve({ send } as Pick as FlueClient), + tracker, + ); + + const submission = transport.sendMessages({ + trigger: "submit-message", + chatId: "conversation-stable", + messageId: undefined, + messages: [ + { + id: "voice-realtime:1:item-1:0", + role: "user", + parts: [{ type: "text", text: "One Voice turn." }], + }, + ], + abortSignal: undefined, + }); + + await expect(submission).rejects.toMatchObject({ + failure: { kind: "ambiguous" }, + name: "FlueChatAdmissionError", + }); + expect(failureListener).toHaveBeenCalledOnce(); + expect(failureListener).toHaveBeenCalledWith( + expect.objectContaining({ + failure: { kind: "ambiguous" }, + name: "FlueChatAdmissionError", + }), + ); + expect(send).toHaveBeenCalledOnce(); +}); diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-panel-transport.ts b/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-panel-transport.ts index 973075debb3..b43c667f6c6 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-panel-transport.ts +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-panel-transport.ts @@ -1,8 +1,12 @@ -import { createFlueChatTransport } from "@hashintel/brunch-agent-transport-aisdk"; +import { + createFlueChatTransport, + FlueChatAdmissionError, +} from "@hashintel/brunch-agent-transport-aisdk"; import { SWEEP_TOOL_NAME } from "@hashintel/brunch-agent/client-tools"; +import { BRUNCH_QUESTION_TOOL_NAME } from "@hashintel/brunch-agent/question-marker"; +import { readPetrinautDocToolName } from "@hashintel/petrinaut-core"; import { sweepOutputSchema } from "../brunch-sweep-output"; -import { brunchClientToolNames } from "./brunch-client-tools"; import type { SweepCapture, @@ -10,7 +14,11 @@ import type { SweepCompletionReport, } from "../brunch-sweep-output"; import type { AgentSendResult, FlueClient } from "@flue/sdk"; -import type { FlueChatTransportOptions } from "@hashintel/brunch-agent-transport-aisdk"; +import type { + FlueChatResponseMessageCompletedEvent, + FlueChatResponseMessageStartedEvent, + FlueChatTransportOptions, +} from "@hashintel/brunch-agent-transport-aisdk"; import type { PetrinautAiChatTransport } from "@hashintel/petrinaut/ui"; import type { UIMessageChunk } from "ai"; @@ -23,6 +31,10 @@ export type BrunchPanelAdmissionTarget = Pick< >; export class BrunchPanelConversationTracker { + readonly #admissionFailureSubscriptions = new Set<{ + readonly listener: (error: FlueChatAdmissionError) => void; + readonly target: BrunchPanelAdmissionTarget; + }>(); readonly #admissionSubscriptions = new Set<{ readonly listener: (admission: BrunchPanelAdmission) => void; readonly target: BrunchPanelAdmissionTarget; @@ -36,6 +48,13 @@ export class BrunchPanelConversationTracker { string, AgentSendResult["submissionId"][] >(); + readonly #responseMessageStartedListeners = new Set< + (event: FlueChatResponseMessageStartedEvent) => void + >(); + readonly #responseMessageCompletedListeners = new Set< + (event: FlueChatResponseMessageCompletedEvent) => void + >(); + readonly #stopRequestedListeners = new Set<() => void>(); public recordAdmission(admission: BrunchPanelAdmission): void { if (admission.kind === "user") { @@ -61,15 +80,29 @@ export class BrunchPanelConversationTracker { * all: Voice correlates a reply by membership, whichever side admitted the * continuation. */ - public recordResponse( - messageId: string, - submissionId: AgentSendResult["submissionId"], - ): void { - const recorded = this.#responseSubmissions.get(messageId); + public recordResponse(event: FlueChatResponseMessageStartedEvent): void { + const recorded = this.#responseSubmissions.get(event.messageId); if (recorded === undefined) { - this.#responseSubmissions.set(messageId, [submissionId]); - } else if (!recorded.includes(submissionId)) { - recorded.push(submissionId); + this.#responseSubmissions.set(event.messageId, [event.submissionId]); + } else if (!recorded.includes(event.submissionId)) { + recorded.push(event.submissionId); + } + for (const listener of this.#responseMessageStartedListeners) { + listener(event); + } + } + + public recordResponseMessageCompleted( + event: FlueChatResponseMessageCompletedEvent, + ): void { + for (const listener of this.#responseMessageCompletedListeners) { + listener(event); + } + } + + public recordStopRequested(): void { + for (const listener of this.#stopRequestedListeners) { + listener(); } } @@ -91,6 +124,21 @@ export class BrunchPanelConversationTracker { return submission; } + public recordAdmissionFailure( + target: BrunchPanelAdmissionTarget, + error: FlueChatAdmissionError, + ): void { + for (const subscription of this.#admissionFailureSubscriptions) { + if ( + subscription.target.kind === target.kind && + subscription.target.messageId === target.messageId + ) { + this.#admissionFailureSubscriptions.delete(subscription); + subscription.listener(error); + } + } + } + public submissionForInput( messageId: string, ): AgentSendResult["submissionId"] | undefined { @@ -111,6 +159,34 @@ export class BrunchPanelConversationTracker { this.#admissionSubscriptions.add(subscription); return () => this.#admissionSubscriptions.delete(subscription); } + + public subscribeToAdmissionFailure( + target: BrunchPanelAdmissionTarget, + listener: (error: FlueChatAdmissionError) => void, + ): () => void { + const subscription = { listener, target }; + this.#admissionFailureSubscriptions.add(subscription); + return () => this.#admissionFailureSubscriptions.delete(subscription); + } + + public subscribeToResponseMessageCompleted( + listener: (event: FlueChatResponseMessageCompletedEvent) => void, + ): () => void { + this.#responseMessageCompletedListeners.add(listener); + return () => this.#responseMessageCompletedListeners.delete(listener); + } + + public subscribeToResponseMessageStarted( + listener: (event: FlueChatResponseMessageStartedEvent) => void, + ): () => void { + this.#responseMessageStartedListeners.add(listener); + return () => this.#responseMessageStartedListeners.delete(listener); + } + + public subscribeToStopRequested(listener: () => void): () => void { + this.#stopRequestedListeners.add(listener); + return () => this.#stopRequestedListeners.delete(listener); + } } const formatFailure = (failure: SweepCompletionFailure): string => { @@ -244,15 +320,40 @@ export const createBrunchPanelTransport = ( const client = await clientPromise; const transport = createFlueChatTransport({ client, - clientToolNames: brunchClientToolNames, + clientToolNames: new Set([readPetrinautDocToolName]), + hiddenToolNames: new Set([BRUNCH_QUESTION_TOOL_NAME]), onAdmission: (event) => { tracker.recordAdmission(event); hooks?.onAdmission?.(event.admission); }, - onResponseMessage: ({ messageId, submissionId }) => - tracker.recordResponse(messageId, submissionId), + onResponseMessage: (event) => tracker.recordResponse(event), + onResponseMessageCompleted: (event) => + tracker.recordResponseMessageCompleted(event), }); - return decorateBrunchStream(await transport.sendMessages(sendOptions)); + try { + return decorateBrunchStream( + await transport.sendMessages(sendOptions), + ); + } catch (error) { + const messageId = + sendOptions.messageId ?? sendOptions.messages.at(-1)?.id; + if ( + error instanceof FlueChatAdmissionError && + messageId !== undefined + ) { + tracker.recordAdmissionFailure( + { + kind: + sendOptions.messageId === undefined + ? "user" + : "client-tool-result", + messageId, + }, + error, + ); + } + throw error; + } })(), ), }); diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx index 4744abfe074..7edbbd16255 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx @@ -1,28 +1,45 @@ /** * @vitest-environment jsdom */ -import { act, cleanup, render } from "@testing-library/react"; +import { act, cleanup, render, waitFor } from "@testing-library/react"; import { isValidElement, type ReactNode } from "react"; import { afterEach, describe, expect, test, vi } from "vitest"; +import { FlueChatAdmissionError } from "@hashintel/brunch-agent-transport-aisdk"; import { defaultPetrinautNavigationHistoryPolicy } from "@hashintel/petrinaut/react"; +import { OpenAIRealtimeSession } from "../voice-interview/openai-realtime-session"; import { VoiceInterviewControl } from "../voice-interview/voice-interview-control"; -import { brunchClientToolNames } from "./brunch-client-tools"; import { BrunchPanelConversationTracker } from "./brunch-panel-transport"; import { - brunchInteractiveTools, getBrunchVoiceMode, LocalStorageDemoApp, requestFlueStop, } from "./local-storage-demo-app"; -import type { FlueClient } from "@flue/sdk"; +import type { + AgentConversationObservationSnapshot, + FlueClient, +} from "@flue/sdk"; import type { PetrinautNavigationController } from "@hashintel/petrinaut/react"; +import type { PetrinautAiAssistant } from "@hashintel/petrinaut/ui"; const defaultTransportOptions = vi.hoisted(() => ({ current: null as unknown, })); +const flueClientMock = vi.hoisted(() => ({ current: null as unknown })); +const renderedPetrinaut = vi.hoisted(() => ({ aiAssistant: null as unknown })); + +vi.mock("@flue/sdk", () => ({ + createFlueClient: () => flueClientMock.current, +})); + +vi.mock("./brunch-preview-config", () => ({ + resolveBrunchPreviewConfig: () => ({ + chatEndpoint: "/agents/chat", + isBrunchConfigured: true, + }), +})); const editorProps = vi.hoisted(() => ({ current: null as { @@ -46,6 +63,7 @@ vi.mock("@hashintel/petrinaut/ui", () => ({ }, Petrinaut: (props: Record) => { editorProps.current = props; + renderedPetrinaut.aiAssistant = props.aiAssistant; return null; }, WalkthroughProvider: ({ children }: { children: ReactNode }) => children, @@ -59,7 +77,8 @@ describe("local storage demo Brunch voice integration", () => { test("installs the app-owned voice control for a configured Brunch transport", () => { const config = { available: true as const, connectionTimeoutMs: 15_000 }; - const voiceMode = getBrunchVoiceMode(config); + const tracker = new BrunchPanelConversationTracker(); + const voiceMode = getBrunchVoiceMode(config, tracker); const control = voiceMode?.({ canAcceptVoiceInput: true, conversationId: "petrinaut-preview:net-1", @@ -86,17 +105,172 @@ describe("local storage demo Brunch voice integration", () => { if (!isValidElement(control)) { throw new Error("Expected the configured composer control to render."); } - expect(control).toMatchObject({ - props: { config }, - type: VoiceInterviewControl, + const failureListener = vi.fn(); + const responseCompletedListener = vi.fn(); + const responseStartedListener = vi.fn(); + const stopListener = vi.fn(); + const target = { kind: "user" as const, messageId: "voice-turn-1" }; + const controlProps = control.props as { + config: typeof config; + subscribeToAdmissionFailure: ( + admissionTarget: typeof target, + listener: (error: FlueChatAdmissionError) => void, + ) => () => void; + subscribeToResponseMessageCompleted: ( + listener: typeof responseCompletedListener, + ) => () => void; + subscribeToResponseMessageStarted: ( + listener: typeof responseStartedListener, + ) => () => void; + subscribeToStopRequested: (listener: () => void) => () => void; + }; + expect(control.type).toBe(VoiceInterviewControl); + expect(controlProps.config).toBe(config); + const unsubscribe = controlProps.subscribeToAdmissionFailure( + target, + failureListener, + ); + const unsubscribeFromStop = + controlProps.subscribeToStopRequested(stopListener); + const unsubscribeFromResponseCompleted = + controlProps.subscribeToResponseMessageCompleted( + responseCompletedListener, + ); + const unsubscribeFromResponseStarted = + controlProps.subscribeToResponseMessageStarted(responseStartedListener); + const admissionError = new FlueChatAdmissionError({ kind: "ambiguous" }); + + tracker.recordAdmissionFailure(target, admissionError); + tracker.recordResponse({ + messageId: "assistant-1", + position: { batch: 1, index: 0 }, + submissionId: "submission-1", }); + tracker.recordResponseMessageCompleted({ + messageId: "assistant-1", + position: { batch: 1, index: 1 }, + submissionId: "submission-1", + }); + tracker.recordStopRequested(); + + expect(failureListener).toHaveBeenCalledWith(admissionError); + expect(responseStartedListener).toHaveBeenCalledOnce(); + expect(responseCompletedListener).toHaveBeenCalledOnce(); + expect(stopListener).toHaveBeenCalledOnce(); + unsubscribe(); + unsubscribeFromResponseCompleted(); + unsubscribeFromResponseStarted(); + unsubscribeFromStop(); }); - test("registers only interactive widgets that answer declared client tools", () => { - expect(brunchInteractiveTools.length).toBeGreaterThan(0); - for (const tool of brunchInteractiveTools) { - expect(brunchClientToolNames.has(tool.toolName)).toBe(true); - } + test("registers no brunch_ask tool in the production Brunch preview", async () => { + renderedPetrinaut.aiAssistant = null; + flueClientMock.current = { + observe: () => ({ + close: vi.fn(), + getSnapshot: () => ({ phase: "absent" }), + refresh: vi.fn(), + subscribe: () => () => undefined, + }), + }; + vi.stubGlobal( + "fetch", + vi.fn(async () => + Response.json({ available: false }), + ), + ); + + const rendered = render( + {}} search={{}} />, + ); + await waitFor(() => expect(renderedPetrinaut.aiAssistant).not.toBeNull()); + const aiAssistant = renderedPetrinaut.aiAssistant as PetrinautAiAssistant; + + expect(aiAssistant.requestStop).toBeTypeOf("function"); + expect(aiAssistant.interactiveTools).toEqual([]); + expect( + aiAssistant.interactiveTools?.some( + ({ toolName }) => toolName === "brunch_ask", + ), + ).toBe(false); + + rendered.unmount(); + vi.unstubAllGlobals(); + }); + + test("keeps durable Flue Stop distinct from local playback cancellation", async () => { + renderedPetrinaut.aiAssistant = null; + let snapshot: AgentConversationObservationSnapshot = { + conversation: { + conversationId: "conversation-stop", + settlements: [], + messages: [], + }, + offset: "offset-before-stop", + phase: "live" as const, + error: undefined, + }; + const listeners = new Set<() => void>(); + const localPlaybackCancellation = vi.spyOn( + OpenAIRealtimeSession.prototype, + "cancelOutput", + ); + const abort = vi.fn(async () => { + snapshot = { + conversation: { + conversationId: "conversation-stop", + settlements: [ + { submissionId: "submission-stop", outcome: "aborted" as const }, + ], + messages: [], + }, + offset: "offset-after-stop", + phase: "live" as const, + error: undefined, + }; + for (const listener of listeners) listener(); + return { aborted: true }; + }); + flueClientMock.current = { + abort, + observe: () => ({ + close: vi.fn(), + getSnapshot: () => snapshot, + refresh: vi.fn(), + subscribe: (listener: () => void) => { + listeners.add(listener); + return () => listeners.delete(listener); + }, + }), + }; + vi.stubGlobal( + "fetch", + vi.fn(async () => + Response.json({ available: false }), + ), + ); + + const rendered = render( + {}} search={{}} />, + ); + await waitFor(() => + expect( + (renderedPetrinaut.aiAssistant as PetrinautAiAssistant).requestStop, + ).toBeTypeOf("function"), + ); + const aiAssistant = renderedPetrinaut.aiAssistant as PetrinautAiAssistant; + + await expect(aiAssistant.requestStop?.()).resolves.toBe("stop-requested"); + expect(abort).toHaveBeenCalledOnce(); + expect(localPlaybackCancellation).not.toHaveBeenCalled(); + expect( + (renderedPetrinaut.aiAssistant as PetrinautAiAssistant) + .renderComposerControl, + ).toBeUndefined(); + + rendered.unmount(); + localPlaybackCancellation.mockRestore(); + vi.unstubAllGlobals(); }); test("correlates the existing Brunch transport request", () => { @@ -132,6 +306,8 @@ describe("local storage demo Brunch voice integration", () => { const abort = vi.fn(async () => ({ aborted: true })); const client = { abort } as Pick as FlueClient; const tracker = new BrunchPanelConversationTracker(); + const stopListener = vi.fn(); + tracker.subscribeToStopRequested(stopListener); let admit: (() => void) | undefined; void tracker.trackSubmission( new Promise((resolve) => { @@ -140,6 +316,7 @@ describe("local storage demo Brunch voice integration", () => { ); const stop = requestFlueStop(Promise.resolve(client), tracker); + expect(stopListener).toHaveBeenCalledOnce(); await Promise.resolve(); await Promise.resolve(); expect(abort).not.toHaveBeenCalled(); diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx index 39e4b466fa3..6e666ee7a70 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx @@ -25,7 +25,6 @@ import { import { DefaultChatTransport, Petrinaut, - type PetrinautAiInteractiveTool, type PetrinautAiMessage, type PetrinautAiStopResult, type PetrinautAiVoiceMode, @@ -45,7 +44,6 @@ import { type OpenAIVoiceConfig, VoiceInterviewControl, } from "../voice-interview/voice-interview-control"; -import { brunchAskInteractiveTool } from "./brunch-ask-interactive-tool"; import { getOrCreateBrunchConversationId } from "./brunch-conversation-id"; import { BrunchPanelConversationTracker, @@ -132,6 +130,23 @@ export const getBrunchVoiceMode = ( resolveResponseSubmission={(messageId) => tracker?.submissionsForResponse(messageId) } + subscribeToResponseMessageCompleted={ + tracker === undefined + ? undefined + : (listener) => + tracker.subscribeToResponseMessageCompleted(listener) + } + subscribeToResponseMessageStarted={ + tracker === undefined + ? undefined + : (listener) => + tracker.subscribeToResponseMessageStarted(listener) + } + subscribeToStopRequested={ + tracker === undefined + ? undefined + : (listener) => tracker.subscribeToStopRequested(listener) + } subscribeToAdmission={ tracker === undefined ? undefined @@ -140,6 +155,12 @@ export const getBrunchVoiceMode = ( listener(admission.submissionId), ) } + subscribeToAdmissionFailure={ + tracker === undefined + ? undefined + : (target, listener) => + tracker.subscribeToAdmissionFailure(target, listener) + } /> ) : undefined; @@ -153,11 +174,6 @@ const createHandle = (net: SDCPNInLocalStorage): PetrinautDocHandle => const brunchPrincipal = getOrCreateBrunchPrincipal(); -/** Every widget here must answer a tool named in `brunchClientToolNames`. */ -export const brunchInteractiveTools: readonly PetrinautAiInteractiveTool[] = [ - brunchAskInteractiveTool, -]; - const stockChatTransport = new DefaultChatTransport({ api: brunchPreviewConfig.chatEndpoint, headers: () => ({ @@ -189,6 +205,7 @@ export const requestFlueStop = async ( clientPromise: Promise>, tracker: BrunchPanelConversationTracker, ): Promise => { + tracker.recordStopRequested(); const client = await clientPromise; await tracker.settleInFlightSubmissions(); const result = await client.abort(); @@ -222,64 +239,6 @@ const createActiveHandle = (net: SDCPNInLocalStorage): ActiveHandle => ({ fallbackNet: net, }); -type FlueChatHistory = ReturnType; - -const errorStatus = (error: Error | undefined): number | undefined => { - if ( - error !== undefined && - "status" in error && - typeof error.status === "number" - ) { - return error.status; - } - return undefined; -}; - -const BrunchConversationStatus = ({ - error, - latestSettlement, - phase, - refresh, -}: Pick< - FlueChatHistory, - "error" | "latestSettlement" | "phase" | "refresh" ->) => { - if (phase === undefined) return null; - - const label = - phase === "loading" - ? "Loading Brunch conversation…" - : phase === "connecting" - ? "Reconnecting to Brunch…" - : phase === "absent" - ? "New Brunch conversation" - : phase === "error" - ? errorStatus(error) === 401 || errorStatus(error) === 403 - ? "Brunch access was denied." - : "Brunch conversation unavailable." - : phase === "closed" - ? "Brunch conversation closed." - : latestSettlement?.outcome === "aborted" - ? "Last Brunch response stopped." - : latestSettlement?.outcome === "failed" - ? "Last Brunch response failed." - : "Brunch conversation ready."; - - return ( - - {label} - {phase === "error" && ( - <> - {" "} - - - )} - - ); -}; - /** * The demo's own palette command, registered beside Petrinaut's: picking it * in the palette starts a fresh net. @@ -549,7 +508,7 @@ export const LocalStorageDemoApp = ({ () => ({ ...(conversationId === null ? {} : { conversationId }), canClearMessages: flueClientPromise === null, - interactiveTools: brunchInteractiveTools, + interactiveTools: [], transport: petrinautAiChatTransport, ...(flueClientPromise === null ? {} @@ -557,18 +516,6 @@ export const LocalStorageDemoApp = ({ requestStop: () => requestFlueStop(flueClientPromise, conversationTracker), }), - ...(flueClientPromise === null - ? {} - : { - renderComposerControl: () => ( - - ), - }), messages: flueClientPromise === null ? currentNetId @@ -609,11 +556,7 @@ export const LocalStorageDemoApp = ({ conversationId, currentNetId, flueClientPromise, - flueHistory.error, - flueHistory.latestSettlement, flueHistory.messages, - flueHistory.phase, - flueHistory.refresh, petrinautAiChatTransport, setAiMessagesByNetId, ], diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/use-flue-chat-history.test.ts b/apps/petrinaut-website/src/main/app/local-storage-demo/use-flue-chat-history.test.ts index f21c7a30033..c29ad8a16d9 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/use-flue-chat-history.test.ts +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/use-flue-chat-history.test.ts @@ -104,6 +104,90 @@ test("exposes the canonical settlement index for Voice correlation", async () => ]); }); +test("preserves every persisted Voice tool origin across hydration and reopen", async () => { + const harness = createObservationHarness({ + conversation: { + conversationId: "conversation-1", + settlements: [], + messages: [ + { + id: "assistant-voice-tools", + role: "assistant", + purpose: "assistant", + display: "visible", + parts: [ + { + type: "dynamic-tool", + toolCallId: "tool-doc-1", + toolName: "readPetrinautDoc", + state: "output-available", + input: { doc: "ai-assistant" }, + output: { awaiting: "client" }, + }, + { + type: "dynamic-tool", + toolCallId: "tool-doc-2", + toolName: "readPetrinautDoc", + state: "output-available", + input: { doc: "ai-assistant" }, + output: { awaiting: "client" }, + }, + ], + }, + { + id: "signal-voice-results", + role: "system", + purpose: "dispatch", + display: "hidden", + signal: { tagName: "client-tool-result" }, + parts: [ + { + type: "text", + text: JSON.stringify([ + { + toolCallId: "tool-doc-1", + toolName: "readPetrinautDoc", + output: "First guide", + source: "voice", + }, + { + toolCallId: "tool-doc-2", + toolName: "readPetrinautDoc", + output: "Second guide", + source: "voice", + }, + ]), + state: "done", + }, + ], + }, + ], + }, + offset: "offset-voice", + phase: "live", + error: undefined, + }); + const firstOpen = renderHook(() => + useFlueChatHistory(harness.clientPromise, "conversation-1"), + ); + + await waitFor(() => expect(firstOpen.result.current.ready).toBe(true)); + expect(firstOpen.result.current.messages?.[0]?.metadata).toEqual({ + source: "voice", + voiceToolCallIds: ["tool-doc-1", "tool-doc-2"], + }); + firstOpen.unmount(); + + const reopened = renderHook(() => + useFlueChatHistory(harness.clientPromise, "conversation-1"), + ); + await waitFor(() => expect(reopened.result.current.ready).toBe(true)); + expect(reopened.result.current.messages?.[0]?.metadata).toEqual({ + source: "voice", + voiceToolCallIds: ["tool-doc-1", "tool-doc-2"], + }); +}); + test("asks nothing of the generic chat route, which keeps no history", () => { const { result } = renderHook(() => useFlueChatHistory(null, "conversation-1"), diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/use-flue-chat-history.ts b/apps/petrinaut-website/src/main/app/local-storage-demo/use-flue-chat-history.ts index 7e3bf752055..65b914a5e3d 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/use-flue-chat-history.ts +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/use-flue-chat-history.ts @@ -1,8 +1,8 @@ import { useCallback, useEffect, useRef, useState } from "react"; import { snapshotToUiMessages } from "@hashintel/brunch-agent-transport-aisdk"; - -import { brunchClientToolNames } from "./brunch-client-tools"; +import { BRUNCH_QUESTION_TOOL_NAME } from "@hashintel/brunch-agent/question-marker"; +import { readPetrinautDocToolName } from "@hashintel/petrinaut-core"; import type { AgentConversationObservation, @@ -22,7 +22,8 @@ const projectPetrinautMessages = ( // The host owns this narrowing: its configured client-tool catalog is the // same catalog Petrinaut's message type exposes. snapshotToUiMessages(conversation, { - clientToolNames: brunchClientToolNames, + clientToolNames: new Set([readPetrinautDocToolName]), + hiddenToolNames: new Set([BRUNCH_QUESTION_TOOL_NAME]), }) as PetrinautAiMessage[]; export const useFlueChatHistory = ( diff --git a/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.test.ts index dc12cda21c6..50807c4ba97 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.test.ts @@ -1,9 +1,8 @@ import { describe, expect, test } from "vitest"; -import { ASK_TOOL_NAME } from "@hashintel/brunch-agent/client-tools"; - import { hashCanonicalSpeechText, + selectCanonicalSpeech, selectCanonicalSpeechSegments, } from "./canonical-speech"; @@ -90,7 +89,7 @@ describe("canonical speech selection", () => { ]); }); - test("selects one exact validated brunch_ask question", () => { + test("does not treat structured tool input as canonical speech", () => { const messages = [ { id: "assistant-ask", @@ -99,21 +98,21 @@ describe("canonical speech selection", () => { { type: "dynamic-tool", toolCallId: "ask-1", - toolName: ASK_TOOL_NAME, + toolName: "brunch_ask", state: "input-available", input: { question: "Which operator confirms the batch?" }, }, { type: "dynamic-tool", toolCallId: "ask-malformed", - toolName: ASK_TOOL_NAME, + toolName: "brunch_ask", state: "input-available", input: { question: 42 }, }, { type: "dynamic-tool", toolCallId: "ask-submitted", - toolName: ASK_TOOL_NAME, + toolName: "brunch_ask", state: "output-available", input: { question: "Do not repeat an answered question." }, output: { answer: "Already answered." }, @@ -129,20 +128,128 @@ describe("canonical speech selection", () => { }, ] satisfies PetrinautAiMessage[]; - const selected = select(messages); - const contentHash = hashCanonicalSpeechText( - "Which operator confirms the batch?", - ); - expect(selected).toEqual([ + expect(select(messages)).toEqual([]); + }); + + test("selects an exact marked question separately from full-response text", () => { + const question = "Which operator confirms the batch?"; + const selection = selectCanonicalSpeech([ { - contentHash, - id: `canonical-speech:assistant-ask:ask-1:${contentHash}`, - messageId: "assistant-ask", - partId: "ask-1", - source: "brunch-ask", - text: "Which operator confirms the batch?", + id: "assistant-question", + role: "assistant", + parts: [ + { + type: "data-brunch-question", + data: { question, toolCallId: "tool-question-1" }, + }, + { + type: "text", + text: `The batch is ready. ${question} I can explain the choices.`, + state: "done", + }, + ], }, ]); + + expect(selection.segments.map(({ text }) => text)).toEqual([ + `The batch is ready. ${question} I can explain the choices.`, + ]); + expect(selection.questionSegment).toEqual({ + contentHash: hashCanonicalSpeechText(question), + id: `canonical-speech:assistant-question:question%3Atool-question-1:${hashCanonicalSpeechText(question)}`, + messageId: "assistant-question", + partId: "question:tool-question-1", + source: "assistant-question", + text: question, + }); + }); + + test.each([ + { + name: "missing exact finalized prose", + parts: [ + { + type: "data-brunch-question" as const, + data: { + question: "Which operator confirms the batch?", + toolCallId: "tool-question-1", + }, + }, + { + type: "text" as const, + text: "A different question appears in the response.", + state: "done" as const, + }, + ], + }, + { + name: "only provisional prose", + parts: [ + { + type: "data-brunch-question" as const, + data: { + question: "Which operator confirms the batch?", + toolCallId: "tool-question-1", + }, + }, + { + type: "text" as const, + text: "Which operator confirms the batch?", + state: "streaming" as const, + }, + ], + }, + { + name: "blank marker identity", + parts: [ + { + type: "data-brunch-question" as const, + data: { + question: "Which operator confirms the batch?", + toolCallId: " ", + }, + }, + { + type: "text" as const, + text: "Which operator confirms the batch?", + state: "done" as const, + }, + ], + }, + ])("rejects a question marker with $name", ({ parts }) => { + expect( + selectCanonicalSpeech([ + { + id: "assistant-invalid-question", + role: "assistant", + parts, + }, + ]).questionSegment, + ).toBeUndefined(); + }); + + test("does not correlate a marker to text from another assistant message", () => { + const question = "Which operator confirms the batch?"; + + expect( + selectCanonicalSpeech([ + { + id: "assistant-marker", + role: "assistant", + parts: [ + { + type: "data-brunch-question", + data: { question, toolCallId: "tool-question-1" }, + }, + ], + }, + { + id: "assistant-text", + role: "assistant", + parts: [{ type: "text", text: question, state: "done" }], + }, + ]).questionSegment, + ).toBeUndefined(); }); test("uses stable source identity plus an exact-text fingerprint", () => { diff --git a/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.ts b/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.ts index b99a0afdf30..fd466e1448d 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.ts @@ -1,7 +1,7 @@ import { - ASK_TOOL_NAME, - parseBrunchAskInput, -} from "@hashintel/brunch-agent/client-tools"; + BRUNCH_QUESTION_DATA_NAME, + parseBrunchQuestionData, +} from "@hashintel/brunch-agent/question-marker"; import { hashCanonicalSpeechText } from "../../../canonical-speech-fingerprint"; @@ -15,7 +15,7 @@ export interface CanonicalSpeechSegment { readonly id: string; readonly messageId: string; readonly partId: string; - readonly source: "assistant-text" | "brunch-ask"; + readonly source: "assistant-question" | "assistant-text"; /** * Every Flue submission that wrote to this segment's message: the one that * started it plus any client-tool continuation projected back onto it. @@ -46,16 +46,28 @@ const createSegment = ( }; }; -export const selectCanonicalSpeechSegments = ( +export interface CanonicalSpeechSelection { + readonly questionSegment?: CanonicalSpeechSegment; + readonly segments: CanonicalSpeechSegment[]; +} + +export const selectCanonicalSpeech = ( messages: PetrinautAiMessage[], -): CanonicalSpeechSegment[] => { +): CanonicalSpeechSelection => { const segments: CanonicalSpeechSegment[] = []; + let questionSegment: CanonicalSpeechSegment | undefined; for (const message of messages) { if (message.role !== "assistant") { continue; } + const finalizedTexts = message.parts.flatMap((part) => + part.type === "text" && part.state !== "streaming" && part.text.trim() + ? [part.text] + : [], + ); + for (const [partIndex, part] of message.parts.entries()) { if ( part.type === "text" && @@ -70,32 +82,36 @@ export const selectCanonicalSpeechSegments = ( part.text, ), ); - continue; } + } - if ( - part.type !== "dynamic-tool" || - part.toolName !== ASK_TOOL_NAME || - part.state !== "input-available" - ) { - continue; + const questionMarkers = message.parts.flatMap((part) => { + if (part.type !== `data-${BRUNCH_QUESTION_DATA_NAME}`) { + return []; } - try { - const input = parseBrunchAskInput(part.input); - segments.push( - createSegment( - message.id, - part.toolCallId, - "brunch-ask", - input.question, - ), - ); - } catch { - // Malformed tool inputs remain visible as tool errors; they are not spoken. - } + const marker = parseBrunchQuestionData(part.data); + + return marker && + finalizedTexts.some((text) => text.includes(marker.question)) + ? [marker] + : []; + }); + const latestQuestionMarker = questionMarkers.at(-1); + + if (latestQuestionMarker) { + questionSegment = createSegment( + message.id, + `question:${latestQuestionMarker.toolCallId}`, + "assistant-question", + latestQuestionMarker.question, + ); } } - return segments; + return { questionSegment, segments }; }; + +export const selectCanonicalSpeechSegments = ( + messages: PetrinautAiMessage[], +): CanonicalSpeechSegment[] => selectCanonicalSpeech(messages).segments; diff --git a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts index 70f7bc4088f..332bf6a3f73 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts @@ -36,7 +36,7 @@ const canonicalSegment = ( id, messageId: `message-${id}`, partId: id, - source: "brunch-ask", + source: "assistant-text", text, }); @@ -221,7 +221,7 @@ describe("OpenAIRealtimeSession", () => { expect(harness.peers[0]!.close).toHaveBeenCalledOnce(); }); - test("keeps the microphone active through playback and reports automatic interruption", async () => { + test("keeps the microphone closed and rejects audio detected during playback", async () => { const harness = createHarness(); await harness.session.connect(); harness.session.setMicrophoneEnabled(true); @@ -242,31 +242,290 @@ describe("OpenAIRealtimeSession", () => { item_id: "item-user", type: "input_audio_buffer.speech_started", }); + channel.receive({ + response_id: "response-canonical", + type: "output_audio_buffer.stopped", + }); + channel.receive({ + content_index: 0, + item_id: "item-user", + transcript: "Assistant echo must not submit.", + type: "conversation.item.input_audio_transcription.completed", + }); + + expect(harness.events).not.toContainEqual( + expect.objectContaining({ itemId: "item-user", type: "completed" }), + ); + expect(harness.events).not.toContainEqual( + expect.objectContaining({ + itemId: "item-user", + type: "input-speech-started", + }), + ); + }); + + test("rejects an accepted input item whose transcript completes after output starts", async () => { + const harness = createHarness(); + await harness.session.connect(); + harness.session.setMicrophoneEnabled(true); + const channel = harness.channels[0]!; + + channel.receive({ + audio_start_ms: 80, + item_id: "item-before-output", + type: "input_audio_buffer.speech_started", + }); + channel.receive({ + content_index: 0, + delta: "This started before output", + item_id: "item-before-output", + type: "conversation.item.input_audio_transcription.delta", + }); + expect(harness.events).toContainEqual({ + key: { + connectionEpoch: 1, + contentIndex: 0, + itemId: "item-before-output", + }, + text: "This started before output", + type: "partial", + }); + + harness.session.speakCanonical([ + canonicalSegment("ask-1", "What happens next?"), + ]); + authorizeLatestSpeechResponse(channel, "response-canonical"); + channel.receive({ + response_id: "response-canonical", + type: "output_audio_buffer.started", + }); + channel.receive({ + content_index: 0, + item_id: "item-before-output", + transcript: "This completed too late.", + type: "conversation.item.input_audio_transcription.completed", + }); + + expect( + harness.events.some( + (event) => + event.type === "completed" && + event.key.itemId === "item-before-output", + ), + ).toBe(false); + expect(harness.localTracks[0]!.enabled).toBe(false); + }); + + test("invalidates accepted input before requesting canonical speech output", async () => { + const harness = createHarness(); + await harness.session.connect(); + harness.session.setMicrophoneEnabled(true); + const channel = harness.channels[0]!; + let microphoneEnabledWhenResponseRequested: boolean | undefined; + channel.send.mockImplementation((payload: string) => { + if (JSON.parse(payload).type === "response.create") { + microphoneEnabledWhenResponseRequested = + harness.localTracks[0]!.enabled; + } + }); + + channel.receive({ + audio_start_ms: 80, + item_id: "item-before-request", + type: "input_audio_buffer.speech_started", + }); + channel.receive({ + content_index: 0, + delta: "This started before canonical speech", + item_id: "item-before-request", + type: "conversation.item.input_audio_transcription.delta", + }); + + harness.session.speakCanonical([ + canonicalSegment("ask-request", "What happens next?"), + ]); + expect(harness.events).toContainEqual( + expect.objectContaining({ type: "canonical-speech-requested" }), + ); + expect(microphoneEnabledWhenResponseRequested).toBe(false); + expect(harness.localTracks[0]!.enabled).toBe(false); + + channel.receive({ + content_index: 0, + item_id: "item-before-request", + transcript: "This completed before output started.", + type: "conversation.item.input_audio_transcription.completed", + }); + expect( + harness.events.some( + (event) => + event.type === "completed" && + event.key.itemId === "item-before-request", + ), + ).toBe(false); + const handoff = harness.session.cancelOutput(); + authorizeLatestSpeechResponse(channel, "response-before-output"); + channel.receive({ type: "input_audio_buffer.cleared" }); + channel.receive({ + response: { + id: "response-before-output", + output: [], + status: "cancelled", + }, + type: "response.done", + }); + await handoff; expect(harness.localTracks[0]!.enabled).toBe(true); - expect(harness.events).toEqual( - expect.arrayContaining([ - { - connectionEpoch: 1, - responseId: "response-canonical", - speechRequestId: "canonical-1-1", - type: "output-started", - }, - { - connectionEpoch: 1, - itemId: "item-user", - type: "input-speech-started", - }, - { + + channel.receive({ + content_index: 0, + item_id: "item-before-request", + transcript: "The stale item cannot recover authority.", + type: "conversation.item.input_audio_transcription.completed", + }); + channel.receive({ + audio_start_ms: 160, + item_id: "item-after-handoff", + type: "input_audio_buffer.speech_started", + }); + channel.receive({ + content_index: 0, + item_id: "item-after-handoff", + transcript: "This is fresh after the handoff.", + type: "conversation.item.input_audio_transcription.completed", + }); + + expect( + harness.events.filter((event) => event.type === "completed"), + ).toEqual([ + { + key: { connectionEpoch: 1, - responseId: "response-canonical", - type: "output-interrupted", + contentIndex: 0, + itemId: "item-after-handoff", }, - ]), - ); + text: "This is fresh after the handoff.", + type: "completed", + }, + ]); }); - test("parses streamed tool arguments and the completed GA response output", async () => { + test("restores only the latest microphone preference after playback", async () => { + const harness = createHarness(); + await harness.session.connect(); + harness.session.setMicrophoneEnabled(true); + harness.session.speakCanonical([ + canonicalSegment("ask-1", "What happens next?"), + ]); + const channel = harness.channels[0]!; + authorizeLatestSpeechResponse(channel, "response-canonical"); + channel.receive({ + response_id: "response-canonical", + type: "output_audio_buffer.started", + }); + + expect(harness.localTracks[0]!.enabled).toBe(false); + harness.session.setMicrophoneEnabled(false); + channel.receive({ + response_id: "response-canonical", + type: "output_audio_buffer.stopped", + }); + + expect(harness.localTracks[0]!.enabled).toBe(false); + }); + + test("waits for input, output, and response settlement before completing handoff", async () => { + const harness = createHarness(); + await harness.session.connect(); + harness.session.setMicrophoneEnabled(true); + const channel = harness.channels[0]!; + channel.receive({ + audio_start_ms: 40, + item_id: "item-before-handoff", + type: "input_audio_buffer.speech_started", + }); + harness.session.speakCanonical([ + canonicalSegment("ask-handoff", "What happens next?"), + ]); + authorizeLatestSpeechResponse(channel, "response-handoff"); + channel.receive({ + response_id: "response-handoff", + type: "output_audio_buffer.started", + }); + + const cancellation = Promise.resolve(harness.session.cancelOutput()); + let settled = false; + void cancellation.then(() => { + settled = true; + }); + + expect(harness.localTracks[0]!.enabled).toBe(false); + expect(sentEvents(channel).slice(-3)).toEqual([ + { type: "input_audio_buffer.clear" }, + expect.objectContaining({ + response_id: "response-handoff", + type: "response.cancel", + }), + { type: "output_audio_buffer.clear" }, + ]); + channel.receive({ + content_index: 0, + item_id: "item-before-handoff", + transcript: "This began too early.", + type: "conversation.item.input_audio_transcription.completed", + }); + channel.receive({ type: "input_audio_buffer.cleared" }); + channel.receive({ + response_id: "response-handoff", + type: "output_audio_buffer.cleared", + }); + await Promise.resolve(); + expect(settled).toBe(false); + expect(harness.localTracks[0]!.enabled).toBe(false); + + channel.receive({ + response: { + id: "response-handoff", + output: [], + status: "cancelled", + }, + type: "response.done", + }); + await cancellation; + + expect(harness.localTracks[0]!.enabled).toBe(true); + expect( + harness.events.some( + (event) => + event.type === "completed" && + event.key.itemId === "item-before-handoff", + ), + ).toBe(false); + + channel.receive({ + audio_start_ms: 120, + item_id: "item-after-handoff", + type: "input_audio_buffer.speech_started", + }); + channel.receive({ + content_index: 0, + item_id: "item-after-handoff", + transcript: "This began after the handoff.", + type: "conversation.item.input_audio_transcription.completed", + }); + expect(harness.events).toContainEqual({ + key: { + connectionEpoch: 1, + contentIndex: 0, + itemId: "item-after-handoff", + }, + text: "This began after the handoff.", + type: "completed", + }); + }); + + test("never exposes model function arguments as user input", async () => { const harness = createHarness(); await harness.session.connect(); const channel = harness.channels[0]!; @@ -287,6 +546,8 @@ describe("OpenAIRealtimeSession", () => { response_id: "response-tool", type: "response.function_call_arguments.delta", }); + expect(harness.events).toEqual([]); + channel.receive({ response: { id: "response-tool", @@ -305,76 +566,7 @@ describe("OpenAIRealtimeSession", () => { }); expect(harness.events).toEqual([ - { - callId: "call-1", - connectionEpoch: 1, - delta: '{"answer":"Approved"}', - itemId: "item-function", - responseId: "response-tool", - type: "tool-arguments-delta", - }, - { - arguments: '{"answer":"Approved"}', - callId: "call-1", - connectionEpoch: 1, - itemId: "item-function", - name: "continue_interview", - responseId: "response-tool", - type: "tool-arguments-done", - }, - { - connectionEpoch: 1, - responseId: "response-tool", - status: "completed", - type: "response-terminal", - }, - ]); - - harness.session.completeFunctionCall("call-1", [ - canonicalSegment("ask-2", "Who acts next?"), - ]); - const [functionOutput, responseCreate] = sentEvents(channel).slice(-2); - expect(functionOutput).toEqual({ - type: "conversation.item.create", - item: { - type: "function_call_output", - call_id: "call-1", - output: JSON.stringify({ response_text: ["Who acts next?"] }), - }, - }); - expect(responseCreate).toMatchObject({ - type: "response.create", - response: { - instructions: - "Speak only the response_text strings supplied by Petrinaut, in array order and verbatim. Deliver them as a warm, calm, curious, confident, concise, and professionally neutral expert interviewer, at a measured conversational pace with natural emphasis. Never sound robotic, fawning, rushed, overenthusiastic, or patronizing. Do not add, remove, paraphrase, acknowledge, or explain anything.", - output_modalities: ["audio"], - parallel_tool_calls: false, - tool_choice: "none", - tools: [], - }, - }); - }); - - test("closes a stopped function call without requesting speech", async () => { - const harness = createHarness(); - await harness.session.connect(); - const channel = harness.channels[0]!; - const sentBefore = sentEvents(channel).length; - - harness.session.completeFunctionCallWithoutResponse( - "call-stopped", - "aborted", - ); - - expect(sentEvents(channel).slice(sentBefore)).toEqual([ - { - type: "conversation.item.create", - item: { - type: "function_call_output", - call_id: "call-stopped", - output: JSON.stringify({ response_text: [], outcome: "aborted" }), - }, - }, + expect.objectContaining({ code: "invalid-response", type: "error" }), ]); }); @@ -383,18 +575,30 @@ describe("OpenAIRealtimeSession", () => { await harness.session.connect(); const channel = harness.channels[0]!; - harness.session.completeFunctionCall("call-exact", [ + harness.session.speakCanonical([ canonicalSegment("ask-exact", " Exact Brunch text.\n"), ]); - expect(sentEvents(channel)[0]).toEqual({ - type: "conversation.item.create", - item: { - type: "function_call_output", - call_id: "call-exact", - output: JSON.stringify({ - response_text: [" Exact Brunch text.\n"], - }), + expect(sentEvents(channel)[0]).toMatchObject({ + type: "response.create", + response: { + conversation: "none", + input: [ + { + type: "message", + role: "system", + content: [ + { + type: "input_text", + text: JSON.stringify({ + response_text: [" Exact Brunch text.\n"], + }), + }, + ], + }, + ], + tool_choice: "none", + tools: [], }, }); const sentCount = sentEvents(channel).length; @@ -459,7 +663,7 @@ describe("OpenAIRealtimeSession", () => { ]); const responseCreate = sentEvents(channel)[0]!; - harness.session.cancelOutput(); + void harness.session.cancelOutput(); expect( sentEvents(channel).filter(({ type }) => type === "response.cancel"), @@ -606,7 +810,7 @@ describe("OpenAIRealtimeSession", () => { type: "output_audio_buffer.started", }); - harness.session.cancelOutput(); + void harness.session.cancelOutput(); const cancelEvent = sentEvents(channel).findLast( ({ type }) => type === "response.cancel", )!; @@ -650,7 +854,7 @@ describe("OpenAIRealtimeSession", () => { type: "output_audio_buffer.started", }); - harness.session.cancelOutput(); + void harness.session.cancelOutput(); const cancelEvent = sentEvents(channel).findLast( ({ type }) => type === "response.cancel", )!; @@ -772,12 +976,31 @@ describe("OpenAIRealtimeSession", () => { expect(harness.peers[0]!.close).toHaveBeenCalledOnce(); }); - test("treats transcripts as display-only and never closes capture", async () => { + test("requires a matching speech-start boundary before exposing transcripts", async () => { const harness = createHarness(); await harness.session.connect(); harness.session.setMicrophoneEnabled(true); const channel = harness.channels[0]!; + channel.receive({ + content_index: 0, + delta: "Missing boundary", + item_id: "item-without-boundary", + type: "conversation.item.input_audio_transcription.delta", + }); + channel.receive({ + content_index: 0, + item_id: "item-without-boundary", + transcript: "This must stay rejected.", + type: "conversation.item.input_audio_transcription.completed", + }); + expect(harness.events).toEqual([]); + + channel.receive({ + audio_start_ms: 100, + item_id: "item-user", + type: "input_audio_buffer.speech_started", + }); channel.receive({ content_index: 0, delta: "The supervisor", @@ -792,6 +1015,11 @@ describe("OpenAIRealtimeSession", () => { }); expect(harness.events).toEqual([ + { + connectionEpoch: 1, + itemId: "item-user", + type: "input-speech-started", + }, { key: { connectionEpoch: 1, contentIndex: 0, itemId: "item-user" }, text: "The supervisor", @@ -806,11 +1034,67 @@ describe("OpenAIRealtimeSession", () => { expect(harness.localTracks[0]!.enabled).toBe(true); }); + test("does not retroactively accept a completion that precedes its speech boundary", async () => { + const harness = createHarness(); + await harness.session.connect(); + harness.session.setMicrophoneEnabled(true); + const channel = harness.channels[0]!; + + channel.receive({ + content_index: 0, + item_id: "item-reordered", + transcript: "This completed before its boundary.", + type: "conversation.item.input_audio_transcription.completed", + }); + channel.receive({ + audio_start_ms: 100, + item_id: "item-reordered", + type: "input_audio_buffer.speech_started", + }); + + expect(harness.events).toEqual([ + { + connectionEpoch: 1, + itemId: "item-reordered", + type: "input-speech-started", + }, + ]); + }); + + test("does not reuse a speech boundary from a previous connection epoch", async () => { + const harness = createHarness(); + await harness.session.connect(); + harness.session.setMicrophoneEnabled(true); + harness.channels[0]!.receive({ + audio_start_ms: 100, + item_id: "reused-item", + type: "input_audio_buffer.speech_started", + }); + + await harness.session.disconnect(); + await harness.session.connect(); + harness.session.setMicrophoneEnabled(true); + harness.events.length = 0; + harness.channels[1]!.receive({ + content_index: 0, + item_id: "reused-item", + transcript: "This lacks a current-epoch boundary.", + type: "conversation.item.input_audio_transcription.completed", + }); + + expect(harness.events).toEqual([]); + }); + test("keeps the duplex session alive when optional input transcription fails", async () => { const harness = createHarness(); await harness.session.connect(); harness.session.setMicrophoneEnabled(true); + harness.channels[0]!.receive({ + audio_start_ms: 100, + item_id: "item-user", + type: "input_audio_buffer.speech_started", + }); harness.channels[0]!.receive({ content_index: 0, error: { message: "private provider detail" }, @@ -819,6 +1103,11 @@ describe("OpenAIRealtimeSession", () => { }); expect(harness.events).toEqual([ + { + connectionEpoch: 1, + itemId: "item-user", + type: "input-speech-started", + }, { key: { connectionEpoch: 1, contentIndex: 0, itemId: "item-user" }, type: "transcription-failed", diff --git a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts index 99b66aac151..6490da54901 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts @@ -19,13 +19,6 @@ export interface OpenAIRealtimeTranscriptKey { readonly itemId: string; } -interface RealtimeToolEventIdentity { - readonly callId: string; - readonly connectionEpoch: number; - readonly itemId: string; - readonly responseId: string; -} - export type OpenAIRealtimeSessionEvent = | { readonly key: OpenAIRealtimeTranscriptKey; @@ -74,15 +67,6 @@ export type OpenAIRealtimeSessionEvent = readonly status: "cancelled" | "completed" | "failed" | "incomplete"; readonly type: "response-terminal"; } - | (RealtimeToolEventIdentity & { - readonly delta: string; - readonly type: "tool-arguments-delta"; - }) - | (RealtimeToolEventIdentity & { - readonly arguments: string; - readonly name: string; - readonly type: "tool-arguments-done"; - }) | { readonly code: VoiceErrorCode; readonly message: string; @@ -192,17 +176,21 @@ const waitForAbort = ( }; export class OpenAIRealtimeSession { + readonly #acceptedInputItemIds = new Set(); readonly #dependencies: OpenAIRealtimeSessionDependencies; readonly #activeResponseIds = new Set(); readonly #listeners = new Set(); readonly #authorizedResponseIds = new Set(); readonly #cancelledCanonicalResponseIds = new Set(); readonly #cancelledSpeechRequestIds = new Set(); + readonly #cancelOutputAwaitingRequestIds = new Set(); + readonly #cancelOutputAwaitingResponseIds = new Set(); readonly #canonicalResponseIds = new Set(); readonly #canonicalSpeechQueue: CanonicalSpeechRequest[] = []; readonly #completedResponseCancelEventIds = new Set(); readonly #pendingClientEvents = new Map(); readonly #pendingSpeechRequests = new Map(); + readonly #playbackOverlappingInputItemIds = new Set(); readonly #remoteStreams = new Set(); readonly #speechRequestIds = new Map(); readonly #speechTimings = new Map(); @@ -214,6 +202,10 @@ export class OpenAIRealtimeSession { #connected = false; #connectedAt: number | null = null; #clientEventSequence = 0; + #cancelOutputAwaitingInputBufferClear = false; + #cancelOutputAwaitingOutputBufferResponseId: string | null = null; + #cancelOutputPromise: Promise | null = null; + #cancelOutputResolve: (() => void) | null = null; #connectionRequestId: string | null = null; #dataChannel: RTCDataChannel | null = null; #epoch = 0; @@ -223,6 +215,7 @@ export class OpenAIRealtimeSession { #meterHasSample = false; #meterLevel = 0; #meterSamples: Uint8Array | null = null; + #microphoneRequested = false; #microphoneTrack: MediaStreamTrack | null = null; #peerConnection: RTCPeerConnection | null = null; #remoteAudio: RemoteAudio | null = null; @@ -393,92 +386,74 @@ export class OpenAIRealtimeSession { } public setMicrophoneEnabled(enabled: boolean): void { - if (!this.#microphoneTrack) { - return; - } - const isEnabled = enabled && this.#connected; - this.#microphoneTrack.enabled = isEnabled; - if (isEnabled) { - this.#startMeter(); - } else { - this.#stopMeter(); - } + this.#microphoneRequested = enabled && this.#connected; + this.#syncMicrophoneTrack(); } public speakCanonical(segments: CanonicalSpeechSegment[]): void { this.#requestCanonicalSpeech(segments, true); } - public completeFunctionCall( - callId: string, - segments: CanonicalSpeechSegment[], - ): void { - if (!callId) { - throw new VoiceError("speech", "invalid-response", ""); + public cancelOutput(): Promise { + if (!this.#connected || this.#dataChannel?.readyState !== "open") { + return Promise.resolve(); } - const responseText = this.#canonicalResponseText(segments); - this.#send({ - type: "conversation.item.create", - item: { - type: "function_call_output", - call_id: callId, - output: JSON.stringify({ response_text: responseText }), - }, - }); - this.#requestCanonicalSpeech(segments, false); - } - - /** - * Close a Realtime function call whose Brunch turn settled without a reply. - * The call output records the settlement so the model does not wait on it, - * and no speech is requested: a stopped turn has no canonical text to speak. - */ - public completeFunctionCallWithoutResponse( - callId: string, - outcome: "aborted" | "failed", - ): void { - if (!callId) { - throw new VoiceError("speech", "invalid-response", ""); + if (this.#cancelOutputPromise) { + return this.#cancelOutputPromise; } - this.#send({ - type: "conversation.item.create", - item: { - type: "function_call_output", - call_id: callId, - output: JSON.stringify({ response_text: [], outcome }), - }, - }); - } - public cancelOutput(): void { - if (!this.#connected || this.#dataChannel?.readyState !== "open") { - return; + const cancelOutputPromise = new Promise((resolve) => { + this.#cancelOutputResolve = resolve; + }); + this.#cancelOutputPromise = cancelOutputPromise; + this.#cancelOutputAwaitingInputBufferClear = true; + this.#cancelOutputAwaitingOutputBufferResponseId = this.#speakingResponseId; + for (const itemId of this.#acceptedInputItemIds) { + this.#playbackOverlappingInputItemIds.add(itemId); } + this.#acceptedInputItemIds.clear(); + this.#syncMicrophoneTrack(); - for (const request of this.#canonicalSpeechQueue.splice(0)) { - this.#cancelPendingSpeechRequest(request.speechRequestId); - } + try { + this.#send({ type: "input_audio_buffer.clear" }); - if (this.#responseCreateEventId !== null) { - const pendingEvent = this.#pendingClientEvents.get( - this.#responseCreateEventId, - ); - if (pendingEvent?.kind === "response-create") { - this.#cancelledSpeechRequestIds.add( - pendingEvent.request.speechRequestId, + for (const request of this.#canonicalSpeechQueue.splice(0)) { + this.#cancelPendingSpeechRequest(request.speechRequestId); + } + + if (this.#responseCreateEventId !== null) { + const pendingEvent = this.#pendingClientEvents.get( + this.#responseCreateEventId, ); + if (pendingEvent?.kind === "response-create") { + this.#cancelledSpeechRequestIds.add( + pendingEvent.request.speechRequestId, + ); + this.#cancelOutputAwaitingRequestIds.add( + pendingEvent.request.speechRequestId, + ); + } } - } - for (const responseId of this.#canonicalResponseIds) { - if ( - this.#activeResponseIds.has(responseId) && - !this.#cancelledCanonicalResponseIds.has(responseId) - ) { - this.#cancelledCanonicalResponseIds.add(responseId); - this.#cancelOutputResponse(responseId); + for (const responseId of this.#canonicalResponseIds) { + if ( + this.#activeResponseIds.has(responseId) && + !this.#cancelledCanonicalResponseIds.has(responseId) + ) { + this.#cancelOutputAwaitingResponseIds.add(responseId); + this.#cancelledCanonicalResponseIds.add(responseId); + this.#cancelResponse(responseId); + } } + + this.#send({ type: "output_audio_buffer.clear" }); + } catch { + this.#finishOutputCancellation(true); + this.#handleConnectionFailure("network", "speech"); } + + this.#finishOutputCancellation(); + return cancelOutputPromise; } #cancelOutputResponse(responseId: string): void { @@ -599,6 +574,11 @@ export class OpenAIRealtimeSession { request, responseTerminalSequence: this.#responseTerminalSequence, }); + for (const itemId of this.#acceptedInputItemIds) { + this.#playbackOverlappingInputItemIds.add(itemId); + } + this.#acceptedInputItemIds.clear(); + this.#syncMicrophoneTrack(); try { this.#send({ event_id: eventId, @@ -615,6 +595,7 @@ export class OpenAIRealtimeSession { } catch (error) { this.#responseCreateEventId = null; this.#pendingClientEvents.delete(eventId); + this.#syncMicrophoneTrack(); throw error; } } @@ -643,6 +624,12 @@ export class OpenAIRealtimeSession { this.#handleResponseDone(parsed, connectionEpoch); return; } + if (parsed.type === "input_audio_buffer.cleared") { + this.#acceptedInputItemIds.clear(); + this.#cancelOutputAwaitingInputBufferClear = false; + this.#finishOutputCancellation(); + return; + } if (parsed.type === "input_audio_buffer.committed") { const itemId = nonEmptyString(parsed.item_id); if (itemId) this.#startTranscription(itemId); @@ -651,23 +638,22 @@ export class OpenAIRealtimeSession { if (parsed.type === "input_audio_buffer.speech_started") { const itemId = nonEmptyString(parsed.item_id); if (!itemId || nonNegativeInteger(parsed.audio_start_ms) === null) return; + if (this.#speakingResponseId || !this.#microphoneTrack?.enabled) { + this.#playbackOverlappingInputItemIds.add(itemId); + return; + } + this.#acceptedInputItemIds.add(itemId); this.#emit({ connectionEpoch, itemId, type: "input-speech-started", }); - if (this.#speakingResponseId) { - this.#emit({ - connectionEpoch, - responseId: this.#speakingResponseId, - type: "output-interrupted", - }); - } return; } if (parsed.type === "input_audio_buffer.speech_stopped") { const itemId = nonEmptyString(parsed.item_id); if (!itemId || nonNegativeInteger(parsed.audio_end_ms) === null) return; + if (this.#playbackOverlappingInputItemIds.has(itemId)) return; this.#emit({ connectionEpoch, itemId, @@ -677,15 +663,12 @@ export class OpenAIRealtimeSession { } if ( parsed.type === "output_audio_buffer.started" || - parsed.type === "output_audio_buffer.stopped" + parsed.type === "output_audio_buffer.stopped" || + parsed.type === "output_audio_buffer.cleared" ) { this.#handleOutputBufferEvent(parsed, connectionEpoch); return; } - if (parsed.type === "response.function_call_arguments.delta") { - this.#handleToolEvent(parsed, connectionEpoch); - return; - } if ( parsed.type === "conversation.item.input_audio_transcription.delta" || parsed.type === "conversation.item.input_audio_transcription.completed" || @@ -707,6 +690,9 @@ export class OpenAIRealtimeSession { if (metadata?.petrinaut_kind !== "canonical-speech" || !speechRequestId) { return; } + if (this.#cancelOutputAwaitingRequestIds.delete(speechRequestId)) { + this.#cancelOutputAwaitingResponseIds.add(responseId); + } this.#completeResponseCreateEvent(speechRequestId); this.#canonicalResponseIds.add(responseId); if (this.#cancelledSpeechRequestIds.delete(speechRequestId)) { @@ -762,6 +748,10 @@ export class OpenAIRealtimeSession { ) { this.#pendingClientEvents.delete(sourceEventId); this.#completedResponseCancelEventIds.delete(sourceEventId); + if (pendingEvent?.kind === "response-cancel") { + this.#cancelOutputAwaitingResponseIds.delete(pendingEvent.responseId); + this.#finishOutputCancellation(); + } return; } @@ -780,7 +770,11 @@ export class OpenAIRealtimeSession { pendingEvent.request.speechRequestId, ) ) { + this.#cancelOutputAwaitingRequestIds.delete( + pendingEvent.request.speechRequestId, + ); this.#cancelPendingSpeechRequest(pendingEvent.request.speechRequestId); + this.#finishOutputCancellation(); return; } this.#canonicalSpeechQueue.unshift(pendingEvent.request); @@ -815,10 +809,19 @@ export class OpenAIRealtimeSession { } this.#responseTerminalSequence += 1; this.#activeResponseIds.delete(responseId); + this.#cancelOutputAwaitingResponseIds.delete(responseId); + this.#finishOutputCancellation(); this.#clearResponseCancelEvents(responseId); this.#waitingForResponseTerminal = false; if (this.#cancelledCanonicalResponseIds.delete(responseId)) { + if (this.#speakingResponseId === responseId) { + this.#emit({ + connectionEpoch, + responseId, + type: "output-interrupted", + }); + } this.#emit({ connectionEpoch, responseId, @@ -836,44 +839,10 @@ export class OpenAIRealtimeSession { this.#handleConnectionFailure("invalid-response", "connection"); return; } - const functionCalls = output - .map(asRecord) - .filter( - (item): item is Record => - item?.type === "function_call", - ); - if ( - functionCalls.length > 1 || - (functionCalls.length > 0 && this.#canonicalResponseIds.has(responseId)) - ) { + if (output.some((item) => asRecord(item)?.type === "function_call")) { this.#handleConnectionFailure("invalid-response", "connection"); return; } - for (const item of functionCalls) { - const argumentsJson = nonEmptyString(item.arguments); - const callId = nonEmptyString(item.call_id); - const itemId = nonEmptyString(item.id); - const name = nonEmptyString(item.name); - if ( - !argumentsJson || - !callId || - !itemId || - !name || - (item.status !== undefined && item.status !== "completed") - ) { - this.#handleConnectionFailure("invalid-response", "connection"); - return; - } - this.#emit({ - arguments: argumentsJson, - callId, - connectionEpoch, - itemId, - name, - responseId, - type: "tool-arguments-done", - }); - } this.#emit({ connectionEpoch, responseId, @@ -941,6 +910,9 @@ export class OpenAIRealtimeSession { if (!responseId) return; if (event.type === "output_audio_buffer.started") { if (this.#cancelledCanonicalResponseIds.has(responseId)) { + if (this.#cancelOutputPromise) { + this.#cancelOutputAwaitingOutputBufferResponseId = responseId; + } this.#send({ type: "output_audio_buffer.clear" }); return; } @@ -949,7 +921,12 @@ export class OpenAIRealtimeSession { this.#handleConnectionFailure("invalid-response", "connection"); return; } + for (const itemId of this.#acceptedInputItemIds) { + this.#playbackOverlappingInputItemIds.add(itemId); + } + this.#acceptedInputItemIds.clear(); this.#speakingResponseId = responseId; + this.#syncMicrophoneTrack(); const speechRequestId = this.#speechRequestIds.get(responseId); if (!speechRequestId) { this.#handleConnectionFailure("invalid-response", "connection"); @@ -964,35 +941,24 @@ export class OpenAIRealtimeSession { return; } const wasSpeaking = this.#speakingResponseId === responseId; + const wasCleared = event.type === "output_audio_buffer.cleared"; this.#finishSpeech( responseId, - this.#cancelledCanonicalResponseIds.has(responseId) + wasCleared || this.#cancelledCanonicalResponseIds.has(responseId) ? "request-aborted" : undefined, ); if (wasSpeaking) { - this.#emit({ connectionEpoch, responseId, type: "output-stopped" }); + this.#emit({ + connectionEpoch, + responseId, + type: wasCleared ? "output-interrupted" : "output-stopped", + }); + } + if (this.#cancelOutputAwaitingOutputBufferResponseId === responseId) { + this.#cancelOutputAwaitingOutputBufferResponseId = null; + this.#finishOutputCancellation(); } - } - - #handleToolEvent( - event: Record, - connectionEpoch: number, - ): void { - const callId = nonEmptyString(event.call_id); - const itemId = nonEmptyString(event.item_id); - const responseId = nonEmptyString(event.response_id); - const outputIndex = nonNegativeInteger(event.output_index); - if (!callId || !itemId || !responseId || outputIndex === null) return; - if (typeof event.delta !== "string") return; - this.#emit({ - callId, - connectionEpoch, - delta: event.delta, - itemId, - responseId, - type: "tool-arguments-delta", - }); } #handleTranscriptEvent( @@ -1003,9 +969,29 @@ export class OpenAIRealtimeSession { const contentIndex = nonNegativeInteger(event.content_index); if (!itemId || contentIndex === null) return; const key = { connectionEpoch, contentIndex, itemId }; + const overlapsPlayback = + this.#playbackOverlappingInputItemIds.has(itemId) || + !this.#acceptedInputItemIds.has(itemId); + if (overlapsPlayback) { + if ( + event.type === + "conversation.item.input_audio_transcription.completed" || + event.type === "conversation.item.input_audio_transcription.failed" + ) { + this.#finishTranscription( + itemId, + event.type === "conversation.item.input_audio_transcription.failed" + ? "invalid-response" + : undefined, + ); + this.#acceptedInputItemIds.delete(itemId); + } + return; + } this.#startTranscription(itemId); if (event.type === "conversation.item.input_audio_transcription.failed") { this.#finishTranscription(itemId, "invalid-response"); + this.#acceptedInputItemIds.delete(itemId); this.#emit({ key, type: "transcription-failed" }); return; } @@ -1018,6 +1004,7 @@ export class OpenAIRealtimeSession { event.type === "conversation.item.input_audio_transcription.completed" ) { this.#finishTranscription(itemId); + this.#acceptedInputItemIds.delete(itemId); } this.#emit({ key, @@ -1058,7 +1045,31 @@ export class OpenAIRealtimeSession { this.#authorizedResponseIds.delete(responseId); if (this.#speakingResponseId === responseId) { this.#speakingResponseId = null; + this.#syncMicrophoneTrack(); + } + } + + #finishOutputCancellation(force = false): void { + if ( + !this.#cancelOutputPromise || + (!force && + (this.#cancelOutputAwaitingInputBufferClear || + this.#cancelOutputAwaitingOutputBufferResponseId !== null || + this.#cancelOutputAwaitingRequestIds.size > 0 || + this.#cancelOutputAwaitingResponseIds.size > 0)) + ) { + return; } + + const resolve = this.#cancelOutputResolve; + this.#cancelOutputPromise = null; + this.#cancelOutputResolve = null; + this.#cancelOutputAwaitingInputBufferClear = false; + this.#cancelOutputAwaitingOutputBufferResponseId = null; + this.#cancelOutputAwaitingRequestIds.clear(); + this.#cancelOutputAwaitingResponseIds.clear(); + this.#syncMicrophoneTrack(); + resolve?.(); } #emit(event: OpenAIRealtimeSessionEvent): void { @@ -1239,6 +1250,24 @@ export class OpenAIRealtimeSession { this.#meterFrame = this.#dependencies.requestAnimationFrame(sample); } + #syncMicrophoneTrack(): void { + if (!this.#microphoneTrack) { + return; + } + const enabled = + this.#microphoneRequested && + this.#connected && + this.#cancelOutputPromise === null && + this.#responseCreateEventId === null && + this.#speakingResponseId === null; + this.#microphoneTrack.enabled = enabled; + if (enabled) { + this.#startMeter(); + } else { + this.#stopMeter(); + } + } + #stopMeter(): void { if (this.#meterFrame === null) return; this.#dependencies.cancelAnimationFrame(this.#meterFrame); @@ -1328,13 +1357,17 @@ export class OpenAIRealtimeSession { ); } this.#transcriptionTimings.clear(); + this.#acceptedInputItemIds.clear(); this.#activeResponseIds.clear(); this.#cancelledCanonicalResponseIds.clear(); this.#cancelledSpeechRequestIds.clear(); + this.#cancelOutputAwaitingRequestIds.clear(); + this.#cancelOutputAwaitingResponseIds.clear(); this.#canonicalSpeechQueue.length = 0; this.#completedResponseCancelEventIds.clear(); this.#pendingClientEvents.clear(); this.#pendingSpeechRequests.clear(); + this.#playbackOverlappingInputItemIds.clear(); this.#speechTimings.clear(); this.#speechRequestIds.clear(); this.#authorizedResponseIds.clear(); @@ -1342,6 +1375,7 @@ export class OpenAIRealtimeSession { this.#responseCreateEventId = null; this.#responseTerminalSequence = 0; this.#speakingResponseId = null; + this.#microphoneRequested = false; this.#waitingForResponseTerminal = false; this.#activeEpoch = null; this.#connected = false; @@ -1388,6 +1422,7 @@ export class OpenAIRealtimeSession { this.#mediaStream = null; } this.#microphoneTrack = null; + this.#finishOutputCancellation(true); } #waitForDataChannelOpen( diff --git a/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.test.ts index b65fc510f9c..e16d0b14c49 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.test.ts @@ -1,5 +1,7 @@ import { describe, expect, test, vi } from "vitest"; +import { FlueChatAdmissionError } from "@hashintel/brunch-agent-transport-aisdk"; + import { createRealtimeSubmissionId, RealtimeBrunchBridge, @@ -7,26 +9,63 @@ import { } from "./realtime-brunch-bridge"; import type { CanonicalSpeechSegment } from "./canonical-speech"; -import type { OpenAIRealtimeSessionEvent } from "./openai-realtime-session"; +import type { + OpenAIRealtimeSessionEvent, + OpenAIRealtimeTranscriptKey, +} from "./openai-realtime-session"; const segment = ( id: string, text: string, - source: CanonicalSpeechSegment["source"] = "brunch-ask", + submissionId?: string, ): CanonicalSpeechSegment => ({ contentHash: "fnv1a32:12345678", id, messageId: `message-${id}`, partId: id, - source, + source: "assistant-text", + ...(submissionId === undefined ? {} : { submissionIds: [submissionId] }), + text, +}); + +const transcriptKey = ( + connectionEpoch: number, + itemId = "user-item-1", + contentIndex = 0, +): OpenAIRealtimeTranscriptKey => ({ connectionEpoch, contentIndex, itemId }); + +const completedTranscript = ( + connectionEpoch: number, + text = "The supervisor approves it.", + itemId = "user-item-1", + contentIndex = 0, +): Extract => ({ + key: transcriptKey(connectionEpoch, itemId, contentIndex), text, + type: "completed", +}); + +const failedTranscript = ( + connectionEpoch: number, + itemId = "user-item-1", +): Extract => ({ + key: transcriptKey(connectionEpoch, itemId), + type: "transcription-failed", +}); + +const completedResponseMessage = ( + messageId: string, + submissionId: string, + index: number, +) => ({ + messageId, + position: { batch: 1, index }, + submissionId, }); const createHarness = () => { let listener: ((event: OpenAIRealtimeSessionEvent) => void) | undefined; const session = { - completeFunctionCall: vi.fn(), - completeFunctionCallWithoutResponse: vi.fn(), speakCanonical: vi.fn(), subscribe: vi.fn((next: (event: OpenAIRealtimeSessionEvent) => void) => { listener = next; @@ -39,10 +78,14 @@ const createHarness = () => { ConstructorParameters< typeof RealtimeBrunchBridge >[0]["submitInterviewAnswer"] - >(async () => ({ - kind: "interactive-tool", - toolCallId: "ask-current", - })); + >(async (input) => { + input.onAdmission("submission-voice-1"); + return { + kind: "message", + messageId: input.id, + submissionId: "submission-voice-1", + }; + }); const bridge = new RealtimeBrunchBridge({ session, submitInterviewAnswer, @@ -59,797 +102,665 @@ const createHarness = () => { }; }; -const toolDelta = ( - connectionEpoch: number, - delta: string, -): Extract => ({ - callId: "call-1", - connectionEpoch, - delta, - itemId: "function-item-1", - responseId: "response-1", - type: "tool-arguments-delta", -}); - -const toolDone = ( - connectionEpoch: number, - argumentsJson = '{"answer":"The supervisor approves it."}', -): Extract => ({ - arguments: argumentsJson, - callId: "call-1", - connectionEpoch, - itemId: "function-item-1", - name: "continue_interview", - responseId: "response-1", - type: "tool-arguments-done", -}); - -const responseTerminal = ( - connectionEpoch: number, - status: "cancelled" | "completed" | "failed" | "incomplete", - responseId = "response-1", -): Extract => ({ - connectionEpoch, - responseId, - status, - type: "response-terminal", -}); +const startReady = ( + harness: ReturnType, + connectionEpoch = 3, +): void => { + harness.bridge.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [], + status: "ready", + }); + harness.bridge.start(connectionEpoch); +}; describe("RealtimeBrunchBridge", () => { - test("speaks the current canonical turn without replaying history", () => { + test("rehydrates settled canonical speech without submission or playback", () => { const harness = createHarness(); - const historical = segment( - "history", - "Do not replay this.", - "assistant-text", - ); - const preamble = { - ...segment("preamble", "Thanks. One more question.", "assistant-text"), - messageId: "message-current-turn", - }; - const question = { - ...segment("ask-current", "What happens after approval?"), - messageId: "message-current-turn", - }; harness.bridge.updateChat({ canAcceptInterviewAnswer: true, - canonicalSegments: [historical, preamble, question], + canonicalSegments: [ + segment("settled", "Already delivered.", "submission-settled"), + ], status: "ready", }); - harness.bridge.start(4); - - expect(harness.session.speakCanonical).toHaveBeenCalledOnce(); - expect(harness.session.speakCanonical).toHaveBeenCalledWith([ - preamble, - question, - ]); - }); - - test("rehydrates the settled Voice turn without resubmission or playback", () => { - const harness = createHarness(); - const settledResponse = { - ...segment( - "settled-response", - "This canonical response was already delivered.", - "assistant-text", - ), - submissionIds: ["submission-settled"], - }; - - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [settledResponse], - status: "ready", - }); harness.bridge.start(9); expect(harness.submitInterviewAnswer).not.toHaveBeenCalled(); expect(harness.session.speakCanonical).not.toHaveBeenCalled(); - expect(harness.session.completeFunctionCall).not.toHaveBeenCalled(); expect(harness.events).toEqual([]); }); - test("streams and validates one tool call, preserves ask correlation, and waits for canonical Brunch output", async () => { + test("submits only a completed transcript through the user admission target", async () => { const harness = createHarness(); - const question = segment("ask-current", "What happens after approval?"); - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [question], - status: "ready", - }); - harness.bridge.start(7); - harness.session.speakCanonical.mockClear(); + startReady(harness, 7); + const key = transcriptKey(7); - harness.emit(toolDelta(7, '{"answer":"The supervisor')); - harness.emit(toolDelta(7, ' approves it."}')); - harness.emit(toolDone(7)); + harness.emit({ key, text: "The supervisor", type: "partial" }); + harness.emit({ + arguments: '{"answer":"Fabricated answer"}', + callId: "legacy-call", + connectionEpoch: 7, + itemId: "legacy-item", + name: "continue_interview", + responseId: "legacy-response", + type: "tool-arguments-done", + } as unknown as OpenAIRealtimeSessionEvent); + expect(harness.submitInterviewAnswer).not.toHaveBeenCalled(); + + harness.emit(completedTranscript(7, " The supervisor\napproves it. ")); await vi.waitFor(() => expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce(), ); + const deliveryId = createRealtimeSubmissionId(key); expect(harness.submitInterviewAnswer).toHaveBeenCalledWith( expect.objectContaining({ - admissionTarget: { - kind: "client-tool-result", - messageId: "message-ask-current", - }, - id: createRealtimeSubmissionId(7, "call-1"), + admissionTarget: { kind: "user", messageId: deliveryId }, + id: deliveryId, text: "The supervisor approves it.", }), ); - expect(harness.session.completeFunctionCall).not.toHaveBeenCalled(); - - harness.bridge.updateChat({ - canAcceptInterviewAnswer: false, - canonicalSegments: [question], - status: "streaming", + expect(harness.events).toContainEqual({ + answer: "The supervisor approves it.", + deliveryId, + type: "submission-started", }); - const acknowledgement = segment( - "acknowledgement", - "Thanks. I have recorded that.", - "assistant-text", - ); - const nextQuestion = segment( - "ask-next", - "Who is informed next?", - "brunch-ask", - ); - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [question, acknowledgement, nextQuestion], - status: "ready", - }); - - await vi.waitFor(() => - expect(harness.session.completeFunctionCall).toHaveBeenCalledWith( - "call-1", - [acknowledgement, nextQuestion], - ), - ); - expect(harness.events.map(({ type }) => type)).toEqual([ - "submission-started", - "submission-accepted", - "canonical-text-ready", - "submission-settled", - "canonical-response-ready", - ]); + expect(JSON.stringify(harness.events)).not.toContain("Fabricated answer"); }); - test("emits the real admission before composer submission completes", async () => { + test("rejects unfinished input invalidated by output and accepts fresh input", async () => { const harness = createHarness(); - let finishSubmission: (() => void) | undefined; - harness.submitInterviewAnswer.mockImplementationOnce(async () => { - await new Promise((resolve) => { - finishSubmission = resolve; - }); - return { - kind: "interactive-tool", - toolCallId: "ask-current", - }; + startReady(harness); + + harness.emit({ + connectionEpoch: 3, + itemId: "item-before-output", + type: "input-speech-started", }); - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [segment("ask-current", "Question")], - status: "ready", + harness.emit({ + connectionEpoch: 3, + responseId: "response-output", + speechRequestId: "speech-output", + type: "output-started", }); - harness.bridge.start(7); - harness.emit(toolDone(7)); - - await vi.waitFor(() => - expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce(), + harness.emit( + completedTranscript(3, "This completed too late.", "item-before-output"), ); - const submission = harness.submitInterviewAnswer.mock.calls[0]?.[0]; - expect(submission).toBeDefined(); - submission?.onAdmission("submission-early"); - submission?.onAdmission("submission-early"); - expect(harness.events).toEqual([ - expect.objectContaining({ type: "submission-started" }), - { - callId: "call-1", - submissionId: "submission-early", - type: "submission-admitted", - }, - ]); + expect(harness.submitInterviewAnswer).not.toHaveBeenCalled(); + expect(harness.events).toContainEqual({ + reason: "unavailable", + type: "transcript-rejected", + }); - finishSubmission?.(); - await vi.waitFor(() => - expect(harness.events).toContainEqual( - expect.objectContaining({ type: "submission-accepted" }), - ), - ); - const question = segment("ask-current", "Question"); - harness.bridge.updateChat({ - canAcceptInterviewAnswer: false, - canonicalSegments: [question], - status: "streaming", + harness.emit({ + connectionEpoch: 3, + responseId: "response-output", + type: "output-stopped", }); - const unrelated = segment("unrelated", "Do not select this."); - const correlated = { - ...segment("correlated", "Select this response."), - submissionIds: ["submission-early"], - }; - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [question, unrelated, correlated], - status: "ready", + harness.emit({ + connectionEpoch: 3, + itemId: "item-after-output", + type: "input-speech-started", }); - expect(harness.session.completeFunctionCall).toHaveBeenCalledWith( - "call-1", - [correlated], + harness.emit(completedTranscript(3, "This is fresh.", "item-after-output")); + await vi.waitFor(() => + expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce(), ); - harness.bridge.stop(); - submission?.onAdmission("submission-stale"); - expect( - harness.events.filter(({ type }) => type === "submission-admitted"), - ).toHaveLength(1); + expect(harness.submitInterviewAnswer).toHaveBeenCalledWith( + expect.objectContaining({ text: "This is fresh." }), + ); + + harness.emit(completedTranscript(3, "Stale replay.", "item-before-output")); + expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce(); }); - test("admits one finalized Realtime answer through Flue once", async () => { + test("rejects unfinished input as soon as canonical speech is requested", async () => { const harness = createHarness(); - harness.submitInterviewAnswer.mockResolvedValueOnce({ - kind: "message", - messageId: "message-kickoff", - submissionId: "submission-voice-1", - }); - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [], - status: "ready", - }); - harness.bridge.start(7); + startReady(harness); - harness.emit(toolDone(7, '{"answer":"Battery charger workflow"}')); - harness.emit(toolDone(7, '{"answer":"Battery charger workflow"}')); - - await vi.waitFor(() => - expect(harness.submitInterviewAnswer).toHaveBeenCalledWith( - expect.objectContaining({ - admissionTarget: { - kind: "user", - messageId: createRealtimeSubmissionId(7, "call-1"), - }, - id: createRealtimeSubmissionId(7, "call-1"), - text: "Battery charger workflow", - }), + harness.emit({ + connectionEpoch: 3, + itemId: "item-before-request", + type: "input-speech-started", + }); + harness.emit({ + connectionEpoch: 3, + speechRequestId: "speech-request", + type: "canonical-speech-requested", + }); + harness.emit( + completedTranscript( + 3, + "This completed before output started.", + "item-before-request", ), ); - harness.bridge.updateChat({ - canAcceptInterviewAnswer: false, - canonicalSegments: [], - status: "submitted", - }); - const unrelated = { - ...segment("unrelated", "Do not speak this response."), - submissionIds: ["submission-other"], - }; - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [unrelated], - status: "ready", - }); - expect(harness.session.completeFunctionCall).not.toHaveBeenCalled(); - const firstQuestion = { - ...segment("ask-first", "What starts the battery charger workflow?"), - submissionIds: ["submission-voice-1"], - }; - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [unrelated, firstQuestion], - status: "ready", + + expect(harness.submitInterviewAnswer).not.toHaveBeenCalled(); + expect(harness.events).toContainEqual({ + reason: "unavailable", + type: "transcript-rejected", }); - expect(harness.session.completeFunctionCall).toHaveBeenCalledWith( - "call-1", - [firstQuestion], + harness.emit( + completedTranscript( + 3, + "The stale item cannot recover authority.", + "item-before-request", + ), ); - expect(harness.events.map(({ type }) => type)).toEqual([ - "submission-started", - "submission-accepted", - "canonical-text-ready", - "submission-settled", - "canonical-response-ready", - ]); - }); + expect(harness.submitInterviewAnswer).not.toHaveBeenCalled(); - test("rejects a submission result that disagrees with the transport admission", async () => { - const harness = createHarness(); - harness.submitInterviewAnswer.mockImplementationOnce(async (input) => { - input.onAdmission("submission-early"); - return { - kind: "message", - messageId: input.id, - submissionId: "submission-other", - }; - }); - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [], - status: "ready", + harness.bridge.completeTurnHandoff(); + harness.emit({ + connectionEpoch: 3, + itemId: "item-after-handoff", + type: "input-speech-started", }); - harness.bridge.start(7); - harness.emit(toolDone(7)); + harness.emit( + completedTranscript(3, "This is fresh.", "item-after-handoff"), + ); await vi.waitFor(() => - expect(harness.events).toContainEqual( - expect.objectContaining({ - code: "interview-correlation", - type: "error", - }), - ), + expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce(), + ); + expect(harness.submitInterviewAnswer).toHaveBeenCalledWith( + expect.objectContaining({ text: "This is fresh." }), ); }); - test("requires a correlated Brunch busy cycle before accepting new canonical segments", async () => { + test("derives stable delivery identity from epoch, item, and content index", () => { + expect( + createRealtimeSubmissionId(transcriptKey(12, "item/with spaces", 4)), + ).toBe("voice-realtime:12:item%2Fwith%20spaces:4"); + }); + + test("submits duplicate completed transcript events exactly once", async () => { const harness = createHarness(); - const question = segment("ask-current", "What happens after approval?"); - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [question], - status: "ready", - }); - harness.bridge.start(7); - harness.emit(toolDone(7)); + startReady(harness); + const transcript = completedTranscript(3); + + harness.emit(transcript); + harness.emit(transcript); + await vi.waitFor(() => expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce(), ); - const unrelated = segment( - "unrelated", - "An unrelated canonical update.", - "assistant-text", - ); - - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [question, unrelated], - status: "ready", + expect(harness.events).toContainEqual({ + reason: "duplicate", + type: "transcript-rejected", }); + }); - expect(harness.session.completeFunctionCall).not.toHaveBeenCalled(); + test.each([ + ["", "empty"], + [" \n\t ", "empty"], + ["a".repeat(32_001), "over-limit"], + ] as const)( + "rejects an invalid completed transcript as %s", + (text, reason) => { + const harness = createHarness(); + startReady(harness); + + harness.emit(completedTranscript(3, text)); + + expect(harness.submitInterviewAnswer).not.toHaveBeenCalled(); + expect(harness.events).toEqual([{ reason, type: "transcript-rejected" }]); + }, + ); + + test("rejects a failed transcript and accepts the next keyed turn", async () => { + const harness = createHarness(); + startReady(harness); - harness.bridge.updateChat({ - canAcceptInterviewAnswer: false, - canonicalSegments: [question, unrelated], - status: "submitted", - }); - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [question, unrelated], - status: "ready", - }); + harness.emit(failedTranscript(3, "failed-item")); + expect(harness.events).toEqual([ + { reason: "failed", type: "transcript-rejected" }, + ]); - expect(harness.session.completeFunctionCall).toHaveBeenCalledWith( - "call-1", - [unrelated], + harness.emit(completedTranscript(3, "Retried answer.", "retry-item")); + await vi.waitFor(() => + expect(harness.submitInterviewAnswer).toHaveBeenCalledWith( + expect.objectContaining({ text: "Retried answer." }), + ), ); }); - test("rejects streamed arguments whose response or item identity changes", async () => { + test("rejects completed transcripts while the shared submission path is unavailable", () => { const harness = createHarness(); harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [segment("ask-current", "Question")], - status: "ready", + canAcceptInterviewAnswer: false, + canonicalSegments: [], + status: "streaming", }); harness.bridge.start(3); - harness.emit(toolDelta(3, '{"answer":"Answer"}')); - harness.emit({ - ...toolDone(3, '{"answer":"Answer"}'), - responseId: "response-2", - }); - await Promise.resolve(); + harness.emit(completedTranscript(3)); expect(harness.submitInterviewAnswer).not.toHaveBeenCalled(); expect(harness.events).toEqual([ - expect.objectContaining({ - code: "interview-correlation", - type: "error", - }), + { reason: "unavailable", type: "transcript-rejected" }, ]); }); - test("rejects concurrent argument streams before either can submit", async () => { + test("ignores transcripts from an inactive connection epoch", () => { const harness = createHarness(); - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [segment("ask-current", "Question")], - status: "ready", - }); - harness.bridge.start(3); - harness.emit(toolDelta(3, '{"answer":"First"}')); + startReady(harness, 2); - harness.emit({ - ...toolDelta(3, '{"answer":"Second"}'), - callId: "call-2", - itemId: "function-item-2", - }); + harness.emit(completedTranscript(1, "Stale answer")); + harness.emit(failedTranscript(1, "stale-failed")); expect(harness.submitInterviewAnswer).not.toHaveBeenCalled(); - expect(harness.events).toEqual([ - expect.objectContaining({ type: "error" }), - ]); + expect(harness.events).toEqual([]); }); - test("discards a cancelled argument stream without poisoning the next answer", async () => { + test("correlates the admitted submission with exact canonical response segments", async () => { const harness = createHarness(); - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [segment("ask-current", "Question")], - status: "ready", - }); - harness.bridge.start(3); - harness.emit(toolDelta(3, '{"answer":"Cancelled"}')); - harness.emit(responseTerminal(3, "cancelled")); - harness.emit(toolDone(3, '{"answer":"Cancelled"}')); - - harness.emit({ - ...toolDelta(3, '{"answer":"Accepted"}'), - callId: "call-2", - itemId: "function-item-2", - responseId: "response-2", - }); - harness.emit({ - ...toolDone(3, '{"answer":"Accepted"}'), - callId: "call-2", - itemId: "function-item-2", - responseId: "response-2", - }); - + startReady(harness, 7); + harness.emit(completedTranscript(7)); await vi.waitFor(() => expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce(), ); - expect(harness.submitInterviewAnswer).toHaveBeenCalledWith( - expect.objectContaining({ - admissionTarget: { - kind: "client-tool-result", - messageId: "message-ask-current", - }, - id: createRealtimeSubmissionId(3, "call-2"), - text: "Accepted", - }), + const input = harness.submitInterviewAnswer.mock.calls[0]?.[0]; + expect(input).toBeDefined(); + + input?.onAdmission("submission-voice-1"); + harness.bridge.updateChat({ + canAcceptInterviewAnswer: false, + canonicalSegments: [], + status: "submitted", + }); + const unrelated = segment( + "unrelated", + "Do not speak this.", + "submission-other", ); - expect(harness.events).not.toContainEqual( - expect.objectContaining({ type: "error" }), + const correlated = segment( + "correlated", + "Speak this canonical response.", + "submission-voice-1", ); - }); - - test("rejects an unfinished argument stream from a completed response", () => { - const harness = createHarness(); + const correlatedQuestion: CanonicalSpeechSegment = { + ...segment( + "correlated-question", + "Which operator confirms the batch?", + "submission-voice-1", + ), + messageId: correlated.messageId, + source: "assistant-question", + }; harness.bridge.updateChat({ canAcceptInterviewAnswer: true, - canonicalSegments: [segment("ask-current", "Question")], + canonicalSegments: [unrelated, correlated], + questionSegment: correlatedQuestion, status: "ready", }); - harness.bridge.start(3); - harness.emit(toolDelta(3, '{"answer":"Incomplete')); - harness.emit(responseTerminal(3, "completed")); - - expect(harness.submitInterviewAnswer).not.toHaveBeenCalled(); - expect(harness.events).toEqual([ - expect.objectContaining({ - code: "interview-correlation", - type: "error", - }), + const deliveryId = createRealtimeSubmissionId(transcriptKey(7)); + expect(harness.session.speakCanonical).toHaveBeenCalledWith([correlated]); + expect(harness.events.map(({ type }) => type)).toEqual([ + "submission-started", + "submission-admitted", + "submission-accepted", + "canonical-text-ready", + "submission-settled", + "canonical-response-ready", ]); + expect(harness.events.at(-1)).toEqual({ + deliveryId, + questionSegment: correlatedQuestion, + segments: [correlated], + type: "canonical-response-ready", + }); }); - test("rejects duplicate, stale, overlapping, and malformed calls without another Brunch submission", async () => { + test("speaks a completed canonical segment while chat remains streaming and settles separately", async () => { const harness = createHarness(); - const question = segment("ask-current", "What happens after approval?"); - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [question], - status: "ready", - }); - harness.bridge.start(2); - - harness.emit(toolDone(1)); - harness.emit(toolDelta(2, '{"answer":"The supervisor approves it."}')); - harness.emit(toolDone(2)); - harness.emit(toolDone(2)); + startReady(harness, 7); + harness.emit(completedTranscript(7)); await vi.waitFor(() => expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce(), ); - - harness.emit({ - ...toolDone(2, '{"answer":"Overlapping"}'), - callId: "call-2", - itemId: "function-item-2", + const correlated = segment( + "correlated", + "Speak this committed response.", + "submission-voice-1", + ); + harness.bridge.updateChat({ + canAcceptInterviewAnswer: false, + canonicalSegments: [correlated], + status: "streaming", }); - expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce(); - expect(harness.events.at(-1)).toMatchObject({ type: "error" }); - }); + expect(harness.session.speakCanonical).not.toHaveBeenCalled(); + harness.bridge.notifyResponseMessageCompleted( + completedResponseMessage(correlated.messageId, "submission-voice-1", 1), + ); + + expect(harness.session.speakCanonical).toHaveBeenCalledWith([correlated]); + expect(harness.events.map(({ type }) => type)).not.toContain( + "submission-settled", + ); + expect(harness.events.map(({ type }) => type)).not.toContain( + "canonical-response-ready", + ); - test.each([ - ["wrong tool", { ...toolDone(3), name: "invent_question" }], - ["invalid JSON", toolDone(3, "not-json")], - ["extra property", toolDone(3, '{"answer":"Valid","extra":true}')], - ["empty answer", toolDone(3, '{"answer":" "}')], - ])("rejects %s arguments", async (_label, event) => { - const harness = createHarness(); harness.bridge.updateChat({ canAcceptInterviewAnswer: true, - canonicalSegments: [segment("ask-current", "Question")], + canonicalSegments: [correlated], status: "ready", }); - harness.bridge.start(3); - - harness.emit(event); - await Promise.resolve(); - expect(harness.submitInterviewAnswer).not.toHaveBeenCalled(); - expect(harness.events).toEqual([ - expect.objectContaining({ type: "error" }), + expect(harness.session.speakCanonical).toHaveBeenCalledOnce(); + expect(harness.events.slice(-2).map(({ type }) => type)).toEqual([ + "submission-settled", + "canonical-response-ready", ]); }); - test("rejects a composer result that does not match the pending brunch_ask", async () => { + test("does not let a completed reasoning-only or tool-only step authorize later text", async () => { const harness = createHarness(); - harness.submitInterviewAnswer.mockResolvedValueOnce({ - kind: "interactive-tool", - toolCallId: "another-ask", + startReady(harness, 7); + harness.emit(completedTranscript(7)); + await vi.waitFor(() => + expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce(), + ); + harness.bridge.updateChat({ + canAcceptInterviewAnswer: false, + canonicalSegments: [], + status: "streaming", }); + + const messageId = "reasoning-or-tool-message"; + harness.bridge.notifyResponseMessageCompleted( + completedResponseMessage(messageId, "submission-voice-1", 1), + ); harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [segment("ask-current", "Question")], - status: "ready", + canAcceptInterviewAnswer: false, + canonicalSegments: [], + status: "streaming", + }); + harness.bridge.notifyResponseMessageStarted({ + messageId, + position: { batch: 1, index: 2 }, + submissionId: "submission-voice-1", + }); + const laterText = { + ...segment( + "not-yet-completed", + "Do not let the earlier completion authorize this text.", + ), + messageId, + submissionIds: ["submission-voice-1"], + }; + harness.bridge.updateChat({ + canAcceptInterviewAnswer: false, + canonicalSegments: [laterText], + status: "streaming", }); - harness.bridge.start(5); - harness.emit(toolDone(5)); + expect(harness.session.speakCanonical).not.toHaveBeenCalled(); - await vi.waitFor(() => - expect(harness.events.at(-1)).toMatchObject({ type: "error" }), + harness.bridge.notifyResponseMessageCompleted( + completedResponseMessage(messageId, "submission-voice-1", 3), ); - expect(harness.session.completeFunctionCall).not.toHaveBeenCalled(); + expect(harness.session.speakCanonical).toHaveBeenCalledWith([laterText]); }); - test("records first canonical text before the turn settles", async () => { + test("speaks later continuation segments once and in canonical order", async () => { const harness = createHarness(); - const question = segment("ask-current", "Question"); - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [question], - status: "ready", - }); - harness.bridge.start(7); - harness.emit(toolDone(7)); + startReady(harness, 7); + harness.emit(completedTranscript(7)); await vi.waitFor(() => expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce(), ); - harness.submitInterviewAnswer.mock.calls[0]?.[0].onAdmission( - "submission-text", - ); - await vi.waitFor(() => - expect(harness.events).toContainEqual( - expect.objectContaining({ type: "submission-accepted" }), - ), + harness.bridge.updateChat({ + canAcceptInterviewAnswer: false, + canonicalSegments: [], + status: "streaming", + }); + const first = { + ...segment("first", "First committed segment."), + messageId: "assistant-response", + submissionIds: ["submission-voice-1"], + }; + harness.bridge.notifyResponseMessageCompleted( + completedResponseMessage(first.messageId, "submission-voice-1", 1), ); harness.bridge.updateChat({ canAcceptInterviewAnswer: false, - canonicalSegments: [question], + canonicalSegments: [first], status: "streaming", }); - const firstText = { - ...segment("first", "First completed block.", "assistant-text"), - submissionIds: ["submission-text"], + harness.bridge.notifyResponseMessageCompleted( + completedResponseMessage(first.messageId, "submission-voice-1", 1), + ); + + const second = { + ...segment("second", "Second committed segment."), + messageId: first.messageId, + submissionIds: ["submission-voice-1", "submission-continuation"], + }; + const third = { + ...segment("third", "Third committed segment."), + messageId: first.messageId, + submissionIds: ["submission-voice-1", "submission-continuation"], }; harness.bridge.updateChat({ canAcceptInterviewAnswer: false, - canonicalSegments: [question, firstText], + canonicalSegments: [first, second, third], status: "streaming", }); + expect(harness.session.speakCanonical).toHaveBeenCalledTimes(1); - const typesWhileStreaming = harness.events.map(({ type }) => type); - expect(typesWhileStreaming).toContain("canonical-text-ready"); - expect(typesWhileStreaming).not.toContain("submission-settled"); - expect(harness.session.completeFunctionCall).not.toHaveBeenCalled(); + harness.bridge.notifyResponseMessageCompleted( + completedResponseMessage(first.messageId, "submission-continuation", 2), + ); + const fourth = { + ...segment("fourth", "Fourth committed segment."), + messageId: first.messageId, + submissionIds: ["submission-voice-1", "submission-continuation"], + }; harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [question, firstText], - status: "ready", + canAcceptInterviewAnswer: false, + canonicalSegments: [first, second, third, fourth], + status: "streaming", }); - expect( - harness.events.filter(({ type }) => type === "canonical-text-ready"), - ).toHaveLength(1); - expect(harness.events.map(({ type }) => type)).toContain( - "submission-settled", - ); - expect(harness.session.completeFunctionCall).toHaveBeenCalledWith( - "call-1", - [firstText], + harness.bridge.notifyResponseMessageCompleted( + completedResponseMessage(first.messageId, "submission-continuation", 3), ); + + expect(harness.session.speakCanonical.mock.calls).toEqual([ + [[first]], + [[second, third]], + [[fourth]], + ]); }); - test("closes a durably stopped Voice turn without speaking", async () => { + test("does not start speech cancelled while its correlated response is pending", async () => { const harness = createHarness(); - const question = segment("ask-current", "Question"); - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [question], - status: "ready", - }); - harness.bridge.start(7); - harness.emit(toolDone(7)); + startReady(harness, 7); + harness.emit(completedTranscript(7)); await vi.waitFor(() => expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce(), ); - harness.submitInterviewAnswer.mock.calls[0]?.[0].onAdmission( - "submission-stopped", + harness.bridge.updateChat({ + canAcceptInterviewAnswer: false, + canonicalSegments: [], + status: "streaming", + }); + + harness.bridge.cancelPendingSpeech(); + + const correlated = segment( + "correlated", + "Retain this without speaking it.", + "submission-voice-1", ); - await vi.waitFor(() => - expect(harness.events).toContainEqual( - expect.objectContaining({ type: "submission-accepted" }), - ), + harness.bridge.notifyResponseMessageCompleted( + completedResponseMessage(correlated.messageId, "submission-voice-1", 1), ); harness.bridge.updateChat({ canAcceptInterviewAnswer: false, - canonicalSegments: [question], + canonicalSegments: [correlated], status: "streaming", }); - - // A completed step with no text is not a stop: the panel may still be - // sending the client-tool follow-up that carries the reply. harness.bridge.updateChat({ canAcceptInterviewAnswer: true, - canonicalSegments: [question], - settlements: [ - { outcome: "completed", submissionId: "submission-stopped" }, - ], + canonicalSegments: [correlated], status: "ready", }); - expect( - harness.session.completeFunctionCallWithoutResponse, - ).not.toHaveBeenCalled(); - expect(harness.events.map(({ type }) => type)).not.toContain( - "submission-settled", - ); - const speechRequestsBeforeStop = - harness.session.speakCanonical.mock.calls.length; - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [question], - settlements: [{ outcome: "aborted", submissionId: "submission-stopped" }], - status: "ready", + expect(harness.session.speakCanonical).not.toHaveBeenCalled(); + expect(harness.events.at(-1)).toMatchObject({ + segments: [correlated], + speechCancelled: true, + type: "canonical-response-ready", }); - expect( - harness.session.completeFunctionCallWithoutResponse, - ).toHaveBeenCalledWith("call-1", "aborted"); - expect(harness.session.completeFunctionCall).not.toHaveBeenCalled(); - expect(harness.session.speakCanonical).toHaveBeenCalledTimes( - speechRequestsBeforeStop, - ); - expect(harness.events.map(({ type }) => type)).toEqual( - expect.arrayContaining(["submission-settled", "submission-stopped"]), - ); - expect(harness.events.some(({ type }) => type === "error")).toBe(false); }); - test("speaks the folded continuation that answers a Voice brunch_ask follow-up", async () => { + test("does not speak a completed segment from an aborted submission", async () => { const harness = createHarness(); - const question = { - ...segment("ask-current", "Question"), - submissionIds: ["submission-question"], - }; - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [question], - status: "ready", - }); - harness.bridge.start(7); - harness.emit(toolDone(7)); + startReady(harness, 7); + harness.emit(completedTranscript(7)); await vi.waitFor(() => expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce(), ); - harness.submitInterviewAnswer.mock.calls[0]?.[0].onAdmission( - "submission-answer", + const aborted = segment( + "aborted", + "Never speak an aborted response.", + "submission-voice-1", ); - await vi.waitFor(() => - expect(harness.events).toContainEqual( - expect.objectContaining({ type: "submission-accepted" }), - ), + harness.bridge.notifyResponseMessageCompleted( + completedResponseMessage(aborted.messageId, "submission-voice-1", 1), ); harness.bridge.updateChat({ canAcceptInterviewAnswer: false, - canonicalSegments: [question], + canonicalSegments: [aborted], + settlements: [{ outcome: "aborted", submissionId: "submission-voice-1" }], status: "streaming", }); - - // The continuation is projected onto the message that asked, so that - // message is now written by both submissions. - const nextQuestion = { - ...segment("ask-next", "Next question"), - messageId: question.messageId, - submissionIds: ["submission-question", "submission-answer"], - }; - const askedAgain = { - ...question, - submissionIds: nextQuestion.submissionIds, - }; harness.bridge.updateChat({ canAcceptInterviewAnswer: true, - canonicalSegments: [askedAgain, nextQuestion], + canonicalSegments: [aborted], + settlements: [{ outcome: "aborted", submissionId: "submission-voice-1" }], status: "ready", }); - expect(harness.session.completeFunctionCall).toHaveBeenCalledWith( - "call-1", - [nextQuestion], - ); + expect(harness.session.speakCanonical).not.toHaveBeenCalled(); + expect(harness.events.at(-1)).toEqual({ + deliveryId: createRealtimeSubmissionId(transcriptKey(7)), + outcome: "aborted", + type: "submission-stopped", + }); }); - test("speaks a reply that arrives through a client-tool follow-up", async () => { + test("rejects a path-B result that does not preserve the delivery identity", async () => { const harness = createHarness(); harness.submitInterviewAnswer.mockResolvedValueOnce({ kind: "message", - messageId: "message-kickoff", + messageId: "different-message", submissionId: "submission-voice-1", }); - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [], - status: "ready", - }); - harness.bridge.start(7); - harness.emit(toolDone(7, '{"answer":"Read the guide first."}')); + startReady(harness); + + harness.emit(completedTranscript(3)); + await vi.waitFor(() => expect(harness.events).toContainEqual( - expect.objectContaining({ type: "submission-accepted" }), + expect.objectContaining({ + code: "interview-correlation", + type: "error", + }), ), ); - harness.bridge.updateChat({ - canAcceptInterviewAnswer: false, - canonicalSegments: [], - status: "submitted", - }); + }); - // Brunch read a doc mid-turn; the panel's follow-up submission finished - // the same assistant message. - const reply = { - ...segment("reply", "The guide says hello.", "assistant-text"), - submissionIds: ["submission-voice-1", "submission-doc-follow-up"], - }; - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [reply], - status: "ready", - }); + test.each([ + { + code: "admission-rejected", + failure: { kind: "rejected", status: 403 } as const, + message: "Brunch rejected the message before admission (HTTP 403).", + }, + { + code: "admission-conflict", + failure: { + kind: "submission-conflict", + status: 409, + submissionId: "submission-existing", + } as const, + message: + "The delivery key already belongs to admitted submission submission-existing; the changed payload was not admitted.", + }, + { + code: "admission-ambiguous", + failure: { kind: "ambiguous" } as const, + message: + "Brunch may have accepted the message, but admission could not be confirmed. Reopen the conversation before trying again.", + }, + { + code: "admission-aborted", + failure: { kind: "aborted" } as const, + message: "The local chat submission was cancelled.", + }, + ])( + "preserves a $failure.kind admission outcome", + async ({ code, failure, message }) => { + const harness = createHarness(); + harness.submitInterviewAnswer.mockRejectedValueOnce( + new FlueChatAdmissionError(failure), + ); + startReady(harness); + + harness.emit(completedTranscript(3)); + + await vi.waitFor(() => + expect(harness.events).toContainEqual({ + code, + failure, + message, + type: "error", + }), + ); + expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce(); + }, + ); - expect(harness.session.completeFunctionCall).toHaveBeenCalledWith( - "call-1", - [reply], + test("requires a shared chat busy cycle before accepting new canonical text", async () => { + const harness = createHarness(); + startReady(harness); + harness.emit(completedTranscript(3)); + await vi.waitFor(() => + expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce(), + ); + const response = segment( + "response", + "Canonical response.", + "submission-voice-1", ); - }); - test("speaks new canonical text turns without creating a Realtime tool result", () => { - const harness = createHarness(); - const question = segment("ask-current", "Question"); harness.bridge.updateChat({ canAcceptInterviewAnswer: true, - canonicalSegments: [question], + canonicalSegments: [response], status: "ready", }); - harness.bridge.start(8); - harness.session.speakCanonical.mockClear(); - const response = segment( - "typed-response", - "Canonical response", - "assistant-text", - ); + expect(harness.session.speakCanonical).not.toHaveBeenCalled(); + harness.bridge.updateChat({ + canAcceptInterviewAnswer: false, + canonicalSegments: [response], + status: "streaming", + }); harness.bridge.updateChat({ canAcceptInterviewAnswer: true, - canonicalSegments: [question, response], + canonicalSegments: [response], status: "ready", }); expect(harness.session.speakCanonical).toHaveBeenCalledWith([response]); - expect(harness.session.completeFunctionCall).not.toHaveBeenCalled(); }); }); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.ts b/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.ts index a7b0014cc59..e835230d26b 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.ts @@ -1,7 +1,17 @@ +import { FlueChatAdmissionError } from "@hashintel/brunch-agent-transport-aisdk"; + import type { CanonicalSpeechSegment } from "./canonical-speech"; -import type { OpenAIRealtimeSessionEvent } from "./openai-realtime-session"; +import type { + OpenAIRealtimeSessionEvent, + OpenAIRealtimeTranscriptKey, +} from "./openai-realtime-session"; import type { AgentSendResult, FlueConversationSettlement } from "@flue/sdk"; -import type { FlueChatTransportOptions } from "@hashintel/brunch-agent-transport-aisdk"; +import type { + FlueChatAdmissionFailure, + FlueChatResponseMessageCompletedEvent, + FlueChatResponseMessageStartedEvent, + FlueChatTransportOptions, +} from "@hashintel/brunch-agent-transport-aisdk"; import type { PetrinautAiComposerSubmitTextResult, PetrinautAiVoiceModeContext, @@ -15,20 +25,13 @@ export type VoiceSubmissionSettlement = Pick< interface ChatUpdate { readonly canAcceptInterviewAnswer: boolean; readonly canonicalSegments: CanonicalSpeechSegment[]; + readonly questionSegment?: CanonicalSpeechSegment; /** Flue's settlement index: the only witness that a turn ended short of a reply. */ readonly settlements?: readonly VoiceSubmissionSettlement[]; readonly status: PetrinautAiVoiceModeContext["status"]; } interface RealtimeBridgeSession { - completeFunctionCall( - callId: string, - segments: CanonicalSpeechSegment[], - ): void; - completeFunctionCallWithoutResponse( - callId: string, - outcome: Exclude, - ): void; speakCanonical(segments: CanonicalSpeechSegment[]): void; subscribe(listener: (event: OpenAIRealtimeSessionEvent) => void): () => void; } @@ -64,61 +67,77 @@ interface RealtimeBrunchBridgeDependencies { ) => Promise; } +interface CompletedResponseMessage extends FlueChatResponseMessageCompletedEvent { + consumed: boolean; +} + interface ActiveSubmission { readonly abortController: AbortController; readonly baselineSegmentIds: ReadonlySet; - readonly callId: string; - readonly epoch: number; - readonly pendingQuestionId: string | null; - readonly pendingQuestionMessageId: string | null; + readonly completedResponseMessages: CompletedResponseMessage[]; + readonly deliveryId: string; correlated: boolean; firstTextEmitted: boolean; sawBusyChatStatus: boolean; + speechCancelled: boolean; submissionId: AgentSendResult["submissionId"] | null; } -interface ArgumentStream { - readonly chunks: string[]; - readonly itemId: string; - readonly responseId: string; -} +type RealtimeAdmissionErrorCode = + | "admission-aborted" + | "admission-ambiguous" + | "admission-conflict" + | "admission-rejected"; -export type RealtimeBridgeErrorCode = +type RealtimeInterviewErrorCode = | "interview-correlation" | "interview-response" | "interview-submission"; +export type RealtimeBridgeErrorCode = + | RealtimeAdmissionErrorCode + | RealtimeInterviewErrorCode; + +export type RealtimeTranscriptRejectionReason = + | "duplicate" + | "empty" + | "failed" + | "over-limit" + | "unavailable"; + export type RealtimeBrunchBridgeEvent = | { readonly answer: string; - readonly callId: string; + readonly deliveryId: string; readonly type: "submission-started"; } | { readonly answer: string; - readonly callId: string; + readonly deliveryId: string; readonly type: "submission-accepted"; } | { - readonly callId: string; + readonly deliveryId: string; readonly submissionId: AgentSendResult["submissionId"]; readonly type: "submission-admitted"; } | { - readonly callId: string; + readonly deliveryId: string; + readonly questionSegment?: CanonicalSpeechSegment; readonly segments: CanonicalSpeechSegment[]; + readonly speechCancelled?: true; readonly type: "canonical-response-ready"; } | { - readonly callId: string; + readonly deliveryId: string; readonly type: "canonical-text-ready"; } | { - readonly callId: string; + readonly deliveryId: string; readonly type: "submission-settled"; } | { - readonly callId: string; + readonly deliveryId: string; readonly outcome: Exclude< VoiceSubmissionSettlement["outcome"], "completed" @@ -126,7 +145,17 @@ export type RealtimeBrunchBridgeEvent = readonly type: "submission-stopped"; } | { - readonly code: RealtimeBridgeErrorCode; + readonly reason: RealtimeTranscriptRejectionReason; + readonly type: "transcript-rejected"; + } + | { + readonly code: RealtimeInterviewErrorCode; + readonly message: string; + readonly type: "error"; + } + | { + readonly code: RealtimeAdmissionErrorCode; + readonly failure: FlueChatAdmissionFailure; readonly message: string; readonly type: "error"; }; @@ -137,45 +166,51 @@ const INVALID_BRIDGE_EVENT = "The voice response could not be matched to the interview. Reconnect voice or use text instead."; const ANSWER_LIMIT = 32_000; -export const createRealtimeSubmissionId = ( - connectionEpoch: number, - callId: string, -): string => `voice-realtime:${connectionEpoch}:${encodeURIComponent(callId)}`; - -const latestPendingQuestion = ( - segments: CanonicalSpeechSegment[], -): CanonicalSpeechSegment | undefined => - segments.findLast(({ source }) => source === "brunch-ask"); - -const parseContinueInterviewArguments = ( - argumentsJson: string, -): string | null => { - try { - const value: unknown = JSON.parse(argumentsJson); - if (typeof value !== "object" || value === null || Array.isArray(value)) { - return null; - } - const record = value as Record; - if (Object.keys(record).length !== 1 || typeof record.answer !== "string") { - return null; - } - const answer = record.answer.trim(); - return answer && Array.from(answer).length <= ANSWER_LIMIT ? answer : null; - } catch { - return null; +export const createRealtimeSubmissionId = ({ + connectionEpoch, + contentIndex, + itemId, +}: OpenAIRealtimeTranscriptKey): string => + `voice-realtime:${connectionEpoch}:${encodeURIComponent(itemId)}:${contentIndex}`; + +const transcriptKeyId = (key: OpenAIRealtimeTranscriptKey): string => + createRealtimeSubmissionId(key); + +const normalizeTranscript = (transcript: string): string => + transcript.trim().replace(/\s+/gu, " "); + +const positionPrecedes = ( + first: FlueChatResponseMessageCompletedEvent["position"], + second: FlueChatResponseMessageStartedEvent["position"], +): boolean => + first.batch < second.batch || + (first.batch === second.batch && first.index < second.index); + +const admissionErrorCode = ( + failure: FlueChatAdmissionFailure, +): RealtimeAdmissionErrorCode => { + switch (failure.kind) { + case "aborted": + return "admission-aborted"; + case "ambiguous": + return "admission-ambiguous"; + case "rejected": + return "admission-rejected"; + case "submission-conflict": + return "admission-conflict"; } }; export class RealtimeBrunchBridge { - readonly #argumentDeltas = new Map(); + readonly #acceptedInputItemIds = new Set(); readonly #listeners = new Set(); - readonly #processedCalls = new Set(); + readonly #playbackOverlappingInputItemIds = new Set(); + readonly #processedTranscripts = new Set(); readonly #session: RealtimeBridgeSession; readonly #submitInterviewAnswer: ( input: SubmitInterviewAnswerInput, ) => Promise; readonly #seenSegmentIds = new Set(); - readonly #terminalResponseIds = new Set(); #activeEpoch: number | null = null; #activeSubmission: ActiveSubmission | null = null; #chat: ChatUpdate = { @@ -184,6 +219,7 @@ export class RealtimeBrunchBridge { status: "ready", }; #generation = 0; + #outputActive = false; public constructor({ session, @@ -199,27 +235,68 @@ export class RealtimeBrunchBridge { return () => this.#listeners.delete(listener); } + public cancelPendingSpeech(): void { + if (this.#activeSubmission) { + this.#activeSubmission.speechCancelled = true; + } + } + + public completeTurnHandoff(): void { + this.#outputActive = false; + } + + public notifyResponseMessageCompleted( + event: FlueChatResponseMessageCompletedEvent, + ): void { + const active = this.#activeSubmission; + if ( + active === null || + active.completedResponseMessages.some( + ({ position }) => + position.batch === event.position.batch && + position.index === event.position.index, + ) + ) { + return; + } + active.completedResponseMessages.push({ + ...event, + consumed: false, + }); + this.#completeCorrelatedSubmission(); + } + + public notifyResponseMessageStarted( + event: FlueChatResponseMessageStartedEvent, + ): void { + const active = this.#activeSubmission; + if (active === null) { + return; + } + for (const completion of active.completedResponseMessages) { + if ( + !completion.consumed && + completion.messageId === event.messageId && + positionPrecedes(completion.position, event.position) + ) { + completion.consumed = true; + } + } + } + public start(connectionEpoch: number): void { ++this.#generation; this.#activeSubmission?.abortController.abort(); this.#activeEpoch = connectionEpoch; this.#activeSubmission = null; - this.#argumentDeltas.clear(); - this.#processedCalls.clear(); + this.#acceptedInputItemIds.clear(); + this.#playbackOverlappingInputItemIds.clear(); + this.#processedTranscripts.clear(); + this.#outputActive = false; this.#seenSegmentIds.clear(); - this.#terminalResponseIds.clear(); for (const segment of this.#chat.canonicalSegments) { this.#seenSegmentIds.add(segment.id); } - - const question = latestPendingQuestion(this.#chat.canonicalSegments); - if (question) { - this.#session.speakCanonical( - this.#chat.canonicalSegments.filter( - ({ messageId }) => messageId === question.messageId, - ), - ); - } } public stop(): void { @@ -227,8 +304,10 @@ export class RealtimeBrunchBridge { this.#activeSubmission?.abortController.abort(); this.#activeEpoch = null; this.#activeSubmission = null; - this.#argumentDeltas.clear(); - this.#terminalResponseIds.clear(); + this.#acceptedInputItemIds.clear(); + this.#playbackOverlappingInputItemIds.clear(); + this.#processedTranscripts.clear(); + this.#outputActive = false; } public updateChat(update: ChatUpdate): void { @@ -276,167 +355,144 @@ export class RealtimeBrunchBridge { } } + #rejectTranscript(reason: RealtimeTranscriptRejectionReason): void { + this.#emit({ reason, type: "transcript-rejected" }); + } + #fail( message: string, - code: RealtimeBridgeErrorCode = "interview-correlation", + code: RealtimeInterviewErrorCode = "interview-correlation", ): void { ++this.#generation; this.#activeSubmission?.abortController.abort(); this.#activeSubmission = null; - this.#argumentDeltas.clear(); this.#emit({ code, message, type: "error" }); } + #failAdmission(error: FlueChatAdmissionError): void { + ++this.#generation; + this.#activeSubmission?.abortController.abort(); + this.#activeSubmission = null; + this.#emit({ + code: admissionErrorCode(error.failure), + failure: error.failure, + message: error.message, + type: "error", + }); + } + #handleSessionEvent(event: OpenAIRealtimeSessionEvent): void { if ( - !("connectionEpoch" in event) || + "connectionEpoch" in event && event.connectionEpoch !== this.#activeEpoch ) { return; } - if (event.type === "response-terminal") { - this.#handleResponseTerminal(event); + if (event.type === "input-speech-started") { + if (this.#outputActive) { + this.#playbackOverlappingInputItemIds.add(event.itemId); + } else { + this.#acceptedInputItemIds.add(event.itemId); + } return; } if ( - event.type !== "tool-arguments-delta" && - event.type !== "tool-arguments-done" + event.type === "canonical-speech-requested" || + event.type === "output-started" ) { + this.#outputActive = true; + for (const itemId of this.#acceptedInputItemIds) { + this.#playbackOverlappingInputItemIds.add(itemId); + } + this.#acceptedInputItemIds.clear(); return; } - - const responseKey = `${event.connectionEpoch}:${event.responseId}`; - if (this.#terminalResponseIds.has(responseKey)) { + if ( + event.type === "output-stopped" || + event.type === "output-interrupted" + ) { + this.#outputActive = false; return; } - const callKey = `${event.connectionEpoch}:${event.callId}`; - if (this.#processedCalls.has(callKey)) { + if (event.type !== "completed" && event.type !== "transcription-failed") { return; } - if (event.type === "tool-arguments-delta") { - const stream = this.#argumentDeltas.get(callKey); - if (!stream && this.#argumentDeltas.size > 0) { - this.#processedCalls.add(callKey); - this.#fail(INVALID_BRIDGE_EVENT); - return; - } - if ( - stream && - (stream.itemId !== event.itemId || - stream.responseId !== event.responseId) - ) { - this.#processedCalls.add(callKey); - this.#fail(INVALID_BRIDGE_EVENT); - return; - } - if (stream) { - stream.chunks.push(event.delta); - } else { - this.#argumentDeltas.set(callKey, { - chunks: [event.delta], - itemId: event.itemId, - responseId: event.responseId, - }); - } + if (event.key.connectionEpoch !== this.#activeEpoch) { return; } - this.#processedCalls.add(callKey); - const stream = this.#argumentDeltas.get(callKey); - if (!stream && this.#argumentDeltas.size > 0) { - this.#fail(INVALID_BRIDGE_EVENT); + const keyId = transcriptKeyId(event.key); + if (this.#processedTranscripts.has(keyId)) { + this.#rejectTranscript("duplicate"); return; } - this.#argumentDeltas.delete(callKey); - if ( - this.#activeSubmission || - event.name !== "continue_interview" || - (stream !== undefined && - (stream.itemId !== event.itemId || - stream.responseId !== event.responseId || - stream.chunks.join("") !== event.arguments)) - ) { - this.#fail(INVALID_BRIDGE_EVENT); + this.#processedTranscripts.add(keyId); + this.#acceptedInputItemIds.delete(event.key.itemId); + + if (this.#playbackOverlappingInputItemIds.has(event.key.itemId)) { + this.#rejectTranscript("unavailable"); return; } - const answer = parseContinueInterviewArguments(event.arguments); - const question = latestPendingQuestion(this.#chat.canonicalSegments); + if (event.type === "transcription-failed") { + this.#rejectTranscript("failed"); + return; + } if ( - !answer || + this.#activeSubmission || !this.#chat.canAcceptInterviewAnswer || - (!question && this.#chat.status !== "ready") + this.#chat.status !== "ready" ) { - this.#fail(INVALID_BRIDGE_EVENT); + this.#rejectTranscript("unavailable"); + return; + } + + const answer = normalizeTranscript(event.text); + if (answer.length === 0) { + this.#rejectTranscript("empty"); + return; + } + if (Array.from(answer).length > ANSWER_LIMIT) { + this.#rejectTranscript("over-limit"); return; } + const deliveryId = createRealtimeSubmissionId(event.key); const generation = this.#generation; this.#activeSubmission = { abortController: new AbortController(), baselineSegmentIds: new Set( this.#chat.canonicalSegments.map(({ id }) => id), ), - callId: event.callId, + completedResponseMessages: [], correlated: false, - epoch: event.connectionEpoch, + deliveryId, firstTextEmitted: false, - pendingQuestionId: question?.partId ?? null, - pendingQuestionMessageId: question?.messageId ?? null, sawBusyChatStatus: false, + speechCancelled: false, submissionId: null, }; - this.#emit({ answer, callId: event.callId, type: "submission-started" }); - void this.#submit(event, answer, generation); - } - - #handleResponseTerminal( - event: Extract, - ): void { - const responseKey = `${event.connectionEpoch}:${event.responseId}`; - const matchingStreams = [...this.#argumentDeltas].filter( - ([, stream]) => stream.responseId === event.responseId, - ); - if (event.status === "completed" && matchingStreams.length > 0) { - this.#fail(INVALID_BRIDGE_EVENT); - return; - } - - for (const [callKey] of matchingStreams) { - this.#argumentDeltas.delete(callKey); - this.#processedCalls.add(callKey); - } - this.#terminalResponseIds.add(responseKey); + this.#emit({ answer, deliveryId, type: "submission-started" }); + void this.#submit(answer, deliveryId, generation); } async #submit( - event: Extract, answer: string, + deliveryId: string, generation: number, ): Promise { try { const activeAtSubmission = this.#activeSubmission; if (!activeAtSubmission) return; - const voiceMessageId = createRealtimeSubmissionId( - event.connectionEpoch, - event.callId, - ); const result = await this.#submitInterviewAnswer({ - admissionTarget: - activeAtSubmission.pendingQuestionMessageId === null - ? { kind: "user", messageId: voiceMessageId } - : { - kind: "client-tool-result", - messageId: activeAtSubmission.pendingQuestionMessageId, - }, - id: voiceMessageId, + admissionTarget: { kind: "user", messageId: deliveryId }, + id: deliveryId, onAdmission: (submissionId) => { const active = this.#activeSubmission; if ( generation !== this.#generation || !active || - active.callId !== event.callId || - active.epoch !== event.connectionEpoch + active.deliveryId !== deliveryId ) { return; } @@ -448,7 +504,7 @@ export class RealtimeBrunchBridge { } active.submissionId = submissionId; this.#emit({ - callId: event.callId, + deliveryId, submissionId, type: "submission-admitted", }); @@ -460,22 +516,15 @@ export class RealtimeBrunchBridge { if ( generation !== this.#generation || !active || - active.callId !== event.callId || - active.epoch !== event.connectionEpoch + active.deliveryId !== deliveryId ) { return; } - const resultMatchesSubmission = - active.pendingQuestionId === null - ? result.kind === "message" - : result.kind === "interactive-tool" && - result.toolCallId === active.pendingQuestionId; - if (!resultMatchesSubmission) { + if (result.kind !== "message" || result.messageId !== deliveryId) { this.#fail(INVALID_BRIDGE_EVENT); return; } - const resultSubmissionId = - result.kind === "message" ? (result.submissionId ?? null) : null; + const resultSubmissionId = result.submissionId ?? null; if ( active.submissionId !== null && resultSubmissionId !== null && @@ -486,18 +535,18 @@ export class RealtimeBrunchBridge { } active.submissionId ??= resultSubmissionId; active.correlated = true; - this.#emit({ - answer, - callId: event.callId, - type: "submission-accepted", - }); + this.#emit({ answer, deliveryId, type: "submission-accepted" }); this.#completeCorrelatedSubmission(); - } catch { + } catch (error) { if (generation === this.#generation) { - this.#fail( - "The interview could not accept that answer. Use the composer to retry.", - "interview-submission", - ); + if (error instanceof FlueChatAdmissionError) { + this.#failAdmission(error); + } else { + this.#fail( + "The interview could not accept that answer. Use the composer to retry.", + "interview-submission", + ); + } } } } @@ -521,7 +570,61 @@ export class RealtimeBrunchBridge { // Completed canonical text can land while the turn is still streaming; // record that instant separately from settlement. active.firstTextEmitted = true; - this.#emit({ callId: active.callId, type: "canonical-text-ready" }); + this.#emit({ + deliveryId: active.deliveryId, + type: "canonical-text-ready", + }); + } + const stoppedSettlement = + active.submissionId === null + ? undefined + : this.#chat.settlements?.find( + ({ submissionId }) => submissionId === active.submissionId, + ); + if (stoppedSettlement && stoppedSettlement.outcome !== "completed") { + if (this.#chat.status === "ready") { + this.#completeStoppedSubmission(active); + } + return; + } + const completionMatchesSegment = ( + completion: CompletedResponseMessage, + segment: CanonicalSpeechSegment, + ): boolean => + completion.messageId === segment.messageId && + (segment.submissionIds?.includes(completion.submissionId) ?? false); + const pendingCompletions = active.completedResponseMessages.filter( + ({ consumed }) => !consumed, + ); + const eligibleCompletions = pendingCompletions.filter((completion) => + responseSegments.some( + (segment) => + !this.#seenSegmentIds.has(segment.id) && + completionMatchesSegment(completion, segment), + ), + ); + const completedSegments = responseSegments.filter( + (segment) => + !this.#seenSegmentIds.has(segment.id) && + eligibleCompletions.some((completion) => + completionMatchesSegment(completion, segment), + ), + ); + if (!active.speechCancelled) { + if (completedSegments.length > 0) { + try { + this.#session.speakCanonical(completedSegments); + for (const segment of completedSegments) { + this.#seenSegmentIds.add(segment.id); + } + } catch { + this.#fail(INVALID_BRIDGE_EVENT); + return; + } + } + } + for (const completion of eligibleCompletions) { + completion.consumed = true; } if (this.#chat.status !== "ready") { return; @@ -531,20 +634,42 @@ export class RealtimeBrunchBridge { return; } - this.#emit({ callId: active.callId, type: "submission-settled" }); - try { - this.#session.completeFunctionCall(active.callId, responseSegments); - } catch { - this.#fail(INVALID_BRIDGE_EVENT); - return; + this.#emit({ + deliveryId: active.deliveryId, + type: "submission-settled", + }); + if (!active.speechCancelled) { + const unscheduledSegments = responseSegments.filter( + ({ id }) => !this.#seenSegmentIds.has(id), + ); + if (unscheduledSegments.length > 0) { + try { + this.#session.speakCanonical(unscheduledSegments); + } catch { + this.#fail(INVALID_BRIDGE_EVENT); + return; + } + } } for (const segment of responseSegments) { this.#seenSegmentIds.add(segment.id); } + const questionSegment = this.#chat.questionSegment; + const correlatedQuestion = + questionSegment && + responseSegments.some( + ({ messageId }) => messageId === questionSegment.messageId, + ) && + (active.submissionId === null || + (questionSegment.submissionIds?.includes(active.submissionId) ?? false)) + ? questionSegment + : undefined; this.#activeSubmission = null; this.#emit({ - callId: active.callId, + deliveryId: active.deliveryId, + ...(correlatedQuestion ? { questionSegment: correlatedQuestion } : {}), segments: responseSegments, + ...(active.speechCancelled ? { speechCancelled: true as const } : {}), type: "canonical-response-ready", }); } @@ -563,19 +688,13 @@ export class RealtimeBrunchBridge { if (settlement === undefined || settlement.outcome === "completed") { return; } - this.#emit({ callId: active.callId, type: "submission-settled" }); - try { - this.#session.completeFunctionCallWithoutResponse( - active.callId, - settlement.outcome, - ); - } catch { - this.#fail(INVALID_BRIDGE_EVENT); - return; - } + this.#emit({ + deliveryId: active.deliveryId, + type: "submission-settled", + }); this.#activeSubmission = null; this.#emit({ - callId: active.callId, + deliveryId: active.deliveryId, outcome: settlement.outcome, type: "submission-stopped", }); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx index 6e94c159fe0..632a4332269 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx @@ -12,6 +12,8 @@ import { import { StrictMode, useState } from "react"; import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; +import { FlueChatAdmissionError } from "@hashintel/brunch-agent-transport-aisdk"; + import { OpenAIRealtimeSession } from "./openai-realtime-session"; import { acknowledgeVoiceInterviewDisclosure, @@ -265,10 +267,55 @@ describe("voice interview control", () => { abortController.abort(); - await expect(resultPromise).rejects.toMatchObject({ name: "AbortError" }); + await expect(resultPromise).rejects.toMatchObject({ + failure: { kind: "aborted" }, + name: "FlueChatAdmissionError", + }); expect(unsubscribe).toHaveBeenCalledOnce(); }); + test("preserves a typed failure reported after the panel submission resolves", async () => { + const admissionError = new FlueChatAdmissionError({ kind: "ambiguous" }); + let reportFailure: ((error: FlueChatAdmissionError) => void) | undefined; + const unsubscribeFromAdmission = vi.fn(); + const unsubscribeFromFailure = vi.fn(); + const resultPromise = submitVoiceInputWithAdmission({ + input: { + admissionTarget: { kind: "user", messageId: "voice-turn-1" }, + id: "voice-turn-1", + onAdmission: vi.fn(), + signal: new AbortController().signal, + text: "One Voice turn.", + }, + submitVoiceInput: async () => ({ + kind: "message", + messageId: "voice-turn-1", + }), + subscribeToAdmission: () => unsubscribeFromAdmission, + subscribeToAdmissionFailure: (_target, listener) => { + reportFailure = listener; + return unsubscribeFromFailure; + }, + }); + let settled = false; + void resultPromise.then( + () => { + settled = true; + }, + () => { + settled = true; + }, + ); + await Promise.resolve(); + expect(settled).toBe(false); + + reportFailure?.(admissionError); + + await expect(resultPromise).rejects.toBe(admissionError); + expect(unsubscribeFromAdmission).toHaveBeenCalledOnce(); + expect(unsubscribeFromFailure).toHaveBeenCalledOnce(); + }); + test("stores and reads the versioned disclosure acknowledgement", () => { const values = new Map(); const storage = { @@ -324,7 +371,7 @@ describe("voice interview control", () => { await expect(loadOpenAIVoiceConfig(fetch)).resolves.toBeNull(); }); - test("keeps the first-use disclosure inline without a text-handoff action", () => { + test("keeps the first-use disclosure inline without a text-handoff action", async () => { render(); fireEvent.click(screen.getByRole("button", { name: "Select Voice" })); @@ -332,16 +379,33 @@ describe("voice interview control", () => { const disclosure = screen.getByRole("region", { name: "Voice mode consent", }); - expect(disclosure).not.toBeNull(); - expect(within(disclosure).getByText("Voice mode")).not.toBeNull(); expect( - screen.getByText("OpenAI processes live audio", { exact: false }), + within(disclosure).getByText("Start a voice conversation"), ).not.toBeNull(); expect( - screen - .getByRole("button", { name: "Start voice mode" }) - .hasAttribute("disabled"), - ).toBe(true); + within(disclosure).getByText( + "OpenAI processes live audio and speaks the interviewer’s words. Petrinaut saves finalized answers—not audio.", + ), + ).not.toBeNull(); + + const consent = within(disclosure).getByRole("checkbox", { + name: "I understand how voice data is handled.", + }); + const start = within(disclosure).getByRole("button", { + name: "Start voice", + }); + expect(start.hasAttribute("disabled")).toBe(true); + fireEvent.click(consent); + await waitFor(() => + expect( + within(disclosure) + .getByRole("button", { name: "Start voice" }) + .hasAttribute("disabled"), + ).toBe(false), + ); + expect( + within(disclosure).getByRole("button", { name: "Test microphone" }), + ).not.toBeNull(); expect( screen.queryByRole("button", { name: "Use text instead" }), ).toBeNull(); @@ -361,7 +425,14 @@ describe("voice interview control", () => { fireEvent.click(screen.getByRole("button", { name: "Select Voice" })); fireEvent.click(screen.getByRole("checkbox")); - fireEvent.click(screen.getByRole("button", { name: "Start voice mode" })); + await waitFor(() => + expect( + screen + .getByRole("button", { name: "Start voice" }) + .hasAttribute("disabled"), + ).toBe(false), + ); + fireEvent.click(screen.getByRole("button", { name: "Start voice" })); expect(await screen.findByText("Session: error")).not.toBeNull(); expect(screen.getByText("Voice active")).not.toBeNull(); @@ -377,6 +448,75 @@ describe("voice interview control", () => { expect(screen.getByText("Session: error")).not.toBeNull(); }); + test("keeps one microphone check pending and reports its result", async () => { + let resolveCheck: ((stream: MediaStream) => void) | undefined; + const getUserMedia = vi.fn( + () => + new Promise((resolve) => { + resolveCheck = resolve; + }), + ); + vi.stubGlobal("navigator", { mediaDevices: { getUserMedia } }); + render(); + + fireEvent.click(screen.getByRole("button", { name: "Select Voice" })); + const check = screen.getByRole("button", { name: "Test microphone" }); + fireEvent.click(check); + fireEvent.click(check); + + expect(getUserMedia).toHaveBeenCalledOnce(); + expect(check.getAttribute("aria-busy")).toBe("true"); + + resolveCheck?.({ getTracks: () => [] } as unknown as MediaStream); + + expect(await screen.findByText("Microphone ready.")).not.toBeNull(); + await waitFor(() => expect(check.getAttribute("aria-busy")).toBe("false")); + }); + + test.each([ + { + failure: "media devices are missing", + stubMedia: () => vi.stubGlobal("navigator", {}), + }, + { + failure: "getUserMedia throws synchronously", + stubMedia: () => + vi.stubGlobal("navigator", { + mediaDevices: { + getUserMedia: () => { + throw new DOMException("Unavailable", "NotSupportedError"); + }, + }, + }), + }, + ])( + "reports an accessible microphone failure when $failure", + async ({ stubMedia }) => { + stubMedia(); + render(); + + fireEvent.click(screen.getByRole("button", { name: "Select Voice" })); + const disclosure = screen.getByRole("region", { + name: "Voice mode consent", + }); + const check = within(disclosure).getByRole("button", { + name: "Test microphone", + }); + + fireEvent.click(check); + + const status = await within(disclosure).findByText( + "Microphone access was not available.", + ); + expect(status.getAttribute("aria-live")).toBe("polite"); + expect(status.getAttribute("aria-atomic")).toBe("true"); + expect(check.getAttribute("aria-describedby")).toBe(status.id); + await waitFor(() => + expect(check.getAttribute("aria-busy")).toBe("false"), + ); + }, + ); + test("starts directly after acknowledgement and ends through the registered control", async () => { window.localStorage.setItem( VOICE_INTERVIEW_DISCLOSURE_STORAGE_KEY, @@ -401,6 +541,18 @@ describe("voice interview control", () => { expect(screen.getByText("Voice inactive")).not.toBeNull(); }); + test("registers replay controls that remain snapshot-gated", async () => { + render(); + + await waitFor(() => expect(registeredVoiceModeControls).toBeDefined()); + + expect(registeredVoiceModeControls?.readFullResponse).toBeTypeOf( + "function", + ); + expect(registeredVoiceModeControls?.takeTurn).toBeTypeOf("function"); + expect(registeredVoiceModeControls?.repeatQuestion).toBeTypeOf("function"); + }); + test("restarts when Voice is reselected before teardown completes", async () => { window.localStorage.setItem( VOICE_INTERVIEW_DISCLOSURE_STORAGE_KEY, @@ -494,18 +646,25 @@ describe("voice interview control", () => { expect(screen.getByText("Panel closed")).not.toBeNull(); }); - test("records acknowledgement only when the interview starts", () => { + test("records acknowledgement only when the interview starts", async () => { stubUnavailableMicrophone(); render(); fireEvent.click(screen.getByRole("button", { name: "Select Voice" })); - fireEvent.click(screen.getByRole("button", { name: "Check microphone" })); + fireEvent.click(screen.getByRole("button", { name: "Test microphone" })); expect( window.localStorage.getItem(VOICE_INTERVIEW_DISCLOSURE_STORAGE_KEY), ).toBeNull(); fireEvent.click(screen.getByRole("checkbox")); - fireEvent.click(screen.getByRole("button", { name: "Start voice mode" })); + await waitFor(() => + expect( + screen + .getByRole("button", { name: "Start voice" }) + .hasAttribute("disabled"), + ).toBe(false), + ); + fireEvent.click(screen.getByRole("button", { name: "Start voice" })); expect( window.localStorage.getItem(VOICE_INTERVIEW_DISCLOSURE_STORAGE_KEY), ).toBe("acknowledged"); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx index 5206ac7d8c1..d9534d4104e 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx @@ -6,11 +6,16 @@ import { useSyncExternalStore, } from "react"; -import { Button } from "@hashintel/ds-components"; +import { + FlueChatAdmissionError, + type FlueChatResponseMessageCompletedEvent, + type FlueChatResponseMessageStartedEvent, +} from "@hashintel/brunch-agent-transport-aisdk"; +import { Button, Checkbox } from "@hashintel/ds-components"; import { css } from "@hashintel/ds-helpers/css"; import { reportVoiceDiagnostic } from "../../../voice-diagnostics"; -import { selectCanonicalSpeechSegments } from "./canonical-speech"; +import { selectCanonicalSpeech } from "./canonical-speech"; import { OpenAIRealtimeSession } from "./openai-realtime-session"; import { RealtimeBrunchBridge, @@ -24,6 +29,7 @@ import { type VoiceTurnSnapshot, } from "./voice-turn-controller"; +import type { CanonicalSpeechSegment } from "./canonical-speech"; import type { AgentSendResult } from "@flue/sdk"; import type { PetrinautAiVoiceModeContext } from "@hashintel/petrinaut/ui"; @@ -37,6 +43,17 @@ type SubscribeToAdmission = ( target: RealtimeBrunchAdmissionTarget, listener: (submissionId: AgentSendResult["submissionId"]) => void, ) => () => void; +type SubscribeToAdmissionFailure = ( + target: RealtimeBrunchAdmissionTarget, + listener: (error: FlueChatAdmissionError) => void, +) => () => void; +type SubscribeToResponseMessageCompleted = ( + listener: (event: FlueChatResponseMessageCompletedEvent) => void, +) => () => void; +type SubscribeToResponseMessageStarted = ( + listener: (event: FlueChatResponseMessageStartedEvent) => void, +) => () => void; +type SubscribeToStopRequested = (listener: () => void) => () => void; type SubmitInterviewAnswer = ConstructorParameters< typeof RealtimeBrunchBridge >[0]["submitInterviewAnswer"]; @@ -48,17 +65,20 @@ export const submitVoiceInputWithAdmission = async ({ resolveInputSubmission, submitVoiceInput, subscribeToAdmission, + subscribeToAdmissionFailure, }: { readonly input: SubmitInterviewAnswerInput; readonly resolveInputSubmission?: ResolveSubmission; readonly submitVoiceInput: PetrinautAiVoiceModeContext["submitVoiceInput"]; readonly subscribeToAdmission?: SubscribeToAdmission; + readonly subscribeToAdmissionFailure?: SubscribeToAdmissionFailure; }): Promise => { let unsubscribe = () => {}; + let unsubscribeFromFailure = () => {}; let removeAbortListener = () => {}; const cancelled = new Promise((_resolve, reject) => { const rejectForAbort = () => - reject(new DOMException("Voice admission cancelled", "AbortError")); + reject(new FlueChatAdmissionError({ kind: "aborted" })); if (input.signal.aborted) { rejectForAbort(); return; @@ -68,16 +88,25 @@ export const submitVoiceInputWithAdmission = async ({ input.signal.removeEventListener("abort", rejectForAbort); }); const admissionObserved = - subscribeToAdmission === undefined + subscribeToAdmission === undefined && + subscribeToAdmissionFailure === undefined ? Promise.resolve() - : new Promise((resolve) => { - unsubscribe = subscribeToAdmission( - input.admissionTarget, - (submissionId) => { - input.onAdmission(submissionId); - resolve(); - }, - ); + : new Promise((resolve, reject) => { + if (subscribeToAdmission !== undefined) { + unsubscribe = subscribeToAdmission( + input.admissionTarget, + (submissionId) => { + input.onAdmission(submissionId); + resolve(); + }, + ); + } + if (subscribeToAdmissionFailure !== undefined) { + unsubscribeFromFailure = subscribeToAdmissionFailure( + input.admissionTarget, + reject, + ); + } }); try { const [result] = await Promise.race([ @@ -96,6 +125,7 @@ export const submitVoiceInputWithAdmission = async ({ } finally { removeAbortListener(); unsubscribe(); + unsubscribeFromFailure(); } }; @@ -186,31 +216,82 @@ export const loadOpenAIVoiceConfig = async ( } }; -const disclosureStyle = css({ - display: "flex", +const VoiceModeIcon = () => ( + +); + +const disclosureFrameStyle = css({ width: "full", - flexDirection: "column", - gap: "2", - paddingX: "2", - paddingY: "2", + padding: "2", borderTopWidth: "thin", borderTopStyle: "solid", borderTopColor: "neutral.a20", + backgroundColor: "neutral.bg.subtle", color: "neutral.s100", + _focus: { outline: "none" }, +}); + +const disclosureCardStyle = css({ + display: "flex", + flexDirection: "column", + gap: "2", + padding: "3", + borderWidth: "thin", + borderStyle: "solid", + borderColor: "neutral.a20", + borderRadius: "xl", + backgroundColor: "neutral.s00", + boxShadow: + "[0px 0px 0px 1px rgba(0,0,0,0.03), 0px 8px 16px -12px rgba(0,0,0,0.18)]", +}); + +const disclosureHeaderStyle = css({ + display: "flex", + alignItems: "center", + gap: "2", +}); + +const disclosureIconStyle = css({ + display: "inline-flex", + width: "7", + height: "7", + flexShrink: "0", + alignItems: "center", + justifyContent: "center", + borderRadius: "lg", + backgroundColor: "blue.a20", + color: "blue.s90", }); const disclosureTitleStyle = css({ display: "flex", + minWidth: "[0]", flexDirection: "column", - gap: "1", + gap: "0.5", +}); + +const disclosureHeadingStyle = css({ fontSize: "sm", fontWeight: "semibold", + lineHeight: "tight", }); const disclosureSubtitleStyle = css({ color: "neutral.s80", fontSize: "xs", - fontWeight: "normal", }); const disclosureCopyStyle = css({ @@ -219,6 +300,14 @@ const disclosureCopyStyle = css({ lineHeight: "relaxed", }); +const disclosureConsentStyle = css({ + width: "full", + padding: "2", + borderRadius: "lg", + backgroundColor: "neutral.a10", + color: "neutral.s100", +}); + const disclosureActionsStyle = css({ display: "flex", flexWrap: "wrap", @@ -226,13 +315,22 @@ const disclosureActionsStyle = css({ gap: "2", }); +const disclosureStatusStyle = css({ + minHeight: "[18px]", + color: "neutral.s80", + fontSize: "xs", + lineHeight: "relaxed", +}); + const VoiceInterviewDisclosure = ({ + checkingMicrophone, consented, microphoneCheck, onCheckMicrophone, onConsentChange, onStart, }: { + readonly checkingMicrophone: boolean; readonly consented: boolean; readonly microphoneCheck: string; readonly onCheckMicrophone: () => void; @@ -248,40 +346,65 @@ const VoiceInterviewDisclosure = ({ return (
-
- Voice mode - - Talk through your process with AI - -
-

- OpenAI processes live audio and speaks the interviewer’s words. - Petrinaut keeps finalized answers in this conversation, not the audio. -

- - {microphoneCheck && ( -

- {microphoneCheck} +

+
+ + + +
+ + Start a voice conversation + + + Talk through your process with AI + +
+
+

+ OpenAI processes live audio and speaks the interviewer’s words. + Petrinaut saves finalized answers—not audio.

- )} -
- - + +
+ + +
+
+ {microphoneCheck} +
); @@ -306,6 +429,10 @@ const AvailableVoiceInterviewControl = ({ resolveResponseSubmission, settlements, subscribeToAdmission, + subscribeToAdmissionFailure, + subscribeToResponseMessageCompleted, + subscribeToResponseMessageStarted, + subscribeToStopRequested, }: { config: OpenAIVoiceConfig; context: PetrinautAiVoiceModeContext; @@ -313,6 +440,10 @@ const AvailableVoiceInterviewControl = ({ resolveResponseSubmission?: ResolveSubmissions; settlements?: readonly VoiceSubmissionSettlement[]; subscribeToAdmission?: SubscribeToAdmission; + subscribeToAdmissionFailure?: SubscribeToAdmissionFailure; + subscribeToResponseMessageCompleted?: SubscribeToResponseMessageCompleted; + subscribeToResponseMessageStarted?: SubscribeToResponseMessageStarted; + subscribeToStopRequested?: SubscribeToStopRequested; }) => { "use no memo"; @@ -323,6 +454,7 @@ const AvailableVoiceInterviewControl = ({ let latestSubmitVoiceInput = context.submitVoiceInput; let latestResolveInputSubmission = resolveInputSubmission; let latestSubscribeToAdmission = subscribeToAdmission; + let latestSubscribeToAdmissionFailure = subscribeToAdmissionFailure; const session = new OpenAIRealtimeSession({ cancelAnimationFrame: (handle) => globalThis.cancelAnimationFrame(handle), connectionTimeoutMs: config.connectionTimeoutMs, @@ -344,6 +476,7 @@ const AvailableVoiceInterviewControl = ({ resolveInputSubmission: latestResolveInputSubmission, submitVoiceInput: latestSubmitVoiceInput, subscribeToAdmission: latestSubscribeToAdmission, + subscribeToAdmissionFailure: latestSubscribeToAdmissionFailure, }), }); const controller = new VoiceTurnController({ @@ -353,6 +486,7 @@ const AvailableVoiceInterviewControl = ({ submitText: (input) => latestSubmitVoiceInput(input), }); return { + bridge, controller, getSnapshot: () => controller.getSnapshot(), subscribe: (listener: (snapshot: VoiceTurnSnapshot) => void) => @@ -363,10 +497,14 @@ const AvailableVoiceInterviewControl = ({ | ((messageId: string) => string | undefined) | undefined, nextSubscribeToAdmission: SubscribeToAdmission | undefined, + nextSubscribeToAdmissionFailure: + | SubscribeToAdmissionFailure + | undefined, ) => { latestSubmitVoiceInput = nextSubmitVoiceInput; latestResolveInputSubmission = nextResolveInputSubmission; latestSubscribeToAdmission = nextSubscribeToAdmission; + latestSubscribeToAdmissionFailure = nextSubscribeToAdmissionFailure; }, }; }); @@ -378,6 +516,7 @@ const AvailableVoiceInterviewControl = ({ const [showDisclosure, setShowDisclosure] = useState(false); const [consented, setConsented] = useState(false); const [microphoneCheck, setMicrophoneCheck] = useState(""); + const [checkingMicrophone, setCheckingMicrophone] = useState(false); const handledVoiceSelectionRef = useRef(false); const { inputMode, @@ -387,22 +526,46 @@ const AvailableVoiceInterviewControl = ({ setVoiceActive, } = context; + useEffect( + () => + subscribeToResponseMessageCompleted?.((event) => + store.bridge.notifyResponseMessageCompleted(event), + ), + [store, subscribeToResponseMessageCompleted], + ); + useEffect( + () => + subscribeToResponseMessageStarted?.((event) => + store.bridge.notifyResponseMessageStarted(event), + ), + [store, subscribeToResponseMessageStarted], + ); + useEffect( + () => + subscribeToStopRequested?.(() => store.controller.cancelPendingSpeech()), + [store, subscribeToStopRequested], + ); + useLayoutEffect(() => { store.updateSubmissionContext( context.submitVoiceInput, resolveInputSubmission, subscribeToAdmission, + subscribeToAdmissionFailure, ); + const canonicalSpeech = selectCanonicalSpeech(context.messages); + const correlateSegment = (segment: CanonicalSpeechSegment) => { + const submissionIds = resolveResponseSubmission?.(segment.messageId); + return submissionIds === undefined || submissionIds.length === 0 + ? segment + : { ...segment, submissionIds }; + }; store.controller.updateChat({ canAcceptInterviewAnswer: context.canAcceptVoiceInput, - canonicalSegments: selectCanonicalSpeechSegments(context.messages).map( - (segment) => { - const submissionIds = resolveResponseSubmission?.(segment.messageId); - return submissionIds === undefined || submissionIds.length === 0 - ? segment - : { ...segment, submissionIds }; - }, - ), + canonicalSegments: canonicalSpeech.segments.map(correlateSegment), + ...(canonicalSpeech.questionSegment + ? { questionSegment: correlateSegment(canonicalSpeech.questionSegment) } + : {}), settlements, status: context.status, }); @@ -415,6 +578,7 @@ const AvailableVoiceInterviewControl = ({ resolveResponseSubmission, settlements, subscribeToAdmission, + subscribeToAdmissionFailure, store, ]); @@ -425,12 +589,17 @@ const AvailableVoiceInterviewControl = ({ registerVoiceModeControls({ end: () => store.controller.end(), pause: () => store.controller.pause(), + readFullResponse: () => store.controller.readFullResponse(), reconnect: () => { void store.controller.reconnect(); }, - resume: () => store.controller.resume(), + repeatQuestion: () => store.controller.repeatQuestion(), + resume: () => { + void store.controller.resume(); + }, setMicrophoneMuted: (muted) => store.controller.setMicrophoneMuted(muted), + takeTurn: () => store.controller.takeTurn(), }), [registerVoiceModeControls, store], ); @@ -497,19 +666,38 @@ const AvailableVoiceInterviewControl = ({ return ( { - setMicrophoneCheck("Checking microphone…"); - void navigator.mediaDevices.getUserMedia({ audio: true }).then( - (stream) => { + if (checkingMicrophone) { + return; + } + setCheckingMicrophone(true); + setMicrophoneCheck(""); + let microphoneCheckPromise: Promise; + try { + const { mediaDevices } = navigator as { + readonly mediaDevices?: MediaDevices; + }; + microphoneCheckPromise = + mediaDevices === undefined + ? Promise.reject(new Error("Microphone access is unavailable.")) + : mediaDevices.getUserMedia({ audio: true }); + } catch (error) { + microphoneCheckPromise = Promise.reject(error); + } + void microphoneCheckPromise + .then((stream) => { for (const track of stream.getTracks()) { track.stop(); } setMicrophoneCheck("Microphone ready."); - }, - () => setMicrophoneCheck("Microphone access was not available."), - ); + }) + .catch(() => + setMicrophoneCheck("Microphone access was not available."), + ) + .finally(() => setCheckingMicrophone(false)); }} onConsentChange={setConsented} onStart={() => { @@ -531,6 +719,10 @@ export const VoiceInterviewControl = ({ resolveResponseSubmission, settlements, subscribeToAdmission, + subscribeToAdmissionFailure, + subscribeToResponseMessageCompleted, + subscribeToResponseMessageStarted, + subscribeToStopRequested, ...context }: PetrinautAiVoiceModeContext & { readonly config: OpenAIVoiceConfig; @@ -538,6 +730,10 @@ export const VoiceInterviewControl = ({ readonly resolveResponseSubmission?: ResolveSubmissions; readonly settlements?: readonly VoiceSubmissionSettlement[]; readonly subscribeToAdmission?: SubscribeToAdmission; + readonly subscribeToAdmissionFailure?: SubscribeToAdmissionFailure; + readonly subscribeToResponseMessageCompleted?: SubscribeToResponseMessageCompleted; + readonly subscribeToResponseMessageStarted?: SubscribeToResponseMessageStarted; + readonly subscribeToStopRequested?: SubscribeToStopRequested; }) => ( ); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts index 6e4a0a63095..b067de20f55 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts @@ -1,3 +1,4 @@ +import { FlueApiError } from "@flue/sdk"; import { describe, expect, test, vi } from "vitest"; import { createOpenAIRealtimeCallHandler } from "../../../server/voice/openai-realtime-call"; @@ -9,11 +10,13 @@ import { BrunchPanelConversationTracker, createBrunchPanelTransport, } from "../local-storage-demo/brunch-panel-transport"; -import { selectCanonicalSpeechSegments } from "./canonical-speech"; +import { selectCanonicalSpeech } from "./canonical-speech"; import { OpenAIRealtimeSession } from "./openai-realtime-session"; import { RealtimeBrunchBridge } from "./realtime-brunch-bridge"; +import { submitVoiceInputWithAdmission } from "./voice-interview-control"; import { VoiceTurnController } from "./voice-turn-controller"; +import type { CanonicalSpeechSegment } from "./canonical-speech"; import type { OpenAIRealtimeSessionEvent } from "./openai-realtime-session"; import type { RealtimeBrunchBridgeEvent } from "./realtime-brunch-bridge"; import type { AgentSendResult, FlueClient } from "@flue/sdk"; @@ -73,11 +76,16 @@ const initialMessages = [ id: "initial-question-message", parts: [ { - input: { question: "What happens after approval?" }, - state: "input-available", - toolCallId: "ask-current", - toolName: "brunch_ask", - type: "dynamic-tool", + data: { + question: "What happens after approval?", + toolCallId: "tool-initial-question", + }, + type: "data-brunch-question", + }, + { + state: "done", + text: "What happens after approval?", + type: "text", }, ], role: "assistant", @@ -95,19 +103,100 @@ const responseMessages = [ id: "next-question-message", parts: [ { - input: { question: canonicalQuestion }, - state: "input-available", - toolCallId: "ask-next", - toolName: "brunch_ask", - type: "dynamic-tool", + data: { + question: canonicalQuestion, + toolCallId: "tool-next-question", + }, + type: "data-brunch-question", + }, + { + state: "done", + text: canonicalQuestion, + type: "text", }, ], role: "assistant", }, ] satisfies PetrinautAiMessage[]; +const createAdmissionOutcomeHarness = ( + client: Pick, +) => { + const tracker = new BrunchPanelConversationTracker(); + const transport = createBrunchPanelTransport( + Promise.resolve(client as FlueClient), + tracker, + ); + let realtimeListener: + | ((event: OpenAIRealtimeSessionEvent) => void) + | undefined; + const bridge = new RealtimeBrunchBridge({ + session: { + speakCanonical: vi.fn(), + subscribe: (listener) => { + realtimeListener = listener; + return () => { + realtimeListener = undefined; + }; + }, + }, + submitInterviewAnswer: (input) => + submitVoiceInputWithAdmission({ + input, + resolveInputSubmission: (messageId) => + tracker.submissionForInput(messageId), + submitVoiceInput: async ({ id, text }) => { + if (id === undefined) { + throw new Error("Voice message identity is required."); + } + void transport + .sendMessages({ + abortSignal: input.signal, + chatId: "conversation-1", + messageId: undefined, + messages: [ + { + id, + metadata: { source: "voice" }, + parts: [{ text, type: "text" }], + role: "user", + }, + ], + trigger: "submit-message", + }) + .catch(() => undefined); + return { kind: "message", messageId: id }; + }, + subscribeToAdmission: (target, listener) => + tracker.subscribeToAdmission(target, ({ admission }) => + listener(admission.submissionId), + ), + subscribeToAdmissionFailure: (target, listener) => + tracker.subscribeToAdmissionFailure(target, listener), + }), + }); + const events: RealtimeBrunchBridgeEvent[] = []; + bridge.subscribe((event) => events.push(event)); + bridge.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [], + status: "ready", + }); + bridge.start(1); + + return { + emitCompletedTranscript: (itemId: string) => + realtimeListener?.({ + key: { connectionEpoch: 1, contentIndex: 0, itemId }, + text: spokenAnswer, + type: "completed", + }), + events, + }; +}; + describe("controlled voice preview", () => { - test("bridges one Realtime tool call through Brunch and back to canonical duplex audio", async () => { + test("bridges one completed transcript through Brunch and back to canonical half-duplex audio", async () => { const diagnostics: VoiceDiagnosticEvent[] = []; const reportDiagnostic = (event: VoiceDiagnosticEvent) => diagnostics.push(event); @@ -211,10 +300,59 @@ describe("controlled voice preview", () => { reportDiagnostic, requestAnimationFrame: vi.fn(() => 1), }); - const submitInterviewAnswer = vi.fn(async () => ({ - kind: "interactive-tool" as const, - toolCallId: "ask-current", - })); + const admission: AgentSendResult = { + offset: "offset-voice-1", + streamUrl: "https://petrinaut.test/agents/chat/instance-1", + submissionId: "submission-voice-1", + uid: "uid-voice-1", + }; + const send = vi.fn(async () => admission); + const wait = vi.fn(async () => undefined); + const tracker = new BrunchPanelConversationTracker(); + const transport = createBrunchPanelTransport( + Promise.resolve({ send, wait } as Pick< + FlueClient, + "send" | "wait" + > as FlueClient), + tracker, + ); + type SubmitInterviewAnswer = ConstructorParameters< + typeof RealtimeBrunchBridge + >[0]["submitInterviewAnswer"]; + const submitInterviewAnswer = vi.fn((input) => + submitVoiceInputWithAdmission({ + input, + resolveInputSubmission: (messageId) => + tracker.submissionForInput(messageId), + submitVoiceInput: async ({ id, text }) => { + if (id === undefined) { + throw new Error("Voice message identity is required."); + } + const stream = await transport.sendMessages({ + abortSignal: input.signal, + chatId: "conversation-1", + messageId: undefined, + messages: [ + { + id, + metadata: { source: "voice" }, + parts: [{ text, type: "text" }], + role: "user", + }, + ], + trigger: "submit-message", + }); + void stream.pipeTo(new WritableStream()); + return { kind: "message", messageId: id }; + }, + subscribeToAdmission: (target, listener) => + tracker.subscribeToAdmission(target, ({ admission: admitted }) => + listener(admitted.submissionId), + ), + subscribeToAdmissionFailure: (target, listener) => + tracker.subscribeToAdmissionFailure(target, listener), + }), + ); const bridge = new RealtimeBrunchBridge({ session, submitInterviewAnswer, @@ -222,32 +360,90 @@ describe("controlled voice preview", () => { const controller = new VoiceTurnController({ bridge, session, - submitText: submitInterviewAnswer, + submitText: vi.fn(async () => ({ kind: "message" as const })), + }); + await controller.start(); + dataChannel.receive({ + audio_start_ms: 200, + item_id: "pre-output-item", + type: "input_audio_buffer.speech_started", }); + dataChannel.receive({ + content_index: 0, + delta: "Speech started before output", + item_id: "pre-output-item", + type: "conversation.item.input_audio_transcription.delta", + }); + expect(controller.getSnapshot().partialText).toBe( + "Speech started before output", + ); + const initialSelection = selectCanonicalSpeech(initialMessages); + const initialSegments = initialSelection.segments; controller.updateChat({ canAcceptInterviewAnswer: true, - canonicalSegments: selectCanonicalSpeechSegments(initialMessages), + canonicalSegments: initialSegments, + questionSegment: initialSelection.questionSegment, status: "ready", }); + dataChannel.receive({ + content_index: 0, + item_id: "pre-output-item", + transcript: "This completed before output started.", + type: "conversation.item.input_audio_transcription.completed", + }); + expect(controller.getSnapshot()).toMatchObject({ + lastCommittedText: "", + microphoneEnabled: true, + partialText: "", + }); + expect(track.enabled).toBe(false); + expect(submitInterviewAnswer).not.toHaveBeenCalled(); + expect(send).not.toHaveBeenCalled(); + + dataChannel.receive({ + content_index: 0, + item_id: "pre-output-item", + transcript: "The stale item cannot recover authority.", + type: "conversation.item.input_audio_transcription.completed", + }); + expect(send).not.toHaveBeenCalled(); - await controller.start(); authorizeLatestSpeechResponse(dataChannel, "response-initial-question"); dataChannel.receive({ response_id: "response-initial-question", type: "output_audio_buffer.started", }); + expect(controller.getSnapshot()).toMatchObject({ + canTakeTurn: true, + output: "speaking", + }); + + const handoff = controller.takeTurn(); dataChannel.receive({ audio_start_ms: 300, - item_id: "user-item", + item_id: "playback-overlap", type: "input_audio_buffer.speech_started", }); + dataChannel.receive({ + content_index: 0, + item_id: "playback-overlap", + transcript: "Playback must not become input.", + type: "conversation.item.input_audio_transcription.completed", + }); + dataChannel.receive({ type: "input_audio_buffer.cleared" }); dataChannel.receive({ response: { id: "response-initial-question", + output: [], status: "cancelled", }, type: "response.done", }); + dataChannel.receive({ + response_id: "response-initial-question", + type: "output_audio_buffer.cleared", + }); + await handoff; expect(controller.getSnapshot()).toMatchObject({ input: "listening", microphoneEnabled: true, @@ -255,75 +451,93 @@ describe("controlled voice preview", () => { }); dataChannel.receive({ - call_id: "call-1", - delta: `{"answer":"${spokenAnswer}"}`, - item_id: "function-item-1", - output_index: 0, - response_id: "response-tool-1", - type: "response.function_call_arguments.delta", + audio_start_ms: 500, + item_id: "user-item", + type: "input_audio_buffer.speech_started", }); dataChannel.receive({ - response: { - id: "response-tool-1", - output: [ - { - arguments: `{"answer":"${spokenAnswer}"}`, - call_id: "call-1", - id: "function-item-1", - name: "continue_interview", - status: "completed", - type: "function_call", - }, - ], - status: "completed", - }, - type: "response.done", + content_index: 0, + delta: "The supervisor", + item_id: "user-item", + type: "conversation.item.input_audio_transcription.delta", + }); + dataChannel.receive({ + content_index: 0, + item_id: "user-item", + transcript: spokenAnswer, + type: "conversation.item.input_audio_transcription.completed", }); await vi.waitFor(() => expect(submitInterviewAnswer).toHaveBeenCalledWith( expect.objectContaining({ admissionTarget: { - kind: "client-tool-result", - messageId: "initial-question-message", + kind: "user", + messageId: "voice-realtime:1:user-item:0", }, - id: "voice-realtime:1:call-1", + id: "voice-realtime:1:user-item:0", text: spokenAnswer, }), ), ); - expect(controller.getSnapshot()).toMatchObject({ - input: "submitting", - lastAnswerDelivery: "delivered", - microphoneEnabled: true, - output: "waiting-for-tool", - }); + await vi.waitFor(() => expect(send).toHaveBeenCalledOnce()); + expect(send).toHaveBeenCalledWith( + expect.objectContaining({ + idempotencyKey: "ai-sdk:voice-realtime:1:user-item:0", + message: { body: spokenAnswer, kind: "user" }, + }), + ); + await vi.waitFor(() => + expect(controller.getSnapshot()).toMatchObject({ + input: "submitting", + lastAnswerDelivery: "delivered", + microphoneEnabled: true, + output: "waiting-for-tool", + }), + ); controller.updateChat({ canAcceptInterviewAnswer: false, - canonicalSegments: selectCanonicalSpeechSegments(initialMessages), + canonicalSegments: initialSegments, status: "streaming", }); + const initialSegmentIds = new Set(initialSegments.map(({ id }) => id)); + const responseSelection = selectCanonicalSpeech(responseMessages); + const correlateResponse = (segment: CanonicalSpeechSegment) => + initialSegmentIds.has(segment.id) + ? segment + : { ...segment, submissionIds: [admission.submissionId] }; + const correlatedSegments = + responseSelection.segments.map(correlateResponse); controller.updateChat({ canAcceptInterviewAnswer: true, - canonicalSegments: selectCanonicalSpeechSegments(responseMessages), + canonicalSegments: correlatedSegments, + questionSegment: responseSelection.questionSegment + ? correlateResponse(responseSelection.questionSegment) + : undefined, status: "ready", }); - const [functionOutput, responseCreate] = sentEvents(dataChannel).slice(-2); - expect(functionOutput).toEqual({ - type: "conversation.item.create", - item: { - type: "function_call_output", - call_id: "call-1", - output: JSON.stringify({ - response_text: [canonicalReply, canonicalQuestion], - }), - }, - }); + const responseCreate = sentEvents(dataChannel).findLast( + ({ type }) => type === "response.create", + ); expect(responseCreate).toMatchObject({ type: "response.create", response: { + input: [ + { + content: [ + { + text: JSON.stringify({ + response_text: [canonicalReply, canonicalQuestion], + }), + type: "input_text", + }, + ], + role: "system", + type: "message", + }, + ], output_modalities: ["audio"], tool_choice: "none", tools: [], @@ -341,6 +555,48 @@ describe("controlled voice preview", () => { microphoneEnabled: true, output: "speaking", }); + expect(track.enabled).toBe(false); + + dataChannel.receive({ + response_id: "response-canonical-reply", + type: "output_audio_buffer.stopped", + }); + dataChannel.receive({ + response: { + id: "response-canonical-reply", + output: [], + status: "completed", + }, + type: "response.done", + }); + expect(controller.getSnapshot()).toMatchObject({ + canReadFullResponse: true, + canRepeatQuestion: true, + output: "idle", + }); + + controller.repeatQuestion(); + + const replayCreate = sentEvents(dataChannel).findLast( + ({ type }) => type === "response.create", + ); + expect(replayCreate).toMatchObject({ + response: { + input: [ + { + content: [ + { + text: JSON.stringify({ response_text: [canonicalQuestion] }), + type: "input_text", + }, + ], + role: "system", + type: "message", + }, + ], + }, + type: "response.create", + }); const remoteTrack = { kind: "audio", stop: vi.fn() }; const remoteStream = { @@ -373,11 +629,13 @@ describe("controlled voice preview", () => { turn_detection: { type: "semantic_vad", eagerness: "low", - create_response: true, - interrupt_response: true, + create_response: false, + interrupt_response: false, }, }, }, + tool_choice: "none", + tools: [], }); expect(diagnostics).toEqual( expect.arrayContaining([ @@ -443,8 +701,6 @@ describe("controlled voice preview", () => { | undefined; const bridge = new RealtimeBrunchBridge({ session: { - completeFunctionCall: vi.fn(), - completeFunctionCallWithoutResponse: vi.fn(), speakCanonical: vi.fn(), subscribe: (listener) => { realtimeListener = listener; @@ -457,6 +713,7 @@ describe("controlled voice preview", () => { admissionTarget, id, onAdmission, + signal, text, }) => { const unsubscribe = tracker.subscribeToAdmission( @@ -475,7 +732,7 @@ describe("controlled voice preview", () => { parts: [{ type: "text", text }], }, ], - abortSignal: undefined, + abortSignal: signal, }); try { await stream.pipeTo(new WritableStream()); @@ -498,14 +755,14 @@ describe("controlled voice preview", () => { }); bridge.start(1); - const finalized = { - arguments: '{"answer":"The supervisor approves it."}', - callId: "call-1", - connectionEpoch: 1, - itemId: "function-item-1", - name: "continue_interview", - responseId: "response-1", - type: "tool-arguments-done" as const, + const finalized: OpenAIRealtimeSessionEvent = { + key: { + connectionEpoch: 1, + contentIndex: 0, + itemId: "input-item-1", + }, + text: spokenAnswer, + type: "completed", }; realtimeListener?.(finalized); realtimeListener?.(finalized); @@ -513,7 +770,7 @@ describe("controlled voice preview", () => { await vi.waitFor(() => expect(send).toHaveBeenCalledOnce()); await vi.waitFor(() => expect(bridgeEvents).toContainEqual({ - callId: "call-1", + deliveryId: "voice-realtime:1:input-item-1:0", submissionId: admission.submissionId, type: "submission-admitted", }), @@ -521,10 +778,13 @@ describe("controlled voice preview", () => { expect(bridgeEvents).not.toContainEqual( expect.objectContaining({ type: "submission-accepted" }), ); - expect(send).toHaveBeenCalledWith({ - message: { kind: "user", body: "The supervisor approves it." }, - signal: undefined, + expect(send).toHaveBeenCalledOnce(); + const sendInput = send.mock.calls[0]?.[0]; + expect(sendInput).toMatchObject({ + idempotencyKey: "ai-sdk:voice-realtime:1:input-item-1:0", + message: { kind: "user", body: spokenAnswer }, }); + expect(sendInput?.signal).toBeInstanceOf(AbortSignal); expect(admission.streamUrl).toContain("/agents/chat/"); settleSubmission?.(); @@ -534,4 +794,80 @@ describe("controlled voice preview", () => { ), ); }); + + test("surfaces an ambiguous Flue admission through the panel observer without retrying", async () => { + const send = vi.fn(async () => { + throw new FlueApiError(500, ""); + }); + const abort = vi.fn(async () => ({ aborted: true })); + const harness = createAdmissionOutcomeHarness({ abort, send }); + + harness.emitCompletedTranscript("input-item-ambiguous"); + + await vi.waitFor(() => + expect(harness.events).toContainEqual({ + code: "admission-ambiguous", + failure: { kind: "ambiguous" }, + message: + "Brunch may have accepted the message, but admission could not be confirmed. Reopen the conversation before trying again.", + type: "error", + }), + ); + expect(send).toHaveBeenCalledOnce(); + expect(abort).not.toHaveBeenCalled(); + }); + + test("preserves a conflicting submission through the production admission path", async () => { + const send = vi.fn(async () => { + throw new FlueApiError(409, { + error: { + details: "", + message: "The delivery key already names another payload.", + meta: { submissionId: "submission-existing" }, + type: "submission_conflict", + }, + }); + }); + const abort = vi.fn(async () => ({ aborted: true })); + const harness = createAdmissionOutcomeHarness({ abort, send }); + + harness.emitCompletedTranscript("input-item-conflict"); + + await vi.waitFor(() => + expect(harness.events).toContainEqual({ + code: "admission-conflict", + failure: { + kind: "submission-conflict", + status: 409, + submissionId: "submission-existing", + }, + message: + "The delivery key already belongs to admitted submission submission-existing; the changed payload was not admitted.", + type: "error", + }), + ); + expect(send).toHaveBeenCalledOnce(); + expect(abort).not.toHaveBeenCalled(); + }); + + test("keeps local admission abort distinct from durable Flue abort", async () => { + const send = vi.fn(async () => { + throw new DOMException("Local admission cancelled", "AbortError"); + }); + const abort = vi.fn(async () => ({ aborted: true })); + const harness = createAdmissionOutcomeHarness({ abort, send }); + + harness.emitCompletedTranscript("input-item-aborted"); + + await vi.waitFor(() => + expect(harness.events).toContainEqual({ + code: "admission-aborted", + failure: { kind: "aborted" }, + message: "The local chat submission was cancelled.", + type: "error", + }), + ); + expect(send).toHaveBeenCalledOnce(); + expect(abort).not.toHaveBeenCalled(); + }); }); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.test.ts index d637a762607..912282facab 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.test.ts @@ -5,6 +5,9 @@ import { toVoiceSessionState } from "./voice-session-state"; import type { VoiceTurnSnapshot } from "./voice-turn-controller"; const listeningSnapshot = { + canReadFullResponse: false, + canRepeatQuestion: false, + canTakeTurn: false, canReviseLastAnswer: false, connection: "connected", currentQuestion: "What happens after approval?", @@ -12,6 +15,7 @@ const listeningSnapshot = { errorMessage: "", errorRequestId: "", input: "listening", + inputNotice: "none", lastAnswerDelivery: "none", lastCommittedText: "", microphoneEnabled: true, @@ -30,13 +34,40 @@ describe("toVoiceSessionState", () => { test("reports a listening turn with its microphone level", () => { expect(mapSnapshot()).toEqual({ + canReadFullResponse: false, + canRepeatQuestion: false, + canTakeTurn: false, errorMessage: null, microphoneLevel: 0.24, microphoneMuted: false, + notice: null, phase: "listening", }); }); + test("publishes safe handoff and canonical playback availability", () => { + expect( + mapSnapshot({ + canReadFullResponse: true, + canRepeatQuestion: true, + canTakeTurn: true, + }), + ).toMatchObject({ + canReadFullResponse: true, + canRepeatQuestion: true, + canTakeTurn: true, + }); + }); + + test("describes recoverable transcript rejections", () => { + expect(mapSnapshot({ inputNotice: "not-heard" })?.notice).toBe( + "We didn't catch that. Please try again.", + ); + expect(mapSnapshot({ inputNotice: "too-long" })?.notice).toBe( + "That answer is too long. Please try a shorter response.", + ); + }); + test("hands the turn to the assistant while it speaks", () => { expect(mapSnapshot({ output: "speaking", partialText: "" })).toMatchObject({ phase: "speaking", diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.ts index e1a06464eb0..5d8bfce2219 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.ts @@ -65,6 +65,7 @@ const phaseOf = ( return "speaking"; } if ( + snapshot.output === "cancelling" || snapshot.output === "waiting-for-tool" || snapshot.input === "submitting" ) { @@ -93,6 +94,9 @@ export const toVoiceSessionState = ({ } return { + canReadFullResponse: snapshot.canReadFullResponse, + canRepeatQuestion: snapshot.canRepeatQuestion, + canTakeTurn: snapshot.canTakeTurn, errorMessage: snapshot.connection === "error" ? errorMessageOf(snapshot) : null, microphoneMuted: @@ -100,6 +104,12 @@ export const toVoiceSessionState = ({ snapshot.input !== "paused" && !snapshot.microphoneEnabled, microphoneLevel: snapshot.microphoneLevel, + notice: + snapshot.inputNotice === "not-heard" + ? "We didn't catch that. Please try again." + : snapshot.inputNotice === "too-long" + ? "That answer is too long. Please try a shorter response." + : null, phase: phaseOf(snapshot), }; }; diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts index 08dbffb2e7a..4f1f528d04b 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts @@ -16,10 +16,11 @@ const createHarness = () => { | undefined; let bridgeListener: ((event: RealtimeBrunchBridgeEvent) => void) | undefined; const session = { - cancelOutput: vi.fn(), + cancelOutput: vi.fn<() => Promise>(async () => undefined), connect: vi.fn(async () => ++epoch), disconnect: vi.fn(async () => undefined), setMicrophoneEnabled: vi.fn(), + speakCanonical: vi.fn(), subscribe: vi.fn( (listener: (event: OpenAIRealtimeSessionEvent) => void) => { sessionListener = listener; @@ -30,6 +31,8 @@ const createHarness = () => { ), }; const bridge = { + cancelPendingSpeech: vi.fn(), + completeTurnHandoff: vi.fn(), start: vi.fn(), stop: vi.fn(), subscribe: vi.fn((listener: (event: RealtimeBrunchBridgeEvent) => void) => { @@ -73,10 +76,18 @@ const question = ( id, messageId: `message-${id}`, partId: id, - source: "brunch-ask", + source: "assistant-text", text, }); +const markedQuestion = ( + id: string, + text = "What happens after approval?", +): CanonicalSpeechSegment => ({ + ...question(id, text), + source: "assistant-question", +}); + describe("VoiceTurnController", () => { test("records the content-free Voice lifecycle once in causal order", async () => { const harness = createHarness(); @@ -84,33 +95,33 @@ describe("VoiceTurnController", () => { harness.emitBridge({ answer: "Private finalized answer", - callId: "call-opaque", + deliveryId: "call-opaque", type: "submission-started", }); harness.advanceTime(10); harness.emitBridge({ - callId: "call-opaque", + deliveryId: "call-opaque", submissionId: "submission-opaque", type: "submission-admitted", }); harness.emitBridge({ - callId: "call-opaque", + deliveryId: "call-opaque", submissionId: "submission-opaque", type: "submission-admitted", }); harness.advanceTime(10); harness.emitBridge({ answer: "Private finalized answer", - callId: "call-opaque", + deliveryId: "call-opaque", type: "submission-accepted", }); harness.emitBridge({ - callId: "call-opaque", + deliveryId: "call-opaque", type: "canonical-text-ready", }); harness.advanceTime(10); harness.emitBridge({ - callId: "call-opaque", + deliveryId: "call-opaque", type: "submission-settled", }); harness.advanceTime(10); @@ -167,7 +178,7 @@ describe("VoiceTurnController", () => { await harness.controller.end(); harness.emitBridge({ - callId: "call-opaque", + deliveryId: "call-opaque", type: "submission-settled", }); harness.emitSession(outputStarted); @@ -184,6 +195,7 @@ describe("VoiceTurnController", () => { harness.controller.updateChat({ canAcceptInterviewAnswer: true, canonicalSegments: [question("ask-1")], + questionSegment: markedQuestion("ask-1"), status: "ready", }); @@ -199,7 +211,7 @@ describe("VoiceTurnController", () => { }); }); - test("keeps capture active while the interviewer speaks and interrupts automatically", async () => { + test("tracks assistant playback without admitting automatic barge-in", async () => { const harness = createHarness(); await harness.controller.start(); @@ -222,18 +234,503 @@ describe("VoiceTurnController", () => { }); expect(harness.controller.getSnapshot()).toMatchObject({ microphoneEnabled: true, - output: "interrupted", + output: "speaking", }); expect(harness.session.cancelOutput).not.toHaveBeenCalled(); }); - test("represents submitting and output independently without closing capture", async () => { + test("clears pre-output capture and only commits fresh post-handoff input", async () => { + const harness = createHarness(); + harness.controller.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [question("ask-late-transcript")], + questionSegment: markedQuestion("ask-late-transcript"), + status: "ready", + }); + await harness.controller.start(); + harness.emitSession({ + connectionEpoch: 1, + itemId: "item-before-output", + type: "input-speech-started", + }); + harness.emitSession({ + key: { + connectionEpoch: 1, + contentIndex: 0, + itemId: "item-before-output", + }, + text: "Pre-output partial", + type: "partial", + }); + expect(harness.controller.getSnapshot().partialText).toBe( + "Pre-output partial", + ); + + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-output", + speechRequestId: "speech-output", + type: "output-started", + }); + harness.emitSession({ + key: { + connectionEpoch: 1, + contentIndex: 0, + itemId: "item-before-output", + }, + text: "This completed too late.", + type: "completed", + }); + + expect(harness.controller.getSnapshot()).toMatchObject({ + lastCommittedText: "", + partialText: "", + }); + + await harness.controller.takeTurn(); + harness.emitSession({ + connectionEpoch: 1, + itemId: "item-after-handoff", + type: "input-speech-started", + }); + harness.emitSession({ + key: { + connectionEpoch: 1, + contentIndex: 0, + itemId: "item-after-handoff", + }, + text: "Fresh post-handoff answer.", + type: "completed", + }); + harness.emitBridge({ + answer: "Fresh post-handoff answer.", + deliveryId: "fresh-delivery", + type: "submission-started", + }); + + expect(harness.controller.getSnapshot()).toMatchObject({ + input: "submitting", + lastCommittedText: "Fresh post-handoff answer.", + partialText: "", + }); + }); + + test("clears capture when canonical speech is requested before output starts", async () => { + const harness = createHarness(); + harness.controller.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [question("ask-request")], + questionSegment: markedQuestion("ask-request"), + status: "ready", + }); + await harness.controller.start(); + harness.emitBridge({ + deliveryId: "voice-request", + segments: [question("ask-request")], + type: "canonical-response-ready", + }); + harness.emitSession({ + connectionEpoch: 1, + itemId: "item-before-request", + type: "input-speech-started", + }); + harness.emitSession({ + key: { + connectionEpoch: 1, + contentIndex: 0, + itemId: "item-before-request", + }, + text: "Provisional pre-request words", + type: "partial", + }); + + harness.emitSession({ + connectionEpoch: 1, + speechRequestId: "speech-request", + type: "canonical-speech-requested", + }); + + expect(harness.session.setMicrophoneEnabled).toHaveBeenLastCalledWith( + false, + ); + expect(harness.controller.getSnapshot()).toMatchObject({ + canTakeTurn: true, + lastCommittedText: "", + partialText: "", + }); + harness.emitSession({ + key: { + connectionEpoch: 1, + contentIndex: 0, + itemId: "item-before-request", + }, + text: "This completed before output started.", + type: "completed", + }); + expect(harness.controller.getSnapshot()).toMatchObject({ + lastCommittedText: "", + partialText: "", + }); + expect(harness.submitText).not.toHaveBeenCalled(); + + await harness.controller.takeTurn(); + expect(harness.bridge.completeTurnHandoff).toHaveBeenCalledOnce(); + harness.emitSession({ + connectionEpoch: 1, + itemId: "item-after-handoff", + type: "input-speech-started", + }); + harness.emitSession({ + key: { + connectionEpoch: 1, + contentIndex: 0, + itemId: "item-after-handoff", + }, + text: "Fresh post-handoff answer.", + type: "completed", + }); + harness.emitBridge({ + answer: "Fresh post-handoff answer.", + deliveryId: "fresh-delivery", + type: "submission-started", + }); + + expect(harness.controller.getSnapshot()).toMatchObject({ + input: "submitting", + lastCommittedText: "Fresh post-handoff answer.", + partialText: "", + }); + }); + + test("hands off an active response once and applies the latest mute preference after cancellation", async () => { + const harness = createHarness(); + let finishCancellation: (() => void) | undefined; + harness.controller.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [question("ask-handoff")], + questionSegment: markedQuestion("ask-handoff"), + status: "ready", + }); + await harness.controller.start(); + harness.session.cancelOutput.mockImplementationOnce( + () => + new Promise((resolve) => { + finishCancellation = resolve; + }), + ); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-handoff", + speechRequestId: "speech-handoff", + type: "output-started", + }); + harness.session.cancelOutput.mockClear(); + + expect(harness.controller.getSnapshot().canTakeTurn).toBe(true); + const handoff = harness.controller.takeTurn(); + const repeatedHandoff = harness.controller.takeTurn(); + + expect(repeatedHandoff).toBe(handoff); + expect(harness.bridge.cancelPendingSpeech).toHaveBeenCalledOnce(); + expect(harness.session.cancelOutput).toHaveBeenCalledOnce(); + expect(harness.session.setMicrophoneEnabled).toHaveBeenLastCalledWith( + false, + ); + expect(harness.controller.getSnapshot()).toMatchObject({ + canTakeTurn: false, + output: "cancelling", + }); + + harness.session.setMicrophoneEnabled.mockClear(); + harness.controller.setMicrophoneMuted(true); + harness.controller.setMicrophoneMuted(false); + expect(harness.session.setMicrophoneEnabled).not.toHaveBeenCalled(); + expect(harness.controller.getSnapshot().microphoneEnabled).toBe(true); + + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-handoff", + type: "output-interrupted", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-handoff", + status: "cancelled", + type: "response-terminal", + }); + expect(harness.controller.getSnapshot().output).toBe("cancelling"); + + finishCancellation?.(); + await handoff; + + expect(harness.session.setMicrophoneEnabled).toHaveBeenCalledOnce(); + expect(harness.session.setMicrophoneEnabled).toHaveBeenCalledWith(true); + expect(harness.controller.getSnapshot()).toMatchObject({ + canTakeTurn: false, + microphoneEnabled: true, + output: "interrupted", + }); + }); + + test("reopens the microphone only after cancellation and Brunch settlement", async () => { + const harness = createHarness(); + harness.controller.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [question("answered-question")], + questionSegment: markedQuestion("answered-question"), + status: "ready", + }); + await harness.controller.start(); + harness.emitBridge({ + answer: "The approved answer.", + deliveryId: "voice-request", + type: "submission-started", + }); + harness.emitBridge({ + answer: "The approved answer.", + deliveryId: "voice-request", + type: "submission-accepted", + }); + harness.controller.updateChat({ + canAcceptInterviewAnswer: false, + canonicalSegments: [question("next-question")], + questionSegment: markedQuestion("next-question"), + status: "streaming", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-handoff", + speechRequestId: "speech-handoff", + type: "output-started", + }); + harness.session.setMicrophoneEnabled.mockClear(); + + const handoff = harness.controller.takeTurn(); + let handoffFinished = false; + void handoff.then(() => { + handoffFinished = true; + }); + await Promise.resolve(); + + expect(handoffFinished).toBe(false); + expect(harness.bridge.completeTurnHandoff).not.toHaveBeenCalled(); + expect(harness.session.setMicrophoneEnabled).toHaveBeenCalledOnce(); + expect(harness.session.setMicrophoneEnabled).toHaveBeenCalledWith(false); + + harness.emitBridge({ + deliveryId: "voice-request", + type: "submission-settled", + }); + await handoff; + + expect(harness.bridge.completeTurnHandoff).toHaveBeenCalledOnce(); + expect(harness.session.setMicrophoneEnabled).toHaveBeenLastCalledWith(true); + }); + + test("cancels queued and later speech when the host stops a response", async () => { + const harness = createHarness(); + await harness.controller.start(); + + harness.controller.cancelPendingSpeech(); + + expect(harness.bridge.cancelPendingSpeech).toHaveBeenCalledOnce(); + expect(harness.session.cancelOutput).toHaveBeenCalledOnce(); + }); + + test("keeps the user turn when cancelled pending speech settles later", async () => { const harness = createHarness(); + harness.controller.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [question("ask-handoff")], + questionSegment: markedQuestion("ask-handoff"), + status: "ready", + }); await harness.controller.start(); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-handoff", + speechRequestId: "speech-handoff", + type: "output-started", + }); + + await harness.controller.takeTurn(); + harness.session.setMicrophoneEnabled.mockClear(); + harness.emitBridge({ + deliveryId: "voice-1", + segments: [question("ask-late", "Retained late response")], + speechCancelled: true, + type: "canonical-response-ready", + }); + + expect(harness.session.setMicrophoneEnabled).not.toHaveBeenCalled(); + expect(harness.controller.getSnapshot()).toMatchObject({ + input: "listening", + microphoneEnabled: true, + output: "interrupted", + }); + }); + + test("replays exact canonical response segments but does not infer a question from the final segment", async () => { + const harness = createHarness(); + const context = question("context", "Approval is required before release."); + const nextQuestion = question("ask-replay", "Who approves release?"); + await harness.controller.start(); + + harness.emitBridge({ + deliveryId: "voice-1", + segments: [context, nextQuestion], + type: "canonical-response-ready", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-source", + speechRequestId: "speech-source", + type: "output-started", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-source", + type: "output-stopped", + }); + + expect(harness.controller.getSnapshot()).toMatchObject({ + canReadFullResponse: false, + canRepeatQuestion: false, + }); + harness.controller.readFullResponse(); + expect(harness.session.speakCanonical).not.toHaveBeenCalled(); + + harness.emitSession({ + connectionEpoch: 1, + responseId: "unrelated-response", + status: "completed", + type: "response-terminal", + }); + expect(harness.controller.getSnapshot().canRepeatQuestion).toBe(false); + + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-source", + status: "completed", + type: "response-terminal", + }); + expect(harness.controller.getSnapshot()).toMatchObject({ + canReadFullResponse: true, + canRepeatQuestion: false, + }); + + expect(harness.session.speakCanonical).not.toHaveBeenCalled(); + + harness.controller.readFullResponse(); + expect(harness.session.speakCanonical).toHaveBeenCalledOnce(); + expect(harness.session.speakCanonical).toHaveBeenCalledWith([ + context, + nextQuestion, + ]); + }); + + test("repeats only the exact Brunch-marked question after replay settles", async () => { + const harness = createHarness(); + const context = question("context", "Approval is required before release."); + const finalProse = question( + "response-prose", + "The approver is recorded. I can explain the escalation path.", + ); + const exactQuestion: CanonicalSpeechSegment = { + ...question("marked-question", "Who approves release?"), + messageId: finalProse.messageId, + source: "assistant-question", + }; + await harness.controller.start(); + + harness.emitBridge({ + deliveryId: "voice-1", + questionSegment: exactQuestion, + segments: [context, finalProse], + type: "canonical-response-ready", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-source", + speechRequestId: "speech-source", + type: "output-started", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-source", + type: "output-stopped", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-source", + status: "completed", + type: "response-terminal", + }); + + expect(harness.controller.getSnapshot()).toMatchObject({ + canReadFullResponse: true, + canRepeatQuestion: true, + }); + + harness.controller.repeatQuestion(); + + expect(harness.session.speakCanonical).toHaveBeenCalledOnce(); + expect(harness.session.speakCanonical).toHaveBeenCalledWith([ + exactQuestion, + ]); + }); + + test("disables replay while the user is capturing input", async () => { + const harness = createHarness(); + const segment = question("ask-capture"); + await harness.controller.start(); + harness.emitBridge({ + deliveryId: "voice-1", + segments: [segment], + type: "canonical-response-ready", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-source", + speechRequestId: "speech-source", + type: "output-started", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-source", + type: "output-stopped", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-source", + status: "completed", + type: "response-terminal", + }); + expect(harness.controller.getSnapshot()).toMatchObject({ + canReadFullResponse: true, + canRepeatQuestion: false, + }); + + harness.emitSession({ + connectionEpoch: 1, + itemId: "item-user", + type: "input-speech-started", + }); + + expect(harness.controller.getSnapshot()).toMatchObject({ + canReadFullResponse: false, + canRepeatQuestion: false, + }); + }); + + test("keeps capture closed from submission until canonical output settles", async () => { + const harness = createHarness(); + await harness.controller.start(); + harness.session.setMicrophoneEnabled.mockClear(); harness.emitBridge({ answer: "The supervisor approves it.", - callId: "call-1", + deliveryId: "call-1", type: "submission-started", }); expect(harness.controller.getSnapshot()).toMatchObject({ @@ -243,13 +740,16 @@ describe("VoiceTurnController", () => { microphoneEnabled: true, output: "waiting-for-tool", }); + expect(harness.session.setMicrophoneEnabled).toHaveBeenLastCalledWith( + false, + ); harness.emitBridge({ answer: "The supervisor approves it.", - callId: "call-1", + deliveryId: "call-1", type: "submission-accepted", }); harness.emitBridge({ - callId: "call-1", + deliveryId: "call-1", segments: [question("ask-2", "Who acts next?")], type: "canonical-response-ready", }); @@ -260,9 +760,33 @@ describe("VoiceTurnController", () => { microphoneEnabled: true, output: "waiting-for-tool", }); - expect(harness.session.setMicrophoneEnabled).not.toHaveBeenCalledWith( - false, - ); + expect(harness.session.setMicrophoneEnabled).not.toHaveBeenCalledWith(true); + + harness.emitSession({ + connectionEpoch: 1, + speechRequestId: "speech-next", + type: "canonical-speech-requested", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-next", + speechRequestId: "speech-next", + type: "output-started", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-next", + status: "completed", + type: "response-terminal", + }); + expect(harness.session.setMicrophoneEnabled).not.toHaveBeenCalledWith(true); + + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-next", + type: "output-stopped", + }); + expect(harness.session.setMicrophoneEnabled).toHaveBeenLastCalledWith(true); }); test("returns to listening after a durably stopped turn without speaking", async () => { @@ -271,18 +795,18 @@ describe("VoiceTurnController", () => { harness.emitBridge({ answer: "Stop this one.", - callId: "call-1", + deliveryId: "voice-1", type: "submission-started", }); harness.emitBridge({ answer: "Stop this one.", - callId: "call-1", + deliveryId: "voice-1", type: "submission-accepted", }); harness.advanceTime(40); - harness.emitBridge({ callId: "call-1", type: "submission-settled" }); + harness.emitBridge({ deliveryId: "voice-1", type: "submission-settled" }); harness.emitBridge({ - callId: "call-1", + deliveryId: "voice-1", outcome: "aborted", type: "submission-stopped", }); @@ -293,7 +817,7 @@ describe("VoiceTurnController", () => { output: "idle", }); expect(harness.latencyEvents).toContainEqual({ - correlationId: "call-1", + correlationId: "voice-1", elapsedMs: 40, name: "submission-settled", }); @@ -304,13 +828,14 @@ describe("VoiceTurnController", () => { await harness.controller.start(); harness.emitBridge({ answer: "The supervisor approves it.", - callId: "call-1", + deliveryId: "call-1", type: "submission-started", }); harness.controller.pause(); - harness.controller.resume(); + await harness.controller.resume(); + expect(harness.bridge.cancelPendingSpeech).toHaveBeenCalledOnce(); expect(harness.controller.getSnapshot()).toMatchObject({ input: "submitting", lastAnswerDelivery: "pending", @@ -319,11 +844,11 @@ describe("VoiceTurnController", () => { harness.emitBridge({ answer: "The supervisor approves it.", - callId: "call-1", + deliveryId: "call-1", type: "submission-accepted", }); harness.emitBridge({ - callId: "call-1", + deliveryId: "call-1", segments: [question("ask-2", "Who acts next?")], type: "canonical-response-ready", }); @@ -339,27 +864,29 @@ describe("VoiceTurnController", () => { harness.controller.updateChat({ canAcceptInterviewAnswer: true, canonicalSegments: [question("ask-1")], + questionSegment: markedQuestion("ask-1"), status: "ready", }); await harness.controller.start(); harness.emitBridge({ answer: "The supervisor approves it.", - callId: "call-1", + deliveryId: "call-1", type: "submission-started", }); harness.controller.updateChat({ canAcceptInterviewAnswer: true, canonicalSegments: [question("ask-2", "Who acts next?")], + questionSegment: markedQuestion("ask-2", "Who acts next?"), status: "ready", }); harness.emitBridge({ answer: "The supervisor approves it.", - callId: "call-1", + deliveryId: "call-1", type: "submission-accepted", }); harness.emitBridge({ - callId: "call-1", + deliveryId: "call-1", segments: [question("ask-2", "Who acts next?")], type: "canonical-response-ready", }); @@ -381,13 +908,13 @@ describe("VoiceTurnController", () => { await harness.controller.start(); harness.emitBridge({ answer: "The supervisor approves it.", - callId: "call-1", + deliveryId: "call-1", type: "submission-started", }); harness.controller.pause(); harness.emitBridge({ - callId: "call-1", + deliveryId: "call-1", segments: [question("ask-2", "Who acts next?")], type: "canonical-response-ready", }); @@ -399,7 +926,7 @@ describe("VoiceTurnController", () => { }); expect(harness.session.cancelOutput).toHaveBeenCalledTimes(2); - harness.controller.resume(); + await harness.controller.resume(); expect(harness.controller.getSnapshot()).toMatchObject({ input: "listening", microphoneEnabled: true, @@ -434,6 +961,33 @@ describe("VoiceTurnController", () => { expect(harness.submitText).not.toHaveBeenCalled(); }); + test.each(["empty", "failed"] as const)( + "reports a recoverable not-heard notice for a %s transcript", + async (reason) => { + const harness = createHarness(); + await harness.controller.start(); + harness.emitSession({ + connectionEpoch: 1, + itemId: "item-1", + type: "input-speech-started", + }); + harness.emitSession({ + key: { connectionEpoch: 1, contentIndex: 0, itemId: "item-1" }, + text: "Provisional words", + type: "partial", + }); + + harness.emitBridge({ reason, type: "transcript-rejected" }); + + expect(harness.controller.getSnapshot()).toMatchObject({ + input: "listening", + inputNotice: "not-heard", + partialText: "", + }); + expect(harness.submitText).not.toHaveBeenCalled(); + }, + ); + test("keeps completed display transcripts until submission and rejects late events", async () => { const harness = createHarness(); await harness.controller.start(); @@ -449,11 +1003,11 @@ describe("VoiceTurnController", () => { }); harness.emitBridge({ answer: "First answer", - callId: "call-1", + deliveryId: "call-1", type: "submission-started", }); harness.emitBridge({ - callId: "call-1", + deliveryId: "call-1", segments: [question("ask-2", "Who acts next?")], type: "canonical-response-ready", }); @@ -539,7 +1093,7 @@ describe("VoiceTurnController", () => { output: "interrupted", }); - harness.controller.resume(); + await harness.controller.resume(); expect(harness.controller.getSnapshot()).toMatchObject({ input: "listening", microphoneEnabled: true, @@ -617,7 +1171,7 @@ describe("VoiceTurnController", () => { output: "idle", }); - harness.controller.resume(); + await harness.controller.resume(); expect(harness.bridge.start).toHaveBeenCalledWith(1); expect(harness.controller.getSnapshot()).toMatchObject({ input: "listening", @@ -735,6 +1289,10 @@ describe("VoiceTurnController", () => { canonicalSegments: [ question("ask-reconnect", "What happens after approval?"), ], + questionSegment: markedQuestion( + "ask-reconnect", + "What happens after approval?", + ), status: "ready", }); await harness.controller.start(); @@ -761,12 +1319,16 @@ describe("VoiceTurnController", () => { canonicalSegments: [ question("ask-failed-delivery", "What happens after approval?"), ], + questionSegment: markedQuestion( + "ask-failed-delivery", + "What happens after approval?", + ), status: "ready", }); await harness.controller.start(); harness.emitBridge({ answer: "The supervisor approves it.", - callId: "call-1", + deliveryId: "call-1", type: "submission-started", }); harness.emitBridge({ @@ -784,6 +1346,52 @@ describe("VoiceTurnController", () => { expect(harness.session.connect).toHaveBeenCalledTimes(2); }); + test.each([ + { + code: "admission-rejected" as const, + failure: { kind: "rejected", status: 403 } as const, + message: "Brunch rejected the message before admission (HTTP 403).", + }, + { + code: "admission-conflict" as const, + failure: { + kind: "submission-conflict", + status: 409, + submissionId: "submission-existing", + } as const, + message: + "The delivery key already belongs to admitted submission submission-existing; the changed payload was not admitted.", + }, + { + code: "admission-ambiguous" as const, + failure: { kind: "ambiguous" } as const, + message: + "Brunch may have accepted the message, but admission could not be confirmed. Reopen the conversation before trying again.", + }, + { + code: "admission-aborted" as const, + failure: { kind: "aborted" } as const, + message: "The local chat submission was cancelled.", + }, + ])("surfaces $failure.kind admission safely", async (admissionFailure) => { + const harness = createHarness(); + await harness.controller.start(); + harness.emitBridge({ + answer: "The supervisor approves it.", + deliveryId: "voice-turn-1", + type: "submission-started", + }); + + harness.emitBridge({ ...admissionFailure, type: "error" }); + + expect(harness.controller.getSnapshot()).toMatchObject({ + connection: "error", + errorCode: admissionFailure.code, + errorMessage: admissionFailure.message, + lastAnswerDelivery: "failed", + }); + }); + test("clears a provisional transcript when the interview fails", async () => { const harness = createHarness(); await harness.controller.start(); @@ -835,7 +1443,7 @@ describe("VoiceTurnController", () => { await bridgeFailure.controller.start(); bridgeFailure.emitBridge({ answer: "Pending answer", - callId: "call-1", + deliveryId: "call-1", type: "submission-started", }); bridgeFailure.emitBridge({ diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts index 0f4e42cbac6..c985197a334 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts @@ -19,13 +19,18 @@ export type VoiceConnectionState = | "error"; export type VoiceInputState = "listening" | "paused" | "submitting"; export type VoiceOutputState = + | "cancelling" | "idle" | "waiting-for-tool" | "speaking" | "interrupted"; export type VoiceAnswerDelivery = "none" | "pending" | "delivered" | "failed"; +export type VoiceInputNotice = "none" | "not-heard" | "too-long"; export interface VoiceTurnSnapshot { + readonly canReadFullResponse: boolean; + readonly canRepeatQuestion: boolean; + readonly canTakeTurn: boolean; readonly canReviseLastAnswer: boolean; readonly connection: VoiceConnectionState; readonly currentQuestion: string; @@ -33,6 +38,7 @@ export interface VoiceTurnSnapshot { readonly errorMessage: string; readonly errorRequestId: string; readonly input: VoiceInputState; + readonly inputNotice: VoiceInputNotice; readonly lastAnswerDelivery: VoiceAnswerDelivery; readonly lastCommittedText: string; readonly microphoneEnabled: boolean; @@ -57,14 +63,17 @@ export interface VoiceLatencyEvent { } interface RealtimeSession { - cancelOutput(): void; + cancelOutput(): Promise; connect(): Promise; disconnect(): Promise; setMicrophoneEnabled(enabled: boolean): void; + speakCanonical(segments: CanonicalSpeechSegment[]): void; subscribe(listener: (event: OpenAIRealtimeSessionEvent) => void): () => void; } interface RealtimeBridge { + cancelPendingSpeech(): void; + completeTurnHandoff(): void; start(connectionEpoch: number): void; stop(): void; subscribe(listener: (event: RealtimeBrunchBridgeEvent) => void): () => void; @@ -90,13 +99,23 @@ interface VoiceTurnControllerDependencies { interface ChatUpdate { readonly canAcceptInterviewAnswer: boolean; readonly canonicalSegments: CanonicalSpeechSegment[]; + readonly questionSegment?: CanonicalSpeechSegment; readonly settlements?: readonly VoiceSubmissionSettlement[]; readonly status: PetrinautAiVoiceModeContext["status"]; } +interface PendingSubmissionSettlement { + readonly deliveryId: string; + readonly promise: Promise; + readonly resolve: () => void; +} + type SnapshotListener = (snapshot: VoiceTurnSnapshot) => void; const initialSnapshot: VoiceTurnSnapshot = { + canReadFullResponse: false, + canRepeatQuestion: false, + canTakeTurn: false, canReviseLastAnswer: false, connection: "idle", currentQuestion: "", @@ -104,6 +123,7 @@ const initialSnapshot: VoiceTurnSnapshot = { errorMessage: "", errorRequestId: "", input: "paused", + inputNotice: "none", lastAnswerDelivery: "none", lastCommittedText: "", microphoneEnabled: false, @@ -112,11 +132,6 @@ const initialSnapshot: VoiceTurnSnapshot = { partialText: "", }; -const latestQuestion = ( - segments: CanonicalSpeechSegment[], -): CanonicalSpeechSegment | undefined => - segments.findLast(({ source }) => source === "brunch-ask"); - export class VoiceTurnController { readonly #bridge: RealtimeBridge; readonly #listeners = new Set(); @@ -125,17 +140,26 @@ export class VoiceTurnController { readonly #session: RealtimeSession; readonly #submitText: (input: SubmitTextInput) => Promise; #activeEpoch: number | null = null; + #activeSpeechOutputEnded = false; + #activeSpeechResponseId: string | null = null; + #activeSpeechResponseTerminal = false; #answerFinalizedAt: number | null = null; #answeredQuestionId: string | null = null; #bridgeStarted = false; #currentQuestionId: string | null = null; #generation = 0; #inputStateOnResume: Exclude | null = null; + #inputTurnPending = false; #latencyCorrelationId: string | null = null; + #lastResponseQuestion: CanonicalSpeechSegment | null = null; + #lastResponseSegments: CanonicalSpeechSegment[] = []; + #outputCancellationPromise: Promise | null = null; #pauseRequested = false; + #pendingSubmissionSettlement: PendingSubmissionSettlement | null = null; readonly #recordedLatencyEvents = new Set(); #snapshot = initialSnapshot; #submittingQuestionId: string | null = null; + #takingTurnPromise: Promise | null = null; #teardownPromise: Promise | null = null; #transcriptItemId: string | null = null; #transcriptKey: string | null = null; @@ -195,8 +219,14 @@ export class VoiceTurnController { } this.#inputStateOnResume = null; + this.#inputTurnPending = false; + this.#outputCancellationPromise = null; this.#pauseRequested = false; + this.#completeSubmissionSettlement(); this.#bridgeStarted = false; + this.#activeSpeechOutputEnded = false; + this.#activeSpeechResponseId = null; + this.#activeSpeechResponseTerminal = false; this.#update({ connection: "connecting", errorCode: null, @@ -234,14 +264,23 @@ export class VoiceTurnController { public async end(): Promise { ++this.#generation; this.#activeEpoch = null; + this.#activeSpeechOutputEnded = false; + this.#activeSpeechResponseId = null; + this.#activeSpeechResponseTerminal = false; this.#answerFinalizedAt = null; this.#answeredQuestionId = null; this.#bridgeStarted = false; this.#currentQuestionId = null; this.#inputStateOnResume = null; + this.#inputTurnPending = false; this.#latencyCorrelationId = null; + this.#lastResponseQuestion = null; + this.#lastResponseSegments = []; + this.#outputCancellationPromise = null; this.#recordedLatencyEvents.clear(); this.#submittingQuestionId = null; + this.#takingTurnPromise = null; + this.#completeSubmissionSettlement(); this.#pauseRequested = false; this.#transcriptItemId = null; this.#transcriptKey = null; @@ -292,7 +331,7 @@ export class VoiceTurnController { this.#inputStateOnResume = this.#snapshot.input; this.#pauseRequested = true; const output = this.#snapshot.output === "idle" ? "idle" : "interrupted"; - this.#session.cancelOutput(); + this.cancelPendingSpeech(); this.#session.setMicrophoneEnabled(false); this.#update({ input: "paused", @@ -315,17 +354,51 @@ export class VoiceTurnController { ) { return; } - this.#session.setMicrophoneEnabled(!muted); + if ( + this.#takingTurnPromise === null && + this.#outputCancellationPromise === null && + this.#activeSpeechResponseId === null && + (this.#snapshot.output === "idle" || + this.#snapshot.output === "interrupted") + ) { + this.#session.setMicrophoneEnabled(!muted); + } this.#update({ microphoneEnabled: !muted, microphoneLevel: 0 }); } - public resume(): void { + public async resume(): Promise { if ( this.#snapshot.connection !== "connected" || this.#snapshot.input !== "paused" ) { return; } + const generation = this.#generation; + while (this.#outputCancellationPromise || this.#takingTurnPromise) { + try { + await (this.#outputCancellationPromise ?? this.#takingTurnPromise); + } catch (error) { + if (generation !== this.#generation) return; + const voiceError = + error instanceof VoiceError + ? error + : new VoiceError("speech", "network", ""); + this.#setError( + voiceError.message, + voiceError.code, + voiceError.requestId, + ); + return; + } + const snapshotAfterCancellation = this.getSnapshot(); + if ( + generation !== this.#generation || + snapshotAfterCancellation.connection !== "connected" || + snapshotAfterCancellation.input !== "paused" + ) { + return; + } + } const input = this.#inputStateOnResume ?? "listening"; this.#inputStateOnResume = null; this.#pauseRequested = false; @@ -350,6 +423,11 @@ export class VoiceTurnController { this.#update({ input, microphoneEnabled: true }); } + public cancelPendingSpeech(): void { + this.#bridge.cancelPendingSpeech(); + void this.#cancelOutput(); + } + public async submitCorrection(correction: string): Promise { const correctedText = correction.trim(); const previousText = this.#snapshot.lastCommittedText; @@ -382,8 +460,80 @@ export class VoiceTurnController { } } + public readFullResponse(): void { + if (!this.#snapshot.canReadFullResponse) return; + this.#update({ output: "waiting-for-tool" }); + this.#session.speakCanonical([...this.#lastResponseSegments]); + } + + public repeatQuestion(): void { + if (!this.#snapshot.canRepeatQuestion || !this.#lastResponseQuestion) + return; + this.#update({ output: "waiting-for-tool" }); + this.#session.speakCanonical([this.#lastResponseQuestion]); + } + + /** + * Hands the turn to the user only after provider cancellation has cleared + * input and output and the active response has reached a terminal state. + */ + public takeTurn(): Promise { + if (this.#takingTurnPromise) return this.#takingTurnPromise; + if (!this.#snapshot.canTakeTurn) return Promise.resolve(); + + const generation = this.#generation; + this.#bridge.cancelPendingSpeech(); + this.#session.setMicrophoneEnabled(false); + this.#inputTurnPending = false; + this.#transcriptItemId = null; + this.#transcriptKey = null; + this.#update({ output: "cancelling", partialText: "" }); + + const submissionSettlement = + this.#pendingSubmissionSettlement?.promise ?? Promise.resolve(); + const takingTurnPromise = Promise.all([ + this.#session.cancelOutput(), + submissionSettlement, + ]) + .then(() => { + if ( + generation !== this.#generation || + this.#snapshot.connection !== "connected" || + this.#snapshot.input === "paused" + ) { + return; + } + this.#activeSpeechOutputEnded = false; + this.#activeSpeechResponseId = null; + this.#activeSpeechResponseTerminal = false; + this.#bridge.completeTurnHandoff(); + this.#session.setMicrophoneEnabled(this.#snapshot.microphoneEnabled); + this.#update({ output: "interrupted" }); + }) + .catch((error: unknown) => { + if (generation !== this.#generation) return; + const voiceError = + error instanceof VoiceError + ? error + : new VoiceError("speech", "network", ""); + this.#setError( + voiceError.message, + voiceError.code, + voiceError.requestId, + ); + }) + .finally(() => { + if (this.#takingTurnPromise === takingTurnPromise) { + this.#takingTurnPromise = null; + } + }); + this.#takingTurnPromise = takingTurnPromise; + this.#update({}); + return takingTurnPromise; + } + public updateChat(update: ChatUpdate): void { - const question = latestQuestion(update.canonicalSegments); + const question = update.questionSegment; if (question && question.id !== this.#currentQuestionId) { this.#currentQuestionId = question.id; this.#update({ currentQuestion: question.text }); @@ -391,30 +541,35 @@ export class VoiceTurnController { } this.#bridge.updateChat(update); if (this.#snapshot.input === "paused") { - this.#session.cancelOutput(); + void this.#cancelOutput(); } } #handleBridgeEvent(event: RealtimeBrunchBridgeEvent): void { if (this.#snapshot.connection !== "connected") return; if (event.type === "error") { + this.#completeSubmissionSettlement(); this.#setError(event.message, event.code); return; } if (event.type === "submission-started") { + this.#beginSubmissionSettlement(event.deliveryId); const paused = this.#snapshot.input === "paused"; if (paused) { this.#inputStateOnResume = "submitting"; } + this.#inputTurnPending = false; this.#answerFinalizedAt = this.#now(); - this.#latencyCorrelationId = event.callId; + this.#latencyCorrelationId = event.deliveryId; this.#recordedLatencyEvents.clear(); this.#submittingQuestionId = this.#currentQuestionId; this.#transcriptItemId = null; this.#transcriptKey = null; this.#ttsSpeechRequestId = null; + this.#session.setMicrophoneEnabled(false); this.#update({ input: paused ? "paused" : "submitting", + inputNotice: "none", lastAnswerDelivery: "pending", lastCommittedText: event.answer, output: "waiting-for-tool", @@ -422,6 +577,18 @@ export class VoiceTurnController { }); return; } + if (event.type === "transcript-rejected") { + if (event.reason === "duplicate" || event.reason === "unavailable") { + return; + } + this.#transcriptItemId = null; + this.#transcriptKey = null; + this.#update({ + inputNotice: event.reason === "over-limit" ? "too-long" : "not-heard", + partialText: "", + }); + return; + } if (event.type === "submission-accepted") { this.#answeredQuestionId = this.#submittingQuestionId; this.#submittingQuestionId = null; @@ -429,18 +596,20 @@ export class VoiceTurnController { return; } if (event.type === "submission-admitted") { - this.#recordLatency("submission-admitted", event.callId); + this.#recordLatency("submission-admitted", event.deliveryId); return; } if (event.type === "canonical-text-ready") { - this.#recordLatency("first-canonical-text", event.callId); + this.#recordLatency("first-canonical-text", event.deliveryId); return; } if (event.type === "submission-settled") { - this.#recordLatency("submission-settled", event.callId); + this.#completeSubmissionSettlement(event.deliveryId); + this.#recordLatency("submission-settled", event.deliveryId); return; } if (event.type === "submission-stopped") { + this.#completeSubmissionSettlement(event.deliveryId); // Brunch was stopped before it replied: nothing to speak, and the // interviewer is free to listen again. const pausedWhileStopped = this.#snapshot.input === "paused"; @@ -453,19 +622,32 @@ export class VoiceTurnController { }); return; } + this.#completeSubmissionSettlement(event.deliveryId); + this.#lastResponseQuestion = event.questionSegment ?? null; + this.#lastResponseSegments = [...event.segments]; + const responseEnd = event.segments.at(-1); + if (event.speechCancelled) { + const paused = this.#snapshot.input === "paused"; + if (paused) { + this.#inputStateOnResume = "listening"; + } + this.#update({ + input: paused ? "paused" : "listening", + output: "interrupted", + }); + if (responseEnd) this.#recordLatency("answer-ready", responseEnd.id); + return; + } const paused = this.#snapshot.input === "paused"; if (paused) { this.#inputStateOnResume = "listening"; - this.#session.cancelOutput(); + void this.#cancelOutput(); } this.#update({ input: paused ? "paused" : "listening", output: paused ? "interrupted" : "waiting-for-tool", }); - const question = event.segments.findLast( - ({ source }) => source === "brunch-ask", - ); - if (question) this.#recordLatency("answer-ready", question.id); + if (responseEnd) this.#recordLatency("answer-ready", responseEnd.id); } #handleSessionEvent(event: OpenAIRealtimeSessionEvent): void { @@ -486,6 +668,11 @@ export class VoiceTurnController { return; } if (event.type === "canonical-speech-requested") { + this.#session.setMicrophoneEnabled(false); + this.#inputTurnPending = false; + this.#transcriptItemId = null; + this.#transcriptKey = null; + this.#update({ partialText: "" }); if ( this.#latencyCorrelationId !== null && this.#ttsSpeechRequestId === null @@ -496,12 +683,18 @@ export class VoiceTurnController { return; } if (event.type === "output-started") { + this.#activeSpeechOutputEnded = false; + this.#activeSpeechResponseId = event.responseId; + this.#activeSpeechResponseTerminal = false; + this.#inputTurnPending = false; + this.#transcriptItemId = null; + this.#transcriptKey = null; if (this.#snapshot.input === "paused") { - this.#session.cancelOutput(); - this.#update({ output: "interrupted" }); + void this.#cancelOutput(); + this.#update({ output: "interrupted", partialText: "" }); return; } - this.#update({ output: "speaking" }); + this.#update({ output: "speaking", partialText: "" }); if ( this.#latencyCorrelationId !== null && event.speechRequestId === this.#ttsSpeechRequestId @@ -514,32 +707,56 @@ export class VoiceTurnController { return; } if (event.type === "output-stopped") { - this.#update({ output: "idle" }); + if (event.responseId !== this.#activeSpeechResponseId) return; + this.#activeSpeechOutputEnded = true; + if (this.#activeSpeechResponseTerminal) { + this.#clearSettledSpeech(); + } + this.#update({ + output: this.#takingTurnPromise ? "cancelling" : "idle", + }); if (this.#currentQuestionId) { this.#recordLatency("question-spoken", this.#currentQuestionId); } return; } if (event.type === "output-interrupted") { - this.#update({ output: "interrupted" }); + if (event.responseId !== this.#activeSpeechResponseId) return; + this.#activeSpeechOutputEnded = true; + if (this.#activeSpeechResponseTerminal) { + this.#clearSettledSpeech(); + } + this.#update({ + output: this.#takingTurnPromise ? "cancelling" : "interrupted", + }); return; } if (event.type === "input-speech-started") { + if ( + this.#takingTurnPromise || + this.#snapshot.output === "speaking" || + this.#snapshot.output === "cancelling" + ) { + return; + } + this.#inputTurnPending = true; this.#transcriptItemId = event.itemId; this.#transcriptKey = null; - if (this.#snapshot.output === "speaking") { - this.#update({ output: "interrupted", partialText: "" }); - } else { - this.#update({ partialText: "" }); + this.#update({ inputNotice: "none", partialText: "" }); + return; + } + if (event.type === "response-terminal") { + if (event.responseId === this.#activeSpeechResponseId) { + if (this.#activeSpeechOutputEnded) { + this.#clearSettledSpeech(); + } else { + this.#activeSpeechResponseTerminal = true; + } + this.#update({}); } return; } - if ( - event.type === "input-speech-stopped" || - event.type === "response-terminal" || - event.type === "tool-arguments-delta" || - event.type === "tool-arguments-done" - ) { + if (event.type === "input-speech-stopped") { return; } @@ -547,6 +764,7 @@ export class VoiceTurnController { if (event.key.connectionEpoch !== this.#activeEpoch) return; if (event.key.itemId !== this.#transcriptItemId) return; if (event.type === "transcription-failed") { + this.#inputTurnPending = false; this.#transcriptItemId = null; this.#transcriptKey = null; this.#update({ partialText: "" }); @@ -560,6 +778,7 @@ export class VoiceTurnController { }); return; } + this.#inputTurnPending = false; this.#transcriptItemId = null; this.#transcriptKey = null; this.#update({ @@ -574,9 +793,16 @@ export class VoiceTurnController { ): void { ++this.#generation; this.#activeEpoch = null; + this.#activeSpeechOutputEnded = false; + this.#activeSpeechResponseId = null; + this.#activeSpeechResponseTerminal = false; this.#inputStateOnResume = null; + this.#inputTurnPending = false; this.#latencyCorrelationId = null; + this.#outputCancellationPromise = null; this.#recordedLatencyEvents.clear(); + this.#takingTurnPromise = null; + this.#completeSubmissionSettlement(); this.#bridgeStarted = false; this.#transcriptItemId = null; this.#transcriptKey = null; @@ -601,6 +827,58 @@ export class VoiceTurnController { }); } + #cancelOutput(): Promise { + const cancellationPromise = this.#session.cancelOutput(); + this.#outputCancellationPromise = cancellationPromise; + void cancellationPromise.then( + () => { + if (this.#outputCancellationPromise === cancellationPromise) { + this.#outputCancellationPromise = null; + } + }, + () => { + if (this.#outputCancellationPromise === cancellationPromise) { + this.#outputCancellationPromise = null; + } + }, + ); + return cancellationPromise; + } + + #beginSubmissionSettlement(deliveryId: string): void { + this.#completeSubmissionSettlement(); + let resolve = () => {}; + const promise = new Promise((resolvePromise) => { + resolve = resolvePromise; + }); + this.#pendingSubmissionSettlement = { deliveryId, promise, resolve }; + } + + #completeSubmissionSettlement(deliveryId?: string): void { + const pending = this.#pendingSubmissionSettlement; + if ( + pending === null || + (deliveryId !== undefined && deliveryId !== pending.deliveryId) + ) { + return; + } + this.#pendingSubmissionSettlement = null; + pending.resolve(); + } + + #clearSettledSpeech(): void { + this.#activeSpeechOutputEnded = false; + this.#activeSpeechResponseId = null; + this.#activeSpeechResponseTerminal = false; + if ( + this.#snapshot.connection === "connected" && + this.#snapshot.input === "listening" && + this.#takingTurnPromise === null + ) { + this.#session.setMicrophoneEnabled(this.#snapshot.microphoneEnabled); + } + } + #recordLatency(name: VoiceLatencyEvent["name"], correlationId: string): void { if (this.#answerFinalizedAt === null) return; const eventKey = `${correlationId}:${name}`; @@ -623,14 +901,42 @@ export class VoiceTurnController { ); } + #canReplay(snapshot: VoiceTurnSnapshot): boolean { + return ( + snapshot.connection === "connected" && + snapshot.input === "listening" && + !this.#inputTurnPending && + this.#activeSpeechResponseId === null && + (snapshot.output === "idle" || snapshot.output === "interrupted") + ); + } + + #canTakeTurn(snapshot: VoiceTurnSnapshot): boolean { + return ( + snapshot.connection === "connected" && + snapshot.input !== "paused" && + (snapshot.output === "waiting-for-tool" || + snapshot.output === "speaking") && + this.#currentQuestionId !== null && + this.#currentQuestionId !== this.#answeredQuestionId && + this.#currentQuestionId !== this.#submittingQuestionId && + Boolean(snapshot.currentQuestion) && + this.#takingTurnPromise === null + ); + } + #isPauseRequested(): boolean { return this.#pauseRequested; } #update(update: Partial): void { const snapshot = { ...this.#snapshot, ...update }; + const canReplay = this.#canReplay(snapshot); this.#snapshot = { ...snapshot, + canReadFullResponse: canReplay && this.#lastResponseSegments.length > 0, + canRepeatQuestion: canReplay && this.#lastResponseQuestion !== null, + canTakeTurn: this.#canTakeTurn(snapshot), canReviseLastAnswer: this.#canReviseLastAnswer(snapshot), }; for (const listener of this.#listeners) listener(this.#snapshot); diff --git a/apps/petrinaut-website/src/server/voice/openai-realtime-call.test.ts b/apps/petrinaut-website/src/server/voice/openai-realtime-call.test.ts index 38188a178c3..c7f4f94c284 100644 --- a/apps/petrinaut-website/src/server/voice/openai-realtime-call.test.ts +++ b/apps/petrinaut-website/src/server/voice/openai-realtime-call.test.ts @@ -86,7 +86,7 @@ describe("OpenAI Realtime call handler", () => { expect(fetch).not.toHaveBeenCalled(); }); - test("forwards only the SDP and server-owned duplex Realtime policy", async () => { + test("forwards only the SDP and server-owned half-duplex Realtime policy", async () => { const reportDiagnostic = vi.fn(); const fetch = vi.fn( async () => @@ -131,16 +131,16 @@ describe("OpenAI Realtime call handler", () => { type: "realtime", model: "gpt-realtime-2", output_modalities: ["audio"], - tool_choice: "required", - tools: [{ name: "continue_interview", type: "function" }], + tool_choice: "none", + tools: [], audio: { input: { transcription: { model: "gpt-4o-transcribe", language: "en" }, turn_detection: { type: "semantic_vad", eagerness: "low", - create_response: true, - interrupt_response: true, + create_response: false, + interrupt_response: false, }, }, }, diff --git a/apps/petrinaut-website/src/server/voice/openai-voice-policy.test.ts b/apps/petrinaut-website/src/server/voice/openai-voice-policy.test.ts index 1cdd3f0fead..523b8c85f17 100644 --- a/apps/petrinaut-website/src/server/voice/openai-voice-policy.test.ts +++ b/apps/petrinaut-website/src/server/voice/openai-voice-policy.test.ts @@ -46,18 +46,18 @@ describe("OpenAI voice policy", () => { ).toEqual({ available: true, connectionTimeoutMs: 15_000 }); }); - test("owns the trusted GPT-Realtime-2 duplex session policy", () => { - expect(OPENAI_REALTIME_POLICY_VERSION).toBe("brunch-control-plane-v1"); + test("owns the trusted GPT-Realtime-2 half-duplex session policy", () => { + expect(OPENAI_REALTIME_POLICY_VERSION).toBe("brunch-control-plane-v3"); expect(createOpenAIRealtimeSession()).toEqual({ type: "realtime", model: "gpt-realtime-2", output_modalities: ["audio"], reasoning: { effort: "low" }, parallel_tool_calls: false, - tool_choice: "required", + tool_choice: "none", instructions: `# Role and objective -You are the realtime voice of an expert interviewer for process-model elicitation. The person speaking is the domain expert. Listen attentively, submit each complete spoken answer to Brunch, and deliver Brunch's next interview turn. +You are the realtime voice of an expert interviewer for process-model elicitation. The person speaking is the domain expert. Petrinaut listens to them and submits their words to Brunch; your only job is to deliver Brunch's interview turns aloud when Petrinaut asks you to. # Personality and delivery @@ -65,29 +65,16 @@ Sound warm, calm, curious, confident, concise, and professionally neutral. Speak # Authority -Brunch is the sole authority for interview state, questions, captures, completion, and business decisions. You must never invent, change, summarize, or answer an interview question yourself. +Brunch is the sole authority for interview state, questions, captures, completion, and business decisions. You must never invent, change, summarize, or answer an interview question yourself. You must never restate, guess, or fill in what the speaker said. # Turn handling -After semantic turn detection finds that the user has finished a complete spoken answer, call continue_interview exactly once with that answer. Do not speak, emit a preamble, or emit conversational text before calling the tool. +Never respond on your own after the speaker stops talking. Petrinaut transcribes their words and decides what happens next. Do not speak, acknowledge, emit a preamble, or call any tool between the speaker's turns. # Canonical output -After the tool result arrives, speak only its response_text strings, in array order and verbatim. Do not add, remove, paraphrase, acknowledge, or explain anything. Never call another tool while speaking a tool result.`, - tools: [ - { - type: "function", - name: "continue_interview", - description: - "Submit the user's complete spoken answer to the authoritative Brunch interview.", - parameters: { - type: "object", - additionalProperties: false, - properties: { answer: { type: "string" } }, - required: ["answer"], - }, - }, - ], +When Petrinaut supplies response_text, speak only those strings, in array order and verbatim. Do not add, remove, paraphrase, acknowledge, or explain anything.`, + tools: [], audio: { input: { noise_reduction: { type: "far_field" }, @@ -100,8 +87,8 @@ After the tool result arrives, speak only its response_text strings, in array or turn_detection: { type: "semantic_vad", eagerness: "low", - create_response: true, - interrupt_response: true, + create_response: false, + interrupt_response: false, }, }, output: { voice: "marin" }, @@ -109,12 +96,17 @@ After the tool result arrives, speak only its response_text strings, in array or }); }); - test("allows no provider-owned interview decisions or unrestricted tools", () => { - const serializedPolicy = JSON.stringify(createOpenAIRealtimeSession()); + test("lets Realtime neither answer for the user nor call tools between turns", () => { + const policy = createOpenAIRealtimeSession(); + const serializedPolicy = JSON.stringify(policy); expect(serializedPolicy).not.toContain("response.create"); expect(serializedPolicy).not.toContain("gpt-realtime-1.5"); + expect(serializedPolicy).not.toContain("continue_interview"); expect(serializedPolicy).not.toContain('"tool_choice":"auto"'); - expect(createOpenAIRealtimeSession().tools).toHaveLength(1); + expect(serializedPolicy).not.toContain('"tool_choice":"required"'); + expect(policy.tools).toHaveLength(0); + expect(policy.audio.input.turn_detection.create_response).toBe(false); + expect(policy.audio.input.transcription.model).toBe("gpt-4o-transcribe"); }); }); diff --git a/apps/petrinaut-website/src/server/voice/openai-voice-policy.ts b/apps/petrinaut-website/src/server/voice/openai-voice-policy.ts index fc401fc0c7b..653b622dc3a 100644 --- a/apps/petrinaut-website/src/server/voice/openai-voice-policy.ts +++ b/apps/petrinaut-website/src/server/voice/openai-voice-policy.ts @@ -1,5 +1,5 @@ export const OPENAI_REALTIME_CONNECTION_TIMEOUT_MS = 15_000; -export const OPENAI_REALTIME_POLICY_VERSION = "brunch-control-plane-v1"; +export const OPENAI_REALTIME_POLICY_VERSION = "brunch-control-plane-v3"; interface VoiceEnvironment { readonly NODE_ENV?: string; @@ -24,7 +24,7 @@ export const getOpenAIVoiceAvailability = (environment: VoiceEnvironment) => ({ const REALTIME_INSTRUCTIONS = `# Role and objective -You are the realtime voice of an expert interviewer for process-model elicitation. The person speaking is the domain expert. Listen attentively, submit each complete spoken answer to Brunch, and deliver Brunch's next interview turn. +You are the realtime voice of an expert interviewer for process-model elicitation. The person speaking is the domain expert. Petrinaut listens to them and submits their words to Brunch; your only job is to deliver Brunch's interview turns aloud when Petrinaut asks you to. # Personality and delivery @@ -32,38 +32,31 @@ Sound warm, calm, curious, confident, concise, and professionally neutral. Speak # Authority -Brunch is the sole authority for interview state, questions, captures, completion, and business decisions. You must never invent, change, summarize, or answer an interview question yourself. +Brunch is the sole authority for interview state, questions, captures, completion, and business decisions. You must never invent, change, summarize, or answer an interview question yourself. You must never restate, guess, or fill in what the speaker said. # Turn handling -After semantic turn detection finds that the user has finished a complete spoken answer, call continue_interview exactly once with that answer. Do not speak, emit a preamble, or emit conversational text before calling the tool. +Never respond on your own after the speaker stops talking. Petrinaut transcribes their words and decides what happens next. Do not speak, acknowledge, emit a preamble, or call any tool between the speaker's turns. # Canonical output -After the tool result arrives, speak only its response_text strings, in array order and verbatim. Do not add, remove, paraphrase, acknowledge, or explain anything. Never call another tool while speaking a tool result.`; +When Petrinaut supplies response_text, speak only those strings, in array order and verbatim. Do not add, remove, paraphrase, acknowledge, or explain anything.`; +/** + * The completed input transcription is the only source of the user's answer. + * Semantic VAD therefore commits audio without creating a response or + * interrupting playback, and the Realtime model has no tools with which to + * manufacture an answer. + */ export const createOpenAIRealtimeSession = () => ({ type: "realtime" as const, model: "gpt-realtime-2", output_modalities: ["audio"] as const, reasoning: { effort: "low" as const }, parallel_tool_calls: false, - tool_choice: "required" as const, + tool_choice: "none" as const, instructions: REALTIME_INSTRUCTIONS, - tools: [ - { - type: "function" as const, - name: "continue_interview", - description: - "Submit the user's complete spoken answer to the authoritative Brunch interview.", - parameters: { - type: "object" as const, - additionalProperties: false, - properties: { answer: { type: "string" as const } }, - required: ["answer"] as const, - }, - }, - ], + tools: [] as const, audio: { input: { noise_reduction: { type: "far_field" as const }, @@ -76,8 +69,8 @@ export const createOpenAIRealtimeSession = () => ({ turn_detection: { type: "semantic_vad" as const, eagerness: "low" as const, - create_response: true, - interrupt_response: true, + create_response: false, + interrupt_response: false, }, }, output: { voice: "marin" as const }, diff --git a/libs/@hashintel/brunch-agent/MISSION.md b/libs/@hashintel/brunch-agent/MISSION.md index 2d09a6215ad..ed8ff9333e4 100644 --- a/libs/@hashintel/brunch-agent/MISSION.md +++ b/libs/@hashintel/brunch-agent/MISSION.md @@ -1,163 +1,197 @@ -# Mission 5 — one Flue conversation route for Voice and the typed panel +# Mission 5 successor — Voice safety and UX parity on the unified Flue route ## Status -**Live as of 2026-09-03** for [FE-1574](https://linear.app/hash/issue/FE-1574/let-voice-speak-through-canonical-brunch-conversations) on `ln/fe-1574-direct-voice-flue`, stacked directly on the closed Mission 4 branch. This is the sole execution authority for the branch. The builder implementation now routes typed panel and finalized Voice turns through one browser `FlueClient` at `/agents/chat/:instanceId`, projects canonical replies into Petrinaut and TTS, uses durable Flue abort for explicit Stop, and rehydrates canonical conversation state through SDK observation. The former Brunch `/api/chat` handler and projector are deleted. Mission acceptance remains open until the required human demo and proof-leaf-8 bundle exist. - -The accepted departure base remains Mission 4's package-composed `ChatAgent`: `useBrunchAgent()` mounts core's independent `elicitation` capability and `useSdcpnPlugin()` mounts the SDCPN job contribution. `@hashintel/brunch-agent-transport-aisdk` is now the browser adapter over public `@flue/sdk`, not a server handler. The current external Voice evidence remains PR [#9496](https://github.com/hashintel/hash/pull/9496) at `c7fe8a2e68e8fdc37018b21ec2e9daf4e9ef7c82`, PR [#9507](https://github.com/hashintel/hash/pull/9507) at `252b9dbb0c77fae8cee45a506f09cac3e20c381c`, and PR [#9512](https://github.com/hashintel/hash/pull/9512) at `d13535d1077b3a78d6a1411031b7d0a0a78e3144`. They are read-only source evidence, not branches to rewrite or wholesale architecture to restore. - -Cold-start reads are [`docs/evidence/implementations/mission-4-voice-integration-handoff.md`](docs/evidence/implementations/mission-4-voice-integration-handoff.md), [`packages/transport-aisdk/src/index.ts`](packages/transport-aisdk/src/index.ts), [`packages/transport-aisdk/src/ui-stream.ts`](packages/transport-aisdk/src/ui-stream.ts), [`packages/transport-aisdk/src/transcript.ts`](packages/transport-aisdk/src/transcript.ts), [`apps/brunch-agent/src/app.ts`](../../../apps/brunch-agent/src/app.ts), [`apps/brunch-agent/src/http/ownership.ts`](../../../apps/brunch-agent/src/http/ownership.ts), [`apps/brunch-agent/test/petrinaut-chat.integration.ts`](../../../apps/brunch-agent/test/petrinaut-chat.integration.ts), [`apps/brunch-agent/test/architecture/boundaries.integration.ts`](../../../apps/brunch-agent/test/architecture/boundaries.integration.ts), [`apps/petrinaut-website/src/main/app/local-storage-demo/`](../../../apps/petrinaut-website/src/main/app/local-storage-demo/), [`apps/petrinaut-website/src/main/app/voice-interview/`](../../../apps/petrinaut-website/src/main/app/voice-interview/), and the installed Flue 2.0.3 documentation at `node_modules/@flue/sdk/docs/sdk/flue-client.md`, `node_modules/@flue/sdk/docs/reference/streaming-protocol.md`, and `node_modules/@flue/sdk/docs/guide/react.md`. +**Live as of 2026-09-04** for [FE-1580](https://linear.app/hash/issue/FE-1580/harden-voice-safety-and-ux-on-the-unified-flue-route) on `kostandin/fe-1580-harden-voice-safety-and-ux-on-the-unified-flue-route`, stacked directly on [PR #9528](https://github.com/hashintel/hash/pull/9528) at the GitHub-verified head `eecbe99e201fd8cb78d9b719e789b6abd373ed1b`. This file is the sole execution authority for the successor branch. The restack adopts the parent's canonical hydration overwrite guard, multi-submission response correlation, settlement-driven durable Stop, aligned live/snapshot projection, queued Voice-input cancellation, and client-tool continuation behavior. The parent remains the authority for defects in those mechanisms; this branch must restack onto further parent fixes rather than repair them. + +The owner selected **half-duplex turn ownership** on 2026-09-03. While canonical assistant audio is pending or playing, the microphone is closed. Ownership transfers away from input as soon as canonical speech is requested, before `response.create` is sent: every accepted unfinished input item becomes stale and provisional transcript state is cleared. Initial automatic speech may begin before Brunch settlement when a new canonical segment is durably completed and correlated to the active Voice submission. The explicit **Your turn** action may cancel that audio immediately, but opens a fresh input turn only after both provider cancellation acknowledgement and Brunch settlement. Automatic duplex barge-in is rejected because assistant playback can become a false user turn. + +On 2026-09-04, the owner approved a non-interactive Brunch-owned question +marker for exact **Repeat question** replay. The marker is a server tool plus a +durable client data part: it identifies exact assistant-authored text but never +suspends for an answer, mounts `brunch_ask`, or creates a second Voice submission +path. For direct-user Voice provenance, the owner selected an upstream Flue +user-metadata contract rather than a local runtime patch or correlated sidecar +signal. The [decision record](docs/evidence/design/mission-5-question-marker-and-provenance-decision-2026-09-04.md) +defines the accepted and rejected mechanics. + +### Turn-ownership decision + +1. **Adopted — half-duplex explicit handoff.** It gives assistant playback exclusive ownership, makes cancellation settlement a visible boundary, and guarantees fresh post-handoff capture. The product cost is one extra **Your turn** action and barrier latency when the user interrupts. +2. **Rejected — automatic duplex barge-in.** It offers the most conversational interruption and avoids an explicit control, but an open microphone can transcribe assistant playback as a user answer, and capture can race an unsettled cancellation. This risk is unacceptable for authoritative completed transcripts. + +The parent now prevents its once-per-conversation canonical hydration from overwriting a locally submitted turn. The real microphone, handoff, Stop, hard-reload, and same-origin witness is therefore executable but remains unproved until the human witness and retained artifact bundle are complete. + +On 2026-09-04, the owner directed PR #9531 to close one bounded launcher +defect exposed while starting that witness. The Brunch panel config now retains +Petrinaut's stock API plugin so `/api/voice/config` and +`/api/voice/realtime-call` remain available beside the unchanged +`/agents/chat/*` Flue proxy. This is an explicit exception to the parent-defect +constraint below; it changes no conversation or Voice-turn semantics. + +On 2026-09-04, the owner directed PR #9531 to close two more bounded Petrinaut +defects: preserve real Flue and browser-tool errors through the transport and +display them in full, and replace deferred Voice-transcript disclosure with +immediate transcript display plus a dock-only collapsed live-session view. +Initial-CTA Voice entry defaults to that compact presentation; before a live +session starts, the consent and microphone-permission card sits above a compact +Voice setup dock, while composer-initiated Voice entry keeps the existing +expanded presentation. Ending Voice from a collapsed dock also closes the AI +panel instead of restoring its expanded text composer; ending from an expanded +dock retains the existing return-to-composer behavior. +These are explicit exceptions to the parent-defect constraint below; they +change no conversation authority, submission path, or Voice-turn semantics, +and local playback cancellation remains separate from durable abort. + +Completed-transcript authority, half-duplex ownership, admission idempotency, +the cancellation barrier, exact full-response replay, and exact marked-question +replay have focused regression coverage, including the interval between a +canonical speech request and output start. **Repeat question** is enabled only +for an approved durable Brunch marker whose exact text appears in finalized +assistant prose from the same message; the final text segment remains invalid +question authority. Proof item 5 is complete +only for the supported client-tool-result path: Flue signals persist each +Voice-origin tool-call id beside its output, and canonical projection +reconstructs multiple surviving origins. Direct spoken user attribution is +blocked because Flue 2.0.3 projects the generated `submissionId` but neither +caller metadata nor the caller idempotency key. The discarded +browser-correlation implementation would have violated the explicit +second-durable-store stop condition. The restacked hydration guard removes the +old parent blocker, but no real witness claim is valid until the retained human +evidence exists. + +The 2026-09-04 corrective verification covers the current 72-file successor +diff against #9528 head `eecbe99e201f`: the four focused race cases pass 4/4 +tests, the filtered production admission-outcome cases pass 3/3 tests, and the +complete seven-workspace Turbo run passes 39/39 tasks and 1,123/1,123 tests, +including 282/282 website tests and the Brunch core package. Architecture +validation passes with 68 layers, 337 edges, 690 files, 69 generated pages, and +38 authored pages. `git diff --check` passes, and the root formatter accepts all +5,527 matched files. The exact commands and dispositions are retained in the +[donor matrix](docs/evidence/implementations/mission-5-voice-safety-parity/donor-behavior-matrix.md#corrective-verification). + +The pinned donor-behavior decision record is the [FE-1580 donor matrix](docs/evidence/implementations/mission-5-voice-safety-parity/donor-behavior-matrix.md). Donor PRs are read-only evidence at their named heads; semantic reimplementation is required, never merge or cherry-pick. ## Imperative -Make the mounted Flue conversation route the only product door into a Brunch conversation, and make Voice a faithful audio projection of that one canonical conversation. One finalized spoken answer and one typed panel message must both enter the owning Flue conversation through `@flue/sdk` against `/agents/chat/:instanceId`, and the corresponding canonical Brunch response must reach visible text and TTS without another model rewriting the text. Do this now because Mission 4 established the canonical agent composition while two transports still exist to the same conversation: the Voice preview's AI SDK composer path and the server-side `/api/chat` adapter, which admits through a different code path than the SDK does. Routing Voice onto Flue while keeping `/api/chat` for typed text would harden the split into two routes, two ownership rules, and two protocols; the least mechanism is one route, with the AI SDK reduced to the panel's rendering contract behind a host-supplied browser `ChatTransport`. - -### Product-manager litmus - -Adopted on restack onto the parent spine's 2026-09-03 litmus reframing. A product manager who did not watch the work must be able to notice the advance; the single-route consolidation, the browser `ChatTransport`, the deleted `/api/chat` door, and the repurposed transport package are internal sequencing and must not be presented as the advance. - -**Release note:** in the Petrinaut Brunch panel you can type or speak to Brunch in one conversation; what you hear is exactly what Brunch wrote; **Stop** really stops Brunch rather than just hiding its answer; and reopening the panel shows the same conversation you left, without re-sending or replaying anything. - -**Demo script (no engineer present), on the deployment posture available at cut time — the local `yarn dev:brunch` pair with the Brunch preview selected:** open the panel and type one message; read the reply. Start Voice mode and speak one answer; see exactly one new user message appear, then see Brunch's reply appear as text and hear the same words read aloud. Speak over it once; playback stops and the text stays. Ask a second question and press **Stop** while Brunch is still working; the conversation shows that turn as stopped, not as an answer. Close the panel and reopen the same conversation: the typed turn, the spoken turn, and the stopped turn are all there exactly as you saw them, nothing replays, and nothing is sent again. +Make Voice safe and product-complete on the one Flue conversation route established by the parent. Only a completed provider transcription may become a spoken answer; one logical typed or Voice delivery must admit at most one Flue turn; assistant output must yield the microphone through an acknowledged cancellation barrier; exact canonical responses must be replayable; and Voice attribution must survive canonical hydration and reopen. -**Previously impossible:** Stop only cancelled the browser request while Brunch kept working, so reopening the panel showed a full answer you had stopped; typed and spoken turns entered Brunch through different doors, so a spoken turn could be held or ordered differently from a typed one. +Voice path B is the **only admissible submission shape**: -**Completion:** the mission is complete at the contract stratum below — when a product manager can run this demo script end to end and proof leaf 8's witness bundle records it — not when the first typed or spoken turn crosses the route. The first green typed-panel tracer and the first green Voice tracer are internal milestones. - -### Recut rationale +```text +Voice completed transcript +→ Voice controller validates one keyed transcript identity +→ panel submitVoiceInputWithAdmission +→ panel submitVoiceInput +→ shared useChat submitText +→ host-supplied Flue ChatTransport +→ client.send({ message, idempotencyKey, signal }) +→ /agents/chat/:instanceId +``` -Inspected at the real boundary on 2026-09-03 (`node_modules/@flue/sdk/docs/reference/streaming-protocol.md`, `packages/transport-aisdk/src/index.ts`, `packages/transport-aisdk/src/ui-stream.ts`, `apps/petrinaut-website/src/main/app/local-storage-demo/brunch-panel-transport.ts`, `node_modules/ai/dist/index.d.ts` `ChatTransport`): +Voice may not call `FlueClient.send()` directly and may not own a second mutable transcript. A direct-send fallback would recreate the second admission path this stack exists to remove. -- Flue's SSE does not remove the need for translation while the panel speaks `useChat`: Flue admits one `DeliveredMessage` with a 202 and streams `ConversationStreamChunk` batches on a separate, never-ending offset-resumed read; the AI SDK posts the whole `messages` array and expects one finite `UIMessageChunk` stream per turn. Request shape, vocabulary, and lifecycle all differ, and `@flue/*` ships no AI SDK adapter. -- At the recut, the adapter was one translation cut across two homes by the transport topology gate: AI SDK request framing, CORS, and principal parsing in `transport-aisdk` (then Flue-free), and the load-bearing `ConversationStreamChunk → UIMessageChunk` projection plus snapshot → UI messages in `apps/brunch-agent/src/conversation/`. -- A Hono-level relocation would have re-expressed the same admit → follow → project → terminate join as a server handler; it would have removed nothing and kept two routes. It was rejected. -- AI SDK `ChatTransport` is a client-side interface, and Petrinaut already accepts and wraps host-supplied transports. The landed browser transport over the same `createFlueClient()` that Voice uses owns the projector, removes `/api/chat` and its handler, and makes typed and spoken turns share one admission path, one ownership guard, and one protocol. +The parent's claim that Flue 2.0.3 cannot accept caller idempotency is false. The installed `@flue/sdk` 2.0.3 typings expose `AgentPromptOptions.idempotencyKey?: string`, `AgentSendResult.deduplicated?: boolean`, and the 409 `submission_conflict` response with the existing `submissionId` in `FlueApiError.body.error.meta.submissionId`. The invariant is **at most one admitted turn**, not exactly one invocation of `send()`. ## Throughline -The real boundary is the local Petrinaut surface driven by `yarn dev:brunch` — both its typed panel and its Voice control — through one same-origin-proxied Flue route into the mounted production `ChatAgent`, back to visible Petrinaut text and, for Voice, audible playback: +The production throughline is the local Petrinaut Brunch surface driven by `yarn dev:brunch`: ```text - Petrinaut typed panel (`useChat`) Petrinaut microphone - → host-supplied browser `ChatTransport` → OpenAI Realtime provisional STT + turn detection - (`sendMessages` → one user text or → one validated finalized `continue_interview` answer - one client-tool-result signal) - └──────────────┬────────────────────────────┘ -→ one browser `createFlueClient()` per selected principal + logical conversation id -→ one supported `send()` admission at the same-origin proxied `/agents/chat/:instanceId` route - (Voice may enter via the panel's transport — preferred, one visible store — or call `send()` directly; see fog-line) -→ `agentOwnershipGuard` (the only ownership check) → `createAgentRouter(ChatAgent)` -→ current `ChatAgent` with `useBrunchAgent()` + `useSdcpnPlugin()` -→ SDK reads: `wait(admission, { onEvent })` for the panel's finite per-turn stream, - `observe({ live: "sse" })` for canonical state and reopen -→ response parts correlated by server-issued `submissionId` -→ panel: existing `ConversationStreamChunk → UIMessageChunk` projector, terminated on `submission-settled` -→ Voice: canonical completed Brunch text displayed and passed unchanged as TTS input -→ local playback/observation cancellation or explicit conversation-wide `abort()` -→ observation rehydration after reopening the same logical conversation +OpenAI Realtime microphone input +→ semantic VAD marks an input boundary but creates no model response +→ conversation.item.input_audio_transcription.completed +→ keyed transcript authority (connection epoch, item id, content index) +→ half-duplex Voice controller and shared panel submission path B +→ browser AI SDK ChatTransport over the memoized FlueClient +→ idempotent Flue admission on the same-origin /agents/chat/:instanceId proxy +→ agentOwnershipGuard → mounted Brunch ChatAgent +→ durably completed, submission-correlated canonical segments +→ each new finalized segment enters the exact speech queue while remaining work continues +→ settlement releases exact replay, the next input turn, and supported client-tool Voice provenance +→ observe({ live: "sse" }) hydration and reopen ``` -The `/api/chat` route, `createPetrinautChatHandler`, the in-process `init().dispatch()/read()` admission path, the `GET ?id=` history door, and the `/api/chat` Vite proxy are removed from the Brunch app; the local launcher proxies `/agents/chat/*` instead. `@hashintel/brunch-agent-transport-aisdk` is repurposed as the browser-side adapter: it exports the projector, the snapshot → UI-message projection, the header names, and a `ChatTransport` factory over a caller-supplied `FlueClient`, and depends on `ai` and the public `@flue/sdk` client only. The Petrinaut panel itself stays on `useChat`; it is not rewritten onto `@flue/react`. +Realtime exposes no tools, uses `tool_choice: "none"`, and configures semantic VAD with `create_response: false`. Model function-call arguments are ignored even if a provider violates the policy. Provisional transcription is display-only and disappears without submission. OpenAI permits transcription completion for any committed audio item and does not guarantee completion order across turns; this mission deliberately accepts only an item whose matching `speech_started` boundary occurred during the current input turn. A boundaryless or completion-before-boundary item remains rejected rather than gaining authority retroactively. Requesting canonical speech ends that input turn before `response.create`: unfinished accepted items and their provisional display state are invalidated even if their transcription completes before output audio starts. -`conversationId` is the stable logical reference selected by the Petrinaut host. The current principal plus that id mechanically derives the Flue instance path and ownership headers. `submissionId` correlates one admitted answer and its settlement. Flue offsets remain opaque. `uid` identifies one current incarnation and may guard a known continuation, but it must not replace the logical conversation id or be presented as durable user identity. +Local playback cancellation, local observation cancellation, the HTTP request `AbortSignal`, and durable conversation-wide `FlueClient.abort()` remain separate operations. The first three never masquerade as durable Stop; durable Stop never appears as a Voice transcription or playback failure. -For maintained Voice state beyond what the panel already holds, use the SDK's materialized `observe()` surface, or `useFlueAgent()` over the same memoized client if the React seam earns it. Use `readSubmissionReply()`, `read()`, or `wait({ onEvent })` only for submission-scoped extraction, not as a parallel transcript reducer; the panel transport's per-submission projection into `useChat`'s own store is that submission-scoped use, not a second transcript. Do not parse SSE, calculate offsets, retry stream chunks, or hand-pick the latest message. The tracer may stream canonical text visibly through the observation, but TTS begins only from completed speakable segments and may remain settlement-gated; token-by-token speech is not part of this claim. +## Proof -The first tracer is text-turn-only at the Brunch boundary. It does not require the temporary `brunch_ask` client-tool shim: a finalized spoken answer is a direct Flue user message, and canonical plain assistant text is sufficient to prove the transport. If the real tracer cannot preserve answer correlation without structured questions, stop and present that observed strain before mounting the suspended capability. +This mission closes the Voice safety and UX-parity stratum on the parent's route. It does not establish production identity, remote deployment, structured questions, a live `brunch_ask` capability, response simplification, workpiece mutation, or fixes for the parent's named defects. -### Contract stratum and readiness gate +### Product-manager litmus -Close the **one-route conversation transport stratum**: one typed turn and one finalized Voice turn each admitted once through `@flue/sdk` at the mounted route, canonical output, client-tool follow-up as a signal, local playback cancellation, durable abort, visible failure, and same-conversation reopen — with no second server-side door remaining. +**Release note:** Voice now submits only what the microphone actually transcribed, waits for a safe **Your turn** handoff before listening over Brunch, and can replay the exact full response or exact Brunch-marked question. Client-tool Voice origins survive canonical reopen. Restoring the Voice chip on direct spoken user messages remains blocked on an upstream Flue user-metadata contract. -Order the tracers so the cheaper one proves the route first: the typed panel over the browser transport (it reuses the existing projector and has an existing integration scenario to re-express), then Voice. After each end-to-end turn works, enumerate the lateral obligations it exposes and close those required to make the visible claim true: duplicate finalization, ambiguous admission, submission/reply correlation, client-tool resume correlation, reconnect and replay, local cancellation versus durable abort races, fatal ownership errors, and canonical text/TTS-input correspondence. Carry broader speech ergonomics, multi-turn barge-in tuning, structured questions, and remote identity and exposure only to the named deferred owners below. +**Demo script:** run `yarn dev:brunch` and select the Brunch preview. Speak one answer and see exactly one matching user turn. While Brunch is speaking, confirm the microphone remains closed, choose **Your turn**, wait for the handoff, and speak again. After the response and audio settle, use the playback menu to read the full response exactly and repeat only the exact Brunch-marked question; a missing or unmatched marker keeps that action disabled. Start another turn, press durable **Stop** before settlement, and see a stopped turn rather than a Voice error. Hard-reload the settled conversation and confirm the canonical turn remains without resubmission or replay; direct-user Voice-chip restoration additionally waits on the Flue projection seam. -## Proof +**Previously impossible:** model-generated function arguments rather than completed audio transcription could become the answer; an accepted transcript could complete after canonical speech was requested but before output started; assistant playback could create a false user turn; cancellation could reopen capture before the provider settled; replay controls and multi-origin client-tool Voice attribution were incomplete. -This proof establishes that one real local typed turn and one real local Voice turn each cross the supported Flue conversation protocol at the single mounted route into the current canonical Brunch agent, that the typed turn returns as one finite AI SDK stream and the Voice turn as one visible and spoken canonical response, both with bounded cancellation and recovery semantics, and that no server-side AI SDK door remains. It does **not** establish trusted production authentication, remote deployment, broad Voice UX, structured-question transport, Petrinaut client-tool mutation, workpiece viability, or that the `useChat` panel itself is removable. +**Completion:** the implemented portions close when their tests and focused checks pass. Exact question replay uses the Brunch-owned marker recorded below; direct-user provenance still needs the Flue re-entry seam recorded below. Mission acceptance additionally requires the real microphone, handoff, Stop, hard-reload, and same-origin route witness, plus the comparative Voice latency gate. Mocked or server-only proof cannot substitute for that witness or for real audible-latency samples. -1. **Typed panel over the browser Flue transport.** A typed panel submission calls `send()` exactly once with one `kind: "user"` message; a completed client-tool follow-up calls `send()` exactly once with one `kind: "signal"` `client-tool-result` message and resumes the same assistant message id; the returned `UIMessageChunk` stream carries the same start/step/part/finish sequence the former `/api/chat` integration asserted, and terminates on that submission's `submission-settled`. Reopen hydration comes from `observe({ live: "sse" })` through `snapshotToUiMessages`. Oracle: the current `apps/brunch-agent/test/petrinaut-chat.integration.ts` scenario re-expressed through the browser transport against the in-process `app.fetch` of the real `app.ts` (Flue route, ownership guard, faux provider), preserving its text, reasoning, server-tool, and client-tool-resume assertions; the relocated projector and transcript unit tests; and the outer witness typing one message in the real panel with the network ledger showing only `/agents/chat/:instanceId` traffic. -2. **Direct finalized admission.** A completed Realtime `continue_interview` call invokes Flue `send()` exactly once with one `kind: "user"` message; provisional transcript events, duplicated provider terminal events, stale epochs, and repeated tool-call delivery never enter history. The admitted server `submissionId` becomes the turn correlation key. A lost or ambiguous admission is surfaced and never blindly resent. Oracle: named cases in `apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.test.ts`, including `admits one finalized Realtime answer through Flue once`, plus canonical snapshot inspection showing exactly one matching visible user message. -3. **No server-side AI SDK door.** The Brunch app no longer mounts `/api/chat`; `createPetrinautChatHandler`, `PETRINAUT_CHAT_ROUTE`, the in-process `init()` admission path, and the `/api/chat` proxy are deleted; the repurposed transport package contains no `Request`/`Response` handler. The real Voice turn reaches the mounted route through `@flue/sdk` — either through the panel's Flue transport or by a direct `send()` — and makes no submission over any non-Flue protocol. Oracle: `apps/brunch-agent/test/build-artifact.test.ts` asserting the built server answers `/api/chat` with Hono's 404 and still serves the Flue route; the retained browser network ledger from the outer witness; and a focused integration case in `apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts` named `admits a Voice turn only through the Flue route`. -4. **Canonical visible and TTS output.** The materialized response selected for the admitted `submissionId` is the source for visible text and TTS input. Completed visible assistant text is preserved in part order; reasoning and non-speech parts are not promoted to spoken text. The exact string array sent for canonical speech equals the selected Brunch text, and no response-preparation or simplification generation call occurs. Oracle: `apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.test.ts`, `openai-realtime-session.test.ts`, and the outer artifact comparison of canonical snapshot text to the recorded TTS request payload; the human witness confirms that playback begins but does not claim synthesized audio is a verbatim recording. -5. **Cancellation and abort stay distinct.** Barge-in, pause, or panel closure stops local playback/observation according to the existing Voice contract without rewriting or deleting canonical history. An explicit durable Stop action uses Flue's conversation-wide `abort()` when work is unsettled, and history/observation exposes an `aborted` settlement; an abort that loses the race to completion remains completed. Oracle: named local-versus-durable cancellation cases in `voice-turn-controller.test.ts` and `voice-preview.integration.test.ts`, plus the retained Voice event ledger and Flue settlement. -6. **Reopen resumes without replay.** Reopening the same selected conversation rehydrates its canonical messages and settlements from Flue, does not submit another user message, and does not automatically replay settled audio. An absent conversation, fatal 401/403, reconnecting stream, and settled conversation are visibly distinguishable. Oracle: a named `rehydrates the settled Voice turn without resubmission or playback` case over the SDK observation in `realtime-brunch-bridge.test.ts`, followed by the outer witness reopening the panel and comparing the second view with the same snapshot. -7. **Architecture and boundary integrity.** The built app still mounts `useBrunchAgent()` plus `useSdcpnPlugin()`, excludes the obsolete app-local stub agent, derives ownership from principal plus logical conversation id in exactly one guard, and reaches Flue locally through a same-origin protocol-preserving proxy rather than a newly public route. The transport package's runtime dependencies are exactly `ai` and `@flue/sdk`; it imports no `@flue/runtime`, core, plugin, or binding module. Oracle: `apps/brunch-agent/test/build-artifact.test.ts`, `apps/brunch-agent/test/agent-ownership.test.ts`, the transport case in `apps/brunch-agent/test/architecture/boundaries.integration.ts` (`transports consume their wire encoder and the public Flue client only — never core, a binding, or the runtime`), SDCPN packaging tests, and browser inspection of the claimed local route and headers. -8. **Real Voice witness and retained proof bundle.** With `yarn dev:brunch`, a human speaks one answer, sees exactly one matching user message, sees and hears the canonical Brunch response begin, interrupts playback once, exercises Stop on one unsettled turn, and reopens the original settled turn. Retain under `docs/evidence/implementations/mission-5-direct-voice-flue/` the witness record, sanitized Voice event ledger, network route summary, canonical Flue snapshot, settlement outcomes, source/build commit, and hashes. Oracle: human adjudication against that bundle; mocked browser or server-only evidence cannot satisfy this leaf. -9. **Focused repository verification and truthful docs.** Brunch app, website, core/plugin, transport, and Petrinaut checks pass; end-user and operator prose describes the single route that actually shipped and preserves the distinction between canonical text and generated audio; no surviving prose or comment names `/api/chat` as a Brunch door. Oracle: `yarn exec turbo run lint:tsc lint:eslint test:unit build --filter @apps/brunch-agent --filter @apps/petrinaut-website --filter @hashintel/petrinaut --filter @hashintel/brunch-agent --filter @hashintel/brunch-agent-plugin-sdcpn --filter @hashintel/brunch-agent-transport-aisdk`, `yarn workspace @local/petrinaut-arch-docs lint:arch-docs` if a Petrinaut architectural boundary changes, `rg -n "api/chat" apps/brunch-agent libs/@hashintel/brunch-agent/packages apps/petrinaut-website/src/main/app/local-storage-demo` returning only the stock Petrinaut route and frozen Mission 4 evidence, inspection of `apps/petrinaut-website/README.md` and `libs/@hashintel/petrinaut/docs/ai-assistant.md`, and a patch changeset if the published Petrinaut package changes. +1. **Completed-transcript authority and half-duplex ownership.** Realtime session configuration has no tools, no model-created semantic-VAD response, and no automatic interruption policy. Only a unique completed transcript can reach the shared panel submission path. Duplicate, empty, failed, unavailable, stale, canonical-speech-overlapping/pre-handoff, playback-overlapping, and over-limit transcripts do not submit and produce the specified passive or recoverable notice. Before sending `response.create`, a canonical speech request invalidates every unfinished accepted item, clears bridge/controller transcript state and provisional UI, and closes the microphone; a completion in the interval before output starts cannot submit or regain authority. **Your turn** may cancel audio immediately but opens only a post-barrier input turn after provider cancellation acknowledgement and Brunch settlement. Oracle: transplanted-first cases in `openai-realtime-session.test.ts`, `realtime-brunch-bridge.test.ts`, `voice-turn-controller.test.ts`, `voice-interview-control.test.tsx`, and `voice-preview.integration.test.ts`. +2. **Idempotent admission.** Typed turns derive a stable key from the AI SDK message id; Voice turns derive it from connection epoch, item id, and content index. A repeated same-payload key converges on the original receipt, including `deduplicated: true`; a 409 `submission_conflict` surfaces the original `submissionId` without admitting another turn. An admission whose outcome cannot be established stays visibly ambiguous and is never automatically retried. Oracle: transport unit tests plus Voice integration tests asserting one admitted submission rather than one `send()` invocation. +3. **Acknowledged cancellation barrier.** `cancelOutput()` resolves only after input/output buffer clears, matching provider acknowledgements, and all targeted response terminal events. The latest mute preference wins while it settles. **Your turn** may request cancellation immediately, but the microphone reopens only after both that provider acknowledgement and correlated Brunch settlement. Audio captured before the handoff cannot submit afterward. Durable Stop remains a stopped Flue turn rather than a Voice failure. Oracle: donor-adapted session/controller race tests and the unsettled-Stop integration case. +4. **Committed canonical speech, full-response replay, and exact question replay.** Initial automatic speech begins from each new, finalized canonical segment as soon as its Flue model step is durably completed and correlated to the active Voice submission; it does not wait for whole-submission settlement. Streaming deltas, unfinished text, reasoning, tool inputs and results, and inferred text are never speech sources. Stable canonical segment ids deduplicate the serialized speech queue and preserve canonical order across client-tool continuations. Hydration seeds existing ids without autoplay; cancellation, pause, Your turn, durable Stop, failure, and abort suppress queued and later continuation speech. `canReadFullResponse`, `readFullResponse()`, and the playback menu retain and enqueue all exact canonical text segments in order without a simplifier. A non-interactive `brunch_mark_question` server tool writes a durable `data-brunch-question` marker containing exact question text and tool-call identity. The selector accepts it only when the same finalized assistant message contains that exact text; there is no final-segment or punctuation fallback. `repeatQuestion()` queues only the accepted marked segment. Both replay actions remain gated until the correlated Brunch response settles, matching Realtime audio is terminal, and input is idle, and remain disabled during submission, capture, cancellation, pause, and errors. The marker never accepts an answer or changes Voice path B. Oracle: transport correlation, live transport, snapshot projection, canonical speech, bridge, controller, panel-host, and production-preview tests proving completed canonical segments can start speech while chat is streaming without admitting any noncanonical source or mounting `brunch_ask`. +5. **Durable Voice provenance.** An assistant message may retain multiple `voiceToolCallIds`; one failed sibling origin does not erase successful origins. Persisted Flue client-tool-result signals support deterministic reconstruction after hydration and reopen. Direct spoken user messages remain Voice-attributed only while live because the canonical snapshot omits their caller origin. Re-entry requires a supported Flue user-message metadata/idempotency projection; browser storage and user-text encoding are rejected. Oracle: snapshot projection and panel partial-failure tests for supported origins, plus the [blocker record](docs/evidence/implementations/mission-5-voice-safety-parity/provenance-blocker.md). +6. **Dormant ask removal.** If still present after restacking, the website does not register `brunchAskInteractiveTool` for Voice and canonical speech does not recognize `"brunch-ask"`. No spoken ask answer can enter a wait state the transport cannot resume. Oracle: registration/canonical-speech negative tests and a repository search showing no mounted Voice `brunch_ask` surface. +7. **Real witness and same-origin route.** A human performs one microphone turn, explicit interruption/handoff, durable Stop on an unsettled turn, and hard reload of a settled turn. The retained network route summary proves the absolute Flue `streamUrl` remains on the same-origin proxy. Oracle: `witness.md`, sanitized `voice-events.jsonl`, `network-routes.json`, canonical `flue-snapshot.json`, `settlements.json`, commit manifest, and hashes under `docs/evidence/implementations/mission-5-voice-safety-parity/`. +8. **Comparative Voice latency.** Ten comparable real-audio trials at pinned donor #9496 head `c7fe8a2e68e8fdc37018b21ec2e9daf4e9ef7c82` and ten at the final restacked candidate use the same machine, browser, microphone/input phrase, model configuration, warm/cold-start policy, and finalized-speech-to-first-audible-canonical-TTS boundary. The candidate median must not regress, and its p95 regression must remain below 20%. The donor runs from an isolated worktree without changing its branch. Oracle: retained raw sanitized samples, calculation method, environment, both commit SHAs, median, and p95; a comparison that cannot be run reliably leaves this proof incomplete. +9. **Focused repository verification and truthful docs.** The requested seven-workspace Turbo command passes, including `@hashintel/brunch-agent`, `@hashintel/brunch-agent-binding-flue`, and `@hashintel/brunch-agent-plugin-sdcpn`. `apps/petrinaut-website/README.md` and `libs/@hashintel/petrinaut/docs/ai-assistant.md` describe half-duplex handoff, exact full-response and marked-question replay, Stop, transcript rejection, and the direct-user attribution limitation. If the published Petrinaut package changes, exactly one patch changeset covers it. Oracle: the command recorded in the PR and changeset inspection. ## Constraints -- Preserve Mission 4's current core/plugin/app composition and authored skill packaging. Voice reconciliation must never restore the deleted app-local `ChatAgent`, concise stub prompt, YAML plugin machinery, or a second model-facing agent. -- Flue history is the sole canonical conversation record. Voice owns media capture, provisional display, turn finalization, TTS, playback, and local interaction state; it owns no durable transcript and may not splice into stock-assistant history. -- Use `@flue/sdk`/`@flue/react` directly for shell-facing conversation transport. No adapter may re-specify Flue offsets, retries, materialization, settlement, or recovery. The browser `ChatTransport` is a projection over the public `FlueClient` (`send()`, `wait()`/`observe()`, `history()`), never a second HTTP client; it reads chunks only through the SDK's `onEvent`/observation surfaces. -- One product route. `/agents/chat/:instanceId` behind `agentOwnershipGuard` is the sole door for typed, Voice, diagnostic, and evaluation traffic; no route, handler, or package may accept a conversation turn over another protocol. The stock Petrinaut `/api/chat` (the website's own OpenAI function) is untouched and must not be borrowed. -- The Petrinaut panel remains on `useChat`; the AI SDK is its rendering contract, supplied a transport by the host. Do not rewrite the panel onto `@flue/react`, and do not add a second transcript store beside `useChat`'s messages for the typed panel. -- Submit only the validated finalized answer. Provisional transcription and audio remain ephemeral. One Voice finalization causes at most one `send()` call; because Flue 2.0.3 does not accept a caller idempotency key on `send()`, ambiguous admission must remain visible and must not trigger an automatic retry. -- Brunch owns canonical response content. TTS may synthesize audio from exact selected text, but no second model may summarize, shorten, paraphrase, or select replacement wording for the tracer. -- Local playback cancellation, local observation cancellation, HTTP request cancellation, and Flue's conversation-wide durable abort are distinct operations and must remain distinguishable in code, UI state, evidence, and tests. -- The current browser-minted local principal is an ownership discriminator, not trusted authentication. The outer proof is local and same-origin; it must not expose `/agents/chat/:id` publicly or claim production identity, authorization, CORS, deployment, or recovery. -- `@hashintel/brunch-agent-transport-aisdk` survives only as the browser-side adapter and the home of the projector, snapshot projection, and header names. Its runtime dependencies are `ai` and `@flue/sdk`; it never imports `@flue/runtime`, core, a plugin, or a binding, and client-tool names reach it as caller-supplied options. The boundary test's transport gate is amended to say exactly that — this is the one accepted topology-gate change of the recut. -- Do not mount the suspended `brunch_ask` capability merely to preserve the divergent preview stack. Re-entry requires observed plain-turn correlation strain and an owner decision consistent with the structured-question planning contract. -- External Voice branches and their issues/PRs remain read-only evidence. Port only behavior that serves this mission, preserve relevant provenance in commits, and do not rewrite, close, or represent those records as accepted wholesale. -- Record admission, first canonical text, first TTS request/audio, and settlement latency without transcript, prompt, tool, SDP, audio, credential, or response-body content in ordinary telemetry. -- No implementation begins until this authority cut is committed separately. Material changes to this contract require owner review and another focused authority commit before dependent implementation. +- Preserve the parent's one product route, memoized Flue client, browser `ChatTransport`, shared panel `useChat`, path-B Voice submission, canonical speech selection, durable Stop seam, and SDK observation hydration. Do not rebuild them. +- Transplant relevant regression tests before implementation. Reimplement donor behavior semantically against the current Flue path; donor branches and PRs are never merged, cherry-picked, rebased, rewritten, retargeted, or closed by this implementation. +- Derive one deterministic admission key per logical delivery. Treat `deduplicated` as successful convergence and `submission_conflict` as evidence of the already-admitted submission. Do not automatically retry an ambiguous admission. +- Normalize completed transcripts exactly once in the Realtime bridge with trim plus Unicode whitespace collapse, then enforce the 32,000-code-point bound. The generic panel validates but does not mutate that already-normalized Voice payload. Provisional text remains ephemeral and display-only. +- The half-duplex microphone is closed from the canonical speech request through output, cancellation, pause, error, and submission states. The request invalidates accepted unfinished input before `response.create`; only a **Your turn** handoff completed by both provider cancellation acknowledgement and Brunch settlement can establish fresh post-request capture. A cancellation promise is part of the turn boundary, not a cosmetic animation state. +- Automatic speech may precede settlement only for new, durably completed canonical segments correlated to the active Voice submission. Settlement remains the authority for replay and next-turn release. Never speak deltas, unfinished text, reasoning, tool material, inferred text, hydrated history, or any segment from a failed or aborted submission. +- Brunch canonical text is never summarized, shortened, paraphrased, or regenerated for speech or replay. +- Preserve every surviving Voice origin independently. Provenance must use supported Flue data or deterministic durable correlation; never encode it in visible user text. +- Do not fix the parent's admission/Stop races, stream cancellation, hydration overwrite, client-tool classification, response/submission correlation, CI, title, or body. Restack onto Lu's fixes; report any blocker. +- Keep local playback cancellation, local observation cancellation, HTTP request cancellation, and durable `abort()` distinguishable in code, UI, tests, and evidence. +- No simplifier, interactive or suspending structured questions, live `brunch_ask`, Petri-net generation/mutation, FE-1575 workpiece work, production identity, CORS/remote deployment, or panel `useChat` removal. The approved non-interactive question marker annotates existing assistant prose only; it is not an answer path or affordance. ### Expected touched paths ```text -~ libs/@hashintel/brunch-agent/packages/transport-aisdk/src/ delete the HTTP handler; add ChatTransport factory over FlueClient; receive ui-stream + snapshotToUiMessages + headers -~ libs/@hashintel/brunch-agent/packages/transport-aisdk/package.json deps become ai + @flue/sdk; drop valibot if unused -~ libs/@hashintel/brunch-agent/packages/transport-aisdk/test/ replace chat-handler/golden with transport + projector + snapshot tests -~ apps/brunch-agent/test/architecture/boundaries.integration.ts amend the transport gate -- apps/brunch-agent/src/http/petrinaut-chat.ts server-side door removed -- apps/brunch-agent/src/conversation/ui-stream.ts moves into the transport package -~ apps/brunch-agent/src/conversation/transcript.ts snapshotToUiMessages moves out; formatFlueTranscript stays for the CLI -~ apps/brunch-agent/src/http/routes.ts, local-origins.ts drop PETRINAUT_CHAT_ROUTE and the /api/chat proxy; proxy /agents/chat/* -~ apps/brunch-agent/src/app.ts remove the /api/chat mount and app-transport closure -~ apps/brunch-agent/package.json drop the `ai` devDependency if nothing else uses it -- apps/brunch-agent/test/petrinaut-chat.test.ts, petrinaut-chat.integration.ts, petrinaut-chat-result.ts, flue-ui-stream.test.ts re-expressed against the browser transport / relocated -~ apps/brunch-agent/test/build-artifact.test.ts, local-dev-origins.test.ts single-route assertions -~ apps/brunch-agent/petrinaut-local.vite.config.ts same-origin Flue-route proxy for the local real surface -~ apps/petrinaut-website/src/main/app/local-storage-demo/ createFlueClient composition, browser transport, delete use-flue-chat-history -~ apps/petrinaut-website/src/main/app/voice-interview/ direct Flue admission, materialized response, cancellation, reopen -~ apps/petrinaut-website/package.json add @flue/sdk -~ yarn.lock workspace dependency update -? apps/brunch-agent/src/http/ownership.ts, src/conversation/identity*.ts only if the identity-contract home (fog-line) moves -? libs/@hashintel/petrinaut/src/ui/ smallest public panel seam only if host composition cannot remain local -~ apps/petrinaut-website/README.md operator-facing route and preview behavior -~ libs/@hashintel/petrinaut/docs/ai-assistant.md user-visible Voice behavior -~ libs/@hashintel/brunch-agent/MISSION.next.md reconcile the production-door, restricted-ingress, and adapter-removal statements -+ libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-direct-voice-flue/ retained proof bundle -? .changeset/ one patch changeset only if the published Petrinaut package changes +~ apps/petrinaut-website/src/main/app/voice-interview/ transcript authority, half-duplex state, cancellation, replay tests/code +~ apps/petrinaut-website/src/main/app/local-storage-demo/ path-B correlation and dormant ask removal if still present +~ apps/petrinaut-website/src/server/voice/ Realtime policy tests/code +~ libs/@hashintel/brunch-agent/packages/transport-aisdk/ stable idempotency and canonical projection/provenance tests/code +~ libs/@hashintel/petrinaut/src/react/voice-session/ public Voice state required by the panel +~ libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ Your turn, replay menu, durable provenance +~ apps/brunch-agent/petrinaut-local.vite.config.ts retain Petrinaut Voice API handlers beside the Flue proxy +~ apps/brunch-agent/test/local-dev-origins.test.ts real merged-config launcher regression +~ apps/petrinaut-website/README.md operator behavior +~ libs/@hashintel/petrinaut/docs/ai-assistant.md end-user behavior +? .changeset/ one patch changeset if published Petrinaut changes ++ libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/ donor matrix and gated witness ``` ## Fog-line -- How a finalized Voice answer enters Flue. Two shapes are admissible: **(B, preferred)** Voice submits through the panel's own Flue `ChatTransport` (`useChat.sendMessage` → `send()`), so the panel's `useChat` messages remain the single visible store and Voice's `observe()` shrinks to selecting completed canonical text for TTS — or reads the panel's completed assistant message and drops `observe()` entirely; **(A)** Voice calls `send()` directly and keeps its own `observe()` state, with the panel rehydrating. Start with B; fall back to A only if the existing hold-while-streaming, epoch, or TTS-correlation semantics demonstrably strain under the panel's transport, and record the observed strain. Either way, one finalization is one `send()`, and if the chosen shape would create two mutable transcript stores or a second custom reducer, stop and reorient at the panel boundary. -- The smallest honest home for the browser-safe principal + logical-conversation-id → Flue-instance-id contract and the two ownership header names. `apps/brunch-agent/src/conversation/identity-web.ts` proves the algorithm; the website must not gain an app-to-app source import, and core must not own HTTP header names. The leading candidate is the repurposed transport package, which already exports the principal header; a website-local copy pinned by an equality test against the app is the fallback. Do not create a new package to hold two strings and a hash. -- Whether `wait(admission, { onEvent })` alone gives the browser transport a clean finite per-turn stream, or whether the panel needs `observe()` for reconnect during a turn. `wait()` rejects on failed/aborted settlement and on `terminal_event_missing`; the transport must map those to `error`/`abort` chunks rather than throwing past `useChat`. The first real disconnect mid-turn decides; do not pre-build reconnect machinery. -- Whether `reconnectToStream` should return `null` (observation-only rehydration, the current behavior) or resume an unsettled submission after reload. Start with `null` plus SDK observation; re-enter only if the witness observes a lost in-flight turn. -- The exact subset of PRs #9496, #9507, and #9512 to port after semantic comparison with the current branch. Their useful Voice state-machine behavior is evidence; their app-local agent topology, temporary ask shim, and generative preparation are not presumed requirements. -- The bounded speech-selection policy if the exercised Brunch response contains multiple completed text blocks or an interactive part. Begin with canonical completed visible text in order; if this produces duplicate, misleading, or unspeakable output, retain the mismatch and seek a Brunch-owned deterministic presentation rule rather than another generator. -- Whether the existing Stop affordance can express both local Voice interruption and explicit conversation-wide durable abort without misleading the user. The first real race decides the smallest UI distinction. -- Whether a same-origin local proxy can carry every SDK history/SSE/send/abort route unchanged. A crisp protocol or middleware blocker permits the thinnest route correction; it does not permit rebuilding the AI SDK adapter under another name. +- **Parent movement.** Lu owns #9528 and may push more commits. Before each implementation phase, compare the GitHub head and restack this branch; an observed parent change is adopted only through restack, never copied into this branch. +- **Conflict normalization.** The installed SDK exposes the 409 contract through `FlueApiError.body: unknown`. Narrow only the documented envelope needed to recover `error.meta.submissionId`; do not create a general error protocol or infer success from prose. +- **Dormant `brunch_ask`.** Remove or gate only the parent surfaces that remain after the next restack. If Lu has already removed them, record the parent commit and make no duplicate change. +- **Question-marker compliance.** The owner selected `brunch_mark_question` plus a durable client data part. The remaining implementation uncertainty is whether the model follows the instruction on every eligible question. Missing or unmatched markers must degrade by leaving **Repeat question** disabled; they never justify inference from final prose. Product proof covers structural correctness, not a universal model-compliance rate. +- **Direct-user Voice provenance.** Flue 2.0.3 and current upstream `main` expose a generated `submissionId` but not caller metadata or `idempotencyKey` on canonical user messages. The owner selected an upstream Flue user-metadata contract. Keep this leaf blocked until a released seam can be adopted; do not patch Flue locally, add a provenance signal admission, add sidecar persistence, or encode origin in user content. +- **Observed timing opportunity.** One pre-change real Voice turn at `f3f5c4ebc7` measured 48.5 ms from finalized transcript to admission, 3,366.6 ms from admission to first canonical text, 0.0 ms from first canonical text to settlement, 0.3 ms from settlement to TTS request, and 628.3 ms from TTS request to the provider audio-start event. The zero measured target gap means this recut would not meaningfully improve that turn; the preceding model/tool/construction latency belongs to #9538 rather than Mission 5. Automated checks cannot establish a latency improvement, and comparable physical microphone-to-audible-TTS trials remain the accepted proof. +- **Human product evidence.** The hydration overwrite guard is present after the restack, so proof leaf 7 may run. Unit/integration tests still cannot substitute for the real microphone/hard-reload witness, and server or synthetic timing cannot substitute for proof leaf 8's first-audible-audio measurements. ## Stop or reorient -Stop and surface the evidence if the implementation creates a second conversation authority, keeps or re-adds any server-side route that accepts a conversation turn over a non-Flue protocol, submits provisional STT, automatically retries an ambiguous admission, rewrites canonical text through another model, hand-rolls stream recovery, computes offsets, restores the old stub agent, or activates `brunch_ask` without observed need and owner approval. - -Stop if the browser transport cannot preserve the current client-tool resume semantics (completed client-tool parts on the referenced assistant message → one signal send → continuation of the same assistant message id) without a server-side helper; that is evidence the resume contract needs redesign, not permission to reintroduce `/api/chat`. +Stop and report if the work would require direct Voice `send()`, a second transcript or conversation authority, hand-rolled stream offsets/recovery, automatic retry after ambiguous admission, canonical text rewriting, a live structured-question path, or any excluded parent fix. -Stop at the boundary if direct Flue state cannot reach the existing visible panel without duplicated mutable history; decide the UI ownership seam before adding synchronization machinery. Stop if local cancellation accidentally aborts durable work, explicit Stop only cancels a browser request while the provider keeps spending, a stale response is spoken after conversation/epoch change, or reopen resubmits or replays a settled turn. +Stop if half-duplex handoff cannot guarantee that pre-handoff audio is rejected and post-barrier audio is fresh, or if provider acknowledgements cannot bound `cancelOutput()` without inventing events. The provenance stop condition has fired for direct spoken user turns: the browser-store implementation was removed and the unsupported leaf is recorded as blocked pending upstream Flue support. For **Repeat question**, stop rather than infer question identity when the approved marker is absent or does not exactly match finalized assistant text. Stop if either replay action can enable before both matching terminal conditions, or if local cancellation invokes durable abort. -Stop rather than widen if the real route requires public unauthenticated exposure, production identity work, remote deployment, Petrinaut mutation tools, workpiece/projection state, or a whole assistant rewrite. Those are not hidden prerequisites to this transport tracer. +Do not manufacture the hard-reload witness or latency samples. If the human/browser environment cannot produce reliable observations, retain an incomplete evidence record and request the missing action explicitly. ## Deferred -- **`useChat` panel removal:** with the server-side door gone, the AI SDK survives only as the Petrinaut panel's rendering contract behind a host-supplied transport. Whether Petrinaut ever drops `useChat` is a Petrinaut product decision, not a Brunch transport question; Brunch carries no further obligation here. -- **Restricted-ingress rule for the Flue route:** Mission 8's landed contract denied `/agents/chat/:id` publicly and routed restricted traffic through `/api/chat`. This recut makes the Flue route the only product route, so that rule must be re-expressed as the FE-1423 gates applying directly to `/agents/chat/:id`. Record the re-expression in `MISSION.next.md`; the release/deployment gate owns its enforcement. -- **Structured questions:** core-owned question semantics, binding, rendering, correlated reply, and resumed tool execution remain in the shared future-planning record. Re-enter when plain Voice turns demonstrably cannot preserve a required interaction. -- **Broader Voice quality:** multi-turn barge-in tuning, long-response ergonomics, optional deterministic spoken presentation, accessibility breadth, and response optimisation re-enter after measured strain on the direct canonical route. -- **Remote/public operation:** trusted identity and authorization, origin policy, hosted Flue reachability, rate/spend controls, replacement recovery, and remote observability remain with the Mission 8 release/deployment gate or a separately cut successor. -- **Product-data work:** prepared workpiece/Petrinaut viability remains Mission 6; capture-backed review remains Mission 7; automatic traceable projection remains Mission 9. This mission carries no document mutation or provenance claim beyond canonical conversation history. -- **Host breadth:** stock/Brunch picker behavior, session switching beyond the selected local Brunch conversation, and HASH embed parity wait for the first visible consumer that makes them load-bearing. +- The real witness, same-origin absolute-`streamUrl` observation, and comparative latency gate require human browser and microphone evidence; they are part of this mission rather than a successor. +- Direct-user Voice attribution after canonical hydration waits on a released upstream Flue caller-metadata projection seam. The owner rejected a local Flue patch and correlated signal sidecar for this mission. +- Donor retirement waits until this replacement is accepted and each donor owner explicitly approves closure. Do not close #9496, #9500, #9507, or #9512 as an implementation side effect, and never close stakeholder-owned H-6763. +- Response preparation/simplification, structured questions, Petri-net work, FE-1575, production identity, CORS/remote deployment, and panel migration away from `useChat` remain in their existing owners or the future mission spine. diff --git a/libs/@hashintel/brunch-agent/docs/evidence/design/mission-5-question-marker-and-provenance-decision-2026-09-04.md b/libs/@hashintel/brunch-agent/docs/evidence/design/mission-5-question-marker-and-provenance-decision-2026-09-04.md new file mode 100644 index 00000000000..f664d2175f4 --- /dev/null +++ b/libs/@hashintel/brunch-agent/docs/evidence/design/mission-5-question-marker-and-provenance-decision-2026-09-04.md @@ -0,0 +1,97 @@ +# Mission 5 question replay and direct-user provenance decision + +## Decision + +The owner approved two changes to the live FE-1580 authority on 2026-09-04: + +1. Brunch may expose a non-interactive, model-facing question-marker tool. The + marker identifies exact assistant-authored question text for Voice replay, + but it never suspends a response, renders an answer affordance, accepts an + answer, or changes Voice path B. +2. Direct-user Voice provenance must wait for an upstream Flue contract that + durably projects caller metadata on a canonical `kind: "user"` message. This + branch must not patch Flue locally or approximate provenance with a second + signal admission, browser storage, or encoded user text. + +## Exact question marker + +Brunch core owns a `brunch_mark_question` server tool and a +`data-brunch-question` client marker. Before asking the user a direct question, +the model calls the tool with the exact question text. The tool writes a durable +data part containing that text and its Flue `toolCallId`, then returns a small +acknowledgement. It does not terminate the response. Brunch instructions require +the same exact text to appear in ordinary assistant prose after the tool call. + +The browser transport hides the marker tool's implementation call while +retaining the data part. This keeps an internal annotation out of Petrinaut's +tool-activity UI without creating another conversation representation. Both the +live stream and canonical snapshot projection apply the same hidden-tool rule. + +Canonical speech accepts a question marker only when all of these facts hold: + +- the marker has a non-empty string question and non-empty `toolCallId`; +- it belongs to an assistant message; +- the same assistant message contains the exact marked string in finalized + ordinary text; and +- the marker data part is complete and canonical, not provisional Voice state. + +Malformed, unmatched, stale, or absent markers do not enable **Repeat +question**. The final text segment and punctuation are never used as fallback +question authority. The selected question segment derives stable identity from +the assistant message id, marker tool-call id, and exact-text hash. Full-response +speech remains the ordered ordinary text segments and is not rewritten or +duplicated by the marker. + +The Voice controller carries the selected question separately from the full +response. **Repeat question** reuses the existing exact canonical queue and the +same settlement, output-completion, idle-input, submission, cancellation, +capture, pause, and error gates as **Read full response**. The control remains +disabled when the settled response has no matching marker. + +## Production proof + +Tests are written and observed failing before implementation. Closing evidence +must cover: + +- Brunch's built Flue agent mounting `brunch_mark_question` while continuing to + omit `brunch_ask`; +- a real server-tool call writing a durable `data-brunch-question` part; +- live transport and snapshot projection hiding the implementation tool while + retaining the marker; +- canonical selection rejecting malformed and unmatched markers and preserving + exact text and stable identity for a valid marker; +- the production Voice host registering `repeatQuestion` and the panel invoking + it only when `canRepeatQuestion` is true; and +- controller and preview integration proving exact question-only replay after + correlated Brunch settlement and matching Realtime output completion, with + every existing replay exclusion still enforced. + +## Direct-user Voice provenance + +Flue 2.0.3 and current upstream `main` accept only `body` and image +`attachments` on `kind: "user"`. The caller's idempotency key is irreversibly +hashed into `submissionId`; canonical snapshots do not expose that key or +caller-authored user metadata. Agent-authored response metadata cannot annotate +the canonical user message. + +The accepted route is an upstream Flue extension that admits caller metadata on +the user delivery, persists it atomically with the canonical user record, and +projects it on live and historical user messages. FE-1580 can adopt that seam +only after a released dependency is available and the branch is explicitly +authorized to upgrade. The closing oracle is a snapshot-only fresh-process test +that restores the Voice marker without browser correlation state. + +Rejected alternatives: + +- a local Yarn patch to Flue, because it forks substrate persistence and wire + projection inside this product PR; +- a correlated provenance signal, because it is a second, non-atomic admission + that can independently fail or wake the agent; +- browser or application sidecar storage, because it becomes a second durable + authority; and +- hidden transcript, attachment, or visible-text encoding, because it changes + the canonical user representation or smuggles metadata through content. + +Until the upstream contract is released and adopted, direct spoken user text +remains canonically durable but its Voice chip after reopen remains blocked and +must not be reported as complete. diff --git a/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/donor-behavior-matrix.md b/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/donor-behavior-matrix.md new file mode 100644 index 00000000000..819e52c82e6 --- /dev/null +++ b/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/donor-behavior-matrix.md @@ -0,0 +1,104 @@ +# FE-1580 donor-behavior matrix + +## Decision frame + +This record pins the semantic disposition of the Voice donor branches for the live [FE-1580 mission](../../../../MISSION.md). The parent and donors are read-only source evidence at these exact heads: + +| Source | Pinned head | Role | +| --- | --- | --- | +| Parent PR [#9528](https://github.com/hashintel/hash/pull/9528) | `eecbe99e201fd8cb78d9b719e789b6abd373ed1b` | Unified Flue route and path-B departure base | +| Donor PR [#9496](https://github.com/hashintel/hash/pull/9496) | `c7fe8a2e68e8fdc37018b21ec2e9daf4e9ef7c82` | Canonical TTS queue and replay mechanics | +| Donor PR [#9500](https://github.com/hashintel/hash/pull/9500) | `935aa9f02a5ac635a50eb8bc130edb3e258af8e4` | Completed-transcript authority | +| Donor PR [#9507](https://github.com/hashintel/hash/pull/9507) | `252b9dbb0c77fae8cee45a506f09cac3e20c381c` | Temporary `brunch_ask` shim, excluded | +| Donor PR [#9512](https://github.com/hashintel/hash/pull/9512) | `d13535d1077b3a78d6a1411031b7d0a0a78e3144` | Half-duplex cancellation, replay UX, and provenance | + +No source is merged, cherry-picked, rebased, retargeted, rewritten, or closed by the implementation. Tests are transplanted first and adapted to the one Flue submission route; production behavior is reimplemented semantically. + +The owner selected half-duplex turn ownership on 2026-09-03: assistant output owns the audio turn until **Your turn** completes an acknowledged cancellation barrier. Automatic duplex is not an admissible fallback. + +## Behavior disposition + +| Source | Behavior | Disposition | Reason | Outstanding adaptation or proof | +| --- | --- | --- | --- | --- | +| #9528 | One `/agents/chat/:instanceId` product route, browser `ChatTransport`, one memoized client, path-B Voice submission through shared `useChat` | **Adopt** | This is the departure architecture and prevents a second admission authority. | Restack onto every new parent head; verify no successor code calls `send()` directly from Voice. | +| #9528 | Direct Voice `send()` as a fog-line fallback | **Reject** | It creates a second admission path and mutable coordination surface. The parent has already proved path B. | Mission authority now permits path B only. | +| #9528 | Claim that Flue 2.0.3 lacks caller idempotency | **Reject as factually false** | Installed typings expose `AgentPromptOptions.idempotencyKey`, `AgentSendResult.deduplicated`, and 409 `submission_conflict` with the existing `submissionId`. | Implemented with transport convergence/conflict tests and typed Voice admission outcomes. | +| #9528 | Canonical hydration guard, multi-submission response correlation, settlement-driven durable Stop, aligned live/snapshot projection, queued Voice-input cancellation, and client-tool continuation | **Adopt through restack** | These mechanisms remain parent-owned and must enter the successor through the stack rather than copied fixes. | Restacked onto `eecbe99e201f`; hydration no longer blocks the real witness. Further defects in these mechanisms remain parent scope. | +| #9496 | Serialized canonical speech queue, retained exact source segments, response/output terminal gating | **Adopt mechanics** | Replay and ordinary TTS need one lifecycle-safe queue, and exact text preserves canonical authority. | Implemented without a preparation/simplifier dependency; exact-segment and queue tests pass. | +| #9496 | `canReadFullResponse`, `readFullResponse()`, exact full-response playback menu | **Adopt** | Exact full-response replay is supported by retained canonical segment identity. | Implemented with idle-state and matching response/output terminal gates. | +| #9496 | `canRepeatQuestion`, `repeatQuestion()`, and playback-menu action | **Adopt UX; reject final-segment inference** | The final segment may be ordinary prose and is not authority for question identity. The approved `brunch_mark_question` data marker now supplies deterministic identity without accepting an answer. | Implemented by replaying only exact marked text found in finalized prose from the same assistant message; a missing or unmatched marker leaves the action disabled. | +| #9496 | Realtime-generated concise response preparation or any fallback that rewrites canonical text | **Reject** | Response simplification is a non-goal and violates exact canonical speech. | Tests compare retained segment ids and exact queued strings; no preparation API remains on this path. | +| #9500 | No Realtime tools, `tool_choice: "none"`, semantic VAD `create_response: false` | **Adopt** | Realtime detects/transcribes and renders supplied TTS only; it must not generate user meaning. | Implemented in policy, session, and production-preview integration tests. | +| #9500 | Only `conversation.item.input_audio_transcription.completed` can submit; model function arguments ignored | **Adopt** | Shape validation cannot prove model-generated arguments match the audio. | Implemented with current-turn speech-boundary, stale, reordered, and late-output rejection tests. | +| #9500 | Transcript identity `(connectionEpoch, itemId, contentIndex)`, stable submission id, trim plus Unicode whitespace collapse, 32,000-code-point limit | **Adopt** | This gives one deterministic logical Voice delivery and one normalization boundary. | Implemented through path B; the panel preserves the bridge-normalized payload unchanged. | +| #9500 | Explicit duplicate, empty, failed, unavailable, and over-limit rejection; passive/recoverable not-heard UI; provisional display only | **Adopt** | Rejected audio must never become a turn, while ordinary silence/failure must not poison the session. | Implemented with reason-specific bridge/controller UI coverage. | +| #9500 | Silently settling ownership by discarding every playback-overlapping utterance without an explicit handoff | **Supersede** | It avoids echo but leaves users without a deliberate way to take the turn. | Use #9512 half-duplex `canTakeTurn`/`takeTurn()` and reject all speech captured before the completed handoff. | +| #9500 | `brunch_ask` answer/tool correlation and preparation code inherited from its base | **Reject** | Structured questions and response preparation are excluded. | Correlate the Voice delivery to its path-B submission and canonical response facts; exact question replay uses the non-interactive marker instead. | +| #9507 | Temporary `brunch_ask` registration, widget, correlated spoken ask answer, transcript formatting | **Reject entire shim** | The current transport only admits the supported follow-up set; a spoken ask can otherwise wait forever. Structured questions are a separate product decision. | Remove or gate dormant `brunchAskInteractiveTool` and `"brunch-ask"` canonical-speech recognition only if still present after restack. | +| #9512 | Half-duplex `canTakeTurn`, `takeTurn()`, `"cancelling"` output state, and **Your turn** control | **Adopt by owner decision** | It makes output/input ownership explicit and prevents assistant playback from becoming a false user turn. | Implemented through the public Voice store and production panel registration path. | +| #9512 | Promise-returning `cancelOutput()` that waits for input/output clears, matching acknowledgements, and response terminal events | **Adopt** | The microphone cannot safely reopen on a fire-and-forget cancel. | Implemented with acknowledgement/race tests, latest-mute behavior, and fresh post-handoff capture. | +| #9512 | Replay availability tied to exact retained source, terminal response, and output completion | **Adopt with #9496 mechanics** | This closes replay races without changing canonical content. | Implemented against parent segment/submission correlation for exact full-response and marked-question replay. | +| #9512 | Voice answer icon/provenance before interactive answers | **Partially adopt; blocked for direct user turns** | Live attribution is useful but one origin per assistant message is insufficient after coalesced or sibling Voice deliveries. Flue's client-tool result signal can durably carry those origins. Its direct-user delivery and snapshot types expose no caller metadata or idempotency key, so a direct spoken user message cannot be identified after reopen without a forbidden second store or text encoding. | Keep `voiceToolCallIds`, preserve successful siblings on partial failure, and reconstruct supported tool-result origins from Flue signals. Re-enter direct-user attribution only when Flue provides a supported durable correlation seam. | +| #9512 | App-local agent topology, temporary ask UI, response preparation, or donor-specific host composition | **Reject** | The parent owns the one Flue route and current host composition; these mechanisms are obsolete or non-goals. | Reuse only state-machine, cancellation, replay, and attribution behavior. | + +## Adopted-behavior replacement coverage + +| Adopted behavior | Replacement implementation | Regression test | Production integration proof | Status | +| --- | --- | --- | --- | --- | +| One path-B Flue admission route | `local-storage-demo-app.tsx`, `brunch-panel-transport.ts`, transport `src/index.ts` | `brunch-panel-transport.test.ts`, `chat-transport.test.ts` | `voice-preview.integration.test.ts` crosses completed transcript → panel submission → Flue transport → canonical speech | **Implemented**; parent defects remain downstack | +| Stable admission identity and typed outcomes | transport `src/index.ts`, `brunch-panel-transport.ts`, `submitVoiceInputWithAdmission`, `realtime-brunch-bridge.ts` | transport admission cases; bridge/controller cases for rejected, conflict, ambiguous, and local abort | production preview carries 409 conflict, 500 ambiguity, and local abort through transport → tracker → `submitVoiceInputWithAdmission` → bridge; each observes one `send()`, and local abort never invokes durable `FlueClient.abort()` | **Implemented** | +| Exact canonical TTS queue and full-response replay | `openai-realtime-session.ts`, `voice-turn-controller.ts`, Petrinaut playback menu | session queue/cancellation cases; controller exact-segment and terminal-gating cases; panel action tests | real host registration exposes `readFullResponse`; panel forwards it through `voiceSessionStore` | **Implemented** | +| Exact question replay | core `brunch_mark_question` tool/data contract; transport hidden-tool projection; `canonical-speech.ts`; bridge/controller; Voice host callback | core marker tests; live/snapshot transport projection tests; canonical selector malformed/unmatched/cross-message cases; controller final-segment negative and exact-marker replay cases | real Flue integration persists and reopens `data-brunch-question` while hiding the marker tool; controlled Voice preview carries the marker through response correlation and queues only the exact question; panel host forwards the action | **Implemented**; missing or unmatched markers fail closed | +| Disabled Realtime generation/tools | `openai-voice-policy.ts`, `openai-realtime-session.ts` | policy/session tests reject tools and function arguments | controlled production preview negotiates the server policy and emits only canonical speech | **Implemented** | +| Completed-transcript authority | `openai-realtime-session.ts`, `realtime-brunch-bridge.ts` | missing/stale/reordered boundary, keyed identity, normalization, duplicate/failure/limit, canonical-request-before-output, and late-output cases | controlled production preview proves a pre-request item cannot call Flue `send()` before output starts and only fresh post-handoff input submits through path B | **Implemented**; provider-valid boundaryless commits are intentionally rejected by mission policy | +| Half-duplex acknowledged handoff | `openai-realtime-session.ts`, `voice-turn-controller.ts`, Voice public store/dock | canonical-request invalidation, input/output clear acknowledgement, targeted response terminal, latest mute, stale/pre-handoff rejection | panel registration tests exercise **Your turn**; preview integration proves the microphone closes before `response.create` and fresh post-handoff capture submits once | **Implemented** | +| Durable Stop distinct from local cancellation | app `requestFlueStop`, panel `stopComposer`, session `cancelOutput` | panel durable-before-local Stop, controller/session local-cancel cases, app host Stop case | configured Brunch app invokes `FlueClient.abort()`, observes an aborted settlement, and does not invoke local playback cancellation | **Implemented**; parent-owned Stop races excluded | +| Multi-origin Voice client-tool provenance | panel `addMappedToolOutput`, transport client-tool-result signal/projection, `useFlueChatHistory` | sibling partial-failure, persisted-signal projection, hydration/reopen cases | configured app consumes the public Flue observation and restores every `voiceToolCallId` | **Implemented for client-tool results**; direct-user marker **blocked** | +| No live `brunch_ask` | Brunch app registers `interactiveTools: []`; canonical speech selector ignores the ask name | canonical-speech negative case and configured-app registration negative case | captured production Brunch `PetrinautAiAssistant` has no ask tool while retaining Flue Stop wiring | **Implemented exclusion** | + +## Outstanding acceptance ledger + +| Area | Required closing evidence | Current state | +| --- | --- | --- | +| Transcript authority | Transplanted-first session, bridge, controller, and integration regressions pass on path B. | Implemented. Matching current-turn `speech_started`, stale/reordered boundaries, canonical-speech-request and late-output invalidation, provisional UI clearing, exact bridge normalization, and unchanged panel payload are covered. | +| Admission idempotency | Typed and Voice logical replays converge on one `submissionId`; conflict metadata is narrowed safely; ambiguous outcome does not retry. | Implemented. Transport tests cover stable typed/Voice keys, deduplicated receipts, 409 conflicts, and non-retried ambiguity; production-path integration preserves the original conflict `submissionId` and keeps local admission abort distinct from durable abort. | +| Cancellation barrier | Buffer acknowledgements and targeted response terminals settle before capture; stale/pre-handoff audio cannot submit; latest mute choice wins. | Implemented. Session/controller races cover the barrier and mute preference; panel registration and configured-app Stop cases cover the production host seams. | +| Canonical replay | Exact segment queue and playback menu pass availability/race tests without a simplifier. | Full-response replay preserves every exact segment. **Repeat question** uses only a durable non-interactive Brunch marker that exactly matches finalized prose in the same assistant message; final-segment inference remains rejected. Both actions share terminal/output/input gating. | +| Durable provenance | Multiple origins and partial failure survive projection, hydration, and reopen without user-text encoding. | Partially implemented for assistant client-tool results through persisted Flue signals; multiple sibling origins survive projection and partial failure. Direct spoken user attribution is blocked because Flue 2.0.3 snapshots do not expose caller idempotency or user-message metadata. The rejected browser store would violate mission authority. | +| Dormant ask | No mounted Voice ask capability remains, or the parent commit that removed it is recorded. | Implemented exclusion. Canonical speech ignores `brunch_ask`, and a configured-app registration test proves the production Brunch assistant supplies no ask tool. Dormant source remains unmounted. | +| Real witness | Microphone, handoff, unsettled Stop, reload, canonical snapshot, settlement, and same-origin absolute-`streamUrl` artifacts are retained with hashes. | Parent hydration blocker resolved by restack; human browser/microphone run and retained artifacts remain outstanding. | +| Comparative latency | Ten pinned #9496 trials and ten final-candidate trials retain raw finalized-speech-to-first-audible-canonical-TTS samples and show no median regression with p95 regression below 20%. | Donor isolated worktree is prepared and its five focused Voice suites pass 108/108 after dependency build. Twenty comparable human audible trials and statistics remain outstanding. | +| Donor retirement | Replacement accepted and each donor owner explicitly approves closure. | Deferred; no donor or stakeholder issue may be closed now. | + +## Corrective verification + +Fresh local checks on 2026-09-04 cover the 72-file successor diff against the +verified #9528 head `eecbe99e201fd8cb78d9b719e789b6abd373ed1b`. Graphite replayed one +repeatedly touched integration-test conflict while restacking: the semantic +resolution keeps the parent's required URL-navigation props and tests together +with the successor's admission, dormant-ask, durable-Stop, and status-removal +proof. No production-source conflict was resolved by choosing either side +wholesale. The earlier verified code head before this evidence-only update is +`9938283a19ab20567ad6b4c96330ea392243c16f`: + +| Command | Result | +| --- | --- | +| `mise exec -- yarn workspace @apps/petrinaut-website test:unit src/main/app/voice-interview/openai-realtime-session.test.ts src/main/app/voice-interview/realtime-brunch-bridge.test.ts src/main/app/voice-interview/voice-turn-controller.test.ts src/main/app/voice-interview/voice-preview.integration.test.ts -t 'invalidates accepted input before requesting canonical speech output\|rejects unfinished input as soon as canonical speech is requested\|clears capture when canonical speech is requested before output starts\|bridges one completed transcript through Brunch and back to canonical half-duplex audio'` | Exit 0; 4/4 selected tests passed and 92 unrelated tests were filtered across four files. This covers the request-before-output race at session, bridge, controller, and production integration layers. | +| `mise exec -- yarn workspace @apps/petrinaut-website test:unit src/main/app/voice-interview/voice-preview.integration.test.ts -t 'ambiguous Flue admission\|conflicting submission\|local admission abort'` | Exit 0; 3/3 selected tests passed and 2 unrelated tests were filtered. Conflict retains the original `submissionId`; local abort remains distinct from durable abort; every path calls `send()` once. | +| `mise exec -- yarn workspace @hashintel/brunch-agent test:unit test/question-marker.test.ts` | Exit 0; 9/9 exact question-marker tests passed. | +| `mise exec -- yarn workspace @hashintel/brunch-agent-transport-aisdk test:unit` | Exit 0; 32/32 transport tests passed, including live and snapshot marker projection plus bounded Flue-error serialization. | +| `mise exec -- yarn workspace @apps/brunch-agent test:unit test/petrinaut-chat.test.ts` | Exit 0; 1/1 real-Flue integration test passed, including exact marker persistence through fresh-process reopen while marker tools remain hidden. | +| `mise exec -- yarn workspace @hashintel/petrinaut test:unit --run src/ui/views/Editor/panels/ai-assistant-panel.test.tsx` | Exit 0; 46/46 production host-registration and panel tests passed. | +| `mise exec -- yarn workspace @apps/petrinaut-website test:unit src/main/app/voice-interview/canonical-speech.test.ts src/main/app/voice-interview/openai-realtime-session.test.ts src/main/app/voice-interview/voice-turn-controller.test.ts` | Exit 0; 79/79 exact replay, queue, terminal-gating, and turn-controller tests passed. | +| `mise exec -- yarn exec turbo run lint:tsc lint:eslint test:unit build --filter @hashintel/brunch-agent` | Exit 0; 5/5 tasks passed, including 10/10 test files and 86/86 tests; the four question-marker mock lint failures are resolved with production-interface signatures. | +| `mise exec -- yarn exec turbo run lint:tsc lint:eslint test:unit build --filter @apps/brunch-agent --filter @apps/petrinaut-website --filter @hashintel/petrinaut --filter @hashintel/brunch-agent --filter @hashintel/brunch-agent-binding-flue --filter @hashintel/brunch-agent-plugin-sdcpn --filter @hashintel/brunch-agent-transport-aisdk` | Exit 0; 39/39 tasks and 1,123/1,123 tests passed: 16/16 Brunch app files with 80/80 tests, 10/10 Brunch core files with 86/86 tests, 3/3 transport files with 32/32 tests, 5/5 binding files with 18/18 tests, 2/2 plugin files with 8/8 tests, 72/72 Petrinaut files with 617/617 tests, and 31/31 website files with 282/282 tests. Website ESLint retains one non-failing warning at `voice-interview-control.tsx:587`; other inherited warnings remain outside this successor's corrective scope. | +| `yarn workspace @local/petrinaut-arch-docs lint:arch-docs` | Exit 0; 68 layers, 337 edges, 690 files, 69 generated pages, and 38 authored pages. | +| `yarn lint:format` | Exit 0; all 5,527 matched repository files use the correct format. | +| `git diff --check` | Exit 0. | +| In isolated detached worktree `/Users/kostandin/Projects/hashdev/worktrees/fe-1580-latency-baseline-9496`: `mise exec -- yarn exec turbo run build --filter '@apps/petrinaut-website^...'`, then `mise exec -- yarn workspace @apps/petrinaut-website test:unit src/main/app/voice-interview/canonical-speech.test.ts src/main/app/voice-interview/openai-realtime-session.test.ts src/main/app/voice-interview/realtime-brunch-bridge.test.ts src/main/app/voice-interview/voice-turn-controller.test.ts src/main/app/voice-interview/voice-preview.integration.test.ts` | Exit 0; dependency build passed 14/14 tasks, then all 5/5 donor Voice files and 108/108 tests passed at pinned #9496 head. The isolated donor and candidate panels return HTTP 200 on ports 4916 and 4915 respectively; real audible samples remain uncollected. | + +No production Voice source under `apps/petrinaut-website/src/main/app/voice-interview` +calls `FlueClient.send()`; its only `.send()` is the OpenAI Realtime data +channel. Production Brunch registration supplies `interactiveTools: []`, and +canonical speech has no `brunch_ask` recognition. The dormant ask source remains +unmounted. diff --git a/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/provenance-blocker.md b/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/provenance-blocker.md new file mode 100644 index 00000000000..b3e9e897b07 --- /dev/null +++ b/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/provenance-blocker.md @@ -0,0 +1,46 @@ +# FE-1580 direct-user Voice provenance blocker + +## Observed boundary + +Flue 2.0.3 can durably preserve Voice provenance for client-tool results: the +existing client-tool result signal carries each Voice-origin tool-call id, and +canonical snapshot projection can reconstruct every surviving sibling origin. +Regression coverage preserves successful siblings after a partial failure, +projects both origins from the persisted signal, and restores them through the +production observation hook after unmount and reopen. + +The corresponding direct-user seam does not exist in the installed public +contract: + +- `DeliveredMessage` user input accepts only `body` and image `attachments`; +- the caller's `idempotencyKey` is accepted for admission but is not projected + into `FlueConversationMessage` or `FlueConversationSettlement`; +- materialized user messages expose the generated `submissionId`, but no Voice + source metadata; and +- snapshot `metadata` is agent-authored response metadata, not caller-authored + user-message metadata. + +The discarded implementation persisted Voice `submissionId` values in browser +storage and correlated them after hydration. That would create a second durable +store, which the mission explicitly names as a stop condition. Encoding the +origin in visible user text is also prohibited. Replacing the canonical direct +user message with a hidden Flue signal would change the delivery semantics and +require a synthetic second transcript projection, so it is not a transparent +representation of the existing path-B turn. + +## Current disposition + +Direct spoken user turns still render with a Voice chip while their AI SDK +message metadata is live. Their canonical text and submission survive Flue +hydration, but the Voice chip cannot be reconstructed after reopen. This portion +of proof item 5 is blocked rather than reported as complete. + +Re-enter only when Flue projects caller metadata or the caller idempotency key +onto the canonical direct-user message, or when the product owner explicitly +authorizes a different durable representation. The oracle is a snapshot-only +test that reconstructs the Voice marker after a fresh process with no browser +correlation state. + +The restacked branch still installs `@flue/sdk` 2.0.3 with this same public +shape. No supported projection seam or owner-approved deferral has been +recorded, so direct-user reopen attribution remains blocked. diff --git a/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/witness-blocker.md b/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/witness-blocker.md new file mode 100644 index 00000000000..e15c60a36d5 --- /dev/null +++ b/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/witness-blocker.md @@ -0,0 +1,61 @@ +# FE-1580 human-evidence gate + +## Current disposition + +The real Voice witness has **not** been run and no witness bundle is claimed. +Completed-transcript authority, admission idempotency, half-duplex handoff, +acknowledged cancellation, exact full-response replay, durable Stop, dormant-ask +exclusion, exact Brunch-marked question replay, and the supported client-tool +portion of Voice provenance have focused automated coverage. Automated coverage +cannot replace the microphone, handoff, unsettled Stop, hard-reload, and +network-route witness required for mission acceptance. Direct-user Voice +attribution has a separate [Flue projection blocker](provenance-blocker.md). + +The successor is restacked onto [PR #9528](https://github.com/hashintel/hash/pull/9528) +head `eecbe99e201fd8cb78d9b719e789b6abd373ed1b`. That parent now guards its +once-per-conversation hydration from replacing a locally visible assistant +response with an older canonical snapshot, so hydration no longer blocks this +witness. The remaining gate is the required human browser and microphone run. + +An owner-directed PR #9531 side quest also removed a local launcher blocker +found at the real boundary on 2026-09-04. The Brunch-specific Vite config had +removed Petrinaut's entire `petrinaut-api-dev` plugin, so +`/api/voice/config` returned transformed module source instead of the handler's +JSON. The launcher now retains the website API plugin while continuing to +proxy only `/agents/chat/*` to Brunch. A config-level regression test loads the +real merged config, and an isolated `yarn dev:brunch` panel process with an +enabled non-secret test environment returned +`{"available":true,"connectionTimeoutMs":15000}`. This proves local Voice API +wiring only; it does not satisfy the human witness below. + +## Re-entry gate + +Using the final source/build commit: + +1. submit one typed turn; +2. run one real microphone turn and confirm exactly one matching user message; +3. confirm visible text and synthesized speech use the same canonical response; +4. use **Your turn** during output and retain cancellation acknowledgements; +5. confirm pre-handoff audio cannot submit and fresh post-handoff speech can; +6. durably stop an unsettled turn and retain its stopped settlement; +7. replay the exact full response and exact marked question; +8. hard-reload the settled conversation and confirm no resubmission or + automatic replay; +9. retain the canonical Flue snapshot and settlement index; +10. retain a network route summary proving the absolute Flue `streamUrl` + remains on the same-origin proxy; and +11. record the exact source/build and evidence commits plus hashes for every + retained artifact. + +The comparative latency proof also requires ten audible trials at pinned donor +#9496 head `c7fe8a2e68e8fdc37018b21ec2e9daf4e9ef7c82` and ten at the final +candidate. Both sets use the same machine, browser, microphone/input phrase, +model configuration, warm/cold-start policy, and finalized-speech-to-first- +audible-canonical-TTS boundary. Raw sanitized samples, the calculation method, +environment, commit identities, median, and p95 must be retained; the candidate +median may not regress and p95 regression must remain below 20%. + +Until then, `witness.md`, `voice-events.jsonl`, `network-routes.json`, +`flue-snapshot.json`, and `settlements.json` are intentionally absent rather +than populated with simulated evidence. Latency samples and statistics are also +intentionally absent until the comparable human trials run. diff --git a/libs/@hashintel/brunch-agent/packages/core/package.json b/libs/@hashintel/brunch-agent/packages/core/package.json index ca72a1d986e..f8c56ce978e 100644 --- a/libs/@hashintel/brunch-agent/packages/core/package.json +++ b/libs/@hashintel/brunch-agent/packages/core/package.json @@ -18,6 +18,10 @@ "types": "./src/flue.ts", "import": "./dist/flue.js" }, + "./question-marker": { + "types": "./src/question-marker.ts", + "import": "./dist/question-marker.js" + }, "./storage": { "types": "./src/storage.ts", "import": "./dist/storage.js" diff --git a/libs/@hashintel/brunch-agent/packages/core/src/flue.ts b/libs/@hashintel/brunch-agent/packages/core/src/flue.ts index cb020aaf9e0..9d57e9ea9e1 100644 --- a/libs/@hashintel/brunch-agent/packages/core/src/flue.ts +++ b/libs/@hashintel/brunch-agent/packages/core/src/flue.ts @@ -1,6 +1,20 @@ -import { useModel, useSkill } from "@flue/runtime"; +import { + defineTool, + useDataWriter, + useModel, + useSkill, + useTool, +} from "@flue/runtime"; +import * as v from "valibot"; import systemPrompt from "./prompts/SYSTEM.md?raw"; +import { + BRUNCH_QUESTION_DATA_NAME, + BRUNCH_QUESTION_TOOL_NAME, + BrunchQuestionDataSchema, + BrunchQuestionInputSchema, + type BrunchQuestionData, +} from "./question-marker"; import { ELICITATION_SKILL_NAME, elicitationSkill, @@ -10,14 +24,32 @@ import { skillFromMarkdown } from "./skills/skill-markdown"; /** * Mount the contributions owned by Brunch core and return its system prompt. * - * Core contributes the always-on universal prompt and one `elicitation` - * capability skill. It owns no model-facing tool; add one here only when it - * applies independently of the selected modelling formalism and host. + * Core contributes the always-on universal prompt, one `elicitation` + * capability skill, and the formalism-independent question marker. */ export function useBrunchAgent(model: string): string { useModel(model); useSkill(elicitationSkill); + const writeQuestion = useDataWriter(BRUNCH_QUESTION_DATA_NAME, { + schema: BrunchQuestionDataSchema, + }); + useTool(createBrunchQuestionMarkerTool(writeQuestion)); return systemPrompt.replace(/^\s+|\s+$/gu, ""); } +export const createBrunchQuestionMarkerTool = ( + writeQuestion: (question: BrunchQuestionData) => void, +) => + defineTool({ + name: BRUNCH_QUESTION_TOOL_NAME, + description: + "Mark the exact text of a direct question for accessible replay. Call this immediately before including that exact question in ordinary assistant prose. This marker does not ask or answer the question itself.", + input: BrunchQuestionInputSchema, + output: v.object({ marked: v.literal(true) }), + run({ data, toolCallId }) { + writeQuestion({ question: data.question, toolCallId }); + return { output: { marked: true as const } }; + }, + }); + export { ELICITATION_SKILL_NAME, elicitationSkill, skillFromMarkdown }; diff --git a/libs/@hashintel/brunch-agent/packages/core/src/index.ts b/libs/@hashintel/brunch-agent/packages/core/src/index.ts index f4021f55ef0..25fd3fdc75c 100644 --- a/libs/@hashintel/brunch-agent/packages/core/src/index.ts +++ b/libs/@hashintel/brunch-agent/packages/core/src/index.ts @@ -41,6 +41,14 @@ export { toolPrefix, type Operation, } from "./conversation/naming"; +export { + BRUNCH_QUESTION_DATA_NAME, + BRUNCH_QUESTION_TOOL_NAME, + BrunchQuestionDataSchema, + BrunchQuestionInputSchema, + parseBrunchQuestionData, + type BrunchQuestionData, +} from "./question-marker"; export { type HarnessReplyEvent, type ReplyPartKind, diff --git a/libs/@hashintel/brunch-agent/packages/core/src/prompts/SYSTEM.md b/libs/@hashintel/brunch-agent/packages/core/src/prompts/SYSTEM.md index 6c8049566df..99a6c65a011 100644 --- a/libs/@hashintel/brunch-agent/packages/core/src/prompts/SYSTEM.md +++ b/libs/@hashintel/brunch-agent/packages/core/src/prompts/SYSTEM.md @@ -10,6 +10,8 @@ Establish what the result must help the person decide, answer, compare, explain, Use the person's vocabulary and follow concrete cases rather than traversing a schema, template, or target representation. Do not open with a battery of independent questions; deepen one answerable thread at a time and group questions only when they share one frame. +Before asking the person a direct question, call `brunch_mark_question` with the exact question text. Then include the exact same question text in ordinary assistant prose. The marker only makes that text available for accessible replay; it does not wait for or accept the answer, so continue the same response normally after calling it. Do not mark headings, rhetorical questions, or prose that you will not present verbatim. + ## Authorship and uncertainty Keep what the person said distinct from your normalization, inference, assumption, proposal, transformation, or default. Do not invent content, silently increase precision, or treat assent to wording you supplied as independent evidence. When accounts differ, establish whether the relationship is correction, conflict, or contextual coexistence before reconciling them. diff --git a/libs/@hashintel/brunch-agent/packages/core/src/question-marker.ts b/libs/@hashintel/brunch-agent/packages/core/src/question-marker.ts new file mode 100644 index 00000000000..ba6194c63f5 --- /dev/null +++ b/libs/@hashintel/brunch-agent/packages/core/src/question-marker.ts @@ -0,0 +1,28 @@ +import * as v from "valibot"; + +export const BRUNCH_QUESTION_TOOL_NAME = "brunch_mark_question"; +export const BRUNCH_QUESTION_DATA_NAME = "brunch-question"; + +const NonBlankStringSchema = v.pipe( + v.string(), + v.check((value) => /\S/u.test(value), "Expected a non-blank string."), +); + +export const BrunchQuestionInputSchema = v.object({ + question: NonBlankStringSchema, +}); + +export const BrunchQuestionDataSchema = v.object({ + question: NonBlankStringSchema, + toolCallId: NonBlankStringSchema, +}); + +export type BrunchQuestionData = v.InferOutput; + +export const parseBrunchQuestionData = ( + value: unknown, +): BrunchQuestionData | undefined => { + const result = v.safeParse(BrunchQuestionDataSchema, value); + + return result.success ? result.output : undefined; +}; diff --git a/libs/@hashintel/brunch-agent/packages/core/test/question-marker.test.ts b/libs/@hashintel/brunch-agent/packages/core/test/question-marker.test.ts new file mode 100644 index 00000000000..b1da49f3049 --- /dev/null +++ b/libs/@hashintel/brunch-agent/packages/core/test/question-marker.test.ts @@ -0,0 +1,93 @@ +import { readFile } from "node:fs/promises"; + +import * as v from "valibot"; +import { describe, expect, test, vi } from "vitest"; + +import { createBrunchQuestionMarkerTool } from "../src/flue"; +import { + BRUNCH_QUESTION_DATA_NAME, + BRUNCH_QUESTION_TOOL_NAME, + BrunchQuestionDataSchema, + BrunchQuestionInputSchema, + parseBrunchQuestionData, + type BrunchQuestionData, +} from "../src/question-marker"; + +import type { FlueLogger } from "@flue/runtime"; + +describe("the Brunch question marker", () => { + test("defines one non-interactive tool and data-part identity", () => { + expect(BRUNCH_QUESTION_TOOL_NAME).toBe("brunch_mark_question"); + expect(BRUNCH_QUESTION_DATA_NAME).toBe("brunch-question"); + }); + + test("preserves exact non-blank question text and tool-call identity", () => { + const question = " Which line should run this order? "; + + expect( + v.parse(BrunchQuestionInputSchema, { + question, + }), + ).toEqual({ question }); + expect( + v.parse(BrunchQuestionDataSchema, { + question, + toolCallId: "tool-question-1", + }), + ).toEqual({ question, toolCallId: "tool-question-1" }); + }); + + test("writes the exact marker without terminating or waiting for an answer", async () => { + const writeQuestion = vi.fn<(question: BrunchQuestionData) => void>(); + const tool = createBrunchQuestionMarkerTool(writeQuestion); + + const result = await tool.run({ + data: { question: "Which line should run this order?" }, + log: { + error: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + }, + toolCallId: "tool-question-1", + }); + + expect(writeQuestion).toHaveBeenCalledOnce(); + expect(writeQuestion).toHaveBeenCalledWith({ + question: "Which line should run this order?", + toolCallId: "tool-question-1", + }); + expect(result).toEqual({ output: { marked: true } }); + }); + + test.each([ + { question: "" }, + { question: " " }, + { question: "What matters?", toolCallId: "" }, + { question: "What matters?", toolCallId: " " }, + ])("rejects an incomplete marker: %j", (marker) => { + expect(v.safeParse(BrunchQuestionDataSchema, marker).success).toBe(false); + expect(parseBrunchQuestionData(marker)).toBeUndefined(); + }); + + test("parses exact question data at the client projection boundary", () => { + const marker = { + question: " Which line should run this order? ", + toolCallId: "tool-question-1", + }; + + expect(parseBrunchQuestionData(marker)).toEqual(marker); + expect(parseBrunchQuestionData(null)).toBeUndefined(); + }); + + test("instructs the model to mark and then reproduce the exact question in ordinary prose", async () => { + const systemPrompt = await readFile( + new URL("../src/prompts/SYSTEM.md", import.meta.url), + "utf8", + ); + + expect(systemPrompt).toContain("brunch_mark_question"); + expect(systemPrompt).toContain("exact same question text"); + expect(systemPrompt).toContain("ordinary assistant prose"); + expect(systemPrompt).toContain("does not wait for or accept the answer"); + }); +}); diff --git a/libs/@hashintel/brunch-agent/packages/core/vite.config.ts b/libs/@hashintel/brunch-agent/packages/core/vite.config.ts index 291e5b9c4bf..cd4e7eedf23 100644 --- a/libs/@hashintel/brunch-agent/packages/core/vite.config.ts +++ b/libs/@hashintel/brunch-agent/packages/core/vite.config.ts @@ -13,6 +13,9 @@ export default defineConfig({ ), flue: fileURLToPath(new URL("src/flue.ts", import.meta.url)), index: fileURLToPath(new URL("src/index.ts", import.meta.url)), + "question-marker": fileURLToPath( + new URL("src/question-marker.ts", import.meta.url), + ), storage: fileURLToPath(new URL("src/storage.ts", import.meta.url)), }, fileName: (_format, entryName) => `${entryName}.js`, diff --git a/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/error-text.ts b/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/error-text.ts new file mode 100644 index 00000000000..16b440c79cc --- /dev/null +++ b/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/error-text.ts @@ -0,0 +1,81 @@ +const maxErrorTextLength = 10_000; + +const nonEmptyText = (value: string): string | null => + value.trim().length > 0 ? value : null; + +const isPlainObject = (value: unknown): value is Record => { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + return false; + } + + const prototype = Object.getPrototypeOf(value) as unknown; + return prototype === Object.prototype || prototype === null; +}; + +const serializePlainObject = ( + value: Record, + seen: WeakSet, +): string | null => { + try { + const serialized: unknown = JSON.stringify( + value, + (_key, nestedValue: unknown) => { + if (typeof nestedValue === "bigint") { + return nestedValue.toString(); + } + if (typeof nestedValue !== "object" || nestedValue === null) { + return nestedValue; + } + if (seen.has(nestedValue)) { + return "[Circular]"; + } + seen.add(nestedValue); + return nestedValue; + }, + ); + return typeof serialized === "string" ? serialized : null; + } catch { + return null; + } +}; + +const serializeErrorValue = ( + value: unknown, + seen: WeakSet, +): string | null => { + if (typeof value === "string") { + return nonEmptyText(value); + } + if (value instanceof Error) { + if (seen.has(value)) { + return "[Circular]"; + } + seen.add(value); + + const message = nonEmptyText(value.message); + const cause = + value.cause === undefined ? null : serializeErrorValue(value.cause, seen); + if (message !== null && cause !== null) { + return `${message}\nCaused by: ${cause}`; + } + return message ?? cause; + } + if (isPlainObject(value)) { + return serializePlainObject(value, seen); + } + return null; +}; + +export const serializeErrorText = ( + error: unknown, + fallback = "The chat turn failed.", +): string => { + const serialized = serializeErrorValue(error, new WeakSet()); + if (serialized === null) { + return fallback; + } + if (serialized.length <= maxErrorTextLength) { + return serialized; + } + return `${serialized.slice(0, maxErrorTextLength - 1)}…`; +}; diff --git a/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/index.ts b/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/index.ts index 93b19f1e1be..14850dea929 100644 --- a/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/index.ts +++ b/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/index.ts @@ -2,9 +2,15 @@ import { FlueApiError, FlueExecutionError } from "@flue/sdk"; import { getToolName, isToolUIPart } from "ai"; import { CLIENT_TOOL_RESULT_SIGNAL } from "./client-tool-result"; +import { serializeErrorText } from "./error-text"; import { createFlueUiStream } from "./ui-stream"; -import type { AgentSendResult, DeliveredMessage, FlueClient } from "@flue/sdk"; +import type { + AgentSendResult, + ConversationStreamChunk, + DeliveredMessage, + FlueClient, +} from "@flue/sdk"; import type { ChatTransport, UIMessage, UIMessageChunk } from "ai"; export { BRUNCH_CONVERSATION_HEADER, BRUNCH_PRINCIPAL_HEADER } from "./headers"; @@ -26,22 +32,86 @@ export interface ClientToolResult { readonly toolCallId: string; readonly toolName: string; readonly output: unknown; + readonly source?: "voice"; +} + +export interface FlueChatResponseMessageEvent { + readonly messageId: string; + readonly submissionId: AgentSendResult["submissionId"]; +} + +export interface FlueChatResponseMessageStartedEvent extends FlueChatResponseMessageEvent { + readonly position: Extract< + ConversationStreamChunk, + { type: "message-started" } + >["position"]; +} + +export interface FlueChatResponseMessageCompletedEvent extends FlueChatResponseMessageEvent { + readonly position: Extract< + ConversationStreamChunk, + { type: "message-completed" } + >["position"]; } export interface FlueChatTransportOptions { readonly client: FlueClient; readonly clientToolNames: ReadonlySet; + readonly hiddenToolNames?: ReadonlySet; readonly onAdmission?: (event: { readonly admission: AgentSendResult; readonly kind: "client-tool-result" | "user"; readonly messageId: string; }) => void; - readonly onResponseMessage?: (event: { - readonly messageId: string; - readonly submissionId: AgentSendResult["submissionId"]; - }) => void; + readonly onResponseMessage?: ( + event: FlueChatResponseMessageStartedEvent, + ) => void; + readonly onResponseMessageCompleted?: ( + event: FlueChatResponseMessageCompletedEvent, + ) => void; +} + +export type FlueChatAdmissionFailure = + | { readonly kind: "aborted" } + | { readonly kind: "ambiguous" } + | { readonly kind: "rejected"; readonly status: number } + | { + readonly kind: "submission-conflict"; + readonly status: 409; + readonly submissionId: AgentSendResult["submissionId"]; + }; + +const admissionFailureMessage = (failure: FlueChatAdmissionFailure): string => { + switch (failure.kind) { + case "aborted": + return "The local chat submission was cancelled."; + case "ambiguous": + return "Brunch may have accepted the message, but admission could not be confirmed. Reopen the conversation before trying again."; + case "rejected": + return `Brunch rejected the message before admission (HTTP ${failure.status}).`; + case "submission-conflict": + return `The delivery key already belongs to admitted submission ${failure.submissionId}; the changed payload was not admitted.`; + } +}; + +export class FlueChatAdmissionError extends Error { + public readonly failure: FlueChatAdmissionFailure; + + public constructor( + failure: FlueChatAdmissionFailure, + options?: { readonly cause?: unknown }, + ) { + super(admissionFailureMessage(failure), options); + this.name = "FlueChatAdmissionError"; + this.failure = failure; + } } +const asRecord = (value: unknown): Record | null => + typeof value === "object" && value !== null && !Array.isArray(value) + ? (value as Record) + : null; + const completedClientToolResults = ( messages: readonly UIMessage[], assistantMessageId: string, @@ -54,6 +124,17 @@ const completedClientToolResults = ( if (assistantMessage === undefined) { return []; } + const metadata = asRecord(assistantMessage.metadata); + const voiceToolCallIds = new Set( + Array.isArray(metadata?.voiceToolCallIds) + ? metadata.voiceToolCallIds.filter( + (toolCallId): toolCallId is string => typeof toolCallId === "string", + ) + : [], + ); + if (typeof metadata?.toolCallId === "string") { + voiceToolCallIds.add(metadata.toolCallId); + } return assistantMessage.parts.flatMap((part): ClientToolResult[] => { if (!isToolUIPart(part)) return []; const toolName = getToolName(part); @@ -70,6 +151,9 @@ const completedClientToolResults = ( toolCallId: part.toolCallId, toolName, output: part.output, + ...(voiceToolCallIds.has(part.toolCallId) + ? { source: "voice" as const } + : {}), }, ]; }); @@ -97,20 +181,49 @@ const finalUserMessage = ( const isAbortError = (error: unknown): boolean => error instanceof Error && error.name === "AbortError"; -const admissionError = (error: unknown): Error => { - if (isAbortError(error)) { - return error as Error; +const conflictingSubmissionId = (error: FlueApiError): string | null => { + if (error.status !== 409) return null; + const body = asRecord(error.body); + const errorBody = asRecord(body?.error); + const metadata = asRecord(errorBody?.meta); + return errorBody?.type === "submission_conflict" && + typeof metadata?.submissionId === "string" && + metadata.submissionId.length > 0 + ? metadata.submissionId + : null; +}; + +const documentedPreAdmissionStatuses = new Set([ + 400, 401, 403, 404, 405, 409, 415, +]); + +const admissionError = ( + error: unknown, + signal: AbortSignal | undefined, +): FlueChatAdmissionError => { + if (signal?.aborted || isAbortError(error)) { + return new FlueChatAdmissionError({ kind: "aborted" }, { cause: error }); } if (error instanceof FlueApiError) { - return new Error( - `Brunch rejected the message before admission (HTTP ${error.status}).`, - { cause: error }, - ); + const existingSubmissionId = conflictingSubmissionId(error); + if (existingSubmissionId !== null) { + return new FlueChatAdmissionError( + { + kind: "submission-conflict", + status: 409, + submissionId: existingSubmissionId, + }, + { cause: error }, + ); + } + if (documentedPreAdmissionStatuses.has(error.status)) { + return new FlueChatAdmissionError( + { kind: "rejected", status: error.status }, + { cause: error }, + ); + } } - return new Error( - "Brunch may have accepted the message, but admission could not be confirmed. Reopen the conversation before trying again.", - { cause: error }, - ); + return new FlueChatAdmissionError({ kind: "ambiguous" }, { cause: error }); }; const streamFailureChunk = ( @@ -136,7 +249,7 @@ const streamFailureChunk = ( error instanceof FlueExecutionError && error.failure === "terminal_event_missing" ? "The chat stream ended before the turn settled." - : "The chat turn failed.", + : serializeErrorText(error), }; }; @@ -159,6 +272,12 @@ const streamSubmission = ( return new ReadableStream({ start(controller) { let terminalEmitted = false; + let responseMessage: + | { + readonly effectiveId: string; + readonly flueId: string; + } + | undefined; const close = (): void => { if (closed) return; closed = true; @@ -182,6 +301,7 @@ const streamSubmission = ( const projector = createFlueUiStream({ submissionId: admission.submissionId, clientToolNames: options.clientToolNames, + hiddenToolNames: options.hiddenToolNames, write, }); @@ -195,12 +315,27 @@ const streamSubmission = ( ) { // Report the id the consumer sees: a client-tool continuation is // projected onto the assistant message it resumes. + responseMessage = { + effectiveId: continuationMessageId ?? event.messageId, + flueId: event.messageId, + }; options.onResponseMessage?.({ - messageId: continuationMessageId ?? event.messageId, + messageId: responseMessage.effectiveId, + position: event.position, submissionId: admission.submissionId, }); } projector.accept(event); + if ( + event.type === "message-completed" && + event.messageId === responseMessage?.flueId + ) { + options.onResponseMessageCompleted?.({ + messageId: responseMessage.effectiveId, + position: event.position, + submissionId: admission.submissionId, + }); + } }, }) .then(close) @@ -262,18 +397,36 @@ export const createFlueChatTransport = < toolCallIds: toolResults .map((result) => result.toolCallId) .join(","), + ...(toolResults.some(({ source }) => source === "voice") + ? { + voiceToolCallIds: toolResults + .filter(({ source }) => source === "voice") + .map(({ toolCallId }) => toolCallId) + .join(","), + } + : {}), }, }; })(); + const idempotencyKey = + messageId === undefined + ? `ai-sdk:${userMessage!.id}` + : `ai-sdk-tool:${messageId}:${toolResults + .map(({ toolCallId }) => toolCallId) + .join(",")}`; + if (Array.from(idempotencyKey).length > 256) { + throw new Error("The submitted message identity is too long."); + } let admission: AgentSendResult; try { admission = await options.client.send({ + idempotencyKey, message, signal: abortSignal, }); } catch (error) { - throw admissionError(error); + throw admissionError(error, abortSignal); } options.onAdmission?.({ admission, diff --git a/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/transcript.ts b/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/transcript.ts index 089fc3dd4bc..8d27f09af1c 100644 --- a/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/transcript.ts +++ b/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/transcript.ts @@ -9,16 +9,23 @@ import type { UIMessage } from "ai"; type UiMessagePart = UIMessage["parts"][number]; +export interface UiHistoryMessageMetadata { + readonly source: "voice"; + readonly voiceToolCallIds?: readonly string[]; +} + export type UiHistoryMessage = Omit< - UIMessage, + UIMessage, "metadata" | "parts" | "role" > & { + metadata?: UiHistoryMessageMetadata; role: Extract; parts: UiMessagePart[]; }; export interface SnapshotToUiMessagesOptions { readonly clientToolNames: ReadonlySet; + readonly hiddenToolNames?: ReadonlySet; } const unhandledConversationPart = (part: never): never => { @@ -33,11 +40,16 @@ const isFlueDataPart = ( const isRecord = (value: unknown): value is Record => typeof value === "object" && value !== null; +interface ClientToolResult { + readonly output: unknown; + readonly source?: "voice"; +} + const clientToolResultsFrom = ( snapshot: Pick, signalName: string, -): ReadonlyMap => { - const outputsByCallId = new Map(); +): ReadonlyMap => { + const resultsByCallId = new Map(); for (const message of snapshot.messages) { if (message.purpose !== "dispatch") continue; if (message.signal?.tagName !== signalName) continue; @@ -63,19 +75,22 @@ const clientToolResultsFrom = ( ) { continue; } - outputsByCallId.set(result.toolCallId, result.output); + resultsByCallId.set(result.toolCallId, { + output: result.output, + ...(result.source === "voice" ? { source: "voice" } : {}), + }); } } - return outputsByCallId; + return resultsByCallId; }; const toolPartFrom = ( part: Extract, clientToolNames: ReadonlySet, - clientOutputs: ReadonlyMap, + clientResults: ReadonlyMap, ): UiMessagePart => { const isClientTool = clientToolNames.has(part.toolName); - const hasClientOutput = clientOutputs.has(part.toolCallId); + const hasClientOutput = clientResults.has(part.toolCallId); if (part.state === "output-error") { return { type: `tool-${part.toolName}`, @@ -95,7 +110,7 @@ const toolPartFrom = ( }; } const output = isClientTool - ? clientOutputs.get(part.toolCallId) + ? clientResults.get(part.toolCallId)?.output : part.state === "output-available" ? part.output : undefined; @@ -121,7 +136,7 @@ const toolPartFrom = ( const partsFrom = ( message: FlueConversationMessage, options: SnapshotToUiMessagesOptions, - clientOutputs: ReadonlyMap, + clientResults: ReadonlyMap, ): UiMessagePart[] => { const parts: UiMessagePart[] = []; for (const part of message.parts) { @@ -134,7 +149,8 @@ const partsFrom = ( continue; } if (part.type === "dynamic-tool") { - parts.push(toolPartFrom(part, options.clientToolNames, clientOutputs)); + if (options.hiddenToolNames?.has(part.toolName) === true) continue; + parts.push(toolPartFrom(part, options.clientToolNames, clientResults)); continue; } if (part.type === "file") { @@ -159,7 +175,7 @@ export const snapshotToUiMessages = ( snapshot: Pick, options: SnapshotToUiMessagesOptions, ): UiHistoryMessage[] => { - const clientOutputs = clientToolResultsFrom( + const clientResults = clientToolResultsFrom( snapshot, CLIENT_TOOL_RESULT_SIGNAL, ); @@ -180,7 +196,7 @@ export const snapshotToUiMessages = ( if (message.display !== "visible") continue; if (message.purpose !== "user" && message.purpose !== "assistant") continue; if (message.role !== "user" && message.role !== "assistant") continue; - const parts = partsFrom(message, options, clientOutputs); + const parts = partsFrom(message, options, clientResults); if (message.role === "user") { resumableAssistant = undefined; continuationPending = false; @@ -195,10 +211,27 @@ export const snapshotToUiMessages = ( continuationPending = false; continue; } + const voiceToolCallIds = + message.role === "assistant" + ? message.parts.flatMap((part) => + part.type === "dynamic-tool" && + clientResults.get(part.toolCallId)?.source === "voice" + ? [part.toolCallId] + : [], + ) + : []; const projected: UiHistoryMessage = { id: message.id, role: message.role, parts, + ...(voiceToolCallIds.length > 0 + ? { + metadata: { + source: "voice", + voiceToolCallIds, + }, + } + : {}), }; messages.push(projected); if (message.role === "assistant") { diff --git a/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/ui-stream.ts b/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/ui-stream.ts index 7270be320b2..860fb2d4c88 100644 --- a/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/ui-stream.ts +++ b/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/ui-stream.ts @@ -1,9 +1,12 @@ +import { serializeErrorText } from "./error-text"; + import type { AgentSendResult, ConversationStreamChunk } from "@flue/sdk"; import type { UIMessageChunk } from "ai"; export interface FlueUiStreamOptions { readonly submissionId: AgentSendResult["submissionId"]; readonly clientToolNames: ReadonlySet; + readonly hiddenToolNames?: ReadonlySet; readonly write: (chunk: UIMessageChunk) => void; } @@ -26,6 +29,7 @@ export const createFlueUiStream = ( let turnId: string | undefined; let partOrdinal = 0; let streamingPart: StreamingPart | undefined; + const hiddenToolCallIds = new Set(); const pendingClientToolCallIds = new Set(); const finishPart = (): void => { @@ -86,7 +90,7 @@ export const createFlueUiStream = ( case "failed": options.write({ type: "error", - errorText: "The chat turn failed.", + errorText: serializeErrorText(chunk.error), }); break; case "aborted": @@ -123,6 +127,10 @@ export const createFlueUiStream = ( if (!accepting || messageId === undefined) return; if (chunk.messageId !== messageId) return; finishPart(); + if (options.hiddenToolNames?.has(chunk.toolName) === true) { + hiddenToolCallIds.add(chunk.toolCallId); + return; + } const isClientTool = options.clientToolNames.has(chunk.toolName); if (isClientTool) pendingClientToolCallIds.add(chunk.toolCallId); options.write({ @@ -136,6 +144,7 @@ export const createFlueUiStream = ( } case "tool-output": { if (!accepting || messageId === undefined) return; + if (hiddenToolCallIds.has(chunk.toolCallId)) return; if (pendingClientToolCallIds.has(chunk.toolCallId)) return; options.write({ type: "tool-output-available", @@ -147,6 +156,7 @@ export const createFlueUiStream = ( } case "tool-output-error": { if (!accepting || messageId === undefined) return; + if (hiddenToolCallIds.has(chunk.toolCallId)) return; if (pendingClientToolCallIds.has(chunk.toolCallId)) return; options.write({ type: "tool-output-error", diff --git a/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/chat-transport.test.ts b/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/chat-transport.test.ts index 3b982b101be..b5d6945226d 100644 --- a/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/chat-transport.test.ts +++ b/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/chat-transport.test.ts @@ -114,6 +114,7 @@ test("admits one user message and projects a finite per-turn stream", async () = expect(send).toHaveBeenCalledOnce(); expect(send).toHaveBeenCalledWith({ + idempotencyKey: "ai-sdk:user-1", message: { kind: "user", body: "Run the transport tracer." }, signal: undefined, }); @@ -158,6 +159,7 @@ test("admits one client-tool result signal and resumes its assistant id", async ); expect(send).toHaveBeenCalledWith({ + idempotencyKey: "ai-sdk-tool:assistant-original:tool-1", message: { kind: "signal", type: "client-tool-result", @@ -194,12 +196,14 @@ test("starts with history-only reconnection", async () => { test.each([ [ "failed", - new FlueExecutionError({ - target: "agent_submission", - targetId: admission.submissionId, - failure: "failed", + new Error("Elicitor tool failed.", { + cause: { field: "answer", reason: "Required" }, }), - { type: "error", errorText: "The chat turn failed." }, + { + type: "error", + errorText: + 'Elicitor tool failed.\nCaused by: {"field":"answer","reason":"Required"}', + }, ], [ "aborted", @@ -283,7 +287,7 @@ test("keeps caller cancellation distinct from durable abort", async () => { ]); }); -test("surfaces rejected and ambiguous admission without retrying", async () => { +test("classifies documented rejection and ambiguous admission without retrying", async () => { const rejectedSend = vi.fn(async () => { throw new FlueApiError(403, ""); }); @@ -305,25 +309,97 @@ test("surfaces rejected and ambiguous admission without retrying", async () => { await expect( createTransport(rejectedSend).sendMessages(options), - ).rejects.toThrow("rejected the message before admission (HTTP 403)"); + ).rejects.toMatchObject({ + failure: { kind: "rejected", status: 403 }, + message: "Brunch rejected the message before admission (HTTP 403).", + name: "FlueChatAdmissionError", + }); await expect( createTransport(ambiguousSend).sendMessages(options), - ).rejects.toThrow("may have accepted the message"); + ).rejects.toMatchObject({ + failure: { kind: "ambiguous" }, + message: + "Brunch may have accepted the message, but admission could not be confirmed. Reopen the conversation before trying again.", + name: "FlueChatAdmissionError", + }); expect(rejectedSend).toHaveBeenCalledOnce(); expect(ambiguousSend).toHaveBeenCalledOnce(); }); -test("reports one admission and its correlated response message", async () => { +test.each([ + ["server failure", new FlueApiError(500, "")], + ["unknown response", new FlueApiError(418, "")], +] as const)( + "treats a %s after request write as ambiguous", + async (_label, error) => { + const send = vi.fn(async () => { + throw error; + }); + const transport = createFlueChatTransport({ + client: { send } as Pick as FlueClient, + clientToolNames: new Set(), + }); + + await expect( + transport.sendMessages( + sendOptions([ + { + id: "user-ambiguous", + role: "user", + parts: [{ type: "text", text: "Do not retry this." }], + }, + ]), + ), + ).rejects.toMatchObject({ + failure: { kind: "ambiguous" }, + name: "FlueChatAdmissionError", + }); + expect(send).toHaveBeenCalledOnce(); + }, +); + +test("classifies an explicit local admission abort without retrying", async () => { + const send = vi.fn(async () => { + throw new DOMException("cancelled", "AbortError"); + }); + const transport = createFlueChatTransport({ + client: { send } as Pick as FlueClient, + clientToolNames: new Set(), + }); + + await expect( + transport.sendMessages( + sendOptions([ + { + id: "user-aborted", + role: "user", + parts: [{ type: "text", text: "Cancel locally." }], + }, + ]), + ), + ).rejects.toMatchObject({ + failure: { kind: "aborted" }, + name: "FlueChatAdmissionError", + }); + expect(send).toHaveBeenCalledOnce(); +}); + +test("reports one admission and its correlated response message completion", async () => { const { client } = clientWith(completedEvents); const onAdmission = vi.fn>(); const onResponseMessage = vi.fn>(); + const onResponseMessageCompleted = + vi.fn< + NonNullable + >(); const transport = createFlueChatTransport({ client, clientToolNames: new Set(), onAdmission, onResponseMessage, + onResponseMessageCompleted, }); const stream = await transport.sendMessages( @@ -346,6 +422,13 @@ test("reports one admission and its correlated response message", async () => { expect(onResponseMessage).toHaveBeenCalledOnce(); expect(onResponseMessage).toHaveBeenCalledWith({ messageId: "assistant-1", + position: position(0), + submissionId: admission.submissionId, + }); + expect(onResponseMessageCompleted).toHaveBeenCalledOnce(); + expect(onResponseMessageCompleted).toHaveBeenCalledWith({ + messageId: "assistant-1", + position: position(2), submissionId: admission.submissionId, }); }); @@ -388,14 +471,19 @@ test("stays silent after the consumer cancels the per-turn stream", async () => await expect(reader.closed).resolves.toBeUndefined(); }); -test("reports a client-tool continuation against the resumed assistant id", async () => { +test("reports a client-tool continuation and completion against the resumed assistant id", async () => { const { client } = clientWith(completedEvents); const onResponseMessage = vi.fn>(); + const onResponseMessageCompleted = + vi.fn< + NonNullable + >(); const transport = createFlueChatTransport({ client, clientToolNames: new Set(["readPetrinautDoc"]), onResponseMessage, + onResponseMessageCompleted, }); const stream = await transport.sendMessages( @@ -424,6 +512,114 @@ test("reports a client-tool continuation against the resumed assistant id", asyn expect(onResponseMessage).toHaveBeenCalledOnce(); expect(onResponseMessage).toHaveBeenCalledWith({ messageId: "assistant-original", + position: position(0), submissionId: admission.submissionId, }); + expect(onResponseMessageCompleted).toHaveBeenCalledOnce(); + expect(onResponseMessageCompleted).toHaveBeenCalledWith({ + messageId: "assistant-original", + position: position(2), + submissionId: admission.submissionId, + }); +}); + +test("replays a stable typed or Voice message with the same idempotency key", async () => { + const seenKeys = new Set(); + let admittedTurns = 0; + const send = vi.fn(async (options) => { + const key = options.idempotencyKey; + if (key === undefined || !seenKeys.has(key)) { + admittedTurns += 1; + if (key !== undefined) seenKeys.add(key); + return admission; + } + return { ...admission, deduplicated: true }; + }); + const wait = vi.fn(async () => undefined); + const onAdmission = + vi.fn>(); + const transport = createFlueChatTransport({ + client: { send, wait } as Pick as FlueClient, + clientToolNames: new Set(), + onAdmission, + }); + const typedTurn = sendOptions([ + { + id: "typed-message-1", + role: "user", + parts: [{ type: "text", text: "Admit this once." }], + }, + ]); + + const firstStream = await transport.sendMessages(typedTurn); + const replayedStream = await transport.sendMessages(typedTurn); + await Promise.all([readChunks(firstStream), readChunks(replayedStream)]); + + expect(send).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ idempotencyKey: "ai-sdk:typed-message-1" }), + ); + expect(send).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ idempotencyKey: "ai-sdk:typed-message-1" }), + ); + expect(admittedTurns).toBe(1); + expect(onAdmission).toHaveBeenNthCalledWith(2, { + admission: { ...admission, deduplicated: true }, + kind: "user", + messageId: "typed-message-1", + }); + + const voiceTurn = sendOptions([ + { + id: "voice-realtime:7:item%2F1:0", + role: "user", + parts: [{ type: "text", text: "Voice transcript." }], + }, + ]); + await readChunks(await transport.sendMessages(voiceTurn)); + expect(send).toHaveBeenLastCalledWith( + expect.objectContaining({ + idempotencyKey: "ai-sdk:voice-realtime:7:item%2F1:0", + }), + ); +}); + +test("reports an idempotency conflict as a definite existing admission", async () => { + const send = vi.fn(async () => { + throw new FlueApiError(409, { + error: { + details: "", + message: "The delivery key already names another payload.", + meta: { submissionId: "submission-existing" }, + type: "submission_conflict", + }, + }); + }); + const transport = createFlueChatTransport({ + client: { send } as Pick as FlueClient, + clientToolNames: new Set(), + }); + + await expect( + transport.sendMessages( + sendOptions([ + { + id: "user-conflict", + role: "user", + parts: [{ type: "text", text: "Changed payload." }], + }, + ]), + ), + ).rejects.toMatchObject({ + failure: { + kind: "submission-conflict", + status: 409, + submissionId: "submission-existing", + }, + message: + "The delivery key already belongs to admitted submission submission-existing; the changed payload was not admitted.", + name: "FlueChatAdmissionError", + }); + expect(send).toHaveBeenCalledOnce(); }); diff --git a/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/transcript.test.ts b/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/transcript.test.ts index fe15a52470a..c41a8de7f0f 100644 --- a/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/transcript.test.ts +++ b/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/transcript.test.ts @@ -31,6 +31,7 @@ const snapshotWithPendingClientTool: FlueConversationSnapshot = { const projectionOptions = { clientToolNames: new Set(["readPetrinautDoc"]), + hiddenToolNames: new Set(["brunch_mark_question"]), }; test("leaves an unfinished client tool available to run", () => { @@ -85,6 +86,65 @@ test("uses a recorded browser result even when it is null", () => { ]); }); +test("reconstructs durable voice provenance for each browser result", () => { + const snapshot: FlueConversationSnapshot = { + ...snapshotWithPendingClientTool, + messages: [ + { + ...snapshotWithPendingClientTool.messages[0]!, + parts: [ + ...snapshotWithPendingClientTool.messages[0]!.parts, + { + type: "dynamic-tool", + toolCallId: "tool-doc-2", + toolName: "readPetrinautDoc", + state: "output-available", + input: { doc: "ai-assistant" }, + output: { awaiting: "client" }, + }, + ], + }, + { + id: "signal-voice-results", + role: "system", + purpose: "dispatch", + display: "hidden", + signal: { tagName: CLIENT_TOOL_RESULT_SIGNAL }, + parts: [ + { + type: "text", + text: JSON.stringify([ + { + toolCallId: "tool-doc-1", + toolName: "readPetrinautDoc", + output: "First guide", + source: "voice", + }, + { + toolCallId: "tool-doc-2", + toolName: "readPetrinautDoc", + output: "Second guide", + source: "voice", + }, + ]), + state: "done", + }, + ], + }, + ], + }; + + expect(snapshotToUiMessages(snapshot, projectionOptions)).toEqual([ + expect.objectContaining({ + id: "assistant-1", + metadata: { + source: "voice", + voiceToolCallIds: ["tool-doc-1", "tool-doc-2"], + }, + }), + ]); +}); + test("keeps Flue data parts on the AI SDK message", () => { const snapshot: FlueConversationSnapshot = { v: 1, @@ -235,3 +295,50 @@ test("folds a client-tool continuation into the assistant message it resumed", ( }, ]); }); + +test("hides a question-marker tool while retaining its durable data", () => { + const question = "Which line should run this order?"; + const snapshot: FlueConversationSnapshot = { + v: 1, + conversationId: "conversation-1", + offset: "0", + messages: [ + { + id: "assistant-question", + role: "assistant", + purpose: "assistant", + display: "visible", + parts: [ + { + type: "dynamic-tool", + toolCallId: "tool-question-1", + toolName: "brunch_mark_question", + state: "output-available", + input: { question }, + output: { marked: true }, + }, + { + type: "data-brunch-question", + data: { question, toolCallId: "tool-question-1" }, + }, + { type: "text", text: question, state: "done" }, + ], + }, + ], + settlements: [], + }; + + expect(snapshotToUiMessages(snapshot, projectionOptions)).toEqual([ + { + id: "assistant-question", + role: "assistant", + parts: [ + { + type: "data-brunch-question", + data: { question, toolCallId: "tool-question-1" }, + }, + { type: "text", text: question, state: "done" }, + ], + }, + ]); +}); diff --git a/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/ui-stream.test.ts b/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/ui-stream.test.ts index c567fcd9b38..999a8ef498a 100644 --- a/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/ui-stream.test.ts +++ b/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/ui-stream.test.ts @@ -9,11 +9,13 @@ const position = (index: number) => ({ batch: 1, index }); const project = ( chunks: readonly ConversationStreamChunk[], + hiddenToolNames: ReadonlySet = new Set(), ): UIMessageChunk[] => { const written: UIMessageChunk[] = []; const projector = createFlueUiStream({ submissionId: "submission-1", clientToolNames: new Set(["readPetrinautDoc"]), + hiddenToolNames, write: (chunk) => written.push(chunk), }); for (const chunk of chunks) projector.accept(chunk); @@ -64,6 +66,72 @@ test("projects data and metadata onto the AI SDK stream", () => { }); }); +test("hides an implementation tool while preserving its data marker", () => { + const written = project( + [ + { + type: "message-started", + conversationId: "conversation-1", + messageId: "message-1", + submissionId: "submission-1", + turnId: "turn-1", + position: position(0), + }, + { + type: "tool-input", + conversationId: "conversation-1", + messageId: "message-1", + toolCallId: "tool-question-1", + toolName: "brunch_mark_question", + input: { question: "Which line should run this order?" }, + position: position(1), + }, + { + type: "data-part", + conversationId: "conversation-1", + messageId: "message-1", + name: "brunch-question", + data: { + question: "Which line should run this order?", + toolCallId: "tool-question-1", + }, + position: position(2), + }, + { + type: "tool-output", + conversationId: "conversation-1", + toolCallId: "tool-question-1", + output: { marked: true }, + position: position(3), + }, + { + type: "submission-settled", + conversationId: "conversation-1", + submissionId: "submission-1", + outcome: "completed", + position: position(4), + }, + ], + new Set(["brunch_mark_question"]), + ); + + expect(written).toContainEqual({ + type: "data-brunch-question", + data: { + question: "Which line should run this order?", + toolCallId: "tool-question-1", + }, + }); + expect( + written.some( + (chunk) => + chunk.type === "tool-input-available" || + chunk.type === "tool-output-available" || + chunk.type === "tool-output-error", + ), + ).toBe(false); +}); + test("ignores observation catch-up chunks in a submission stream", () => { const written = project([ { @@ -118,3 +186,71 @@ test("ignores observation catch-up chunks in a submission stream", () => { "finish", ]); }); + +test.each([ + { + error: new Error("Elicitor failed.", { + cause: "The requested field is required.", + }), + expected: "Elicitor failed.\nCaused by: The requested field is required.", + shape: "Error with cause", + }, + { + error: "The elicitor rejected the answer.", + expected: "The elicitor rejected the answer.", + shape: "string", + }, + { + error: { field: "answer", reason: "Required" }, + expected: '{"field":"answer","reason":"Required"}', + shape: "plain object", + }, + { + error: 503, + expected: "The chat turn failed.", + shape: "unsupported value", + }, + { + error: "", + expected: "The chat turn failed.", + shape: "empty string", + }, +])("preserves a failed submission's $shape error", ({ error, expected }) => { + const written = project([ + { + type: "submission-settled", + conversationId: "conversation-1", + submissionId: "submission-1", + outcome: "failed", + error, + position: position(0), + }, + ]); + + expect(written).toEqual([{ type: "error", errorText: expected }]); + expect(written).not.toContainEqual({ + type: "error", + errorText: "[object Object]", + }); +}); + +test("bounds cyclic failed-submission objects", () => { + const cyclicError: Record = { reason: "Recursive failure" }; + cyclicError.self = cyclicError; + cyclicError.payload = "x".repeat(20_000); + + const written = project([ + { + type: "submission-settled", + conversationId: "conversation-1", + submissionId: "submission-1", + outcome: "failed", + error: cyclicError, + position: position(0), + }, + ]); + + const failure = written.find((chunk) => chunk.type === "error"); + expect(failure?.errorText).toContain('"self":"[Circular]"'); + expect(failure?.errorText.length).toBeLessThanOrEqual(10_000); +}); diff --git a/libs/@hashintel/petrinaut/docs/ai-assistant.md b/libs/@hashintel/petrinaut/docs/ai-assistant.md index 6d56acb96e8..0cf3ec43ca4 100644 --- a/libs/@hashintel/petrinaut/docs/ai-assistant.md +++ b/libs/@hashintel/petrinaut/docs/ai-assistant.md @@ -20,22 +20,25 @@ While a response is streaming you can: - Type your next message in the composer -- it is queued for after the current response ends. The application embedding Petrinaut may place an additional control beside the message box. For example, a host can offer another way to enter finalized text. Text submitted by that control behaves like text sent with the keyboard: it joins the same conversation and, when an inline question is waiting for an answer, completes that question rather than starting an unrelated message. A host can explicitly submit a separate message instead when the text is a correction or other follow-up that must not answer the pending question. -If the host offers voice input, a finalized spoken turn is held while an existing response finishes and is submitted when the conversation is ready. +If the host offers voice input, only a finalized transcript captured while Voice owns the input turn can be submitted. Voice waits while an existing response finishes or yields through the host's handoff control. -If an assistant request fails, Petrinaut shows the error in a brief toast rather than adding it to the conversation. Retry from the composer when the assistant is ready. +If an assistant request fails, Petrinaut shows the complete error in a persistent toast rather than adding it to the conversation. Long errors wrap, diagnostic details can be copied, and the toast stays open until you close it. Retry from the composer when the assistant is ready. -Hosts may provide canonical conversation rehydration. In that case, reopening the same assistant shows its settled and stopped turns without resubmitting a message or replaying Voice audio. +Hosts may provide canonical conversation rehydration. In that case, reopening the same assistant shows its settled and stopped turns without resubmitting a message or replaying Voice audio. Voice markers attached to client-tool results survive that history. A direct spoken user message remains in the transcript after reopening, but its **Voice** chip may not be restored by the current Brunch host. When the Brunch voice preview is enabled and available, an empty composer shows a waveform action titled **Start voice mode**. Typing non-whitespace text replaces it with **Send**. The same dynamic action appears in the first-run prompt and the assistant panel; if voice is unavailable, the empty composer retains a disabled **Send** action. Starting Voice mode keeps the transcript in place and -opens the existing one-time disclosure above the composer. Review that OpenAI processes live -audio and speaks the interviewer's words while Petrinaut keeps finalized answers in the conversation -rather than the audio. You can check your microphone before confirming that you understand and -selecting **Start voice mode**. Petrinaut remembers that acknowledgement in this browser for the -current disclosure version, so later uses of **Start voice mode** start directly. If browser storage -is unavailable or the disclosure changes, Petrinaut asks again. +opens the existing one-time disclosure. Voice selected from the first-run prompt starts compact: the +disclosure and microphone check appear in a card immediately above a **Voice setup** dock, while the +AI header, transcript, and composer stay hidden. Select **Expand voice setup** to restore the full +panel. Voice started from the composer keeps that full panel visible. Review that OpenAI processes +live audio and speaks the interviewer's words while Petrinaut keeps finalized answers in the +conversation rather than the audio. You can check your microphone before confirming that you +understand and selecting **Start voice mode**. Petrinaut remembers that acknowledgement in this +browser for the current disclosure version, so later uses of **Start voice mode** start directly. If +browser storage is unavailable or the disclosure changes, Petrinaut asks again. While a session runs, the composer is replaced by a low-profile Voice dock at the foot of the panel: a ribbon that fades out at both ends and one short state -- **Connecting**, **Listening**, @@ -46,32 +49,42 @@ flicker above the line. While the assistant speaks the ribbon takes on a restrai motion instead, colour crossfading as the turn changes hands, so which side holds it is readable at a glance. It flattens to near a line whenever nobody holds the turn. -The conversation itself stays still. Spoken turns are written to it as they happen, because that is -what runs the tools that edit the net, but they stay hidden until the session ends rather than -scrolling the transcript mid-sentence. **Show transcription in chat** lets them through as they land -instead; turning it off holds them back again, and it starts off with each session. Two things are -never held back either way: anything you typed, and any inline question waiting for your answer. When -the session ends, the held turns appear together under a **Voice session · N turns** divider. Only -finalized answers and canonical Brunch text become chat history; provisional transcription and -Realtime audio are ephemeral. Finalized spoken user messages carry a small **Voice** chip in front of -the words themselves, and the exact inline answer completed by speech carries the same chip, so Voice -provenance remains visible without duplicating an answer. - -The microphone stays on while the interviewer speaks, so speaking naturally interrupts the audio -and starts listening to you; you do not need to select an interrupt action. Semantic voice detection -finishes each answer automatically after a natural pause and is tuned to allow longer thinking -pauses. There is no required done-speaking action. - -Every session control lives in the dock: **Show transcription in chat** on the left, and on the right -**Mute microphone** (**Unmute microphone** once muted) beside **End voice mode**. Muting stops -sending audio without ending the turn, so the assistant plays out whatever it is saying and unmuting -drops you straight back into the conversation. **Resume voice mode** replaces the microphone action -while a session is paused, and **Reconnect voice mode** replaces it after a failure. Nothing is added -to the canvas toolbar. Sending non-empty typed text from the +Spoken turns appear in the conversation as soon as their finalized text arrives, so the transcript +stays current while the session runs and tools that edit the net remain visible. Select **Collapse +voice session** to reduce the panel to the Voice dock alone; this hides the AI header, transcript, and +host Voice region without ending the session. Select **Expand voice session** to restore them. Ending +Voice while collapsed also closes the AI panel; ending Voice while expanded returns to the text +composer. Only finalized answers and canonical Brunch text become chat history; provisional +transcription and Realtime audio are ephemeral. Finalized spoken user messages carry a small +**Voice** chip in front of the words themselves, and the exact inline answer completed by speech +carries the same chip, so Voice provenance remains visible without duplicating an answer while the +session is mounted. + +Voice is half-duplex. The microphone is closed while the interviewer speaks or the assistant is +working, which prevents playback from becoming a false answer. Select **Your turn** to interrupt: +the dock shows the handoff as thinking while it clears pending audio and waits for the provider to +finish cancellation, then opens a fresh input turn. Audio captured before that completed handoff is +discarded. Semantic voice detection finishes your answer automatically after a natural pause, so +there is no required done-speaking action. Duplicate, empty, failed, or unavailable transcripts are +not submitted; the dock asks you to try again. An overlong answer instead asks for a shorter response. +Provisional words remain display-only until the provider completes their transcript. + +Every session control lives in the dock: **Collapse voice session** / **Expand voice session** and +**Voice playback options** on the left, and the available handoff, microphone, recovery, and end +actions on the right. +**Read full response** becomes available after the matching response and speech have both finished +and replays every exact retained canonical segment in order. **Repeat question** uses the same +availability gates and replays only exact question text explicitly marked by Brunch. It stays +disabled when that marker is missing or does not match finalized assistant text rather than +guessing that the final segment is a question. +Playback stays unavailable during active capture, submission, cancellation, pause, and errors. **Mute microphone** becomes +**Unmute microphone** once muted, and your latest choice applies when a handoff settles. **Resume voice mode** +replaces the microphone action while a session is paused, and **Reconnect voice mode** replaces it +after a failure. Nothing is added to the canvas toolbar. Sending non-empty typed text from the composer or first-run prompt ends Voice mode before it sends the message once through the same conversation; repeated send actions are ignored while that short handoff completes. -The interviewer uses a warm, calm, curious, and professionally neutral voice and treats you as the authority on your system. Brunch still chooses every question and interview decision; OpenAI only delivers its words. The question and finalized response shown in the Petrinaut conversation are authoritative. The speech request receives that exact Brunch text in part order; synthesized audio is generated from it but is not a verbatim recording. Interrupting audio does not undo the visible response or change the interview's saved history. +The interviewer uses a warm, calm, curious, and professionally neutral voice and treats you as the authority on your system. Brunch still chooses every question and interview decision; OpenAI only transcribes your completed input and delivers Brunch's words. The question and finalized response shown in the Petrinaut conversation are authoritative. The speech request receives that exact Brunch text in part order; synthesized audio is generated from it but is not a verbatim recording. Interrupting audio does not undo the visible response or change the interview's saved history. Closing the AI panel pauses microphone capture and active speech, then hides the dock until you reopen the panel. The same mounted session stays paused; choose **Resume voice mode** when you are @@ -79,8 +92,8 @@ ready. **Clear AI chat** is unavailable while a Voice session is active. If voice cannot continue, the status reads **Voice interrupted** and the actionable error arrives as -a toast that names the microphone, connection, or Voice failure in one sentence, followed by any -diagnostic reference in parentheses. **Reconnect voice mode** replaces the microphone action until +a persistent toast that names the microphone, connection, or Voice failure in one sentence, followed +by any diagnostic reference in parentheses. **Reconnect voice mode** replaces the microphone action until the session recovers. For microphone permission or device errors, allow access or connect/select a microphone before reconnecting. For an interrupted request, network error, or timeout, check the connection and reconnect. If the preview is unavailable, continue with the text composer. An invalid @@ -93,7 +106,7 @@ When no interview is active and the host permits clearing, **Clear AI chat** via ## What the assistant can do -The assistant has tools for inspecting and modifying the current net. You'll see one card per tool call inline in the conversation: +The assistant has tools for inspecting and modifying the current net. You'll see one card per tool call inline in the conversation. A failed tool card leads with its complete error instead of hiding it behind a hover tooltip: - **Read tools** (neutral, expandable) –– for checking the current net state and active Petrinaut extensions at any point, for compilation errors, and for reading the user guide. - **Mutation tools** (green for additions/updates, red for deletions) -- "Added place X", "Updated transition Y", "Removed metric Z", and so on. Multiple successive mutations group under a collapsible "N changes" header. diff --git a/libs/@hashintel/petrinaut/src/panda-preset.ts b/libs/@hashintel/petrinaut/src/panda-preset.ts index 2cdc120abf8..7d4ab7bd23c 100644 --- a/libs/@hashintel/petrinaut/src/panda-preset.ts +++ b/libs/@hashintel/petrinaut/src/panda-preset.ts @@ -122,10 +122,6 @@ export const petrinautPandaPreset = { from: { opacity: "1", transform: "translateX(0)" }, to: { opacity: "0", transform: "translateX(100px)" }, }, - petrinautVoiceReveal: { - from: { opacity: "0", transform: "translateY(10px)" }, - to: { opacity: "1", transform: "translateY(0)" }, - }, petrinautVoiceSwap: { from: { opacity: "0" }, to: { opacity: "1" }, diff --git a/libs/@hashintel/petrinaut/src/react/notifications/context.ts b/libs/@hashintel/petrinaut/src/react/notifications/context.ts index 9911f709022..5f508982ae4 100644 --- a/libs/@hashintel/petrinaut/src/react/notifications/context.ts +++ b/libs/@hashintel/petrinaut/src/react/notifications/context.ts @@ -3,6 +3,7 @@ import { createContext } from "react"; export type NotificationTone = "error" | "neutral" | "success"; export type AddNotificationInput = { + detail?: string; message: string; tone?: NotificationTone; durationMs?: number; diff --git a/libs/@hashintel/petrinaut/src/react/notifications/provider.test.tsx b/libs/@hashintel/petrinaut/src/react/notifications/provider.test.tsx new file mode 100644 index 00000000000..1dda6521daa --- /dev/null +++ b/libs/@hashintel/petrinaut/src/react/notifications/provider.test.tsx @@ -0,0 +1,76 @@ +/** + * @vitest-environment jsdom + */ +import { + cleanup, + fireEvent, + render, + screen, + waitFor, +} from "@testing-library/react"; +import { use } from "react"; +import { afterEach, expect, test, vi } from "vitest"; + +import { NotificationsContext } from "./context"; +import { NotificationsProvider } from "./provider"; +import { notificationsToaster } from "./toaster"; + +afterEach(() => { + cleanup(); + vi.restoreAllMocks(); +}); + +test("keeps error notifications open while preserving the default for other tones", async () => { + const createToast = vi.spyOn(notificationsToaster, "create"); + const Trigger = () => { + const { addNotification } = use(NotificationsContext); + + return ( + <> + + + + ); + }; + + render( + + + , + ); + + fireEvent.click(screen.getByRole("button", { name: "Error" })); + fireEvent.click(screen.getByRole("button", { name: "Success" })); + + await waitFor(() => expect(createToast).toHaveBeenCalledTimes(2)); + expect(createToast).toHaveBeenNthCalledWith(1, { + description: "The complete elicitor failure.", + duration: Infinity, + id: "notification-0", + title: "AI assistant error", + type: "error", + }); + expect(createToast).toHaveBeenNthCalledWith(2, { + description: undefined, + duration: 3000, + id: "notification-1", + title: "Saved", + type: "success", + }); +}); diff --git a/libs/@hashintel/petrinaut/src/react/notifications/provider.tsx b/libs/@hashintel/petrinaut/src/react/notifications/provider.tsx index f6d39467b8b..945584fe104 100644 --- a/libs/@hashintel/petrinaut/src/react/notifications/provider.tsx +++ b/libs/@hashintel/petrinaut/src/react/notifications/provider.tsx @@ -16,22 +16,28 @@ export const NotificationsProvider = ({ }: { children: ReactNode; }) => { - function dismissNotification(id: string) { + const dismissNotification = (id: string) => { queueMicrotask(() => { notificationsToaster.dismiss(id); }); - } + }; - function addNotification({ + const addNotification = ({ + detail, durationMs, message, tone = "success", - }: AddNotificationInput) { - const id = `notification-${nextNotificationId++}`; - const effectiveDurationMs = durationMs ?? DEFAULT_NOTIFICATION_DURATION_MS; + }: AddNotificationInput) => { + const id = `notification-${nextNotificationId}`; + nextNotificationId += 1; + const effectiveDurationMs = + tone === "error" + ? Infinity + : (durationMs ?? DEFAULT_NOTIFICATION_DURATION_MS); queueMicrotask(() => { notificationsToaster.create({ + description: detail, duration: effectiveDurationMs, id, title: message, @@ -40,7 +46,7 @@ export const NotificationsProvider = ({ }); return id; - } + }; useEffect(() => { return () => { diff --git a/libs/@hashintel/petrinaut/src/react/notifications/toaster.tsx b/libs/@hashintel/petrinaut/src/react/notifications/toaster.tsx index bc26dc8403b..fd2261b2033 100644 --- a/libs/@hashintel/petrinaut/src/react/notifications/toaster.tsx +++ b/libs/@hashintel/petrinaut/src/react/notifications/toaster.tsx @@ -5,7 +5,7 @@ import { createToaster, } from "@ark-ui/react/toast"; -import { usePortalContainerRef } from "@hashintel/ds-components"; +import { Button, usePortalContainerRef } from "@hashintel/ds-components"; import { css } from "@hashintel/ds-helpers/css"; export const notificationsToaster = createToaster({ @@ -24,10 +24,11 @@ const toastRootStyle = css({ transition: "[translate 300ms, scale 300ms, opacity 300ms, box-shadow 300ms]", transitionTimingFunction: "[cubic-bezier(0.21, 1.02, 0.73, 1)]", display: "flex", - alignItems: "center", + alignItems: "flex-start", + gap: "2", minHeight: "[26px]", width: "[max-content]", - maxWidth: "[320px]", + maxWidth: "[min(480px, calc(100vw - 32px))]", borderRadius: "lg", boxShadow: "[0 8px 24px rgba(0, 0, 0, 0.24)]", paddingX: "4", @@ -44,23 +45,93 @@ const toastRootStyle = css({ }, }); +const toastContentStyle = css({ + display: "flex", + flex: "[1]", + minWidth: "[0]", + flexDirection: "column", + gap: "1", +}); + const toastTitleStyle = css({ overflow: "hidden", - textOverflow: "ellipsis", - whiteSpace: "nowrap", + overflowWrap: "anywhere", + lineClamp: "4", fontSize: "xs", fontWeight: "medium", lineHeight: "[14px]", }); +const toastDescriptionStyle = css({ + maxHeight: "[240px]", + overflow: "auto", + overflowWrap: "anywhere", + whiteSpace: "pre-wrap", + fontSize: "xs", + lineHeight: "[18px]", + userSelect: "text", +}); + +const toastActionsStyle = css({ + display: "flex", + flexShrink: "[0]", + gap: "1", +}); + +const toastActionStyle = css({ + color: "neutral.s00", + _hover: { + color: "neutral.s00", + }, +}); + export const NotificationsToaster = () => ( - {(toast) => ( - - {toast.title} - - )} + {(toast) => { + const detail = + typeof toast.description === "string" ? toast.description : undefined; + + return ( + +
+ + {toast.title} + + {detail && ( + + {detail} + + )} +
+
+ {detail && ( +
+ + ); + }} ); diff --git a/libs/@hashintel/petrinaut/src/react/voice-session/store.ts b/libs/@hashintel/petrinaut/src/react/voice-session/store.ts index bcdc4e0ee48..73e502c20bb 100644 --- a/libs/@hashintel/petrinaut/src/react/voice-session/store.ts +++ b/libs/@hashintel/petrinaut/src/react/voice-session/store.ts @@ -7,9 +7,12 @@ import type { export type VoiceSessionActions = { end: () => void; pause: () => void; + readFullResponse?: () => void; reconnect: () => void; + repeatQuestion?: () => void; resume: () => void; setMicrophoneMuted: (muted: boolean) => void; + takeTurn?: () => Promise | void; }; export type VoiceSessionSnapshot = { diff --git a/libs/@hashintel/petrinaut/src/react/voice-session/types.ts b/libs/@hashintel/petrinaut/src/react/voice-session/types.ts index bd1a26f425e..b72bee75f8c 100644 --- a/libs/@hashintel/petrinaut/src/react/voice-session/types.ts +++ b/libs/@hashintel/petrinaut/src/react/voice-session/types.ts @@ -16,10 +16,18 @@ export type PetrinautAiVoiceSessionPhase = * effect: it changes at microphone-sampling rate. */ export type PetrinautAiVoiceSessionState = { + /** Whether the current canonical assistant response is safe to replay. */ + canReadFullResponse?: boolean; + /** Whether the final segment of the canonical response is safe to repeat. */ + canRepeatQuestion?: boolean; + /** Whether the user can cancel Voice output and start their turn. */ + canTakeTurn?: boolean; errorMessage: string | null; /** Whether microphone capture is muted independently of whose turn it is. */ microphoneMuted: boolean; /** Normalized 0–1 input level driving the listening indicator. */ microphoneLevel: number; + /** Recoverable feedback about an utterance which was not submitted. */ + notice?: string | null; phase: PetrinautAiVoiceSessionPhase; }; diff --git a/libs/@hashintel/petrinaut/src/react/voice-session/use-voice-session.ts b/libs/@hashintel/petrinaut/src/react/voice-session/use-voice-session.ts index a95671ce61d..3f88e964621 100644 --- a/libs/@hashintel/petrinaut/src/react/voice-session/use-voice-session.ts +++ b/libs/@hashintel/petrinaut/src/react/voice-session/use-voice-session.ts @@ -62,3 +62,43 @@ export const useVoiceSessionActions = (): VoiceSessionActions | null => { () => null, ); }; + +export const useVoiceSessionCanReadFullResponse = (): boolean => { + const store = use(VoiceSessionContext); + + return useSyncExternalStore( + store.subscribe, + () => store.getSnapshot().state?.canReadFullResponse ?? false, + () => false, + ); +}; + +export const useVoiceSessionCanRepeatQuestion = (): boolean => { + const store = use(VoiceSessionContext); + + return useSyncExternalStore( + store.subscribe, + () => store.getSnapshot().state?.canRepeatQuestion ?? false, + () => false, + ); +}; + +export const useVoiceSessionCanTakeTurn = (): boolean => { + const store = use(VoiceSessionContext); + + return useSyncExternalStore( + store.subscribe, + () => store.getSnapshot().state?.canTakeTurn ?? false, + () => false, + ); +}; + +export const useVoiceSessionNotice = (): string | null => { + const store = use(VoiceSessionContext); + + return useSyncExternalStore( + store.subscribe, + () => store.getSnapshot().state?.notice ?? null, + () => null, + ); +}; diff --git a/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts b/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts index 7ef27d515f5..de3acc3a4ae 100644 --- a/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts +++ b/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts @@ -61,12 +61,18 @@ export type PetrinautAiVoiceModeControls = { reconnect: () => void; /** Resumes microphone capture after `pause`. */ resume: () => void; + /** Replays the exact retained canonical assistant response when available. */ + readFullResponse?: () => void; + /** Replays the final segment of the retained canonical response. */ + repeatQuestion?: () => void; /** * Stops or restarts microphone capture while the session keeps running, so * the assistant carries on speaking. Unlike `pause`, which suspends the * whole session when Petrinaut closes the panel. */ setMicrophoneMuted: (muted: boolean) => void; + /** Cancels Voice output and hands the live microphone turn to the user. */ + takeTurn?: () => Promise | void; }; /** Stable controls and conversation state supplied to a host-owned Voice mode. */ diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/components/voice-session-labels.ts b/libs/@hashintel/petrinaut/src/ui/views/Editor/components/voice-session-labels.ts index eaf5dabb078..0e2f843924b 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/components/voice-session-labels.ts +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/components/voice-session-labels.ts @@ -26,10 +26,23 @@ export const voiceSessionStatusLabel = ( }; export const voiceSessionActionLabels = { + collapse: "Collapse voice session", end: "End voice mode", + expand: "Expand voice session", mute: "Mute microphone", pause: "Pause voice mode", + playbackOptions: "Voice playback options", + readFullResponse: "Read full response", reconnect: "Reconnect voice mode", + repeatQuestion: "Repeat question", resume: "Resume voice mode", + takeTurn: "Your turn", unmute: "Unmute microphone", } as const; + +export const voiceSetupLabels = { + collapse: "Collapse voice setup", + expand: "Expand voice setup", + region: "Voice setup", + status: "Voice setup", +} as const; diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx index b4659e40a8c..a8cd9b9d608 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx @@ -7,15 +7,17 @@ import { fireEvent, render, screen, + within, waitFor, } from "@testing-library/react"; import { useEffect } from "react"; -import { afterEach, describe, expect, test, vi } from "vitest"; +import { afterEach, beforeAll, describe, expect, test, vi } from "vitest"; import { DEFAULT_PETRINAUT_EXTENSIONS, createJsonDocHandle, createPetrinaut, + getLatestNetDefinitionToolName, type SDCPN, } from "@hashintel/petrinaut-core"; @@ -31,7 +33,11 @@ import { type SDCPNContextValue, } from "../../../../react/state/sdcpn-context"; import { definePetrinautAiInteractiveTool } from "../../../types/ai-interactive-tool"; -import { addMappedToolOutput, AiAssistantPanel } from "./ai-assistant-panel"; +import { + addMappedToolOutput, + AiAssistantPanel, + safelyAddToolOutput, +} from "./ai-assistant-panel"; import type { PetrinautAiAssistant } from "../../../petrinaut"; import type { @@ -48,6 +54,18 @@ import type { UIMessageChunk } from "ai"; let voiceModeMounts = 0; let voiceModeUnmounts = 0; +beforeAll(() => { + vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue(null); + vi.stubGlobal( + "ResizeObserver", + class { + public disconnect() {} + public observe() {} + public unobserve() {} + }, + ); +}); + const emptySDCPN: SDCPN = { places: [], transitions: [], @@ -1148,6 +1166,181 @@ describe("AiAssistantPanel composer submissions", () => { expect(voiceModeUnmounts).toBe(0); }); + test("forwards optional host Voice actions to the production dock", async () => { + const takeTurn = vi.fn(); + const repeatQuestion = vi.fn(); + const readFullResponse = vi.fn(); + const VoiceMode = ({ + context, + replayAllowed, + }: { + context: PetrinautAiVoiceModeContext; + replayAllowed: boolean; + }) => { + const { registerVoiceModeControls, reportVoiceSessionState } = context; + + useEffect( + () => + registerVoiceModeControls({ + end: async () => undefined, + pause: vi.fn(), + readFullResponse, + reconnect: vi.fn(), + repeatQuestion, + resume: vi.fn(), + setMicrophoneMuted: vi.fn(), + takeTurn, + }), + [registerVoiceModeControls], + ); + useEffect(() => { + reportVoiceSessionState({ + canReadFullResponse: replayAllowed, + canRepeatQuestion: replayAllowed, + canTakeTurn: true, + errorMessage: null, + microphoneLevel: 0, + microphoneMuted: false, + phase: "speaking", + }); + return () => reportVoiceSessionState(null); + }, [replayAllowed, reportVoiceSessionState]); + + return null; + }; + const aiAssistant = (replayAllowed: boolean): PetrinautAiAssistant => ({ + renderVoiceMode: (context) => ( + + ), + transport: { + reconnectToStream: () => Promise.resolve(null), + sendMessages: vi.fn(), + }, + }); + + const rendered = renderTestPanel({ aiAssistant: aiAssistant(true) }); + + fireEvent.click(await screen.findByRole("button", { name: "Your turn" })); + expect(takeTurn).toHaveBeenCalledOnce(); + + fireEvent.click( + screen.getByRole("button", { name: "Voice playback options" }), + ); + const repeatQuestionItem = await screen.findByRole("menuitem", { + name: "Repeat question", + }); + expect(repeatQuestionItem.getAttribute("aria-disabled")).not.toBe("true"); + const repeatQuestionMenu = screen.getByRole("menu"); + fireEvent.keyDown(repeatQuestionMenu, { key: "ArrowDown" }); + await waitFor(() => + expect(repeatQuestionMenu.getAttribute("aria-activedescendant")).toBe( + repeatQuestionItem.id, + ), + ); + fireEvent.keyDown(repeatQuestionMenu, { key: "Enter" }); + await waitFor(() => expect(repeatQuestion).toHaveBeenCalledOnce()); + + fireEvent.click( + screen.getByRole("button", { name: "Voice playback options" }), + ); + const readFullResponseItem = await screen.findByRole("menuitem", { + name: "Read full response", + }); + expect(readFullResponseItem.getAttribute("aria-disabled")).not.toBe("true"); + const readFullResponseMenu = screen.getByRole("menu"); + fireEvent.keyDown(readFullResponseMenu, { key: "End" }); + await waitFor(() => + expect(readFullResponseMenu.getAttribute("aria-activedescendant")).toBe( + readFullResponseItem.id, + ), + ); + fireEvent.keyDown(readFullResponseMenu, { key: "Enter" }); + await waitFor(() => expect(readFullResponse).toHaveBeenCalledOnce()); + + rendered.rerenderPanel(aiAssistant(false), editorContextValue); + fireEvent.click( + await screen.findByRole("button", { name: "Voice playback options" }), + ); + expect( + ( + await screen.findByRole("menuitem", { name: "Repeat question" }) + ).getAttribute("aria-disabled"), + ).toBe("true"); + expect( + screen + .getByRole("menuitem", { name: "Read full response" }) + .getAttribute("aria-disabled"), + ).toBe("true"); + }); + + test("retires missing and unmounted optional host Voice actions", async () => { + const VoiceMode = ({ + context, + }: { + context: PetrinautAiVoiceModeContext; + }) => { + const { registerVoiceModeControls, reportVoiceSessionState } = context; + + useEffect( + () => + registerVoiceModeControls({ + end: async () => undefined, + pause: vi.fn(), + reconnect: vi.fn(), + resume: vi.fn(), + setMicrophoneMuted: vi.fn(), + }), + [registerVoiceModeControls], + ); + useEffect(() => { + reportVoiceSessionState({ + canReadFullResponse: true, + canRepeatQuestion: true, + canTakeTurn: true, + errorMessage: null, + microphoneLevel: 0, + microphoneMuted: false, + phase: "speaking", + }); + return () => reportVoiceSessionState(null); + }, [reportVoiceSessionState]); + + return null; + }; + const aiAssistant = (mounted: boolean): PetrinautAiAssistant => ({ + renderVoiceMode: (context) => + mounted ? : null, + transport: { + reconnectToStream: () => Promise.resolve(null), + sendMessages: vi.fn(), + }, + }); + const rendered = renderTestPanel({ aiAssistant: aiAssistant(true) }); + + expect(screen.queryByRole("button", { name: "Your turn" })).toBeNull(); + fireEvent.click( + await screen.findByRole("button", { name: "Voice playback options" }), + ); + expect( + ( + await screen.findByRole("menuitem", { name: "Repeat question" }) + ).getAttribute("aria-disabled"), + ).toBe("true"); + expect( + screen + .getByRole("menuitem", { name: "Read full response" }) + .getAttribute("aria-disabled"), + ).toBe("true"); + + rendered.rerenderPanel(aiAssistant(false), editorContextValue); + + await waitFor(() => + expect( + screen.queryByRole("region", { name: "Voice session" }), + ).toBeNull(), + ); + }); + test("ends active Voice mode when the unified composer returns to text", () => { voiceModeMounts = 0; voiceModeUnmounts = 0; @@ -1195,6 +1388,10 @@ describe("AiAssistantPanel composer submissions", () => { fireEvent.click(screen.getByRole("button", { name: "Start voice mode" })); expect(screen.getByText("Voice mode voice")).not.toBeNull(); + expect(screen.queryByRole("region", { name: "Voice setup" })).toBeNull(); + expect( + screen.getByRole("textbox", { name: "Message AI assistant" }), + ).not.toBeNull(); fireEvent.click(screen.getByRole("button", { name: "Voice mode voice" })); expect( @@ -1206,7 +1403,7 @@ describe("AiAssistantPanel composer submissions", () => { expect(sendMessages).not.toHaveBeenCalled(); }); - test("defers and consumes an initial Voice mode once, then falls back to text", () => { + test("opens initial Voice setup compact once, then falls back to text", () => { let latestInputMode = "text"; const onInitialInteractionModeConsumed = vi.fn(); const aiAssistant: PetrinautAiAssistant = { @@ -1237,6 +1434,27 @@ describe("AiAssistantPanel composer submissions", () => { expect(latestInputMode).toBe("voice"); expect(onInitialInteractionModeConsumed).toHaveBeenCalledOnce(); + expect(screen.getByText("Voice mode")).not.toBeNull(); + const composer = screen.getByRole("textbox", { + hidden: true, + name: "Message AI assistant", + }); + const composerWrap = composer.closest("form")?.parentElement; + expect(composerWrap?.className).toContain("d_none"); + + const setupDock = screen.getByRole("region", { name: "Voice setup" }); + fireEvent.click( + within(setupDock).getByRole("button", { name: "Expand voice setup" }), + ); + + expect(screen.queryByRole("region", { name: "Voice setup" })).toBeNull(); + expect(composerWrap?.className).not.toContain("d_none"); + expect(screen.getByRole("textbox", { name: "Message AI assistant" })).toBe( + composer, + ); + expect( + screen.getByRole("button", { name: "Close AI assistant" }), + ).not.toBeNull(); const unavailableAssistant: PetrinautAiAssistant = { transport: aiAssistant.transport, @@ -1250,6 +1468,78 @@ describe("AiAssistantPanel composer submissions", () => { expect(onInitialInteractionModeConsumed).toHaveBeenCalledOnce(); }); + test("ends collapsed Voice and closes the panel without pausing", async () => { + const events: string[] = []; + const endVoice = vi.fn(async () => { + events.push("end"); + }); + const pauseVoice = vi.fn(() => events.push("pause")); + const setAiAssistantOpen = vi.fn(() => events.push("close")); + const VoiceMode = ({ + context, + }: { + context: PetrinautAiVoiceModeContext; + }) => { + const { + inputMode, + registerVoiceModeControls, + reportVoiceSessionState, + setVoiceActive, + } = context; + + useEffect( + () => + registerVoiceModeControls({ + end: endVoice, + pause: pauseVoice, + reconnect: vi.fn(), + resume: vi.fn(), + setMicrophoneMuted: vi.fn(), + }), + [registerVoiceModeControls], + ); + useEffect(() => { + if (inputMode !== "voice") { + return; + } + setVoiceActive(true); + reportVoiceSessionState({ + errorMessage: null, + microphoneLevel: 0, + microphoneMuted: false, + phase: "listening", + }); + }, [inputMode, reportVoiceSessionState, setVoiceActive]); + + return null; + }; + + renderTestPanel({ + aiAssistant: { + renderVoiceMode: (context) => , + transport: { + reconnectToStream: () => Promise.resolve(null), + sendMessages: vi.fn(), + }, + }, + editorContext: { + ...editorContextValue, + setAiAssistantOpen, + }, + initialInteractionMode: "voice", + }); + + const dock = await screen.findByRole("region", { name: "Voice session" }); + fireEvent.click( + within(dock).getByRole("button", { name: "End voice mode" }), + ); + + expect(events).toEqual(["end", "close"]); + expect(endVoice).toHaveBeenCalledOnce(); + expect(pauseVoice).not.toHaveBeenCalled(); + expect(setAiAssistantOpen).toHaveBeenCalledWith(false); + }); + test("accepts one voice input while generic chat is streaming and submits it after settlement", async () => { let firstStreamController: | ReadableStreamDefaultController @@ -1352,7 +1642,10 @@ describe("AiAssistantPanel composer submissions", () => { expect(latestVoiceContext?.status).toBe("streaming"); expect(latestVoiceContext?.canAcceptVoiceInput).toBe(true); expect(requests[1]?.at(-1)).toMatchObject({ - metadata: { source: "voice", toolCallId: "queued-question" }, + metadata: { + source: "voice", + voiceToolCallIds: ["queued-question"], + }, role: "assistant", }); expect( @@ -2347,6 +2640,51 @@ describe("AiAssistantPanel composer submissions", () => { ); }); + test("preserves the already-normalized Voice payload at the panel boundary", async () => { + const requestMessages: PetrinautAiMessage[][] = []; + const transport: PetrinautAiTransport = { + reconnectToStream: () => Promise.resolve(null), + sendMessages: vi.fn(({ messages }) => { + requestMessages.push(structuredClone(messages)); + return Promise.resolve( + streamChunks(textChunks("voice-response", "Voice message accepted")), + ); + }), + }; + + renderTestPanel({ + aiAssistant: { + renderComposerControl: ({ submitText }) => ( + + ), + transport, + }, + }); + + fireEvent.click( + screen.getByRole("button", { name: "Submit normalized Voice payload" }), + ); + await screen.findByText("Voice message accepted"); + + expect(requestMessages[0]?.at(-1)).toMatchObject({ + id: "voice-realtime:3:item-1:0", + metadata: { source: "voice" }, + parts: [{ text: " Already normalized upstream ", type: "text" }], + role: "user", + }); + }); + test("marks the exact pending tool as voice-origin without a user message", async () => { const requestMessages: PetrinautAiMessage[][] = []; const onMessages = vi.fn(); @@ -2425,7 +2763,10 @@ describe("AiAssistantPanel composer submissions", () => { ), ); expect(containingMessage).toMatchObject({ - metadata: { source: "voice", toolCallId: "question-voice" }, + metadata: { + source: "voice", + voiceToolCallIds: ["question-voice"], + }, }); expect( containingMessage?.parts.find( @@ -2449,12 +2790,139 @@ describe("AiAssistantPanel composer submissions", () => { expect(onMessages.mock.lastCall?.[0]).toEqual( expect.arrayContaining([ expect.objectContaining({ - metadata: { source: "voice", toolCallId: "question-voice" }, + metadata: { + source: "voice", + voiceToolCallIds: ["question-voice"], + }, }), ]), ); }); + test("retains every voice tool origin on one assistant message", async () => { + let latestMessages = [ + { + id: "assistant-voice-questions", + parts: [ + { + input: { question: "Who approves it?" }, + state: "input-available", + toolCallId: "voice-question-1", + toolName: "answerQuestion", + type: "dynamic-tool", + }, + { + input: { question: "Who acts next?" }, + state: "input-available", + toolCallId: "voice-question-2", + toolName: "answerQuestion", + type: "dynamic-tool", + }, + ], + role: "assistant", + }, + ] as unknown as PetrinautAiMessage[]; + const updateMessages = ( + updater: (messages: PetrinautAiMessage[]) => PetrinautAiMessage[], + ) => { + latestMessages = updater(latestMessages); + }; + const addToolOutput = vi.fn().mockResolvedValue(undefined); + + for (const toolCallId of ["voice-question-1", "voice-question-2"]) { + await addMappedToolOutput({ + addToolOutput, + currentMessages: latestMessages, + params: { + output: { answer: toolCallId }, + tool: "answerQuestion", + toolCallId, + }, + source: "voice", + updateMessages, + }); + } + + expect(latestMessages[0]?.metadata).toEqual({ + source: "voice", + voiceToolCallIds: ["voice-question-1", "voice-question-2"], + }); + }); + + test("preserves sibling voice provenance when another tool output rejects", async () => { + let latestMessages = [ + { + id: "assistant-voice-questions", + parts: [ + { + input: { question: "Who approves it?" }, + state: "input-available", + toolCallId: "voice-question-1", + toolName: "answerQuestion", + type: "dynamic-tool", + }, + { + input: { question: "Who acts next?" }, + state: "input-available", + toolCallId: "voice-question-2", + toolName: "answerQuestion", + type: "dynamic-tool", + }, + ], + role: "assistant", + }, + ] as unknown as PetrinautAiMessage[]; + const updateMessages = ( + updater: (messages: PetrinautAiMessage[]) => PetrinautAiMessage[], + ) => { + latestMessages = updater(latestMessages); + }; + let rejectFirstSubmission: ((reason?: unknown) => void) | undefined; + const addToolOutput = vi + .fn() + .mockImplementationOnce( + () => + new Promise((_resolve, reject) => { + rejectFirstSubmission = reject; + }), + ) + .mockResolvedValueOnce(undefined); + + const firstSubmission = addMappedToolOutput({ + addToolOutput, + currentMessages: latestMessages, + params: { + output: { answer: "The shift lead" }, + tool: "answerQuestion", + toolCallId: "voice-question-1", + }, + source: "voice", + updateMessages, + }); + const firstSubmissionRejection = expect(firstSubmission).rejects.toThrow( + "First voice tool output rejected.", + ); + + await addMappedToolOutput({ + addToolOutput, + currentMessages: latestMessages, + params: { + output: { answer: "The release manager" }, + tool: "answerQuestion", + toolCallId: "voice-question-2", + }, + source: "voice", + updateMessages, + }); + rejectFirstSubmission?.(new Error("First voice tool output rejected.")); + await firstSubmissionRejection; + + expect(latestMessages[0]?.metadata).toEqual({ + source: "voice", + voiceToolCallIds: ["voice-question-2"], + }); + }); + test("rolls back failed tool provenance before a typed retry", async () => { let latestMessages = [ { @@ -2529,6 +2997,34 @@ describe("AiAssistantPanel composer submissions", () => { expect(latestMessages[0]?.metadata).toBeUndefined(); }); + test("reports browser tool-output rejections through the AI SDK error state", async () => { + const addToolOutput = vi + .fn() + .mockRejectedValueOnce(new Error("The browser tool rejected its output.")) + .mockResolvedValueOnce(undefined); + + safelyAddToolOutput( + addToolOutput as Parameters[0], + { + tool: getLatestNetDefinitionToolName, + toolCallId: "tool-browser-failure", + output: { + definition: emptySDCPN, + extensions: DEFAULT_PETRINAUT_EXTENSIONS, + title: "Failure fixture", + }, + }, + ); + + await waitFor(() => expect(addToolOutput).toHaveBeenCalledTimes(2)); + expect(addToolOutput).toHaveBeenLastCalledWith({ + errorText: "The browser tool rejected its output.", + state: "output-error", + tool: getLatestNetDefinitionToolName, + toolCallId: "tool-browser-failure", + }); + }); + test("sends review chips as messages while an interactive tool is pending", async () => { const requestMessages: PetrinautAiMessage[][] = []; const transport: PetrinautAiTransport = { diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx index 6c515cc2adf..cf59be3487a 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx @@ -119,10 +119,30 @@ const markVoiceToolOrigin = ( ): PetrinautAiMessage[] => messages.map((message) => message.id === messageId - ? { - ...message, - metadata: { ...message.metadata, source: "voice", toolCallId }, - } + ? (() => { + const previousToolCallIds = + message.metadata?.source === "voice" + ? [ + ...(message.metadata.voiceToolCallIds ?? []), + ...(message.metadata.toolCallId + ? [message.metadata.toolCallId] + : []), + ] + : []; + const { toolCallId: _legacyToolCallId, ...previousMetadata } = + message.metadata ?? {}; + + return { + ...message, + metadata: { + ...previousMetadata, + source: "voice", + voiceToolCallIds: [ + ...new Set([...previousToolCallIds, toolCallId]), + ], + }, + }; + })() : message, ); @@ -135,7 +155,25 @@ const isPetrinautAiCommandToolName = ( toolName: string, ): toolName is AiCommandActionName => toolName in aiCommandActionInputSchemas; -const safelyAddToolOutput = ( +const browserToolErrorText = (error: unknown): string => { + if (error instanceof Error && error.message.trim().length > 0) { + return error.message; + } + if (typeof error === "string" && error.trim().length > 0) { + return error; + } + try { + const serialized: unknown = JSON.stringify(error); + if (typeof serialized === "string" && serialized.length > 0) { + return serialized; + } + } catch { + // Fall through to the stable fallback for cyclic values. + } + return "The browser tool failed."; +}; + +export const safelyAddToolOutput = ( addToolOutput: ReturnType< typeof useChat >["addToolOutput"], @@ -143,10 +181,16 @@ const safelyAddToolOutput = ( ReturnType>["addToolOutput"] >[0], ) => { - // Failures here surface in the UI as an errored tool call (with the - // error message on hover), so we just swallow the rejection to avoid an - // unhandled-promise warning. - void Promise.resolve(addToolOutput(params)).catch(() => {}); + void Promise.resolve(addToolOutput(params)).catch((error: unknown) => { + void Promise.resolve( + addToolOutput({ + errorText: browserToolErrorText(error), + state: "output-error", + tool: params.tool, + toolCallId: params.toolCallId, + }), + ).catch(() => {}); + }); }; const addDynamicToolOutput = ( @@ -211,8 +255,42 @@ export const addMappedToolOutput = async ({ latestMessages.map((message) => message.id === containingMessage.id && message.metadata?.source === "voice" && - message.metadata.toolCallId === params.toolCallId - ? { ...message, metadata: previousMetadata } + (message.metadata.voiceToolCallIds?.includes(params.toolCallId) === + true || + message.metadata.toolCallId === params.toolCallId) + ? (() => { + const attributionAlreadyPresent = + previousMetadata?.source === "voice" && + (previousMetadata.voiceToolCallIds?.includes( + params.toolCallId, + ) === true || + previousMetadata.toolCallId === params.toolCallId); + const voiceToolCallIds = [ + ...(message.metadata.voiceToolCallIds ?? []), + ...(message.metadata.toolCallId + ? [message.metadata.toolCallId] + : []), + ]; + const remainingVoiceToolCallIds = attributionAlreadyPresent + ? voiceToolCallIds + : voiceToolCallIds.filter( + (candidateToolCallId) => + candidateToolCallId !== params.toolCallId, + ); + if (remainingVoiceToolCallIds.length === 0) { + return { ...message, metadata: previousMetadata }; + } + const { toolCallId: _legacyToolCallId, ...metadata } = + message.metadata; + + return { + ...message, + metadata: { + ...metadata, + voiceToolCallIds: [...new Set(remainingVoiceToolCallIds)], + }, + }; + })() : message, ), ); @@ -342,12 +420,19 @@ export const AiAssistantPanel = ({ const [composerFocusRequest, setComposerFocusRequest] = useState(0); const [interactionMode, setInteractionMode] = useState("text"); + const [voiceDockCollapsed, setVoiceDockCollapsed] = useState(false); const interactionModeRef = useRef("text"); const selectInteractionMode = useCallback( - (nextMode: PetrinautAiInputMode) => { + ( + nextMode: PetrinautAiInputMode, + options: { collapseVoiceDock?: boolean } = {}, + ) => { const previousMode = interactionModeRef.current; interactionModeRef.current = nextMode; setInteractionMode(nextMode); + setVoiceDockCollapsed( + nextMode === "voice" && options.collapseVoiceDock === true, + ); if (previousMode === "voice" && nextMode === "text") { setComposerFocusRequest((request) => request + 1); } @@ -495,9 +580,16 @@ export const AiAssistantPanel = ({ // invalidates the host's active generation. end: () => requestInputMode("text"), pause: () => controls.pause(), + ...(controls.readFullResponse + ? { readFullResponse: () => controls.readFullResponse?.() } + : {}), reconnect: () => controls.reconnect(), + ...(controls.repeatQuestion + ? { repeatQuestion: () => controls.repeatQuestion?.() } + : {}), resume: () => controls.resume(), setMicrophoneMuted: (muted) => controls.setMicrophoneMuted(muted), + ...(controls.takeTurn ? { takeTurn: () => controls.takeTurn?.() } : {}), }); return () => { @@ -838,8 +930,8 @@ export const AiAssistantPanel = ({ target?: "auto" | "message"; text: string; }): Promise => { - const trimmed = text.trim(); - if (!trimmed) { + const submissionText = source === "voice" ? text : text.trim(); + if (!submissionText.trim()) { const submissionError = new Error( "AI assistant text must not be empty.", ); @@ -918,7 +1010,7 @@ export const AiAssistantPanel = ({ try { output = mappedToolCall.mapText({ input: mappedToolCall.input, - text: trimmed, + text: submissionText, }); } catch (caught) { const submissionError = @@ -972,7 +1064,7 @@ export const AiAssistantPanel = ({ await submitMessage({ id: messageId, ...(source === "voice" ? { metadata: { source } } : {}), - parts: [{ text: trimmed, type: "text" }], + parts: [{ text: submissionText, type: "text" }], role: "user", }); return { kind: "message", messageId }; @@ -1205,12 +1297,14 @@ export const AiAssistantPanel = ({ return; } - selectInteractionMode( + const nextMode = initialInteractionMode === "voice" && - aiAssistant.renderVoiceMode === undefined + aiAssistant.renderVoiceMode === undefined ? "text" - : initialInteractionMode, - ); + : initialInteractionMode; + selectInteractionMode(nextMode, { + collapseVoiceDock: nextMode === "voice", + }); consumedInitialInteractionModeRef.current = initialInteractionMode; onInitialInteractionModeConsumed?.(); }, [ @@ -1338,6 +1432,7 @@ export const AiAssistantPanel = ({ voiceModeControlsRef.current?.pause(); setAiAssistantOpen(false); }} + onCollapsedVoiceEnd={() => setAiAssistantOpen(false)} onInputChange={setInput} onInputModeChange={selectInteractionMode} onInteractiveToolSubmit={({ toolCallId, toolName, output }) => { @@ -1416,11 +1511,13 @@ export const AiAssistantPanel = ({ void stopComposer(); }} onSubmit={submitComposerInput} + onVoiceDockCollapsedChange={setVoiceDockCollapsed} promptChips={promptChips} rightOffset={hasSelection ? propertiesPanelWidth + PANEL_MARGIN : 0} status={status} stopped={stopped} voiceHandoffPending={voiceHandoffPending} + voiceDockCollapsed={voiceDockCollapsed} voiceMode={voiceMode} voiceModeAvailable={aiAssistant.renderVoiceMode !== undefined} /> diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.stories.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.stories.tsx index c21fe5af927..c6e9c98b7b3 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.stories.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.stories.tsx @@ -1,4 +1,5 @@ import { type ReactNode, useState } from "react"; +import { userEvent, within } from "storybook/test"; import { Button } from "@hashintel/ds-components"; import { css } from "@hashintel/ds-helpers/css"; @@ -244,6 +245,7 @@ const HostVoiceSlotPreview = () => ( const Frame = ({ error, + initialVoiceDockCollapsed = false, inputMode = "text", messages, status = "ready", @@ -253,6 +255,7 @@ const Frame = ({ voiceSession, }: { error?: Error; + initialVoiceDockCollapsed?: boolean; inputMode?: "text" | "voice"; messages: PetrinautAiMessage[]; status?: "submitted" | "streaming" | "ready" | "error"; @@ -262,6 +265,9 @@ const Frame = ({ voiceSession?: PetrinautAiVoiceSessionState; }) => { const [input, setInput] = useState(""); + const [voiceDockCollapsed, setVoiceDockCollapsed] = useState( + initialVoiceDockCollapsed, + ); // Stands in for the host, which reports session state rather than rendering // the live surfaces itself. const [voiceSessionStore] = useState(() => { @@ -291,8 +297,10 @@ const Frame = ({ onInputModeChange={() => {}} onStop={() => {}} onSubmit={() => setInput("")} + onVoiceDockCollapsedChange={setVoiceDockCollapsed} status={status} stopped={stopped} + voiceDockCollapsed={voiceDockCollapsed} voiceMode={voiceMode} voiceModeAvailable={voiceModeAvailable} /> @@ -330,6 +338,18 @@ export const VoiceModeAwaitingConsent: Story = { ), }; +export const VoiceModeAwaitingConsentCompact: Story = { + render: () => ( + } + voiceModeAvailable + /> + ), +}; + export const VoiceSessionListening: Story = { render: () => ( ( + } + voiceModeAvailable + voiceSession={liveSession({ microphoneLevel: 0.6 })} + /> + ), + play: async ({ canvasElement }) => { + await userEvent.click( + within(canvasElement).getByRole("button", { + name: "Collapse voice session", + }), + ); + }, +}; + export const VoiceSessionSpeaking: Story = { render: () => ( + ...singleToolCallMessage, + parts: singleToolCallMessage.parts.map((part) => part.type.startsWith("tool-") ? { ...part, diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx index ee2d2d8c6f7..025d3d6ca81 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx @@ -44,26 +44,52 @@ vi.mock("react-markdown", async (importOriginal) => { }); const noop = () => {}; +const initialClipboardDescriptor = Object.getOwnPropertyDescriptor( + navigator, + "clipboard", +); // The voice ribbon asks for a 2D context on mount. jsdom has no canvas, and // answering with `null` takes the same branch a browser without one would, // instead of letting jsdom log a not-implemented error per render. beforeAll(() => { vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue(null); + vi.stubGlobal( + "ResizeObserver", + class { + public disconnect() {} + public observe() {} + public unobserve() {} + }, + ); }); afterEach(() => { cleanup(); vi.clearAllMocks(); vi.useRealTimers(); + if (initialClipboardDescriptor === undefined) { + Reflect.deleteProperty(navigator, "clipboard"); + } else { + Object.defineProperty(navigator, "clipboard", initialClipboardDescriptor); + } }); describe("AiAssistantContents", () => { test("shows assistant errors as toasts instead of transcript messages", async () => { + const writeText = vi.fn().mockResolvedValue(undefined); + Object.defineProperty(navigator, "clipboard", { + configurable: true, + value: { writeText }, + }); render( { ); const toast = await waitFor(() => { - const element = document.querySelector( + const element = document.querySelector( '[data-scope="toast"][data-part="root"]', ); expect(element).not.toBeNull(); return element!; }); - expect(toast.textContent).toBe("Failed to fetch"); + expect( + toast.querySelector('[data-scope="toast"][data-part="title"]') + ?.textContent, + ).toBe("AI assistant error"); + expect( + toast.querySelector('[data-scope="toast"][data-part="description"]') + ?.textContent, + ).toBe( + 'Elicitor failed.\nCaused by: {"field":"answer","reason":"Required"}', + ); + fireEvent.click( + within(toast).getByRole("button", { name: "Copy details" }), + ); + expect(writeText).toHaveBeenCalledWith( + 'Elicitor failed.\nCaused by: {"field":"answer","reason":"Required"}', + ); expect( within(screen.getByTestId("ai-transcript")).queryByText( - "Failed to fetch", + "AI assistant error", ), ).toBeNull(); + fireEvent.click( + within(toast).getByRole("button", { name: "Close notification" }), + ); + await waitFor(() => + expect(toast.getAttribute("data-state")).toBe("closed"), + ); }); test("keeps one Voice mode slot mounted above the composer when the panel closes", () => { @@ -137,8 +184,9 @@ describe("AiAssistantContents", () => { expect(voiceModeUnmounts).toBe(0); }); - test("swaps the composer for the dock while a session runs, and defers its spoken turns", () => { + test("shows spoken turns live and collapses an active session without unmounting the panel", () => { const store = createVoiceSessionStore(); + const onCollapsedVoiceEnd = vi.fn(); const actions = { end: vi.fn(), pause: vi.fn(), @@ -160,21 +208,55 @@ describe("AiAssistantContents", () => { parts: [{ type: "text", text: "Earlier answer" }], }, ] as PetrinautAiMessage[]; - const renderWith = (messages: PetrinautAiMessage[]) => ( - - - - ); + const liveMessages = [ + ...earlierMessages, + { + id: "spoken-user", + metadata: { source: "voice" }, + role: "user", + parts: [{ type: "text", text: "Spoken request" }], + }, + { + id: "spoken-assistant", + role: "assistant", + parts: [{ type: "text", text: "Spoken reply" }], + }, + { + id: "typed-user", + role: "user", + parts: [{ type: "text", text: "Typed aside" }], + }, + ] as PetrinautAiMessage[]; + const VoiceContents = ({ + inputMode = "voice", + messages, + }: { + inputMode?: "text" | "voice"; + messages: PetrinautAiMessage[]; + }) => { + const [collapsed, setCollapsed] = useState(false); - const { rerender } = render(renderWith(earlierMessages)); + return ( + + Host Voice controls} + /> + + ); + }; + + const { rerender } = render(); const dock = screen.getByRole("region", { name: "Voice session" }); expect(within(dock).getByText("Listening")).not.toBeNull(); @@ -182,69 +264,139 @@ describe("AiAssistantContents", () => { screen.queryByRole("textbox", { name: "Message AI assistant" }), ).toBeNull(); expect(screen.getByText("Earlier answer")).not.toBeNull(); + expect( + within(dock) + .getByRole("button", { name: "Collapse voice session" }) + .getAttribute("aria-expanded"), + ).toBeNull(); - rerender( - renderWith([ - ...earlierMessages, - { - id: "spoken-user", - metadata: { source: "voice" }, - role: "user", - parts: [{ type: "text", text: "Spoken request" }], - }, - { - id: "spoken-assistant", - role: "assistant", - parts: [{ type: "text", text: "Spoken reply" }], - }, - { - id: "typed-user", - role: "user", - parts: [{ type: "text", text: "Typed aside" }], - }, - ] as PetrinautAiMessage[]), - ); + rerender(); - expect(screen.queryByText("Spoken request")).toBeNull(); - expect(screen.queryByText("Spoken reply")).toBeNull(); + expect(screen.getByText("Spoken request")).not.toBeNull(); + expect(screen.getByText("Spoken reply")).not.toBeNull(); expect(screen.getByText("Typed aside")).not.toBeNull(); - // The dock's transcription action writes the held turns into the chat - // mid-session, and holds them back again when it is turned off. + const transcript = screen.getByTestId("ai-transcript"); + const voiceMode = screen.getByTestId("ai-voice-mode"); + const header = screen + .getByRole("button", { name: "Close AI assistant" }) + .closest("div")!; + + fireEvent.click( + within(dock).getByRole("button", { name: "Collapse voice session" }), + ); + + expect(screen.getByTestId("ai-transcript")).toBe(transcript); + expect(screen.getByTestId("ai-voice-mode")).toBe(voiceMode); + expect( + screen + .getByRole("button", { name: "Close AI assistant", hidden: true }) + .closest("div"), + ).toBe(header); + expect(transcript.className).toContain("d_none"); + expect(voiceMode.className).toContain("d_none"); + expect(header.className).toContain("d_none"); + + fireEvent.click( + within(dock).getByRole("button", { name: "End voice mode" }), + ); + + expect(actions.end).toHaveBeenCalledOnce(); + expect(onCollapsedVoiceEnd).toHaveBeenCalledOnce(); + fireEvent.click( - within(dock).getByRole("button", { name: "Show transcription in chat" }), + within(dock).getByRole("button", { name: "Expand voice session" }), ); + + expect(transcript.className).not.toContain("d_none"); + expect(voiceMode.className).not.toContain("d_none"); + expect(header.className).not.toContain("d_none"); expect(screen.getByText("Spoken request")).not.toBeNull(); - expect(screen.getByText("Spoken reply")).not.toBeNull(); - expect(screen.queryByText("Voice session · 1 turn")).toBeNull(); fireEvent.click( - within(dock).getByRole("button", { name: "Hide transcription in chat" }), + within(dock).getByRole("button", { name: "End voice mode" }), ); - expect(screen.queryByText("Spoken request")).toBeNull(); + + expect(actions.end).toHaveBeenCalledTimes(2); + expect(onCollapsedVoiceEnd).toHaveBeenCalledOnce(); act(() => store.setState(null)); + rerender(); expect(screen.getByText("Spoken request")).not.toBeNull(); expect(screen.getByText("Spoken reply")).not.toBeNull(); - expect(screen.getByText("Voice session · 1 turn")).not.toBeNull(); expect(screen.queryByRole("region", { name: "Voice session" })).toBeNull(); expect( screen.getByRole("textbox", { name: "Message AI assistant" }), ).not.toBeNull(); }); - test("keeps the session's controls in the dock", () => { + test("stacks Voice setup above its compact dock while keeping the full panel mounted", () => { + const onVoiceDockCollapsedChange = vi.fn(); + render( + Permission + } + />, + ); + + const permission = screen.getByRole("region", { + name: "Voice mode consent", + }); + const setupDock = screen.getByRole("region", { name: "Voice setup" }); + expect(permission.parentElement?.nextElementSibling).toBe( + setupDock.parentElement, + ); + expect(screen.getByTestId("ai-transcript").className).toContain("d_none"); + expect( + screen + .getByRole("button", { name: "Close AI assistant", hidden: true }) + .closest("div")?.className, + ).toContain("d_none"); + expect( + screen.getByRole("textbox", { + hidden: true, + name: "Message AI assistant", + }), + ).not.toBeNull(); + + const expandButton = within(setupDock).getByRole("button", { + name: "Expand voice setup", + }); + expect(expandButton.getAttribute("aria-expanded")).toBeNull(); + fireEvent.click(expandButton); + + expect(onVoiceDockCollapsedChange).toHaveBeenCalledWith(false); + }); + + test("keeps handoff and canonical playback controls in the Voice dock", async () => { const store = createVoiceSessionStore(); const actions = { end: vi.fn(), pause: vi.fn(), + readFullResponse: vi.fn(), reconnect: vi.fn(), + repeatQuestion: vi.fn(), resume: vi.fn(), setMicrophoneMuted: vi.fn(), + takeTurn: vi.fn(), }; store.setActions(actions); store.setState({ + canReadFullResponse: true, + canRepeatQuestion: true, + canTakeTurn: true, errorMessage: null, microphoneLevel: 0.4, microphoneMuted: false, @@ -273,9 +425,45 @@ describe("AiAssistantContents", () => { fireEvent.click( within(dock).getByRole("button", { name: "End voice mode" }), ); + fireEvent.click(within(dock).getByRole("button", { name: "Your turn" })); expect(actions.setMicrophoneMuted).toHaveBeenCalledWith(true); expect(actions.end).toHaveBeenCalledOnce(); + expect(actions.takeTurn).toHaveBeenCalledOnce(); + + fireEvent.click( + within(dock).getByRole("button", { name: "Voice playback options" }), + ); + const repeatQuestion = await screen.findByRole("menuitem", { + name: "Repeat question", + }); + const repeatMenu = screen.getByRole("menu"); + fireEvent.keyDown(repeatMenu, { key: "ArrowDown" }); + await waitFor(() => + expect(repeatMenu.getAttribute("aria-activedescendant")).toBe( + repeatQuestion.id, + ), + ); + fireEvent.keyDown(repeatMenu, { key: "Enter" }); + await waitFor(() => expect(actions.repeatQuestion).toHaveBeenCalledOnce()); + + fireEvent.click( + within(dock).getByRole("button", { name: "Voice playback options" }), + ); + const readFullResponse = await screen.findByRole("menuitem", { + name: "Read full response", + }); + const fullResponseMenu = screen.getByRole("menu"); + fireEvent.keyDown(fullResponseMenu, { key: "End" }); + await waitFor(() => + expect(fullResponseMenu.getAttribute("aria-activedescendant")).toBe( + readFullResponse.id, + ), + ); + fireEvent.keyDown(fullResponseMenu, { key: "Enter" }); + await waitFor(() => + expect(actions.readFullResponse).toHaveBeenCalledOnce(), + ); act(() => { store.setState({ @@ -292,6 +480,20 @@ describe("AiAssistantContents", () => { ); expect(actions.setMicrophoneMuted).toHaveBeenLastCalledWith(false); + + act(() => { + store.setState({ + errorMessage: null, + microphoneLevel: 0, + microphoneMuted: false, + notice: "We didn't catch that. Please try again.", + phase: "listening", + }); + }); + expect( + within(dock).getAllByText("We didn't catch that. Please try again."), + ).not.toHaveLength(0); + expect(dock.getAttribute("data-voice-notice")).toBe("visible"); }); test("shows a voice recovery failure as a toast", async () => { @@ -319,15 +521,16 @@ describe("AiAssistantContents", () => { ); const toast = await waitFor(() => { - const element = document.querySelector( + const element = document.querySelector( '[data-scope="toast"][data-part="root"]', ); expect(element).not.toBeNull(); return element!; }); - expect(toast.textContent).toBe( - "Microphone unavailable. Check your browser permissions.", - ); + expect( + toast.querySelector('[data-scope="toast"][data-part="title"]') + ?.textContent, + ).toBe("Microphone unavailable. Check your browser permissions."); }); test("does not repeat a voice error toast until the session recovers", () => { @@ -554,7 +757,7 @@ describe("AiAssistantContents", () => { ).toBeNull(); }); - test("marks only the exact submitted interactive-tool answer named by voice metadata", () => { + test("marks every submitted interactive-tool answer named by voice metadata", () => { const hostTool = definePetrinautAiInteractiveTool({ toolName: "answerQuestion", inputSchema: { @@ -570,7 +773,10 @@ describe("AiAssistantContents", () => { const messages = [ { id: "assistant-questions", - metadata: { source: "voice", toolCallId: "question-voice" }, + metadata: { + source: "voice", + voiceToolCallIds: ["question-voice-1", "question-voice-2"], + }, role: "assistant", parts: [ { @@ -585,10 +791,18 @@ describe("AiAssistantContents", () => { type: "dynamic-tool", toolName: "answerQuestion", state: "output-available", - toolCallId: "question-voice", + toolCallId: "question-voice-1", input: { question: "Who approves it?" }, output: { answer: "The shift lead" }, }, + { + type: "dynamic-tool", + toolName: "answerQuestion", + state: "output-available", + toolCallId: "question-voice-2", + input: { question: "Who acts next?" }, + output: { answer: "The dispatcher" }, + }, ], }, ] as unknown as PetrinautAiMessage[]; @@ -606,13 +820,18 @@ describe("AiAssistantContents", () => { />, ); - expect( - within( - screen - .getByText("question-voice: The shift lead") - .closest("[data-tool-call-id]")!, - ).getByTestId("voice-input-provenance"), - ).not.toBeNull(); + for (const [toolCallId, answer] of [ + ["question-voice-1", "The shift lead"], + ["question-voice-2", "The dispatcher"], + ]) { + expect( + within( + screen + .getByText(`${toolCallId}: ${answer}`) + .closest("[data-tool-call-id]")!, + ).getByTestId("voice-input-provenance"), + ).not.toBeNull(); + } expect( within( screen @@ -620,7 +839,7 @@ describe("AiAssistantContents", () => { .closest("[data-tool-call-id]")!, ).queryByTestId("voice-input-provenance"), ).toBeNull(); - expect(screen.getAllByTestId("voice-input-provenance")).toHaveLength(1); + expect(screen.getAllByTestId("voice-input-provenance")).toHaveLength(2); expect(screen.queryByText("The shift lead", { exact: true })).toBeNull(); expect(container.querySelectorAll('[data-role="user"]')).toHaveLength(0); }); @@ -1594,7 +1813,7 @@ describe("AiAssistantContents", () => { expect(screen.getByRole("button", { name: /2 changes/u })).not.toBeNull(); }); - test("labels failed tool calls as errored", () => { + test("shows failed tool-call errors inline", () => { const messages: PetrinautAiMessage[] = [ { id: "assistant-1", @@ -1625,9 +1844,11 @@ describe("AiAssistantContents", () => { />, ); - expect( - screen.getByRole("button", { name: /deleteItemsByIds errored/u }), - ).not.toBeNull(); + const tool = screen.getByRole("button", { + name: /Validation failed.*deleteItemsByIds/u, + }); + expect(tool).not.toBeNull(); + expect(tool.getAttribute("title")).toBeNull(); }); test("expands deleted item summaries", () => { diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.tsx index 42cd71bc79e..e607bc34122 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.tsx @@ -1,5 +1,4 @@ import { - Fragment, memo, type ReactNode, type RefObject, @@ -14,8 +13,10 @@ import ReactMarkdown from "react-markdown"; import { Button, Icon } from "@hashintel/ds-components"; import { css, cva } from "@hashintel/ds-helpers/css"; -import { NotificationsContext } from "../../../../../react/notifications/context"; -import { VoiceSessionContext } from "../../../../../react/voice-session/context"; +import { + NotificationsContext, + type AddNotificationInput, +} from "../../../../../react/notifications/context"; import { useVoiceSessionErrorMessage, useVoiceSessionPhase, @@ -23,7 +24,7 @@ import { import { AiAssistantIcon } from "../../../../components/ai-assistant-icon"; import { ResizeHandle } from "../../../../resize/resize-handle"; import { AiVoiceModeIcon } from "../../components/ai-voice-mode-button"; -import { partitionVoiceSessionMessages } from "./ai-assistant-contents/defer-voice-messages"; +import { voiceSetupLabels } from "../../components/voice-session-labels"; import { aiFooterMinHeight } from "./ai-assistant-contents/footer-height"; import { getMessageRenderItems } from "./ai-assistant-contents/get-message-render-items"; import { @@ -36,7 +37,7 @@ import { AiAssistantToolList, type OnInteractiveToolSubmit, } from "./ai-assistant-contents/tool-list"; -import { LiveVoiceDock } from "./ai-assistant-contents/voice-dock"; +import { LiveVoiceDock, VoiceDock } from "./ai-assistant-contents/voice-dock"; import { VoiceInputProvenance } from "./ai-assistant-contents/voice-input-provenance"; import type { PetrinautAiInputMode } from "../../../../types/ai-assistant-composer-control"; @@ -48,6 +49,11 @@ type AiAssistantStatus = "submitted" | "streaming" | "ready" | "error"; const EMPTY_INTERACTIVE_TOOLS: readonly PetrinautAiInteractiveTool[] = []; +const errorNotification = ( + message: string, + detail?: string, +): AddNotificationInput => ({ detail, message, tone: "error" }); + export type AiAssistantContentsProps = { clearMessagesDisabled?: boolean; composerControl?: ReactNode; @@ -60,6 +66,7 @@ export type AiAssistantContentsProps = { messages: PetrinautAiMessage[]; onClearMessages?: () => void; onClose: () => void; + onCollapsedVoiceEnd?: () => void; onInputModeChange?: (mode: PetrinautAiInputMode) => void; onInputChange: (value: string) => void; onInteractiveToolSubmit?: OnInteractiveToolSubmit; @@ -67,11 +74,13 @@ export type AiAssistantContentsProps = { onSendPrompt?: (prompt: string) => void; onStop: () => void; onSubmit: () => void; + onVoiceDockCollapsedChange?: (collapsed: boolean) => void; promptChips?: PromptChip[]; rightOffset?: number; status: AiAssistantStatus; stopped?: boolean; voiceHandoffPending?: boolean; + voiceDockCollapsed?: boolean; voiceMode?: ReactNode; voiceModeAvailable?: boolean; }; @@ -90,6 +99,9 @@ const shellStyle = cva({ }, }, variants: { + collapsed: { + true: {}, + }, open: { true: { top: "0", @@ -117,6 +129,16 @@ const shellStyle = cva({ }, }, }, + compoundVariants: [ + { + collapsed: true, + open: true, + css: { + top: "[auto]", + height: "auto", + }, + }, + ], }); // Tracks the card's inset within the padded shell, so the resize handle @@ -261,40 +283,6 @@ const messageStyle = cva({ textAlign: "right", }, }, - // Spoken turns land in the transcript together once the session ends, so - // they arrive with a single entrance rather than appearing out of nowhere. - revealed: { - true: { - animationName: "[petrinautVoiceReveal]", - animationDuration: "[420ms]", - animationTimingFunction: "[cubic-bezier(0.22, 0.9, 0.3, 1)]", - "@media (prefers-reduced-motion: reduce)": { - animationName: "[none]", - }, - }, - }, - }, -}); - -const voiceSessionMetaStyle = css({ - display: "flex", - alignItems: "center", - gap: "2", - paddingX: "1", - color: "neutral.s90", - fontSize: "xs", - fontWeight: "medium", - _before: { - flex: "[1]", - height: "[1px]", - backgroundColor: "neutral.a30", - content: '""', - }, - _after: { - flex: "[1]", - height: "[1px]", - backgroundColor: "neutral.a30", - content: '""', }, }); @@ -441,12 +429,10 @@ const AiAssistantMessage = memo( handlersRef, interactiveTools, message, - revealed = false, }: { handlersRef: MessageHandlersRef; interactiveTools: readonly PetrinautAiInteractiveTool[]; message: PetrinautAiMessage; - revealed?: boolean; }) => { const role = message.role === "user" ? "user" : "assistant"; const renderItems = getMessageRenderItems(message, interactiveTools); @@ -459,7 +445,7 @@ const AiAssistantMessage = memo( return (
@@ -526,6 +512,7 @@ export const AiAssistantContents = ({ messages, onClearMessages, onClose, + onCollapsedVoiceEnd, onInputModeChange, onInputChange, onInteractiveToolSubmit, @@ -533,16 +520,17 @@ export const AiAssistantContents = ({ onSendPrompt, onStop, onSubmit, + onVoiceDockCollapsedChange, promptChips, rightOffset = 0, status, stopped = false, voiceHandoffPending = false, + voiceDockCollapsed = false, voiceMode, voiceModeAvailable = false, }: AiAssistantContentsProps) => { const { addNotification } = use(NotificationsContext); - const voiceSessionStore = use(VoiceSessionContext); const voiceSessionPhase = useVoiceSessionPhase(); const voiceSessionErrorMessage = useVoiceSessionErrorMessage(); const isVoiceSessionLive = voiceSessionPhase !== null; @@ -601,76 +589,8 @@ export const AiAssistantContents = ({ variant: "solid", }; - // Index of the first message belonging to the current or most recent voice - // session. Everything from here on is held back while that session runs, and - // revealed together once it ends. - const [sessionBaselineIndex, setSessionBaselineIndex] = useState< - number | null - >(() => - voiceSessionStore.getSnapshot().state === null ? null : messages.length, - ); - - // Off by default: the dock's transcription action writes spoken turns into - // the conversation as they land instead of holding them to the end. - const [transcriptionShown, setTranscriptionShown] = useState(false); - - const messageCountRef = useRef(messages.length); - useEffect(() => { - messageCountRef.current = messages.length; - }, [messages]); - - // Read from the store rather than from a render effect, so the baseline is - // captured on the event that starts the session instead of a render that - // happens to observe it. - useEffect(() => { - let wasLive = voiceSessionStore.getSnapshot().state !== null; - - return voiceSessionStore.subscribe(() => { - const isLive = voiceSessionStore.getSnapshot().state !== null; - if (isLive === wasLive) { - return; - } - wasLive = isLive; - - if (isLive) { - setSessionBaselineIndex(messageCountRef.current); - setTranscriptionShown(false); - } - }); - }, [voiceSessionStore]); - - const isHoldingVoiceTurns = isVoiceSessionLive && !transcriptionShown; - - const sessionPartition = - sessionBaselineIndex === null - ? null - : partitionVoiceSessionMessages({ - deferredFromIndex: sessionBaselineIndex, - interactiveTools, - messages, - }); - - const visibleMessages = - isHoldingVoiceTurns && sessionPartition !== null - ? sessionPartition.visible - : messages; - - // Held turns become "revealed" once they are let through — by the - // transcription action mid-session, or by the session ending — so they carry - // the entrance animation either way. - const revealedIds = new Set( - isHoldingVoiceTurns || sessionPartition === null - ? [] - : sessionPartition.deferred.map((message) => message.id), - ); - // The divider counts a finished session, so it waits for the session to end - // rather than growing a turn at a time under a live transcription. - const firstRevealedMessageId = isVoiceSessionLive - ? undefined - : visibleMessages.find((message) => revealedIds.has(message.id))?.id; - const revealedVoiceTurnCount = visibleMessages.filter( - (message) => revealedIds.has(message.id) && message.role === "user", - ).length; + const isVoiceDockCollapsed = + voiceDockCollapsed && (isVoiceSessionLive || inputMode === "voice"); const [assistantWidth, setAssistantWidth] = useState(defaultAssistantWidth); @@ -686,10 +606,7 @@ export const AiAssistantContents = ({ return; } notifiedErrorRef.current = error; - addNotification({ - message: error.message, - tone: "error", - }); + addNotification(errorNotification("AI assistant error", error.message)); }, [addNotification, error]); // Voice failures (microphone denied, connection dropped) are reported by the @@ -709,10 +626,7 @@ export const AiAssistantContents = ({ } notifiedVoiceErrorRef.current = voiceSessionErrorMessage; - addNotification({ - message: voiceSessionErrorMessage, - tone: "error", - }); + addNotification(errorNotification(voiceSessionErrorMessage)); }, [addNotification, voiceSessionErrorMessage, voiceSessionPhase]); const inputRef = useRef(null); @@ -810,7 +724,10 @@ export const AiAssistantContents = ({