diff --git a/.changeset/stable-composer-controls.md b/.changeset/stable-composer-controls.md index b6aa782f07b..a648d9c4a80 100644 --- a/.changeset/stable-composer-controls.md +++ b/.changeset/stable-composer-controls.md @@ -39,3 +39,11 @@ End Voice mode before submitting typed text exactly once through the shared comp draft if handoff fails. Pause active media before the AI panel closes and reopen the mounted session paused. Provisional transcription and Realtime audio remain ephemeral rather than becoming persisted chat history. + +Prepare concise spoken context within a strict 50-word budget while preserving Brunch's exact +protected question, which application code appends unchanged for tool-disabled audio rendering. +Fall back to canonical context and question whenever preparation is unavailable, invalid, or times +out. + +Add canonical replay controls for repeating the question or reading the full response without +speech preparation. diff --git a/apps/petrinaut-website/README.md b/apps/petrinaut-website/README.md index 958a1b0b8b2..2d47a9d81bc 100644 --- a/apps/petrinaut-website/README.md +++ b/apps/petrinaut-website/README.md @@ -97,12 +97,12 @@ bubble is replaced by the finalized message or pending-question tool output, which retains a waveform indicator without duplicating the answer. Provisional transcription and Realtime audio are not persisted as chat history. -The text composer remains available. Sending typed text ends Voice mode first, -then submits the draft exactly once through the same conversation; a failed -handoff restores the draft. Closing the assistant pauses capture and speech -before hiding it. Reopening preserves the mounted session in **Paused** state. -**Pause** and **End voice mode** live under **Voice mode actions**, while -**Resume** or **Reconnect** appears as the primary action when applicable. +The active session replaces the text composer with the Voice dock, and all +session controls are direct dock controls. Closing the assistant pauses capture +and speech before hiding it. Reopening preserves the mounted session in +**Paused** state. A compact **Voice playback options** menu beside the +transcription control provides **Repeat question** and **Read full response**; +both replay canonical Brunch content and bypass speech preparation. The browser sends its SDP offer to this app; the server initializes a trusted `gpt-realtime-2` audio-input/audio-output session through OpenAI's unified @@ -118,12 +118,19 @@ and durable history. The browser bridge accepts only the configured duplicate or stale calls, and submits the answer through Petrinaut's shared composer path with pending-`brunch_ask` correlation. -The bridge waits for the correlated Brunch turn before returning canonical -speech segments to Realtime. It then requests audio with tools disabled and -instructs Realtime to speak only those segments. Generated audio is not a -verbatim record: canonical Brunch text remains visible and authoritative. The -microphone stays active while the interviewer speaks and while Brunch is -working. Speaking over assistant audio interrupts playback automatically; +The experimental **Approach D** design waits for the correlated Brunch turn, +then gives Realtime two bounded roles. First, an out-of-band, text-only request +semantically prepares the canonical context within the portion of a strict +50-word spoken budget left after reserving the exact Brunch question. +Application code validates the prepared context and appends the protected +question exactly. Second, a +tool-disabled audio request renders only those supplied words verbatim. If +preparation is unavailable, invalid, or times out, the bridge supplies the +canonical context and question instead. Brunch's canonical transcript and +exact question are never rewritten and remain authoritative. Preparation, +provisional transcription, and Realtime audio are ephemeral and are not +persisted. The microphone stays active while the interviewer speaks and while +Brunch is working. Speaking over assistant audio interrupts playback automatically; WebRTC truncates provider-side unheard audio without changing Brunch history. The Brunch deployment must allow the website origin through its diff --git a/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.test.ts index dc12cda21c6..c520a70f822 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.test.ts @@ -5,6 +5,7 @@ import { ASK_TOOL_NAME } from "@hashintel/brunch-agent/client-tools"; import { hashCanonicalSpeechText, selectCanonicalSpeechSegments, + selectInterviewSpeech, } from "./canonical-speech"; import type { PetrinautAiMessage } from "@hashintel/petrinaut/ui"; @@ -13,6 +14,167 @@ const select = (messages: PetrinautAiMessage[]) => selectCanonicalSpeechSegments(messages); describe("canonical speech selection", () => { + test("separates finalized context from an exact pending question", () => { + const completeCanonicalExplanation = + "The release needs one named approver before the batch can proceed."; + const question = "Who approves it: the manager or quality lead?"; + const messages = [ + { + id: "assistant-turn", + role: "assistant", + parts: [ + { + type: "text", + text: completeCanonicalExplanation, + state: "done", + }, + { + type: "dynamic-tool", + toolCallId: "ask-1", + toolName: ASK_TOOL_NAME, + state: "input-available", + input: { question }, + }, + ], + }, + ] satisfies PetrinautAiMessage[]; + + const selection = selectInterviewSpeech(messages); + + expect(selection.automaticSource).toMatchObject({ + contextSegments: [ + { source: "assistant-text", text: completeCanonicalExplanation }, + ], + messageId: "assistant-turn", + questionSegment: { source: "brunch-ask", text: question }, + fullResponseSegments: [ + { source: "assistant-text", text: completeCanonicalExplanation }, + { source: "brunch-ask", text: question }, + ], + }); + expect(selection.canonicalSegments).toEqual( + selection.automaticSource?.fullResponseSegments, + ); + }); + + test("groups multiple finalized text parts into the latest assistant turn", () => { + const selection = selectInterviewSpeech([ + { + id: "assistant-old", + role: "assistant", + parts: [{ type: "text", text: "Previous turn.", state: "done" }], + }, + { + id: "assistant-latest", + role: "assistant", + parts: [ + { type: "text", text: "First explanation.", state: "done" }, + { type: "text", text: "Second explanation.", state: "done" }, + ], + }, + ]); + + expect(selection.automaticSource).toMatchObject({ + contextSegments: [ + { text: "First explanation." }, + { text: "Second explanation." }, + ], + fullResponseSegments: [ + { text: "First explanation." }, + { text: "Second explanation." }, + ], + messageId: "assistant-latest", + questionSegment: null, + }); + expect(selection.canonicalSegments).toHaveLength(3); + }); + + test("selects standalone finalized assistant text for automatic speech", () => { + const selection = selectInterviewSpeech([ + { + id: "assistant-complete", + role: "assistant", + parts: [ + { type: "text", text: "The interview is complete.", state: "done" }, + ], + }, + ]); + + expect(selection.automaticSource).toMatchObject({ + contextSegments: [{ text: "The interview is complete." }], + fullResponseSegments: [{ text: "The interview is complete." }], + questionSegment: null, + }); + }); + + test("excludes answered and malformed asks from the protected question", () => { + const selection = selectInterviewSpeech([ + { + id: "assistant-ask", + role: "assistant", + parts: [ + { type: "text", text: "Canonical context.", state: "done" }, + { + type: "dynamic-tool", + toolCallId: "ask-answered", + toolName: ASK_TOOL_NAME, + state: "output-available", + input: { question: "Already answered?" }, + output: { answer: "Yes." }, + }, + { + type: "dynamic-tool", + toolCallId: "ask-malformed", + toolName: ASK_TOOL_NAME, + state: "input-available", + input: { question: 42 }, + }, + ], + }, + ]); + + expect(selection.automaticSource).toBeNull(); + expect(selection.canonicalSegments).toHaveLength(1); + }); + + test("excludes streaming text, reasoning, diagnostics, tool output, and tool errors", () => { + const selection = selectInterviewSpeech([ + { + id: "assistant-filtered", + role: "assistant", + parts: [ + { + type: "text", + text: "Do not speak streaming text.", + state: "streaming", + }, + { type: "reasoning", text: "Do not speak reasoning.", state: "done" }, + { + type: "dynamic-tool", + toolCallId: "tool-output", + toolName: "diagnostic", + state: "output-available", + input: {}, + output: { text: "Do not speak tool output." }, + }, + { + type: "dynamic-tool", + toolCallId: "tool-error", + toolName: "diagnostic", + state: "output-error", + input: {}, + errorText: "Do not speak tool errors.", + }, + ], + }, + ]); + + expect(selection).toEqual({ + automaticSource: null, + canonicalSegments: [], + }); + }); + test("selects only finalized assistant text without changing it", () => { const messages = [ { @@ -174,5 +336,22 @@ describe("canonical speech selection", () => { expect(changed[0]?.partId).toBe(first[0]?.partId); expect(changed[0]?.contentHash).not.toBe(first[0]?.contentHash); expect(changed[0]?.id).not.toBe(first[0]?.id); + + const firstSelection = selectInterviewSpeech([ + { + id: "assistant/id", + role: "assistant", + parts: [{ type: "text", text: "Exact text", state: "done" }], + }, + ]); + expect( + selectInterviewSpeech([ + { + id: "assistant/id", + role: "assistant", + parts: [{ type: "text", text: "Exact text", state: "done" }], + }, + ]), + ).toEqual(firstSelection); }); }); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.ts b/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.ts index bb111385b7e..7252cef8569 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.ts @@ -18,6 +18,18 @@ export interface CanonicalSpeechSegment { readonly text: string; } +export interface InterviewSpeechSource { + readonly contextSegments: readonly CanonicalSpeechSegment[]; + readonly fullResponseSegments: readonly CanonicalSpeechSegment[]; + readonly messageId: string; + readonly questionSegment: CanonicalSpeechSegment | null; +} + +export interface InterviewSpeechSelection { + readonly automaticSource: InterviewSpeechSource | null; + readonly canonicalSegments: readonly CanonicalSpeechSegment[]; +} + const createSegment = ( messageId: string, partId: string, @@ -40,56 +52,83 @@ const createSegment = ( }; }; -export const selectCanonicalSpeechSegments = ( +export const selectInterviewSpeech = ( messages: PetrinautAiMessage[], -): CanonicalSpeechSegment[] => { - const segments: CanonicalSpeechSegment[] = []; +): InterviewSpeechSelection => { + const canonicalSegments: CanonicalSpeechSegment[] = []; + let automaticSource: InterviewSpeechSource | null = null; for (const message of messages) { if (message.role !== "assistant") { continue; } + const contextSegments: CanonicalSpeechSegment[] = []; + const questionSegments: CanonicalSpeechSegment[] = []; + let hasAskPart = false; + for (const [partIndex, part] of message.parts.entries()) { if ( part.type === "text" && part.state !== "streaming" && part.text.trim() ) { - segments.push( - createSegment( - message.id, - `text:${partIndex}`, - "assistant-text", - part.text, - ), + const segment = createSegment( + message.id, + `text:${partIndex}`, + "assistant-text", + part.text, ); + canonicalSegments.push(segment); + contextSegments.push(segment); continue; } - if ( - part.type !== "dynamic-tool" || - part.toolName !== ASK_TOOL_NAME || - part.state !== "input-available" - ) { + if (part.type !== "dynamic-tool" || part.toolName !== ASK_TOOL_NAME) { + continue; + } + hasAskPart = true; + if (part.state !== "input-available") { continue; } try { const input = parseBrunchAskInput(part.input); - segments.push( - createSegment( - message.id, - part.toolCallId, - "brunch-ask", - input.question, - ), + const segment = createSegment( + message.id, + part.toolCallId, + "brunch-ask", + input.question, ); + canonicalSegments.push(segment); + questionSegments.push(segment); } catch { // Malformed tool inputs remain visible as tool errors; they are not spoken. } } + + const questionSegment = questionSegments.at(-1) ?? null; + if (contextSegments.length > 0 || hasAskPart) { + automaticSource = + questionSegment || !hasAskPart + ? { + contextSegments, + fullResponseSegments: [ + ...contextSegments, + ...(questionSegment ? [questionSegment] : []), + ], + messageId: message.id, + questionSegment, + } + : null; + } } - return segments; + return { automaticSource, canonicalSegments }; }; + +export const selectCanonicalSpeechSegments = ( + messages: PetrinautAiMessage[], +): CanonicalSpeechSegment[] => [ + ...selectInterviewSpeech(messages).canonicalSegments, +]; diff --git a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts index 2e486daae5d..3798423848c 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts @@ -2,6 +2,7 @@ import { afterEach, describe, expect, test, vi } from "vitest"; import { OpenAIRealtimeSession, + type InterviewSpeechPreparationRequest, type OpenAIRealtimeSessionEvent, } from "./openai-realtime-session"; @@ -40,6 +41,16 @@ const canonicalSegment = ( text, }); +const preparationRequest = ( + overrides: Partial = {}, +): InterviewSpeechPreparationRequest => ({ + cacheKey: "preparation-cache-key", + contextText: ["The batch requires a named approver before release."], + contextWordBudget: 12, + sourceSegmentIds: ["context-segment-1"], + ...overrides, +}); + const createHarness = ({ connectionTimeoutMs = 15_000, }: { @@ -329,9 +340,7 @@ describe("OpenAIRealtimeSession", () => { }, ]); - harness.session.completeFunctionCall("call-1", [ - canonicalSegment("ask-2", "Who acts next?"), - ]); + harness.session.completeFunctionCall("call-1", ["Who acts next?"]); const [functionOutput, responseCreate] = sentEvents(channel).slice(-2); expect(functionOutput).toEqual({ type: "conversation.item.create", @@ -352,6 +361,58 @@ describe("OpenAIRealtimeSession", () => { tools: [], }, }); + + const responseCreateCount = sentEvents(channel).filter( + ({ type }) => type === "response.create", + ).length; + harness.session.completeFunctionCall( + "call-2", + ["Response cancelled before speech."], + { speakResponse: false }, + ); + expect(sentEvents(channel).at(-1)).toEqual({ + type: "conversation.item.create", + item: { + type: "function_call_output", + call_id: "call-2", + output: JSON.stringify({ + response_text: ["Response cancelled before speech."], + }), + }, + }); + expect( + sentEvents(channel).filter(({ type }) => type === "response.create"), + ).toHaveLength(responseCreateCount); + }); + + test("renders prepared strings through the verbatim out-of-band audio response", async () => { + const harness = createHarness(); + await harness.session.connect(); + + harness.session.speakPrepared([ + "Prepared concise context.", + "Which operator confirms the batch?", + ]); + + const responseCreate = sentEvents(harness.channels[0]!).at(-1)!; + expect(responseCreate).toMatchObject({ + type: "response.create", + response: { + conversation: "none", + output_modalities: ["audio"], + tool_choice: "none", + tools: [], + }, + }); + const response = responseCreate.response as { + input: Array<{ content: Array<{ text: string }> }>; + }; + expect(JSON.parse(response.input[0]!.content[0]!.text)).toEqual({ + response_text: [ + "Prepared concise context.", + "Which operator confirms the batch?", + ], + }); }); test("queues canonical speech behind an active Realtime response", async () => { @@ -392,6 +453,358 @@ describe("OpenAIRealtimeSession", () => { }); }); + test("prepares context with an out-of-band text-only response", async () => { + const harness = createHarness(); + await harness.session.connect(); + const channel = harness.channels[0]!; + const request = preparationRequest({ + contextText: [ + "The batch requires a named approver.", + "Release remains blocked until approval.", + ], + sourceSegmentIds: ["message-1:text-1", "message-1:text-2"], + }); + + const preparation = harness.session.prepareInterviewSpeech(request); + const responseCreate = sentEvents(channel).at(-1)!; + expect(responseCreate).toMatchObject({ + type: "response.create", + response: { + conversation: "none", + max_output_tokens: 120, + metadata: { + petrinaut_kind: "speech-preparation", + petrinaut_request_id: request.cacheKey, + }, + output_modalities: ["text"], + parallel_tool_calls: false, + tool_choice: "none", + tools: [], + }, + }); + const response = responseCreate.response as { + input: Array<{ content: Array<{ text: string }> }>; + }; + expect(JSON.parse(response.input[0]!.content[0]!.text)).toEqual({ + context_text: request.contextText, + maximum_words: request.contextWordBudget, + }); + expect(response.input[0]!.content[0]!.text).not.toContain("message-1"); + expect(response.input[0]!.content[0]!.text).not.toContain("fnv1a32"); + expect(response.input[0]!.content[0]!.text).not.toContain("question"); + + channel.receive({ + response: { + id: "response-preparation", + metadata: (responseCreate.response as Record).metadata, + }, + type: "response.created", + }); + channel.receive({ + delta: "Ignore unrelated output.", + response_id: "response-unrelated", + type: "response.output_text.delta", + }); + channel.receive({ + delta: "Approval is required before release.", + response_id: "response-preparation", + type: "response.output_text.delta", + }); + channel.receive({ + response: { + id: "response-preparation", + output: [], + status: "completed", + }, + type: "response.done", + }); + + await expect(preparation).resolves.toEqual({ + context: "Approval is required before release.", + kind: "prepared", + sourceSegmentIds: request.sourceSegmentIds, + }); + expect(harness.reportDiagnostic).toHaveBeenLastCalledWith({ + durationMs: 0, + inputWordCount: 11, + operation: "preparation", + outcome: "success", + outputWordCount: 5, + requestId: "voice-request-2", + sourceSegmentCount: 2, + stage: "browser", + }); + }); + + test.each([ + ["empty output", "", 12], + ["over-budget output", "one two three", 2], + ["over-400-character output", "word ".repeat(81), 100], + ["Markdown heading", "# Approval is required", 12], + ["Markdown list", "- Approval is required", 12], + ["JSON fence", "```json\n{}\n```", 12], + ["tool syntax", 'continue_interview({"answer":"yes"})', 12], + ])("falls back for invalid prepared %s", async (_label, output, budget) => { + const harness = createHarness(); + await harness.session.connect(); + const channel = harness.channels[0]!; + const request = preparationRequest({ contextWordBudget: budget }); + const preparation = harness.session.prepareInterviewSpeech(request); + const responseCreate = sentEvents(channel).at(-1)!; + channel.receive({ + response: { + id: "response-preparation", + metadata: (responseCreate.response as Record).metadata, + }, + type: "response.created", + }); + if (output) { + channel.receive({ + delta: output, + response_id: "response-preparation", + type: "response.output_text.delta", + }); + } + channel.receive({ + response: { + id: "response-preparation", + output: [], + status: "completed", + }, + type: "response.done", + }); + + await expect(preparation).resolves.toEqual({ + kind: "fallback", + reason: "invalid-output", + sourceSegmentIds: request.sourceSegmentIds, + }); + }); + + test("falls back without closing the session after a correlated provider error", async () => { + const harness = createHarness(); + await harness.session.connect(); + const channel = harness.channels[0]!; + const request = preparationRequest(); + const preparation = harness.session.prepareInterviewSpeech(request); + const responseCreate = sentEvents(channel).at(-1)!; + + channel.receive({ + error: { + event_id: responseCreate.event_id, + message: "private provider detail", + type: "server_error", + }, + type: "error", + }); + + await expect(preparation).resolves.toEqual({ + kind: "fallback", + reason: "provider-error", + sourceSegmentIds: request.sourceSegmentIds, + }); + expect(harness.localTracks[0]!.stop).not.toHaveBeenCalled(); + expect(JSON.stringify(harness.events)).not.toContain( + "private provider detail", + ); + }); + + test("interrupts preparation and ignores its late completion", async () => { + const harness = createHarness(); + await harness.session.connect(); + const channel = harness.channels[0]!; + const request = preparationRequest(); + const preparation = harness.session.prepareInterviewSpeech(request); + const responseCreate = sentEvents(channel).at(-1)!; + channel.receive({ + response: { + id: "response-preparation", + metadata: (responseCreate.response as Record).metadata, + }, + type: "response.created", + }); + + harness.session.cancelOutput(); + + await expect(preparation).resolves.toEqual({ + kind: "fallback", + reason: "interrupted", + sourceSegmentIds: request.sourceSegmentIds, + }); + expect(sentEvents(channel).slice(-1)[0]).toMatchObject({ + response_id: "response-preparation", + type: "response.cancel", + }); + channel.receive({ + delta: "Late output must be ignored.", + response_id: "response-preparation", + type: "response.output_text.delta", + }); + channel.receive({ + response: { + id: "response-preparation", + output: [], + status: "cancelled", + }, + type: "response.done", + }); + expect(harness.localTracks[0]!.stop).not.toHaveBeenCalled(); + }); + + test("times out preparation after two seconds and cancels late provider work", async () => { + vi.useFakeTimers(); + const harness = createHarness(); + const connection = harness.session.connect(); + await vi.runAllTimersAsync(); + await connection; + const channel = harness.channels[0]!; + const request = preparationRequest(); + const preparation = harness.session.prepareInterviewSpeech(request); + const responseCreate = sentEvents(channel).at(-1)!; + channel.receive({ + response: { + id: "response-preparation", + metadata: (responseCreate.response as Record).metadata, + }, + type: "response.created", + }); + + await vi.advanceTimersByTimeAsync(2_000); + + await expect(preparation).resolves.toEqual({ + kind: "fallback", + reason: "timeout", + sourceSegmentIds: request.sourceSegmentIds, + }); + expect(sentEvents(channel).slice(-1)[0]).toMatchObject({ + response_id: "response-preparation", + type: "response.cancel", + }); + channel.receive({ + delta: "Late output must be ignored.", + response_id: "response-preparation", + type: "response.output_text.delta", + }); + channel.receive({ + response: { + id: "response-preparation", + output: [], + status: "cancelled", + }, + type: "response.done", + }); + expect(harness.localTracks[0]!.stop).not.toHaveBeenCalled(); + }); + + test("releases a timed-out preparation create so canonical speech can send", async () => { + vi.useFakeTimers(); + const harness = createHarness(); + const connection = harness.session.connect(); + await vi.runAllTimersAsync(); + await connection; + const channel = harness.channels[0]!; + const request = preparationRequest(); + const preparation = harness.session.prepareInterviewSpeech(request); + const preparationCreate = sentEvents(channel).at(-1)!; + + await vi.advanceTimersByTimeAsync(2_000); + await expect(preparation).resolves.toMatchObject({ + kind: "fallback", + reason: "timeout", + }); + harness.session.speakCanonical([ + canonicalSegment("question-after-timeout", "What happens next?"), + ]); + + const responseCreates = sentEvents(channel).filter( + ({ type }) => type === "response.create", + ); + expect(responseCreates).toHaveLength(2); + expect(responseCreates[1]).toMatchObject({ + response: { metadata: { petrinaut_kind: "canonical-speech" } }, + }); + + channel.receive({ + response: { + id: "late-preparation-response", + metadata: (preparationCreate.response as Record) + .metadata, + }, + type: "response.created", + }); + expect(sentEvents(channel)).toContainEqual( + expect.objectContaining({ + response_id: "late-preparation-response", + type: "response.cancel", + }), + ); + expect(harness.localTracks[0]!.stop).not.toHaveBeenCalled(); + }); + + test("rejects preparation metadata for a non-current pending request", async () => { + vi.useFakeTimers(); + const harness = createHarness(); + const connection = harness.session.connect(); + await vi.runAllTimersAsync(); + await connection; + const channel = harness.channels[0]!; + const firstRequest = preparationRequest({ cacheKey: "first-request" }); + const secondRequest = preparationRequest({ cacheKey: "second-request" }); + const firstPreparation = + harness.session.prepareInterviewSpeech(firstRequest); + const secondPreparation = + harness.session.prepareInterviewSpeech(secondRequest); + + channel.receive({ + response: { + id: "mismatched-response", + metadata: { + petrinaut_kind: "speech-preparation", + petrinaut_request_id: secondRequest.cacheKey, + }, + }, + type: "response.created", + }); + channel.receive({ + delta: "This output must not be accepted.", + response_id: "mismatched-response", + type: "response.output_text.delta", + }); + channel.receive({ + response: { + id: "mismatched-response", + output: [], + status: "completed", + }, + type: "response.done", + }); + + await vi.advanceTimersByTimeAsync(2_000); + await expect(firstPreparation).resolves.toMatchObject({ + reason: "timeout", + }); + await expect(secondPreparation).resolves.toMatchObject({ + reason: "timeout", + }); + expect(harness.localTracks[0]!.stop).not.toHaveBeenCalled(); + }); + + test("settles active preparation during disconnect cleanup", async () => { + const harness = createHarness(); + await harness.session.connect(); + const request = preparationRequest(); + const preparation = harness.session.prepareInterviewSpeech(request); + + await harness.session.disconnect(); + + await expect(preparation).resolves.toEqual({ + kind: "fallback", + reason: "interrupted", + sourceSegmentIds: request.sourceSegmentIds, + }); + expect(harness.localTracks[0]!.stop).toHaveBeenCalledOnce(); + }); + test("cancels canonical speech before the response starts", async () => { const harness = createHarness(); await harness.session.connect(); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts index f7c13d9c28c..0475308cf52 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts @@ -112,11 +112,61 @@ interface RequestTiming { readonly startedAt: number; } +export interface InterviewSpeechPreparationRequest { + readonly cacheKey: string; + readonly contextText: readonly string[]; + readonly contextWordBudget: number; + readonly sourceSegmentIds: readonly string[]; +} + +export type InterviewSpeechPreparationResult = + | { + readonly context: string; + readonly kind: "prepared"; + readonly sourceSegmentIds: readonly string[]; + } + | { + readonly kind: "fallback"; + readonly reason: + | "empty-context" + | "invalid-output" + | "provider-error" + | "timeout" + | "interrupted"; + readonly sourceSegmentIds: readonly string[]; + }; + +type InterviewSpeechPreparationFallbackReason = Extract< + InterviewSpeechPreparationResult, + { kind: "fallback" } +>["reason"]; + interface CanonicalSpeechRequest { + readonly kind: "canonical-speech"; readonly response: Record; readonly speechRequestId: string; } +interface InterviewPreparationResponseRequest { + readonly kind: "speech-preparation"; + readonly preparationRequestId: string; + readonly response: Record; +} + +type SerializedResponseRequest = + | CanonicalSpeechRequest + | InterviewPreparationResponseRequest; + +interface PendingInterviewPreparation { + readonly inputWordCount: number; + readonly outputChunks: string[]; + readonly request: InterviewSpeechPreparationRequest; + readonly requestId: string; + readonly resolve: (result: InterviewSpeechPreparationResult) => void; + readonly startedAt: number; + readonly timeout: ReturnType; +} + type PendingClientEvent = | { readonly kind: "response-cancel"; @@ -124,7 +174,7 @@ type PendingClientEvent = } | { readonly kind: "response-create"; - readonly request: CanonicalSpeechRequest; + readonly request: SerializedResponseRequest; readonly responseTerminalSequence: number; }; @@ -132,7 +182,20 @@ type SessionListener = (event: OpenAIRealtimeSessionEvent) => void; const CANONICAL_RESPONSE_INSTRUCTIONS = "Speak only the response_text strings supplied by Petrinaut, in array order and verbatim. Deliver them as a warm, calm, curious, confident, concise, and professionally neutral expert interviewer, at a measured conversational pace with natural emphasis. Never sound robotic, fawning, rushed, overenthusiastic, or patronizing. Do not add, remove, paraphrase, acknowledge, or explain anything."; +const INTERVIEW_PREPARATION_INSTRUCTIONS = [ + "Rewrite the supplied context as a concise spoken interviewer response.", + "Output plain text only.", + "Use one or two short sentences.", + "Do not ask a question.", + "Do not add facts, promises, praise, choices, or next steps.", + "Preserve explicit negations, warnings, commitments, and required actions.", + "Stay within the supplied maximum word count.", +].join(" "); const MAX_CANONICAL_SEGMENTS = 64; +const MAX_PREPARED_CONTEXT_CHARACTERS = 400; +const PREPARATION_TIMEOUT_MS = 2_000; +const INVALID_PREPARED_CONTEXT_SYNTAX = + /(^|\n)\s*(?:#{1,6}\s|[-*+]\s|\d+[.)]\s)|```(?:json)?|\b(?:brunch_ask|continue_interview)\s*\(| | null => typeof value === "object" && value !== null && !Array.isArray(value) @@ -145,6 +208,11 @@ const nonEmptyString = (value: unknown): string | null => const nonNegativeInteger = (value: unknown): number | null => Number.isInteger(value) && (value as number) >= 0 ? (value as number) : null; +const countWords = (text: string): number => { + const trimmed = text.trim(); + return trimmed ? trimmed.split(/\s+/u).length : 0; +}; + const parseRealtimeEvent = (value: unknown): Record | null => { if (typeof value !== "string") { return null; @@ -191,13 +259,19 @@ export class OpenAIRealtimeSession { readonly #listeners = new Set(); readonly #authorizedResponseIds = new Set(); readonly #cancelledCanonicalResponseIds = new Set(); + readonly #cancelledPreparationRequestIds = new Set(); readonly #cancelledSpeechRequestIds = new Set(); readonly #canonicalResponseIds = new Set(); - readonly #canonicalSpeechQueue: CanonicalSpeechRequest[] = []; readonly #completedResponseCancelEventIds = new Set(); readonly #pendingClientEvents = new Map(); + readonly #pendingPreparations = new Map< + string, + PendingInterviewPreparation + >(); readonly #pendingSpeechRequests = new Map(); + readonly #preparationResponseIds = new Map(); readonly #remoteStreams = new Set(); + readonly #responseQueue: SerializedResponseRequest[] = []; readonly #speechTimings = new Map(); readonly #transcriptionTimings = new Map(); #abortController: AbortController | null = null; @@ -399,17 +473,26 @@ export class OpenAIRealtimeSession { } public speakCanonical(segments: CanonicalSpeechSegment[]): void { - this.#requestCanonicalSpeech(segments, true); + this.#requestSpeech(this.#canonicalResponseText(segments), true); + } + + public speakPrepared(responseText: readonly string[]): void { + this.#requestSpeech(this.#validResponseText(responseText), true); } public completeFunctionCall( callId: string, - segments: CanonicalSpeechSegment[], + responseTextInput: readonly string[], + { + speakResponse = true, + }: { + readonly speakResponse?: boolean; + } = {}, ): void { if (!callId) { throw new VoiceError("speech", "invalid-response", ""); } - const responseText = this.#canonicalResponseText(segments); + const responseText = this.#validResponseText(responseTextInput); this.#send({ type: "conversation.item.create", item: { @@ -418,7 +501,103 @@ export class OpenAIRealtimeSession { output: JSON.stringify({ response_text: responseText }), }, }); - this.#requestCanonicalSpeech(segments, false); + if (speakResponse) { + this.#requestSpeech(responseText, false); + } + } + + public prepareInterviewSpeech( + request: InterviewSpeechPreparationRequest, + ): Promise { + const sourceSegmentIds = [...request.sourceSegmentIds]; + const contextText = request.contextText.filter((text) => text.trim()); + if (contextText.length === 0) { + return Promise.resolve({ + kind: "fallback", + reason: "empty-context", + sourceSegmentIds, + }); + } + if (request.contextWordBudget === 0) { + return Promise.resolve({ + context: "", + kind: "prepared", + sourceSegmentIds, + }); + } + if ( + !request.cacheKey || + !Number.isInteger(request.contextWordBudget) || + request.contextWordBudget < 0 || + this.#pendingPreparations.has(request.cacheKey) + ) { + return Promise.resolve({ + kind: "fallback", + reason: "invalid-output", + sourceSegmentIds, + }); + } + + return new Promise((resolve) => { + const timeout = globalThis.setTimeout( + () => this.#timeoutPreparation(request.cacheKey), + PREPARATION_TIMEOUT_MS, + ); + this.#pendingPreparations.set(request.cacheKey, { + inputWordCount: countWords(contextText.join(" ")), + outputChunks: [], + request, + requestId: + this.#dependencies.createRequestId?.() ?? createVoiceRequestId(), + resolve, + startedAt: this.#now(), + timeout, + }); + + const serializedRequest: InterviewPreparationResponseRequest = { + kind: "speech-preparation", + preparationRequestId: request.cacheKey, + response: { + conversation: "none", + input: [ + { + type: "message", + role: "user", + content: [ + { + type: "input_text", + text: JSON.stringify({ + context_text: contextText, + maximum_words: request.contextWordBudget, + }), + }, + ], + }, + ], + instructions: INTERVIEW_PREPARATION_INSTRUCTIONS, + max_output_tokens: 120, + output_modalities: ["text"], + parallel_tool_calls: false, + tool_choice: "none", + tools: [], + metadata: { + petrinaut_kind: "speech-preparation", + petrinaut_request_id: request.cacheKey, + }, + }, + }; + this.#responseQueue.push(serializedRequest); + try { + this.#sendNextSerializedResponse(); + } catch { + const queuedRequestIndex = + this.#responseQueue.indexOf(serializedRequest); + if (queuedRequestIndex >= 0) { + this.#responseQueue.splice(queuedRequestIndex, 1); + } + this.#settlePreparation(request.cacheKey, "provider-error"); + } + }); } public cancelOutput(): void { @@ -426,8 +605,12 @@ export class OpenAIRealtimeSession { return; } - for (const request of this.#canonicalSpeechQueue.splice(0)) { - this.#cancelPendingSpeechRequest(request.speechRequestId); + for (const request of this.#responseQueue.splice(0)) { + if (request.kind === "canonical-speech") { + this.#cancelPendingSpeechRequest(request.speechRequestId); + } else { + this.#settlePreparation(request.preparationRequestId, "interrupted"); + } } if (this.#responseCreateEventId !== null) { @@ -435,9 +618,29 @@ export class OpenAIRealtimeSession { this.#responseCreateEventId, ); if (pendingEvent?.kind === "response-create") { - this.#cancelledSpeechRequestIds.add( - pendingEvent.request.speechRequestId, - ); + if (pendingEvent.request.kind === "canonical-speech") { + this.#cancelledSpeechRequestIds.add( + pendingEvent.request.speechRequestId, + ); + } else { + this.#cancelledPreparationRequestIds.add( + pendingEvent.request.preparationRequestId, + ); + this.#settlePreparation( + pendingEvent.request.preparationRequestId, + "interrupted", + ); + } + } + } + + for (const [ + responseId, + preparationRequestId, + ] of this.#preparationResponseIds) { + if (this.#activeResponseIds.has(responseId)) { + this.#settlePreparation(preparationRequestId, "interrupted"); + this.#cancelResponse(responseId); } } @@ -462,21 +665,24 @@ export class OpenAIRealtimeSession { } #canonicalResponseText(segments: CanonicalSpeechSegment[]): string[] { - const responseText = segments + return this.#validResponseText(segments.map(({ text }) => text)); + } + + #validResponseText(responseTextInput: readonly string[]): string[] { + const responseText = responseTextInput .slice(0, MAX_CANONICAL_SEGMENTS) - .map(({ text }) => text.trim()) + .map((text) => text.trim()) .filter(Boolean); - if (responseText.length === 0 || responseText.length !== segments.length) { + if ( + responseText.length === 0 || + responseText.length !== responseTextInput.length + ) { throw new VoiceError("speech", "invalid-response", ""); } return responseText; } - #requestCanonicalSpeech( - segments: CanonicalSpeechSegment[], - outOfBand: boolean, - ): void { - const responseText = this.#canonicalResponseText(segments); + #requestSpeech(responseText: string[], outOfBand: boolean): void { const speechRequestId = `canonical-${this.#activeEpoch}-${++this.#speechRequestSequence}`; this.#pendingSpeechRequests.set(speechRequestId, { requestId: @@ -511,14 +717,18 @@ export class OpenAIRealtimeSession { petrinaut_request_id: speechRequestId, }, }; - const request = { response, speechRequestId }; - this.#canonicalSpeechQueue.push(request); + const request: CanonicalSpeechRequest = { + kind: "canonical-speech", + response, + speechRequestId, + }; + this.#responseQueue.push(request); try { - this.#sendNextCanonicalSpeech(); + this.#sendNextSerializedResponse(); } catch (error) { - const queuedRequestIndex = this.#canonicalSpeechQueue.indexOf(request); + const queuedRequestIndex = this.#responseQueue.indexOf(request); if (queuedRequestIndex >= 0) { - this.#canonicalSpeechQueue.splice(queuedRequestIndex, 1); + this.#responseQueue.splice(queuedRequestIndex, 1); } this.#pendingSpeechRequests.delete(speechRequestId); throw error; @@ -547,7 +757,7 @@ export class OpenAIRealtimeSession { return `petrinaut-${this.#activeEpoch}-${++this.#clientEventSequence}`; } - #sendNextCanonicalSpeech(): void { + #sendNextSerializedResponse(): void { if ( this.#activeResponseIds.size > 0 || this.#responseCreateEventId !== null || @@ -555,7 +765,7 @@ export class OpenAIRealtimeSession { ) { return; } - const request = this.#canonicalSpeechQueue.shift(); + const request = this.#responseQueue.shift(); if (!request) { return; } @@ -604,6 +814,10 @@ export class OpenAIRealtimeSession { this.#handleResponseDone(parsed, connectionEpoch); return; } + if (parsed.type === "response.output_text.delta") { + this.#handlePreparationDelta(parsed); + return; + } if (parsed.type === "input_audio_buffer.committed") { const itemId = nonEmptyString(parsed.item_id); if (itemId) this.#startTranscription(itemId); @@ -664,42 +878,99 @@ export class OpenAIRealtimeSession { } this.#activeResponseIds.add(responseId); const metadata = asRecord(response?.metadata); - const speechRequestId = nonEmptyString(metadata?.petrinaut_request_id); - if (metadata?.petrinaut_kind !== "canonical-speech" || !speechRequestId) { + const correlatedRequestId = nonEmptyString(metadata?.petrinaut_request_id); + if (!correlatedRequestId) { + return; + } + + if (metadata?.petrinaut_kind === "speech-preparation") { + const correlated = this.#completeResponseCreateEvent( + "speech-preparation", + correlatedRequestId, + ); + if (!correlated) { + this.#cancelResponse(responseId); + return; + } + this.#preparationResponseIds.set(responseId, correlatedRequestId); + if ( + this.#cancelledPreparationRequestIds.delete(correlatedRequestId) || + !this.#pendingPreparations.has(correlatedRequestId) + ) { + this.#cancelResponse(responseId); + } + return; + } + if (metadata?.petrinaut_kind !== "canonical-speech") { + return; + } + + if ( + !this.#completeResponseCreateEvent( + "canonical-speech", + correlatedRequestId, + ) + ) { + this.#cancelResponse(responseId); return; } - this.#completeResponseCreateEvent(speechRequestId); this.#canonicalResponseIds.add(responseId); - if (this.#cancelledSpeechRequestIds.delete(speechRequestId)) { - this.#cancelPendingSpeechRequest(speechRequestId); + if (this.#cancelledSpeechRequestIds.delete(correlatedRequestId)) { + this.#cancelPendingSpeechRequest(correlatedRequestId); this.#cancelledCanonicalResponseIds.add(responseId); this.#cancelOutputResponse(responseId); return; } - const timing = this.#pendingSpeechRequests.get(speechRequestId); + const timing = this.#pendingSpeechRequests.get(correlatedRequestId); if (!timing) { return; } - this.#pendingSpeechRequests.delete(speechRequestId); + this.#pendingSpeechRequests.delete(correlatedRequestId); this.#authorizedResponseIds.add(responseId); this.#speechTimings.set(responseId, timing); } - #completeResponseCreateEvent(speechRequestId: string): void { - if (!this.#responseCreateEventId) { - return; - } - const pendingEvent = this.#pendingClientEvents.get( - this.#responseCreateEventId, - ); + #completeResponseCreateEvent( + kind: SerializedResponseRequest["kind"], + requestId: string, + ): boolean { + const eventIds = this.#responseCreateEventId + ? [this.#responseCreateEventId] + : []; if ( - pendingEvent?.kind !== "response-create" || - pendingEvent.request.speechRequestId !== speechRequestId + kind === "speech-preparation" && + this.#cancelledPreparationRequestIds.has(requestId) ) { - return; + eventIds.push( + ...[...this.#pendingClientEvents.entries()] + .filter( + ([eventId, pendingEvent]) => + eventId !== this.#responseCreateEventId && + pendingEvent.kind === "response-create" && + pendingEvent.request.kind === "speech-preparation" && + pendingEvent.request.preparationRequestId === requestId, + ) + .map(([eventId]) => eventId), + ); } - this.#pendingClientEvents.delete(this.#responseCreateEventId); - this.#responseCreateEventId = null; + for (const eventId of eventIds) { + const pendingEvent = this.#pendingClientEvents.get(eventId); + if ( + pendingEvent?.kind !== "response-create" || + pendingEvent.request.kind !== kind || + (pendingEvent.request.kind === "canonical-speech" + ? pendingEvent.request.speechRequestId + : pendingEvent.request.preparationRequestId) !== requestId + ) { + continue; + } + this.#pendingClientEvents.delete(eventId); + if (this.#responseCreateEventId === eventId) { + this.#responseCreateEventId = null; + } + return true; + } + return false; } #handleProviderError(event: Record): void { @@ -736,6 +1007,7 @@ export class OpenAIRealtimeSession { this.#responseCreateEventId = null; } if ( + pendingEvent.request.kind === "canonical-speech" && this.#cancelledSpeechRequestIds.delete( pendingEvent.request.speechRequestId, ) @@ -743,11 +1015,40 @@ export class OpenAIRealtimeSession { this.#cancelPendingSpeechRequest(pendingEvent.request.speechRequestId); return; } - this.#canonicalSpeechQueue.unshift(pendingEvent.request); + if ( + pendingEvent.request.kind === "speech-preparation" && + this.#cancelledPreparationRequestIds.delete( + pendingEvent.request.preparationRequestId, + ) + ) { + this.#settlePreparation( + pendingEvent.request.preparationRequestId, + "interrupted", + ); + return; + } + this.#responseQueue.unshift(pendingEvent.request); this.#waitingForResponseTerminal = this.#responseTerminalSequence === pendingEvent.responseTerminalSequence; - this.#resumeCanonicalSpeechQueue(); + this.#resumeSerializedResponseQueue(); + return; + } + + if ( + pendingEvent?.kind === "response-create" && + pendingEvent.request.kind === "speech-preparation" && + sourceEventId + ) { + this.#pendingClientEvents.delete(sourceEventId); + if (this.#responseCreateEventId === sourceEventId) { + this.#responseCreateEventId = null; + } + this.#settlePreparation( + pendingEvent.request.preparationRequestId, + "provider-error", + ); + this.#resumeSerializedResponseQueue(); return; } @@ -778,6 +1079,21 @@ export class OpenAIRealtimeSession { this.#clearResponseCancelEvents(responseId); this.#waitingForResponseTerminal = false; + const preparationRequestId = this.#preparationResponseIds.get(responseId); + if (preparationRequestId) { + this.#preparationResponseIds.delete(responseId); + if (status === "completed") { + this.#finishPreparation(preparationRequestId); + } else { + this.#settlePreparation( + preparationRequestId, + status === "cancelled" ? "interrupted" : "provider-error", + ); + } + this.#resumeSerializedResponseQueue(); + return; + } + if (this.#cancelledCanonicalResponseIds.delete(responseId)) { this.#emit({ connectionEpoch, @@ -786,7 +1102,7 @@ export class OpenAIRealtimeSession { type: "response-terminal", }); this.#finishSpeech(responseId, "request-aborted"); - this.#resumeCanonicalSpeechQueue(); + this.#resumeSerializedResponseQueue(); return; } @@ -840,7 +1156,7 @@ export class OpenAIRealtimeSession { status, type: "response-terminal", }); - this.#resumeCanonicalSpeechQueue(); + this.#resumeSerializedResponseQueue(); return; } if (status === "cancelled") { @@ -858,7 +1174,7 @@ export class OpenAIRealtimeSession { type: "response-terminal", }); this.#finishSpeech(responseId, "request-aborted"); - this.#resumeCanonicalSpeechQueue(); + this.#resumeSerializedResponseQueue(); return; } this.#emit({ @@ -885,14 +1201,123 @@ export class OpenAIRealtimeSession { } } - #resumeCanonicalSpeechQueue(): void { + #resumeSerializedResponseQueue(): void { try { - this.#sendNextCanonicalSpeech(); + this.#sendNextSerializedResponse(); } catch { this.#handleConnectionFailure("network", "speech"); } } + #handlePreparationDelta(event: Record): void { + const responseId = nonEmptyString(event.response_id); + const delta = event.delta; + if (!responseId || typeof delta !== "string") { + return; + } + const preparationRequestId = this.#preparationResponseIds.get(responseId); + if (!preparationRequestId) { + return; + } + this.#pendingPreparations + .get(preparationRequestId) + ?.outputChunks.push(delta); + } + + #finishPreparation(preparationRequestId: string): void { + const pending = this.#pendingPreparations.get(preparationRequestId); + if (!pending) { + return; + } + const context = pending.outputChunks.join("").trim(); + if ( + !context || + countWords(context) > pending.request.contextWordBudget || + Array.from(context).length > MAX_PREPARED_CONTEXT_CHARACTERS || + INVALID_PREPARED_CONTEXT_SYNTAX.test(context) + ) { + this.#settlePreparation(preparationRequestId, "invalid-output", context); + return; + } + this.#settlePreparation(preparationRequestId, undefined, context); + } + + #settlePreparation( + preparationRequestId: string, + reason?: InterviewSpeechPreparationFallbackReason, + context = "", + ): void { + const pending = this.#pendingPreparations.get(preparationRequestId); + if (!pending) { + return; + } + this.#pendingPreparations.delete(preparationRequestId); + globalThis.clearTimeout(pending.timeout); + const errorCode: VoiceErrorCode | undefined = + reason === undefined + ? undefined + : reason === "timeout" + ? "timeout" + : reason === "interrupted" + ? "request-aborted" + : "invalid-response"; + this.#dependencies.reportDiagnostic?.({ + durationMs: voiceDurationMs(pending.startedAt, this.#now()), + ...(errorCode === undefined ? {} : { errorCode }), + inputWordCount: pending.inputWordCount, + operation: "preparation", + outcome: voiceDiagnosticOutcome(errorCode), + outputWordCount: countWords(context), + requestId: pending.requestId, + sourceSegmentCount: pending.request.sourceSegmentIds.length, + stage: "browser", + }); + pending.resolve( + reason === undefined + ? { + context, + kind: "prepared", + sourceSegmentIds: [...pending.request.sourceSegmentIds], + } + : { + kind: "fallback", + reason, + sourceSegmentIds: [...pending.request.sourceSegmentIds], + }, + ); + } + + #timeoutPreparation(preparationRequestId: string): void { + const queuedRequestIndex = this.#responseQueue.findIndex( + (request) => + request.kind === "speech-preparation" && + request.preparationRequestId === preparationRequestId, + ); + if (queuedRequestIndex >= 0) { + this.#responseQueue.splice(queuedRequestIndex, 1); + } + + const pendingCreateEvent = this.#responseCreateEventId + ? this.#pendingClientEvents.get(this.#responseCreateEventId) + : undefined; + if ( + pendingCreateEvent?.kind === "response-create" && + pendingCreateEvent.request.kind === "speech-preparation" && + pendingCreateEvent.request.preparationRequestId === preparationRequestId + ) { + this.#cancelledPreparationRequestIds.add(preparationRequestId); + this.#responseCreateEventId = null; + } + const activeResponse = [...this.#preparationResponseIds].find( + ([, requestId]) => requestId === preparationRequestId, + ); + if (activeResponse) { + this.#cancelResponse(activeResponse[0]); + } + this.#settlePreparation(preparationRequestId, "timeout"); + this.#resumeSerializedResponseQueue(); + } + #handleOutputBufferEvent( event: Record, connectionEpoch: number, @@ -1257,6 +1682,9 @@ export class OpenAIRealtimeSession { } #releaseResources(): void { + for (const preparationRequestId of this.#pendingPreparations.keys()) { + this.#settlePreparation(preparationRequestId, "interrupted"); + } for (const timing of this.#transcriptionTimings.values()) { this.#reportDiagnostic( "transcription", @@ -1279,11 +1707,14 @@ export class OpenAIRealtimeSession { this.#transcriptionTimings.clear(); this.#activeResponseIds.clear(); this.#cancelledCanonicalResponseIds.clear(); + this.#cancelledPreparationRequestIds.clear(); this.#cancelledSpeechRequestIds.clear(); - this.#canonicalSpeechQueue.length = 0; this.#completedResponseCancelEventIds.clear(); this.#pendingClientEvents.clear(); + this.#pendingPreparations.clear(); this.#pendingSpeechRequests.clear(); + this.#preparationResponseIds.clear(); + this.#responseQueue.length = 0; this.#speechTimings.clear(); this.#authorizedResponseIds.clear(); this.#canonicalResponseIds.clear(); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.test.ts index a723f2dfe5a..37c1f796a22 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.test.ts @@ -1,13 +1,21 @@ import { describe, expect, test, vi } from "vitest"; import { + assemblePreparedInterviewSpeech, createRealtimeSubmissionId, RealtimeBrunchBridge, type RealtimeBrunchBridgeEvent, } from "./realtime-brunch-bridge"; -import type { CanonicalSpeechSegment } from "./canonical-speech"; -import type { OpenAIRealtimeSessionEvent } from "./openai-realtime-session"; +import type { + CanonicalSpeechSegment, + InterviewSpeechSource, +} from "./canonical-speech"; +import type { + InterviewSpeechPreparationRequest, + InterviewSpeechPreparationResult, + OpenAIRealtimeSessionEvent, +} from "./openai-realtime-session"; const segment = ( id: string, @@ -22,11 +30,46 @@ const segment = ( text, }); +const speechSource = ({ + context = [], + messageId = "message-current-turn", + question = null, +}: { + readonly context?: readonly CanonicalSpeechSegment[]; + readonly messageId?: string; + readonly question?: CanonicalSpeechSegment | null; +}): InterviewSpeechSource => { + const contextSegments = context.map((contextSegment) => ({ + ...contextSegment, + messageId, + })); + const questionSegment = question ? { ...question, messageId } : null; + return { + contextSegments, + fullResponseSegments: [ + ...contextSegments, + ...(questionSegment ? [questionSegment] : []), + ], + messageId, + questionSegment, + }; +}; + const createHarness = () => { let listener: ((event: OpenAIRealtimeSessionEvent) => void) | undefined; const session = { completeFunctionCall: vi.fn(), + prepareInterviewSpeech: vi.fn( + async ( + request: InterviewSpeechPreparationRequest, + ): Promise => ({ + context: "Prepared concise context.", + kind: "prepared", + sourceSegmentIds: request.sourceSegmentIds, + }), + ), speakCanonical: vi.fn(), + speakPrepared: vi.fn(), subscribe: vi.fn((next: (event: OpenAIRealtimeSessionEvent) => void) => { listener = next; return () => { @@ -96,6 +139,331 @@ const responseTerminal = ( }); describe("RealtimeBrunchBridge", () => { + test("prepares only context and appends the exact canonical question", async () => { + const harness = createHarness(); + const context = segment( + "context", + "This complete canonical explanation is deliberately long enough to be condensed before automatic speech.", + "assistant-text", + ); + const question = segment( + "ask-current", + "Who approves it: the manager or quality lead?", + ); + const source = speechSource({ context: [context], question }); + harness.bridge.updateChat({ + automaticSource: source, + canAcceptInterviewAnswer: true, + canonicalSegments: [...source.fullResponseSegments], + status: "ready", + }); + + harness.bridge.start(4); + + await vi.waitFor(() => + expect(harness.session.speakPrepared).toHaveBeenCalledOnce(), + ); + const preparationRequest = + harness.session.prepareInterviewSpeech.mock.calls[0]?.[0]; + expect(typeof preparationRequest?.cacheKey).toBe("string"); + expect(preparationRequest).toEqual({ + cacheKey: preparationRequest?.cacheKey, + contextText: [context.text], + contextWordBudget: 42, + sourceSegmentIds: [context.id], + }); + expect( + JSON.stringify(harness.session.prepareInterviewSpeech.mock.calls), + ).not.toContain(question.text); + expect(harness.session.speakPrepared).toHaveBeenCalledWith([ + "Prepared concise context.", + question.text, + ]); + expect(source.fullResponseSegments.map(({ text }) => text)).toEqual([ + context.text, + question.text, + ]); + expect(JSON.stringify(harness.events)).not.toContain( + "Prepared concise context.", + ); + }); + + test("skips preparation when a turn contains only a protected question", async () => { + const harness = createHarness(); + const question = segment("ask-current", "What happens after approval?"); + const source = speechSource({ question }); + harness.bridge.updateChat({ + automaticSource: source, + canAcceptInterviewAnswer: true, + canonicalSegments: [...source.fullResponseSegments], + status: "ready", + }); + + harness.bridge.start(4); + + await vi.waitFor(() => + expect(harness.session.speakPrepared).toHaveBeenCalledWith([ + question.text, + ]), + ); + expect(harness.session.prepareInterviewSpeech).not.toHaveBeenCalled(); + }); + + test("prepares every human-facing segment in a standalone completion", async () => { + const harness = createHarness(); + const first = segment("first", "First result.", "assistant-text"); + const second = segment("second", "Second result.", "assistant-text"); + const source = speechSource({ context: [first, second] }); + harness.bridge.updateChat({ + automaticSource: source, + canAcceptInterviewAnswer: false, + canonicalSegments: [...source.fullResponseSegments], + status: "ready", + }); + + harness.bridge.start(4); + + await vi.waitFor(() => + expect(harness.session.prepareInterviewSpeech).toHaveBeenCalledWith( + expect.objectContaining({ + contextText: [first.text, second.text], + contextWordBudget: 50, + }), + ), + ); + expect(harness.session.speakPrepared).toHaveBeenCalledWith([ + "Prepared concise context.", + ]); + }); + + test.each([ + "empty-context", + "invalid-output", + "provider-error", + "timeout", + "interrupted", + ] as const)( + "reads complete canonical speech after %s fallback", + async (reason) => { + const harness = createHarness(); + const context = segment("context", "Complete context.", "assistant-text"); + const question = segment("ask-current", "Exact question?"); + const source = speechSource({ context: [context], question }); + harness.session.prepareInterviewSpeech.mockResolvedValueOnce({ + kind: "fallback", + reason, + sourceSegmentIds: [context.id], + }); + harness.bridge.updateChat({ + automaticSource: source, + canAcceptInterviewAnswer: true, + canonicalSegments: [...source.fullResponseSegments], + status: "ready", + }); + + harness.bridge.start(4); + + await vi.waitFor(() => + expect(harness.session.speakPrepared).toHaveBeenCalledWith([ + context.text, + question.text, + ]), + ); + }, + ); + + test("assembles prepared context with the protected question identity", () => { + const context = segment("context", "Complete context.", "assistant-text"); + const question = segment("ask-current", "Exact question?"); + const source = speechSource({ context: [context], question }); + + expect( + assemblePreparedInterviewSpeech({ + preparation: { + context: "Prepared context.", + kind: "prepared", + sourceSegmentIds: [context.id], + }, + source, + }), + ).toEqual({ + mode: "realtime-processed", + sourceSegmentIds: [context.id, question.id], + text: ["Prepared context.", question.text], + }); + }); + + test("reuses prepared context only while its epoch, content, and budget match", async () => { + const harness = createHarness(); + const context = segment("context", "Complete context.", "assistant-text"); + const question = segment("ask-current", "Exact question?"); + const source = speechSource({ context: [context], question }); + harness.bridge.updateChat({ + automaticSource: source, + canAcceptInterviewAnswer: true, + canonicalSegments: [...source.fullResponseSegments], + status: "ready", + }); + + harness.bridge.start(4); + await vi.waitFor(() => + expect(harness.session.speakPrepared).toHaveBeenCalledOnce(), + ); + harness.bridge.start(4); + await vi.waitFor(() => + expect(harness.session.speakPrepared).toHaveBeenCalledTimes(2), + ); + expect(harness.session.prepareInterviewSpeech).toHaveBeenCalledOnce(); + + const changedContext = { ...context, contentHash: "fnv1a32:changed" }; + const changedContentSource = speechSource({ + context: [changedContext], + question, + }); + harness.bridge.updateChat({ + automaticSource: changedContentSource, + canAcceptInterviewAnswer: true, + canonicalSegments: [...changedContentSource.fullResponseSegments], + status: "ready", + }); + harness.bridge.start(4); + await vi.waitFor(() => + expect(harness.session.prepareInterviewSpeech).toHaveBeenCalledTimes(2), + ); + + const longerQuestion = { ...question, text: "What is the exact question?" }; + const changedBudgetSource = speechSource({ + context: [changedContext], + question: longerQuestion, + }); + harness.bridge.updateChat({ + automaticSource: changedBudgetSource, + canAcceptInterviewAnswer: true, + canonicalSegments: [...changedBudgetSource.fullResponseSegments], + status: "ready", + }); + harness.bridge.start(4); + await vi.waitFor(() => + expect(harness.session.prepareInterviewSpeech).toHaveBeenCalledTimes(3), + ); + + harness.bridge.start(5); + await vi.waitFor(() => + expect(harness.session.prepareInterviewSpeech).toHaveBeenCalledTimes(4), + ); + harness.bridge.stop(); + harness.bridge.start(5); + await vi.waitFor(() => + expect(harness.session.prepareInterviewSpeech).toHaveBeenCalledTimes(5), + ); + }); + + test("does not cache fallback or deliver preparation cancelled by lifecycle", async () => { + const harness = createHarness(); + const context = segment("context", "Complete context.", "assistant-text"); + const question = segment("ask-current", "Exact question?"); + const source = speechSource({ context: [context], question }); + let finishPreparation: + | ((result: InterviewSpeechPreparationResult) => void) + | undefined; + harness.session.prepareInterviewSpeech.mockImplementationOnce( + () => + new Promise((resolve) => { + finishPreparation = resolve; + }), + ); + harness.bridge.updateChat({ + automaticSource: source, + canAcceptInterviewAnswer: true, + canonicalSegments: [...source.fullResponseSegments], + status: "ready", + }); + harness.bridge.start(4); + + harness.bridge.cancelPendingSpeech(); + finishPreparation?.({ + kind: "fallback", + reason: "interrupted", + sourceSegmentIds: [context.id], + }); + await Promise.resolve(); + + expect(harness.session.speakPrepared).not.toHaveBeenCalled(); + harness.bridge.start(4); + await vi.waitFor(() => + expect(harness.session.prepareInterviewSpeech).toHaveBeenCalledTimes(2), + ); + }); + + test("completes a function call without speech when preparation is cancelled", async () => { + const harness = createHarness(); + const question = segment("ask-current", "What happens after approval?"); + harness.bridge.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [question], + status: "ready", + }); + harness.bridge.start(7); + harness.emit(toolDone(7)); + await vi.waitFor(() => + expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce(), + ); + harness.bridge.updateChat({ + canAcceptInterviewAnswer: false, + canonicalSegments: [question], + status: "streaming", + }); + + let finishPreparation: + | ((result: InterviewSpeechPreparationResult) => void) + | undefined; + harness.session.prepareInterviewSpeech.mockImplementationOnce( + () => + new Promise((resolve) => { + finishPreparation = resolve; + }), + ); + const acknowledgement = segment( + "acknowledgement", + "Thanks. I have recorded that.", + "assistant-text", + ); + const nextQuestion = segment( + "ask-next", + "Who is informed next?", + "brunch-ask", + ); + const source = speechSource({ + context: [acknowledgement], + question: nextQuestion, + }); + harness.bridge.updateChat({ + automaticSource: source, + canAcceptInterviewAnswer: true, + canonicalSegments: [question, ...source.fullResponseSegments], + status: "ready", + }); + await vi.waitFor(() => + expect(harness.session.prepareInterviewSpeech).toHaveBeenCalledOnce(), + ); + + harness.bridge.cancelPendingSpeech(); + finishPreparation?.({ + context: "Prepared concise context.", + kind: "prepared", + sourceSegmentIds: [acknowledgement.id], + }); + + await vi.waitFor(() => + expect(harness.session.completeFunctionCall).toHaveBeenCalledWith( + "call-1", + [acknowledgement.text, nextQuestion.text], + { speakResponse: false }, + ), + ); + expect(harness.session.speakPrepared).not.toHaveBeenCalled(); + }); + test("speaks the current canonical turn without replaying history", () => { const harness = createHarness(); const historical = segment( @@ -174,7 +542,7 @@ describe("RealtimeBrunchBridge", () => { await vi.waitFor(() => expect(harness.session.completeFunctionCall).toHaveBeenCalledWith( "call-1", - [acknowledgement, nextQuestion], + [acknowledgement.text, nextQuestion.text], ), ); expect(harness.events.map(({ type }) => type)).toEqual([ @@ -222,7 +590,7 @@ describe("RealtimeBrunchBridge", () => { expect(harness.session.completeFunctionCall).toHaveBeenCalledWith( "call-1", - [firstQuestion], + [firstQuestion.text], ); expect(harness.events.map(({ type }) => type)).toEqual([ "submission-started", @@ -271,7 +639,7 @@ describe("RealtimeBrunchBridge", () => { expect(harness.session.completeFunctionCall).toHaveBeenCalledWith( "call-1", - [unrelated], + [unrelated.text], ); }); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.ts b/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.ts index 1a84fa74c62..5bf988dbfe0 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.ts @@ -1,9 +1,19 @@ -import type { CanonicalSpeechSegment } from "./canonical-speech"; -import type { OpenAIRealtimeSessionEvent } from "./openai-realtime-session"; +import { + hashCanonicalSpeechText, + type CanonicalSpeechSegment, + type InterviewSpeechSource, +} from "./canonical-speech"; + +import type { + InterviewSpeechPreparationRequest, + InterviewSpeechPreparationResult, + OpenAIRealtimeSessionEvent, +} from "./openai-realtime-session"; type ChatStatus = "ready" | "submitted" | "streaming" | "error"; interface ChatUpdate { + readonly automaticSource?: InterviewSpeechSource | null; readonly canAcceptInterviewAnswer: boolean; readonly canonicalSegments: CanonicalSpeechSegment[]; readonly status: ChatStatus; @@ -12,9 +22,14 @@ interface ChatUpdate { interface RealtimeBridgeSession { completeFunctionCall( callId: string, - segments: CanonicalSpeechSegment[], + responseText: readonly string[], + options?: { readonly speakResponse?: boolean }, ): void; + prepareInterviewSpeech( + request: InterviewSpeechPreparationRequest, + ): Promise; speakCanonical(segments: CanonicalSpeechSegment[]): void; + speakPrepared(responseText: readonly string[]): void; subscribe(listener: (event: OpenAIRealtimeSessionEvent) => void): () => void; } @@ -49,6 +64,10 @@ interface ArgumentStream { readonly responseId: string; } +type SpeechDelivery = + | { readonly kind: "automatic" } + | { readonly callId: string; readonly kind: "function-call" }; + export type RealtimeBridgeErrorCode = | "interview-correlation" | "interview-response" @@ -74,8 +93,45 @@ export type RealtimeBrunchBridgeEvent = readonly code: RealtimeBridgeErrorCode; readonly message: string; readonly type: "error"; + } + | { + readonly type: "speech-delivery-pending"; }; +export interface PreparedInterviewSpeech { + readonly mode: "realtime-processed" | "canonical-fallback"; + readonly sourceSegmentIds: readonly string[]; + readonly text: readonly string[]; +} + +export const assemblePreparedInterviewSpeech = ({ + preparation, + source, +}: { + preparation: InterviewSpeechPreparationResult; + source: InterviewSpeechSource; +}): PreparedInterviewSpeech => { + if (preparation.kind === "prepared") { + return { + mode: "realtime-processed", + sourceSegmentIds: [ + ...preparation.sourceSegmentIds, + ...(source.questionSegment ? [source.questionSegment.id] : []), + ], + text: [ + ...(preparation.context ? [preparation.context] : []), + ...(source.questionSegment ? [source.questionSegment.text] : []), + ], + }; + } + + return { + mode: "canonical-fallback", + sourceSegmentIds: source.fullResponseSegments.map(({ id }) => id), + text: source.fullResponseSegments.map(({ text }) => text), + }; +}; + type BridgeListener = (event: RealtimeBrunchBridgeEvent) => void; const INVALID_BRIDGE_EVENT = @@ -92,6 +148,22 @@ const latestPendingQuestion = ( ): CanonicalSpeechSegment | undefined => segments.findLast(({ source }) => source === "brunch-ask"); +const spokenWordCount = (text: string): number => { + const trimmedText = text.trim(); + return trimmedText ? trimmedText.split(/\s+/u).length : 0; +}; + +const preparationCacheKey = ( + source: InterviewSpeechSource, + contextWordBudget: number, +): string => { + const sourceIdentity = JSON.stringify([ + ...source.contextSegments.map(({ contentHash, id }) => [id, contentHash]), + contextWordBudget, + ]); + return `speech-preparation:${hashCanonicalSpeechText(sourceIdentity)}`; +}; + const parseContinueInterviewArguments = ( argumentsJson: string, ): string | null => { @@ -114,6 +186,7 @@ const parseContinueInterviewArguments = ( export class RealtimeBrunchBridge { readonly #argumentDeltas = new Map(); readonly #listeners = new Set(); + readonly #preparedContextCache = new Map(); readonly #processedCalls = new Set(); readonly #session: RealtimeBridgeSession; readonly #submitInterviewAnswer: ( @@ -129,6 +202,7 @@ export class RealtimeBrunchBridge { status: "ready", }; #generation = 0; + #speechGeneration = 0; public constructor({ session, @@ -146,6 +220,9 @@ export class RealtimeBrunchBridge { public start(connectionEpoch: number): void { ++this.#generation; + if (this.#activeEpoch !== connectionEpoch) { + this.#preparedContextCache.clear(); + } this.#activeEpoch = connectionEpoch; this.#activeSubmission = null; this.#argumentDeltas.clear(); @@ -156,13 +233,18 @@ export class RealtimeBrunchBridge { this.#seenSegmentIds.add(segment.id); } - const question = latestPendingQuestion(this.#chat.canonicalSegments); - if (question) { - this.#session.speakCanonical( - this.#chat.canonicalSegments.filter( - ({ messageId }) => messageId === question.messageId, - ), + const source = this.#chat.automaticSource; + if (source) { + this.#prepareAndDeliver(source, { kind: "automatic" }); + } else { + const question = latestPendingQuestion(this.#chat.canonicalSegments); + if (!question) { + return; + } + const currentTurnSegments = this.#chat.canonicalSegments.filter( + ({ messageId }) => messageId === question.messageId, ); + this.#session.speakCanonical(currentTurnSegments); } } @@ -171,9 +253,14 @@ export class RealtimeBrunchBridge { this.#activeEpoch = null; this.#activeSubmission = null; this.#argumentDeltas.clear(); + this.#preparedContextCache.clear(); this.#terminalResponseIds.clear(); } + public cancelPendingSpeech(): void { + ++this.#speechGeneration; + } + public updateChat(update: ChatUpdate): void { this.#chat = update; if (this.#activeEpoch === null) { @@ -204,10 +291,15 @@ export class RealtimeBrunchBridge { return; } try { - this.#session.speakCanonical(newSegments); for (const segment of newSegments) { this.#seenSegmentIds.add(segment.id); } + const source = update.automaticSource; + if (source && this.#sourceMatchesSegments(source, newSegments)) { + this.#prepareAndDeliver(source, { kind: "automatic" }); + } else { + this.#session.speakCanonical(newSegments); + } } catch { this.#fail(INVALID_BRIDGE_EVENT); } @@ -408,20 +500,142 @@ export class RealtimeBrunchBridge { return; } - try { - this.#session.completeFunctionCall(active.callId, responseSegments); - } catch { - this.#fail(INVALID_BRIDGE_EVENT); - return; - } for (const segment of responseSegments) { this.#seenSegmentIds.add(segment.id); } this.#activeSubmission = null; + const source = this.#chat.automaticSource; + if (source && this.#sourceMatchesSegments(source, responseSegments)) { + this.#prepareAndDeliver(source, { + callId: active.callId, + kind: "function-call", + }); + } else { + try { + this.#session.completeFunctionCall( + active.callId, + responseSegments.map(({ text }) => text), + ); + } catch { + this.#fail(INVALID_BRIDGE_EVENT); + return; + } + } this.#emit({ callId: active.callId, segments: responseSegments, type: "canonical-response-ready", }); } + + #sourceMatchesSegments( + source: InterviewSpeechSource, + segments: readonly CanonicalSpeechSegment[], + ): boolean { + return ( + source.fullResponseSegments.length === segments.length && + source.fullResponseSegments.every( + ({ id }, index) => segments[index]?.id === id, + ) + ); + } + + #prepareAndDeliver( + source: InterviewSpeechSource, + delivery: SpeechDelivery, + ): void { + this.#emit({ type: "speech-delivery-pending" }); + const generation = this.#generation; + const speechGeneration = this.#speechGeneration; + const questionWordCount = source.questionSegment + ? spokenWordCount(source.questionSegment.text) + : 0; + const contextWordBudget = Math.max(0, 50 - questionWordCount); + + if (source.contextSegments.length === 0 || contextWordBudget === 0) { + const questionOnlySource: InterviewSpeechSource = { + ...source, + contextSegments: [], + fullResponseSegments: source.questionSegment + ? [source.questionSegment] + : [], + }; + this.#deliverPreparedSpeech( + assemblePreparedInterviewSpeech({ + preparation: { + context: "", + kind: "prepared", + sourceSegmentIds: [], + }, + source: questionOnlySource, + }).text, + delivery, + ); + return; + } + + const request: InterviewSpeechPreparationRequest = { + cacheKey: preparationCacheKey(source, contextWordBudget), + contextText: source.contextSegments.map(({ text }) => text), + contextWordBudget, + sourceSegmentIds: source.contextSegments.map(({ id }) => id), + }; + const cachedContext = this.#preparedContextCache.get(request.cacheKey); + if (cachedContext !== undefined) { + this.#deliverPreparedSpeech( + assemblePreparedInterviewSpeech({ + preparation: { + context: cachedContext, + kind: "prepared", + sourceSegmentIds: request.sourceSegmentIds, + }, + source, + }).text, + delivery, + ); + return; + } + void this.#session.prepareInterviewSpeech(request).then((preparation) => { + if (generation !== this.#generation) { + return; + } + const preparedSpeech = assemblePreparedInterviewSpeech({ + preparation, + source, + }); + if (speechGeneration !== this.#speechGeneration) { + if (delivery.kind === "function-call") { + try { + this.#session.completeFunctionCall( + delivery.callId, + source.fullResponseSegments.map(({ text }) => text), + { speakResponse: false }, + ); + } catch { + this.#fail(INVALID_BRIDGE_EVENT); + } + } + return; + } + if (preparation.kind === "prepared") { + this.#preparedContextCache.set(request.cacheKey, preparation.context); + } + this.#deliverPreparedSpeech(preparedSpeech.text, delivery); + }); + } + + #deliverPreparedSpeech( + responseText: readonly string[], + delivery: SpeechDelivery, + ): void { + try { + if (delivery.kind === "function-call") { + this.#session.completeFunctionCall(delivery.callId, responseText); + } else { + this.#session.speakPrepared(responseText); + } + } catch { + this.#fail(INVALID_BRIDGE_EVENT); + } + } } diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx index b4e37880a96..4be3b7bcce4 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx @@ -213,6 +213,24 @@ describe("voice interview control", () => { ).not.toThrow(); }); + test("registers canonical replay controls with the host", () => { + const repeatQuestion = vi.spyOn( + VoiceTurnController.prototype, + "repeatQuestion", + ); + const readFullResponse = vi.spyOn( + VoiceTurnController.prototype, + "readFullResponse", + ); + render(); + + registeredVoiceModeControls?.repeatQuestion?.(); + registeredVoiceModeControls?.readFullResponse?.(); + + expect(repeatQuestion).toHaveBeenCalledOnce(); + expect(readFullResponse).toHaveBeenCalledOnce(); + }); + test("loads only a schema-valid available server configuration", async () => { const fetch = vi.fn(async () => Response.json({ available: true, connectionTimeoutMs: 15_000 }), diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx index ee911bc6d4d..8e030fb725a 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx @@ -10,7 +10,7 @@ import { Button } from "@hashintel/ds-components"; import { css } from "@hashintel/ds-helpers/css"; import { reportVoiceDiagnostic } from "../../../voice-diagnostics"; -import { selectCanonicalSpeechSegments } from "./canonical-speech"; +import { selectInterviewSpeech } from "./canonical-speech"; import { OpenAIRealtimeSession } from "./openai-realtime-session"; import { RealtimeBrunchBridge } from "./realtime-brunch-bridge"; import { toVoiceSessionState } from "./voice-session-state"; @@ -290,9 +290,11 @@ const AvailableVoiceInterviewControl = ({ useLayoutEffect(() => { store.updateSubmissionContext(context.submitVoiceInput); + const speechSelection = selectInterviewSpeech(context.messages); store.controller.updateChat({ + automaticSource: speechSelection.automaticSource, canAcceptInterviewAnswer: context.canAcceptVoiceInput, - canonicalSegments: selectCanonicalSpeechSegments(context.messages), + canonicalSegments: [...speechSelection.canonicalSegments], status: context.status, }); }, [ @@ -310,9 +312,11 @@ const AvailableVoiceInterviewControl = ({ registerVoiceModeControls({ end: () => store.controller.end(), pause: () => store.controller.pause(), + readFullResponse: () => store.controller.readFullResponse(), reconnect: () => { void store.controller.reconnect(); }, + repeatQuestion: () => store.controller.repeatQuestion(), resume: () => store.controller.resume(), setMicrophoneMuted: (muted) => store.controller.setMicrophoneMuted(muted), diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts index 596389aad50..a45eee6542a 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts @@ -5,7 +5,7 @@ import { VOICE_REQUEST_ID_HEADER, type VoiceDiagnosticEvent, } from "../../../voice-diagnostics"; -import { selectCanonicalSpeechSegments } from "./canonical-speech"; +import { selectInterviewSpeech } from "./canonical-speech"; import { OpenAIRealtimeSession } from "./openai-realtime-session"; import { RealtimeBrunchBridge } from "./realtime-brunch-bridge"; import { VoiceTurnController } from "./voice-turn-controller"; @@ -16,12 +16,25 @@ const origin = "https://petrinaut.test"; const browserOffer = "v=0\r\na=private-browser-sdp\r\n"; const providerAnswer = "v=0\r\na=private-provider-sdp\r\n"; const spokenAnswer = "The supervisor approves it."; -const canonicalReply = "Thanks. I have recorded that."; -const canonicalQuestion = "Who is informed next?"; +const canonicalReply = + "Thanks. I have recorded that the supervisor approves each release before the operations team schedules the batch, and that the quality lead must receive the signed checklist, inspect every exception, preserve the audit record, and notify the manager before any delayed item can move into production."; +const preparedReply = "Approval recorded."; +const canonicalQuestion = + "Who is informed next: the manager, the quality lead, or both? Choose one."; +const toolMetadata = "internal-tool-metadata-must-not-be-spoken"; +const toolError = "internal-tool-error-must-not-be-spoken"; +const fallbackReply = + "The complete canonical fallback says that rejected batches remain quarantined until an authorized reviewer documents the resolution."; +const fallbackQuestion = + "Who documents the resolution: the manager or the quality lead?"; const requestIds = [ "00000000-0000-4000-8000-000000000011", "00000000-0000-4000-8000-000000000012", "00000000-0000-4000-8000-000000000013", + "00000000-0000-4000-8000-000000000014", + "00000000-0000-4000-8000-000000000015", + "00000000-0000-4000-8000-000000000016", + "00000000-0000-4000-8000-000000000017", ] as const; class FakeDataChannel extends EventTarget { @@ -81,16 +94,46 @@ const responseMessages = [ ...initialMessages, { id: "canonical-response-message", - parts: [{ state: "done", text: canonicalReply, type: "text" }], + parts: [ + { state: "done", text: canonicalReply, type: "text" }, + { + input: { metadata: toolMetadata }, + output: { result: toolMetadata }, + state: "output-available", + toolCallId: "metadata-tool", + toolName: "internal_metadata", + type: "dynamic-tool", + }, + { + errorText: toolError, + input: { metadata: toolMetadata }, + state: "output-error", + toolCallId: "error-tool", + toolName: "internal_error", + type: "dynamic-tool", + }, + { + input: { question: canonicalQuestion }, + state: "input-available", + toolCallId: "ask-next", + toolName: "brunch_ask", + type: "dynamic-tool", + }, + ], role: "assistant", }, +] satisfies PetrinautAiMessage[]; + +const fallbackMessages = [ + ...responseMessages, { - id: "next-question-message", + id: "canonical-fallback-message", parts: [ + { state: "done", text: fallbackReply, type: "text" }, { - input: { question: canonicalQuestion }, + input: { question: fallbackQuestion }, state: "input-available", - toolCallId: "ask-next", + toolCallId: "ask-fallback", toolName: "brunch_ask", type: "dynamic-tool", }, @@ -101,6 +144,7 @@ const responseMessages = [ describe("controlled voice preview", () => { test("bridges one Realtime tool call through Brunch and back to canonical duplex audio", async () => { + const responseMessagesSnapshot = structuredClone(responseMessages); const diagnostics: VoiceDiagnosticEvent[] = []; const reportDiagnostic = (event: VoiceDiagnosticEvent) => diagnostics.push(event); @@ -217,9 +261,11 @@ describe("controlled voice preview", () => { session, submitText: submitInterviewAnswer, }); + const initialSpeech = selectInterviewSpeech(initialMessages); controller.updateChat({ + automaticSource: initialSpeech.automaticSource, canAcceptInterviewAnswer: true, - canonicalSegments: selectCanonicalSpeechSegments(initialMessages), + canonicalSegments: [...initialSpeech.canonicalSegments], status: "ready", }); @@ -286,17 +332,75 @@ describe("controlled voice preview", () => { output: "waiting-for-tool", }); + const pendingSpeech = selectInterviewSpeech(initialMessages); controller.updateChat({ + automaticSource: pendingSpeech.automaticSource, canAcceptInterviewAnswer: false, - canonicalSegments: selectCanonicalSpeechSegments(initialMessages), + canonicalSegments: [...pendingSpeech.canonicalSegments], status: "streaming", }); + const responseSpeech = selectInterviewSpeech(responseMessages); + expect(responseSpeech.automaticSource).toMatchObject({ + contextSegments: [{ text: canonicalReply }], + questionSegment: { text: canonicalQuestion }, + }); + expect(responseSpeech.canonicalSegments.map(({ text }) => text)).toEqual([ + "What happens after approval?", + canonicalReply, + canonicalQuestion, + ]); controller.updateChat({ + automaticSource: responseSpeech.automaticSource, canAcceptInterviewAnswer: true, - canonicalSegments: selectCanonicalSpeechSegments(responseMessages), + canonicalSegments: [...responseSpeech.canonicalSegments], status: "ready", }); + const preparationCreate = sentEvents(dataChannel).at(-1)!; + expect(preparationCreate).toMatchObject({ + type: "response.create", + response: { + metadata: { petrinaut_kind: "speech-preparation" }, + output_modalities: ["text"], + }, + }); + const preparationResponse = preparationCreate.response as { + input: Array<{ content: Array<{ text: string }> }>; + metadata: Record; + }; + expect(JSON.parse(preparationResponse.input[0]!.content[0]!.text)).toEqual({ + context_text: [canonicalReply], + maximum_words: 50 - canonicalQuestion.trim().split(/\s+/u).length, + }); + expect(JSON.stringify(preparationResponse)).not.toContain(toolMetadata); + expect(JSON.stringify(preparationResponse)).not.toContain(toolError); + dataChannel.receive({ + response: { + id: "response-prepared-reply", + metadata: preparationResponse.metadata, + }, + type: "response.created", + }); + dataChannel.receive({ + delta: preparedReply, + response_id: "response-prepared-reply", + type: "response.output_text.delta", + }); + dataChannel.receive({ + response: { + id: "response-prepared-reply", + output: [], + status: "completed", + }, + type: "response.done", + }); + + await vi.waitFor(() => + expect(sentEvents(dataChannel)).toContainEqual( + expect.objectContaining({ type: "conversation.item.create" }), + ), + ); + const [functionOutput, responseCreate] = sentEvents(dataChannel).slice(-2); expect(functionOutput).toEqual({ type: "conversation.item.create", @@ -304,7 +408,7 @@ describe("controlled voice preview", () => { type: "function_call_output", call_id: "call-1", output: JSON.stringify({ - response_text: [canonicalReply, canonicalQuestion], + response_text: [preparedReply, canonicalQuestion], }), }, }); @@ -316,6 +420,12 @@ describe("controlled voice preview", () => { tools: [], }, }); + expect(JSON.stringify([functionOutput, responseCreate])).not.toContain( + toolMetadata, + ); + expect(JSON.stringify([functionOutput, responseCreate])).not.toContain( + toolError, + ); authorizeLatestSpeechResponse(dataChannel, "response-canonical-reply"); dataChannel.receive({ @@ -343,6 +453,116 @@ describe("controlled voice preview", () => { }); expect(remoteAudio.play).toHaveBeenCalledOnce(); + dataChannel.receive({ + response_id: "response-canonical-reply", + type: "output_audio_buffer.stopped", + }); + dataChannel.receive({ + response: { + id: "response-canonical-reply", + output: [], + status: "completed", + }, + type: "response.done", + }); + + controller.readFullResponse(); + await vi.waitFor(() => + expect(sentEvents(dataChannel).at(-1)).toMatchObject({ + response: { + metadata: { petrinaut_kind: "canonical-speech" }, + }, + type: "response.create", + }), + ); + const replayCreate = sentEvents(dataChannel).at(-1)!; + const replayResponse = replayCreate.response as { + input: Array<{ content: Array<{ text: string }> }>; + }; + expect(JSON.parse(replayResponse.input[0]!.content[0]!.text)).toEqual({ + response_text: [canonicalReply, canonicalQuestion], + }); + expect(JSON.stringify(replayCreate)).not.toContain(preparedReply); + + authorizeLatestSpeechResponse(dataChannel, "response-canonical-replay"); + dataChannel.receive({ + response_id: "response-canonical-replay", + type: "output_audio_buffer.started", + }); + dataChannel.receive({ + response_id: "response-canonical-replay", + type: "output_audio_buffer.stopped", + }); + dataChannel.receive({ + response: { + id: "response-canonical-replay", + output: [], + status: "completed", + }, + type: "response.done", + }); + + const canonicalSpeechCountBeforeFallback = sentEvents(dataChannel).filter( + (event) => + event.type === "response.create" && + (event.response as { metadata?: { petrinaut_kind?: string } }).metadata + ?.petrinaut_kind === "canonical-speech", + ).length; + const fallbackSpeech = selectInterviewSpeech(fallbackMessages); + controller.updateChat({ + automaticSource: fallbackSpeech.automaticSource, + canAcceptInterviewAnswer: true, + canonicalSegments: [...fallbackSpeech.canonicalSegments], + status: "ready", + }); + const fallbackPreparationCreate = sentEvents(dataChannel).at(-1)!; + expect(fallbackPreparationCreate).toMatchObject({ + response: { + metadata: { petrinaut_kind: "speech-preparation" }, + }, + type: "response.create", + }); + const fallbackPreparationResponse = fallbackPreparationCreate.response as { + input: Array<{ content: Array<{ text: string }> }>; + metadata: Record; + }; + expect( + JSON.parse(fallbackPreparationResponse.input[0]!.content[0]!.text), + ).toMatchObject({ context_text: [fallbackReply] }); + dataChannel.receive({ + response: { + id: "response-fallback-preparation", + metadata: fallbackPreparationResponse.metadata, + }, + type: "response.created", + }); + dataChannel.receive({ + response: { + id: "response-fallback-preparation", + status: "failed", + }, + type: "response.done", + }); + + await vi.waitFor(() => + expect( + sentEvents(dataChannel).filter( + (event) => + event.type === "response.create" && + (event.response as { metadata?: { petrinaut_kind?: string } }) + .metadata?.petrinaut_kind === "canonical-speech", + ), + ).toHaveLength(canonicalSpeechCountBeforeFallback + 1), + ); + const fallbackAudioCreate = sentEvents(dataChannel).at(-1)!; + const fallbackAudioResponse = fallbackAudioCreate.response as { + input: Array<{ content: Array<{ text: string }> }>; + }; + expect( + JSON.parse(fallbackAudioResponse.input[0]!.content[0]!.text), + ).toEqual({ response_text: [fallbackReply, fallbackQuestion] }); + expect(responseMessages).toEqual(responseMessagesSnapshot); + expect(browserRequests).toEqual([ { path: "/api/voice/realtime-call", @@ -378,7 +598,12 @@ describe("controlled voice preview", () => { providerAnswer, spokenAnswer, canonicalReply, + preparedReply, canonicalQuestion, + toolMetadata, + toolError, + fallbackReply, + fallbackQuestion, environment.OPENAI_VOICE_API_KEY, ]) { expect(serializedDiagnostics).not.toContain(privateValue); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.test.ts index d637a762607..c24b89af036 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.test.ts @@ -5,6 +5,8 @@ import { toVoiceSessionState } from "./voice-session-state"; import type { VoiceTurnSnapshot } from "./voice-turn-controller"; const listeningSnapshot = { + canReadFullResponse: true, + canRepeatQuestion: true, canReviseLastAnswer: false, connection: "connected", currentQuestion: "What happens after approval?", @@ -30,6 +32,8 @@ describe("toVoiceSessionState", () => { test("reports a listening turn with its microphone level", () => { expect(mapSnapshot()).toEqual({ + canReadFullResponse: true, + canRepeatQuestion: true, errorMessage: null, microphoneLevel: 0.24, microphoneMuted: false, diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.ts index e1a06464eb0..07d6fede838 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.ts @@ -93,6 +93,8 @@ export const toVoiceSessionState = ({ } return { + canReadFullResponse: snapshot.canReadFullResponse, + canRepeatQuestion: snapshot.canRepeatQuestion, errorMessage: snapshot.connection === "error" ? errorMessageOf(snapshot) : null, microphoneMuted: diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts index 76c09ccf495..b2a730ea3d9 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts @@ -3,7 +3,10 @@ import { describe, expect, test, vi } from "vitest"; import { VoiceError } from "../../../voice-diagnostics"; import { VoiceTurnController } from "./voice-turn-controller"; -import type { CanonicalSpeechSegment } from "./canonical-speech"; +import type { + CanonicalSpeechSegment, + InterviewSpeechSource, +} from "./canonical-speech"; import type { OpenAIRealtimeSessionEvent } from "./openai-realtime-session"; import type { RealtimeBrunchBridgeEvent } from "./realtime-brunch-bridge"; @@ -18,6 +21,7 @@ const createHarness = () => { connect: vi.fn(async () => ++epoch), disconnect: vi.fn(async () => undefined), setMicrophoneEnabled: vi.fn(), + speakCanonical: vi.fn(), subscribe: vi.fn( (listener: (event: OpenAIRealtimeSessionEvent) => void) => { sessionListener = listener; @@ -28,6 +32,7 @@ const createHarness = () => { ), }; const bridge = { + cancelPendingSpeech: vi.fn(), start: vi.fn(), stop: vi.fn(), subscribe: vi.fn((listener: (event: RealtimeBrunchBridgeEvent) => void) => { @@ -64,6 +69,20 @@ const question = ( text, }); +const speechSource = (): InterviewSpeechSource => { + const context = { + ...question("context", "Approval is required before release."), + source: "assistant-text" as const, + }; + const nextQuestion = question("ask-replay", "Who approves release?"); + return { + contextSegments: [context], + fullResponseSegments: [context, nextQuestion], + messageId: "message-replay", + questionSegment: nextQuestion, + }; +}; + describe("VoiceTurnController", () => { test("opens a continuous microphone before starting canonical question speech", async () => { const harness = createHarness(); @@ -115,6 +134,341 @@ describe("VoiceTurnController", () => { output: "interrupted", }); expect(harness.session.cancelOutput).not.toHaveBeenCalled(); + expect(harness.bridge.cancelPendingSpeech).toHaveBeenCalledOnce(); + }); + + test("invalidates pending preparation before pausing or cancelling paused output", async () => { + const harness = createHarness(); + const order: string[] = []; + harness.bridge.cancelPendingSpeech.mockImplementation(() => + order.push("bridge-cancel"), + ); + harness.bridge.updateChat.mockImplementation(() => + order.push("chat-update"), + ); + harness.session.cancelOutput.mockImplementation(() => + order.push("session-cancel"), + ); + await harness.controller.start(); + + harness.controller.pause(); + expect(order).toEqual(["bridge-cancel", "session-cancel"]); + + order.length = 0; + harness.controller.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [question("ask-paused")], + status: "ready", + }); + expect(order).toEqual(["chat-update", "bridge-cancel", "session-cancel"]); + }); + + test("replays canonical speech only when preparation and turn state are safe", async () => { + const harness = createHarness(); + const source = speechSource(); + harness.controller.updateChat({ + automaticSource: source, + canAcceptInterviewAnswer: true, + canonicalSegments: [...source.fullResponseSegments], + status: "ready", + }); + await harness.controller.start(); + + harness.emitBridge({ type: "speech-delivery-pending" }); + expect(harness.controller.getSnapshot()).toMatchObject({ + canReadFullResponse: false, + canRepeatQuestion: false, + }); + harness.controller.repeatQuestion(); + harness.controller.readFullResponse(); + expect(harness.session.speakCanonical).not.toHaveBeenCalled(); + + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-replay-source", + type: "output-started", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-replay-source", + type: "output-stopped", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-replay-source", + status: "completed", + type: "response-terminal", + }); + expect(harness.controller.getSnapshot()).toMatchObject({ + canReadFullResponse: true, + canRepeatQuestion: true, + }); + harness.controller.repeatQuestion(); + harness.controller.readFullResponse(); + + expect(harness.session.speakCanonical).toHaveBeenCalledOnce(); + expect(harness.session.speakCanonical).toHaveBeenCalledWith([ + source.questionSegment, + ]); + expect(harness.controller.getSnapshot()).toMatchObject({ + canReadFullResponse: false, + canRepeatQuestion: false, + output: "waiting-for-tool", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-repeat-question", + type: "output-started", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-repeat-question", + type: "output-stopped", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-repeat-question", + status: "completed", + type: "response-terminal", + }); + harness.controller.readFullResponse(); + + expect(harness.session.speakCanonical).toHaveBeenNthCalledWith( + 2, + source.fullResponseSegments, + ); + }); + + test("keeps replay disabled after barge-in until the input turn settles", async () => { + const harness = createHarness(); + const source = speechSource(); + harness.controller.updateChat({ + automaticSource: source, + canAcceptInterviewAnswer: true, + canonicalSegments: [...source.fullResponseSegments], + status: "ready", + }); + await harness.controller.start(); + harness.emitBridge({ type: "speech-delivery-pending" }); + + harness.emitSession({ + connectionEpoch: 1, + itemId: "item-user", + type: "input-speech-started", + }); + harness.emitSession({ + connectionEpoch: 1, + itemId: "item-user", + type: "input-speech-stopped", + }); + + expect(harness.controller.getSnapshot()).toMatchObject({ + canReadFullResponse: false, + canRepeatQuestion: false, + output: "interrupted", + }); + + harness.emitSession({ + key: { connectionEpoch: 1, contentIndex: 0, itemId: "item-user" }, + text: "The supervisor approves it.", + type: "completed", + }); + + expect(harness.controller.getSnapshot()).toMatchObject({ + canReadFullResponse: true, + canRepeatQuestion: true, + }); + }); + + test("refuses replay after audio stops until the response is terminal", async () => { + const harness = createHarness(); + const source = speechSource(); + harness.controller.updateChat({ + automaticSource: source, + canAcceptInterviewAnswer: true, + canonicalSegments: [...source.fullResponseSegments], + status: "ready", + }); + await harness.controller.start(); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-source", + type: "output-started", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-source", + type: "output-stopped", + }); + + harness.controller.repeatQuestion(); + expect(harness.session.speakCanonical).not.toHaveBeenCalled(); + + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-source", + status: "completed", + type: "response-terminal", + }); + harness.controller.repeatQuestion(); + expect(harness.session.speakCanonical).toHaveBeenCalledWith([ + source.questionSegment, + ]); + }); + + test("refuses replay when an unrelated response becomes terminal", async () => { + const harness = createHarness(); + const source = speechSource(); + harness.controller.updateChat({ + automaticSource: source, + canAcceptInterviewAnswer: true, + canonicalSegments: [...source.fullResponseSegments], + status: "ready", + }); + await harness.controller.start(); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-a", + type: "output-started", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-a", + type: "output-stopped", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-b", + status: "completed", + type: "response-terminal", + }); + + harness.controller.repeatQuestion(); + expect(harness.session.speakCanonical).not.toHaveBeenCalled(); + + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-a", + status: "completed", + type: "response-terminal", + }); + harness.controller.repeatQuestion(); + expect(harness.session.speakCanonical).toHaveBeenCalledWith([ + source.questionSegment, + ]); + }); + + test("refuses interrupted replay until the response is terminal", async () => { + const harness = createHarness(); + const source = speechSource(); + harness.controller.updateChat({ + automaticSource: source, + canAcceptInterviewAnswer: true, + canonicalSegments: [...source.fullResponseSegments], + status: "ready", + }); + await harness.controller.start(); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-source", + type: "output-started", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-source", + type: "output-interrupted", + }); + + harness.controller.readFullResponse(); + expect(harness.session.speakCanonical).not.toHaveBeenCalled(); + + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-source", + status: "cancelled", + type: "response-terminal", + }); + harness.controller.readFullResponse(); + expect(harness.session.speakCanonical).toHaveBeenCalledWith( + source.fullResponseSegments, + ); + }); + + test("keeps replay unavailable when terminal arrives before audio stops", async () => { + const harness = createHarness(); + const source = speechSource(); + harness.controller.updateChat({ + automaticSource: source, + canAcceptInterviewAnswer: true, + canonicalSegments: [...source.fullResponseSegments], + status: "ready", + }); + await harness.controller.start(); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-source", + type: "output-started", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-source", + status: "completed", + type: "response-terminal", + }); + + harness.controller.repeatQuestion(); + expect(harness.session.speakCanonical).not.toHaveBeenCalled(); + + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-source", + type: "output-stopped", + }); + harness.controller.repeatQuestion(); + expect(harness.session.speakCanonical).toHaveBeenCalledWith([ + source.questionSegment, + ]); + }); + + test("refuses replay when unrelated audio stops after the tracked response is terminal", async () => { + const harness = createHarness(); + const source = speechSource(); + harness.controller.updateChat({ + automaticSource: source, + canAcceptInterviewAnswer: true, + canonicalSegments: [...source.fullResponseSegments], + status: "ready", + }); + await harness.controller.start(); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-a", + type: "output-started", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-a", + status: "completed", + type: "response-terminal", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-b", + type: "output-stopped", + }); + + harness.controller.repeatQuestion(); + expect(harness.session.speakCanonical).not.toHaveBeenCalled(); + + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-a", + type: "output-stopped", + }); + harness.controller.repeatQuestion(); + expect(harness.session.speakCanonical).toHaveBeenCalledWith([ + source.questionSegment, + ]); }); test("represents submitting and output independently without closing capture", async () => { diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts index 4277717f7aa..666f9745ffc 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts @@ -1,6 +1,9 @@ import { VoiceError, type VoiceErrorCode } from "../../../voice-diagnostics"; -import type { CanonicalSpeechSegment } from "./canonical-speech"; +import type { + CanonicalSpeechSegment, + InterviewSpeechSource, +} from "./canonical-speech"; import type { OpenAIRealtimeSessionEvent } from "./openai-realtime-session"; import type { RealtimeBridgeErrorCode, @@ -21,6 +24,8 @@ export type VoiceOutputState = export type VoiceAnswerDelivery = "none" | "pending" | "delivered" | "failed"; export interface VoiceTurnSnapshot { + readonly canReadFullResponse: boolean; + readonly canRepeatQuestion: boolean; readonly canReviseLastAnswer: boolean; readonly connection: VoiceConnectionState; readonly currentQuestion: string; @@ -53,10 +58,12 @@ interface RealtimeSession { connect(): Promise; disconnect(): Promise; setMicrophoneEnabled(enabled: boolean): void; + speakCanonical(segments: CanonicalSpeechSegment[]): void; subscribe(listener: (event: OpenAIRealtimeSessionEvent) => void): () => void; } interface RealtimeBridge { + cancelPendingSpeech(): void; start(connectionEpoch: number): void; stop(): void; subscribe(listener: (event: RealtimeBrunchBridgeEvent) => void): () => void; @@ -77,6 +84,7 @@ interface VoiceTurnControllerDependencies { } interface ChatUpdate { + readonly automaticSource?: InterviewSpeechSource | null; readonly canAcceptInterviewAnswer: boolean; readonly canonicalSegments: CanonicalSpeechSegment[]; readonly status: ChatStatus; @@ -85,6 +93,8 @@ interface ChatUpdate { type SnapshotListener = (snapshot: VoiceTurnSnapshot) => void; const initialSnapshot: VoiceTurnSnapshot = { + canReadFullResponse: false, + canRepeatQuestion: false, canReviseLastAnswer: false, connection: "idle", currentQuestion: "", @@ -113,14 +123,19 @@ export class VoiceTurnController { readonly #session: RealtimeSession; readonly #submitText: (input: SubmitTextInput) => Promise; #activeEpoch: number | null = null; + #activeSpeechOutputEnded = false; + #activeSpeechResponseId: string | null = null; + #activeSpeechResponseTerminal = false; #answerFinalizedAt: number | null = null; #answeredQuestionId: string | null = null; #bridgeStarted = false; #currentQuestionId: string | null = null; #generation = 0; #inputStateOnResume: Exclude | null = null; + #inputTurnPending = false; #pauseRequested = false; #snapshot = initialSnapshot; + #speechSource: InterviewSpeechSource | null = null; #submittingQuestionId: string | null = null; #teardownPromise: Promise | null = null; #transcriptItemId: string | null = null; @@ -180,8 +195,12 @@ export class VoiceTurnController { } this.#inputStateOnResume = null; + this.#inputTurnPending = false; this.#pauseRequested = false; this.#bridgeStarted = false; + this.#activeSpeechOutputEnded = false; + this.#activeSpeechResponseId = null; + this.#activeSpeechResponseTerminal = false; this.#update({ connection: "connecting", errorCode: null, @@ -219,11 +238,15 @@ export class VoiceTurnController { public async end(): Promise { ++this.#generation; this.#activeEpoch = null; + this.#activeSpeechOutputEnded = false; + this.#activeSpeechResponseId = null; + this.#activeSpeechResponseTerminal = false; this.#answerFinalizedAt = null; this.#answeredQuestionId = null; this.#bridgeStarted = false; this.#currentQuestionId = null; this.#inputStateOnResume = null; + this.#inputTurnPending = false; this.#submittingQuestionId = null; this.#pauseRequested = false; this.#transcriptItemId = null; @@ -274,7 +297,7 @@ export class VoiceTurnController { this.#inputStateOnResume = this.#snapshot.input; this.#pauseRequested = true; const output = this.#snapshot.output === "idle" ? "idle" : "interrupted"; - this.#session.cancelOutput(); + this.#cancelOutput(); this.#session.setMicrophoneEnabled(false); this.#update({ input: "paused", @@ -364,16 +387,44 @@ export class VoiceTurnController { } } + public readFullResponse(): void { + const source = this.#speechSource; + if (!source || !this.#snapshot.canReadFullResponse) { + return; + } + this.#update({ output: "waiting-for-tool" }); + this.#session.speakCanonical([...source.fullResponseSegments]); + } + + public repeatQuestion(): void { + const question = this.#speechSource?.questionSegment; + if (!question || !this.#snapshot.canRepeatQuestion) { + return; + } + this.#update({ output: "waiting-for-tool" }); + this.#session.speakCanonical([question]); + } + public updateChat(update: ChatUpdate): void { + this.#speechSource = update.automaticSource ?? null; + const canReplay = this.#canReplay(this.#snapshot); + const replayAvailabilityChanged = + this.#snapshot.canReadFullResponse !== + (canReplay && + Boolean(this.#speechSource?.fullResponseSegments.length)) || + this.#snapshot.canRepeatQuestion !== + (canReplay && Boolean(this.#speechSource?.questionSegment)); const question = latestQuestion(update.canonicalSegments); if (question && question.id !== this.#currentQuestionId) { this.#currentQuestionId = question.id; this.#update({ currentQuestion: question.text }); this.#recordLatency("question-visible", question.id); + } else if (replayAvailabilityChanged) { + this.#update({}); } this.#bridge.updateChat(update); if (this.#snapshot.input === "paused") { - this.#session.cancelOutput(); + this.#cancelOutput(); } } @@ -388,6 +439,7 @@ export class VoiceTurnController { if (paused) { this.#inputStateOnResume = "submitting"; } + this.#inputTurnPending = false; this.#answerFinalizedAt = this.#now(); this.#submittingQuestionId = this.#currentQuestionId; this.#transcriptItemId = null; @@ -407,10 +459,14 @@ export class VoiceTurnController { this.#update({ lastAnswerDelivery: "delivered" }); return; } + if (event.type === "speech-delivery-pending") { + this.#update({ output: "waiting-for-tool" }); + return; + } const paused = this.#snapshot.input === "paused"; if (paused) { this.#inputStateOnResume = "listening"; - this.#session.cancelOutput(); + this.#cancelOutput(); } this.#update({ input: paused ? "paused" : "listening", @@ -440,8 +496,11 @@ export class VoiceTurnController { return; } if (event.type === "output-started") { + this.#activeSpeechOutputEnded = false; + this.#activeSpeechResponseId = event.responseId; + this.#activeSpeechResponseTerminal = false; if (this.#snapshot.input === "paused") { - this.#session.cancelOutput(); + this.#cancelOutput(); this.#update({ output: "interrupted" }); return; } @@ -452,6 +511,13 @@ export class VoiceTurnController { return; } if (event.type === "output-stopped") { + if (event.responseId !== this.#activeSpeechResponseId) return; + this.#activeSpeechOutputEnded = true; + if (this.#activeSpeechResponseTerminal) { + this.#activeSpeechOutputEnded = false; + this.#activeSpeechResponseId = null; + this.#activeSpeechResponseTerminal = false; + } this.#update({ output: "idle" }); if (this.#currentQuestionId) { this.#recordLatency("question-spoken", this.#currentQuestionId); @@ -459,22 +525,43 @@ export class VoiceTurnController { return; } if (event.type === "output-interrupted") { + if (event.responseId !== this.#activeSpeechResponseId) return; + this.#activeSpeechOutputEnded = true; + if (this.#activeSpeechResponseTerminal) { + this.#activeSpeechOutputEnded = false; + this.#activeSpeechResponseId = null; + this.#activeSpeechResponseTerminal = false; + } this.#update({ output: "interrupted" }); return; } if (event.type === "input-speech-started") { + this.#inputTurnPending = true; + this.#bridge.cancelPendingSpeech(); this.#transcriptItemId = event.itemId; this.#transcriptKey = null; - if (this.#snapshot.output === "speaking") { + if (this.#snapshot.output !== "idle") { this.#update({ output: "interrupted", partialText: "" }); } else { this.#update({ partialText: "" }); } return; } + if (event.type === "response-terminal") { + if (event.responseId === this.#activeSpeechResponseId) { + if (this.#activeSpeechOutputEnded) { + this.#activeSpeechOutputEnded = false; + this.#activeSpeechResponseId = null; + this.#activeSpeechResponseTerminal = false; + } else { + this.#activeSpeechResponseTerminal = true; + } + this.#update({}); + } + return; + } if ( event.type === "input-speech-stopped" || - event.type === "response-terminal" || event.type === "tool-arguments-delta" || event.type === "tool-arguments-done" ) { @@ -485,6 +572,7 @@ export class VoiceTurnController { if (event.key.connectionEpoch !== this.#activeEpoch) return; if (event.key.itemId !== this.#transcriptItemId) return; if (event.type === "transcription-failed") { + this.#inputTurnPending = false; this.#transcriptItemId = null; this.#transcriptKey = null; this.#update({ partialText: "" }); @@ -498,6 +586,7 @@ export class VoiceTurnController { }); return; } + this.#inputTurnPending = false; this.#transcriptItemId = null; this.#transcriptKey = null; this.#update({ @@ -512,7 +601,11 @@ export class VoiceTurnController { ): void { ++this.#generation; this.#activeEpoch = null; + this.#activeSpeechOutputEnded = false; + this.#activeSpeechResponseId = null; + this.#activeSpeechResponseTerminal = false; this.#inputStateOnResume = null; + this.#inputTurnPending = false; this.#bridgeStarted = false; this.#transcriptItemId = null; this.#transcriptKey = null; @@ -536,6 +629,11 @@ export class VoiceTurnController { }); } + #cancelOutput(): void { + this.#bridge.cancelPendingSpeech(); + this.#session.cancelOutput(); + } + #recordLatency(name: VoiceLatencyEvent["name"], questionId: string): void { if (this.#answerFinalizedAt === null) return; this.#onLatencyEvent?.({ @@ -555,6 +653,16 @@ export class VoiceTurnController { ); } + #canReplay(snapshot: VoiceTurnSnapshot): boolean { + return ( + snapshot.connection === "connected" && + snapshot.input === "listening" && + !this.#inputTurnPending && + this.#activeSpeechResponseId === null && + (snapshot.output === "idle" || snapshot.output === "interrupted") + ); + } + #isPauseRequested(): boolean { return this.#pauseRequested; } @@ -563,6 +671,12 @@ export class VoiceTurnController { const snapshot = { ...this.#snapshot, ...update }; this.#snapshot = { ...snapshot, + canReadFullResponse: + this.#canReplay(snapshot) && + Boolean(this.#speechSource?.fullResponseSegments.length), + canRepeatQuestion: + this.#canReplay(snapshot) && + Boolean(this.#speechSource?.questionSegment), canReviseLastAnswer: this.#canReviseLastAnswer(snapshot), }; for (const listener of this.#listeners) listener(this.#snapshot); diff --git a/apps/petrinaut-website/src/voice-diagnostics.ts b/apps/petrinaut-website/src/voice-diagnostics.ts index be74ef0b911..e7eadb40d82 100644 --- a/apps/petrinaut-website/src/voice-diagnostics.ts +++ b/apps/petrinaut-website/src/voice-diagnostics.ts @@ -12,14 +12,21 @@ export const voiceErrorCodes = [ ] as const; export type VoiceErrorCode = (typeof voiceErrorCodes)[number]; -export type VoiceOperation = "connection" | "transcription" | "speech"; +export type VoiceOperation = + | "connection" + | "preparation" + | "transcription" + | "speech"; export interface VoiceDiagnosticEvent { readonly durationMs: number; readonly errorCode?: VoiceErrorCode; + readonly inputWordCount?: number; readonly operation: VoiceOperation; readonly outcome: "success" | "failure" | "aborted"; + readonly outputWordCount?: number; readonly requestId: string; + readonly sourceSegmentCount?: number; readonly stage: "browser" | "playback" | "server"; readonly status?: number; } diff --git a/libs/@hashintel/petrinaut/docs/ai-assistant.md b/libs/@hashintel/petrinaut/docs/ai-assistant.md index 4d4d3828282..6fca195ecfa 100644 --- a/libs/@hashintel/petrinaut/docs/ai-assistant.md +++ b/libs/@hashintel/petrinaut/docs/ai-assistant.md @@ -72,15 +72,20 @@ Every session control lives in the dock: **Show transcription in chat** on the l sending audio without ending the turn, so the assistant plays out whatever it is saying and unmuting drops you straight back into the conversation. **Resume voice mode** replaces the microphone action while a session is paused, and **Reconnect voice mode** replaces it after a failure. Nothing is added -to the canvas toolbar. Sending non-empty typed text from the -composer or first-run prompt ends Voice mode before it sends the message once through the same -conversation; repeated send actions are ignored while that short handoff completes. +to the canvas toolbar. To send a typed message, first select **End voice mode** to restore the +composer. The interviewer uses a warm, calm, curious, and professionally neutral voice and treats you as the -authority on your system. Brunch still chooses every question and interview decision; OpenAI only -delivers its words. The question and finalized response shown in the Petrinaut conversation are -authoritative. Spoken audio is generated from that Brunch text but may not be verbatim. Interrupting -audio does not undo the visible response or change the interview's saved history. +authority on your system. Brunch still chooses every question and interview decision. Its exact +question and canonical response in the conversation remain unchanged and authoritative. Voice may +prepare a shorter context for listening, with a strict 50-word allowance that includes the protected +question. The exact question is always appended to that context before it is spoken. If preparation +is unavailable, invalid, or takes too long, Voice reads Brunch's canonical context and question. +Preparation and generated audio are ephemeral and never become chat history. + +Beside **Show transcription in chat**, the compact **Voice playback options** menu offers **Repeat +question** and **Read full response**. These replay Brunch's exact canonical question or its canonical +context and question, bypassing preparation. Closing the AI panel pauses microphone capture and active speech, then hides the dock until you reopen the panel. The same mounted session stays paused; choose **Resume voice mode** when you are diff --git a/libs/@hashintel/petrinaut/src/react/voice-session/store.ts b/libs/@hashintel/petrinaut/src/react/voice-session/store.ts index bcdc4e0ee48..345e080b176 100644 --- a/libs/@hashintel/petrinaut/src/react/voice-session/store.ts +++ b/libs/@hashintel/petrinaut/src/react/voice-session/store.ts @@ -7,7 +7,9 @@ import type { export type VoiceSessionActions = { end: () => void; pause: () => void; + readFullResponse?: () => void; reconnect: () => void; + repeatQuestion?: () => void; resume: () => void; setMicrophoneMuted: (muted: boolean) => void; }; diff --git a/libs/@hashintel/petrinaut/src/react/voice-session/types.ts b/libs/@hashintel/petrinaut/src/react/voice-session/types.ts index bd1a26f425e..1ee8e679bd2 100644 --- a/libs/@hashintel/petrinaut/src/react/voice-session/types.ts +++ b/libs/@hashintel/petrinaut/src/react/voice-session/types.ts @@ -16,6 +16,10 @@ export type PetrinautAiVoiceSessionPhase = * effect: it changes at microphone-sampling rate. */ export type PetrinautAiVoiceSessionState = { + /** Whether the current canonical assistant response is safe to replay. */ + canReadFullResponse?: boolean; + /** Whether the current canonical interview question is safe to repeat. */ + canRepeatQuestion?: boolean; errorMessage: string | null; /** Whether microphone capture is muted independently of whose turn it is. */ microphoneMuted: boolean; diff --git a/libs/@hashintel/petrinaut/src/react/voice-session/use-voice-session.ts b/libs/@hashintel/petrinaut/src/react/voice-session/use-voice-session.ts index a95671ce61d..916c5a39d17 100644 --- a/libs/@hashintel/petrinaut/src/react/voice-session/use-voice-session.ts +++ b/libs/@hashintel/petrinaut/src/react/voice-session/use-voice-session.ts @@ -62,3 +62,23 @@ export const useVoiceSessionActions = (): VoiceSessionActions | null => { () => null, ); }; + +export const useVoiceSessionCanReadFullResponse = (): boolean => { + const store = use(VoiceSessionContext); + + return useSyncExternalStore( + store.subscribe, + () => store.getSnapshot().state?.canReadFullResponse ?? false, + () => false, + ); +}; + +export const useVoiceSessionCanRepeatQuestion = (): boolean => { + const store = use(VoiceSessionContext); + + return useSyncExternalStore( + store.subscribe, + () => store.getSnapshot().state?.canRepeatQuestion ?? false, + () => false, + ); +}; diff --git a/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts b/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts index 5369be4c639..c2778346770 100644 --- a/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts +++ b/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts @@ -57,8 +57,12 @@ export type PetrinautAiVoiceModeControls = { end: () => Promise; /** Pauses microphone capture and active Voice output synchronously. */ pause: () => void; + /** Reads the complete canonical assistant response without rewriting it. */ + readFullResponse?: () => void; /** Re-establishes a session that dropped, keeping the conversation. */ reconnect: () => void; + /** Repeats only the current canonical interview question. */ + repeatQuestion?: () => void; /** Resumes microphone capture after `pause`. */ resume: () => void; /** diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/components/voice-session-labels.ts b/libs/@hashintel/petrinaut/src/ui/views/Editor/components/voice-session-labels.ts index eaf5dabb078..f4ea8f4fa3e 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/components/voice-session-labels.ts +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/components/voice-session-labels.ts @@ -29,7 +29,10 @@ export const voiceSessionActionLabels = { end: "End voice mode", mute: "Mute microphone", pause: "Pause voice mode", + playbackOptions: "Voice playback options", + readFullResponse: "Read full response", reconnect: "Reconnect voice mode", + repeatQuestion: "Repeat question", resume: "Resume voice mode", unmute: "Unmute microphone", } as const; diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx index b197c585cfd..eab9bafbff7 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx @@ -10,7 +10,7 @@ import { waitFor, } from "@testing-library/react"; import { useEffect } from "react"; -import { afterEach, describe, expect, test, vi } from "vitest"; +import { afterEach, beforeAll, describe, expect, test, vi } from "vitest"; import { DEFAULT_PETRINAUT_EXTENSIONS, @@ -156,6 +156,29 @@ const SubmitForSecondConversation = ({ const testInstances: ReturnType[] = []; +beforeAll(() => { + vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue(null); + vi.stubGlobal( + "PointerEvent", + class extends MouseEvent { + public readonly pointerType: string; + + public constructor(type: string, init: PointerEventInit = {}) { + super(type, init); + this.pointerType = init.pointerType ?? ""; + } + }, + ); + vi.stubGlobal( + "ResizeObserver", + class { + public disconnect() {} + public observe() {} + public unobserve() {} + }, + ); +}); + const renderTestPanel = ({ aiAssistant, editorContext = editorContextValue, @@ -1032,6 +1055,96 @@ describe("AiAssistantPanel composer submissions", () => { expect(submitVoiceInputReferences.size).toBe(1); }); + test("forwards registered canonical replay controls to the Voice dock", async () => { + const readFullResponse = vi.fn(); + const repeatQuestion = vi.fn(); + const VoiceMode = (context: PetrinautAiVoiceModeContext) => { + const { + inputMode, + registerVoiceModeControls, + reportVoiceSessionState, + setVoiceActive, + } = context; + + useEffect(() => { + if (inputMode !== "voice") { + return; + } + const unregister = registerVoiceModeControls({ + end: async () => undefined, + pause: vi.fn(), + readFullResponse, + reconnect: vi.fn(), + repeatQuestion, + resume: vi.fn(), + setMicrophoneMuted: vi.fn(), + }); + reportVoiceSessionState({ + canReadFullResponse: true, + canRepeatQuestion: true, + errorMessage: null, + microphoneLevel: 0, + microphoneMuted: false, + phase: "listening", + }); + setVoiceActive(true); + + return unregister; + }, [ + inputMode, + registerVoiceModeControls, + reportVoiceSessionState, + setVoiceActive, + ]); + + return null; + }; + + renderTestPanel({ + aiAssistant: { + renderVoiceMode: (context) => , + transport: { + reconnectToStream: () => Promise.resolve(null), + sendMessages: vi.fn(), + }, + }, + }); + + fireEvent.click(screen.getByRole("button", { name: "Start voice mode" })); + fireEvent.click( + screen.getByRole("button", { name: "Voice playback options" }), + ); + const readFullResponseItem = await screen.findByRole("menuitem", { + name: "Read full response", + }); + fireEvent.pointerMove(readFullResponseItem, { pointerType: "mouse" }); + await waitFor(() => + expect(readFullResponseItem.hasAttribute("data-highlighted")).toBe(true), + ); + fireEvent.click(readFullResponseItem); + await waitFor(() => expect(readFullResponse).toHaveBeenCalledOnce()); + await waitFor(() => + expect(screen.queryByRole("menu", { hidden: true })).toBeNull(), + ); + fireEvent.click( + screen.getByRole("button", { name: "Voice playback options" }), + ); + const repeatQuestionItem = await screen.findByRole("menuitem", { + name: "Repeat question", + }); + const playbackMenu = screen.getByRole("menu"); + fireEvent.keyDown(playbackMenu, { key: "ArrowDown" }); + await waitFor(() => + expect(playbackMenu.getAttribute("aria-activedescendant")).toBe( + repeatQuestionItem.id, + ), + ); + fireEvent.keyDown(playbackMenu, { key: "Enter" }); + + await waitFor(() => expect(repeatQuestion).toHaveBeenCalledOnce()); + expect(readFullResponse).toHaveBeenCalledOnce(); + }); + test("keeps one mounted voice mode when the panel closes and reopens", () => { voiceModeMounts = 0; voiceModeUnmounts = 0; diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx index df83042ed1b..cc4f11010db 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx @@ -486,7 +486,13 @@ export const AiAssistantPanel = ({ // invalidates the host's active generation. end: () => requestInputMode("text"), pause: () => controls.pause(), + ...(controls.readFullResponse + ? { readFullResponse: controls.readFullResponse } + : {}), reconnect: () => controls.reconnect(), + ...(controls.repeatQuestion + ? { repeatQuestion: controls.repeatQuestion } + : {}), resume: () => controls.resume(), setMicrophoneMuted: (muted) => controls.setMicrophoneMuted(muted), }); diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.stories.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.stories.tsx index c21fe5af927..f6687f6ac18 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.stories.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.stories.tsx @@ -304,6 +304,8 @@ const Frame = ({ const liveSession = ( overrides: Partial, ): PetrinautAiVoiceSessionState => ({ + canReadFullResponse: true, + canRepeatQuestion: true, errorMessage: null, microphoneLevel: 0, microphoneMuted: false, diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx index ee2d2d8c6f7..84af4b577f0 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx @@ -50,6 +50,25 @@ const noop = () => {}; // instead of letting jsdom log a not-implemented error per render. beforeAll(() => { vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue(null); + vi.stubGlobal( + "PointerEvent", + class extends MouseEvent { + public readonly pointerType: string; + + public constructor(type: string, init: PointerEventInit = {}) { + super(type, init); + this.pointerType = init.pointerType ?? ""; + } + }, + ); + vi.stubGlobal( + "ResizeObserver", + class { + public disconnect() {} + public observe() {} + public unobserve() {} + }, + ); }); afterEach(() => { @@ -142,12 +161,16 @@ describe("AiAssistantContents", () => { const actions = { end: vi.fn(), pause: vi.fn(), + readFullResponse: vi.fn(), reconnect: vi.fn(), + repeatQuestion: vi.fn(), resume: vi.fn(), setMicrophoneMuted: vi.fn(), }; store.setActions(actions); store.setState({ + canReadFullResponse: true, + canRepeatQuestion: true, errorMessage: null, microphoneLevel: 0, microphoneMuted: false, @@ -234,17 +257,21 @@ describe("AiAssistantContents", () => { ).not.toBeNull(); }); - test("keeps the session's controls in the dock", () => { + test("keeps the session's controls in the dock", async () => { const store = createVoiceSessionStore(); const actions = { end: vi.fn(), pause: vi.fn(), + readFullResponse: vi.fn(), reconnect: vi.fn(), + repeatQuestion: vi.fn(), resume: vi.fn(), setMicrophoneMuted: vi.fn(), }; store.setActions(actions); store.setState({ + canReadFullResponse: true, + canRepeatQuestion: true, errorMessage: null, microphoneLevel: 0.4, microphoneMuted: false, @@ -277,6 +304,39 @@ describe("AiAssistantContents", () => { expect(actions.setMicrophoneMuted).toHaveBeenCalledWith(true); expect(actions.end).toHaveBeenCalledOnce(); + fireEvent.click( + within(dock).getByRole("button", { name: "Voice playback options" }), + ); + const readFullResponse = await screen.findByRole("menuitem", { + name: "Read full response", + }); + expect(readFullResponse.hasAttribute("data-disabled")).toBe(false); + fireEvent.pointerMove(readFullResponse, { pointerType: "mouse" }); + await waitFor(() => + expect(readFullResponse.hasAttribute("data-highlighted")).toBe(true), + ); + fireEvent.click(readFullResponse); + await waitFor(() => + expect(actions.readFullResponse).toHaveBeenCalledOnce(), + ); + fireEvent.click( + within(dock).getByRole("button", { name: "Voice playback options" }), + ); + const repeatQuestion = await screen.findByRole("menuitem", { + name: "Repeat question", + }); + const playbackMenu = screen.getByRole("menu"); + fireEvent.keyDown(playbackMenu, { key: "ArrowDown" }); + await waitFor(() => + expect(playbackMenu.getAttribute("aria-activedescendant")).toBe( + repeatQuestion.id, + ), + ); + fireEvent.keyDown(playbackMenu, { key: "Enter" }); + + await waitFor(() => expect(actions.repeatQuestion).toHaveBeenCalledOnce()); + expect(actions.readFullResponse).toHaveBeenCalledOnce(); + act(() => { store.setState({ errorMessage: null, @@ -294,6 +354,53 @@ describe("AiAssistantContents", () => { expect(actions.setMicrophoneMuted).toHaveBeenLastCalledWith(false); }); + test("disables unavailable Voice playback actions", async () => { + const store = createVoiceSessionStore(); + store.setActions({ + end: vi.fn(), + pause: vi.fn(), + reconnect: vi.fn(), + resume: vi.fn(), + setMicrophoneMuted: vi.fn(), + }); + store.setState({ + canReadFullResponse: false, + canRepeatQuestion: false, + errorMessage: null, + microphoneLevel: 0, + microphoneMuted: false, + phase: "listening", + }); + render( + + + , + ); + + fireEvent.click( + screen.getByRole("button", { name: "Voice playback options" }), + ); + + expect( + ( + await screen.findByRole("menuitem", { name: "Repeat question" }) + ).hasAttribute("data-disabled"), + ).toBe(true); + expect( + screen + .getByRole("menuitem", { name: "Read full response" }) + .hasAttribute("data-disabled"), + ).toBe(true); + }); + test("shows a voice recovery failure as a toast", async () => { const store = createVoiceSessionStore(); store.setState({ diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/voice-dock.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/voice-dock.tsx index db2177be73f..823bbed7fac 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/voice-dock.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/voice-dock.tsx @@ -3,6 +3,8 @@ import { css, cva } from "@hashintel/ds-helpers/css"; import { useVoiceSessionActions, + useVoiceSessionCanReadFullResponse, + useVoiceSessionCanRepeatQuestion, useVoiceSessionMicrophoneMuted, useVoiceSessionPhase, } from "../../../../../../react/voice-session/use-voice-session"; @@ -13,6 +15,7 @@ import { } from "../../../components/voice-session-labels"; import { aiFooterMinHeight } from "./footer-height"; import { MicrophoneIcon } from "./voice-dock/microphone-icon"; +import { VoicePlaybackMenu } from "./voice-dock/playback-menu"; import { TranscriptionIcon } from "./voice-dock/transcription-icon"; import type { VoiceSessionActions } from "../../../../../../react/voice-session/store"; @@ -100,6 +103,8 @@ const visuallyHiddenStyle = css({ export type VoiceDockProps = { actions: VoiceSessionActions | null; + canReadFullResponse: boolean; + canRepeatQuestion: boolean; /** Rendered instead of the live indicator when the caller supplies one. */ indicator?: ReactNode; microphoneMuted: boolean; @@ -117,6 +122,8 @@ export type VoiceDockProps = { */ export const VoiceDock = ({ actions, + canReadFullResponse, + canRepeatQuestion, indicator, microphoneMuted, onTranscriptionToggle, @@ -139,16 +146,23 @@ export const VoiceDock = ({ > {actions !== null && ( -