From 9ad0fa537cb09d2b2110480d49c4fdec1d5eef8b Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Wed, 2 Sep 2026 12:02:41 +0200 Subject: [PATCH 01/15] Separate Voice context from canonical questions Amp-Thread-ID: https://ampcode.com/threads/T-01a0618e-4184-736b-a426-c802f086dc83 Co-authored-by: Amp --- .../voice-interview/canonical-speech.test.ts | 183 ++++++++++++++++++ .../app/voice-interview/canonical-speech.ts | 71 +++++-- 2 files changed, 236 insertions(+), 18 deletions(-) diff --git a/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.test.ts index dc12cda21c6..a82a7e6fb43 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.test.ts @@ -5,6 +5,7 @@ import { ASK_TOOL_NAME } from "@hashintel/brunch-agent/client-tools"; import { hashCanonicalSpeechText, selectCanonicalSpeechSegments, + selectInterviewSpeech, } from "./canonical-speech"; import type { PetrinautAiMessage } from "@hashintel/petrinaut/ui"; @@ -13,6 +14,171 @@ const select = (messages: PetrinautAiMessage[]) => selectCanonicalSpeechSegments(messages); describe("canonical speech selection", () => { + test("separates finalized context from an exact pending question", () => { + const completeCanonicalExplanation = + "The release needs one named approver before the batch can proceed."; + const question = "Who approves it: the manager or quality lead?"; + const messages = [ + { + id: "assistant-turn", + role: "assistant", + parts: [ + { + type: "text", + text: completeCanonicalExplanation, + state: "done", + }, + { + type: "dynamic-tool", + toolCallId: "ask-1", + toolName: ASK_TOOL_NAME, + state: "input-available", + input: { question }, + }, + ], + }, + ] satisfies PetrinautAiMessage[]; + + const selection = selectInterviewSpeech(messages); + + expect(selection.automaticSource).toMatchObject({ + contextSegments: [ + { source: "assistant-text", text: completeCanonicalExplanation }, + ], + messageId: "assistant-turn", + questionSegment: { source: "brunch-ask", text: question }, + fullResponseSegments: [ + { source: "assistant-text", text: completeCanonicalExplanation }, + { source: "brunch-ask", text: question }, + ], + }); + expect(selection.canonicalSegments).toEqual( + selection.automaticSource?.fullResponseSegments, + ); + }); + + test("groups multiple finalized text parts into the latest assistant turn", () => { + const selection = selectInterviewSpeech([ + { + id: "assistant-old", + role: "assistant", + parts: [{ type: "text", text: "Previous turn.", state: "done" }], + }, + { + id: "assistant-latest", + role: "assistant", + parts: [ + { type: "text", text: "First explanation.", state: "done" }, + { type: "text", text: "Second explanation.", state: "done" }, + ], + }, + ]); + + expect(selection.automaticSource).toMatchObject({ + contextSegments: [ + { text: "First explanation." }, + { text: "Second explanation." }, + ], + fullResponseSegments: [ + { text: "First explanation." }, + { text: "Second explanation." }, + ], + messageId: "assistant-latest", + questionSegment: null, + }); + expect(selection.canonicalSegments).toHaveLength(3); + }); + + test("selects standalone finalized assistant text for automatic speech", () => { + const selection = selectInterviewSpeech([ + { + id: "assistant-complete", + role: "assistant", + parts: [ + { type: "text", text: "The interview is complete.", state: "done" }, + ], + }, + ]); + + expect(selection.automaticSource).toMatchObject({ + contextSegments: [{ text: "The interview is complete." }], + fullResponseSegments: [{ text: "The interview is complete." }], + questionSegment: null, + }); + }); + + test("excludes answered and malformed asks from the protected question", () => { + const selection = selectInterviewSpeech([ + { + id: "assistant-ask", + role: "assistant", + parts: [ + { type: "text", text: "Canonical context.", state: "done" }, + { + type: "dynamic-tool", + toolCallId: "ask-answered", + toolName: ASK_TOOL_NAME, + state: "output-available", + input: { question: "Already answered?" }, + output: { answer: "Yes." }, + }, + { + type: "dynamic-tool", + toolCallId: "ask-malformed", + toolName: ASK_TOOL_NAME, + state: "input-available", + input: { question: 42 }, + }, + ], + }, + ]); + + expect(selection.automaticSource).toMatchObject({ + contextSegments: [{ text: "Canonical context." }], + fullResponseSegments: [{ text: "Canonical context." }], + questionSegment: null, + }); + expect(selection.canonicalSegments).toHaveLength(1); + }); + + test("excludes streaming text, reasoning, diagnostics, tool output, and tool errors", () => { + const selection = selectInterviewSpeech([ + { + id: "assistant-filtered", + role: "assistant", + parts: [ + { + type: "text", + text: "Do not speak streaming text.", + state: "streaming", + }, + { type: "reasoning", text: "Do not speak reasoning.", state: "done" }, + { + type: "dynamic-tool", + toolCallId: "tool-output", + toolName: "diagnostic", + state: "output-available", + input: {}, + output: { text: "Do not speak tool output." }, + }, + { + type: "dynamic-tool", + toolCallId: "tool-error", + toolName: "diagnostic", + state: "output-error", + input: {}, + errorText: "Do not speak tool errors.", + }, + ], + }, + ]); + + expect(selection).toEqual({ + automaticSource: null, + canonicalSegments: [], + }); + }); + test("selects only finalized assistant text without changing it", () => { const messages = [ { @@ -174,5 +340,22 @@ describe("canonical speech selection", () => { expect(changed[0]?.partId).toBe(first[0]?.partId); expect(changed[0]?.contentHash).not.toBe(first[0]?.contentHash); expect(changed[0]?.id).not.toBe(first[0]?.id); + + const firstSelection = selectInterviewSpeech([ + { + id: "assistant/id", + role: "assistant", + parts: [{ type: "text", text: "Exact text", state: "done" }], + }, + ]); + expect( + selectInterviewSpeech([ + { + id: "assistant/id", + role: "assistant", + parts: [{ type: "text", text: "Exact text", state: "done" }], + }, + ]), + ).toEqual(firstSelection); }); }); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.ts b/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.ts index bb111385b7e..2972a03a1dc 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.ts @@ -18,6 +18,18 @@ export interface CanonicalSpeechSegment { readonly text: string; } +export interface InterviewSpeechSource { + readonly contextSegments: readonly CanonicalSpeechSegment[]; + readonly fullResponseSegments: readonly CanonicalSpeechSegment[]; + readonly messageId: string; + readonly questionSegment: CanonicalSpeechSegment | null; +} + +export interface InterviewSpeechSelection { + readonly automaticSource: InterviewSpeechSource | null; + readonly canonicalSegments: readonly CanonicalSpeechSegment[]; +} + const createSegment = ( messageId: string, partId: string, @@ -40,30 +52,34 @@ const createSegment = ( }; }; -export const selectCanonicalSpeechSegments = ( +export const selectInterviewSpeech = ( messages: PetrinautAiMessage[], -): CanonicalSpeechSegment[] => { - const segments: CanonicalSpeechSegment[] = []; +): InterviewSpeechSelection => { + const canonicalSegments: CanonicalSpeechSegment[] = []; + let automaticSource: InterviewSpeechSource | null = null; for (const message of messages) { if (message.role !== "assistant") { continue; } + const contextSegments: CanonicalSpeechSegment[] = []; + const questionSegments: CanonicalSpeechSegment[] = []; + for (const [partIndex, part] of message.parts.entries()) { if ( part.type === "text" && part.state !== "streaming" && part.text.trim() ) { - segments.push( - createSegment( - message.id, - `text:${partIndex}`, - "assistant-text", - part.text, - ), + const segment = createSegment( + message.id, + `text:${partIndex}`, + "assistant-text", + part.text, ); + canonicalSegments.push(segment); + contextSegments.push(segment); continue; } @@ -77,19 +93,38 @@ export const selectCanonicalSpeechSegments = ( try { const input = parseBrunchAskInput(part.input); - segments.push( - createSegment( - message.id, - part.toolCallId, - "brunch-ask", - input.question, - ), + const segment = createSegment( + message.id, + part.toolCallId, + "brunch-ask", + input.question, ); + canonicalSegments.push(segment); + questionSegments.push(segment); } catch { // Malformed tool inputs remain visible as tool errors; they are not spoken. } } + + const questionSegment = questionSegments.at(-1) ?? null; + if (contextSegments.length > 0 || questionSegment) { + automaticSource = { + contextSegments, + fullResponseSegments: [ + ...contextSegments, + ...(questionSegment ? [questionSegment] : []), + ], + messageId: message.id, + questionSegment, + }; + } } - return segments; + return { automaticSource, canonicalSegments }; }; + +export const selectCanonicalSpeechSegments = ( + messages: PetrinautAiMessage[], +): CanonicalSpeechSegment[] => [ + ...selectInterviewSpeech(messages).canonicalSegments, +]; From 1c08e050886467880db41eb8790838362ca3b212 Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Wed, 2 Sep 2026 12:06:40 +0200 Subject: [PATCH 02/15] Prepare concise interview speech with Realtime Amp-Thread-ID: https://ampcode.com/threads/T-01a0618e-4184-736b-a426-c802f086dc83 Co-authored-by: Amp --- .../openai-realtime-session.test.ts | 270 +++++++++++ .../openai-realtime-session.ts | 445 ++++++++++++++++-- .../src/voice-diagnostics.ts | 9 +- 3 files changed, 692 insertions(+), 32 deletions(-) diff --git a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts index 2e486daae5d..ad2d50bb5d5 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts @@ -2,6 +2,7 @@ import { afterEach, describe, expect, test, vi } from "vitest"; import { OpenAIRealtimeSession, + type InterviewSpeechPreparationRequest, type OpenAIRealtimeSessionEvent, } from "./openai-realtime-session"; @@ -40,6 +41,16 @@ const canonicalSegment = ( text, }); +const preparationRequest = ( + overrides: Partial = {}, +): InterviewSpeechPreparationRequest => ({ + cacheKey: "preparation-cache-key", + contextText: ["The batch requires a named approver before release."], + contextWordBudget: 12, + sourceSegmentIds: ["context-segment-1"], + ...overrides, +}); + const createHarness = ({ connectionTimeoutMs = 15_000, }: { @@ -392,6 +403,265 @@ describe("OpenAIRealtimeSession", () => { }); }); + test("prepares context with an out-of-band text-only response", async () => { + const harness = createHarness(); + await harness.session.connect(); + const channel = harness.channels[0]!; + const request = preparationRequest({ + contextText: [ + "The batch requires a named approver.", + "Release remains blocked until approval.", + ], + sourceSegmentIds: ["message-1:text-1", "message-1:text-2"], + }); + + const preparation = harness.session.prepareInterviewSpeech(request); + const responseCreate = sentEvents(channel).at(-1)!; + expect(responseCreate).toMatchObject({ + type: "response.create", + response: { + conversation: "none", + max_output_tokens: 120, + metadata: { + petrinaut_kind: "speech-preparation", + petrinaut_request_id: request.cacheKey, + }, + output_modalities: ["text"], + parallel_tool_calls: false, + tool_choice: "none", + tools: [], + }, + }); + const response = responseCreate.response as { + input: Array<{ content: Array<{ text: string }> }>; + }; + expect(JSON.parse(response.input[0]!.content[0]!.text)).toEqual({ + context_text: request.contextText, + maximum_words: request.contextWordBudget, + }); + expect(response.input[0]!.content[0]!.text).not.toContain("message-1"); + expect(response.input[0]!.content[0]!.text).not.toContain("fnv1a32"); + expect(response.input[0]!.content[0]!.text).not.toContain("question"); + + channel.receive({ + response: { + id: "response-preparation", + metadata: (responseCreate.response as Record).metadata, + }, + type: "response.created", + }); + channel.receive({ + delta: "Ignore unrelated output.", + response_id: "response-unrelated", + type: "response.output_text.delta", + }); + channel.receive({ + delta: "Approval is required before release.", + response_id: "response-preparation", + type: "response.output_text.delta", + }); + channel.receive({ + response: { + id: "response-preparation", + output: [], + status: "completed", + }, + type: "response.done", + }); + + await expect(preparation).resolves.toEqual({ + context: "Approval is required before release.", + kind: "prepared", + sourceSegmentIds: request.sourceSegmentIds, + }); + expect(harness.reportDiagnostic).toHaveBeenLastCalledWith({ + durationMs: 0, + inputWordCount: 11, + operation: "preparation", + outcome: "success", + outputWordCount: 5, + requestId: "voice-request-2", + sourceSegmentCount: 2, + stage: "browser", + }); + }); + + test.each([ + ["empty output", "", 12], + ["over-budget output", "one two three", 2], + ["over-400-character output", "word ".repeat(81), 100], + ["Markdown heading", "# Approval is required", 12], + ["Markdown list", "- Approval is required", 12], + ["JSON fence", "```json\n{}\n```", 12], + ["tool syntax", 'continue_interview({"answer":"yes"})', 12], + ])("falls back for invalid prepared %s", async (_label, output, budget) => { + const harness = createHarness(); + await harness.session.connect(); + const channel = harness.channels[0]!; + const request = preparationRequest({ contextWordBudget: budget }); + const preparation = harness.session.prepareInterviewSpeech(request); + const responseCreate = sentEvents(channel).at(-1)!; + channel.receive({ + response: { + id: "response-preparation", + metadata: (responseCreate.response as Record).metadata, + }, + type: "response.created", + }); + if (output) { + channel.receive({ + delta: output, + response_id: "response-preparation", + type: "response.output_text.delta", + }); + } + channel.receive({ + response: { + id: "response-preparation", + output: [], + status: "completed", + }, + type: "response.done", + }); + + await expect(preparation).resolves.toEqual({ + kind: "fallback", + reason: "invalid-output", + sourceSegmentIds: request.sourceSegmentIds, + }); + }); + + test("falls back without closing the session after a correlated provider error", async () => { + const harness = createHarness(); + await harness.session.connect(); + const channel = harness.channels[0]!; + const request = preparationRequest(); + const preparation = harness.session.prepareInterviewSpeech(request); + const responseCreate = sentEvents(channel).at(-1)!; + + channel.receive({ + error: { + event_id: responseCreate.event_id, + message: "private provider detail", + type: "server_error", + }, + type: "error", + }); + + await expect(preparation).resolves.toEqual({ + kind: "fallback", + reason: "provider-error", + sourceSegmentIds: request.sourceSegmentIds, + }); + expect(harness.localTracks[0]!.stop).not.toHaveBeenCalled(); + expect(JSON.stringify(harness.events)).not.toContain( + "private provider detail", + ); + }); + + test("interrupts preparation and ignores its late completion", async () => { + const harness = createHarness(); + await harness.session.connect(); + const channel = harness.channels[0]!; + const request = preparationRequest(); + const preparation = harness.session.prepareInterviewSpeech(request); + const responseCreate = sentEvents(channel).at(-1)!; + channel.receive({ + response: { + id: "response-preparation", + metadata: (responseCreate.response as Record).metadata, + }, + type: "response.created", + }); + + harness.session.cancelOutput(); + + await expect(preparation).resolves.toEqual({ + kind: "fallback", + reason: "interrupted", + sourceSegmentIds: request.sourceSegmentIds, + }); + expect(sentEvents(channel).slice(-1)[0]).toMatchObject({ + response_id: "response-preparation", + type: "response.cancel", + }); + channel.receive({ + delta: "Late output must be ignored.", + response_id: "response-preparation", + type: "response.output_text.delta", + }); + channel.receive({ + response: { + id: "response-preparation", + output: [], + status: "cancelled", + }, + type: "response.done", + }); + expect(harness.localTracks[0]!.stop).not.toHaveBeenCalled(); + }); + + test("times out preparation after two seconds and cancels late provider work", async () => { + vi.useFakeTimers(); + const harness = createHarness(); + const connection = harness.session.connect(); + await vi.runAllTimersAsync(); + await connection; + const channel = harness.channels[0]!; + const request = preparationRequest(); + const preparation = harness.session.prepareInterviewSpeech(request); + const responseCreate = sentEvents(channel).at(-1)!; + channel.receive({ + response: { + id: "response-preparation", + metadata: (responseCreate.response as Record).metadata, + }, + type: "response.created", + }); + + await vi.advanceTimersByTimeAsync(2_000); + + await expect(preparation).resolves.toEqual({ + kind: "fallback", + reason: "timeout", + sourceSegmentIds: request.sourceSegmentIds, + }); + expect(sentEvents(channel).slice(-1)[0]).toMatchObject({ + response_id: "response-preparation", + type: "response.cancel", + }); + channel.receive({ + delta: "Late output must be ignored.", + response_id: "response-preparation", + type: "response.output_text.delta", + }); + channel.receive({ + response: { + id: "response-preparation", + output: [], + status: "cancelled", + }, + type: "response.done", + }); + expect(harness.localTracks[0]!.stop).not.toHaveBeenCalled(); + }); + + test("settles active preparation during disconnect cleanup", async () => { + const harness = createHarness(); + await harness.session.connect(); + const request = preparationRequest(); + const preparation = harness.session.prepareInterviewSpeech(request); + + await harness.session.disconnect(); + + await expect(preparation).resolves.toEqual({ + kind: "fallback", + reason: "interrupted", + sourceSegmentIds: request.sourceSegmentIds, + }); + expect(harness.localTracks[0]!.stop).toHaveBeenCalledOnce(); + }); + test("cancels canonical speech before the response starts", async () => { const harness = createHarness(); await harness.session.connect(); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts index f7c13d9c28c..53adf70ac75 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts @@ -112,11 +112,61 @@ interface RequestTiming { readonly startedAt: number; } +export interface InterviewSpeechPreparationRequest { + readonly cacheKey: string; + readonly contextText: readonly string[]; + readonly contextWordBudget: number; + readonly sourceSegmentIds: readonly string[]; +} + +export type InterviewSpeechPreparationResult = + | { + readonly context: string; + readonly kind: "prepared"; + readonly sourceSegmentIds: readonly string[]; + } + | { + readonly kind: "fallback"; + readonly reason: + | "empty-context" + | "invalid-output" + | "provider-error" + | "timeout" + | "interrupted"; + readonly sourceSegmentIds: readonly string[]; + }; + +type InterviewSpeechPreparationFallbackReason = Extract< + InterviewSpeechPreparationResult, + { kind: "fallback" } +>["reason"]; + interface CanonicalSpeechRequest { + readonly kind: "canonical-speech"; readonly response: Record; readonly speechRequestId: string; } +interface InterviewPreparationResponseRequest { + readonly kind: "speech-preparation"; + readonly preparationRequestId: string; + readonly response: Record; +} + +type SerializedResponseRequest = + | CanonicalSpeechRequest + | InterviewPreparationResponseRequest; + +interface PendingInterviewPreparation { + readonly inputWordCount: number; + readonly outputChunks: string[]; + readonly request: InterviewSpeechPreparationRequest; + readonly requestId: string; + readonly resolve: (result: InterviewSpeechPreparationResult) => void; + readonly startedAt: number; + readonly timeout: ReturnType; +} + type PendingClientEvent = | { readonly kind: "response-cancel"; @@ -124,7 +174,7 @@ type PendingClientEvent = } | { readonly kind: "response-create"; - readonly request: CanonicalSpeechRequest; + readonly request: SerializedResponseRequest; readonly responseTerminalSequence: number; }; @@ -132,7 +182,20 @@ type SessionListener = (event: OpenAIRealtimeSessionEvent) => void; const CANONICAL_RESPONSE_INSTRUCTIONS = "Speak only the response_text strings supplied by Petrinaut, in array order and verbatim. Deliver them as a warm, calm, curious, confident, concise, and professionally neutral expert interviewer, at a measured conversational pace with natural emphasis. Never sound robotic, fawning, rushed, overenthusiastic, or patronizing. Do not add, remove, paraphrase, acknowledge, or explain anything."; +const INTERVIEW_PREPARATION_INSTRUCTIONS = [ + "Rewrite the supplied context as a concise spoken interviewer response.", + "Output plain text only.", + "Use one or two short sentences.", + "Do not ask a question.", + "Do not add facts, promises, praise, choices, or next steps.", + "Preserve explicit negations, warnings, commitments, and required actions.", + "Stay within the supplied maximum word count.", +].join(" "); const MAX_CANONICAL_SEGMENTS = 64; +const MAX_PREPARED_CONTEXT_CHARACTERS = 400; +const PREPARATION_TIMEOUT_MS = 2_000; +const INVALID_PREPARED_CONTEXT_SYNTAX = + /(^|\n)\s*(?:#{1,6}\s|[-*+]\s|\d+[.)]\s)|```(?:json)?|\b(?:brunch_ask|continue_interview)\s*\(| | null => typeof value === "object" && value !== null && !Array.isArray(value) @@ -145,6 +208,11 @@ const nonEmptyString = (value: unknown): string | null => const nonNegativeInteger = (value: unknown): number | null => Number.isInteger(value) && (value as number) >= 0 ? (value as number) : null; +const countWords = (text: string): number => { + const trimmed = text.trim(); + return trimmed ? trimmed.split(/\s+/u).length : 0; +}; + const parseRealtimeEvent = (value: unknown): Record | null => { if (typeof value !== "string") { return null; @@ -191,13 +259,19 @@ export class OpenAIRealtimeSession { readonly #listeners = new Set(); readonly #authorizedResponseIds = new Set(); readonly #cancelledCanonicalResponseIds = new Set(); + readonly #cancelledPreparationRequestIds = new Set(); readonly #cancelledSpeechRequestIds = new Set(); readonly #canonicalResponseIds = new Set(); - readonly #canonicalSpeechQueue: CanonicalSpeechRequest[] = []; readonly #completedResponseCancelEventIds = new Set(); readonly #pendingClientEvents = new Map(); + readonly #pendingPreparations = new Map< + string, + PendingInterviewPreparation + >(); readonly #pendingSpeechRequests = new Map(); + readonly #preparationResponseIds = new Map(); readonly #remoteStreams = new Set(); + readonly #responseQueue: SerializedResponseRequest[] = []; readonly #speechTimings = new Map(); readonly #transcriptionTimings = new Map(); #abortController: AbortController | null = null; @@ -421,13 +495,111 @@ export class OpenAIRealtimeSession { this.#requestCanonicalSpeech(segments, false); } + public prepareInterviewSpeech( + request: InterviewSpeechPreparationRequest, + ): Promise { + const sourceSegmentIds = [...request.sourceSegmentIds]; + const contextText = request.contextText.filter((text) => text.trim()); + if (contextText.length === 0) { + return Promise.resolve({ + kind: "fallback", + reason: "empty-context", + sourceSegmentIds, + }); + } + if (request.contextWordBudget === 0) { + return Promise.resolve({ + context: "", + kind: "prepared", + sourceSegmentIds, + }); + } + if ( + !request.cacheKey || + !Number.isInteger(request.contextWordBudget) || + request.contextWordBudget < 0 || + this.#pendingPreparations.has(request.cacheKey) + ) { + return Promise.resolve({ + kind: "fallback", + reason: "invalid-output", + sourceSegmentIds, + }); + } + + return new Promise((resolve) => { + const timeout = globalThis.setTimeout( + () => this.#timeoutPreparation(request.cacheKey), + PREPARATION_TIMEOUT_MS, + ); + this.#pendingPreparations.set(request.cacheKey, { + inputWordCount: countWords(contextText.join(" ")), + outputChunks: [], + request, + requestId: + this.#dependencies.createRequestId?.() ?? createVoiceRequestId(), + resolve, + startedAt: this.#now(), + timeout, + }); + + const serializedRequest: InterviewPreparationResponseRequest = { + kind: "speech-preparation", + preparationRequestId: request.cacheKey, + response: { + conversation: "none", + input: [ + { + type: "message", + role: "user", + content: [ + { + type: "input_text", + text: JSON.stringify({ + context_text: contextText, + maximum_words: request.contextWordBudget, + }), + }, + ], + }, + ], + instructions: INTERVIEW_PREPARATION_INSTRUCTIONS, + max_output_tokens: 120, + output_modalities: ["text"], + parallel_tool_calls: false, + tool_choice: "none", + tools: [], + metadata: { + petrinaut_kind: "speech-preparation", + petrinaut_request_id: request.cacheKey, + }, + }, + }; + this.#responseQueue.push(serializedRequest); + try { + this.#sendNextSerializedResponse(); + } catch { + const queuedRequestIndex = + this.#responseQueue.indexOf(serializedRequest); + if (queuedRequestIndex >= 0) { + this.#responseQueue.splice(queuedRequestIndex, 1); + } + this.#settlePreparation(request.cacheKey, "provider-error"); + } + }); + } + public cancelOutput(): void { if (!this.#connected || this.#dataChannel?.readyState !== "open") { return; } - for (const request of this.#canonicalSpeechQueue.splice(0)) { - this.#cancelPendingSpeechRequest(request.speechRequestId); + for (const request of this.#responseQueue.splice(0)) { + if (request.kind === "canonical-speech") { + this.#cancelPendingSpeechRequest(request.speechRequestId); + } else { + this.#settlePreparation(request.preparationRequestId, "interrupted"); + } } if (this.#responseCreateEventId !== null) { @@ -435,9 +607,29 @@ export class OpenAIRealtimeSession { this.#responseCreateEventId, ); if (pendingEvent?.kind === "response-create") { - this.#cancelledSpeechRequestIds.add( - pendingEvent.request.speechRequestId, - ); + if (pendingEvent.request.kind === "canonical-speech") { + this.#cancelledSpeechRequestIds.add( + pendingEvent.request.speechRequestId, + ); + } else { + this.#cancelledPreparationRequestIds.add( + pendingEvent.request.preparationRequestId, + ); + this.#settlePreparation( + pendingEvent.request.preparationRequestId, + "interrupted", + ); + } + } + } + + for (const [ + responseId, + preparationRequestId, + ] of this.#preparationResponseIds) { + if (this.#activeResponseIds.has(responseId)) { + this.#settlePreparation(preparationRequestId, "interrupted"); + this.#cancelResponse(responseId); } } @@ -511,14 +703,18 @@ export class OpenAIRealtimeSession { petrinaut_request_id: speechRequestId, }, }; - const request = { response, speechRequestId }; - this.#canonicalSpeechQueue.push(request); + const request: CanonicalSpeechRequest = { + kind: "canonical-speech", + response, + speechRequestId, + }; + this.#responseQueue.push(request); try { - this.#sendNextCanonicalSpeech(); + this.#sendNextSerializedResponse(); } catch (error) { - const queuedRequestIndex = this.#canonicalSpeechQueue.indexOf(request); + const queuedRequestIndex = this.#responseQueue.indexOf(request); if (queuedRequestIndex >= 0) { - this.#canonicalSpeechQueue.splice(queuedRequestIndex, 1); + this.#responseQueue.splice(queuedRequestIndex, 1); } this.#pendingSpeechRequests.delete(speechRequestId); throw error; @@ -547,7 +743,7 @@ export class OpenAIRealtimeSession { return `petrinaut-${this.#activeEpoch}-${++this.#clientEventSequence}`; } - #sendNextCanonicalSpeech(): void { + #sendNextSerializedResponse(): void { if ( this.#activeResponseIds.size > 0 || this.#responseCreateEventId !== null || @@ -555,7 +751,7 @@ export class OpenAIRealtimeSession { ) { return; } - const request = this.#canonicalSpeechQueue.shift(); + const request = this.#responseQueue.shift(); if (!request) { return; } @@ -604,6 +800,10 @@ export class OpenAIRealtimeSession { this.#handleResponseDone(parsed, connectionEpoch); return; } + if (parsed.type === "response.output_text.delta") { + this.#handlePreparationDelta(parsed); + return; + } if (parsed.type === "input_audio_buffer.committed") { const itemId = nonEmptyString(parsed.item_id); if (itemId) this.#startTranscription(itemId); @@ -664,28 +864,50 @@ export class OpenAIRealtimeSession { } this.#activeResponseIds.add(responseId); const metadata = asRecord(response?.metadata); - const speechRequestId = nonEmptyString(metadata?.petrinaut_request_id); - if (metadata?.petrinaut_kind !== "canonical-speech" || !speechRequestId) { + const correlatedRequestId = nonEmptyString(metadata?.petrinaut_request_id); + if (!correlatedRequestId) { + return; + } + + if (metadata?.petrinaut_kind === "speech-preparation") { + this.#completeResponseCreateEvent( + "speech-preparation", + correlatedRequestId, + ); + this.#preparationResponseIds.set(responseId, correlatedRequestId); + if ( + this.#cancelledPreparationRequestIds.delete(correlatedRequestId) || + !this.#pendingPreparations.has(correlatedRequestId) + ) { + this.#cancelResponse(responseId); + } + return; + } + if (metadata?.petrinaut_kind !== "canonical-speech") { return; } - this.#completeResponseCreateEvent(speechRequestId); + + this.#completeResponseCreateEvent("canonical-speech", correlatedRequestId); this.#canonicalResponseIds.add(responseId); - if (this.#cancelledSpeechRequestIds.delete(speechRequestId)) { - this.#cancelPendingSpeechRequest(speechRequestId); + if (this.#cancelledSpeechRequestIds.delete(correlatedRequestId)) { + this.#cancelPendingSpeechRequest(correlatedRequestId); this.#cancelledCanonicalResponseIds.add(responseId); this.#cancelOutputResponse(responseId); return; } - const timing = this.#pendingSpeechRequests.get(speechRequestId); + const timing = this.#pendingSpeechRequests.get(correlatedRequestId); if (!timing) { return; } - this.#pendingSpeechRequests.delete(speechRequestId); + this.#pendingSpeechRequests.delete(correlatedRequestId); this.#authorizedResponseIds.add(responseId); this.#speechTimings.set(responseId, timing); } - #completeResponseCreateEvent(speechRequestId: string): void { + #completeResponseCreateEvent( + kind: SerializedResponseRequest["kind"], + requestId: string, + ): void { if (!this.#responseCreateEventId) { return; } @@ -694,7 +916,10 @@ export class OpenAIRealtimeSession { ); if ( pendingEvent?.kind !== "response-create" || - pendingEvent.request.speechRequestId !== speechRequestId + pendingEvent.request.kind !== kind || + (pendingEvent.request.kind === "canonical-speech" + ? pendingEvent.request.speechRequestId + : pendingEvent.request.preparationRequestId) !== requestId ) { return; } @@ -736,6 +961,7 @@ export class OpenAIRealtimeSession { this.#responseCreateEventId = null; } if ( + pendingEvent.request.kind === "canonical-speech" && this.#cancelledSpeechRequestIds.delete( pendingEvent.request.speechRequestId, ) @@ -743,11 +969,40 @@ export class OpenAIRealtimeSession { this.#cancelPendingSpeechRequest(pendingEvent.request.speechRequestId); return; } - this.#canonicalSpeechQueue.unshift(pendingEvent.request); + if ( + pendingEvent.request.kind === "speech-preparation" && + this.#cancelledPreparationRequestIds.delete( + pendingEvent.request.preparationRequestId, + ) + ) { + this.#settlePreparation( + pendingEvent.request.preparationRequestId, + "interrupted", + ); + return; + } + this.#responseQueue.unshift(pendingEvent.request); this.#waitingForResponseTerminal = this.#responseTerminalSequence === pendingEvent.responseTerminalSequence; - this.#resumeCanonicalSpeechQueue(); + this.#resumeSerializedResponseQueue(); + return; + } + + if ( + pendingEvent?.kind === "response-create" && + pendingEvent.request.kind === "speech-preparation" && + sourceEventId + ) { + this.#pendingClientEvents.delete(sourceEventId); + if (this.#responseCreateEventId === sourceEventId) { + this.#responseCreateEventId = null; + } + this.#settlePreparation( + pendingEvent.request.preparationRequestId, + "provider-error", + ); + this.#resumeSerializedResponseQueue(); return; } @@ -778,6 +1033,21 @@ export class OpenAIRealtimeSession { this.#clearResponseCancelEvents(responseId); this.#waitingForResponseTerminal = false; + const preparationRequestId = this.#preparationResponseIds.get(responseId); + if (preparationRequestId) { + this.#preparationResponseIds.delete(responseId); + if (status === "completed") { + this.#finishPreparation(preparationRequestId); + } else { + this.#settlePreparation( + preparationRequestId, + status === "cancelled" ? "interrupted" : "provider-error", + ); + } + this.#resumeSerializedResponseQueue(); + return; + } + if (this.#cancelledCanonicalResponseIds.delete(responseId)) { this.#emit({ connectionEpoch, @@ -786,7 +1056,7 @@ export class OpenAIRealtimeSession { type: "response-terminal", }); this.#finishSpeech(responseId, "request-aborted"); - this.#resumeCanonicalSpeechQueue(); + this.#resumeSerializedResponseQueue(); return; } @@ -840,7 +1110,7 @@ export class OpenAIRealtimeSession { status, type: "response-terminal", }); - this.#resumeCanonicalSpeechQueue(); + this.#resumeSerializedResponseQueue(); return; } if (status === "cancelled") { @@ -858,7 +1128,7 @@ export class OpenAIRealtimeSession { type: "response-terminal", }); this.#finishSpeech(responseId, "request-aborted"); - this.#resumeCanonicalSpeechQueue(); + this.#resumeSerializedResponseQueue(); return; } this.#emit({ @@ -885,14 +1155,121 @@ export class OpenAIRealtimeSession { } } - #resumeCanonicalSpeechQueue(): void { + #resumeSerializedResponseQueue(): void { try { - this.#sendNextCanonicalSpeech(); + this.#sendNextSerializedResponse(); } catch { this.#handleConnectionFailure("network", "speech"); } } + #handlePreparationDelta(event: Record): void { + const responseId = nonEmptyString(event.response_id); + const delta = event.delta; + if (!responseId || typeof delta !== "string") { + return; + } + const preparationRequestId = this.#preparationResponseIds.get(responseId); + if (!preparationRequestId) { + return; + } + this.#pendingPreparations + .get(preparationRequestId) + ?.outputChunks.push(delta); + } + + #finishPreparation(preparationRequestId: string): void { + const pending = this.#pendingPreparations.get(preparationRequestId); + if (!pending) { + return; + } + const context = pending.outputChunks.join("").trim(); + if ( + !context || + countWords(context) > pending.request.contextWordBudget || + Array.from(context).length > MAX_PREPARED_CONTEXT_CHARACTERS || + INVALID_PREPARED_CONTEXT_SYNTAX.test(context) + ) { + this.#settlePreparation(preparationRequestId, "invalid-output", context); + return; + } + this.#settlePreparation(preparationRequestId, undefined, context); + } + + #settlePreparation( + preparationRequestId: string, + reason?: InterviewSpeechPreparationFallbackReason, + context = "", + ): void { + const pending = this.#pendingPreparations.get(preparationRequestId); + if (!pending) { + return; + } + this.#pendingPreparations.delete(preparationRequestId); + globalThis.clearTimeout(pending.timeout); + const errorCode: VoiceErrorCode | undefined = + reason === undefined + ? undefined + : reason === "timeout" + ? "timeout" + : reason === "interrupted" + ? "request-aborted" + : "invalid-response"; + this.#dependencies.reportDiagnostic?.({ + durationMs: voiceDurationMs(pending.startedAt, this.#now()), + ...(errorCode === undefined ? {} : { errorCode }), + inputWordCount: pending.inputWordCount, + operation: "preparation", + outcome: voiceDiagnosticOutcome(errorCode), + outputWordCount: countWords(context), + requestId: pending.requestId, + sourceSegmentCount: pending.request.sourceSegmentIds.length, + stage: "browser", + }); + pending.resolve( + reason === undefined + ? { + context, + kind: "prepared", + sourceSegmentIds: [...pending.request.sourceSegmentIds], + } + : { + kind: "fallback", + reason, + sourceSegmentIds: [...pending.request.sourceSegmentIds], + }, + ); + } + + #timeoutPreparation(preparationRequestId: string): void { + const queuedRequestIndex = this.#responseQueue.findIndex( + (request) => + request.kind === "speech-preparation" && + request.preparationRequestId === preparationRequestId, + ); + if (queuedRequestIndex >= 0) { + this.#responseQueue.splice(queuedRequestIndex, 1); + } + + const pendingCreateEvent = this.#responseCreateEventId + ? this.#pendingClientEvents.get(this.#responseCreateEventId) + : undefined; + if ( + pendingCreateEvent?.kind === "response-create" && + pendingCreateEvent.request.kind === "speech-preparation" && + pendingCreateEvent.request.preparationRequestId === preparationRequestId + ) { + this.#cancelledPreparationRequestIds.add(preparationRequestId); + } + const activeResponse = [...this.#preparationResponseIds].find( + ([, requestId]) => requestId === preparationRequestId, + ); + if (activeResponse) { + this.#cancelResponse(activeResponse[0]); + } + this.#settlePreparation(preparationRequestId, "timeout"); + } + #handleOutputBufferEvent( event: Record, connectionEpoch: number, @@ -1257,6 +1634,9 @@ export class OpenAIRealtimeSession { } #releaseResources(): void { + for (const preparationRequestId of this.#pendingPreparations.keys()) { + this.#settlePreparation(preparationRequestId, "interrupted"); + } for (const timing of this.#transcriptionTimings.values()) { this.#reportDiagnostic( "transcription", @@ -1279,11 +1659,14 @@ export class OpenAIRealtimeSession { this.#transcriptionTimings.clear(); this.#activeResponseIds.clear(); this.#cancelledCanonicalResponseIds.clear(); + this.#cancelledPreparationRequestIds.clear(); this.#cancelledSpeechRequestIds.clear(); - this.#canonicalSpeechQueue.length = 0; this.#completedResponseCancelEventIds.clear(); this.#pendingClientEvents.clear(); + this.#pendingPreparations.clear(); this.#pendingSpeechRequests.clear(); + this.#preparationResponseIds.clear(); + this.#responseQueue.length = 0; this.#speechTimings.clear(); this.#authorizedResponseIds.clear(); this.#canonicalResponseIds.clear(); diff --git a/apps/petrinaut-website/src/voice-diagnostics.ts b/apps/petrinaut-website/src/voice-diagnostics.ts index be74ef0b911..e7eadb40d82 100644 --- a/apps/petrinaut-website/src/voice-diagnostics.ts +++ b/apps/petrinaut-website/src/voice-diagnostics.ts @@ -12,14 +12,21 @@ export const voiceErrorCodes = [ ] as const; export type VoiceErrorCode = (typeof voiceErrorCodes)[number]; -export type VoiceOperation = "connection" | "transcription" | "speech"; +export type VoiceOperation = + | "connection" + | "preparation" + | "transcription" + | "speech"; export interface VoiceDiagnosticEvent { readonly durationMs: number; readonly errorCode?: VoiceErrorCode; + readonly inputWordCount?: number; readonly operation: VoiceOperation; readonly outcome: "success" | "failure" | "aborted"; + readonly outputWordCount?: number; readonly requestId: string; + readonly sourceSegmentCount?: number; readonly stage: "browser" | "playback" | "server"; readonly status?: number; } From 7d0c78f86275834d9e9145b5d48ef7b59d62402d Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Wed, 2 Sep 2026 12:12:27 +0200 Subject: [PATCH 03/15] Speak prepared context with exact Brunch questions Amp-Thread-ID: https://ampcode.com/threads/T-01a0618e-4184-736b-a426-c802f086dc83 Co-authored-by: Amp --- .../openai-realtime-session.test.ts | 34 ++- .../openai-realtime-session.ts | 31 +-- .../realtime-brunch-bridge.test.ts | 201 +++++++++++++++++- .../voice-interview/realtime-brunch-bridge.ts | 196 +++++++++++++++-- 4 files changed, 423 insertions(+), 39 deletions(-) diff --git a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts index ad2d50bb5d5..46837f9babf 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts @@ -340,9 +340,7 @@ describe("OpenAIRealtimeSession", () => { }, ]); - harness.session.completeFunctionCall("call-1", [ - canonicalSegment("ask-2", "Who acts next?"), - ]); + harness.session.completeFunctionCall("call-1", ["Who acts next?"]); const [functionOutput, responseCreate] = sentEvents(channel).slice(-2); expect(functionOutput).toEqual({ type: "conversation.item.create", @@ -365,6 +363,36 @@ describe("OpenAIRealtimeSession", () => { }); }); + test("renders prepared strings through the verbatim out-of-band audio response", async () => { + const harness = createHarness(); + await harness.session.connect(); + + harness.session.speakPrepared([ + "Prepared concise context.", + "Which operator confirms the batch?", + ]); + + const responseCreate = sentEvents(harness.channels[0]!).at(-1)!; + expect(responseCreate).toMatchObject({ + type: "response.create", + response: { + conversation: "none", + output_modalities: ["audio"], + tool_choice: "none", + tools: [], + }, + }); + const response = responseCreate.response as { + input: Array<{ content: Array<{ text: string }> }>; + }; + expect(JSON.parse(response.input[0]!.content[0]!.text)).toEqual({ + response_text: [ + "Prepared concise context.", + "Which operator confirms the batch?", + ], + }); + }); + test("queues canonical speech behind an active Realtime response", async () => { const harness = createHarness(); await harness.session.connect(); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts index 53adf70ac75..20c729a8b78 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts @@ -473,17 +473,21 @@ export class OpenAIRealtimeSession { } public speakCanonical(segments: CanonicalSpeechSegment[]): void { - this.#requestCanonicalSpeech(segments, true); + this.#requestSpeech(this.#canonicalResponseText(segments), true); + } + + public speakPrepared(responseText: readonly string[]): void { + this.#requestSpeech(this.#validResponseText(responseText), true); } public completeFunctionCall( callId: string, - segments: CanonicalSpeechSegment[], + responseTextInput: readonly string[], ): void { if (!callId) { throw new VoiceError("speech", "invalid-response", ""); } - const responseText = this.#canonicalResponseText(segments); + const responseText = this.#validResponseText(responseTextInput); this.#send({ type: "conversation.item.create", item: { @@ -492,7 +496,7 @@ export class OpenAIRealtimeSession { output: JSON.stringify({ response_text: responseText }), }, }); - this.#requestCanonicalSpeech(segments, false); + this.#requestSpeech(responseText, false); } public prepareInterviewSpeech( @@ -654,21 +658,24 @@ export class OpenAIRealtimeSession { } #canonicalResponseText(segments: CanonicalSpeechSegment[]): string[] { - const responseText = segments + return this.#validResponseText(segments.map(({ text }) => text)); + } + + #validResponseText(responseTextInput: readonly string[]): string[] { + const responseText = responseTextInput .slice(0, MAX_CANONICAL_SEGMENTS) - .map(({ text }) => text.trim()) + .map((text) => text.trim()) .filter(Boolean); - if (responseText.length === 0 || responseText.length !== segments.length) { + if ( + responseText.length === 0 || + responseText.length !== responseTextInput.length + ) { throw new VoiceError("speech", "invalid-response", ""); } return responseText; } - #requestCanonicalSpeech( - segments: CanonicalSpeechSegment[], - outOfBand: boolean, - ): void { - const responseText = this.#canonicalResponseText(segments); + #requestSpeech(responseText: string[], outOfBand: boolean): void { const speechRequestId = `canonical-${this.#activeEpoch}-${++this.#speechRequestSequence}`; this.#pendingSpeechRequests.set(speechRequestId, { requestId: diff --git a/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.test.ts index a723f2dfe5a..6116a7b125c 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.test.ts @@ -1,13 +1,20 @@ import { describe, expect, test, vi } from "vitest"; import { + assemblePreparedInterviewSpeech, createRealtimeSubmissionId, RealtimeBrunchBridge, type RealtimeBrunchBridgeEvent, } from "./realtime-brunch-bridge"; -import type { CanonicalSpeechSegment } from "./canonical-speech"; -import type { OpenAIRealtimeSessionEvent } from "./openai-realtime-session"; +import type { + CanonicalSpeechSegment, + InterviewSpeechSource, +} from "./canonical-speech"; +import type { + InterviewSpeechPreparationResult, + OpenAIRealtimeSessionEvent, +} from "./openai-realtime-session"; const segment = ( id: string, @@ -22,11 +29,44 @@ const segment = ( text, }); +const speechSource = ({ + context = [], + messageId = "message-current-turn", + question = null, +}: { + readonly context?: readonly CanonicalSpeechSegment[]; + readonly messageId?: string; + readonly question?: CanonicalSpeechSegment | null; +}): InterviewSpeechSource => { + const contextSegments = context.map((contextSegment) => ({ + ...contextSegment, + messageId, + })); + const questionSegment = question ? { ...question, messageId } : null; + return { + contextSegments, + fullResponseSegments: [ + ...contextSegments, + ...(questionSegment ? [questionSegment] : []), + ], + messageId, + questionSegment, + }; +}; + const createHarness = () => { let listener: ((event: OpenAIRealtimeSessionEvent) => void) | undefined; const session = { completeFunctionCall: vi.fn(), + prepareInterviewSpeech: vi.fn( + async (request): Promise => ({ + context: "Prepared concise context.", + kind: "prepared", + sourceSegmentIds: request.sourceSegmentIds, + }), + ), speakCanonical: vi.fn(), + speakPrepared: vi.fn(), subscribe: vi.fn((next: (event: OpenAIRealtimeSessionEvent) => void) => { listener = next; return () => { @@ -96,6 +136,157 @@ const responseTerminal = ( }); describe("RealtimeBrunchBridge", () => { + test("prepares only context and appends the exact canonical question", async () => { + const harness = createHarness(); + const context = segment( + "context", + "This complete canonical explanation is deliberately long enough to be condensed before automatic speech.", + "assistant-text", + ); + const question = segment( + "ask-current", + "Who approves it: the manager or quality lead?", + ); + const source = speechSource({ context: [context], question }); + harness.bridge.updateChat({ + automaticSource: source, + canAcceptInterviewAnswer: true, + canonicalSegments: [...source.fullResponseSegments], + status: "ready", + }); + + harness.bridge.start(4); + + await vi.waitFor(() => + expect(harness.session.speakPrepared).toHaveBeenCalledOnce(), + ); + expect(harness.session.prepareInterviewSpeech).toHaveBeenCalledWith({ + cacheKey: expect.any(String), + contextText: [context.text], + contextWordBudget: 42, + sourceSegmentIds: [context.id], + }); + expect( + JSON.stringify(harness.session.prepareInterviewSpeech.mock.calls), + ).not.toContain(question.text); + expect(harness.session.speakPrepared).toHaveBeenCalledWith([ + "Prepared concise context.", + question.text, + ]); + expect(source.fullResponseSegments.map(({ text }) => text)).toEqual([ + context.text, + question.text, + ]); + expect(JSON.stringify(harness.events)).not.toContain( + "Prepared concise context.", + ); + }); + + test("skips preparation when a turn contains only a protected question", async () => { + const harness = createHarness(); + const question = segment("ask-current", "What happens after approval?"); + const source = speechSource({ question }); + harness.bridge.updateChat({ + automaticSource: source, + canAcceptInterviewAnswer: true, + canonicalSegments: [...source.fullResponseSegments], + status: "ready", + }); + + harness.bridge.start(4); + + await vi.waitFor(() => + expect(harness.session.speakPrepared).toHaveBeenCalledWith([ + question.text, + ]), + ); + expect(harness.session.prepareInterviewSpeech).not.toHaveBeenCalled(); + }); + + test("prepares every human-facing segment in a standalone completion", async () => { + const harness = createHarness(); + const first = segment("first", "First result.", "assistant-text"); + const second = segment("second", "Second result.", "assistant-text"); + const source = speechSource({ context: [first, second] }); + harness.bridge.updateChat({ + automaticSource: source, + canAcceptInterviewAnswer: false, + canonicalSegments: [...source.fullResponseSegments], + status: "ready", + }); + + harness.bridge.start(4); + + await vi.waitFor(() => + expect(harness.session.prepareInterviewSpeech).toHaveBeenCalledWith( + expect.objectContaining({ + contextText: [first.text, second.text], + contextWordBudget: 50, + }), + ), + ); + expect(harness.session.speakPrepared).toHaveBeenCalledWith([ + "Prepared concise context.", + ]); + }); + + test.each([ + "empty-context", + "invalid-output", + "provider-error", + "timeout", + "interrupted", + ] as const)( + "reads complete canonical speech after %s fallback", + async (reason) => { + const harness = createHarness(); + const context = segment("context", "Complete context.", "assistant-text"); + const question = segment("ask-current", "Exact question?"); + const source = speechSource({ context: [context], question }); + harness.session.prepareInterviewSpeech.mockResolvedValueOnce({ + kind: "fallback", + reason, + sourceSegmentIds: [context.id], + }); + harness.bridge.updateChat({ + automaticSource: source, + canAcceptInterviewAnswer: true, + canonicalSegments: [...source.fullResponseSegments], + status: "ready", + }); + + harness.bridge.start(4); + + await vi.waitFor(() => + expect(harness.session.speakPrepared).toHaveBeenCalledWith([ + context.text, + question.text, + ]), + ); + }, + ); + + test("assembles prepared context with the protected question identity", () => { + const context = segment("context", "Complete context.", "assistant-text"); + const question = segment("ask-current", "Exact question?"); + const source = speechSource({ context: [context], question }); + + expect( + assemblePreparedInterviewSpeech({ + preparation: { + context: "Prepared context.", + kind: "prepared", + sourceSegmentIds: [context.id], + }, + source, + }), + ).toEqual({ + mode: "realtime-processed", + sourceSegmentIds: [context.id, question.id], + text: ["Prepared context.", question.text], + }); + }); + test("speaks the current canonical turn without replaying history", () => { const harness = createHarness(); const historical = segment( @@ -174,7 +365,7 @@ describe("RealtimeBrunchBridge", () => { await vi.waitFor(() => expect(harness.session.completeFunctionCall).toHaveBeenCalledWith( "call-1", - [acknowledgement, nextQuestion], + [acknowledgement.text, nextQuestion.text], ), ); expect(harness.events.map(({ type }) => type)).toEqual([ @@ -222,7 +413,7 @@ describe("RealtimeBrunchBridge", () => { expect(harness.session.completeFunctionCall).toHaveBeenCalledWith( "call-1", - [firstQuestion], + [firstQuestion.text], ); expect(harness.events.map(({ type }) => type)).toEqual([ "submission-started", @@ -271,7 +462,7 @@ describe("RealtimeBrunchBridge", () => { expect(harness.session.completeFunctionCall).toHaveBeenCalledWith( "call-1", - [unrelated], + [unrelated.text], ); }); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.ts b/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.ts index 1a84fa74c62..badcddd8b28 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.ts @@ -1,20 +1,29 @@ -import type { CanonicalSpeechSegment } from "./canonical-speech"; -import type { OpenAIRealtimeSessionEvent } from "./openai-realtime-session"; +import type { + CanonicalSpeechSegment, + InterviewSpeechSource, +} from "./canonical-speech"; +import type { + InterviewSpeechPreparationRequest, + InterviewSpeechPreparationResult, + OpenAIRealtimeSessionEvent, +} from "./openai-realtime-session"; type ChatStatus = "ready" | "submitted" | "streaming" | "error"; interface ChatUpdate { + readonly automaticSource?: InterviewSpeechSource | null; readonly canAcceptInterviewAnswer: boolean; readonly canonicalSegments: CanonicalSpeechSegment[]; readonly status: ChatStatus; } interface RealtimeBridgeSession { - completeFunctionCall( - callId: string, - segments: CanonicalSpeechSegment[], - ): void; + completeFunctionCall(callId: string, responseText: readonly string[]): void; + prepareInterviewSpeech( + request: InterviewSpeechPreparationRequest, + ): Promise; speakCanonical(segments: CanonicalSpeechSegment[]): void; + speakPrepared(responseText: readonly string[]): void; subscribe(listener: (event: OpenAIRealtimeSessionEvent) => void): () => void; } @@ -49,6 +58,10 @@ interface ArgumentStream { readonly responseId: string; } +type SpeechDelivery = + | { readonly kind: "automatic" } + | { readonly callId: string; readonly kind: "function-call" }; + export type RealtimeBridgeErrorCode = | "interview-correlation" | "interview-response" @@ -76,6 +89,40 @@ export type RealtimeBrunchBridgeEvent = readonly type: "error"; }; +export interface PreparedInterviewSpeech { + readonly mode: "realtime-processed" | "canonical-fallback"; + readonly sourceSegmentIds: readonly string[]; + readonly text: readonly string[]; +} + +export const assemblePreparedInterviewSpeech = ({ + preparation, + source, +}: { + preparation: InterviewSpeechPreparationResult; + source: InterviewSpeechSource; +}): PreparedInterviewSpeech => { + if (preparation.kind === "prepared") { + return { + mode: "realtime-processed", + sourceSegmentIds: [ + ...preparation.sourceSegmentIds, + ...(source.questionSegment ? [source.questionSegment.id] : []), + ], + text: [ + ...(preparation.context ? [preparation.context] : []), + ...(source.questionSegment ? [source.questionSegment.text] : []), + ], + }; + } + + return { + mode: "canonical-fallback", + sourceSegmentIds: source.fullResponseSegments.map(({ id }) => id), + text: source.fullResponseSegments.map(({ text }) => text), + }; +}; + type BridgeListener = (event: RealtimeBrunchBridgeEvent) => void; const INVALID_BRIDGE_EVENT = @@ -92,6 +139,20 @@ const latestPendingQuestion = ( ): CanonicalSpeechSegment | undefined => segments.findLast(({ source }) => source === "brunch-ask"); +const spokenWordCount = (text: string): number => { + const trimmedText = text.trim(); + return trimmedText ? trimmedText.split(/\s+/u).length : 0; +}; + +const preparationCacheKey = ( + source: InterviewSpeechSource, + contextWordBudget: number, +): string => + JSON.stringify([ + ...source.contextSegments.map(({ contentHash, id }) => [id, contentHash]), + contextWordBudget, + ]); + const parseContinueInterviewArguments = ( argumentsJson: string, ): string | null => { @@ -156,13 +217,18 @@ export class RealtimeBrunchBridge { this.#seenSegmentIds.add(segment.id); } - const question = latestPendingQuestion(this.#chat.canonicalSegments); - if (question) { - this.#session.speakCanonical( - this.#chat.canonicalSegments.filter( - ({ messageId }) => messageId === question.messageId, - ), + const source = this.#chat.automaticSource; + if (source) { + this.#prepareAndDeliver(source, { kind: "automatic" }); + } else { + const question = latestPendingQuestion(this.#chat.canonicalSegments); + if (!question) { + return; + } + const currentTurnSegments = this.#chat.canonicalSegments.filter( + ({ messageId }) => messageId === question.messageId, ); + this.#session.speakCanonical(currentTurnSegments); } } @@ -204,10 +270,15 @@ export class RealtimeBrunchBridge { return; } try { - this.#session.speakCanonical(newSegments); for (const segment of newSegments) { this.#seenSegmentIds.add(segment.id); } + const source = update.automaticSource; + if (source && this.#sourceMatchesSegments(source, newSegments)) { + this.#prepareAndDeliver(source, { kind: "automatic" }); + } else { + this.#session.speakCanonical(newSegments); + } } catch { this.#fail(INVALID_BRIDGE_EVENT); } @@ -408,20 +479,107 @@ export class RealtimeBrunchBridge { return; } - try { - this.#session.completeFunctionCall(active.callId, responseSegments); - } catch { - this.#fail(INVALID_BRIDGE_EVENT); - return; - } for (const segment of responseSegments) { this.#seenSegmentIds.add(segment.id); } this.#activeSubmission = null; + const source = this.#chat.automaticSource; + if (source && this.#sourceMatchesSegments(source, responseSegments)) { + this.#prepareAndDeliver(source, { + callId: active.callId, + kind: "function-call", + }); + } else { + try { + this.#session.completeFunctionCall( + active.callId, + responseSegments.map(({ text }) => text), + ); + } catch { + this.#fail(INVALID_BRIDGE_EVENT); + return; + } + } this.#emit({ callId: active.callId, segments: responseSegments, type: "canonical-response-ready", }); } + + #sourceMatchesSegments( + source: InterviewSpeechSource, + segments: readonly CanonicalSpeechSegment[], + ): boolean { + return ( + source.fullResponseSegments.length === segments.length && + source.fullResponseSegments.every( + ({ id }, index) => segments[index]?.id === id, + ) + ); + } + + #prepareAndDeliver( + source: InterviewSpeechSource, + delivery: SpeechDelivery, + ): void { + const generation = this.#generation; + const questionWordCount = source.questionSegment + ? spokenWordCount(source.questionSegment.text) + : 0; + const contextWordBudget = Math.max(0, 50 - questionWordCount); + + if (source.contextSegments.length === 0 || contextWordBudget === 0) { + const questionOnlySource: InterviewSpeechSource = { + ...source, + contextSegments: [], + fullResponseSegments: source.questionSegment + ? [source.questionSegment] + : [], + }; + this.#deliverPreparedSpeech( + assemblePreparedInterviewSpeech({ + preparation: { + context: "", + kind: "prepared", + sourceSegmentIds: [], + }, + source: questionOnlySource, + }).text, + delivery, + ); + return; + } + + const request: InterviewSpeechPreparationRequest = { + cacheKey: preparationCacheKey(source, contextWordBudget), + contextText: source.contextSegments.map(({ text }) => text), + contextWordBudget, + sourceSegmentIds: source.contextSegments.map(({ id }) => id), + }; + void this.#session.prepareInterviewSpeech(request).then((preparation) => { + if (generation !== this.#generation) { + return; + } + this.#deliverPreparedSpeech( + assemblePreparedInterviewSpeech({ preparation, source }).text, + delivery, + ); + }); + } + + #deliverPreparedSpeech( + responseText: readonly string[], + delivery: SpeechDelivery, + ): void { + try { + if (delivery.kind === "function-call") { + this.#session.completeFunctionCall(delivery.callId, responseText); + } else { + this.#session.speakPrepared(responseText); + } + } catch { + this.#fail(INVALID_BRIDGE_EVENT); + } + } } From 19e5c43ab164623d79887694684dcf96bdfaa05f Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Wed, 2 Sep 2026 12:17:02 +0200 Subject: [PATCH 04/15] Keep prepared Voice responses lifecycle-safe Amp-Thread-ID: https://ampcode.com/threads/T-01a0618e-4184-736b-a426-c802f086dc83 Co-authored-by: Amp --- .../realtime-brunch-bridge.test.ts | 114 +++++++++++++++++- .../voice-interview/realtime-brunch-bridge.ts | 41 ++++++- .../voice-interview-control.tsx | 6 +- .../voice-preview.integration.test.ts | 68 +++++++++-- .../voice-turn-controller.test.ts | 28 +++++ .../voice-interview/voice-turn-controller.ts | 21 +++- 6 files changed, 253 insertions(+), 25 deletions(-) diff --git a/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.test.ts index 6116a7b125c..8cdb705a9d7 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.test.ts @@ -12,6 +12,7 @@ import type { InterviewSpeechSource, } from "./canonical-speech"; import type { + InterviewSpeechPreparationRequest, InterviewSpeechPreparationResult, OpenAIRealtimeSessionEvent, } from "./openai-realtime-session"; @@ -59,7 +60,9 @@ const createHarness = () => { const session = { completeFunctionCall: vi.fn(), prepareInterviewSpeech: vi.fn( - async (request): Promise => ({ + async ( + request: InterviewSpeechPreparationRequest, + ): Promise => ({ context: "Prepared concise context.", kind: "prepared", sourceSegmentIds: request.sourceSegmentIds, @@ -160,8 +163,11 @@ describe("RealtimeBrunchBridge", () => { await vi.waitFor(() => expect(harness.session.speakPrepared).toHaveBeenCalledOnce(), ); - expect(harness.session.prepareInterviewSpeech).toHaveBeenCalledWith({ - cacheKey: expect.any(String), + const preparationRequest = + harness.session.prepareInterviewSpeech.mock.calls[0]?.[0]; + expect(typeof preparationRequest?.cacheKey).toBe("string"); + expect(preparationRequest).toEqual({ + cacheKey: preparationRequest?.cacheKey, contextText: [context.text], contextWordBudget: 42, sourceSegmentIds: [context.id], @@ -287,6 +293,108 @@ describe("RealtimeBrunchBridge", () => { }); }); + test("reuses prepared context only while its epoch, content, and budget match", async () => { + const harness = createHarness(); + const context = segment("context", "Complete context.", "assistant-text"); + const question = segment("ask-current", "Exact question?"); + const source = speechSource({ context: [context], question }); + harness.bridge.updateChat({ + automaticSource: source, + canAcceptInterviewAnswer: true, + canonicalSegments: [...source.fullResponseSegments], + status: "ready", + }); + + harness.bridge.start(4); + await vi.waitFor(() => + expect(harness.session.speakPrepared).toHaveBeenCalledOnce(), + ); + harness.bridge.start(4); + await vi.waitFor(() => + expect(harness.session.speakPrepared).toHaveBeenCalledTimes(2), + ); + expect(harness.session.prepareInterviewSpeech).toHaveBeenCalledOnce(); + + const changedContext = { ...context, contentHash: "fnv1a32:changed" }; + const changedContentSource = speechSource({ + context: [changedContext], + question, + }); + harness.bridge.updateChat({ + automaticSource: changedContentSource, + canAcceptInterviewAnswer: true, + canonicalSegments: [...changedContentSource.fullResponseSegments], + status: "ready", + }); + harness.bridge.start(4); + await vi.waitFor(() => + expect(harness.session.prepareInterviewSpeech).toHaveBeenCalledTimes(2), + ); + + const longerQuestion = { ...question, text: "What is the exact question?" }; + const changedBudgetSource = speechSource({ + context: [changedContext], + question: longerQuestion, + }); + harness.bridge.updateChat({ + automaticSource: changedBudgetSource, + canAcceptInterviewAnswer: true, + canonicalSegments: [...changedBudgetSource.fullResponseSegments], + status: "ready", + }); + harness.bridge.start(4); + await vi.waitFor(() => + expect(harness.session.prepareInterviewSpeech).toHaveBeenCalledTimes(3), + ); + + harness.bridge.start(5); + await vi.waitFor(() => + expect(harness.session.prepareInterviewSpeech).toHaveBeenCalledTimes(4), + ); + harness.bridge.stop(); + harness.bridge.start(5); + await vi.waitFor(() => + expect(harness.session.prepareInterviewSpeech).toHaveBeenCalledTimes(5), + ); + }); + + test("does not cache fallback or deliver preparation cancelled by lifecycle", async () => { + const harness = createHarness(); + const context = segment("context", "Complete context.", "assistant-text"); + const question = segment("ask-current", "Exact question?"); + const source = speechSource({ context: [context], question }); + let finishPreparation: + | ((result: InterviewSpeechPreparationResult) => void) + | undefined; + harness.session.prepareInterviewSpeech.mockImplementationOnce( + () => + new Promise((resolve) => { + finishPreparation = resolve; + }), + ); + harness.bridge.updateChat({ + automaticSource: source, + canAcceptInterviewAnswer: true, + canonicalSegments: [...source.fullResponseSegments], + status: "ready", + }); + harness.bridge.start(4); + + harness.bridge.cancelPendingSpeech(); + finishPreparation?.({ + kind: "fallback", + reason: "interrupted", + sourceSegmentIds: [context.id], + }); + await Promise.resolve(); + + expect(harness.session.speakPrepared).not.toHaveBeenCalled(); + harness.bridge.start(4); + await vi.waitFor(() => + expect(harness.session.prepareInterviewSpeech).toHaveBeenCalledTimes(2), + ); + }); + test("speaks the current canonical turn without replaying history", () => { const harness = createHarness(); const historical = segment( diff --git a/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.ts b/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.ts index badcddd8b28..33cdcdeb7a4 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.ts @@ -1,7 +1,9 @@ -import type { - CanonicalSpeechSegment, - InterviewSpeechSource, +import { + hashCanonicalSpeechText, + type CanonicalSpeechSegment, + type InterviewSpeechSource, } from "./canonical-speech"; + import type { InterviewSpeechPreparationRequest, InterviewSpeechPreparationResult, @@ -147,11 +149,13 @@ const spokenWordCount = (text: string): number => { const preparationCacheKey = ( source: InterviewSpeechSource, contextWordBudget: number, -): string => - JSON.stringify([ +): string => { + const sourceIdentity = JSON.stringify([ ...source.contextSegments.map(({ contentHash, id }) => [id, contentHash]), contextWordBudget, ]); + return `speech-preparation:${hashCanonicalSpeechText(sourceIdentity)}`; +}; const parseContinueInterviewArguments = ( argumentsJson: string, @@ -175,6 +179,7 @@ const parseContinueInterviewArguments = ( export class RealtimeBrunchBridge { readonly #argumentDeltas = new Map(); readonly #listeners = new Set(); + readonly #preparedContextCache = new Map(); readonly #processedCalls = new Set(); readonly #session: RealtimeBridgeSession; readonly #submitInterviewAnswer: ( @@ -207,6 +212,9 @@ export class RealtimeBrunchBridge { public start(connectionEpoch: number): void { ++this.#generation; + if (this.#activeEpoch !== connectionEpoch) { + this.#preparedContextCache.clear(); + } this.#activeEpoch = connectionEpoch; this.#activeSubmission = null; this.#argumentDeltas.clear(); @@ -237,9 +245,14 @@ export class RealtimeBrunchBridge { this.#activeEpoch = null; this.#activeSubmission = null; this.#argumentDeltas.clear(); + this.#preparedContextCache.clear(); this.#terminalResponseIds.clear(); } + public cancelPendingSpeech(): void { + ++this.#generation; + } + public updateChat(update: ChatUpdate): void { this.#chat = update; if (this.#activeEpoch === null) { @@ -557,10 +570,28 @@ export class RealtimeBrunchBridge { contextWordBudget, sourceSegmentIds: source.contextSegments.map(({ id }) => id), }; + const cachedContext = this.#preparedContextCache.get(request.cacheKey); + if (cachedContext !== undefined) { + this.#deliverPreparedSpeech( + assemblePreparedInterviewSpeech({ + preparation: { + context: cachedContext, + kind: "prepared", + sourceSegmentIds: request.sourceSegmentIds, + }, + source, + }).text, + delivery, + ); + return; + } void this.#session.prepareInterviewSpeech(request).then((preparation) => { if (generation !== this.#generation) { return; } + if (preparation.kind === "prepared") { + this.#preparedContextCache.set(request.cacheKey, preparation.context); + } this.#deliverPreparedSpeech( assemblePreparedInterviewSpeech({ preparation, source }).text, delivery, diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx index ee911bc6d4d..138aab3864e 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx @@ -10,7 +10,7 @@ import { Button } from "@hashintel/ds-components"; import { css } from "@hashintel/ds-helpers/css"; import { reportVoiceDiagnostic } from "../../../voice-diagnostics"; -import { selectCanonicalSpeechSegments } from "./canonical-speech"; +import { selectInterviewSpeech } from "./canonical-speech"; import { OpenAIRealtimeSession } from "./openai-realtime-session"; import { RealtimeBrunchBridge } from "./realtime-brunch-bridge"; import { toVoiceSessionState } from "./voice-session-state"; @@ -290,9 +290,11 @@ const AvailableVoiceInterviewControl = ({ useLayoutEffect(() => { store.updateSubmissionContext(context.submitVoiceInput); + const speechSelection = selectInterviewSpeech(context.messages); store.controller.updateChat({ + automaticSource: speechSelection.automaticSource, canAcceptInterviewAnswer: context.canAcceptVoiceInput, - canonicalSegments: selectCanonicalSpeechSegments(context.messages), + canonicalSegments: [...speechSelection.canonicalSegments], status: context.status, }); }, [ diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts index 596389aad50..0134f4fa10e 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts @@ -5,7 +5,7 @@ import { VOICE_REQUEST_ID_HEADER, type VoiceDiagnosticEvent, } from "../../../voice-diagnostics"; -import { selectCanonicalSpeechSegments } from "./canonical-speech"; +import { selectInterviewSpeech } from "./canonical-speech"; import { OpenAIRealtimeSession } from "./openai-realtime-session"; import { RealtimeBrunchBridge } from "./realtime-brunch-bridge"; import { VoiceTurnController } from "./voice-turn-controller"; @@ -17,11 +17,13 @@ const browserOffer = "v=0\r\na=private-browser-sdp\r\n"; const providerAnswer = "v=0\r\na=private-provider-sdp\r\n"; const spokenAnswer = "The supervisor approves it."; const canonicalReply = "Thanks. I have recorded that."; +const preparedReply = "Approval recorded."; const canonicalQuestion = "Who is informed next?"; const requestIds = [ "00000000-0000-4000-8000-000000000011", "00000000-0000-4000-8000-000000000012", "00000000-0000-4000-8000-000000000013", + "00000000-0000-4000-8000-000000000014", ] as const; class FakeDataChannel extends EventTarget { @@ -81,12 +83,8 @@ const responseMessages = [ ...initialMessages, { id: "canonical-response-message", - parts: [{ state: "done", text: canonicalReply, type: "text" }], - role: "assistant", - }, - { - id: "next-question-message", parts: [ + { state: "done", text: canonicalReply, type: "text" }, { input: { question: canonicalQuestion }, state: "input-available", @@ -217,9 +215,11 @@ describe("controlled voice preview", () => { session, submitText: submitInterviewAnswer, }); + const initialSpeech = selectInterviewSpeech(initialMessages); controller.updateChat({ + automaticSource: initialSpeech.automaticSource, canAcceptInterviewAnswer: true, - canonicalSegments: selectCanonicalSpeechSegments(initialMessages), + canonicalSegments: [...initialSpeech.canonicalSegments], status: "ready", }); @@ -286,17 +286,64 @@ describe("controlled voice preview", () => { output: "waiting-for-tool", }); + const pendingSpeech = selectInterviewSpeech(initialMessages); controller.updateChat({ + automaticSource: pendingSpeech.automaticSource, canAcceptInterviewAnswer: false, - canonicalSegments: selectCanonicalSpeechSegments(initialMessages), + canonicalSegments: [...pendingSpeech.canonicalSegments], status: "streaming", }); + const responseSpeech = selectInterviewSpeech(responseMessages); controller.updateChat({ + automaticSource: responseSpeech.automaticSource, canAcceptInterviewAnswer: true, - canonicalSegments: selectCanonicalSpeechSegments(responseMessages), + canonicalSegments: [...responseSpeech.canonicalSegments], status: "ready", }); + const preparationCreate = sentEvents(dataChannel).at(-1)!; + expect(preparationCreate).toMatchObject({ + type: "response.create", + response: { + metadata: { petrinaut_kind: "speech-preparation" }, + output_modalities: ["text"], + }, + }); + const preparationResponse = preparationCreate.response as { + input: Array<{ content: Array<{ text: string }> }>; + metadata: Record; + }; + expect(JSON.parse(preparationResponse.input[0]!.content[0]!.text)).toEqual({ + context_text: [canonicalReply], + maximum_words: 46, + }); + dataChannel.receive({ + response: { + id: "response-prepared-reply", + metadata: preparationResponse.metadata, + }, + type: "response.created", + }); + dataChannel.receive({ + delta: preparedReply, + response_id: "response-prepared-reply", + type: "response.output_text.delta", + }); + dataChannel.receive({ + response: { + id: "response-prepared-reply", + output: [], + status: "completed", + }, + type: "response.done", + }); + + await vi.waitFor(() => + expect(sentEvents(dataChannel)).toContainEqual( + expect.objectContaining({ type: "conversation.item.create" }), + ), + ); + const [functionOutput, responseCreate] = sentEvents(dataChannel).slice(-2); expect(functionOutput).toEqual({ type: "conversation.item.create", @@ -304,7 +351,7 @@ describe("controlled voice preview", () => { type: "function_call_output", call_id: "call-1", output: JSON.stringify({ - response_text: [canonicalReply, canonicalQuestion], + response_text: [preparedReply, canonicalQuestion], }), }, }); @@ -378,6 +425,7 @@ describe("controlled voice preview", () => { providerAnswer, spokenAnswer, canonicalReply, + preparedReply, canonicalQuestion, environment.OPENAI_VOICE_API_KEY, ]) { diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts index 76c09ccf495..0f21827565e 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts @@ -28,6 +28,7 @@ const createHarness = () => { ), }; const bridge = { + cancelPendingSpeech: vi.fn(), start: vi.fn(), stop: vi.fn(), subscribe: vi.fn((listener: (event: RealtimeBrunchBridgeEvent) => void) => { @@ -115,6 +116,33 @@ describe("VoiceTurnController", () => { output: "interrupted", }); expect(harness.session.cancelOutput).not.toHaveBeenCalled(); + expect(harness.bridge.cancelPendingSpeech).toHaveBeenCalledOnce(); + }); + + test("invalidates pending preparation before pausing or cancelling paused output", async () => { + const harness = createHarness(); + const order: string[] = []; + harness.bridge.cancelPendingSpeech.mockImplementation(() => + order.push("bridge-cancel"), + ); + harness.bridge.updateChat.mockImplementation(() => + order.push("chat-update"), + ); + harness.session.cancelOutput.mockImplementation(() => + order.push("session-cancel"), + ); + await harness.controller.start(); + + harness.controller.pause(); + expect(order).toEqual(["bridge-cancel", "session-cancel"]); + + order.length = 0; + harness.controller.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [question("ask-paused")], + status: "ready", + }); + expect(order).toEqual(["chat-update", "bridge-cancel", "session-cancel"]); }); test("represents submitting and output independently without closing capture", async () => { diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts index 4277717f7aa..fa2903db9ed 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts @@ -1,6 +1,9 @@ import { VoiceError, type VoiceErrorCode } from "../../../voice-diagnostics"; -import type { CanonicalSpeechSegment } from "./canonical-speech"; +import type { + CanonicalSpeechSegment, + InterviewSpeechSource, +} from "./canonical-speech"; import type { OpenAIRealtimeSessionEvent } from "./openai-realtime-session"; import type { RealtimeBridgeErrorCode, @@ -57,6 +60,7 @@ interface RealtimeSession { } interface RealtimeBridge { + cancelPendingSpeech(): void; start(connectionEpoch: number): void; stop(): void; subscribe(listener: (event: RealtimeBrunchBridgeEvent) => void): () => void; @@ -77,6 +81,7 @@ interface VoiceTurnControllerDependencies { } interface ChatUpdate { + readonly automaticSource?: InterviewSpeechSource | null; readonly canAcceptInterviewAnswer: boolean; readonly canonicalSegments: CanonicalSpeechSegment[]; readonly status: ChatStatus; @@ -274,7 +279,7 @@ export class VoiceTurnController { this.#inputStateOnResume = this.#snapshot.input; this.#pauseRequested = true; const output = this.#snapshot.output === "idle" ? "idle" : "interrupted"; - this.#session.cancelOutput(); + this.#cancelOutput(); this.#session.setMicrophoneEnabled(false); this.#update({ input: "paused", @@ -373,7 +378,7 @@ export class VoiceTurnController { } this.#bridge.updateChat(update); if (this.#snapshot.input === "paused") { - this.#session.cancelOutput(); + this.#cancelOutput(); } } @@ -410,7 +415,7 @@ export class VoiceTurnController { const paused = this.#snapshot.input === "paused"; if (paused) { this.#inputStateOnResume = "listening"; - this.#session.cancelOutput(); + this.#cancelOutput(); } this.#update({ input: paused ? "paused" : "listening", @@ -441,7 +446,7 @@ export class VoiceTurnController { } if (event.type === "output-started") { if (this.#snapshot.input === "paused") { - this.#session.cancelOutput(); + this.#cancelOutput(); this.#update({ output: "interrupted" }); return; } @@ -463,6 +468,7 @@ export class VoiceTurnController { return; } if (event.type === "input-speech-started") { + this.#bridge.cancelPendingSpeech(); this.#transcriptItemId = event.itemId; this.#transcriptKey = null; if (this.#snapshot.output === "speaking") { @@ -536,6 +542,11 @@ export class VoiceTurnController { }); } + #cancelOutput(): void { + this.#bridge.cancelPendingSpeech(); + this.#session.cancelOutput(); + } + #recordLatency(name: VoiceLatencyEvent["name"], questionId: string): void { if (this.#answerFinalizedAt === null) return; this.#onLatencyEvent?.({ From 73f1aec37abe6351e3e73b2d73a9245bf96648ea Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Wed, 2 Sep 2026 12:36:41 +0200 Subject: [PATCH 05/15] Add canonical Voice replay actions Amp-Thread-ID: https://ampcode.com/threads/T-01a0618e-4184-736b-a426-c802f086dc83 Co-authored-by: Amp --- .../voice-interview/realtime-brunch-bridge.ts | 4 + .../voice-interview-control.test.tsx | 18 +++ .../voice-interview-control.tsx | 2 + .../voice-session-state.test.ts | 4 + .../voice-interview/voice-session-state.ts | 2 + .../voice-turn-controller.test.ts | 84 ++++++++++++- .../voice-interview/voice-turn-controller.ts | 54 ++++++++- .../src/react/voice-session/store.ts | 2 + .../src/react/voice-session/types.ts | 4 + .../react/voice-session/use-voice-session.ts | 20 +++ .../ui/types/ai-assistant-composer-control.ts | 4 + .../Editor/components/voice-session-labels.ts | 3 + .../Editor/panels/ai-assistant-panel.test.tsx | 114 +++++++++++++++++- .../Editor/panels/ai-assistant-panel.tsx | 6 + .../ai-assistant-contents.stories.tsx | 2 + .../ai-assistant-contents.test.tsx | 108 ++++++++++++++++- .../ai-assistant-contents/voice-dock.tsx | 38 ++++-- .../voice-dock/playback-menu.tsx | 47 ++++++++ 18 files changed, 502 insertions(+), 14 deletions(-) create mode 100644 libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/voice-dock/playback-menu.tsx diff --git a/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.ts b/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.ts index 33cdcdeb7a4..d75549e6e67 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.ts @@ -89,6 +89,9 @@ export type RealtimeBrunchBridgeEvent = readonly code: RealtimeBridgeErrorCode; readonly message: string; readonly type: "error"; + } + | { + readonly type: "speech-delivery-pending"; }; export interface PreparedInterviewSpeech { @@ -536,6 +539,7 @@ export class RealtimeBrunchBridge { source: InterviewSpeechSource, delivery: SpeechDelivery, ): void { + this.#emit({ type: "speech-delivery-pending" }); const generation = this.#generation; const questionWordCount = source.questionSegment ? spokenWordCount(source.questionSegment.text) diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx index b4e37880a96..4be3b7bcce4 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx @@ -213,6 +213,24 @@ describe("voice interview control", () => { ).not.toThrow(); }); + test("registers canonical replay controls with the host", () => { + const repeatQuestion = vi.spyOn( + VoiceTurnController.prototype, + "repeatQuestion", + ); + const readFullResponse = vi.spyOn( + VoiceTurnController.prototype, + "readFullResponse", + ); + render(); + + registeredVoiceModeControls?.repeatQuestion?.(); + registeredVoiceModeControls?.readFullResponse?.(); + + expect(repeatQuestion).toHaveBeenCalledOnce(); + expect(readFullResponse).toHaveBeenCalledOnce(); + }); + test("loads only a schema-valid available server configuration", async () => { const fetch = vi.fn(async () => Response.json({ available: true, connectionTimeoutMs: 15_000 }), diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx index 138aab3864e..8e030fb725a 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx @@ -312,9 +312,11 @@ const AvailableVoiceInterviewControl = ({ registerVoiceModeControls({ end: () => store.controller.end(), pause: () => store.controller.pause(), + readFullResponse: () => store.controller.readFullResponse(), reconnect: () => { void store.controller.reconnect(); }, + repeatQuestion: () => store.controller.repeatQuestion(), resume: () => store.controller.resume(), setMicrophoneMuted: (muted) => store.controller.setMicrophoneMuted(muted), diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.test.ts index d637a762607..c24b89af036 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.test.ts @@ -5,6 +5,8 @@ import { toVoiceSessionState } from "./voice-session-state"; import type { VoiceTurnSnapshot } from "./voice-turn-controller"; const listeningSnapshot = { + canReadFullResponse: true, + canRepeatQuestion: true, canReviseLastAnswer: false, connection: "connected", currentQuestion: "What happens after approval?", @@ -30,6 +32,8 @@ describe("toVoiceSessionState", () => { test("reports a listening turn with its microphone level", () => { expect(mapSnapshot()).toEqual({ + canReadFullResponse: true, + canRepeatQuestion: true, errorMessage: null, microphoneLevel: 0.24, microphoneMuted: false, diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.ts index e1a06464eb0..07d6fede838 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.ts @@ -93,6 +93,8 @@ export const toVoiceSessionState = ({ } return { + canReadFullResponse: snapshot.canReadFullResponse, + canRepeatQuestion: snapshot.canRepeatQuestion, errorMessage: snapshot.connection === "error" ? errorMessageOf(snapshot) : null, microphoneMuted: diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts index 0f21827565e..5d85ee340a2 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts @@ -3,7 +3,10 @@ import { describe, expect, test, vi } from "vitest"; import { VoiceError } from "../../../voice-diagnostics"; import { VoiceTurnController } from "./voice-turn-controller"; -import type { CanonicalSpeechSegment } from "./canonical-speech"; +import type { + CanonicalSpeechSegment, + InterviewSpeechSource, +} from "./canonical-speech"; import type { OpenAIRealtimeSessionEvent } from "./openai-realtime-session"; import type { RealtimeBrunchBridgeEvent } from "./realtime-brunch-bridge"; @@ -18,6 +21,7 @@ const createHarness = () => { connect: vi.fn(async () => ++epoch), disconnect: vi.fn(async () => undefined), setMicrophoneEnabled: vi.fn(), + speakCanonical: vi.fn(), subscribe: vi.fn( (listener: (event: OpenAIRealtimeSessionEvent) => void) => { sessionListener = listener; @@ -65,6 +69,20 @@ const question = ( text, }); +const speechSource = (): InterviewSpeechSource => { + const context = { + ...question("context", "Approval is required before release."), + source: "assistant-text" as const, + }; + const nextQuestion = question("ask-replay", "Who approves release?"); + return { + contextSegments: [context], + fullResponseSegments: [context, nextQuestion], + messageId: "message-replay", + questionSegment: nextQuestion, + }; +}; + describe("VoiceTurnController", () => { test("opens a continuous microphone before starting canonical question speech", async () => { const harness = createHarness(); @@ -145,6 +163,70 @@ describe("VoiceTurnController", () => { expect(order).toEqual(["chat-update", "bridge-cancel", "session-cancel"]); }); + test("replays canonical speech only when preparation and turn state are safe", async () => { + const harness = createHarness(); + const source = speechSource(); + harness.controller.updateChat({ + automaticSource: source, + canAcceptInterviewAnswer: true, + canonicalSegments: [...source.fullResponseSegments], + status: "ready", + }); + await harness.controller.start(); + + harness.emitBridge({ type: "speech-delivery-pending" }); + expect(harness.controller.getSnapshot()).toMatchObject({ + canReadFullResponse: false, + canRepeatQuestion: false, + }); + harness.controller.repeatQuestion(); + harness.controller.readFullResponse(); + expect(harness.session.speakCanonical).not.toHaveBeenCalled(); + + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-replay-source", + type: "output-started", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-replay-source", + type: "output-stopped", + }); + expect(harness.controller.getSnapshot()).toMatchObject({ + canReadFullResponse: true, + canRepeatQuestion: true, + }); + harness.controller.repeatQuestion(); + harness.controller.readFullResponse(); + + expect(harness.session.speakCanonical).toHaveBeenCalledOnce(); + expect(harness.session.speakCanonical).toHaveBeenCalledWith([ + source.questionSegment, + ]); + expect(harness.controller.getSnapshot()).toMatchObject({ + canReadFullResponse: false, + canRepeatQuestion: false, + output: "waiting-for-tool", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-repeat-question", + type: "output-started", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-repeat-question", + type: "output-stopped", + }); + harness.controller.readFullResponse(); + + expect(harness.session.speakCanonical).toHaveBeenNthCalledWith( + 2, + source.fullResponseSegments, + ); + }); + test("represents submitting and output independently without closing capture", async () => { const harness = createHarness(); await harness.controller.start(); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts index fa2903db9ed..523e2671f2e 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts @@ -24,6 +24,8 @@ export type VoiceOutputState = export type VoiceAnswerDelivery = "none" | "pending" | "delivered" | "failed"; export interface VoiceTurnSnapshot { + readonly canReadFullResponse: boolean; + readonly canRepeatQuestion: boolean; readonly canReviseLastAnswer: boolean; readonly connection: VoiceConnectionState; readonly currentQuestion: string; @@ -56,6 +58,7 @@ interface RealtimeSession { connect(): Promise; disconnect(): Promise; setMicrophoneEnabled(enabled: boolean): void; + speakCanonical(segments: CanonicalSpeechSegment[]): void; subscribe(listener: (event: OpenAIRealtimeSessionEvent) => void): () => void; } @@ -90,6 +93,8 @@ interface ChatUpdate { type SnapshotListener = (snapshot: VoiceTurnSnapshot) => void; const initialSnapshot: VoiceTurnSnapshot = { + canReadFullResponse: false, + canRepeatQuestion: false, canReviseLastAnswer: false, connection: "idle", currentQuestion: "", @@ -126,6 +131,7 @@ export class VoiceTurnController { #inputStateOnResume: Exclude | null = null; #pauseRequested = false; #snapshot = initialSnapshot; + #speechSource: InterviewSpeechSource | null = null; #submittingQuestionId: string | null = null; #teardownPromise: Promise | null = null; #transcriptItemId: string | null = null; @@ -369,12 +375,40 @@ export class VoiceTurnController { } } + public readFullResponse(): void { + const source = this.#speechSource; + if (!source || !this.#snapshot.canReadFullResponse) { + return; + } + this.#update({ output: "waiting-for-tool" }); + this.#session.speakCanonical([...source.fullResponseSegments]); + } + + public repeatQuestion(): void { + const question = this.#speechSource?.questionSegment; + if (!question || !this.#snapshot.canRepeatQuestion) { + return; + } + this.#update({ output: "waiting-for-tool" }); + this.#session.speakCanonical([question]); + } + public updateChat(update: ChatUpdate): void { + this.#speechSource = update.automaticSource ?? null; + const canReplay = this.#canReplay(this.#snapshot); + const replayAvailabilityChanged = + this.#snapshot.canReadFullResponse !== + (canReplay && + Boolean(this.#speechSource?.fullResponseSegments.length)) || + this.#snapshot.canRepeatQuestion !== + (canReplay && Boolean(this.#speechSource?.questionSegment)); const question = latestQuestion(update.canonicalSegments); if (question && question.id !== this.#currentQuestionId) { this.#currentQuestionId = question.id; this.#update({ currentQuestion: question.text }); this.#recordLatency("question-visible", question.id); + } else if (replayAvailabilityChanged) { + this.#update({}); } this.#bridge.updateChat(update); if (this.#snapshot.input === "paused") { @@ -412,6 +446,10 @@ export class VoiceTurnController { this.#update({ lastAnswerDelivery: "delivered" }); return; } + if (event.type === "speech-delivery-pending") { + this.#update({ output: "waiting-for-tool" }); + return; + } const paused = this.#snapshot.input === "paused"; if (paused) { this.#inputStateOnResume = "listening"; @@ -471,7 +509,7 @@ export class VoiceTurnController { this.#bridge.cancelPendingSpeech(); this.#transcriptItemId = event.itemId; this.#transcriptKey = null; - if (this.#snapshot.output === "speaking") { + if (this.#snapshot.output !== "idle") { this.#update({ output: "interrupted", partialText: "" }); } else { this.#update({ partialText: "" }); @@ -566,6 +604,14 @@ export class VoiceTurnController { ); } + #canReplay(snapshot: VoiceTurnSnapshot): boolean { + return ( + snapshot.connection === "connected" && + snapshot.input === "listening" && + (snapshot.output === "idle" || snapshot.output === "interrupted") + ); + } + #isPauseRequested(): boolean { return this.#pauseRequested; } @@ -574,6 +620,12 @@ export class VoiceTurnController { const snapshot = { ...this.#snapshot, ...update }; this.#snapshot = { ...snapshot, + canReadFullResponse: + this.#canReplay(snapshot) && + Boolean(this.#speechSource?.fullResponseSegments.length), + canRepeatQuestion: + this.#canReplay(snapshot) && + Boolean(this.#speechSource?.questionSegment), canReviseLastAnswer: this.#canReviseLastAnswer(snapshot), }; for (const listener of this.#listeners) listener(this.#snapshot); diff --git a/libs/@hashintel/petrinaut/src/react/voice-session/store.ts b/libs/@hashintel/petrinaut/src/react/voice-session/store.ts index bcdc4e0ee48..345e080b176 100644 --- a/libs/@hashintel/petrinaut/src/react/voice-session/store.ts +++ b/libs/@hashintel/petrinaut/src/react/voice-session/store.ts @@ -7,7 +7,9 @@ import type { export type VoiceSessionActions = { end: () => void; pause: () => void; + readFullResponse?: () => void; reconnect: () => void; + repeatQuestion?: () => void; resume: () => void; setMicrophoneMuted: (muted: boolean) => void; }; diff --git a/libs/@hashintel/petrinaut/src/react/voice-session/types.ts b/libs/@hashintel/petrinaut/src/react/voice-session/types.ts index bd1a26f425e..1ee8e679bd2 100644 --- a/libs/@hashintel/petrinaut/src/react/voice-session/types.ts +++ b/libs/@hashintel/petrinaut/src/react/voice-session/types.ts @@ -16,6 +16,10 @@ export type PetrinautAiVoiceSessionPhase = * effect: it changes at microphone-sampling rate. */ export type PetrinautAiVoiceSessionState = { + /** Whether the current canonical assistant response is safe to replay. */ + canReadFullResponse?: boolean; + /** Whether the current canonical interview question is safe to repeat. */ + canRepeatQuestion?: boolean; errorMessage: string | null; /** Whether microphone capture is muted independently of whose turn it is. */ microphoneMuted: boolean; diff --git a/libs/@hashintel/petrinaut/src/react/voice-session/use-voice-session.ts b/libs/@hashintel/petrinaut/src/react/voice-session/use-voice-session.ts index a95671ce61d..916c5a39d17 100644 --- a/libs/@hashintel/petrinaut/src/react/voice-session/use-voice-session.ts +++ b/libs/@hashintel/petrinaut/src/react/voice-session/use-voice-session.ts @@ -62,3 +62,23 @@ export const useVoiceSessionActions = (): VoiceSessionActions | null => { () => null, ); }; + +export const useVoiceSessionCanReadFullResponse = (): boolean => { + const store = use(VoiceSessionContext); + + return useSyncExternalStore( + store.subscribe, + () => store.getSnapshot().state?.canReadFullResponse ?? false, + () => false, + ); +}; + +export const useVoiceSessionCanRepeatQuestion = (): boolean => { + const store = use(VoiceSessionContext); + + return useSyncExternalStore( + store.subscribe, + () => store.getSnapshot().state?.canRepeatQuestion ?? false, + () => false, + ); +}; diff --git a/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts b/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts index 5369be4c639..c2778346770 100644 --- a/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts +++ b/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts @@ -57,8 +57,12 @@ export type PetrinautAiVoiceModeControls = { end: () => Promise; /** Pauses microphone capture and active Voice output synchronously. */ pause: () => void; + /** Reads the complete canonical assistant response without rewriting it. */ + readFullResponse?: () => void; /** Re-establishes a session that dropped, keeping the conversation. */ reconnect: () => void; + /** Repeats only the current canonical interview question. */ + repeatQuestion?: () => void; /** Resumes microphone capture after `pause`. */ resume: () => void; /** diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/components/voice-session-labels.ts b/libs/@hashintel/petrinaut/src/ui/views/Editor/components/voice-session-labels.ts index eaf5dabb078..f4ea8f4fa3e 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/components/voice-session-labels.ts +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/components/voice-session-labels.ts @@ -29,7 +29,10 @@ export const voiceSessionActionLabels = { end: "End voice mode", mute: "Mute microphone", pause: "Pause voice mode", + playbackOptions: "Voice playback options", + readFullResponse: "Read full response", reconnect: "Reconnect voice mode", + repeatQuestion: "Repeat question", resume: "Resume voice mode", unmute: "Unmute microphone", } as const; diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx index b197c585cfd..a7e0a2ab7ca 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx @@ -10,7 +10,7 @@ import { waitFor, } from "@testing-library/react"; import { useEffect } from "react"; -import { afterEach, describe, expect, test, vi } from "vitest"; +import { afterEach, beforeAll, describe, expect, test, vi } from "vitest"; import { DEFAULT_PETRINAUT_EXTENSIONS, @@ -156,6 +156,29 @@ const SubmitForSecondConversation = ({ const testInstances: ReturnType[] = []; +beforeAll(() => { + vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue(null); + vi.stubGlobal( + "PointerEvent", + class extends MouseEvent { + public readonly pointerType: string; + + public constructor(type: string, init: PointerEventInit = {}) { + super(type, init); + this.pointerType = init.pointerType ?? ""; + } + }, + ); + vi.stubGlobal( + "ResizeObserver", + class { + public disconnect() {} + public observe() {} + public unobserve() {} + }, + ); +}); + const renderTestPanel = ({ aiAssistant, editorContext = editorContextValue, @@ -1032,6 +1055,95 @@ describe("AiAssistantPanel composer submissions", () => { expect(submitVoiceInputReferences.size).toBe(1); }); + test("forwards registered canonical replay controls to the Voice dock", async () => { + const readFullResponse = vi.fn(); + const repeatQuestion = vi.fn(); + const VoiceMode = (context: PetrinautAiVoiceModeContext) => { + const { + inputMode, + registerVoiceModeControls, + reportVoiceSessionState, + setVoiceActive, + } = context; + + useEffect(() => { + if (inputMode !== "voice") { + return; + } + const unregister = registerVoiceModeControls({ + end: async () => undefined, + pause: vi.fn(), + readFullResponse, + reconnect: vi.fn(), + repeatQuestion, + resume: vi.fn(), + setMicrophoneMuted: vi.fn(), + }); + reportVoiceSessionState({ + canReadFullResponse: true, + canRepeatQuestion: true, + errorMessage: null, + microphoneLevel: 0, + phase: "listening", + }); + setVoiceActive(true); + + return unregister; + }, [ + inputMode, + registerVoiceModeControls, + reportVoiceSessionState, + setVoiceActive, + ]); + + return null; + }; + + renderTestPanel({ + aiAssistant: { + renderVoiceMode: (context) => , + transport: { + reconnectToStream: () => Promise.resolve(null), + sendMessages: vi.fn(), + }, + }, + }); + + fireEvent.click(screen.getByRole("button", { name: "Start voice mode" })); + fireEvent.click( + screen.getByRole("button", { name: "Voice playback options" }), + ); + const readFullResponseItem = await screen.findByRole("menuitem", { + name: "Read full response", + }); + fireEvent.pointerMove(readFullResponseItem, { pointerType: "mouse" }); + await waitFor(() => + expect(readFullResponseItem.hasAttribute("data-highlighted")).toBe(true), + ); + fireEvent.click(readFullResponseItem); + await waitFor(() => expect(readFullResponse).toHaveBeenCalledOnce()); + await waitFor(() => + expect(screen.queryByRole("menu", { hidden: true })).toBeNull(), + ); + fireEvent.click( + screen.getByRole("button", { name: "Voice playback options" }), + ); + const repeatQuestionItem = await screen.findByRole("menuitem", { + name: "Repeat question", + }); + const playbackMenu = screen.getByRole("menu"); + fireEvent.keyDown(playbackMenu, { key: "ArrowDown" }); + await waitFor(() => + expect(playbackMenu.getAttribute("aria-activedescendant")).toBe( + repeatQuestionItem.id, + ), + ); + fireEvent.keyDown(playbackMenu, { key: "Enter" }); + + await waitFor(() => expect(repeatQuestion).toHaveBeenCalledOnce()); + expect(readFullResponse).toHaveBeenCalledOnce(); + }); + test("keeps one mounted voice mode when the panel closes and reopens", () => { voiceModeMounts = 0; voiceModeUnmounts = 0; diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx index df83042ed1b..cc4f11010db 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx @@ -486,7 +486,13 @@ export const AiAssistantPanel = ({ // invalidates the host's active generation. end: () => requestInputMode("text"), pause: () => controls.pause(), + ...(controls.readFullResponse + ? { readFullResponse: controls.readFullResponse } + : {}), reconnect: () => controls.reconnect(), + ...(controls.repeatQuestion + ? { repeatQuestion: controls.repeatQuestion } + : {}), resume: () => controls.resume(), setMicrophoneMuted: (muted) => controls.setMicrophoneMuted(muted), }); diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.stories.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.stories.tsx index c21fe5af927..f6687f6ac18 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.stories.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.stories.tsx @@ -304,6 +304,8 @@ const Frame = ({ const liveSession = ( overrides: Partial, ): PetrinautAiVoiceSessionState => ({ + canReadFullResponse: true, + canRepeatQuestion: true, errorMessage: null, microphoneLevel: 0, microphoneMuted: false, diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx index ee2d2d8c6f7..5ba1c5d65f2 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx @@ -50,6 +50,25 @@ const noop = () => {}; // instead of letting jsdom log a not-implemented error per render. beforeAll(() => { vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue(null); + vi.stubGlobal( + "PointerEvent", + class extends MouseEvent { + public readonly pointerType: string; + + public constructor(type: string, init: PointerEventInit = {}) { + super(type, init); + this.pointerType = init.pointerType ?? ""; + } + }, + ); + vi.stubGlobal( + "ResizeObserver", + class { + public disconnect() {} + public observe() {} + public unobserve() {} + }, + ); }); afterEach(() => { @@ -142,12 +161,16 @@ describe("AiAssistantContents", () => { const actions = { end: vi.fn(), pause: vi.fn(), + readFullResponse: vi.fn(), reconnect: vi.fn(), + repeatQuestion: vi.fn(), resume: vi.fn(), setMicrophoneMuted: vi.fn(), }; store.setActions(actions); store.setState({ + canReadFullResponse: true, + canRepeatQuestion: true, errorMessage: null, microphoneLevel: 0, microphoneMuted: false, @@ -234,17 +257,21 @@ describe("AiAssistantContents", () => { ).not.toBeNull(); }); - test("keeps the session's controls in the dock", () => { + test("keeps the session's controls in the dock", async () => { const store = createVoiceSessionStore(); const actions = { end: vi.fn(), pause: vi.fn(), + readFullResponse: vi.fn(), reconnect: vi.fn(), + repeatQuestion: vi.fn(), resume: vi.fn(), setMicrophoneMuted: vi.fn(), }; store.setActions(actions); store.setState({ + canReadFullResponse: true, + canRepeatQuestion: true, errorMessage: null, microphoneLevel: 0.4, microphoneMuted: false, @@ -277,6 +304,39 @@ describe("AiAssistantContents", () => { expect(actions.setMicrophoneMuted).toHaveBeenCalledWith(true); expect(actions.end).toHaveBeenCalledOnce(); + fireEvent.click( + within(dock).getByRole("button", { name: "Voice playback options" }), + ); + const readFullResponse = await screen.findByRole("menuitem", { + name: "Read full response", + }); + expect(readFullResponse.hasAttribute("data-disabled")).toBe(false); + fireEvent.pointerMove(readFullResponse, { pointerType: "mouse" }); + await waitFor(() => + expect(readFullResponse.hasAttribute("data-highlighted")).toBe(true), + ); + fireEvent.click(readFullResponse); + await waitFor(() => + expect(actions.readFullResponse).toHaveBeenCalledOnce(), + ); + fireEvent.click( + within(dock).getByRole("button", { name: "Voice playback options" }), + ); + const repeatQuestion = await screen.findByRole("menuitem", { + name: "Repeat question", + }); + const playbackMenu = screen.getByRole("menu"); + fireEvent.keyDown(playbackMenu, { key: "ArrowDown" }); + await waitFor(() => + expect(playbackMenu.getAttribute("aria-activedescendant")).toBe( + repeatQuestion.id, + ), + ); + fireEvent.keyDown(playbackMenu, { key: "Enter" }); + + await waitFor(() => expect(actions.repeatQuestion).toHaveBeenCalledOnce()); + expect(actions.readFullResponse).toHaveBeenCalledOnce(); + act(() => { store.setState({ errorMessage: null, @@ -294,6 +354,52 @@ describe("AiAssistantContents", () => { expect(actions.setMicrophoneMuted).toHaveBeenLastCalledWith(false); }); + test("disables unavailable Voice playback actions", async () => { + const store = createVoiceSessionStore(); + store.setActions({ + end: vi.fn(), + pause: vi.fn(), + reconnect: vi.fn(), + resume: vi.fn(), + setMicrophoneMuted: vi.fn(), + }); + store.setState({ + canReadFullResponse: false, + canRepeatQuestion: false, + errorMessage: null, + microphoneLevel: 0, + phase: "listening", + }); + render( + + + , + ); + + fireEvent.click( + screen.getByRole("button", { name: "Voice playback options" }), + ); + + expect( + ( + await screen.findByRole("menuitem", { name: "Repeat question" }) + ).hasAttribute("data-disabled"), + ).toBe(true); + expect( + screen + .getByRole("menuitem", { name: "Read full response" }) + .hasAttribute("data-disabled"), + ).toBe(true); + }); + test("shows a voice recovery failure as a toast", async () => { const store = createVoiceSessionStore(); store.setState({ diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/voice-dock.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/voice-dock.tsx index db2177be73f..823bbed7fac 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/voice-dock.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/voice-dock.tsx @@ -3,6 +3,8 @@ import { css, cva } from "@hashintel/ds-helpers/css"; import { useVoiceSessionActions, + useVoiceSessionCanReadFullResponse, + useVoiceSessionCanRepeatQuestion, useVoiceSessionMicrophoneMuted, useVoiceSessionPhase, } from "../../../../../../react/voice-session/use-voice-session"; @@ -13,6 +15,7 @@ import { } from "../../../components/voice-session-labels"; import { aiFooterMinHeight } from "./footer-height"; import { MicrophoneIcon } from "./voice-dock/microphone-icon"; +import { VoicePlaybackMenu } from "./voice-dock/playback-menu"; import { TranscriptionIcon } from "./voice-dock/transcription-icon"; import type { VoiceSessionActions } from "../../../../../../react/voice-session/store"; @@ -100,6 +103,8 @@ const visuallyHiddenStyle = css({ export type VoiceDockProps = { actions: VoiceSessionActions | null; + canReadFullResponse: boolean; + canRepeatQuestion: boolean; /** Rendered instead of the live indicator when the caller supplies one. */ indicator?: ReactNode; microphoneMuted: boolean; @@ -117,6 +122,8 @@ export type VoiceDockProps = { */ export const VoiceDock = ({ actions, + canReadFullResponse, + canRepeatQuestion, indicator, microphoneMuted, onTranscriptionToggle, @@ -139,16 +146,23 @@ export const VoiceDock = ({ > {actions !== null && ( -