FE-1570: Optimize spoken Brunch responses for Voice mode

.changeset/stable-composer-controls.md

@@ -39,3 +39,11 @@ End Voice mode before submitting typed text exactly once through the shared comp
 draft if handoff fails. Pause active media before the AI panel closes and reopen the mounted session
 paused. Provisional transcription and Realtime audio remain ephemeral rather than becoming
 persisted chat history.
+
+Prepare concise spoken context within a strict 50-word budget while preserving Brunch's exact
+protected question, which application code appends unchanged for tool-disabled audio rendering.
+Fall back to canonical context and question whenever preparation is unavailable, invalid, or times
+out.
+
+Add canonical replay controls for repeating the question or reading the full response without
+speech preparation.

apps/petrinaut-website/README.md

@@ -97,12 +97,12 @@ bubble is replaced by the finalized message or pending-question tool output,
 which retains a waveform indicator without duplicating the answer. Provisional
 transcription and Realtime audio are not persisted as chat history.
 
-The text composer remains available. Sending typed text ends Voice mode first,
-then submits the draft exactly once through the same conversation; a failed
-handoff restores the draft. Closing the assistant pauses capture and speech
-before hiding it. Reopening preserves the mounted session in **Paused** state.
-**Pause** and **End voice mode** live under **Voice mode actions**, while
-**Resume** or **Reconnect** appears as the primary action when applicable.
+The active session replaces the text composer with the Voice dock, and all
+session controls are direct dock controls. Closing the assistant pauses capture
+and speech before hiding it. Reopening preserves the mounted session in
+**Paused** state. A compact **Voice playback options** menu beside the
+transcription control provides **Repeat question** and **Read full response**;
+both replay canonical Brunch content and bypass speech preparation.
 
 The browser sends its SDP offer to this app; the server initializes a trusted
 `gpt-realtime-2` audio-input/audio-output session through OpenAI's unified
@@ -118,12 +118,19 @@ and durable history. The browser bridge accepts only the configured
 duplicate or stale calls, and submits the answer through Petrinaut's shared
 composer path with pending-`brunch_ask` correlation.
 
-The bridge waits for the correlated Brunch turn before returning canonical
-speech segments to Realtime. It then requests audio with tools disabled and
-instructs Realtime to speak only those segments. Generated audio is not a
-verbatim record: canonical Brunch text remains visible and authoritative. The
-microphone stays active while the interviewer speaks and while Brunch is
-working. Speaking over assistant audio interrupts playback automatically;
+The experimental **Approach D** design waits for the correlated Brunch turn,
+then gives Realtime two bounded roles. First, an out-of-band, text-only request
+semantically prepares the canonical context within the portion of a strict
+50-word spoken budget left after reserving the exact Brunch question.
+Application code validates the prepared context and appends the protected
+question exactly. Second, a
+tool-disabled audio request renders only those supplied words verbatim. If
+preparation is unavailable, invalid, or times out, the bridge supplies the
+canonical context and question instead. Brunch's canonical transcript and
+exact question are never rewritten and remain authoritative. Preparation,
+provisional transcription, and Realtime audio are ephemeral and are not
+persisted. The microphone stays active while the interviewer speaks and while
+Brunch is working. Speaking over assistant audio interrupts playback automatically;
 WebRTC truncates provider-side unheard audio without changing Brunch history.
 
 The Brunch deployment must allow the website origin through its

apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.test.ts

@@ -5,6 +5,7 @@ import { ASK_TOOL_NAME } from "@hashintel/brunch-agent/client-tools";
 import {
   hashCanonicalSpeechText,
   selectCanonicalSpeechSegments,
+  selectInterviewSpeech,
 } from "./canonical-speech";
 
 import type { PetrinautAiMessage } from "@hashintel/petrinaut/ui";
@@ -13,6 +14,167 @@ const select = (messages: PetrinautAiMessage[]) =>
   selectCanonicalSpeechSegments(messages);
 
 describe("canonical speech selection", () => {
+  test("separates finalized context from an exact pending question", () => {
+    const completeCanonicalExplanation =
+      "The release needs one named approver before the batch can proceed.";
+    const question = "Who approves it: the manager or quality lead?";
+    const messages = [
+      {
+        id: "assistant-turn",
+        role: "assistant",
+        parts: [
+          {
+            type: "text",
+            text: completeCanonicalExplanation,
+            state: "done",
+          },
+          {
+            type: "dynamic-tool",
+            toolCallId: "ask-1",
+            toolName: ASK_TOOL_NAME,
+            state: "input-available",
+            input: { question },
+          },
+        ],
+      },
+    ] satisfies PetrinautAiMessage[];
+
+    const selection = selectInterviewSpeech(messages);
+
+    expect(selection.automaticSource).toMatchObject({
+      contextSegments: [
+        { source: "assistant-text", text: completeCanonicalExplanation },
+      ],
+      messageId: "assistant-turn",
+      questionSegment: { source: "brunch-ask", text: question },
+      fullResponseSegments: [
+        { source: "assistant-text", text: completeCanonicalExplanation },
+        { source: "brunch-ask", text: question },
+      ],
+    });
+    expect(selection.canonicalSegments).toEqual(
+      selection.automaticSource?.fullResponseSegments,
+    );
+  });
+
+  test("groups multiple finalized text parts into the latest assistant turn", () => {
+    const selection = selectInterviewSpeech([
+      {
+        id: "assistant-old",
+        role: "assistant",
+        parts: [{ type: "text", text: "Previous turn.", state: "done" }],
+      },
+      {
+        id: "assistant-latest",
+        role: "assistant",
+        parts: [
+          { type: "text", text: "First explanation.", state: "done" },
+          { type: "text", text: "Second explanation.", state: "done" },
+        ],
+      },
+    ]);
+
+    expect(selection.automaticSource).toMatchObject({
+      contextSegments: [
+        { text: "First explanation." },
+        { text: "Second explanation." },
+      ],
+      fullResponseSegments: [
+        { text: "First explanation." },
+        { text: "Second explanation." },
+      ],
+      messageId: "assistant-latest",
+      questionSegment: null,
+    });
+    expect(selection.canonicalSegments).toHaveLength(3);
+  });
+
+  test("selects standalone finalized assistant text for automatic speech", () => {
+    const selection = selectInterviewSpeech([
+      {
+        id: "assistant-complete",
+        role: "assistant",
+        parts: [
+          { type: "text", text: "The interview is complete.", state: "done" },
+        ],
+      },
+    ]);
+
+    expect(selection.automaticSource).toMatchObject({
+      contextSegments: [{ text: "The interview is complete." }],
+      fullResponseSegments: [{ text: "The interview is complete." }],
+      questionSegment: null,
+    });
+  });
+
+  test("excludes answered and malformed asks from the protected question", () => {
+    const selection = selectInterviewSpeech([
+      {
+        id: "assistant-ask",
+        role: "assistant",
+        parts: [
+          { type: "text", text: "Canonical context.", state: "done" },
+          {
+            type: "dynamic-tool",
+            toolCallId: "ask-answered",
+            toolName: ASK_TOOL_NAME,
+            state: "output-available",
+            input: { question: "Already answered?" },
+            output: { answer: "Yes." },
+          },
+          {
+            type: "dynamic-tool",
+            toolCallId: "ask-malformed",
+            toolName: ASK_TOOL_NAME,
+            state: "input-available",
+            input: { question: 42 },
+          },
+        ],
+      },
+    ]);
+
+    expect(selection.automaticSource).toBeNull();
+    expect(selection.canonicalSegments).toHaveLength(1);
+  });
+
+  test("excludes streaming text, reasoning, diagnostics, tool output, and tool errors", () => {
+    const selection = selectInterviewSpeech([
+      {
+        id: "assistant-filtered",
+        role: "assistant",
+        parts: [
+          {
+            type: "text",
+            text: "Do not speak streaming text.",
+            state: "streaming",
+          },
+          { type: "reasoning", text: "Do not speak reasoning.", state: "done" },
+          {
+            type: "dynamic-tool",
+            toolCallId: "tool-output",
+            toolName: "diagnostic",
+            state: "output-available",
+            input: {},
+            output: { text: "Do not speak tool output." },
+          },
+          {
+            type: "dynamic-tool",
+            toolCallId: "tool-error",
+            toolName: "diagnostic",
+            state: "output-error",
+            input: {},
+            errorText: "Do not speak tool errors.",
+          },
+        ],
+      },
+    ]);
+
+    expect(selection).toEqual({
+      automaticSource: null,
+      canonicalSegments: [],
+    });
+  });
+
   test("selects only finalized assistant text without changing it", () => {
     const messages = [
       {
@@ -174,5 +336,22 @@ describe("canonical speech selection", () => {
     expect(changed[0]?.partId).toBe(first[0]?.partId);
     expect(changed[0]?.contentHash).not.toBe(first[0]?.contentHash);
     expect(changed[0]?.id).not.toBe(first[0]?.id);
+
+    const firstSelection = selectInterviewSpeech([
+      {
+        id: "assistant/id",
+        role: "assistant",
+        parts: [{ type: "text", text: "Exact text", state: "done" }],
+      },
+    ]);
+    expect(
+      selectInterviewSpeech([
+        {
+          id: "assistant/id",
+          role: "assistant",
+          parts: [{ type: "text", text: "Exact text", state: "done" }],
+        },
+      ]),
+    ).toEqual(firstSelection);
   });
 });

apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.ts

@@ -18,6 +18,18 @@ export interface CanonicalSpeechSegment {
   readonly text: string;
 }
 
+export interface InterviewSpeechSource {
+  readonly contextSegments: readonly CanonicalSpeechSegment[];
+  readonly fullResponseSegments: readonly CanonicalSpeechSegment[];
+  readonly messageId: string;
+  readonly questionSegment: CanonicalSpeechSegment | null;
+}
+
+export interface InterviewSpeechSelection {
+  readonly automaticSource: InterviewSpeechSource | null;
+  readonly canonicalSegments: readonly CanonicalSpeechSegment[];
+}
+
 const createSegment = (
   messageId: string,
   partId: string,
@@ -40,56 +52,83 @@ const createSegment = (
   };
 };
 
-export const selectCanonicalSpeechSegments = (
+export const selectInterviewSpeech = (
   messages: PetrinautAiMessage[],
-): CanonicalSpeechSegment[] => {
-  const segments: CanonicalSpeechSegment[] = [];
+): InterviewSpeechSelection => {
+  const canonicalSegments: CanonicalSpeechSegment[] = [];
+  let automaticSource: InterviewSpeechSource | null = null;
 
   for (const message of messages) {
     if (message.role !== "assistant") {
       continue;
     }
 
+    const contextSegments: CanonicalSpeechSegment[] = [];
+    const questionSegments: CanonicalSpeechSegment[] = [];
+    let hasAskPart = false;
+
     for (const [partIndex, part] of message.parts.entries()) {
       if (
         part.type === "text" &&
         part.state !== "streaming" &&
         part.text.trim()
       ) {
-        segments.push(
-          createSegment(
-            message.id,
-            `text:${partIndex}`,
-            "assistant-text",
-            part.text,
-          ),
+        const segment = createSegment(
+          message.id,
+          `text:${partIndex}`,
+          "assistant-text",
+          part.text,
         );
+        canonicalSegments.push(segment);
+        contextSegments.push(segment);
         continue;
       }
 
-      if (
-        part.type !== "dynamic-tool" ||
-        part.toolName !== ASK_TOOL_NAME ||
-        part.state !== "input-available"
-      ) {
+      if (part.type !== "dynamic-tool" || part.toolName !== ASK_TOOL_NAME) {
+        continue;
+      }
+      hasAskPart = true;
+      if (part.state !== "input-available") {
         continue;
       }
 
       try {
         const input = parseBrunchAskInput(part.input);
-        segments.push(
-          createSegment(
-            message.id,
-            part.toolCallId,
-            "brunch-ask",
-            input.question,
-          ),
+        const segment = createSegment(
+          message.id,
+          part.toolCallId,
+          "brunch-ask",
+          input.question,
         );
+        canonicalSegments.push(segment);
+        questionSegments.push(segment);
       } catch {
         // Malformed tool inputs remain visible as tool errors; they are not spoken.
       }
     }
+
+    const questionSegment = questionSegments.at(-1) ?? null;
+    if (contextSegments.length > 0 || hasAskPart) {
+      automaticSource =
+        questionSegment || !hasAskPart
+          ? {
+              contextSegments,
+              fullResponseSegments: [
+                ...contextSegments,
+                ...(questionSegment ? [questionSegment] : []),
+              ],
+              messageId: message.id,
+              questionSegment,
+            }
+          : null;
+    }
   }
 
-  return segments;
+  return { automaticSource, canonicalSegments };
 };
+
+export const selectCanonicalSpeechSegments = (
+  messages: PetrinautAiMessage[],
+): CanonicalSpeechSegment[] => [
+  ...selectInterviewSpeech(messages).canonicalSegments,
+];