FE-1580: Port omitted Voice settlement behavior

.changeset/flue-voice-safety.md

@@ -2,4 +2,4 @@
 "@hashintel/petrinaut": patch
 ---
 
-Add half-duplex Voice handoff, exact response and marked-question replay, live transcripts, compact Voice setup and playback controls, and persistent copyable errors. Keep the conversation busy through browser-tool continuations, withhold pending work on Stop, surface automatic-tool failures to Voice, and display stopped entries and surviving client-tool Voice origins supplied by canonical history.
+Add half-duplex Voice handoff, exact response and marked-question replay, live transcripts, compact Voice setup and playback controls, and persistent copyable errors. Keep the conversation busy through browser-tool continuations, withhold pending work on Stop, preserve rejected durable Stop failures through late provider completion, surface automatic-tool failures to Voice, and display stopped entries and surviving client-tool Voice origins supplied by canonical history.

apps/petrinaut-website/src/main/app/local-storage-demo/brunch-client-tools.ts

@@ -1,14 +1,12 @@
-import { ASK_TOOL_NAME } from "@hashintel/brunch-agent/client-tools";
 import { readPetrinautDocToolName } from "@hashintel/petrinaut-core";
 
 /**
  * The one catalog of tools the browser answers on Brunch's behalf. The panel
- * transport admits their results, the history projection leaves them runnable,
- * and every interactive widget the demo registers must name one of them so a
- * composer answer reaches Flue as a `client-tool-result` rather than an error.
+ * transport admits their results and the history projection leaves them runnable.
+ * The production preview has no interactive ask handler; fixture-specific tools
+ * extend this default catalog without restoring the suspended ask path.
  * Kept free of React imports so the transport can load outside the DOM.
  */
 export const brunchClientToolNames: ReadonlySet<string> = new Set([
   readPetrinautDocToolName,
-  ASK_TOOL_NAME,
 ]);

apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx

@@ -10,6 +10,7 @@ import { defaultPetrinautNavigationHistoryPolicy } from "@hashintel/petrinaut/re
 
 import { OpenAIRealtimeSession } from "../voice-interview/openai-realtime-session";
 import { VoiceInterviewControl } from "../voice-interview/voice-interview-control";
+import { brunchClientToolNames } from "./brunch-client-tools";
 import { BrunchPanelConversationTracker } from "./brunch-panel-transport";
 import {
   getBrunchVoiceMode,
@@ -233,6 +234,7 @@ describe("local storage demo Brunch voice integration", () => {
     const aiAssistant = renderedPetrinaut.aiAssistant as PetrinautAiAssistant;
 
     expect(aiAssistant.requestStop).toBeTypeOf("function");
+    expect([...brunchClientToolNames]).toEqual(["readPetrinautDoc"]);
     expect(aiAssistant.interactiveTools).toEqual([]);
     expect(
       aiAssistant.interactiveTools?.some(

apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts

@@ -683,12 +683,95 @@ describe("OpenAIRealtimeSession", () => {
     });
     expect(harness.events).toContainEqual({
       connectionEpoch: 1,
+      playbackExpected: false,
       responseId: "response-active",
       status: "completed",
       type: "response-terminal",
     });
   });
 
+  test("reopens capture when a completed canonical response has no audio", async () => {
+    const harness = createHarness();
+    await harness.session.connect();
+    harness.session.setMicrophoneEnabled(true);
+    harness.session.speakCanonical([
+      canonicalSegment("silent", "This response produced no audio."),
+    ]);
+    const channel = harness.channels[0]!;
+    authorizeLatestSpeechResponse(channel, "response-silent");
+
+    channel.receive({
+      response: {
+        id: "response-silent",
+        output: [],
+        status: "completed",
+      },
+      type: "response.done",
+    });
+
+    expect(harness.events).toContainEqual({
+      connectionEpoch: 1,
+      playbackExpected: false,
+      responseId: "response-silent",
+      speechRequestId: "canonical-1-1",
+      status: "completed",
+      type: "response-terminal",
+    });
+    expect(
+      harness.events.some(
+        (event) =>
+          event.type === "output-started" || event.type === "output-stopped",
+      ),
+    ).toBe(false);
+    expect(harness.localTracks[0]!.enabled).toBe(true);
+  });
+
+  test("keeps active audio owned when completed output omits audio metadata", async () => {
+    const harness = createHarness();
+    await harness.session.connect();
+    harness.session.setMicrophoneEnabled(true);
+    harness.session.speakCanonical([
+      canonicalSegment("playing", "This response is already playing."),
+    ]);
+    const channel = harness.channels[0]!;
+    authorizeLatestSpeechResponse(channel, "response-playing");
+    channel.receive({
+      response_id: "response-playing",
+      type: "output_audio_buffer.started",
+    });
+
+    channel.receive({
+      response: {
+        id: "response-playing",
+        output: [],
+        status: "completed",
+      },
+      type: "response.done",
+    });
+
+    expect(harness.events).toContainEqual({
+      connectionEpoch: 1,
+      playbackExpected: true,
+      responseId: "response-playing",
+      speechRequestId: "canonical-1-1",
+      status: "completed",
+      type: "response-terminal",
+    });
+    expect(harness.localTracks[0]!.enabled).toBe(false);
+
+    channel.receive({
+      response_id: "response-playing",
+      type: "output_audio_buffer.stopped",
+    });
+
+    expect(harness.events).toContainEqual({
+      connectionEpoch: 1,
+      responseId: "response-playing",
+      type: "output-stopped",
+    });
+    expect(harness.localTracks[0]!.enabled).toBe(true);
+  });
+
   test("keeps the microphone closed when an earlier stop follows a queued response request", async () => {
     const harness = createHarness();
     await harness.session.connect();
@@ -709,7 +792,15 @@ describe("OpenAIRealtimeSession", () => {
     channel.receive({
       response: {
         id: "response-early",
-        output: [],
+        output: [
+          {
+            content: [
+              { transcript: "First canonical segment.", type: "output_audio" },
+            ],
+            role: "assistant",
+            type: "message",
+          },
+        ],
         status: "completed",
       },
       type: "response.done",
@@ -722,7 +813,15 @@ describe("OpenAIRealtimeSession", () => {
     channel.receive({
       response: {
         id: "response-follow-on",
-        output: [],
+        output: [
+          {
+            content: [
+              { transcript: "Second canonical segment.", type: "output_audio" },
+            ],
+            role: "assistant",
+            type: "message",
+          },
+        ],
         status: "completed",
       },
       type: "response.done",
@@ -745,6 +844,7 @@ describe("OpenAIRealtimeSession", () => {
     });
     expect(harness.events).toContainEqual({
       connectionEpoch: 1,
+      playbackExpected: true,
       responseId: "response-follow-on",
       speechRequestId: "canonical-1-2",
       status: "completed",
@@ -803,12 +903,29 @@ describe("OpenAIRealtimeSession", () => {
     channel.receive({
       response: {
         id: "response-generated",
-        output: [],
+        output: [
+          {
+            content: [
+              { transcript: "Generated canonical segment.", type: "audio" },
+            ],
+            role: "assistant",
+            type: "message",
+          },
+        ],
         status: "completed",
       },
       type: "response.done",
     });
 
+    expect(harness.events).toContainEqual({
+      connectionEpoch: 1,
+      playbackExpected: true,
+      responseId: "response-generated",
+      speechRequestId: "canonical-1-1",
+      status: "completed",
+      type: "response-terminal",
+    });
+
     const cancellation = harness.session.cancelOutput();
     let settled = false;
     void cancellation.then(() => {

apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts

@@ -63,6 +63,7 @@ export type OpenAIRealtimeSessionEvent =
     }
   | {
       readonly connectionEpoch: number;
+      readonly playbackExpected: boolean;
       readonly responseId: string;
       readonly speechRequestId?: string;
       readonly status: "cancelled" | "completed" | "failed" | "incomplete";
@@ -140,6 +141,19 @@ const nonEmptyString = (value: unknown): string | null =>
 const nonNegativeInteger = (value: unknown): number | null =>
   Number.isInteger(value) && (value as number) >= 0 ? (value as number) : null;
 
+const responseContainsAudio = (output: unknown[]): boolean =>
+  output.some((item) => {
+    const outputItem = asRecord(item);
+    if (outputItem?.type !== "message" || !Array.isArray(outputItem.content)) {
+      return false;
+    }
+    return outputItem.content.some(
+      (contentItem) =>
+        asRecord(contentItem)?.type === "output_audio" ||
+        asRecord(contentItem)?.type === "audio",
+    );
+  });
+
 const parseRealtimeEvent = (value: unknown): Record<string, unknown> | null => {
   if (typeof value !== "string") {
     return null;
@@ -820,6 +834,7 @@ export class OpenAIRealtimeSession {
       return;
     }
     const terminalStatus = status as ResponseTerminalStatus;
+    let playbackExpected = false;
     this.#responseTerminalSequence += 1;
     this.#activeResponseIds.delete(responseId);
     this.#cancelOutputAwaitingResponseIds.delete(responseId);
@@ -829,6 +844,7 @@ export class OpenAIRealtimeSession {
     const speechRequestId = this.#speechRequestIds.get(responseId);
     const terminalEvent = {
       connectionEpoch,
+      playbackExpected,
       responseId,
       ...(speechRequestId === undefined ? {} : { speechRequestId }),
       status: terminalStatus,
@@ -859,10 +875,18 @@ export class OpenAIRealtimeSession {
         this.#handleConnectionFailure("invalid-response", "connection");
         return;
       }
+      playbackExpected =
+        this.#speakingResponseId === responseId ||
+        responseContainsAudio(output);
       if (this.#authorizedResponseIds.has(responseId)) {
-        this.#terminalCanonicalResponseIds.add(responseId);
+        if (playbackExpected) {
+          this.#terminalCanonicalResponseIds.add(responseId);
+        }
+      }
+      this.#emit({ ...terminalEvent, playbackExpected });
+      if (this.#authorizedResponseIds.has(responseId) && !playbackExpected) {
+        this.#finishSpeech(responseId);
       }
-      this.#emit(terminalEvent);
       this.#resumeCanonicalSpeechQueue();
       return;
     }
@@ -874,12 +898,12 @@ export class OpenAIRealtimeSession {
           type: "output-interrupted",
         });
       }
-      this.#emit(terminalEvent);
+      this.#emit({ ...terminalEvent, playbackExpected });
       this.#finishSpeech(responseId, "request-aborted");
       this.#resumeCanonicalSpeechQueue();
       return;
     }
-    this.#emit(terminalEvent);
+    this.#emit({ ...terminalEvent, playbackExpected });
     if (this.#authorizedResponseIds.has(responseId)) {
       this.#finishSpeech(responseId, "invalid-response");
     }