FE-1654: Test Realtime rephrasing of completed Brunch responses

.changeset/realtime-voice-queue.md

@@ -0,0 +1,5 @@
+---
+"@hashintel/petrinaut": patch
+---
+
+Retain finalized Voice inputs in FIFO order while the assistant works, show a compact follow-up count with resume and discard controls, and expose whole-turn completion snapshots to Voice hosts. Stop withdraws queued inputs, while failed or aborted work holds them for explicit recovery.

apps/brunch-agent/src/agents/chat-agent/agent.ts

@@ -38,11 +38,9 @@ export function ChatAgent() {
     "responseMode" in context &&
     context.responseMode === "voice"
   ) {
-    useInstruction(`Voice response style for this delivery only:
-Respond conversationally and concisely. Put the necessary question or conclusion first.
-Avoid unnecessary preambles and repetition; preserve consequential qualifications.
-For a short clarification, prefer one or two spoken sentences, with any consequential qualification, rather than an unsolicited report or a repeated summary. Expand only when the question requires it.
-When a detailed report is needed, keep it complete in the visible canonical response; the application offers to read long responses on request.
+    useInstruction(`Voice response presentation for this delivery only:
+Write the complete canonical on-screen response normally, with the same content and detail you would provide for typed delivery. Do not shorten or reshape it for speech: Realtime rephrases the completed response later.
+Present any marked question in its exact wording so its authoritative text remains available for exact delivery.
 These are presentation instructions only. Retain all domain, evidence, workpiece, and tool obligations.`);
   }
 

apps/brunch-agent/test/voice-context.test.ts

@@ -63,9 +63,17 @@ test("ChatAgent scopes its fixed Voice instructions to the current delivery", as
       .then((receipt) => handle.read(receipt));
     expect(prompts).toHaveLength(5);
     expect(prompts[0]).not.toContain("Voice response style");
-    expect(prompts[1]).toContain("Voice response style");
-    expect(prompts[1]).toContain("consequential qualifications");
-    expect(prompts[1]).toContain("visible canonical response");
+    expect(prompts[1]).toContain("Voice response presentation");
+    expect(prompts[1]).toContain(
+      "complete canonical on-screen response normally",
+    );
+    expect(prompts[1]).toContain("marked question in its exact wording");
+    expect(prompts[1]).toContain(
+      "Realtime rephrases the completed response later",
+    );
+    expect(prompts[1]).not.toContain("Respond conversationally and concisely");
+    expect(prompts[1]).not.toContain("one or two spoken sentences");
+    expect(prompts[1]).not.toContain("offers to read long responses");
     expect(prompts[2]).toBe(prompts[1]);
     expect(prompts[3]).toBe(prompts[0]);
     expect(prompts[4]).toBe(prompts[0]);

apps/petrinaut-website/src/main/app/local-storage-demo/brunch-panel-transport.test.ts

@@ -162,6 +162,69 @@ test("matches client-tool admissions once and supports unsubscribe", () => {
   expect(unsubscribedListener).not.toHaveBeenCalled();
 });
 
+test("publishes every admission globally and records continuation membership immediately", () => {
+  const tracker = new BrunchPanelConversationTracker();
+  const listener = vi.fn();
+  tracker.subscribeToAdmissionEvents(listener);
+  const userEvent = {
+    admission: {
+      streamUrl: "http://brunch.test/user",
+      offset: "offset-user",
+      submissionId: "submission-user",
+      uid: "uid-user",
+    },
+    kind: "user" as const,
+    messageId: "user-1",
+  };
+  const continuationEvent = {
+    admission: {
+      streamUrl: "http://brunch.test/continuation",
+      offset: "offset-continuation",
+      submissionId: "submission-continuation",
+      uid: "uid-continuation",
+    },
+    kind: "client-tool-result" as const,
+    messageId: "assistant-1",
+  };
+
+  tracker.recordAdmission(userEvent);
+  tracker.recordAdmission(continuationEvent);
+  tracker.recordAdmission(continuationEvent);
+
+  expect(listener.mock.calls).toEqual([
+    [userEvent],
+    [continuationEvent],
+    [continuationEvent],
+  ]);
+  expect(tracker.submissionsForResponse("assistant-1")).toEqual([
+    "submission-continuation",
+  ]);
+});
+
+test("publishes repeated explicit submission settlements and supports unsubscribe", () => {
+  const tracker = new BrunchPanelConversationTracker();
+  const listener = vi.fn();
+  const unsubscribedListener = vi.fn();
+  tracker.subscribeToSubmissionSettled(listener);
+  const unsubscribe =
+    tracker.subscribeToSubmissionSettled(unsubscribedListener);
+  unsubscribe();
+  const event = {
+    type: "submission-settled" as const,
+    conversationId: "conversation-1",
+    submissionId: "submission-1",
+    outcome: "failed" as const,
+    position: { batch: 1, index: 0 },
+  };
+
+  tracker.recordSubmissionSettled(event);
+  tracker.recordSubmissionSettled(event);
+
+  expect(listener).toHaveBeenCalledTimes(2);
+  expect(listener).toHaveBeenNthCalledWith(1, event);
+  expect(unsubscribedListener).not.toHaveBeenCalled();
+});
+
 test("records every submission that wrote a resumed assistant message", () => {
   const tracker = new BrunchPanelConversationTracker();
   const responseStartedListener = vi.fn();

apps/petrinaut-website/src/main/app/local-storage-demo/brunch-panel-transport.ts

@@ -19,6 +19,7 @@ import type { AgentSendResult, FlueClient } from "@flue/sdk";
 import type {
   FlueChatResponseMessageCompletedEvent,
   FlueChatResponseMessageStartedEvent,
+  FlueChatSubmissionSettledEvent,
   FlueChatTransportOptions,
 } from "@hashintel/brunch-agent-transport-aisdk";
 import type { PetrinautAiChatTransport } from "@hashintel/petrinaut/ui";
@@ -41,6 +42,9 @@ export class BrunchPanelConversationTracker {
     readonly listener: (admission: BrunchPanelAdmission) => void;
     readonly target: BrunchPanelAdmissionTarget;
   }>();
+  readonly #admissionEventListeners = new Set<
+    (admission: BrunchPanelAdmission) => void
+  >();
   readonly #inFlightSubmissions = new Set<Promise<unknown>>();
   readonly #inputSubmissions = new Map<
     string,
@@ -56,6 +60,9 @@ export class BrunchPanelConversationTracker {
   readonly #responseMessageCompletedListeners = new Set<
     (event: FlueChatResponseMessageCompletedEvent) => void
   >();
+  readonly #submissionSettledListeners = new Set<
+    (event: FlueChatSubmissionSettledEvent) => void
+  >();
   readonly #stopRequestedListeners = new Set<() => void>();
 
   public recordAdmission(admission: BrunchPanelAdmission): void {
@@ -64,6 +71,14 @@ export class BrunchPanelConversationTracker {
         admission.messageId,
         admission.admission.submissionId,
       );
+    } else {
+      this.#recordResponseSubmission(
+        admission.messageId,
+        admission.admission.submissionId,
+      );
+    }
+    for (const listener of this.#admissionEventListeners) {
+      listener(admission);
     }
     for (const subscription of this.#admissionSubscriptions) {
       if (
@@ -83,17 +98,24 @@ export class BrunchPanelConversationTracker {
    * continuation.
    */
   public recordResponse(event: FlueChatResponseMessageStartedEvent): void {
-    const recorded = this.#responseSubmissions.get(event.messageId);
-    if (recorded === undefined) {
-      this.#responseSubmissions.set(event.messageId, [event.submissionId]);
-    } else if (!recorded.includes(event.submissionId)) {
-      recorded.push(event.submissionId);
-    }
+    this.#recordResponseSubmission(event.messageId, event.submissionId);
     for (const listener of this.#responseMessageStartedListeners) {
       listener(event);
     }
   }
 
+  #recordResponseSubmission(
+    messageId: string,
+    submissionId: AgentSendResult["submissionId"],
+  ): void {
+    const recorded = this.#responseSubmissions.get(messageId);
+    if (recorded === undefined) {
+      this.#responseSubmissions.set(messageId, [submissionId]);
+    } else if (!recorded.includes(submissionId)) {
+      recorded.push(submissionId);
+    }
+  }
+
   public recordResponseMessageCompleted(
     event: FlueChatResponseMessageCompletedEvent,
   ): void {
@@ -102,6 +124,12 @@ export class BrunchPanelConversationTracker {
     }
   }
 
+  public recordSubmissionSettled(event: FlueChatSubmissionSettledEvent): void {
+    for (const listener of this.#submissionSettledListeners) {
+      listener(event);
+    }
+  }
+
   public recordStopRequested(): void {
     for (const listener of this.#stopRequestedListeners) {
       listener();
@@ -162,6 +190,13 @@ export class BrunchPanelConversationTracker {
     return () => this.#admissionSubscriptions.delete(subscription);
   }
 
+  public subscribeToAdmissionEvents(
+    listener: (admission: BrunchPanelAdmission) => void,
+  ): () => void {
+    this.#admissionEventListeners.add(listener);
+    return () => this.#admissionEventListeners.delete(listener);
+  }
+
   public subscribeToAdmissionFailure(
     target: BrunchPanelAdmissionTarget,
     listener: (error: FlueChatAdmissionError) => void,
@@ -189,6 +224,13 @@ export class BrunchPanelConversationTracker {
     this.#stopRequestedListeners.add(listener);
     return () => this.#stopRequestedListeners.delete(listener);
   }
+
+  public subscribeToSubmissionSettled(
+    listener: (event: FlueChatSubmissionSettledEvent) => void,
+  ): () => void {
+    this.#submissionSettledListeners.add(listener);
+    return () => this.#submissionSettledListeners.delete(listener);
+  }
 }
 
 const formatFailure = (failure: SweepCompletionFailure): string => {
@@ -319,6 +361,9 @@ export const createBrunchPanelTransport = (
       readonly toolName: string;
     }) => unknown;
     readonly onAdmission?: (admission: AgentSendResult) => void;
+    readonly onSubmissionSettled?: (
+      event: FlueChatSubmissionSettledEvent,
+    ) => void;
   },
 ): PetrinautAiChatTransport => ({
   reconnectToStream: async () => null,
@@ -340,6 +385,10 @@ export const createBrunchPanelTransport = (
           onResponseMessage: (event) => tracker.recordResponse(event),
           onResponseMessageCompleted: (event) =>
             tracker.recordResponseMessageCompleted(event),
+          onSubmissionSettled: (event) => {
+            tracker.recordSubmissionSettled(event);
+            options?.onSubmissionSettled?.(event);
+          },
         });
         try {
           return decorateBrunchStream(

apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx

@@ -114,9 +114,11 @@ describe("local storage demo Brunch voice integration", () => {
       throw new Error("Expected the configured composer control to render.");
     }
     const failureListener = vi.fn();
+    const admissionEventListener = vi.fn();
     const responseCompletedListener = vi.fn();
     const responseStartedListener = vi.fn();
     const stopListener = vi.fn();
+    const submissionSettledListener = vi.fn();
     const target = { kind: "user" as const, messageId: "voice-turn-1" };
     const controlProps = control.props as {
       config: typeof config;
@@ -132,13 +134,19 @@ describe("local storage demo Brunch voice integration", () => {
         admissionTarget: typeof target,
         listener: (error: FlueChatAdmissionError) => void,
       ) => () => void;
+      subscribeToAdmissionEvents: (
+        listener: typeof admissionEventListener,
+      ) => () => void;
       subscribeToResponseMessageCompleted: (
         listener: typeof responseCompletedListener,
       ) => () => void;
       subscribeToResponseMessageStarted: (
         listener: typeof responseStartedListener,
       ) => () => void;
       subscribeToStopRequested: (listener: () => void) => () => void;
+      subscribeToSubmissionSettled: (
+        listener: typeof submissionSettledListener,
+      ) => () => void;
     };
     expect(control.type).toBe(VoiceInterviewControl);
     expect(controlProps.config).toBe(config);
@@ -162,6 +170,9 @@ describe("local storage demo Brunch voice integration", () => {
     expect(rerenderedControlProps.subscribeToAdmissionFailure).toBe(
       controlProps.subscribeToAdmissionFailure,
     );
+    expect(rerenderedControlProps.subscribeToAdmissionEvents).toBe(
+      controlProps.subscribeToAdmissionEvents,
+    );
     expect(rerenderedControlProps.subscribeToResponseMessageCompleted).toBe(
       controlProps.subscribeToResponseMessageCompleted,
     );
@@ -171,13 +182,20 @@ describe("local storage demo Brunch voice integration", () => {
     expect(rerenderedControlProps.subscribeToStopRequested).toBe(
       controlProps.subscribeToStopRequested,
     );
+    expect(rerenderedControlProps.subscribeToSubmissionSettled).toBe(
+      controlProps.subscribeToSubmissionSettled,
+    );
 
     const unsubscribe = controlProps.subscribeToAdmissionFailure(
       target,
       failureListener,
     );
     const unsubscribeFromStop =
       controlProps.subscribeToStopRequested(stopListener);
+    const unsubscribeFromAdmissionEvents =
+      controlProps.subscribeToAdmissionEvents(admissionEventListener);
+    const unsubscribeFromSubmissionSettled =
+      controlProps.subscribeToSubmissionSettled(submissionSettledListener);
     const unsubscribeFromResponseCompleted =
       controlProps.subscribeToResponseMessageCompleted(
         responseCompletedListener,
@@ -198,15 +216,38 @@ describe("local storage demo Brunch voice integration", () => {
       submissionId: "submission-1",
     });
     tracker.recordStopRequested();
+    const admissionEvent = {
+      admission: {
+        offset: "offset-1",
+        streamUrl: "http://brunch.test/stream",
+        submissionId: "submission-1",
+        uid: "uid-1",
+      },
+      kind: "user" as const,
+      messageId: "voice-turn-1",
+    };
+    tracker.recordAdmission(admissionEvent);
+    const settlementEvent = {
+      conversationId: "conversation-1",
+      outcome: "completed" as const,
+      position: { batch: 1, index: 0 },
+      submissionId: "submission-1",
+      type: "submission-settled" as const,
+    };
+    tracker.recordSubmissionSettled(settlementEvent);
 
     expect(failureListener).toHaveBeenCalledWith(admissionError);
     expect(responseStartedListener).toHaveBeenCalledOnce();
     expect(responseCompletedListener).toHaveBeenCalledOnce();
     expect(stopListener).toHaveBeenCalledOnce();
+    expect(admissionEventListener).toHaveBeenCalledWith(admissionEvent);
+    expect(submissionSettledListener).toHaveBeenCalledWith(settlementEvent);
     unsubscribe();
     unsubscribeFromResponseCompleted();
     unsubscribeFromResponseStarted();
     unsubscribeFromStop();
+    unsubscribeFromAdmissionEvents();
+    unsubscribeFromSubmissionSettled();
   });
 
   test("registers no brunch_ask tool in the production Brunch preview", async () => {

apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx

@@ -130,6 +130,10 @@ export const getBrunchVoiceMode = (
           );
   const subscribeToAdmissionFailure =
     tracker?.subscribeToAdmissionFailure.bind(tracker);
+  const subscribeToAdmissionEvents =
+    tracker?.subscribeToAdmissionEvents.bind(tracker);
+  const subscribeToSubmissionSettled =
+    tracker?.subscribeToSubmissionSettled.bind(tracker);
 
   return (context: PetrinautAiVoiceModeContext) => (
     <VoiceInterviewControl
@@ -143,6 +147,8 @@ export const getBrunchVoiceMode = (
       subscribeToStopRequested={subscribeToStopRequested}
       subscribeToAdmission={subscribeToAdmission}
       subscribeToAdmissionFailure={subscribeToAdmissionFailure}
+      subscribeToAdmissionEvents={subscribeToAdmissionEvents}
+      subscribeToSubmissionSettled={subscribeToSubmissionSettled}
     />
   );
 };