diff --git a/.changeset/petrinaut-voice-interruption.md b/.changeset/petrinaut-voice-interruption.md
new file mode 100644
index 00000000000..65e2f4b7561
--- /dev/null
+++ b/.changeset/petrinaut-voice-interruption.md
@@ -0,0 +1,5 @@
+---
+"@hashintel/petrinaut": patch
+---
+
+Allow Voice users to interrupt assistant playback by speaking, with a browser-saved preference and optional manual handoff.
diff --git a/apps/petrinaut-website/README.md b/apps/petrinaut-website/README.md
index 9e86d13ca13..19e96a685f4 100644
--- a/apps/petrinaut-website/README.md
+++ b/apps/petrinaut-website/README.md
@@ -145,7 +145,8 @@ The text composer remains available. Sending typed text ends Voice mode first,
then submits the draft exactly once through the same conversation; a failed
handoff restores the draft. Closing the assistant pauses capture and speech
before hiding it. Reopening preserves the mounted session in **Paused** state.
-The dock exposes **Your turn** while canonical audio owns the turn. That action
+With **Interruption by speaking** disabled, the dock exposes **Your turn** while
+canonical audio owns the turn. That action
clears pending input and output, waits for the provider's matching
acknowledgements and response terminal event, and only then opens the
microphone for fresh capture. Its playback menu offers **Repeat question** and
@@ -160,7 +161,8 @@ action stays disabled rather than guessing from the final segment.
The browser sends its SDP offer to this app; the server initializes a trusted
`gpt-realtime-2` audio-input/audio-output session through OpenAI's unified
Realtime call endpoint. The provider key, model, instructions, language, and
-vocabulary policy stay server-side. Realtime exposes no tools, uses
+session configuration stay server-side; the transcription vocabulary is shared
+with the browser's local admission filter. Realtime exposes no tools, uses
`tool_choice: "none"`, and configures semantic VAD to detect an input boundary
without creating a model response.
@@ -174,21 +176,35 @@ Petrinaut's shared composer path. Connection epoch, item id, and content index
form its stable identity. Duplicate, empty, failed, unavailable, and over-limit
transcripts never submit; recoverable failures leave a not-heard or too-long
notice in the dock. Provisional transcription remains display-only.
+Only interruption-originated completions receive local prompt-regurgitation
+and self-echo checks before admission or pending-answer retention. Comparison
+uses NFKC, lowercase, punctuation removal, and whitespace collapse. Exact
+normalized active-playback echoes are rejected at any length; fuzzy comparison
+requires at least 80% ordered bigram overlap and minimum lengths of eight tokens
+for the vocabulary prompt or six for active canonical playback. The playback
+reference is captured when interruption starts, excludes queued speech and
+history, and is released on completion or lifecycle cleanup. Rejections produce
+only content-free diagnostics; they create no answer, error, or pending-answer
+notice. Short novel answers remain valid and the admitted payload keeps its
+original casing and punctuation.
The bridge waits for the correlated Brunch turn before returning canonical
speech segments to Realtime. It instructs Realtime to speak only those
segments. Generated audio is not a verbatim recording: canonical Brunch text
-remains visible and authoritative. Voice is half-duplex: the physical
-microphone is closed while the interviewer speaks, while Brunch is working, and
-through cancellation. Audio captured before a **Your turn** handoff is
-discarded and cannot become a later answer.
+remains visible and authoritative. **Interruption by speaking** is enabled by
+default: speech detection immediately cancels generation and clears output audio,
+never the input buffer. The completed answer waits if Brunch is still busy.
+False speech detection may still stop playback even if the transcript is later
+discarded. Disable this browser-saved preference for half-duplex capture: the
+microphone closes during assistant output, and audio captured before a completed
+**Your turn** handoff cannot become a later answer.
The local Brunch preview reaches the mounted route through its same-origin, protocol-preserving proxy; this does not establish remote authentication or public ingress. Denying microphone permission leaves the text composer available and submits nothing to Brunch. When Voice mode cannot continue, the inline recovery state distinguishes microphone, connection, and other Voice failures, explains the next action, and offers **Reconnect** where appropriate. Sanitized error codes and diagnostic references remain collapsed under **Technical details**.
Realtime connection, transcription, and canonical speech timings use random
request IDs, and the existing Brunch transport provides its own request
correlation. Browser and server diagnostics report only operation, stage,
-outcome, duration, request ID, and—where applicable—status or a sanitized error
+outcome, duration, request ID, and—where applicable—status, rejection reason, or a sanitized error
code. Voice responses also expose privacy-safe `Server-Timing` metrics. These
diagnostics never record audio, SDP, transcript or prompt contents, canonical
speech text, credentials, or provider response bodies. Production Voice remains
diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/voice-history-continuity.integration.test.tsx b/apps/petrinaut-website/src/main/app/local-storage-demo/voice-history-continuity.integration.test.tsx
new file mode 100644
index 00000000000..82742b15a79
--- /dev/null
+++ b/apps/petrinaut-website/src/main/app/local-storage-demo/voice-history-continuity.integration.test.tsx
@@ -0,0 +1,407 @@
+/** @vitest-environment jsdom */
+import {
+ act,
+ cleanup,
+ fireEvent,
+ render,
+ screen,
+ waitFor,
+ within,
+} from "@testing-library/react";
+import { useEffect, useState } from "react";
+import { afterEach, expect, test, vi } from "vitest";
+
+import { CLIENT_TOOL_RESULT_SIGNAL } from "@hashintel/brunch-agent-transport-aisdk";
+import { createJsonDocHandle } from "@hashintel/petrinaut-core";
+import {
+ definePetrinautAiInteractiveTool,
+ Petrinaut,
+} from "@hashintel/petrinaut/ui";
+
+import { useFlueChatHistory } from "./use-flue-chat-history";
+
+import type {
+ AgentConversationObservation,
+ AgentConversationObservationSnapshot,
+ FlueClient,
+} from "@flue/sdk";
+import type {
+ PetrinautAiChatTransport,
+ PetrinautAiVoiceModeContext,
+} from "@hashintel/petrinaut/ui";
+
+vi.hoisted(() => {
+ window.matchMedia = (media) => ({
+ media,
+ matches: false,
+ onchange: null,
+ addListener() {},
+ removeListener() {},
+ addEventListener() {},
+ removeEventListener() {},
+ dispatchEvent: () => true,
+ });
+});
+
+const conversationId = "voice-continuity";
+const voiceAnswerToolName = "answerQuestion";
+const voiceClientToolNames = new Set([voiceAnswerToolName]);
+const voiceAnswerTool = definePetrinautAiInteractiveTool({
+ component: ({ submittedOutput, toolCallId }) => (
+ {`${toolCallId}: ${submittedOutput?.answer}`}
+ ),
+ inputSchema: {
+ parse: (raw: unknown) => raw as { question: string },
+ },
+ outputSchema: {
+ parse: (raw: unknown) => raw as { answer: string },
+ },
+ toolName: voiceAnswerToolName,
+});
+const emptyDefinition = {
+ places: [],
+ transitions: [],
+ types: [],
+ parameters: [],
+ differentialEquations: [],
+};
+const inertWorker = () => ({
+ addEventListener() {},
+ postMessage() {},
+ removeEventListener() {},
+ terminate() {},
+});
+
+const createObservationHarness = (
+ initialSnapshot: AgentConversationObservationSnapshot,
+) => {
+ let snapshot = initialSnapshot;
+ const activeListeners = new Set void>>();
+ const observe = vi.fn((): AgentConversationObservation => {
+ const listeners = new Set<() => void>();
+ activeListeners.add(listeners);
+ return {
+ close: () => activeListeners.delete(listeners),
+ getSnapshot: () => snapshot,
+ refresh: vi.fn(),
+ subscribe: (listener) => {
+ listeners.add(listener);
+ return () => listeners.delete(listener);
+ },
+ };
+ });
+ return {
+ clientPromise: Promise.resolve({
+ observe,
+ } as Pick as FlueClient),
+ observe,
+ publish(nextSnapshot: AgentConversationObservationSnapshot) {
+ snapshot = nextSnapshot;
+ for (const listeners of activeListeners) {
+ for (const listener of listeners) listener();
+ }
+ },
+ };
+};
+
+const VoiceMode = ({
+ context,
+ endVoice,
+}: {
+ context: PetrinautAiVoiceModeContext;
+ endVoice: () => Promise;
+}) => {
+ const {
+ inputMode,
+ registerVoiceModeControls,
+ reportVoiceSessionState,
+ setVoiceActive,
+ } = context;
+ useEffect(
+ () =>
+ registerVoiceModeControls({
+ end: endVoice,
+ pause: vi.fn(),
+ reconnect: vi.fn(),
+ resume: vi.fn(),
+ setMicrophoneMuted: vi.fn(),
+ }),
+ [endVoice, registerVoiceModeControls],
+ );
+ useEffect(() => {
+ if (inputMode !== "voice") return;
+ setVoiceActive(true);
+ reportVoiceSessionState({
+ errorMessage: null,
+ microphoneLevel: 0,
+ microphoneMuted: false,
+ phase: "listening",
+ });
+ }, [inputMode, reportVoiceSessionState, setVoiceActive]);
+ return null;
+};
+
+const ContinuityPanel = ({
+ clientPromise,
+ endVoice,
+ handleId,
+ requestStop,
+ transport,
+}: {
+ clientPromise: Promise;
+ endVoice: () => Promise;
+ handleId: string;
+ requestStop: () => Promise<"already-settled" | "stop-requested">;
+ transport: PetrinautAiChatTransport;
+}) => {
+ const [handle] = useState(() =>
+ createJsonDocHandle({
+ id: handleId,
+ initial: emptyDefinition,
+ }),
+ );
+ const history = useFlueChatHistory(
+ clientPromise,
+ conversationId,
+ voiceClientToolNames,
+ );
+ if (!history.ready || history.messages === undefined) return null;
+ return (
+ (
+
+ ),
+ transport,
+ }}
+ handle={handle}
+ lspWorkerFactory={inertWorker}
+ />
+ );
+};
+
+afterEach(() => {
+ cleanup();
+ vi.restoreAllMocks();
+ vi.unstubAllGlobals();
+});
+
+test("projects typed, Voice-tool, and stopped fixture history after remount", async () => {
+ const storageEntries = new Map();
+ vi.stubGlobal("localStorage", {
+ get length() {
+ return storageEntries.size;
+ },
+ clear: () => storageEntries.clear(),
+ getItem: (key: string) => storageEntries.get(key) ?? null,
+ key: (index: number) => [...storageEntries.keys()].at(index) ?? null,
+ removeItem: (key: string) => storageEntries.delete(key),
+ setItem: (key: string, value: string) => storageEntries.set(key, value),
+ } satisfies Storage);
+ vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue(null);
+ vi.stubGlobal(
+ "ResizeObserver",
+ class {
+ observe() {}
+ unobserve() {}
+ disconnect() {}
+ },
+ );
+ const initialSnapshot: AgentConversationObservationSnapshot = {
+ conversation: {
+ conversationId,
+ messages: [
+ {
+ display: "visible",
+ id: "typed-user",
+ parts: [{ state: "done", text: "Typed planning note", type: "text" }],
+ purpose: "user",
+ role: "user",
+ },
+ {
+ display: "visible",
+ id: "voice-tool-response",
+ parts: [
+ {
+ input: { question: "Who approves this?" },
+ output: { awaiting: "client" },
+ state: "output-available",
+ toolCallId: "voice-tool-1",
+ toolName: voiceAnswerToolName,
+ type: "dynamic-tool",
+ },
+ ],
+ purpose: "assistant",
+ role: "assistant",
+ submissionId: "voice-submission",
+ },
+ {
+ display: "hidden",
+ id: "voice-tool-result",
+ parts: [
+ {
+ state: "done",
+ text: JSON.stringify([
+ {
+ output: { answer: "The supervisor" },
+ source: "voice",
+ toolCallId: "voice-tool-1",
+ toolName: voiceAnswerToolName,
+ },
+ ]),
+ type: "text",
+ },
+ ],
+ purpose: "dispatch",
+ role: "system",
+ signal: { tagName: CLIENT_TOOL_RESULT_SIGNAL },
+ },
+ ],
+ settlements: [{ outcome: "completed", submissionId: "voice-submission" }],
+ },
+ error: undefined,
+ offset: "before-stop",
+ phase: "live",
+ };
+ const stoppedSnapshot: AgentConversationObservationSnapshot = {
+ conversation: {
+ conversationId,
+ messages: [
+ ...initialSnapshot.conversation!.messages,
+ {
+ display: "visible",
+ id: "stop-user",
+ parts: [
+ {
+ state: "done",
+ text: "Start a stoppable response",
+ type: "text",
+ },
+ ],
+ purpose: "user",
+ role: "user",
+ },
+ {
+ display: "visible",
+ id: "stopped-response",
+ parts: [
+ {
+ state: "done",
+ text: "Durably interrupted response",
+ type: "text",
+ },
+ ],
+ purpose: "assistant",
+ role: "assistant",
+ submissionId: "stop-submission",
+ },
+ ],
+ settlements: [
+ ...initialSnapshot.conversation!.settlements,
+ { outcome: "aborted", submissionId: "stop-submission" },
+ ],
+ },
+ error: undefined,
+ offset: "after-stop",
+ phase: "live",
+ };
+ const observation = createObservationHarness(initialSnapshot);
+ const endVoice = vi.fn(async () => undefined);
+ const transport: PetrinautAiChatTransport = {
+ reconnectToStream: () => Promise.resolve(null),
+ sendMessages: vi.fn(() =>
+ Promise.resolve(
+ new ReadableStream({
+ start(controller) {
+ controller.enqueue({ type: "start-step" });
+ controller.enqueue({ id: "partial", type: "text-start" });
+ controller.enqueue({
+ delta: "Durably interrupted response",
+ id: "partial",
+ type: "text-delta",
+ });
+ },
+ cancel() {},
+ }),
+ ),
+ ),
+ };
+ const requestStop = vi.fn(async () => {
+ observation.publish(stoppedSnapshot);
+ return "stop-requested" as const;
+ });
+
+ const firstMount = render(
+ ,
+ );
+ const showFirstPanel = await screen.findByRole("button", {
+ name: "Show AI assistant",
+ });
+ await act(async () => fireEvent.click(showFirstPanel));
+ const composer = await screen.findByRole("textbox", {
+ name: "Message AI assistant",
+ });
+ fireEvent.change(composer, {
+ target: { value: "Start a stoppable response" },
+ });
+ fireEvent.click(screen.getByRole("button", { name: "Send message" }));
+ await screen.findByText("Durably interrupted response");
+ await act(async () =>
+ fireEvent.click(screen.getByRole("button", { name: "Stop AI response" })),
+ );
+ await waitFor(() => expect(requestStop).toHaveBeenCalledOnce());
+ firstMount.unmount();
+
+ const secondMount = render(
+ ,
+ );
+ const showSecondPanel = await screen.findByRole("button", {
+ name: "Show AI assistant",
+ });
+ await act(async () => fireEvent.click(showSecondPanel));
+ await screen.findByText("Typed planning note");
+ expect(observation.observe).toHaveBeenCalledTimes(2);
+ expect(
+ within(
+ screen.getByText("Typed planning note").closest("[data-role]")!,
+ ).queryByTestId("voice-input-provenance"),
+ ).toBeNull();
+ expect(
+ within(
+ secondMount.container.querySelector(
+ '[data-tool-call-id="voice-tool-1"]',
+ )!,
+ ).getByTestId("voice-input-provenance"),
+ ).not.toBeNull();
+ expect(screen.getByText("Durably interrupted response")).not.toBeNull();
+ expect(screen.getByText("Response stopped")).not.toBeNull();
+
+ await act(async () =>
+ fireEvent.click(screen.getByRole("button", { name: "Start voice mode" })),
+ );
+ const voiceDock = await screen.findByRole("region", {
+ name: "Voice session",
+ });
+ fireEvent.click(
+ within(voiceDock).getByRole("button", { name: "End voice mode" }),
+ );
+
+ await waitFor(() => expect(endVoice).toHaveBeenCalledOnce());
+ expect(requestStop).toHaveBeenCalledOnce();
+});
diff --git a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts
index 12d189375be..339175de4b5 100644
--- a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts
+++ b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts
@@ -4,6 +4,8 @@ import {
OpenAIRealtimeSession,
type OpenAIRealtimeSessionEvent,
} from "./openai-realtime-session";
+import { RealtimeBrunchBridge } from "./realtime-brunch-bridge";
+import { VoiceTurnController } from "./voice-turn-controller";
import type { CanonicalSpeechSegment } from "./canonical-speech";
@@ -175,6 +177,406 @@ describe("OpenAIRealtimeSession", () => {
vi.useRealTimers();
});
+ test.each([
+ [
+ "SDCPN, stochastic Petri net, place, transition, arc, token, marking, guard, rate, distribution, parameter, subnet, scenario, and metric.",
+ "prompt-regurgitation",
+ ],
+ [
+ "The supervisor reviews the request before the manager approves it.",
+ "self-echo",
+ ],
+ ["stop", null],
+ ["no", null],
+ ["wait", null],
+ ["The reviewer sends the signed form to the accounts department.", null],
+ ["The applicant receives an email after the review is complete.", null],
+ [
+ "The auditor reviews all requests before the manager receives them.",
+ null,
+ ],
+ ])(
+ "cancels immediately and validates completed interruption through the real stack: %s",
+ async (text, rejectionReason) => {
+ const harness = createHarness();
+ const submitInterviewAnswer = vi.fn<
+ ConstructorParameters<
+ typeof RealtimeBrunchBridge
+ >[0]["submitInterviewAnswer"]
+ >(async (input) => ({ kind: "message", messageId: input.id }));
+ const reportDiagnostic = vi.fn();
+ const bridge = new RealtimeBrunchBridge({
+ session: harness.session,
+ submitInterviewAnswer,
+ reportDiagnostic,
+ });
+ const controller = new VoiceTurnController({
+ bridge,
+ session: harness.session,
+ submitText: vi.fn(async () => undefined),
+ });
+ controller.setInterruptionBySpeaking(true);
+ const history = canonicalSegment(
+ "history",
+ "The reviewer sends the signed form to the accounts department.",
+ );
+ controller.updateChat({
+ canAcceptInterviewAnswer: true,
+ canonicalSegments: [history],
+ status: "ready",
+ });
+ await controller.start();
+ const channel = harness.channels.at(-1)!;
+ controller.updateChat({
+ canAcceptInterviewAnswer: true,
+ canonicalSegments: [
+ history,
+ canonicalSegment(
+ "playing",
+ "The supervisor reviews the request before the manager approves it.",
+ ),
+ ],
+ status: "ready",
+ });
+ authorizeLatestSpeechResponse(channel, "playing-response");
+ channel.receive({
+ type: "output_audio_buffer.started",
+ response_id: "playing-response",
+ });
+ harness.session.speakCanonical([
+ canonicalSegment(
+ "queued",
+ "The applicant receives an email after the review is complete.",
+ ),
+ ]);
+ channel.send.mockClear();
+ channel.receive({
+ type: "input_audio_buffer.speech_started",
+ item_id: "interruption",
+ audio_start_ms: 100,
+ });
+ expect(sentEvents(channel).map(({ type }) => type)).toEqual([
+ "response.cancel",
+ "output_audio_buffer.clear",
+ ]);
+ expect(submitInterviewAnswer).not.toHaveBeenCalled();
+ expect(reportDiagnostic).not.toHaveBeenCalled();
+ channel.receive({
+ type: "conversation.item.input_audio_transcription.delta",
+ item_id: "interruption",
+ content_index: 0,
+ delta: text,
+ });
+ expect(submitInterviewAnswer).not.toHaveBeenCalled();
+ channel.receive({
+ type: "response.done",
+ response: { id: "playing-response", status: "cancelled", output: [] },
+ });
+ channel.receive({
+ type: "output_audio_buffer.cleared",
+ response_id: "playing-response",
+ });
+ const completed = {
+ type: "conversation.item.input_audio_transcription.completed",
+ item_id: "interruption",
+ content_index: 0,
+ transcript: text,
+ };
+ channel.receive(completed);
+ channel.receive(completed);
+ if (rejectionReason) {
+ expect(submitInterviewAnswer).not.toHaveBeenCalled();
+ expect(controller.getSnapshot()).toMatchObject({
+ connection: "connected",
+ errorCode: null,
+ inputNotice: "none",
+ lastAnswerDelivery: "none",
+ lastCommittedText: "",
+ partialText: "",
+ });
+ expect(reportDiagnostic).toHaveBeenCalledOnce();
+ expect(reportDiagnostic).toHaveBeenCalledWith({
+ durationMs: expect.any(Number) as unknown,
+ operation: "transcription",
+ outcome: "rejected",
+ rejectionReason,
+ requestId: expect.any(String) as unknown,
+ stage: "browser",
+ });
+ expect(JSON.stringify(reportDiagnostic.mock.calls)).not.toContain(text);
+ } else {
+ expect(submitInterviewAnswer).toHaveBeenCalledOnce();
+ expect(submitInterviewAnswer).toHaveBeenCalledWith(
+ expect.objectContaining({
+ text,
+ id: "voice-realtime:1:interruption:0",
+ }),
+ );
+ }
+ expect(
+ sentEvents(channel).some(
+ ({ type }) => type === "input_audio_buffer.clear",
+ ),
+ ).toBe(false);
+ await controller.end();
+ },
+ );
+
+ test.each(["playing", "generated", "creating"] as const)(
+ "preserves an interrupting answer through the real Voice stack while %s",
+ async (phase) => {
+ const harness = createHarness();
+ const submitInterviewAnswer = vi.fn<
+ ConstructorParameters<
+ typeof RealtimeBrunchBridge
+ >[0]["submitInterviewAnswer"]
+ >(async (input) => ({ kind: "message", messageId: input.id }));
+ const bridge = new RealtimeBrunchBridge({
+ session: harness.session,
+ submitInterviewAnswer,
+ });
+ const controller = new VoiceTurnController({
+ bridge,
+ session: harness.session,
+ submitText: vi.fn(async () => undefined),
+ });
+ controller.setInterruptionBySpeaking(true);
+ await controller.start();
+ const channel = harness.channels.at(-1)!;
+ const question = canonicalSegment("question", "Who approves this?");
+ controller.updateChat({
+ canAcceptInterviewAnswer: true,
+ canonicalSegments: [question],
+ questionSegment: question,
+ status: "ready",
+ });
+ if (phase !== "creating") {
+ authorizeLatestSpeechResponse(channel, "question-response");
+ }
+ if (phase === "playing") {
+ channel.receive({
+ type: "output_audio_buffer.started",
+ response_id: "question-response",
+ });
+ }
+ if (phase === "generated") {
+ channel.receive({
+ type: "response.done",
+ response: {
+ id: "question-response",
+ status: "completed",
+ output: [
+ {
+ content: [
+ {
+ transcript: "Who approves this?",
+ type: "output_audio",
+ },
+ ],
+ role: "assistant",
+ type: "message",
+ },
+ ],
+ },
+ });
+ }
+ expect(harness.localTracks.at(-1)?.enabled).toBe(true);
+ const pendingResponse = sentEvents(channel).findLast(
+ ({ type }) => type === "response.create",
+ )?.response as Record;
+ harness.session.speakCanonical([
+ canonicalSegment("queued", "This must not play."),
+ ]);
+ channel.send.mockClear();
+ channel.receive({
+ type: "input_audio_buffer.speech_started",
+ item_id: "interrupting-answer",
+ audio_start_ms: 100,
+ });
+ if (phase !== "creating") {
+ expect(sentEvents(channel).map(({ type }) => type)).toEqual(
+ phase === "playing"
+ ? ["response.cancel", "output_audio_buffer.clear"]
+ : ["output_audio_buffer.clear"],
+ );
+ }
+ expect(harness.localTracks.at(-1)?.enabled).toBe(true);
+ channel.receive({
+ type: "conversation.item.input_audio_transcription.delta",
+ item_id: "interrupting-answer",
+ content_index: 0,
+ delta: "The supervisor",
+ });
+ expect(controller.getSnapshot().partialText).toBe("The supervisor");
+ if (phase === "creating") {
+ channel.receive({
+ type: "response.created",
+ response: {
+ id: "question-response",
+ metadata: pendingResponse.metadata,
+ },
+ });
+ expect(sentEvents(channel).map(({ type }) => type)).toEqual([
+ "response.cancel",
+ "output_audio_buffer.clear",
+ ]);
+ }
+ const terminal = {
+ type: "response.done",
+ response: { id: "question-response", status: "cancelled", output: [] },
+ };
+ const cleared = {
+ type: "output_audio_buffer.cleared",
+ response_id: "question-response",
+ };
+ if (phase === "generated") {
+ channel.receive(cleared);
+ expect(harness.events).toContainEqual({
+ connectionEpoch: 1,
+ responseId: "question-response",
+ speechRequestId: (pendingResponse.metadata as Record)
+ .petrinaut_request_id,
+ type: "output-interrupted",
+ });
+ } else {
+ channel.receive(phase === "playing" ? terminal : cleared);
+ channel.receive(phase === "playing" ? cleared : terminal);
+ }
+ channel.receive({
+ type: "output_audio_buffer.started",
+ response_id: "question-response",
+ });
+ expect(controller.getSnapshot().connection).toBe("connected");
+ expect(harness.localTracks.at(-1)?.enabled).toBe(true);
+ expect(
+ sentEvents(channel).some(({ type }) => type === "response.create"),
+ ).toBe(false);
+ channel.receive({
+ type: "conversation.item.input_audio_transcription.completed",
+ item_id: "interrupting-answer",
+ content_index: 0,
+ transcript: "The supervisor approves it.",
+ });
+ await vi.waitFor(() =>
+ expect(submitInterviewAnswer).toHaveBeenCalledOnce(),
+ );
+ expect(submitInterviewAnswer).toHaveBeenCalledWith(
+ expect.objectContaining({
+ id: "voice-realtime:1:interrupting-answer:0",
+ text: "The supervisor approves it.",
+ }),
+ );
+ expect(
+ sentEvents(channel).some(
+ ({ type }) => type === "input_audio_buffer.clear",
+ ),
+ ).toBe(false);
+ expect(controller.getSnapshot().lastCommittedText).toBe(
+ "The supervisor approves it.",
+ );
+ await controller.end();
+ },
+ );
+
+ test("reopens capture for a streamed reply and submits the retained interruption after settlement", async () => {
+ const harness = createHarness();
+ const submitInterviewAnswer = vi.fn<
+ ConstructorParameters<
+ typeof RealtimeBrunchBridge
+ >[0]["submitInterviewAnswer"]
+ >(async (input) => {
+ input.onAdmission("submission-1");
+ return {
+ kind: "message",
+ messageId: input.id,
+ submissionId: "submission-1",
+ };
+ });
+ const bridge = new RealtimeBrunchBridge({
+ session: harness.session,
+ submitInterviewAnswer,
+ });
+ const controller = new VoiceTurnController({
+ bridge,
+ session: harness.session,
+ submitText: vi.fn(async () => undefined),
+ });
+ controller.setInterruptionBySpeaking(true);
+ await controller.start();
+ controller.updateChat({
+ canAcceptInterviewAnswer: true,
+ canonicalSegments: [],
+ status: "ready",
+ });
+ const channel = harness.channels.at(-1)!;
+ channel.receive({
+ type: "input_audio_buffer.speech_started",
+ item_id: "first-answer",
+ audio_start_ms: 0,
+ });
+ channel.receive({
+ type: "conversation.item.input_audio_transcription.completed",
+ item_id: "first-answer",
+ content_index: 0,
+ transcript: "We need an approval.",
+ });
+ await vi.waitFor(() =>
+ expect(controller.getSnapshot().lastAnswerDelivery).toBe("delivered"),
+ );
+ const reply = {
+ ...canonicalSegment("reply", "Who approves this?"),
+ submissionIds: ["submission-1"],
+ };
+ bridge.notifyResponseMessageCompleted({
+ messageId: reply.messageId,
+ submissionId: "submission-1",
+ position: { batch: 1, index: 0 },
+ });
+ controller.updateChat({
+ canAcceptInterviewAnswer: false,
+ canonicalSegments: [reply],
+ status: "streaming",
+ });
+ authorizeLatestSpeechResponse(channel, "streamed-reply");
+ channel.receive({
+ type: "output_audio_buffer.started",
+ response_id: "streamed-reply",
+ });
+ expect(harness.localTracks.at(-1)?.enabled).toBe(true);
+ controller.setMicrophoneMuted(true);
+ expect(harness.localTracks.at(-1)?.enabled).toBe(false);
+ controller.setMicrophoneMuted(false);
+ expect(harness.localTracks.at(-1)?.enabled).toBe(true);
+ channel.receive({
+ type: "input_audio_buffer.speech_started",
+ item_id: "second-answer",
+ audio_start_ms: 100,
+ });
+ channel.receive({
+ type: "conversation.item.input_audio_transcription.completed",
+ item_id: "second-answer",
+ content_index: 0,
+ transcript: "The supervisor approves it.",
+ });
+ expect(submitInterviewAnswer).toHaveBeenCalledOnce();
+ expect(controller.getSnapshot().inputNotice).toBe("answer-pending");
+ controller.updateChat({
+ canAcceptInterviewAnswer: true,
+ canonicalSegments: [reply],
+ status: "ready",
+ });
+ await vi.waitFor(() =>
+ expect(submitInterviewAnswer).toHaveBeenCalledTimes(2),
+ );
+ expect(submitInterviewAnswer).toHaveBeenLastCalledWith(
+ expect.objectContaining({
+ id: "voice-realtime:1:second-answer:0",
+ text: "The supervisor approves it.",
+ }),
+ );
+ await controller.end();
+ });
+
test("negotiates duplex WebRTC, attaches remote audio, and cleans all media", async () => {
const harness = createHarness();
@@ -264,7 +666,7 @@ describe("OpenAIRealtimeSession", () => {
);
});
- test("rejects an accepted input item whose transcript completes after output starts", async () => {
+ test("preserves a stopped input item whose transcript completes after output starts", async () => {
const harness = createHarness();
await harness.session.connect();
harness.session.setMicrophoneEnabled(true);
@@ -290,6 +692,11 @@ describe("OpenAIRealtimeSession", () => {
text: "This started before output",
type: "partial",
});
+ channel.receive({
+ audio_end_ms: 160,
+ item_id: "item-before-output",
+ type: "input_audio_buffer.speech_stopped",
+ });
harness.session.speakCanonical([
canonicalSegment("ask-1", "What happens next?"),
@@ -306,17 +713,19 @@ describe("OpenAIRealtimeSession", () => {
type: "conversation.item.input_audio_transcription.completed",
});
- expect(
- harness.events.some(
- (event) =>
- event.type === "completed" &&
- event.key.itemId === "item-before-output",
- ),
- ).toBe(false);
+ expect(harness.events).toContainEqual({
+ key: {
+ connectionEpoch: 1,
+ contentIndex: 0,
+ itemId: "item-before-output",
+ },
+ text: "This completed too late.",
+ type: "completed",
+ });
expect(harness.localTracks[0]!.enabled).toBe(false);
});
- test("invalidates accepted input before requesting canonical speech output", async () => {
+ test("waits for accepted input before requesting canonical speech output", async () => {
const harness = createHarness();
await harness.session.connect();
harness.session.setMicrophoneEnabled(true);
@@ -344,11 +753,11 @@ describe("OpenAIRealtimeSession", () => {
harness.session.speakCanonical([
canonicalSegment("ask-request", "What happens next?"),
]);
- expect(harness.events).toContainEqual(
+ expect(harness.events).not.toContainEqual(
expect.objectContaining({ type: "canonical-speech-requested" }),
);
- expect(microphoneEnabledWhenResponseRequested).toBe(false);
- expect(harness.localTracks[0]!.enabled).toBe(false);
+ expect(microphoneEnabledWhenResponseRequested).toBeUndefined();
+ expect(harness.localTracks[0]!.enabled).toBe(true);
channel.receive({
content_index: 0,
@@ -356,13 +765,20 @@ describe("OpenAIRealtimeSession", () => {
transcript: "This completed before output started.",
type: "conversation.item.input_audio_transcription.completed",
});
- expect(
- harness.events.some(
- (event) =>
- event.type === "completed" &&
- event.key.itemId === "item-before-request",
- ),
- ).toBe(false);
+ expect(harness.events).toContainEqual(
+ expect.objectContaining({ type: "canonical-speech-requested" }),
+ );
+ expect(microphoneEnabledWhenResponseRequested).toBe(false);
+ expect(harness.localTracks[0]!.enabled).toBe(false);
+ expect(harness.events).toContainEqual({
+ key: {
+ connectionEpoch: 1,
+ contentIndex: 0,
+ itemId: "item-before-request",
+ },
+ text: "This completed before output started.",
+ type: "completed",
+ });
const handoff = harness.session.cancelOutput();
let handoffSettled = false;
@@ -412,6 +828,15 @@ describe("OpenAIRealtimeSession", () => {
expect(
harness.events.filter((event) => event.type === "completed"),
).toEqual([
+ {
+ key: {
+ connectionEpoch: 1,
+ contentIndex: 0,
+ itemId: "item-before-request",
+ },
+ text: "This completed before output started.",
+ type: "completed",
+ },
{
key: {
connectionEpoch: 1,
@@ -480,6 +905,11 @@ describe("OpenAIRealtimeSession", () => {
item_id: "item-before-handoff",
type: "input_audio_buffer.speech_started",
});
+ channel.receive({
+ audio_end_ms: 80,
+ item_id: "item-before-handoff",
+ type: "input_audio_buffer.speech_stopped",
+ });
harness.session.speakCanonical([
canonicalSegment("ask-handoff", "What happens next?"),
]);
@@ -853,6 +1283,79 @@ describe("OpenAIRealtimeSession", () => {
expect(harness.localTracks[0]!.enabled).toBe(true);
});
+ test("releases every buffered response after one spoken-interruption clear", async () => {
+ const harness = createHarness();
+ await harness.session.connect();
+ harness.session.setInterruptionBySpeaking(true);
+ harness.session.setMicrophoneEnabled(true);
+ const channel = harness.channels[0]!;
+ harness.session.speakCanonical([
+ canonicalSegment("first", "First canonical segment."),
+ ]);
+ authorizeLatestSpeechResponse(channel, "response-first");
+ channel.receive({
+ response_id: "response-first",
+ type: "output_audio_buffer.started",
+ });
+ harness.session.speakCanonical([
+ canonicalSegment("second", "Second canonical segment."),
+ ]);
+ channel.receive({
+ response: {
+ id: "response-first",
+ output: [
+ {
+ content: [
+ { transcript: "First canonical segment.", type: "output_audio" },
+ ],
+ role: "assistant",
+ type: "message",
+ },
+ ],
+ status: "completed",
+ },
+ type: "response.done",
+ });
+ authorizeLatestSpeechResponse(channel, "response-second");
+ channel.receive({
+ response: {
+ id: "response-second",
+ output: [
+ {
+ content: [
+ { transcript: "Second canonical segment.", type: "output_audio" },
+ ],
+ role: "assistant",
+ type: "message",
+ },
+ ],
+ status: "completed",
+ },
+ type: "response.done",
+ });
+ channel.send.mockClear();
+
+ channel.receive({
+ audio_start_ms: 100,
+ item_id: "interrupting-answer",
+ type: "input_audio_buffer.speech_started",
+ });
+ expect(sentEvents(channel)).toEqual([
+ { type: "output_audio_buffer.clear" },
+ ]);
+ channel.receive({
+ response_id: "response-first",
+ type: "output_audio_buffer.cleared",
+ });
+
+ expect(harness.events).toContainEqual({
+ connectionEpoch: 1,
+ responseId: "response-second",
+ speechRequestId: "canonical-1-2",
+ type: "output-interrupted",
+ });
+ });
+
test("releases active canonical ownership after acknowledged cancellation", async () => {
const harness = createHarness();
await harness.session.connect();
@@ -974,6 +1477,9 @@ describe("OpenAIRealtimeSession", () => {
canonicalSegment("question", "Canonical question"),
]);
const responseCreate = sentEvents(channel)[0]!;
+ const response = responseCreate.response as Record;
+ const responseMetadata = response.metadata as Record;
+ const speechRequestId = responseMetadata.petrinaut_request_id;
void harness.session.cancelOutput();
@@ -984,7 +1490,7 @@ describe("OpenAIRealtimeSession", () => {
channel.receive({
response: {
id: "response-canonical",
- metadata: (responseCreate.response as Record).metadata,
+ metadata: responseMetadata,
},
type: "response.created",
});
@@ -1010,6 +1516,14 @@ describe("OpenAIRealtimeSession", () => {
type: "response.done",
});
+ expect(harness.events).toContainEqual({
+ connectionEpoch: 1,
+ playbackExpected: false,
+ responseId: "response-canonical",
+ speechRequestId,
+ status: "cancelled",
+ type: "response-terminal",
+ });
expect(harness.events).not.toContainEqual(
expect.objectContaining({ type: "output-started" }),
);
@@ -1062,85 +1576,213 @@ describe("OpenAIRealtimeSession", () => {
});
});
- test("retries a correlated canonical response after the active response ends", async () => {
+ test("terminalizes cancelled canonical creation rejected behind an active response", async () => {
const harness = createHarness();
await harness.session.connect();
+ harness.session.setInterruptionBySpeaking(true);
+ harness.session.setMicrophoneEnabled(true);
const channel = harness.channels[0]!;
harness.session.speakCanonical([
canonicalSegment("question", "Canonical question"),
]);
- const firstCreate = sentEvents(channel)[0]!;
+ const responseCreate = sentEvents(channel)[0]!;
+ channel.receive({
+ audio_start_ms: 100,
+ item_id: "interrupting-item",
+ type: "input_audio_buffer.speech_started",
+ });
channel.receive({
error: {
code: "conversation_already_has_active_response",
- event_id: firstCreate.event_id,
+ event_id: responseCreate.event_id,
message: "private provider detail",
type: "invalid_request_error",
},
type: "error",
});
- channel.receive({
- response: { id: "response-active" },
- type: "response.created",
- });
- channel.receive({
- response: {
- id: "response-active",
- output: [],
- status: "completed",
- },
- type: "response.done",
- });
- const responseCreates = sentEvents(channel).filter(
- ({ type }) => type === "response.create",
- );
- expect(responseCreates).toHaveLength(2);
- expect(responseCreates[1]?.response).toEqual(firstCreate.response);
- expect(harness.events).not.toContainEqual(
- expect.objectContaining({ type: "error" }),
- );
- expect(harness.localTracks[0]!.stop).not.toHaveBeenCalled();
+ expect(harness.events).toContainEqual({
+ connectionEpoch: 1,
+ playbackExpected: false,
+ speechRequestId: "canonical-1-1",
+ status: "cancelled",
+ type: "response-terminal",
+ });
});
- test("retries canonical speech when the active response ends before the correlated error arrives", async () => {
+ test("terminalizes a collided canonical request cancelled after it is requeued", async () => {
const harness = createHarness();
await harness.session.connect();
+ harness.session.setInterruptionBySpeaking(true);
+ harness.session.setMicrophoneEnabled(true);
const channel = harness.channels[0]!;
harness.session.speakCanonical([
canonicalSegment("question", "Canonical question"),
]);
- const firstCreate = sentEvents(channel)[0]!;
+ const responseCreate = sentEvents(channel)[0]!;
- channel.receive({
- response: { id: "response-active" },
- type: "response.created",
- });
- channel.receive({
- response: {
- id: "response-active",
- output: [],
- status: "completed",
- },
- type: "response.done",
- });
channel.receive({
error: {
code: "conversation_already_has_active_response",
- event_id: firstCreate.event_id,
+ event_id: responseCreate.event_id,
message: "private provider detail",
type: "invalid_request_error",
},
type: "error",
});
+ channel.receive({
+ audio_start_ms: 100,
+ item_id: "interrupting-item",
+ type: "input_audio_buffer.speech_started",
+ });
- const responseCreates = sentEvents(channel).filter(
- ({ type }) => type === "response.create",
- );
- expect(responseCreates).toHaveLength(2);
- expect(responseCreates[1]?.response).toEqual(firstCreate.response);
- expect(harness.events).not.toContainEqual(
+ expect(harness.events).toContainEqual({
+ connectionEpoch: 1,
+ playbackExpected: false,
+ speechRequestId: "canonical-1-1",
+ status: "cancelled",
+ type: "response-terminal",
+ });
+ expect(
+ harness.events.filter(
+ (event) => event.type === "canonical-speech-requested",
+ ),
+ ).toHaveLength(1);
+ });
+
+ test("starts queued speech when a competing response terminates before the cancelled request collision", async () => {
+ const harness = createHarness();
+ await harness.session.connect();
+ harness.session.setInterruptionBySpeaking(true);
+ harness.session.setMicrophoneEnabled(true);
+ const channel = harness.channels[0]!;
+ harness.session.speakCanonical([
+ canonicalSegment("first", "First canonical response"),
+ ]);
+ const firstResponseCreate = sentEvents(channel)[0]!;
+
+ channel.receive({
+ audio_start_ms: 100,
+ item_id: "interrupting-item",
+ type: "input_audio_buffer.speech_started",
+ });
+ channel.receive({
+ audio_end_ms: 200,
+ item_id: "interrupting-item",
+ type: "input_audio_buffer.speech_stopped",
+ });
+ harness.session.speakCanonical([
+ canonicalSegment("second", "Second canonical response"),
+ ]);
+ channel.receive({
+ response: {
+ id: "response-competing",
+ output: [],
+ status: "completed",
+ },
+ type: "response.done",
+ });
+ channel.receive({
+ error: {
+ code: "conversation_already_has_active_response",
+ event_id: firstResponseCreate.event_id,
+ message: "private provider detail",
+ type: "invalid_request_error",
+ },
+ type: "error",
+ });
+
+ const responseCreates = sentEvents(channel).filter(
+ ({ type }) => type === "response.create",
+ );
+ expect(responseCreates).toHaveLength(2);
+ expect(responseCreates[1]).toMatchObject({
+ response: {
+ metadata: { petrinaut_request_id: "canonical-1-2" },
+ },
+ });
+ });
+
+ test("retries a correlated canonical response after the active response ends", async () => {
+ const harness = createHarness();
+ await harness.session.connect();
+ const channel = harness.channels[0]!;
+ harness.session.speakCanonical([
+ canonicalSegment("question", "Canonical question"),
+ ]);
+ const firstCreate = sentEvents(channel)[0]!;
+
+ channel.receive({
+ error: {
+ code: "conversation_already_has_active_response",
+ event_id: firstCreate.event_id,
+ message: "private provider detail",
+ type: "invalid_request_error",
+ },
+ type: "error",
+ });
+ channel.receive({
+ response: { id: "response-active" },
+ type: "response.created",
+ });
+ channel.receive({
+ response: {
+ id: "response-active",
+ output: [],
+ status: "completed",
+ },
+ type: "response.done",
+ });
+
+ const responseCreates = sentEvents(channel).filter(
+ ({ type }) => type === "response.create",
+ );
+ expect(responseCreates).toHaveLength(2);
+ expect(responseCreates[1]?.response).toEqual(firstCreate.response);
+ expect(harness.events).not.toContainEqual(
+ expect.objectContaining({ type: "error" }),
+ );
+ expect(harness.localTracks[0]!.stop).not.toHaveBeenCalled();
+ });
+
+ test("retries canonical speech when the active response ends before the correlated error arrives", async () => {
+ const harness = createHarness();
+ await harness.session.connect();
+ const channel = harness.channels[0]!;
+ harness.session.speakCanonical([
+ canonicalSegment("question", "Canonical question"),
+ ]);
+ const firstCreate = sentEvents(channel)[0]!;
+
+ channel.receive({
+ response: { id: "response-active" },
+ type: "response.created",
+ });
+ channel.receive({
+ response: {
+ id: "response-active",
+ output: [],
+ status: "completed",
+ },
+ type: "response.done",
+ });
+ channel.receive({
+ error: {
+ code: "conversation_already_has_active_response",
+ event_id: firstCreate.event_id,
+ message: "private provider detail",
+ type: "invalid_request_error",
+ },
+ type: "error",
+ });
+
+ const responseCreates = sentEvents(channel).filter(
+ ({ type }) => type === "response.create",
+ );
+ expect(responseCreates).toHaveLength(2);
+ expect(responseCreates[1]?.response).toEqual(firstCreate.response);
+ expect(harness.events).not.toContainEqual(
expect.objectContaining({ type: "error" }),
);
});
@@ -1416,6 +2058,397 @@ describe("OpenAIRealtimeSession", () => {
]);
});
+ test("accepts a new provider input item after the previous item stops", async () => {
+ const harness = createHarness();
+ await harness.session.connect();
+ harness.session.setMicrophoneEnabled(true);
+ const channel = harness.channels[0]!;
+
+ channel.receive({
+ audio_start_ms: 100,
+ item_id: "first-item",
+ type: "input_audio_buffer.speech_started",
+ });
+ channel.receive({
+ audio_end_ms: 200,
+ item_id: "first-item",
+ type: "input_audio_buffer.speech_stopped",
+ });
+ channel.receive({
+ audio_start_ms: 220,
+ item_id: "second-item",
+ type: "input_audio_buffer.speech_started",
+ });
+ channel.receive({
+ content_index: 0,
+ item_id: "first-item",
+ transcript: "Submit this first answer.",
+ type: "conversation.item.input_audio_transcription.completed",
+ });
+ channel.receive({
+ audio_end_ms: 320,
+ item_id: "second-item",
+ type: "input_audio_buffer.speech_stopped",
+ });
+ channel.receive({
+ content_index: 0,
+ item_id: "second-item",
+ transcript: "Submit this second answer.",
+ type: "conversation.item.input_audio_transcription.completed",
+ });
+
+ expect(
+ harness.events.filter(
+ ({ type }) =>
+ type === "input-speech-started" ||
+ type === "input-speech-stopped" ||
+ type === "completed",
+ ),
+ ).toEqual([
+ {
+ connectionEpoch: 1,
+ itemId: "first-item",
+ type: "input-speech-started",
+ },
+ {
+ connectionEpoch: 1,
+ itemId: "first-item",
+ type: "input-speech-stopped",
+ },
+ {
+ connectionEpoch: 1,
+ itemId: "second-item",
+ type: "input-speech-started",
+ },
+ {
+ key: {
+ connectionEpoch: 1,
+ contentIndex: 0,
+ itemId: "first-item",
+ },
+ text: "Submit this first answer.",
+ type: "completed",
+ },
+ {
+ connectionEpoch: 1,
+ itemId: "second-item",
+ type: "input-speech-stopped",
+ },
+ {
+ key: {
+ connectionEpoch: 1,
+ contentIndex: 0,
+ itemId: "second-item",
+ },
+ text: "Submit this second answer.",
+ type: "completed",
+ },
+ ]);
+ });
+
+ test("ignores a duplicate speech start after the input item completes", async () => {
+ const harness = createHarness();
+ await harness.session.connect();
+ harness.session.setInterruptionBySpeaking(true);
+ harness.session.setMicrophoneEnabled(true);
+ const channel = harness.channels[0]!;
+
+ channel.receive({
+ audio_start_ms: 100,
+ item_id: "completed-item",
+ type: "input_audio_buffer.speech_started",
+ });
+ channel.receive({
+ audio_end_ms: 200,
+ item_id: "completed-item",
+ type: "input_audio_buffer.speech_stopped",
+ });
+ channel.receive({
+ content_index: 0,
+ item_id: "completed-item",
+ transcript: "Completed answer.",
+ type: "conversation.item.input_audio_transcription.completed",
+ });
+ harness.events.length = 0;
+ channel.send.mockClear();
+
+ channel.receive({
+ audio_start_ms: 100,
+ item_id: "completed-item",
+ type: "input_audio_buffer.speech_started",
+ });
+ channel.receive({
+ audio_start_ms: 220,
+ item_id: "next-item",
+ type: "input_audio_buffer.speech_started",
+ });
+ channel.receive({
+ audio_end_ms: 320,
+ item_id: "next-item",
+ type: "input_audio_buffer.speech_stopped",
+ });
+ harness.session.speakCanonical([
+ canonicalSegment("follow-up", "Continue with the next question."),
+ ]);
+
+ expect(
+ harness.events.filter(({ type }) => type === "input-speech-started"),
+ ).toEqual([
+ {
+ connectionEpoch: 1,
+ interruptionBySpeaking: true,
+ itemId: "next-item",
+ type: "input-speech-started",
+ },
+ ]);
+ expect(sentEvents(channel)).toEqual([
+ expect.objectContaining({ type: "response.create" }),
+ ]);
+ });
+
+ test("preserves stopped half-duplex input when playback starts before transcription completes", async () => {
+ const harness = createHarness();
+ await harness.session.connect();
+ harness.session.setMicrophoneEnabled(true);
+ const channel = harness.channels[0]!;
+
+ channel.receive({
+ audio_start_ms: 100,
+ item_id: "stopped-item",
+ type: "input_audio_buffer.speech_started",
+ });
+ channel.receive({
+ audio_end_ms: 200,
+ item_id: "stopped-item",
+ type: "input_audio_buffer.speech_stopped",
+ });
+ harness.session.speakCanonical([
+ canonicalSegment("follow-up", "Continue with the next question."),
+ ]);
+ channel.receive({
+ content_index: 0,
+ item_id: "stopped-item",
+ transcript: "Keep this completed answer.",
+ type: "conversation.item.input_audio_transcription.completed",
+ });
+
+ expect(
+ harness.events.filter(({ type }) => type === "completed"),
+ ).toContainEqual({
+ key: {
+ connectionEpoch: 1,
+ contentIndex: 0,
+ itemId: "stopped-item",
+ },
+ text: "Keep this completed answer.",
+ type: "completed",
+ });
+ });
+
+ test("waits for active half-duplex follow-up speech before starting playback", async () => {
+ const harness = createHarness();
+ await harness.session.connect();
+ harness.session.setMicrophoneEnabled(true);
+ const channel = harness.channels[0]!;
+
+ channel.receive({
+ audio_start_ms: 100,
+ item_id: "first-item",
+ type: "input_audio_buffer.speech_started",
+ });
+ channel.receive({
+ audio_end_ms: 200,
+ item_id: "first-item",
+ type: "input_audio_buffer.speech_stopped",
+ });
+ channel.receive({
+ audio_start_ms: 220,
+ item_id: "follow-up-item",
+ type: "input_audio_buffer.speech_started",
+ });
+ channel.receive({
+ content_index: 0,
+ item_id: "first-item",
+ transcript: "First answer.",
+ type: "conversation.item.input_audio_transcription.completed",
+ });
+ harness.session.speakCanonical([
+ canonicalSegment("follow-up", "Continue with the next question."),
+ ]);
+
+ expect(sentEvents(channel)).toEqual([]);
+
+ channel.receive({
+ audio_end_ms: 320,
+ item_id: "follow-up-item",
+ type: "input_audio_buffer.speech_stopped",
+ });
+ expect(sentEvents(channel)).toEqual([
+ expect.objectContaining({ type: "response.create" }),
+ ]);
+
+ channel.receive({
+ content_index: 0,
+ item_id: "follow-up-item",
+ transcript: "Follow-up answer.",
+ type: "conversation.item.input_audio_transcription.completed",
+ });
+ expect(harness.events).toContainEqual({
+ key: {
+ connectionEpoch: 1,
+ contentIndex: 0,
+ itemId: "follow-up-item",
+ },
+ text: "Follow-up answer.",
+ type: "completed",
+ });
+ });
+
+ test.each([
+ [" ", 0],
+ ["Submit this first answer.", 1],
+ ])(
+ "keeps follow-up speech tracked when the previous transcript settles as %j",
+ async (firstTranscript, expectedSubmissions) => {
+ const harness = createHarness();
+ const submitInterviewAnswer = vi.fn<
+ ConstructorParameters<
+ typeof RealtimeBrunchBridge
+ >[0]["submitInterviewAnswer"]
+ >(async (input) => ({ kind: "message", messageId: input.id }));
+ const bridge = new RealtimeBrunchBridge({
+ session: harness.session,
+ submitInterviewAnswer,
+ });
+ const controller = new VoiceTurnController({
+ bridge,
+ session: harness.session,
+ submitText: vi.fn(async () => undefined),
+ });
+ controller.setInterruptionBySpeaking(true);
+ controller.updateChat({
+ canAcceptInterviewAnswer: true,
+ canonicalSegments: [],
+ status: "ready",
+ });
+ await controller.start();
+ const channel = harness.channels[0]!;
+
+ channel.receive({
+ audio_start_ms: 100,
+ item_id: "first-item",
+ type: "input_audio_buffer.speech_started",
+ });
+ channel.receive({
+ audio_end_ms: 200,
+ item_id: "first-item",
+ type: "input_audio_buffer.speech_stopped",
+ });
+ channel.receive({
+ audio_start_ms: 220,
+ item_id: "follow-up-item",
+ type: "input_audio_buffer.speech_started",
+ });
+ channel.receive({
+ content_index: 0,
+ item_id: "first-item",
+ transcript: firstTranscript,
+ type: "conversation.item.input_audio_transcription.completed",
+ });
+ channel.receive({
+ content_index: 0,
+ delta: "Follow-up answer",
+ item_id: "follow-up-item",
+ type: "conversation.item.input_audio_transcription.delta",
+ });
+
+ expect(submitInterviewAnswer).toHaveBeenCalledTimes(expectedSubmissions);
+ expect(harness.localTracks[0]!.enabled).toBe(true);
+ expect(controller.getSnapshot()).toMatchObject({
+ inputNotice: "none",
+ partialText: "Follow-up answer",
+ });
+ await controller.end();
+ },
+ );
+
+ test("queues canonical playback until in-progress speech stops", async () => {
+ const harness = createHarness();
+ await harness.session.connect();
+ harness.session.setInterruptionBySpeaking(true);
+ harness.session.setMicrophoneEnabled(true);
+ const channel = harness.channels[0]!;
+
+ channel.receive({
+ audio_start_ms: 100,
+ item_id: "in-progress-answer",
+ type: "input_audio_buffer.speech_started",
+ });
+ harness.session.speakCanonical([
+ canonicalSegment("follow-up", "This must wait for the user to finish."),
+ ]);
+
+ expect(sentEvents(channel)).toEqual([]);
+
+ channel.receive({
+ audio_end_ms: 200,
+ item_id: "in-progress-answer",
+ type: "input_audio_buffer.speech_stopped",
+ });
+
+ expect(sentEvents(channel)).toEqual([
+ expect.objectContaining({ type: "response.create" }),
+ ]);
+ });
+
+ test("exposes only the first concurrently speaking provider input item", async () => {
+ const harness = createHarness();
+ await harness.session.connect();
+ harness.session.setMicrophoneEnabled(true);
+ const channel = harness.channels[0]!;
+
+ for (const itemId of ["first-item", "overlapping-item"]) {
+ channel.receive({
+ audio_start_ms: 100,
+ item_id: itemId,
+ type: "input_audio_buffer.speech_started",
+ });
+ }
+ channel.receive({
+ content_index: 0,
+ item_id: "first-item",
+ transcript: "Submit only this answer.",
+ type: "conversation.item.input_audio_transcription.completed",
+ });
+ channel.receive({
+ content_index: 0,
+ item_id: "overlapping-item",
+ transcript: "Do not submit this overlap.",
+ type: "conversation.item.input_audio_transcription.completed",
+ });
+
+ expect(
+ harness.events.filter(
+ ({ type }) => type === "input-speech-started" || type === "completed",
+ ),
+ ).toEqual([
+ {
+ connectionEpoch: 1,
+ itemId: "first-item",
+ type: "input-speech-started",
+ },
+ {
+ key: {
+ connectionEpoch: 1,
+ contentIndex: 0,
+ itemId: "first-item",
+ },
+ text: "Submit only this answer.",
+ type: "completed",
+ },
+ ]);
+ });
+
test("does not reuse a speech boundary from a previous connection epoch", async () => {
const harness = createHarness();
await harness.session.connect();
diff --git a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts
index c01d16b8984..82c8c4a97a8 100644
--- a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts
+++ b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts
@@ -34,6 +34,8 @@ export type OpenAIRealtimeSessionEvent =
readonly connectionEpoch: number;
readonly itemId: string;
readonly type: "input-speech-started";
+ /** Capture was accepted with interruption by speaking enabled. */
+ readonly interruptionBySpeaking?: true;
}
| {
readonly connectionEpoch: number;
@@ -44,6 +46,8 @@ export type OpenAIRealtimeSessionEvent =
readonly connectionEpoch: number;
readonly responseId: string;
readonly speechRequestId: string;
+ /** Exact canonical text for this playback, never generated audio text. */
+ readonly canonicalText?: readonly string[];
readonly type: "output-started";
}
| {
@@ -59,12 +63,15 @@ export type OpenAIRealtimeSessionEvent =
| {
readonly connectionEpoch: number;
readonly responseId: string;
+ /** Present when interruption ends a request before playback starts. */
+ readonly speechRequestId?: string;
readonly type: "output-interrupted";
}
| {
readonly connectionEpoch: number;
readonly playbackExpected: boolean;
- readonly responseId: string;
+ /** Absent when a cancelled response.create was rejected before creation. */
+ readonly responseId?: string;
readonly speechRequestId?: string;
readonly status: "cancelled" | "completed" | "failed" | "incomplete";
readonly type: "response-terminal";
@@ -104,7 +111,12 @@ interface RequestTiming {
readonly startedAt: number;
}
+interface SpeechTiming extends RequestTiming {
+ readonly canonicalText: readonly string[];
+}
+
interface CanonicalSpeechRequest {
+ ownershipAnnounced: boolean;
readonly response: Record;
readonly speechRequestId: string;
}
@@ -208,11 +220,14 @@ export class OpenAIRealtimeSession {
readonly #canonicalSpeechQueue: CanonicalSpeechRequest[] = [];
readonly #completedResponseCancelEventIds = new Set();
readonly #pendingClientEvents = new Map();
- readonly #pendingSpeechRequests = new Map();
+ readonly #pendingSpeechRequests = new Map();
readonly #playbackOverlappingInputItemIds = new Set();
readonly #remoteStreams = new Set();
+ readonly #responseSpeechRequestIds = new Map();
+ readonly #seenInputItemIds = new Set();
readonly #speechRequestIds = new Map();
- readonly #speechTimings = new Map();
+ readonly #speechTimings = new Map();
+ readonly #stoppedInputItemIds = new Set();
readonly #terminalCanonicalResponseIds = new Set();
readonly #transcriptionTimings = new Map();
#abortController: AbortController | null = null;
@@ -235,6 +250,7 @@ export class OpenAIRealtimeSession {
#meterHasSample = false;
#meterLevel = 0;
#meterSamples: Uint8Array | null = null;
+ #interruptionBySpeaking = false;
#microphoneRequested = false;
#microphoneTrack: MediaStreamTrack | null = null;
#peerConnection: RTCPeerConnection | null = null;
@@ -242,6 +258,7 @@ export class OpenAIRealtimeSession {
#responseCreateEventId: string | null = null;
#responseTerminalSequence = 0;
#speakingResponseId: string | null = null;
+ #speakingInputItemId: string | null = null;
#speechRequestSequence = 0;
#unexpectedCloseListener: (() => void) | null = null;
#waitingForResponseTerminal = false;
@@ -410,6 +427,45 @@ export class OpenAIRealtimeSession {
this.#syncMicrophoneTrack();
}
+ public setInterruptionBySpeaking(enabled: boolean): void {
+ this.#interruptionBySpeaking = enabled;
+ this.#syncMicrophoneTrack();
+ }
+
+ /** Cancel only assistant output; the utterance which caused this stays alive. */
+ #interruptOutputBySpeaking(): void {
+ for (const request of this.#canonicalSpeechQueue.splice(0)) {
+ this.#settleCancelledSpeechRequest(request);
+ }
+ if (this.#responseCreateEventId !== null) {
+ const pending = this.#pendingClientEvents.get(
+ this.#responseCreateEventId,
+ );
+ if (pending?.kind === "response-create") {
+ this.#cancelledSpeechRequestIds.add(pending.request.speechRequestId);
+ }
+ }
+ let cancelledOutput = false;
+ for (const responseId of this.#canonicalResponseIds) {
+ const responseIsActive = this.#activeResponseIds.has(responseId);
+ const responseHasBufferedOutput =
+ this.#terminalCanonicalResponseIds.has(responseId) ||
+ this.#speakingResponseId === responseId;
+ if (
+ this.#cancelledCanonicalResponseIds.has(responseId) ||
+ (!responseIsActive && !responseHasBufferedOutput)
+ ) {
+ continue;
+ }
+ this.#cancelledCanonicalResponseIds.add(responseId);
+ if (responseIsActive) {
+ this.#cancelResponse(responseId);
+ }
+ cancelledOutput = true;
+ }
+ if (cancelledOutput) this.#send({ type: "output_audio_buffer.clear" });
+ }
+
public speakCanonical(segments: CanonicalSpeechSegment[]): void {
this.#requestCanonicalSpeech(segments, true);
}
@@ -435,13 +491,15 @@ export class OpenAIRealtimeSession {
this.#playbackOverlappingInputItemIds.add(itemId);
}
this.#acceptedInputItemIds.clear();
+ this.#stoppedInputItemIds.clear();
+ this.#speakingInputItemId = null;
this.#syncMicrophoneTrack();
try {
this.#send({ type: "input_audio_buffer.clear" });
for (const request of this.#canonicalSpeechQueue.splice(0)) {
- this.#cancelPendingSpeechRequest(request.speechRequestId);
+ this.#settleCancelledSpeechRequest(request);
}
if (this.#responseCreateEventId !== null) {
@@ -512,6 +570,7 @@ export class OpenAIRealtimeSession {
const responseText = this.#canonicalResponseText(segments);
const speechRequestId = `canonical-${this.#activeEpoch}-${++this.#speechRequestSequence}`;
this.#pendingSpeechRequests.set(speechRequestId, {
+ canonicalText: responseText,
requestId:
this.#dependencies.createRequestId?.() ?? createVoiceRequestId(),
startedAt: this.#now(),
@@ -544,7 +603,11 @@ export class OpenAIRealtimeSession {
petrinaut_request_id: speechRequestId,
},
};
- const request = { response, speechRequestId };
+ const request = {
+ ownershipAnnounced: false,
+ response,
+ speechRequestId,
+ };
this.#canonicalSpeechQueue.push(request);
try {
this.#sendNextCanonicalSpeech();
@@ -580,11 +643,21 @@ export class OpenAIRealtimeSession {
return `petrinaut-${this.#activeEpoch}-${++this.#clientEventSequence}`;
}
+ #markUnfinishedInputItemsAsPlaybackOverlaps(): void {
+ for (const itemId of this.#acceptedInputItemIds) {
+ if (!this.#stoppedInputItemIds.has(itemId)) {
+ this.#playbackOverlappingInputItemIds.add(itemId);
+ this.#acceptedInputItemIds.delete(itemId);
+ }
+ }
+ }
+
#sendNextCanonicalSpeech(): void {
if (
this.#activeResponseIds.size > 0 ||
this.#responseCreateEventId !== null ||
- this.#waitingForResponseTerminal
+ this.#waitingForResponseTerminal ||
+ this.#speakingInputItemId !== null
) {
return;
}
@@ -600,10 +673,10 @@ export class OpenAIRealtimeSession {
request,
responseTerminalSequence: this.#responseTerminalSequence,
});
- for (const itemId of this.#acceptedInputItemIds) {
- this.#playbackOverlappingInputItemIds.add(itemId);
+ if (!this.#interruptionBySpeaking) {
+ this.#markUnfinishedInputItemsAsPlaybackOverlaps();
+ this.#speakingInputItemId = null;
}
- this.#acceptedInputItemIds.clear();
this.#syncMicrophoneTrack();
try {
this.#send({
@@ -611,7 +684,8 @@ export class OpenAIRealtimeSession {
response: request.response,
type: "response.create",
});
- if (this.#activeEpoch !== null) {
+ if (this.#activeEpoch !== null && !request.ownershipAnnounced) {
+ request.ownershipAnnounced = true;
this.#emit({
connectionEpoch: this.#activeEpoch,
speechRequestId: request.speechRequestId,
@@ -652,6 +726,8 @@ export class OpenAIRealtimeSession {
}
if (parsed.type === "input_audio_buffer.cleared") {
this.#acceptedInputItemIds.clear();
+ this.#stoppedInputItemIds.clear();
+ this.#speakingInputItemId = null;
this.#cancelOutputAwaitingInputBufferClear = false;
this.#finishOutputCancellation();
return;
@@ -664,12 +740,39 @@ export class OpenAIRealtimeSession {
if (parsed.type === "input_audio_buffer.speech_started") {
const itemId = nonEmptyString(parsed.item_id);
if (!itemId || nonNegativeInteger(parsed.audio_start_ms) === null) return;
- if (this.#speakingResponseId || !this.#microphoneTrack?.enabled) {
+ if (this.#seenInputItemIds.has(itemId)) return;
+ this.#seenInputItemIds.add(itemId);
+ if (
+ (!this.#interruptionBySpeaking && this.#speakingResponseId) ||
+ !this.#microphoneTrack?.enabled
+ ) {
+ this.#playbackOverlappingInputItemIds.add(itemId);
+ return;
+ }
+ if (
+ this.#acceptedInputItemIds.has(itemId) ||
+ this.#playbackOverlappingInputItemIds.has(itemId)
+ ) {
+ return;
+ }
+ if (this.#speakingInputItemId !== null) {
this.#playbackOverlappingInputItemIds.add(itemId);
return;
}
this.#acceptedInputItemIds.add(itemId);
+ this.#speakingInputItemId = itemId;
+ if (this.#interruptionBySpeaking) {
+ try {
+ this.#interruptOutputBySpeaking();
+ } catch {
+ this.#handleConnectionFailure("network", "speech");
+ return;
+ }
+ }
this.#emit({
+ ...(this.#interruptionBySpeaking
+ ? { interruptionBySpeaking: true as const }
+ : {}),
connectionEpoch,
itemId,
type: "input-speech-started",
@@ -680,11 +783,21 @@ export class OpenAIRealtimeSession {
const itemId = nonEmptyString(parsed.item_id);
if (!itemId || nonNegativeInteger(parsed.audio_end_ms) === null) return;
if (this.#playbackOverlappingInputItemIds.has(itemId)) return;
+ if (this.#acceptedInputItemIds.has(itemId)) {
+ this.#stoppedInputItemIds.add(itemId);
+ }
+ const shouldResumeCanonicalSpeech = this.#speakingInputItemId === itemId;
+ if (this.#speakingInputItemId === itemId) {
+ this.#speakingInputItemId = null;
+ }
this.#emit({
connectionEpoch,
itemId,
type: "input-speech-stopped",
});
+ if (shouldResumeCanonicalSpeech) {
+ this.#resumeCanonicalSpeechQueue();
+ }
return;
}
if (
@@ -721,6 +834,7 @@ export class OpenAIRealtimeSession {
}
this.#completeResponseCreateEvent(speechRequestId);
this.#canonicalResponseIds.add(responseId);
+ this.#responseSpeechRequestIds.set(responseId, speechRequestId);
if (this.#cancelledSpeechRequestIds.delete(speechRequestId)) {
this.#cancelPendingSpeechRequest(speechRequestId);
this.#cancelledCanonicalResponseIds.add(responseId);
@@ -796,11 +910,11 @@ export class OpenAIRealtimeSession {
pendingEvent.request.speechRequestId,
)
) {
- this.#cancelOutputAwaitingRequestIds.delete(
- pendingEvent.request.speechRequestId,
- );
- this.#cancelPendingSpeechRequest(pendingEvent.request.speechRequestId);
+ const { speechRequestId } = pendingEvent.request;
+ this.#cancelOutputAwaitingRequestIds.delete(speechRequestId);
+ this.#settleCancelledSpeechRequest(pendingEvent.request);
this.#finishOutputCancellation();
+ this.#resumeCanonicalSpeechQueue();
return;
}
this.#canonicalSpeechQueue.unshift(pendingEvent.request);
@@ -841,7 +955,8 @@ export class OpenAIRealtimeSession {
this.#finishOutputCancellation();
this.#clearResponseCancelEvents(responseId);
this.#waitingForResponseTerminal = false;
- const speechRequestId = this.#speechRequestIds.get(responseId);
+ const speechRequestId = this.#responseSpeechRequestIds.get(responseId);
+ this.#responseSpeechRequestIds.delete(responseId);
const terminalEvent = {
connectionEpoch,
playbackExpected,
@@ -949,10 +1064,10 @@ export class OpenAIRealtimeSession {
this.#handleConnectionFailure("invalid-response", "connection");
return;
}
- for (const itemId of this.#acceptedInputItemIds) {
- this.#playbackOverlappingInputItemIds.add(itemId);
+ if (!this.#interruptionBySpeaking) {
+ this.#markUnfinishedInputItemsAsPlaybackOverlaps();
+ this.#speakingInputItemId = null;
}
- this.#acceptedInputItemIds.clear();
this.#speakingResponseId = responseId;
this.#syncMicrophoneTrack();
const speechRequestId = this.#speechRequestIds.get(responseId);
@@ -961,6 +1076,7 @@ export class OpenAIRealtimeSession {
return;
}
this.#emit({
+ canonicalText: this.#speechTimings.get(responseId)?.canonicalText ?? [],
connectionEpoch,
responseId,
speechRequestId,
@@ -970,23 +1086,47 @@ export class OpenAIRealtimeSession {
}
const wasSpeaking = this.#speakingResponseId === responseId;
const wasCleared = event.type === "output_audio_buffer.cleared";
+ const speechRequestId = this.#speechRequestIds.get(responseId);
+ const additionallyClearedResponses = wasCleared
+ ? [...this.#terminalCanonicalResponseIds]
+ .filter((terminalResponseId) => terminalResponseId !== responseId)
+ .map((terminalResponseId) => ({
+ responseId: terminalResponseId,
+ speechRequestId: this.#speechRequestIds.get(terminalResponseId),
+ }))
+ : [];
+ const interruptedBeforePlayback =
+ wasCleared &&
+ !wasSpeaking &&
+ speechRequestId !== undefined &&
+ this.#cancelledCanonicalResponseIds.has(responseId);
this.#finishSpeech(
responseId,
wasCleared || this.#cancelledCanonicalResponseIds.has(responseId)
? "request-aborted"
: undefined,
);
- if (wasCleared && this.#cancelOutputAwaitingOutputBufferClear) {
- for (const terminalResponseId of this.#terminalCanonicalResponseIds) {
- this.#finishSpeech(terminalResponseId, "request-aborted");
- }
- }
if (wasSpeaking) {
this.#emit({
connectionEpoch,
responseId,
type: wasCleared ? "output-interrupted" : "output-stopped",
});
+ } else if (interruptedBeforePlayback) {
+ this.#emit({
+ connectionEpoch,
+ responseId,
+ speechRequestId,
+ type: "output-interrupted",
+ });
+ }
+ for (const clearedResponse of additionallyClearedResponses) {
+ this.#finishSpeech(clearedResponse.responseId, "request-aborted");
+ this.#emit({
+ connectionEpoch,
+ ...clearedResponse,
+ type: "output-interrupted",
+ });
}
if (wasCleared && this.#cancelOutputAwaitingOutputBufferClear) {
this.#cancelOutputAwaitingOutputBufferClear = false;
@@ -1017,15 +1157,21 @@ export class OpenAIRealtimeSession {
? "invalid-response"
: undefined,
);
- this.#acceptedInputItemIds.delete(itemId);
+ if (this.#finishTranscribedInputItem(itemId)) {
+ this.#resumeCanonicalSpeechQueue();
+ }
}
return;
}
this.#startTranscription(itemId);
if (event.type === "conversation.item.input_audio_transcription.failed") {
this.#finishTranscription(itemId, "invalid-response");
- this.#acceptedInputItemIds.delete(itemId);
+ const shouldResumeCanonicalSpeech =
+ this.#finishTranscribedInputItem(itemId);
this.#emit({ key, type: "transcription-failed" });
+ if (shouldResumeCanonicalSpeech) {
+ this.#resumeCanonicalSpeechQueue();
+ }
return;
}
const text =
@@ -1033,11 +1179,12 @@ export class OpenAIRealtimeSession {
? event.delta
: event.transcript;
if (typeof text !== "string") return;
+ let shouldResumeCanonicalSpeech = false;
if (
event.type === "conversation.item.input_audio_transcription.completed"
) {
this.#finishTranscription(itemId);
- this.#acceptedInputItemIds.delete(itemId);
+ shouldResumeCanonicalSpeech = this.#finishTranscribedInputItem(itemId);
}
this.#emit({
key,
@@ -1047,6 +1194,19 @@ export class OpenAIRealtimeSession {
? "partial"
: "completed",
});
+ if (shouldResumeCanonicalSpeech) {
+ this.#resumeCanonicalSpeechQueue();
+ }
+ }
+
+ #finishTranscribedInputItem(itemId: string): boolean {
+ this.#acceptedInputItemIds.delete(itemId);
+ this.#stoppedInputItemIds.delete(itemId);
+ if (this.#speakingInputItemId !== itemId) {
+ return false;
+ }
+ this.#speakingInputItemId = null;
+ return true;
}
#cancelPendingSpeechRequest(speechRequestId: string): void {
@@ -1063,6 +1223,20 @@ export class OpenAIRealtimeSession {
);
}
+ #settleCancelledSpeechRequest(request: CanonicalSpeechRequest): void {
+ this.#cancelPendingSpeechRequest(request.speechRequestId);
+ if (!request.ownershipAnnounced || this.#activeEpoch === null) {
+ return;
+ }
+ this.#emit({
+ connectionEpoch: this.#activeEpoch,
+ playbackExpected: false,
+ speechRequestId: request.speechRequestId,
+ status: "cancelled",
+ type: "response-terminal",
+ });
+ }
+
#finishSpeech(responseId: string, errorCode?: VoiceErrorCode): void {
const timing = this.#speechTimings.get(responseId);
if (timing) {
@@ -1292,10 +1466,11 @@ export class OpenAIRealtimeSession {
this.#microphoneRequested &&
this.#connected &&
this.#cancelOutputPromise === null &&
- this.#authorizedResponseIds.size === 0 &&
- this.#canonicalSpeechQueue.length === 0 &&
- this.#responseCreateEventId === null &&
- this.#speakingResponseId === null;
+ (this.#interruptionBySpeaking ||
+ (this.#authorizedResponseIds.size === 0 &&
+ this.#canonicalSpeechQueue.length === 0 &&
+ this.#responseCreateEventId === null &&
+ this.#speakingResponseId === null));
this.#microphoneTrack.enabled = enabled;
if (enabled) {
this.#startMeter();
@@ -1404,14 +1579,18 @@ export class OpenAIRealtimeSession {
this.#pendingClientEvents.clear();
this.#pendingSpeechRequests.clear();
this.#playbackOverlappingInputItemIds.clear();
+ this.#responseSpeechRequestIds.clear();
+ this.#seenInputItemIds.clear();
this.#speechTimings.clear();
this.#speechRequestIds.clear();
+ this.#stoppedInputItemIds.clear();
this.#terminalCanonicalResponseIds.clear();
this.#authorizedResponseIds.clear();
this.#canonicalResponseIds.clear();
this.#responseCreateEventId = null;
this.#responseTerminalSequence = 0;
this.#speakingResponseId = null;
+ this.#speakingInputItemId = null;
this.#microphoneRequested = false;
this.#waitingForResponseTerminal = false;
this.#activeEpoch = null;
diff --git a/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.test.ts
index 24277f28af1..b59b1ecae9e 100644
--- a/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.test.ts
+++ b/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.test.ts
@@ -89,6 +89,7 @@ const createHarness = () => {
const bridge = new RealtimeBrunchBridge({
session,
submitInterviewAnswer,
+ reportDiagnostic: vi.fn(),
});
const events: RealtimeBrunchBridgeEvent[] = [];
bridge.subscribe((event) => events.push(event));
@@ -115,6 +116,904 @@ const startReady = (
};
describe("RealtimeBrunchBridge", () => {
+ const vocabularyLeak =
+ "SDCPN, stochastic Petri net, place, transition, arc, token, marking, guard, rate, distribution, parameter, subnet, scenario, and metric.";
+ const assistantText =
+ "The supervisor reviews the request before the manager approves it.";
+
+ test.each([
+ ["prompt-regurgitation", vocabularyLeak],
+ ["prompt-regurgitation", vocabularyLeak.normalize("NFKC").toUpperCase()],
+ [
+ "prompt-regurgitation",
+ "place, transition, arc, token, marking, guard, rate, distribution, parameter, subnet",
+ ],
+ ["self-echo", assistantText],
+ [
+ "self-echo",
+ "ＴＨＥ supervisor—reviews the request, before\n the manager approves it!",
+ ],
+ ["self-echo", "reviews the request before the manager approves it"],
+ ])("silently discards %s before pending admission: %s", (reason, text) => {
+ const harness = createHarness();
+ startReady(harness);
+ harness.bridge.updateChat({
+ canAcceptInterviewAnswer: false,
+ canonicalSegments: [],
+ status: "streaming",
+ });
+ harness.emit({
+ type: "output-started",
+ connectionEpoch: 3,
+ responseId: "playing",
+ speechRequestId: "speech",
+ canonicalText: [assistantText],
+ });
+ harness.emit({
+ type: "input-speech-started",
+ connectionEpoch: 3,
+ itemId: "false-vad",
+ interruptionBySpeaking: true,
+ });
+ // Output and canonical chat may change before transcription completes.
+ harness.emit({
+ type: "output-interrupted",
+ connectionEpoch: 3,
+ responseId: "playing",
+ });
+ harness.bridge.updateChat({
+ canAcceptInterviewAnswer: false,
+ canonicalSegments: [
+ segment("later", "Which department handles the invoice?"),
+ ],
+ status: "streaming",
+ });
+ // Repeated starts cannot replace the original playback snapshot.
+ harness.emit({
+ type: "input-speech-started",
+ connectionEpoch: 3,
+ itemId: "false-vad",
+ interruptionBySpeaking: true,
+ });
+ harness.emit(completedTranscript(3, text, "false-vad"));
+ expect(harness.events).toEqual([
+ { itemId: "false-vad", type: "transcript-rejected", reason },
+ ]);
+ expect(harness.submitInterviewAnswer).not.toHaveBeenCalled();
+ harness.emit(completedTranscript(3, text, "false-vad"));
+ expect(harness.events.at(-1)).toEqual({
+ itemId: "false-vad",
+ type: "transcript-rejected",
+ reason: "duplicate",
+ });
+
+ // A false transcript must not occupy the single pending-answer slot.
+ harness.emit({
+ type: "input-speech-started",
+ connectionEpoch: 3,
+ itemId: "real",
+ interruptionBySpeaking: true,
+ });
+ harness.emit(completedTranscript(3, "wait", "real"));
+ expect(harness.events.at(-1)).toEqual({
+ type: "transcript-retained",
+ answer: "wait",
+ });
+ const ready = {
+ canAcceptInterviewAnswer: true,
+ canonicalSegments: [],
+ status: "ready" as const,
+ };
+ harness.bridge.updateChat(ready);
+ harness.bridge.updateChat(ready);
+ harness.emit(completedTranscript(3, "wait", "real"));
+ expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce();
+ expect(harness.submitInterviewAnswer).toHaveBeenCalledWith(
+ expect.objectContaining({ text: "wait" }),
+ );
+ });
+
+ test.each([
+ "stop",
+ "no",
+ "wait",
+ "ＳＴＯＰ!",
+ "place",
+ "transition",
+ "The supervisor does not approve it; the auditor makes that decision.",
+ "Use a place and transition with a token and a guard for approval.",
+ "metric scenario subnet parameter distribution rate guard marking token arc transition place net Petri stochastic SDCPN",
+ ])(
+ "admits novel interruption exactly once without changing its text: %s",
+ (text) => {
+ const harness = createHarness();
+ startReady(harness);
+ harness.emit({
+ type: "output-started",
+ connectionEpoch: 3,
+ responseId: "playing",
+ speechRequestId: "speech",
+ canonicalText: [assistantText],
+ });
+ harness.emit({
+ type: "input-speech-started",
+ connectionEpoch: 3,
+ itemId: "real",
+ interruptionBySpeaking: true,
+ });
+ harness.emit(completedTranscript(3, text, "real"));
+ harness.emit(completedTranscript(3, text, "real"));
+ expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce();
+ expect(harness.submitInterviewAnswer).toHaveBeenCalledWith(
+ expect.objectContaining({ text }),
+ );
+ },
+ );
+
+ test("rejects a short exact self-echo while admitting short novel speech", () => {
+ const harness = createHarness();
+ startReady(harness);
+ harness.emit({
+ type: "output-started",
+ connectionEpoch: 3,
+ responseId: "playing",
+ speechRequestId: "speech",
+ canonicalText: ["Who approves this?"],
+ });
+ harness.emit({
+ type: "input-speech-started",
+ connectionEpoch: 3,
+ itemId: "short-echo",
+ interruptionBySpeaking: true,
+ });
+ harness.emit(completedTranscript(3, "ＷＨＯ approves this!", "short-echo"));
+
+ expect(harness.events.at(-1)).toEqual({
+ itemId: "short-echo",
+ type: "transcript-rejected",
+ reason: "self-echo",
+ });
+ expect(harness.submitInterviewAnswer).not.toHaveBeenCalled();
+
+ harness.emit({
+ type: "input-speech-started",
+ connectionEpoch: 3,
+ itemId: "short-novel",
+ interruptionBySpeaking: true,
+ });
+ harness.emit(completedTranscript(3, "The supervisor does.", "short-novel"));
+
+ expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce();
+ expect(harness.submitInterviewAnswer).toHaveBeenCalledWith(
+ expect.objectContaining({ text: "The supervisor does." }),
+ );
+ });
+
+ test.each([vocabularyLeak, assistantText])(
+ "leaves ordinary transcripts unchanged: %s",
+ (text) => {
+ const harness = createHarness();
+ startReady(harness);
+ harness.emit(completedTranscript(3, text));
+ expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce();
+ expect(harness.submitInterviewAnswer).toHaveBeenCalledWith(
+ expect.objectContaining({ text }),
+ );
+ },
+ );
+
+ test("does not classify ordinary capture merely because interruption is enabled", () => {
+ const harness = createHarness();
+ startReady(harness);
+ harness.emit({
+ type: "input-speech-started",
+ connectionEpoch: 3,
+ itemId: "ordinary",
+ interruptionBySpeaking: true,
+ });
+ harness.emit(completedTranscript(3, vocabularyLeak, "ordinary"));
+ expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce();
+ });
+
+ test.each(["stop", "cancelPendingSpeech", "failure", "reconnect"] as const)(
+ "cleans playback snapshots on %s",
+ (action) => {
+ const harness = createHarness();
+ startReady(harness);
+ harness.emit({
+ type: "output-started",
+ connectionEpoch: 3,
+ responseId: "old",
+ speechRequestId: "speech",
+ canonicalText: [assistantText],
+ });
+ harness.emit({
+ type: "input-speech-started",
+ connectionEpoch: 3,
+ itemId: "unfinished",
+ interruptionBySpeaking: true,
+ });
+ if (action === "failure")
+ harness.bridge.updateChat({
+ canAcceptInterviewAnswer: false,
+ canonicalSegments: [],
+ status: "error",
+ });
+ else if (action === "reconnect") harness.bridge.start(4);
+ else harness.bridge[action]();
+ if (action === "cancelPendingSpeech")
+ harness.bridge.completeTurnHandoff();
+ else harness.bridge.start(4);
+ harness.bridge.updateChat({
+ canAcceptInterviewAnswer: true,
+ canonicalSegments: [],
+ status: "ready",
+ });
+ const epoch = action === "cancelPendingSpeech" ? 3 : 4;
+ const itemId =
+ action === "cancelPendingSpeech"
+ ? "fresh-after-cancellation"
+ : "unfinished";
+ harness.emit({
+ type: "input-speech-started",
+ connectionEpoch: epoch,
+ itemId,
+ interruptionBySpeaking: true,
+ });
+ harness.emit(completedTranscript(epoch, assistantText, itemId));
+ expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce();
+ },
+ );
+
+ test("does not classify a prompt leak while playback creation is still pending", () => {
+ const harness = createHarness();
+ startReady(harness);
+ harness.emit({
+ type: "canonical-speech-requested",
+ connectionEpoch: 3,
+ speechRequestId: "creating",
+ });
+ harness.emit({
+ type: "input-speech-started",
+ connectionEpoch: 3,
+ itemId: "false-vad",
+ interruptionBySpeaking: true,
+ });
+ harness.emit(completedTranscript(3, vocabularyLeak, "false-vad"));
+ expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce();
+ expect(harness.submitInterviewAnswer).toHaveBeenCalledWith(
+ expect.objectContaining({ text: vocabularyLeak }),
+ );
+ expect(harness.events).not.toContainEqual(
+ expect.objectContaining({ type: "transcript-rejected" }),
+ );
+ });
+
+ test("keeps an accepted interruption through a later speech request", async () => {
+ const harness = createHarness();
+ startReady(harness);
+ harness.emit({
+ canonicalText: [assistantText],
+ connectionEpoch: 3,
+ responseId: "playing",
+ speechRequestId: "speech-playing",
+ type: "output-started",
+ });
+ harness.emit({
+ connectionEpoch: 3,
+ interruptionBySpeaking: true,
+ itemId: "interruption",
+ type: "input-speech-started",
+ });
+ harness.emit({
+ connectionEpoch: 3,
+ speechRequestId: "speech-follow-on",
+ type: "canonical-speech-requested",
+ });
+
+ harness.emit(
+ completedTranscript(
+ 3,
+ "Actually, the auditor approves it.",
+ "interruption",
+ ),
+ );
+
+ await vi.waitFor(() =>
+ expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce(),
+ );
+ expect(harness.events).not.toContainEqual({
+ reason: "unavailable",
+ type: "transcript-rejected",
+ });
+ });
+
+ test.each(["output-stopped", "output-interrupted"] as const)(
+ "does not compare against playback that already %s",
+ (type) => {
+ const harness = createHarness();
+ startReady(harness);
+ harness.emit({
+ type: "output-started",
+ connectionEpoch: 3,
+ responseId: "old",
+ speechRequestId: "speech",
+ canonicalText: [assistantText],
+ });
+ harness.emit({ type, connectionEpoch: 3, responseId: "old" });
+ harness.emit({
+ type: "input-speech-started",
+ connectionEpoch: 3,
+ itemId: "real",
+ interruptionBySpeaking: true,
+ });
+ harness.emit(completedTranscript(3, assistantText, "real"));
+ expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce();
+ },
+ );
+
+ test("retains an interrupting transcript until the previous Brunch submission settles", async () => {
+ const harness = createHarness();
+ startReady(harness);
+ harness.emit(completedTranscript(3));
+ await vi.waitFor(() =>
+ expect(
+ harness.events.some(({ type }) => type === "submission-accepted"),
+ ).toBe(true),
+ );
+ harness.bridge.updateChat({
+ canAcceptInterviewAnswer: false,
+ canonicalSegments: [],
+ status: "streaming",
+ });
+ harness.emit({
+ type: "canonical-speech-requested",
+ connectionEpoch: 3,
+ speechRequestId: "speech-1",
+ });
+ harness.emit({
+ type: "input-speech-started",
+ connectionEpoch: 3,
+ itemId: "interruption",
+ interruptionBySpeaking: true,
+ });
+ harness.emit(
+ completedTranscript(
+ 3,
+ "Actually, the manager approves it.",
+ "interruption",
+ ),
+ );
+ expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce();
+ const update = {
+ canAcceptInterviewAnswer: true,
+ canonicalSegments: [
+ segment("reply", "Who is informed?", "submission-voice-1"),
+ ],
+ status: "ready" as const,
+ };
+ harness.bridge.updateChat(update);
+ await vi.waitFor(() =>
+ expect(harness.submitInterviewAnswer).toHaveBeenCalledTimes(2),
+ );
+ expect(harness.submitInterviewAnswer).toHaveBeenLastCalledWith(
+ expect.objectContaining({
+ id: "voice-realtime:3:interruption:0",
+ text: "Actually, the manager approves it.",
+ }),
+ );
+ harness.emit(
+ completedTranscript(
+ 3,
+ "Actually, the manager approves it.",
+ "interruption",
+ ),
+ );
+ harness.bridge.updateChat(update);
+ expect(harness.submitInterviewAnswer).toHaveBeenCalledTimes(2);
+ expect(harness.session.speakCanonical).not.toHaveBeenCalled();
+ });
+
+ test("submits sequential utterances in speech order when transcription completes out of order", async () => {
+ const harness = createHarness();
+ startReady(harness);
+ for (const itemId of ["first-item", "second-item"]) {
+ harness.emit({
+ connectionEpoch: 3,
+ interruptionBySpeaking: true,
+ itemId,
+ type: "input-speech-started",
+ });
+ harness.emit({
+ connectionEpoch: 3,
+ itemId,
+ type: "input-speech-stopped",
+ });
+ }
+
+ harness.emit(completedTranscript(3, "Second answer.", "second-item"));
+ expect(harness.submitInterviewAnswer).not.toHaveBeenCalled();
+
+ harness.emit(completedTranscript(3, "First answer.", "first-item"));
+ await vi.waitFor(() =>
+ expect(harness.events).toContainEqual(
+ expect.objectContaining({
+ answer: "First answer.",
+ type: "submission-accepted",
+ }),
+ ),
+ );
+ expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce();
+ expect(harness.submitInterviewAnswer).toHaveBeenNthCalledWith(
+ 1,
+ expect.objectContaining({ text: "First answer." }),
+ );
+ expect(harness.events).toContainEqual({
+ answer: "Second answer.",
+ type: "transcript-retained",
+ });
+
+ harness.bridge.updateChat({
+ canAcceptInterviewAnswer: false,
+ canonicalSegments: [],
+ status: "streaming",
+ });
+ harness.bridge.updateChat({
+ canAcceptInterviewAnswer: true,
+ canonicalSegments: [
+ segment("first-reply", "Who acts next?", "submission-voice-1"),
+ ],
+ status: "ready",
+ });
+
+ await vi.waitFor(() =>
+ expect(harness.submitInterviewAnswer).toHaveBeenCalledTimes(2),
+ );
+ expect(harness.submitInterviewAnswer).toHaveBeenNthCalledWith(
+ 2,
+ expect.objectContaining({ text: "Second answer." }),
+ );
+ });
+
+ test("retains a half-duplex follow-up accepted before the first submission starts", async () => {
+ const harness = createHarness();
+ startReady(harness);
+ for (const itemId of ["first-item", "second-item"]) {
+ harness.emit({
+ connectionEpoch: 3,
+ itemId,
+ type: "input-speech-started",
+ });
+ harness.emit({
+ connectionEpoch: 3,
+ itemId,
+ type: "input-speech-stopped",
+ });
+ }
+
+ harness.emit(completedTranscript(3, "First answer.", "first-item"));
+ await vi.waitFor(() =>
+ expect(harness.events).toContainEqual(
+ expect.objectContaining({
+ answer: "First answer.",
+ type: "submission-accepted",
+ }),
+ ),
+ );
+ harness.emit(completedTranscript(3, "Second answer.", "second-item"));
+
+ expect(harness.events).toContainEqual({
+ answer: "Second answer.",
+ type: "transcript-retained",
+ });
+ expect(harness.events).not.toContainEqual({
+ itemId: "second-item",
+ reason: "unavailable",
+ type: "transcript-rejected",
+ });
+
+ harness.bridge.updateChat({
+ canAcceptInterviewAnswer: false,
+ canonicalSegments: [],
+ status: "streaming",
+ });
+ harness.bridge.updateChat({
+ canAcceptInterviewAnswer: true,
+ canonicalSegments: [
+ segment("first-reply", "Who acts next?", "submission-voice-1"),
+ ],
+ status: "ready",
+ });
+
+ await vi.waitFor(() =>
+ expect(harness.submitInterviewAnswer).toHaveBeenCalledTimes(2),
+ );
+ expect(harness.submitInterviewAnswer).toHaveBeenNthCalledWith(
+ 2,
+ expect.objectContaining({ text: "Second answer." }),
+ );
+ });
+
+ test("treats transcript completion as the stop boundary for a half-duplex follow-up", async () => {
+ const harness = createHarness();
+ startReady(harness);
+ harness.emit({
+ connectionEpoch: 3,
+ itemId: "first-item",
+ type: "input-speech-started",
+ });
+ harness.emit({
+ connectionEpoch: 3,
+ itemId: "first-item",
+ type: "input-speech-stopped",
+ });
+ harness.emit({
+ connectionEpoch: 3,
+ itemId: "second-item",
+ type: "input-speech-started",
+ });
+
+ harness.emit(completedTranscript(3, "First answer.", "first-item"));
+ await vi.waitFor(() =>
+ expect(harness.events).toContainEqual(
+ expect.objectContaining({
+ answer: "First answer.",
+ type: "submission-accepted",
+ }),
+ ),
+ );
+ harness.emit(completedTranscript(3, "Second answer.", "second-item"));
+
+ expect(harness.events).toContainEqual({
+ answer: "Second answer.",
+ type: "transcript-retained",
+ });
+ harness.bridge.updateChat({
+ canAcceptInterviewAnswer: false,
+ canonicalSegments: [],
+ status: "streaming",
+ });
+ harness.bridge.updateChat({
+ canAcceptInterviewAnswer: true,
+ canonicalSegments: [
+ segment("first-reply", "Who acts next?", "submission-voice-1"),
+ ],
+ status: "ready",
+ });
+ await vi.waitFor(() =>
+ expect(harness.submitInterviewAnswer).toHaveBeenCalledTimes(2),
+ );
+ expect(harness.submitInterviewAnswer).toHaveBeenNthCalledWith(
+ 2,
+ expect.objectContaining({ text: "Second answer." }),
+ );
+ });
+
+ test.each([false, true])(
+ "retires unfinished input across cancellation when pending answers are discarded: %s",
+ (discardPendingInterruption) => {
+ const harness = createHarness();
+ startReady(harness);
+ harness.emit({
+ connectionEpoch: 3,
+ interruptionBySpeaking: true,
+ itemId: "cancelled-item",
+ type: "input-speech-started",
+ });
+
+ harness.bridge.cancelPendingSpeech({ discardPendingInterruption });
+ harness.bridge.completeTurnHandoff();
+ harness.emit({
+ connectionEpoch: 3,
+ itemId: "fresh-item",
+ type: "input-speech-started",
+ });
+ harness.emit({
+ connectionEpoch: 3,
+ itemId: "fresh-item",
+ type: "input-speech-stopped",
+ });
+ harness.emit(completedTranscript(3, "A fresh answer.", "fresh-item"));
+
+ expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce();
+ expect(harness.submitInterviewAnswer).toHaveBeenCalledWith(
+ expect.objectContaining({ text: "A fresh answer." }),
+ );
+ },
+ );
+
+ test.each(["stop", "reconnect"] as const)(
+ "clears a retained interruption on %s",
+ (action) => {
+ const harness = createHarness();
+ startReady(harness);
+ harness.bridge.updateChat({
+ canAcceptInterviewAnswer: false,
+ canonicalSegments: [],
+ status: "streaming",
+ });
+ harness.emit({
+ type: "input-speech-started",
+ connectionEpoch: 3,
+ itemId: "pending",
+ interruptionBySpeaking: true,
+ });
+ harness.emit(completedTranscript(3, "Pending answer", "pending"));
+ expect(harness.events).toContainEqual({
+ type: "transcript-retained",
+ answer: "Pending answer",
+ });
+ if (action === "reconnect") harness.bridge.start(4);
+ else harness.bridge[action]();
+ harness.bridge.updateChat({
+ canAcceptInterviewAnswer: true,
+ canonicalSegments: [],
+ status: "ready",
+ });
+ expect(harness.submitInterviewAnswer).not.toHaveBeenCalled();
+ },
+ );
+
+ test.each(["stop-before-cancellation", "cancellation-before-stop"] as const)(
+ "preserves a retained interruption through host Stop when %s settles first",
+ async (order) => {
+ const harness = createHarness();
+ startReady(harness);
+ harness.emit(completedTranscript(3, "Initial answer", "initial"));
+ await vi.waitFor(() =>
+ expect(harness.events).toContainEqual(
+ expect.objectContaining({ type: "submission-accepted" }),
+ ),
+ );
+ harness.bridge.updateChat({
+ canAcceptInterviewAnswer: false,
+ canonicalSegments: [],
+ status: "streaming",
+ });
+ harness.emit({
+ type: "input-speech-started",
+ connectionEpoch: 3,
+ itemId: "pending",
+ interruptionBySpeaking: true,
+ });
+ harness.emit(completedTranscript(3, "Pending answer", "pending"));
+
+ harness.bridge.cancelPendingSpeech();
+ const stoppedChat = {
+ canAcceptInterviewAnswer: true,
+ canonicalSegments: [],
+ status: "ready" as const,
+ settlements: [
+ { outcome: "aborted" as const, submissionId: "submission-voice-1" },
+ ],
+ stopped: true,
+ };
+ if (order === "stop-before-cancellation") {
+ harness.bridge.updateChat(stoppedChat);
+ expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce();
+ harness.bridge.completeTurnHandoff();
+ } else {
+ harness.bridge.completeTurnHandoff();
+ expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce();
+ harness.bridge.updateChat(stoppedChat);
+ }
+
+ await vi.waitFor(() =>
+ expect(harness.submitInterviewAnswer).toHaveBeenCalledTimes(2),
+ );
+ expect(harness.submitInterviewAnswer).toHaveBeenLastCalledWith(
+ expect.objectContaining({ text: "Pending answer" }),
+ );
+ },
+ );
+
+ test("preserves a completed answer buffered behind input cancelled by host Stop", () => {
+ const harness = createHarness();
+ startReady(harness);
+ for (const itemId of ["first-item", "completed-item"]) {
+ harness.emit({
+ connectionEpoch: 3,
+ interruptionBySpeaking: true,
+ itemId,
+ type: "input-speech-started",
+ });
+ harness.emit({
+ connectionEpoch: 3,
+ itemId,
+ type: "input-speech-stopped",
+ });
+ }
+ harness.emit(completedTranscript(3, "Keep this answer.", "completed-item"));
+ expect(harness.submitInterviewAnswer).not.toHaveBeenCalled();
+
+ harness.bridge.cancelPendingSpeech();
+
+ expect(harness.events).toContainEqual({
+ answer: "Keep this answer.",
+ type: "transcript-retained",
+ });
+ expect(harness.events).not.toContainEqual({
+ itemId: "completed-item",
+ reason: "unavailable",
+ type: "transcript-rejected",
+ });
+
+ harness.bridge.completeTurnHandoff();
+ expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce();
+ expect(harness.submitInterviewAnswer).toHaveBeenCalledWith(
+ expect.objectContaining({ text: "Keep this answer." }),
+ );
+ });
+
+ test("discards a retained interruption for an explicit turn handoff", () => {
+ const harness = createHarness();
+ startReady(harness);
+ harness.bridge.updateChat({
+ canAcceptInterviewAnswer: false,
+ canonicalSegments: [],
+ status: "streaming",
+ });
+ harness.emit({
+ type: "input-speech-started",
+ connectionEpoch: 3,
+ itemId: "pending",
+ interruptionBySpeaking: true,
+ });
+ harness.emit(completedTranscript(3, "Stale pending answer", "pending"));
+
+ harness.bridge.cancelPendingSpeech({ discardPendingInterruption: true });
+ harness.bridge.completeTurnHandoff();
+ harness.bridge.updateChat({
+ canAcceptInterviewAnswer: true,
+ canonicalSegments: [],
+ status: "ready",
+ });
+
+ expect(harness.submitInterviewAnswer).not.toHaveBeenCalled();
+ });
+
+ test("releases generated output ownership after interruption before playback", async () => {
+ const harness = createHarness();
+ startReady(harness);
+ harness.emit({
+ type: "canonical-speech-requested",
+ connectionEpoch: 3,
+ speechRequestId: "speech-generated",
+ });
+ harness.emit({
+ type: "response-terminal",
+ connectionEpoch: 3,
+ playbackExpected: true,
+ responseId: "response-generated",
+ speechRequestId: "speech-generated",
+ status: "completed",
+ });
+ harness.emit({
+ type: "output-interrupted",
+ connectionEpoch: 3,
+ responseId: "response-generated",
+ speechRequestId: "speech-generated",
+ });
+
+ harness.emit({
+ type: "input-speech-started",
+ connectionEpoch: 3,
+ itemId: "ordinary-answer",
+ });
+ harness.emit(
+ completedTranscript(3, "A fresh ordinary answer", "ordinary-answer"),
+ );
+
+ await vi.waitFor(() =>
+ expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce(),
+ );
+ expect(harness.submitInterviewAnswer).toHaveBeenCalledWith(
+ expect.objectContaining({ text: "A fresh ordinary answer" }),
+ );
+ });
+
+ test("drains a retained interruption once the panel reopens voice input", async () => {
+ const harness = createHarness();
+ startReady(harness);
+ harness.bridge.updateChat({
+ canAcceptInterviewAnswer: false,
+ canonicalSegments: [],
+ status: "streaming",
+ });
+ harness.emit({
+ type: "input-speech-started",
+ connectionEpoch: 3,
+ itemId: "interruption",
+ interruptionBySpeaking: true,
+ });
+ harness.emit(
+ completedTranscript(3, "The auditor approves it.", "interruption"),
+ );
+
+ // The panel reports ready before it releases the queued voice input, so
+ // the first ready update cannot deliver the retained answer.
+ const heldSegments = [
+ segment("held", "Who signs it off?", "submission-held"),
+ ];
+ harness.bridge.updateChat({
+ canAcceptInterviewAnswer: false,
+ canonicalSegments: heldSegments,
+ status: "ready",
+ });
+ expect(harness.submitInterviewAnswer).not.toHaveBeenCalled();
+
+ harness.bridge.updateChat({
+ canAcceptInterviewAnswer: true,
+ canonicalSegments: heldSegments,
+ status: "ready",
+ });
+ await vi.waitFor(() =>
+ expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce(),
+ );
+ expect(harness.submitInterviewAnswer).toHaveBeenCalledWith(
+ expect.objectContaining({ text: "The auditor approves it." }),
+ );
+ expect(harness.session.speakCanonical).not.toHaveBeenCalled();
+ });
+
+ test("refuses ordinary capture while a submission is active", async () => {
+ const harness = createHarness();
+ startReady(harness);
+ harness.emit(completedTranscript(3));
+ await vi.waitFor(() =>
+ expect(
+ harness.events.some(({ type }) => type === "submission-accepted"),
+ ).toBe(true),
+ );
+ harness.bridge.updateChat({
+ canAcceptInterviewAnswer: false,
+ canonicalSegments: [],
+ status: "streaming",
+ });
+
+ harness.emit(completedTranscript(3, " ", "second-item"));
+
+ expect(harness.events).toContainEqual({
+ itemId: "second-item",
+ type: "transcript-rejected",
+ reason: "unavailable",
+ });
+ expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce();
+ });
+
+ test("retains only the first pending interruption and reports the extra utterance", () => {
+ const harness = createHarness();
+ startReady(harness);
+ harness.bridge.updateChat({
+ canAcceptInterviewAnswer: false,
+ canonicalSegments: [],
+ status: "streaming",
+ });
+ for (const itemId of ["first", "second"]) {
+ harness.emit({
+ type: "input-speech-started",
+ connectionEpoch: 3,
+ itemId,
+ interruptionBySpeaking: true,
+ });
+ harness.emit(completedTranscript(3, itemId, itemId));
+ }
+ expect(harness.events).toContainEqual({
+ itemId: "second",
+ type: "transcript-rejected",
+ reason: "pending",
+ });
+ harness.bridge.updateChat({
+ canAcceptInterviewAnswer: true,
+ canonicalSegments: [],
+ status: "ready",
+ });
+ expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce();
+ expect(harness.submitInterviewAnswer).toHaveBeenCalledWith(
+ expect.objectContaining({ text: "first" }),
+ );
+ });
+
test("rehydrates settled canonical speech without submission or playback", () => {
const harness = createHarness();
harness.bridge.updateChat({
@@ -194,6 +1093,7 @@ describe("RealtimeBrunchBridge", () => {
expect(harness.events).toContainEqual({
answer: "The supervisor approves it.",
deliveryId,
+ itemId: "user-item-1",
type: "submission-started",
});
expect(JSON.stringify(harness.events)).not.toContain("Fabricated answer");
@@ -220,6 +1120,7 @@ describe("RealtimeBrunchBridge", () => {
expect(harness.submitInterviewAnswer).not.toHaveBeenCalled();
expect(harness.events).toContainEqual({
+ itemId: "item-before-output",
reason: "unavailable",
type: "transcript-rejected",
});
@@ -270,6 +1171,7 @@ describe("RealtimeBrunchBridge", () => {
expect(harness.submitInterviewAnswer).not.toHaveBeenCalled();
expect(harness.events).toContainEqual({
+ itemId: "item-before-request",
reason: "unavailable",
type: "transcript-rejected",
});
@@ -356,6 +1258,7 @@ describe("RealtimeBrunchBridge", () => {
expect(harness.submitInterviewAnswer).not.toHaveBeenCalled();
expect(harness.events).toContainEqual({
+ itemId: "item-during-follow-on",
reason: "unavailable",
type: "transcript-rejected",
});
@@ -378,7 +1281,7 @@ describe("RealtimeBrunchBridge", () => {
);
});
- test("releases pending output ownership when cancellation settles before playback", async () => {
+ test("releases pending output ownership when creation is cancelled before playback", async () => {
const harness = createHarness();
startReady(harness);
harness.emit({
@@ -389,11 +1292,11 @@ describe("RealtimeBrunchBridge", () => {
harness.emit({
connectionEpoch: 3,
- responseId: "response-cancelled",
speechRequestId: "speech-cancelled",
+ playbackExpected: false,
status: "cancelled",
type: "response-terminal",
- } as OpenAIRealtimeSessionEvent);
+ });
harness.emit({
connectionEpoch: 3,
itemId: "item-after-cancellation",
@@ -475,6 +1378,7 @@ describe("RealtimeBrunchBridge", () => {
expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce(),
);
expect(harness.events).toContainEqual({
+ itemId: "user-item-1",
reason: "duplicate",
type: "transcript-rejected",
});
@@ -493,7 +1397,9 @@ describe("RealtimeBrunchBridge", () => {
harness.emit(completedTranscript(3, text));
expect(harness.submitInterviewAnswer).not.toHaveBeenCalled();
- expect(harness.events).toEqual([{ reason, type: "transcript-rejected" }]);
+ expect(harness.events).toEqual([
+ { itemId: "user-item-1", reason, type: "transcript-rejected" },
+ ]);
},
);
@@ -503,7 +1409,11 @@ describe("RealtimeBrunchBridge", () => {
harness.emit(failedTranscript(3, "failed-item"));
expect(harness.events).toEqual([
- { reason: "failed", type: "transcript-rejected" },
+ {
+ itemId: "failed-item",
+ reason: "failed",
+ type: "transcript-rejected",
+ },
]);
harness.emit(completedTranscript(3, "Retried answer.", "retry-item"));
@@ -527,7 +1437,11 @@ describe("RealtimeBrunchBridge", () => {
expect(harness.submitInterviewAnswer).not.toHaveBeenCalled();
expect(harness.events).toEqual([
- { reason: "unavailable", type: "transcript-rejected" },
+ {
+ itemId: "user-item-1",
+ reason: "unavailable",
+ type: "transcript-rejected",
+ },
]);
});
@@ -648,6 +1562,49 @@ describe("RealtimeBrunchBridge", () => {
);
});
+ test("drains a retained interruption when the completed submission has no canonical response", async () => {
+ const harness = createHarness();
+ startReady(harness, 7);
+ harness.emit(completedTranscript(7, "The silent answer."));
+ await vi.waitFor(() =>
+ expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce(),
+ );
+ harness.bridge.updateChat({
+ canAcceptInterviewAnswer: false,
+ canonicalSegments: [],
+ status: "submitted",
+ });
+ harness.emit({
+ connectionEpoch: 7,
+ interruptionBySpeaking: true,
+ itemId: "retained-item",
+ type: "input-speech-started",
+ });
+ harness.emit(
+ completedTranscript(7, "The retained answer.", "retained-item"),
+ );
+ expect(harness.events).toContainEqual({
+ answer: "The retained answer.",
+ type: "transcript-retained",
+ });
+
+ harness.bridge.updateChat({
+ canAcceptInterviewAnswer: true,
+ canonicalSegments: [],
+ settlements: [
+ { outcome: "completed", submissionId: "submission-voice-1" },
+ ],
+ status: "ready",
+ });
+
+ await vi.waitFor(() =>
+ expect(harness.submitInterviewAnswer).toHaveBeenCalledTimes(2),
+ );
+ expect(harness.submitInterviewAnswer).toHaveBeenLastCalledWith(
+ expect.objectContaining({ text: "The retained answer." }),
+ );
+ });
+
test("speaks a completed canonical segment while chat remains streaming and settles separately", async () => {
const harness = createHarness();
startReady(harness, 7);
diff --git a/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.ts b/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.ts
index 820b67c925e..ef5a7f2b41a 100644
--- a/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.ts
+++ b/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.ts
@@ -1,5 +1,13 @@
import { FlueChatAdmissionError } from "@hashintel/brunch-agent-transport-aisdk";
+import {
+ createVoiceRequestId,
+ reportVoiceDiagnostic,
+ voiceDurationMs,
+ type VoiceDiagnosticReporter,
+} from "../../../voice-diagnostics";
+import { classifyInterruption } from "./realtime-brunch-bridge/classify-interruption";
+
import type { CanonicalSpeechSegment } from "./canonical-speech";
import type {
OpenAIRealtimeSessionEvent,
@@ -22,6 +30,10 @@ export type VoiceSubmissionSettlement = Pick<
"outcome" | "submissionId"
>;
+export interface CancelPendingSpeechOptions {
+ readonly discardPendingInterruption?: boolean;
+}
+
interface ChatUpdate {
readonly canAcceptInterviewAnswer: boolean;
readonly canonicalSegments: CanonicalSpeechSegment[];
@@ -63,6 +75,7 @@ type SubmitInterviewAnswerResult =
});
interface RealtimeBrunchBridgeDependencies {
+ readonly reportDiagnostic?: VoiceDiagnosticReporter;
readonly session: RealtimeBridgeSession;
readonly submitInterviewAnswer: (
input: SubmitInterviewAnswerInput,
@@ -73,6 +86,22 @@ interface CompletedResponseMessage extends FlueChatResponseMessageCompletedEvent
consumed: boolean;
}
+type TerminalTranscriptEvent =
+ | {
+ readonly key: OpenAIRealtimeTranscriptKey;
+ readonly text: string;
+ readonly type: "completed";
+ }
+ | {
+ readonly key: OpenAIRealtimeTranscriptKey;
+ readonly type: "transcription-failed";
+ };
+
+interface PendingInputItem {
+ readonly ordinaryAcceptedWhileReady: boolean;
+ stopped: boolean;
+}
+
interface ActiveSubmission {
readonly abortController: AbortController;
readonly baselineSegmentIds: ReadonlySet;
@@ -105,12 +134,17 @@ export type RealtimeTranscriptRejectionReason =
| "empty"
| "failed"
| "over-limit"
+ | "pending"
+ | "prompt-regurgitation"
+ | "self-echo"
| "unavailable";
export type RealtimeBrunchBridgeEvent =
+ | { readonly answer: string; readonly type: "transcript-retained" }
| {
readonly answer: string;
readonly deliveryId: string;
+ readonly itemId: string;
readonly type: "submission-started";
}
| {
@@ -146,6 +180,7 @@ export type RealtimeBrunchBridgeEvent =
readonly type: "submission-stopped";
}
| {
+ readonly itemId: string;
readonly reason: RealtimeTranscriptRejectionReason;
readonly type: "transcript-rejected";
}
@@ -204,16 +239,30 @@ const admissionErrorCode = (
export class RealtimeBrunchBridge {
readonly #acceptedInputItemIds = new Set();
- readonly #activeOutputResponseIds = new Set();
+ readonly #activePlaybackText = new Map();
+ readonly #completedInputEvents = new Map();
+ readonly #inputItemOrder: string[] = [];
readonly #listeners = new Set();
+ readonly #pendingInputItems = new Map();
readonly #pendingSpeechRequestIds = new Set();
readonly #playbackOverlappingInputItemIds = new Set();
readonly #processedTranscripts = new Set();
+ readonly #reportDiagnostic: VoiceDiagnosticReporter;
readonly #session: RealtimeBridgeSession;
readonly #submitInterviewAnswer: (
input: SubmitInterviewAnswerInput,
) => Promise;
readonly #seenSegmentIds = new Set();
+ // null preserves enabled-mode admission without classifying ordinary capture.
+ readonly #interruptionPlaybackText = new Map<
+ string,
+ readonly string[] | null
+ >();
+ #pendingInterruption: {
+ answer: string;
+ deliveryId: string;
+ itemId: string;
+ } | null = null;
#activeEpoch: number | null = null;
#activeSubmission: ActiveSubmission | null = null;
#chat: ChatUpdate = {
@@ -225,9 +274,11 @@ export class RealtimeBrunchBridge {
#outputCancellationPending = false;
public constructor({
+ reportDiagnostic = reportVoiceDiagnostic,
session,
submitInterviewAnswer,
}: RealtimeBrunchBridgeDependencies) {
+ this.#reportDiagnostic = reportDiagnostic;
this.#session = session;
this.#submitInterviewAnswer = submitInterviewAnswer;
session.subscribe((event) => this.#handleSessionEvent(event));
@@ -238,17 +289,28 @@ export class RealtimeBrunchBridge {
return () => this.#listeners.delete(listener);
}
- public cancelPendingSpeech(): void {
+ public cancelPendingSpeech({
+ discardPendingInterruption = false,
+ }: CancelPendingSpeechOptions = {}): void {
this.#outputCancellationPending = true;
+ this.#retirePendingInputItems(discardPendingInterruption);
+ this.#interruptionPlaybackText.clear();
+ for (const responseId of this.#activePlaybackText.keys()) {
+ this.#activePlaybackText.set(responseId, []);
+ }
+ if (discardPendingInterruption) {
+ this.#pendingInterruption = null;
+ }
if (this.#activeSubmission) {
this.#activeSubmission.speechCancelled = true;
}
}
public completeTurnHandoff(): void {
- this.#activeOutputResponseIds.clear();
+ this.#activePlaybackText.clear();
this.#outputCancellationPending = false;
this.#pendingSpeechRequestIds.clear();
+ this.#drainPendingInterruption();
}
public notifyResponseMessageCompleted(
@@ -296,9 +358,14 @@ export class RealtimeBrunchBridge {
this.#activeEpoch = connectionEpoch;
this.#activeSubmission = null;
this.#acceptedInputItemIds.clear();
+ this.#completedInputEvents.clear();
+ this.#inputItemOrder.length = 0;
+ this.#interruptionPlaybackText.clear();
+ this.#pendingInputItems.clear();
+ this.#pendingInterruption = null;
this.#playbackOverlappingInputItemIds.clear();
this.#processedTranscripts.clear();
- this.#activeOutputResponseIds.clear();
+ this.#activePlaybackText.clear();
this.#outputCancellationPending = false;
this.#pendingSpeechRequestIds.clear();
this.#seenSegmentIds.clear();
@@ -313,9 +380,14 @@ export class RealtimeBrunchBridge {
this.#activeEpoch = null;
this.#activeSubmission = null;
this.#acceptedInputItemIds.clear();
+ this.#completedInputEvents.clear();
+ this.#inputItemOrder.length = 0;
+ this.#interruptionPlaybackText.clear();
+ this.#pendingInputItems.clear();
+ this.#pendingInterruption = null;
this.#playbackOverlappingInputItemIds.clear();
this.#processedTranscripts.clear();
- this.#activeOutputResponseIds.clear();
+ this.#activePlaybackText.clear();
this.#outputCancellationPending = false;
this.#pendingSpeechRequestIds.clear();
}
@@ -343,11 +415,12 @@ export class RealtimeBrunchBridge {
for (const segment of update.canonicalSegments) {
this.#seenSegmentIds.add(segment.id);
}
- return;
}
- if (update.status !== "ready") {
+ if (this.#outputCancellationPending) {
return;
}
+ if (this.#drainPendingInterruption()) return;
+ if (update.stopped || update.status !== "ready") return;
const newSegments = update.canonicalSegments.filter(
({ id }) => !this.#seenSegmentIds.has(id),
@@ -371,8 +444,11 @@ export class RealtimeBrunchBridge {
}
}
- #rejectTranscript(reason: RealtimeTranscriptRejectionReason): void {
- this.#emit({ reason, type: "transcript-rejected" });
+ #rejectTranscript(
+ itemId: string,
+ reason: RealtimeTranscriptRejectionReason,
+ ): void {
+ this.#emit({ itemId, reason, type: "transcript-rejected" });
}
#fail(
@@ -382,6 +458,9 @@ export class RealtimeBrunchBridge {
++this.#generation;
this.#activeSubmission?.abortController.abort();
this.#activeSubmission = null;
+ this.#pendingInterruption = null;
+ this.#interruptionPlaybackText.clear();
+ this.#activePlaybackText.clear();
this.#emit({ code, message, type: "error" });
}
@@ -389,6 +468,9 @@ export class RealtimeBrunchBridge {
++this.#generation;
this.#activeSubmission?.abortController.abort();
this.#activeSubmission = null;
+ this.#pendingInterruption = null;
+ this.#interruptionPlaybackText.clear();
+ this.#activePlaybackText.clear();
this.#emit({
code: admissionErrorCode(error.failure),
failure: error.failure,
@@ -405,35 +487,66 @@ export class RealtimeBrunchBridge {
return;
}
if (event.type === "input-speech-started") {
- if (this.#ownsOutputTurn()) {
+ if (event.interruptionBySpeaking) {
+ // The session has already sent output cancellation. Snapshot only text
+ // whose playback started, not queued speech or canonical chat history.
+ if (!this.#interruptionPlaybackText.has(event.itemId)) {
+ const activePlaybackText = Array.from(
+ this.#activePlaybackText.values(),
+ ).flat();
+ this.#interruptionPlaybackText.set(
+ event.itemId,
+ activePlaybackText.length > 0 ? activePlaybackText : null,
+ );
+ }
+ if (this.#activeSubmission) {
+ this.#activeSubmission.speechCancelled = true;
+ }
+ }
+ if (!event.interruptionBySpeaking && this.#ownsOutputTurn()) {
this.#playbackOverlappingInputItemIds.add(event.itemId);
} else {
this.#acceptedInputItemIds.add(event.itemId);
+ if (!this.#pendingInputItems.has(event.itemId)) {
+ this.#pendingInputItems.set(event.itemId, {
+ ordinaryAcceptedWhileReady:
+ !event.interruptionBySpeaking && this.#canSubmitAnswerNow(),
+ stopped: false,
+ });
+ this.#inputItemOrder.push(event.itemId);
+ }
+ }
+ return;
+ }
+ if (event.type === "input-speech-stopped") {
+ const pendingInput = this.#pendingInputItems.get(event.itemId);
+ if (this.#acceptedInputItemIds.has(event.itemId) && pendingInput) {
+ pendingInput.stopped = true;
}
return;
}
if (event.type === "canonical-speech-requested") {
this.#pendingSpeechRequestIds.add(event.speechRequestId);
- for (const itemId of this.#acceptedInputItemIds) {
- this.#playbackOverlappingInputItemIds.add(itemId);
- }
- this.#acceptedInputItemIds.clear();
+ this.#markPlaybackOverlappingInputItems();
return;
}
if (event.type === "output-started") {
this.#pendingSpeechRequestIds.delete(event.speechRequestId);
- this.#activeOutputResponseIds.add(event.responseId);
- for (const itemId of this.#acceptedInputItemIds) {
- this.#playbackOverlappingInputItemIds.add(itemId);
- }
- this.#acceptedInputItemIds.clear();
+ this.#activePlaybackText.set(event.responseId, event.canonicalText ?? []);
+ this.#markPlaybackOverlappingInputItems();
return;
}
if (
event.type === "output-stopped" ||
event.type === "output-interrupted"
) {
- this.#activeOutputResponseIds.delete(event.responseId);
+ if (
+ event.type === "output-interrupted" &&
+ event.speechRequestId !== undefined
+ ) {
+ this.#pendingSpeechRequestIds.delete(event.speechRequestId);
+ }
+ this.#activePlaybackText.delete(event.responseId);
return;
}
if (event.type === "response-terminal") {
@@ -451,44 +564,198 @@ export class RealtimeBrunchBridge {
if (event.key.connectionEpoch !== this.#activeEpoch) {
return;
}
+ const terminalEvent: TerminalTranscriptEvent =
+ event.type === "transcription-failed"
+ ? { key: event.key, type: "transcription-failed" }
+ : { key: event.key, text: event.text, type: "completed" };
const keyId = transcriptKeyId(event.key);
if (this.#processedTranscripts.has(keyId)) {
- this.#rejectTranscript("duplicate");
+ this.#rejectTranscript(event.key.itemId, "duplicate");
return;
}
this.#processedTranscripts.add(keyId);
+ if (this.#pendingInputItems.has(event.key.itemId)) {
+ this.#completedInputEvents.set(event.key.itemId, terminalEvent);
+ this.#drainCompletedInputEvents();
+ return;
+ }
+ this.#processCompletedInputEvent(terminalEvent);
+ }
+
+ #drainCompletedInputEvents(): void {
+ let itemId = this.#inputItemOrder.at(0);
+ let event =
+ itemId === undefined ? undefined : this.#completedInputEvents.get(itemId);
+ while (itemId !== undefined && event !== undefined) {
+ this.#inputItemOrder.shift();
+ this.#completedInputEvents.delete(itemId);
+ this.#processCompletedInputEvent(event);
+ itemId = this.#inputItemOrder.at(0);
+ event =
+ itemId === undefined
+ ? undefined
+ : this.#completedInputEvents.get(itemId);
+ }
+ }
+
+ #processCompletedInputEvent(event: TerminalTranscriptEvent): void {
this.#acceptedInputItemIds.delete(event.key.itemId);
+ const ordinaryInputWasAcceptedWhileReady = Boolean(
+ this.#pendingInputItems.get(event.key.itemId)?.ordinaryAcceptedWhileReady,
+ );
+ this.#pendingInputItems.delete(event.key.itemId);
+ const interruptionPlaybackText = this.#interruptionPlaybackText.get(
+ event.key.itemId,
+ );
+ this.#interruptionPlaybackText.delete(event.key.itemId);
if (this.#playbackOverlappingInputItemIds.has(event.key.itemId)) {
- this.#rejectTranscript("unavailable");
+ this.#rejectTranscript(event.key.itemId, "unavailable");
return;
}
if (event.type === "transcription-failed") {
- this.#rejectTranscript("failed");
+ this.#rejectTranscript(event.key.itemId, "failed");
return;
}
+
+ // An interruption, or ordinary speech that finished while admission was
+ // open, waits for an earlier spoken answer rather than losing speech-order
+ // authority to asynchronous transcription completion.
if (
- this.#activeSubmission ||
- !this.#chat.canAcceptInterviewAnswer ||
- this.#chat.status !== "ready"
+ interruptionPlaybackText === undefined &&
+ !ordinaryInputWasAcceptedWhileReady &&
+ !this.#canSubmitAnswerNow()
) {
- this.#rejectTranscript("unavailable");
+ this.#rejectTranscript(event.key.itemId, "unavailable");
return;
}
const answer = normalizeTranscript(event.text);
if (answer.length === 0) {
- this.#rejectTranscript("empty");
+ this.#rejectTranscript(event.key.itemId, "empty");
return;
}
if (Array.from(answer).length > ANSWER_LIMIT) {
- this.#rejectTranscript("over-limit");
+ this.#rejectTranscript(event.key.itemId, "over-limit");
return;
}
+ if (
+ interruptionPlaybackText !== undefined &&
+ interruptionPlaybackText !== null
+ ) {
+ const startedAt = performance.now();
+ const rejectionReason = classifyInterruption(
+ answer,
+ interruptionPlaybackText,
+ );
+ if (rejectionReason !== null) {
+ this.#reportDiagnostic({
+ durationMs: voiceDurationMs(startedAt, performance.now()),
+ operation: "transcription",
+ outcome: "rejected",
+ rejectionReason,
+ requestId: createVoiceRequestId(),
+ stage: "browser",
+ });
+ this.#rejectTranscript(event.key.itemId, rejectionReason);
+ return;
+ }
+ }
+
const deliveryId = createRealtimeSubmissionId(event.key);
+ if (this.#pendingInterruption) {
+ this.#rejectTranscript(event.key.itemId, "pending");
+ return;
+ }
+ if (!this.#canSubmitAnswerNow()) {
+ this.#pendingInterruption = {
+ answer,
+ deliveryId,
+ itemId: event.key.itemId,
+ };
+ this.#emit({ answer, type: "transcript-retained" });
+ return;
+ }
+ this.#submitAnswer(answer, deliveryId, event.key.itemId);
+ }
+
+ #ordinaryInputFinishedWhileReady(itemId: string): boolean {
+ const pendingInput = this.#pendingInputItems.get(itemId);
+ return Boolean(
+ pendingInput?.ordinaryAcceptedWhileReady &&
+ (pendingInput.stopped || this.#completedInputEvents.has(itemId)),
+ );
+ }
+
+ #retirePendingInputItems(discardCompletedInput: boolean): void {
+ for (const itemId of this.#pendingInputItems.keys()) {
+ if (!discardCompletedInput && this.#completedInputEvents.has(itemId)) {
+ continue;
+ }
+ this.#acceptedInputItemIds.delete(itemId);
+ this.#playbackOverlappingInputItemIds.add(itemId);
+ this.#pendingInputItems.delete(itemId);
+ const orderIndex = this.#inputItemOrder.indexOf(itemId);
+ if (orderIndex >= 0) {
+ this.#inputItemOrder.splice(orderIndex, 1);
+ }
+ if (this.#completedInputEvents.delete(itemId)) {
+ this.#rejectTranscript(itemId, "unavailable");
+ }
+ }
+ this.#drainCompletedInputEvents();
+ }
+
+ #markPlaybackOverlappingInputItems(): void {
+ for (const itemId of this.#acceptedInputItemIds) {
+ if (
+ this.#interruptionPlaybackText.has(itemId) ||
+ this.#ordinaryInputFinishedWhileReady(itemId)
+ ) {
+ continue;
+ }
+ this.#playbackOverlappingInputItemIds.add(itemId);
+ this.#acceptedInputItemIds.delete(itemId);
+ this.#pendingInputItems.delete(itemId);
+ const orderIndex = this.#inputItemOrder.indexOf(itemId);
+ if (orderIndex >= 0) {
+ this.#inputItemOrder.splice(orderIndex, 1);
+ }
+ if (this.#completedInputEvents.delete(itemId)) {
+ this.#rejectTranscript(itemId, "unavailable");
+ }
+ }
+ this.#drainCompletedInputEvents();
+ }
+
+ #canSubmitAnswerNow(): boolean {
+ return (
+ !this.#activeSubmission &&
+ !this.#outputCancellationPending &&
+ this.#chat.canAcceptInterviewAnswer &&
+ this.#chat.status === "ready"
+ );
+ }
+
+ #drainPendingInterruption(): boolean {
+ const pending = this.#pendingInterruption;
+ if (!pending) return false;
+ if (
+ this.#activeEpoch === null ||
+ this.#outputCancellationPending ||
+ !this.#canSubmitAnswerNow()
+ ) {
+ return true;
+ }
+ this.#pendingInterruption = null;
+ this.#submitAnswer(pending.answer, pending.deliveryId, pending.itemId);
+ return true;
+ }
+
+ #submitAnswer(answer: string, deliveryId: string, itemId: string): void {
const generation = this.#generation;
this.#activeSubmission = {
abortController: new AbortController(),
@@ -503,13 +770,13 @@ export class RealtimeBrunchBridge {
speechCancelled: false,
submissionId: null,
};
- this.#emit({ answer, deliveryId, type: "submission-started" });
+ this.#emit({ answer, deliveryId, itemId, type: "submission-started" });
void this.#submit(answer, deliveryId, generation);
}
#ownsOutputTurn(): boolean {
return (
- this.#activeOutputResponseIds.size > 0 ||
+ this.#activePlaybackText.size > 0 ||
this.#pendingSpeechRequestIds.size > 0
);
}
@@ -613,6 +880,7 @@ export class RealtimeBrunchBridge {
: "withheld",
type: "submission-stopped",
});
+ this.#drainPendingInterruption();
return;
}
// A reply may be written by the admitted submission itself or by a
@@ -700,6 +968,7 @@ export class RealtimeBrunchBridge {
segments: [],
type: "canonical-response-ready",
});
+ this.#drainPendingInterruption();
}
return;
}
@@ -742,6 +1011,7 @@ export class RealtimeBrunchBridge {
...(active.speechCancelled ? { speechCancelled: true as const } : {}),
type: "canonical-response-ready",
});
+ this.#drainPendingInterruption();
}
/**
@@ -768,5 +1038,6 @@ export class RealtimeBrunchBridge {
outcome: settlement.outcome,
type: "submission-stopped",
});
+ this.#drainPendingInterruption();
}
}
diff --git a/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge/classify-interruption.ts b/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge/classify-interruption.ts
new file mode 100644
index 00000000000..d2eeeba1a3d
--- /dev/null
+++ b/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge/classify-interruption.ts
@@ -0,0 +1,64 @@
+import { voiceTranscriptionPrompt } from "../../../../shared/voice-transcription";
+
+const tokensOf = (text: string): string[] =>
+ text
+ .normalize("NFKC")
+ .toLowerCase()
+ .replace(/[^\p{L}\p{M}\p{N}\s]/gu, " ")
+ .trim()
+ .split(/\s+/u)
+ .filter(Boolean);
+
+const bigramsOf = (tokens: readonly string[]): string[] => {
+ const bigrams: string[] = [];
+ let previous: string | undefined;
+ for (const token of tokens) {
+ if (previous !== undefined) bigrams.push(`${previous} ${token}`);
+ previous = token;
+ }
+ return bigrams;
+};
+
+const promptBigrams = bigramsOf(tokensOf(voiceTranscriptionPrompt));
+
+/** Require matching adjacent words in reference order, not a bag of vocabulary. */
+const hasStrongOrderedOverlap = (
+ candidate: readonly string[],
+ reference: readonly string[],
+): boolean => {
+ let matched = 0;
+ let referenceOffset = 0;
+ for (const bigram of candidate) {
+ const position = reference.indexOf(bigram, referenceOffset);
+ if (position !== -1) {
+ matched++;
+ referenceOffset = position + 1;
+ }
+ }
+ return matched / candidate.length >= 0.8;
+};
+
+/** Comparison normalization never changes the admitted user's words. */
+export const classifyInterruption = (
+ transcript: string,
+ canonicalPlaybackText: readonly string[],
+): "prompt-regurgitation" | "self-echo" | null => {
+ const tokens = tokensOf(transcript);
+ const canonicalPlaybackTokens = tokensOf(canonicalPlaybackText.join(" "));
+ if (
+ tokens.length > 0 &&
+ tokens.join(" ") === canonicalPlaybackTokens.join(" ")
+ ) {
+ return "self-echo";
+ }
+ // Short answers and isolated domain terms are not enough evidence of echo.
+ if (tokens.length < 6) return null;
+ const bigrams = bigramsOf(tokens);
+ if (tokens.length >= 8 && hasStrongOrderedOverlap(bigrams, promptBigrams)) {
+ return "prompt-regurgitation";
+ }
+ if (hasStrongOrderedOverlap(bigrams, bigramsOf(canonicalPlaybackTokens))) {
+ return "self-echo";
+ }
+ return null;
+};
diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx
index 632a4332269..17f7317a30a 100644
--- a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx
+++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx
@@ -19,6 +19,8 @@ import {
acknowledgeVoiceInterviewDisclosure,
isVoiceInterviewDisclosureAcknowledged,
loadOpenAIVoiceConfig,
+ readInterruptionBySpeakingPreference,
+ saveInterruptionBySpeakingPreference,
submitVoiceInputWithAdmission,
VOICE_INTERVIEW_DISCLOSURE_STORAGE_KEY,
VoiceInterviewControl,
@@ -670,3 +672,35 @@ describe("voice interview control", () => {
).toBe("acknowledged");
});
});
+
+describe("interruption by speaking preference", () => {
+ test("defaults on and remembers both settings across reads", () => {
+ const values = new Map();
+ const storage = {
+ getItem: (key: string) => values.get(key) ?? null,
+ setItem: (key: string, value: string) => {
+ values.set(key, value);
+ },
+ };
+ expect(readInterruptionBySpeakingPreference(storage)).toBe(true);
+ saveInterruptionBySpeakingPreference(false, storage);
+ expect(readInterruptionBySpeakingPreference(storage)).toBe(false);
+ saveInterruptionBySpeakingPreference(true, storage);
+ expect(readInterruptionBySpeakingPreference(storage)).toBe(true);
+ });
+ test("works when browser storage is unavailable", () => {
+ const storage = {
+ getItem: () => {
+ throw new Error("denied");
+ },
+ setItem: () => {
+ throw new Error("denied");
+ },
+ };
+ expect(readInterruptionBySpeakingPreference(storage)).toBe(true);
+ expect(() =>
+ saveInterruptionBySpeakingPreference(false, storage),
+ ).not.toThrow();
+ expect(readInterruptionBySpeakingPreference(null)).toBe(true);
+ });
+});
diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx
index 7094a09f9aa..ac362a52fdc 100644
--- a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx
+++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx
@@ -149,6 +149,36 @@ const getVoiceInterviewDisclosureStorage = (): Storage | null => {
}
};
+const interruptionBySpeakingStorageKey =
+ "petrinaut:interruption-by-speaking:v1";
+
+export const readInterruptionBySpeakingPreference = (
+ storage: Pick<
+ Storage,
+ "getItem"
+ > | null = getVoiceInterviewDisclosureStorage(),
+): boolean => {
+ try {
+ return storage?.getItem(interruptionBySpeakingStorageKey) !== "false";
+ } catch {
+ return true;
+ }
+};
+
+export const saveInterruptionBySpeakingPreference = (
+ enabled: boolean,
+ storage: Pick<
+ Storage,
+ "setItem"
+ > | null = getVoiceInterviewDisclosureStorage(),
+): void => {
+ try {
+ storage?.setItem(interruptionBySpeakingStorageKey, String(enabled));
+ } catch {
+ // The preference still applies to this session when storage is unavailable.
+ }
+};
+
export const isVoiceInterviewDisclosureAcknowledged = (
storage: Pick<
Storage,
@@ -485,6 +515,9 @@ const AvailableVoiceInterviewControl = ({
session,
submitText: (input) => latestSubmitVoiceInput(input),
});
+ controller.setInterruptionBySpeaking(
+ readInterruptionBySpeakingPreference(),
+ );
return {
bridge,
controller,
@@ -599,6 +632,10 @@ const AvailableVoiceInterviewControl = ({
resume: () => {
void store.controller.resume();
},
+ setInterruptionBySpeaking: (enabled) => {
+ store.controller.setInterruptionBySpeaking(enabled);
+ saveInterruptionBySpeakingPreference(enabled);
+ },
setMicrophoneMuted: (muted) =>
store.controller.setMicrophoneMuted(muted),
takeTurn: () => store.controller.takeTurn(),
diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts
index f0fb32056ad..93740ae72d7 100644
--- a/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts
+++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts
@@ -388,8 +388,7 @@ describe("controlled voice preview", () => {
dataChannel.receive({
content_index: 0,
item_id: "pre-output-item",
- transcript: "This completed before output started.",
- type: "conversation.item.input_audio_transcription.completed",
+ type: "conversation.item.input_audio_transcription.failed",
});
expect(controller.getSnapshot()).toMatchObject({
lastCommittedText: "",
@@ -634,7 +633,7 @@ describe("controlled voice preview", () => {
input: {
turn_detection: {
type: "semantic_vad",
- eagerness: "low",
+ eagerness: "medium",
create_response: false,
interrupt_response: false,
},
diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.test.ts
index 912282facab..4d5033a88e7 100644
--- a/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.test.ts
+++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.test.ts
@@ -16,6 +16,7 @@ const listeningSnapshot = {
errorRequestId: "",
input: "listening",
inputNotice: "none",
+ interruptionBySpeaking: false,
lastAnswerDelivery: "none",
lastCommittedText: "",
microphoneEnabled: true,
@@ -38,6 +39,7 @@ describe("toVoiceSessionState", () => {
canRepeatQuestion: false,
canTakeTurn: false,
errorMessage: null,
+ interruptionBySpeaking: false,
microphoneLevel: 0.24,
microphoneMuted: false,
notice: null,
diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.ts
index 5d8bfce2219..5a713fc2440 100644
--- a/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.ts
+++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.ts
@@ -97,6 +97,7 @@ export const toVoiceSessionState = ({
canReadFullResponse: snapshot.canReadFullResponse,
canRepeatQuestion: snapshot.canRepeatQuestion,
canTakeTurn: snapshot.canTakeTurn,
+ interruptionBySpeaking: snapshot.interruptionBySpeaking,
errorMessage:
snapshot.connection === "error" ? errorMessageOf(snapshot) : null,
microphoneMuted:
@@ -105,11 +106,15 @@ export const toVoiceSessionState = ({
!snapshot.microphoneEnabled,
microphoneLevel: snapshot.microphoneLevel,
notice:
- snapshot.inputNotice === "not-heard"
- ? "We didn't catch that. Please try again."
- : snapshot.inputNotice === "too-long"
- ? "That answer is too long. Please try a shorter response."
- : null,
+ snapshot.inputNotice === "answer-pending"
+ ? "Answer captured. Waiting for Brunch."
+ : snapshot.inputNotice === "answer-already-pending"
+ ? "Previous answer waiting. Please try again after it is sent."
+ : snapshot.inputNotice === "not-heard"
+ ? "We didn't catch that. Please try again."
+ : snapshot.inputNotice === "too-long"
+ ? "That answer is too long. Please try a shorter response."
+ : null,
phase: phaseOf(snapshot),
};
};
diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts
index 25abbf1a631..44f233ef891 100644
--- a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts
+++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts
@@ -8,8 +8,23 @@ import type { OpenAIRealtimeSessionEvent } from "./openai-realtime-session";
import type { RealtimeBrunchBridgeEvent } from "./realtime-brunch-bridge";
import type { VoiceLatencyEvent } from "./voice-turn-controller";
+type TestBridgeEvent =
+ | Exclude<
+ RealtimeBrunchBridgeEvent,
+ { type: "submission-started" | "transcript-rejected" }
+ >
+ | (Omit<
+ Extract,
+ "itemId"
+ > & { readonly itemId?: string })
+ | (Omit<
+ Extract,
+ "itemId"
+ > & { readonly itemId?: string });
+
const createHarness = () => {
let epoch = 0;
+ let latestInputItemId: string | null = null;
let now = 0;
let sessionListener:
| ((event: OpenAIRealtimeSessionEvent) => void)
@@ -19,6 +34,7 @@ const createHarness = () => {
cancelOutput: vi.fn<() => Promise>(async () => undefined),
connect: vi.fn(async () => ++epoch),
disconnect: vi.fn(async () => undefined),
+ setInterruptionBySpeaking: vi.fn(),
setMicrophoneEnabled: vi.fn(),
speakCanonical: vi.fn(),
subscribe: vi.fn(
@@ -59,9 +75,25 @@ const createHarness = () => {
},
bridge,
controller,
- emitBridge: (event: RealtimeBrunchBridgeEvent) => bridgeListener?.(event),
- emitSession: (event: OpenAIRealtimeSessionEvent) =>
- sessionListener?.(event),
+ emitBridge: (event: TestBridgeEvent) => {
+ if (
+ event.type === "submission-started" ||
+ event.type === "transcript-rejected"
+ ) {
+ bridgeListener?.({
+ ...event,
+ itemId: event.itemId ?? latestInputItemId ?? "test-input",
+ });
+ } else {
+ bridgeListener?.(event);
+ }
+ },
+ emitSession: (event: OpenAIRealtimeSessionEvent) => {
+ if (event.type === "input-speech-started") {
+ latestInputItemId = event.itemId;
+ }
+ sessionListener?.(event);
+ },
latencyEvents,
session,
submitText,
@@ -89,6 +121,35 @@ const markedQuestion = (
});
describe("VoiceTurnController", () => {
+ test("keeps interruption preference through end and reconnect and disables manual handover", async () => {
+ const harness = createHarness();
+ harness.controller.setInterruptionBySpeaking(true);
+ await harness.controller.start();
+ harness.controller.updateChat({
+ canAcceptInterviewAnswer: true,
+ canonicalSegments: [],
+ questionSegment: markedQuestion("question"),
+ status: "ready",
+ });
+ harness.emitSession({
+ type: "output-started",
+ connectionEpoch: 1,
+ responseId: "response",
+ speechRequestId: "speech",
+ });
+ await harness.controller.takeTurn();
+ expect(harness.session.cancelOutput).not.toHaveBeenCalled();
+ await harness.controller.reconnect();
+ expect(harness.controller.getSnapshot().interruptionBySpeaking).toBe(true);
+ await harness.controller.end();
+ await harness.controller.start();
+ expect(harness.controller.getSnapshot().interruptionBySpeaking).toBe(true);
+ harness.controller.setInterruptionBySpeaking(false);
+ expect(harness.session.setInterruptionBySpeaking).toHaveBeenLastCalledWith(
+ false,
+ );
+ });
+
test("records the content-free Voice lifecycle once in causal order", async () => {
const harness = createHarness();
await harness.controller.start();
@@ -402,6 +463,38 @@ describe("VoiceTurnController", () => {
});
});
+ test("preserves in-flight interruption text through queued canonical speech", async () => {
+ const harness = createHarness();
+ harness.controller.setInterruptionBySpeaking(true);
+ await harness.controller.start();
+ harness.emitSession({
+ connectionEpoch: 1,
+ interruptionBySpeaking: true,
+ itemId: "interruption",
+ type: "input-speech-started",
+ });
+ harness.emitSession({
+ key: { connectionEpoch: 1, contentIndex: 0, itemId: "interruption" },
+ text: "The supervisor",
+ type: "partial",
+ });
+
+ harness.emitSession({
+ connectionEpoch: 1,
+ speechRequestId: "speech-follow-on",
+ type: "canonical-speech-requested",
+ });
+ expect(harness.controller.getSnapshot().partialText).toBe("The supervisor");
+
+ harness.emitSession({
+ connectionEpoch: 1,
+ responseId: "response-follow-on",
+ speechRequestId: "speech-follow-on",
+ type: "output-started",
+ });
+ expect(harness.controller.getSnapshot().partialText).toBe("The supervisor");
+ });
+
test("offers handoff for canonical output without a question marker", async () => {
const harness = createHarness();
await harness.controller.start();
@@ -508,6 +601,9 @@ describe("VoiceTurnController", () => {
expect(repeatedHandoff).toBe(handoff);
expect(harness.bridge.cancelPendingSpeech).toHaveBeenCalledOnce();
+ expect(harness.bridge.cancelPendingSpeech).toHaveBeenCalledWith({
+ discardPendingInterruption: true,
+ });
expect(harness.session.cancelOutput).toHaveBeenCalledOnce();
expect(harness.session.setMicrophoneEnabled).toHaveBeenLastCalledWith(
false,
@@ -697,6 +793,47 @@ describe("VoiceTurnController", () => {
expect(harness.session.setMicrophoneEnabled).toHaveBeenLastCalledWith(true);
});
+ test("restores capture when generated speech is interrupted before playback", async () => {
+ const harness = createHarness();
+ harness.controller.setInterruptionBySpeaking(true);
+ await harness.controller.start();
+ harness.emitSession({
+ connectionEpoch: 1,
+ speechRequestId: "speech-generated",
+ type: "canonical-speech-requested",
+ });
+ harness.emitSession({
+ connectionEpoch: 1,
+ playbackExpected: true,
+ responseId: "response-generated",
+ speechRequestId: "speech-generated",
+ status: "completed",
+ type: "response-terminal",
+ });
+ harness.emitSession({
+ connectionEpoch: 1,
+ interruptionBySpeaking: true,
+ itemId: "interrupting-answer",
+ type: "input-speech-started",
+ });
+ harness.session.setMicrophoneEnabled.mockClear();
+
+ harness.emitSession({
+ connectionEpoch: 1,
+ responseId: "response-generated",
+ speechRequestId: "speech-generated",
+ type: "output-interrupted",
+ });
+
+ expect(harness.controller.getSnapshot()).toMatchObject({
+ input: "listening",
+ microphoneEnabled: true,
+ output: "interrupted",
+ });
+ expect(harness.session.setMicrophoneEnabled).toHaveBeenCalledOnce();
+ expect(harness.session.setMicrophoneEnabled).toHaveBeenCalledWith(true);
+ });
+
test("restores capture when cancelled settlement arrives after interrupted early speech", async () => {
const harness = createHarness();
await harness.controller.start();
@@ -1203,6 +1340,31 @@ describe("VoiceTurnController", () => {
expect(harness.session.setMicrophoneEnabled).toHaveBeenLastCalledWith(true);
});
+ test("returns to idle when canonical creation is cancelled before a response exists", async () => {
+ const harness = createHarness();
+ await harness.controller.start();
+ harness.emitSession({
+ connectionEpoch: 1,
+ speechRequestId: "speech-cancelled",
+ type: "canonical-speech-requested",
+ });
+
+ harness.emitSession({
+ connectionEpoch: 1,
+ playbackExpected: false,
+ speechRequestId: "speech-cancelled",
+ status: "cancelled",
+ type: "response-terminal",
+ });
+
+ expect(harness.controller.getSnapshot()).toMatchObject({
+ input: "listening",
+ microphoneEnabled: true,
+ output: "idle",
+ });
+ expect(harness.session.setMicrophoneEnabled).toHaveBeenLastCalledWith(true);
+ });
+
test("keeps capture closed when more canonical speech starts at settlement", async () => {
const harness = createHarness();
const finalSegment = markedQuestion("ask-final", "Who acts next?");
@@ -1701,6 +1863,106 @@ describe("VoiceTurnController", () => {
},
);
+ test.each(["prompt-regurgitation", "self-echo"] as const)(
+ "preserves a retained answer after a later %s rejection",
+ async (reason) => {
+ const harness = createHarness();
+ await harness.controller.start();
+ harness.emitBridge({
+ answer: "The retained answer",
+ type: "transcript-retained",
+ });
+
+ harness.emitBridge({ reason, type: "transcript-rejected" });
+
+ expect(harness.controller.getSnapshot()).toMatchObject({
+ inputNotice: "answer-pending",
+ partialText: "The retained answer",
+ });
+ },
+ );
+
+ test.each(["empty", "failed"] as const)(
+ "preserves a retained answer through a later %s transcript",
+ async (reason) => {
+ const harness = createHarness();
+ harness.controller.setInterruptionBySpeaking(true);
+ await harness.controller.start();
+ harness.emitBridge({
+ answer: "The retained answer",
+ type: "transcript-retained",
+ });
+ harness.emitSession({
+ connectionEpoch: 1,
+ interruptionBySpeaking: true,
+ itemId: "false-interruption",
+ type: "input-speech-started",
+ });
+ harness.emitSession({
+ key: {
+ connectionEpoch: 1,
+ contentIndex: 0,
+ itemId: "false-interruption",
+ },
+ text: "Provisional words",
+ type: "partial",
+ });
+
+ const key = {
+ connectionEpoch: 1,
+ contentIndex: 0,
+ itemId: "false-interruption",
+ };
+ if (reason === "failed") {
+ harness.emitSession({ key, type: "transcription-failed" });
+ harness.emitBridge({ reason, type: "transcript-rejected" });
+ } else {
+ harness.emitBridge({ reason, type: "transcript-rejected" });
+ harness.emitSession({ key, text: " ", type: "completed" });
+ }
+
+ expect(harness.controller.getSnapshot()).toMatchObject({
+ inputNotice: "answer-pending",
+ partialText: "The retained answer",
+ });
+ },
+ );
+
+ test("keeps a retained answer visible through a later rejected transcript", async () => {
+ const harness = createHarness();
+ harness.controller.setInterruptionBySpeaking(true);
+ await harness.controller.start();
+ harness.emitBridge({
+ answer: "The retained answer",
+ type: "transcript-retained",
+ });
+ harness.emitSession({
+ connectionEpoch: 1,
+ interruptionBySpeaking: true,
+ itemId: "false-echo",
+ type: "input-speech-started",
+ });
+ harness.emitSession({
+ key: { connectionEpoch: 1, contentIndex: 0, itemId: "false-echo" },
+ text: "Assistant echo",
+ type: "partial",
+ });
+ harness.emitBridge({
+ reason: "self-echo",
+ type: "transcript-rejected",
+ });
+ harness.emitSession({
+ key: { connectionEpoch: 1, contentIndex: 0, itemId: "false-echo" },
+ text: "Assistant echo",
+ type: "completed",
+ });
+
+ expect(harness.controller.getSnapshot()).toMatchObject({
+ inputNotice: "answer-pending",
+ partialText: "The retained answer",
+ });
+ });
+
test("keeps completed display transcripts until submission and rejects late events", async () => {
const harness = createHarness();
await harness.controller.start();
diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts
index d8298d8d523..d23483947ba 100644
--- a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts
+++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts
@@ -3,6 +3,7 @@ import { VoiceError, type VoiceErrorCode } from "../../../voice-diagnostics";
import type { CanonicalSpeechSegment } from "./canonical-speech";
import type { OpenAIRealtimeSessionEvent } from "./openai-realtime-session";
import type {
+ CancelPendingSpeechOptions,
RealtimeBridgeErrorCode,
RealtimeBrunchBridgeEvent,
VoiceSubmissionSettlement,
@@ -25,7 +26,12 @@ export type VoiceOutputState =
| "speaking"
| "interrupted";
export type VoiceAnswerDelivery = "none" | "pending" | "delivered" | "failed";
-export type VoiceInputNotice = "none" | "not-heard" | "too-long";
+export type VoiceInputNotice =
+ | "none"
+ | "not-heard"
+ | "too-long"
+ | "answer-pending"
+ | "answer-already-pending";
export interface VoiceTurnSnapshot {
readonly canReadFullResponse: boolean;
@@ -39,6 +45,7 @@ export interface VoiceTurnSnapshot {
readonly errorRequestId: string;
readonly input: VoiceInputState;
readonly inputNotice: VoiceInputNotice;
+ readonly interruptionBySpeaking: boolean;
readonly lastAnswerDelivery: VoiceAnswerDelivery;
readonly lastCommittedText: string;
readonly microphoneEnabled: boolean;
@@ -66,13 +73,14 @@ interface RealtimeSession {
cancelOutput(): Promise;
connect(): Promise;
disconnect(): Promise;
+ setInterruptionBySpeaking(enabled: boolean): void;
setMicrophoneEnabled(enabled: boolean): void;
speakCanonical(segments: CanonicalSpeechSegment[]): void;
subscribe(listener: (event: OpenAIRealtimeSessionEvent) => void): () => void;
}
interface RealtimeBridge {
- cancelPendingSpeech(): void;
+ cancelPendingSpeech(options?: CancelPendingSpeechOptions): void;
completeTurnHandoff(): void;
start(connectionEpoch: number): void;
stop(): void;
@@ -125,6 +133,7 @@ const initialSnapshot: VoiceTurnSnapshot = {
errorRequestId: "",
input: "paused",
inputNotice: "none",
+ interruptionBySpeaking: false,
lastAnswerDelivery: "none",
lastCommittedText: "",
microphoneEnabled: false,
@@ -184,6 +193,19 @@ export class VoiceTurnController {
bridge.subscribe((event) => this.#handleBridgeEvent(event));
}
+ public setInterruptionBySpeaking(enabled: boolean): void {
+ this.#session.setInterruptionBySpeaking(enabled);
+ this.#update({ interruptionBySpeaking: enabled });
+ if (
+ this.#snapshot.connection === "connected" &&
+ this.#snapshot.input !== "paused" &&
+ !this.#takingTurnPromise &&
+ !this.#outputCancellationPromise
+ ) {
+ this.#session.setMicrophoneEnabled(this.#snapshot.microphoneEnabled);
+ }
+ }
+
public getSnapshot(): VoiceTurnSnapshot {
return this.#snapshot;
}
@@ -296,7 +318,10 @@ export class VoiceTurnController {
this.#session.setMicrophoneEnabled(false);
const teardownPromise = this.#teardownPromise ?? this.#session.disconnect();
this.#teardownPromise = teardownPromise;
- this.#update({ ...initialSnapshot });
+ this.#update({
+ ...initialSnapshot,
+ interruptionBySpeaking: this.#snapshot.interruptionBySpeaking,
+ });
try {
await teardownPromise;
} finally {
@@ -364,9 +389,10 @@ export class VoiceTurnController {
if (
this.#takingTurnPromise === null &&
this.#outputCancellationPromise === null &&
- this.#activeSpeechResponseId === null &&
- (this.#snapshot.output === "idle" ||
- this.#snapshot.output === "interrupted")
+ (this.#snapshot.interruptionBySpeaking ||
+ (this.#activeSpeechResponseId === null &&
+ (this.#snapshot.output === "idle" ||
+ this.#snapshot.output === "interrupted")))
) {
this.#session.setMicrophoneEnabled(!muted);
}
@@ -486,10 +512,11 @@ export class VoiceTurnController {
*/
public takeTurn(): Promise {
if (this.#takingTurnPromise) return this.#takingTurnPromise;
- if (!this.#snapshot.canTakeTurn) return Promise.resolve();
+ if (this.#snapshot.interruptionBySpeaking || !this.#snapshot.canTakeTurn)
+ return Promise.resolve();
const generation = this.#generation;
- this.#bridge.cancelPendingSpeech();
+ this.#bridge.cancelPendingSpeech({ discardPendingInterruption: true });
this.#session.setMicrophoneEnabled(false);
this.#inputTurnPending = false;
this.#transcriptItemId = null;
@@ -569,36 +596,82 @@ export class VoiceTurnController {
if (event.type === "submission-started") {
this.#beginSubmissionSettlement(event.deliveryId);
const paused = this.#snapshot.input === "paused";
+ const preservePendingInput =
+ this.#transcriptItemId !== null &&
+ this.#transcriptItemId !== event.itemId;
if (paused) {
this.#inputStateOnResume = "submitting";
}
- this.#inputTurnPending = false;
+ if (!preservePendingInput) {
+ this.#inputTurnPending = false;
+ this.#transcriptItemId = null;
+ this.#transcriptKey = null;
+ }
this.#answerFinalizedAt = this.#now();
this.#latencyCorrelationId = event.deliveryId;
this.#recordedLatencyEvents.clear();
this.#submittingQuestionId = this.#currentQuestionId;
- this.#transcriptItemId = null;
- this.#transcriptKey = null;
this.#ttsSpeechRequestId = null;
- this.#session.setMicrophoneEnabled(false);
+ if (!preservePendingInput) {
+ this.#session.setMicrophoneEnabled(false);
+ }
this.#update({
input: paused ? "paused" : "submitting",
- inputNotice: "none",
+ inputNotice: preservePendingInput ? this.#snapshot.inputNotice : "none",
lastAnswerDelivery: "pending",
lastCommittedText: event.answer,
output: "waiting-for-tool",
- partialText: "",
+ partialText: preservePendingInput ? this.#snapshot.partialText : "",
+ });
+ return;
+ }
+ if (event.type === "transcript-retained") {
+ this.#update({
+ inputNotice: "answer-pending",
+ partialText: event.answer,
});
return;
}
if (event.type === "transcript-rejected") {
+ if (event.itemId !== this.#transcriptItemId) {
+ return;
+ }
if (event.reason === "duplicate" || event.reason === "unavailable") {
return;
}
+ if (
+ this.#snapshot.inputNotice === "answer-pending" &&
+ event.reason !== "pending"
+ ) {
+ this.#inputTurnPending = false;
+ this.#transcriptItemId = null;
+ this.#transcriptKey = null;
+ this.#update({});
+ return;
+ }
+ if (
+ event.reason === "prompt-regurgitation" ||
+ event.reason === "self-echo"
+ ) {
+ this.#inputTurnPending = false;
+ this.#transcriptItemId = null;
+ this.#transcriptKey = null;
+ this.#update(
+ this.#snapshot.inputNotice === "answer-pending"
+ ? {}
+ : { inputNotice: "none", partialText: "" },
+ );
+ return;
+ }
this.#transcriptItemId = null;
this.#transcriptKey = null;
this.#update({
- inputNotice: event.reason === "over-limit" ? "too-long" : "not-heard",
+ inputNotice:
+ event.reason === "pending"
+ ? "answer-already-pending"
+ : event.reason === "over-limit"
+ ? "too-long"
+ : "not-heard",
partialText: "",
});
return;
@@ -697,11 +770,27 @@ export class VoiceTurnController {
}
if (event.type === "canonical-speech-requested") {
this.#pendingSpeechRequestIds.add(event.speechRequestId);
- this.#session.setMicrophoneEnabled(false);
- this.#inputTurnPending = false;
- this.#transcriptItemId = null;
- this.#transcriptKey = null;
- this.#update({ output: "waiting-for-tool", partialText: "" });
+ if (
+ this.#snapshot.interruptionBySpeaking &&
+ this.#snapshot.input !== "paused" &&
+ !this.#takingTurnPromise &&
+ !this.#outputCancellationPromise
+ ) {
+ this.#session.setMicrophoneEnabled(this.#snapshot.microphoneEnabled);
+ }
+ if (!this.#snapshot.interruptionBySpeaking) {
+ this.#session.setMicrophoneEnabled(false);
+ this.#inputTurnPending = false;
+ this.#transcriptItemId = null;
+ this.#transcriptKey = null;
+ }
+ this.#update({
+ output: "waiting-for-tool",
+ partialText:
+ this.#snapshot.interruptionBySpeaking && this.#inputTurnPending
+ ? this.#snapshot.partialText
+ : "",
+ });
if (
this.#latencyCorrelationId !== null &&
this.#ttsSpeechRequestId === null
@@ -717,15 +806,23 @@ export class VoiceTurnController {
this.#activeSpeechResponseId = event.responseId;
this.#activeSpeechResponseTerminal =
this.#terminalSpeechRequestIds.delete(event.speechRequestId);
- this.#inputTurnPending = false;
- this.#transcriptItemId = null;
- this.#transcriptKey = null;
+ if (!this.#snapshot.interruptionBySpeaking) {
+ this.#inputTurnPending = false;
+ this.#transcriptItemId = null;
+ this.#transcriptKey = null;
+ }
if (this.#snapshot.input === "paused") {
void this.#cancelOutput();
this.#update({ output: "interrupted", partialText: "" });
return;
}
- this.#update({ output: "speaking", partialText: "" });
+ this.#update({
+ output: "speaking",
+ partialText:
+ this.#snapshot.interruptionBySpeaking && this.#inputTurnPending
+ ? this.#snapshot.partialText
+ : "",
+ });
if (
this.#latencyCorrelationId !== null &&
event.speechRequestId === this.#ttsSpeechRequestId
@@ -753,7 +850,21 @@ export class VoiceTurnController {
return;
}
if (event.type === "output-interrupted") {
- if (event.responseId !== this.#activeSpeechResponseId) return;
+ const interruptedPendingSpeech =
+ event.speechRequestId !== undefined &&
+ this.#pendingSpeechRequestIds.delete(event.speechRequestId);
+ if (event.speechRequestId !== undefined) {
+ this.#terminalSpeechRequestIds.delete(event.speechRequestId);
+ }
+ if (event.responseId !== this.#activeSpeechResponseId) {
+ if (interruptedPendingSpeech) {
+ this.#update({
+ output: this.#outputAfterPlaybackEnds("interrupted"),
+ });
+ this.#restoreMicrophoneIfCaptureAvailable();
+ }
+ return;
+ }
this.#activeSpeechOutputEnded = true;
if (this.#activeSpeechResponseTerminal) {
this.#clearSettledSpeech();
@@ -767,15 +878,27 @@ export class VoiceTurnController {
if (event.type === "input-speech-started") {
if (
this.#takingTurnPromise ||
- this.#snapshot.output === "speaking" ||
+ this.#snapshot.input === "paused" ||
+ (!event.interruptionBySpeaking &&
+ this.#snapshot.output === "speaking") ||
this.#snapshot.output === "cancelling"
) {
return;
}
+ if (event.interruptionBySpeaking) {
+ this.#activeSpeechResponseId = null;
+ this.#activeSpeechOutputEnded = false;
+ this.#activeSpeechResponseTerminal = false;
+ this.#update({ output: "interrupted" });
+ }
this.#inputTurnPending = true;
this.#transcriptItemId = event.itemId;
this.#transcriptKey = null;
- this.#update({ inputNotice: "none", partialText: "" });
+ this.#update(
+ this.#snapshot.inputNotice === "answer-pending"
+ ? {}
+ : { inputNotice: "none", partialText: "" },
+ );
return;
}
if (event.type === "response-terminal") {
@@ -820,12 +943,17 @@ export class VoiceTurnController {
this.#inputTurnPending = false;
this.#transcriptItemId = null;
this.#transcriptKey = null;
- this.#update({ partialText: "" });
+ this.#update(
+ this.#snapshot.inputNotice === "answer-pending"
+ ? {}
+ : { partialText: "" },
+ );
return;
}
if (this.#transcriptKey !== null && this.#transcriptKey !== key) return;
this.#transcriptKey = key;
if (event.type === "partial") {
+ if (this.#snapshot.inputNotice === "answer-pending") return;
this.#update({
partialText: `${this.#snapshot.partialText}${event.text}`,
});
@@ -834,9 +962,11 @@ export class VoiceTurnController {
this.#inputTurnPending = false;
this.#transcriptItemId = null;
this.#transcriptKey = null;
- this.#update({
- partialText: event.text.trim() || this.#snapshot.partialText,
- });
+ this.#update(
+ this.#snapshot.inputNotice === "answer-pending"
+ ? {}
+ : { partialText: event.text.trim() || this.#snapshot.partialText },
+ );
}
#setError(
diff --git a/apps/petrinaut-website/src/server/voice/openai-realtime-call.test.ts b/apps/petrinaut-website/src/server/voice/openai-realtime-call.test.ts
index c7f4f94c284..b463d2ebb55 100644
--- a/apps/petrinaut-website/src/server/voice/openai-realtime-call.test.ts
+++ b/apps/petrinaut-website/src/server/voice/openai-realtime-call.test.ts
@@ -138,7 +138,7 @@ describe("OpenAI Realtime call handler", () => {
transcription: { model: "gpt-4o-transcribe", language: "en" },
turn_detection: {
type: "semantic_vad",
- eagerness: "low",
+ eagerness: "medium",
create_response: false,
interrupt_response: false,
},
diff --git a/apps/petrinaut-website/src/server/voice/openai-voice-policy.test.ts b/apps/petrinaut-website/src/server/voice/openai-voice-policy.test.ts
index eef273d2b16..6b8dae58653 100644
--- a/apps/petrinaut-website/src/server/voice/openai-voice-policy.test.ts
+++ b/apps/petrinaut-website/src/server/voice/openai-voice-policy.test.ts
@@ -109,7 +109,7 @@ When Petrinaut supplies response_text, speak only those strings, in array order
},
turn_detection: {
type: "semantic_vad",
- eagerness: "low",
+ eagerness: "medium",
create_response: false,
interrupt_response: false,
},
diff --git a/apps/petrinaut-website/src/server/voice/openai-voice-policy.ts b/apps/petrinaut-website/src/server/voice/openai-voice-policy.ts
index 25e2a13bca5..41caa881b89 100644
--- a/apps/petrinaut-website/src/server/voice/openai-voice-policy.ts
+++ b/apps/petrinaut-website/src/server/voice/openai-voice-policy.ts
@@ -1,3 +1,5 @@
+import { voiceTranscriptionPrompt } from "../../shared/voice-transcription.js";
+
export const OPENAI_REALTIME_CONNECTION_TIMEOUT_MS = 15_000;
export const OPENAI_REALTIME_POLICY_VERSION = "brunch-control-plane-v3";
@@ -59,12 +61,11 @@ export const createOpenAIRealtimeSession = () => ({
transcription: {
model: "gpt-4o-transcribe",
language: "en",
- prompt:
- "Expect English process-modeling vocabulary including SDCPN, stochastic Petri net, place, transition, arc, token, marking, guard, rate, distribution, parameter, subnet, scenario, and metric.",
+ prompt: voiceTranscriptionPrompt,
},
turn_detection: {
type: "semantic_vad" as const,
- eagerness: "low" as const,
+ eagerness: "medium" as const,
create_response: false,
interrupt_response: false,
},
diff --git a/apps/petrinaut-website/src/shared/voice-transcription.ts b/apps/petrinaut-website/src/shared/voice-transcription.ts
new file mode 100644
index 00000000000..65492107f69
--- /dev/null
+++ b/apps/petrinaut-website/src/shared/voice-transcription.ts
@@ -0,0 +1,3 @@
+/** Shared by provider configuration and local completed-transcript admission. */
+export const voiceTranscriptionPrompt =
+ "Expect English process-modeling vocabulary including SDCPN, stochastic Petri net, place, transition, arc, token, marking, guard, rate, distribution, parameter, subnet, scenario, and metric.";
diff --git a/apps/petrinaut-website/src/voice-diagnostics.ts b/apps/petrinaut-website/src/voice-diagnostics.ts
index be74ef0b911..2b6f2d4b1ee 100644
--- a/apps/petrinaut-website/src/voice-diagnostics.ts
+++ b/apps/petrinaut-website/src/voice-diagnostics.ts
@@ -18,7 +18,8 @@ export interface VoiceDiagnosticEvent {
readonly durationMs: number;
readonly errorCode?: VoiceErrorCode;
readonly operation: VoiceOperation;
- readonly outcome: "success" | "failure" | "aborted";
+ readonly outcome: "success" | "failure" | "aborted" | "rejected";
+ readonly rejectionReason?: "prompt-regurgitation" | "self-echo";
readonly requestId: string;
readonly stage: "browser" | "playback" | "server";
readonly status?: number;
diff --git a/libs/@hashintel/brunch-agent/MISSION.md b/libs/@hashintel/brunch-agent/MISSION.md
index b463d16025a..28e89831830 100644
--- a/libs/@hashintel/brunch-agent/MISSION.md
+++ b/libs/@hashintel/brunch-agent/MISSION.md
@@ -1,120 +1,191 @@
-# Brunch architecture-check ownership
+# Voice interruption by speaking
## Status
**Live as of 2026-09-09** for
-[SRE-1010](https://linear.app/hash/issue/SRE-1010/move-brunch-architecture-checks-out-of-core-unit-tests)
-on `ln/sre-1010-move-brunch-checks`. This file is the branch's sole execution authority.
+[FE-1604](https://linear.app/hash/issue/FE-1604/allow-voice-interruption-by-speaking)
+on `kostandin/fe-1604-recut-voice-interruption`, cut from post-deployment `main` at
+`ef0f4449876d63d82657147fb4e29cdf024e9f79`.
+
+This is an independent semantic recut of the interruption-only delta from the
+stale, conflicting [PR #9550](https://github.com/hashintel/hash/pull/9550) head
+`f69ac17034dfe4290691d34b64930e5a07245480`. It must not merge the old branch or
+carry its unrelated Brunch stack. The open settlement port in
+[PR #9588](https://github.com/hashintel/hash/pull/9588) is a separate sibling and
+is not part of this branch.
+
+On 2026-09-10 the owner explicitly approved closing this interruption-only
+recut without the production Voice/Stop/second-tab continuity witness. The
+installed Flue 2.0.3 contract cannot durably project direct-user Voice origin,
+and FE-1604 does not recut that transport or history boundary. This is a scoped
+deferral, not continuity evidence: the post-FE-1604 re-entry gate remains in
+`MISSION.next.md`.
+
+## Supplemental FE-1580 settlement follow-up
+
+**Live as of 2026-09-08** for
+[PR #9588](https://github.com/hashintel/hash/pull/9588) on
+`kostandin/fe-1580-port-voice-settlement-fixes`, based directly on current
+`main` after #9564 and #9537 merged. This supplement preserves the accepted
+Voice contract without changing the CORS authority in this file.
+
+- **Imperative:** semantically port the omitted #9531 commit `9415e1b007`;
+ release silent Voice ownership and settle completed submissions without
+ canonical prose. Preserve failed durable Stop errors and remove the stale
+ browser `brunch_ask` catalogue entry.
+- **Throughline:** OpenAI terminal output → session/bridge/controller ownership;
+ correlated Brunch settlement → next Voice turn; panel Stop rejection →
+ deferred browser-tool termination; shared browser catalogue →
+ transport/history.
+- **Proof:** donor session/bridge/controller and preview regressions; panel DOM
+ tests for persistent Stop failure and withheld continuation; catalogue and
+ fixture tests; focused unit, build, TypeScript, ESLint and formatting checks.
+ These tests establish local settlement behavior, not paid-provider behavior,
+ audible latency or a new microphone witness.
+- **Constraints:** preserve current `main`'s accepted Voice and CORS joins; no
+ #9538 grounding, #9550 VAD/interruption, snapshot-overlay or provenance
+ rollback work, generic interactive tools, obsolete shim, or `brunch_ask`
+ restoration.
+- **Stop or reorient:** stop if the port erases errors, releases unrelated
+ playback, revives withheld tools, weakens the CORS policy, or disturbs other
+ work.
## Imperative
-Remove static-policy source walkers from Brunch unit tests and let each existing enforcement
-mechanism own the contract it can prove: Oxlint owns authored imports, Yarn owns manifests, and
-behavioral tests own emitted behavior. Do this now because the SRE-1007 hotfix modeled sibling
-packages and the app as inputs to core, creating a reverse `libs -> apps` edge and requiring
-downstream trees in a core prune.
+Let a person interrupt Voice assistant playback by speaking without losing the
+interrupting utterance. Keep the existing **Your turn** handoff as a
+browser-saved half-duplex fallback, and reject likely prompt regurgitation or
+assistant self-echo before a completed interruption transcript becomes an
+answer.
+
+The interruption must stop playback immediately while preserving completed
+transcription as the sole answer authority and the existing Brunch admission
+path as the sole submission authority.
## Throughline
```text
-authored dependency-boundary violation
-→ the owning workspace's package-local Oxlint configuration
-→ an AST-backed lint diagnostic in the existing package lint task
-
-forbidden manifest dependency
-→ the repository's Yarn constraints
-→ install and Global constraint lint fail
-
-change in apps/brunch-agent
-→ @apps/brunch-agent#test:unit
-→ the built artifact must contain every declared agent registration
-
-change in Brunch core
-→ @hashintel/brunch-agent#test:unit
-→ core-local tests and utilities, without context-root, app, or sibling-package inputs
+OpenAI Realtime microphone input remains enabled during canonical playback
+→ input_audio_buffer.speech_started
+→ response.cancel + output_audio_buffer.clear, without input_audio_buffer.clear
+→ completed transcription for the same input item
+→ interruption-only prompt-regurgitation and active-playback self-echo checks
+→ retain while the previous Brunch turn settles, if necessary
+→ existing Voice bridge and panel admission path exactly once
+→ canonical Brunch turn and ordinary Voice lifecycle
```
-No replacement family scanner is introduced. Assertions that restate package manifests, filenames,
-source strings, review inventories, or tool configuration are deleted when the existing tool,
-compiler, build, or review is already authoritative.
+The playback menu owns a default-on **Interruption by speaking** preference.
+Disabling it restores the existing half-duplex microphone closure and
+acknowledged **Your turn** handoff.
## Proof
-This mission establishes native enforcement ownership and prune closure. It does **not** add a new
-policy surface merely to preserve every historical assertion.
-
-1. **Import boundaries use existing lint infrastructure.** Each Brunch workspace's existing
- `no-restricted-imports` policy catches forbidden authored imports using Oxlint's parser. Oracle:
- representative negative lint probes in the owning packages.
-2. **Manifest boundaries use existing repository infrastructure.** The Brunch transport Yarn
- constraint continues to reject forbidden runtime edges. Oracle: `yarn constraints` and a
- representative negative manifest probe.
-3. **Agent registration is behavioral.** The app build-artifact test proves every declared agent
- reaches the emitted registration bundle; source-walking directive and filename checks are
- removed. Oracle: `yarn workspace @apps/brunch-agent test:unit`.
-4. **A core prune owns no downstream or context-root Brunch tree.** The core task has no app,
- sibling-package, or context-root input glob, and requesting only core no longer adds them through
- prune exceptions. The Linear graph utility and its tests are co-located in core. Oracle:
- `.github/actions/prune-repository/prune_test.py` plus inspection of
- `turbo run test:unit --filter @hashintel/brunch-agent --dry=json`.
-5. **No bespoke checker remains.** The temporary architecture workspace and repo-chores command,
- task, scanner, CI step, and coverage constraint are absent. Oracle: repository search plus
- relevant package lint, typecheck, tests, constraints, and formatting.
-
-## Constraints
-
-- Keep Yarn constraints and package-local Oxlint rules with their current owning packages; do not
- duplicate them in tests or repo-chores.
-- Preserve behavioral tests that can fail while source and manifests remain unchanged.
-- Delete static assertions whose only oracle is a hard-coded mirror of source, package metadata,
- review inventory, or file layout.
-- Move the Linear graph utility into core so its real unit tests use static imports and never skip
- based on checkout shape.
-- Never make a governed workspace depend on a checker or recreate a family-wide source walker.
-- No implementation begins until this authority cut is committed separately. Material changes to
- this contract require owner review and another focused authority commit.
+1. **Immediate, input-preserving cancellation.** Session tests observe
+ `speech_started → response.cancel → output_audio_buffer.clear`, no input
+ buffer clear, and completion of the same input item.
+2. **Exactly-once admission.** Bridge and controller tests cover duplicate
+ completions, delayed Brunch admission, an unsettled previous turn, follow-on
+ canonical speech, queued playback, and lifecycle cleanup.
+3. **Local false-transcript rejection.** Tests cover configured transcription
+ prompt regurgitation and exact active canonical self-echo, while preserving
+ short novel answers and leaving ordinary non-interruption capture unchanged.
+4. **Retained-answer visibility.** Controller tests prove that later empty,
+ failed, prompt-regurgitated, or self-echo transcripts cannot erase an
+ earlier retained answer or submit a replacement.
+5. **User control.** Shared Petrinaut tests prove the preference is default-on,
+ browser-saved, exposed in the existing playback menu, and controls whether
+ **Your turn** is visible.
+6. **Continuity projection guard.**
+ `local-storage-demo/voice-history-continuity.integration.test.tsx` mounts the
+ real history projector and Petrinaut panel against prepared before/after
+ observations, then remounts the observer. It proves typed history, supported
+ Voice client-tool attribution, aborted-settlement rendering, and the local
+ **Exit voice mode** versus injected **Stop** port remain distinct at those
+ component boundaries. It does not prove production Voice provenance
+ creation, `requestFlueStop`/Flue abort persistence, a fresh Flue client or
+ second browser tab, or direct-user Voice source reconstruction after reopen.
+ The owner-approved FE-1604 deferral above leaves those claims open rather
+ than treating this prepared fixture as a substitute.
+7. **Package integrity.** Focused Voice unit tests, Petrinaut unit tests,
+ TypeScript checks, ESLint, the website and library builds, architecture-doc
+ lint, repository formatting, and `git diff --check` distinguish a working
+ recut from code presence alone.
+
+Mocked protocol tests establish event ordering and state behavior; they do not
+establish real microphone latency, speaker echo cancellation, or acoustic
+classifier accuracy.
### Expected touched paths
```text
-~ libs/@hashintel/brunch-agent/MISSION.md
-~ apps/brunch-agent/test/build-artifact.test.ts
-- apps/brunch-agent/test/architecture/*
-> libs/@hashintel/brunch-agent/scripts/linear-project-graph.ts
- -> libs/@hashintel/brunch-agent/packages/core/src/linear-project-graph.ts
-~ libs/@hashintel/brunch-agent/packages/core/test/architecture/linear-project-graph.test.ts
-- libs/@hashintel/brunch-agent/packages/core/test/architecture/context-root.ts
-- libs/@hashintel/brunch-agent/packages/core/test/architecture/open-gaps*
-~ libs/@hashintel/brunch-agent/packages/core/package.json
-~ libs/@hashintel/brunch-agent/packages/core/turbo.json
-~ apps/brunch-agent/turbo.json
-~ .github/actions/prune-repository/prune.py
-~ .github/actions/prune-repository/prune_test.py
-~ yarn.lock
+~ apps/petrinaut-website/src/main/app/voice-interview/ session, bridge, controller, preference, tests
+~ apps/petrinaut-website/src/main/app/local-storage-demo/ history projection and remount guard
+~ apps/petrinaut-website/src/server/voice/ Realtime VAD and transcription policy
+~ apps/petrinaut-website/src/shared/ shared transcription vocabulary
+~ apps/petrinaut-website/README.md website behavior
+~ libs/@hashintel/petrinaut/ shared state, playback control, user guide
+~ libs/@hashintel/brunch-agent/docs/adr/ Voice turn-shell decision
++ .changeset/ Petrinaut patch release note
```
+## Constraints
+
+- A completed provider transcription is the only Voice-answer authority.
+ Provisional text remains display-only.
+- Interruption cancellation is immediate and is never gated on transcript
+ classification. It clears output, never the interrupting input buffer.
+- Only input that began while canonical playback was active is classified as an
+ interruption. Ordinary capture behavior remains unchanged.
+- Self-echo compares only with the exact canonical text active when speech
+ started, not queued speech or conversation history.
+- Comparison may normalize Unicode, case, punctuation, and whitespace, but
+ admitted wording, casing, and punctuation remain unchanged.
+- Rejection diagnostics contain operational metadata and a reason, never the
+ transcript, transcription prompt, or assistant text.
+- Short novel answers such as “stop”, “no”, and “wait” remain admissible.
+- Mute, pause, Stop, end, reconnect, exact question replay, and exact full
+ response replay retain their current behavior in both preference modes.
+- Keep FE-1604 independent from PR #9588. If that sibling lands, update from
+ `main` and resolve overlap semantically instead of importing its branch.
+- Update the Petrinaut user guide and retain exactly one Petrinaut patch
+ changeset.
+
## Fog-line
-- If an import boundary lacks package-local lint coverage, extend the owning workspace's Oxlint
- rule rather than introducing a cross-workspace scanner.
-- Some app unit tests independently read non-workspace Brunch evaluation and documentation assets.
- This mission removes family architecture ownership from the app but does not remove inputs those
- product tests still genuinely consume.
-- The non-blocking review identifies similar tautological tests outside this architecture cluster.
- They are evidence of the same defect pattern, but broad product-test cleanup is deferred unless a
- touched test blocks this ownership change.
+- Browser echo cancellation may still allow speaker feedback to trigger VAD or
+ transcription. A deterministic completed-transcript classifier reduces false
+ admission but cannot prevent playback from stopping after a false VAD event.
+- Real interruption latency and acoustic behavior remain unmeasured until a
+ human browser/microphone witness is retained.
+- The classifier is intentionally conservative. Evidence of rejected novel
+ speech or admitted repeated playback requires threshold or feature
+ re-evaluation before release.
## Stop or reorient
-Stop if deleting a source-walking assertion removes the only enforcement of a security boundary or
-observable runtime behavior. Preserve that contract in its native tool before deleting the mirror.
+Stop if the recut requires unrelated stale-branch files, a second Voice
+submission path, delayed cancellation, input-buffer clearing, transcript
+logging, or assistant-generated classification.
-Stop if removing a prune exception makes a non-architecture core or app test lose a genuine
-fixture. Preserve that proven dependency narrowly and report it instead of deleting it to satisfy
-the desired graph shape.
+Stop if current-main APIs cannot preserve the same input item through
+cancellation and completion, if an interruption can submit twice, if a rejected
+completion can replace or erase a retained answer, or if disabling the
+preference no longer restores the acknowledged half-duplex handoff.
## Deferred
-The canonical future planning record remains [`MISSION.next.md`](MISSION.next.md). SRE-1008 owns
-repository-wide detection of undeclared Turbo task inputs. The broader non-blocking cleanup of
-tautological core tests is deferred.
+- PR #9588 owns the omitted Voice settlement port and remains a separate
+ mainline update.
+- A human browser/microphone witness owns claims about speaker feedback,
+ acoustic false interruption, and audible interruption latency.
+- Preventing a false VAD event from stopping playback is outside FE-1604.
+- The projection guard in Proof 6 does not discharge the combined durable
+ continuity witness required by `MISSION.next.md`. That witness still needs a
+ production Voice input, production **Stop**/Flue abort, destroyed browser
+ client, independently created second-tab client, and persisted-history
+ hydration. Direct-user Voice source reconstruction remains blocked because
+ Flue 2.0.3 projects neither caller metadata nor idempotency keys on canonical
+ user messages, while browser-side correlation and visible text encoding
+ remain prohibited. Re-entry remains governed by
+ `docs/evidence/implementations/mission-5-voice-safety-parity/provenance-blocker.md`.
diff --git a/libs/@hashintel/brunch-agent/MISSION.next.md b/libs/@hashintel/brunch-agent/MISSION.next.md
index c65e005691c..2598a6daa20 100644
--- a/libs/@hashintel/brunch-agent/MISSION.next.md
+++ b/libs/@hashintel/brunch-agent/MISSION.next.md
@@ -371,7 +371,13 @@ Before claiming long-running provenance, prove panel/transcript/workpiece recove
The Mission 5 contract, recut on 2026-09-03, owns the single-route consolidation: the typed panel's browser `ChatTransport` over `@flue/sdk`, removal of the server-side `/api/chat` door, repurposing `transport-aisdk` as the browser-side adapter, direct Voice/Flue reconciliation, its selected external-PR evidence, and the bounded local tracer. Its 2026-09-04 human witness passed typed and Voice admission, spoken playback, barge-in, and durable Stop, then failed faithful reopen: per-message typed/Voice provenance disappeared and the stopped entry returned as ordinary truncated content. On 2026-09-04 the owner explicitly waived the fresh-human re-check and closed Mission 6; its fresh product-manager conversation contained neither record. A subsequent source/artifact audit could not substantiate the earlier mechanical-coverage claim: both retained outer-witness bundles contain only completed settlements and no recorded Voice origins, and the analyzed history projector did not reconstruct either per-message property. Preserve the historical close and immutable records, but neither the waiver nor those bundles establish a presentation pass. Mission 6b's root authority owns the combined foundation check and distinguishes supported client-tool attribution from blocked direct-user attribution.
-A later mission that exercises Voice, exact conversation resume, or pre-release scenario breadth must include one reproducible scenario containing at least one typed-origin message, one Voice-origin message, and one durably aborted assistant entry. After closing and reopening in a second tab, the oracle must verify per-message typed/Voice provenance, render the aborted entry as stopped rather than ordinary truncated content, and distinguish local **Exit voice mode** from durable composer **Stop**. Fold this scenario into that mission's named test portfolio before closure; do not treat Mission 6's prepared fixture or mechanical witness as a permanent substitute for the skipped human check.
+On 2026-09-10 the owner explicitly deferred this gate for FE-1604's
+interruption-only recut because Flue 2.0.3 exposes no supported durable
+direct-user Voice-origin projection seam and that recut does not change the
+transport or history boundary. This exception is not a passing witness and
+does not promote FE-1604's prepared fixture to production evidence.
+
+The next mission after FE-1604 that exercises Voice, exact conversation resume, or pre-release scenario breadth must include one reproducible scenario containing at least one typed-origin message, one Voice-origin message, and one durably aborted assistant entry. After closing and reopening in a second tab, the oracle must verify per-message typed/Voice provenance, render the aborted entry as stopped rather than ordinary truncated content, and distinguish local **Exit voice mode** from durable composer **Stop**. Fold this scenario into that mission's named test portfolio before closure; do not treat Mission 6's prepared fixture or mechanical witness as a permanent substitute for the skipped human check.
The small transcript reveal control remains observed discoverability strain for that surface. This future record otherwise retains only work beyond the direct cut: whether Petrinaut ever drops `useChat` itself is a Petrinaut product decision with no Brunch obligation; the structured-question route re-enters only after plain-turn strain and owner acceptance; broader barge-in, long-response, speech-selection, and accessibility quality require observations from the direct route; and trusted remote identity, origin policy, deployment, and spend controls remain release work. The inherited seam map remains in [`mission-4-voice-integration-handoff.md`](docs/evidence/implementations/mission-4-voice-integration-handoff.md).
diff --git a/libs/@hashintel/brunch-agent/docs/adr/0009-openai-voice-ui-turn-shell.md b/libs/@hashintel/brunch-agent/docs/adr/0009-openai-voice-ui-turn-shell.md
index 12aad812639..5697bb559f8 100644
--- a/libs/@hashintel/brunch-agent/docs/adr/0009-openai-voice-ui-turn-shell.md
+++ b/libs/@hashintel/brunch-agent/docs/adr/0009-openai-voice-ui-turn-shell.md
@@ -29,7 +29,7 @@ complete voice telemetry and final projection contracts are not production-ready
and there is no generic voice-provider abstraction.
3. **Server policy is fixed.** The website server combines browser SDP with a trusted Realtime
session and calls the unified `/v1/realtime/calls` endpoint. The session enables audio output,
- low reasoning effort, the `marin` voice and semantic VAD with low eagerness, automatic response
+ low reasoning effort, the `marin` voice and semantic VAD with medium eagerness, automatic response
creation and interruption. Optional provisional input transcription is display-only; neither it
nor Realtime audio is persisted as chat history. API credentials, model selection, instructions
and tools never enter browser-controlled configuration.
diff --git a/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/provenance-blocker.md b/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/provenance-blocker.md
index b3e9e897b07..d99d3b8211a 100644
--- a/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/provenance-blocker.md
+++ b/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/provenance-blocker.md
@@ -42,5 +42,8 @@ test that reconstructs the Voice marker after a fresh process with no browser
correlation state.
The restacked branch still installs `@flue/sdk` 2.0.3 with this same public
-shape. No supported projection seam or owner-approved deferral has been
-recorded, so direct-user reopen attribution remains blocked.
+shape. On 2026-09-10 the owner approved a scoped deferral for FE-1604's
+interruption-only recut because it does not change this transport or history
+boundary. Direct-user reopen attribution remains blocked, and the real
+Voice/Stop/second-tab witness re-enters with the next post-FE-1604 mission that
+exercises Voice, exact conversation resume, or pre-release scenario breadth.
diff --git a/libs/@hashintel/ds-components/src/components/Menu/SelectableList/selectable-list.tsx b/libs/@hashintel/ds-components/src/components/Menu/SelectableList/selectable-list.tsx
index f1034de73d6..fbf7056d8c6 100644
--- a/libs/@hashintel/ds-components/src/components/Menu/SelectableList/selectable-list.tsx
+++ b/libs/@hashintel/ds-components/src/components/Menu/SelectableList/selectable-list.tsx
@@ -226,6 +226,23 @@ const ItemRow = ({ item, ctx }: { item: Item; ctx: RenderCtx }) => {
}
};
+ if (item.selectedStyle === "checkbox" && "onClick" in item) {
+ return (
+
+ {body}
+
+ );
+ }
+
return (
void;
repeatQuestion?: () => void;
resume: () => void;
+ setInterruptionBySpeaking?: (enabled: boolean) => void;
setMicrophoneMuted: (muted: boolean) => void;
takeTurn?: () => Promise | void;
};
diff --git a/libs/@hashintel/petrinaut/src/react/voice-session/types.ts b/libs/@hashintel/petrinaut/src/react/voice-session/types.ts
index b72bee75f8c..78aeacbc2b6 100644
--- a/libs/@hashintel/petrinaut/src/react/voice-session/types.ts
+++ b/libs/@hashintel/petrinaut/src/react/voice-session/types.ts
@@ -22,6 +22,8 @@ export type PetrinautAiVoiceSessionState = {
canRepeatQuestion?: boolean;
/** Whether the user can cancel Voice output and start their turn. */
canTakeTurn?: boolean;
+ /** Whether speaking can interrupt assistant audio. */
+ interruptionBySpeaking?: boolean;
errorMessage: string | null;
/** Whether microphone capture is muted independently of whose turn it is. */
microphoneMuted: boolean;
diff --git a/libs/@hashintel/petrinaut/src/react/voice-session/use-voice-session.ts b/libs/@hashintel/petrinaut/src/react/voice-session/use-voice-session.ts
index 3f88e964621..4660ec2513e 100644
--- a/libs/@hashintel/petrinaut/src/react/voice-session/use-voice-session.ts
+++ b/libs/@hashintel/petrinaut/src/react/voice-session/use-voice-session.ts
@@ -102,3 +102,12 @@ export const useVoiceSessionNotice = (): string | null => {
() => null,
);
};
+
+export const useVoiceSessionInterruptionBySpeaking = (): boolean => {
+ const store = use(VoiceSessionContext);
+ return useSyncExternalStore(
+ store.subscribe,
+ () => store.getSnapshot().state?.interruptionBySpeaking ?? false,
+ () => false,
+ );
+};
diff --git a/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts b/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts
index 0f7f8da56f0..54c70070cd1 100644
--- a/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts
+++ b/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts
@@ -77,6 +77,8 @@ export type PetrinautAiVoiceModeControls = {
* whole session when Petrinaut closes the panel.
*/
setMicrophoneMuted: (muted: boolean) => void;
+ /** Allows speech to interrupt assistant playback without clearing input. */
+ setInterruptionBySpeaking?: (enabled: boolean) => void;
/** Cancels Voice output and hands the live microphone turn to the user. */
takeTurn?: () => Promise | void;
};
diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/components/voice-session-labels.ts b/libs/@hashintel/petrinaut/src/ui/views/Editor/components/voice-session-labels.ts
index 0e2f843924b..7dd389c9e4e 100644
--- a/libs/@hashintel/petrinaut/src/ui/views/Editor/components/voice-session-labels.ts
+++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/components/voice-session-labels.ts
@@ -29,6 +29,7 @@ export const voiceSessionActionLabels = {
collapse: "Collapse voice session",
end: "End voice mode",
expand: "Expand voice session",
+ interruptionBySpeaking: "Interruption by speaking",
mute: "Mute microphone",
pause: "Pause voice mode",
playbackOptions: "Voice playback options",
diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx
index d82e6a0ec79..803a3e83051 100644
--- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx
+++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx
@@ -709,6 +709,12 @@ const ConversationAiAssistantPanel = ({
? { repeatQuestion: () => controls.repeatQuestion?.() }
: {}),
resume: () => controls.resume(),
+ ...(controls.setInterruptionBySpeaking
+ ? {
+ setInterruptionBySpeaking: (enabled: boolean) =>
+ controls.setInterruptionBySpeaking?.(enabled),
+ }
+ : {}),
setMicrophoneMuted: (muted) => controls.setMicrophoneMuted(muted),
...(controls.takeTurn ? { takeTurn: () => controls.takeTurn?.() } : {}),
});
diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx
index 7bfb76a8df4..076d72d8639 100644
--- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx
+++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx
@@ -412,6 +412,72 @@ describe("AiAssistantContents", () => {
expect(onVoiceDockCollapsedChange).toHaveBeenCalledWith(false);
});
+ test("toggles interruption by speaking in the playback menu and reveals manual handover", async () => {
+ const store = createVoiceSessionStore();
+ const state = {
+ canTakeTurn: true,
+ interruptionBySpeaking: true,
+ errorMessage: null,
+ microphoneLevel: 0,
+ microphoneMuted: false,
+ phase: "speaking" as const,
+ };
+ const setInterruptionBySpeaking = vi.fn((enabled: boolean) =>
+ store.setState({ ...state, interruptionBySpeaking: enabled }),
+ );
+ store.setActions({
+ end: vi.fn(),
+ pause: vi.fn(),
+ reconnect: vi.fn(),
+ resume: vi.fn(),
+ setMicrophoneMuted: vi.fn(),
+ takeTurn: vi.fn(),
+ setInterruptionBySpeaking,
+ });
+ store.setState(state);
+ render(
+
+
+ ,
+ );
+ expect(screen.queryByRole("button", { name: "Your turn" })).toBeNull();
+ fireEvent.click(
+ screen.getByRole("button", { name: "Voice playback options" }),
+ );
+ const preference = await screen.findByRole("menuitemcheckbox", {
+ name: "Interruption by speaking",
+ });
+ expect(preference.getAttribute("aria-checked")).toBe("true");
+ expect(preference.hasAttribute("data-selected")).toBe(true);
+ const menu = screen.getByRole("menu");
+ fireEvent.keyDown(menu, { key: "End" });
+ await waitFor(() =>
+ expect(menu.getAttribute("aria-activedescendant")).toBe(preference.id),
+ );
+ fireEvent.keyDown(menu, { key: "Enter" });
+ await waitFor(() =>
+ expect(setInterruptionBySpeaking).toHaveBeenCalledWith(false),
+ );
+ expect(screen.getByRole("menu")).not.toBeNull();
+ expect(preference.getAttribute("aria-checked")).toBe("false");
+ expect(preference.hasAttribute("data-selected")).toBe(false);
+ expect(screen.getByRole("button", { name: "Your turn" })).not.toBeNull();
+ fireEvent.keyDown(menu, { key: "Enter" });
+ await waitFor(() =>
+ expect(setInterruptionBySpeaking).toHaveBeenLastCalledWith(true),
+ );
+ expect(preference.getAttribute("aria-checked")).toBe("true");
+ expect(screen.queryByRole("button", { name: "Your turn" })).toBeNull();
+ });
+
test("keeps handoff and canonical playback controls in the Voice dock", async () => {
const store = createVoiceSessionStore();
const actions = {
diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/voice-dock.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/voice-dock.tsx
index 8eb347ea695..fa88dda5029 100644
--- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/voice-dock.tsx
+++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/voice-dock.tsx
@@ -6,6 +6,7 @@ import {
useVoiceSessionCanReadFullResponse,
useVoiceSessionCanRepeatQuestion,
useVoiceSessionCanTakeTurn,
+ useVoiceSessionInterruptionBySpeaking,
useVoiceSessionMicrophoneMuted,
useVoiceSessionNotice,
useVoiceSessionPhase,
@@ -111,6 +112,7 @@ export type VoiceDockProps = {
collapsed: boolean;
/** Rendered instead of the live indicator when the caller supplies one. */
indicator?: ReactNode;
+ interruptionBySpeaking?: boolean;
microphoneMuted: boolean;
notice: string | null;
onCollapsedEnd?: () => void;
@@ -130,6 +132,7 @@ export const VoiceDock = ({
canTakeTurn,
collapsed,
indicator,
+ interruptionBySpeaking = false,
microphoneMuted,
notice,
onCollapsedEnd,
@@ -174,6 +177,7 @@ export const VoiceDock = ({
actions={actions}
canReadFullResponse={canReadFullResponse}
canRepeatQuestion={canRepeatQuestion}
+ interruptionBySpeaking={interruptionBySpeaking}
/>
)}
@@ -188,7 +192,7 @@ export const VoiceDock = ({
{actions !== null && (
<>
- {canTakeTurn && actions.takeTurn && (
+ {!interruptionBySpeaking && canTakeTurn && actions.takeTurn && (