diff --git a/.changeset/flue-voice-safety.md b/.changeset/flue-voice-safety.md index 5540dee7b70..bf64ec164db 100644 --- a/.changeset/flue-voice-safety.md +++ b/.changeset/flue-voice-safety.md @@ -2,4 +2,4 @@ "@hashintel/petrinaut": patch --- -Add half-duplex Voice handoff, exact response and marked-question replay, live transcripts, compact Voice setup and playback controls, and persistent copyable errors. Keep the conversation busy through browser-tool continuations, withhold pending work on Stop, surface automatic-tool failures to Voice, and display stopped entries and surviving client-tool Voice origins supplied by canonical history. +Add half-duplex Voice handoff, exact response and marked-question replay, live transcripts, compact Voice setup and playback controls, and persistent copyable errors. Keep the conversation busy through browser-tool continuations, withhold pending work on Stop, preserve rejected durable Stop failures through late provider completion, surface automatic-tool failures to Voice, and display stopped entries and surviving client-tool Voice origins supplied by canonical history. diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-client-tools.ts b/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-client-tools.ts index e3226db07ec..68675c09775 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-client-tools.ts +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-client-tools.ts @@ -1,14 +1,12 @@ -import { ASK_TOOL_NAME } from "@hashintel/brunch-agent/client-tools"; import { readPetrinautDocToolName } from "@hashintel/petrinaut-core"; /** * The one catalog of tools the browser answers on Brunch's behalf. The panel - * transport admits their results, the history projection leaves them runnable, - * and every interactive widget the demo registers must name one of them so a - * composer answer reaches Flue as a `client-tool-result` rather than an error. + * transport admits their results and the history projection leaves them runnable. + * The production preview has no interactive ask handler; fixture-specific tools + * extend this default catalog without restoring the suspended ask path. * Kept free of React imports so the transport can load outside the DOM. */ export const brunchClientToolNames: ReadonlySet = new Set([ readPetrinautDocToolName, - ASK_TOOL_NAME, ]); diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx index 878bd25c03e..88f03b3e060 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx @@ -10,6 +10,7 @@ import { defaultPetrinautNavigationHistoryPolicy } from "@hashintel/petrinaut/re import { OpenAIRealtimeSession } from "../voice-interview/openai-realtime-session"; import { VoiceInterviewControl } from "../voice-interview/voice-interview-control"; +import { brunchClientToolNames } from "./brunch-client-tools"; import { BrunchPanelConversationTracker } from "./brunch-panel-transport"; import { getBrunchVoiceMode, @@ -233,6 +234,7 @@ describe("local storage demo Brunch voice integration", () => { const aiAssistant = renderedPetrinaut.aiAssistant as PetrinautAiAssistant; expect(aiAssistant.requestStop).toBeTypeOf("function"); + expect([...brunchClientToolNames]).toEqual(["readPetrinautDoc"]); expect(aiAssistant.interactiveTools).toEqual([]); expect( aiAssistant.interactiveTools?.some( diff --git a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts index 245058d31d2..12d189375be 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts @@ -683,12 +683,95 @@ describe("OpenAIRealtimeSession", () => { }); expect(harness.events).toContainEqual({ connectionEpoch: 1, + playbackExpected: false, responseId: "response-active", status: "completed", type: "response-terminal", }); }); + test("reopens capture when a completed canonical response has no audio", async () => { + const harness = createHarness(); + await harness.session.connect(); + harness.session.setMicrophoneEnabled(true); + harness.session.speakCanonical([ + canonicalSegment("silent", "This response produced no audio."), + ]); + const channel = harness.channels[0]!; + authorizeLatestSpeechResponse(channel, "response-silent"); + + channel.receive({ + response: { + id: "response-silent", + output: [], + status: "completed", + }, + type: "response.done", + }); + + expect(harness.events).toContainEqual({ + connectionEpoch: 1, + playbackExpected: false, + responseId: "response-silent", + speechRequestId: "canonical-1-1", + status: "completed", + type: "response-terminal", + }); + expect( + harness.events.some( + (event) => + event.type === "output-started" || event.type === "output-stopped", + ), + ).toBe(false); + expect(harness.localTracks[0]!.enabled).toBe(true); + }); + + test("keeps active audio owned when completed output omits audio metadata", async () => { + const harness = createHarness(); + await harness.session.connect(); + harness.session.setMicrophoneEnabled(true); + harness.session.speakCanonical([ + canonicalSegment("playing", "This response is already playing."), + ]); + const channel = harness.channels[0]!; + authorizeLatestSpeechResponse(channel, "response-playing"); + channel.receive({ + response_id: "response-playing", + type: "output_audio_buffer.started", + }); + + channel.receive({ + response: { + id: "response-playing", + output: [], + status: "completed", + }, + type: "response.done", + }); + + expect(harness.events).toContainEqual({ + connectionEpoch: 1, + playbackExpected: true, + responseId: "response-playing", + speechRequestId: "canonical-1-1", + status: "completed", + type: "response-terminal", + }); + expect(harness.localTracks[0]!.enabled).toBe(false); + + channel.receive({ + response_id: "response-playing", + type: "output_audio_buffer.stopped", + }); + + expect(harness.events).toContainEqual({ + connectionEpoch: 1, + responseId: "response-playing", + type: "output-stopped", + }); + expect(harness.localTracks[0]!.enabled).toBe(true); + }); + test("keeps the microphone closed when an earlier stop follows a queued response request", async () => { const harness = createHarness(); await harness.session.connect(); @@ -709,7 +792,15 @@ describe("OpenAIRealtimeSession", () => { channel.receive({ response: { id: "response-early", - output: [], + output: [ + { + content: [ + { transcript: "First canonical segment.", type: "output_audio" }, + ], + role: "assistant", + type: "message", + }, + ], status: "completed", }, type: "response.done", @@ -722,7 +813,15 @@ describe("OpenAIRealtimeSession", () => { channel.receive({ response: { id: "response-follow-on", - output: [], + output: [ + { + content: [ + { transcript: "Second canonical segment.", type: "output_audio" }, + ], + role: "assistant", + type: "message", + }, + ], status: "completed", }, type: "response.done", @@ -745,6 +844,7 @@ describe("OpenAIRealtimeSession", () => { }); expect(harness.events).toContainEqual({ connectionEpoch: 1, + playbackExpected: true, responseId: "response-follow-on", speechRequestId: "canonical-1-2", status: "completed", @@ -803,12 +903,29 @@ describe("OpenAIRealtimeSession", () => { channel.receive({ response: { id: "response-generated", - output: [], + output: [ + { + content: [ + { transcript: "Generated canonical segment.", type: "audio" }, + ], + role: "assistant", + type: "message", + }, + ], status: "completed", }, type: "response.done", }); + expect(harness.events).toContainEqual({ + connectionEpoch: 1, + playbackExpected: true, + responseId: "response-generated", + speechRequestId: "canonical-1-1", + status: "completed", + type: "response-terminal", + }); + const cancellation = harness.session.cancelOutput(); let settled = false; void cancellation.then(() => { diff --git a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts index 0051a56fdb9..c01d16b8984 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts @@ -63,6 +63,7 @@ export type OpenAIRealtimeSessionEvent = } | { readonly connectionEpoch: number; + readonly playbackExpected: boolean; readonly responseId: string; readonly speechRequestId?: string; readonly status: "cancelled" | "completed" | "failed" | "incomplete"; @@ -140,6 +141,19 @@ const nonEmptyString = (value: unknown): string | null => const nonNegativeInteger = (value: unknown): number | null => Number.isInteger(value) && (value as number) >= 0 ? (value as number) : null; +const responseContainsAudio = (output: unknown[]): boolean => + output.some((item) => { + const outputItem = asRecord(item); + if (outputItem?.type !== "message" || !Array.isArray(outputItem.content)) { + return false; + } + return outputItem.content.some( + (contentItem) => + asRecord(contentItem)?.type === "output_audio" || + asRecord(contentItem)?.type === "audio", + ); + }); + const parseRealtimeEvent = (value: unknown): Record | null => { if (typeof value !== "string") { return null; @@ -820,6 +834,7 @@ export class OpenAIRealtimeSession { return; } const terminalStatus = status as ResponseTerminalStatus; + let playbackExpected = false; this.#responseTerminalSequence += 1; this.#activeResponseIds.delete(responseId); this.#cancelOutputAwaitingResponseIds.delete(responseId); @@ -829,6 +844,7 @@ export class OpenAIRealtimeSession { const speechRequestId = this.#speechRequestIds.get(responseId); const terminalEvent = { connectionEpoch, + playbackExpected, responseId, ...(speechRequestId === undefined ? {} : { speechRequestId }), status: terminalStatus, @@ -859,10 +875,18 @@ export class OpenAIRealtimeSession { this.#handleConnectionFailure("invalid-response", "connection"); return; } + playbackExpected = + this.#speakingResponseId === responseId || + responseContainsAudio(output); if (this.#authorizedResponseIds.has(responseId)) { - this.#terminalCanonicalResponseIds.add(responseId); + if (playbackExpected) { + this.#terminalCanonicalResponseIds.add(responseId); + } + } + this.#emit({ ...terminalEvent, playbackExpected }); + if (this.#authorizedResponseIds.has(responseId) && !playbackExpected) { + this.#finishSpeech(responseId); } - this.#emit(terminalEvent); this.#resumeCanonicalSpeechQueue(); return; } @@ -874,12 +898,12 @@ export class OpenAIRealtimeSession { type: "output-interrupted", }); } - this.#emit(terminalEvent); + this.#emit({ ...terminalEvent, playbackExpected }); this.#finishSpeech(responseId, "request-aborted"); this.#resumeCanonicalSpeechQueue(); return; } - this.#emit(terminalEvent); + this.#emit({ ...terminalEvent, playbackExpected }); if (this.#authorizedResponseIds.has(responseId)) { this.#finishSpeech(responseId, "invalid-response"); } diff --git a/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.test.ts index 3f2596e316d..24277f28af1 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.test.ts @@ -317,6 +317,7 @@ describe("RealtimeBrunchBridge", () => { }); harness.emit({ connectionEpoch: 3, + playbackExpected: true, responseId: "response-early", status: "completed", type: "response-terminal", @@ -328,6 +329,7 @@ describe("RealtimeBrunchBridge", () => { }); harness.emit({ connectionEpoch: 3, + playbackExpected: true, responseId: "response-follow-on", speechRequestId: "speech-follow-on", status: "completed", @@ -415,6 +417,46 @@ describe("RealtimeBrunchBridge", () => { ); }); + test("releases the matching pending request when completed output has no audio", async () => { + const harness = createHarness(); + startReady(harness); + harness.emit({ + connectionEpoch: 3, + speechRequestId: "speech-silent", + type: "canonical-speech-requested", + }); + + harness.emit({ + connectionEpoch: 3, + playbackExpected: false, + responseId: "response-silent", + speechRequestId: "speech-silent", + status: "completed", + type: "response-terminal", + }); + harness.emit({ + connectionEpoch: 3, + itemId: "item-after-silent-response", + type: "input-speech-started", + }); + harness.emit( + completedTranscript( + 3, + "This follows a completed response without audio.", + "item-after-silent-response", + ), + ); + + await vi.waitFor(() => + expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce(), + ); + expect(harness.submitInterviewAnswer).toHaveBeenCalledWith( + expect.objectContaining({ + text: "This follows a completed response without audio.", + }), + ); + }); + test("derives stable delivery identity from epoch, item, and content index", () => { expect( createRealtimeSubmissionId(transcriptKey(12, "item/with spaces", 4)), @@ -560,6 +602,52 @@ describe("RealtimeBrunchBridge", () => { }); }); + test("settles a completed submission with no canonical response and accepts the next turn", async () => { + const harness = createHarness(); + startReady(harness, 7); + harness.emit(completedTranscript(7, "The silent answer.")); + await vi.waitFor(() => + expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce(), + ); + + harness.bridge.updateChat({ + canAcceptInterviewAnswer: false, + canonicalSegments: [], + status: "submitted", + }); + const eventCountBeforeUnrelatedSettlement = harness.events.length; + harness.bridge.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [], + settlements: [{ outcome: "completed", submissionId: "submission-other" }], + status: "ready", + }); + expect(harness.events).toHaveLength(eventCountBeforeUnrelatedSettlement); + + const deliveryId = createRealtimeSubmissionId(transcriptKey(7)); + harness.bridge.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [], + settlements: [ + { outcome: "completed", submissionId: "submission-other" }, + { outcome: "completed", submissionId: "submission-voice-1" }, + ], + status: "ready", + }); + expect(harness.events.slice(-2)).toEqual([ + { deliveryId, type: "submission-settled" }, + { deliveryId, segments: [], type: "canonical-response-ready" }, + ]); + + harness.emit(completedTranscript(7, "The next answer.", "next-item")); + await vi.waitFor(() => + expect(harness.submitInterviewAnswer).toHaveBeenCalledTimes(2), + ); + expect(harness.submitInterviewAnswer).toHaveBeenLastCalledWith( + expect.objectContaining({ text: "The next answer." }), + ); + }); + test("speaks a completed canonical segment while chat remains streaming and settles separately", async () => { const harness = createHarness(); startReady(harness, 7); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.ts b/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.ts index 3172b93b15d..820b67c925e 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.ts @@ -437,7 +437,10 @@ export class RealtimeBrunchBridge { return; } if (event.type === "response-terminal") { - if (event.status !== "completed" && event.speechRequestId !== undefined) { + if ( + event.speechRequestId !== undefined && + (event.status !== "completed" || !event.playbackExpected) + ) { this.#pendingSpeechRequestIds.delete(event.speechRequestId); } return; @@ -686,7 +689,18 @@ export class RealtimeBrunchBridge { return; } if (responseSegments.length === 0) { - this.#completeStoppedSubmission(active); + if (stoppedSettlement?.outcome === "completed") { + this.#emit({ + deliveryId: active.deliveryId, + type: "submission-settled", + }); + this.#activeSubmission = null; + this.#emit({ + deliveryId: active.deliveryId, + segments: [], + type: "canonical-response-ready", + }); + } return; } diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts index 6f81ef30b4b..f0fb32056ad 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts @@ -564,7 +564,13 @@ describe("controlled voice preview", () => { dataChannel.receive({ response: { id: "response-canonical-reply", - output: [], + output: [ + { + content: [{ transcript: canonicalReply, type: "output_audio" }], + role: "assistant", + type: "message", + }, + ], status: "completed", }, type: "response.done", diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts index 7152451dcd0..25abbf1a631 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts @@ -531,6 +531,7 @@ describe("VoiceTurnController", () => { harness.emitSession({ connectionEpoch: 1, responseId: "response-handoff", + playbackExpected: false, status: "cancelled", type: "response-terminal", }); @@ -550,6 +551,7 @@ describe("VoiceTurnController", () => { test("reopens the microphone only after cancellation and Brunch settlement", async () => { const harness = createHarness(); + let finishCancellation: (() => void) | undefined; harness.controller.updateChat({ canAcceptInterviewAnswer: true, canonicalSegments: [question("answered-question")], @@ -573,6 +575,12 @@ describe("VoiceTurnController", () => { questionSegment: markedQuestion("next-question"), status: "streaming", }); + harness.session.cancelOutput.mockImplementationOnce( + () => + new Promise((resolve) => { + finishCancellation = resolve; + }), + ); harness.emitSession({ connectionEpoch: 1, responseId: "response-handoff", @@ -593,10 +601,28 @@ describe("VoiceTurnController", () => { expect(harness.session.setMicrophoneEnabled).toHaveBeenCalledOnce(); expect(harness.session.setMicrophoneEnabled).toHaveBeenCalledWith(false); + finishCancellation?.(); + await Promise.resolve(); + expect(handoffFinished).toBe(false); + expect(harness.bridge.completeTurnHandoff).not.toHaveBeenCalled(); + harness.emitBridge({ deliveryId: "voice-request", type: "submission-settled", }); + harness.emitBridge({ + deliveryId: "voice-request", + segments: [], + type: "canonical-response-ready", + }); + expect(handoffFinished).toBe(false); + expect(harness.controller.getSnapshot()).toMatchObject({ + canReadFullResponse: false, + canRepeatQuestion: false, + input: "listening", + microphoneEnabled: true, + output: "idle", + }); await handoff; expect(harness.bridge.completeTurnHandoff).toHaveBeenCalledOnce(); @@ -703,6 +729,7 @@ describe("VoiceTurnController", () => { harness.emitSession({ connectionEpoch: 1, responseId: "response-interrupted-early", + playbackExpected: false, status: "cancelled", type: "response-terminal", }); @@ -755,6 +782,7 @@ describe("VoiceTurnController", () => { harness.emitSession({ connectionEpoch: 1, + playbackExpected: false, responseId: "unrelated-response", status: "completed", type: "response-terminal", @@ -763,6 +791,7 @@ describe("VoiceTurnController", () => { harness.emitSession({ connectionEpoch: 1, + playbackExpected: true, responseId: "response-source", status: "completed", type: "response-terminal", @@ -815,6 +844,7 @@ describe("VoiceTurnController", () => { }); harness.emitSession({ connectionEpoch: 1, + playbackExpected: true, responseId: "response-source", status: "completed", type: "response-terminal", @@ -875,6 +905,7 @@ describe("VoiceTurnController", () => { harness.emitSession({ connectionEpoch: 1, responseId: "response-replay", + playbackExpected: true, status: "completed", type: "response-terminal", }); @@ -930,6 +961,7 @@ describe("VoiceTurnController", () => { harness.emitSession({ connectionEpoch: 1, responseId: "response-source", + playbackExpected: true, status: "completed", type: "response-terminal", }); @@ -1002,6 +1034,7 @@ describe("VoiceTurnController", () => { }); harness.emitSession({ connectionEpoch: 1, + playbackExpected: true, responseId: "response-next", status: "completed", type: "response-terminal", @@ -1016,6 +1049,42 @@ describe("VoiceTurnController", () => { expect(harness.session.setMicrophoneEnabled).toHaveBeenLastCalledWith(true); }); + test("returns to idle after a completed submission with no canonical response", async () => { + const harness = createHarness(); + await harness.controller.start(); + harness.session.setMicrophoneEnabled.mockClear(); + + harness.emitBridge({ + answer: "The silent answer.", + deliveryId: "voice-silent", + type: "submission-started", + }); + harness.emitBridge({ + answer: "The silent answer.", + deliveryId: "voice-silent", + type: "submission-accepted", + }); + harness.emitBridge({ + deliveryId: "voice-silent", + type: "submission-settled", + }); + harness.emitBridge({ + deliveryId: "voice-silent", + segments: [], + type: "canonical-response-ready", + }); + + expect(harness.controller.getSnapshot()).toMatchObject({ + canReadFullResponse: false, + canRepeatQuestion: false, + input: "listening", + lastAnswerDelivery: "delivered", + microphoneEnabled: true, + output: "idle", + }); + expect(harness.session.setMicrophoneEnabled).toHaveBeenLastCalledWith(true); + }); + test("returns to listening after a durably stopped turn without speaking", async () => { const harness = createHarness(); await harness.controller.start(); @@ -1081,6 +1150,7 @@ describe("VoiceTurnController", () => { }); harness.emitSession({ connectionEpoch: 1, + playbackExpected: true, responseId: "response-early", status: "completed", type: "response-terminal", @@ -1107,6 +1177,32 @@ describe("VoiceTurnController", () => { expect(harness.session.setMicrophoneEnabled).toHaveBeenLastCalledWith(true); }); + test("returns to idle and restores capture when completed output has no audio", async () => { + const harness = createHarness(); + await harness.controller.start(); + harness.emitSession({ + connectionEpoch: 1, + speechRequestId: "speech-silent", + type: "canonical-speech-requested", + }); + + harness.emitSession({ + connectionEpoch: 1, + playbackExpected: false, + responseId: "response-silent", + speechRequestId: "speech-silent", + status: "completed", + type: "response-terminal", + }); + + expect(harness.controller.getSnapshot()).toMatchObject({ + input: "listening", + microphoneEnabled: true, + output: "idle", + }); + expect(harness.session.setMicrophoneEnabled).toHaveBeenLastCalledWith(true); + }); + test("keeps capture closed when more canonical speech starts at settlement", async () => { const harness = createHarness(); const finalSegment = markedQuestion("ask-final", "Who acts next?"); @@ -1130,6 +1226,7 @@ describe("VoiceTurnController", () => { harness.emitSession({ connectionEpoch: 1, responseId: "response-early", + playbackExpected: true, status: "completed", type: "response-terminal", }); @@ -1168,6 +1265,7 @@ describe("VoiceTurnController", () => { }); harness.emitSession({ connectionEpoch: 1, + playbackExpected: true, responseId: "response-final", status: "completed", type: "response-terminal", @@ -1209,6 +1307,7 @@ describe("VoiceTurnController", () => { }); harness.emitSession({ connectionEpoch: 1, + playbackExpected: true, responseId: "response-early", status: "completed", type: "response-terminal", @@ -1222,6 +1321,7 @@ describe("VoiceTurnController", () => { connectionEpoch: 1, responseId: "response-follow-on", speechRequestId: "speech-follow-on", + playbackExpected: true, status: "completed", type: "response-terminal", }); @@ -1257,6 +1357,90 @@ describe("VoiceTurnController", () => { expect(harness.session.setMicrophoneEnabled).toHaveBeenLastCalledWith(true); }); + test("keeps active playback speaking when a queued silent response settles", async () => { + const harness = createHarness(); + await harness.controller.start(); + + harness.emitSession({ + connectionEpoch: 1, + speechRequestId: "speech-playing", + type: "canonical-speech-requested", + }); + harness.emitSession({ + connectionEpoch: 1, + playbackExpected: true, + responseId: "response-playing", + status: "completed", + type: "response-terminal", + }); + harness.emitSession({ + connectionEpoch: 1, + speechRequestId: "speech-queued", + type: "canonical-speech-requested", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-playing", + speechRequestId: "speech-playing", + type: "output-started", + }); + harness.session.setMicrophoneEnabled.mockClear(); + + harness.emitSession({ + connectionEpoch: 1, + playbackExpected: false, + responseId: "response-queued", + speechRequestId: "speech-queued", + status: "completed", + type: "response-terminal", + }); + + expect(harness.controller.getSnapshot()).toMatchObject({ + canTakeTurn: true, + output: "speaking", + }); + expect(harness.session.setMicrophoneEnabled).not.toHaveBeenCalledWith(true); + }); + + test("keeps active playback owned when terminal metadata omits audio", async () => { + const harness = createHarness(); + await harness.controller.start(); + + harness.emitSession({ + connectionEpoch: 1, + speechRequestId: "speech-playing", + type: "canonical-speech-requested", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-playing", + speechRequestId: "speech-playing", + type: "output-started", + }); + harness.session.setMicrophoneEnabled.mockClear(); + + harness.emitSession({ + connectionEpoch: 1, + playbackExpected: false, + responseId: "response-playing", + speechRequestId: "speech-playing", + status: "completed", + type: "response-terminal", + }); + + expect(harness.controller.getSnapshot().output).toBe("speaking"); + expect(harness.session.setMicrophoneEnabled).not.toHaveBeenCalledWith(true); + + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-playing", + type: "output-stopped", + }); + + expect(harness.controller.getSnapshot().output).toBe("idle"); + expect(harness.session.setMicrophoneEnabled).toHaveBeenLastCalledWith(true); + }); + test("preserves speaking output when canonical settlement arrives during playback", async () => { const harness = createHarness(); const nextQuestion = markedQuestion("ask-playing", "Who acts next?"); @@ -1296,6 +1480,7 @@ describe("VoiceTurnController", () => { harness.emitSession({ connectionEpoch: 1, + playbackExpected: true, responseId: "response-playing", status: "completed", type: "response-terminal", diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts index 4c8c5950c3a..d8298d8d523 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts @@ -668,9 +668,11 @@ export class VoiceTurnController { input: paused ? "paused" : "listening", output: paused ? "interrupted" - : preserveSettledOutput - ? this.#snapshot.output - : "waiting-for-tool", + : event.segments.length === 0 + ? "idle" + : preserveSettledOutput + ? this.#snapshot.output + : "waiting-for-tool", }); this.#restoreMicrophoneIfCaptureAvailable(); if (responseEnd) this.#recordLatency("answer-ready", responseEnd.id); @@ -778,7 +780,10 @@ export class VoiceTurnController { } if (event.type === "response-terminal") { if (event.responseId === this.#activeSpeechResponseId) { - if (this.#activeSpeechOutputEnded) { + if ( + this.#activeSpeechOutputEnded || + (event.status !== "completed" && !event.playbackExpected) + ) { this.#clearSettledSpeech(); } else { this.#activeSpeechResponseTerminal = true; @@ -789,11 +794,17 @@ export class VoiceTurnController { event.speechRequestId !== undefined && this.#pendingSpeechRequestIds.has(event.speechRequestId) ) { - if (event.status === "completed") { + if (event.status === "completed" && event.playbackExpected) { this.#terminalSpeechRequestIds.add(event.speechRequestId); } else { this.#pendingSpeechRequestIds.delete(event.speechRequestId); this.#terminalSpeechRequestIds.delete(event.speechRequestId); + if (this.#activeSpeechResponseId === null) { + this.#update({ + output: this.#outputAfterPlaybackEnds("idle"), + }); + this.#restoreMicrophoneIfCaptureAvailable(); + } } } return; diff --git a/libs/@hashintel/brunch-agent/MISSION.md b/libs/@hashintel/brunch-agent/MISSION.md index ff515a804a6..ebafd2340a9 100644 --- a/libs/@hashintel/brunch-agent/MISSION.md +++ b/libs/@hashintel/brunch-agent/MISSION.md @@ -15,6 +15,35 @@ new same-origin proxy. CORS governs whether a conforming browser exposes a cross to client code; it does not authenticate or restrict non-browser callers, authorize a conversation, or make public exposure safe by itself. +## Supplemental FE-1580 settlement follow-up + +**Live as of 2026-09-08** for +[PR #9588](https://github.com/hashintel/hash/pull/9588) on +`kostandin/fe-1580-port-voice-settlement-fixes`, based directly on current +`main` after #9564 and #9537 merged. This supplement preserves the accepted +Voice contract without changing the CORS authority in this file. + +- **Imperative:** semantically port the omitted #9531 commit `9415e1b007`; + release silent Voice ownership and settle completed submissions without + canonical prose. Preserve failed durable Stop errors and remove the stale + browser `brunch_ask` catalogue entry. +- **Throughline:** OpenAI terminal output → session/bridge/controller ownership; + correlated Brunch settlement → next Voice turn; panel Stop rejection → + deferred browser-tool termination; shared browser catalogue → + transport/history. +- **Proof:** donor session/bridge/controller and preview regressions; panel DOM + tests for persistent Stop failure and withheld continuation; catalogue and + fixture tests; focused unit, build, TypeScript, ESLint and formatting checks. + These tests establish local settlement behavior, not paid-provider behavior, + audible latency or a new microphone witness. +- **Constraints:** preserve current `main`'s accepted Voice and CORS joins; no + #9538 grounding, #9550 VAD/interruption, snapshot-overlay or provenance + rollback work, generic interactive tools, obsolete shim, or `brunch_ask` + restoration. +- **Stop or reorient:** stop if the port erases errors, releases unrelated + playback, revives withheld tools, weakens the CORS policy, or disturbs other + work. + ## Imperative Let a deployed Petrinaut website use the Brunch `/agents/*` Flue routes from an explicitly trusted diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx index a680b181f08..1bb8b7a3de4 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx @@ -23,6 +23,7 @@ import { import { PetrinautInstanceContext } from "../../../../react/instance-context"; import { NotificationsProvider } from "../../../../react/notifications/provider"; +import { notificationsToaster } from "../../../../react/notifications/toaster"; import { EditorContext, initialEditorState, @@ -263,6 +264,7 @@ const renderTestPanel = ({ afterEach(() => { cleanup(); + notificationsToaster.remove(); for (const instance of testInstances.splice(0)) { instance.dispose(); } @@ -2516,6 +2518,83 @@ describe("AiAssistantPanel composer submissions", () => { ); }); + test.each([false, true])( + "preserves a failed durable Stop after completion (pending tool: %s)", + async (withTool) => { + let streamController: + | ReadableStreamDefaultController + | undefined; + let latest: PetrinautAiComposerControlContext | undefined; + const failure = new Error("Durable stop failed"); + const requestStop = vi.fn(async () => { + throw failure; + }); + const sendMessages = vi.fn( + async () => + new ReadableStream({ + start(controller) { + streamController = controller; + controller.enqueue({ type: "start-step" }); + controller.enqueue({ type: "text-start", id: "preamble" }); + controller.enqueue({ + type: "text-delta", + id: "preamble", + delta: "Work in progress", + }); + }, + }), + ); + renderTestPanel({ + aiAssistant: { + requestStop, + transport: { reconnectToStream: async () => null, sendMessages }, + renderComposerControl: (context) => { + latest = context; + return null; + }, + }, + initialMessage: "Start work", + petriNetDefinition: nonEmptySDCPN, + }); + await screen.findByText("Work in progress"); + await act(async () => latest?.stop()); + expect(requestStop).toHaveBeenCalledOnce(); + expect( + screen.getAllByText(/Durable stop failed/u).length, + ).toBeGreaterThan(0); + + await act(async () => { + streamController?.enqueue({ type: "text-end", id: "preamble" }); + if (withTool) { + streamController?.enqueue({ + type: "tool-input-available", + toolCallId: "withheld-mutation", + toolName: "updatePlace", + input: { placeId: "place-1", update: { name: "MustNotApply" } }, + }); + } + streamController?.enqueue({ type: "finish-step" }); + streamController?.enqueue({ + type: "finish", + finishReason: withTool ? "tool-calls" : "stop", + }); + streamController?.close(); + }); + await act(() => new Promise((resolve) => setTimeout(resolve, 20))); + + expect(latest?.status).toBe("error"); + expect(latest?.stopped).toBe(false); + expect(screen.queryByText("Response stopped")).toBeNull(); + expect( + screen.getAllByText(/Durable stop failed/u).length, + ).toBeGreaterThan(0); + expect(testInstances.at(-1)?.definition.get().places[0]?.name).toBe( + "PlaceOne", + ); + expect(sendMessages).toHaveBeenCalledOnce(); + }, + ); + test("reports a textless automatic browser failure to hosts and its matching tool", async () => { let latest: PetrinautAiComposerControlContext | undefined; const sendMessages = vi.fn(async () => diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx index 0979fd72005..6252edcd451 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx @@ -942,6 +942,9 @@ const ConversationAiAssistantPanel = ({ ) { return false; } + if (automaticToolTurnIsTerminated(submissionGenerationRef.current)) { + return false; + } if (!stopRequestedRef.current) { // Left pending until the follow-up's own status change lands, so hosts // never observe the `ready` between this check and that request. @@ -966,10 +969,15 @@ const ConversationAiAssistantPanel = ({ }, onFinish: ({ messages: finishedMessages, isAbort, isError }) => { pendingSubmissionRecoveryRef.current = null; + const termination = automaticToolTerminationRef.current; + const failed = + termination?.generation === submissionGenerationRef.current && + termination.kind === "failed"; // A step that ended in client tool calls is followed automatically by // the SDK unless it was aborted or errored; that follow-up is still part // of this turn. const followUpPending = + !failed && !isAbort && !isError && (lastAssistantMessageIsCompleteWithToolCalls({ @@ -1001,6 +1009,9 @@ const ConversationAiAssistantPanel = ({ } aiAssistant.onMessages?.(finishedMessages); + // A rejected durable Stop remains an error even if the provider later + // completes. Neither completion nor deferred tools may report success. + if (failed) return; if (followUpPending) { // The turn is not over: a Stop pressed during this step must still be // able to withhold the follow-up, so its intent survives this step. @@ -1535,7 +1546,10 @@ const ConversationAiAssistantPanel = ({ if (submissionGenerationRef.current !== generation) { return; } + automaticToolTerminationRef.current = { generation, kind: "failed" }; stopRequestedRef.current = false; + setContinuationPending(false); + setStopped(false); setStreamError( caught instanceof Error ? caught : new Error(String(caught)), );