From 5f66ac39ea131e2b4b00badaa99bc7327ac443b8 Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Wed, 2 Sep 2026 20:32:30 +0200 Subject: [PATCH 01/10] Add manual Voice turn handoff Amp-Thread-ID: https://ampcode.com/threads/T-01a06344-05c3-75c8-ac3a-d8a6fd3ede6c Co-authored-by: Amp --- .changeset/manual-voice-turn.md | 5 + .../openai-realtime-session.test.ts | 110 ++++++++- .../openai-realtime-session.ts | 210 ++++++++++++++---- .../voice-interview-control.test.tsx | 5 +- .../voice-interview-control.tsx | 1 + .../voice-preview.integration.test.ts | 52 ++++- .../voice-session-state.test.ts | 12 +- .../voice-interview/voice-session-state.ts | 2 + .../voice-turn-controller.test.ts | 95 +++++++- .../voice-interview/voice-turn-controller.ts | 94 +++++++- .../src/react/voice-session/store.ts | 1 + .../src/react/voice-session/types.ts | 2 + .../react/voice-session/use-voice-session.ts | 10 + .../ui/types/ai-assistant-composer-control.ts | 2 + .../Editor/components/voice-session-labels.ts | 1 + .../Editor/panels/ai-assistant-panel.test.tsx | 7 +- .../Editor/panels/ai-assistant-panel.tsx | 1 + .../ai-assistant-contents/voice-dock.tsx | 15 ++ 18 files changed, 555 insertions(+), 70 deletions(-) create mode 100644 .changeset/manual-voice-turn.md diff --git a/.changeset/manual-voice-turn.md b/.changeset/manual-voice-turn.md new file mode 100644 index 00000000000..e1bcac4e251 --- /dev/null +++ b/.changeset/manual-voice-turn.md @@ -0,0 +1,5 @@ +--- +"@hashintel/petrinaut": patch +--- + +Add an accessible manual Your turn control for cancelling Voice playback before speaking. diff --git a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts index 9f260841a96..5f42eda2f9b 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts @@ -376,6 +376,89 @@ describe("OpenAIRealtimeSession", () => { expect(harness.localTracks[0]!.enabled).toBe(false); }); + test("cancels buffered speech before accepting a fresh post-handoff utterance", async () => { + const harness = createHarness(); + await harness.session.connect(); + harness.session.setMicrophoneEnabled(true); + const channel = harness.channels[0]!; + channel.receive({ + audio_start_ms: 40, + item_id: "item-before-handoff", + type: "input_audio_buffer.speech_started", + }); + harness.session.speakCanonical([ + canonicalSegment("ask-handoff", "What happens next?"), + ]); + authorizeLatestSpeechResponse(channel, "response-handoff"); + channel.receive({ + response_id: "response-handoff", + type: "output_audio_buffer.started", + }); + + const cancellation = harness.session.cancelOutput(); + + expect(harness.localTracks[0]!.enabled).toBe(false); + expect(sentEvents(channel).slice(-3)).toEqual([ + { type: "input_audio_buffer.clear" }, + expect.objectContaining({ + response_id: "response-handoff", + type: "response.cancel", + }), + { type: "output_audio_buffer.clear" }, + ]); + channel.receive({ + content_index: 0, + item_id: "item-before-handoff", + transcript: "This began too early.", + type: "conversation.item.input_audio_transcription.completed", + }); + channel.receive({ type: "input_audio_buffer.cleared" }); + channel.receive({ + response_id: "response-handoff", + type: "output_audio_buffer.cleared", + }); + channel.receive({ + response: { + id: "response-handoff", + output: [], + status: "cancelled", + }, + type: "response.done", + }); + await cancellation; + + expect(harness.localTracks[0]!.enabled).toBe(true); + expect( + harness.events.some( + (event) => + event.type === "completed" && + event.key.itemId === "item-before-handoff", + ), + ).toBe(false); + channel.receive({ + audio_start_ms: 120, + item_id: "item-after-handoff", + type: "input_audio_buffer.speech_started", + }); + channel.receive({ + content_index: 0, + item_id: "item-after-handoff", + transcript: "This began after the handoff.", + type: "conversation.item.input_audio_transcription.completed", + }); + + expect(harness.events).toContainEqual({ + key: { + connectionEpoch: 1, + contentIndex: 0, + itemId: "item-after-handoff", + }, + text: "This began after the handoff.", + type: "completed", + }); + expect(harness.localTracks[0]!.stop).not.toHaveBeenCalled(); + }); + test("never surfaces model-generated function-call arguments as user speech", async () => { const harness = createHarness(); await harness.session.connect(); @@ -670,17 +753,20 @@ describe("OpenAIRealtimeSession", () => { type: "response.created", }); - harness.session.cancelOutput(); + void harness.session.cancelOutput(); await expect(preparation).resolves.toEqual({ kind: "fallback", reason: "interrupted", sourceSegmentIds: request.sourceSegmentIds, }); - expect(sentEvents(channel).slice(-1)[0]).toMatchObject({ - response_id: "response-preparation", - type: "response.cancel", - }); + expect(sentEvents(channel).slice(-2)).toEqual([ + expect.objectContaining({ + response_id: "response-preparation", + type: "response.cancel", + }), + { type: "output_audio_buffer.clear" }, + ]); channel.receive({ delta: "Late output must be ignored.", response_id: "response-preparation", @@ -860,7 +946,7 @@ describe("OpenAIRealtimeSession", () => { ]); const responseCreate = sentEvents(channel)[0]!; - harness.session.cancelOutput(); + void harness.session.cancelOutput(); expect( sentEvents(channel).filter(({ type }) => type === "response.cancel"), @@ -990,6 +1076,7 @@ describe("OpenAIRealtimeSession", () => { test("ignores a correlated cancel for an already-finished response", async () => { const harness = createHarness(); await harness.session.connect(); + harness.session.setMicrophoneEnabled(true); const channel = harness.channels[0]!; harness.session.speakCanonical([ canonicalSegment("question", "Canonical question"), @@ -1007,7 +1094,7 @@ describe("OpenAIRealtimeSession", () => { type: "output_audio_buffer.started", }); - harness.session.cancelOutput(); + const cancellation = harness.session.cancelOutput(); const cancelEvent = sentEvents(channel).findLast( ({ type }) => type === "response.cancel", )!; @@ -1024,10 +1111,17 @@ describe("OpenAIRealtimeSession", () => { }, type: "error", }); + channel.receive({ type: "input_audio_buffer.cleared" }); + channel.receive({ + response_id: "response-canonical", + type: "output_audio_buffer.cleared", + }); + await cancellation; expect(harness.events).not.toContainEqual( expect.objectContaining({ type: "error" }), ); + expect(harness.localTracks[0]!.enabled).toBe(true); expect(harness.localTracks[0]!.stop).not.toHaveBeenCalled(); }); @@ -1051,7 +1145,7 @@ describe("OpenAIRealtimeSession", () => { type: "output_audio_buffer.started", }); - harness.session.cancelOutput(); + void harness.session.cancelOutput(); const cancelEvent = sentEvents(channel).findLast( ({ type }) => type === "response.cancel", )!; diff --git a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts index 6bbd9e36be6..0312d2d7013 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts @@ -243,6 +243,7 @@ const waitForAbort = ( }; export class OpenAIRealtimeSession { + readonly #acceptedInputItemIds = new Set(); readonly #dependencies: OpenAIRealtimeSessionDependencies; readonly #activeResponseIds = new Set(); readonly #listeners = new Set(); @@ -250,6 +251,8 @@ export class OpenAIRealtimeSession { readonly #cancelledCanonicalResponseIds = new Set(); readonly #cancelledPreparationRequestIds = new Set(); readonly #cancelledSpeechRequestIds = new Set(); + readonly #cancelOutputAwaitingRequestIds = new Set(); + readonly #cancelOutputAwaitingResponseIds = new Set(); readonly #canonicalResponseIds = new Set(); readonly #completedResponseCancelEventIds = new Set(); readonly #pendingClientEvents = new Map(); @@ -271,6 +274,10 @@ export class OpenAIRealtimeSession { #connected = false; #connectedAt: number | null = null; #clientEventSequence = 0; + #cancelOutputAwaitingInputBufferClear = false; + #cancelOutputAwaitingOutputBufferResponseId: string | null = null; + #cancelOutputPromise: Promise | null = null; + #cancelOutputResolve: (() => void) | null = null; #connectionRequestId: string | null = null; #dataChannel: RTCDataChannel | null = null; #epoch = 0; @@ -284,6 +291,7 @@ export class OpenAIRealtimeSession { #microphoneTrack: MediaStreamTrack | null = null; #peerConnection: RTCPeerConnection | null = null; #remoteAudio: RemoteAudio | null = null; + #requireInputSpeechStart = false; #responseCreateEventId: string | null = null; #responseTerminalSequence = 0; #speakingResponseId: string | null = null; @@ -557,59 +565,95 @@ export class OpenAIRealtimeSession { }); } - public cancelOutput(): void { + public cancelOutput(): Promise { if (!this.#connected || this.#dataChannel?.readyState !== "open") { - return; + return Promise.resolve(); } - - for (const request of this.#responseQueue.splice(0)) { - if (request.kind === "canonical-speech") { - this.#cancelPendingSpeechRequest(request.speechRequestId); - } else { - this.#settlePreparation(request.preparationRequestId, "interrupted"); - } + if (this.#cancelOutputPromise) { + return this.#cancelOutputPromise; } - if (this.#responseCreateEventId !== null) { - const pendingEvent = this.#pendingClientEvents.get( - this.#responseCreateEventId, - ); - if (pendingEvent?.kind === "response-create") { - if (pendingEvent.request.kind === "canonical-speech") { - this.#cancelledSpeechRequestIds.add( - pendingEvent.request.speechRequestId, - ); + const cancelOutputPromise = new Promise((resolve) => { + this.#cancelOutputResolve = resolve; + }); + this.#cancelOutputPromise = cancelOutputPromise; + this.#cancelOutputAwaitingInputBufferClear = true; + this.#cancelOutputAwaitingOutputBufferResponseId = this.#speakingResponseId; + this.#requireInputSpeechStart = true; + for (const itemId of this.#acceptedInputItemIds) { + this.#playbackOverlappingInputItemIds.add(itemId); + } + this.#acceptedInputItemIds.clear(); + this.#syncMicrophoneTrack(); + + try { + this.#send({ type: "input_audio_buffer.clear" }); + + for (const request of this.#responseQueue.splice(0)) { + if (request.kind === "canonical-speech") { + this.#cancelPendingSpeechRequest(request.speechRequestId); } else { - this.#cancelledPreparationRequestIds.add( - pendingEvent.request.preparationRequestId, - ); - this.#settlePreparation( - pendingEvent.request.preparationRequestId, - "interrupted", - ); + this.#settlePreparation(request.preparationRequestId, "interrupted"); } } - } - for (const [ - responseId, - preparationRequestId, - ] of this.#preparationResponseIds) { - if (this.#activeResponseIds.has(responseId)) { - this.#settlePreparation(preparationRequestId, "interrupted"); - this.#cancelResponse(responseId); + if (this.#responseCreateEventId !== null) { + const pendingEvent = this.#pendingClientEvents.get( + this.#responseCreateEventId, + ); + if (pendingEvent?.kind === "response-create") { + if (pendingEvent.request.kind === "canonical-speech") { + this.#cancelledSpeechRequestIds.add( + pendingEvent.request.speechRequestId, + ); + this.#cancelOutputAwaitingRequestIds.add( + pendingEvent.request.speechRequestId, + ); + } else { + this.#cancelledPreparationRequestIds.add( + pendingEvent.request.preparationRequestId, + ); + this.#cancelOutputAwaitingRequestIds.add( + pendingEvent.request.preparationRequestId, + ); + this.#settlePreparation( + pendingEvent.request.preparationRequestId, + "interrupted", + ); + } + } } - } - for (const responseId of this.#canonicalResponseIds) { - if ( - this.#activeResponseIds.has(responseId) && - !this.#cancelledCanonicalResponseIds.has(responseId) - ) { - this.#cancelledCanonicalResponseIds.add(responseId); - this.#cancelOutputResponse(responseId); + for (const [ + responseId, + preparationRequestId, + ] of this.#preparationResponseIds) { + if (this.#activeResponseIds.has(responseId)) { + this.#cancelOutputAwaitingResponseIds.add(responseId); + this.#settlePreparation(preparationRequestId, "interrupted"); + this.#cancelResponse(responseId); + } } + + for (const responseId of this.#canonicalResponseIds) { + if ( + this.#activeResponseIds.has(responseId) && + !this.#cancelledCanonicalResponseIds.has(responseId) + ) { + this.#cancelOutputAwaitingResponseIds.add(responseId); + this.#cancelledCanonicalResponseIds.add(responseId); + this.#cancelResponse(responseId); + } + } + + this.#send({ type: "output_audio_buffer.clear" }); + } catch { + this.#finishOutputCancellation(true); + this.#handleConnectionFailure("network", "speech"); } + + this.#finishOutputCancellation(); + return cancelOutputPromise; } #cancelOutputResponse(responseId: string): void { @@ -776,6 +820,12 @@ export class OpenAIRealtimeSession { this.#handlePreparationDelta(parsed); return; } + if (parsed.type === "input_audio_buffer.cleared") { + this.#acceptedInputItemIds.clear(); + this.#cancelOutputAwaitingInputBufferClear = false; + this.#finishOutputCancellation(); + return; + } if (parsed.type === "input_audio_buffer.committed") { const itemId = nonEmptyString(parsed.item_id); if (itemId) this.#startTranscription(itemId); @@ -784,10 +834,11 @@ export class OpenAIRealtimeSession { if (parsed.type === "input_audio_buffer.speech_started") { const itemId = nonEmptyString(parsed.item_id); if (!itemId || nonNegativeInteger(parsed.audio_start_ms) === null) return; - if (this.#speakingResponseId) { + if (this.#speakingResponseId || !this.#microphoneTrack?.enabled) { this.#playbackOverlappingInputItemIds.add(itemId); return; } + this.#acceptedInputItemIds.add(itemId); this.#emit({ connectionEpoch, itemId, @@ -808,7 +859,8 @@ export class OpenAIRealtimeSession { } if ( parsed.type === "output_audio_buffer.started" || - parsed.type === "output_audio_buffer.stopped" + parsed.type === "output_audio_buffer.stopped" || + parsed.type === "output_audio_buffer.cleared" ) { this.#handleOutputBufferEvent(parsed, connectionEpoch); return; @@ -834,6 +886,9 @@ export class OpenAIRealtimeSession { if (!correlatedRequestId) { return; } + if (this.#cancelOutputAwaitingRequestIds.delete(correlatedRequestId)) { + this.#cancelOutputAwaitingResponseIds.add(responseId); + } if (metadata?.petrinaut_kind === "speech-preparation") { const correlated = this.#completeResponseCreateEvent( @@ -945,6 +1000,10 @@ export class OpenAIRealtimeSession { ) { this.#pendingClientEvents.delete(sourceEventId); this.#completedResponseCancelEventIds.delete(sourceEventId); + if (pendingEvent?.kind === "response-cancel") { + this.#cancelOutputAwaitingResponseIds.delete(pendingEvent.responseId); + this.#finishOutputCancellation(); + } return; } @@ -964,7 +1023,11 @@ export class OpenAIRealtimeSession { pendingEvent.request.speechRequestId, ) ) { + this.#cancelOutputAwaitingRequestIds.delete( + pendingEvent.request.speechRequestId, + ); this.#cancelPendingSpeechRequest(pendingEvent.request.speechRequestId); + this.#finishOutputCancellation(); return; } if ( @@ -973,10 +1036,14 @@ export class OpenAIRealtimeSession { pendingEvent.request.preparationRequestId, ) ) { + this.#cancelOutputAwaitingRequestIds.delete( + pendingEvent.request.preparationRequestId, + ); this.#settlePreparation( pendingEvent.request.preparationRequestId, "interrupted", ); + this.#finishOutputCancellation(); return; } this.#responseQueue.unshift(pendingEvent.request); @@ -1000,6 +1067,10 @@ export class OpenAIRealtimeSession { pendingEvent.request.preparationRequestId, "provider-error", ); + this.#cancelOutputAwaitingRequestIds.delete( + pendingEvent.request.preparationRequestId, + ); + this.#finishOutputCancellation(); this.#resumeSerializedResponseQueue(); return; } @@ -1028,6 +1099,8 @@ export class OpenAIRealtimeSession { } this.#responseTerminalSequence += 1; this.#activeResponseIds.delete(responseId); + this.#cancelOutputAwaitingResponseIds.delete(responseId); + this.#finishOutputCancellation(); this.#clearResponseCancelEvents(responseId); this.#waitingForResponseTerminal = false; @@ -1247,6 +1320,9 @@ export class OpenAIRealtimeSession { if (!responseId) return; if (event.type === "output_audio_buffer.started") { if (this.#cancelledCanonicalResponseIds.has(responseId)) { + if (this.#cancelOutputPromise) { + this.#cancelOutputAwaitingOutputBufferResponseId = responseId; + } this.#send({ type: "output_audio_buffer.clear" }); return; } @@ -1261,14 +1337,23 @@ export class OpenAIRealtimeSession { return; } const wasSpeaking = this.#speakingResponseId === responseId; + const wasCleared = event.type === "output_audio_buffer.cleared"; this.#finishSpeech( responseId, - this.#cancelledCanonicalResponseIds.has(responseId) + wasCleared || this.#cancelledCanonicalResponseIds.has(responseId) ? "request-aborted" : undefined, ); if (wasSpeaking) { - this.#emit({ connectionEpoch, responseId, type: "output-stopped" }); + this.#emit({ + connectionEpoch, + responseId, + type: wasCleared ? "output-interrupted" : "output-stopped", + }); + } + if (this.#cancelOutputAwaitingOutputBufferResponseId === responseId) { + this.#cancelOutputAwaitingOutputBufferResponseId = null; + this.#finishOutputCancellation(); } } @@ -1280,7 +1365,10 @@ export class OpenAIRealtimeSession { const contentIndex = nonNegativeInteger(event.content_index); if (!itemId || contentIndex === null) return; const key = { connectionEpoch, contentIndex, itemId }; - const overlapsPlayback = this.#playbackOverlappingInputItemIds.has(itemId); + const overlapsPlayback = + this.#playbackOverlappingInputItemIds.has(itemId) || + (this.#requireInputSpeechStart && + !this.#acceptedInputItemIds.has(itemId)); if (overlapsPlayback) { if ( event.type === @@ -1293,12 +1381,14 @@ export class OpenAIRealtimeSession { ? "invalid-response" : undefined, ); + this.#acceptedInputItemIds.delete(itemId); } return; } this.#startTranscription(itemId); if (event.type === "conversation.item.input_audio_transcription.failed") { this.#finishTranscription(itemId, "invalid-response"); + this.#acceptedInputItemIds.delete(itemId); this.#emit({ key, type: "transcription-failed" }); return; } @@ -1311,6 +1401,7 @@ export class OpenAIRealtimeSession { event.type === "conversation.item.input_audio_transcription.completed" ) { this.#finishTranscription(itemId); + this.#acceptedInputItemIds.delete(itemId); } this.#emit({ key, @@ -1354,6 +1445,29 @@ export class OpenAIRealtimeSession { } } + #finishOutputCancellation(force = false): void { + if ( + !this.#cancelOutputPromise || + (!force && + (this.#cancelOutputAwaitingInputBufferClear || + this.#cancelOutputAwaitingOutputBufferResponseId !== null || + this.#cancelOutputAwaitingRequestIds.size > 0 || + this.#cancelOutputAwaitingResponseIds.size > 0)) + ) { + return; + } + + const resolve = this.#cancelOutputResolve; + this.#cancelOutputPromise = null; + this.#cancelOutputResolve = null; + this.#cancelOutputAwaitingInputBufferClear = false; + this.#cancelOutputAwaitingOutputBufferResponseId = null; + this.#cancelOutputAwaitingRequestIds.clear(); + this.#cancelOutputAwaitingResponseIds.clear(); + this.#syncMicrophoneTrack(); + resolve?.(); + } + #emit(event: OpenAIRealtimeSessionEvent): void { for (const listener of this.#listeners) listener(event); } @@ -1539,6 +1653,7 @@ export class OpenAIRealtimeSession { const enabled = this.#microphoneRequested && this.#connected && + this.#cancelOutputPromise === null && this.#speakingResponseId === null; this.#microphoneTrack.enabled = enabled; if (enabled) { @@ -1640,6 +1755,8 @@ export class OpenAIRealtimeSession { ); } this.#transcriptionTimings.clear(); + this.#acceptedInputItemIds.clear(); + this.#requireInputSpeechStart = false; this.#activeResponseIds.clear(); this.#cancelledCanonicalResponseIds.clear(); this.#cancelledPreparationRequestIds.clear(); @@ -1660,6 +1777,7 @@ export class OpenAIRealtimeSession { this.#waitingForResponseTerminal = false; this.#activeEpoch = null; this.#connected = false; + this.#finishOutputCancellation(true); this.#microphoneRequested = false; this.#connectedAt = null; this.#connectionRequestId = null; diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx index 4be3b7bcce4..57f8f842156 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx @@ -213,7 +213,7 @@ describe("voice interview control", () => { ).not.toThrow(); }); - test("registers canonical replay controls with the host", () => { + test("registers canonical replay and manual handoff controls with the host", () => { const repeatQuestion = vi.spyOn( VoiceTurnController.prototype, "repeatQuestion", @@ -222,13 +222,16 @@ describe("voice interview control", () => { VoiceTurnController.prototype, "readFullResponse", ); + const takeTurn = vi.spyOn(VoiceTurnController.prototype, "takeTurn"); render(); registeredVoiceModeControls?.repeatQuestion?.(); registeredVoiceModeControls?.readFullResponse?.(); + void registeredVoiceModeControls?.takeTurn?.(); expect(repeatQuestion).toHaveBeenCalledOnce(); expect(readFullResponse).toHaveBeenCalledOnce(); + expect(takeTurn).toHaveBeenCalledOnce(); }); test("loads only a schema-valid available server configuration", async () => { diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx index 8e030fb725a..6e732b33bf7 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx @@ -320,6 +320,7 @@ const AvailableVoiceInterviewControl = ({ resume: () => store.controller.resume(), setMicrophoneMuted: (muted) => store.controller.setMicrophoneMuted(muted), + takeTurn: () => store.controller.takeTurn(), }), [registerVoiceModeControls, store], ); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts index 7b857490f34..9892e2bbcaf 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts @@ -271,30 +271,76 @@ describe("controlled voice preview", () => { await controller.start(); authorizeLatestSpeechResponse(dataChannel, "response-initial-question"); + dataChannel.receive({ + audio_start_ms: 100, + item_id: "item-before-handoff", + type: "input_audio_buffer.speech_started", + }); dataChannel.receive({ response_id: "response-initial-question", type: "output_audio_buffer.started", }); + + expect(controller.getSnapshot()).toMatchObject({ + canTakeTurn: true, + currentQuestion: "What happens after approval?", + output: "speaking", + }); + const handoff = controller.takeTurn(); + const repeatedHandoff = controller.takeTurn(); + expect(repeatedHandoff).toBe(handoff); + expect(sentEvents(dataChannel).slice(-3)).toEqual([ + { type: "input_audio_buffer.clear" }, + expect.objectContaining({ + response_id: "response-initial-question", + type: "response.cancel", + }), + { type: "output_audio_buffer.clear" }, + ]); + expect(track.enabled).toBe(false); + expect(peer.close).not.toHaveBeenCalled(); + + dataChannel.receive({ + content_index: 0, + item_id: "item-before-handoff", + transcript: "This began before the handoff.", + type: "conversation.item.input_audio_transcription.completed", + }); + dataChannel.receive({ type: "input_audio_buffer.cleared" }); dataChannel.receive({ response_id: "response-initial-question", - type: "output_audio_buffer.stopped", + type: "output_audio_buffer.cleared", }); dataChannel.receive({ response: { id: "response-initial-question", output: [], - status: "completed", + status: "cancelled", }, type: "response.done", }); + await handoff; + await repeatedHandoff; + + expect(submitInterviewAnswer).not.toHaveBeenCalled(); + expect(track.enabled).toBe(true); expect(controller.getSnapshot()).toMatchObject({ + canRepeatQuestion: true, + canTakeTurn: false, + connection: "connected", + currentQuestion: "What happens after approval?", input: "listening", microphoneEnabled: true, - output: "idle", + output: "interrupted", }); // Silence or noise: Realtime completes an empty transcript. Nothing is // submitted and the session keeps listening with a recoverable notice. + dataChannel.receive({ + audio_start_ms: 200, + item_id: "noise-item", + type: "input_audio_buffer.speech_started", + }); dataChannel.receive({ content_index: 0, item_id: "noise-item", diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.test.ts index a0f878c29d5..afb2013e8d8 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.test.ts @@ -7,6 +7,7 @@ import type { VoiceTurnSnapshot } from "./voice-turn-controller"; const listeningSnapshot = { canReadFullResponse: true, canRepeatQuestion: true, + canTakeTurn: false, canReviseLastAnswer: false, connection: "connected", currentQuestion: "What happens after approval?", @@ -35,6 +36,7 @@ describe("toVoiceSessionState", () => { expect(mapSnapshot()).toEqual({ canReadFullResponse: true, canRepeatQuestion: true, + canTakeTurn: false, errorMessage: null, microphoneLevel: 0.24, microphoneMuted: false, @@ -43,15 +45,19 @@ describe("toVoiceSessionState", () => { }); test("hands the turn to the assistant while it speaks", () => { - expect(mapSnapshot({ output: "speaking", partialText: "" })).toMatchObject({ - phase: "speaking", - }); + expect( + mapSnapshot({ canTakeTurn: true, output: "speaking", partialText: "" }), + ).toMatchObject({ canTakeTurn: true, phase: "speaking" }); }); test("treats a pending tool and a submitting turn as thinking", () => { expect( mapSnapshot({ output: "waiting-for-tool", partialText: "" }), ).toMatchObject({ phase: "thinking" }); + expect(mapSnapshot({ output: "cancelling" })).toMatchObject({ + canTakeTurn: false, + phase: "thinking", + }); expect(mapSnapshot({ input: "submitting", partialText: "" })).toMatchObject( { phase: "thinking" }, ); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.ts index 07d6fede838..03e8d341ded 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.ts @@ -65,6 +65,7 @@ const phaseOf = ( return "speaking"; } if ( + snapshot.output === "cancelling" || snapshot.output === "waiting-for-tool" || snapshot.input === "submitting" ) { @@ -95,6 +96,7 @@ export const toVoiceSessionState = ({ return { canReadFullResponse: snapshot.canReadFullResponse, canRepeatQuestion: snapshot.canRepeatQuestion, + canTakeTurn: snapshot.canTakeTurn, errorMessage: snapshot.connection === "error" ? errorMessageOf(snapshot) : null, microphoneMuted: diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts index 513a768a77a..b7018ac760e 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts @@ -17,7 +17,7 @@ const createHarness = () => { | undefined; let bridgeListener: ((event: RealtimeBrunchBridgeEvent) => void) | undefined; const session = { - cancelOutput: vi.fn(), + cancelOutput: vi.fn<() => Promise>(async () => undefined), connect: vi.fn(async () => ++epoch), disconnect: vi.fn(async () => undefined), setMicrophoneEnabled: vi.fn(), @@ -126,6 +126,93 @@ describe("VoiceTurnController", () => { }); }); + test("hands over an active question once and reopens capture after cancellation", async () => { + const harness = createHarness(); + const source = speechSource(); + let finishCancellation: (() => void) | undefined; + harness.session.cancelOutput.mockImplementationOnce( + () => + new Promise((resolve) => { + finishCancellation = resolve; + }), + ); + harness.controller.updateChat({ + automaticSource: source, + canAcceptInterviewAnswer: true, + canonicalSegments: [...source.fullResponseSegments], + status: "ready", + }); + await harness.controller.start(); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-question", + type: "output-started", + }); + harness.bridge.cancelPendingSpeech.mockClear(); + harness.session.cancelOutput.mockClear(); + + expect(harness.controller.getSnapshot().canTakeTurn).toBe(true); + const firstHandoff = harness.controller.takeTurn(); + const repeatedHandoff = harness.controller.takeTurn(); + + expect(repeatedHandoff).toBe(firstHandoff); + expect(harness.bridge.cancelPendingSpeech).toHaveBeenCalledOnce(); + expect(harness.session.cancelOutput).toHaveBeenCalledOnce(); + expect(harness.session.disconnect).not.toHaveBeenCalled(); + expect(harness.session.setMicrophoneEnabled).toHaveBeenLastCalledWith( + false, + ); + expect(harness.controller.getSnapshot()).toMatchObject({ + canTakeTurn: false, + connection: "connected", + currentQuestion: "Who approves release?", + output: "cancelling", + }); + + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-question", + type: "output-interrupted", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-question", + status: "cancelled", + type: "response-terminal", + }); + expect(harness.controller.getSnapshot().output).toBe("cancelling"); + finishCancellation?.(); + await firstHandoff; + + expect(harness.session.setMicrophoneEnabled).toHaveBeenLastCalledWith(true); + expect(harness.controller.getSnapshot()).toMatchObject({ + canRepeatQuestion: true, + canTakeTurn: false, + connection: "connected", + currentQuestion: "Who approves release?", + microphoneEnabled: true, + output: "interrupted", + }); + }); + + test("offers handoff while speech is being prepared for an unanswered question", async () => { + const harness = createHarness(); + harness.controller.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [question("ask-preparing")], + status: "ready", + }); + await harness.controller.start(); + + harness.emitBridge({ type: "speech-delivery-pending" }); + + expect(harness.controller.getSnapshot()).toMatchObject({ + canTakeTurn: true, + currentQuestion: "What happens after approval?", + output: "waiting-for-tool", + }); + }); + test("invalidates pending preparation before pausing or cancelling paused output", async () => { const harness = createHarness(); const order: string[] = []; @@ -135,9 +222,9 @@ describe("VoiceTurnController", () => { harness.bridge.updateChat.mockImplementation(() => order.push("chat-update"), ); - harness.session.cancelOutput.mockImplementation(() => - order.push("session-cancel"), - ); + harness.session.cancelOutput.mockImplementation(async () => { + order.push("session-cancel"); + }); await harness.controller.start(); harness.controller.pause(); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts index 171d86fa120..d02c82987dd 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts @@ -17,6 +17,7 @@ export type VoiceConnectionState = | "error"; export type VoiceInputState = "listening" | "paused" | "submitting"; export type VoiceOutputState = + | "cancelling" | "idle" | "waiting-for-tool" | "speaking" @@ -33,6 +34,7 @@ export type VoiceInputNotice = "none" | "not-heard"; export interface VoiceTurnSnapshot { readonly canReadFullResponse: boolean; readonly canRepeatQuestion: boolean; + readonly canTakeTurn: boolean; readonly canReviseLastAnswer: boolean; readonly connection: VoiceConnectionState; readonly currentQuestion: string; @@ -62,7 +64,7 @@ export interface VoiceLatencyEvent { type ChatStatus = "ready" | "submitted" | "streaming" | "error"; interface RealtimeSession { - cancelOutput(): void; + cancelOutput(): Promise; connect(): Promise; disconnect(): Promise; setMicrophoneEnabled(enabled: boolean): void; @@ -104,6 +106,7 @@ type SnapshotListener = (snapshot: VoiceTurnSnapshot) => void; const initialSnapshot: VoiceTurnSnapshot = { canReadFullResponse: false, canRepeatQuestion: false, + canTakeTurn: false, canReviseLastAnswer: false, connection: "idle", currentQuestion: "", @@ -147,6 +150,7 @@ export class VoiceTurnController { #snapshot = initialSnapshot; #speechSource: InterviewSpeechSource | null = null; #submittingQuestionId: string | null = null; + #takingTurnPromise: Promise | null = null; #teardownPromise: Promise | null = null; #transcriptItemId: string | null = null; #transcriptKey: string | null = null; @@ -258,6 +262,7 @@ export class VoiceTurnController { this.#inputStateOnResume = null; this.#inputTurnPending = false; this.#submittingQuestionId = null; + this.#takingTurnPromise = null; this.#pauseRequested = false; this.#transcriptItemId = null; this.#transcriptKey = null; @@ -415,6 +420,66 @@ export class VoiceTurnController { this.#session.speakCanonical([question]); } + /** + * Ends the assistant's pending or active speech without ending Realtime. + * Capture stays closed until the session confirms its input and output + * buffers are clear, so the next accepted utterance begins after the click. + */ + public takeTurn(): Promise { + if (this.#takingTurnPromise) { + return this.#takingTurnPromise; + } + if (!this.#snapshot.canTakeTurn) { + return Promise.resolve(); + } + + const generation = this.#generation; + this.#bridge.cancelPendingSpeech(); + this.#session.setMicrophoneEnabled(false); + this.#transcriptItemId = null; + this.#transcriptKey = null; + this.#update({ output: "cancelling", partialText: "" }); + + const takingTurnPromise = this.#session + .cancelOutput() + .then(() => { + if ( + generation !== this.#generation || + this.#snapshot.connection !== "connected" || + this.#snapshot.input === "paused" + ) { + return; + } + this.#activeSpeechOutputEnded = false; + this.#activeSpeechResponseId = null; + this.#activeSpeechResponseTerminal = false; + this.#session.setMicrophoneEnabled(true); + this.#update({ microphoneEnabled: true, output: "interrupted" }); + }) + .catch((error: unknown) => { + if (generation !== this.#generation) { + return; + } + const voiceError = + error instanceof VoiceError + ? error + : new VoiceError("speech", "network", ""); + this.#setError( + voiceError.message, + voiceError.code, + voiceError.requestId, + ); + }) + .finally(() => { + if (this.#takingTurnPromise === takingTurnPromise) { + this.#takingTurnPromise = null; + } + }); + this.#takingTurnPromise = takingTurnPromise; + this.#update({}); + return takingTurnPromise; + } + public updateChat(update: ChatUpdate): void { this.#speechSource = update.automaticSource ?? null; const canReplay = this.#canReplay(this.#snapshot); @@ -555,7 +620,9 @@ export class VoiceTurnController { this.#activeSpeechResponseId = null; this.#activeSpeechResponseTerminal = false; } - this.#update({ output: "idle" }); + this.#update({ + output: this.#takingTurnPromise ? "cancelling" : "idle", + }); if (this.#currentQuestionId) { this.#recordLatency("question-spoken", this.#currentQuestionId); } @@ -569,10 +636,13 @@ export class VoiceTurnController { this.#activeSpeechResponseId = null; this.#activeSpeechResponseTerminal = false; } - this.#update({ output: "interrupted" }); + this.#update({ + output: this.#takingTurnPromise ? "cancelling" : "interrupted", + }); return; } if (event.type === "input-speech-started") { + if (this.#takingTurnPromise) return; this.#inputTurnPending = true; this.#bridge.cancelPendingSpeech(); this.#transcriptItemId = event.itemId; @@ -643,6 +713,7 @@ export class VoiceTurnController { this.#activeSpeechResponseTerminal = false; this.#inputStateOnResume = null; this.#inputTurnPending = false; + this.#takingTurnPromise = null; this.#bridgeStarted = false; this.#transcriptItemId = null; this.#transcriptKey = null; @@ -668,7 +739,7 @@ export class VoiceTurnController { #cancelOutput(): void { this.#bridge.cancelPendingSpeech(); - this.#session.cancelOutput(); + void this.#session.cancelOutput(); } #recordLatency(name: VoiceLatencyEvent["name"], questionId: string): void { @@ -700,6 +771,20 @@ export class VoiceTurnController { ); } + #canTakeTurn(snapshot: VoiceTurnSnapshot): boolean { + return ( + snapshot.connection === "connected" && + snapshot.input !== "paused" && + (snapshot.output === "waiting-for-tool" || + snapshot.output === "speaking") && + this.#currentQuestionId !== null && + this.#currentQuestionId !== this.#answeredQuestionId && + this.#currentQuestionId !== this.#submittingQuestionId && + Boolean(snapshot.currentQuestion) && + this.#takingTurnPromise === null + ); + } + #isPauseRequested(): boolean { return this.#pauseRequested; } @@ -714,6 +799,7 @@ export class VoiceTurnController { canRepeatQuestion: this.#canReplay(snapshot) && Boolean(this.#speechSource?.questionSegment), + canTakeTurn: this.#canTakeTurn(snapshot), canReviseLastAnswer: this.#canReviseLastAnswer(snapshot), }; for (const listener of this.#listeners) listener(this.#snapshot); diff --git a/libs/@hashintel/petrinaut/src/react/voice-session/store.ts b/libs/@hashintel/petrinaut/src/react/voice-session/store.ts index 345e080b176..73e502c20bb 100644 --- a/libs/@hashintel/petrinaut/src/react/voice-session/store.ts +++ b/libs/@hashintel/petrinaut/src/react/voice-session/store.ts @@ -12,6 +12,7 @@ export type VoiceSessionActions = { repeatQuestion?: () => void; resume: () => void; setMicrophoneMuted: (muted: boolean) => void; + takeTurn?: () => Promise | void; }; export type VoiceSessionSnapshot = { diff --git a/libs/@hashintel/petrinaut/src/react/voice-session/types.ts b/libs/@hashintel/petrinaut/src/react/voice-session/types.ts index 1ee8e679bd2..004e57b7a28 100644 --- a/libs/@hashintel/petrinaut/src/react/voice-session/types.ts +++ b/libs/@hashintel/petrinaut/src/react/voice-session/types.ts @@ -20,6 +20,8 @@ export type PetrinautAiVoiceSessionState = { canReadFullResponse?: boolean; /** Whether the current canonical interview question is safe to repeat. */ canRepeatQuestion?: boolean; + /** Whether the user can manually cancel Voice output and start their turn. */ + canTakeTurn?: boolean; errorMessage: string | null; /** Whether microphone capture is muted independently of whose turn it is. */ microphoneMuted: boolean; diff --git a/libs/@hashintel/petrinaut/src/react/voice-session/use-voice-session.ts b/libs/@hashintel/petrinaut/src/react/voice-session/use-voice-session.ts index 916c5a39d17..91ee7b73ac1 100644 --- a/libs/@hashintel/petrinaut/src/react/voice-session/use-voice-session.ts +++ b/libs/@hashintel/petrinaut/src/react/voice-session/use-voice-session.ts @@ -82,3 +82,13 @@ export const useVoiceSessionCanRepeatQuestion = (): boolean => { () => false, ); }; + +export const useVoiceSessionCanTakeTurn = (): boolean => { + const store = use(VoiceSessionContext); + + return useSyncExternalStore( + store.subscribe, + () => store.getSnapshot().state?.canTakeTurn ?? false, + () => false, + ); +}; diff --git a/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts b/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts index c2778346770..b536f7e11bb 100644 --- a/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts +++ b/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts @@ -71,6 +71,8 @@ export type PetrinautAiVoiceModeControls = { * whole session when Petrinaut closes the panel. */ setMicrophoneMuted: (muted: boolean) => void; + /** Cancels Voice output and hands the live microphone turn to the user. */ + takeTurn?: () => Promise | void; }; /** Stable controls and conversation state supplied to a host-owned Voice mode. */ diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/components/voice-session-labels.ts b/libs/@hashintel/petrinaut/src/ui/views/Editor/components/voice-session-labels.ts index f4ea8f4fa3e..a6b6fb3f0c2 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/components/voice-session-labels.ts +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/components/voice-session-labels.ts @@ -34,5 +34,6 @@ export const voiceSessionActionLabels = { reconnect: "Reconnect voice mode", repeatQuestion: "Repeat question", resume: "Resume voice mode", + takeTurn: "Your turn", unmute: "Unmute microphone", } as const; diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx index eab9bafbff7..5fb1408a4d5 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx @@ -1055,9 +1055,10 @@ describe("AiAssistantPanel composer submissions", () => { expect(submitVoiceInputReferences.size).toBe(1); }); - test("forwards registered canonical replay controls to the Voice dock", async () => { + test("forwards registered replay and handoff controls to the Voice dock", async () => { const readFullResponse = vi.fn(); const repeatQuestion = vi.fn(); + const takeTurn = vi.fn(); const VoiceMode = (context: PetrinautAiVoiceModeContext) => { const { inputMode, @@ -1078,10 +1079,12 @@ describe("AiAssistantPanel composer submissions", () => { repeatQuestion, resume: vi.fn(), setMicrophoneMuted: vi.fn(), + takeTurn, }); reportVoiceSessionState({ canReadFullResponse: true, canRepeatQuestion: true, + canTakeTurn: true, errorMessage: null, microphoneLevel: 0, microphoneMuted: false, @@ -1111,6 +1114,8 @@ describe("AiAssistantPanel composer submissions", () => { }); fireEvent.click(screen.getByRole("button", { name: "Start voice mode" })); + fireEvent.click(screen.getByRole("button", { name: "Your turn" })); + expect(takeTurn).toHaveBeenCalledOnce(); fireEvent.click( screen.getByRole("button", { name: "Voice playback options" }), ); diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx index cc4f11010db..50074c1ba32 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx @@ -495,6 +495,7 @@ export const AiAssistantPanel = ({ : {}), resume: () => controls.resume(), setMicrophoneMuted: (muted) => controls.setMicrophoneMuted(muted), + ...(controls.takeTurn ? { takeTurn: controls.takeTurn } : {}), }); return () => { diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/voice-dock.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/voice-dock.tsx index 823bbed7fac..6ba096d89cf 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/voice-dock.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/voice-dock.tsx @@ -5,6 +5,7 @@ import { useVoiceSessionActions, useVoiceSessionCanReadFullResponse, useVoiceSessionCanRepeatQuestion, + useVoiceSessionCanTakeTurn, useVoiceSessionMicrophoneMuted, useVoiceSessionPhase, } from "../../../../../../react/voice-session/use-voice-session"; @@ -105,6 +106,7 @@ export type VoiceDockProps = { actions: VoiceSessionActions | null; canReadFullResponse: boolean; canRepeatQuestion: boolean; + canTakeTurn: boolean; /** Rendered instead of the live indicator when the caller supplies one. */ indicator?: ReactNode; microphoneMuted: boolean; @@ -124,6 +126,7 @@ export const VoiceDock = ({ actions, canReadFullResponse, canRepeatQuestion, + canTakeTurn, indicator, microphoneMuted, onTranscriptionToggle, @@ -176,6 +179,16 @@ export const VoiceDock = ({ {actions !== null && ( <> + {canTakeTurn && actions.takeTurn && ( + + )} {phase === "error" ? ( ) : ( -

{submittedOutput.approved ? "Approved" : "Declined"}

+

+ {submittedOutputPrefix} + {submittedOutput.approved ? "Approved" : "Declined"} +

), }); @@ -84,11 +95,14 @@ checked before Petrinaut calls the AI SDK's `addToolOutput`. The component receives a stable `toolCallId` plus a discriminated lifecycle: `state: "awaiting"` has no submitted output, while `state: "submitted"` -includes the validated `submittedOutput`. While a submission is in flight, -duplicate `submit` calls are ignored. An accepted submission stays one-shot; -if the AI SDK rejects it, the awaiting component can submit again. Once every -pending tool call has output, the existing AI SDK automatic follow-up runs as -usual. +includes the validated `submittedOutput`. A tool can set +`supportsSubmittedOutputPrefix: true` and render `submittedOutputPrefix` +immediately before its submitted value; Petrinaut uses that slot for inline +provenance when available. Widgets that omit the opt-in keep the trailing +provenance fallback. While a submission is in flight, duplicate `submit` calls +are ignored. An accepted submission stays one-shot; if the AI SDK rejects it, +the awaiting component can submit again. Once every pending tool call has +output, the existing AI SDK automatic follow-up runs as usual. Tool names must be unique within the host registry and must not collide with a built-in Petrinaut tool such as `applyAutoLayout`. A dynamic tool call with no diff --git a/libs/@hashintel/petrinaut/docs/ai-assistant.md b/libs/@hashintel/petrinaut/docs/ai-assistant.md index 53939f99f3a..51e13cb0838 100644 --- a/libs/@hashintel/petrinaut/docs/ai-assistant.md +++ b/libs/@hashintel/petrinaut/docs/ai-assistant.md @@ -59,8 +59,9 @@ never held back either way: anything you typed, and any inline question waiting the session ends, the held turns appear together under a **Voice session · N turns** divider. Only finalized answers and canonical Brunch text become chat history; provisional transcription and Realtime audio are ephemeral. Finalized spoken user messages carry a small **Voice** chip in front of -the words themselves, and the exact inline answer completed by speech carries the same chip, so Voice -provenance remains visible without duplicating an answer. +the words themselves. The submitted-answer box for the exact inline question completed by speech +also begins with the same chip immediately before the answer, so Voice provenance remains visible +without duplicating an answer as a separate user message. Microphone capture pauses while the interviewer speaks so speaker echo cannot be submitted as your answer. Wait for playback to finish; Voice resumes listening automatically in your previous mute diff --git a/libs/@hashintel/petrinaut/src/ui/types/ai-interactive-tool.ts b/libs/@hashintel/petrinaut/src/ui/types/ai-interactive-tool.ts index ccd5aa95fff..7d576f26e52 100644 --- a/libs/@hashintel/petrinaut/src/ui/types/ai-interactive-tool.ts +++ b/libs/@hashintel/petrinaut/src/ui/types/ai-interactive-tool.ts @@ -1,4 +1,4 @@ -import type { ComponentType } from "react"; +import type { ComponentType, ReactNode } from "react"; /** A runtime parser such as a Zod schema. */ export type PetrinautAiInteractiveToolSchema = { @@ -21,10 +21,13 @@ export type PetrinautAiInteractiveToolWidgetProps = | { state: "awaiting"; submittedOutput?: never; + submittedOutputPrefix?: never; } | { state: "submitted"; submittedOutput: Output; + /** Optional host-positioned content rendered before the submitted value. */ + submittedOutputPrefix?: ReactNode; } ); @@ -45,6 +48,11 @@ export type PetrinautAiInteractiveToolDefinition = { * output before completing the tool call. */ fromComposerText?: (params: { input: Input; text: string }) => Output; + /** + * Opt in to positioning Petrinaut-provided provenance inside the widget, + * immediately before its submitted output. + */ + supportsSubmittedOutputPrefix?: true; /** Inline component shown while awaiting input and after submission. */ component: ComponentType< PetrinautAiInteractiveToolWidgetProps @@ -56,6 +64,7 @@ type ErasedInteractiveToolDefinition = { parseInput: (value: unknown) => unknown; parseOutput: (value: unknown) => unknown; fromComposerText?: (params: { input: unknown; text: string }) => unknown; + supportsSubmittedOutputPrefix?: true; component: ComponentType< PetrinautAiInteractiveToolWidgetProps >; @@ -93,6 +102,7 @@ export const definePetrinautAiInteractiveTool = ( }), ) : undefined, + supportsSubmittedOutputPrefix: definition.supportsSubmittedOutputPrefix, component: definition.component as ComponentType< PetrinautAiInteractiveToolWidgetProps >, diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx index 84af4b577f0..b244dd2df2e 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx @@ -649,11 +649,11 @@ describe("AiAssistantContents", () => { />, ); - expect( - within( - screen.getByText("Spoken workflow").closest("[data-role]")!, - ).getByTestId("voice-input-provenance"), - ).not.toBeNull(); + const spokenText = screen.getByText("Spoken workflow"); + const spokenProvenance = within( + spokenText.closest("[data-role]")!, + ).getByTestId("voice-input-provenance"); + expect(spokenProvenance.nextElementSibling).toBe(spokenText); expect( within( screen.getByText("Typed follow-up").closest("[data-role]")!, @@ -670,8 +670,17 @@ describe("AiAssistantContents", () => { outputSchema: { parse: (raw: unknown) => raw as { answer: string }, }, - component: ({ submittedOutput, toolCallId }) => ( - {`${toolCallId}: ${submittedOutput?.answer}`} + supportsSubmittedOutputPrefix: true, + component: ({ + state, + submittedOutput, + submittedOutputPrefix, + toolCallId, + }) => ( + + {state === "submitted" ? submittedOutputPrefix : null} + {`${toolCallId}: ${submittedOutput?.answer}`} + ), }); const messages = [ @@ -713,25 +722,76 @@ describe("AiAssistantContents", () => { />, ); + const voiceAnswer = screen.getByTestId("answer-question-voice"); + const voiceProvenance = within(voiceAnswer).getByTestId( + "voice-input-provenance", + ); + const voiceText = screen.getByText("question-voice: The shift lead"); + expect(voiceProvenance.nextElementSibling).toBe(voiceText); expect( - within( - screen - .getByText("question-voice: The shift lead") - .closest("[data-tool-call-id]")!, - ).getByTestId("voice-input-provenance"), - ).not.toBeNull(); - expect( - within( - screen - .getByText("question-typed: The operator") - .closest("[data-tool-call-id]")!, - ).queryByTestId("voice-input-provenance"), + within(screen.getByTestId("answer-question-typed")).queryByTestId( + "voice-input-provenance", + ), ).toBeNull(); expect(screen.getAllByTestId("voice-input-provenance")).toHaveLength(1); expect(screen.queryByText("The shift lead", { exact: true })).toBeNull(); expect(container.querySelectorAll('[data-role="user"]')).toHaveLength(0); }); + test("keeps trailing voice provenance for interactive tools without a submitted-output prefix", () => { + const hostTool = definePetrinautAiInteractiveTool({ + toolName: "legacyAnswerQuestion", + inputSchema: { + parse: (raw: unknown) => raw as { question: string }, + }, + outputSchema: { + parse: (raw: unknown) => raw as { answer: string }, + }, + component: ({ submittedOutput }) => ( + {submittedOutput?.answer} + ), + }); + const messages = [ + { + id: "assistant-legacy-question", + metadata: { source: "voice", toolCallId: "legacy-question" }, + role: "assistant", + parts: [ + { + type: "dynamic-tool", + toolName: "legacyAnswerQuestion", + state: "output-available", + toolCallId: "legacy-question", + input: { question: "Who approves it?" }, + output: { answer: "The shift lead" }, + }, + ], + }, + ] as unknown as PetrinautAiMessage[]; + + render( + , + ); + + const legacyAnswer = screen.getByTestId("legacy-answer"); + const toolContainer = legacyAnswer.closest( + "[data-tool-call-id]", + ) as HTMLElement; + const voiceProvenance = within(toolContainer).getByTestId( + "voice-input-provenance", + ); + expect(legacyAnswer.nextElementSibling).toBe(voiceProvenance); + }); + test("keeps completed messages memoized when interactive tools are omitted", () => { const messages: PetrinautAiMessage[] = [ { diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/tool-list.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/tool-list.tsx index f566adab421..d0129bd9d53 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/tool-list.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/tool-list.tsx @@ -525,6 +525,9 @@ const InteractiveToolItem = ({ const typedInput = definition.parseInput(input); if (submitted) { + const positionsVoicePrefix = + tool.voiceOrigin && definition.supportsSubmittedOutputPrefix === true; + return (
{}} submittedOutput={definition.parseOutput(submittedOutput)} + submittedOutputPrefix={ + positionsVoicePrefix ? : undefined + } toolCallId={tool.id} /> - {tool.voiceOrigin && } + {tool.voiceOrigin && !positionsVoicePrefix && }
); } diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/interactive-tools/registry.test.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/interactive-tools/registry.test.tsx index 4204e6a3a34..1bc86260b2a 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/interactive-tools/registry.test.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/interactive-tools/registry.test.tsx @@ -20,6 +20,7 @@ const hostTool = definePetrinautAiInteractiveTool({ outputSchema: { parse: (raw: unknown) => raw as { approved: boolean }, }, + supportsSubmittedOutputPrefix: true, component: () => null, }); @@ -35,6 +36,7 @@ describe("interactive tool registry", () => { expect(definition.parseInput({ question: "Ship this change?" })).toEqual({ question: "Ship this change?", }); + expect(definition.supportsSubmittedOutputPrefix).toBe(true); expect(() => definition.parseInput({ question: 42 })).toThrow( "Expected a question", ); diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/interactive-tools/registry.ts b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/interactive-tools/registry.ts index 6f75651ceca..00799661528 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/interactive-tools/registry.ts +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/interactive-tools/registry.ts @@ -63,6 +63,8 @@ export const getInteractiveTool = ( parseInput: hostDefinition.parseInput, parseOutput: hostDefinition.parseOutput, fromComposerText: hostDefinition.fromComposerText, + supportsSubmittedOutputPrefix: + hostDefinition.supportsSubmittedOutputPrefix, Widget: hostDefinition.component, } : undefined; diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/interactive-tools/types.ts b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/interactive-tools/types.ts index 5de09166f9a..9e13ba1a9d6 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/interactive-tools/types.ts +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/interactive-tools/types.ts @@ -31,5 +31,7 @@ export type InteractiveToolDefinition = { parseOutput: (raw: unknown) => Output; /** Map composer text to a validated tool output when the host opts in. */ fromComposerText?: (params: { input: Input; text: string }) => Output; + /** Whether the widget positions a submitted-output prefix supplied by Petrinaut. */ + supportsSubmittedOutputPrefix?: true; Widget: ComponentType>; }; From a6cfd99617952cd393c229467a964fa5998449fd Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Wed, 2 Sep 2026 20:35:07 +0200 Subject: [PATCH 03/10] Document manual Voice turn handoff Amp-Thread-ID: https://ampcode.com/threads/T-01a06344-05c3-75c8-ac3a-d8a6fd3ede6c Co-authored-by: Amp --- .../@hashintel/petrinaut/docs/ai-assistant.md | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/libs/@hashintel/petrinaut/docs/ai-assistant.md b/libs/@hashintel/petrinaut/docs/ai-assistant.md index 51e13cb0838..72aaa92a8d9 100644 --- a/libs/@hashintel/petrinaut/docs/ai-assistant.md +++ b/libs/@hashintel/petrinaut/docs/ai-assistant.md @@ -64,17 +64,22 @@ also begins with the same chip immediately before the answer, so Voice provenanc without duplicating an answer as a separate user message. Microphone capture pauses while the interviewer speaks so speaker echo cannot be submitted as your -answer. Wait for playback to finish; Voice resumes listening automatically in your previous mute -state. Semantic voice detection finishes each answer after a natural pause and is tuned to allow -longer thinking pauses. There is no required done-speaking action. +answer. Wait for playback to finish, or select **Your turn** while the interviewer is preparing or +speaking. **Your turn** cancels pending speech, stops playback, and clears audio already buffered for +the turn without disconnecting Voice. The microphone reopens only after that cancellation finishes, +and only speech that begins after you selected **Your turn** is accepted as your answer. The +interrupted question remains visible and available from **Repeat question** if you want to hear it +again. Voice otherwise resumes listening automatically in your previous mute state. Semantic voice +detection finishes each answer after a natural pause and is tuned to allow longer thinking pauses. +There is no required done-speaking action. Every session control lives in the dock: **Show transcription in chat** on the left, and on the right -**Mute microphone** (**Unmute microphone** once muted) beside **End voice mode**. Muting stops -sending audio without ending the turn, so the assistant plays out whatever it is saying and unmuting -drops you straight back into the conversation. **Resume voice mode** replaces the microphone action -while a session is paused, and **Reconnect voice mode** replaces it after a failure. Nothing is added -to the canvas toolbar. To send a typed message, first select **End voice mode** to restore the -composer. +**Your turn** while the interviewer is preparing or speaking, **Mute microphone** (**Unmute +microphone** once muted), and **End voice mode**. Muting stops sending audio without ending the turn, +so the assistant plays out whatever it is saying and unmuting drops you straight back into the +conversation. **Resume voice mode** replaces the microphone action while a session is paused, and +**Reconnect voice mode** replaces it after a failure. Nothing is added to the canvas toolbar. To send +a typed message, first select **End voice mode** to restore the composer. The interviewer uses a warm, calm, curious, and professionally neutral voice and treats you as the authority on your system. Brunch still chooses every question and interview decision. Its exact From a6eb1e688b317dcf9dcf2284cad2d52d0a83b785 Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Wed, 2 Sep 2026 20:40:58 +0200 Subject: [PATCH 04/10] Consolidate Petrinaut release notes Amp-Thread-ID: https://ampcode.com/threads/T-01a06357-5d4a-728b-8b49-57d6d1458733 Co-authored-by: Amp --- .changeset/manual-voice-turn.md | 5 ----- .changeset/stable-composer-controls.md | 14 ++++++++------ 2 files changed, 8 insertions(+), 11 deletions(-) delete mode 100644 .changeset/manual-voice-turn.md diff --git a/.changeset/manual-voice-turn.md b/.changeset/manual-voice-turn.md deleted file mode 100644 index e1bcac4e251..00000000000 --- a/.changeset/manual-voice-turn.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@hashintel/petrinaut": patch ---- - -Add an accessible manual Your turn control for cancelling Voice playback before speaking. diff --git a/.changeset/stable-composer-controls.md b/.changeset/stable-composer-controls.md index 2b0ee67cdb7..85afa146df0 100644 --- a/.changeset/stable-composer-controls.md +++ b/.changeset/stable-composer-controls.md @@ -30,12 +30,14 @@ per-session Show transcription in chat action write those turns into the convers instead. Keep every session control -- transcription, the microphone toggle, Resume, Reconnect, and End -- in the dock, leaving the canvas toolbar untouched. Add `setMicrophoneMuted` to the Voice mode controls and a `muted` session phase, so muting stops capture without interrupting what the assistant -is saying, unlike pausing. Surface voice recovery failures as toasts with privacy-safe diagnostic -references, and request one-time consent before the host starts the microphone. Mark persisted spoken -messages and the exact interactive-tool answer completed by Voice with an inline Voice chip ahead of -the words themselves. Add a backwards-compatible submitted-output prefix slot to interactive-tool -widgets: opted-in widgets can place Voice provenance inside their submitted-value box, while existing -widgets retain the trailing fallback. +is saying, unlike pausing. Add an accessible manual Your turn control that cancels pending Voice +speech and hands the live microphone turn back to the user without disconnecting the session. Surface +voice recovery failures as toasts with privacy-safe diagnostic references, and request one-time +consent before the host starts the microphone. Mark persisted spoken messages and the exact +interactive-tool answer completed by Voice with an inline Voice chip ahead of the words themselves. +Add a backwards-compatible submitted-output prefix slot to interactive-tool widgets: opted-in widgets +can place Voice provenance inside their submitted-value box, while existing widgets retain the +trailing fallback. End Voice mode before submitting typed text exactly once through the shared composer, preserving the draft if handoff fails. Pause active media before the AI panel closes and reopen the mounted session From a573816e81ea9705b60f4cd08299730b45198427 Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Wed, 2 Sep 2026 21:45:09 +0200 Subject: [PATCH 05/10] Match Your turn to Voice dock controls Amp-Thread-ID: https://ampcode.com/threads/T-01a06344-05c3-75c8-ac3a-d8a6fd3ede6c Co-authored-by: Amp --- .../ui/views/Editor/panels/ai-assistant-panel.test.tsx | 5 ++++- .../ai-assistant-contents/voice-dock.tsx | 9 +++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx index 5fb1408a4d5..a90930f176e 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx @@ -1114,7 +1114,10 @@ describe("AiAssistantPanel composer submissions", () => { }); fireEvent.click(screen.getByRole("button", { name: "Start voice mode" })); - fireEvent.click(screen.getByRole("button", { name: "Your turn" })); + const takeTurnButton = screen.getByRole("button", { name: "Your turn" }); + expect(takeTurnButton.getAttribute("data-variant")).toBe("ghost"); + expect(takeTurnButton.querySelector("svg")).not.toBeNull(); + fireEvent.click(takeTurnButton); expect(takeTurn).toHaveBeenCalledOnce(); fireEvent.click( screen.getByRole("button", { name: "Voice playback options" }), diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/voice-dock.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/voice-dock.tsx index 6ba096d89cf..d2ad1219ec7 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/voice-dock.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/voice-dock.tsx @@ -181,13 +181,14 @@ export const VoiceDock = ({ <> {canTakeTurn && actions.takeTurn && ( + variant="ghost" + /> )} {phase === "error" ? (