From 8f6b16f0151e48540fc1241455bb3269db67531b Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Mon, 31 Aug 2026 15:25:29 +0200 Subject: [PATCH 01/16] Integrate voice mode into the AI assistant transcript Keep text and voice in one conversation with inline controls, provenance, lifecycle recovery, and accessible live feedback. Co-authored-by: Cursor --- .changeset/stable-composer-controls.md | 33 +- apps/petrinaut-website/README.md | 85 +- .../local-storage-demo-app.test.tsx | 23 +- .../local-storage-demo-app.tsx | 20 +- .../voice-interview-control.test.tsx | 1142 +++----------- .../voice-interview-control.tsx | 1370 +++-------------- .../voice-interview-inline-view.test.tsx | 537 +++++++ .../voice-interview-inline-view.tsx | 534 +++++++ .../voice-turn-controller.test.ts | 55 + .../voice-interview/voice-turn-controller.ts | 25 +- .../adr/0009-openai-voice-ui-turn-shell.md | 25 +- .../docs/specs/petrinaut-integration.md | 55 +- libs/@hashintel/petrinaut-core/src/ai.ts | 2 +- libs/@hashintel/petrinaut/CHANGELOG.md | 23 +- libs/@hashintel/petrinaut/docs/README.md | 2 +- .../@hashintel/petrinaut/docs/ai-assistant.md | 101 +- libs/@hashintel/petrinaut/src/ui/index.ts | 12 +- .../@hashintel/petrinaut/src/ui/petrinaut.tsx | 6 +- .../ui/types/ai-assistant-composer-control.ts | 76 +- .../Editor/components/ai-cta-modal.test.tsx | 112 +- .../views/Editor/components/ai-cta-modal.tsx | 183 +-- .../ai-interaction-mode-tabs.test.tsx | 50 - .../components/ai-interaction-mode-tabs.tsx | 91 -- .../components/ai-voice-mode-button.tsx | 42 + .../src/ui/views/Editor/editor-view.tsx | 14 +- .../Editor/panels/ai-assistant-panel.test.tsx | 1271 ++++++++++++--- .../Editor/panels/ai-assistant-panel.tsx | 393 +++-- .../ai-assistant-contents.stories.tsx | 250 ++- .../ai-assistant-contents.test.tsx | 329 +++- .../ai-assistant-contents.tsx | 271 ++-- .../ai-assistant-contents/tool-list.tsx | 83 +- .../voice-input-provenance.tsx | 41 + .../Editor/panels/ai-assistant-panel/types.ts | 9 +- 33 files changed, 4174 insertions(+), 3091 deletions(-) create mode 100644 apps/petrinaut-website/src/main/app/voice-interview/voice-interview-inline-view.test.tsx create mode 100644 apps/petrinaut-website/src/main/app/voice-interview/voice-interview-inline-view.tsx delete mode 100644 libs/@hashintel/petrinaut/src/ui/views/Editor/components/ai-interaction-mode-tabs.test.tsx delete mode 100644 libs/@hashintel/petrinaut/src/ui/views/Editor/components/ai-interaction-mode-tabs.tsx create mode 100644 libs/@hashintel/petrinaut/src/ui/views/Editor/components/ai-voice-mode-button.tsx create mode 100644 libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/voice-input-provenance.tsx diff --git a/.changeset/stable-composer-controls.md b/.changeset/stable-composer-controls.md index 36cc6644f20..f2c6e44ecb9 100644 --- a/.changeset/stable-composer-controls.md +++ b/.changeset/stable-composer-controls.md @@ -2,16 +2,25 @@ "@hashintel/petrinaut": patch --- -Add generic host-rendered AI composer controls and a persistent interview stage with docked and -detached placements, protected active conversations, keyboard fallback, and one-answer buffering -while the normal chat stream settles. Include stable finalized-text submission, conversation -identity, stop handling, schema-validated interactive-tool text mapping, explicit separate-message -targeting for corrections, and a queue-aware voice submission path. Add the Chat / Interview mode -switch and export `PetrinautAiInteractionMode`, with the selected interaction mode and mode-change -callback available to host-rendered interview stages. `renderComposerControl` remains a supported -public seam for hosts that only need their own control beside the message box, independently of the -interview stage. +Add generic host-rendered AI composer controls and a persistent inline Voice session, protected +active conversations, keyboard fallback, and one-answer buffering while the normal chat stream +settles. Add the provider-neutral `renderVoiceMode` contract and export `PetrinautAiInputMode`, with +the selected input mode and mode-change callback available to host-rendered Voice modes. Include +stable finalized-text submission, conversation identity, stop handling, schema-validated +interactive-tool text mapping, explicit separate-message targeting for corrections, and a +queue-aware voice submission path. Present text and voice through one transcript and composer whose +trailing action switches between waveform, Send, and Stop. `renderComposerControl` remains a +supported public seam for hosts that only need their own control beside the message box, +independently of Voice mode. -Simplify Interview mode with a circular microphone waveform, compact transcript states that -distinguish recording, sending, sent, and undelivered answers, phase-specific icon controls, and -recovery that names the kind of failure before offering reconnect. +Show partial speech as an ephemeral user-style bubble and render compact waveform provenance on +persisted spoken messages and the exact interactive-tool answer completed by Voice. Keep session +state in a subtle transcript divider with input-responsive listening bars, deterministic speaking +motion, throttled announcements, reduced-motion behavior, and compact recovery controls. Keep Pause +and End voice mode in an overflow menu, expose collapsed privacy-safe technical details, and request +one-time consent before the host starts the microphone. + +End Voice mode before submitting typed text exactly once through the shared composer, preserving the +draft if handoff fails. Pause active media before the AI panel closes and reopen the mounted session +paused. Provisional transcription and Realtime audio remain ephemeral rather than becoming +persisted chat history. diff --git a/apps/petrinaut-website/README.md b/apps/petrinaut-website/README.md index 8951b4136b6..958a1b0b8b2 100644 --- a/apps/petrinaut-website/README.md +++ b/apps/petrinaut-website/README.md @@ -74,60 +74,73 @@ provides a fake optimizer for isolated UI development. Local values live in `.env.local`; Vite's `loadEnv` (see [`vite.config.ts`](vite.config.ts)) copies them into `process.env` for both the dev server and the API functions. In production, set these in the Vercel project settings. -### Brunch voice preview +### Brunch Voice mode preview -Voice input is disabled by default and always unavailable when `VERCEL_ENV` is +Voice mode is disabled by default and always unavailable when `VERCEL_ENV` is `production`. To exercise the preview locally or in a Vercel preview, set a real `VITE_BRUNCH_CHAT_ENDPOINT`, `PETRINAUT_OPENAI_VOICE_ENABLED=true`, and a -dedicated `OPENAI_VOICE_API_KEY`. The browser sends its SDP offer to this app; -the server initializes a trusted `gpt-realtime-2` audio-input/audio-output -session through OpenAI's unified Realtime call endpoint. The provider key, -model, instructions, tools, language and vocabulary policy stay server-side. -The session uses semantic VAD with low eagerness so natural thinking pauses are -less likely to end an answer early. +dedicated `OPENAI_VOICE_API_KEY`. + +Text and Voice mode use one assistant transcript and composer. When Voice mode +is available, the empty first-run prompt and empty composer show a waveform +action; non-whitespace text replaces it with **Send**, and a busy assistant +shows **Stop**. Starting Voice mode opens an inline, versioned consent +disclosure before requesting microphone access. The disclosure also provides a +microphone check and is remembered in browser storage only after Voice mode +starts. + +An active session stays at the end of the transcript. Its compact divider shows +a waveform and **Connecting**, **Listening**, **Speaking**, **Paused**, or a +recovery state. Listening levels follow microphone input; provisional words +appear immediately above the divider in an ephemeral user-style bubble. The +bubble is replaced by the finalized message or pending-question tool output, +which retains a waveform indicator without duplicating the answer. Provisional +transcription and Realtime audio are not persisted as chat history. + +The text composer remains available. Sending typed text ends Voice mode first, +then submits the draft exactly once through the same conversation; a failed +handoff restores the draft. Closing the assistant pauses capture and speech +before hiding it. Reopening preserves the mounted session in **Paused** state. +**Pause** and **End voice mode** live under **Voice mode actions**, while +**Resume** or **Reconnect** appears as the primary action when applicable. + +The browser sends its SDP offer to this app; the server initializes a trusted +`gpt-realtime-2` audio-input/audio-output session through OpenAI's unified +Realtime call endpoint. The provider key, model, instructions, tools, language, +and vocabulary policy stay server-side. The session uses semantic VAD with low +eagerness so natural thinking pauses are less likely to end an answer early. Realtime is the disposable media plane: it carries continuous microphone and -remote audio, detects complete turns and handles barge-in. Brunch remains the -control plane and sole authority for interview questions, captures, state, -completion and durable history. The browser bridge accepts only the configured +remote audio, detects complete turns, and handles barge-in. Brunch remains the +control plane and sole authority for questions, captures, state, completion, +and durable history. The browser bridge accepts only the configured `continue_interview` function, validates and serializes its arguments, rejects -duplicate or stale calls and submits the answer through Petrinaut's existing +duplicate or stale calls, and submits the answer through Petrinaut's shared composer path with pending-`brunch_ask` correlation. The bridge waits for the correlated Brunch turn before returning canonical speech segments to Realtime. It then requests audio with tools disabled and -instructs Realtime to speak only those segments. Generated audio is not treated -as a verbatim record: canonical Brunch text remains visible and authoritative. -Partial input transcripts are display-only and never submit or disable capture. - -The microphone remains active while the interviewer speaks and while Brunch is +instructs Realtime to speak only those segments. Generated audio is not a +verbatim record: canonical Brunch text remains visible and authoritative. The +microphone stays active while the interviewer speaks and while Brunch is working. Speaking over assistant audio interrupts playback automatically; WebRTC truncates provider-side unheard audio without changing Brunch history. -**Pause** temporarily turns off input, while **End interview**, failure and -reconnect release or replace all media and invalidate events from old -connections. There is no separate Speech API, MP3 playback queue, required -**Done speaking** action or manual **Interrupt and speak** action. The Brunch deployment must allow the website origin through its -`BRUNCH_PETRINAUT_ORIGINS` setting. Starting voice input requests browser -microphone permission. Denying permission leaves the existing text composer -available and does not submit anything to Brunch. - -When the preview cannot continue, the status panel distinguishes microphone -permission, microphone device, interrupted request, network, timeout, invalid -response, and unavailable/disabled failures. Permission and device failures -identify what to fix; network, timeout, and interrupted requests offer a -reconnect; invalid responses include a diagnostic reference for an operator; -and unavailable voice leaves the text composer as the fallback. Voice failures -always leave the canonical response visible to read. - -Realtime connection, transcription and canonical speech timings use random +`BRUNCH_PETRINAUT_ORIGINS` setting. Denying microphone permission leaves the +text composer available and submits nothing to Brunch. When Voice mode cannot +continue, the inline recovery state distinguishes microphone, connection, and +other Voice failures, explains the next action, and offers **Reconnect** where +appropriate. Sanitized error codes and diagnostic references remain collapsed +under **Technical details**. + +Realtime connection, transcription, and canonical speech timings use random request IDs, and the existing Brunch transport provides its own request correlation. Browser and server diagnostics report only operation, stage, -outcome, duration, request ID and—where applicable—status or a sanitized error +outcome, duration, request ID, and—where applicable—status or a sanitized error code. Voice responses also expose privacy-safe `Server-Timing` metrics. These diagnostics never record audio, SDP, transcript or prompt contents, canonical -speech text, credentials or provider response bodies. This controlled-preview +speech text, credentials, or provider response bodies. This controlled-preview evidence does not enable production: production remains unconditionally disabled by the server policy. diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx index 53948c0074a..b069556fca2 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx @@ -5,7 +5,7 @@ import { isValidElement, type ReactNode } from "react"; import { describe, expect, test, vi } from "vitest"; import { VoiceInterviewControl } from "../voice-interview/voice-interview-control"; -import { getBrunchVoiceInterviewStage } from "./local-storage-demo-app"; +import { getBrunchVoiceMode } from "./local-storage-demo-app"; const defaultTransportOptions = vi.hoisted(() => ({ current: null as unknown, @@ -28,25 +28,24 @@ vi.mock("@hashintel/petrinaut/ui", () => ({ describe("local storage demo Brunch voice integration", () => { test("does not install voice on the generic local chat fallback", () => { - expect(getBrunchVoiceInterviewStage(null)).toBeUndefined(); + expect(getBrunchVoiceMode(null)).toBeUndefined(); }); test("installs the app-owned voice control for a configured Brunch transport", () => { const config = { available: true as const, connectionTimeoutMs: 15_000 }; - const stage = getBrunchVoiceInterviewStage(config); - const control = stage?.({ - canAcceptInterviewAnswer: true, + const voiceMode = getBrunchVoiceMode(config); + const control = voiceMode?.({ + canAcceptVoiceInput: true, conversationId: "petrinaut-preview:net-1", - focusComposer: vi.fn(), - interactionMode: "chat", + inputMode: "text", + isAiAssistantOpen: true, messages: [], - openSidebar: vi.fn(), - placement: "sidebar", - setActive: vi.fn(), - setInteractionMode: vi.fn(), + registerVoiceModeControls: vi.fn(() => () => undefined), + setInputMode: vi.fn(), + setVoiceActive: vi.fn(), status: "ready", stop: vi.fn(async () => undefined), - submitInterviewAnswer: vi.fn(async () => ({ + submitVoiceInput: vi.fn(async () => ({ kind: "message" as const, messageId: "message-1", })), diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx index adf36eb0860..dd79961d013 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx @@ -17,9 +17,9 @@ import { import { DefaultChatTransport, Petrinaut, - type PetrinautAiInterviewStage, - type PetrinautAiInterviewStageContext, type PetrinautAiMessage, + type PetrinautAiVoiceMode, + type PetrinautAiVoiceModeContext, WalkthroughProvider, } from "@hashintel/petrinaut/ui"; @@ -95,11 +95,11 @@ const brunchPreviewConfig = resolveBrunchPreviewConfig( import.meta.env.VITE_BRUNCH_CHAT_ENDPOINT, ); -export const getBrunchVoiceInterviewStage = ( +export const getBrunchVoiceMode = ( config: OpenAIVoiceConfig | null | undefined, -): PetrinautAiInterviewStage | undefined => +): PetrinautAiVoiceMode | undefined => config - ? (context: PetrinautAiInterviewStageContext) => ( + ? (context: PetrinautAiVoiceModeContext) => ( ) : undefined; @@ -181,8 +181,8 @@ export const LocalStorageDemoApp = () => { return () => abortController.abort(); }, []); - const brunchVoiceInterviewStage = useMemo( - () => getBrunchVoiceInterviewStage(openAIVoiceConfig), + const brunchVoiceMode = useMemo( + () => getBrunchVoiceMode(openAIVoiceConfig), [openAIVoiceConfig], ); @@ -361,15 +361,15 @@ export const LocalStorageDemoApp = () => { return next; }); }, - ...(brunchVoiceInterviewStage + ...(brunchVoiceMode ? { - renderInterviewStage: brunchVoiceInterviewStage, + renderVoiceMode: brunchVoiceMode, } : {}), }), [ aiMessagesByNetId, - brunchVoiceInterviewStage, + brunchVoiceMode, conversationId, currentNetId, flueHistory.messages, diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx index 7ee6d71f1d9..3ac1e4bbf2e 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx @@ -6,103 +6,59 @@ import { fireEvent, render, screen, + within, waitFor, } from "@testing-library/react"; import { StrictMode, useState } from "react"; -import { renderToStaticMarkup } from "react-dom/server"; import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; import { OpenAIRealtimeSession } from "./openai-realtime-session"; import { acknowledgeVoiceInterviewDisclosure, + isVoiceInputRepresented, isVoiceInterviewDisclosureAcknowledged, loadOpenAIVoiceConfig, VOICE_INTERVIEW_DISCLOSURE_STORAGE_KEY, VoiceInterviewControl, - VoiceInterviewControlView, - type VoiceInterviewControlViewProps, } from "./voice-interview-control"; +import { VoiceTurnController } from "./voice-turn-controller"; -import type { VoiceTurnSnapshot } from "./voice-turn-controller"; -import type { PetrinautAiInterviewStageContext } from "@hashintel/petrinaut/ui"; - -const snapshot = { - canReviseLastAnswer: false, - connection: "connected", - currentQuestion: "What happens after approval?", - errorCode: null, - errorMessage: "", - errorRequestId: "", - input: "listening", - lastAnswerDelivery: "none" as const, - lastCommittedText: "", - microphoneEnabled: true, - microphoneLevel: 0.24, - output: "idle", - partialText: "The request goes to", -} satisfies VoiceTurnSnapshot; +import type { PetrinautAiVoiceModeContext } from "@hashintel/petrinaut/ui"; const config = { available: true as const, connectionTimeoutMs: 15_000 }; -const viewProps = ( - overrides: Partial = {}, -): VoiceInterviewControlViewProps => ({ - consented: true, - correction: "", - coverage: null, - editing: false, - microphoneCheck: "", - onCheckMicrophone: vi.fn(), - onConsentChange: vi.fn(), - onCorrectionChange: vi.fn(), - onEdit: vi.fn(), - onEnd: vi.fn(), - onExpand: vi.fn(), - onMinimize: vi.fn(), - onPause: vi.fn(), - onReconnect: vi.fn(), - onResume: vi.fn(), - onStart: vi.fn(), - onSubmitCorrection: vi.fn(), - onTypeInstead: vi.fn(), - placement: "sidebar", - presentation: "full", - snapshot, - ...overrides, -}); +let registeredVoiceModeControls: + | { + end: () => Promise; + pause: () => void; + } + | undefined; -const StatefulVoiceInterviewHarness = ({ - onFocusComposer = vi.fn(), - onOpenSidebar, -}: { - onFocusComposer?: () => void; - onOpenSidebar: () => void; -}) => { +const VoiceInterviewHarness = () => { "use no memo"; const [active, setActive] = useState(false); - const [interactionMode, setInteractionMode] = - useState("chat"); - const [sidebarOpenRequests, setSidebarOpenRequests] = useState(0); - const context: PetrinautAiInterviewStageContext = { - canAcceptInterviewAnswer: true, - conversationId: "interview-test", - focusComposer: onFocusComposer, - interactionMode, + const [inputMode, setInputMode] = + useState("text"); + const [isAiAssistantOpen, setAiAssistantOpen] = useState(true); + const context: PetrinautAiVoiceModeContext = { + canAcceptVoiceInput: true, + conversationId: "voice-control-test", + inputMode, + isAiAssistantOpen, messages: [], - openSidebar: () => { - onOpenSidebar(); - setSidebarOpenRequests((requests) => requests + 1); + registerVoiceModeControls: (controls) => { + registeredVoiceModeControls = controls; + return () => { + if (registeredVoiceModeControls === controls) { + registeredVoiceModeControls = undefined; + } + }; }, - placement: "sidebar", - setActive, - setInteractionMode, + setInputMode, + setVoiceActive: setActive, status: "ready", stop: vi.fn(async () => undefined), - submitInterviewAnswer: vi.fn(async () => ({ - kind: "message" as const, - messageId: "voice-answer", - })), submitText: vi.fn(async () => ({ kind: "message" as const, messageId: "typed-answer", @@ -115,17 +71,21 @@ const StatefulVoiceInterviewHarness = ({ return ( <> - - - {active ? "Interview active" : "Interview inactive"} - - {interactionMode === "chat" ? "Chat mode" : "Interview mode"} - - {sidebarOpenRequests} sidebar open requests + + {active ? "Voice active" : "Voice inactive"} + {inputMode === "voice" ? "Voice mode" : "Text mode"} + {isAiAssistantOpen ? "Panel open" : "Panel closed"} ); @@ -156,6 +116,26 @@ const stubUnavailableMicrophone = () => { }; beforeEach(() => { + registeredVoiceModeControls = undefined; + vi.stubGlobal( + "PointerEvent", + class extends MouseEvent { + public readonly pointerType: string; + + public constructor(type: string, init: PointerEventInit = {}) { + super(type, init); + this.pointerType = init.pointerType ?? ""; + } + }, + ); + vi.stubGlobal( + "ResizeObserver", + class { + public disconnect() {} + public observe() {} + public unobserve() {} + }, + ); const values = new Map(); Object.defineProperty(window, "localStorage", { configurable: true, @@ -174,12 +154,89 @@ beforeEach(() => { afterEach(() => { cleanup(); + vi.restoreAllMocks(); window.localStorage.clear(); vi.unstubAllGlobals(); }); -describe("voice interview stage", () => { - test("stores and reads the current disclosure acknowledgement", () => { +describe("voice interview control", () => { + test("recognizes only the current canonical voice input representation", () => { + const baselineToolCallIds = new Set(["previous-question"]); + const ordinaryVoiceMessage = { + id: "voice-realtime:2:call-4", + metadata: { source: "voice" }, + parts: [{ text: "Send it to dispatch", type: "text" }], + role: "user", + } as PetrinautAiVoiceModeContext["messages"][number]; + const previousToolMessage = { + id: "assistant-previous", + metadata: { source: "voice", toolCallId: "previous-question" }, + parts: [ + { + input: { question: "Previous question?" }, + output: { answer: "Previous answer" }, + state: "output-available", + toolCallId: "previous-question", + toolName: "answerQuestion", + type: "dynamic-tool", + }, + ], + role: "assistant", + } as unknown as PetrinautAiVoiceModeContext["messages"][number]; + const currentToolMessage = { + id: "assistant-current", + metadata: { source: "voice", toolCallId: "current-question" }, + parts: [ + { + input: { question: "Current question?" }, + output: { answer: "Current answer" }, + state: "output-available", + toolCallId: "current-question", + toolName: "answerQuestion", + type: "dynamic-tool", + }, + ], + role: "assistant", + } as unknown as PetrinautAiVoiceModeContext["messages"][number]; + + expect( + isVoiceInputRepresented([ordinaryVoiceMessage], { + baselineToolCallIds, + messageId: ordinaryVoiceMessage.id, + }), + ).toBe(true); + expect( + isVoiceInputRepresented([previousToolMessage], { + baselineToolCallIds, + messageId: "voice-realtime:2:call-5", + }), + ).toBe(false); + expect( + isVoiceInputRepresented([previousToolMessage, currentToolMessage], { + baselineToolCallIds, + messageId: "voice-realtime:2:call-5", + }), + ).toBe(true); + expect( + isVoiceInputRepresented( + [ + { + ...currentToolMessage, + parts: currentToolMessage.parts.map((part) => ({ + ...part, + state: "input-available", + })) as typeof currentToolMessage.parts, + }, + ], + { + baselineToolCallIds, + messageId: "voice-realtime:2:call-5", + }, + ), + ).toBe(false); + }); + + test("stores and reads the versioned disclosure acknowledgement", () => { const values = new Map(); const storage = { getItem: (key: string) => values.get(key) ?? null, @@ -216,14 +273,13 @@ describe("voice interview stage", () => { const fetch = vi.fn(async () => Response.json({ available: true, connectionTimeoutMs: 15_000 }), ); - await expect(loadOpenAIVoiceConfig(fetch)).resolves.toEqual({ - available: true, - connectionTimeoutMs: 15_000, + + await expect(loadOpenAIVoiceConfig(fetch)).resolves.toEqual(config); + expect(fetch.mock.calls[0]?.[0]).toBe("/api/voice/config"); + expect(fetch.mock.calls[0]?.[1]).toMatchObject({ + cache: "no-store", + method: "GET", }); - const [url, request] = fetch.mock.calls[0]!; - expect(url).toBe("/api/voice/config"); - expect(request).toMatchObject({ cache: "no-store", method: "GET" }); - expect(request?.signal).toBeInstanceOf(AbortSignal); fetch.mockResolvedValueOnce( Response.json({ available: false, connectionTimeoutMs: 15_000 }), @@ -235,869 +291,179 @@ describe("voice interview stage", () => { await expect(loadOpenAIVoiceConfig(fetch)).resolves.toBeNull(); }); - test("shows disclosure and requires consent before starting", () => { - const html = renderToStaticMarkup( - , - ); - render( - , - ); + test("keeps the first-use disclosure inline without a text-handoff action", () => { + render(); - expect(html).toContain("Voice interview"); - expect(html).toContain("Talk through your process with AI"); - expect(html).toContain("OpenAI processes live audio"); - expect(html).toContain("keeps finalized answers"); - expect(html).toContain("not the audio"); - expect(html.indexOf("Start interview")).toBeLessThan( - html.indexOf("Check microphone"), - ); - expect(html).toMatch(/]*disabled[^>]*>Start interview/u); - expect(html).toContain('aria-label="Use text instead"'); - expect(html).toContain("Check microphone"); - expect(html).toContain("pos_absolute"); - expect(html).not.toContain("pos_fixed"); - - const textButton = screen.getByRole("button", { name: "Use text instead" }); - expect(textButton.querySelector("svg")).not.toBeNull(); - expect(textButton.parentElement?.getAttribute("data-scope")).toBe( - "tooltip", - ); - expect(textButton.textContent.replaceAll("\u200B", "").trim()).toBe(""); - }); - - test("keeps diagnostic recovery details visible without reopening the microphone", () => { - const html = renderToStaticMarkup( - , - ); + fireEvent.click(screen.getByRole("button", { name: "Select Voice" })); - expect(html).toContain("We couldn’t reconnect the microphone"); - expect(html).toContain( - "Allow microphone access in your browser settings, then reconnect voice input.", - ); - expect(html).toContain("Technical details"); - expect(html).toContain("microphone-permission"); - expect(html).toContain("voice-request-permission"); - expect(html).toContain(">Reconnect<"); - expect(html).toContain('aria-label="Use text instead"'); - expect(html).not.toContain(">Type instead<"); + const disclosure = screen.getByRole("region", { + name: "Voice mode consent", + }); + expect(disclosure).not.toBeNull(); + expect(within(disclosure).getByText("Voice mode")).not.toBeNull(); + expect( + screen.getByText("OpenAI processes live audio", { exact: false }), + ).not.toBeNull(); + expect( + screen + .getByRole("button", { name: "Start voice mode" }) + .hasAttribute("disabled"), + ).toBe(true); + expect( + screen.queryByRole("button", { name: "Use text instead" }), + ).toBeNull(); + expect( + screen.queryByRole("region", { name: "Voice interview stage" }), + ).toBeNull(); + expect(document.activeElement).toBe(disclosure); }); - test("starts in the full stage and keeps recovery visible under Strict Mode", async () => { + test("starts one inline session after consent and keeps it inline across host presentation changes", async () => { const getUserMedia = stubUnavailableMicrophone(); - const openSidebar = vi.fn(); - render( - + , ); - fireEvent.click(screen.getByRole("button", { name: "Select Interview" })); + fireEvent.click(screen.getByRole("button", { name: "Select Voice" })); fireEvent.click(screen.getByRole("checkbox")); - fireEvent.click(screen.getByRole("button", { name: "Start interview" })); + fireEvent.click(screen.getByRole("button", { name: "Start voice mode" })); - expect( - await screen.findByRole("region", { name: "Voice interview stage" }), - ).not.toBeNull(); - expect( - await screen.findByText( - /Microphone off · Allow microphone access in your browser settings, then reconnect voice input\./u, - ), - ).not.toBeNull(); - expect(screen.getByRole("button", { name: "Reconnect" })).not.toBeNull(); - expect(screen.getByText("Interview active")).not.toBeNull(); - expect(screen.getByText("Interview mode")).not.toBeNull(); - expect(screen.getByText("1 sidebar open requests")).not.toBeNull(); - expect(openSidebar).toHaveBeenCalledOnce(); + const voiceSession = await screen.findByRole("region", { + name: "Voice session", + }); + expect(voiceSession).not.toBeNull(); + expect(screen.getByText("Microphone unavailable")).not.toBeNull(); + expect(screen.getByText("Voice active")).not.toBeNull(); expect(getUserMedia).toHaveBeenCalledOnce(); - - fireEvent.click(screen.getByRole("button", { name: "Select Chat" })); - expect( - screen.getByRole("region", { name: "Voice interview mini bar" }), - ).not.toBeNull(); - expect(screen.getByText("Chat mode")).not.toBeNull(); - expect(openSidebar).toHaveBeenCalledOnce(); - }); - - test("uses full Interview and compact Chat presentations without ending", async () => { - window.localStorage.setItem( - VOICE_INTERVIEW_DISCLOSURE_STORAGE_KEY, - "acknowledged", + expect(document.activeElement?.getAttribute("aria-label")).toBe( + "Voice session", ); - const getUserMedia = stubUnavailableMicrophone(); - render(); - fireEvent.click(screen.getByRole("button", { name: "Select Interview" })); - expect( - await screen.findByRole("region", { name: "Voice interview stage" }), - ).not.toBeNull(); + fireEvent.click(screen.getByRole("button", { name: "Select Text" })); + fireEvent.click(screen.getByRole("button", { name: "Toggle panel" })); - fireEvent.click( - screen.getByRole("button", { name: "Minimize voice interview" }), - ); + expect(screen.getByText("Text mode")).not.toBeNull(); + expect(screen.getByText("Panel closed")).not.toBeNull(); expect( - screen.getByRole("region", { name: "Voice interview mini bar" }), + screen.getByRole("region", { name: "Voice session" }), ).not.toBeNull(); - expect(screen.getByText("Interview active")).not.toBeNull(); - - fireEvent.click( - screen.getByRole("button", { name: /Expand voice interview/u }), - ); expect( - screen.getByRole("region", { name: "Voice interview stage" }), - ).not.toBeNull(); - expect(screen.getByText("Interview mode")).not.toBeNull(); - expect(getUserMedia).toHaveBeenCalledOnce(); + screen.queryByRole("region", { name: "Voice interview mini bar" }), + ).toBeNull(); + expect( + screen.queryByRole("button", { name: /Expand voice interview/u }), + ).toBeNull(); }); - test("ends the interview and returns to Chat", async () => { + test("starts directly after acknowledgement and ends through the inline action", async () => { window.localStorage.setItem( VOICE_INTERVIEW_DISCLOSURE_STORAGE_KEY, "acknowledged", ); stubUnavailableMicrophone(); - render(); + render(); - fireEvent.click(screen.getByRole("button", { name: "Select Interview" })); + fireEvent.click(screen.getByRole("button", { name: "Select Voice" })); + + expect( + screen.queryByRole("region", { name: "Voice mode consent" }), + ).toBeNull(); expect( - await screen.findByRole("region", { name: "Voice interview stage" }), + await screen.findByRole("region", { name: "Voice session" }), ).not.toBeNull(); - fireEvent.click(screen.getByRole("button", { name: "End interview" })); + fireEvent.click(screen.getByRole("button", { name: "Voice mode actions" })); + const endItem = await screen.findByRole("menuitem", { + name: "End voice mode", + }); + fireEvent.pointerMove(endItem, { pointerType: "mouse" }); + await waitFor(() => + expect(endItem.hasAttribute("data-highlighted")).toBe(true), + ); + fireEvent.click(endItem); - expect(screen.getByText("Chat mode")).not.toBeNull(); - expect(screen.getByText("Interview inactive")).not.toBeNull(); await waitFor(() => { expect( - screen.queryByRole("region", { name: "Voice interview stage" }), - ).toBeNull(); - expect( - screen.queryByRole("region", { name: "Voice interview mini bar" }), + screen.queryByRole("region", { name: "Voice session" }), ).toBeNull(); }); + expect(screen.getByText("Text mode")).not.toBeNull(); + expect(screen.getByText("Voice inactive")).not.toBeNull(); }); - test("restarts when Interview is reselected before teardown completes", async () => { + test("pauses once when the panel closes and stays paused after reopening", async () => { window.localStorage.setItem( VOICE_INTERVIEW_DISCLOSURE_STORAGE_KEY, "acknowledged", ); - const connect = vi - .spyOn(OpenAIRealtimeSession.prototype, "connect") - .mockResolvedValue(1); - let finishDisconnect: (() => void) | undefined; - vi.spyOn(OpenAIRealtimeSession.prototype, "disconnect") - .mockImplementationOnce( - () => - new Promise((resolve) => { - finishDisconnect = resolve; - }), - ) - .mockResolvedValue(undefined); + vi.spyOn(OpenAIRealtimeSession.prototype, "connect").mockResolvedValue(1); + vi.spyOn(OpenAIRealtimeSession.prototype, "disconnect").mockResolvedValue(); vi.spyOn( OpenAIRealtimeSession.prototype, "setMicrophoneEnabled", ).mockImplementation(() => {}); - render(); + const pause = vi.spyOn(VoiceTurnController.prototype, "pause"); + render(); - fireEvent.click(screen.getByRole("button", { name: "Select Interview" })); + fireEvent.click(screen.getByRole("button", { name: "Select Voice" })); await screen.findByText("Listening"); - fireEvent.click(screen.getByRole("button", { name: "End interview" })); - - expect(screen.getByText("Chat mode")).not.toBeNull(); - expect(screen.getByText("Interview inactive")).not.toBeNull(); - expect( - screen.queryByRole("region", { name: "Voice interview mini bar" }), - ).toBeNull(); - - fireEvent.click(screen.getByRole("button", { name: "Select Interview" })); - - expect(connect).toHaveBeenCalledOnce(); - expect(screen.getByText("Interview inactive")).not.toBeNull(); - finishDisconnect?.(); - - await waitFor(() => expect(connect).toHaveBeenCalledTimes(2)); - expect(screen.getByText("Interview active")).not.toBeNull(); - expect(screen.getByText("Listening")).not.toBeNull(); - }); - - test("records acknowledgement only when the interview starts", async () => { - window.localStorage.clear(); - stubUnavailableMicrophone(); - render(); - - fireEvent.click(screen.getByRole("button", { name: "Select Interview" })); - fireEvent.click(screen.getByRole("button", { name: "Check microphone" })); - expect( - window.localStorage.getItem(VOICE_INTERVIEW_DISCLOSURE_STORAGE_KEY), - ).toBeNull(); - - fireEvent.click(screen.getByRole("checkbox")); - fireEvent.click(screen.getByRole("button", { name: "Start interview" })); - expect( - window.localStorage.getItem(VOICE_INTERVIEW_DISCLOSURE_STORAGE_KEY), - ).toBe("acknowledged"); - }); + expect(registeredVoiceModeControls).toBeDefined(); - test("does not record acknowledgement when choosing text instead", async () => { - window.localStorage.clear(); - const focusComposer = vi.fn(); - vi.stubGlobal( - "fetch", - vi.fn(async () => - Response.json({ available: true, connectionTimeoutMs: 15_000 }), - ), - ); - render( - , - ); + fireEvent.click(screen.getByRole("button", { name: "Toggle panel" })); + await screen.findByText("Paused"); + expect(pause).toHaveBeenCalledOnce(); - fireEvent.click(screen.getByRole("button", { name: "Select Interview" })); - fireEvent.click(screen.getByRole("button", { name: "Use text instead" })); - expect( - window.localStorage.getItem(VOICE_INTERVIEW_DISCLOSURE_STORAGE_KEY), - ).toBeNull(); - expect(screen.getByText("Chat mode")).not.toBeNull(); - expect(focusComposer).toHaveBeenCalledOnce(); + fireEvent.click(screen.getByRole("button", { name: "Toggle panel" })); + expect(screen.getByText("Paused")).not.toBeNull(); + expect(screen.getByRole("button", { name: "Resume" })).not.toBeNull(); + expect(pause).toHaveBeenCalledOnce(); }); - test("uses text from an active interview without ending the session", async () => { + test("latches panel closure while the Voice connection is pending", async () => { window.localStorage.setItem( VOICE_INTERVIEW_DISCLOSURE_STORAGE_KEY, "acknowledged", ); - const focusComposer = vi.fn(); - stubUnavailableMicrophone(); - render( - , + let finishConnection: ((epoch: number) => void) | undefined; + vi.spyOn(OpenAIRealtimeSession.prototype, "connect").mockImplementation( + () => + new Promise((resolve) => { + finishConnection = resolve; + }), ); + vi.spyOn(OpenAIRealtimeSession.prototype, "disconnect").mockResolvedValue(); + const setMicrophoneEnabled = vi + .spyOn(OpenAIRealtimeSession.prototype, "setMicrophoneEnabled") + .mockImplementation(() => {}); + render(); - fireEvent.click(screen.getByRole("button", { name: "Select Interview" })); - expect( - await screen.findByRole("region", { name: "Voice interview stage" }), - ).not.toBeNull(); + fireEvent.click(screen.getByRole("button", { name: "Select Voice" })); + await screen.findByText("Connecting"); + fireEvent.click(screen.getByRole("button", { name: "Toggle panel" })); + finishConnection?.(1); - fireEvent.click(screen.getByRole("button", { name: "Use text instead" })); - - expect(screen.getByText("Chat mode")).not.toBeNull(); - expect(screen.getByText("Interview active")).not.toBeNull(); - expect( - screen.getByRole("region", { name: "Voice interview mini bar" }), - ).not.toBeNull(); - expect(focusComposer).toHaveBeenCalledOnce(); + await screen.findByText("Paused"); + expect(setMicrophoneEnabled).toHaveBeenCalledWith(false); + expect(setMicrophoneEnabled).not.toHaveBeenCalledWith(true); + expect(screen.getByText("Panel closed")).not.toBeNull(); }); - test("skips the disclosure after it has been acknowledged", async () => { + test("records acknowledgement only when the interview starts", () => { stubUnavailableMicrophone(); - window.localStorage.setItem( - VOICE_INTERVIEW_DISCLOSURE_STORAGE_KEY, - "acknowledged", - ); - render(); + render(); - fireEvent.click(screen.getByRole("button", { name: "Select Interview" })); - - expect( - screen.queryByRole("region", { name: "Start voice interview" }), - ).toBeNull(); - expect( - await screen.findByRole("region", { name: "Voice interview stage" }), - ).not.toBeNull(); - }); - - test("keeps the question visible and names microphone level", () => { - const html = renderToStaticMarkup( - , - ); - - expect(html).toContain("What happens after approval?"); - expect(html).toContain("Live transcript"); - expect(html).toContain("Listening"); - expect(html).toContain("Microphone input level: Medium"); - expect(html).not.toContain('aria-label="Done speaking"'); - expect(html).not.toContain('aria-label="Interrupt and speak"'); - expect(html).toContain("motionReduce:vis_hidden"); - expect(html).toContain("pos_relative"); - expect(html).not.toContain("pos_fixed"); - expect(html).toContain('aria-live="polite"'); - }); - - test("invites a spoken kickoff when no interview question exists", () => { - render( - , - ); - - expect( - screen.getByText("Tell me about the process you want to model."), - ).not.toBeNull(); - }); - - test("centers a circular microphone and waveform without visible level copy", () => { - render(); - - expect(screen.getByTestId("voice-microphone-focal")).not.toBeNull(); - expect(screen.getByTestId("voice-waveform")).not.toBeNull(); - expect(screen.getByText("Listening")).not.toBeNull(); - - const accessibleLevel = screen.getByText("Microphone input level: Medium"); - expect(accessibleLevel.className).toContain("pos_absolute"); - expect( - screen.queryByText("Microphone on · Listening", { - selector: ":not([role='status'])", - }), - ).toBeNull(); - }); - - test("shows compact recording and sent transcript statuses", () => { - const rendered = render(); - - expect(screen.getByText("Live transcript")).not.toBeNull(); - expect(screen.getByText("Recording")).not.toBeNull(); - expect(screen.queryByText("What we’re hearing · Not sent yet")).toBeNull(); - expect(screen.getByRole("status").textContent).toContain("Not sent yet"); - - rendered.rerender( - , - ); - - expect(screen.getByText("Your answer")).not.toBeNull(); - expect(screen.getByText("Sent")).not.toBeNull(); - expect(screen.getByText("The shift lead assigns an owner.")).not.toBeNull(); - }); - - test("shows a sending status while the answer is still being delivered", () => { - render( - , - ); - - expect(screen.getByText("Your answer")).not.toBeNull(); - expect(screen.getByText("Sending")).not.toBeNull(); - expect(screen.queryByText("Sent")).toBeNull(); - }); - - test("shows a not-sent status when delivery failed", () => { - render( - , - ); - - expect(screen.getByText("Not sent")).not.toBeNull(); - expect(screen.queryByText("Sent")).toBeNull(); - expect(screen.getByText("The shift lead assigns an owner.")).not.toBeNull(); - }); - - test("uses voice-app icon controls while listening", () => { - render(); - - for (const name of ["Use text instead", "Pause"]) { - const button = screen.getByRole("button", { name }); - expect(button.querySelector("svg")).not.toBeNull(); - expect(button.parentElement?.getAttribute("data-scope")).toBe("tooltip"); - } - expect(screen.queryByRole("button", { name: "Done speaking" })).toBeNull(); - expect( - screen.queryByRole("button", { name: "Interrupt and speak" }), - ).toBeNull(); - }); - - test("orders the full listening actions as keyboard then pause", () => { - const html = renderToStaticMarkup( - , - ); - - expect(html.indexOf('aria-label="Use text instead"')).toBeLessThan( - html.indexOf('aria-label="Pause"'), - ); - expect(html).not.toContain('aria-label="Done speaking"'); - }); - - test("offers only resume and keyboard actions while paused", () => { - render( - , - ); - - expect(screen.getByText("Paused")).not.toBeNull(); - expect( - screen.getByRole("button", { name: "Resume listening" }), - ).not.toBeNull(); - expect( - screen.getByRole("button", { name: "Use text instead" }), - ).not.toBeNull(); - expect(screen.queryByRole("button", { name: "Done speaking" })).toBeNull(); - expect(screen.queryByRole("button", { name: "Pause" })).toBeNull(); - expect( - screen.queryByRole("button", { name: "Interrupt and speak" }), - ).toBeNull(); - }); - - test("shows the waveform whenever the continuous microphone is active", () => { - const rendered = render(); - - expect(screen.getByTestId("voice-waveform")).not.toBeNull(); - - rendered.rerender( - , - ); - expect(screen.getByTestId("voice-waveform")).not.toBeNull(); - expect(screen.getByText("Interviewer speaking")).not.toBeNull(); - - rendered.rerender( - , - ); - expect(screen.queryByTestId("voice-waveform")).toBeNull(); - expect(screen.queryByText(/Microphone input level:/u)).toBeNull(); - }); - - test("keeps reconnect visible and makes secondary recovery icon-only", () => { - render( - , - ); - - expect( - screen.getByText("We couldn’t reconnect the microphone"), - ).not.toBeNull(); - expect( - screen.getByRole("button", { name: "Reconnect" }).textContent, - ).toContain("Reconnect"); - expect( - screen - .getByRole("button", { name: "Use text instead" }) - .querySelector("svg"), - ).not.toBeNull(); - expect(screen.getByText("Technical details")).not.toBeNull(); - }); - - test("names the recovery problem for each error family", () => { - const recovery = ( - errorCode: VoiceTurnSnapshot["errorCode"], - errorMessage: string, - ) => ( - - ); - - const rendered = render( - recovery( - "microphone-device", - "Connect or select a microphone, then reconnect voice input.", - ), - ); - expect( - screen.getByText("We couldn’t reconnect the microphone"), - ).not.toBeNull(); - expect(screen.getByText("Microphone unavailable")).not.toBeNull(); - expect( - screen.getByText( - "Connect or select a microphone, then reconnect voice input.", - ), - ).not.toBeNull(); - - rendered.rerender( - recovery( - "timeout", - "The voice connection timed out. Check your connection, then reconnect voice input.", - ), - ); - expect(screen.getByText("We lost the voice connection")).not.toBeNull(); - expect(screen.getByText("Connection paused")).not.toBeNull(); - - rendered.rerender( - recovery( - "invalid-response", - "The interview could not accept that answer. Use the composer to retry.", - ), - ); - expect(screen.getByText("The interview couldn’t continue")).not.toBeNull(); - expect(screen.getByText("Interview paused")).not.toBeNull(); - - rendered.rerender( - recovery( - null, - "The interview could not accept that answer. Use the composer to retry.", - ), - ); - expect(screen.getByText("The interview couldn’t continue")).not.toBeNull(); - expect(screen.getByText("Interview paused")).not.toBeNull(); - expect( - screen.queryByText("We couldn’t reconnect the microphone"), - ).toBeNull(); - }); - - test("renders icons for the listening controls", () => { - render(); - - for (const name of ["Minimize voice interview", "End interview", "Pause"]) { - expect( - screen.getByRole("button", { name }).querySelector("svg"), - ).not.toBeNull(); - } - }); - - test("does not describe an unavailable meter while the microphone is off", () => { - const waitingHtml = renderToStaticMarkup( - , - ); - - expect(waitingHtml).not.toContain( - "Microphone input level unavailable while microphone is off", - ); - }); - - test("renders committed edit separately from pause, minimize, and end", () => { - const html = renderToStaticMarkup( - , - ); - - for (const name of [ - "Minimize voice interview", - "End interview", - "Edit text", - "Use text instead", - ]) { - expect(html).toContain(name); - } - }); - - test("enables repair actions only while the last answer can be revised", () => { - const rendered = render( - , - ); - - expect( - screen - .getByRole("button", { name: "Edit text" }) - .hasAttribute("disabled"), - ).toBe(true); - - rendered.rerender( - , - ); - - expect( - screen - .getByRole("button", { name: "Edit text" }) - .hasAttribute("disabled"), - ).toBe(false); - }); - - test("keeps listening and offers no manual interrupt during playback", () => { - const html = renderToStaticMarkup( - , - ); - - expect(html).toContain( - "Microphone on · Interviewer speaking · Speak to interrupt", - ); - expect(html).not.toContain('aria-label="Interrupt and speak"'); - expect(html).toContain('aria-label="Pause"'); - }); - - test("uses a detached bottom mini bar with independent expand, type, pause, and end controls", () => { - render( - , - ); - - expect( - screen.getByRole("region", { name: "Voice interview mini bar" }), - ).not.toBeNull(); - expect( - screen.getByRole("button", { - name: "Expand voice interview. Microphone on · Listening. Question: What happens after approval?", - }), - ).not.toBeNull(); - expect(screen.getByText("Listening")).not.toBeNull(); - expect(screen.queryByText("Microphone on · Listening")).toBeNull(); - expect(screen.getByText("What happens after approval?")).not.toBeNull(); - expect(screen.getByRole("button", { name: "Pause" })).not.toBeNull(); - expect( - screen.getByRole("button", { name: "Use text instead" }), - ).not.toBeNull(); - expect( - screen.getByRole("button", { name: "End interview" }), - ).not.toBeNull(); - - for (const name of ["Pause", "Use text instead", "End interview"]) { - const button = screen.getByRole("button", { name }); - expect(button.querySelector("svg")).not.toBeNull(); - expect(button.parentElement?.getAttribute("data-scope")).toBe("tooltip"); - } - - const html = renderToStaticMarkup( - , - ); - expect(html).toContain("--voice-interview-right"); - expect(html).toContain("[@media_(min-width:_768px)]"); - expect(html).not.toContain("md:right_4"); - }); - - test("shows only the valid compact phase action", () => { - const rendered = render( - , - ); - - expect( - screen.getByRole("button", { name: "Resume listening" }), - ).not.toBeNull(); - expect(screen.queryByRole("button", { name: "Done speaking" })).toBeNull(); - expect( - screen - .getByRole("button", { name: "Resume listening" }) - .querySelector("svg"), - ).not.toBeNull(); - expect( - screen - .getByRole("button", { name: "Resume listening" }) - .parentElement?.getAttribute("data-scope"), - ).toBe("tooltip"); - - rendered.rerender( - , - ); - expect(screen.getByRole("button", { name: "Pause" })).not.toBeNull(); - expect( - screen.getByRole("button", { name: "Pause" }).querySelector("svg"), - ).not.toBeNull(); - expect( - screen - .getByRole("button", { name: "Pause" }) - .parentElement?.getAttribute("data-scope"), - ).toBe("tooltip"); + fireEvent.click(screen.getByRole("button", { name: "Select Voice" })); + fireEvent.click(screen.getByRole("button", { name: "Check microphone" })); expect( - screen.queryByRole("button", { name: "Interrupt and speak" }), + window.localStorage.getItem(VOICE_INTERVIEW_DISCLOSURE_STORAGE_KEY), ).toBeNull(); - }); - - test("announces compact question and provisional transcript context", () => { - render( - , - ); + fireEvent.click(screen.getByRole("checkbox")); + fireEvent.click(screen.getByRole("button", { name: "Start voice mode" })); expect( - screen.getByRole("button", { - name: "Expand voice interview. Microphone on · Listening. Question: What happens after approval?", - }), - ).not.toBeNull(); - expect(screen.getByRole("status").textContent).toBe( - "Microphone on · Listening. Question: What happens after approval? Not sent yet: The request goes to", - ); - }); - - test("shows authoritative covered and still-exploring facts without a question count", () => { - const html = renderToStaticMarkup( - , - ); - - expect(html).toContain("Covered"); - expect(html).toContain("Still exploring"); - expect(html).not.toMatch(/\d+ of \d+/u); - }); - - test("keeps interview coverage as a low-emphasis details row", () => { - const html = renderToStaticMarkup( - , - ); - - expect(html).toMatch(/
; + readonly messageId: string | undefined; +}; + +const representedVoiceToolCallIds = ( + messages: PetrinautAiVoiceModeContext["messages"], +): Set => { + const toolCallIds = new Set(); + for (const message of messages) { + const toolCallId = message.metadata?.toolCallId; + if ( + message.metadata?.source === "voice" && + toolCallId !== undefined && + message.parts.some( + (part) => + part.type === "dynamic-tool" && + part.toolCallId === toolCallId && + part.state === "output-available", + ) + ) { + toolCallIds.add(toolCallId); + } + } + return toolCallIds; +}; + +export const isVoiceInputRepresented = ( + messages: PetrinautAiVoiceModeContext["messages"], + pendingInput: PendingVoiceInputRepresentation, +): boolean => { + if ( + pendingInput.messageId !== undefined && + messages.some( + (message) => + message.id === pendingInput.messageId && + message.role === "user" && + message.metadata?.source === "voice", + ) + ) { + return true; + } + + return [...representedVoiceToolCallIds(messages)].some( + (toolCallId) => !pendingInput.baselineToolCallIds.has(toolCallId), + ); +}; const isRecord = (value: unknown): value is Record => typeof value === "object" && value !== null; @@ -110,7 +143,9 @@ export const loadOpenAIVoiceConfig = async ( method: "GET", signal, }); - if (!response.ok) return null; + if (!response.ok) { + return null; + } const body: unknown = await response.json(); if ( !isRecord(body) || @@ -130,1008 +165,107 @@ export const loadOpenAIVoiceConfig = async ( } }; -const rootStyle = cva({ - base: { - zIndex: "overlay", - pointerEvents: "auto", - }, - variants: { - presentation: { - start: { - position: "absolute", - right: "0", - bottom: "[-2px]", - width: "full", - }, - full: { - position: "relative", - width: "full", - }, - mini: { - position: "relative", - width: "full", - }, - detached: { - position: "fixed", - "--voice-interview-right": "0px", - "--voice-interview-bottom": "0px", - "--voice-interview-left": "0px", - "--voice-interview-width": "100%", - right: "[var(--voice-interview-right)]", - bottom: "[var(--voice-interview-bottom)]", - left: "[var(--voice-interview-left)]", - width: "[var(--voice-interview-width)]", - "@media (min-width: 768px)": { - "--voice-interview-right": "var(--spacing-4)", - "--voice-interview-bottom": "var(--spacing-4)", - "--voice-interview-left": "auto", - "--voice-interview-width": "440px", - }, - }, - }, - }, -}); - -const cardStyle = css({ - display: "flex", - flexDirection: "column", - gap: "3", - padding: "4", - borderWidth: "thin", - borderStyle: "solid", - borderColor: "neutral.a20", - borderTopLeftRadius: "xl", - borderTopRightRadius: "xl", - borderBottomRightRadius: "xl", - borderBottomLeftRadius: "xl", - backgroundColor: "neutral.s00", - boxShadow: "xl", -}); - -const stageStyle = css({ +const disclosureStyle = css({ display: "flex", - maxHeight: "[72vh]", + width: "full", flexDirection: "column", - gap: "3", - padding: "3", - overflowY: "auto", - borderWidth: "thin", - borderStyle: "solid", - borderColor: "neutral.a20", - backgroundColor: "neutral.s00", - boxShadow: "[0 -8px 24px rgba(0,0,0,0.06)]", - borderRadius: "lg", -}); - -const headerStyle = css({ - display: "flex", - alignItems: "center", - justifyContent: "flex-end", gap: "2", -}); - -const startHeaderStyle = css({ - display: "flex", - alignItems: "center", - gap: "2", -}); - -const titleStyle = css({ - flex: "1", + paddingX: "2", + paddingY: "2", + borderTopWidth: "thin", + borderTopStyle: "solid", + borderTopColor: "neutral.a20", color: "neutral.s100", - fontSize: "sm", - fontWeight: "semibold", -}); - -const subtitleStyle = css({ - color: "neutral.s80", - fontSize: "xs", - lineHeight: "snug", -}); - -const questionStyle = css({ - color: "neutral.s110", - fontSize: "lg", - fontWeight: "semibold", - lineHeight: "snug", }); -const contextStyle = css({ - display: "block", - overflow: "hidden", - color: "neutral.s80", - fontSize: "xs", - lineHeight: "snug", - textOverflow: "ellipsis", - whiteSpace: "nowrap", -}); - -const miniTextStyle = css({ - display: "flex", - minWidth: "0", - flexDirection: "column", -}); - -const focalAreaStyle = css({ - display: "flex", - minHeight: "[150px]", - flexDirection: "column", - alignItems: "center", - justifyContent: "center", - gap: "3", -}); - -const focalCircleStyle = cva({ - base: { - position: "relative", - display: "flex", - width: "[108px]", - height: "[108px]", - flexDirection: "column", - alignItems: "center", - justifyContent: "center", - gap: "2", - borderWidth: "thin", - borderStyle: "solid", - borderRadius: "full", - _before: { - content: '""', - position: "absolute", - inset: "[-10px]", - borderWidth: "thin", - borderStyle: "solid", - borderColor: "blue.a20", - borderRadius: "full", - }, - }, - variants: { - tone: { - active: { - borderColor: "blue.a30", - backgroundColor: "blue.a10", - color: "blue.s90", - boxShadow: "[0 14px 30px rgba(35,125,181,0.14)]", - }, - idle: { - borderColor: "neutral.a20", - backgroundColor: "neutral.s20", - color: "neutral.s80", - }, - success: { - borderColor: "green.a30", - backgroundColor: "green.a10", - color: "green.s90", - }, - error: { - borderColor: "red.a30", - backgroundColor: "red.a10", - color: "red.s90", - }, - }, - }, -}); - -const shortStateStyle = css({ - display: "flex", - alignItems: "center", - gap: "2", - color: "neutral.s90", - fontSize: "xs", - fontWeight: "semibold", -}); - -const recordingDotStyle = css({ - width: "[7px]", - height: "[7px]", - borderRadius: "full", - backgroundColor: "green.s70", - boxShadow: "[0 0 0 4px rgba(24,168,120,0.10)]", -}); - -const transcriptHeaderStyle = css({ - display: "flex", - alignItems: "center", - justifyContent: "space-between", - gap: "2", -}); - -const recordingTranscriptDotStyle = css({ - width: "[7px]", - height: "[7px]", - borderRadius: "full", - backgroundColor: "red.s70", -}); - -const transcriptActionsStyle = css({ - display: "flex", - justifyContent: "flex-end", - gap: "1", -}); - -const transcriptStateStyle = cva({ - base: { - display: "flex", - alignItems: "center", - gap: "1", - fontSize: "xs", - fontWeight: "semibold", - }, - variants: { - state: { - recording: { color: "red.s80" }, - sending: { color: "neutral.s80" }, - sent: { color: "green.s90" }, - unsent: { color: "red.s90" }, - }, - }, -}); - -const meterStyle = css({ - display: "flex", - height: "[34px]", - alignItems: "center", - gap: "1", - _motionReduce: { visibility: "hidden" }, -}); - -const meterBarStyle = css({ - width: "[5px]", - minHeight: "[4px]", - borderRadius: "full", - backgroundColor: "blue.s70", - transition: "[height 80ms linear]", - _motionReduce: { transition: "[none]" }, -}); - -const statusStyle = css({ - color: "neutral.s90", - fontSize: "sm", - fontWeight: "medium", -}); - -const transcriptStyle = css({ +const disclosureTitleStyle = css({ display: "flex", flexDirection: "column", gap: "1", - padding: "2.5", - borderRadius: "lg", - backgroundColor: "neutral.s10", - color: "neutral.s100", fontSize: "sm", + fontWeight: "semibold", }); -const labelStyle = css({ +const disclosureSubtitleStyle = css({ color: "neutral.s80", fontSize: "xs", - fontWeight: "semibold", -}); - -const recoveryStyle = css({ - display: "flex", - flexDirection: "column", - gap: "1", - padding: "2.5", - borderRadius: "lg", - backgroundColor: "red.a10", - color: "neutral.s100", - fontSize: "sm", + fontWeight: "normal", }); -const secondaryDetailsStyle = css({ - color: "neutral.s80", +const disclosureCopyStyle = css({ + color: "neutral.s90", fontSize: "xs", - _open: { color: "neutral.s90" }, + lineHeight: "relaxed", }); -const actionsStyle = css({ +const disclosureActionsStyle = css({ display: "flex", flexWrap: "wrap", alignItems: "center", gap: "2", }); -const inputStyle = css({ - width: "full", - paddingX: "2", - paddingY: "1.5", - borderWidth: "thin", - borderStyle: "solid", - borderColor: "neutral.a30", - borderRadius: "md", - backgroundColor: "neutral.s00", - color: "neutral.s100", - fontSize: "sm", - _focusVisible: { - outline: "2px solid", - outlineColor: "blue.a40", - outlineOffset: "[1px]", - }, -}); - -const miniStyle = css({ - display: "flex", - minHeight: "[60px]", - alignItems: "center", - gap: "2", - padding: "2", - borderWidth: "thin", - borderStyle: "solid", - borderColor: "neutral.a20", - borderTopLeftRadius: "lg", - borderTopRightRadius: "lg", - borderBottomRightRadius: "lg", - borderBottomLeftRadius: "lg", - backgroundColor: "neutral.s00", - boxShadow: "lg", -}); - -const miniExpandStyle = css({ - display: "flex", - minWidth: "0", - flex: "1", - alignItems: "center", - gap: "2", - padding: "2", - color: "neutral.s100", - textAlign: "left", - background: "[transparent]", - border: "none", - cursor: "pointer", - _focusVisible: { outline: "2px solid", outlineColor: "blue.a50" }, -}); - -const liveRegionStyle = css({ - position: "absolute", - width: "[1px]", - height: "[1px]", - padding: "0", - margin: "[-1px]", - overflow: "hidden", - clip: "[rect(0,0,0,0)]", - whiteSpace: "nowrap", - borderWidth: "0", -}); - -const statusText = (snapshot: VoiceTurnSnapshot): string => { - if (snapshot.connection === "idle") - return "Microphone off · Interview not started"; - if (snapshot.connection === "connecting") - return "Microphone off · Joining the interview"; - if (snapshot.connection === "error") - return `Microphone off · ${snapshot.errorMessage}`; - if (snapshot.input === "paused") return "Microphone off · Paused"; - if (snapshot.output === "speaking") - return "Microphone on · Interviewer speaking · Speak to interrupt"; - if (snapshot.input === "submitting") - return "Microphone on · Answer recorded · Writing that down"; - if (snapshot.output === "waiting-for-tool") - return "Microphone on · Preparing the next question"; - if (snapshot.output === "interrupted") - return "Microphone on · Listening after interruption"; - return "Microphone on · Listening"; -}; - -type RecoveryErrorFamily = "connection" | "interview" | "microphone"; - -const recoveryErrorFamily = ( - errorCode: VoiceTurnSnapshot["errorCode"], -): RecoveryErrorFamily => { - switch (errorCode) { - case "microphone-permission": - case "microphone-device": - return "microphone"; - case "network": - case "timeout": - case "request-aborted": - return "connection"; - default: - return "interview"; - } -}; - -const shortStatusText = (snapshot: VoiceTurnSnapshot): string => { - if (snapshot.connection === "idle") return "Ready"; - if (snapshot.connection === "connecting") return "Connecting"; - if (snapshot.connection === "error") { - switch (recoveryErrorFamily(snapshot.errorCode)) { - case "microphone": - return "Microphone unavailable"; - case "connection": - return "Connection paused"; - case "interview": - return "Interview paused"; - } - } - if (snapshot.input === "paused") return "Paused"; - if (snapshot.output === "speaking") return "Interviewer speaking"; - if (snapshot.input === "submitting") return "Writing that down"; - if (snapshot.output === "waiting-for-tool") return "Preparing next question"; - return "Listening"; -}; - -const inputLevelText = (level: number): string => - level >= 0.35 - ? "High" - : level >= 0.12 - ? "Medium" - : level > 0 - ? "Low" - : "Quiet"; - -const Meter = ({ snapshot }: { snapshot: VoiceTurnSnapshot }) => { - const level = snapshot.microphoneLevel; - return ( - <> - - - {`Microphone input level: ${inputLevelText(level)}`} - - - ); -}; - -const focalIcon = (snapshot: VoiceTurnSnapshot): ReactNode => { - if (snapshot.connection === "error") - return