diff --git a/.changeset/stable-composer-controls.md b/.changeset/stable-composer-controls.md index b6aa782f07b..553de6f0c71 100644 --- a/.changeset/stable-composer-controls.md +++ b/.changeset/stable-composer-controls.md @@ -30,10 +30,13 @@ per-session Show transcription in chat action write those turns into the convers instead. Keep every session control -- transcription, the microphone toggle, Resume, Reconnect, and End -- in the dock, leaving the canvas toolbar untouched. Add `setMicrophoneMuted` to the Voice mode controls and a `muted` session phase, so muting stops capture without interrupting what the assistant -is saying, unlike pausing. Surface voice recovery failures as toasts with privacy-safe diagnostic -references, and request one-time consent before the host starts the microphone. Mark persisted spoken -messages and the exact interactive-tool answer completed by Voice with an inline Voice chip ahead of -the words themselves. +is saying, unlike pausing. Keep the complete legacy Voice controls and context registration contract +required while a separate optional registration seam accepts provider-specific capabilities. Offer +an explicit playback retry when a browser blocks session audio, keeping the warning visible until +that user gesture succeeds. Surface voice recovery failures as toasts with privacy-safe diagnostic +references, and request one-time consent before the host starts the microphone. Mark persisted +spoken messages and the exact interactive-tool answer completed by Voice with an inline Voice chip +ahead of the words themselves. End Voice mode before submitting typed text exactly once through the shared composer, preserving the draft if handoff fails. Pause active media before the AI panel closes and reopen the mounted session diff --git a/.changeset/tidy-voice-dock.md b/.changeset/tidy-voice-dock.md new file mode 100644 index 00000000000..66b86c2b692 --- /dev/null +++ b/.changeset/tidy-voice-dock.md @@ -0,0 +1,5 @@ +--- +"@hashintel/petrinaut": patch +--- + +Allow host-owned Voice sessions to report a neutral connected state and omit unsupported microphone, resume, reconnect, and playback controls from the dock. Keep the viewport controls above a collapsed Voice dock instead of reserving full-height chat panel space. diff --git a/apps/petrinaut-website/MISSION.md b/apps/petrinaut-website/MISSION.md new file mode 100644 index 00000000000..69347722a67 --- /dev/null +++ b/apps/petrinaut-website/MISSION.md @@ -0,0 +1,92 @@ +# Experiment Live Full-Duplex Migration + +## Status + +Live experiment scope; local implementation prepared for Kostandin's manual +witness, not conversational-quality acceptance. +[FE-1663](https://linear.app/hash/issue/FE-1663/experiment-live-full-duplex-migration) +is related to FE-1661, which remains the migration-effort assessment. +This website experiment uses Petrinaut's existing Voice dock, with optional +session controls and local audio-activity indicators, not inferred turn boundaries. +It does not recut Brunch's existing Mission 7a or promote its future-planning drafts. + +## Imperative + +Learn whether GPT-Live-1 natively makes process interviewing feel fluid: +concise relevant follow-ups, room for hesitation and elaboration, and natural +interruption/correction without losing meaning. Smooth audio alone is not success. + +## Throughline + +Existing website Voice entry → explicit experimental consent/Start → trusted +website `/api/voice/live-session` → client-delegated GPT-Live-1 WebRTC → microphone +and speaker → existing Voice dock → local End/Exit. Local disconnection is +not a remote-closure claim. +`main → FE-1663 → Experiment Live Full Brunch Integration` is the intended stack; +only FE-1663 is authorized now. No relay, rephrasing, or harness code is a donor. + +## Proof + +- `openai-live-session.test.ts` checks provider/default selection, enablement, + trusted session payload, origin/content validation, credential privacy, and + no retry. Existing Realtime policy and transport tests remain regression oracles. +- `live-conversation.test.ts` checks waiting for `session.started`, no duplicate + start, no transcript/delegation execution, remote audio attachment, connection + failures, late microphone permission, and capture/playback/transport cleanup. +- `live-conversation-control.test.tsx` checks explicit consent/Start, provider + pinning, canonical isolation, host state reporting, panel closure and stale callbacks. +- Petrinaut's assistant-panel and contents tests check optional action visibility + and retain regression coverage for Realtime's playback and lifecycle controls. +- Rendered real-editor consent, simulated error, and mocked active controls + must be inspected without a real microphone or provider session. +- Kostandin's [10–15 minute manual procedure](README.md#manual-test--1015-minutes) + is the oracle for conversational quality and actual media behavior. Provider-free + tests and screenshots do not establish these or end-to-end Brunch compatibility. + +## Constraints + +Unset `PETRINAUT_VOICE_PROVIDER` means `realtime`; only `realtime` and `live` are +valid. Keep existing enablement/credential plumbing. Pin configuration per +mounted conversation; never switch providers or resubmit input automatically. +Keep Realtime's prompts, admission, queue, settlement and playback unchanged. + +Live has no access to canonical submission, Brunch/Petrinaut tools or chat +history. Its guidance does not grant domain authority. Transcript deltas are +not finalized utterances; delegation events are metadata; commentary (unused +here) is paraphrasable context limited to 500 tokens, and append acknowledgements +are not speech/playback completion. Do not invent response terminals or infer +authoritative completion from silence. Stop ends local media, not canonical work. + +No paid sessions, synthetic recordings, audio evaluation harness, deployment, +Notion writes, second issue, or integration implementation are authorized. +The follow-up permits cleanup, a clean commit without Amp thread IDs, and a draft PR. +Preserve other worktrees and uncommitted work. + +## Fog-line + +Native finalization and enforceable output gating remain unresolved for PR 2. +Observe whether interruption preserves corrections, whether the user feels +heard, whether questions advance process elicitation, whether pauses/one-word +answers survive, and whether output is concise. Observe transport failures and +remote closure separately. A pleasant standalone conversation cannot answer +whether canonical admission/settlement delays retain this quality. + +## Stop or reorient + +Stop at an authority change: autonomous domain speech, local substantive +follow-ups in the integrated workflow, hybrid authoritative transcription, or +buffered/gated output requires an explicit recut for Lu's approval. Neither +this prompt nor success here reopens FE-1624. Provider-free verification cannot +be replaced with paid calls or a synthetic audio campaign. + +## Deferred + +PR 2 must preserve: finalized input → existing composer → Flue → Brunch → +authorized Petrinaut execution → complete-turn settlement → frozen canonical +reply → Live delivery. Brunch alone authors substantive domain answers and +questions; Flue is canonical conversation; Petrinaut is execution authority. +Proposal approval, full-turn settlement and playback eligibility remain distinct. +Unknown execution outcomes must not be reapplied or spoken as success. Workpiece, +basis and tool payloads are not assistant prose. See the existing Brunch +[future spine](../../libs/@hashintel/brunch-agent/MISSION.next.md#voice-after-the-live-transport-cut) +for retained context, not implementation authority. diff --git a/apps/petrinaut-website/README.md b/apps/petrinaut-website/README.md index ed360f7a67d..d4cac073c99 100644 --- a/apps/petrinaut-website/README.md +++ b/apps/petrinaut-website/README.md @@ -134,20 +134,115 @@ from jsDelivr and Optuna from PyPI; later runs use the browser cache. ## Environment variables -| Name | Required | Used by | Notes | -| -------------------------------- | ---------------- | ---------------- | ---------------------------------------------------------- | -| `OPENAI_API_KEY` | for chat to work | `api/chat.ts` | OpenAI key the function uses to call `streamText`. | -| `OPENAI_VOICE_API_KEY` | for voice | voice API | Dedicated OpenAI key used to create Realtime WebRTC calls. | -| `PETRINAUT_OPENAI_VOICE_ENABLED` | no | voice API | Set to `true` to enable voice, including in production. | -| `PETRINAUT_AI_MODEL` | no | `api/chat.ts` | Overrides the default OpenAI model id. | -| `VITE_BRUNCH_CHAT_ENDPOINT` | for Brunch | website | Base URL of the mounted Brunch Flue route. | -| `SENTRY_DSN` | no | `vite.config.ts` | Wired into the bundle via `__SENTRY_DSN__` at build time. | +| Name | Required | Used by | Notes | +| -------------------------------- | ---------------- | ---------------- | --------------------------------------------------------------------------------------------- | +| `OPENAI_API_KEY` | for chat to work | `api/chat.ts` | OpenAI key the function uses to call `streamText`. | +| `OPENAI_VOICE_API_KEY` | for voice | voice API | Dedicated OpenAI key used to create Voice WebRTC sessions. | +| `PETRINAUT_OPENAI_VOICE_ENABLED` | no | voice API | Set to `true` to enable voice, including in production. | +| `PETRINAUT_VOICE_PROVIDER` | no | voice API | `realtime` (default) or `live` (detached experiment). Invalid values disable Voice discovery. | +| `PETRINAUT_AI_MODEL` | no | `api/chat.ts` | Overrides the default OpenAI model id. | +| `VITE_BRUNCH_CHAT_ENDPOINT` | for Brunch | website | Base URL of the mounted Brunch Flue route. | +| `SENTRY_DSN` | no | `vite.config.ts` | Wired into the bundle via `__SENTRY_DSN__` at build time. | Local values live in `.env.local`; Vite's `loadEnv` (see [`vite.config.ts`](vite.config.ts)) copies them into `process.env` for both the dev server and the API functions. In production, set these in the Vercel project settings. +### Experimental Live interview (FE-1663) + +`PETRINAUT_VOICE_PROVIDER=live` selects a standalone GPT-Live-1 conversation +inside the existing Voice entry. **It is not Brunch output.** It cannot see or +submit chat, execute tools, or change the model/workpiece. No experimental +transcripts are displayed or saved. The server uses client delegation, not +managed Responses; transcript deltas and delegation metadata are ignored. +The short process-interview prompt is conversational guidance, not a domain +system or a guarantee of model compliance. + +From the repository root, with `OPENAI_VOICE_API_KEY` already exported (or in +this worktree's `apps/petrinaut-website/.env.local`): + +```sh +# Initial local preparation, without inference: +turbo run build --filter '@apps/brunch-agent^...' --filter '@apps/petrinaut-website^...' +yarn workspace @apps/petrinaut-website codegen +yarn workspace @apps/petrinaut-website examples:generate + +# Standalone Live needs only the existing panel launcher, not a running Brunch server. +PETRINAUT_OPENAI_VOICE_ENABLED=true PETRINAUT_VOICE_PROVIDER=live yarn dev:brunch:panel +``` + +Open [http://localhost:4915/new](http://localhost:4915/new), dismiss the tour if shown, open the AI panel, +and select the waveform **Start voice mode** action in the empty composer. +Read the experimental label, check consent, then choose **Start voice**. +Only that last action requests microphone access and a billable Live session. +Use headphones for the first trial. HTTPS or localhost and an OpenAI project +with GPT-Live-1 access are required. + +Once connected, the existing Voice dock replaces the composer with +**Listening** or **Speaking**, collapse/expand and **End voice mode**. +The experimental label appears only during consent. Local WebRTC audio levels +drive the microphone ribbon and Speaking indicator; Listening means the session +is open for input, including while output is active. These are activity indicators, +not authoritative turn boundaries or proof of heard playback. Browsers without +audio-level telemetry retain Listening without an animated input level. +There is no separate experiment panel, replay menu, or microphone toggle. +Connection errors return to setup; starting again requires fresh consent. +Brief WebRTC interruptions show **Connecting** while the existing session has up +to the connection timeout (15 seconds by default) to recover. Media stays open; +no new session is created and no input is replayed. End still stops both directions +immediately. A failed connection or an expired recovery deadline ends the session. + +**End voice mode**, **Exit experiment** during setup, closing the panel, switching to text, +changing conversation, and leaving the page stop local experimental capture +and playback. Stop requests `session.close` only after `session.started` and +waits up to two seconds for `session.closed` before releasing the transport. +Local silence is not proof of remote closure or final usage. Connection failures are not retried. +Starting again creates a new session with fresh consent; there is no resume, +replay, "Your turn", or fabricated Realtime terminal lifecycle. + +To return to the **unchanged integrated Realtime path**, Exit, stop the panel +dev command with Ctrl-C, configure the existing local Brunch environment, and run: + +```sh +PETRINAUT_OPENAI_VOICE_ENABLED=true PETRINAUT_VOICE_PROVIDER=realtime yarn dev:brunch +``` + +Reload the page before starting a new session. Unsetting +`PETRINAUT_VOICE_PROVIDER` also selects Realtime. Provider/config selection is +pinned for the mounted conversation; there is no provider switching or input +resubmission mid-session. The launcher sets the existing `/agents/chat` route; +the ordinary website launcher still needs `VITE_BRUNCH_CHAT_ENDPOINT` configured +to expose Voice. Export variables to the launcher directly or use `.env.local`; +the website's generic Turbo `dev` task does not forward arbitrary shell variables. + +#### Manual test — 10–15 minutes + +1. **3 minutes:** explain a familiar process. Let Live ask relevant follow-ups. +2. **2 minutes:** hesitate, pause, answer with one word, then elaborate. Note + whether it leaves room and follows the meaning rather than guessing a turn end. +3. **2 minutes:** interrupt mid-sentence and correct an earlier detail. Note + whether the correction is retained and the interview advances. +4. **2 minutes:** speak while Live responds. Listen for lost words, overlap, + unwanted acknowledgements, and long monologues. A button click is not an + acoustic-interruption test. +5. **2–4 minutes:** End voice mode and check the browser microphone indicator and actual + speaker silence. Change to Realtime using the commands above, reload, + and begin a fresh session. Confirm the experiment added no canonical chat + messages or model changes. + +Record **feeling heard and advancing the interview** separately from transport +correctness. Note browser/headset, representative pauses/corrections, and which +responses were excessive or useful. Prior relay/rephrasing/harness evidence is +not a passing comparative baseline. This experiment has no audio harness or +synthetic recordings. + +The intended successor is **Experiment Live Full Brunch Integration**, stacked +on this branch; it is not implemented here. See [MISSION.md](MISSION.md) for +the authority boundary and unresolved finalization/output-control questions. +The existing unauthenticated Voice endpoint risk below also applies to Live; +do not expose this local experiment publicly without addressing that boundary. + ### Brunch Voice mode -Voice mode is disabled by default. To enable it, configure a real +The following describes Realtime, the default provider. Voice mode is disabled by default. To enable it, configure a real `VITE_BRUNCH_CHAT_ENDPOINT`, set `PETRINAUT_OPENAI_VOICE_ENABLED=true`, and provide a dedicated `OPENAI_VOICE_API_KEY`. diff --git a/apps/petrinaut-website/api/voice/live-session.ts b/apps/petrinaut-website/api/voice/live-session.ts new file mode 100644 index 00000000000..a11884f34cc --- /dev/null +++ b/apps/petrinaut-website/api/voice/live-session.ts @@ -0,0 +1,10 @@ +import { createOpenAILiveSessionHandler } from "../../src/server/voice/openai-live-session.js"; + +declare const process: { env: Record }; + +export default { + fetch: createOpenAILiveSessionHandler({ + environment: process.env, + fetch: globalThis.fetch.bind(globalThis), + }), +}; diff --git a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.test.tsx b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.test.tsx new file mode 100644 index 00000000000..5ee9062a026 --- /dev/null +++ b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.test.tsx @@ -0,0 +1,272 @@ +// @vitest-environment jsdom +import { + act, + cleanup, + fireEvent, + render, + screen, + waitFor, +} from "@testing-library/react"; +import { afterEach, expect, test, vi } from "vitest"; + +import { createLiveConversation } from "./live-conversation"; +import { + loadOpenAIVoiceConfig, + VoiceInterviewControl, +} from "./voice-interview-control"; + +import type { PetrinautAiVoiceModeContext } from "@hashintel/petrinaut/ui"; + +vi.mock("./live-conversation", () => ({ + createLiveConversation: vi.fn(() => ({ + retryPlayback: vi.fn(async () => {}), + start: vi.fn(async () => {}), + stop: vi.fn(async () => {}), + })), +})); +afterEach(() => { + cleanup(); + vi.clearAllMocks(); +}); + +const context = (): PetrinautAiVoiceModeContext => ({ + conversationId: "standalone", + messages: [], + status: "ready", + canAcceptVoiceInput: true, + inputMode: "voice", + isAiAssistantOpen: true, + stop: vi.fn(async () => {}), + submitText: vi.fn(), + submitVoiceInput: vi.fn(), + registerVoiceModeControls: vi.fn(() => () => {}), + registerVoiceModeSessionControls: vi.fn(() => () => {}), + reportVoiceSessionState: vi.fn(), + setInputMode: vi.fn(), + setVoiceActive: vi.fn(), +}); +const config = { + available: true as const, + provider: "live" as const, + connectionTimeoutMs: 15_000, +}; +const start = async () => { + fireEvent.click(screen.getByRole("checkbox")); + await waitFor(() => + expect( + screen + .getByRole("button", { name: "Start voice" }) + .hasAttribute("disabled"), + ).toBe(false), + ); + fireEvent.click(screen.getByRole("button", { name: "Start voice" })); +}; + +test("starts only one Live session when Start is activated twice", async () => { + render(); + fireEvent.click(screen.getByRole("checkbox")); + const startButton = await screen.findByRole("button", { + name: "Start voice", + }); + await waitFor(() => expect(startButton.hasAttribute("disabled")).toBe(false)); + act(() => { + startButton.click(); + startButton.click(); + }); + expect(createLiveConversation).toHaveBeenCalledOnce(); +}); + +test("reuses setup, reports listening and speaking to the host dock, and clears it on failure", async () => { + const props = context(); + render(); + expect( + screen.getByRole("region", { name: "Voice mode consent" }), + ).toBeTruthy(); + expect(screen.getByText("GPT-Live · Experimental interview")).toBeTruthy(); + expect(screen.queryByText(/Petrinaut saves finalized/)).toBeNull(); + expect( + screen + .getByRole("button", { name: "Start voice" }) + .hasAttribute("disabled"), + ).toBe(true); + expect(createLiveConversation).not.toHaveBeenCalled(); + await start(); + expect(props.reportVoiceSessionState).toHaveBeenLastCalledWith( + expect.objectContaining({ phase: "connecting", notice: null }), + ); + const onState = vi.mocked(createLiveConversation).mock.calls[0]![0]; + act(() => onState({ phase: "connected", message: null })); + expect( + screen.queryByRole("region", { name: "Voice mode consent" }), + ).toBeNull(); + expect(props.reportVoiceSessionState).toHaveBeenLastCalledWith({ + phase: "listening", + microphoneLevel: 0, + microphoneMuted: false, + errorMessage: null, + notice: null, + }); + act(() => + onState({ + phase: "connected", + message: null, + activity: { + microphoneLevel: 0.24, + outputActive: true, + }, + }), + ); + expect(props.reportVoiceSessionState).toHaveBeenLastCalledWith( + expect.objectContaining({ + phase: "speaking", + microphoneLevel: 0.24, + microphoneMuted: false, + }), + ); + act(() => onState({ phase: "connecting", message: null })); + expect(props.reportVoiceSessionState).toHaveBeenLastCalledWith( + expect.objectContaining({ phase: "connecting", microphoneLevel: 0 }), + ); + expect( + screen.queryByRole("region", { name: "Voice mode consent" }), + ).toBeNull(); + expect(createLiveConversation).toHaveBeenCalledOnce(); + act(() => + onState({ + phase: "connected", + message: null, + activity: { + microphoneLevel: 0.12, + outputActive: false, + }, + }), + ); + expect(props.reportVoiceSessionState).toHaveBeenLastCalledWith( + expect.objectContaining({ + phase: "listening", + microphoneLevel: 0.12, + }), + ); + const playbackNotice = + "Audio playback is blocked. Select Play voice audio to hear Live."; + act(() => + onState({ + phase: "connected", + message: playbackNotice, + playbackBlocked: true, + }), + ); + expect(props.reportVoiceSessionState).toHaveBeenLastCalledWith({ + canRetryPlayback: true, + phase: "listening", + microphoneLevel: 0, + microphoneMuted: false, + errorMessage: null, + notice: playbackNotice, + }); + if (!props.registerVoiceModeSessionControls) + throw new Error("Session control registration was not provided"); + const controls = vi.mocked(props.registerVoiceModeSessionControls).mock + .lastCall![0]; + expect(props.registerVoiceModeControls).not.toHaveBeenCalled(); + expect(Object.keys(controls).sort()).toEqual([ + "end", + "pause", + "retryPlayback", + ]); + controls.retryPlayback?.(); + const liveSession = vi.mocked(createLiveConversation).mock.results[0]! + .value as ReturnType; + expect(liveSession.retryPlayback).toHaveBeenCalledOnce(); + act(() => + onState({ + phase: "error", + message: "Connection failed. Remote session closure was not confirmed.", + }), + ); + expect(props.reportVoiceSessionState).toHaveBeenLastCalledWith(null); + expect( + screen.getByText( + "Connection error. Check microphone and server configuration.", + ), + ).toBeTruthy(); + expect( + screen + .getByRole("button", { name: "Start voice" }) + .hasAttribute("disabled"), + ).toBe(true); + expect(createLiveConversation).toHaveBeenCalledOnce(); +}); + +test("pins provider, ends through host controls, and never submits or stops canonical work", async () => { + const props = context(); + const subscribeToAdmission = vi.fn(); + const { rerender, unmount } = render( + , + ); + await start(); + const session = vi.mocked(createLiveConversation).mock.results[0]! + .value as ReturnType; + rerender( + , + ); + expect(session.start).toHaveBeenCalledOnce(); + if (!props.registerVoiceModeSessionControls) + throw new Error("Session control registration was not provided"); + const controls = vi.mocked(props.registerVoiceModeSessionControls).mock + .lastCall![0]; + await act(() => controls.end()); + expect(session.stop).toHaveBeenCalled(); + expect(props.submitText).not.toHaveBeenCalled(); + expect(props.submitVoiceInput).not.toHaveBeenCalled(); + expect(props.stop).not.toHaveBeenCalled(); + expect(subscribeToAdmission).not.toHaveBeenCalled(); + unmount(); + expect(props.reportVoiceSessionState).toHaveBeenLastCalledWith(null); +}); + +test("closing the panel ends Live; reopening cannot restart it; stale callbacks cannot reset the next conversation", async () => { + const props = context(); + const { rerender, unmount } = render( + , + ); + await start(); + const session = vi.mocked(createLiveConversation).mock.results[0]! + .value as ReturnType; + const onState = vi.mocked(createLiveConversation).mock.calls[0]![0]; + rerender( + , + ); + await waitFor(() => expect(session.stop).toHaveBeenCalled()); + rerender(); + expect(session.start).toHaveBeenCalledOnce(); + unmount(); + vi.mocked(props.setVoiceActive).mockClear(); + vi.mocked(props.reportVoiceSessionState).mockClear(); + onState({ phase: "ended", message: "Late close" }); + expect(props.setVoiceActive).not.toHaveBeenCalled(); + expect(props.reportVoiceSessionState).not.toHaveBeenCalled(); +}); + +test.each(["live", "realtime", "live-experience"])( + "validates provider config %s", + async (provider) => { + const fetch = vi.fn(async () => + Response.json({ ...config, provider }), + ); + expect(await loadOpenAIVoiceConfig(fetch)).toEqual( + provider === "live-experience" ? null : { ...config, provider }, + ); + }, +); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.tsx b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.tsx new file mode 100644 index 00000000000..bde5238a6f7 --- /dev/null +++ b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.tsx @@ -0,0 +1,155 @@ +import { useCallback, useEffect, useRef, useState } from "react"; + +import { + createLiveConversation, + type LiveConversationState, +} from "./live-conversation"; +import { VoiceInterviewDisclosure } from "./voice-interview-disclosure"; + +import type { PetrinautAiVoiceModeContext } from "@hashintel/petrinaut/ui"; + +// No canonical messages, submission, settlement, or tools cross this boundary. +type LiveControlsContext = Pick< + PetrinautAiVoiceModeContext, + | "inputMode" + | "isAiAssistantOpen" + | "reportVoiceSessionState" + | "setVoiceActive" + | "setInputMode" +> & + Required< + Pick + >; + +export const LiveConversationControl = ({ + inputMode, + isAiAssistantOpen, + registerVoiceModeSessionControls, + reportVoiceSessionState, + setVoiceActive, + setInputMode, + connectionTimeoutMs, +}: LiveControlsContext & { readonly connectionTimeoutMs: number }) => { + const [consented, setConsented] = useState(false); + const [state, setState] = useState({ + phase: "idle", + message: null, + }); + const { phase, activity, message, playbackBlocked } = state; + const session = useRef | null>( + null, + ); + const sessionActive = useRef(false); + const end = useCallback(async () => { + const closing = session.current?.stop(); + setVoiceActive(false); + setConsented(false); + await closing; + }, [setVoiceActive]); + + useEffect( + () => + registerVoiceModeSessionControls({ + end, + // Closing the panel ends Live. Reopening requires consent and a new session. + pause: () => { + void end(); + }, + retryPlayback: () => { + void session.current?.retryPlayback(); + }, + }), + [end, registerVoiceModeSessionControls], + ); + + useEffect(() => { + reportVoiceSessionState( + inputMode === "voice" && + isAiAssistantOpen && + (phase === "connecting" || phase === "connected") + ? { + phase: + phase === "connecting" + ? "connecting" + : activity?.outputActive + ? "speaking" + : "listening", + microphoneLevel: activity?.microphoneLevel ?? 0, + microphoneMuted: false, + errorMessage: null, + notice: playbackBlocked ? message : null, + ...(playbackBlocked ? { canRetryPlayback: true } : {}), + } + : null, + ); + }, [ + inputMode, + isAiAssistantOpen, + phase, + activity, + message, + playbackBlocked, + reportVoiceSessionState, + ]); + + useEffect(() => { + if (inputMode !== "voice" || !isAiAssistantOpen) + void session.current?.stop(); + }, [inputMode, isAiAssistantOpen]); + useEffect(() => { + const leave = () => { + void session.current?.stop(); + }; + window.addEventListener("pagehide", leave); + return () => { + window.removeEventListener("pagehide", leave); + const current = session.current; + session.current = null; + sessionActive.current = false; + void current?.stop(); + setVoiceActive(false); + reportVoiceSessionState(null); + }; + }, [reportVoiceSessionState, setVoiceActive]); + + if (inputMode !== "voice" || phase === "connecting" || phase === "connected") + return null; + return ( + { + if (!consented || phase === "stopping" || sessionActive.current) return; + sessionActive.current = true; + setConsented(false); + setState({ phase: "connecting", message: null }); + const next = createLiveConversation((nextState) => { + if (session.current !== next) return; + if (nextState.phase === "ended" || nextState.phase === "error") { + sessionActive.current = false; + } + setState(nextState); + setVoiceActive( + nextState.phase === "connecting" || nextState.phase === "connected", + ); + }, connectionTimeoutMs); + session.current = next; + setVoiceActive(true); + void next.start(); + }} + onExit={() => { + void end(); + setInputMode("text"); + }} + /> + ); +}; diff --git a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation.test.ts new file mode 100644 index 00000000000..c743241869a --- /dev/null +++ b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation.test.ts @@ -0,0 +1,558 @@ +// @vitest-environment jsdom +import { afterEach, expect, test, vi } from "vitest"; + +import { createLiveConversation } from "./live-conversation"; + +const setup = () => { + const sent: string[] = []; + const channel = Object.assign(new EventTarget(), { + readyState: "open", + send: (data: string) => sent.push(data), + close: vi.fn(), + }); + const input = Object.assign(new EventTarget(), { stop: vi.fn() }); + const output = { stop: vi.fn() }; + const stream = { getTracks: () => [input] }; + const peer = Object.assign(new EventTarget(), { + connectionState: "new", + iceGatheringState: "complete", + localDescription: { type: "offer", sdp: "v=0\r\no=complete-offer" }, + createDataChannel: vi.fn(() => channel), + addTrack: vi.fn(), + createOffer: vi.fn(async () => ({ type: "offer", sdp: "v=0" })), + setLocalDescription: vi.fn(async () => undefined), + setRemoteDescription: vi.fn(async () => undefined), + close: vi.fn(), + getReceivers: () => [{ track: output }], + }); + const audio = { + srcObject: null, + autoplay: false, + muted: false, + paused: false, + play: vi.fn(async () => undefined), + pause: vi.fn(), + }; + vi.stubGlobal( + "RTCPeerConnection", + class { + constructor() { + return peer; + } + }, + ); + vi.stubGlobal( + "Audio", + class { + constructor() { + return audio; + } + }, + ); + const getUserMedia = vi.fn(async () => stream); + vi.stubGlobal("navigator", { mediaDevices: { getUserMedia } }); + const fetch = vi.fn(async () => + Response.json({ sessionId: "opaque/id", sdp: "v=0\r\no=answer" }), + ); + vi.stubGlobal("fetch", fetch); + const onState = vi.fn[0]>(); + const conversation = createLiveConversation(onState, 15_000); + const emit = (data: unknown) => + channel.dispatchEvent( + new MessageEvent("message", { data: JSON.stringify(data) }), + ); + return { + conversation, + emit, + sent, + channel, + input, + output, + peer, + audio, + getUserMedia, + fetch, + onState, + }; +}; + +afterEach(() => { + vi.useRealTimers(); + vi.unstubAllGlobals(); +}); + +test("waits for session.started after SDP; never sends session.start or handles transcript/delegation as tasks", async () => { + const fixture = setup(); + expect(fixture.getUserMedia).not.toHaveBeenCalled(); + await fixture.conversation.start(); + expect(fixture.onState).toHaveBeenLastCalledWith( + expect.objectContaining({ phase: "connecting" }), + ); + expect(fixture.fetch).toHaveBeenCalledWith( + "/api/voice/live-session", + expect.objectContaining({ body: "v=0\r\no=complete-offer" }), + ); + expect(fixture.peer.createDataChannel).toHaveBeenCalledWith("oai-events"); + fixture.emit({ type: "session.started" }); + expect(fixture.onState).toHaveBeenLastCalledWith({ + phase: "connected", + message: null, + }); + fixture.emit({ + type: "session.input_transcript.delta", + delta: "Change the model", + }); + fixture.emit({ type: "session.output_transcript.delta", delta: "Done" }); + fixture.emit({ + type: "session.delegation.created", + delegation: { id: "metadata-only", target: "client" }, + }); + expect(fixture.sent).toEqual([]); + expect(fixture.fetch).toHaveBeenCalledTimes(1); + const stopped = fixture.conversation.stop(); + expect(fixture.input.stop).toHaveBeenCalled(); + expect(fixture.output.stop).toHaveBeenCalled(); + expect(fixture.audio.pause).toHaveBeenCalled(); + expect(fixture.audio.srcObject).toBeNull(); + expect(fixture.sent.map((event) => JSON.parse(event))).toEqual([ + { type: "session.close" }, + ]); + expect(fixture.peer.close).not.toHaveBeenCalled(); + fixture.emit({ type: "session.closed" }); + await stopped; + expect(fixture.peer.close).toHaveBeenCalledOnce(); + expect(fixture.onState).toHaveBeenLastCalledWith({ + phase: "ended", + message: "Microphone and playback stopped. Live confirmed session closure.", + }); +}); + +test("releases late microphone permission without ever creating a provider session", async () => { + const fixture = setup(); + let release!: ( + stream: Awaited>, + ) => void; + fixture.getUserMedia.mockImplementation( + () => + new Promise((resolve) => { + release = resolve; + }), + ); + const started = fixture.conversation.start(); + await fixture.conversation.stop(); + release({ getTracks: () => [fixture.input] }); + await started; + expect(fixture.input.stop).toHaveBeenCalled(); + expect(fixture.fetch).not.toHaveBeenCalled(); +}); + +test("reports missing remote closure honestly and cleans transport after bounded wait", async () => { + vi.useFakeTimers(); + const fixture = setup(); + await fixture.conversation.start(); + fixture.emit({ type: "session.started" }); + const stopped = fixture.conversation.stop(); + await vi.advanceTimersByTimeAsync(2_000); + await stopped; + expect(fixture.peer.close).toHaveBeenCalled(); + expect(fixture.onState.mock.lastCall?.[0].phase).toBe("ended"); + expect(fixture.onState.mock.lastCall?.[0].message).toContain("not confirmed"); +}); + +test("times out connection without inventing turn completion or retrying", async () => { + vi.useFakeTimers(); + const fixture = setup(); + await fixture.conversation.start(); + await vi.advanceTimersByTimeAsync(17_000); + expect(fixture.input.stop).toHaveBeenCalled(); + expect(fixture.onState.mock.lastCall?.[0].phase).toBe("error"); + expect(fixture.onState.mock.lastCall?.[0].message).toContain("timed out"); + expect(fixture.fetch).toHaveBeenCalledTimes(1); +}); + +test("a temporary media disconnect recovers on the same session without replaying input", async () => { + vi.useFakeTimers(); + const fixture = setup(); + let release!: (stats: Map) => void; + const getStats = vi.fn( + () => + new Promise>((resolve) => { + release = resolve; + }), + ); + Object.assign(fixture.peer, { getStats }); + await fixture.conversation.start(); + fixture.emit({ type: "session.started" }); + await vi.advanceTimersByTimeAsync(100); + fixture.peer.connectionState = "disconnected"; + fixture.peer.dispatchEvent(new Event("connectionstatechange")); + expect(fixture.input.stop).not.toHaveBeenCalled(); + expect(fixture.audio.pause).not.toHaveBeenCalled(); + expect(fixture.onState.mock.lastCall?.[0].phase).toBe("connecting"); + // A sample requested before the disconnect must not restore Listening. + release( + new Map([ + ["input", { type: "media-source", kind: "audio", audioLevel: 0.4 }], + ]), + ); + getStats.mockResolvedValue(new Map()); + await vi.advanceTimersByTimeAsync(1_000); + expect(fixture.onState.mock.lastCall?.[0].phase).toBe("connecting"); + fixture.peer.connectionState = "connected"; + fixture.peer.dispatchEvent(new Event("connectionstatechange")); + expect(fixture.onState.mock.lastCall?.[0].phase).toBe("connected"); + await vi.advanceTimersByTimeAsync(16_000); + expect(fixture.input.stop).not.toHaveBeenCalled(); + expect(fixture.onState.mock.lastCall?.[0]).toMatchObject({ + phase: "connected", + activity: { microphoneLevel: 0, outputActive: false }, + }); + expect(fixture.sent).toEqual([]); + expect(fixture.fetch).toHaveBeenCalledOnce(); + expect(fixture.getUserMedia).toHaveBeenCalledOnce(); + const stopped = fixture.conversation.stop(); + fixture.emit({ type: "session.closed" }); + await stopped; + expect(vi.getTimerCount()).toBe(0); +}); + +test("an unrecovered disconnect has a fixed deadline and cannot revive after cleanup", async () => { + vi.useFakeTimers(); + const fixture = setup(); + await fixture.conversation.start(); + fixture.emit({ type: "session.started" }); + fixture.peer.connectionState = "disconnected"; + fixture.peer.dispatchEvent(new Event("connectionstatechange")); + await vi.advanceTimersByTimeAsync(5_000); + fixture.peer.dispatchEvent(new Event("connectionstatechange")); + await vi.advanceTimersByTimeAsync(9_999); + expect(fixture.input.stop).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(1); + expect(fixture.input.stop).toHaveBeenCalled(); + expect(fixture.output.stop).toHaveBeenCalled(); + expect(fixture.audio.pause).toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(2_000); + expect(fixture.onState.mock.lastCall?.[0].phase).toBe("error"); + expect(fixture.onState.mock.lastCall?.[0].message).toContain( + "did not recover", + ); + const updates = fixture.onState.mock.calls.length; + fixture.peer.connectionState = "connected"; + fixture.peer.dispatchEvent(new Event("connectionstatechange")); + await vi.advanceTimersByTimeAsync(20_000); + expect(fixture.onState).toHaveBeenCalledTimes(updates); + expect(fixture.fetch).toHaveBeenCalledOnce(); + expect(vi.getTimerCount()).toBe(0); +}); + +test.each(["failed", "closed"])( + "terminal media state %s still ends immediately", + async (state) => { + const fixture = setup(); + await fixture.conversation.start(); + fixture.emit({ type: "session.started" }); + fixture.peer.connectionState = state; + fixture.peer.dispatchEvent(new Event("connectionstatechange")); + expect(fixture.input.stop).toHaveBeenCalled(); + expect(fixture.audio.pause).toHaveBeenCalled(); + fixture.emit({ type: "session.closed" }); + expect(fixture.onState.mock.lastCall?.[0].phase).toBe("error"); + }, +); + +test("End during a disconnect cancels the recovery deadline", async () => { + vi.useFakeTimers(); + const fixture = setup(); + await fixture.conversation.start(); + fixture.emit({ type: "session.started" }); + fixture.peer.connectionState = "disconnected"; + fixture.peer.dispatchEvent(new Event("connectionstatechange")); + const stopped = fixture.conversation.stop(); + fixture.emit({ type: "session.closed" }); + await stopped; + await vi.advanceTimersByTimeAsync(20_000); + expect(fixture.onState.mock.lastCall?.[0].phase).toBe("ended"); + expect(vi.getTimerCount()).toBe(0); +}); + +test.each([false, true])( + "a handshake disconnect remains bounded (session.started received: %s)", + async (ready) => { + vi.useFakeTimers(); + const fixture = setup(); + await fixture.conversation.start(); + fixture.peer.connectionState = "disconnected"; + fixture.peer.dispatchEvent(new Event("connectionstatechange")); + await vi.advanceTimersByTimeAsync(5_000); + if (ready) fixture.emit({ type: "session.started" }); + await vi.advanceTimersByTimeAsync(ready ? 14_999 : 9_999); + expect(fixture.input.stop).not.toHaveBeenCalled(); + expect(fixture.onState.mock.lastCall?.[0].phase).toBe("connecting"); + await vi.advanceTimersByTimeAsync(1); + expect(fixture.input.stop).toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(2_000); + expect(fixture.onState.mock.lastCall?.[0].phase).toBe("error"); + expect(fixture.fetch).toHaveBeenCalledOnce(); + expect(vi.getTimerCount()).toBe(0); + }, +); + +test("provider errors stop both media directions without exposing payloads", async () => { + const fixture = setup(); + await fixture.conversation.start(); + fixture.emit({ type: "session.started" }); + fixture.emit({ + type: "error", + error: { message: "sensitive provider detail" }, + }); + expect(fixture.input.stop).toHaveBeenCalled(); + expect(fixture.output.stop).toHaveBeenCalled(); + fixture.emit({ type: "session.closed" }); + expect(fixture.onState.mock.lastCall?.[0].phase).toBe("error"); + expect(fixture.onState.mock.lastCall?.[0].message).not.toContain("sensitive"); +}); + +test.each([false, true])( + "microphone loss stops playback and closes without restarting (session started: %s)", + async (ready) => { + const fixture = setup(); + await fixture.conversation.start(); + if (ready) fixture.emit({ type: "session.started" }); + fixture.input.dispatchEvent(new Event("ended")); + expect(fixture.audio.pause).toHaveBeenCalled(); + expect(fixture.audio.muted).toBe(true); + expect(fixture.output.stop).toHaveBeenCalled(); + expect(fixture.sent).toEqual(ready ? ['{"type":"session.close"}'] : []); + if (ready) fixture.emit({ type: "session.closed" }); + await fixture.conversation.stop(); + expect(fixture.peer.close).toHaveBeenCalledOnce(); + expect(fixture.onState.mock.lastCall?.[0].phase).toBe("error"); + expect(fixture.onState.mock.lastCall?.[0].message).toContain( + "Microphone disconnected", + ); + expect(fixture.getUserMedia).toHaveBeenCalledOnce(); + expect(fixture.fetch).toHaveBeenCalledOnce(); + }, +); + +test("Stop before session.started never sends application commands", async () => { + vi.useFakeTimers(); + const fixture = setup(); + await fixture.conversation.start(); + const stopped = fixture.conversation.stop(); + await vi.advanceTimersByTimeAsync(2_000); + await stopped; + expect(fixture.sent).toEqual([]); + expect(fixture.onState.mock.lastCall?.[0].message).toContain("not confirmed"); +}); + +test("plays the remote track while microphone remains attached, then stops both", async () => { + const fixture = setup(); + await fixture.conversation.start(); + fixture.emit({ type: "session.started" }); + const remoteStream = { getTracks: () => [fixture.output] }; + fixture.peer.dispatchEvent( + Object.assign(new Event("track"), { + track: fixture.output, + streams: [remoteStream], + }), + ); + expect(fixture.audio.srcObject).toBe(remoteStream); + expect(fixture.audio.autoplay).toBe(true); + expect(fixture.audio.play).toHaveBeenCalledOnce(); + expect(fixture.peer.addTrack).toHaveBeenCalledWith( + fixture.input, + expect.anything(), + ); + expect(fixture.input.stop).not.toHaveBeenCalled(); + const stopped = fixture.conversation.stop(); + fixture.emit({ type: "session.closed" }); + await stopped; + expect(fixture.audio.pause).toHaveBeenCalled(); + expect(fixture.input.stop).toHaveBeenCalled(); +}); + +test.each(["rejects", "throws"] as const)( + "reports playback that %s and retries it without restarting the Live session", + async (failureMode) => { + vi.useFakeTimers(); + const fixture = setup(); + Object.assign(fixture.peer, { + getStats: vi.fn( + async () => + new Map([ + [ + "input", + { type: "media-source", kind: "audio", audioLevel: 0.42 }, + ], + ]), + ), + }); + if (failureMode === "rejects") { + fixture.audio.play.mockRejectedValueOnce(new Error("Playback blocked")); + } else { + fixture.audio.play.mockImplementationOnce(() => { + throw new Error("Playback blocked"); + }); + } + await fixture.conversation.start(); + fixture.emit({ type: "session.started" }); + await vi.advanceTimersByTimeAsync(100); + fixture.peer.dispatchEvent( + Object.assign(new Event("track"), { + track: fixture.output, + streams: [{ getTracks: () => [fixture.output] }], + }), + ); + await Promise.resolve(); + + expect(fixture.onState).toHaveBeenLastCalledWith({ + phase: "connected", + message: + "Audio playback is blocked. Select Play voice audio to hear Live.", + playbackBlocked: true, + activity: { microphoneLevel: 0.42, outputActive: false }, + }); + await vi.advanceTimersByTimeAsync(100); + expect(fixture.onState.mock.lastCall?.[0]).toMatchObject({ + phase: "connected", + message: + "Audio playback is blocked. Select Play voice audio to hear Live.", + playbackBlocked: true, + }); + expect(fixture.input.stop).not.toHaveBeenCalled(); + expect(fixture.output.stop).not.toHaveBeenCalled(); + await fixture.conversation.retryPlayback(); + expect(fixture.audio.play).toHaveBeenCalledTimes(2); + expect(fixture.onState).toHaveBeenLastCalledWith({ + phase: "connected", + message: null, + activity: { microphoneLevel: 0.42, outputActive: false }, + }); + expect(fixture.fetch).toHaveBeenCalledOnce(); + expect(fixture.getUserMedia).toHaveBeenCalledOnce(); + const stopped = fixture.conversation.stop(); + fixture.emit({ type: "session.closed" }); + await stopped; + expect(vi.getTimerCount()).toBe(0); + }, +); + +test("reports local audio activity without treating silence or transcripts as turn completion", async () => { + vi.useFakeTimers(); + const fixture = setup(); + const levels = { input: 0.24, output: 0 }; + const getStats = vi.fn( + async () => + new Map([ + [ + "input", + { type: "media-source", kind: "audio", audioLevel: levels.input }, + ], + [ + "output", + { type: "inbound-rtp", kind: "audio", audioLevel: levels.output }, + ], + ]), + ); + Object.assign(fixture.peer, { getStats }); + await fixture.conversation.start(); + await vi.advanceTimersByTimeAsync(100); + expect(getStats).not.toHaveBeenCalled(); + fixture.emit({ type: "session.started" }); + fixture.peer.dispatchEvent( + Object.assign(new Event("track"), { + track: fixture.output, + streams: [{ getTracks: () => [fixture.output] }], + }), + ); + await vi.advanceTimersByTimeAsync(100); + expect(fixture.onState).toHaveBeenLastCalledWith( + expect.objectContaining({ + phase: "connected", + activity: { microphoneLevel: 0.24, outputActive: false }, + }), + ); + levels.output = 0.2; + await vi.advanceTimersByTimeAsync(100); + expect(fixture.onState).toHaveBeenLastCalledWith( + expect.objectContaining({ + activity: { microphoneLevel: 0.24, outputActive: true }, + }), + ); + expect(fixture.input.stop).not.toHaveBeenCalled(); + levels.output = 0; + await vi.advanceTimersByTimeAsync(100); + expect(fixture.onState.mock.lastCall?.[0]).toMatchObject({ + activity: { outputActive: true }, + }); + await vi.advanceTimersByTimeAsync(400); + fixture.emit({ + type: "session.output_transcript.delta", + delta: "Not playback", + }); + expect(fixture.onState.mock.lastCall?.[0]).toMatchObject({ + phase: "connected", + activity: { outputActive: false }, + }); + levels.output = 0.2; + fixture.audio.paused = true; + await vi.advanceTimersByTimeAsync(100); + expect(fixture.onState.mock.lastCall?.[0]).toMatchObject({ + activity: { outputActive: false }, + }); + fixture.audio.paused = false; + fixture.audio.muted = true; + await vi.advanceTimersByTimeAsync(100); + expect(fixture.onState.mock.lastCall?.[0]).toMatchObject({ + activity: { outputActive: false }, + }); + expect(fixture.sent).toEqual([]); + expect(fixture.fetch).toHaveBeenCalledTimes(1); + const stopped = fixture.conversation.stop(); + const samples = getStats.mock.calls.length; + await vi.advanceTimersByTimeAsync(100); + expect(getStats).toHaveBeenCalledTimes(samples); + fixture.emit({ type: "session.closed" }); + await stopped; +}); + +test("missing or failed telemetry does not end the session; a late sample cannot revive it after Stop", async () => { + vi.useFakeTimers(); + const fixture = setup(); + const getStats = vi.fn(async (): Promise> => new Map()); + Object.assign(fixture.peer, { getStats }); + await fixture.conversation.start(); + fixture.emit({ type: "session.started" }); + await vi.advanceTimersByTimeAsync(100); + expect(fixture.onState.mock.lastCall?.[0]).toMatchObject({ + phase: "connected", + activity: { microphoneLevel: 0, outputActive: false }, + }); + getStats.mockRejectedValueOnce(new Error("Telemetry unavailable")); + await vi.advanceTimersByTimeAsync(100); + expect(fixture.input.stop).not.toHaveBeenCalled(); + expect(fixture.sent).toEqual([]); + let release!: (stats: Map) => void; + getStats.mockImplementationOnce( + () => + new Promise((resolve) => { + release = resolve; + }), + ); + await vi.advanceTimersByTimeAsync(100); + const stopped = fixture.conversation.stop(); + fixture.emit({ type: "session.closed" }); + await stopped; + const updates = fixture.onState.mock.calls.length; + const samples = getStats.mock.calls.length; + release( + new Map([ + ["late", { type: "media-source", kind: "audio", audioLevel: 0.9 }], + ]), + ); + await vi.advanceTimersByTimeAsync(500); + expect(fixture.onState).toHaveBeenCalledTimes(updates); + expect(getStats).toHaveBeenCalledTimes(samples); + expect(fixture.fetch).toHaveBeenCalledTimes(1); +}); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation.ts b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation.ts new file mode 100644 index 00000000000..66cb3337722 --- /dev/null +++ b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation.ts @@ -0,0 +1,356 @@ +export interface LiveConversationState { + readonly phase: + | "idle" + | "connecting" + | "connected" + | "stopping" + | "ended" + | "error"; + readonly message: string | null; + readonly playbackBlocked?: boolean; + /** Local media activity for the dock, never a turn or playback-completion signal. */ + readonly activity?: { + readonly microphoneLevel: number; + readonly outputActive: boolean; + }; +} + +/** One disposable Live session. No composer, tool, transcript or turn-settlement interface. */ +export const createLiveConversation = ( + onState: (state: LiveConversationState) => void, + connectionTimeoutMs: number, +) => { + const abort = new AbortController(); + let peer: RTCPeerConnection | undefined; + let channel: RTCDataChannel | undefined; + let microphone: MediaStream | undefined; + let audio: HTMLAudioElement | undefined; + let started = false; + let ready = false; + let recovering = false; + let playbackBlocked = false; + let playbackAttempt = 0; + let stopping = false; + let finished = false; + let creationRequested = false; + let failure: string | undefined; + let connectionTimer: ReturnType | undefined; + let closeTimer: ReturnType | undefined; + let activityTimer: ReturnType | undefined; + let lastOutputActivity = -Infinity; + let lastActivity: LiveConversationState["activity"]; + let resolveStopped: () => void = () => {}; + const stopped = new Promise((resolve) => { + resolveStopped = resolve; + }); + + const activeState = ( + phase: "connecting" | "connected", + activity?: LiveConversationState["activity"], + ): LiveConversationState => ({ + phase, + message: playbackBlocked + ? "Audio playback is blocked. Select Play voice audio to hear Live." + : null, + ...(playbackBlocked ? { playbackBlocked: true } : {}), + ...(activity ? { activity } : {}), + }); + + const sampleActivity = async () => { + if (stopping || !peer) return; + let microphoneLevel = 0; + let outputLevel = 0; + try { + const stats = await peer.getStats(); + stats.forEach((report: unknown) => { + if ( + typeof report !== "object" || + report === null || + !("kind" in report) || + report.kind !== "audio" || + !("audioLevel" in report) || + typeof report.audioLevel !== "number" || + !("type" in report) + ) + return; + if (report.type === "media-source") microphoneLevel = report.audioLevel; + if (report.type === "inbound-rtp") outputLevel = report.audioLevel; + }); + } catch { + // Optional browser telemetry must not terminate or retry the conversation. + } + if (abort.signal.aborted) return; + activityTimer = setTimeout(() => void sampleActivity(), 100); + if (recovering) return; + const playing = audio?.srcObject && !audio.paused && !audio.muted; + if (playing && outputLevel > 0.01) lastOutputActivity = Date.now(); + const activity = { + microphoneLevel: Math.round(microphoneLevel * 100) / 100, + // Brief hold avoids flicker between syllables. This never settles a turn; + // received audio energy also cannot prove that the user heard playback. + outputActive: Boolean(playing) && Date.now() - lastOutputActivity < 300, + }; + if ( + !lastActivity || + activity.microphoneLevel !== lastActivity.microphoneLevel || + activity.outputActive !== lastActivity.outputActive + ) { + lastActivity = activity; + onState(activeState("connected", activity)); + } + }; + + const stopMedia = () => { + clearTimeout(activityTimer); + microphone?.getTracks().forEach((track) => track.stop()); + if (audio) { + audio.muted = true; + audio.pause(); + audio.srcObject = null; + } + peer?.getReceivers().forEach((receiver) => receiver.track.stop()); + }; + + const finish = (confirmed: boolean) => { + if (finished) return; + finished = true; + stopping = true; + clearTimeout(connectionTimer); + clearTimeout(closeTimer); + abort.abort(); + stopMedia(); + channel?.close(); + peer?.close(); + const closure = confirmed + ? "Live confirmed session closure." + : creationRequested + ? "Remote session closure was not confirmed." + : "No provider session was requested."; + onState({ + phase: failure ? "error" : "ended", + message: `${failure ? `${failure} ` : ""}Microphone and playback stopped. ${closure}`, + }); + resolveStopped(); + }; + + const stop = (): Promise => { + if (stopping) return stopped; + stopping = true; + clearTimeout(connectionTimer); + abort.abort(); + stopMedia(); + onState({ + phase: "stopping", + message: "Microphone and playback stopped. Closing Live…", + }); + if (channel?.readyState === "open" && ready) { + // Registered message listener remains until session.closed or this cleanup deadline. + closeTimer = setTimeout(() => finish(false), 2_000); + try { + channel.send(JSON.stringify({ type: "session.close" })); + } catch { + finish(false); + } + } else { + finish(false); + } + return stopped; + }; + + const fail = (message: string) => { + if (stopping) return; + failure = message; + void stop(); + }; + + const playAudio = async (): Promise => { + const currentAudio = audio; + if (stopping || !currentAudio?.srcObject) return; + const attempt = ++playbackAttempt; + try { + await currentAudio.play(); + } catch { + if (abort.signal.aborted || attempt !== playbackAttempt) return; + playbackBlocked = true; + onState( + activeState( + ready && !recovering ? "connected" : "connecting", + lastActivity, + ), + ); + return; + } + if (abort.signal.aborted || attempt !== playbackAttempt || !playbackBlocked) + return; + playbackBlocked = false; + onState( + activeState( + ready && !recovering ? "connected" : "connecting", + lastActivity, + ), + ); + }; + + const start = async (): Promise => { + if (started || stopping) return; + started = true; + onState(activeState("connecting")); + connectionTimer = setTimeout( + () => fail("Live connection timed out. No automatic retry was made."), + connectionTimeoutMs, + ); + try { + audio = new Audio(); + audio.autoplay = true; + const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + if (abort.signal.aborted) { + stream.getTracks().forEach((track) => track.stop()); + return; + } + microphone = stream; + peer = new RTCPeerConnection(); + const connection = peer; + const handleConnectionState = () => { + if (stopping) return; + if (["failed", "closed"].includes(connection.connectionState)) { + fail("Live media connection ended."); + } else if ( + connection.connectionState === "disconnected" && + ready && + !recovering + ) { + // ICE can recover on this connection. Do not create a session or replay input. + recovering = true; + lastActivity = undefined; + lastOutputActivity = -Infinity; + onState(activeState("connecting")); + connectionTimer = setTimeout( + () => + fail( + "Live media connection did not recover. No automatic retry was made.", + ), + connectionTimeoutMs, + ); + } else if (connection.connectionState === "connected" && recovering) { + recovering = false; + clearTimeout(connectionTimer); + onState(activeState("connected")); + } + }; + channel = connection.createDataChannel("oai-events"); + channel.addEventListener("message", (event: MessageEvent) => { + if (finished) return; + let data: unknown; + try { + data = JSON.parse(event.data); + } catch { + fail("Live sent an unreadable event."); + return; + } + if (typeof data !== "object" || data === null || !("type" in data)) + return; + if (data.type === "session.closed") { + finish(true); + return; + } + if (stopping) return; + if (data.type === "session.started" && !ready) { + ready = true; + clearTimeout(connectionTimer); + activityTimer = setTimeout(() => void sampleActivity(), 100); + onState(activeState("connected")); + handleConnectionState(); + } else if (data.type === "error" || data.type === "session.error") { + fail("Live reported an error. No automatic retry was made."); + } + // Transcript deltas are not finalized utterances. Delegations contain metadata, + // not task text. Neither is forwarded, persisted, or used to execute anything. + }); + channel.addEventListener("close", () => { + if (!stopping) failure = "Live disconnected."; + finish(false); + }); + channel.addEventListener("error", () => + fail("Live data connection failed."), + ); + connection.addEventListener( + "connectionstatechange", + handleConnectionState, + ); + connection.addEventListener("track", (event) => { + if (stopping) { + event.track.stop(); + return; + } + if (!audio) return; + audio.srcObject = event.streams[0] ?? new MediaStream([event.track]); + void playAudio(); + }); + stream.getTracks().forEach((track) => { + track.addEventListener( + "ended", + () => fail("Microphone disconnected."), + { + signal: abort.signal, + }, + ); + connection.addTrack(track, stream); + }); + await connection.setLocalDescription(await connection.createOffer()); + abort.signal.throwIfAborted(); + if (connection.iceGatheringState !== "complete") { + const listeners = new AbortController(); + await new Promise((resolve, reject) => { + const check = () => { + if (connection.iceGatheringState === "complete") { + resolve(); + } + }; + const cancelled = () => { + reject(abort.signal.reason); + }; + connection.addEventListener("icegatheringstatechange", check, { + signal: listeners.signal, + }); + abort.signal.addEventListener("abort", cancelled, { + once: true, + signal: listeners.signal, + }); + check(); + }).finally(() => listeners.abort()); + } + abort.signal.throwIfAborted(); + const sdp = connection.localDescription?.sdp; + if (!sdp) throw new Error("Missing local SDP"); + creationRequested = true; + const response = await fetch("/api/voice/live-session", { + method: "POST", + headers: { "content-type": "application/sdp" }, + body: sdp, + signal: abort.signal, + }); + if (!response.ok) throw new Error("Session creation failed"); + const answer: unknown = await response.json(); + abort.signal.throwIfAborted(); + if ( + typeof answer !== "object" || + answer === null || + !("sdp" in answer) || + typeof answer.sdp !== "string" || + !answer.sdp.trimStart().startsWith("v=0") + ) + throw new Error("Invalid SDP answer"); + await connection.setRemoteDescription({ + type: "answer", + sdp: answer.sdp, + }); + // WebRTC creation already starts Live. Wait for session.started; never send session.start. + } catch { + fail( + "Live could not connect. Check microphone, audio permissions and server configuration. No automatic retry was made.", + ); + } + }; + + return { retryPlayback: playAudio, start, stop }; +}; diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx index 17f7317a30a..0be154c6c07 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx @@ -30,13 +30,15 @@ import { VoiceTurnController } from "./voice-turn-controller"; import type { AgentSendResult } from "@flue/sdk"; import type { PetrinautAiVoiceModeContext, - PetrinautAiVoiceModeControls, + PetrinautAiVoiceModeSessionControls, PetrinautAiVoiceSessionState, } from "@hashintel/petrinaut/ui"; const config = { available: true as const, connectionTimeoutMs: 15_000 }; -let registeredVoiceModeControls: PetrinautAiVoiceModeControls | undefined; +let registeredVoiceModeControls: + | PetrinautAiVoiceModeSessionControls + | undefined; const VoiceInterviewHarness = () => { "use no memo"; diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx index 1641df27470..ee465471ad8 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx @@ -11,17 +11,17 @@ import { type FlueChatResponseMessageCompletedEvent, type FlueChatResponseMessageStartedEvent, } from "@hashintel/brunch-agent-transport-aisdk"; -import { Button, Checkbox } from "@hashintel/ds-components"; -import { css } from "@hashintel/ds-helpers/css"; import { reportVoiceDiagnostic } from "../../../voice-diagnostics"; import { selectCanonicalSpeech } from "./canonical-speech"; +import { LiveConversationControl } from "./live-conversation-control"; import { OpenAIRealtimeSession } from "./openai-realtime-session"; import { RealtimeBrunchBridge, type RealtimeBrunchAdmissionTarget, type VoiceSubmissionSettlement, } from "./realtime-brunch-bridge"; +import { VoiceInterviewDisclosure } from "./voice-interview-disclosure"; import { toVoiceSessionState } from "./voice-session-state"; import { VoiceTurnController, @@ -132,6 +132,7 @@ export const submitVoiceInputWithAdmission = async ({ export interface OpenAIVoiceConfig { readonly available: true; readonly connectionTimeoutMs: number; + readonly provider?: "realtime" | "live"; } export const VOICE_INTERVIEW_DISCLOSURE_STORAGE_KEY = @@ -231,6 +232,9 @@ export const loadOpenAIVoiceConfig = async ( if ( !isRecord(body) || body.available !== true || + (body.provider !== undefined && + body.provider !== "realtime" && + body.provider !== "live") || !Number.isInteger(body.connectionTimeoutMs) || (body.connectionTimeoutMs as number) < 1_000 || (body.connectionTimeoutMs as number) > 60_000 @@ -240,206 +244,15 @@ export const loadOpenAIVoiceConfig = async ( return { available: true, connectionTimeoutMs: body.connectionTimeoutMs as number, + ...(body.provider === undefined + ? {} + : { provider: body.provider as "realtime" | "live" }), }; } catch { return null; } }; -const VoiceModeIcon = () => ( - -); - -const disclosureFrameStyle = css({ - width: "full", - padding: "2", - borderTopWidth: "thin", - borderTopStyle: "solid", - borderTopColor: "neutral.a20", - backgroundColor: "neutral.bg.subtle", - color: "neutral.s100", - _focus: { outline: "none" }, -}); - -const disclosureCardStyle = css({ - display: "flex", - flexDirection: "column", - gap: "2", - padding: "3", - borderWidth: "thin", - borderStyle: "solid", - borderColor: "neutral.a20", - borderRadius: "xl", - backgroundColor: "neutral.s00", - boxShadow: - "[0px 0px 0px 1px rgba(0,0,0,0.03), 0px 8px 16px -12px rgba(0,0,0,0.18)]", -}); - -const disclosureHeaderStyle = css({ - display: "flex", - alignItems: "center", - gap: "2", -}); - -const disclosureIconStyle = css({ - display: "inline-flex", - width: "7", - height: "7", - flexShrink: "0", - alignItems: "center", - justifyContent: "center", - borderRadius: "lg", - backgroundColor: "blue.a20", - color: "blue.s90", -}); - -const disclosureTitleStyle = css({ - display: "flex", - minWidth: "[0]", - flexDirection: "column", - gap: "0.5", -}); - -const disclosureHeadingStyle = css({ - fontSize: "sm", - fontWeight: "semibold", - lineHeight: "tight", -}); - -const disclosureSubtitleStyle = css({ - color: "neutral.s80", - fontSize: "xs", -}); - -const disclosureCopyStyle = css({ - color: "neutral.s90", - fontSize: "xs", - lineHeight: "relaxed", -}); - -const disclosureConsentStyle = css({ - width: "full", - padding: "2", - borderRadius: "lg", - backgroundColor: "neutral.a10", - color: "neutral.s100", -}); - -const disclosureActionsStyle = css({ - display: "flex", - flexWrap: "wrap", - alignItems: "center", - gap: "2", -}); - -const disclosureStatusStyle = css({ - minHeight: "[18px]", - color: "neutral.s80", - fontSize: "xs", - lineHeight: "relaxed", -}); - -const VoiceInterviewDisclosure = ({ - checkingMicrophone, - consented, - microphoneCheck, - onCheckMicrophone, - onConsentChange, - onStart, -}: { - readonly checkingMicrophone: boolean; - readonly consented: boolean; - readonly microphoneCheck: string; - readonly onCheckMicrophone: () => void; - readonly onConsentChange: (consented: boolean) => void; - readonly onStart: () => void; -}) => { - const disclosureRef = useRef(null); - - useEffect(() => { - disclosureRef.current?.focus(); - }, []); - - return ( -
-
-
- - - -
- - Start a voice conversation - - - Talk through your process with AI - -
-
-

- OpenAI processes live audio and speaks the interviewer’s words. - Petrinaut saves finalized answers—not audio. -

- -
- - -
-
- {microphoneCheck} -
-
-
- ); -}; - const recordLatency = (event: VoiceLatencyEvent): void => { try { performance.measure(`voice-interview:${event.name}`, { @@ -753,7 +566,7 @@ const AvailableVoiceInterviewControl = ({ return null; }; -export const VoiceInterviewControl = ({ +const PinnedVoiceInterviewControl = ({ config, resolveInputSubmission, resolveResponseSubmission, @@ -774,18 +587,43 @@ export const VoiceInterviewControl = ({ readonly subscribeToResponseMessageCompleted?: SubscribeToResponseMessageCompleted; readonly subscribeToResponseMessageStarted?: SubscribeToResponseMessageStarted; readonly subscribeToStopRequested?: SubscribeToStopRequested; -}) => ( - -); +}) => { + // Configuration changes only apply after a new conversation mount / page load. + // Never replace a running provider or resubmit its input. + const [sessionConfig] = useState(config); + if (sessionConfig.provider === "live") { + if (!context.registerVoiceModeSessionControls) return null; + return ( + + ); + } + return ( + + ); +}; + +export const VoiceInterviewControl = ( + props: Parameters[0], +) => ; diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-disclosure.tsx b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-disclosure.tsx new file mode 100644 index 00000000000..0f0f22f63c5 --- /dev/null +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-disclosure.tsx @@ -0,0 +1,209 @@ +import { useEffect, useRef } from "react"; + +import { Button, Checkbox } from "@hashintel/ds-components"; +import { css } from "@hashintel/ds-helpers/css"; + +const VoiceModeIcon = () => ( + +); + +const disclosureFrameStyle = css({ + width: "full", + padding: "2", + borderTopWidth: "thin", + borderTopStyle: "solid", + borderTopColor: "neutral.a20", + backgroundColor: "neutral.bg.subtle", + color: "neutral.s100", + _focus: { outline: "none" }, +}); +const disclosureCardStyle = css({ + display: "flex", + flexDirection: "column", + gap: "2", + padding: "3", + borderWidth: "thin", + borderStyle: "solid", + borderColor: "neutral.a20", + borderRadius: "xl", + backgroundColor: "neutral.s00", + boxShadow: + "[0px 0px 0px 1px rgba(0,0,0,0.03), 0px 8px 16px -12px rgba(0,0,0,0.18)]", +}); +const disclosureHeaderStyle = css({ + display: "flex", + alignItems: "center", + gap: "2", +}); +const disclosureIconStyle = css({ + display: "inline-flex", + width: "7", + height: "7", + flexShrink: "0", + alignItems: "center", + justifyContent: "center", + borderRadius: "lg", + backgroundColor: "blue.a20", + color: "blue.s90", +}); +const disclosureTitleStyle = css({ + display: "flex", + minWidth: "[0]", + flexDirection: "column", + gap: "0.5", +}); +const disclosureHeadingStyle = css({ + fontSize: "sm", + fontWeight: "semibold", + lineHeight: "tight", +}); +const disclosureSubtitleStyle = css({ color: "neutral.s80", fontSize: "xs" }); +const disclosureCopyStyle = css({ + color: "neutral.s90", + fontSize: "xs", + lineHeight: "relaxed", +}); +const disclosureConsentStyle = css({ + width: "full", + padding: "2", + borderRadius: "lg", + backgroundColor: "neutral.a10", + color: "neutral.s100", +}); +const disclosureActionsStyle = css({ + display: "flex", + flexWrap: "wrap", + alignItems: "center", + gap: "2", +}); +const disclosureStatusStyle = css({ + minHeight: "[18px]", + color: "neutral.s80", + fontSize: "xs", + lineHeight: "relaxed", +}); + +export const VoiceInterviewDisclosure = ({ + checkingMicrophone = false, + consented, + microphoneCheck, + onCheckMicrophone, + onConsentChange, + onStart, + experimental = false, + startDisabled = false, + onExit, +}: { + readonly checkingMicrophone?: boolean; + readonly consented: boolean; + readonly microphoneCheck: string; + readonly onCheckMicrophone?: () => void; + readonly onConsentChange: (consented: boolean) => void; + readonly onStart: () => void; + readonly experimental?: boolean; + readonly startDisabled?: boolean; + readonly onExit?: () => void; +}) => { + const disclosureRef = useRef(null); + useEffect(() => { + disclosureRef.current?.focus(); + }, []); + + return ( +
+
+
+ + + +
+ + {experimental + ? "GPT-Live · Experimental interview" + : "Start a voice conversation"} + + {!experimental && ( + + Talk through your process with AI + + )} +
+
+ {!experimental && ( +

+ OpenAI processes live audio and speaks the interviewer’s words. + Petrinaut saves finalized answers—not audio. +

+ )} + +
+ + {onCheckMicrophone && ( + + )} + {onExit && ( + + )} +
+ {(!experimental || microphoneCheck) && ( +
+ {microphoneCheck} +
+ )} +
+
+ ); +}; diff --git a/apps/petrinaut-website/src/server/voice/openai-live-session.test.ts b/apps/petrinaut-website/src/server/voice/openai-live-session.test.ts new file mode 100644 index 00000000000..e066d2e7300 --- /dev/null +++ b/apps/petrinaut-website/src/server/voice/openai-live-session.test.ts @@ -0,0 +1,265 @@ +import { describe, expect, test, vi } from "vitest"; + +import { createOpenAILiveSessionHandler } from "./openai-live-session"; +import { createOpenAIVoiceConfigHandler } from "./openai-voice-config"; + +const environment = { + PETRINAUT_OPENAI_VOICE_ENABLED: "true", + PETRINAUT_VOICE_PROVIDER: "live", + OPENAI_VOICE_API_KEY: "server-only-secret", +}; +const request = (overrides: RequestInit & { duplex?: "half" } = {}) => + new Request("https://petrinaut.test/api/voice/live-session", { + method: "POST", + headers: { + origin: "https://petrinaut.test", + "content-type": "application/sdp", + }, + body: "v=0\r\no=offer", + ...overrides, + }); + +describe("Live configuration and session creation", () => { + test.each([ + [undefined, "realtime", true], + ["realtime", "realtime", true], + ["live", "live", true], + ["live-brunch", null, false], + ])( + "selects %s without silently falling back", + async (value, provider, available) => { + const handler = createOpenAIVoiceConfigHandler({ + ...environment, + PETRINAUT_VOICE_PROVIDER: value, + }); + const response = await handler( + new Request("https://petrinaut.test/api/voice/config"), + ); + expect(await response.json()).toEqual({ + available, + provider, + connectionTimeoutMs: 15_000, + }); + }, + ); + + test.each([ + [{ method: "GET", body: undefined }, 405], + [ + { + headers: { + origin: "https://attacker.test", + "content-type": "application/sdp", + }, + }, + 403, + ], + [{ headers: { "content-type": "application/sdp" } }, 403], + [ + { + headers: { + origin: "https://petrinaut.test", + "content-type": "application/json", + }, + }, + 415, + ], + [{ body: "not SDP" }, 400], + [{ body: "v=0" + "x".repeat(65_536) }, 413], + ] satisfies [RequestInit, number][])( + "rejects unsafe requests before fetching: %j", + async (overrides, status) => { + const fetch = vi.fn(); + const response = await createOpenAILiveSessionHandler({ + environment, + fetch, + })(request(overrides)); + expect(response.status).toBe(status); + expect(response.headers.get("cache-control")).toBe("no-store"); + expect(fetch).not.toHaveBeenCalled(); + }, + ); + + test("cancels a chunked offer as soon as it exceeds the byte limit", async () => { + const fetch = vi.fn(); + const cancel = vi.fn(); + const chunks = [new TextEncoder().encode("v=0"), new Uint8Array(65_534)]; + const pull = vi.fn( + (controller: ReadableStreamDefaultController) => { + const chunk = chunks.shift(); + if (chunk) controller.enqueue(chunk); + else controller.close(); + }, + ); + const body = new ReadableStream({ pull, cancel }, { highWaterMark: 0 }); + const response = await createOpenAILiveSessionHandler({ + environment, + fetch, + })( + request({ + body, + duplex: "half", + }), + ); + expect(response.status).toBe(413); + expect(cancel).toHaveBeenCalledOnce(); + expect(pull).toHaveBeenCalledTimes(2); + expect(fetch).not.toHaveBeenCalled(); + }); + + test("preserves an offer at the byte limit across split UTF-8 chunks", async () => { + const sdp = "v=0\r\n" + "x".repeat(65_529) + "é"; + const encoded = new TextEncoder().encode(sdp); + expect(encoded.byteLength).toBe(65_536); + const body = new ReadableStream({ + start(controller) { + controller.enqueue(encoded.subarray(0, 65_535)); + controller.enqueue(encoded.subarray(65_535)); + controller.close(); + }, + }); + const fetch = vi.fn(async () => + Response.json({ + session: { id: "session" }, + transport: { type: "webrtc", sdp: "v=0\r\no=answer" }, + }), + ); + const response = await createOpenAILiveSessionHandler({ + environment, + fetch, + })( + request({ + body, + duplex: "half", + }), + ); + expect(response.status).toBe(201); + expect(fetch).toHaveBeenCalledOnce(); + expect(fetch.mock.calls[0]?.[1]?.body).toContain( + JSON.stringify({ type: "webrtc", sdp }), + ); + }); + + test.each(["request", "deadline"])( + "cancels a stalled offer on %s abort without contacting the provider", + async (source) => { + const abort = new AbortController(); + const timeout = vi.spyOn(AbortSignal, "timeout"); + if (source === "deadline") timeout.mockReturnValue(abort.signal); + const fetch = vi.fn(); + const cancel = vi.fn(); + let controller!: ReadableStreamDefaultController; + const body = new ReadableStream({ + start(streamController) { + controller = streamController; + controller.enqueue(new TextEncoder().encode("v=0")); + }, + cancel, + }); + const pending = createOpenAILiveSessionHandler({ environment, fetch })( + request({ + body, + signal: source === "request" ? abort.signal : undefined, + duplex: "half", + }), + ); + try { + abort.abort(); + await vi.waitFor(() => expect(cancel).toHaveBeenCalledOnce()); + expect((await pending).status).toBe(502); + expect(fetch).not.toHaveBeenCalled(); + } finally { + // Release the broken implementation's read too, so a regression cannot leak work. + if (!cancel.mock.calls.length) controller.close(); + await pending; + timeout.mockRestore(); + } + }, + ); + + test.each([ + { PETRINAUT_VOICE_PROVIDER: undefined }, + { PETRINAUT_VOICE_PROVIDER: "realtime" }, + { PETRINAUT_OPENAI_VOICE_ENABLED: "false" }, + { OPENAI_VOICE_API_KEY: " " }, + ])( + "requires Live selection and existing enablement/credentials: %j", + async (override) => { + const fetch = vi.fn(); + expect( + ( + await createOpenAILiveSessionHandler({ + environment: { ...environment, ...override }, + fetch, + })(request()) + ).status, + ).toBe(404); + expect(fetch).not.toHaveBeenCalled(); + }, + ); + + test("creates one client-delegated WebRTC session with trusted instructions and no tools", async () => { + const fetch = vi.fn(async () => + Response.json( + { + session: { id: "opaque/session:id" }, + transport: { type: "webrtc", sdp: "v=0\r\no=answer" }, + secret: "must-not-leak", + }, + { status: 201 }, + ), + ); + const response = await createOpenAILiveSessionHandler({ + environment, + fetch, + })(request()); + expect(response.status).toBe(201); + expect(await response.json()).toEqual({ + sessionId: "opaque/session:id", + sdp: "v=0\r\no=answer", + }); + expect(fetch).toHaveBeenCalledTimes(1); + const [url, init] = fetch.mock.calls[0]!; + expect(url).toBe("https://api.openai.com/v1/live/sessions"); + expect(typeof init?.body).toBe("string"); + const body: unknown = JSON.parse(init?.body as string); + expect(body).toEqual({ + session: { + model: "gpt-live-1", + delegation: { type: "client" }, + store: false, + instructions: expect.stringMatching( + /Interview approach:[\s\S]*Backchannel policy:[\s\S]*Interruption policy:[\s\S]*Delegation policy:\nBackend tools:\n- None\.[\s\S]*Delegate to the backend when:\n- Never in this experiment[\s\S]*Do not delegate to the backend when:[\s\S]*Never claim that anything was changed, executed, or saved\./, + ) as unknown, + audio: { output: { voice: "marin" } }, + }, + transport: { type: "webrtc", sdp: "v=0\r\no=offer" }, + }); + expect(new Headers(init?.headers).get("authorization")).toBe( + "Bearer server-only-secret", + ); + }); + + test("sanitizes provider failure and never retries", async () => { + const fetch = vi.fn( + async () => new Response("server-only-secret", { status: 429 }), + ); + const response = await createOpenAILiveSessionHandler({ + environment, + fetch, + })(request()); + expect(response.status).toBe(502); + expect(await response.text()).not.toContain("server-only-secret"); + expect(fetch).toHaveBeenCalledTimes(1); + }); + + test("does not contact the provider for an already aborted request", async () => { + const fetch = vi.fn(); + const response = await createOpenAILiveSessionHandler({ + environment, + fetch, + })(request({ signal: AbortSignal.abort() })); + expect(response.status).toBe(502); + expect(fetch).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/petrinaut-website/src/server/voice/openai-live-session.ts b/apps/petrinaut-website/src/server/voice/openai-live-session.ts new file mode 100644 index 00000000000..b4e8d270221 --- /dev/null +++ b/apps/petrinaut-website/src/server/voice/openai-live-session.ts @@ -0,0 +1,172 @@ +import { getVoiceProvider } from "./openai-voice-config.js"; +import { getOpenAIVoiceAvailability } from "./openai-voice-policy.js"; + +const instructions = `You are a calm, curious process interviewer. The person speaking is +the domain expert. Help them explain how their process works and +what they want to understand or improve. + +Speak naturally at an unhurried pace. Be clear and direct, not overly +cheerful. If they are unsure or frustrated, acknowledge it briefly +and make the next question easier to answer. + +Interview approach: +Follow their active account and use their vocabulary. Learn their +purpose naturally, without restarting an intake if they have already +begun. Prefer walking through a recent concrete case. + +Notice triggers, sequence, decisions, dependencies, waiting, and +outcomes. Explore exceptions when they matter to the person's purpose. +These guide your attention; they are not a questionnaire. + +Deepen one thread with one focused question at a time. Do not supply +answers or invent precision. Accept “I don't know.” Restate only when +checking an important interpretation, not after every answer. +Keep routine contributions to one or two short sentences. +When the person wants to finish, open no new topic. + +Backchannel policy: Use moderate backchannels. Acknowledge naturally +without competing with the main response. Avoid repetitive praise. + +Interruption policy: Stop speaking when the user interrupts. Listen +to what they say. Follow their correction rather than finishing your +previous point. Keep listening while they pause to think. + +Delegation policy: +Backend tools: +- None. This standalone interview has no application, chat, model, + storage, or execution capabilities. + +Delegate to the backend when: +- Never in this experiment; no backend handles delegated work. + +Do not delegate to the backend when: +- Conducting the interview, clarifying an answer, or incorporating + a correction. + +If asked to operate the application, briefly explain that you cannot. +Never claim that anything was changed, executed, or saved.`; + +/** Uses the existing website credential boundary; this switch is not authentication. */ +export const createOpenAILiveSessionHandler = + ({ + environment, + fetch, + }: { + environment: Parameters[0]; + fetch: typeof globalThis.fetch; + }) => + async (request: Request): Promise => { + const respond = ( + body: string, + status: number, + headers?: Record, + ) => + new Response(body, { + status, + headers: { "cache-control": "no-store", ...headers }, + }); + if (request.method !== "POST") + return respond("Method not allowed.", 405, { allow: "POST" }); + if (request.headers.get("origin") !== new URL(request.url).origin) + return respond("Forbidden.", 403); + if ( + request.headers + .get("content-type") + ?.split(";")[0] + ?.trim() + .toLowerCase() !== "application/sdp" + ) + return respond("Expected SDP.", 415); + const availability = getOpenAIVoiceAvailability(environment); + if (!availability.available || getVoiceProvider(environment) !== "live") + return respond("Live is unavailable.", 404); + + const signal = AbortSignal.any([ + request.signal, + AbortSignal.timeout(availability.connectionTimeoutMs), + ]); + try { + signal.throwIfAborted(); + if (Number(request.headers.get("content-length")) > 65_536) + return respond("SDP too large.", 413); + const body = new Uint8Array(65_536); + let length = 0; + try { + // Bound memory while reading, including chunked offers, and cancel stalled uploads. + await request.body?.pipeTo( + new WritableStream({ + write(chunk) { + length += chunk.byteLength; + if (length > body.byteLength) throw new Error("SDP too large"); + body.set(chunk, length - chunk.byteLength); + }, + }), + { signal }, + ); + } catch (error) { + if (length > body.byteLength) return respond("SDP too large.", 413); + throw error; + } + const sdp = new TextDecoder().decode(body.subarray(0, length)); + if (!sdp.trimStart().startsWith("v=0")) + return respond("Invalid SDP.", 400); + signal.throwIfAborted(); + const upstream = await fetch("https://api.openai.com/v1/live/sessions", { + method: "POST", + signal, + headers: { + authorization: `Bearer ${environment.OPENAI_VOICE_API_KEY!.trim()}`, + "content-type": "application/json", + }, + body: JSON.stringify({ + session: { + model: "gpt-live-1", + instructions, + delegation: { type: "client" }, + store: false, + audio: { output: { voice: "marin" } }, + }, + transport: { type: "webrtc", sdp }, + }), + }); + if (!upstream.ok) { + await upstream.body?.cancel(); + return respond( + "Live session creation failed. No automatic retry was made.", + 502, + ); + } + const answer: unknown = await upstream.json(); + if ( + typeof answer !== "object" || + answer === null || + !("session" in answer) || + !("transport" in answer) + ) + throw new Error("Invalid answer"); + const { session, transport } = answer; + if ( + typeof session !== "object" || + session === null || + !("id" in session) || + typeof session.id !== "string" || + typeof transport !== "object" || + transport === null || + !("type" in transport) || + transport.type !== "webrtc" || + !("sdp" in transport) || + typeof transport.sdp !== "string" || + !transport.sdp.trimStart().startsWith("v=0") + ) + throw new Error("Invalid answer"); + return Response.json( + { sessionId: session.id, sdp: transport.sdp }, + { status: 201, headers: { "cache-control": "no-store" } }, + ); + } catch { + return respond( + "Live connection failed. The remote session outcome may be unknown; no automatic retry was made.", + 502, + ); + } + }; diff --git a/apps/petrinaut-website/src/server/voice/openai-voice-config.test.ts b/apps/petrinaut-website/src/server/voice/openai-voice-config.test.ts index 51537c3345c..f60a5b3e317 100644 --- a/apps/petrinaut-website/src/server/voice/openai-voice-config.test.ts +++ b/apps/petrinaut-website/src/server/voice/openai-voice-config.test.ts @@ -19,6 +19,7 @@ describe("OpenAI voice config handler", () => { expect(await response.json()).toEqual({ available: true, connectionTimeoutMs: 15_000, + provider: "realtime", }); }); diff --git a/apps/petrinaut-website/src/server/voice/openai-voice-config.ts b/apps/petrinaut-website/src/server/voice/openai-voice-config.ts index 699002753df..62483b1e118 100644 --- a/apps/petrinaut-website/src/server/voice/openai-voice-config.ts +++ b/apps/petrinaut-website/src/server/voice/openai-voice-config.ts @@ -3,9 +3,15 @@ import { getOpenAIVoiceAvailability } from "./openai-voice-policy.js"; interface VoiceEnvironment { readonly OPENAI_VOICE_API_KEY?: string; readonly PETRINAUT_OPENAI_VOICE_ENABLED?: string; + readonly PETRINAUT_VOICE_PROVIDER?: string; readonly VERCEL_ENV?: string; } +export const getVoiceProvider = (environment: VoiceEnvironment) => { + const provider = environment.PETRINAUT_VOICE_PROVIDER ?? "realtime"; + return provider === "realtime" || provider === "live" ? provider : null; +}; + export const createOpenAIVoiceConfigHandler = (environment: VoiceEnvironment) => async (request: Request): Promise => { @@ -16,7 +22,16 @@ export const createOpenAIVoiceConfigHandler = }); } - return Response.json(getOpenAIVoiceAvailability(environment), { - headers: { "cache-control": "no-store" }, - }); + const availability = getOpenAIVoiceAvailability(environment); + const provider = getVoiceProvider(environment); + return Response.json( + { + ...availability, + provider, + available: availability.available && provider !== null, + }, + { + headers: { "cache-control": "no-store" }, + }, + ); }; diff --git a/apps/petrinaut-website/vite.config.ts b/apps/petrinaut-website/vite.config.ts index 027a6924172..d0a454a19ed 100644 --- a/apps/petrinaut-website/vite.config.ts +++ b/apps/petrinaut-website/vite.config.ts @@ -26,6 +26,7 @@ const apiModules = [ ["/api/oembed", "/api/oembed.ts"], ["/api/voice/config", "/api/voice/config.ts"], ["/api/voice/realtime-call", "/api/voice/realtime-call.ts"], + ["/api/voice/live-session", "/api/voice/live-session.ts"], ] as const; // Plugin required to serve the Vercel fetch handlers in dev. In production, diff --git a/libs/@hashintel/petrinaut/docs/ai-assistant.md b/libs/@hashintel/petrinaut/docs/ai-assistant.md index b828d8223ff..3f79e314dbb 100644 --- a/libs/@hashintel/petrinaut/docs/ai-assistant.md +++ b/libs/@hashintel/petrinaut/docs/ai-assistant.md @@ -64,6 +64,20 @@ understand and selecting **Start voice mode**. Petrinaut remembers that acknowle browser for the current disclosure version, so later uses of **Start voice mode** start directly. If browser storage is unavailable or the disclosure changes, Petrinaut asks again. +Some hosts offer a **GPT-Live · Experimental interview** instead. It uses the same +Voice setup and dock, but is a standalone conversation: it does not add chat messages +or edit the net. Consent and **Start voice** are required for each session. Its dock +normally shows **Listening** or **Speaking**, with collapse/expand and **End voice +mode**. Speaking reflects local audio activity, not a completed turn; the microphone +stays open while it speaks. The input ribbon follows your microphone level where +the browser supports it. The experimental label appears only during consent. +If the browser blocks remote playback, the dock keeps the warning visible and +offers **Play voice audio**; selecting it retries playback from that user gesture. +Ending Voice stops microphone capture and playback; closing the panel ends +the experiment rather than pausing it. A connection error returns to setup without +retrying. The following turn-taking and playback controls apply to integrated +Brunch Voice, not that experiment. + While a session runs, the composer is replaced by a low-profile Voice dock at the foot of the panel: a ribbon that fades out at both ends and one short state -- **Connecting**, **Listening**, **Muted**, **Thinking**, **Speaking**, **Paused**, or **Voice interrupted**. The ribbon is three @@ -78,7 +92,8 @@ stays current while the session runs and tools that edit the net remain visible. voice session** to reduce the panel to the Voice dock alone; this hides the AI header, transcript, and host Voice region without ending the session. Select **Expand voice session** to restore them. Ending Voice while collapsed also closes the AI panel; ending Voice while expanded returns to the text -composer. Only finalized answers and canonical Brunch text become chat history; provisional +composer. When Voice is collapsed, the zoom and fullscreen controls sit above the compact dock +at the right edge; the bottom toolbar stays beside it. Only finalized answers and canonical Brunch text become chat history; provisional transcription and Realtime audio are ephemeral. Finalized spoken user messages carry a small **Voice** chip in front of the words themselves, and the exact inline answer completed by speech carries the same chip, so Voice provenance remains visible without duplicating an answer while the diff --git a/libs/@hashintel/petrinaut/src/react/hooks/use-petrinaut-commands.test.tsx b/libs/@hashintel/petrinaut/src/react/hooks/use-petrinaut-commands.test.tsx index 368f3f93d6c..e16c46824e0 100644 --- a/libs/@hashintel/petrinaut/src/react/hooks/use-petrinaut-commands.test.tsx +++ b/libs/@hashintel/petrinaut/src/react/hooks/use-petrinaut-commands.test.tsx @@ -47,6 +47,7 @@ const editorContextValue = ( setLeftSidebarWidth: () => {}, setPropertiesPanelWidth: () => {}, setAiAssistantWidth: () => {}, + setAiAssistantDockHeight: () => {}, setBottomPanelOpen: () => {}, toggleBottomPanel: () => {}, setBottomPanelHeight: () => {}, diff --git a/libs/@hashintel/petrinaut/src/react/hooks/use-petrinaut-mutations.test.tsx b/libs/@hashintel/petrinaut/src/react/hooks/use-petrinaut-mutations.test.tsx index cf435c6f7f0..21800a6d140 100644 --- a/libs/@hashintel/petrinaut/src/react/hooks/use-petrinaut-mutations.test.tsx +++ b/libs/@hashintel/petrinaut/src/react/hooks/use-petrinaut-mutations.test.tsx @@ -50,6 +50,7 @@ const editorContextValue = ( setLeftSidebarWidth: () => {}, setPropertiesPanelWidth: () => {}, setAiAssistantWidth: () => {}, + setAiAssistantDockHeight: () => {}, setBottomPanelOpen: () => {}, toggleBottomPanel: () => {}, setBottomPanelHeight: () => {}, diff --git a/libs/@hashintel/petrinaut/src/react/state/editor-context.ts b/libs/@hashintel/petrinaut/src/react/state/editor-context.ts index 49825ed1eee..44e661beb46 100644 --- a/libs/@hashintel/petrinaut/src/react/state/editor-context.ts +++ b/libs/@hashintel/petrinaut/src/react/state/editor-context.ts @@ -82,6 +82,8 @@ export type EditorState = { * the surfaces that have to keep clear of it can read it. */ aiAssistantWidth: number; + /** Rendered compact dock height; null when expanded or closed. */ + aiAssistantDockHeight: number | null; activeBottomPanelTab: BottomPanelTab; componentSubnetId: string | null; selection: SelectionMap; @@ -140,6 +142,7 @@ export type EditorActions = { setLeftSidebarWidth: (width: number) => void; setPropertiesPanelWidth: (width: number) => void; setAiAssistantWidth: (width: number) => void; + setAiAssistantDockHeight: (height: number | null) => void; setBottomPanelOpen: (isOpen: boolean) => void; toggleBottomPanel: () => void; setBottomPanelHeight: (height: number) => void; @@ -195,6 +198,7 @@ export const initialEditorState: EditorState = { isBottomPanelOpen: false, bottomPanelHeight: DEFAULT_BOTTOM_PANEL_HEIGHT, aiAssistantWidth: DEFAULT_AI_ASSISTANT_WIDTH, + aiAssistantDockHeight: null, activeBottomPanelTab: "diagnostics", componentSubnetId: null, selection: new Map(), @@ -223,6 +227,7 @@ const DEFAULT_CONTEXT_VALUE: EditorContextValue = { setLeftSidebarWidth: () => {}, setPropertiesPanelWidth: () => {}, setAiAssistantWidth: () => {}, + setAiAssistantDockHeight: () => {}, setBottomPanelOpen: () => {}, toggleBottomPanel: () => {}, setBottomPanelHeight: () => {}, diff --git a/libs/@hashintel/petrinaut/src/react/state/editor-provider.tsx b/libs/@hashintel/petrinaut/src/react/state/editor-provider.tsx index 504d3802d81..c6b90d5b134 100644 --- a/libs/@hashintel/petrinaut/src/react/state/editor-provider.tsx +++ b/libs/@hashintel/petrinaut/src/react/state/editor-provider.tsx @@ -388,6 +388,12 @@ export const EditorProvider: React.FC = ({ children }) => { setState((prev) => ({ ...prev, propertiesPanelWidth: width })), setAiAssistantWidth: (width) => setState((prev) => ({ ...prev, aiAssistantWidth: width })), + setAiAssistantDockHeight: (height) => + setState((prev) => + prev.aiAssistantDockHeight === height + ? prev + : { ...prev, aiAssistantDockHeight: height }, + ), setBottomPanelOpen: (isOpen) => { scheduleAnimationEnd(); setState((prev) => ({ diff --git a/libs/@hashintel/petrinaut/src/react/voice-session/store.ts b/libs/@hashintel/petrinaut/src/react/voice-session/store.ts index bc03867ca28..6bb3ad5230d 100644 --- a/libs/@hashintel/petrinaut/src/react/voice-session/store.ts +++ b/libs/@hashintel/petrinaut/src/react/voice-session/store.ts @@ -8,11 +8,12 @@ export type VoiceSessionActions = { end: () => void; pause: () => void; readFullResponse?: () => void; - reconnect: () => void; + reconnect?: () => void; repeatQuestion?: () => void; - resume: () => void; + retryPlayback?: () => void; + resume?: () => void; setInterruptionBySpeaking?: (enabled: boolean) => void; - setMicrophoneMuted: (muted: boolean) => void; + setMicrophoneMuted?: (muted: boolean) => void; takeTurn?: () => Promise | void; }; diff --git a/libs/@hashintel/petrinaut/src/react/voice-session/types.ts b/libs/@hashintel/petrinaut/src/react/voice-session/types.ts index 78aeacbc2b6..bb885456aef 100644 --- a/libs/@hashintel/petrinaut/src/react/voice-session/types.ts +++ b/libs/@hashintel/petrinaut/src/react/voice-session/types.ts @@ -1,5 +1,6 @@ -/** Which side of a Voice session currently holds the turn. */ +/** Session connectivity or, when known, which side holds the turn. */ export type PetrinautAiVoiceSessionPhase = + | "connected" | "connecting" | "error" | "listening" @@ -20,6 +21,8 @@ export type PetrinautAiVoiceSessionState = { canReadFullResponse?: boolean; /** Whether the final segment of the canonical response is safe to repeat. */ canRepeatQuestion?: boolean; + /** Whether browser-blocked session audio can be retried by the user. */ + canRetryPlayback?: boolean; /** Whether the user can cancel Voice output and start their turn. */ canTakeTurn?: boolean; /** Whether speaking can interrupt assistant audio. */ diff --git a/libs/@hashintel/petrinaut/src/react/voice-session/use-voice-session.ts b/libs/@hashintel/petrinaut/src/react/voice-session/use-voice-session.ts index 4660ec2513e..5de03d283e8 100644 --- a/libs/@hashintel/petrinaut/src/react/voice-session/use-voice-session.ts +++ b/libs/@hashintel/petrinaut/src/react/voice-session/use-voice-session.ts @@ -83,6 +83,16 @@ export const useVoiceSessionCanRepeatQuestion = (): boolean => { ); }; +export const useVoiceSessionCanRetryPlayback = (): boolean => { + const store = use(VoiceSessionContext); + + return useSyncExternalStore( + store.subscribe, + () => store.getSnapshot().state?.canRetryPlayback ?? false, + () => false, + ); +}; + export const useVoiceSessionCanTakeTurn = (): boolean => { const store = use(VoiceSessionContext); diff --git a/libs/@hashintel/petrinaut/src/ui/hooks/use-canvas-insets.test.ts b/libs/@hashintel/petrinaut/src/ui/hooks/use-canvas-insets.test.ts index 73d82896401..25b1c55f975 100644 --- a/libs/@hashintel/petrinaut/src/ui/hooks/use-canvas-insets.test.ts +++ b/libs/@hashintel/petrinaut/src/ui/hooks/use-canvas-insets.test.ts @@ -10,6 +10,7 @@ const closed: PanelLayoutState = { propertiesPanelWidth: 450, isAiAssistantOpen: false, aiAssistantWidth: 500, + aiAssistantDockHeight: null, isBottomPanelOpen: false, bottomPanelHeight: 180, }; @@ -53,4 +54,49 @@ describe("getCanvasInsets", () => { }).bottom, ).toBe(240); }); + + it("lets the viewport column clear a compact dock above, while the toolbar stays beside it", () => { + const compact = { + ...closed, + isAiAssistantOpen: true, + aiAssistantDockHeight: 84, + }; + expect(getCanvasInsets(compact)).toEqual({ + left: 0, + right: 500, + bottom: 0, + }); + expect(getCanvasInsets(compact, { aboveCollapsedDock: true })).toEqual({ + left: 0, + right: 0, + bottom: 84, + }); + expect( + getCanvasInsets( + { ...compact, hasSelection: true, isBottomPanelOpen: true }, + { aboveCollapsedDock: true }, + ), + ).toEqual({ left: 0, right: 450, bottom: 180 }); + expect( + getCanvasInsets( + { ...compact, aiAssistantDockHeight: 320, isBottomPanelOpen: true }, + { aboveCollapsedDock: true }, + ), + ).toEqual({ left: 0, right: 0, bottom: 320 }); + }); + + it("keeps expanded positioning and ignores a closing dock's last measurement", () => { + expect( + getCanvasInsets( + { ...closed, isAiAssistantOpen: true }, + { aboveCollapsedDock: true }, + ), + ).toEqual({ left: 0, right: 500, bottom: 0 }); + expect( + getCanvasInsets( + { ...closed, aiAssistantDockHeight: 84 }, + { aboveCollapsedDock: true }, + ), + ).toEqual({ left: 0, right: 0, bottom: 0 }); + }); }); diff --git a/libs/@hashintel/petrinaut/src/ui/hooks/use-canvas-insets.ts b/libs/@hashintel/petrinaut/src/ui/hooks/use-canvas-insets.ts index 11f374daa17..fa502949dba 100644 --- a/libs/@hashintel/petrinaut/src/ui/hooks/use-canvas-insets.ts +++ b/libs/@hashintel/petrinaut/src/ui/hooks/use-canvas-insets.ts @@ -20,26 +20,46 @@ export interface PanelLayoutState { readonly propertiesPanelWidth: number; readonly isAiAssistantOpen: boolean; readonly aiAssistantWidth: number; + readonly aiAssistantDockHeight: number | null; readonly isBottomPanelOpen: boolean; readonly bottomPanelHeight: number; } +interface CanvasInsetOptions { + /** The viewport column clears the compact dock above, not beside it. */ + readonly aboveCollapsedDock?: boolean; +} + /** * Each edge's rule is the one the panel on it renders by: search opens the * left sidebar without the toggle, a selection opens the properties panel, and * the assistant docks beside the properties panel rather than over it, so an * open pair covers the sum of the two. */ -export const getCanvasInsets = (state: PanelLayoutState): CanvasInsets => ({ - left: - state.isLeftSidebarOpen || state.isSearchOpen - ? state.leftSidebarWidth + PANEL_MARGIN - : 0, - right: - (state.hasSelection ? state.propertiesPanelWidth + PANEL_MARGIN : 0) + - (state.isAiAssistantOpen ? state.aiAssistantWidth : 0), - bottom: state.isBottomPanelOpen ? state.bottomPanelHeight + PANEL_MARGIN : 0, -}); +export const getCanvasInsets = ( + state: PanelLayoutState, + { aboveCollapsedDock = false }: CanvasInsetOptions = {}, +): CanvasInsets => { + const dockHeight = + aboveCollapsedDock && state.isAiAssistantOpen + ? state.aiAssistantDockHeight + : null; + return { + left: + state.isLeftSidebarOpen || state.isSearchOpen + ? state.leftSidebarWidth + PANEL_MARGIN + : 0, + right: + (state.hasSelection ? state.propertiesPanelWidth + PANEL_MARGIN : 0) + + (state.isAiAssistantOpen && dockHeight === null + ? state.aiAssistantWidth + : 0), + bottom: Math.max( + state.isBottomPanelOpen ? state.bottomPanelHeight + PANEL_MARGIN : 0, + dockHeight ?? 0, + ), + }; +}; /** * What the docked panels take out of the canvas, for the controls that float @@ -50,11 +70,13 @@ export const getCanvasInsets = (state: PanelLayoutState): CanvasInsets => ({ */ const NO_INSETS: CanvasInsets = { left: 0, right: 0, bottom: 0 }; -export const useCanvasInsets = (): CanvasInsets => { +export const useCanvasInsets = (options?: CanvasInsetOptions): CanvasInsets => { const presentation = usePetrinautPresentation(); const editor = use(EditorContext); // Where the panels sit beside the canvas rather than over it, the canvas is // already the space it occupies and there is nothing to keep clear of. - return presentation.panelsOverlayCanvas ? getCanvasInsets(editor) : NO_INSETS; + return presentation.panelsOverlayCanvas + ? getCanvasInsets(editor, options) + : NO_INSETS; }; diff --git a/libs/@hashintel/petrinaut/src/ui/index.ts b/libs/@hashintel/petrinaut/src/ui/index.ts index 0f36e19ce7c..e6975730ee9 100644 --- a/libs/@hashintel/petrinaut/src/ui/index.ts +++ b/libs/@hashintel/petrinaut/src/ui/index.ts @@ -32,6 +32,7 @@ export type { PetrinautAiVoiceMode, PetrinautAiVoiceModeContext, PetrinautAiVoiceModeControls, + PetrinautAiVoiceModeSessionControls, PetrinautAiVoiceSessionPhase, PetrinautAiVoiceSessionState, } from "./types/ai-assistant-composer-control"; diff --git a/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.test.ts b/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.test.ts new file mode 100644 index 00000000000..37ec0af7bbc --- /dev/null +++ b/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.test.ts @@ -0,0 +1,45 @@ +import { expectTypeOf, test } from "vitest"; + +import type { + PetrinautAiVoiceModeContext, + PetrinautAiVoiceModeControls, + PetrinautAiVoiceModeSessionControls, +} from "./ai-assistant-composer-control"; + +test("keeps legacy Voice controls required while sessions advertise capabilities", () => { + expectTypeOf().toEqualTypeOf< + () => void + >(); + expectTypeOf().toEqualTypeOf< + () => void + >(); + expectTypeOf< + PetrinautAiVoiceModeControls["setMicrophoneMuted"] + >().toEqualTypeOf<(muted: boolean) => void>(); + + expectTypeOf< + PetrinautAiVoiceModeSessionControls["reconnect"] + >().toEqualTypeOf<(() => void) | undefined>(); + expectTypeOf().toEqualTypeOf< + (() => void) | undefined + >(); + expectTypeOf< + PetrinautAiVoiceModeSessionControls["setMicrophoneMuted"] + >().toEqualTypeOf<((muted: boolean) => void) | undefined>(); +}); + +test("accepts an existing context implementation with complete control registration", () => { + type ExistingVoiceModeContext = Omit< + PetrinautAiVoiceModeContext, + "registerVoiceModeControls" + > & { + registerVoiceModeControls: ( + controls: PetrinautAiVoiceModeControls, + ) => () => void; + }; + + const existingContext = {} as ExistingVoiceModeContext; + const currentContext: PetrinautAiVoiceModeContext = existingContext; + + expectTypeOf(currentContext).toEqualTypeOf(); +}); diff --git a/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts b/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts index 54c70070cd1..1716a4ca3e2 100644 --- a/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts +++ b/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts @@ -54,7 +54,7 @@ export type PetrinautAiComposerControl = ( context: PetrinautAiComposerControlContext, ) => ReactNode; -/** Imperative lifecycle controls registered by a host-owned Voice mode. */ +/** Complete lifecycle controls retained for existing host-owned Voice modes. */ export type PetrinautAiVoiceModeControls = { /** * Invalidates the active Voice generation synchronously, then finishes @@ -83,6 +83,21 @@ export type PetrinautAiVoiceModeControls = { takeTurn?: () => Promise | void; }; +/** Lifecycle controls registered by a session with provider-specific capabilities. */ +export type PetrinautAiVoiceModeSessionControls = Omit< + PetrinautAiVoiceModeControls, + "reconnect" | "resume" | "setMicrophoneMuted" +> & + Partial< + Pick< + PetrinautAiVoiceModeControls, + "reconnect" | "resume" | "setMicrophoneMuted" + > + > & { + /** Retries browser-blocked session audio from a user gesture. */ + retryPlayback?: () => void; + }; + /** Stable controls and conversation state supplied to a host-owned Voice mode. */ export type PetrinautAiVoiceModeContext = PetrinautAiComposerControlContext & { /** True when Petrinaut can retain one next voice turn while chat settles. */ @@ -96,6 +111,13 @@ export type PetrinautAiVoiceModeContext = PetrinautAiComposerControlContext & { registerVoiceModeControls: ( controls: PetrinautAiVoiceModeControls, ) => () => void; + /** + * Registers a Voice session whose provider supports only a subset of the + * complete lifecycle controls. + */ + registerVoiceModeSessionControls?: ( + controls: PetrinautAiVoiceModeSessionControls, + ) => () => void; /** * Publishes the live session state Petrinaut renders from. Pass `null` once * no session is running so Petrinaut can retire its Voice surfaces. diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/components/voice-session-indicator.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/components/voice-session-indicator.tsx index 1994e3e7784..3856b4c2b36 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/components/voice-session-indicator.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/components/voice-session-indicator.tsx @@ -63,6 +63,8 @@ const phaseLevel: Record< PetrinautAiVoiceSessionPhase, (seconds: number, microphoneLevel: number) => number > = { + // Connectivity alone is not evidence that either side is speaking. + connected: () => 0, connecting: (seconds) => 0.1 + Math.sin(seconds * 2.2) * 0.04, error: () => 0, listening: (_seconds, microphoneLevel) => @@ -90,6 +92,7 @@ const ribbonStyle = cva({ }, variants: { phase: { + connected: { color: "blue.s90" }, connecting: { color: "neutral.s80" }, error: { color: "neutral.s80" }, listening: { color: "blue.s90" }, diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/components/voice-session-labels.ts b/libs/@hashintel/petrinaut/src/ui/views/Editor/components/voice-session-labels.ts index 7dd389c9e4e..b5b170792ca 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/components/voice-session-labels.ts +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/components/voice-session-labels.ts @@ -8,6 +8,8 @@ export const voiceSessionStatusLabel = ( phase: PetrinautAiVoiceSessionPhase, ): string => { switch (phase) { + case "connected": + return "Connected"; case "connecting": return "Connecting"; case "error": @@ -36,6 +38,7 @@ export const voiceSessionActionLabels = { readFullResponse: "Read full response", reconnect: "Reconnect voice mode", repeatQuestion: "Repeat question", + retryPlayback: "Play voice audio", resume: "Resume voice mode", takeTurn: "Your turn", unmute: "Unmute microphone", diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiments-story-fixtures.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiments-story-fixtures.tsx index ee853f374d8..69d2e7d386f 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiments-story-fixtures.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiments-story-fixtures.tsx @@ -847,6 +847,7 @@ export function FakeEditorProvider({ setLeftSidebarWidth: () => {}, setPropertiesPanelWidth: () => {}, setAiAssistantWidth: () => {}, + setAiAssistantDockHeight: () => {}, setBottomPanelOpen: () => {}, toggleBottomPanel: () => {}, setBottomPanelHeight: () => {}, diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx index 81cd533b4bf..66168b6b3ef 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx @@ -115,6 +115,7 @@ const editorContextValue: EditorContextValue = { setLeftSidebarWidth: () => {}, setPropertiesPanelWidth: () => {}, setAiAssistantWidth: () => {}, + setAiAssistantDockHeight: () => {}, setBottomPanelOpen: () => {}, toggleBottomPanel: () => {}, setBottomPanelHeight: () => {}, @@ -2462,19 +2463,16 @@ describe("AiAssistantPanel composer submissions", () => { }: { context: PetrinautAiVoiceModeContext; }) => { - const { registerVoiceModeControls, reportVoiceSessionState } = context; + const { registerVoiceModeSessionControls, reportVoiceSessionState } = + context; - useEffect( - () => - registerVoiceModeControls({ - end: async () => undefined, - pause: vi.fn(), - reconnect: vi.fn(), - resume: vi.fn(), - setMicrophoneMuted: vi.fn(), - }), - [registerVoiceModeControls], - ); + useEffect(() => { + if (!registerVoiceModeSessionControls) return; + return registerVoiceModeSessionControls({ + end: async () => undefined, + pause: vi.fn(), + }); + }, [registerVoiceModeSessionControls]); useEffect(() => { reportVoiceSessionState({ canReadFullResponse: true, @@ -2501,19 +2499,13 @@ describe("AiAssistantPanel composer submissions", () => { const rendered = renderTestPanel({ aiAssistant: aiAssistant(true) }); expect(screen.queryByRole("button", { name: "Your turn" })).toBeNull(); - fireEvent.click( - await screen.findByRole("button", { name: "Voice playback options" }), - ); + await screen.findByRole("region", { name: "Voice session" }); expect( - ( - await screen.findByRole("menuitem", { name: "Repeat question" }) - ).getAttribute("aria-disabled"), - ).toBe("true"); + screen.queryByRole("button", { name: "Voice playback options" }), + ).toBeNull(); expect( - screen - .getByRole("menuitem", { name: "Read full response" }) - .getAttribute("aria-disabled"), - ).toBe("true"); + screen.queryByRole("button", { name: "Mute microphone" }), + ).toBeNull(); rendered.rerenderPanel(aiAssistant(false), editorContextValue); diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx index 3f86c40d2f1..7298250a4b7 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx @@ -74,6 +74,7 @@ import type { PetrinautAiInputMode, PetrinautAiVoiceModeContext, PetrinautAiVoiceModeControls, + PetrinautAiVoiceModeSessionControls, PetrinautAiVoiceSessionState, } from "../../../types/ai-assistant-composer-control"; import type { PetrinautAiMessage } from "./ai-assistant-panel/types"; @@ -571,9 +572,8 @@ const ConversationAiAssistantPanel = ({ setVoiceActiveState(active); }, []); const voiceHandoffPendingRef = useRef(false); - const voiceModeControlsRef = useRef( - null, - ); + const voiceModeControlsRef = + useRef(null); const queuedVoiceInputRef = useRef(null); const consumedInitialInteractionModeRef = useRef( null, @@ -666,8 +666,8 @@ const ConversationAiAssistantPanel = ({ [voiceSessionStore], ); - const registerVoiceModeControls = useCallback( - (controls: PetrinautAiVoiceModeControls) => { + const registerVoiceModeSessionControls = useCallback( + (controls: PetrinautAiVoiceModeSessionControls) => { voiceModeControlsRef.current = controls; voiceSessionStore.setActions({ // Ending returns the composer to text, which is also the path that @@ -677,18 +677,28 @@ const ConversationAiAssistantPanel = ({ ...(controls.readFullResponse ? { readFullResponse: () => controls.readFullResponse?.() } : {}), - reconnect: () => controls.reconnect(), + ...(controls.reconnect + ? { reconnect: () => controls.reconnect?.() } + : {}), ...(controls.repeatQuestion ? { repeatQuestion: () => controls.repeatQuestion?.() } : {}), - resume: () => controls.resume(), + ...(controls.retryPlayback + ? { retryPlayback: () => controls.retryPlayback?.() } + : {}), + ...(controls.resume ? { resume: () => controls.resume?.() } : {}), ...(controls.setInterruptionBySpeaking ? { setInterruptionBySpeaking: (enabled: boolean) => controls.setInterruptionBySpeaking?.(enabled), } : {}), - setMicrophoneMuted: (muted) => controls.setMicrophoneMuted(muted), + ...(controls.setMicrophoneMuted + ? { + setMicrophoneMuted: (muted: boolean) => + controls.setMicrophoneMuted?.(muted), + } + : {}), ...(controls.takeTurn ? { takeTurn: () => controls.takeTurn?.() } : {}), }); @@ -702,6 +712,11 @@ const ConversationAiAssistantPanel = ({ }, [requestInputMode, voiceSessionStore], ); + const registerVoiceModeControls = useCallback( + (controls: PetrinautAiVoiceModeControls) => + registerVoiceModeSessionControls(controls), + [registerVoiceModeSessionControls], + ); const stopRequestedRef = useRef(false); // Advances on composer submissions and conversation changes so late work @@ -1974,6 +1989,7 @@ const ConversationAiAssistantPanel = ({ inputMode: interactionMode, isAiAssistantOpen, registerVoiceModeControls, + registerVoiceModeSessionControls, reportVoiceSessionState, setInputMode: requestInputMode, setVoiceActive, diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx index effd27f37c1..25ad55d7026 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx @@ -24,6 +24,7 @@ import { VoiceSessionContext } from "../../../../../react/voice-session/context" import { createVoiceSessionStore } from "../../../../../react/voice-session/store"; import { definePetrinautAiInteractiveTool } from "../../../../types/ai-interactive-tool"; import { AiAssistantContents } from "./ai-assistant-contents"; +import { VoiceDock } from "./ai-assistant-contents/voice-dock"; import type { PetrinautAiMessage } from "./types"; @@ -80,6 +81,62 @@ const HostContent = ({ onMount }: { onMount: () => void }) => { return

Saved account

; }; +test("session-only dock shows Connected and End without unsupported controls", () => { + const end = vi.fn(); + const collapse = vi.fn(); + render( + } + microphoneMuted={false} + notice={null} + onCollapsedToggle={collapse} + phase="connected" + />, + ); + expect(screen.getByText("Connected")).toBeTruthy(); + expect( + screen.queryByRole("button", { name: "Voice playback options" }), + ).toBeNull(); + expect(screen.queryByRole("button", { name: "Mute microphone" })).toBeNull(); + expect(screen.queryByRole("button", { name: "Your turn" })).toBeNull(); + fireEvent.click( + screen.getByRole("button", { name: "Collapse voice session" }), + ); + expect(collapse).toHaveBeenCalledOnce(); + fireEvent.click(screen.getByRole("button", { name: "End voice mode" })); + expect(end).toHaveBeenCalledOnce(); +}); + +test("offers a user-gesture retry while session audio is blocked", () => { + const retryPlayback = vi.fn(); + const commonProps = { + actions: { end: noop, pause: noop, retryPlayback }, + canReadFullResponse: false, + canRepeatQuestion: false, + canTakeTurn: false, + collapsed: false, + indicator: , + microphoneMuted: false, + notice: "Audio playback is blocked. Select Play voice audio to hear Live.", + onCollapsedToggle: noop, + phase: "connected" as const, + }; + const rendered = render( + , + ); + + fireEvent.click(screen.getByRole("button", { name: "Play voice audio" })); + expect(retryPlayback).toHaveBeenCalledOnce(); + + rendered.rerender(); + expect(screen.queryByRole("button", { name: "Play voice audio" })).toBeNull(); +}); + describe("AiAssistantContents", () => { test("switches to host content without unmounting chat or losing its draft and Stop control", () => { const onStop = vi.fn(); diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.tsx index 167e835b509..fa06911a913 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.tsx @@ -4,6 +4,7 @@ import { type RefObject, use, useEffect, + useEffectEvent, useId, useLayoutEffect, useRef, @@ -608,8 +609,29 @@ export const AiAssistantContents = ({ const { aiAssistantWidth: assistantWidth, setAiAssistantWidth: setAssistantWidth, + setAiAssistantDockHeight, } = use(EditorContext); + const shellRef = useRef(null); + const reportDockHeight = useEffectEvent((height: number | null) => { + setAiAssistantDockHeight(height); + }); + useLayoutEffect(() => { + const shell = shellRef.current; + if (!isOpen || !isVoiceDockCollapsed || !shell) { + return; + } + const measure = () => + reportDockHeight(shell.getBoundingClientRect().height); + measure(); + const observer = new ResizeObserver(measure); + observer.observe(shell); + return () => { + observer.disconnect(); + reportDockHeight(null); + }; + }, [isOpen, isVoiceDockCollapsed]); + const [chipsDismissed, setChipsDismissed] = useState(false); const notifiedErrorRef = useRef(undefined); @@ -738,6 +760,7 @@ export const AiAssistantContents = ({ return (