From 81571ed80ee013fec69d2b252e7de4a48bec83c0 Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Fri, 11 Sep 2026 14:10:55 +0200 Subject: [PATCH 01/13] Add isolated GPT-Live interview experiment to Voice controls Co-authored-by: Amp --- .changeset/tidy-voice-dock.md | 5 + apps/petrinaut-website/MISSION.md | 92 +++++++ apps/petrinaut-website/README.md | 105 ++++++- .../api/voice/live-session.ts | 10 + .../live-conversation-control.test.tsx | 183 +++++++++++++ .../live-conversation-control.tsx | 126 +++++++++ .../voice-interview/live-conversation.test.ts | 221 +++++++++++++++ .../app/voice-interview/live-conversation.ts | 230 ++++++++++++++++ .../voice-interview-control.tsx | 259 ++++-------------- .../voice-interview-disclosure.tsx | 209 ++++++++++++++ .../server/voice/openai-live-session.test.ts | 166 +++++++++++ .../src/server/voice/openai-live-session.ts | 113 ++++++++ .../server/voice/openai-voice-config.test.ts | 1 + .../src/server/voice/openai-voice-config.ts | 21 +- apps/petrinaut-website/vite.config.ts | 1 + .../@hashintel/petrinaut/docs/ai-assistant.md | 9 + .../src/react/voice-session/store.ts | 6 +- .../src/react/voice-session/types.ts | 3 +- .../ui/types/ai-assistant-composer-control.ts | 6 +- .../components/voice-session-indicator.tsx | 3 + .../Editor/components/voice-session-labels.ts | 2 + .../Editor/panels/ai-assistant-panel.test.tsx | 19 +- .../Editor/panels/ai-assistant-panel.tsx | 13 +- .../ai-assistant-contents.test.tsx | 32 +++ .../ai-assistant-contents/voice-dock.tsx | 92 ++++--- 25 files changed, 1638 insertions(+), 289 deletions(-) create mode 100644 .changeset/tidy-voice-dock.md create mode 100644 apps/petrinaut-website/MISSION.md create mode 100644 apps/petrinaut-website/api/voice/live-session.ts create mode 100644 apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.test.tsx create mode 100644 apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.tsx create mode 100644 apps/petrinaut-website/src/main/app/voice-interview/live-conversation.test.ts create mode 100644 apps/petrinaut-website/src/main/app/voice-interview/live-conversation.ts create mode 100644 apps/petrinaut-website/src/main/app/voice-interview/voice-interview-disclosure.tsx create mode 100644 apps/petrinaut-website/src/server/voice/openai-live-session.test.ts create mode 100644 apps/petrinaut-website/src/server/voice/openai-live-session.ts diff --git a/.changeset/tidy-voice-dock.md b/.changeset/tidy-voice-dock.md new file mode 100644 index 00000000000..2932c104106 --- /dev/null +++ b/.changeset/tidy-voice-dock.md @@ -0,0 +1,5 @@ +--- +"@hashintel/petrinaut": patch +--- + +Allow host-owned Voice sessions to report a neutral connected state and omit unsupported microphone, resume, reconnect, and playback controls from the dock. diff --git a/apps/petrinaut-website/MISSION.md b/apps/petrinaut-website/MISSION.md new file mode 100644 index 00000000000..e6101ed685e --- /dev/null +++ b/apps/petrinaut-website/MISSION.md @@ -0,0 +1,92 @@ +# Experiment Live Full-Duplex Migration + +## Status + +Live experiment scope; local implementation prepared for Kostandin's manual +witness, not conversational-quality acceptance. +[FE-1663](https://linear.app/hash/issue/FE-1663/experiment-live-full-duplex-migration) +is related to FE-1661, which remains the migration-effort assessment. +This website experiment uses Petrinaut's existing Voice dock, with optional +session controls and a connectivity-only state. It does not recut Brunch's existing Mission 7a or +promote its future-planning drafts. + +## Imperative + +Learn whether GPT-Live-1 natively makes process interviewing feel fluid: +concise relevant follow-ups, room for hesitation and elaboration, and natural +interruption/correction without losing meaning. Smooth audio alone is not success. + +## Throughline + +Existing website Voice entry → explicit experimental consent/Start → trusted +website `/api/voice/live-session` → client-delegated GPT-Live-1 WebRTC → microphone +and speaker → existing Voice dock → local End/Exit. Local disconnection is +not a remote-closure claim. +`main → FE-1663 → Experiment Live Full Brunch Integration` is the intended stack; +only FE-1663 is authorized now. No relay, rephrasing, or harness code is a donor. + +## Proof + +- `openai-live-session.test.ts` checks provider/default selection, enablement, + trusted session payload, origin/content validation, credential privacy, and + no retry. Existing Realtime policy and transport tests remain regression oracles. +- `live-conversation.test.ts` checks waiting for `session.started`, no duplicate + start, no transcript/delegation execution, remote audio attachment, connection + failures, late microphone permission, and capture/playback/transport cleanup. +- `live-conversation-control.test.tsx` checks explicit consent/Start, provider + pinning, canonical isolation, host state reporting, panel closure and stale callbacks. +- Petrinaut's assistant-panel and contents tests check optional action visibility + and retain regression coverage for Realtime's playback and lifecycle controls. +- Rendered real-editor consent, simulated error, and mocked active controls + must be inspected without a real microphone or provider session. +- Kostandin's [10–15 minute manual procedure](README.md#manual-test--1015-minutes) + is the oracle for conversational quality and actual media behavior. Provider-free + tests and screenshots do not establish these or end-to-end Brunch compatibility. + +## Constraints + +Unset `PETRINAUT_VOICE_PROVIDER` means `realtime`; only `realtime` and `live` are +valid. Keep existing enablement/credential plumbing. Pin configuration per +mounted conversation; never switch providers or resubmit input automatically. +Keep Realtime's prompts, admission, queue, settlement and playback unchanged. + +Live has no access to canonical submission, Brunch/Petrinaut tools or chat +history. Its guidance does not grant domain authority. Transcript deltas are +not finalized utterances; delegation events are metadata; commentary (unused +here) is paraphrasable context limited to 500 tokens, and append acknowledgements +are not speech/playback completion. Do not invent response terminals or infer +authoritative completion from silence. Stop ends local media, not canonical work. + +No paid sessions, synthetic recordings, audio evaluation harness, deployment, +Notion writes, second issue, or integration implementation are authorized. +The follow-up permits cleanup, a clean commit without Amp thread IDs, and a draft PR. +Preserve other worktrees and uncommitted work. + +## Fog-line + +Native finalization and enforceable output gating remain unresolved for PR 2. +Observe whether interruption preserves corrections, whether the user feels +heard, whether questions advance process elicitation, whether pauses/one-word +answers survive, and whether output is concise. Observe transport failures and +remote closure separately. A pleasant standalone conversation cannot answer +whether canonical admission/settlement delays retain this quality. + +## Stop or reorient + +Stop at an authority change: autonomous domain speech, local substantive +follow-ups in the integrated workflow, hybrid authoritative transcription, or +buffered/gated output requires an explicit recut for Lu's approval. Neither +this prompt nor success here reopens FE-1624. Provider-free verification cannot +be replaced with paid calls or a synthetic audio campaign. + +## Deferred + +PR 2 must preserve: finalized input → existing composer → Flue → Brunch → +authorized Petrinaut execution → complete-turn settlement → frozen canonical +reply → Live delivery. Brunch alone authors substantive domain answers and +questions; Flue is canonical conversation; Petrinaut is execution authority. +Proposal approval, full-turn settlement and playback eligibility remain distinct. +Unknown execution outcomes must not be reapplied or spoken as success. Workpiece, +basis and tool payloads are not assistant prose. See the existing Brunch +[future spine](../../libs/@hashintel/brunch-agent/MISSION.next.md#voice-after-the-live-transport-cut) +for retained context, not implementation authority. diff --git a/apps/petrinaut-website/README.md b/apps/petrinaut-website/README.md index ed360f7a67d..d699d9ebbd1 100644 --- a/apps/petrinaut-website/README.md +++ b/apps/petrinaut-website/README.md @@ -134,20 +134,107 @@ from jsDelivr and Optuna from PyPI; later runs use the browser cache. ## Environment variables -| Name | Required | Used by | Notes | -| -------------------------------- | ---------------- | ---------------- | ---------------------------------------------------------- | -| `OPENAI_API_KEY` | for chat to work | `api/chat.ts` | OpenAI key the function uses to call `streamText`. | -| `OPENAI_VOICE_API_KEY` | for voice | voice API | Dedicated OpenAI key used to create Realtime WebRTC calls. | -| `PETRINAUT_OPENAI_VOICE_ENABLED` | no | voice API | Set to `true` to enable voice, including in production. | -| `PETRINAUT_AI_MODEL` | no | `api/chat.ts` | Overrides the default OpenAI model id. | -| `VITE_BRUNCH_CHAT_ENDPOINT` | for Brunch | website | Base URL of the mounted Brunch Flue route. | -| `SENTRY_DSN` | no | `vite.config.ts` | Wired into the bundle via `__SENTRY_DSN__` at build time. | +| Name | Required | Used by | Notes | +| -------------------------------- | ---------------- | ---------------- | --------------------------------------------------------------------------------------------- | +| `OPENAI_API_KEY` | for chat to work | `api/chat.ts` | OpenAI key the function uses to call `streamText`. | +| `OPENAI_VOICE_API_KEY` | for voice | voice API | Dedicated OpenAI key used to create Voice WebRTC sessions. | +| `PETRINAUT_OPENAI_VOICE_ENABLED` | no | voice API | Set to `true` to enable voice, including in production. | +| `PETRINAUT_VOICE_PROVIDER` | no | voice API | `realtime` (default) or `live` (detached experiment). Invalid values disable Voice discovery. | +| `PETRINAUT_AI_MODEL` | no | `api/chat.ts` | Overrides the default OpenAI model id. | +| `VITE_BRUNCH_CHAT_ENDPOINT` | for Brunch | website | Base URL of the mounted Brunch Flue route. | +| `SENTRY_DSN` | no | `vite.config.ts` | Wired into the bundle via `__SENTRY_DSN__` at build time. | Local values live in `.env.local`; Vite's `loadEnv` (see [`vite.config.ts`](vite.config.ts)) copies them into `process.env` for both the dev server and the API functions. In production, set these in the Vercel project settings. +### Experimental Live interview (FE-1663) + +`PETRINAUT_VOICE_PROVIDER=live` selects a standalone GPT-Live-1 conversation +inside the existing Voice entry. **It is not Brunch output.** It cannot see or +submit chat, execute tools, or change the model/workpiece. No experimental +transcripts are displayed or saved. The server uses client delegation, not +managed Responses; transcript deltas and delegation metadata are ignored. +The short process-interview prompt is conversational guidance, not a domain +system or a guarantee of model compliance. + +From the repository root, with `OPENAI_VOICE_API_KEY` already exported (or in +this worktree's `apps/petrinaut-website/.env.local`): + +```sh +# Initial local preparation, without inference: +turbo run build --filter '@apps/brunch-agent^...' --filter '@apps/petrinaut-website^...' +yarn workspace @apps/petrinaut-website codegen +yarn workspace @apps/petrinaut-website examples:generate + +# Standalone Live needs only the existing panel launcher, not a running Brunch server. +PETRINAUT_OPENAI_VOICE_ENABLED=true PETRINAUT_VOICE_PROVIDER=live yarn dev:brunch:panel +``` + +Open , dismiss the tour if shown, open the AI panel, +and select the waveform **Start voice mode** action in the empty composer. +Read the experimental label, check consent, then choose **Start voice**. +Only that last action requests microphone access and a billable Live session. +Use headphones for the first trial. HTTPS or localhost and an OpenAI project +with GPT-Live-1 access are required. + +Once connected, the existing Voice dock replaces the composer with +**GPT-Live · Experimental · Connected**, collapse/expand and **End voice mode**. +There is no separate experiment panel, replay menu, microphone toggle, or +inferred listening/speaking turn state. Connection errors return to setup; +starting again requires fresh consent. + +**End voice mode**, **Exit experiment** during setup, closing the panel, switching to text, +changing conversation, and leaving the page stop local experimental capture +and playback. Stop requests `session.close` only after `session.started` and +waits up to two seconds for `session.closed` before releasing the transport. +Local silence is not proof of remote closure or final usage. Connection failures are not retried. +Starting again creates a new session with fresh consent; there is no resume, +replay, "Your turn", or fabricated Realtime terminal lifecycle. + +To return to the **unchanged integrated Realtime path**, Exit, stop the panel +dev command with Ctrl-C, configure the existing local Brunch environment, and run: + +```sh +PETRINAUT_OPENAI_VOICE_ENABLED=true PETRINAUT_VOICE_PROVIDER=realtime yarn dev:brunch +``` + +Reload the page before starting a new session. Unsetting +`PETRINAUT_VOICE_PROVIDER` also selects Realtime. Provider/config selection is +pinned for the mounted conversation; there is no provider switching or input +resubmission mid-session. The launcher sets the existing `/agents/chat` route; +the ordinary website launcher still needs `VITE_BRUNCH_CHAT_ENDPOINT` configured +to expose Voice. Export variables to the launcher directly or use `.env.local`; +the website's generic Turbo `dev` task does not forward arbitrary shell variables. + +#### Manual test — 10–15 minutes + +1. **3 minutes:** explain a familiar process. Let Live ask relevant follow-ups. +2. **2 minutes:** hesitate, pause, answer with one word, then elaborate. Note + whether it leaves room and follows the meaning rather than guessing a turn end. +3. **2 minutes:** interrupt mid-sentence and correct an earlier detail. Note + whether the correction is retained and the interview advances. +4. **2 minutes:** speak while Live responds. Listen for lost words, overlap, + unwanted acknowledgements, and long monologues. A button click is not an + acoustic-interruption test. +5. **2–4 minutes:** End voice mode and check the browser microphone indicator and actual + speaker silence. Change to Realtime using the commands above, reload, + and begin a fresh session. Confirm the experiment added no canonical chat + messages or model changes. + +Record **feeling heard and advancing the interview** separately from transport +correctness. Note browser/headset, representative pauses/corrections, and which +responses were excessive or useful. Prior relay/rephrasing/harness evidence is +not a passing comparative baseline. This experiment has no audio harness or +synthetic recordings. + +The intended successor is **Experiment Live Full Brunch Integration**, stacked +on this branch; it is not implemented here. See [MISSION.md](MISSION.md) for +the authority boundary and unresolved finalization/output-control questions. +The existing unauthenticated Voice endpoint risk below also applies to Live; +do not expose this local experiment publicly without addressing that boundary. + ### Brunch Voice mode -Voice mode is disabled by default. To enable it, configure a real +The following describes Realtime, the default provider. Voice mode is disabled by default. To enable it, configure a real `VITE_BRUNCH_CHAT_ENDPOINT`, set `PETRINAUT_OPENAI_VOICE_ENABLED=true`, and provide a dedicated `OPENAI_VOICE_API_KEY`. diff --git a/apps/petrinaut-website/api/voice/live-session.ts b/apps/petrinaut-website/api/voice/live-session.ts new file mode 100644 index 00000000000..a11884f34cc --- /dev/null +++ b/apps/petrinaut-website/api/voice/live-session.ts @@ -0,0 +1,10 @@ +import { createOpenAILiveSessionHandler } from "../../src/server/voice/openai-live-session.js"; + +declare const process: { env: Record }; + +export default { + fetch: createOpenAILiveSessionHandler({ + environment: process.env, + fetch: globalThis.fetch.bind(globalThis), + }), +}; diff --git a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.test.tsx b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.test.tsx new file mode 100644 index 00000000000..a5d06b8f37e --- /dev/null +++ b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.test.tsx @@ -0,0 +1,183 @@ +// @vitest-environment jsdom +import { + act, + cleanup, + fireEvent, + render, + screen, + waitFor, +} from "@testing-library/react"; +import { afterEach, expect, test, vi } from "vitest"; + +import { createLiveConversation } from "./live-conversation"; +import { + loadOpenAIVoiceConfig, + VoiceInterviewControl, +} from "./voice-interview-control"; + +import type { PetrinautAiVoiceModeContext } from "@hashintel/petrinaut/ui"; + +vi.mock("./live-conversation", () => ({ + createLiveConversation: vi.fn(() => ({ + start: vi.fn(async () => {}), + stop: vi.fn(async () => {}), + })), +})); +afterEach(() => { + cleanup(); + vi.clearAllMocks(); +}); + +const context = (): PetrinautAiVoiceModeContext => ({ + conversationId: "standalone", + messages: [], + status: "ready", + canAcceptVoiceInput: true, + inputMode: "voice", + isAiAssistantOpen: true, + stop: vi.fn(async () => {}), + submitText: vi.fn(), + submitVoiceInput: vi.fn(), + registerVoiceModeControls: vi.fn(() => () => {}), + reportVoiceSessionState: vi.fn(), + setInputMode: vi.fn(), + setVoiceActive: vi.fn(), +}); +const config = { + available: true as const, + provider: "live" as const, + connectionTimeoutMs: 15_000, +}; +const start = async () => { + fireEvent.click(screen.getByRole("checkbox")); + await waitFor(() => + expect( + screen + .getByRole("button", { name: "Start voice" }) + .hasAttribute("disabled"), + ).toBe(false), + ); + fireEvent.click(screen.getByRole("button", { name: "Start voice" })); +}; + +test("reuses setup, reports connectivity to the host dock, and clears it on failure", async () => { + const props = context(); + render(); + expect( + screen.getByRole("region", { name: "Voice mode consent" }), + ).toBeTruthy(); + expect(screen.getByText("GPT-Live · Experimental interview")).toBeTruthy(); + expect(screen.queryByText(/Petrinaut saves finalized/)).toBeNull(); + expect( + screen + .getByRole("button", { name: "Start voice" }) + .hasAttribute("disabled"), + ).toBe(true); + expect(createLiveConversation).not.toHaveBeenCalled(); + await start(); + expect(props.reportVoiceSessionState).toHaveBeenLastCalledWith( + expect.objectContaining({ phase: "connecting" }), + ); + const onState = vi.mocked(createLiveConversation).mock.calls[0]![0]; + act(() => onState({ phase: "connected", message: null })); + expect( + screen.queryByRole("region", { name: "Voice mode consent" }), + ).toBeNull(); + expect(props.reportVoiceSessionState).toHaveBeenLastCalledWith({ + phase: "connected", + microphoneLevel: 0, + microphoneMuted: false, + errorMessage: null, + notice: "GPT-Live · Experimental · Connected", + }); + const controls = vi.mocked(props.registerVoiceModeControls).mock.lastCall![0]; + expect(Object.keys(controls).sort()).toEqual(["end", "pause"]); + act(() => + onState({ + phase: "error", + message: "Connection failed. Remote session closure was not confirmed.", + }), + ); + expect(props.reportVoiceSessionState).toHaveBeenLastCalledWith(null); + expect( + screen.getByText( + "Connection error. Check microphone and server configuration.", + ), + ).toBeTruthy(); + expect( + screen + .getByRole("button", { name: "Start voice" }) + .hasAttribute("disabled"), + ).toBe(true); + expect(createLiveConversation).toHaveBeenCalledOnce(); +}); + +test("pins provider, ends through host controls, and never submits or stops canonical work", async () => { + const props = context(); + const subscribeToAdmission = vi.fn(); + const { rerender, unmount } = render( + , + ); + await start(); + const session = vi.mocked(createLiveConversation).mock.results[0]! + .value as ReturnType; + rerender( + , + ); + expect(session.start).toHaveBeenCalledOnce(); + const controls = vi.mocked(props.registerVoiceModeControls).mock.lastCall![0]; + await act(() => controls.end()); + expect(session.stop).toHaveBeenCalled(); + expect(props.submitText).not.toHaveBeenCalled(); + expect(props.submitVoiceInput).not.toHaveBeenCalled(); + expect(props.stop).not.toHaveBeenCalled(); + expect(subscribeToAdmission).not.toHaveBeenCalled(); + unmount(); + expect(props.reportVoiceSessionState).toHaveBeenLastCalledWith(null); +}); + +test("closing the panel ends Live; reopening cannot restart it; stale callbacks cannot reset the next conversation", async () => { + const props = context(); + const { rerender, unmount } = render( + , + ); + await start(); + const session = vi.mocked(createLiveConversation).mock.results[0]! + .value as ReturnType; + const onState = vi.mocked(createLiveConversation).mock.calls[0]![0]; + rerender( + , + ); + await waitFor(() => expect(session.stop).toHaveBeenCalled()); + rerender(); + expect(session.start).toHaveBeenCalledOnce(); + unmount(); + vi.mocked(props.setVoiceActive).mockClear(); + vi.mocked(props.reportVoiceSessionState).mockClear(); + onState({ phase: "ended", message: "Late close" }); + expect(props.setVoiceActive).not.toHaveBeenCalled(); + expect(props.reportVoiceSessionState).not.toHaveBeenCalled(); +}); + +test.each(["live", "realtime", "live-experience"])( + "validates provider config %s", + async (provider) => { + const fetch = vi.fn(async () => + Response.json({ ...config, provider }), + ); + expect(await loadOpenAIVoiceConfig(fetch)).toEqual( + provider === "live-experience" ? null : { ...config, provider }, + ); + }, +); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.tsx b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.tsx new file mode 100644 index 00000000000..cb7869cc837 --- /dev/null +++ b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.tsx @@ -0,0 +1,126 @@ +import { useCallback, useEffect, useRef, useState } from "react"; + +import { + createLiveConversation, + type LiveConversationState, +} from "./live-conversation"; +import { VoiceInterviewDisclosure } from "./voice-interview-disclosure"; + +import type { PetrinautAiVoiceModeContext } from "@hashintel/petrinaut/ui"; + +// No canonical messages, submission, settlement, or tools cross this boundary. +type LiveControlsContext = Pick< + PetrinautAiVoiceModeContext, + | "inputMode" + | "isAiAssistantOpen" + | "registerVoiceModeControls" + | "reportVoiceSessionState" + | "setVoiceActive" + | "setInputMode" +>; + +export const LiveConversationControl = ({ + inputMode, + isAiAssistantOpen, + registerVoiceModeControls, + reportVoiceSessionState, + setVoiceActive, + setInputMode, + connectionTimeoutMs, +}: LiveControlsContext & { readonly connectionTimeoutMs: number }) => { + const [consented, setConsented] = useState(false); + const [phase, setPhase] = useState("idle"); + const session = useRef | null>( + null, + ); + const end = useCallback(async () => { + const closing = session.current?.stop(); + setVoiceActive(false); + setConsented(false); + await closing; + }, [setVoiceActive]); + + useEffect( + () => + registerVoiceModeControls({ + end, + // Closing the panel ends Live. Reopening requires consent and a new session. + pause: () => { + void end(); + }, + }), + [end, registerVoiceModeControls], + ); + + useEffect(() => { + reportVoiceSessionState( + inputMode === "voice" && + isAiAssistantOpen && + (phase === "connecting" || phase === "connected") + ? { + phase, + microphoneLevel: 0, + microphoneMuted: false, + errorMessage: null, + notice: `GPT-Live · Experimental · ${phase === "connected" ? "Connected" : "Connecting"}`, + } + : null, + ); + }, [inputMode, isAiAssistantOpen, phase, reportVoiceSessionState]); + + useEffect(() => { + if (inputMode !== "voice" || !isAiAssistantOpen) + void session.current?.stop(); + }, [inputMode, isAiAssistantOpen]); + useEffect(() => { + const leave = () => { + void session.current?.stop(); + }; + window.addEventListener("pagehide", leave); + return () => { + window.removeEventListener("pagehide", leave); + const current = session.current; + session.current = null; + void current?.stop(); + setVoiceActive(false); + reportVoiceSessionState(null); + }; + }, [reportVoiceSessionState, setVoiceActive]); + + if (inputMode !== "voice" || phase === "connecting" || phase === "connected") + return null; + return ( + { + if (!consented || phase === "stopping") return; + setConsented(false); + setPhase("connecting"); + const next = createLiveConversation((nextState) => { + if (session.current !== next) return; + setPhase(nextState.phase); + setVoiceActive( + nextState.phase === "connecting" || nextState.phase === "connected", + ); + }, connectionTimeoutMs); + session.current = next; + setVoiceActive(true); + void next.start(); + }} + onExit={() => { + void end(); + setInputMode("text"); + }} + /> + ); +}; diff --git a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation.test.ts new file mode 100644 index 00000000000..0a11a7b04ce --- /dev/null +++ b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation.test.ts @@ -0,0 +1,221 @@ +// @vitest-environment jsdom +import { afterEach, expect, test, vi } from "vitest"; + +import { createLiveConversation } from "./live-conversation"; + +const setup = () => { + const sent: string[] = []; + const channel = Object.assign(new EventTarget(), { + readyState: "open", + send: (data: string) => sent.push(data), + close: vi.fn(), + }); + const input = { stop: vi.fn() }; + const output = { stop: vi.fn() }; + const stream = { getTracks: () => [input] }; + const peer = Object.assign(new EventTarget(), { + connectionState: "new", + iceGatheringState: "complete", + localDescription: { type: "offer", sdp: "v=0\r\no=complete-offer" }, + createDataChannel: vi.fn(() => channel), + addTrack: vi.fn(), + createOffer: vi.fn(async () => ({ type: "offer", sdp: "v=0" })), + setLocalDescription: vi.fn(async () => undefined), + setRemoteDescription: vi.fn(async () => undefined), + close: vi.fn(), + getReceivers: () => [{ track: output }], + }); + const audio = { + srcObject: null, + autoplay: false, + muted: false, + play: vi.fn(async () => undefined), + pause: vi.fn(), + }; + vi.stubGlobal( + "RTCPeerConnection", + class { + constructor() { + return peer; + } + }, + ); + vi.stubGlobal( + "Audio", + class { + constructor() { + return audio; + } + }, + ); + const getUserMedia = vi.fn(async () => stream); + vi.stubGlobal("navigator", { mediaDevices: { getUserMedia } }); + const fetch = vi.fn(async () => + Response.json({ sessionId: "opaque/id", sdp: "v=0\r\no=answer" }), + ); + vi.stubGlobal("fetch", fetch); + const onState = vi.fn[0]>(); + const conversation = createLiveConversation(onState, 15_000); + const emit = (data: unknown) => + channel.dispatchEvent( + new MessageEvent("message", { data: JSON.stringify(data) }), + ); + return { + conversation, + emit, + sent, + channel, + input, + output, + peer, + audio, + getUserMedia, + fetch, + onState, + }; +}; + +afterEach(() => { + vi.useRealTimers(); + vi.unstubAllGlobals(); +}); + +test("waits for session.started after SDP; never sends session.start or handles transcript/delegation as tasks", async () => { + const fixture = setup(); + expect(fixture.getUserMedia).not.toHaveBeenCalled(); + await fixture.conversation.start(); + expect(fixture.onState).toHaveBeenLastCalledWith( + expect.objectContaining({ phase: "connecting" }), + ); + expect(fixture.fetch).toHaveBeenCalledWith( + "/api/voice/live-session", + expect.objectContaining({ body: "v=0\r\no=complete-offer" }), + ); + expect(fixture.peer.createDataChannel).toHaveBeenCalledWith("oai-events"); + fixture.emit({ type: "session.started" }); + expect(fixture.onState).toHaveBeenLastCalledWith({ + phase: "connected", + message: null, + }); + fixture.emit({ + type: "session.input_transcript.delta", + delta: "Change the model", + }); + fixture.emit({ type: "session.output_transcript.delta", delta: "Done" }); + fixture.emit({ + type: "session.delegation.created", + delegation: { id: "metadata-only", target: "client" }, + }); + expect(fixture.sent).toEqual([]); + expect(fixture.fetch).toHaveBeenCalledTimes(1); + const stopped = fixture.conversation.stop(); + expect(fixture.input.stop).toHaveBeenCalled(); + expect(fixture.output.stop).toHaveBeenCalled(); + expect(fixture.audio.pause).toHaveBeenCalled(); + expect(fixture.audio.srcObject).toBeNull(); + expect(fixture.sent.map((event) => JSON.parse(event))).toEqual([ + { type: "session.close" }, + ]); + expect(fixture.peer.close).not.toHaveBeenCalled(); + fixture.emit({ type: "session.closed" }); + await stopped; + expect(fixture.peer.close).toHaveBeenCalledOnce(); + expect(fixture.onState).toHaveBeenLastCalledWith({ + phase: "ended", + message: "Microphone and playback stopped. Live confirmed session closure.", + }); +}); + +test("releases late microphone permission without ever creating a provider session", async () => { + const fixture = setup(); + let release!: ( + stream: Awaited>, + ) => void; + fixture.getUserMedia.mockImplementation( + () => + new Promise((resolve) => { + release = resolve; + }), + ); + const started = fixture.conversation.start(); + await fixture.conversation.stop(); + release({ getTracks: () => [fixture.input] }); + await started; + expect(fixture.input.stop).toHaveBeenCalled(); + expect(fixture.fetch).not.toHaveBeenCalled(); +}); + +test("reports missing remote closure honestly and cleans transport after bounded wait", async () => { + vi.useFakeTimers(); + const fixture = setup(); + await fixture.conversation.start(); + fixture.emit({ type: "session.started" }); + const stopped = fixture.conversation.stop(); + await vi.advanceTimersByTimeAsync(2_000); + await stopped; + expect(fixture.peer.close).toHaveBeenCalled(); + expect(fixture.onState.mock.lastCall?.[0].phase).toBe("ended"); + expect(fixture.onState.mock.lastCall?.[0].message).toContain("not confirmed"); +}); + +test("times out connection without inventing turn completion or retrying", async () => { + vi.useFakeTimers(); + const fixture = setup(); + await fixture.conversation.start(); + await vi.advanceTimersByTimeAsync(17_000); + expect(fixture.input.stop).toHaveBeenCalled(); + expect(fixture.onState.mock.lastCall?.[0].phase).toBe("error"); + expect(fixture.onState.mock.lastCall?.[0].message).toContain("timed out"); + expect(fixture.fetch).toHaveBeenCalledTimes(1); +}); + +test("provider errors stop both media directions without exposing payloads", async () => { + const fixture = setup(); + await fixture.conversation.start(); + fixture.emit({ type: "session.started" }); + fixture.emit({ + type: "error", + error: { message: "sensitive provider detail" }, + }); + expect(fixture.input.stop).toHaveBeenCalled(); + expect(fixture.output.stop).toHaveBeenCalled(); + fixture.emit({ type: "session.closed" }); + expect(fixture.onState.mock.lastCall?.[0].phase).toBe("error"); + expect(fixture.onState.mock.lastCall?.[0].message).not.toContain("sensitive"); +}); + +test("Stop before session.started never sends application commands", async () => { + vi.useFakeTimers(); + const fixture = setup(); + await fixture.conversation.start(); + const stopped = fixture.conversation.stop(); + await vi.advanceTimersByTimeAsync(2_000); + await stopped; + expect(fixture.sent).toEqual([]); + expect(fixture.onState.mock.lastCall?.[0].message).toContain("not confirmed"); +}); + +test("plays the remote track while microphone remains attached, then stops both", async () => { + const fixture = setup(); + await fixture.conversation.start(); + fixture.emit({ type: "session.started" }); + const remoteStream = { getTracks: () => [fixture.output] }; + fixture.peer.dispatchEvent( + Object.assign(new Event("track"), { + track: fixture.output, + streams: [remoteStream], + }), + ); + expect(fixture.audio.srcObject).toBe(remoteStream); + expect(fixture.audio.play).toHaveBeenCalledOnce(); + expect(fixture.peer.addTrack).toHaveBeenCalledWith( + fixture.input, + expect.anything(), + ); + expect(fixture.input.stop).not.toHaveBeenCalled(); + const stopped = fixture.conversation.stop(); + fixture.emit({ type: "session.closed" }); + await stopped; + expect(fixture.audio.pause).toHaveBeenCalled(); + expect(fixture.input.stop).toHaveBeenCalled(); +}); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation.ts b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation.ts new file mode 100644 index 00000000000..91df16d4cbb --- /dev/null +++ b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation.ts @@ -0,0 +1,230 @@ +export interface LiveConversationState { + readonly phase: + | "idle" + | "connecting" + | "connected" + | "stopping" + | "ended" + | "error"; + readonly message: string | null; +} + +/** One disposable Live session. No composer, tool, transcript or turn-settlement interface. */ +export const createLiveConversation = ( + onState: (state: LiveConversationState) => void, + connectionTimeoutMs: number, +) => { + const abort = new AbortController(); + let peer: RTCPeerConnection | undefined; + let channel: RTCDataChannel | undefined; + let microphone: MediaStream | undefined; + let audio: HTMLAudioElement | undefined; + let started = false; + let ready = false; + let stopping = false; + let finished = false; + let creationRequested = false; + let failure: string | undefined; + let connectionTimer: ReturnType | undefined; + let closeTimer: ReturnType | undefined; + let resolveStopped: () => void = () => {}; + const stopped = new Promise((resolve) => { + resolveStopped = resolve; + }); + + const stopMedia = () => { + microphone?.getTracks().forEach((track) => track.stop()); + if (audio) { + audio.muted = true; + audio.pause(); + audio.srcObject = null; + } + peer?.getReceivers().forEach((receiver) => receiver.track.stop()); + }; + + const finish = (confirmed: boolean) => { + if (finished) return; + finished = true; + stopping = true; + clearTimeout(connectionTimer); + clearTimeout(closeTimer); + abort.abort(); + stopMedia(); + channel?.close(); + peer?.close(); + const closure = confirmed + ? "Live confirmed session closure." + : creationRequested + ? "Remote session closure was not confirmed." + : "No provider session was requested."; + onState({ + phase: failure ? "error" : "ended", + message: `${failure ? `${failure} ` : ""}Microphone and playback stopped. ${closure}`, + }); + resolveStopped(); + }; + + const stop = (): Promise => { + if (stopping) return stopped; + stopping = true; + clearTimeout(connectionTimer); + abort.abort(); + stopMedia(); + onState({ + phase: "stopping", + message: "Microphone and playback stopped. Closing Live…", + }); + if (channel?.readyState === "open" && ready) { + // Registered message listener remains until session.closed or this cleanup deadline. + closeTimer = setTimeout(() => finish(false), 2_000); + try { + channel.send(JSON.stringify({ type: "session.close" })); + } catch { + finish(false); + } + } else { + finish(false); + } + return stopped; + }; + + const fail = (message: string) => { + if (stopping) return; + failure = message; + void stop(); + }; + + const start = async (): Promise => { + if (started || stopping) return; + started = true; + onState({ phase: "connecting", message: null }); + connectionTimer = setTimeout( + () => fail("Live connection timed out. No automatic retry was made."), + connectionTimeoutMs, + ); + try { + audio = new Audio(); + const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + if (abort.signal.aborted) { + stream.getTracks().forEach((track) => track.stop()); + return; + } + microphone = stream; + peer = new RTCPeerConnection(); + const connection = peer; + channel = connection.createDataChannel("oai-events"); + channel.addEventListener("message", (event: MessageEvent) => { + if (finished) return; + let data: unknown; + try { + data = JSON.parse(event.data); + } catch { + fail("Live sent an unreadable event."); + return; + } + if (typeof data !== "object" || data === null || !("type" in data)) + return; + if (data.type === "session.closed") { + finish(true); + return; + } + if (stopping) return; + if (data.type === "session.started") { + ready = true; + clearTimeout(connectionTimer); + onState({ phase: "connected", message: null }); + } else if (data.type === "error" || data.type === "session.error") { + fail("Live reported an error. No automatic retry was made."); + } + // Transcript deltas are not finalized utterances. Delegations contain metadata, + // not task text. Neither is forwarded, persisted, or used to execute anything. + }); + channel.addEventListener("close", () => { + if (!stopping) failure = "Live disconnected."; + finish(false); + }); + channel.addEventListener("error", () => + fail("Live data connection failed."), + ); + connection.addEventListener("connectionstatechange", () => { + if ( + ["failed", "disconnected", "closed"].includes( + connection.connectionState, + ) + ) + fail("Live media connection ended."); + }); + connection.addEventListener("track", (event) => { + if (stopping) { + event.track.stop(); + return; + } + if (!audio) return; + audio.srcObject = event.streams[0] ?? new MediaStream([event.track]); + void audio + .play() + .catch(() => + fail( + "Browser blocked Live playback. Start a new session after checking audio permissions.", + ), + ); + }); + stream.getTracks().forEach((track) => connection.addTrack(track, stream)); + await connection.setLocalDescription(await connection.createOffer()); + abort.signal.throwIfAborted(); + if (connection.iceGatheringState !== "complete") { + const listeners = new AbortController(); + await new Promise((resolve, reject) => { + const check = () => { + if (connection.iceGatheringState === "complete") { + resolve(); + } + }; + const cancelled = () => { + reject(abort.signal.reason); + }; + connection.addEventListener("icegatheringstatechange", check, { + signal: listeners.signal, + }); + abort.signal.addEventListener("abort", cancelled, { + once: true, + signal: listeners.signal, + }); + check(); + }).finally(() => listeners.abort()); + } + abort.signal.throwIfAborted(); + const sdp = connection.localDescription?.sdp; + if (!sdp) throw new Error("Missing local SDP"); + creationRequested = true; + const response = await fetch("/api/voice/live-session", { + method: "POST", + headers: { "content-type": "application/sdp" }, + body: sdp, + signal: abort.signal, + }); + if (!response.ok) throw new Error("Session creation failed"); + const answer: unknown = await response.json(); + abort.signal.throwIfAborted(); + if ( + typeof answer !== "object" || + answer === null || + !("sdp" in answer) || + typeof answer.sdp !== "string" || + !answer.sdp.trimStart().startsWith("v=0") + ) + throw new Error("Invalid SDP answer"); + await connection.setRemoteDescription({ + type: "answer", + sdp: answer.sdp, + }); + // WebRTC creation already starts Live. Wait for session.started; never send session.start. + } catch { + fail( + "Live could not connect. Check microphone, audio permissions and server configuration. No automatic retry was made.", + ); + } + }; + + return { start, stop }; +}; diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx index 1641df27470..f43b2603e71 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx @@ -11,17 +11,17 @@ import { type FlueChatResponseMessageCompletedEvent, type FlueChatResponseMessageStartedEvent, } from "@hashintel/brunch-agent-transport-aisdk"; -import { Button, Checkbox } from "@hashintel/ds-components"; -import { css } from "@hashintel/ds-helpers/css"; import { reportVoiceDiagnostic } from "../../../voice-diagnostics"; import { selectCanonicalSpeech } from "./canonical-speech"; +import { LiveConversationControl } from "./live-conversation-control"; import { OpenAIRealtimeSession } from "./openai-realtime-session"; import { RealtimeBrunchBridge, type RealtimeBrunchAdmissionTarget, type VoiceSubmissionSettlement, } from "./realtime-brunch-bridge"; +import { VoiceInterviewDisclosure } from "./voice-interview-disclosure"; import { toVoiceSessionState } from "./voice-session-state"; import { VoiceTurnController, @@ -132,6 +132,7 @@ export const submitVoiceInputWithAdmission = async ({ export interface OpenAIVoiceConfig { readonly available: true; readonly connectionTimeoutMs: number; + readonly provider?: "realtime" | "live"; } export const VOICE_INTERVIEW_DISCLOSURE_STORAGE_KEY = @@ -231,6 +232,9 @@ export const loadOpenAIVoiceConfig = async ( if ( !isRecord(body) || body.available !== true || + (body.provider !== undefined && + body.provider !== "realtime" && + body.provider !== "live") || !Number.isInteger(body.connectionTimeoutMs) || (body.connectionTimeoutMs as number) < 1_000 || (body.connectionTimeoutMs as number) > 60_000 @@ -240,206 +244,15 @@ export const loadOpenAIVoiceConfig = async ( return { available: true, connectionTimeoutMs: body.connectionTimeoutMs as number, + ...(body.provider === undefined + ? {} + : { provider: body.provider as "realtime" | "live" }), }; } catch { return null; } }; -const VoiceModeIcon = () => ( - -); - -const disclosureFrameStyle = css({ - width: "full", - padding: "2", - borderTopWidth: "thin", - borderTopStyle: "solid", - borderTopColor: "neutral.a20", - backgroundColor: "neutral.bg.subtle", - color: "neutral.s100", - _focus: { outline: "none" }, -}); - -const disclosureCardStyle = css({ - display: "flex", - flexDirection: "column", - gap: "2", - padding: "3", - borderWidth: "thin", - borderStyle: "solid", - borderColor: "neutral.a20", - borderRadius: "xl", - backgroundColor: "neutral.s00", - boxShadow: - "[0px 0px 0px 1px rgba(0,0,0,0.03), 0px 8px 16px -12px rgba(0,0,0,0.18)]", -}); - -const disclosureHeaderStyle = css({ - display: "flex", - alignItems: "center", - gap: "2", -}); - -const disclosureIconStyle = css({ - display: "inline-flex", - width: "7", - height: "7", - flexShrink: "0", - alignItems: "center", - justifyContent: "center", - borderRadius: "lg", - backgroundColor: "blue.a20", - color: "blue.s90", -}); - -const disclosureTitleStyle = css({ - display: "flex", - minWidth: "[0]", - flexDirection: "column", - gap: "0.5", -}); - -const disclosureHeadingStyle = css({ - fontSize: "sm", - fontWeight: "semibold", - lineHeight: "tight", -}); - -const disclosureSubtitleStyle = css({ - color: "neutral.s80", - fontSize: "xs", -}); - -const disclosureCopyStyle = css({ - color: "neutral.s90", - fontSize: "xs", - lineHeight: "relaxed", -}); - -const disclosureConsentStyle = css({ - width: "full", - padding: "2", - borderRadius: "lg", - backgroundColor: "neutral.a10", - color: "neutral.s100", -}); - -const disclosureActionsStyle = css({ - display: "flex", - flexWrap: "wrap", - alignItems: "center", - gap: "2", -}); - -const disclosureStatusStyle = css({ - minHeight: "[18px]", - color: "neutral.s80", - fontSize: "xs", - lineHeight: "relaxed", -}); - -const VoiceInterviewDisclosure = ({ - checkingMicrophone, - consented, - microphoneCheck, - onCheckMicrophone, - onConsentChange, - onStart, -}: { - readonly checkingMicrophone: boolean; - readonly consented: boolean; - readonly microphoneCheck: string; - readonly onCheckMicrophone: () => void; - readonly onConsentChange: (consented: boolean) => void; - readonly onStart: () => void; -}) => { - const disclosureRef = useRef(null); - - useEffect(() => { - disclosureRef.current?.focus(); - }, []); - - return ( -
-
-
- - - -
- - Start a voice conversation - - - Talk through your process with AI - -
-
-

- OpenAI processes live audio and speaks the interviewer’s words. - Petrinaut saves finalized answers—not audio. -

- -
- - -
-
- {microphoneCheck} -
-
-
- ); -}; - const recordLatency = (event: VoiceLatencyEvent): void => { try { performance.measure(`voice-interview:${event.name}`, { @@ -753,7 +566,7 @@ const AvailableVoiceInterviewControl = ({ return null; }; -export const VoiceInterviewControl = ({ +const PinnedVoiceInterviewControl = ({ config, resolveInputSubmission, resolveResponseSubmission, @@ -774,18 +587,40 @@ export const VoiceInterviewControl = ({ readonly subscribeToResponseMessageCompleted?: SubscribeToResponseMessageCompleted; readonly subscribeToResponseMessageStarted?: SubscribeToResponseMessageStarted; readonly subscribeToStopRequested?: SubscribeToStopRequested; -}) => ( - -); +}) => { + // Configuration changes only apply after a new conversation mount / page load. + // Never replace a running provider or resubmit its input. + const [sessionConfig] = useState(config); + if (sessionConfig.provider === "live") { + return ( + + ); + } + return ( + + ); +}; + +export const VoiceInterviewControl = ( + props: Parameters[0], +) => ; diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-disclosure.tsx b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-disclosure.tsx new file mode 100644 index 00000000000..0f0f22f63c5 --- /dev/null +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-disclosure.tsx @@ -0,0 +1,209 @@ +import { useEffect, useRef } from "react"; + +import { Button, Checkbox } from "@hashintel/ds-components"; +import { css } from "@hashintel/ds-helpers/css"; + +const VoiceModeIcon = () => ( + +); + +const disclosureFrameStyle = css({ + width: "full", + padding: "2", + borderTopWidth: "thin", + borderTopStyle: "solid", + borderTopColor: "neutral.a20", + backgroundColor: "neutral.bg.subtle", + color: "neutral.s100", + _focus: { outline: "none" }, +}); +const disclosureCardStyle = css({ + display: "flex", + flexDirection: "column", + gap: "2", + padding: "3", + borderWidth: "thin", + borderStyle: "solid", + borderColor: "neutral.a20", + borderRadius: "xl", + backgroundColor: "neutral.s00", + boxShadow: + "[0px 0px 0px 1px rgba(0,0,0,0.03), 0px 8px 16px -12px rgba(0,0,0,0.18)]", +}); +const disclosureHeaderStyle = css({ + display: "flex", + alignItems: "center", + gap: "2", +}); +const disclosureIconStyle = css({ + display: "inline-flex", + width: "7", + height: "7", + flexShrink: "0", + alignItems: "center", + justifyContent: "center", + borderRadius: "lg", + backgroundColor: "blue.a20", + color: "blue.s90", +}); +const disclosureTitleStyle = css({ + display: "flex", + minWidth: "[0]", + flexDirection: "column", + gap: "0.5", +}); +const disclosureHeadingStyle = css({ + fontSize: "sm", + fontWeight: "semibold", + lineHeight: "tight", +}); +const disclosureSubtitleStyle = css({ color: "neutral.s80", fontSize: "xs" }); +const disclosureCopyStyle = css({ + color: "neutral.s90", + fontSize: "xs", + lineHeight: "relaxed", +}); +const disclosureConsentStyle = css({ + width: "full", + padding: "2", + borderRadius: "lg", + backgroundColor: "neutral.a10", + color: "neutral.s100", +}); +const disclosureActionsStyle = css({ + display: "flex", + flexWrap: "wrap", + alignItems: "center", + gap: "2", +}); +const disclosureStatusStyle = css({ + minHeight: "[18px]", + color: "neutral.s80", + fontSize: "xs", + lineHeight: "relaxed", +}); + +export const VoiceInterviewDisclosure = ({ + checkingMicrophone = false, + consented, + microphoneCheck, + onCheckMicrophone, + onConsentChange, + onStart, + experimental = false, + startDisabled = false, + onExit, +}: { + readonly checkingMicrophone?: boolean; + readonly consented: boolean; + readonly microphoneCheck: string; + readonly onCheckMicrophone?: () => void; + readonly onConsentChange: (consented: boolean) => void; + readonly onStart: () => void; + readonly experimental?: boolean; + readonly startDisabled?: boolean; + readonly onExit?: () => void; +}) => { + const disclosureRef = useRef(null); + useEffect(() => { + disclosureRef.current?.focus(); + }, []); + + return ( +
+
+
+ + + +
+ + {experimental + ? "GPT-Live · Experimental interview" + : "Start a voice conversation"} + + {!experimental && ( + + Talk through your process with AI + + )} +
+
+ {!experimental && ( +

+ OpenAI processes live audio and speaks the interviewer’s words. + Petrinaut saves finalized answers—not audio. +

+ )} + +
+ + {onCheckMicrophone && ( + + )} + {onExit && ( + + )} +
+ {(!experimental || microphoneCheck) && ( +
+ {microphoneCheck} +
+ )} +
+
+ ); +}; diff --git a/apps/petrinaut-website/src/server/voice/openai-live-session.test.ts b/apps/petrinaut-website/src/server/voice/openai-live-session.test.ts new file mode 100644 index 00000000000..8bf7d927967 --- /dev/null +++ b/apps/petrinaut-website/src/server/voice/openai-live-session.test.ts @@ -0,0 +1,166 @@ +import { describe, expect, test, vi } from "vitest"; + +import { createOpenAILiveSessionHandler } from "./openai-live-session"; +import { createOpenAIVoiceConfigHandler } from "./openai-voice-config"; + +const environment = { + PETRINAUT_OPENAI_VOICE_ENABLED: "true", + PETRINAUT_VOICE_PROVIDER: "live", + OPENAI_VOICE_API_KEY: "server-only-secret", +}; +const request = (overrides: RequestInit = {}) => + new Request("https://petrinaut.test/api/voice/live-session", { + method: "POST", + headers: { + origin: "https://petrinaut.test", + "content-type": "application/sdp", + }, + body: "v=0\r\no=offer", + ...overrides, + }); + +describe("Live configuration and session creation", () => { + test.each([ + [undefined, "realtime", true], + ["realtime", "realtime", true], + ["live", "live", true], + ["live-brunch", null, false], + ])( + "selects %s without silently falling back", + async (value, provider, available) => { + const handler = createOpenAIVoiceConfigHandler({ + ...environment, + PETRINAUT_VOICE_PROVIDER: value, + }); + const response = await handler( + new Request("https://petrinaut.test/api/voice/config"), + ); + expect(await response.json()).toEqual({ + available, + provider, + connectionTimeoutMs: 15_000, + }); + }, + ); + + test.each([ + [{ method: "GET", body: undefined }, 405], + [ + { + headers: { + origin: "https://attacker.test", + "content-type": "application/sdp", + }, + }, + 403, + ], + [{ headers: { "content-type": "application/sdp" } }, 403], + [ + { + headers: { + origin: "https://petrinaut.test", + "content-type": "application/json", + }, + }, + 415, + ], + [{ body: "not SDP" }, 400], + [{ body: "v=0" + "x".repeat(65_536) }, 413], + ] satisfies [RequestInit, number][])( + "rejects unsafe requests before fetching: %j", + async (overrides, status) => { + const fetch = vi.fn(); + const response = await createOpenAILiveSessionHandler({ + environment, + fetch, + })(request(overrides)); + expect(response.status).toBe(status); + expect(response.headers.get("cache-control")).toBe("no-store"); + expect(fetch).not.toHaveBeenCalled(); + }, + ); + + test.each([ + { PETRINAUT_VOICE_PROVIDER: undefined }, + { PETRINAUT_VOICE_PROVIDER: "realtime" }, + { PETRINAUT_OPENAI_VOICE_ENABLED: "false" }, + { OPENAI_VOICE_API_KEY: " " }, + ])( + "requires Live selection and existing enablement/credentials: %j", + async (override) => { + const fetch = vi.fn(); + expect( + ( + await createOpenAILiveSessionHandler({ + environment: { ...environment, ...override }, + fetch, + })(request()) + ).status, + ).toBe(404); + expect(fetch).not.toHaveBeenCalled(); + }, + ); + + test("creates one client-delegated WebRTC session with trusted instructions and no tools", async () => { + const fetch = vi.fn(async () => + Response.json( + { + session: { id: "opaque/session:id" }, + transport: { type: "webrtc", sdp: "v=0\r\no=answer" }, + secret: "must-not-leak", + }, + { status: 201 }, + ), + ); + const response = await createOpenAILiveSessionHandler({ + environment, + fetch, + })(request()); + expect(response.status).toBe(201); + expect(await response.json()).toEqual({ + sessionId: "opaque/session:id", + sdp: "v=0\r\no=answer", + }); + expect(fetch).toHaveBeenCalledTimes(1); + const [url, init] = fetch.mock.calls[0]!; + expect(url).toBe("https://api.openai.com/v1/live/sessions"); + expect(typeof init?.body).toBe("string"); + const body: unknown = JSON.parse(init?.body as string); + expect(body).toEqual({ + session: { + model: "gpt-live-1", + delegation: { type: "client" }, + store: false, + instructions: expect.stringContaining("Do not operate") as unknown, + audio: { output: { voice: "marin" } }, + }, + transport: { type: "webrtc", sdp: "v=0\r\no=offer" }, + }); + expect(new Headers(init?.headers).get("authorization")).toBe( + "Bearer server-only-secret", + ); + }); + + test("sanitizes provider failure and never retries", async () => { + const fetch = vi.fn( + async () => new Response("server-only-secret", { status: 429 }), + ); + const response = await createOpenAILiveSessionHandler({ + environment, + fetch, + })(request()); + expect(response.status).toBe(502); + expect(await response.text()).not.toContain("server-only-secret"); + expect(fetch).toHaveBeenCalledTimes(1); + }); + + test("does not contact the provider for an already aborted request", async () => { + const fetch = vi.fn(); + const response = await createOpenAILiveSessionHandler({ + environment, + fetch, + })(request({ signal: AbortSignal.abort() })); + expect(response.status).toBe(502); + expect(fetch).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/petrinaut-website/src/server/voice/openai-live-session.ts b/apps/petrinaut-website/src/server/voice/openai-live-session.ts new file mode 100644 index 00000000000..822077217e6 --- /dev/null +++ b/apps/petrinaut-website/src/server/voice/openai-live-session.ts @@ -0,0 +1,113 @@ +import { getVoiceProvider } from "./openai-voice-config.js"; +import { getOpenAIVoiceAvailability } from "./openai-voice-policy.js"; + +const instructions = `You are an experimental conversational stand-in for a process interviewer, not Brunch or the application's agent. Help the speaker explain a familiar process. Listen to their meaning, allow hesitation and elaboration, and incorporate corrections. Ask one concise, relevant follow-up when uncertainty matters. Avoid repetitive acknowledgements and long monologues. Do not operate the application, call tools, or claim that changes were executed or saved. You have no access to the application's chat, model, or workpiece. Keep this a conversation, not a modelling system.`; + +/** Uses the existing website credential boundary; this switch is not authentication. */ +export const createOpenAILiveSessionHandler = + ({ + environment, + fetch, + }: { + environment: Parameters[0]; + fetch: typeof globalThis.fetch; + }) => + async (request: Request): Promise => { + const respond = ( + body: string, + status: number, + headers?: Record, + ) => + new Response(body, { + status, + headers: { "cache-control": "no-store", ...headers }, + }); + if (request.method !== "POST") + return respond("Method not allowed.", 405, { allow: "POST" }); + if (request.headers.get("origin") !== new URL(request.url).origin) + return respond("Forbidden.", 403); + if ( + request.headers + .get("content-type") + ?.split(";")[0] + ?.trim() + .toLowerCase() !== "application/sdp" + ) + return respond("Expected SDP.", 415); + const availability = getOpenAIVoiceAvailability(environment); + if (!availability.available || getVoiceProvider(environment) !== "live") + return respond("Live is unavailable.", 404); + + const signal = AbortSignal.any([ + request.signal, + AbortSignal.timeout(availability.connectionTimeoutMs), + ]); + try { + signal.throwIfAborted(); + if (Number(request.headers.get("content-length")) > 65_536) + return respond("SDP too large.", 413); + const body = await request.arrayBuffer(); + if (body.byteLength > 65_536) return respond("SDP too large.", 413); + const sdp = new TextDecoder().decode(body); + if (!sdp.trimStart().startsWith("v=0")) + return respond("Invalid SDP.", 400); + signal.throwIfAborted(); + const upstream = await fetch("https://api.openai.com/v1/live/sessions", { + method: "POST", + signal, + headers: { + authorization: `Bearer ${environment.OPENAI_VOICE_API_KEY!.trim()}`, + "content-type": "application/json", + }, + body: JSON.stringify({ + session: { + model: "gpt-live-1", + instructions, + delegation: { type: "client" }, + store: false, + audio: { output: { voice: "marin" } }, + }, + transport: { type: "webrtc", sdp }, + }), + }); + if (!upstream.ok) { + await upstream.body?.cancel(); + return respond( + "Live session creation failed. No automatic retry was made.", + 502, + ); + } + const answer: unknown = await upstream.json(); + if ( + typeof answer !== "object" || + answer === null || + !("session" in answer) || + !("transport" in answer) + ) + throw new Error("Invalid answer"); + const { session, transport } = answer; + if ( + typeof session !== "object" || + session === null || + !("id" in session) || + typeof session.id !== "string" || + typeof transport !== "object" || + transport === null || + !("type" in transport) || + transport.type !== "webrtc" || + !("sdp" in transport) || + typeof transport.sdp !== "string" || + !transport.sdp.trimStart().startsWith("v=0") + ) + throw new Error("Invalid answer"); + return Response.json( + { sessionId: session.id, sdp: transport.sdp }, + { status: 201, headers: { "cache-control": "no-store" } }, + ); + } catch { + return respond( + "Live connection failed. The remote session outcome may be unknown; no automatic retry was made.", + 502, + ); + } + }; diff --git a/apps/petrinaut-website/src/server/voice/openai-voice-config.test.ts b/apps/petrinaut-website/src/server/voice/openai-voice-config.test.ts index 51537c3345c..f60a5b3e317 100644 --- a/apps/petrinaut-website/src/server/voice/openai-voice-config.test.ts +++ b/apps/petrinaut-website/src/server/voice/openai-voice-config.test.ts @@ -19,6 +19,7 @@ describe("OpenAI voice config handler", () => { expect(await response.json()).toEqual({ available: true, connectionTimeoutMs: 15_000, + provider: "realtime", }); }); diff --git a/apps/petrinaut-website/src/server/voice/openai-voice-config.ts b/apps/petrinaut-website/src/server/voice/openai-voice-config.ts index 699002753df..62483b1e118 100644 --- a/apps/petrinaut-website/src/server/voice/openai-voice-config.ts +++ b/apps/petrinaut-website/src/server/voice/openai-voice-config.ts @@ -3,9 +3,15 @@ import { getOpenAIVoiceAvailability } from "./openai-voice-policy.js"; interface VoiceEnvironment { readonly OPENAI_VOICE_API_KEY?: string; readonly PETRINAUT_OPENAI_VOICE_ENABLED?: string; + readonly PETRINAUT_VOICE_PROVIDER?: string; readonly VERCEL_ENV?: string; } +export const getVoiceProvider = (environment: VoiceEnvironment) => { + const provider = environment.PETRINAUT_VOICE_PROVIDER ?? "realtime"; + return provider === "realtime" || provider === "live" ? provider : null; +}; + export const createOpenAIVoiceConfigHandler = (environment: VoiceEnvironment) => async (request: Request): Promise => { @@ -16,7 +22,16 @@ export const createOpenAIVoiceConfigHandler = }); } - return Response.json(getOpenAIVoiceAvailability(environment), { - headers: { "cache-control": "no-store" }, - }); + const availability = getOpenAIVoiceAvailability(environment); + const provider = getVoiceProvider(environment); + return Response.json( + { + ...availability, + provider, + available: availability.available && provider !== null, + }, + { + headers: { "cache-control": "no-store" }, + }, + ); }; diff --git a/apps/petrinaut-website/vite.config.ts b/apps/petrinaut-website/vite.config.ts index 027a6924172..d0a454a19ed 100644 --- a/apps/petrinaut-website/vite.config.ts +++ b/apps/petrinaut-website/vite.config.ts @@ -26,6 +26,7 @@ const apiModules = [ ["/api/oembed", "/api/oembed.ts"], ["/api/voice/config", "/api/voice/config.ts"], ["/api/voice/realtime-call", "/api/voice/realtime-call.ts"], + ["/api/voice/live-session", "/api/voice/live-session.ts"], ] as const; // Plugin required to serve the Vercel fetch handlers in dev. In production, diff --git a/libs/@hashintel/petrinaut/docs/ai-assistant.md b/libs/@hashintel/petrinaut/docs/ai-assistant.md index b828d8223ff..2097be2bca0 100644 --- a/libs/@hashintel/petrinaut/docs/ai-assistant.md +++ b/libs/@hashintel/petrinaut/docs/ai-assistant.md @@ -64,6 +64,15 @@ understand and selecting **Start voice mode**. Petrinaut remembers that acknowle browser for the current disclosure version, so later uses of **Start voice mode** start directly. If browser storage is unavailable or the disclosure changes, Petrinaut asks again. +Some hosts offer a **GPT-Live · Experimental interview** instead. It uses the same +Voice setup and dock, but is a standalone conversation: it does not add chat messages +or edit the net. Consent and **Start voice** are required for each session. Its dock +shows **Connected**, not whose turn it is, with collapse/expand and **End voice mode** +only. Ending Voice stops microphone capture and playback; closing the panel ends +the experiment rather than pausing it. A connection error returns to setup without +retrying. The following turn-taking and playback controls apply to integrated +Brunch Voice, not that experiment. + While a session runs, the composer is replaced by a low-profile Voice dock at the foot of the panel: a ribbon that fades out at both ends and one short state -- **Connecting**, **Listening**, **Muted**, **Thinking**, **Speaking**, **Paused**, or **Voice interrupted**. The ribbon is three diff --git a/libs/@hashintel/petrinaut/src/react/voice-session/store.ts b/libs/@hashintel/petrinaut/src/react/voice-session/store.ts index bc03867ca28..a24fd47cca2 100644 --- a/libs/@hashintel/petrinaut/src/react/voice-session/store.ts +++ b/libs/@hashintel/petrinaut/src/react/voice-session/store.ts @@ -8,11 +8,11 @@ export type VoiceSessionActions = { end: () => void; pause: () => void; readFullResponse?: () => void; - reconnect: () => void; + reconnect?: () => void; repeatQuestion?: () => void; - resume: () => void; + resume?: () => void; setInterruptionBySpeaking?: (enabled: boolean) => void; - setMicrophoneMuted: (muted: boolean) => void; + setMicrophoneMuted?: (muted: boolean) => void; takeTurn?: () => Promise | void; }; diff --git a/libs/@hashintel/petrinaut/src/react/voice-session/types.ts b/libs/@hashintel/petrinaut/src/react/voice-session/types.ts index 78aeacbc2b6..6e2c32bce0d 100644 --- a/libs/@hashintel/petrinaut/src/react/voice-session/types.ts +++ b/libs/@hashintel/petrinaut/src/react/voice-session/types.ts @@ -1,5 +1,6 @@ -/** Which side of a Voice session currently holds the turn. */ +/** Session connectivity or, when known, which side holds the turn. */ export type PetrinautAiVoiceSessionPhase = + | "connected" | "connecting" | "error" | "listening" diff --git a/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts b/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts index 54c70070cd1..6df008f8e32 100644 --- a/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts +++ b/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts @@ -64,9 +64,9 @@ export type PetrinautAiVoiceModeControls = { /** Pauses microphone capture and active Voice output synchronously. */ pause: () => void; /** Re-establishes a session that dropped, keeping the conversation. */ - reconnect: () => void; + reconnect?: () => void; /** Resumes microphone capture after `pause`. */ - resume: () => void; + resume?: () => void; /** Replays the exact retained canonical assistant response when available. */ readFullResponse?: () => void; /** Replays only the exact question selected by the host's canonical marker. */ @@ -76,7 +76,7 @@ export type PetrinautAiVoiceModeControls = { * the assistant carries on speaking. Unlike `pause`, which suspends the * whole session when Petrinaut closes the panel. */ - setMicrophoneMuted: (muted: boolean) => void; + setMicrophoneMuted?: (muted: boolean) => void; /** Allows speech to interrupt assistant playback without clearing input. */ setInterruptionBySpeaking?: (enabled: boolean) => void; /** Cancels Voice output and hands the live microphone turn to the user. */ diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/components/voice-session-indicator.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/components/voice-session-indicator.tsx index 1994e3e7784..3856b4c2b36 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/components/voice-session-indicator.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/components/voice-session-indicator.tsx @@ -63,6 +63,8 @@ const phaseLevel: Record< PetrinautAiVoiceSessionPhase, (seconds: number, microphoneLevel: number) => number > = { + // Connectivity alone is not evidence that either side is speaking. + connected: () => 0, connecting: (seconds) => 0.1 + Math.sin(seconds * 2.2) * 0.04, error: () => 0, listening: (_seconds, microphoneLevel) => @@ -90,6 +92,7 @@ const ribbonStyle = cva({ }, variants: { phase: { + connected: { color: "blue.s90" }, connecting: { color: "neutral.s80" }, error: { color: "neutral.s80" }, listening: { color: "blue.s90" }, diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/components/voice-session-labels.ts b/libs/@hashintel/petrinaut/src/ui/views/Editor/components/voice-session-labels.ts index 7dd389c9e4e..ad18e8c43e5 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/components/voice-session-labels.ts +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/components/voice-session-labels.ts @@ -8,6 +8,8 @@ export const voiceSessionStatusLabel = ( phase: PetrinautAiVoiceSessionPhase, ): string => { switch (phase) { + case "connected": + return "Connected"; case "connecting": return "Connecting"; case "error": diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx index 81cd533b4bf..d122d3250d0 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx @@ -2469,9 +2469,6 @@ describe("AiAssistantPanel composer submissions", () => { registerVoiceModeControls({ end: async () => undefined, pause: vi.fn(), - reconnect: vi.fn(), - resume: vi.fn(), - setMicrophoneMuted: vi.fn(), }), [registerVoiceModeControls], ); @@ -2501,19 +2498,13 @@ describe("AiAssistantPanel composer submissions", () => { const rendered = renderTestPanel({ aiAssistant: aiAssistant(true) }); expect(screen.queryByRole("button", { name: "Your turn" })).toBeNull(); - fireEvent.click( - await screen.findByRole("button", { name: "Voice playback options" }), - ); + await screen.findByRole("region", { name: "Voice session" }); expect( - ( - await screen.findByRole("menuitem", { name: "Repeat question" }) - ).getAttribute("aria-disabled"), - ).toBe("true"); + screen.queryByRole("button", { name: "Voice playback options" }), + ).toBeNull(); expect( - screen - .getByRole("menuitem", { name: "Read full response" }) - .getAttribute("aria-disabled"), - ).toBe("true"); + screen.queryByRole("button", { name: "Mute microphone" }), + ).toBeNull(); rendered.rerenderPanel(aiAssistant(false), editorContextValue); diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx index 3f86c40d2f1..bd782c7ec01 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx @@ -677,18 +677,25 @@ const ConversationAiAssistantPanel = ({ ...(controls.readFullResponse ? { readFullResponse: () => controls.readFullResponse?.() } : {}), - reconnect: () => controls.reconnect(), + ...(controls.reconnect + ? { reconnect: () => controls.reconnect?.() } + : {}), ...(controls.repeatQuestion ? { repeatQuestion: () => controls.repeatQuestion?.() } : {}), - resume: () => controls.resume(), + ...(controls.resume ? { resume: () => controls.resume?.() } : {}), ...(controls.setInterruptionBySpeaking ? { setInterruptionBySpeaking: (enabled: boolean) => controls.setInterruptionBySpeaking?.(enabled), } : {}), - setMicrophoneMuted: (muted) => controls.setMicrophoneMuted(muted), + ...(controls.setMicrophoneMuted + ? { + setMicrophoneMuted: (muted: boolean) => + controls.setMicrophoneMuted?.(muted), + } + : {}), ...(controls.takeTurn ? { takeTurn: () => controls.takeTurn?.() } : {}), }); diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx index effd27f37c1..9a872087efc 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx @@ -24,6 +24,7 @@ import { VoiceSessionContext } from "../../../../../react/voice-session/context" import { createVoiceSessionStore } from "../../../../../react/voice-session/store"; import { definePetrinautAiInteractiveTool } from "../../../../types/ai-interactive-tool"; import { AiAssistantContents } from "./ai-assistant-contents"; +import { VoiceDock } from "./ai-assistant-contents/voice-dock"; import type { PetrinautAiMessage } from "./types"; @@ -80,6 +81,37 @@ const HostContent = ({ onMount }: { onMount: () => void }) => { return

Saved account

; }; +test("session-only dock shows Connected and End without unsupported controls", () => { + const end = vi.fn(); + const collapse = vi.fn(); + render( + } + microphoneMuted={false} + notice={null} + onCollapsedToggle={collapse} + phase="connected" + />, + ); + expect(screen.getByText("Connected")).toBeTruthy(); + expect( + screen.queryByRole("button", { name: "Voice playback options" }), + ).toBeNull(); + expect(screen.queryByRole("button", { name: "Mute microphone" })).toBeNull(); + expect(screen.queryByRole("button", { name: "Your turn" })).toBeNull(); + fireEvent.click( + screen.getByRole("button", { name: "Collapse voice session" }), + ); + expect(collapse).toHaveBeenCalledOnce(); + fireEvent.click(screen.getByRole("button", { name: "End voice mode" })); + expect(end).toHaveBeenCalledOnce(); +}); + describe("AiAssistantContents", () => { test("switches to host content without unmounting chat or losing its draft and Stop control", () => { const onStop = vi.fn(); diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/voice-dock.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/voice-dock.tsx index fa88dda5029..c3b04268ffc 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/voice-dock.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/voice-dock.tsx @@ -74,6 +74,7 @@ const statusStyle = cva({ }, variants: { phase: { + connected: { color: "blue.s90" }, connecting: { color: "neutral.s90" }, error: { color: "neutral.s100" }, listening: { color: "blue.s90" }, @@ -172,14 +173,17 @@ export const VoiceDock = ({ type="button" variant="ghost" /> - {actions !== null && ( - - )} + {actions !== null && + (actions.readFullResponse || + actions.repeatQuestion || + actions.setInterruptionBySpeaking) && ( + + )}
@@ -203,39 +207,45 @@ export const VoiceDock = ({ variant="ghost" /> )} - {phase === "error" ? ( -