From e07f8f0cddadd25f7db9f9a8f01ee2ca98af6c7a Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Mon, 14 Sep 2026 19:10:23 +0200 Subject: [PATCH 1/8] Integrate Live delivery with canonical Brunch turns Co-authored-by: Amp Co-authored-by: Cursor --- apps/petrinaut-website/MISSION.md | 101 +- apps/petrinaut-website/README.md | 80 +- .../api/voice/transcription-session.ts | 10 + .../local-storage-demo-app.test.tsx | 28 +- .../local-storage-demo-app.tsx | 12 +- .../live-brunch-bridge.test.ts | 906 ++++++++++++ .../app/voice-interview/live-brunch-bridge.ts | 522 +++++++ .../live-conversation-control.test.tsx | 441 +++++- .../live-conversation-control.tsx | 231 ++- .../voice-interview/live-conversation.test.ts | 1267 ++++++++++++----- .../app/voice-interview/live-conversation.ts | 820 ++++++++--- .../voice-interview/realtime-brunch-bridge.ts | 2 +- .../voice-interview/shared/live-diagnostic.ts | 14 + .../voice-browser-tools.integration.test.tsx | 9 +- .../voice-interview-control.tsx | 27 +- .../voice-interview-disclosure.tsx | 19 +- .../server/voice/openai-live-session.test.ts | 5 +- .../src/server/voice/openai-live-session.ts | 58 +- .../openai-transcription-session.test.ts | 586 ++++++++ .../voice/openai-transcription-session.ts | 311 ++++ apps/petrinaut-website/vite.config.ts | 1 + libs/@hashintel/brunch-agent/MISSION.md | 565 +++++--- libs/@hashintel/brunch-agent/MISSION.next.md | 31 +- .../@hashintel/petrinaut/docs/ai-assistant.md | 34 +- 24 files changed, 5069 insertions(+), 1011 deletions(-) create mode 100644 apps/petrinaut-website/api/voice/transcription-session.ts create mode 100644 apps/petrinaut-website/src/main/app/voice-interview/live-brunch-bridge.test.ts create mode 100644 apps/petrinaut-website/src/main/app/voice-interview/live-brunch-bridge.ts create mode 100644 apps/petrinaut-website/src/main/app/voice-interview/shared/live-diagnostic.ts create mode 100644 apps/petrinaut-website/src/server/voice/openai-transcription-session.test.ts create mode 100644 apps/petrinaut-website/src/server/voice/openai-transcription-session.ts diff --git a/apps/petrinaut-website/MISSION.md b/apps/petrinaut-website/MISSION.md index 69347722a67..637e72532d4 100644 --- a/apps/petrinaut-website/MISSION.md +++ b/apps/petrinaut-website/MISSION.md @@ -1,92 +1,15 @@ -# Experiment Live Full-Duplex Migration +# Experiment Live Full Brunch Integration -## Status +The child branch's sole execution authority is the +[Brunch mission](../../libs/@hashintel/brunch-agent/MISSION.md). +This file is a pointer, not a second mission. -Live experiment scope; local implementation prepared for Kostandin's manual -witness, not conversational-quality acceptance. -[FE-1663](https://linear.app/hash/issue/FE-1663/experiment-live-full-duplex-migration) -is related to FE-1661, which remains the migration-effort assessment. -This website experiment uses Petrinaut's existing Voice dock, with optional -session controls and local audio-activity indicators, not inferred turn boundaries. -It does not recut Brunch's existing Mission 7a or promote its future-planning drafts. +The accepted experiment uses native Live with best-effort speech instructions and +separate authoritative transcription. Brunch retains canonical answers and tool +authority; settlement gates Brunch context sent to Live, not all audible speech. +Independent questions and unsupported claims are manual-test failures. -## Imperative - -Learn whether GPT-Live-1 natively makes process interviewing feel fluid: -concise relevant follow-ups, room for hesitation and elaboration, and natural -interruption/correction without losing meaning. Smooth audio alone is not success. - -## Throughline - -Existing website Voice entry → explicit experimental consent/Start → trusted -website `/api/voice/live-session` → client-delegated GPT-Live-1 WebRTC → microphone -and speaker → existing Voice dock → local End/Exit. Local disconnection is -not a remote-closure claim. -`main → FE-1663 → Experiment Live Full Brunch Integration` is the intended stack; -only FE-1663 is authorized now. No relay, rephrasing, or harness code is a donor. - -## Proof - -- `openai-live-session.test.ts` checks provider/default selection, enablement, - trusted session payload, origin/content validation, credential privacy, and - no retry. Existing Realtime policy and transport tests remain regression oracles. -- `live-conversation.test.ts` checks waiting for `session.started`, no duplicate - start, no transcript/delegation execution, remote audio attachment, connection - failures, late microphone permission, and capture/playback/transport cleanup. -- `live-conversation-control.test.tsx` checks explicit consent/Start, provider - pinning, canonical isolation, host state reporting, panel closure and stale callbacks. -- Petrinaut's assistant-panel and contents tests check optional action visibility - and retain regression coverage for Realtime's playback and lifecycle controls. -- Rendered real-editor consent, simulated error, and mocked active controls - must be inspected without a real microphone or provider session. -- Kostandin's [10–15 minute manual procedure](README.md#manual-test--1015-minutes) - is the oracle for conversational quality and actual media behavior. Provider-free - tests and screenshots do not establish these or end-to-end Brunch compatibility. - -## Constraints - -Unset `PETRINAUT_VOICE_PROVIDER` means `realtime`; only `realtime` and `live` are -valid. Keep existing enablement/credential plumbing. Pin configuration per -mounted conversation; never switch providers or resubmit input automatically. -Keep Realtime's prompts, admission, queue, settlement and playback unchanged. - -Live has no access to canonical submission, Brunch/Petrinaut tools or chat -history. Its guidance does not grant domain authority. Transcript deltas are -not finalized utterances; delegation events are metadata; commentary (unused -here) is paraphrasable context limited to 500 tokens, and append acknowledgements -are not speech/playback completion. Do not invent response terminals or infer -authoritative completion from silence. Stop ends local media, not canonical work. - -No paid sessions, synthetic recordings, audio evaluation harness, deployment, -Notion writes, second issue, or integration implementation are authorized. -The follow-up permits cleanup, a clean commit without Amp thread IDs, and a draft PR. -Preserve other worktrees and uncommitted work. - -## Fog-line - -Native finalization and enforceable output gating remain unresolved for PR 2. -Observe whether interruption preserves corrections, whether the user feels -heard, whether questions advance process elicitation, whether pauses/one-word -answers survive, and whether output is concise. Observe transport failures and -remote closure separately. A pleasant standalone conversation cannot answer -whether canonical admission/settlement delays retain this quality. - -## Stop or reorient - -Stop at an authority change: autonomous domain speech, local substantive -follow-ups in the integrated workflow, hybrid authoritative transcription, or -buffered/gated output requires an explicit recut for Lu's approval. Neither -this prompt nor success here reopens FE-1624. Provider-free verification cannot -be replaced with paid calls or a synthetic audio campaign. - -## Deferred - -PR 2 must preserve: finalized input → existing composer → Flue → Brunch → -authorized Petrinaut execution → complete-turn settlement → frozen canonical -reply → Live delivery. Brunch alone authors substantive domain answers and -questions; Flue is canonical conversation; Petrinaut is execution authority. -Proposal approval, full-turn settlement and playback eligibility remain distinct. -Unknown execution outcomes must not be reapplied or spoken as success. Workpiece, -basis and tool payloads are not assistant prose. See the existing Brunch -[future spine](../../libs/@hashintel/brunch-agent/MISSION.next.md#voice-after-the-live-transport-cut) -for retained context, not implementation authority. +Keep `PETRINAUT_VOICE_PROVIDER=realtime|live`, with unset meaning `realtime`. +The Brunch mission records current proof, risks and acceptance blockers. No +agent-started paid sessions, automatic microphone access, merge or deployment are +authorized; Kostandin performs live testing manually. diff --git a/apps/petrinaut-website/README.md b/apps/petrinaut-website/README.md index d4cac073c99..fe6049904b3 100644 --- a/apps/petrinaut-website/README.md +++ b/apps/petrinaut-website/README.md @@ -146,15 +146,14 @@ from jsDelivr and Optuna from PyPI; later runs use the browser cache. Local values live in `.env.local`; Vite's `loadEnv` (see [`vite.config.ts`](vite.config.ts)) copies them into `process.env` for both the dev server and the API functions. In production, set these in the Vercel project settings. -### Experimental Live interview (FE-1663) +### Experimental Brunch-backed Live interview (FE-1664) -`PETRINAUT_VOICE_PROVIDER=live` selects a standalone GPT-Live-1 conversation -inside the existing Voice entry. **It is not Brunch output.** It cannot see or -submit chat, execute tools, or change the model/workpiece. No experimental -transcripts are displayed or saved. The server uses client delegation, not -managed Responses; transcript deltas and delegation metadata are ignored. -The short process-interview prompt is conversational guidance, not a domain -system or a guarantee of model compliance. +`PETRINAUT_VOICE_PROVIDER=live` uses GPT-Live-1 for conversational audio while a +separate `gpt-4o-transcribe` session supplies finalized user text to Brunch. +Brunch remains the canonical conversation, domain and tool authority. Settled +Brunch prose is offered to Live as delegation-correlated commentary; Live has no +tools and must not answer domain questions independently. These instructions are +best effort, not an enforced speech boundary. From the repository root, with `OPENAI_VOICE_API_KEY` already exported (or in this worktree's `apps/petrinaut-website/.env.local`): @@ -165,24 +164,27 @@ turbo run build --filter '@apps/brunch-agent^...' --filter '@apps/petrinaut-webs yarn workspace @apps/petrinaut-website codegen yarn workspace @apps/petrinaut-website examples:generate -# Standalone Live needs only the existing panel launcher, not a running Brunch server. -PETRINAUT_OPENAI_VOICE_ENABLED=true PETRINAUT_VOICE_PROVIDER=live yarn dev:brunch:panel +# Live input and answers use the existing Brunch route. +PETRINAUT_OPENAI_VOICE_ENABLED=true PETRINAUT_VOICE_PROVIDER=live yarn dev:brunch ``` Open [http://localhost:4915/new](http://localhost:4915/new), dismiss the tour if shown, open the AI panel, and select the waveform **Start voice mode** action in the empty composer. -Read the experimental label, check consent, then choose **Start voice**. +Read the short audio-processing disclosure, allow microphone audio for voice and +transcription, then choose **Start voice**. **Cancel** returns to text without starting a session. Only that last action requests microphone access and a billable Live session. Use headphones for the first trial. HTTPS or localhost and an OpenAI project with GPT-Live-1 access are required. -Once connected, the existing Voice dock replaces the composer with -**Listening** or **Speaking**, collapse/expand and **End voice mode**. -The experimental label appears only during consent. Local WebRTC audio levels +Once connected, the Voice dock replaces the composer. It shows **Thinking** while +a Brunch request is submitted or streaming, **Speaking** during active playback, +and **Listening** when both are idle. Connection and error states take precedence. +Thinking is a local work indicator, not a spoken progress update. +The consent panel uses a plain voice-permission heading. Local WebRTC audio levels drive the microphone ribbon and Speaking indicator; Listening means the session is open for input, including while output is active. These are activity indicators, not authoritative turn boundaries or proof of heard playback. Browsers without -audio-level telemetry retain Listening without an animated input level. +audio-level telemetry omit the animated input level; Brunch work still shows Thinking. There is no separate experiment panel, replay menu, or microphone toggle. Connection errors return to setup; starting again requires fresh consent. Brief WebRTC interruptions show **Connecting** while the existing session has up @@ -190,13 +192,10 @@ to the connection timeout (15 seconds by default) to recover. Media stays open; no new session is created and no input is replayed. End still stops both directions immediately. A failed connection or an expired recovery deadline ends the session. -**End voice mode**, **Exit experiment** during setup, closing the panel, switching to text, -changing conversation, and leaving the page stop local experimental capture -and playback. Stop requests `session.close` only after `session.started` and -waits up to two seconds for `session.closed` before releasing the transport. -Local silence is not proof of remote closure or final usage. Connection failures are not retried. -Starting again creates a new session with fresh consent; there is no resume, -replay, "Your turn", or fabricated Realtime terminal lifecycle. +**End voice mode**, **Cancel** during setup, closing the panel, switching to text, +changing conversation, and leaving the page stop both Voice transports and local +capture. Starting again creates fresh Live and transcription sessions; there is +no resume, replay, automatic retry, or input resubmission. To return to the **unchanged integrated Realtime path**, Exit, stop the panel dev command with Ctrl-C, configure the existing local Brunch environment, and run: @@ -215,28 +214,21 @@ the website's generic Turbo `dev` task does not forward arbitrary shell variable #### Manual test — 10–15 minutes -1. **3 minutes:** explain a familiar process. Let Live ask relevant follow-ups. -2. **2 minutes:** hesitate, pause, answer with one word, then elaborate. Note - whether it leaves room and follows the meaning rather than guessing a turn end. -3. **2 minutes:** interrupt mid-sentence and correct an earlier detail. Note - whether the correction is retained and the interview advances. -4. **2 minutes:** speak while Live responds. Listen for lost words, overlap, - unwanted acknowledgements, and long monologues. A button click is not an - acoustic-interruption test. -5. **2–4 minutes:** End voice mode and check the browser microphone indicator and actual - speaker silence. Change to Realtime using the commands above, reload, - and begin a fresh session. Confirm the experiment added no canonical chat - messages or model changes. - -Record **feeling heard and advancing the interview** separately from transport -correctness. Note browser/headset, representative pauses/corrections, and which -responses were excessive or useful. Prior relay/rephrasing/harness evidence is -not a passing comparative baseline. This experiment has no audio harness or -synthetic recordings. - -The intended successor is **Experiment Live Full Brunch Integration**, stacked -on this branch; it is not implemented here. See [MISSION.md](MISSION.md) for -the authority boundary and unresolved finalization/output-control questions. +1. Explain a familiar process and answer one Brunch follow-up. +2. Hesitate, elaborate, and correct a consequential detail. Confirm each retained + finalized utterance appears once in canonical history and later questions use + the correction. +3. Request one available model operation. Compare the spoken result with settled + Brunch text and inspect the actual workpiece/model effect. +4. Speak while Live responds and interrupt it acoustically. Record lost input, + overlap, unsupported acknowledgements, independent questions, or unsupported + completion claims separately from canonical Brunch behavior. +5. End Voice mode and confirm microphone capture and speaker playback stop. + +Use headphones while the known phantom-input risk is investigated. Server VAD can +split hesitation into multiple finalized items, and the existing one-waiting-input +policy may not retain all of them. See [MISSION.md](MISSION.md) for the current +acceptance limits and manual proof obligations. The existing unauthenticated Voice endpoint risk below also applies to Live; do not expose this local experiment publicly without addressing that boundary. diff --git a/apps/petrinaut-website/api/voice/transcription-session.ts b/apps/petrinaut-website/api/voice/transcription-session.ts new file mode 100644 index 00000000000..fc5a2f59612 --- /dev/null +++ b/apps/petrinaut-website/api/voice/transcription-session.ts @@ -0,0 +1,10 @@ +import { createOpenAITranscriptionSessionHandler } from "../../src/server/voice/openai-transcription-session.js"; + +declare const process: { env: Record }; + +export default { + fetch: createOpenAITranscriptionSessionHandler({ + environment: process.env, + fetch: globalThis.fetch.bind(globalThis), + }), +}; diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx index 1fee18594b9..07dbcbb31f0 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx @@ -334,10 +334,20 @@ describe("local storage demo Brunch voice integration", () => { expect(getBrunchVoiceMode(null)).toBeUndefined(); }); - test("installs the app-owned voice control for a configured Brunch transport", () => { + test("installs the app-owned voice control for a configured Brunch transport", async () => { const config = { available: true as const, connectionTimeoutMs: 15_000 }; const tracker = new BrunchPanelConversationTracker(); - const voiceMode = getBrunchVoiceMode(config, tracker); + const snapshot = { + conversationId: "petrinaut-preview:net-1", + messages: [], + settlements: [], + }; + const voiceMode = getBrunchVoiceMode( + config, + tracker, + snapshot.settlements, + snapshot, + ); const renderControl = () => voiceMode?.({ canAcceptVoiceInput: true, @@ -366,6 +376,20 @@ describe("local storage demo Brunch voice integration", () => { if (!isValidElement(control)) { throw new Error("Expected the configured composer control to render."); } + expect(control.props).toHaveProperty("snapshot", snapshot); + let finishSubmission = () => {}; + const pending = tracker.trackSubmission( + new Promise((resolve) => { + finishSubmission = resolve; + }), + ); + const whilePending = renderControl(); + expect(isValidElement(whilePending) && whilePending.props).toHaveProperty( + "snapshot", + snapshot, + ); + finishSubmission(); + await pending; const failureListener = vi.fn(); const responseCompletedListener = vi.fn(); const responseStartedListener = vi.fn(); diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx index dbe4b541fc6..91fa2625398 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx @@ -3,7 +3,11 @@ * @role Editable demo shell: nets in local storage, one live document handle */ -import { createFlueClient, type FlueConversationSettlement } from "@flue/sdk"; +import { + createFlueClient, + type FlueConversationSettlement, + type FlueConversationState, +} from "@flue/sdk"; import { use, useCallback, @@ -134,6 +138,7 @@ export const getBrunchVoiceMode = ( config: OpenAIVoiceConfig | null | undefined, tracker?: BrunchPanelConversationTracker, settlements?: readonly FlueConversationSettlement[], + snapshot?: FlueConversationState, ): PetrinautAiVoiceMode | undefined => { if (!config) return undefined; @@ -161,6 +166,9 @@ export const getBrunchVoiceMode = ( {...context} config={config} settlements={settlements} + // Voice only observes this snapshot. Message replacement remains gated + // independently by followMessages.canReplace below. + snapshot={snapshot} resolveInputSubmission={resolveInputSubmission} resolveResponseSubmission={resolveResponseSubmission} subscribeToResponseMessageCompleted={subscribeToResponseMessageCompleted} @@ -787,11 +795,13 @@ export const LocalStorageDemoApp = ({ brunchSelected ? openAIVoiceConfig : null, conversationTracker, flueHistory.settlements, + flueHistory.snapshot, ), [ brunchSelected, conversationTracker, flueHistory.settlements, + flueHistory.snapshot, openAIVoiceConfig, ], ); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/live-brunch-bridge.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/live-brunch-bridge.test.ts new file mode 100644 index 00000000000..02ebb572981 --- /dev/null +++ b/apps/petrinaut-website/src/main/app/voice-interview/live-brunch-bridge.test.ts @@ -0,0 +1,906 @@ +import { afterEach, beforeEach, expect, test, vi } from "vitest"; + +import { LiveBrunchBridge } from "./live-brunch-bridge"; + +import type { FlueConversationState } from "@flue/sdk"; + +beforeEach(() => { + vi.spyOn(console, "debug").mockImplementation(() => {}); +}); + +afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllEnvs(); +}); + +const setup = () => { + const appendCommentary = vi.fn< + ConstructorParameters[0]["appendCommentary"] + >(() => true); + const appendInstructions = vi.fn< + ConstructorParameters[0]["appendInstructions"] + >(() => true); + const notice = vi.fn(); + const submit = vi.fn(async (input: { onAdmission: (id: string) => void }) => { + input.onAdmission("root"); + return { + kind: "message" as const, + messageId: "user", + submissionId: "root", + }; + }); + const bridge = new LiveBrunchBridge({ + appendCommentary, + appendInstructions, + notice, + submit, + }); + const update = ( + overrides: Partial[0]> = {}, + ) => + bridge.update({ + status: "ready", + canAcceptVoiceInput: true, + segments: [], + settlements: [], + ...overrides, + }); + update(); + return { + bridge, + appendCommentary, + appendInstructions, + notice, + submit, + update, + }; +}; + +const segment = ( + text = "There are 7 reviewers, not 4. Is approval optional?", +) => ({ + id: text, + contentHash: text, + messageId: "answer", + partId: "text:0", + source: "assistant-text" as const, + text, + submissionIds: ["root"], +}); +const completed = [{ submissionId: "root", outcome: "completed" as const }]; +const started = { + messageId: "answer", + submissionId: "root", + position: { batch: 1, index: 0 }, +}; + +test("trace distinguishes ungated admission, later delegation matching, settlement and dropped speech", async () => { + vi.stubEnv("DEV", true); + const debug = vi.spyOn(console, "debug").mockImplementation(() => {}); + const fixture = setup(); + await fixture.bridge.accept({ id: "first-input", text: "PRIVATE INPUT" }); + fixture.bridge.acceptDelegation("late-delegation"); + fixture.bridge.responseStarted(started); + fixture.update({ status: "streaming" }); + fixture.bridge.responseCompleted({ + ...started, + position: { batch: 2, index: 0 }, + }); + fixture.update({ + segments: [segment("PRIVATE ANSWER")], + settlements: completed, + }); + fixture.update({ canAcceptVoiceInput: false }); + await fixture.bridge.accept({ id: "dropped-input", text: "PRIVATE DROPPED" }); + const records = debug.mock.calls.map( + ([line]) => + JSON.parse(String(line).replace("[Petrinaut Live trace] ", "")) as Record< + string, + unknown + >, + ); + expect(records).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + event: "brunch.submit", + inputId: "first-input", + delegationId: null, + }), + expect.objectContaining({ + event: "brunch.admitted", + inputId: "first-input", + submissionId: "root", + }), + expect.objectContaining({ + event: "delegation.matched", + inputId: "first-input", + delegationId: "late-delegation", + }), + expect.objectContaining({ + event: "brunch.offer", + inputId: "first-input", + submissionId: "root", + delegationId: "late-delegation", + }), + expect.objectContaining({ + event: "input.dropped", + inputId: "dropped-input", + admissionUnavailable: true, + waitingForComposer: false, + oversized: false, + }), + ]), + ); + expect(JSON.stringify(records)).not.toContain("PRIVATE"); + expect(fixture.submit).toHaveBeenCalledOnce(); + expect(fixture.appendCommentary).toHaveBeenCalledExactlyOnceWith( + "PRIVATE ANSWER", + "late-delegation", + ); +}); + +test("offers one frozen correlated answer only after complete settlement, never streamed or historical prose", async () => { + const fixture = setup(); + fixture.update({ segments: [segment("Old answer")] }); + await fixture.bridge.accept({ id: "one", text: "Seven, not four" }); + fixture.bridge.responseStarted(started); + fixture.update({ status: "streaming", segments: [segment()] }); + fixture.bridge.responseCompleted({ + ...started, + position: { batch: 2, index: 0 }, + }); + fixture.update({ + status: "streaming", + segments: [segment()], + settlements: completed, + }); + expect(fixture.appendCommentary).not.toHaveBeenCalled(); + fixture.update({ segments: [segment()], settlements: [] }); + expect(fixture.appendCommentary).not.toHaveBeenCalled(); + fixture.update({ segments: [segment()], settlements: completed }); + fixture.update({ segments: [segment()], settlements: completed }); + expect(fixture.appendCommentary).toHaveBeenCalledExactlyOnceWith( + segment().text, + null, + ); +}); + +test("waits for observed prose rendered after ready settlement", async () => { + const fixture = setup(); + await fixture.bridge.accept({ id: "one", text: "Seven, not four" }); + fixture.bridge.responseStarted(started); + fixture.update({ status: "streaming" }); + fixture.bridge.responseCompleted({ + ...started, + position: { batch: 2, index: 0 }, + }); + const snapshot: FlueConversationState = { + conversationId: "conversation", + settlements: completed, + messages: [ + { + id: "answer", + role: "assistant", + purpose: "assistant", + display: "visible", + submissionId: "root", + parts: [{ type: "text", state: "done", text: segment().text }], + }, + ], + }; + + fixture.update({ settlements: completed, snapshot }); + expect(fixture.appendCommentary).not.toHaveBeenCalled(); + expect(fixture.notice).not.toHaveBeenCalledWith( + expect.stringContaining("without a spoken answer"), + ); + + fixture.update({ segments: [segment()], settlements: completed, snapshot }); + expect(fixture.appendCommentary).toHaveBeenCalledExactlyOnceWith( + segment().text, + null, + ); +}); + +test("a finalized snapshot confirms an observed response is textless", async () => { + const fixture = setup(); + await fixture.bridge.accept({ id: "one", text: "Seven, not four" }); + fixture.bridge.responseStarted(started); + fixture.update({ status: "streaming" }); + fixture.bridge.responseCompleted({ + ...started, + position: { batch: 2, index: 0 }, + }); + + fixture.update({ + settlements: completed, + snapshot: { + conversationId: "conversation", + settlements: completed, + messages: [ + { + id: "answer", + role: "assistant", + purpose: "assistant", + display: "visible", + submissionId: "root", + parts: [], + }, + ], + }, + }); + + expect(fixture.notice).toHaveBeenLastCalledWith( + "Brunch settled without a spoken answer. Check the conversation.", + ); + expect(fixture.appendCommentary).not.toHaveBeenCalled(); +}); + +test("completion retries settlement against already rendered prose", async () => { + const fixture = setup(); + await fixture.bridge.accept({ id: "one", text: "Seven, not four" }); + fixture.bridge.responseStarted(started); + fixture.update({ status: "streaming" }); + fixture.update({ segments: [segment()], settlements: completed }); + expect(fixture.appendCommentary).not.toHaveBeenCalled(); + + fixture.bridge.responseCompleted({ + ...started, + position: { batch: 2, index: 0 }, + }); + expect(fixture.appendCommentary).toHaveBeenCalledExactlyOnceWith( + segment().text, + null, + ); +}); + +test("a stale completion cannot finish a newer response and a duplicate start cannot reopen it", async () => { + const fixture = setup(); + await fixture.bridge.accept({ id: "one", text: "Seven, not four" }); + fixture.update({ status: "streaming" }); + fixture.bridge.responseStarted(started); + const newer = { ...started, position: { batch: 3, index: 2 } }; + fixture.bridge.responseStarted(newer); + fixture.bridge.responseCompleted({ + ...started, + position: { batch: 3, index: 1 }, + }); + fixture.update({ segments: [segment()], settlements: completed }); + expect(fixture.appendCommentary).not.toHaveBeenCalled(); + fixture.bridge.responseCompleted({ + ...started, + position: { batch: 3, index: 3 }, + }); + fixture.bridge.responseStarted(newer); + fixture.update({ segments: [segment()], settlements: completed }); + expect(fixture.appendCommentary).toHaveBeenCalledExactlyOnceWith( + segment().text, + null, + ); +}); + +test("Stop aborts pending admission and its late resolution cannot produce commentary or a new notice", async () => { + const fixture = setup(); + let release = () => {}; + let signal: AbortSignal | undefined; + const bridge = new LiveBrunchBridge({ + appendCommentary: fixture.appendCommentary, + appendInstructions: fixture.appendInstructions, + notice: fixture.notice, + submit: async (input) => { + signal = input.signal; + await new Promise((resolve) => { + release = resolve; + }); + input.onAdmission("root"); + return { kind: "message", messageId: "user", submissionId: "root" }; + }, + }); + bridge.update({ + status: "ready", + canAcceptVoiceInput: true, + segments: [], + settlements: [], + }); + const pending = bridge.accept({ id: "one", text: "First" }); + bridge.stop(); + expect(signal?.aborted).toBe(true); + fixture.notice.mockClear(); + release(); + await pending; + expect(fixture.notice).not.toHaveBeenCalled(); + expect(fixture.appendCommentary).not.toHaveBeenCalled(); +}); + +test.each(["failed", "aborted", "completed"] as const)( + "textless continuation settlement %s cannot be skipped", + async (outcome) => { + const fixture = setup(); + await fixture.bridge.accept({ id: "one", text: "Please explain" }); + fixture.bridge.responseStarted(started); + fixture.bridge.responseCompleted({ + ...started, + position: { batch: 2, index: 0 }, + }); + fixture.update({ + status: "streaming", + segments: [segment()], + settlements: completed, + }); + const continuation = { + ...started, + submissionId: "continuation", + position: { batch: 3, index: 0 }, + }; + fixture.bridge.responseStarted(continuation); + fixture.bridge.responseCompleted({ + ...continuation, + position: { batch: 4, index: 0 }, + }); + fixture.update({ segments: [segment()], settlements: completed }); + expect(fixture.appendCommentary).not.toHaveBeenCalled(); + fixture.update({ + segments: [segment()], + settlements: [...completed, { submissionId: "continuation", outcome }], + }); + expect(fixture.appendCommentary).toHaveBeenCalledTimes( + outcome === "completed" ? 1 : 0, + ); + }, +); + +test("duplicates, empty input and one waiting composer submission never create another admission", async () => { + const fixture = setup(); + let release = () => {}; + fixture.submit.mockImplementationOnce(async () => { + await new Promise((resolve) => { + release = resolve; + }); + return { kind: "message", messageId: "user", submissionId: "root" }; + }); + const pending = fixture.bridge.accept({ id: "one", text: "First" }); + await fixture.bridge.accept({ id: "one", text: "First" }); + await fixture.bridge.accept({ id: "empty", text: " " }); + await fixture.bridge.accept({ id: "two", text: "Follow-up" }); + expect(fixture.submit).toHaveBeenCalledOnce(); + expect(fixture.notice).toHaveBeenLastCalledWith( + expect.stringContaining("not retained"), + ); + release(); + await pending; + await fixture.bridge.accept({ id: "three", text: "First" }); + expect(fixture.submit).toHaveBeenCalledTimes(2); +}); + +test("uncertain admission is visible and never automatically replayed", async () => { + const fixture = setup(); + fixture.submit.mockRejectedValueOnce(new Error("Unknown admission")); + await fixture.bridge.accept({ id: "one", text: "First" }); + await fixture.bridge.accept({ id: "one", text: "First" }); + expect(fixture.submit).toHaveBeenCalledOnce(); + expect(fixture.notice).toHaveBeenLastCalledWith( + expect.stringContaining("Check canonical history"), + ); +}); + +test("a response failure after confirmed admission does not report uncertain admission or replay input", async () => { + const fixture = setup(); + fixture.submit.mockImplementationOnce(async (input) => { + input.onAdmission("root"); + throw new Error("401 invalid x-api-key"); + }); + await fixture.bridge.accept({ id: "one", text: "First" }); + await fixture.bridge.accept({ id: "one", text: "First" }); + expect(fixture.submit).toHaveBeenCalledOnce(); + expect(fixture.appendCommentary).not.toHaveBeenCalled(); + expect(fixture.notice).toHaveBeenLastCalledWith( + "Your message was admitted, but its response could not be confirmed. Check canonical history; no automatic retry was made.", + ); +}); + +test("admission frees the existing waiting-input slot, but finishing an earlier turn cannot free a newer wait", async () => { + const fixture = setup(); + let finishFirst = () => {}; + let admitCorrection = () => {}; + let finishCorrection = () => {}; + fixture.submit.mockImplementationOnce(async (input) => { + input.onAdmission("first"); + await new Promise((resolve) => { + finishFirst = resolve; + }); + return { kind: "message", messageId: "one", submissionId: "first" }; + }); + fixture.submit.mockImplementationOnce(async (input) => { + admitCorrection = () => input.onAdmission("correction"); + await new Promise((resolve) => { + finishCorrection = resolve; + }); + return { kind: "message", messageId: "two", submissionId: "correction" }; + }); + const first = fixture.bridge.accept({ id: "one", text: "Four reviewers" }); + fixture.update({ status: "streaming", canAcceptVoiceInput: true }); + const correction = fixture.bridge.accept({ + id: "two", + text: "Seven, not four", + }); + expect(fixture.submit).toHaveBeenCalledTimes(2); + + finishFirst(); + await first; + await fixture.bridge.accept({ id: "three", text: "Another pending input" }); + expect(fixture.submit).toHaveBeenCalledTimes(2); + expect(fixture.notice).toHaveBeenLastCalledWith( + expect.stringContaining("not retained"), + ); + + admitCorrection(); + await fixture.bridge.accept({ id: "four", text: "Approval is optional" }); + expect(fixture.submit).toHaveBeenCalledTimes(3); + finishCorrection(); + await correction; +}); + +test.each(["completed", "failed", "aborted"] as const)( + "follows the answering submission and waits for its %s settlement and textless continuation", + async (outcome) => { + const fixture = setup(); + await fixture.bridge.accept({ id: "one", text: "Explain the result" }); + fixture.update({ status: "streaming" }); + const answer = { ...started, submissionId: "answering" }; + fixture.bridge.responseStarted(answer); + fixture.bridge.responseCompleted({ + ...answer, + position: { batch: 2, index: 0 }, + }); + const continuation = { + ...answer, + submissionId: "continuation", + position: { batch: 3, index: 0 }, + }; + fixture.bridge.responseStarted(continuation); + fixture.bridge.responseCompleted({ + ...continuation, + position: { batch: 4, index: 0 }, + }); + const segments = [ + { ...segment(), submissionIds: ["answering", "continuation"] }, + ]; + const rootSettlement = { + submissionId: "root", + outcome: "completed" as const, + answeredBySubmissionId: "answering", + }; + fixture.update({ segments, settlements: [rootSettlement] }); + expect(fixture.appendCommentary).not.toHaveBeenCalled(); + expect(fixture.notice).not.toHaveBeenCalledWith( + expect.stringContaining("without a spoken answer"), + ); + fixture.update({ + segments, + settlements: [ + rootSettlement, + { submissionId: "answering", outcome: "completed" }, + ], + }); + expect(fixture.appendCommentary).not.toHaveBeenCalled(); + const settlements = [ + rootSettlement, + { submissionId: "answering", outcome: "completed" as const }, + { submissionId: "continuation", outcome }, + ]; + fixture.update({ segments, settlements }); + fixture.update({ segments, settlements }); + expect(fixture.appendCommentary.mock.calls).toEqual( + outcome === "completed" ? [[segment().text, null]] : [], + ); + }, +); + +test("two inputs answered by one submission offer its canonical prose only once", async () => { + const fixture = setup(); + fixture.bridge.acceptDelegation("first-delegation"); + await fixture.bridge.accept({ id: "one", text: "First question" }); + fixture.submit.mockImplementationOnce(async (input) => { + input.onAdmission("second"); + return { kind: "message", messageId: "two", submissionId: "second" }; + }); + fixture.bridge.acceptDelegation("second-delegation"); + await fixture.bridge.accept({ id: "two", text: "Clarification" }); + fixture.update({ status: "streaming" }); + const answer = { ...started, submissionId: "answering" }; + fixture.bridge.responseStarted(answer); + fixture.bridge.responseCompleted({ + ...answer, + position: { batch: 2, index: 0 }, + }); + fixture.update({ + segments: [{ ...segment(), submissionIds: ["answering"] }], + settlements: [ + { + submissionId: "root", + outcome: "completed", + answeredBySubmissionId: "answering", + }, + { + submissionId: "second", + outcome: "completed", + answeredBySubmissionId: "answering", + }, + { submissionId: "answering", outcome: "completed" }, + ], + }); + expect(fixture.appendCommentary).toHaveBeenCalledExactlyOnceWith( + segment().text, + "first-delegation", + ); + expect(fixture.notice).not.toHaveBeenCalledWith( + expect.stringContaining("without a spoken answer"), + ); +}); + +test.each([ + "deliver", + "stop", + "historical", + "historical-render", + "failed", + "aborted", +] as const)( + "snapshot-only answered-by source: %s, never a stale or partial snapshot", + async (mode) => { + const fixture = setup(); + const prose = segment(); + const settlements = [ + { + submissionId: "root", + outcome: "completed" as const, + answeredBySubmissionId: "answering", + }, + { submissionId: "answering", outcome: "completed" as const }, + ]; + const snapshot: FlueConversationState = { + conversationId: "conversation", + settlements, + messages: [ + { + id: "answer", + role: "assistant", + purpose: "assistant", + display: "visible", + submissionId: "answering", + parts: [{ type: "text", state: "done", text: prose.text }], + }, + ], + }; + const segments = [{ ...prose, submissionIds: undefined }]; + if (mode === "historical") fixture.update({ segments, snapshot }); + if (mode === "historical-render") fixture.update({ segments }); + await fixture.bridge.accept({ id: "input", text: "Seven, not four" }); + fixture.update({ status: "streaming" }); + // Settlement can arrive before the snapshot/render. Do not discard the turn. + fixture.update({ settlements }); + fixture.update({ + settlements, + segments, + snapshot: { ...snapshot, settlements: [settlements[0]!] }, + }); + fixture.update({ + settlements, + segments, + snapshot: { + ...snapshot, + messages: [{ ...snapshot.messages[0]!, submissionId: "root" }], + }, + }); + expect(fixture.appendCommentary).not.toHaveBeenCalled(); + fixture.update({ + settlements, + segments, + snapshot: { + ...snapshot, + messages: [ + { + ...snapshot.messages[0]!, + parts: [{ type: "text", state: "streaming", text: prose.text }], + }, + ], + }, + }); + fixture.update({ settlements, segments: [], snapshot }); + expect(fixture.appendCommentary).not.toHaveBeenCalled(); + if (mode === "stop") fixture.bridge.stop(); + if (mode === "failed" || mode === "aborted") { + fixture.update({ + segments, + snapshot, + settlements: [ + settlements[0]!, + { submissionId: "answering", outcome: mode }, + ], + }); + } + fixture.update({ settlements, segments, snapshot }); + fixture.update({ settlements, segments, snapshot }); + expect(fixture.appendCommentary.mock.calls).toEqual( + mode === "deliver" ? [[prose.text, null]] : [], + ); + }, +); + +test("a finalized textless unobserved answer cannot fall through to an observed response", async () => { + const fixture = setup(); + fixture.bridge.acceptDelegation("delegation"); + await fixture.bridge.accept({ id: "input", text: "Explain the result" }); + fixture.update({ status: "streaming" }); + fixture.bridge.responseStarted(started); + fixture.bridge.responseCompleted({ + ...started, + position: { batch: 2, index: 0 }, + }); + const settlements = [ + { + submissionId: "root", + outcome: "completed" as const, + answeredBySubmissionId: "answering", + }, + { submissionId: "answering", outcome: "completed" as const }, + ]; + const textlessAnswer = { + id: "answering-message", + role: "assistant" as const, + purpose: "assistant" as const, + display: "visible" as const, + submissionId: "answering", + parts: [], + }; + + fixture.update({ + settlements, + snapshot: { + conversationId: "conversation", + settlements, + messages: [textlessAnswer], + }, + }); + fixture.update({ + settlements, + segments: [segment("Unrelated observed response")], + snapshot: { + conversationId: "conversation", + settlements, + messages: [ + textlessAnswer, + { + id: "answer", + role: "assistant", + purpose: "assistant", + display: "visible", + submissionId: "root", + parts: [ + { + type: "text", + state: "done", + text: "Unrelated observed response", + }, + ], + }, + ], + }, + }); + + expect(fixture.notice).toHaveBeenLastCalledWith( + "Brunch settled without a spoken answer. Check the conversation.", + ); + expect(fixture.appendInstructions).toHaveBeenCalledExactlyOnceWith( + expect.stringContaining("Ask the person to continue"), + "delegation", + ); + expect(fixture.appendCommentary).not.toHaveBeenCalled(); +}); + +test("Stop withdraws only unsubmitted input and suppresses late settlements and later input", async () => { + const fixture = setup(); + await fixture.bridge.accept({ id: "one", text: "First" }); + const signal = vi.mocked(fixture.submit).mock.calls[0]?.[0]; + fixture.bridge.stop(); + fixture.bridge.responseStarted(started); + fixture.bridge.responseCompleted({ + ...started, + position: { batch: 2, index: 0 }, + }); + fixture.update({ segments: [segment()], settlements: completed }); + await fixture.bridge.accept({ id: "late", text: "Late" }); + expect(signal).toBeDefined(); + expect(fixture.appendCommentary).not.toHaveBeenCalled(); + expect(fixture.submit).toHaveBeenCalledOnce(); +}); + +test("a locally stopped or failed turn cannot turn earlier successful prose into success speech", async () => { + const fixture = setup(); + await fixture.bridge.accept({ id: "one", text: "First" }); + fixture.bridge.responseStarted(started); + fixture.bridge.responseCompleted({ + ...started, + position: { batch: 2, index: 0 }, + }); + fixture.update({ status: "streaming", segments: [segment()] }); + fixture.update({ + stopped: true, + segments: [segment()], + settlements: completed, + }); + fixture.update({ segments: [segment()], settlements: completed }); + expect(fixture.appendCommentary).not.toHaveBeenCalled(); +}); + +test("a locally refused long commentary is offered intact once without truncation or replay", async () => { + const fixture = setup(); + fixture.appendCommentary.mockReturnValue(false); + fixture.bridge.acceptDelegation("long-answer"); + await fixture.bridge.accept({ id: "one", text: "First" }); + fixture.bridge.responseStarted(started); + fixture.bridge.responseCompleted({ + ...started, + position: { batch: 2, index: 0 }, + }); + fixture.update({ status: "streaming" }); + const prose = segment("Seven, not four. ".repeat(100)); + fixture.update({ segments: [prose], settlements: completed }); + fixture.update({ segments: [prose], settlements: completed }); + expect(fixture.appendCommentary).toHaveBeenCalledExactlyOnceWith( + prose.text, + "long-answer", + ); + expect(fixture.appendInstructions).not.toHaveBeenCalled(); +}); + +test.each(["before", "after"] as const)( + "claims the most recent unclaimed delegation %s admission without admitting new work", + async (timing) => { + const fixture = setup(); + if (timing === "before") { + fixture.bridge.acceptDelegation("older"); + fixture.bridge.acceptDelegation("newer"); + } + expect(fixture.submit).not.toHaveBeenCalled(); + await fixture.bridge.accept({ id: "first", text: "Seven reviewers" }); + fixture.submit.mockImplementationOnce(async (input) => { + input.onAdmission("second"); + return { + kind: "message", + messageId: "second-input", + submissionId: "second", + }; + }); + await fixture.bridge.accept({ + id: "second-input", + text: "Approval is optional", + }); + if (timing === "after") { + fixture.bridge.acceptDelegation("newer"); + fixture.bridge.acceptDelegation("older"); + } + expect(fixture.submit).toHaveBeenCalledTimes(2); + expect(fixture.appendCommentary).not.toHaveBeenCalled(); + fixture.update({ status: "streaming" }); + for (const response of [ + started, + { ...started, messageId: "second-answer", submissionId: "second" }, + ]) { + fixture.bridge.responseStarted(response); + fixture.bridge.responseCompleted({ + ...response, + position: { batch: 2, index: 0 }, + }); + } + const chat = { + segments: [ + segment(), + { + ...segment("Approval is optional."), + messageId: "second-answer", + submissionIds: ["second"], + }, + ], + settlements: [ + ...completed, + { submissionId: "second", outcome: "completed" as const }, + ], + }; + fixture.update(chat); + fixture.update(chat); + expect(fixture.appendCommentary.mock.calls).toEqual([ + [segment().text, timing === "before" ? "newer" : "older"], + ["Approval is optional.", timing === "before" ? "older" : "newer"], + ]); + expect(fixture.appendInstructions).not.toHaveBeenCalled(); + }, +); + +test("dropped input resolves only its claimed delegation, once, without changing the waiting-input policy", async () => { + const fixture = setup(); + fixture.bridge.acceptDelegation("waiting"); + let release = () => {}; + fixture.submit.mockImplementationOnce(async () => { + await new Promise((resolve) => { + release = resolve; + }); + return { kind: "message", messageId: "first", submissionId: "root" }; + }); + const pending = fixture.bridge.accept({ id: "first", text: "First" }); + fixture.bridge.acceptDelegation("dropped"); + await fixture.bridge.accept({ id: "second", text: "Second" }); + await fixture.bridge.accept({ id: "second", text: "Second" }); + expect(fixture.appendInstructions).toHaveBeenCalledExactlyOnceWith( + "The backend could not take that request now. Ask the person to continue. Do not claim the work completed or was cancelled.", + "dropped", + ); + expect(fixture.submit).toHaveBeenCalledOnce(); + expect(fixture.appendCommentary).not.toHaveBeenCalled(); + release(); + await pending; + fixture.bridge.stop(); +}); + +test.each([ + "admission", + "response", + "failed", + "failed-without-busy", + "textless", + "chat-error", + "stop", +] as const)( + "%s resolves an attached delegation without premature commentary or replay", + async (failure) => { + const fixture = setup(); + fixture.bridge.acceptDelegation("request"); + if (failure === "admission") + fixture.submit.mockRejectedValueOnce(new Error("Unknown")); + if (failure === "response") + fixture.submit.mockImplementationOnce(async (input) => { + input.onAdmission("root"); + throw new Error("Response failed"); + }); + await fixture.bridge.accept({ id: "one", text: "Describe the process" }); + if (failure !== "failed-without-busy") + fixture.update({ status: "streaming" }); + expect(fixture.appendCommentary).not.toHaveBeenCalled(); + if (failure === "stop") fixture.bridge.stop(); + if (failure === "chat-error") fixture.update({ status: "error" }); + const settlements = [ + { + submissionId: "root", + outcome: + failure === "failed" || failure === "failed-without-busy" + ? ("failed" as const) + : ("completed" as const), + }, + ]; + fixture.update({ settlements }); + fixture.update({ settlements }); + await fixture.bridge.accept({ id: "one", text: "Describe the process" }); + expect(fixture.appendInstructions.mock.calls).toEqual( + failure === "stop" + ? [] + : [ + [ + "The backend could not take that request now. Ask the person to continue. Do not claim the work completed or was cancelled.", + "request", + ], + ], + ); + expect(fixture.appendCommentary).not.toHaveBeenCalled(); + expect(fixture.submit).toHaveBeenCalledOnce(); + }, +); + +test("textless and dropped inputs without delegations do not inject session-wide instructions", async () => { + const fixture = setup(); + await fixture.bridge.accept({ id: "one", text: "First" }); + fixture.update({ status: "streaming" }); + fixture.update({ settlements: completed, canAcceptVoiceInput: false }); + await fixture.bridge.accept({ id: "two", text: "Dropped" }); + expect(fixture.appendInstructions).not.toHaveBeenCalled(); + expect(fixture.appendCommentary).not.toHaveBeenCalled(); +}); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/live-brunch-bridge.ts b/apps/petrinaut-website/src/main/app/voice-interview/live-brunch-bridge.ts new file mode 100644 index 00000000000..19a5eb40362 --- /dev/null +++ b/apps/petrinaut-website/src/main/app/voice-interview/live-brunch-bridge.ts @@ -0,0 +1,522 @@ +import { selectCanonicalSpeech } from "./canonical-speech"; +import { logLiveDiagnostic } from "./shared/live-diagnostic"; + +import type { CanonicalSpeechSegment } from "./canonical-speech"; +import type { + RealtimeBrunchBridge, + VoiceSubmissionSettlement, +} from "./realtime-brunch-bridge"; +import type { FlueConversationState } from "@flue/sdk"; +import type { + FlueChatResponseMessageCompletedEvent, + FlueChatResponseMessageStartedEvent, +} from "@hashintel/brunch-agent-transport-aisdk"; +import type { PetrinautAiVoiceModeContext } from "@hashintel/petrinaut/ui"; + +interface Chat { + readonly canAcceptVoiceInput: boolean; + readonly segments: readonly CanonicalSpeechSegment[]; + readonly settlements: readonly VoiceSubmissionSettlement[]; + readonly snapshot?: FlueConversationState; + readonly status: PetrinautAiVoiceModeContext["status"]; + readonly stopped?: boolean; +} + +interface Turn { + readonly inputId: string; + readonly baseline: ReadonlySet; + readonly baselineMessages: ReadonlySet; + delegationId: string | null; + submissionId?: string; + sawBusy: boolean; +} + +type Submit = ConstructorParameters< + typeof RealtimeBrunchBridge +>[0]["submitInterviewAnswer"]; +interface Dependencies { + readonly submit: Submit; + readonly appendCommentary: ( + text: string, + delegationId: string | null, + ) => boolean; + readonly appendInstructions: (text: string, delegationId: string) => boolean; + readonly notice: (message: string | null) => void; +} + +/** Session-local correlation only. Flue and the composer retain all canonical ownership. */ +export class LiveBrunchBridge { + readonly #dependencies: Dependencies; + readonly #abort = new AbortController(); + readonly #seenInputs = new Set(); + readonly #offeredSegments = new Set(); + readonly #turns = new Set(); + readonly #unclaimedDelegations = new Set(); + readonly #responses = new Map< + string, + Map< + string, + { + completed: boolean; + position: FlueChatResponseMessageStartedEvent["position"]; + } + > + >(); + #waitingForComposer: Turn | undefined; + #chat: Chat = { + canAcceptVoiceInput: false, + segments: [], + settlements: [], + status: "ready", + }; + + public constructor(dependencies: Dependencies) { + this.#dependencies = dependencies; + } + + public stop(): void { + this.#abort.abort(); + this.#turns.clear(); + this.#unclaimedDelegations.clear(); + } + + public acceptDelegation(delegationId: string): void { + if (this.#abort.signal.aborted) return; + const turn = [...this.#turns].findLast( + (candidate) => candidate.delegationId === null, + ); + if (turn) turn.delegationId = delegationId; + else this.#unclaimedDelegations.add(delegationId); + logLiveDiagnostic(turn ? "delegation.matched" : "delegation.unclaimed", { + delegationId, + inputId: turn?.inputId, + submissionId: turn?.submissionId, + }); + } + + #unserved(delegationId: string | null): void { + if (delegationId === null || this.#abort.signal.aborted) return; + this.#dependencies.appendInstructions( + "The backend could not take that request now. Ask the person to continue. Do not claim the work completed or was cancelled.", + delegationId, + ); + } + + public async accept(input: { + readonly id: string; + readonly text: string; + }): Promise { + if (this.#abort.signal.aborted) return; + if (this.#seenInputs.has(input.id)) { + logLiveDiagnostic("input.ignored", { + inputId: input.id, + reason: "duplicate", + }); + return; + } + this.#seenInputs.add(input.id); + if (!input.text.trim()) { + logLiveDiagnostic("input.ignored", { + inputId: input.id, + reason: "empty", + }); + return; + } + const delegationId = [...this.#unclaimedDelegations].at(-1) ?? null; + if (delegationId !== null) this.#unclaimedDelegations.delete(delegationId); + if ( + input.text.length > 32_000 || + this.#waitingForComposer || + !this.#chat.canAcceptVoiceInput + ) { + logLiveDiagnostic("input.dropped", { + inputId: input.id, + delegationId, + oversized: input.text.length > 32_000, + waitingForComposer: this.#waitingForComposer !== undefined, + admissionUnavailable: !this.#chat.canAcceptVoiceInput, + }); + this.#dependencies.notice( + "That utterance was not retained. Wait for the pending input, then use the composer to send it.", + ); + this.#unserved(delegationId); + return; + } + this.#dependencies.notice(null); + const turn: Turn = { + inputId: input.id, + delegationId, + baseline: new Set(this.#chat.segments.map((segment) => segment.id)), + baselineMessages: new Set([ + ...this.#chat.segments.map((segment) => segment.messageId), + ...(this.#chat.snapshot?.messages.map((message) => message.id) ?? []), + ]), + sawBusy: + this.#chat.status === "submitted" || this.#chat.status === "streaming", + }; + this.#waitingForComposer = turn; + this.#turns.add(turn); + try { + logLiveDiagnostic("brunch.submit", { inputId: input.id, delegationId }); + const result = await this.#dependencies.submit({ + ...input, + admissionTarget: { kind: "user", messageId: input.id }, + signal: this.#abort.signal, + onAdmission: (submissionId) => { + turn.submissionId = submissionId; + logLiveDiagnostic("brunch.admitted", { + inputId: input.id, + submissionId, + delegationId: turn.delegationId, + afterStop: this.#abort.signal.aborted, + }); + // The submission promise includes the response stream. Admission, + // not response completion, frees the composer's waiting-input slot. + if (this.#waitingForComposer === turn) + this.#waitingForComposer = undefined; + }, + }); + this.#abort.signal.throwIfAborted(); + if ( + result.kind !== "message" || + !result.submissionId || + (turn.submissionId && turn.submissionId !== result.submissionId) + ) { + throw new Error("Uncorrelated admission"); + } + turn.submissionId = result.submissionId; + this.#settle(); + } catch { + if (this.#turns.delete(turn)) { + logLiveDiagnostic("brunch.unconfirmed", { + inputId: input.id, + submissionId: turn.submissionId, + delegationId: turn.delegationId, + }); + this.#dependencies.notice( + turn.submissionId + ? "Your message was admitted, but its response could not be confirmed. Check canonical history; no automatic retry was made." + : "Voice admission could not be confirmed. Check canonical history before sending again; no automatic retry was made.", + ); + this.#unserved(turn.delegationId); + } + } finally { + if (this.#waitingForComposer === turn) + this.#waitingForComposer = undefined; + } + } + + public responseStarted(event: FlueChatResponseMessageStartedEvent): void { + this.#recordResponse(event, false); + } + + public responseCompleted(event: FlueChatResponseMessageCompletedEvent): void { + this.#recordResponse(event, true); + // Rendering and complete-turn status are supplied independently by the + // host, so completion must retry any state that arrived first. + this.#settle(); + } + + #recordResponse( + event: FlueChatResponseMessageStartedEvent, + completed: boolean, + ): void { + if (this.#abort.signal.aborted) return; + let submissions = this.#responses.get(event.messageId); + const previous = submissions?.get(event.submissionId); + if (completed && !previous) return; + if ( + previous && + (event.position.batch < previous.position.batch || + (event.position.batch === previous.position.batch && + event.position.index <= previous.position.index)) + ) + return; + if (!submissions) { + submissions = new Map(); + this.#responses.set(event.messageId, submissions); + } + submissions.set(event.submissionId, { + completed, + position: event.position, + }); + logLiveDiagnostic( + completed ? "brunch.response-completed" : "brunch.response-started", + { + submissionId: event.submissionId, + messageId: event.messageId, + batch: event.position.batch, + index: event.position.index, + }, + ); + } + + public update(chat: Chat): void { + if (this.#abort.signal.aborted) return; + this.#chat = chat; + if (chat.stopped || chat.status === "error") { + for (const turn of this.#turns) { + logLiveDiagnostic("brunch.interrupted", { + inputId: turn.inputId, + submissionId: turn.submissionId, + stopped: chat.stopped === true, + status: chat.status, + }); + } + if (!chat.stopped) { + for (const turn of this.#turns) this.#unserved(turn.delegationId); + } + this.#turns.clear(); + return; + } + if (chat.status === "submitted" || chat.status === "streaming") { + for (const turn of this.#turns) turn.sawBusy = true; + } + this.#settle(); + } + + #settle(): void { + if ( + this.#abort.signal.aborted || + this.#chat.status !== "ready" || + this.#chat.stopped + ) + return; + for (const turn of this.#turns) { + if (!turn.submissionId) continue; + // Client-tool continuations are projected onto their original message. + // Follow that shared identity, including steps that contribute no prose. + const required = new Set([turn.submissionId]); + const messages = new Set(); + let expanded = true; + while (expanded) { + expanded = false; + for (const settlement of this.#chat.settlements) { + if ( + required.has(settlement.submissionId) && + settlement.answeredBySubmissionId && + !required.has(settlement.answeredBySubmissionId) + ) { + required.add(settlement.answeredBySubmissionId); + expanded = true; + } + } + for (const [messageId, submissions] of this.#responses) { + if ( + messages.has(messageId) || + ![...submissions.keys()].some((id) => required.has(id)) + ) + continue; + messages.add(messageId); + for (const id of submissions.keys()) required.add(id); + expanded = true; + } + } + const settlements = [...required].map((id) => + this.#chat.settlements.find( + (settlement) => settlement.submissionId === id, + ), + ); + if ( + settlements.some( + (settlement) => settlement && settlement.outcome !== "completed", + ) + ) { + this.#turns.delete(turn); + logLiveDiagnostic("brunch.not-completed", { + inputId: turn.inputId, + submissionId: turn.submissionId, + delegationId: turn.delegationId, + }); + this.#dependencies.notice( + "Brunch did not complete this turn. Check the conversation; no result was offered to Live.", + ); + this.#unserved(turn.delegationId); + continue; + } + if (!turn.sawBusy) continue; + if (settlements.some((settlement) => !settlement)) continue; + if ( + [...messages].some((id) => + [...this.#responses.get(id)!.values()].some( + (response) => !response.completed, + ), + ) + ) + continue; + let sourceSegments = this.#chat.segments.filter( + (segment) => + messages.has(segment.messageId) && + !turn.baseline.has(segment.id) && + segment.submissionIds?.some((id) => required.has(id)), + ); + const unobservedAnswer = settlements.some( + (settlement) => + settlement?.answeredBySubmissionId && + ![...this.#responses.values()].some((submissions) => + submissions.has(settlement.answeredBySubmissionId!), + ), + ); + if (unobservedAnswer) { + // Another submission's stream is not projected onto this admission. + // Recover from one host-approved snapshot, never invent response events + // or combine newer settlements with older snapshot prose. + const snapshot = this.#chat.snapshot; + if ( + !snapshot || + settlements.some( + (settlement) => + !snapshot.settlements.some( + (entry) => + entry.submissionId === settlement!.submissionId && + entry.outcome === "completed" && + entry.answeredBySubmissionId === + settlement!.answeredBySubmissionId, + ), + ) + ) + continue; + const snapshotMessages = snapshot.messages.filter( + (message) => + message.submissionId && + required.has(message.submissionId) && + message.role === "assistant" && + message.purpose === "assistant" && + message.display === "visible", + ); + if ( + !snapshotMessages.length || + settlements.some( + (settlement) => + settlement?.answeredBySubmissionId && + !snapshotMessages.some( + (message) => + message.submissionId === settlement.answeredBySubmissionId, + ), + ) || + snapshotMessages.some((message) => + message.parts.some( + (part) => part.type === "text" && part.state === "streaming", + ), + ) + ) + continue; + const recovered = selectCanonicalSpeech( + snapshotMessages.map(({ id, parts }) => ({ + id, + role: "assistant", + // Preserve canonical part indices without exposing tools/reasoning. + parts: parts.map((part) => + part.type === "text" ? part : { type: "step-start" as const }, + ), + })), + ).segments; + // The canonical snapshot supplies identity; rendering only confirms + // that the same finalized prose is visible (continuations may fold IDs). + if ( + recovered.some( + (segment) => + !this.#chat.segments.some( + (visible) => visible.text === segment.text, + ), + ) + ) + continue; + sourceSegments = recovered.filter( + (segment) => + !turn.baselineMessages.has(segment.messageId) && + !turn.baseline.has(segment.id), + ); + } else if (!sourceSegments.length && messages.size > 0) { + // Response events, Flue history and Petrinaut rendering are independent + // projections. A ready settlement cannot prove that canonical prose has + // already rendered. Use the matching finalized snapshot to distinguish + // a genuinely textless response from prose that is still catching up. + const snapshot = this.#chat.snapshot; + if (!snapshot) continue; + const snapshotMessages = snapshot.messages.filter( + (message) => + messages.has(message.id) && + message.role === "assistant" && + message.purpose === "assistant" && + message.display === "visible", + ); + if ( + settlements.some( + (settlement) => + !snapshot.settlements.some( + (entry) => + entry.submissionId === settlement!.submissionId && + entry.outcome === "completed" && + entry.answeredBySubmissionId === + settlement!.answeredBySubmissionId, + ), + ) || + [...messages].some( + (messageId) => + !snapshotMessages.some((message) => message.id === messageId), + ) || + snapshotMessages.some((message) => + message.parts.some( + (part) => part.type === "text" && part.state === "streaming", + ), + ) + ) + continue; + const recovered = selectCanonicalSpeech( + snapshotMessages.map(({ id, parts }) => ({ + id, + role: "assistant", + parts: parts.map((part) => + part.type === "text" ? part : { type: "step-start" as const }, + ), + })), + ).segments.filter( + (segment) => + !turn.baselineMessages.has(segment.messageId) && + !turn.baseline.has(segment.id), + ); + if ( + recovered.some( + (segment) => + !this.#chat.segments.some( + (visible) => visible.text === segment.text, + ), + ) + ) + continue; + sourceSegments = recovered; + } + this.#turns.delete(turn); + if (!sourceSegments.length) { + logLiveDiagnostic("brunch.no-prose", { + inputId: turn.inputId, + submissionId: turn.submissionId, + delegationId: turn.delegationId, + }); + this.#dependencies.notice( + "Brunch settled without a spoken answer. Check the conversation.", + ); + this.#unserved(turn.delegationId); + continue; + } + // Coalesced admissions can share an answer. Offer each frozen segment + // only once, even if the connection or provider refuses it. + const segments = sourceSegments.filter( + (segment) => !this.#offeredSegments.has(segment.id), + ); + if (!segments.length) continue; + for (const segment of segments) this.#offeredSegments.add(segment.id); + // Freeze complete prose once. Sending is neither exact relay nor playback proof. + const source = segments.map((segment) => segment.text).join("\n\n"); + logLiveDiagnostic("brunch.offer", { + inputId: turn.inputId, + submissionId: turn.submissionId, + delegationId: turn.delegationId, + segmentCount: segments.length, + characters: source.length, + }); + this.#dependencies.appendCommentary(source, turn.delegationId); + } + } +} diff --git a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.test.tsx b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.test.tsx index 5ee9062a026..8c314811712 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.test.tsx +++ b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.test.tsx @@ -9,12 +9,17 @@ import { } from "@testing-library/react"; import { afterEach, expect, test, vi } from "vitest"; +import { + BrunchPanelConversationTracker, + createBrunchPanelTransport, +} from "../local-storage-demo/brunch-panel-transport"; import { createLiveConversation } from "./live-conversation"; import { loadOpenAIVoiceConfig, VoiceInterviewControl, } from "./voice-interview-control"; +import type { FlueClient, FlueConversationState } from "@flue/sdk"; import type { PetrinautAiVoiceModeContext } from "@hashintel/petrinaut/ui"; vi.mock("./live-conversation", () => ({ @@ -22,6 +27,8 @@ vi.mock("./live-conversation", () => ({ retryPlayback: vi.fn(async () => {}), start: vi.fn(async () => {}), stop: vi.fn(async () => {}), + appendCommentary: vi.fn(() => true), + appendInstructions: vi.fn(() => true), })), })); afterEach(() => { @@ -76,14 +83,99 @@ test("starts only one Live session when Start is activated twice", async () => { expect(createLiveConversation).toHaveBeenCalledOnce(); }); -test("reuses setup, reports listening and speaking to the host dock, and clears it on failure", async () => { +test("Cancel leaves consent without starting a provider session", () => { + const props = context(); + render(); + fireEvent.click(screen.getByRole("checkbox")); + expect(createLiveConversation).not.toHaveBeenCalled(); + fireEvent.click(screen.getByRole("button", { name: "Cancel" })); + expect(props.setInputMode).toHaveBeenCalledWith("text"); + expect(createLiveConversation).not.toHaveBeenCalled(); +}); + +test.each(["submitted", "streaming"] as const)( + "maps Brunch %s to thinking without replacing playback or connection status", + async (status) => { + const props = context(); + const { rerender } = render( + , + ); + await start(); + const onState = vi.mocked(createLiveConversation).mock.calls[0]![0]; + act(() => onState({ phase: "connected", message: null })); + + // A composer update must change the dock without another audio event. + rerender( + , + ); + expect(props.reportVoiceSessionState).toHaveBeenLastCalledWith( + expect.objectContaining({ phase: "thinking" }), + ); + act(() => + onState({ + phase: "connected", + message: null, + activity: { microphoneLevel: 0.2, outputActive: true }, + }), + ); + expect(props.reportVoiceSessionState).toHaveBeenLastCalledWith( + expect.objectContaining({ phase: "speaking" }), + ); + act(() => + onState({ + phase: "connected", + message: null, + activity: { microphoneLevel: 0.2, outputActive: false }, + }), + ); + expect(props.reportVoiceSessionState).toHaveBeenLastCalledWith( + expect.objectContaining({ phase: "thinking" }), + ); + rerender( + , + ); + expect(props.reportVoiceSessionState).toHaveBeenLastCalledWith( + expect.objectContaining({ phase: "listening" }), + ); + rerender( + , + ); + expect(props.reportVoiceSessionState).toHaveBeenLastCalledWith( + expect.objectContaining({ phase: "listening" }), + ); + rerender( + , + ); + act(() => onState({ phase: "connecting", message: null })); + expect(props.reportVoiceSessionState).toHaveBeenLastCalledWith( + expect.objectContaining({ phase: "connecting" }), + ); + act(() => onState({ phase: "error", message: "Connection lost" })); + expect(props.reportVoiceSessionState).toHaveBeenLastCalledWith( + expect.objectContaining({ phase: "error" }), + ); + }, +); + +test("reuses setup and reports failure to the host dock and notification surface", async () => { const props = context(); render(); expect( screen.getByRole("region", { name: "Voice mode consent" }), ).toBeTruthy(); - expect(screen.getByText("GPT-Live · Experimental interview")).toBeTruthy(); - expect(screen.queryByText(/Petrinaut saves finalized/)).toBeNull(); + expect(screen.getByText("Start a voice conversation")).toBeTruthy(); + expect(screen.getByText(/OpenAI processes microphone audio/)).toBeTruthy(); + expect(screen.queryByText(/experimental|best-effort/i)).toBeNull(); + expect(screen.getByRole("button", { name: "Cancel" })).toBeTruthy(); expect( screen .getByRole("button", { name: "Start voice" }) @@ -178,18 +270,22 @@ test("reuses setup, reports listening and speaking to the host dock, and clears const liveSession = vi.mocked(createLiveConversation).mock.results[0]! .value as ReturnType; expect(liveSession.retryPlayback).toHaveBeenCalledOnce(); + const connectionError = + "live session request failed (HTTP 502, provider HTTP 401). No automatic retry was made."; act(() => onState({ phase: "error", - message: "Connection failed. Remote session closure was not confirmed.", + message: connectionError, }), ); - expect(props.reportVoiceSessionState).toHaveBeenLastCalledWith(null); - expect( - screen.getByText( - "Connection error. Check microphone and server configuration.", - ), - ).toBeTruthy(); + expect(props.reportVoiceSessionState).toHaveBeenLastCalledWith({ + phase: "error", + errorMessage: connectionError, + microphoneLevel: 0, + microphoneMuted: true, + notice: null, + }); + expect(screen.getByText(connectionError)).toBeTruthy(); expect( screen .getByRole("button", { name: "Start voice" }) @@ -198,6 +294,21 @@ test("reuses setup, reports listening and speaking to the host dock, and clears expect(createLiveConversation).toHaveBeenCalledOnce(); }); +test("does not show a successful prior session close on the next consent card", async () => { + render(); + await start(); + const onState = vi.mocked(createLiveConversation).mock.calls[0]![0]; + act(() => onState({ phase: "connected", message: null })); + const closureMessage = + "Microphone and playback stopped. Live confirmed session closure."; + act(() => onState({ phase: "ended", message: closureMessage })); + + expect( + screen.getByRole("region", { name: "Voice mode consent" }), + ).toBeTruthy(); + expect(screen.queryByText(closureMessage)).toBeNull(); +}); + test("pins provider, ends through host controls, and never submits or stops canonical work", async () => { const props = context(); const subscribeToAdmission = vi.fn(); @@ -270,3 +381,313 @@ test.each(["live", "realtime", "live-experience"])( ); }, ); + +test("final transcription enters the real admission helper and only its settled canonical prose reaches Live", async () => { + const tracker = new BrunchPanelConversationTracker(); + const props = context(); + props.submitVoiceInput = vi.fn< + PetrinautAiVoiceModeContext["submitVoiceInput"] + >(async ({ id }) => { + if (!id) throw new Error("Missing stable input identity"); + tracker.recordAdmission({ + kind: "user", + messageId: id, + admission: { + submissionId: "root", + uid: "test", + offset: "opaque", + streamUrl: "http://local/stream", + }, + }); + return { kind: "message", messageId: id }; + }); + const wiring = { + resolveInputSubmission: tracker.submissionForInput.bind(tracker), + resolveResponseSubmission: tracker.submissionsForResponse.bind(tracker), + subscribeToAdmission: ( + target: Parameters[0], + listener: (id: string) => void, + ) => + tracker.subscribeToAdmission(target, (event) => + listener(event.admission.submissionId), + ), + subscribeToAdmissionFailure: + tracker.subscribeToAdmissionFailure.bind(tracker), + subscribeToResponseMessageStarted: + tracker.subscribeToResponseMessageStarted.bind(tracker), + subscribeToResponseMessageCompleted: + tracker.subscribeToResponseMessageCompleted.bind(tracker), + subscribeToStopRequested: tracker.subscribeToStopRequested.bind(tracker), + }; + const { rerender } = render( + , + ); + await start(); + const call = vi.mocked(createLiveConversation).mock.lastCall!; + const session = vi.mocked(createLiveConversation).mock.results.at(-1)! + .value as ReturnType; + act(() => call[0]({ phase: "connected", message: null })); + act(() => call[3]("delegation-1")); + await act(async () => + call[2]({ id: "utterance-1", text: "Seven reviewers, not four." }), + ); + expect(props.submitVoiceInput).toHaveBeenCalledOnce(); + expect(props.submitVoiceInput).toHaveBeenCalledWith( + expect.objectContaining({ text: "Seven reviewers, not four." }), + ); + const response = { + messageId: "answer", + submissionId: "root", + position: { batch: 1, index: 0 }, + }; + act(() => tracker.recordResponse(response)); + const messages: PetrinautAiVoiceModeContext["messages"] = [ + { + id: "answer", + role: "assistant", + parts: [ + { + type: "text", + text: "Are all seven reviewers required?", + state: "done", + }, + ], + }, + ]; + rerender( + , + ); + act(() => + tracker.recordResponseMessageCompleted({ + ...response, + position: { batch: 2, index: 0 }, + }), + ); + expect(session.appendCommentary).not.toHaveBeenCalled(); + rerender( + , + ); + expect(session.appendCommentary).toHaveBeenCalledExactlyOnceWith( + "Are all seven reviewers required?", + "delegation-1", + ); + act(() => tracker.recordStopRequested()); + expect(session.stop).toHaveBeenCalled(); + await act(async () => call[2]({ id: "late", text: "Late transcription" })); + expect(props.submitVoiceInput).toHaveBeenCalledOnce(); +}); + +test.each(["answer", "folded-answer"])( + "the real transport's unobserved answered-by response reaches Live with rendered ID %s", + async (renderedId) => { + const tracker = new BrunchPanelConversationTracker(); + const props = context(); + const text = "Seven reviewers, not four. Is approval optional?"; + const snapshot: FlueConversationState = { + conversationId: props.conversationId, + messages: [ + { + id: "answer", + role: "assistant", + purpose: "assistant", + display: "visible", + submissionId: "answering", + parts: [ + { + type: "reasoning", + state: "done", + text: "Private reasoning must not be spoken", + }, + { type: "text", state: "done", text }, + ], + }, + ], + settlements: [ + { + submissionId: "root", + outcome: "completed", + answeredBySubmissionId: "answering", + }, + { submissionId: "answering", outcome: "completed" }, + ], + }; + const client = { + send: vi.fn(async () => ({ + submissionId: "root", + uid: "test", + offset: "opaque", + streamUrl: "http://local/stream", + })), + wait: vi.fn(async (_admission, options) => { + await options?.onEvent?.({ + type: "message-started", + conversationId: props.conversationId, + messageId: "answer", + submissionId: "answering", + turnId: "turn", + position: { batch: 1, index: 0 }, + }); + await options?.onEvent?.({ + type: "message-completed", + conversationId: props.conversationId, + messageId: "answer", + position: { batch: 1, index: 1 }, + }); + await options?.onEvent?.({ + type: "submission-settled", + conversationId: props.conversationId, + submissionId: "root", + outcome: "completed", + answeredBySubmissionId: "answering", + position: { batch: 1, index: 2 }, + }); + }), + } as Pick as FlueClient; + const transport = createBrunchPanelTransport( + Promise.resolve(client), + tracker, + ); + props.submitVoiceInput = async ({ id, text: input }) => { + if (!id) throw new Error("Missing identity"); + const stream = await transport.sendMessages({ + trigger: "submit-message", + chatId: props.conversationId, + messageId: undefined, + messages: [ + { id, role: "user", parts: [{ type: "text", text: input }] }, + ], + abortSignal: undefined, + }); + const reader = stream.getReader(); + while (!(await reader.read()).done) { + /* drain the real transport */ + } + return { kind: "message", messageId: id }; + }; + const wiring = { + resolveInputSubmission: tracker.submissionForInput.bind(tracker), + resolveResponseSubmission: tracker.submissionsForResponse.bind(tracker), + subscribeToResponseMessageStarted: + tracker.subscribeToResponseMessageStarted.bind(tracker), + subscribeToResponseMessageCompleted: + tracker.subscribeToResponseMessageCompleted.bind(tracker), + }; + const { rerender } = render( + , + ); + await start(); + const call = vi.mocked(createLiveConversation).mock.lastCall!; + const session = vi.mocked(createLiveConversation).mock.results.at(-1)! + .value as ReturnType; + act(() => call[0]({ phase: "connected", message: null })); + await act(async () => + call[2]({ id: "utterance", text: "Seven, not four" }), + ); + rerender( + , + ); + expect(tracker.submissionsForResponse("answer")).toBeUndefined(); + expect(tracker.canReplaceMessages(snapshot)).toBe(true); + rerender( + , + ); + expect(session.appendCommentary).not.toHaveBeenCalled(); + rerender( + , + ); + expect(session.appendCommentary).toHaveBeenCalledExactlyOnceWith( + text, + null, + ); + }, +); + +test.each(["commentary", "instructions"] as const)( + "%s pending and accepted appends do not raise errors or replace actual failures", + async (kind) => { + const props = context(); + const { unmount } = render( + , + ); + await start(); + const call = vi.mocked(createLiveConversation).mock.lastCall!; + act(() => call[0]({ phase: "connected", message: null })); + const result = { + eventId: "first", + kind, + delegationId: "delegation", + }; + for (const eventId of ["first", "second"]) { + for (const status of ["unknown", "accepted"] as const) { + act(() => call[4]({ ...result, eventId, status })); + expect(props.reportVoiceSessionState).toHaveBeenLastCalledWith( + expect.objectContaining({ + phase: "listening", + errorMessage: null, + notice: null, + }), + ); + } + } + for (const [status, text] of [ + ["local-failure", "could not be sent to Live locally"], + ["rejected", "was rejected by Live"], + ] as const) { + act(() => call[4]({ ...result, eventId: status, status })); + expect(props.reportVoiceSessionState).toHaveBeenLastCalledWith( + expect.objectContaining({ + phase: "listening", + errorMessage: null, + notice: expect.stringContaining(text) as unknown, + }), + ); + for (const nextStatus of ["unknown", "accepted"] as const) { + act(() => call[4]({ ...result, eventId: "later", status: nextStatus })); + expect(props.reportVoiceSessionState).toHaveBeenLastCalledWith( + expect.objectContaining({ + notice: expect.stringContaining(text) as unknown, + }), + ); + } + } + unmount(); + vi.mocked(props.reportVoiceSessionState).mockClear(); + act(() => call[4]({ ...result, status: "rejected" })); + expect(props.reportVoiceSessionState).not.toHaveBeenCalled(); + }, +); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.tsx b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.tsx index bde5238a6f7..b244bb8e653 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.tsx +++ b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.tsx @@ -1,25 +1,40 @@ -import { useCallback, useEffect, useRef, useState } from "react"; +import { + useCallback, + useEffect, + useLayoutEffect, + useRef, + useState, +} from "react"; +import { selectCanonicalSpeech } from "./canonical-speech"; +import { LiveBrunchBridge } from "./live-brunch-bridge"; import { createLiveConversation, type LiveConversationState, } from "./live-conversation"; import { VoiceInterviewDisclosure } from "./voice-interview-disclosure"; +import type { VoiceInterviewControl } from "./voice-interview-control"; import type { PetrinautAiVoiceModeContext } from "@hashintel/petrinaut/ui"; -// No canonical messages, submission, settlement, or tools cross this boundary. -type LiveControlsContext = Pick< - PetrinautAiVoiceModeContext, - | "inputMode" - | "isAiAssistantOpen" - | "reportVoiceSessionState" - | "setVoiceActive" - | "setInputMode" -> & +type LiveControlsContext = PetrinautAiVoiceModeContext & Required< Pick - >; + > & + Pick< + Parameters[0], + | "resolveResponseSubmission" + | "settlements" + | "snapshot" + | "subscribeToResponseMessageStarted" + | "subscribeToResponseMessageCompleted" + | "subscribeToStopRequested" + > & { + readonly submit: ConstructorParameters< + typeof LiveBrunchBridge + >[0]["submit"]; + readonly connectionTimeoutMs: number; + }; export const LiveConversationControl = ({ inputMode, @@ -29,8 +44,20 @@ export const LiveConversationControl = ({ setVoiceActive, setInputMode, connectionTimeoutMs, -}: LiveControlsContext & { readonly connectionTimeoutMs: number }) => { + submit, + messages, + status, + stopped, + canAcceptVoiceInput, + resolveResponseSubmission, + settlements, + snapshot, + subscribeToResponseMessageStarted, + subscribeToResponseMessageCompleted, + subscribeToStopRequested, +}: LiveControlsContext) => { const [consented, setConsented] = useState(false); + const [notice, setNotice] = useState(null); const [state, setState] = useState({ phase: "idle", message: null, @@ -40,12 +67,83 @@ export const LiveConversationControl = ({ null, ); const sessionActive = useRef(false); + const bridge = useRef(null); + const latest = useRef({ + submit, + chat: { + status, + stopped, + canAcceptVoiceInput, + segments: selectCanonicalSpeech(messages).segments, + settlements: settlements ?? [], + snapshot, + }, + }); + useLayoutEffect(() => { + const segments = selectCanonicalSpeech(messages).segments.map( + (segment) => ({ + ...segment, + submissionIds: resolveResponseSubmission?.(segment.messageId), + }), + ); + latest.current = { + submit, + chat: { + status, + stopped, + canAcceptVoiceInput, + segments, + settlements: settlements ?? [], + snapshot, + }, + }; + bridge.current?.update(latest.current.chat); + }, [ + submit, + messages, + status, + stopped, + canAcceptVoiceInput, + resolveResponseSubmission, + settlements, + snapshot, + ]); + + useEffect( + () => + subscribeToResponseMessageStarted?.((event) => + bridge.current?.responseStarted(event), + ), + [subscribeToResponseMessageStarted], + ); + useEffect( + () => + subscribeToResponseMessageCompleted?.((event) => + bridge.current?.responseCompleted(event), + ), + [subscribeToResponseMessageCompleted], + ); const end = useCallback(async () => { + bridge.current?.stop(); const closing = session.current?.stop(); setVoiceActive(false); setConsented(false); await closing; }, [setVoiceActive]); + useEffect( + () => + subscribeToStopRequested?.(() => { + void end(); + }), + [end, subscribeToStopRequested], + ); + + useEffect(() => { + if (stopped || inputMode !== "voice" || !isAiAssistantOpen) { + bridge.current?.stop(); + void session.current?.stop(); + } + }, [stopped, inputMode, isAiAssistantOpen]); useEffect( () => @@ -66,18 +164,24 @@ export const LiveConversationControl = ({ reportVoiceSessionState( inputMode === "voice" && isAiAssistantOpen && - (phase === "connecting" || phase === "connected") + (phase === "connecting" || phase === "connected" || phase === "error") ? { phase: - phase === "connecting" - ? "connecting" - : activity?.outputActive - ? "speaking" - : "listening", - microphoneLevel: activity?.microphoneLevel ?? 0, - microphoneMuted: false, - errorMessage: null, - notice: playbackBlocked ? message : null, + phase === "error" + ? "error" + : phase === "connecting" + ? "connecting" + : activity?.outputActive + ? "speaking" + : !stopped && + (status === "submitted" || status === "streaming") + ? "thinking" + : "listening", + microphoneLevel: + phase === "error" ? 0 : (activity?.microphoneLevel ?? 0), + microphoneMuted: phase === "error", + errorMessage: phase === "error" ? message : null, + notice: playbackBlocked ? message : notice, ...(playbackBlocked ? { canRetryPlayback: true } : {}), } : null, @@ -86,18 +190,18 @@ export const LiveConversationControl = ({ inputMode, isAiAssistantOpen, phase, - activity, message, playbackBlocked, + activity, + status, + stopped, + notice, reportVoiceSessionState, ]); - useEffect(() => { - if (inputMode !== "voice" || !isAiAssistantOpen) - void session.current?.stop(); - }, [inputMode, isAiAssistantOpen]); useEffect(() => { const leave = () => { + bridge.current?.stop(); void session.current?.stop(); }; window.addEventListener("pagehide", leave); @@ -106,6 +210,7 @@ export const LiveConversationControl = ({ const current = session.current; session.current = null; sessionActive.current = false; + bridge.current?.stop(); void current?.stop(); setVoiceActive(false); reportVoiceSessionState(null); @@ -120,28 +225,66 @@ export const LiveConversationControl = ({ consented={consented} onConsentChange={setConsented} startDisabled={phase === "stopping"} - microphoneCheck={ - phase === "error" - ? "Connection error. Check microphone and server configuration." - : phase === "stopping" - ? "Disconnecting…" - : "" - } + microphoneCheck={phase === "error" ? (state.message ?? "") : ""} onStart={() => { if (!consented || phase === "stopping" || sessionActive.current) return; sessionActive.current = true; setConsented(false); + setNotice(null); setState({ phase: "connecting", message: null }); - const next = createLiveConversation((nextState) => { - if (session.current !== next) return; - if (nextState.phase === "ended" || nextState.phase === "error") { - sessionActive.current = false; - } - setState(nextState); - setVoiceActive( - nextState.phase === "connecting" || nextState.phase === "connected", - ); - }, connectionTimeoutMs); + const next = createLiveConversation( + (nextState) => { + if (session.current !== next) return; + if (nextState.phase === "ended" || nextState.phase === "error") { + sessionActive.current = false; + } + if ( + nextState.phase === "error" || + nextState.phase === "ended" || + nextState.phase === "stopping" + ) + bridge.current?.stop(); + setState(nextState); + setVoiceActive( + nextState.phase === "connecting" || + nextState.phase === "connected", + ); + }, + connectionTimeoutMs, + (input) => { + if (session.current === next) void bridge.current?.accept(input); + }, + (delegationId) => { + if (session.current === next) + bridge.current?.acceptDelegation(delegationId); + }, + (result) => { + if (session.current !== next) return; + // Every successful local send starts as unknown. Neither waiting + // for acceptance nor acceptance itself is an error or resolves a + // failure from another append. + if (result.status === "unknown" || result.status === "accepted") + return; + const label = + result.kind === "commentary" + ? "answer" + : "continuation instruction"; + const outcome = + result.status === "local-failure" + ? "could not be sent to Live locally" + : "was rejected by Live"; + setNotice( + `The ${label} ${outcome}. Check the conversation; no automatic retry or replay was made. Acceptance does not confirm playback.`, + ); + }, + ); + bridge.current = new LiveBrunchBridge({ + submit: (input) => latest.current.submit(input), + appendCommentary: next.appendCommentary, + appendInstructions: next.appendInstructions, + notice: setNotice, + }); + bridge.current.update(latest.current.chat); session.current = next; setVoiceActive(true); void next.start(); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation.test.ts index c743241869a..2475ef39793 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation.test.ts @@ -1,46 +1,51 @@ // @vitest-environment jsdom -import { afterEach, expect, test, vi } from "vitest"; +import { afterEach, beforeEach, expect, test, vi } from "vitest"; import { createLiveConversation } from "./live-conversation"; +beforeEach(() => { + vi.spyOn(console, "debug").mockImplementation(() => {}); +}); + const setup = () => { - const sent: string[] = []; - const channel = Object.assign(new EventTarget(), { - readyState: "open", - send: (data: string) => sent.push(data), - close: vi.fn(), - }); + const sent = [[], []] as [string[], string[]]; + const createChannel = (events: string[]) => + Object.assign(new EventTarget(), { + readyState: "open", + send: (data: string) => events.push(data), + close: vi.fn(), + }); + const channels = [createChannel(sent[0]), createChannel(sent[1])] as const; const input = Object.assign(new EventTarget(), { stop: vi.fn() }); - const output = { stop: vi.fn() }; + const outputs = [{ stop: vi.fn() }, { stop: vi.fn() }]; const stream = { getTracks: () => [input] }; - const peer = Object.assign(new EventTarget(), { - connectionState: "new", - iceGatheringState: "complete", - localDescription: { type: "offer", sdp: "v=0\r\no=complete-offer" }, - createDataChannel: vi.fn(() => channel), - addTrack: vi.fn(), - createOffer: vi.fn(async () => ({ type: "offer", sdp: "v=0" })), - setLocalDescription: vi.fn(async () => undefined), - setRemoteDescription: vi.fn(async () => undefined), - close: vi.fn(), - getReceivers: () => [{ track: output }], + const peers = channels.map((channel, index) => + Object.assign(new EventTarget(), { + connectionState: "new", + iceGatheringState: "complete", + localDescription: { type: "offer", sdp: `v=0\r\no=offer-${index}` }, + createDataChannel: vi.fn(() => channel), + addTrack: vi.fn(), + createOffer: vi.fn(async () => ({ type: "offer", sdp: "v=0" })), + setLocalDescription: vi.fn(async () => undefined), + setRemoteDescription: vi.fn(async () => undefined), + close: vi.fn(), + getReceivers: () => [{ track: outputs[index]! }], + getStats: vi.fn(async () => new Map()), + }), + ); + let peerIndex = 0; + const RTCPeerConnectionMock = vi.fn(function createPeer(this: unknown) { + return peers[peerIndex++]!; }); + vi.stubGlobal("RTCPeerConnection", RTCPeerConnectionMock); const audio = { srcObject: null, - autoplay: false, muted: false, paused: false, play: vi.fn(async () => undefined), pause: vi.fn(), }; - vi.stubGlobal( - "RTCPeerConnection", - class { - constructor() { - return peer; - } - }, - ); vi.stubGlobal( "Audio", class { @@ -51,343 +56,742 @@ const setup = () => { ); const getUserMedia = vi.fn(async () => stream); vi.stubGlobal("navigator", { mediaDevices: { getUserMedia } }); - const fetch = vi.fn(async () => - Response.json({ sessionId: "opaque/id", sdp: "v=0\r\no=answer" }), + const fetch = vi.fn(async (url: string) => + Response.json( + url.endsWith("transcription-session") + ? { sdp: "v=0\r\no=transcription-answer" } + : { sessionId: "opaque", sdp: "v=0\r\no=live-answer" }, + ), ); vi.stubGlobal("fetch", fetch); const onState = vi.fn[0]>(); - const conversation = createLiveConversation(onState, 15_000); - const emit = (data: unknown) => - channel.dispatchEvent( + const onFinalizedInput = + vi.fn[2]>(); + const onDelegation = vi.fn[3]>(); + const onAppendResult = vi.fn[4]>(); + const conversation = createLiveConversation( + onState, + 15_000, + onFinalizedInput, + onDelegation, + onAppendResult, + ); + const emit = (connection: 0 | 1, data: unknown) => + channels[connection].dispatchEvent( new MessageEvent("message", { data: JSON.stringify(data) }), ); return { conversation, emit, sent, - channel, + channels, input, - output, - peer, + outputs, + stream, + peers, audio, getUserMedia, fetch, onState, + onFinalizedInput, + onDelegation, + onAppendResult, }; }; afterEach(() => { vi.useRealTimers(); + vi.restoreAllMocks(); vi.unstubAllGlobals(); + vi.unstubAllEnvs(); }); -test("waits for session.started after SDP; never sends session.start or handles transcript/delegation as tasks", async () => { - const fixture = setup(); - expect(fixture.getUserMedia).not.toHaveBeenCalled(); +const connect = async (fixture: ReturnType) => { await fixture.conversation.start(); - expect(fixture.onState).toHaveBeenLastCalledWith( - expect.objectContaining({ phase: "connecting" }), - ); - expect(fixture.fetch).toHaveBeenCalledWith( - "/api/voice/live-session", - expect.objectContaining({ body: "v=0\r\no=complete-offer" }), - ); - expect(fixture.peer.createDataChannel).toHaveBeenCalledWith("oai-events"); - fixture.emit({ type: "session.started" }); - expect(fixture.onState).toHaveBeenLastCalledWith({ - phase: "connected", - message: null, - }); - fixture.emit({ - type: "session.input_transcript.delta", - delta: "Change the model", - }); - fixture.emit({ type: "session.output_transcript.delta", delta: "Done" }); - fixture.emit({ - type: "session.delegation.created", - delegation: { id: "metadata-only", target: "client" }, - }); - expect(fixture.sent).toEqual([]); - expect(fixture.fetch).toHaveBeenCalledTimes(1); - const stopped = fixture.conversation.stop(); - expect(fixture.input.stop).toHaveBeenCalled(); - expect(fixture.output.stop).toHaveBeenCalled(); - expect(fixture.audio.pause).toHaveBeenCalled(); - expect(fixture.audio.srcObject).toBeNull(); - expect(fixture.sent.map((event) => JSON.parse(event))).toEqual([ - { type: "session.close" }, - ]); - expect(fixture.peer.close).not.toHaveBeenCalled(); - fixture.emit({ type: "session.closed" }); - await stopped; - expect(fixture.peer.close).toHaveBeenCalledOnce(); - expect(fixture.onState).toHaveBeenLastCalledWith({ - phase: "ended", - message: "Microphone and playback stopped. Live confirmed session closure.", - }); -}); - -test("releases late microphone permission without ever creating a provider session", async () => { - const fixture = setup(); - let release!: ( - stream: Awaited>, - ) => void; - fixture.getUserMedia.mockImplementation( - () => - new Promise((resolve) => { - release = resolve; - }), - ); - const started = fixture.conversation.start(); - await fixture.conversation.stop(); - release({ getTracks: () => [fixture.input] }); - await started; - expect(fixture.input.stop).toHaveBeenCalled(); - expect(fixture.fetch).not.toHaveBeenCalled(); -}); + fixture.emit(0, { type: "session.started" }); + fixture.emit(1, { type: "session.created" }); +}; -test("reports missing remote closure honestly and cleans transport after bounded wait", async () => { - vi.useFakeTimers(); - const fixture = setup(); - await fixture.conversation.start(); - fixture.emit({ type: "session.started" }); - const stopped = fixture.conversation.stop(); - await vi.advanceTimersByTimeAsync(2_000); - await stopped; - expect(fixture.peer.close).toHaveBeenCalled(); - expect(fixture.onState.mock.lastCall?.[0].phase).toBe("ended"); - expect(fixture.onState.mock.lastCall?.[0].message).toContain("not confirmed"); -}); +test.each([true, false])( + "diagnostic trace is development-only (%s) and excludes provider content", + async (development) => { + vi.stubEnv("DEV", development); + const debug = vi.spyOn(console, "debug").mockImplementation(() => {}); + const fixture = setup(); + await connect(fixture); + fixture.emit(1, { + type: "conversation.item.input_audio_transcription.completed", + item_id: "diagnostic-item", + content_index: 0, + transcript: "PRIVATE TRANSCRIPT", + }); + fixture.emit(0, { + type: "session.delegation.created", + offset_ms: 710, + delegation: { + id: "diagnostic-delegation", + target: "client", + metadata: { secret: "PRIVATE DELEGATION" }, + }, + }); + fixture.emit(1, { + type: "input_audio_buffer.committed", + item_id: "diagnostic-item", + previous_item_id: null, + }); + fixture.conversation.appendCommentary( + "PRIVATE ANSWER", + "diagnostic-delegation", + ); + const append = fixture.onAppendResult.mock.lastCall![0]; + fixture.emit(0, { + type: "session.commentary.appended", + client_event_id: append.eventId, + }); + const stopped = fixture.conversation.stop(); + fixture.emit(0, { type: "session.closed" }); + await stopped; + if (!development) { + expect(debug).not.toHaveBeenCalled(); + return; + } + const records = debug.mock.calls.map( + ([line]) => + JSON.parse( + String(line).replace("[Petrinaut Live trace] ", ""), + ) as Record, + ); + expect(records).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + event: "provider.event", + connection: "live", + type: "session.delegation.created", + delegationId: "diagnostic-delegation", + offsetMs: 710, + }), + expect.objectContaining({ + event: "input.finalized", + itemId: "diagnostic-item", + }), + expect.objectContaining({ + event: "append.result", + eventId: append.eventId, + status: "unknown", + }), + expect.objectContaining({ + event: "append.result", + eventId: append.eventId, + status: "accepted", + }), + expect.objectContaining({ + event: "session.finished", + liveConfirmed: true, + }), + ]), + ); + expect( + records.find((record) => record.event === "input.finalized")?.inputId, + ).toMatch(/^voice-live:[^:]+:diagnostic-item:0$/u); + expect( + records.every( + (record) => + typeof record.sessionId === "string" && typeof record.at === "string", + ), + ).toBe(true); + expect(JSON.stringify(records)).not.toContain("PRIVATE"); + expect(fixture.fetch).toHaveBeenCalledTimes(2); + expect(fixture.getUserMedia).toHaveBeenCalledOnce(); + }, +); -test("times out connection without inventing turn completion or retrying", async () => { - vi.useFakeTimers(); - const fixture = setup(); - await fixture.conversation.start(); - await vi.advanceTimersByTimeAsync(17_000); - expect(fixture.input.stop).toHaveBeenCalled(); - expect(fixture.onState.mock.lastCall?.[0].phase).toBe("error"); - expect(fixture.onState.mock.lastCall?.[0].message).toContain("timed out"); - expect(fixture.fetch).toHaveBeenCalledTimes(1); -}); +test.each([0, 1] as const)( + "connection %s recovers without replacing sessions or replaying finalized input", + async (connection) => { + vi.useFakeTimers(); + const fixture = setup(); + let release = (_stats: Map) => {}; + fixture.peers[0]!.getStats.mockImplementationOnce( + () => + new Promise((resolve) => { + release = resolve; + }), + ); + await connect(fixture); + await vi.advanceTimersByTimeAsync(100); + const peer = fixture.peers[connection]!; + peer.connectionState = "disconnected"; + peer.dispatchEvent(new Event("connectionstatechange")); + expect(fixture.input.stop).not.toHaveBeenCalled(); + expect(fixture.audio.pause).not.toHaveBeenCalled(); + release(new Map()); + await vi.advanceTimersByTimeAsync(1_000); + expect(fixture.onState.mock.lastCall?.[0].phase).toBe("connecting"); + // Completed, ordered events still use the existing admission path. + const completed = { + type: "conversation.item.input_audio_transcription.completed", + item_id: "one", + content_index: 0, + transcript: "Actually, three machines", + }; + fixture.emit(1, { + type: "input_audio_buffer.committed", + item_id: "one", + previous_item_id: null, + }); + fixture.emit(1, completed); + expect(fixture.onFinalizedInput).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ text: "Actually, three machines" }), + ); + expect(fixture.conversation.appendCommentary("Settled answer", null)).toBe( + true, + ); + peer.connectionState = "connected"; + peer.dispatchEvent(new Event("connectionstatechange")); + expect(fixture.onState.mock.lastCall?.[0].phase).toBe("connected"); + fixture.emit(1, completed); + await vi.advanceTimersByTimeAsync(16_000); + expect(fixture.input.stop).not.toHaveBeenCalled(); + expect(fixture.onFinalizedInput).toHaveBeenCalledOnce(); + expect(fixture.sent[0]).toHaveLength(1); + expect(fixture.sent[1]).toEqual([]); + expect(fixture.fetch).toHaveBeenCalledTimes(2); + expect(fixture.getUserMedia).toHaveBeenCalledOnce(); + const stopped = fixture.conversation.stop(); + fixture.emit(0, { type: "session.closed" }); + await stopped; + expect(vi.getTimerCount()).toBe(0); + }, +); -test("a temporary media disconnect recovers on the same session without replaying input", async () => { - vi.useFakeTimers(); - const fixture = setup(); - let release!: (stats: Map) => void; - const getStats = vi.fn( - () => - new Promise>((resolve) => { - release = resolve; - }), - ); - Object.assign(fixture.peer, { getStats }); - await fixture.conversation.start(); - fixture.emit({ type: "session.started" }); - await vi.advanceTimersByTimeAsync(100); - fixture.peer.connectionState = "disconnected"; - fixture.peer.dispatchEvent(new Event("connectionstatechange")); - expect(fixture.input.stop).not.toHaveBeenCalled(); - expect(fixture.audio.pause).not.toHaveBeenCalled(); - expect(fixture.onState.mock.lastCall?.[0].phase).toBe("connecting"); - // A sample requested before the disconnect must not restore Listening. - release( - new Map([ - ["input", { type: "media-source", kind: "audio", audioLevel: 0.4 }], - ]), - ); - getStats.mockResolvedValue(new Map()); - await vi.advanceTimersByTimeAsync(1_000); - expect(fixture.onState.mock.lastCall?.[0].phase).toBe("connecting"); - fixture.peer.connectionState = "connected"; - fixture.peer.dispatchEvent(new Event("connectionstatechange")); - expect(fixture.onState.mock.lastCall?.[0].phase).toBe("connected"); - await vi.advanceTimersByTimeAsync(16_000); - expect(fixture.input.stop).not.toHaveBeenCalled(); - expect(fixture.onState.mock.lastCall?.[0]).toMatchObject({ - phase: "connected", - activity: { microphoneLevel: 0, outputActive: false }, - }); - expect(fixture.sent).toEqual([]); - expect(fixture.fetch).toHaveBeenCalledOnce(); - expect(fixture.getUserMedia).toHaveBeenCalledOnce(); - const stopped = fixture.conversation.stop(); - fixture.emit({ type: "session.closed" }); - await stopped; - expect(vi.getTimerCount()).toBe(0); -}); +test.each([0, 1] as const)( + "connection %s has a fixed recovery deadline and cannot revive after timeout", + async (connection) => { + vi.useFakeTimers(); + const fixture = setup(); + await connect(fixture); + const peer = fixture.peers[connection]!; + peer.connectionState = "disconnected"; + peer.dispatchEvent(new Event("connectionstatechange")); + await vi.advanceTimersByTimeAsync(5_000); + peer.dispatchEvent(new Event("connectionstatechange")); + await vi.advanceTimersByTimeAsync(9_999); + expect(fixture.input.stop).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(1); + expect(fixture.input.stop).toHaveBeenCalled(); + expect(fixture.audio.pause).toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(2_000); + expect(fixture.onState.mock.lastCall?.[0].phase).toBe("error"); + expect(fixture.onState.mock.lastCall?.[0].message).toContain( + "did not recover", + ); + const updates = fixture.onState.mock.calls.length; + peer.connectionState = "connected"; + peer.dispatchEvent(new Event("connectionstatechange")); + await vi.advanceTimersByTimeAsync(20_000); + expect(fixture.onState).toHaveBeenCalledTimes(updates); + expect(fixture.fetch).toHaveBeenCalledTimes(2); + expect(vi.getTimerCount()).toBe(0); + }, +); -test("an unrecovered disconnect has a fixed deadline and cannot revive after cleanup", async () => { +test("recovering one peer does not clear the other peer's deadline or show connected", async () => { vi.useFakeTimers(); const fixture = setup(); - await fixture.conversation.start(); - fixture.emit({ type: "session.started" }); - fixture.peer.connectionState = "disconnected"; - fixture.peer.dispatchEvent(new Event("connectionstatechange")); + await connect(fixture); + const live = fixture.peers[0]!; + const transcription = fixture.peers[1]!; + live.connectionState = "disconnected"; + live.dispatchEvent(new Event("connectionstatechange")); await vi.advanceTimersByTimeAsync(5_000); - fixture.peer.dispatchEvent(new Event("connectionstatechange")); - await vi.advanceTimersByTimeAsync(9_999); + transcription.connectionState = "disconnected"; + transcription.dispatchEvent(new Event("connectionstatechange")); + live.connectionState = "connected"; + live.dispatchEvent(new Event("connectionstatechange")); + await vi.advanceTimersByTimeAsync(14_999); expect(fixture.input.stop).not.toHaveBeenCalled(); + expect(fixture.onState.mock.lastCall?.[0].phase).toBe("connecting"); await vi.advanceTimersByTimeAsync(1); expect(fixture.input.stop).toHaveBeenCalled(); - expect(fixture.output.stop).toHaveBeenCalled(); - expect(fixture.audio.pause).toHaveBeenCalled(); - await vi.advanceTimersByTimeAsync(2_000); - expect(fixture.onState.mock.lastCall?.[0].phase).toBe("error"); + fixture.emit(0, { type: "session.closed" }); expect(fixture.onState.mock.lastCall?.[0].message).toContain( - "did not recover", + "Transcription media connection did not recover", ); - const updates = fixture.onState.mock.calls.length; - fixture.peer.connectionState = "connected"; - fixture.peer.dispatchEvent(new Event("connectionstatechange")); - await vi.advanceTimersByTimeAsync(20_000); - expect(fixture.onState).toHaveBeenCalledTimes(updates); - expect(fixture.fetch).toHaveBeenCalledOnce(); expect(vi.getTimerCount()).toBe(0); }); -test.each(["failed", "closed"])( - "terminal media state %s still ends immediately", - async (state) => { +test.each([0, 1] as const)( + "a handshake disconnect on connection %s stays bounded after both ready events", + async (connection) => { + vi.useFakeTimers(); const fixture = setup(); await fixture.conversation.start(); - fixture.emit({ type: "session.started" }); - fixture.peer.connectionState = state; - fixture.peer.dispatchEvent(new Event("connectionstatechange")); + const peer = fixture.peers[connection]!; + peer.connectionState = "disconnected"; + peer.dispatchEvent(new Event("connectionstatechange")); + await vi.advanceTimersByTimeAsync(5_000); + fixture.emit(0, { type: "session.started" }); + fixture.emit(1, { type: "session.created" }); + await vi.advanceTimersByTimeAsync(14_999); + expect(fixture.input.stop).not.toHaveBeenCalled(); + expect(fixture.onState.mock.lastCall?.[0].phase).toBe("connecting"); + await vi.advanceTimersByTimeAsync(1); expect(fixture.input.stop).toHaveBeenCalled(); - expect(fixture.audio.pause).toHaveBeenCalled(); - fixture.emit({ type: "session.closed" }); - expect(fixture.onState.mock.lastCall?.[0].phase).toBe("error"); + fixture.emit(0, { type: "session.closed" }); + expect(vi.getTimerCount()).toBe(0); }, ); -test("End during a disconnect cancels the recovery deadline", async () => { +test("Stop cancels both recovery deadlines and ignores late connections", async () => { vi.useFakeTimers(); const fixture = setup(); - await fixture.conversation.start(); - fixture.emit({ type: "session.started" }); - fixture.peer.connectionState = "disconnected"; - fixture.peer.dispatchEvent(new Event("connectionstatechange")); + await connect(fixture); + for (const peer of fixture.peers) { + peer.connectionState = "disconnected"; + peer.dispatchEvent(new Event("connectionstatechange")); + } + expect(fixture.input.stop).not.toHaveBeenCalled(); const stopped = fixture.conversation.stop(); - fixture.emit({ type: "session.closed" }); + fixture.emit(0, { type: "session.closed" }); await stopped; + const updates = fixture.onState.mock.calls.length; + for (const peer of fixture.peers) { + peer.connectionState = "connected"; + peer.dispatchEvent(new Event("connectionstatechange")); + } await vi.advanceTimersByTimeAsync(20_000); + expect(fixture.onState).toHaveBeenCalledTimes(updates); expect(fixture.onState.mock.lastCall?.[0].phase).toBe("ended"); expect(vi.getTimerCount()).toBe(0); }); -test.each([false, true])( - "a handshake disconnect remains bounded (session.started received: %s)", - async (ready) => { - vi.useFakeTimers(); +test.each([ + [0, "failed"], + [0, "closed"], + [1, "failed"], + [1, "closed"], +] as const)( + "terminal connection %s state %s still stops immediately", + async (connection, state) => { const fixture = setup(); - await fixture.conversation.start(); - fixture.peer.connectionState = "disconnected"; - fixture.peer.dispatchEvent(new Event("connectionstatechange")); - await vi.advanceTimersByTimeAsync(5_000); - if (ready) fixture.emit({ type: "session.started" }); - await vi.advanceTimersByTimeAsync(ready ? 14_999 : 9_999); - expect(fixture.input.stop).not.toHaveBeenCalled(); - expect(fixture.onState.mock.lastCall?.[0].phase).toBe("connecting"); - await vi.advanceTimersByTimeAsync(1); + await connect(fixture); + const peer = fixture.peers[connection]!; + peer.connectionState = state; + peer.dispatchEvent(new Event("connectionstatechange")); expect(fixture.input.stop).toHaveBeenCalled(); - await vi.advanceTimersByTimeAsync(2_000); + expect(fixture.audio.pause).toHaveBeenCalled(); + fixture.emit(0, { type: "session.closed" }); expect(fixture.onState.mock.lastCall?.[0].phase).toBe("error"); - expect(fixture.fetch).toHaveBeenCalledOnce(); - expect(vi.getTimerCount()).toBe(0); }, ); -test("provider errors stop both media directions without exposing payloads", async () => { +test("starts Live and transcription WebRTC from one consented capture and connects only when both are usable", async () => { const fixture = setup(); await fixture.conversation.start(); - fixture.emit({ type: "session.started" }); - fixture.emit({ - type: "error", - error: { message: "sensitive provider detail" }, + expect(fixture.getUserMedia).toHaveBeenCalledOnce(); + expect(fixture.fetch.mock.calls.map(([url]) => url)).toEqual([ + "/api/voice/live-session", + "/api/voice/transcription-session", + ]); + expect(fixture.peers[0]!.addTrack).toHaveBeenCalledWith( + fixture.input, + fixture.stream, + ); + expect(fixture.peers[1]!.addTrack).toHaveBeenCalledWith( + fixture.input, + fixture.stream, + ); + fixture.emit(0, { type: "session.started" }); + expect(fixture.onState.mock.lastCall?.[0].phase).toBe("connecting"); + fixture.emit(1, { type: "session.created" }); + expect(fixture.onState).toHaveBeenLastCalledWith({ + phase: "connected", + message: null, }); - expect(fixture.input.stop).toHaveBeenCalled(); - expect(fixture.output.stop).toHaveBeenCalled(); - fixture.emit({ type: "session.closed" }); - expect(fixture.onState.mock.lastCall?.[0].phase).toBe("error"); - expect(fixture.onState.mock.lastCall?.[0].message).not.toContain("sensitive"); }); -test.each([false, true])( - "microphone loss stops playback and closes without restarting (session started: %s)", - async (ready) => { +test("emits only completed transcripts in committed provider order and deduplicates identical events", async () => { + const fixture = setup(); + await connect(fixture); + fixture.emit(1, { + type: "conversation.item.input_audio_transcription.completed", + item_id: "second", + content_index: 0, + transcript: "Second", + }); + fixture.emit(1, { + type: "conversation.item.input_audio_transcription.completed", + item_id: "first", + content_index: 0, + transcript: "First", + }); + fixture.emit(1, { + type: "input_audio_buffer.committed", + item_id: "first", + previous_item_id: null, + }); + expect(fixture.onFinalizedInput.mock.lastCall?.[0].id).toMatch( + /^voice-live:[^:]+:first:0$/u, + ); + expect(fixture.onFinalizedInput.mock.lastCall?.[0].text).toBe("First"); + fixture.emit(1, { + type: "input_audio_buffer.committed", + item_id: "second", + previous_item_id: "first", + }); + fixture.emit(1, { + type: "conversation.item.input_audio_transcription.completed", + item_id: "second", + content_index: 0, + transcript: "Second", + }); + expect( + fixture.onFinalizedInput.mock.calls.map(([input]) => input.text), + ).toEqual(["First", "Second"]); + expect(fixture.onFinalizedInput.mock.lastCall?.[0].id).toMatch( + /^voice-live:[^:]+:second:0$/u, + ); + fixture.emit(1, { + type: "conversation.item.input_audio_transcription.delta", + item_id: "third", + delta: "ignored", + }); + fixture.emit(0, { type: "session.input_transcript.delta", delta: "ignored" }); + expect(fixture.onFinalizedInput).toHaveBeenCalledTimes(2); +}); + +test("conflicting transcript identity fails visibly, stops both connections, and ignores late callbacks", async () => { + const fixture = setup(); + await connect(fixture); + fixture.emit(1, { + type: "input_audio_buffer.committed", + item_id: "one", + previous_item_id: null, + }); + fixture.emit(1, { + type: "conversation.item.input_audio_transcription.completed", + item_id: "one", + content_index: 0, + transcript: "One", + }); + fixture.emit(1, { + type: "conversation.item.input_audio_transcription.completed", + item_id: "one", + content_index: 0, + transcript: "Changed", + }); + expect(fixture.input.stop).toHaveBeenCalledOnce(); + expect(fixture.audio.pause).toHaveBeenCalled(); + fixture.emit(0, { type: "session.closed" }); + expect(fixture.onState.mock.lastCall?.[0]).toMatchObject({ phase: "error" }); + fixture.emit(1, { + type: "conversation.item.input_audio_transcription.completed", + item_id: "late", + content_index: 0, + transcript: "Late", + }); + expect(fixture.onFinalizedInput).toHaveBeenCalledTimes(1); +}); + +test.each([true, false])( + "transcription failure silences immediately and gives ready Live bounded graceful closure (confirmed: %s)", + async (confirmed) => { + vi.useFakeTimers(); const fixture = setup(); await fixture.conversation.start(); - if (ready) fixture.emit({ type: "session.started" }); - fixture.input.dispatchEvent(new Event("ended")); - expect(fixture.audio.pause).toHaveBeenCalled(); + fixture.emit(0, { type: "session.started" }); + fixture.emit(1, { + type: "conversation.item.input_audio_transcription.failed", + item_id: "one", + }); + expect(fixture.input.stop).toHaveBeenCalledOnce(); expect(fixture.audio.muted).toBe(true); - expect(fixture.output.stop).toHaveBeenCalled(); - expect(fixture.sent).toEqual(ready ? ['{"type":"session.close"}'] : []); - if (ready) fixture.emit({ type: "session.closed" }); - await fixture.conversation.stop(); - expect(fixture.peer.close).toHaveBeenCalledOnce(); - expect(fixture.onState.mock.lastCall?.[0].phase).toBe("error"); + expect(fixture.channels[1].close).toHaveBeenCalledOnce(); + expect(fixture.channels[0].close).not.toHaveBeenCalled(); + expect(fixture.sent[0]).toEqual([ + JSON.stringify({ type: "session.close" }), + ]); + expect(fixture.onState.mock.lastCall?.[0].phase).toBe("stopping"); expect(fixture.onState.mock.lastCall?.[0].message).toContain( - "Microphone disconnected", + "Transcription failed", ); - expect(fixture.getUserMedia).toHaveBeenCalledOnce(); - expect(fixture.fetch).toHaveBeenCalledOnce(); + fixture.emit(1, { type: "session.created" }); + fixture.emit(0, { type: "session.started" }); + expect(fixture.conversation.appendCommentary("Late answer", null)).toBe( + false, + ); + if (confirmed) fixture.emit(0, { type: "session.closed" }); + else await vi.advanceTimersByTimeAsync(2_000); + expect(fixture.onState.mock.lastCall?.[0]).toMatchObject({ + phase: "error", + }); + expect(fixture.onState.mock.lastCall?.[0].message).toContain( + "Transcription failed", + ); + expect(fixture.onState.mock.lastCall?.[0].message).toContain( + confirmed + ? "Live confirmed session closure" + : "Remote Live session closure was not confirmed", + ); + expect(fixture.fetch).toHaveBeenCalledTimes(2); + expect(fixture.peers[0]!.close).toHaveBeenCalledOnce(); + expect(fixture.peers[1]!.close).toHaveBeenCalled(); + }, +); + +test("tracks only client delegations once without admitting their metadata as input", async () => { + const fixture = setup(); + await connect(fixture); + for (const delegation of [ + { id: "opaque/first", target: "client" }, + { id: "responses", target: "responses" }, + { id: "opaque/second", target: "client" }, + { id: "opaque/first", target: "client" }, + { id: 1, target: "client" }, + null, + ]) + fixture.emit(0, { type: "session.delegation.created", delegation }); + expect(fixture.onDelegation.mock.calls).toEqual([ + ["opaque/first"], + ["opaque/second"], + ]); + expect([...fixture.conversation.openDelegations]).toEqual([ + "opaque/first", + "opaque/second", + ]); + expect(fixture.onFinalizedInput).not.toHaveBeenCalled(); + expect(fixture.sent[0]).toEqual([]); +}); + +test.each(["commentary", "instructions"] as const)( + "%s sends intact >500-byte content with unique IDs and correlates only matching acknowledgements", + async (kind) => { + const fixture = setup(); + await connect(fixture); + fixture.emit(0, { + type: "session.delegation.created", + delegation: { id: "opaque", target: "client" }, + }); + const append = + kind === "commentary" + ? fixture.conversation.appendCommentary + : fixture.conversation.appendInstructions; + const content = "Seven reviewers, not four. ".repeat(30) + "😀"; + expect(new TextEncoder().encode(content).byteLength).toBeGreaterThan(500); + expect(append(content, "opaque")).toBe(true); + const first = fixture.onAppendResult.mock.lastCall![0]; + expect(first.status).toBe("unknown"); + expect(JSON.parse(fixture.sent[0][0]!)).toEqual({ + type: `session.${kind}.append`, + event_id: first.eventId, + delegation_id: "opaque", + content, + }); + expect(append("Second", null)).toBe(true); + const second = fixture.onAppendResult.mock.lastCall![0]; + expect(second.eventId).not.toBe(first.eventId); + expect(JSON.parse(fixture.sent[0][1]!)).toEqual({ + type: `session.${kind}.append`, + event_id: second.eventId, + delegation_id: null, + content: "Second", + }); + const acknowledgement = { + type: `session.${kind}.appended`, + client_event_id: first.eventId, + }; + fixture.emit(0, { ...acknowledgement, client_event_id: "unrelated" }); + fixture.emit(0, { type: acknowledgement.type, event_id: first.eventId }); + fixture.emit(0, { + ...acknowledgement, + type: + kind === "commentary" + ? "session.instructions.appended" + : "session.commentary.appended", + }); + expect(fixture.onAppendResult).toHaveBeenCalledTimes(2); + expect(fixture.conversation.openDelegations.has("opaque")).toBe(true); + fixture.emit(0, acknowledgement); + expect(fixture.onAppendResult.mock.lastCall![0]).toEqual({ + ...first, + status: "accepted", + }); + expect(fixture.conversation.openDelegations.size).toBe(0); + fixture.emit(0, acknowledgement); + fixture.emit(0, { + type: "session.delegation.created", + delegation: { id: "opaque", target: "client" }, + }); + expect(fixture.onAppendResult).toHaveBeenCalledTimes(3); + expect(fixture.onDelegation).toHaveBeenCalledOnce(); + expect(fixture.sent[0]).toHaveLength(2); }, ); -test("Stop before session.started never sends application commands", async () => { +test.each([ + ["commentary", "nested"], + ["commentary", "top-level"], + ["instructions", "nested"], + ["instructions", "top-level"], +] as const)( + "%s distinguishes local failure from %s provider rejection without exposing provider text or retrying", + async (kind, correlationLocation) => { + const fixture = setup(); + const append = + kind === "commentary" + ? fixture.conversation.appendCommentary + : fixture.conversation.appendInstructions; + expect(append("Not connected", "opaque")).toBe(false); + expect(fixture.onAppendResult.mock.lastCall![0].status).toBe( + "local-failure", + ); + await connect(fixture); + const send = vi + .spyOn(fixture.channels[0], "send") + .mockImplementationOnce(() => { + throw new Error("Local failure"); + }); + expect(append("Cannot send", "opaque")).toBe(false); + const failed = fixture.onAppendResult.mock.lastCall![0]; + expect(failed.status).toBe("local-failure"); + fixture.emit(0, { + type: `session.${kind}.appended`, + client_event_id: failed.eventId, + }); + expect(fixture.onAppendResult).toHaveBeenCalledTimes(2); + expect(append("Accepted locally", "opaque")).toBe(true); + const pending = fixture.onAppendResult.mock.lastCall![0]; + const rejection = (eventId: string) => ({ + type: "error", + ...(correlationLocation === "top-level" + ? { client_event_id: eventId } + : {}), + error: { + message: "secret", + ...(correlationLocation === "nested" + ? { client_event_id: eventId } + : {}), + }, + }); + fixture.emit(0, rejection("unrelated")); + expect(fixture.onAppendResult).toHaveBeenCalledTimes(3); + expect(fixture.onState.mock.lastCall![0].phase).toBe("connected"); + fixture.emit(0, rejection(pending.eventId)); + expect(fixture.onAppendResult.mock.lastCall![0]).toEqual({ + ...pending, + status: "rejected", + }); + fixture.emit(0, rejection(pending.eventId)); + fixture.emit(0, { + type: `session.${kind}.appended`, + client_event_id: pending.eventId, + }); + expect(fixture.onAppendResult).toHaveBeenCalledTimes(4); + expect(fixture.onState.mock.lastCall![0].phase).toBe("connected"); + expect(send).toHaveBeenCalledTimes(2); + expect(fixture.sent[0]).toHaveLength(1); + }, +); + +test("Stop invalidates pending appends and delegations before closure; late events cannot change outcomes", async () => { + const fixture = setup(); + await connect(fixture); + fixture.emit(0, { + type: "session.delegation.created", + delegation: { id: "opaque", target: "client" }, + }); + fixture.conversation.appendCommentary("Settled answer", "opaque"); + fixture.conversation.appendInstructions("Ask the person to continue", null); + const pending = fixture.onAppendResult.mock.calls.map(([result]) => result); + const stop = fixture.conversation.stop(); + expect(fixture.conversation.openDelegations.size).toBe(0); + for (const closed of [false, true]) { + if (closed) { + fixture.emit(0, { type: "session.closed" }); + await stop; + } + for (const result of pending) { + fixture.emit(0, { + type: `session.${result.kind}.appended`, + client_event_id: result.eventId, + }); + fixture.emit(0, { + type: "error", + error: { client_event_id: result.eventId }, + }); + fixture.emit(0, { + type: "error", + client_event_id: result.eventId, + error: { message: "Late rejection" }, + }); + } + fixture.emit(0, { + type: "session.delegation.created", + delegation: { id: "late", target: "client" }, + }); + expect(fixture.conversation.appendCommentary("Late", null)).toBe(false); + expect(fixture.conversation.appendInstructions("Late", "opaque")).toBe( + false, + ); + } + expect(fixture.onAppendResult).toHaveBeenCalledTimes(2); + expect(fixture.onDelegation).toHaveBeenCalledOnce(); + expect(fixture.conversation.openDelegations.size).toBe(0); + expect(fixture.sent[0]).toHaveLength(3); // Two appends and session.close, no replay. +}); + +test("Stop synchronously silences playback and capture, closes both transports, and does not claim transcription closure", async () => { vi.useFakeTimers(); const fixture = setup(); - await fixture.conversation.start(); + await connect(fixture); const stopped = fixture.conversation.stop(); - await vi.advanceTimersByTimeAsync(2_000); + expect(fixture.input.stop).toHaveBeenCalledOnce(); + expect(fixture.audio.pause).toHaveBeenCalledOnce(); + expect(fixture.sent[0].map((event) => JSON.parse(event))).toContainEqual({ + type: "session.close", + }); + expect(fixture.channels[1].close).toHaveBeenCalledOnce(); + fixture.emit(0, { type: "session.closed" }); await stopped; - expect(fixture.sent).toEqual([]); - expect(fixture.onState.mock.lastCall?.[0].message).toContain("not confirmed"); + expect(fixture.onState.mock.lastCall?.[0].message).toBe( + "Microphone and playback stopped. Live confirmed session closure.", + ); }); -test("plays the remote track while microphone remains attached, then stops both", async () => { +test("a failure on either media connection stops both and remote audio remains native and unbuffered", async () => { const fixture = setup(); - await fixture.conversation.start(); - fixture.emit({ type: "session.started" }); - const remoteStream = { getTracks: () => [fixture.output] }; - fixture.peer.dispatchEvent( + await connect(fixture); + const remoteStream = { getTracks: () => [fixture.outputs[0]!] }; + fixture.peers[0]!.dispatchEvent( Object.assign(new Event("track"), { - track: fixture.output, + track: fixture.outputs[0]!, streams: [remoteStream], }), ); expect(fixture.audio.srcObject).toBe(remoteStream); expect(fixture.audio.autoplay).toBe(true); expect(fixture.audio.play).toHaveBeenCalledOnce(); - expect(fixture.peer.addTrack).toHaveBeenCalledWith( - fixture.input, - expect.anything(), - ); - expect(fixture.input.stop).not.toHaveBeenCalled(); - const stopped = fixture.conversation.stop(); - fixture.emit({ type: "session.closed" }); - await stopped; - expect(fixture.audio.pause).toHaveBeenCalled(); - expect(fixture.input.stop).toHaveBeenCalled(); + Object.assign(fixture.peers[1]!, { connectionState: "failed" }); + fixture.peers[1]!.dispatchEvent(new Event("connectionstatechange")); + expect(fixture.input.stop).toHaveBeenCalledOnce(); + expect(fixture.peers[1]!.close).toHaveBeenCalledOnce(); + fixture.emit(0, { type: "session.closed" }); + expect(fixture.peers[0]!.close).toHaveBeenCalledOnce(); }); test.each(["rejects", "throws"] as const)( - "reports playback that %s and retries it without restarting the Live session", + "reports playback that %s and retries it without restarting either Live session", async (failureMode) => { vi.useFakeTimers(); const fixture = setup(); - Object.assign(fixture.peer, { - getStats: vi.fn( - async () => - new Map([ - [ - "input", - { type: "media-source", kind: "audio", audioLevel: 0.42 }, - ], - ]), - ), - }); + fixture.peers[0]!.getStats.mockResolvedValue( + new Map([ + [ + "input", + { type: "media-source", kind: "audio", audioLevel: 0.42 }, + ], + ]), + ); if (failureMode === "rejects") { fixture.audio.play.mockRejectedValueOnce(new Error("Playback blocked")); } else { @@ -395,13 +799,12 @@ test.each(["rejects", "throws"] as const)( throw new Error("Playback blocked"); }); } - await fixture.conversation.start(); - fixture.emit({ type: "session.started" }); + await connect(fixture); await vi.advanceTimersByTimeAsync(100); - fixture.peer.dispatchEvent( + fixture.peers[0]!.dispatchEvent( Object.assign(new Event("track"), { - track: fixture.output, - streams: [{ getTracks: () => [fixture.output] }], + track: fixture.outputs[0], + streams: [fixture.stream], }), ); await Promise.resolve(); @@ -421,7 +824,7 @@ test.each(["rejects", "throws"] as const)( playbackBlocked: true, }); expect(fixture.input.stop).not.toHaveBeenCalled(); - expect(fixture.output.stop).not.toHaveBeenCalled(); + expect(fixture.outputs[0]!.stop).not.toHaveBeenCalled(); await fixture.conversation.retryPlayback(); expect(fixture.audio.play).toHaveBeenCalledTimes(2); expect(fixture.onState).toHaveBeenLastCalledWith({ @@ -429,111 +832,230 @@ test.each(["rejects", "throws"] as const)( message: null, activity: { microphoneLevel: 0.42, outputActive: false }, }); - expect(fixture.fetch).toHaveBeenCalledOnce(); + expect(fixture.fetch).toHaveBeenCalledTimes(2); expect(fixture.getUserMedia).toHaveBeenCalledOnce(); const stopped = fixture.conversation.stop(); - fixture.emit({ type: "session.closed" }); + fixture.emit(0, { type: "session.closed" }); await stopped; expect(vi.getTimerCount()).toBe(0); }, ); -test("reports local audio activity without treating silence or transcripts as turn completion", async () => { - vi.useFakeTimers(); +test("late transcription while waiting for Live closure cannot submit or revive the session", async () => { const fixture = setup(); - const levels = { input: 0.24, output: 0 }; - const getStats = vi.fn( - async () => - new Map([ - [ - "input", - { type: "media-source", kind: "audio", audioLevel: levels.input }, - ], - [ - "output", - { type: "inbound-rtp", kind: "audio", audioLevel: levels.output }, - ], - ]), - ); - Object.assign(fixture.peer, { getStats }); - await fixture.conversation.start(); - await vi.advanceTimersByTimeAsync(100); - expect(getStats).not.toHaveBeenCalled(); - fixture.emit({ type: "session.started" }); - fixture.peer.dispatchEvent( - Object.assign(new Event("track"), { - track: fixture.output, - streams: [{ getTracks: () => [fixture.output] }], - }), - ); - await vi.advanceTimersByTimeAsync(100); - expect(fixture.onState).toHaveBeenLastCalledWith( - expect.objectContaining({ - phase: "connected", - activity: { microphoneLevel: 0.24, outputActive: false }, - }), - ); - levels.output = 0.2; - await vi.advanceTimersByTimeAsync(100); - expect(fixture.onState).toHaveBeenLastCalledWith( - expect.objectContaining({ - activity: { microphoneLevel: 0.24, outputActive: true }, - }), - ); - expect(fixture.input.stop).not.toHaveBeenCalled(); - levels.output = 0; - await vi.advanceTimersByTimeAsync(100); - expect(fixture.onState.mock.lastCall?.[0]).toMatchObject({ - activity: { outputActive: true }, + await connect(fixture); + const stopped = fixture.conversation.stop(); + fixture.emit(1, { + type: "input_audio_buffer.committed", + item_id: "late", + previous_item_id: null, }); - await vi.advanceTimersByTimeAsync(400); - fixture.emit({ - type: "session.output_transcript.delta", - delta: "Not playback", + fixture.emit(1, { + type: "conversation.item.input_audio_transcription.completed", + item_id: "late", + content_index: 0, + transcript: "Do not submit", }); - expect(fixture.onState.mock.lastCall?.[0]).toMatchObject({ - phase: "connected", - activity: { outputActive: false }, + expect(fixture.onFinalizedInput).not.toHaveBeenCalled(); + fixture.emit(0, { type: "session.closed" }); + await stopped; +}); + +test("provider item identities are namespaced to the consented local session", async () => { + const ids: string[] = []; + for (let index = 0; index < 2; index++) { + const fixture = setup(); + await connect(fixture); + fixture.emit(1, { + type: "input_audio_buffer.committed", + item_id: "one", + previous_item_id: null, + }); + fixture.emit(1, { + type: "conversation.item.input_audio_transcription.completed", + item_id: "one", + content_index: 0, + transcript: "Same words", + }); + ids.push(fixture.onFinalizedInput.mock.calls[0]![0].id); + fixture.emit(0, { type: "session.closed" }); + } + expect(ids[0]).not.toBe(ids[1]); +}); + +test("a conflicting committed successor fails rather than silently dropping a correction", async () => { + const fixture = setup(); + await connect(fixture); + fixture.emit(1, { + type: "input_audio_buffer.committed", + item_id: "one", + previous_item_id: null, }); - levels.output = 0.2; - fixture.audio.paused = true; - await vi.advanceTimersByTimeAsync(100); - expect(fixture.onState.mock.lastCall?.[0]).toMatchObject({ - activity: { outputActive: false }, + fixture.emit(1, { + type: "input_audio_buffer.committed", + item_id: "two", + previous_item_id: "one", }); - fixture.audio.paused = false; - fixture.audio.muted = true; - await vi.advanceTimersByTimeAsync(100); - expect(fixture.onState.mock.lastCall?.[0]).toMatchObject({ - activity: { outputActive: false }, + fixture.emit(1, { + type: "input_audio_buffer.committed", + item_id: "three", + previous_item_id: "one", }); - expect(fixture.sent).toEqual([]); - expect(fixture.fetch).toHaveBeenCalledTimes(1); - const stopped = fixture.conversation.stop(); - const samples = getStats.mock.calls.length; - await vi.advanceTimersByTimeAsync(100); - expect(getStats).toHaveBeenCalledTimes(samples); - fixture.emit({ type: "session.closed" }); - await stopped; + fixture.emit(0, { type: "session.closed" }); + expect(fixture.onState.mock.lastCall?.[0].phase).toBe("error"); }); -test("missing or failed telemetry does not end the session; a late sample cannot revive it after Stop", async () => { - vi.useFakeTimers(); +test("permission resolving after Stop cannot start either provider", async () => { + const fixture = setup(); + let release = (_stream: typeof fixture.stream) => {}; + fixture.getUserMedia.mockImplementationOnce( + () => + new Promise((resolve) => { + release = resolve; + }), + ); + const start = fixture.conversation.start(); + await fixture.conversation.stop(); + release(fixture.stream); + await start; + expect(fixture.input.stop).toHaveBeenCalledOnce(); + expect(fixture.fetch).not.toHaveBeenCalled(); +}); + +test.each([0, 1] as const)( + "identifies the unfinished connection after the other session is ready: %s", + async (readyConnection) => { + vi.useFakeTimers(); + const fixture = setup(); + await fixture.conversation.start(); + fixture.emit(readyConnection, { + type: readyConnection === 0 ? "session.started" : "session.created", + }); + await vi.advanceTimersByTimeAsync(15_000); + const message = fixture.onState.mock.lastCall?.[0].message; + expect(message).toContain( + readyConnection === 0 + ? "transcription: waiting for session.created" + : "live: waiting for session.started", + ); + expect(message).not.toContain( + readyConnection === 0 ? "live: waiting" : "transcription: waiting", + ); + expect(fixture.fetch).toHaveBeenCalledTimes(2); + }, +); + +test("reports the failed endpoint and HTTP statuses without reflecting response content", async () => { + const warning = vi.spyOn(console, "warn").mockImplementation(() => {}); const fixture = setup(); - const getStats = vi.fn(async (): Promise> => new Map()); - Object.assign(fixture.peer, { getStats }); + fixture.fetch.mockImplementation(async (url) => + url.endsWith("live-session") + ? new Response("sensitive provider response", { + status: 502, + headers: { "x-voice-upstream-status": "401" }, + }) + : new Promise(() => {}), + ); await fixture.conversation.start(); - fixture.emit({ type: "session.started" }); + expect(fixture.onState.mock.lastCall?.[0].phase).toBe("error"); + expect(fixture.onState.mock.lastCall?.[0].message).toContain( + "live session request failed (HTTP 502, provider HTTP 401)", + ); + expect(fixture.onState.mock.lastCall?.[0].message).not.toContain("sensitive"); + expect(warning).toHaveBeenCalledExactlyOnceWith( + "[Petrinaut Live]", + "live session request failed (HTTP 502, provider HTTP 401). No automatic retry was made.", + ); + expect(fixture.input.stop).toHaveBeenCalledOnce(); + expect( + fixture.peers.every((peer) => peer.close.mock.calls.length === 1), + ).toBe(true); +}); + +test("timeout identifies a pending transcription HTTP response rather than blaming microphone permission", async () => { + vi.useFakeTimers(); + const warning = vi.spyOn(console, "warn").mockImplementation(() => {}); + const fixture = setup(); + fixture.fetch.mockImplementation(async (url) => + url.endsWith("transcription-session") + ? new Promise(() => {}) + : Response.json({ sdp: "v=0\r\no=answer" }), + ); + void fixture.conversation.start(); + await vi.advanceTimersByTimeAsync(15_000); + expect(fixture.onState.mock.lastCall?.[0].message).toContain( + "transcription: waiting for session HTTP response", + ); + expect(warning).toHaveBeenCalledExactlyOnceWith( + "[Petrinaut Live]", + expect.stringContaining("transcription: waiting for session HTTP response"), + ); + expect(fixture.input.stop).toHaveBeenCalledOnce(); +}); + +test("both connection timeout and unconfirmed closure remain bounded without retry", async () => { + vi.useFakeTimers(); + const timedOut = setup(); + await timedOut.conversation.start(); + await vi.advanceTimersByTimeAsync(15_000); + expect(timedOut.onState.mock.lastCall?.[0].phase).toBe("error"); + expect(timedOut.onState.mock.lastCall?.[0].message).toContain("timed out"); + expect(timedOut.fetch).toHaveBeenCalledTimes(2); + const fixture = setup(); + await connect(fixture); + const stop = fixture.conversation.stop(); + await vi.advanceTimersByTimeAsync(2_000); + await stop; + expect(fixture.onState.mock.lastCall?.[0].message).toContain("not confirmed"); + expect(fixture.fetch).toHaveBeenCalledTimes(2); +}); + +test("microphone loss shuts both transports and does not expose provider error text", async () => { + const fixture = setup(); + await connect(fixture); + fixture.input.dispatchEvent(new Event("ended")); + fixture.emit(0, { type: "error", error: { message: "secret" } }); + expect(fixture.audio.muted).toBe(true); + fixture.emit(0, { type: "session.closed" }); + expect(fixture.onState.mock.lastCall?.[0].phase).toBe("error"); + expect(fixture.onState.mock.lastCall?.[0].message).toContain( + "Microphone disconnected", + ); + expect(fixture.onState.mock.lastCall?.[0].message).not.toContain("secret"); + expect(fixture.getUserMedia).toHaveBeenCalledOnce(); +}); + +test("telemetry shows activity but silence and late samples never settle or revive speech", async () => { + vi.useFakeTimers(); + const fixture = setup(); + const getStats = vi.fn( + async (): Promise> => + new Map([ + ["input", { type: "media-source", kind: "audio", audioLevel: 0.24 }], + ["output", { type: "inbound-rtp", kind: "audio", audioLevel: 0.2 }], + ]), + ); + Object.assign(fixture.peers[0]!, { getStats }); + await connect(fixture); + fixture.peers[0]!.dispatchEvent( + Object.assign(new Event("track"), { + track: fixture.outputs[0], + streams: [fixture.stream], + }), + ); await vi.advanceTimersByTimeAsync(100); - expect(fixture.onState.mock.lastCall?.[0]).toMatchObject({ - phase: "connected", - activity: { microphoneLevel: 0, outputActive: false }, + expect(fixture.onState.mock.lastCall?.[0].activity).toEqual({ + microphoneLevel: 0.24, + outputActive: true, }); - getStats.mockRejectedValueOnce(new Error("Telemetry unavailable")); + getStats.mockResolvedValue(new Map()); + await vi.advanceTimersByTimeAsync(500); + expect(fixture.onState.mock.lastCall?.[0].activity?.outputActive).toBe(false); + expect(fixture.onFinalizedInput).not.toHaveBeenCalled(); + getStats.mockRejectedValueOnce(new Error("Optional telemetry failed")); await vi.advanceTimersByTimeAsync(100); expect(fixture.input.stop).not.toHaveBeenCalled(); - expect(fixture.sent).toEqual([]); - let release!: (stats: Map) => void; + let release = (_stats: Map) => {}; getStats.mockImplementationOnce( () => new Promise((resolve) => { @@ -541,18 +1063,11 @@ test("missing or failed telemetry does not end the session; a late sample cannot }), ); await vi.advanceTimersByTimeAsync(100); - const stopped = fixture.conversation.stop(); - fixture.emit({ type: "session.closed" }); - await stopped; - const updates = fixture.onState.mock.calls.length; - const samples = getStats.mock.calls.length; - release( - new Map([ - ["late", { type: "media-source", kind: "audio", audioLevel: 0.9 }], - ]), - ); + const stop = fixture.conversation.stop(); + fixture.emit(0, { type: "session.closed" }); + await stop; + const calls = fixture.onState.mock.calls.length; + release(new Map()); await vi.advanceTimersByTimeAsync(500); - expect(fixture.onState).toHaveBeenCalledTimes(updates); - expect(getStats).toHaveBeenCalledTimes(samples); - expect(fixture.fetch).toHaveBeenCalledTimes(1); + expect(fixture.onState).toHaveBeenCalledTimes(calls); }); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation.ts b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation.ts index 66cb3337722..d821cd22f05 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation.ts @@ -1,3 +1,5 @@ +import { logLiveDiagnostic } from "./shared/live-diagnostic"; + export interface LiveConversationState { readonly phase: | "idle" @@ -15,24 +17,60 @@ export interface LiveConversationState { }; } -/** One disposable Live session. No composer, tool, transcript or turn-settlement interface. */ +interface FinalizedInput { + readonly id: string; + readonly text: string; +} + +type ConnectionKind = "live" | "transcription"; + +export interface LiveAppendResult { + readonly eventId: string; + readonly kind: "commentary" | "instructions"; + readonly delegationId: string | null; + /** Unknown means sent locally, but provider acceptance is not yet confirmed. */ + readonly status: "local-failure" | "unknown" | "accepted" | "rejected"; +} + +/** One disposable Live plus transcription session sharing one consented capture. */ export const createLiveConversation = ( onState: (state: LiveConversationState) => void, connectionTimeoutMs: number, + onFinalizedInput: (input: FinalizedInput) => void, + onDelegation: (delegationId: string) => void, + onAppendResult: (result: LiveAppendResult) => void, ) => { const abort = new AbortController(); - let peer: RTCPeerConnection | undefined; - let channel: RTCDataChannel | undefined; + const sessionId = crypto.randomUUID(); + const seenDelegations = new Set(); + const openDelegations = new Set(); + const pendingAppends = new Map(); + const peers = new Map(); + const channels = new Map(); + const ready = new Set(); + const recoveryTimers = new Map< + ConnectionKind, + ReturnType + >(); + const connectionStages = new Map(); + const connectionProgress = () => + connectionStages.size === 0 + ? "waiting for microphone access" + : [...connectionStages] + .filter(([kind]) => !ready.has(kind)) + .map(([kind, stage]) => `${kind}: ${stage}`) + .join("; "); + const committedPrevious = new Map(); + const completed = new Map(); + const emitted = new Set(); let microphone: MediaStream | undefined; let audio: HTMLAudioElement | undefined; let started = false; - let ready = false; - let recovering = false; let playbackBlocked = false; let playbackAttempt = 0; let stopping = false; let finished = false; - let creationRequested = false; + let liveCreationRequested = false; let failure: string | undefined; let connectionTimer: ReturnType | undefined; let closeTimer: ReturnType | undefined; @@ -56,76 +94,48 @@ export const createLiveConversation = ( ...(activity ? { activity } : {}), }); - const sampleActivity = async () => { - if (stopping || !peer) return; - let microphoneLevel = 0; - let outputLevel = 0; - try { - const stats = await peer.getStats(); - stats.forEach((report: unknown) => { - if ( - typeof report !== "object" || - report === null || - !("kind" in report) || - report.kind !== "audio" || - !("audioLevel" in report) || - typeof report.audioLevel !== "number" || - !("type" in report) - ) - return; - if (report.type === "media-source") microphoneLevel = report.audioLevel; - if (report.type === "inbound-rtp") outputLevel = report.audioLevel; - }); - } catch { - // Optional browser telemetry must not terminate or retry the conversation. - } - if (abort.signal.aborted) return; - activityTimer = setTimeout(() => void sampleActivity(), 100); - if (recovering) return; - const playing = audio?.srcObject && !audio.paused && !audio.muted; - if (playing && outputLevel > 0.01) lastOutputActivity = Date.now(); - const activity = { - microphoneLevel: Math.round(microphoneLevel * 100) / 100, - // Brief hold avoids flicker between syllables. This never settles a turn; - // received audio energy also cannot prove that the user heard playback. - outputActive: Boolean(playing) && Date.now() - lastOutputActivity < 300, - }; - if ( - !lastActivity || - activity.microphoneLevel !== lastActivity.microphoneLevel || - activity.outputActive !== lastActivity.outputActive - ) { - lastActivity = activity; - onState(activeState("connected", activity)); - } + const reportAppendResult = (result: LiveAppendResult) => { + logLiveDiagnostic("append.result", { sessionId, ...result }); + onAppendResult(result); }; const stopMedia = () => { + pendingAppends.clear(); + openDelegations.clear(); clearTimeout(activityTimer); + recoveryTimers.forEach((timer) => clearTimeout(timer)); + recoveryTimers.clear(); microphone?.getTracks().forEach((track) => track.stop()); if (audio) { audio.muted = true; audio.pause(); audio.srcObject = null; } - peer?.getReceivers().forEach((receiver) => receiver.track.stop()); + peers.forEach((peer) => + peer.getReceivers().forEach((receiver) => receiver.track.stop()), + ); }; - const finish = (confirmed: boolean) => { + const finish = (liveConfirmed: boolean) => { if (finished) return; + logLiveDiagnostic("session.finished", { + sessionId, + liveConfirmed, + failed: failure !== undefined, + }); finished = true; stopping = true; clearTimeout(connectionTimer); clearTimeout(closeTimer); abort.abort(); stopMedia(); - channel?.close(); - peer?.close(); - const closure = confirmed + channels.forEach((channel) => channel.close()); + peers.forEach((peer) => peer.close()); + const closure = liveConfirmed ? "Live confirmed session closure." - : creationRequested - ? "Remote session closure was not confirmed." - : "No provider session was requested."; + : liveCreationRequested + ? "Remote Live session closure was not confirmed." + : "No Live provider session was requested."; onState({ phase: failure ? "error" : "ended", message: `${failure ? `${failure} ` : ""}Microphone and playback stopped. ${closure}`, @@ -135,19 +145,26 @@ export const createLiveConversation = ( const stop = (): Promise => { if (stopping) return stopped; + logLiveDiagnostic("session.stopping", { + sessionId, + pendingAppends: pendingAppends.size, + openDelegations: openDelegations.size, + }); stopping = true; clearTimeout(connectionTimer); abort.abort(); stopMedia(); + channels.get("transcription")?.close(); + peers.get("transcription")?.close(); onState({ phase: "stopping", - message: "Microphone and playback stopped. Closing Live…", + message: `${failure ? `${failure} ` : ""}Microphone and playback stopped. Closing Live…`, }); - if (channel?.readyState === "open" && ready) { - // Registered message listener remains until session.closed or this cleanup deadline. + const liveChannel = channels.get("live"); + if (liveChannel?.readyState === "open" && ready.has("live")) { closeTimer = setTimeout(() => finish(false), 2_000); try { - channel.send(JSON.stringify({ type: "session.close" })); + liveChannel.send(JSON.stringify({ type: "session.close" })); } catch { finish(false); } @@ -159,8 +176,377 @@ export const createLiveConversation = ( const fail = (message: string) => { if (stopping) return; + logLiveDiagnostic("session.failure", { sessionId, reason: message }); + // Only application-authored errors reach this function, never raw provider + // events, SDP, credentials or transcripts. Preserve the first failure even + // when the panel is collapsed or no longer showing the consent card. + // oxlint-disable-next-line no-console -- local connection diagnostics for the manual experiment. + console.warn("[Petrinaut Live]", message); failure = message; - void stop(); + if (ready.has("live")) void stop(); + else finish(false); + }; + + const handleConnectionState = (kind: ConnectionKind) => { + if (stopping) return; + const state = peers.get(kind)?.connectionState; + logLiveDiagnostic("connection.state", { + sessionId, + connection: kind, + state, + }); + const label = kind === "live" ? "Live" : "Transcription"; + if (state === "failed" || state === "closed") { + fail(`${label} media connection ended.`); + } else if ( + state === "disconnected" && + ready.size === 2 && + !recoveryTimers.has(kind) + ) { + // ICE can recover on the existing peer. Each peer keeps its own fixed + // deadline; neither recovery creates a session or replays input/output. + recoveryTimers.set( + kind, + setTimeout( + () => + fail( + `${label} media connection did not recover. No automatic retry was made.`, + ), + connectionTimeoutMs, + ), + ); + lastActivity = undefined; + lastOutputActivity = -Infinity; + onState(activeState("connecting")); + } else if (state === "connected" && recoveryTimers.has(kind)) { + clearTimeout(recoveryTimers.get(kind)); + recoveryTimers.delete(kind); + if (recoveryTimers.size === 0) + onState(activeState("connected")); + } + }; + + const sampleActivity = async () => { + const livePeer = peers.get("live"); + if (stopping || !livePeer) return; + let microphoneLevel = 0; + let outputLevel = 0; + try { + const stats = await livePeer.getStats(); + stats.forEach((report: unknown) => { + if ( + typeof report !== "object" || + report === null || + !("kind" in report) || + report.kind !== "audio" || + !("audioLevel" in report) || + typeof report.audioLevel !== "number" || + !("type" in report) + ) + return; + if (report.type === "media-source") microphoneLevel = report.audioLevel; + if (report.type === "inbound-rtp") outputLevel = report.audioLevel; + }); + } catch { + // Optional telemetry must not affect the session lifetime. + } + if (abort.signal.aborted) return; + activityTimer = setTimeout(() => void sampleActivity(), 100); + if (recoveryTimers.size > 0) return; + const playing = audio?.srcObject && !audio.paused && !audio.muted; + if (playing && outputLevel > 0.01) lastOutputActivity = Date.now(); + const activity = { + microphoneLevel: Math.round(microphoneLevel * 100) / 100, + outputActive: Boolean(playing) && Date.now() - lastOutputActivity < 300, + }; + if ( + !lastActivity || + activity.microphoneLevel !== lastActivity.microphoneLevel || + activity.outputActive !== lastActivity.outputActive + ) { + lastActivity = activity; + onState(activeState("connected", activity)); + } + }; + + const committedAfter = (previousItemId: string) => + [...committedPrevious].find( + ([, previous]) => previous === previousItemId, + )?.[0]; + + const flushFinalizedInputs = () => { + if (stopping || ready.size !== 2) return; + const roots = [...committedPrevious].filter( + ([, previous]) => previous === null, + ); + if (roots.length > 1) { + fail( + "Transcription item ordering conflicted. No automatic retry was made.", + ); + return; + } + let itemId = roots[0]?.[0]; + while (itemId && !abort.signal.aborted) { + const input = completed.get(itemId); + if (!input) return; + if (!emitted.has(input.id)) { + emitted.add(input.id); + logLiveDiagnostic("input.finalized", { + sessionId, + itemId, + inputId: input.id, + characters: input.text.length, + }); + onFinalizedInput(input); + } + itemId = committedAfter(itemId); + } + }; + + const markReady = (kind: ConnectionKind) => { + ready.add(kind); + logLiveDiagnostic("connection.ready", { sessionId, connection: kind }); + if (ready.size !== 2) return; + clearTimeout(connectionTimer); + // A peer may have disconnected before the last session-ready event. + peers.forEach((_, connectionKind) => handleConnectionState(connectionKind)); + if (stopping) return; + if (recoveryTimers.size === 0) + onState(activeState("connected")); + activityTimer = setTimeout(() => void sampleActivity(), 100); + flushFinalizedInputs(); + }; + + const handleTranscriptionEvent = (data: Record) => { + if (data.type === "session.created" || data.type === "session.updated") { + if (!ready.has("transcription")) markReady("transcription"); + return; + } + if (data.type === "input_audio_buffer.committed") { + if ( + typeof data.item_id !== "string" || + !( + data.previous_item_id === null || + typeof data.previous_item_id === "string" + ) + ) { + fail("Transcription sent an invalid committed item."); + return; + } + const previous = committedPrevious.get(data.item_id); + const conflictingSuccessor = [...committedPrevious].some( + ([id, predecessor]) => + id !== data.item_id && predecessor === data.previous_item_id, + ); + let ancestor: string | null | undefined = data.previous_item_id; + const ancestors = new Set([data.item_id]); + while ( + ancestor !== null && + ancestor !== undefined && + !ancestors.has(ancestor) + ) { + ancestors.add(ancestor); + ancestor = committedPrevious.get(ancestor); + } + if ( + (previous !== undefined && previous !== data.previous_item_id) || + conflictingSuccessor || + (ancestor !== null && ancestor !== undefined) + ) { + fail( + "Transcription item ordering conflicted. No automatic retry was made.", + ); + return; + } + committedPrevious.set(data.item_id, data.previous_item_id); + flushFinalizedInputs(); + return; + } + if (data.type === "conversation.item.input_audio_transcription.completed") { + if ( + typeof data.item_id !== "string" || + data.content_index !== 0 || + typeof data.transcript !== "string" + ) { + fail("Transcription sent an invalid completed item."); + return; + } + const input = { + id: `voice-live:${sessionId}:${encodeURIComponent(data.item_id)}:${data.content_index}`, + text: data.transcript, + }; + const existing = completed.get(data.item_id); + if ( + existing && + (existing.id !== input.id || existing.text !== input.text) + ) { + fail("Transcription identity conflicted. No automatic retry was made."); + return; + } + completed.set(data.item_id, input); + flushFinalizedInputs(); + return; + } + if ( + data.type === "conversation.item.input_audio_transcription.failed" || + data.type === "error" || + data.type === "session.error" + ) + fail("Transcription failed. No fallback or automatic retry was made."); + }; + + const parseEvent = (kind: ConnectionKind, event: MessageEvent) => { + if (finished) return; + let data: unknown; + try { + data = JSON.parse(event.data); + } catch { + fail( + `${kind === "live" ? "Live" : "Transcription"} sent an unreadable event.`, + ); + return; + } + if (typeof data !== "object" || data === null || !("type" in data)) return; + if (stopping && !(kind === "live" && data.type === "session.closed")) + return; + if ( + import.meta.env.DEV && + typeof data.type === "string" && + [ + "session.started", + "session.created", + "session.updated", + "session.closed", + "session.delegation.created", + "session.commentary.appended", + "session.instructions.appended", + "input_audio_buffer.speech_started", + "input_audio_buffer.speech_stopped", + "input_audio_buffer.committed", + "conversation.item.input_audio_transcription.completed", + "conversation.item.input_audio_transcription.failed", + "error", + "session.error", + ].includes(data.type) + ) { + const fields = data as Record; + const delegation = + typeof fields.delegation === "object" && fields.delegation !== null + ? (fields.delegation as Record) + : undefined; + // Receipt time and provider offsets are observations, not a shared audio clock. + // Never spread provider data: transcripts, metadata, SDP and errors may contain secrets. + logLiveDiagnostic("provider.event", { + sessionId, + connection: kind, + type: data.type, + itemId: typeof fields.item_id === "string" ? fields.item_id : undefined, + previousItemId: + typeof fields.previous_item_id === "string" || + fields.previous_item_id === null + ? fields.previous_item_id + : undefined, + contentIndex: + typeof fields.content_index === "number" + ? fields.content_index + : undefined, + audioStartMs: + typeof fields.audio_start_ms === "number" + ? fields.audio_start_ms + : undefined, + audioEndMs: + typeof fields.audio_end_ms === "number" + ? fields.audio_end_ms + : undefined, + offsetMs: + typeof fields.offset_ms === "number" ? fields.offset_ms : undefined, + delegationId: + typeof delegation?.id === "string" ? delegation.id : undefined, + target: + typeof delegation?.target === "string" + ? delegation.target + : undefined, + clientEventId: + typeof fields.client_event_id === "string" + ? fields.client_event_id + : undefined, + }); + } + if (kind === "transcription") { + handleTranscriptionEvent(data as Record); + return; + } + if (data.type === "session.closed") { + finish(true); + } else if ( + !stopping && + data.type === "session.started" && + !ready.has("live") + ) { + markReady("live"); + } else if (data.type === "session.delegation.created") { + if (!("delegation" in data)) return; + const { delegation } = data; + if ( + typeof delegation !== "object" || + delegation === null || + !("target" in delegation) || + delegation.target !== "client" || + !("id" in delegation) || + typeof delegation.id !== "string" || + !delegation.id || + seenDelegations.has(delegation.id) + ) + return; + seenDelegations.add(delegation.id); + openDelegations.add(delegation.id); + logLiveDiagnostic("delegation.accepted", { + sessionId, + delegationId: delegation.id, + }); + onDelegation(delegation.id); + } else if ( + data.type === "session.commentary.appended" || + data.type === "session.instructions.appended" + ) { + if ( + !("client_event_id" in data) || + typeof data.client_event_id !== "string" + ) + return; + const pending = pendingAppends.get(data.client_event_id); + if (!pending || data.type !== `session.${pending.kind}.appended`) return; + pendingAppends.delete(pending.eventId); + if (pending.delegationId !== null) + openDelegations.delete(pending.delegationId); + reportAppendResult({ ...pending, status: "accepted" }); + } else if ( + !stopping && + (data.type === "error" || data.type === "session.error") + ) { + let clientEventId = + "client_event_id" in data && typeof data.client_event_id === "string" + ? data.client_event_id + : undefined; + if ( + "error" in data && + typeof data.error === "object" && + data.error !== null && + "client_event_id" in data.error && + typeof data.error.client_event_id === "string" + ) { + clientEventId = data.error.client_event_id; + } + if (clientEventId !== undefined) { + const pending = pendingAppends.get(clientEventId); + // A stale or unrelated command error cannot reject another append. + if (!pending) return; + pendingAppends.delete(pending.eventId); + reportAppendResult({ ...pending, status: "rejected" }); + return; + } + fail("Live reported an error. No automatic retry was made."); + } }; const playAudio = async (): Promise => { @@ -174,7 +560,9 @@ export const createLiveConversation = ( playbackBlocked = true; onState( activeState( - ready && !recovering ? "connected" : "connecting", + ready.size === 2 && recoveryTimers.size === 0 + ? "connected" + : "connecting", lastActivity, ), ); @@ -185,18 +573,131 @@ export const createLiveConversation = ( playbackBlocked = false; onState( activeState( - ready && !recovering ? "connected" : "connecting", + ready.size === 2 && recoveryTimers.size === 0 + ? "connected" + : "connecting", lastActivity, ), ); }; + const createConnection = async ( + kind: ConnectionKind, + stream: MediaStream, + ) => { + abort.signal.throwIfAborted(); + connectionStages.set(kind, "creating local WebRTC offer"); + const connection = new RTCPeerConnection(); + peers.set(kind, connection); + const channel = connection.createDataChannel("oai-events"); + channels.set(kind, channel); + channel.addEventListener("message", (event: MessageEvent) => + parseEvent(kind, event), + ); + channel.addEventListener("close", () => { + if (!stopping) + fail(`${kind === "live" ? "Live" : "Transcription"} disconnected.`); + }); + channel.addEventListener("error", () => + fail( + `${kind === "live" ? "Live" : "Transcription"} data connection failed.`, + ), + ); + connection.addEventListener("connectionstatechange", () => + handleConnectionState(kind), + ); + if (kind === "live") { + connection.addEventListener("track", (event) => { + if (stopping) { + event.track.stop(); + return; + } + if (!audio) return; + audio.srcObject = event.streams[0] ?? new MediaStream([event.track]); + void playAudio(); + }); + } + stream.getTracks().forEach((track) => connection.addTrack(track, stream)); + await connection.setLocalDescription(await connection.createOffer()); + abort.signal.throwIfAborted(); + connectionStages.set(kind, "gathering ICE candidates"); + if (connection.iceGatheringState !== "complete") { + await new Promise((resolve, reject) => { + const check = () => { + if (connection.iceGatheringState === "complete") resolve(); + }; + connection.addEventListener("icegatheringstatechange", check, { + signal: abort.signal, + }); + abort.signal.addEventListener( + "abort", + () => reject(abort.signal.reason), + { + once: true, + }, + ); + check(); + }); + } + abort.signal.throwIfAborted(); + const sdp = connection.localDescription?.sdp; + if (!sdp) throw new Error("Missing local SDP"); + if (kind === "live") liveCreationRequested = true; + connectionStages.set(kind, "waiting for session HTTP response"); + const response = await fetch(`/api/voice/${kind}-session`, { + method: "POST", + headers: { "content-type": "application/sdp" }, + body: sdp, + signal: abort.signal, + }); + logLiveDiagnostic("connection.http", { + sessionId, + connection: kind, + status: response.status, + }); + if (!response.ok) { + // Only expose numeric status metadata, never provider text, credentials or SDP. + const upstreamStatus = response.headers.get("x-voice-upstream-status"); + const providerStatus = + upstreamStatus && /^[45]\d{2}$/u.test(upstreamStatus) + ? `, provider HTTP ${upstreamStatus}` + : ""; + fail( + `${kind} session request failed (HTTP ${response.status}${providerStatus}). No automatic retry was made.`, + ); + throw new Error("Session creation failed"); + } + connectionStages.set(kind, "reading session HTTP response"); + const answer: unknown = await response.json(); + abort.signal.throwIfAborted(); + if ( + typeof answer !== "object" || + answer === null || + !("sdp" in answer) || + typeof answer.sdp !== "string" || + !answer.sdp.trimStart().startsWith("v=0") + ) + throw new Error("Invalid SDP answer"); + connectionStages.set(kind, "applying remote SDP answer"); + await connection.setRemoteDescription({ type: "answer", sdp: answer.sdp }); + connectionStages.set( + kind, + kind === "live" + ? "waiting for session.started" + : "waiting for session.created", + ); + }; + const start = async (): Promise => { if (started || stopping) return; started = true; + logLiveDiagnostic("session.starting", { sessionId }); onState(activeState("connecting")); connectionTimer = setTimeout( - () => fail("Live connection timed out. No automatic retry was made."), + () => + fail( + `Voice connections timed out (${connectionProgress()}). No automatic retry was made.`, + ), connectionTimeoutMs, ); try { @@ -208,149 +709,74 @@ export const createLiveConversation = ( return; } microphone = stream; - peer = new RTCPeerConnection(); - const connection = peer; - const handleConnectionState = () => { - if (stopping) return; - if (["failed", "closed"].includes(connection.connectionState)) { - fail("Live media connection ended."); - } else if ( - connection.connectionState === "disconnected" && - ready && - !recovering - ) { - // ICE can recover on this connection. Do not create a session or replay input. - recovering = true; - lastActivity = undefined; - lastOutputActivity = -Infinity; - onState(activeState("connecting")); - connectionTimer = setTimeout( - () => - fail( - "Live media connection did not recover. No automatic retry was made.", - ), - connectionTimeoutMs, - ); - } else if (connection.connectionState === "connected" && recovering) { - recovering = false; - clearTimeout(connectionTimer); - onState(activeState("connected")); - } - }; - channel = connection.createDataChannel("oai-events"); - channel.addEventListener("message", (event: MessageEvent) => { - if (finished) return; - let data: unknown; - try { - data = JSON.parse(event.data); - } catch { - fail("Live sent an unreadable event."); - return; - } - if (typeof data !== "object" || data === null || !("type" in data)) - return; - if (data.type === "session.closed") { - finish(true); - return; - } - if (stopping) return; - if (data.type === "session.started" && !ready) { - ready = true; - clearTimeout(connectionTimer); - activityTimer = setTimeout(() => void sampleActivity(), 100); - onState(activeState("connected")); - handleConnectionState(); - } else if (data.type === "error" || data.type === "session.error") { - fail("Live reported an error. No automatic retry was made."); - } - // Transcript deltas are not finalized utterances. Delegations contain metadata, - // not task text. Neither is forwarded, persisted, or used to execute anything. - }); - channel.addEventListener("close", () => { - if (!stopping) failure = "Live disconnected."; - finish(false); - }); - channel.addEventListener("error", () => - fail("Live data connection failed."), - ); - connection.addEventListener( - "connectionstatechange", - handleConnectionState, - ); - connection.addEventListener("track", (event) => { - if (stopping) { - event.track.stop(); - return; - } - if (!audio) return; - audio.srcObject = event.streams[0] ?? new MediaStream([event.track]); - void playAudio(); - }); - stream.getTracks().forEach((track) => { + stream.getTracks().forEach((track) => track.addEventListener( "ended", () => fail("Microphone disconnected."), { signal: abort.signal, }, - ); - connection.addTrack(track, stream); - }); - await connection.setLocalDescription(await connection.createOffer()); - abort.signal.throwIfAborted(); - if (connection.iceGatheringState !== "complete") { - const listeners = new AbortController(); - await new Promise((resolve, reject) => { - const check = () => { - if (connection.iceGatheringState === "complete") { - resolve(); - } - }; - const cancelled = () => { - reject(abort.signal.reason); - }; - connection.addEventListener("icegatheringstatechange", check, { - signal: listeners.signal, - }); - abort.signal.addEventListener("abort", cancelled, { - once: true, - signal: listeners.signal, - }); - check(); - }).finally(() => listeners.abort()); - } - abort.signal.throwIfAborted(); - const sdp = connection.localDescription?.sdp; - if (!sdp) throw new Error("Missing local SDP"); - creationRequested = true; - const response = await fetch("/api/voice/live-session", { - method: "POST", - headers: { "content-type": "application/sdp" }, - body: sdp, - signal: abort.signal, - }); - if (!response.ok) throw new Error("Session creation failed"); - const answer: unknown = await response.json(); - abort.signal.throwIfAborted(); - if ( - typeof answer !== "object" || - answer === null || - !("sdp" in answer) || - typeof answer.sdp !== "string" || - !answer.sdp.trimStart().startsWith("v=0") - ) - throw new Error("Invalid SDP answer"); - await connection.setRemoteDescription({ - type: "answer", - sdp: answer.sdp, - }); - // WebRTC creation already starts Live. Wait for session.started; never send session.start. + ), + ); + await Promise.all([ + createConnection("live", stream), + createConnection("transcription", stream), + ]); } catch { fail( - "Live could not connect. Check microphone, audio permissions and server configuration. No automatic retry was made.", + `Voice could not connect (${connectionProgress()}). No automatic retry was made.`, + ); + } + }; + + const append = ( + kind: LiveAppendResult["kind"], + text: string, + delegationId: string | null, + ): boolean => { + if (stopping) return false; + const result: LiveAppendResult = { + eventId: crypto.randomUUID(), + kind, + delegationId, + status: "unknown", + }; + const liveChannel = channels.get("live"); + if ( + ready.size !== 2 || + liveChannel?.readyState !== "open" || + !text.trim() + ) { + reportAppendResult({ ...result, status: "local-failure" }); + return false; + } + pendingAppends.set(result.eventId, result); + try { + liveChannel.send( + JSON.stringify({ + type: `session.${kind}.append`, + event_id: result.eventId, + delegation_id: delegationId, + content: text, + }), ); + } catch { + pendingAppends.delete(result.eventId); + reportAppendResult({ ...result, status: "local-failure" }); + return false; } + reportAppendResult(result); + return true; }; - return { retryPlayback: playAudio, start, stop }; + return { + retryPlayback: playAudio, + start, + stop, + openDelegations: openDelegations as ReadonlySet, + appendCommentary: (text: string, delegationId: string | null) => + append("commentary", text, delegationId), + appendInstructions: (text: string, delegationId: string | null) => + append("instructions", text, delegationId), + }; }; diff --git a/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.ts b/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.ts index ef5a7f2b41a..a0c379739b4 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.ts @@ -27,7 +27,7 @@ import type { export type VoiceSubmissionSettlement = Pick< FlueConversationSettlement, - "outcome" | "submissionId" + "outcome" | "submissionId" | "answeredBySubmissionId" >; export interface CancelPendingSpeechOptions { diff --git a/apps/petrinaut-website/src/main/app/voice-interview/shared/live-diagnostic.ts b/apps/petrinaut-website/src/main/app/voice-interview/shared/live-diagnostic.ts new file mode 100644 index 00000000000..590b6ea9a22 --- /dev/null +++ b/apps/petrinaut-website/src/main/app/voice-interview/shared/live-diagnostic.ts @@ -0,0 +1,14 @@ +/** Local development trace only. Pass explicit metadata, never payloads or text. */ +export const logLiveDiagnostic = ( + event: string, + metadata: Readonly< + Record + >, +): void => { + if (!import.meta.env.DEV) return; + // Serialize now so DevTools cannot show later mutations as earlier state. + // oxlint-disable-next-line no-console -- owner-requested, metadata-only local diagnosis. + console.debug( + `[Petrinaut Live trace] ${JSON.stringify({ at: new Date().toISOString(), event, ...metadata })}`, + ); +}; diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-browser-tools.integration.test.tsx b/apps/petrinaut-website/src/main/app/voice-interview/voice-browser-tools.integration.test.tsx index 01cb36c2206..158336faa8e 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-browser-tools.integration.test.tsx +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-browser-tools.integration.test.tsx @@ -1,7 +1,7 @@ /** @vitest-environment jsdom */ import { act, cleanup, render, waitFor } from "@testing-library/react"; import { useLayoutEffect } from "react"; -import { afterEach, expect, test, vi } from "vitest"; +import { afterEach, beforeAll, expect, test, vi } from "vitest"; import { createJsonDocHandle } from "@hashintel/petrinaut-core"; import { Petrinaut } from "@hashintel/petrinaut/ui"; @@ -47,6 +47,13 @@ vi.hoisted(() => { }); }); +beforeAll(async () => { + // The real panel loads Monaco lazily. Resolve its browser capability checks + // during setup, rather than letting an import failure race a later test. + // Cold transforms exceeded the default 10s when the full suite ran in parallel. + await import("monaco-editor"); +}, 30_000); + const VoiceObserver = ({ current, onUpdate, diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx index ee465471ad8..e73fea68f52 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx @@ -30,7 +30,7 @@ import { } from "./voice-turn-controller"; import type { CanonicalSpeechSegment } from "./canonical-speech"; -import type { AgentSendResult } from "@flue/sdk"; +import type { AgentSendResult, FlueConversationState } from "@flue/sdk"; import type { PetrinautAiVoiceModeContext } from "@hashintel/petrinaut/ui"; type ResolveSubmission = ( @@ -571,6 +571,7 @@ const PinnedVoiceInterviewControl = ({ resolveInputSubmission, resolveResponseSubmission, settlements, + snapshot, subscribeToAdmission, subscribeToAdmissionFailure, subscribeToResponseMessageCompleted, @@ -582,6 +583,7 @@ const PinnedVoiceInterviewControl = ({ readonly resolveInputSubmission?: ResolveSubmission; readonly resolveResponseSubmission?: ResolveSubmissions; readonly settlements?: readonly VoiceSubmissionSettlement[]; + readonly snapshot?: FlueConversationState; readonly subscribeToAdmission?: SubscribeToAdmission; readonly subscribeToAdmissionFailure?: SubscribeToAdmissionFailure; readonly subscribeToResponseMessageCompleted?: SubscribeToResponseMessageCompleted; @@ -595,15 +597,28 @@ const PinnedVoiceInterviewControl = ({ if (!context.registerVoiceModeSessionControls) return null; return ( + submitVoiceInputWithAdmission({ + input, + resolveInputSubmission, + subscribeToAdmission, + subscribeToAdmissionFailure, + submitVoiceInput: context.submitVoiceInput, + }) + } /> ); } diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-disclosure.tsx b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-disclosure.tsx index 0f0f22f63c5..26b3420b85c 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-disclosure.tsx +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-disclosure.tsx @@ -136,9 +136,7 @@ export const VoiceInterviewDisclosure = ({
- {experimental - ? "GPT-Live · Experimental interview" - : "Start a voice conversation"} + Start a voice conversation {!experimental && ( @@ -147,17 +145,16 @@ export const VoiceInterviewDisclosure = ({ )}
- {!experimental && ( -

- OpenAI processes live audio and speaks the interviewer’s words. - Petrinaut saves finalized answers—not audio. -

- )} +

+ {experimental + ? "OpenAI processes microphone audio for voice and transcription. Petrinaut saves text, not audio." + : "OpenAI processes live audio and speaks the interviewer’s words. Petrinaut saves finalized answers—not audio."} +

- Exit experiment + Cancel )} diff --git a/apps/petrinaut-website/src/server/voice/openai-live-session.test.ts b/apps/petrinaut-website/src/server/voice/openai-live-session.test.ts index e066d2e7300..46b4f95ec7a 100644 --- a/apps/petrinaut-website/src/server/voice/openai-live-session.test.ts +++ b/apps/petrinaut-website/src/server/voice/openai-live-session.test.ts @@ -198,7 +198,7 @@ describe("Live configuration and session creation", () => { }, ); - test("creates one client-delegated WebRTC session with trusted instructions and no tools", async () => { + test("creates one client-delegated WebRTC session with Brunch-authoritative delivery instructions and no tools", async () => { const fetch = vi.fn(async () => Response.json( { @@ -229,7 +229,7 @@ describe("Live configuration and session creation", () => { delegation: { type: "client" }, store: false, instructions: expect.stringMatching( - /Interview approach:[\s\S]*Backchannel policy:[\s\S]*Interruption policy:[\s\S]*Delegation policy:\nBackend tools:\n- None\.[\s\S]*Delegate to the backend when:\n- Never in this experiment[\s\S]*Do not delegate to the backend when:[\s\S]*Never claim that anything was changed, executed, or saved\./, + /Backchannel policy:[\s\S]*Interruption policy:[\s\S]*Delegation policy:[\s\S]*Backend tools:[\s\S]*Brunch:[\s\S]*Delegate to the backend when:[\s\S]*Do not delegate to the backend when:[\s\S]*Delegate before giving an answer that depends on backend work\. Do not guess the result while waiting\.[\s\S]*Brunch is the sole authority[\s\S]*You have no tools[\s\S]*supplied settled Brunch context[\s\S]*best-effort/i, ) as unknown, audio: { output: { voice: "marin" } }, }, @@ -249,6 +249,7 @@ describe("Live configuration and session creation", () => { fetch, })(request()); expect(response.status).toBe(502); + expect(response.headers.get("x-voice-upstream-status")).toBe("429"); expect(await response.text()).not.toContain("server-only-secret"); expect(fetch).toHaveBeenCalledTimes(1); }); diff --git a/apps/petrinaut-website/src/server/voice/openai-live-session.ts b/apps/petrinaut-website/src/server/voice/openai-live-session.ts index b4e8d270221..f0c7ea3204b 100644 --- a/apps/petrinaut-website/src/server/voice/openai-live-session.ts +++ b/apps/petrinaut-website/src/server/voice/openai-live-session.ts @@ -1,50 +1,39 @@ import { getVoiceProvider } from "./openai-voice-config.js"; import { getOpenAIVoiceAvailability } from "./openai-voice-policy.js"; -const instructions = `You are a calm, curious process interviewer. The person speaking is -the domain expert. Help them explain how their process works and -what they want to understand or improve. +const instructions = `You are the voice of Brunch, a calm, friendly process interview assistant. +Speak warmly and naturally, at an unhurried pace. Be clear and direct, not overly +cheerful. If the person is frustrated, acknowledge it briefly and listen. -Speak naturally at an unhurried pace. Be clear and direct, not overly -cheerful. If they are unsure or frustrated, acknowledge it briefly -and make the next question easier to answer. +Backchannel policy: Use brief, sparse backchannels. Acknowledge naturally without +competing with the main response or repeating praise. Do not invent progress. -Interview approach: -Follow their active account and use their vocabulary. Learn their -purpose naturally, without restarting an intake if they have already -begun. Prefer walking through a recent concrete case. - -Notice triggers, sequence, decisions, dependencies, waiting, and -outcomes. Explore exceptions when they matter to the person's purpose. -These guide your attention; they are not a questionnaire. - -Deepen one thread with one focused question at a time. Do not supply -answers or invent precision. Accept “I don't know.” Restate only when -checking an important interpretation, not after every answer. -Keep routine contributions to one or two short sentences. -When the person wants to finish, open no new topic. - -Backchannel policy: Use moderate backchannels. Acknowledge naturally -without competing with the main response. Avoid repetitive praise. - -Interruption policy: Stop speaking when the user interrupts. Listen -to what they say. Follow their correction rather than finishing your -previous point. Keep listening while they pause to think. +Interruption policy: Stop speaking when the person interrupts. Listen to what they +say and follow later supplied corrections. Do not claim backend work was cancelled. Delegation policy: Backend tools: -- None. This standalone interview has no application, chat, model, - storage, or execution capabilities. +- Brunch: interviews the person about their process, asks substantive follow-up + questions, and builds or updates the model through authorized application work. Delegate to the backend when: -- Never in this experiment; no backend handles delegated work. +- The person describes their process or answers an interview question. +- They ask a substantive question or request building or updating the model. +- A correction changes the process or work already requested. Do not delegate to the backend when: -- Conducting the interview, clarifying an answer, or incorporating - a correction. +- The person greets you or only needs a brief listening acknowledgement. +- They ask you to repeat a still-current Brunch result already supplied. + +Delegate before giving an answer that depends on backend work. Do not guess the result while waiting. -If asked to operate the application, briefly explain that you cannot. -Never claim that anything was changed, executed, or saved.`; +Brunch is the sole authority for domain interpretation, substantive interview +questions and answers, application work, and completion. You have no tools. Do not +use or call tools or give independent substantive answers or follow-up questions. +Convey supplied settled Brunch context faithfully: preserve facts, quantities, +negation, uncertainty, corrections and Brunch-authored questions. Never present +unreported work as complete. These are best-effort speech policies, not mechanically +enforced boundaries or a guarantee of exact relay.`; /** Uses the existing website credential boundary; this switch is not authentication. */ export const createOpenAILiveSessionHandler = @@ -134,6 +123,7 @@ export const createOpenAILiveSessionHandler = return respond( "Live session creation failed. No automatic retry was made.", 502, + { "x-voice-upstream-status": String(upstream.status) }, ); } const answer: unknown = await upstream.json(); diff --git a/apps/petrinaut-website/src/server/voice/openai-transcription-session.test.ts b/apps/petrinaut-website/src/server/voice/openai-transcription-session.test.ts new file mode 100644 index 00000000000..265d0b773fe --- /dev/null +++ b/apps/petrinaut-website/src/server/voice/openai-transcription-session.test.ts @@ -0,0 +1,586 @@ +import { createCustomAbortControllerSignal } from "@whatwg-node/server"; +import { afterEach, describe, expect, test, vi } from "vitest"; + +import { createOpenAITranscriptionSessionHandler } from "./openai-transcription-session"; + +const environment = { + PETRINAUT_OPENAI_VOICE_ENABLED: "true", + PETRINAUT_VOICE_PROVIDER: "live", + OPENAI_VOICE_API_KEY: "server-only-secret", +}; + +const transcriptionSecret = () => + Response.json({ + value: "ephemeral-transcription-secret", + session: { type: "transcription" }, + }); + +const request = (overrides: RequestInit & { duplex?: "half" } = {}) => + new Request("https://petrinaut.test/api/voice/transcription-session", { + method: "POST", + headers: { + origin: "https://petrinaut.test", + "content-type": "application/sdp", + }, + body: "v=0\r\no=offer", + ...overrides, + }); + +describe("OpenAI transcription WebRTC session", () => { + afterEach(() => vi.restoreAllMocks()); + + test.each([ + [{ method: "GET", body: undefined }, 405], + [ + { + headers: { + origin: "https://attacker.test", + "content-type": "application/sdp", + }, + }, + 403, + ], + [ + { + headers: { + origin: "https://petrinaut.test", + "content-type": "application/json", + }, + }, + 415, + ], + [{ body: "not SDP" }, 400], + [{ body: "v=0" + "x".repeat(65_536) }, 413], + ] satisfies [RequestInit, number][])( + "rejects unsafe requests before fetching: %j", + async (overrides, status) => { + const fetch = vi.fn(); + const response = await createOpenAITranscriptionSessionHandler({ + environment, + fetch, + })(request(overrides)); + + expect(response.status).toBe(status); + expect(response.headers.get("cache-control")).toBe("no-store"); + expect(fetch).not.toHaveBeenCalled(); + }, + ); + + test.each([ + { PETRINAUT_VOICE_PROVIDER: undefined }, + { PETRINAUT_VOICE_PROVIDER: "realtime" }, + { PETRINAUT_OPENAI_VOICE_ENABLED: "false" }, + { OPENAI_VOICE_API_KEY: " " }, + ])( + "requires Live selection and existing enablement/credentials: %j", + async (override) => { + const fetch = vi.fn(); + const response = await createOpenAITranscriptionSessionHandler({ + environment: { ...environment, ...override }, + fetch, + })(request()); + + expect(response.status).toBe(404); + expect(fetch).not.toHaveBeenCalled(); + }, + ); + + test("configures transcription through a server-only client secret before exchanging raw SDP", async () => { + const log = vi.spyOn(console, "info").mockImplementation(() => {}); + const fetch = vi + .fn( + async () => + new Response("v=0\r\no=answer", { + headers: { "content-type": "application/sdp" }, + }), + ) + .mockResolvedValueOnce(transcriptionSecret()); + const response = await createOpenAITranscriptionSessionHandler({ + environment, + fetch, + })(request()); + + expect(response.status).toBe(201); + expect(await response.json()).toEqual({ sdp: "v=0\r\no=answer" }); + expect(fetch).toHaveBeenCalledTimes(2); + const [url, init] = fetch.mock.calls[0]!; + expect(url).toBe("https://api.openai.com/v1/realtime/client_secrets"); + expect(init?.method).toBe("POST"); + expect(new Headers(init?.headers).get("authorization")).toBe( + "Bearer server-only-secret", + ); + expect(new Headers(init?.headers).get("content-type")).toBe( + "application/json", + ); + expect(JSON.parse(init?.body as string)).toEqual({ + session: { + type: "transcription", + audio: { + input: { + transcription: { model: "gpt-4o-transcribe" }, + turn_detection: { type: "server_vad" }, + }, + }, + }, + }); + const [callUrl, callInit] = fetch.mock.calls[1]!; + expect(callUrl).toBe("https://api.openai.com/v1/realtime/calls"); + expect(callInit?.method).toBe("POST"); + expect(callInit?.body).toBe("v=0\r\no=offer"); + expect(new Headers(callInit?.headers).get("authorization")).toBe( + "Bearer ephemeral-transcription-secret", + ); + expect(new Headers(callInit?.headers).get("content-type")).toBe( + "application/sdp", + ); + expect(callInit?.signal).toBe(init?.signal); + expect(JSON.stringify(log.mock.calls)).not.toContain( + "ephemeral-transcription-secret", + ); + expect(JSON.stringify(log.mock.calls)).not.toContain("server-only-secret"); + }); + + test.each([ + {}, + { value: "" }, + { value: "ephemeral-transcription-secret", session: { type: "realtime" } }, + ])( + "does not create a call with an invalid or non-transcription credential: %j", + async (secret) => { + const fetch = vi + .fn() + .mockResolvedValueOnce(Response.json(secret)); + const response = await createOpenAITranscriptionSessionHandler({ + environment, + fetch, + })(request()); + expect(response.status).toBe(502); + expect(fetch).toHaveBeenCalledOnce(); + expect(await response.text()).not.toContain( + "ephemeral-transcription-secret", + ); + }, + ); + + test("does not proceed to SDP exchange when cancelled during credential creation", async () => { + const abort = new AbortController(); + const fetch = vi.fn(async () => { + abort.abort(); + return transcriptionSecret(); + }); + const response = await createOpenAITranscriptionSessionHandler({ + environment, + fetch, + })(request({ signal: abort.signal })); + expect(response.status).toBe(502); + expect(fetch).toHaveBeenCalledOnce(); + }); + + test("does not retry or expose credentials when the SDP exchange is rejected", async () => { + const log = vi.spyOn(console, "info").mockImplementation(() => {}); + const fetch = vi + .fn() + .mockResolvedValueOnce(transcriptionSecret()) + .mockResolvedValueOnce( + new Response("ephemeral-transcription-secret", { status: 403 }), + ); + const response = await createOpenAITranscriptionSessionHandler({ + environment, + fetch, + })(request()); + expect(response.status).toBe(502); + expect(response.headers.get("x-voice-upstream-status")).toBe("403"); + expect(fetch).toHaveBeenCalledTimes(2); + expect(await response.text()).not.toContain( + "ephemeral-transcription-secret", + ); + expect(JSON.stringify(log.mock.calls)).not.toContain( + "ephemeral-transcription-secret", + ); + }); + + test("sanitizes invalid provider responses and never retries", async () => { + const fetch = vi.fn( + async () => new Response("server-only-secret", { status: 429 }), + ); + const response = await createOpenAITranscriptionSessionHandler({ + environment, + fetch, + })(request()); + + expect(response.status).toBe(502); + expect(response.headers.get("x-voice-upstream-status")).toBe("429"); + expect(await response.text()).not.toContain("server-only-secret"); + expect(fetch).toHaveBeenCalledOnce(); + }); + + test.each([ + ["invalid_value", "session.audio.input.turn_detection.type"], + ["model_not_found", "session.audio.input.transcription.model"], + ["server-only-secret", "private-transcript"], + ])( + "logs only allowlisted credential rejection metadata: %s", + async (code, param) => { + const log = vi.spyOn(console, "info").mockImplementation(() => {}); + const fetch = vi.fn(async () => + Response.json( + { + error: { + code, + param, + message: "server-only-secret private-transcript", + }, + }, + { status: 400 }, + ), + ); + const response = await createOpenAITranscriptionSessionHandler({ + environment, + fetch, + })(request()); + + expect(response.status).toBe(502); + expect(response.headers.get("x-voice-upstream-status")).toBe("400"); + expect(fetch).toHaveBeenCalledOnce(); + const terminal: unknown = JSON.parse(log.mock.calls.at(-1)![1] as string); + expect(terminal).toMatchObject({ + event: "finished", + stage: "reading-transcription-secret", + upstreamErrorCode: + code === "server-only-secret" ? "unrecognized" : code, + upstreamErrorParam: + param === "private-transcript" ? "unrecognized" : param, + }); + const exposed = JSON.stringify(log.mock.calls) + (await response.text()); + expect(exposed).not.toContain("server-only-secret"); + expect(exposed).not.toContain("private-transcript"); + }, + ); + + test("local turn-detection diagnostics preserve the explanation and redact credentials before truncation", async () => { + const log = vi.spyOn(console, "info").mockImplementation(() => {}); + const explanation = + "This transcription model requires turn_detection to be null."; + const fetch = vi.fn(async () => + Response.json( + { + error: { + code: "invalid_value", + param: "session.audio.input.turn_detection", + message: `${explanation} server-only-secret sk-proj-testtoken ek-testtoken ${"x".repeat(1024)} server-only-secret`, + }, + }, + { status: 400 }, + ), + ); + const response = await createOpenAITranscriptionSessionHandler({ + environment: { ...environment, NODE_ENV: "development" }, + fetch, + })( + new Request("http://localhost:4915/api/voice/transcription-session", { + method: "POST", + headers: { + origin: "http://localhost:4915", + "content-type": "application/sdp", + }, + body: "v=0\r\no=offer", + }), + ); + + const terminal: unknown = JSON.parse(log.mock.calls.at(-1)![1] as string); + expect(terminal).toMatchObject({ + upstreamErrorMessage: + `${explanation} [redacted] [redacted] [redacted] ${"x".repeat(1024)}`.slice( + 0, + 1024, + ), + }); + const exposed = JSON.stringify(log.mock.calls); + expect(exposed).not.toContain("server-only-secret"); + expect(exposed).not.toContain("sk-proj-testtoken"); + expect(exposed).not.toContain("ek-testtoken"); + expect(await response.text()).toBe( + "Transcription credential creation failed.", + ); + expect(response.status).toBe(502); + expect(fetch).toHaveBeenCalledOnce(); + }); + + test.each([ + [ + "production", + "http://localhost:4915", + "invalid_value", + "session.audio.input.turn_detection", + ], + [ + "development", + "https://petrinaut.test", + "invalid_value", + "session.audio.input.turn_detection", + ], + [ + "development", + "http://localhost:4915", + "invalid_api_key", + "session.audio.input.turn_detection", + ], + [ + "development", + "http://localhost:4915", + "invalid_value", + "session.audio.input.transcription.model", + ], + ])( + "does not expose provider wording outside the local turn-detection rejection: %s %s %s %s", + async (nodeEnvironment, origin, code, param) => { + const log = vi.spyOn(console, "info").mockImplementation(() => {}); + const fetch = vi.fn(async () => + Response.json( + { + error: { code, param, message: "private-provider-wording" }, + }, + { status: 400 }, + ), + ); + const response = await createOpenAITranscriptionSessionHandler({ + environment: { ...environment, NODE_ENV: nodeEnvironment }, + fetch, + })( + new Request(`${origin}/api/voice/transcription-session`, { + method: "POST", + headers: { origin, "content-type": "application/sdp" }, + body: "v=0\r\no=offer", + }), + ); + + expect(JSON.stringify(log.mock.calls)).not.toContain( + "upstreamErrorMessage", + ); + expect(JSON.stringify(log.mock.calls)).not.toContain( + "private-provider-wording", + ); + expect(await response.text()).not.toContain("private-provider-wording"); + expect(fetch).toHaveBeenCalledOnce(); + }, + ); + + test("bounds rejected credential bodies and keeps the received HTTP status", async () => { + const log = vi.spyOn(console, "info").mockImplementation(() => {}); + const cancel = vi.fn(); + const fetch = vi.fn( + async () => + new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode("x".repeat(8193))); + }, + cancel, + }), + { status: 400 }, + ), + ); + const response = await createOpenAITranscriptionSessionHandler({ + environment, + fetch, + })(request()); + + expect(response.headers.get("x-voice-upstream-status")).toBe("400"); + expect(cancel).toHaveBeenCalledOnce(); + expect(JSON.parse(log.mock.calls.at(-1)![1] as string)).toMatchObject({ + upstreamErrorCode: "unrecognized", + upstreamErrorParam: "unrecognized", + }); + expect(fetch).toHaveBeenCalledOnce(); + }); + + test("uses the existing deadline for a stalled credential error body", async () => { + const log = vi.spyOn(console, "info").mockImplementation(() => {}); + const abort = new AbortController(); + vi.spyOn(AbortSignal, "timeout").mockReturnValue(abort.signal); + const fetch = vi.fn( + async () => new Response(new ReadableStream(), { status: 400 }), + ); + const pending = createOpenAITranscriptionSessionHandler({ + environment, + fetch, + })(request()); + await vi.waitFor(() => + expect(log).toHaveBeenCalledWith( + "[Petrinaut transcription]", + expect.stringContaining('"stage":"reading-transcription-secret"'), + ), + ); + abort.abort(); + + const response = await pending; + expect(response.headers.get("x-voice-upstream-status")).toBe("400"); + expect(JSON.parse(log.mock.calls.at(-1)![1] as string)).toMatchObject({ + upstreamStatus: 400, + timedOut: true, + upstreamErrorCode: "unrecognized", + }); + }); + + test("does not contact the provider after request cancellation", async () => { + const fetch = vi.fn(); + const response = await createOpenAITranscriptionSessionHandler({ + environment, + fetch, + })(request({ signal: AbortSignal.abort() })); + + expect(response.status).toBe(502); + expect(fetch).not.toHaveBeenCalled(); + }); + + test("aborts a stalled provider call at the existing connection timeout", async () => { + const log = vi.spyOn(console, "info").mockImplementation(() => {}); + const abort = new AbortController(); + vi.spyOn(AbortSignal, "timeout").mockReturnValue(abort.signal); + const fetch = vi + .fn( + (_input, init) => + new Promise((_resolve, reject) => { + init?.signal?.addEventListener("abort", () => + reject(new DOMException("aborted", "AbortError")), + ); + }), + ) + .mockResolvedValueOnce(transcriptionSecret()); + const pending = createOpenAITranscriptionSessionHandler({ + environment, + fetch, + })(request()); + await vi.waitFor(() => expect(fetch).toHaveBeenCalledTimes(2)); + abort.abort(); + + expect((await pending).status).toBe(502); + expect(log).toHaveBeenLastCalledWith( + "[Petrinaut transcription]", + expect.stringContaining('"stage":"awaiting-provider-headers"'), + ); + expect(log).toHaveBeenLastCalledWith( + "[Petrinaut transcription]", + expect.stringContaining('"timedOut":true'), + ); + }); + + test("distinguishes an unfinished browser upload from a stalled provider request", async () => { + const log = vi.spyOn(console, "info").mockImplementation(() => {}); + const abort = new AbortController(); + vi.spyOn(AbortSignal, "timeout").mockReturnValue(abort.signal); + const fetch = vi.fn(); + const pending = createOpenAITranscriptionSessionHandler({ + environment, + fetch, + })( + request({ + body: new ReadableStream({ + start(controller) { + controller.enqueue( + new TextEncoder().encode("v=0\r\no=private-offer"), + ); + }, + }), + duplex: "half", + }), + ); + abort.abort(); + + expect((await pending).status).toBe(502); + expect(fetch).not.toHaveBeenCalled(); + expect(log).toHaveBeenLastCalledWith( + "[Petrinaut transcription]", + expect.stringContaining('"stage":"reading-offer"'), + ); + expect(JSON.stringify(log.mock.calls)).not.toContain("private-offer"); + }); + + test("reports combined cancellation with the dev adapter's lazy native signal", async () => { + const log = vi.spyOn(console, "info").mockImplementation(() => {}); + const controller = createCustomAbortControllerSignal(); + const offer = request(); + // The dev adapter's Request retains this signal before AbortSignal.any + // materializes its native controller. Its original aborted flag can stay false. + Object.defineProperty(offer, "signal", { value: controller.signal }); + const fetch = vi.fn( + (_input, init) => + new Promise((_resolve, reject) => { + init?.signal?.addEventListener( + "abort", + () => reject(init.signal?.reason), + { once: true }, + ); + }), + ); + const pending = createOpenAITranscriptionSessionHandler({ + environment, + fetch, + })(offer); + await vi.waitFor(() => expect(fetch).toHaveBeenCalledOnce()); + controller.abort(); + + expect((await pending).status).toBe(502); + expect(log).toHaveBeenLastCalledWith( + "[Petrinaut transcription]", + expect.stringContaining('"aborted":true'), + ); + expect(log).toHaveBeenLastCalledWith( + "[Petrinaut transcription]", + expect.stringContaining('"timedOut":false'), + ); + expect(fetch).toHaveBeenCalledOnce(); + }); + + test("reports received provider headers before a cancelled response body without logging content", async () => { + const log = vi.spyOn(console, "info").mockImplementation(() => {}); + const abort = new AbortController(); + const fetch = vi + .fn( + async (_input, init) => + new Response( + new ReadableStream({ + start(controller) { + controller.enqueue( + new TextEncoder().encode("v=0\r\no=private-answer"), + ); + init?.signal?.addEventListener( + "abort", + () => { + controller.error(new Error("private-provider-error")); + }, + { once: true }, + ); + }, + }), + { status: 201, headers: { "content-type": "application/sdp" } }, + ), + ) + .mockResolvedValueOnce(transcriptionSecret()); + const pending = createOpenAITranscriptionSessionHandler({ + environment, + fetch, + })(request({ signal: abort.signal })); + await vi.waitFor(() => expect(fetch).toHaveBeenCalledTimes(2)); + // Headers must be observable while the body is still pending. + const progress = log.mock.calls.at(-1); + abort.abort(); + expect((await pending).status).toBe(502); + expect(progress).toEqual([ + "[Petrinaut transcription]", + expect.stringContaining('"stage":"reading-provider-answer"'), + ]); + expect(log).toHaveBeenLastCalledWith( + "[Petrinaut transcription]", + expect.stringContaining('"upstreamStatus":201'), + ); + expect(log).toHaveBeenLastCalledWith( + "[Petrinaut transcription]", + expect.stringContaining('"requestAborted":true'), + ); + const logs = JSON.stringify(log.mock.calls); + expect(logs).not.toContain("private-answer"); + expect(logs).not.toContain("private-provider-error"); + expect(logs).not.toContain("server-only-secret"); + }); +}); diff --git a/apps/petrinaut-website/src/server/voice/openai-transcription-session.ts b/apps/petrinaut-website/src/server/voice/openai-transcription-session.ts new file mode 100644 index 00000000000..ab402fd1dd3 --- /dev/null +++ b/apps/petrinaut-website/src/server/voice/openai-transcription-session.ts @@ -0,0 +1,311 @@ +import { + createVoiceRequestId, + voiceDurationMs, +} from "../../voice-diagnostics.js"; +import { getVoiceProvider } from "./openai-voice-config.js"; +import { getOpenAIVoiceAvailability } from "./openai-voice-policy.js"; + +const maxSdpBytes = 65_536; + +const respond = (body: string, status: number, headers?: HeadersInit) => { + const responseHeaders = new Headers(headers); + responseHeaders.set("cache-control", "no-store"); + return new Response(body, { status, headers: responseHeaders }); +}; + +/** Creates the authoritative transcription-only WebRTC session server-side. */ +export const createOpenAITranscriptionSessionHandler = + ({ + environment, + fetch, + }: { + environment: Parameters[0] & { + readonly NODE_ENV?: string; + }; + fetch: typeof globalThis.fetch; + }) => + async (request: Request): Promise => { + if (request.method !== "POST") + return respond("Method not allowed.", 405, { allow: "POST" }); + if (request.headers.get("origin") !== new URL(request.url).origin) + return respond("Forbidden.", 403); + if ( + request.headers + .get("content-type") + ?.split(";", 1)[0] + ?.trim() + .toLowerCase() !== "application/sdp" + ) + return respond("Expected SDP.", 415); + + const availability = getOpenAIVoiceAvailability(environment); + if (!availability.available || getVoiceProvider(environment) !== "live") + return respond("Transcription is unavailable.", 404); + + const requestId = createVoiceRequestId(); + const startedAt = performance.now(); + let stage: + | "reading-offer" + | "awaiting-transcription-secret" + | "reading-transcription-secret" + | "awaiting-provider-headers" + | "reading-provider-answer" + | "answer-ready" = "reading-offer"; + let upstreamStatus: number | undefined; + let upstreamErrorCode: string | undefined; + let upstreamErrorParam: string | undefined; + let upstreamErrorMessage: string | undefined; + const timeoutSignal = AbortSignal.timeout(availability.connectionTimeoutMs); + const signal = AbortSignal.any([request.signal, timeoutSignal]); + const report = (event: "progress" | "finished") => { + // Operational fields plus a bounded, redacted, local-only configuration + // rejection. Never log full provider bodies, SDP or exception messages. + // Terminal logs survive a browser abort that prevents an HTTP response. + // oxlint-disable-next-line no-console -- diagnose the manual experiment's stalled transcription handshake. + console.info( + "[Petrinaut transcription]", + JSON.stringify({ + requestId, + event, + stage, + durationMs: voiceDurationMs(startedAt, performance.now()), + upstreamStatus, + upstreamErrorCode, + upstreamErrorParam, + upstreamErrorMessage, + // The dev adapter's lazy signal can retain a stale aborted flag. + // This is the actual signal supplied to fetch and the upload reader. + aborted: signal.aborted, + requestAborted: request.signal.aborted, + timedOut: timeoutSignal.aborted, + }), + ); + }; + try { + report("progress"); + signal.throwIfAborted(); + if (Number(request.headers.get("content-length")) > maxSdpBytes) + return respond("SDP too large.", 413); + + const bytes = new Uint8Array(maxSdpBytes); + let length = 0; + try { + await request.body?.pipeTo( + new WritableStream({ + write(chunk) { + length += chunk.byteLength; + if (length > bytes.byteLength) throw new Error("SDP too large"); + bytes.set(chunk, length - chunk.byteLength); + }, + }), + { signal }, + ); + } catch (error) { + if (length > bytes.byteLength) return respond("SDP too large.", 413); + throw error; + } + + const sdp = new TextDecoder().decode(bytes.subarray(0, length)); + if (!sdp.trimStart().startsWith("v=0")) + return respond("Invalid SDP.", 400); + signal.throwIfAborted(); + + // The unified multipart calls schema only configures realtime sessions. + // Configure transcription on a client secret, retained server-side, then + // use that scoped credential for the documented raw-SDP exchange. + stage = "awaiting-transcription-secret"; + report("progress"); + const secretResponse = await fetch( + "https://api.openai.com/v1/realtime/client_secrets", + { + method: "POST", + signal, + headers: { + authorization: `Bearer ${environment.OPENAI_VOICE_API_KEY!.trim()}`, + "content-type": "application/json", + }, + body: JSON.stringify({ + session: { + type: "transcription", + audio: { + input: { + transcription: { model: "gpt-4o-transcribe" }, + turn_detection: { type: "server_vad" }, + }, + }, + }, + }), + }, + ); + upstreamStatus = secretResponse.status; + stage = "reading-transcription-secret"; + report("progress"); + if (!secretResponse.ok) { + upstreamErrorCode = "unrecognized"; + upstreamErrorParam = "unrecognized"; + try { + // Error messages may echo credentials or input. Read at most 8 KiB + // under the existing deadline and retain only exact allowlisted fields. + const errorBytes = new Uint8Array(8192); + let errorLength = 0; + await secretResponse.body?.pipeTo( + new WritableStream({ + write(chunk) { + if (errorLength + chunk.byteLength > errorBytes.byteLength) + throw new Error("Provider error body too large"); + errorBytes.set(chunk, errorLength); + errorLength += chunk.byteLength; + }, + }), + { signal }, + ); + const body: unknown = JSON.parse( + new TextDecoder().decode(errorBytes.subarray(0, errorLength)), + ); + if ( + typeof body === "object" && + body !== null && + "error" in body && + typeof body.error === "object" && + body.error !== null + ) { + const error = body.error; + if ( + "code" in error && + typeof error.code === "string" && + [ + "invalid_value", + "invalid_parameter", + "invalid_request_error", + "unknown_parameter", + "missing_required_parameter", + "unsupported_parameter", + "unsupported_value", + "model_not_found", + "unsupported_model", + "invalid_api_key", + "insufficient_quota", + "rate_limit_exceeded", + ].includes(error.code) + ) + upstreamErrorCode = error.code; + if ( + "param" in error && + typeof error.param === "string" && + [ + "session", + "session.type", + "session.audio", + "session.audio.input", + "session.audio.input.transcription", + "session.audio.input.transcription.model", + "session.audio.input.turn_detection", + "session.audio.input.turn_detection.type", + "audio.input.transcription", + "audio.input.transcription.model", + "audio.input.turn_detection", + "audio.input.turn_detection.type", + "transcription.model", + "turn_detection", + "turn_detection.type", + "model", + ].includes(error.param) + ) + upstreamErrorParam = error.param; + + // This request sends only fixed session configuration, not SDP or + // audio. Inspect this specific rejection locally without exposing + // arbitrary provider wording in deployed logs or HTTP responses. + if ( + environment.NODE_ENV === "development" && + ["localhost", "127.0.0.1", "[::1]"].includes( + new URL(request.url).hostname, + ) && + secretResponse.status === 400 && + upstreamErrorCode === "invalid_value" && + upstreamErrorParam === "session.audio.input.turn_detection" && + "message" in error && + typeof error.message === "string" + ) { + upstreamErrorMessage = error.message + .replaceAll( + environment.OPENAI_VOICE_API_KEY!.trim(), + "[redacted]", + ) + .replaceAll(sdp, "[redacted]") + .replace(/\b(?:sk|ek)-[A-Za-z0-9_-]+/gu, "[redacted]") + .slice(0, 1024); + } + } + } catch { + // A malformed, oversized or aborted diagnostic body must not hide the + // rejection status already received, trigger a retry, or expose text. + } + return respond("Transcription credential creation failed.", 502, { + "x-voice-upstream-status": String(secretResponse.status), + }); + } + const secret: unknown = await secretResponse.json(); + signal.throwIfAborted(); + if ( + typeof secret !== "object" || + secret === null || + !("value" in secret) || + typeof secret.value !== "string" || + !secret.value.trim() || + !("session" in secret) || + typeof secret.session !== "object" || + secret.session === null || + !("type" in secret.session) || + secret.session.type !== "transcription" + ) + return respond("Invalid transcription credential response.", 502); + + stage = "awaiting-provider-headers"; + upstreamStatus = undefined; + report("progress"); + const upstream = await fetch("https://api.openai.com/v1/realtime/calls", { + method: "POST", + signal, + headers: { + authorization: `Bearer ${secret.value}`, + "content-type": "application/sdp", + }, + body: sdp, + }); + upstreamStatus = upstream.status; + stage = "reading-provider-answer"; + report("progress"); + if (!upstream.ok) { + await upstream.body?.cancel(); + return respond("Transcription session creation failed.", 502, { + "x-voice-upstream-status": String(upstream.status), + }); + } + const contentType = upstream.headers + .get("content-type") + ?.split(";", 1)[0] + ?.trim() + .toLowerCase(); + if (contentType !== "application/sdp" && contentType !== "text/plain") { + await upstream.body?.cancel(); + return respond("Transcription session creation failed.", 502); + } + const answer = await upstream.text(); + if (!answer.trimStart().startsWith("v=0")) + return respond("Transcription session creation failed.", 502); + + stage = "answer-ready"; + return Response.json( + { sdp: answer }, + { status: 201, headers: { "cache-control": "no-store" } }, + ); + } catch { + return respond( + "Transcription connection failed. The remote session outcome may be unknown; no automatic retry was made.", + 502, + ); + } finally { + report("finished"); + } + }; diff --git a/apps/petrinaut-website/vite.config.ts b/apps/petrinaut-website/vite.config.ts index d0a454a19ed..ae4389f8e44 100644 --- a/apps/petrinaut-website/vite.config.ts +++ b/apps/petrinaut-website/vite.config.ts @@ -27,6 +27,7 @@ const apiModules = [ ["/api/voice/config", "/api/voice/config.ts"], ["/api/voice/realtime-call", "/api/voice/realtime-call.ts"], ["/api/voice/live-session", "/api/voice/live-session.ts"], + ["/api/voice/transcription-session", "/api/voice/transcription-session.ts"], ] as const; // Plugin required to serve the Vercel fetch handlers in dev. In production, diff --git a/libs/@hashintel/brunch-agent/MISSION.md b/libs/@hashintel/brunch-agent/MISSION.md index 3d32931c5e8..1de4523d1f4 100644 --- a/libs/@hashintel/brunch-agent/MISSION.md +++ b/libs/@hashintel/brunch-agent/MISSION.md @@ -1,203 +1,424 @@ -# Mission 7c — Record an Inventory worked example from scratch (FE-1573 / FE-1478) +# Experiment Live Full Brunch Integration ## Status -Provisionally closed for engineering review by Lu on 2026-09-14, on `ln/fe-1573-mission-7c`, [PR #9667](https://github.com/hashintel/hash/pull/9667). **The worked example is not accepted.** The original contract below records the attempted outcome and its unmet proof; closure does not turn those obligations into passes or imply PR approval or merge. +The provider-free implementation and manual Brunch-backed sessions establish the +dual WebRTC handshake, canonical input admission, tool-backed Brunch work and +settled commentary handoff. They do not establish naturalness, transcription +fidelity or production readiness. Acceptance is blocked by phantom input observed +during silence and by server-VAD splits that can exceed the existing one-waiting-input +policy. FE-1712 owns capture-constraint investigation; Kostandin owns further live +testing. The PR records exact checks and limitations. No agent-started provider +session is authorized. -The parent delivers the browser-visible persona/construction mechanism and the verified schema, streaming, Stop, diagnostics and tool-progress repairs. Mission 7d, `ln/fe-1573-mission-7d-provider-worked-example`, takes one alternative-provider qualification and completion of the retained example: diagnostics/repair, bounded correction, provenance questions, original-session reopen and Lu's review. Fixture distribution and portfolio breadth remain subsequent missions. FE-1573 is reused under Lu's explicit branch-split exception; no tracker state is changed. - -This file owns the contract and current progress dispositions. The PR holds detailed verification results and residuals; native run records hold execution evidence. Update the state below in place, not as a running log. - -- **Established base:** `yarn brunch:persona` is the canonical browser-visible persona method for any case pack; the [operator guide](../../../apps/brunch-agent/.pi/extensions/brunch-persona-testing/README.md) owns launch, recording and resume. The SDK/spectator persona path is removed. The synthetic browser proof passes: opening and subsequent client continuations, two visible net/workpiece updates, tab switching during execution, abort settlement and no replay on reload. This is [mechanism coverage](#throughline-proof-floor), not Inventory acceptance. The Inventory opening and persona request construction from scratch; operational source facts and the separate hand-built reference are unchanged. -- **Acceptance open:** one retained run must connect ordinary-language elicitation to an agent-built net, explanation, correction and original-session reopen, followed by Lu's review. Distribution and portfolio breadth are [next-mission work](docs/mission-drafts/worked-example-distribution-and-breadth.md), not blockers on this run. -- **Observed run, stopped after resume:** local-only `apps/brunch-agent/.data-wipe-me/persona-runs/run-PUmmN6/` retains the original database, Chrome profile, Pi session and allocation. Native recovery settled the old interrupted submission as `submission_timeout`; workpiece revision 10 and construction-resource reads survived. Pi's new request to proceed was admitted without replaying the correction. After 446.6 seconds without admitted assistant/tool output, the builder used ordinary panel Stop following Lu's stall report; native history confirms `aborted`. No net tools executed; the net remains empty. `evidence/snapshot.json` and its derived records now include the resumed/aborted turn. Pi is idle after accounting refusal; browser and services remain open. Requests 61 and 63 remain unknown, each retaining US$7.92, inside the original US$100 allocation; Lu accepts both holds for continuation. Do not resume this old run concurrently with the fresh observation. -- **Streaming and Stop regressions repaired:** the admission wrapper now forwards text/reasoning progress while withholding executable tool inputs and completion until validation. The real panel shows partial replies before completion and tool operations after admission. Ordinary panel Stop settles natively as `aborted`, retains partial prose, prevents the pending write and reports a stopped turn to the persona; `/abort` is no longer misclassified as admission. Passed 2026-09-13: 75 targeted tests, 5 production admission-control tests, app typecheck/lint and the synthetic browser proof under `apps/brunch-agent/.data-wipe-me/persona-runs/persona-construction-6qqOeJ/` (native snapshots, screenshots and execution log). Partial tool arguments still do not project into the panel: a not-yet-admitted proposal can be generating arguments without a tool card. The cause of the original long generation remains unresolved. -- **Failed opening retained:** local-only `apps/brunch-agent/.data-wipe-me/persona-runs/run-qb10He/` started with an empty canvas and paused for recording. Its first Brunch request was rejected because `query_workpiece` serialized a top-level `anyOf`; Pi had not started and has no session to resume. The launcher stopped its owned browser/services. Request 1 remains unknown with US$7.92 reserved inside this run's US$20 suballocation. The schema now uses an object containing `selector`, preserving the full alternatives. The free provider preflight passes; this is schema acceptance, not another construction observation. -- **Earlier construction stop retained:** local-only `apps/brunch-agent/.data-wipe-me/persona-runs/run-093ijt/` retains the Sonaflozin recording's native history. The first batch applied all 30 operations (4 types, 3 differential equations, 23 parameters), but no places or transitions. Three diagnostics reads returned `pending`; the next inference was refused by the accounting reserve: US$4.17237525 recorded, US$7.90762475 remaining, US$7.92 required. This was not a mutation failure or exhausted actual spend. `evidence/snapshot.json` includes the terminal turn. The accounting stop did not explain delayed construction, long reasoning or the diagnostics/progress regressions repaired below. -- **Accounting interruption removed:** persona launch/resume no longer creates or consults reservations, accepts unknown usage, or installs Pi's accounting provider. The launcher explicitly overrides inherited campaign accounting for its services and Pi. Native usage, old ledgers, identity/replay protections, recording pause and operator Stop remain. Verification: 51 targeted tests, app build/typecheck, and two synthetic requests through the built ChatAgent under OS network denial; native usage persisted while an invalid old ledger stayed untouched. No paid run restarted. -- **Incremental construction guidance installed, cadence not established:** the SDCPN append, job skill, construction reference and checks direct construction alongside meaning-bearing workpiece settlements once an activity and adjacent state or relationship are supported. Readiness and dependency ordering apply to the next connected fragment, not a complete process or whole-model catalogue. Wording-only changes need no mutation; unsupported operational defaults remain unauthorized. All 88 plugin tests and the production build pass. The latest run still waited roughly 12 minutes before its first fragment; packaging checks do not establish progressive behavior. -- **Latest observation stopped on provider refusal:** local-only `apps/brunch-agent/.data-wipe-me/persona-runs/run-1vFeVo/` retains the native conversation, workpiece and document. Brunch reached workpiece ordinal 15, applied batches of 14 and 9 operations and reached provenance querying, but diagnostics repeatedly returned pending, layout did not apply and the final correction did not complete. Native settlement records an Anthropic refusal with fallback guidance, not evidence of a network outage. The builder stopped this launch's owned services/browser/Pi after Lu reported the run stopped. No fallback is configured; selecting one remains an owner decision. The authorized continuation is recorded below. -- **Diagnostics and tool progress repaired — builder verification:** explicit worker requests now check the captured net independently of whether pushed diagnostics changed, and superseded results cannot claim current success. The real-browser compiler tracer passes dirty → repaired → changed-but-still-clean, with recorded position-only layout effects. Running tool cards show a spinner/status that clears on completion; rendered captures were inspected. Core and UI suites pass (1,719 and 1,048 tests), along with affected builds/typechecks/lint and architecture checks. The synthetic persona regression also passes browser construction, tab switching, Stop and same-session recovery without replay (`persona-construction-WfHC3Y/` in the system temp directory). Partial argument progress still cannot reach Brunch's panel through Flue's remote stream; tools appear after admission. This is synthetic mechanism evidence, not a successful live rerun. -- **Continuation also refused:** the original profile reopened with 7 places and 8 transitions, but the next Brunch request settled with the same Anthropic refusal at `2026-09-14T10:06:09.315Z`, following the original refusal at `2026-09-14T09:19:06.797Z`. The [provider documentation](https://platform.claude.com/docs/en/build-with-claude/refusals-and-fallback) identifies this response as refusal, not an outage; the retained error does not preserve `stop_details`, so the specific classifier/category is unknown. The launcher is stopped and its original stores remain local-only under `run-1vFeVo`. No fallback or different provider is configured. Resume did not complete the remaining acceptance gates or establish improved fresh-run cadence. - -### Owner decisions - -- **2026-09-14 — Lu:** provisionally tie off Mission 7c for review without worked-example acceptance and create a stacked successor for one different provider and example completion. Reuse FE-1573 if no existing issue fits. This authorizes the branch/mission split, not a new paid run, production provider switch, fixture distribution or portfolio expansion. -- **2026-09-13 — Lu:** close this branch on a recorded persona-driven worked example starting from scratch. Defer fixture distribution and portfolio breadth to the next mission; the later reusable demo depends on first producing this example. This cut changes scope, not acceptance of the unrun example or authorization of a paid allocation. -- **2026-09-13 — Lu:** build the first browser-executed persona proof after oracle feasibility review. The persona supplies ordinary utterances behind the scenes; the real UI streams replies and executes Brunch's tool calls, without human operation or screenshot-based AI control. AI/Workpiece tab switching must not interrupt it. The intended live run spans roughly 15–25 turns or more as needed, not a fixed turn-count acceptance rule. -- **2026-09-13 — Lu:** use at least Sonnet-level models on both sides with a US$100 budget limit. The builder selects the launcher's existing `claude-sonnet-4-6` for both and treats US$100 as the combined ceiling, including continuation, compaction, failures and retries—not an allocation per participant. -- **2026-09-13 — Lu:** add same-run resume using the original database, Chrome profile and Pi session. Accept the interrupted request's unknown usage for continuation while retaining its full US$7.92 reservation inside the original budget; reopen at a recording pause before continuing. -- **2026-09-14 — Lu:** canonicalize the browser-visible, background-driven persona method, make any context pack launchable through the same command, and remove superseded persona code paths and operating instructions. This authorizes instrument cleanup, not a new paid run or acceptance of the worked example. -- **2026-09-14 — Lu:** accept request 63's unknown usage for continuation with its full US$7.92 hold retained; allocate US$20 of the original budget's remainder to the smaller fresh Sonaflozin observation specified in Status. Pause its identified Chrome window before inference for recording. -- **2026-09-14 — Lu:** make free Anthropic tool-definition acceptance part of the ongoing harness. The [schema acceptance contract](evaluations/README.md#tool-schema-acceptance) uses actual native catalogues; acceptance is provider-specific, not a universal compatibility claim or settlement of the failed run. -- **2026-09-14 — Lu:** commit the schema remediation and return to the Sonaflozin persona observation with Chrome relaunched. Continue within the existing allocation while retaining the disclosed failed-request hold; no unknown cost is settled or discarded. -- **2026-09-14 — Lu:** remove the accounting checks that interrupt persona recordings. This supersedes automatic request reservations, budget refusals and unknown-usage acceptance gates for the persona method; retain usage as observation in native records, not permission to dispatch. Historical ledgers remain evidence, not gates on continuation. This change does not itself start another paid run or change worked-example acceptance. +[FE-1664](https://linear.app/hash/issue/FE-1664/experiment-live-full-brunch-integration) +depends on [FE-1663 / #9671](https://github.com/hashintel/hash/pull/9671) and retains +[FE-1661](https://linear.app/hash/issue/FE-1661/evaluate-gpt-live-1-migration-effort-before-the-demo) +as the migration assessment. This is the child branch's sole execution authority. ## Imperative -Produce one recorded Inventory purchasing worked example through the local Pi persona setup and the actual Brunch/Petrinaut product route. Start with a fresh conversation, no prior workpiece and an empty net. The persona supplies operational knowledge in ordinary language; Brunch elicits and records it, constructs a connected compiler-clean SDCPN, explains consequential content and makes one bounded correction from a changed operational fact or explicit policy choice. Close and reopen that same local document/session and demonstrate continuity. +Preserve a fluid, realistic interview while Brunch directs it and performs real +application work. The person has room to hesitate, elaborate, correct consequential +details and continue while Brunch reasons or executes. Avoid repetitive +acknowledgements, monologues, irrelevant questions, lost corrections and unsupported +progress/completion claims. A possible release claim is “Talk naturally while Brunch +updates the model, without losing corrections.” It is not established yet. -Retain the conversation, workpiece revisions, mutation/provenance records and final net in the original run for Lu's semantic review and the next mission's input. Recording a useful example does not require packaging, seeding, copying or exporting its session. Local-only evidence remains valid within that stated limit. - -The established Inventory reference SDCPN was built by hand and has no associated workpiece or session. Brunch can interpret its visible structure, but cannot recover a recorded construction basis that does not exist. Keep it as an evaluator-side comparator, not a seed, an elicitor input or a source of prewritten mutations. The result need not copy its IDs or layout; it must faithfully represent the elicited operation. Reusable guidance and construction architecture must remain independent of Inventory-specific nouns and IDs. +## Throughline -This proves one worked example, not portfolio breadth, repeatability across runs, simulation correctness or fixture distribution. An unsupported request must refuse visibly and specifically rather than silently omit meaning or claim success. +Finalized transcription → existing composer/submission path → one Flue admission +→ Brunch reasoning and authorized Petrinaut execution → complete-turn settlement +→ frozen canonical speech source → delegation-correlated Live commentary and native +unbuffered delivery. Live's native client-delegation loop supplies timing metadata, +never canonical input or another backend invocation. An unserved delegation receives +an instruction to ask the person to continue, not a fabricated backend answer. +Settlement gates supplied Brunch context, not all audible speech. -## Throughline +### Owner decisions -The full acceptance path is below. The next authorized action is in [Status](#status); the full path is not an execution schedule. - -```text -fresh browser/session + empty local net; no preloaded reference or workpiece -→ persona speaks in ordinary language; workpiece revisions settle -→ Brunch recognizes explanation, construction or correction intent -→ read_petrinaut_net supplies a fresh, verified base -→ mutate_petrinaut_net adds, edits or removes admitted root-net parts by ID -→ website applies the committed prefix and records complete verified effects -→ read_petrinaut_diagnostics returns clean | errors | pending for that version -→ Brunch repairs against a fresh observation when errors remain -→ layout_petrinaut_net records its pre/post hashes and position-only effects -→ query_workpiece maps selected Petrinaut elements to their recorded mutation - revisions, workpiece passages and session turns, or reports absence -→ an ordinary-language correction changes the workpiece and bounded net region -→ close/reopen resumes the original local document and conversation -→ Lu reviews the retained run, operational account and agent-constructed net -``` - -Assistant selection is host-owned. Each mode has its own transport, tool manifest and conversation history; no history or tool result is spliced across modes. +- **2026-09-14:** Kostandin requests the existing Thinking dock status during + Brunch work. Map submitted/streaming composer status to Thinking only while Live + is connected, not stopped, and not playing output; preserve connection/error + precedence and Speaking during playback. This is local status presentation, + not `session.thinking.append`, progress speech, a new invocation or proof of + completion. +- **2026-09-14:** Kostandin accepts the bounded consent UI correction on an isolated + FE-1664 checkout: use a plain voice-permission heading, concise OpenAI voice and + transcription disclosure, permission checkbox, Start voice and Cancel. Remove + experiment-specific warnings from this surface, not from the retained mission + risks. Keep the compact dock and viewport controls stationary while consent opens + above them; show Voice setup rather than Connecting before Start. +- **2026-09-12:** Kostandin accepts a Live prompt delegation policy describing Brunch's + interview, follow-up and model-building capabilities; delegation-id correlation of + settled commentary; `session.instructions.append` for delegations Brunch cannot + serve; and replacing the local 500-byte commentary cap with provider validation + correlated through `client_event_id`. `session.thinking.append` for event-backed + progress is not accepted. +- **2026-09-11:** Kostandin accepts `gpt-4o-transcribe` with default `server_vad` + for canonical transcription, including the risk of splitting hesitation into + separate submissions. Native Live speech, canonical admission and queue policy + remain unchanged. + +### Selected experiment + +- **Input:** use a separate OpenAI transcription-only session for canonical user + text. The model is `gpt-4o-transcribe`, using browser WebRTC and the + existing server credential boundary. One consented microphone capture supplies + Live and transcription; neither session starts automatically. Disclose the second + stream in the concise voice-permission surface. Separate transcription still + incurs additional provider usage; simplifying the disclosure does not change it. +- **Finalization:** consume `conversation.item.input_audio_transcription.completed` + for its `item_id` and `content_index`; deltas and Live delegation notices cannot + admit text. Use provider-supported turn detection to commit audio chunks, not + application silence timers. Use `server_vad` with provider defaults rather than + unmeasured tuning. A final transcript is final for that audio item, not proof + the person has finished their thought or that transcription is semantically correct. +- **Ordering:** transcription completions can arrive out of order. Reconcile them + against provider committed-item ordering before offering finalized text to the + existing composer. Keep session/item identity through duplicate detection and + admission correlation. This transport reconciliation is not a new application + FIFO: preserve the existing one-waiting-input policy, expose inability to retain + further input, and never silently replay uncertain admission or substitute Live's + transcript when transcription fails. +- **Speech:** play native Live audio without an approval buffer. Instruct Live to + delegate substantive questions/answers to Brunch using the official personality, + backchannel, interruption and Delegation policy structure. Brunch interviews the + person, asks follow-ups and builds/updates the model. Live has no tools or independent + substantive answers; it delegates before answering and does not guess while waiting. + These are best-effort instructions, not an enforced output boundary. Independent + questions, lost corrections and unsupported claims are manual failure observations, + not canonical answers or evidence that work succeeded. No Live tool authority. +- **Canonical result handoff:** retain complete-turn settlement and source freezing + before sending Brunch prose as commentary. A finalized input claims the most recent + unclaimed client delegation; a later delegation attaches to the newest turn without + one. Send its opaque ID unchanged, or `null` when none is attached. Dropped input, + failed turns and textless settlements with a delegation receive + `session.instructions.append` saying the backend could not take that request now + and asking the person to continue. No delegation event admits input itself. + Commentary is paraphrasable and provider-limited to 500 tokens: send the frozen + source intact once, without a local byte cap, truncation, chunking or replay. + Match commentary/instruction acknowledgements by `client_event_id` and rejections + by `error.client_event_id`; distinguish local send failure, provider rejection and + unknown acceptance. Stop invalidates pending correlations and ignores late events. + Acceptance is not playback. Full reports stay on screen; longer-source delivery + after provider rejection remains unresolved, not grounds for a new queue. + +The browser path avoids a new service and reuses existing credentials. Transcription +configuration belongs on `/v1/realtime/client_secrets`; the server retains the returned +short-lived credential and exchanges raw SDP at `/v1/realtime/calls`. The SDK's +multipart calls configuration only types `realtime`, not `transcription`. This is a +handshake correction within the selected WebRTC/credential boundary, not a new input +policy. Both requests share the existing deadline and neither is retried. +Keep both connections under one session lifetime: failure/Stop invalidates late +callbacks and tears down media +without an automatic standalone or Realtime fallback. Do not infer playback completion +from commentary acceptance, use it to release pending work, or restore historical audio. +Acoustic interruption remains native Live behavior; durable composer Stop still +cancels canonical work separately and must immediately silence local playback. + +### Inspected departure and stack + +- Base: `ln/fe-1573-mission-7c`, [#9667](https://github.com/hashintel/hash/pull/9667), + pinned for this restack at [dee90599e9](https://github.com/hashintel/hash/commit/dee90599e9a07d9fa3e55d0711c14491e9ce5c7c). + Its host-owned tools, document lifecycle, revision tracking and conversation + binding are inherited, not replaced by the old voice host. +- Parent: `kostandin/fe-1663-experiment-live-full-duplex-migration`, #9671, based + on #9667. It retains the standalone Live experiment, activity indicators, + microphone-loss shutdown, bounded uploads and disconnect recovery. +- Child: `kostandin/fe-1664-experiment-live-full-brunch-integration`, #9673, + targets #9671 and adds canonical Brunch integration. Activity remains telemetry, + never a turn or playback-completion signal. +- Kostandin reports that the standalone experience looks fine. The exact tested + revision was not supplied. This supports proceeding, not comprehensive + acceptance. Keep the original comparison revision separately from the restacked base. + +Mission 7c remains owned by #9667 and preserved in its pinned source, including +`libs/@hashintel/brunch-agent/MISSION.md`; the standalone contract remains in +#9671's `apps/petrinaut-website/MISSION.md`. This child closes neither mission. +The website mission points to this voice authority. The newer future spine and +successor drafts are retained; consumed Mission 7b drafts are not resurrected. ### Cold-start reads -- [`evaluations/cases/inventory-purchasing/`](evaluations/cases/inventory-purchasing/) — private persona situation pack and shared opening; `reference-sdcpn.json` is evaluator-only. -- [Persona operator guide](../../../apps/brunch-agent/.pi/extensions/brunch-persona-testing/README.md) and [launcher](../../../apps/brunch-agent/src/evaluations/persona/launch.ts) — `yarn brunch:persona --case inventory-purchasing` defaults to the ordinary `/` route and empty document, without an automatic accounting cutoff. Omit `--initial-net` and verify actual initial state rather than infer it from flags. Read [execution safety](evaluations/README.md#execution-safety) before provider checks or paid runs. -- [`docs/mission-archive/7b-ordinary-batched-construction-provenance.md`](docs/mission-archive/7b-ordinary-batched-construction-provenance.md) — accepted ordinary batch and provenance base. -- [`docs/reference/architecture/mutation-capability-matrix.md`](docs/reference/architecture/mutation-capability-matrix.md) — operation ownership, admission, execution and refusal authority. -- [`../../../apps/brunch-agent/src/conversation/net-freshness.ts`](../../../apps/brunch-agent/src/conversation/net-freshness.ts) and [`../../../apps/brunch-agent/src/conversation/net-ledger.ts`](../../../apps/brunch-agent/src/conversation/net-ledger.ts) — current-net freshness and the candidate shared-history projection, including its authority constraints. -- [`packages/plugin-sdcpn/src/mutate-petrinet.ts`](packages/plugin-sdcpn/src/mutate-petrinet.ts) and [`packages/plugin-sdcpn/src/mutation-record.ts`](packages/plugin-sdcpn/src/mutation-record.ts) — selected carrier and receiving-boundary verification. -- [`../petrinaut-core/src/action-schemas.ts`](../petrinaut-core/src/action-schemas.ts), [`../petrinaut-core/src/selected-mutation-batch.ts`](../petrinaut-core/src/selected-mutation-batch.ts) and [`../petrinaut-core/src/diagnostics.ts`](../petrinaut-core/src/diagnostics.ts) — canonical actions, batch schema and TypeScript diagnostics. -- [`../../../apps/petrinaut-website/src/main/app/local-storage-demo/documents/`](../../../apps/petrinaut-website/src/main/app/local-storage-demo/documents/) and [`../../../apps/petrinaut-website/src/main/app/local-storage-demo/assistants/brunch/use-process-agent-binding.ts`](../../../apps/petrinaut-website/src/main/app/local-storage-demo/assistants/brunch/use-process-agent-binding.ts) — storage-neutral lifecycle, source crossing and typed conversation identity. -- [`docs/reference/architecture/topology.md`](docs/reference/architecture/topology.md) — current tool and document-lifecycle topology. +Read [AGENTS.md](AGENTS.md), this mission and the +[retained Voice/recovery record](MISSION.next.md#voice-after-the-live-transport-cut). +The base and parent above preserve their respective contracts at +`libs/@hashintel/brunch-agent/MISSION.md` and `apps/petrinaut-website/MISSION.md`. +Use the actual seams below for current signatures. Mission 6b's accepted causal +Voice/mutation/Stop behavior is regression input, not proof of the explicitly +deferred direct-user hydration, post-settlement withholding or latency claims. + +Reuse Ex1's session handler, WebRTC, consent, provider selection and cleanup; do +not rebuild the standalone experience or add another panel. Use client delegation, +not managed Responses. The first milestone is one short, no-tool exchange, not +a tool-heavy interview. Only after inspecting that exchange should the line add +one already-authorized operation, correction during work, acoustic interruption +and durable Stop. Kostandin performs live testing; the agent runs mocks only. + +### Actual seams, not a ready-made adapter + +Paths below are relative to the repository root. + +| Boundary | Inspected contract and consequence | +| --- | --- | +| Live startup | `apps/petrinaut-website/api/voice/live-session.ts` composes `createOpenAILiveSessionHandler({environment, fetch})`. The server validates origin/SDP/enablement, creates client-delegated `gpt-live-1`, returns `{sessionId, sdp}` and does not retry unknown outcomes. | +| Live media | `createLiveConversation(onState, connectionTimeoutMs, onFinalizedInput)` returns `{start, stop, appendCommentary}`. One consented capture feeds two WebRTC sessions. Transcription completed items are reconciled against committed predecessor IDs; Live deltas/delegations never admit input. Audio remains native and unbuffered. | +| Live controls | `LiveConversationControl` receives canonical messages, submission resolution, settlements and response/Stop subscriptions. `LiveBrunchBridge` calls the existing admission helper and freezes correlated prose only at complete-turn settlement. Local end/pause silences both sessions; canonical Stop also invokes that local teardown. | +| Transcription creation | `api/voice/transcription-session.ts` composes `createOpenAITranscriptionSessionHandler`, with the existing credentials and enablement boundary. The accepted recut requires `/v1/realtime/client_secrets` to configure `type: transcription`, `gpt-4o-transcribe` and server VAD with provider defaults. The server uses that credential for a raw-SDP `/v1/realtime/calls` exchange. Neither credential reaches the browser. Origin, upload size, abort and timeout failures are bounded; unknown creation is never retried automatically. | +| Canonical entry | `voice-interview-control.tsx` exposes `submitVoiceInputWithAdmission`: subscribe before composer submission, correlate stable message identity to Flue admission, race abort, distinguish ambiguous/rejected/conflicting admission. `submitVoiceInput` retains at most one waiting input; abort withdraws only unsubmitted work. | +| Canonical source | `canonical-speech.ts` selects non-streaming assistant prose, with message/part/hash/submission identity and a marked question only if it occurs in finalized prose. This is source selection, not whole-turn or playback approval. Workpiece, reasoning, basis and raw tool payloads are excluded. | +| Baseline delivery | `realtime-brunch-bridge.ts` can call `speakCanonical` for completed segments while chat is streaming. Its test “speaks a completed canonical segment while chat remains streaming and settles separately” makes that intentional. Preserve Realtime; do not label this whole-turn-gated Live delivery. | +| Host history | `BrunchPanelConversationTracker.canReplaceMessages` rejects absent/in-flight/incomplete snapshots and resolves `answeredBySubmissionId`. `local-storage-demo-app.tsx` joins admissions, settlements, history and durable Stop. Keep these source contracts rather than substituting a latest-message lookup. | +| Execution | `apps/brunch-agent/src/provider-admission.ts` buffers complete proposals and refuses mixed server/browser calls, multiple browser calls and inconsistent streamed/final arguments. Construction bindings and observed effects remain authoritative; a proposal approval is not settlement or speech eligibility. | ## Proof -### Claim discipline - -Four evidence levels remain distinct: - -1. **Structural:** the mutation applied and its record verifies at the receiving boundary. -2. **Compiled:** Petrinaut's TypeScript diagnostics are clean for the exact version the batch produced. -3. **Semantic:** the model corresponds to the workpiece and operational account, established by human review of the flagship. -4. **Behavioral:** the model behaves correctly when executed. - -Mission 7c proves levels 1 and 2 mechanically and obtains level 3 through Lu's review of Inventory. It makes no level-4 claim. A structurally applied mutation is not thereby compiled; a compiled model is not thereby faithful; a timeout is not clean; and a single successful recording is not robustness. - -### Visible product advance - -**Release-note sentence:** Brunch builds and corrects an operational-process model in ordinary conversation, keeps it compiler-clean and legible, and answers where visible content came from, with Inventory purchasing as the recorded flagship. - -**Product-manager script:** watch the retained persona session start with an empty canvas and develop the Inventory operational account and model. In that conversation the persona asks why two consequential elements exist and changes one operational fact or policy choice in ordinary language. Observe a bounded, compiler-clean model change and legible layout, then reopen the same local document/session and inspect the continuing account and explanation. Lu reviews the resulting model against actual testimony; no tool vocabulary or developer-authored model repair is needed. A reusable template launch is not part of this script. - -### Throughline proof floor - -These are existing mechanism checks to reuse while attempting the persona throughline, not an instruction to complete a subsystem checklist before the first informative run. Repair the first boundary that blocks or falsifies the selected run; use affected regression checks for each change. - -Except where a fresh execution is dated below, dispositions are based on inspected coverage artifacts and [PR #9667's reported verification and known failures](https://github.com/hashintel/hash/pull/9667). **Coverage present** means an instrument exists, not that the whole obligation passed. The PR reports passing affected suites but a failing full Brunch integration run, including compiler-feedback; it does not establish an all-green baseline. - -| Required result | Oracle | Current disposition / evidence | -| --- | --- | --- | -| Model-facing tool definitions survive serialization and provider acceptance | `yarn workspace @apps/brunch-agent test:anthropic-tools` rebuilds the app, captures both native entrypoints across all four mounted modes and checks each distinct catalogue through Anthropic count-tokens, with a rejected top-level union control. | Passed 2026-09-14: five catalogues accepted (HTTP 200); negative control received the specific HTTP 400 rejection. Local-only `/var/folders/2c/ptn6jcrj61lck_yzfz_p3b5m0000gn/T/brunch-anthropic-tools-WMprfl/` retains safe results and captures. The expanded offline oracle passes under OS network denial: 28 synthetic requests, zero network attempts, nested-selector parsing and mounted executor refusal when no workpiece exists. Root-creation and typed-state browser tests fail on missing `getLatestNetDefinition` results; construction-progression times out before the query. Those tests do not establish query success or an all-green browser baseline. No paid generation or cross-provider proof. | -| Same-run persona recovery retains identity and history | Persona construction integration restarts the built backend after an aborted turn, reconciles without sending, retains the net/workpiece and executes a fresh browser-tool turn. Launcher tests read original Pi/session stores with no accounting fields or with unusable legacy accounting. | Recovery mechanism passed 2026-09-13 in local-only `apps/brunch-agent/.data-wipe-me/persona-runs/persona-construction-qab33H/`. Legacy/new resume parsing passed 2026-09-14 with the original Pi session selected and old ledger bytes unchanged. Accounting holds are retired for persona runs, not settled or erased. Does not establish successful real-run continuation or Inventory construction. | -| Valid first-workpiece bases survive argument validation; stale bases still refuse | [Persona construction integration](../../../apps/brunch-agent/test/persona-construction.integration.ts) sends explicit `null`, then the settled revision ID, then a stale `null` through the built ChatAgent/native provider adapter and real browser. | Passed 2026-09-13 after reproducing `null` becoming `""` before tool execution. Backported Pi's upstream nullable-union fix; no revision-guard weakening. Local-only browser records: `apps/brunch-agent/.data-wipe-me/persona-runs/persona-construction-K3eGQX/`. Native schema-carriage integration also passes (22 synthetic SDK requests, zero network attempts); 18 workpiece unit tests pass. | -| Usage observation cannot interrupt persona inference; recording starts before inference | `test/provider-accounting.integration.ts --disabled` sends two synthetic requests through the built ChatAgent with an invalid historical ledger, checks retained native usage and unchanged ledger bytes. Launcher tests verify inherited accounting is disabled; installed-Pi lifecycle test checks extension initialization without inference. `test/persona-construction.integration.ts` owns the recording pause. | Passed 2026-09-14: both synthetic requests complete under OS network denial and retain 320 total tokens in native records. All 51 targeted tests pass, including installed Pi lifecycle and legacy/new resume. The recording pause's earlier synthetic browser proof remains applicable; no fresh paid generation or run-quality claim. | -| Persona turns execute through the visible browser | [Persona construction integration](../../../apps/brunch-agent/test/persona-construction.integration.ts): built ChatAgent, synthetic provider, registered Pi extension, private IPC and ordinary Chrome composer; inspect native settlements, actual net/workpiece and screenshots. | Passed 2026-09-14 after canonicalization under loopback-only OS networking plus private Unix IPC. Local-only evidence: `apps/brunch-agent/.data-wipe-me/persona-runs/persona-construction-LOUt95/` (initial, final, stopped and resumed snapshots, net, screenshots and execution log). Includes pre-completion prose, admitted tools, two net/workpiece revisions, tab independence, Stop and same-conversation continuation after backend restart without replay. Workpiece/canvas capture inspected. All seven packs load; root help/list-cases and caller-relative directory check pass. Thirty targeted tests including installed Pi lifecycle and accounting pass; affected build, app typecheck/lint and the independent synthetic schema-carrier probe pass. No live Pi model or construction-quality claim; synthetic nodes use visible coordinates, so automatic viewport framing is not proven. | -| Inventory-derived code-bearing slice fits the carrier | A frozen fixture names a coloured type, parameters, places and arcs, a stochastic transition and a differential equation; it parses, applies canonically and reaches clean TypeScript diagnostics. It is mechanism evidence only. | Coverage present: [Inventory slice test](packages/plugin-sdcpn/test/inventory-slice.test.ts). Persona construction remains open. | -| The run's operations apply or refuse honestly | Use the [capability matrix](docs/reference/architecture/mutation-capability-matrix.md) for admission and canonical execution. Receiving-boundary records verify applied effects; an unadmitted shape refuses at its position before application. A failed supported operation is a finding to repair, not a satisfied construction result. | Coverage present: [carrier tests](packages/plugin-sdcpn/test/mutate-petrinet.test.ts), [admission controls](../../../apps/brunch-agent/test/integration/admission-controls.test.ts). Full-envelope and portfolio proof moves to the successor. | -| Compiler feedback is version-correlated | A structurally applied dirty batch reports errors or pending, never stale success; repair begins from a fresh observation and reaches diagnostics for the repaired definition. Dependency changes invalidate all affected code. | Passed 2026-09-14: [browser compiler tracer](../../../apps/brunch-agent/test/compiler-feedback.integration.ts), built Brunch/Chrome and real language worker under loopback-only OS networking. Undefined equation symbol reaches the model as TS2304, repair returns clean, and a position-changing layout followed by another check returns clean despite unchanged diagnostics. Captured-definition race and worker rejection are covered by panel helper tests. Local-only `m7c-compiler-feedback-iYjOy7/` under the system temp directory retains native history and tool-progress captures; running/completed captures from the preceding `m7c-compiler-feedback-5WRlDu/` and the final running-row capture were inspected. Live-run confirmation remains open. | -| Layout is a recorded document mutation | `layout_petrinaut_net` is separate from the semantic batch; its pre-hash equals the batch's final definition, its post-hash equals a fresh observation, and its effects are positions only. Existing user-arranged content uses the confirmation policy. | Covered by [mutation-record tests](packages/plugin-sdcpn/test/mutation-record.test.ts), [freshness tests](../../../apps/brunch-agent/test/net-freshness.test.ts), and the passing compiler tracer's pre/post hashes and four position effects. This does not prove viewport framing: the synthetic root-creation capture leaves Store outside the visible canvas after layout. Flagship witness remains open. | -| Workpiece query uses recorded current-revision evidence | An ordinary question about visible Petrinaut elements obtains a fresh observation, resolves element IDs to existing mutation-attempt revision IDs, maps those to current workpiece passages and relevant session turns, and reports missing or ambiguous provenance without inventing a link. | Coverage present: [root-creation provenance cases](../../../apps/brunch-agent/test/root-creation.integration.ts). Sampled flagship why answers remain open. | -| Existing mode and tool boundaries remain intact | Reuse [host selection tests](../../../apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx), [catalogue](../../../apps/brunch-agent/src/agents/chat-agent/tool-catalogue.ts) and [schema-carriage comparison](../../../apps/brunch-agent/test/integration/native-schema-carriage.integration.ts) if run repairs touch those boundaries. | Schema remediation passed 2026-09-14: complete query alternatives survive native SDK serialization and reject incomplete/mixed selectors; canonical mutation metadata and documentation schemas survive wrapping. Construction guidance now teaches the mounted batch and provenance lookups; stale workpiece errors direct reconciliation. Core/plugin/aggregate-query tests: 110/88/12 passed; affected build, typecheck and lint passed. Local-only `apps/brunch-agent/.data-wipe-me/evaluations/TEST-schema-remediation-be40bb93-3234-4d70-9259-03c27b7c4062/` holds 22 synthetic SDK requests with zero network attempts. Required presentation fields and the full operation set remain intact; no model-effectiveness or all-mode continuity claim. Full topology-envelope adjudication remains in the successor. | - -Host-executor tests are not persona construction evidence. The readiness gate below must use model-originated calls through the visible product. - -### Readiness gate - -These product acceptance obligations remain unmet at the owner-directed engineering close and transfer to Mission 7d. The worked example is accepted only when the recorded product-manager script works without developer model repair and Lu accepts it. The rows below are judgments over the same run, not separate feature workstreams: - -| Acceptance result | Required oracle | Current disposition | -| --- | --- | --- | -| Inventory is connected and operationally coherent | Lu reviews procurement, supplier disruption, transit, quality/quarantine, expiry/recall, production and demand decisions. Structural and compiled evidence cannot pass this gate. | Open: Lu's flagship acceptance not recorded. | -| Ordinary construction and correction succeed | The retained persona run builds from the elicited account, then updates the workpiece and bounded net region for one changed operational fact or explicit policy choice without unrelated rebuilding. Unsupported work is reported explicitly, never silently omitted or falsely successful. A refusal that prevents a coherent Inventory example leaves this gate open. | Open: requires the from-scratch product run. | -| Code-bearing construction is compiler-clean and legible | The exact final constructed/corrected definition has clean version-correlated diagnostics and recorded layout. Intermediate errors/pending remain visible and repairs start from fresh observations; a timeout never counts as clean. | Open: needs the exact flagship version and diagnostics. | -| Consequential content has a recorded basis | Why questions about two consequential agent-constructed elements trace actual mutation records to workpiece passages and session testimony, checked against native records. Missing or ambiguous basis is disclosed honestly; such disclosures alone do not demonstrate provenance-backed explanation. | Open: needs flagship questions and native records. | -| The flagship starts from scratch and is persona-driven | Initial document/session evidence shows no preloaded net, prior workpiece or retained conversation. A local Pi-harness recording shows ordinary-language elicitation, recurring workpiece revisions, model-originated construction, repair where needed, layout, explanation and correction. The persona's private pack and evaluator reference net never enter the elicitor's inputs. Browser-only scripts, fixed batches and operator-authored repairs are not this proof. | Open: [launcher](../../../apps/brunch-agent/src/evaluations/persona/launch.ts) exists; fresh-state verification and accepted recording remain. | -| The original worked session resumes | Reopen the same local document and conversation in their original stores; recover the final net/workpiece and answer a current-basis question from native records. This is original-session continuity, not export, template copy or identity remapping. | Open: requires the retained run and reopen witness. | -| Compaction dependence is disclosed | If the flagship crosses compaction, reopen, current-workpiece recovery and explanation are proven afterward. If it does not, dependence on uncompacted history is stated at closure and remains required before Mission 9 or any hosted long-lived provenance claim. | Open: depends on the retained flagship run. | +### First real-turn milestone + +One finalized input enters the existing composer with stable identity, produces +one Flue admission and one correlated Brunch answer, settles completely, and is +offered to Live as frozen commentary. Kostandin compares the actual audible answer +with that source, including whether it was heard at all; append acceptance is not +that witness. Manual sessions crossed this path, including tool-backed work, but +also exposed phantom and lost input. Mocked tests establish item ordering, +deduplication, admission correlation, complete-turn settlement and teardown; they +do not establish transcription accuracy, speech adherence or naturalness. Strict +native output eligibility is deliberately not a claim. + +### Provider-free regression portfolio + +Use the existing suites, extending only the newly crossed boundary. The portfolio +below names the intended checks; the executed subset and its limits follow it: + +| Claim to test | Existing oracle / required discriminator | +| --- | --- | +| Realtime default, pinning, consent, isolation and cleanup | Website `src/server/voice/openai-live-session.test.ts`, `src/main/app/voice-interview/live-conversation.test.ts`, `live-conversation-control.test.tsx`, `voice-interview-control.test.tsx`. No provider creation before consent; no mid-session provider change or fallback. | +| Input identity and admission | `realtime-brunch-bridge.test.ts`, `voice-interview-control.test.tsx`; approved Live input tests must distinguish incomplete input, repeated text with different identity, duplicate identity, stale session and uncertain admission without replay. | +| Complete-turn source freeze | `canonical-speech.test.ts`, `buffered-admission.integration.test.ts`, bridge/controller tests; add Live-specific root plus textless/failed continuation cases. A completed earlier segment, a later correction, or `ready` without positive correlated settlement must not authorize sending the final Brunch source to Live. This does not gate independent native speech. | +| Stop and reconstruction | `voice-turn-controller.test.ts`, `brunch-panel-transport.test.ts`; Stop before admission, during work and after settlement; late events cannot revive speech, stale replacement cannot erase current output, reconnect cannot replay history. | +| Rejected proposals and effects | Brunch `test/provider-admission.test.ts`, `test/reconciliation.test.ts`, `test/integration/admission-controls.test.ts`; reject before browser execution and never narrate conflicting/unknown effects as successful or reapply them. | +| Mounted UI | Petrinaut `ai-assistant-panel.test.tsx` and `ai-assistant-panel/ai-assistant-contents.test.tsx`; switch AI/Workpiece during work, retain draft/Voice/Stop and verify cleanup. Inspect rendered changed states with provider endpoints mocked and real microphone blocked. | + +Run affected `test:unit`, `lint:tsc`, `lint:eslint` and formatting checks via Yarn/Turbo +after actual code changes; read evaluation execution safety before hermetic runs. +Do not port the synthetic audio harness. Provider-free tests do not establish +speech fidelity, native full duplex, naturalness or migration readiness. + +Current provider-free evidence lives in the named website, Brunch and Petrinaut +suites above; the PR records the latest command results. It establishes boundary +behavior with mocked provider and media, not a real Brunch interview. + +The accepted tracer sends each frozen commentary source intact once and relies on +provider validation of its 500-token cap, with correlated application-authored +notices for send failure, rejection or unknown acceptance. No progress commentary +is injected before settlement. Cross-submission `answeredBySubmissionId` and +multi-turn recovery still rely on the existing host and need the Live +operation/correction/recovery witness; these checks do not establish that broader +path. + +### Manual acceptance obligations + +Allow about 15–20 minutes, with a familiar process and a disposable local document: + +1. Give an account and answer one short Brunch clarification. Compare spoken + questions and answers against the canonical Brunch text. +2. Hesitate, give a short answer, then elaborate. Correct a consequential number, + negation or condition. Check that subsequent questions preserve it. +3. Request one operation available in the current authorized mode. Add one + follow-up while work proceeds. Inspect admission order, workpiece and actual + tool effects; a queued correction does not retroactively cancel execution. +4. Interrupt speech acoustically, then separately exercise composer Stop during + work. Verify the microphone/audio state, queued input and canonical stop state + separately. “Your turn” is not an acoustic-interruption witness. +5. Compare history, effects and audible responses. Record excess acknowledgements, + missed answers, lost corrections and whether the interview advances. Separate + backend work duration, queue wait and provider/playback delay. No numerical + latency/naturalness threshold has been approved. +6. Before migration-readiness closure, include typed-origin, Voice-origin and + durably aborted entries, close and reopen in a second tab without concurrent + editing, and inspect per-message provenance/stopped presentation. Separately + reproduce locally withheld post-settlement work. No autoplay or automatic + resubmission is allowed. These remain unproved; FE-1604's waiver does not transfer. + +The recovery checks gate readiness. Failure remains visible and returns to the owner +rather than becoming a new store, cross-store transaction or concurrency project +by default. ## Constraints -### Product boundary - -Brunch is Petrinaut's default assistant for understanding, constructing, explaining and revising operational processes as SDCPNs, including organizational, software and cyber-physical operations. It does not claim universal Petri-net assistance. Petrinaut's stock assistant is the feature-flagged alternate; its canonical frontend tool surface and history remain independent. - -### Authority and execution +### Continuing boundaries + +- Native client delegation changes timing/correlation only. Keep `gpt-4o-transcribe` + canonical input and Brunch as sole authority, with prompts/models inherited from + #9667 unchanged by voice work. Add no + session, store, queue, service or provider call. `session.instructions.append` is + admitted only to resolve attached delegations Brunch cannot serve; thinking/progress + appends remain unaccepted. Provider validation replaces only the 500-byte safeguard, + not no-truncation, no-chunking, no-retry, no-replay or Stop/cleanup requirements. +- Brunch owns interpretation, substantive questions/answers, evidence sufficiency, + workpiece meaning and tool selection. Flue owns canonical identity, admission, + ordering, settlement and reconstruction. Petrinaut owns authorized native + execution and observed effects. Live is conversational delivery, not a second + domain agent in the intended division of responsibility. The selected experiment + permits unbuffered speech that may violate that intention; + measure those violations without granting canonical or tool authority. Never + insert its paraphrase as another canonical assistant answer. +- Exactly `PETRINAUT_VOICE_PROVIDER=realtime|live`, unset `realtime`; reuse current + enablement/credentials. Pin provider and delivery policy for the session. No + `live-experience`, `live-brunch`, new provider framework, automatic replay or + standalone fallback. The child's `live` path is the local integration; the pinned + parent remains the standalone comparison. Neither is a migration-readiness claim. +- Retain current host queue policy. The one-waiting-input contract is not #9638's + FIFO. Realtime's interruption preference and echo/noise filtering are transport + behavior, not proof that native Live establishes input ordering. Interrupting + audio, retaining/admitting speech, correcting pending work and durable Stop are + four different actions. Corrections are subsequent Brunch input, not silent + modification of an in-flight operation; if that strains the interview, surface + the policy decision instead of changing it. +- Preserve #9562's complete proposal validation, bindings, actual effects, tool + result metadata, history replacement and mounted controls. Unknown/conflicting + execution cannot be reapplied or spoken as success. A completed explanation of + a rejected operation differs from a failed/unfinished agent continuation. +- Exact relay and settlement-gated faithful rephrasing are different policies. + The selected policy is best-effort paraphrasing from settlement-gated context, + not settlement-gated audio. Fidelity is a manual criterion: preserve facts, + quantities, negation, uncertainty, corrections and Brunch-authored questions; + prompting cannot guarantee that. Keep complete + reports visible. No truncation to fit 500 tokens and no private reasoning/raw + payloads sent as “progress.” Only real application events may ground permitted + acknowledgements. Keep them brief and non-repetitive as a prompting objective, + with no claimed deterministic wording/frequency enforcement. +- No agent-started provider sessions/inference/microphone, synthetic recordings, + audio harness, Brunch model/prompt edits, new service/infrastructure, donor host + overwrite, merge, deploy or Notion writes. Only the explicitly authorized restack + may rewrite the two voice branches; #9667 and unrelated worktree files stay + untouched. Provider-free verification and draft stack publication are allowed. -- Petrinaut Core owns canonical mutation and command schemas, including `getNetCompilationErrors` and `applyAutoLayout`. Brunch selects or projects them and does not copy their field contracts. -- Ordinary construction exposes one model-facing `mutate_petrinaut_net` carrier, not a parallel catalogue of individual mutations. Its admitted set is governed by the capability matrix, not by a schema-size threshold. -- Every code-bearing batch, or batch that changes a code dependency, reaches a version-correlated diagnostics result before Brunch relies on it. A bounded wait may return `pending`; it never becomes clean by timeout. -- Petrinaut's ELK layout is authoritative. Coordinates do not inherit operational basis, and nothing may mutate after the recorded final hash. -- `query_workpiece` is the one model-facing current-basis operation. Its plugin-contributed selector accepts Petrinaut element IDs; the plugin resolves them through recorded effects to existing per-operation mutation-attempt tool-call IDs, which are the target mutation revision identities. Generic workpiece code maps those IDs to workpiece revisions, passages and relevant session turns. The Brunch app supplies authorized canonical history and current-document reconciliation. Stable semantic identity across arbitrary workpiece rewrites belongs to Missions 9 and 10. -- Safeguards remain only when earned by an observed failure, external constraint or explicit owner requirement. The 30-operation maximum remains provisional; the retired 64 KiB schema threshold is not a provider limit. - -### Persistence and identity - -- Flue history is canonical conversation history; the workpiece is the recoverable operational account; Petrinaut is the model authority. -- Retain the original run's session, workpiece and net through existing local persistence and native evidence. Label local-only records and name their actual locations at handoff; export and Postgres delivery are not prerequisites to believing an inspected run. Do not introduce a second persistence system. -- A projection over Flue history remains recomputable and unpersisted. It cannot introduce identities, repair or drop ambiguous records, consult a live Petrinaut state as hidden input, reorder history, or become another authority. - -### Ownership +## Fog-line -- Brunch core owns universal workpiece tools and formalism-independent guidance. -- Petrinaut Core owns model actions, commands and canonical schemas. -- The SDCPN plugin owns the selected carrier, formalism-specific operation policy, basis/effect interpretation and construction guidance. -- The Brunch app owns composition, authorized history, browser/document reconciliation, freshness, workpiece-query history access and operational diagnostics. The retained Postgres catalogue/copy path is successor work. -- The Petrinaut website owns browser execution, diagnostics/layout host integration, assistant selection and document routing. Preserve stock transport/tool/history independence during any run-driven repair; deferred remote-route contracts live in the successor draft. -- Workpiece operations use action names rather than ownership prefixes: `read_workpiece` reads the current workpiece and source/locator material; `mutate_workpiece` submits a complete next revision and records its verified delta from the cited base. Retained histories may recognize the legacy `brunch_workpiece` and `update_workpiece` names, but new conversations mount only the current names. Canonical Petrinaut action and command names remain unchanged. Definition homes, mounts, execution hosts, display consumers and persistence must agree before any other tool is renamed or moved. +- `session.thinking.append` for event-backed progress is **not yet accepted**. Whether + progress improves the interview without duplicate or unsupported speech requires + owner selection after the first manual exchange; do not implement it now. +- Delegation/transcription timing correlation is a selected heuristic, not proof of + semantic alignment. Provider-free tests can establish the claim rule and append + outcomes, not that Live waits, avoids repetition or speaks a settled answer. +- Phantom input during silence remains unexplained. Server VAD can split hesitation + into separate inputs, and the unchanged one-waiting-input policy can lose further + speech. FE-1712 must distinguish speaker feedback from headphone input and record + whether capture constraints change either failure. + +Official documentation inspected 2026-09-11: +[migration](https://developers.openai.com/api/docs/guides/live-migration), +[Live reference](https://developers.openai.com/api/reference/resources/live/), +[server controls](https://developers.openai.com/api/docs/guides/voice-server-controls?api=live). + +The selected input proposal additionally uses the current official +[transcription guide](https://developers.openai.com/api/docs/guides/realtime-transcription) +and [VAD guide](https://developers.openai.com/api/docs/guides/realtime-vad). +The [client-secret reference](https://developers.openai.com/api/reference/resources/realtime/subresources/client_secrets) +and SDK transcription schemas expose GPT-4o transcription models and server VAD; +the [model card](https://developers.openai.com/api/docs/models/gpt-4o-transcribe.md) +lists Realtime support. This makes `gpt-4o-transcribe` with server VAD the selected +candidate, not proof of the exact transcription-only WebRTC handshake. Documentation +examples do not establish that entire path. Completed-transcript events, out-of-order +reconciliation and dual-session cleanup are checked with mocks. Actual provider +compatibility and transcription/Live disagreement remain manual-test obligations. + +### Input: no supported native finalization marker + +`session.input_transcript.delta` explicitly has no transcript-done event. +`session.delegation.created` carries an ID/timing/target, not final text. The +migration example's `readContext()` is an application callback that assumes a +ready context; it does not provide the missing finalizer. Silence, timeout, +delegation and transcript timestamps cannot serve as authoritative admission. + +Input alternatives considered: + +1. **Explicit review and composer submission:** the user approves the displayed + draft text, making that frozen text authoritative. Late deltas cannot silently + extend a submitted message. Not selected: it changes hands-free interaction + and does not solve output eligibility. +2. **Separate authoritative transcription:** route consented input to a supported + transcription lifecycle as well as Live, with stable capture identity and an + explicit final event. This is a hybrid, with extra cost, timing/ordering, + disagreement and cleanup obligations. Selected by Kostandin, with the concrete + OpenAI path above. Live testing established the path and exposed phantom and + lost input that still block acceptance. + +### Output: context acceptance is not permission to hear speech + +Live chooses when to speak even while Brunch works. `session.commentary.append` +supplies up to 500 tokens of paraphrasable context; `session.commentary.appended` +only acknowledges acceptance. It supplies neither exact relay, a response-scoped +audio association, nor speech/playback completion. `session.thinking.append` is +not a secrecy boundary. Sideband output timestamps describe audio ranges, not +which frozen Brunch answer authorized them or word-level playback. Live has no +Realtime-style per-response terminal event. Input mute does not stop output. + +Output alternatives considered: + +1. **Keep strict ownership:** retaining Realtime remains the application fallback, + not an automatic session fallback or the selected experiment. A controlled hybrid + could use authoritative transcription and bounded speech delivery; Live audio + must remain muted/dropped unless eligible. Replacing Live's audible path would + change what the experiment tests. Buffering native Live is only a feasibility + candidate: source/audio association, coverage of late transcripts, approval, + cancellation, completion, stale-audio removal and suppressed-context recovery + must be justified before release. Its latency may defeat the goal. +2. **Change the speech policy explicitly:** permit native unbuffered Live speech + with best-effort prompting and post-hoc inspection. This accepts the possibility + of independent substantive questions/claims before settlement. This is the + accepted experimental exception. Protected + tool execution does not itself enforce speech ownership. This is not exact relay + or enforced faithful rephrasing. + +Neither input alternative fixes output, and output approval does not finalize input. +Both alternatives are explicitly selected by Kostandin. Neither the selected policy +nor draft publication establishes production acceptance. -### Scope boundary and external owners +## Stop or reorient -- No Petrinaut simulation scenarios or metrics, structured-question widgets or questionnaires enter 7c unless PM explicitly recuts the objective. -- The established reference net's scenarios and metrics remain evaluator reference content, not preloaded model content, supported creation/editing or behavioral evidence. -- Fixture packaging/seeding, template distribution, copy/reset/remapping, all six-pack probes and the non-Inventory end-to-end witness belong to the [next mission](docs/mission-drafts/worked-example-distribution-and-breadth.md). Preserve existing implementations and regression pins; deferral is neither deletion authority nor a completion claim. -- No public deployment, hosted authentication, spend control, backup/recovery or multi-replica safety claim enters 7c. Tim owns hosted infrastructure and remote readiness under the Mission 8 successor and [FE-1569](https://linear.app/hash/issue/FE-1569). -- Voice limitations remain Kostandin's. General optimization handoff is Mission 11's. -- Crew reservation is a legacy, test-authored Mission 6 resume fixture: regression evidence only, not demo content, provenance evidence, a worked-model template, an owned-copy implementation or a precedent for the Inventory path. +If `gpt-4o-transcribe` rejects the current client-secret or SDP exchange, inspect the +provider failure rather than silently changing VAD, switching models or adding +application silence timers. Any model, transport or interaction-policy change +requires an approved recut. -## Fog-line +Reorient if live testing reproduces phantom input, loses or reorders finalized +speech, revives stopped work, invents progress, or lets Live answer substantive +questions independently. FE-1712 must first distinguish capture feedback from +transcription behavior. If the experiment cannot meet its interaction bar, retaining +Realtime is a recommendation, not an automatic session fallback. -- **Assumption-based preview — PM decision:** decide whether Brunch may offer a provisional model when operational evidence is incomplete. The recommended policy is evidence-first; offer only when blocked; require explicit assent; distinguish assumptions from testimony in workpiece, explanation and provenance; keep them confirmable, replaceable and rejectable. Settle what assent authorizes, which assumptions are acceptable, how provisional content appears in the UI and what review makes it accepted meaning. This is a candidate policy, not permission to implement it. -- **Live inference — builder:** both participants produced paid replies. The launcher owns its services and fresh local store; it does not reuse unverified servers. Persona budget enforcement is removed under owner direction; native usage remains an estimate, not an invoice, and historical unknown usage stays unknown. The latest run exercised the incremental-construction guidance and produced a net, but construction still began late and a provider refusal prevented completion. The unresolved questions are cadence, live confirmation of the diagnostics repair, and explicit fallback selection; see Status for the retained run and next decision. -- **Empty-start product path:** the launcher supports an omitted `--initial-net`, but flags alone do not prove the initial canvas, workpiece or history. Check the ordinary product route and starting evidence before claiming a from-scratch run; repair only a demonstrated setup blocker. -- **Carrier shape:** provider/product probes decide whether one full union, capability-grouped carriers or supported deferred loading is simplest. -- **Shared history projection:** shared interpretation of canonical Flue history is the product contract, not a predetermined module. The candidate projection is retained only if parity tests show that it removes duplicate interpretation without creating a store, identity scheme or authority. -- **Question marker reliability:** `brunch_mark_question` supports Voice question replay when the model calls it with exact matching prose. Plumbing is proven; autonomous activation reliability is not. Decide whether this model-compliance mechanism remains mounted, moves behind a deterministic response contract, or is removed. -- **Diagnostics live confirmation:** the explicit captured-definition worker request is implemented and passes the browser compiler tracer. Confirm it completes checks and supports repair on the retained persona model; protocol selection is no longer open. +### Carry-over classification -## Stop or reorient +Heads below were rechecked through GitHub; no donor code is imported. -- Stop carrier expansion if the provider/product route cannot reliably select and populate representative operations; compare grouped or deferred carriers rather than imposing an arbitrary byte cap. -- Stop code-bearing construction if diagnostics cannot be correlated to the exact post-mutation definition. -- Stop automatic layout if it can silently move user-arranged content, escape effect accounting or change the document after its recorded final hash. -- Stop the assistant flag if it requires Brunch-specific behavior inside `@hashintel/petrinaut` beyond a generic host extension or merges provider histories. -- Stop the from-scratch claim if the run starts from a prebuilt net, existing workpiece/history, or requires operator-authored mutations to count as success. -- Stop Inventory acceptance for an inert, flattened, illegible, compiler-broken or operator-authored model, or a path that only works with Inventory-specific language. -- Keep separate history walks rather than extracting a shared projection that fails the authority constraints. +| Source | Required data/authority contract | Realtime mechanism or policy disposition | +| --- | --- | --- | +| [#9585](https://github.com/hashintel/hash/pull/9585), [038e7471be](https://github.com/hashintel/hash/commit/038e7471be3f654d3612e411e8a3fc868eb348cd), open/paused | Full reports stay canonical and visible; delivery is not another answer. | Bounded offers/exact replay and Voice-specific prompting are experiment mechanisms, not copied. Negative short-answer naturalness is not a passing baseline. Brunch prompts stay unchanged. | +| [#9638](https://github.com/hashintel/hash/pull/9638), [a2d01de0b2](https://github.com/hashintel/hash/commit/a2d01de0b2a500164448b8d215609edde0e799e0), open/paused | Whole-turn settlement including textless continuations; frozen source; event-backed notices; no history autoplay or late speech revival. | Isolated `response.create`/response terminal machinery is Realtime-specific and cannot be fabricated in Live. Rephrasing fidelity and FIFO/Resume/Discard are explicit policies, not inherited authority here. | +| [#9651](https://github.com/hashintel/hash/pull/9651), [6af148d2eb](https://github.com/hashintel/hash/commit/6af148d2eb05e7d78a534aafa325189c3c309654), open | Distinguish canonical evidence, acknowledgements, actual audible answer and application effects. | No harness or synthetic recordings. Published evidence does not establish a successful live sweep; attribution, silent containers, pre-admission failures and playback wait must not become success claims. | +| [#9622](https://github.com/hashintel/hash/pull/9622), [1677453c87](https://github.com/hashintel/hash/commit/1677453c874eb94300e01f05ce79be1f543609d8), closed/unmerged | Model adherence is not proved by mocked routing. | Negative speech-authoring control, not a donor or permission for another Brunch prompt/tool variant. | +| [#9571](https://github.com/hashintel/hash/pull/9571), [8e45c9edc1](https://github.com/hashintel/hash/commit/8e45c9edc19171ea3721fdc8ca07fcf2bdf6e1de), closed/unmerged | Domain/conversation/execution boundaries and honest reconstruction remain required. | Planning only, not permission for autonomous interviewing or a second dialogue store. | ## Deferred -- **Mission 7d — one alternative provider and worked-example completion:** all open readiness rows above, live diagnostics confirmation and the retained `run-1vFeVo` continue on `ln/fe-1573-mission-7d-provider-worked-example`. Qualify actual schema carriage, browser-tool execution and native history continuity for one selected provider; do not build a general fallback framework or compatibility matrix. Provider/model selection and another paid run remain to be agreed with Lu. -- [Next mission — worked-example distribution and portfolio breadth](docs/mission-drafts/worked-example-distribution-and-breadth.md): complete versioned fixtures, build/Postgres seeding, connected-bundle copy/reset/reopen, identity/provenance remapping, template/sibling isolation, remote-mode continuity, six-pack capability probes, a non-Inventory end-to-end run and full-envelope/topology adjudication. Consumes the accepted original example; unresolved fork capability and expected-failure pins remain open there. -- [Mission 9](docs/mission-drafts/9-traceable-projection.md) / [FE-1438](https://linear.app/hash/issue/FE-1438/project-an-evidence-backed-workpiece-into-a-traceable-live-sdcpn): unchanged repeat without duplication; changed-input impact; retirement and identity epochs; concurrent/manual-edit reconciliation; cross-revision passage identity; repeated construction beyond the next mission's portfolio probes. -- [Mission 10](docs/mission-drafts/10-bounded-reviewer-revision.md) / [FE-1394](https://linear.app/hash/issue/FE-1394/revise-one-traceable-net-region-through-targeted-reviewer-elicitation): general reviewer authority and revision cadence. -- [Mission 11](docs/mission-drafts/11-optimisation-handoff.md): consumer-accepted optimization handoff. -- [After-demo evaluation](docs/mission-drafts/7-explainable-construction.md): broader semantic, behavioral, provenance and lifecycle evaluation. -- [Future spine](MISSION.next.md): deployment, provider migration and unallocated product concerns. +The shared [recovery obligation](MISSION.next.md#voice-after-the-live-transport-cut) +remains in the future spine. This experiment consumes its readiness witness, not +the old waiver. Broader construction, demo content, deployment, concurrency and +optimisation remain in their existing planning homes. Native naturalness and live +speech/transcription fidelity remain Kostandin's manual observations, not agent +verdicts. + +- [Mission 7c on #9667](https://github.com/hashintel/hash/blob/dee90599e9a07d9fa3e55d0711c14491e9ce5c7c/libs/%40hashintel/brunch-agent/MISSION.md) + is provisionally closed for engineering review, not worked-example acceptance. + Mission 7d inherits the unresolved example obligations; voice does not claim or + change that acceptance. +- [Distribution and breadth](docs/mission-drafts/worked-example-distribution-and-breadth.md) + retains complete-bundle copying, fixture distribution and portfolio obligations. +- [After-demo evaluation](docs/mission-drafts/7-explainable-construction.md) retains + full-region semantic, behavioural, passage, adversarial and lifecycle evaluation. +- [Future spine](MISSION.next.md) retains general repeat/change/retirement, + concurrency/reviewer, optimisation and other source/plugin work. Its Mission 7c + references describe the upstream mission, not another live mission on this child. diff --git a/libs/@hashintel/brunch-agent/MISSION.next.md b/libs/@hashintel/brunch-agent/MISSION.next.md index 7b884d11892..005a1132949 100644 --- a/libs/@hashintel/brunch-agent/MISSION.next.md +++ b/libs/@hashintel/brunch-agent/MISSION.next.md @@ -2,6 +2,12 @@ > Future sequence and decision register only; not execution authority. [`MISSION.md`](MISSION.md) owns live scope and progress. Successor drafts become executable only after an owner-authorized cut; archives and git history retain prior contracts. +On this stacked voice branch, `MISSION.md` owns FE-1664. The inherited Mission 7c +map and its mission-section references below belong to the +[upstream contract](https://github.com/hashintel/hash/blob/dee90599e9a07d9fa3e55d0711c14491e9ce5c7c/libs/%40hashintel/brunch-agent/MISSION.md) +on #9667, not to a second execution authority here. The stack does not close that +mission or grant its paid-run permissions to voice work. + ## How to use this spine Read this file to answer four questions: @@ -30,7 +36,10 @@ The first composed product is `process-sdcpn`: operational processes represented Brunch is intended to become Petrinaut's default operational-process assistant. Petrinaut's stock assistant remains an alternate selected by a host feature flag. Stock mode retains its canonical frontend tools and separate history; Brunch uses its own projected or adapted tool surface. The host must not splice histories, reinterpret prior tool calls across modes or make stock behavior depend on Brunch. -For live route and assistant-selection behavior, see [mission ownership constraints](MISSION.md#ownership). Future deployment policy and remote switching are the [host-choice fork](#host-choice-and-continuity). +For live route and assistant-selection behavior, see the +[Mission 7c ownership constraints](https://github.com/hashintel/hash/blob/dee90599e9a07d9fa3e55d0711c14491e9ce5c7c/libs/%40hashintel/brunch-agent/MISSION.md#ownership). +Future deployment policy and remote switching are the +[host-choice fork](#host-choice-and-continuity). The accepted naming target is: @@ -67,7 +76,7 @@ A flagship proves one accepted product path. It does not prove every operational - Mission 7 tracks [FE-1573](https://linear.app/hash/issue/FE-1573/construct-and-explain-one-real-net-region-from-a-genuine-conversation) and partially advances [FE-1478](https://linear.app/hash/issue/FE-1478/provide-provenance-from-a-generated-net-back-to-the-requirements-graph) without closing the broader provenance objective. - [Mission 7a](docs/mission-archive/7a-workpiece-construction-explanation-groundwork.md) established workpiece, construction-record and explanation groundwork and landed on `main`. - [Mission 7b](docs/mission-archive/7b-ordinary-batched-construction-provenance.md) established the ordinary selected structural batch, correction, recorded basis/effects, reopen and experimental create-new seam. Its engineering [PR #9649](https://github.com/hashintel/hash/pull/9649) remains a separate external closeout. -- [Mission 7c](MISSION.md) is provisionally closed for engineering review with browser-visible persona construction and verified repairs. The Inventory worked example remains unaccepted; consult its [Status](MISSION.md#status) and [proof dispositions](MISSION.md#proof). +- [Mission 7c](https://github.com/hashintel/hash/blob/dee90599e9a07d9fa3e55d0711c14491e9ce5c7c/libs/%40hashintel/brunch-agent/MISSION.md) is provisionally closed for engineering review with browser-visible persona construction and verified repairs. The Inventory worked example remains unaccepted; consult its [Status](https://github.com/hashintel/hash/blob/dee90599e9a07d9fa3e55d0711c14491e9ce5c7c/libs/%40hashintel/brunch-agent/MISSION.md#status) and [proof dispositions](https://github.com/hashintel/hash/blob/dee90599e9a07d9fa3e55d0711c14491e9ce5c7c/libs/%40hashintel/brunch-agent/MISSION.md#proof). - **Authorized next cut — Mission 7d:** `ln/fe-1573-mission-7d-provider-worked-example` qualifies one alternative provider and completes the retained Inventory example. It consumes all open 7c readiness obligations, not fixture distribution or portfolio breadth. Lu authorizes reuse of FE-1573; no tracker state change is implied. - [After-demo construction and explanation evaluation](docs/mission-drafts/7-explainable-construction.md) owns broader cross-scenario quality, behavioral correspondence, explanation usefulness, provenance stress and lifecycle evaluation after a useful flagship exists. @@ -123,22 +132,22 @@ This register records product consequences, not every engineering idea. A scope ### Decisions to report or confirm now -- **Assistant scope — PM communication required:** communicate the accepted [product boundary](MISSION.md#product-boundary). +- **Assistant scope — PM communication required:** communicate the accepted [product boundary](https://github.com/hashintel/hash/blob/dee90599e9a07d9fa3e55d0711c14491e9ce5c7c/libs/%40hashintel/brunch-agent/MISSION.md#product-boundary). - **Assistant deployment policy — future owner decision:** resolve the [host-choice fork](#host-choice-and-continuity). -- **Live exclusions:** [MISSION.md](MISSION.md#scope-boundary-and-external-owners) settles the current boundary; [distribution and breadth](docs/mission-drafts/worked-example-distribution-and-breadth.md) owns the deferred portfolio and bundle scope. These are not pending confirmations here. -- **Assumption-based preview — open PM decision:** the candidate policy and unanswered questions have one home in the [live Fog-line](MISSION.md#fog-line). -- **Behavioral evaluation:** use the [after-demo draft](docs/mission-drafts/7-explainable-construction.md); the live mission's [claim discipline](MISSION.md#claim-discipline) determines its evidence tier. +- **Live exclusions:** [Mission 7c](https://github.com/hashintel/hash/blob/dee90599e9a07d9fa3e55d0711c14491e9ce5c7c/libs/%40hashintel/brunch-agent/MISSION.md#scope-boundary-and-external-owners) settles the current boundary; [distribution and breadth](docs/mission-drafts/worked-example-distribution-and-breadth.md) owns the deferred portfolio and bundle scope. These are not pending confirmations here. +- **Assumption-based preview — open PM decision:** the candidate policy and unanswered questions have one home in the [Mission 7c Fog-line](https://github.com/hashintel/hash/blob/dee90599e9a07d9fa3e55d0711c14491e9ce5c7c/libs/%40hashintel/brunch-agent/MISSION.md#fog-line). +- **Behavioral evaluation:** use the [after-demo draft](docs/mission-drafts/7-explainable-construction.md); [Mission 7c's claim discipline](https://github.com/hashintel/hash/blob/dee90599e9a07d9fa3e55d0711c14491e9ce5c7c/libs/%40hashintel/brunch-agent/MISSION.md#claim-discipline) determines its evidence tier. ### Capability and lifecycle strains -- **Capability and portfolio obligations:** [Mission 7c](MISSION.md#proof) owns the selected run's evidence; [its successor](docs/mission-drafts/worked-example-distribution-and-breadth.md#outcome-2--establish-portfolio-breadth) owns breadth and full-envelope adjudication. +- **Capability and portfolio obligations:** [Mission 7c](https://github.com/hashintel/hash/blob/dee90599e9a07d9fa3e55d0711c14491e9ce5c7c/libs/%40hashintel/brunch-agent/MISSION.md#proof) owns the selected run's evidence; [its successor](docs/mission-drafts/worked-example-distribution-and-breadth.md#outcome-2--establish-portfolio-breadth) owns breadth and full-envelope adjudication. - **Repeat/change/retirement/concurrency — Mission 9.** Mission 7c should leave stable IDs, fresh-base discipline, ordinary correction and current-state why as a usable handoff. - **General reviewer revision — Mission 10.** Mission 7c's ordinary correction does not establish reviewer authority, qualification, conflict handling or general patch locality. - **Optimization handoff — Mission 11.** Do not infer an optimization product from code-bearing dynamics. ### Conditional technical strains -- **Compaction survival:** consume the live mission's [compaction disposition](MISSION.md#readiness-gate) before Mission 9 or a long-lived hosted provenance claim. If proof remains open, exercise recovery and explanation across compaction first. +- **Compaction survival:** consume [Mission 7c's compaction disposition](https://github.com/hashintel/hash/blob/dee90599e9a07d9fa3e55d0711c14491e9ce5c7c/libs/%40hashintel/brunch-agent/MISSION.md#readiness-gate) before Mission 9 or a long-lived hosted provenance claim. If proof remains open, exercise recovery and explanation across compaction first. - **Passage identity across revisions:** rename/move/paraphrase/split/merge/delete/reintroduce continuity belongs to Mission 9/10; consume the live mission's current-revision evidence without inferring continuity. - **Arbitrary import/clone:** re-enter general import, attachment rebinding or complete effect-history migration only for a named portability consumer; the planned fixture-copy boundary is defined in the [successor draft](docs/mission-drafts/worked-example-distribution-and-breadth.md#connected-bundle-contract). - **Provider qualification — Mission 7d.** Repeated Anthropic refusals blocked the retained example. The next cut owns canonical schema carriage, tool selection/arguments, native-history continuity and compiler repair for one alternative provider on that example, with observed latency and cost. A general fallback framework and portfolio-wide provider comparison remain outside the cut; re-enter those only for a named broader consumer. Provider success does not establish semantic or behavioral correctness, and no production default switch is authorized by the branch transition. @@ -194,6 +203,12 @@ Immediate switching from a review or gap report into renewed elicitation remains ### Voice after the live transport cut +The FE-1664 integration is governed by this branch's [mission](MISSION.md). +Native Live delivery and canonical transcription do not waive the recovery +obligation below or establish live provider compatibility. Historical waiver and +attribution rationale remains in the +[pre-restack voice record](https://github.com/hashintel/hash/blob/14cad8904de351166c1d58ad0973e643082e747a/libs/%40hashintel/brunch-agent/MISSION.next.md#voice-after-the-live-transport-cut). + Kostandin owns the current Voice continuation. The accepted path covers microphone input, mutation, resume and durable Stop. Direct spoken-user attribution after hydration, durable recovery of locally withheld post-settlement browser work and comparative latency remain unproved. Before a mission claims Voice plus exact resume or broad pre-release continuity, run one reproducible product scenario containing typed-origin and Voice-origin messages and a durably stopped assistant entry. After reopening, verify per-message origin and stopped presentation, and distinguish local Exit voice mode from durable Stop. diff --git a/libs/@hashintel/petrinaut/docs/ai-assistant.md b/libs/@hashintel/petrinaut/docs/ai-assistant.md index fcc789c0f7e..c0f5c69ef82 100644 --- a/libs/@hashintel/petrinaut/docs/ai-assistant.md +++ b/libs/@hashintel/petrinaut/docs/ai-assistant.md @@ -66,19 +66,22 @@ understand and selecting **Start voice mode**. Petrinaut remembers that acknowle browser for the current disclosure version, so later uses of **Start voice mode** start directly. If browser storage is unavailable or the disclosure changes, Petrinaut asks again. -Some hosts offer a **GPT-Live · Experimental interview** instead. It uses the same -Voice setup and dock, but is a standalone conversation: it does not add chat messages -or edit the net. Consent and **Start voice** are required for each session. Its dock -normally shows **Listening** or **Speaking**, with collapse/expand and **End voice -mode**. Speaking reflects local audio activity, not a completed turn; the microphone -stays open while it speaks. The input ribbon follows your microphone level where -the browser supports it. The experimental label appears only during consent. -If the browser blocks remote playback, the dock keeps the warning visible and -offers **Play voice audio**; selecting it retries playback from that user gesture. -Ending Voice stops microphone capture and playback; closing the panel ends -the experiment rather than pausing it. A connection error returns to setup without -retrying. The following turn-taking and playback controls apply to integrated -Brunch Voice, not that experiment. +Some hosts offer a Brunch-backed GPT-Live voice interview instead. It uses one +GPT-Live session for conversational audio and a separate transcription session for +canonical user messages. Brunch remains responsible for domain answers, chat history, +and model operations; settled Brunch prose is supplied to Live for best-effort spoken +delivery. Consent and **Start voice** are required for each session. The permission +panel explains both OpenAI audio streams and text retention, with a permission +checkbox, **Start voice**, and **Cancel**. Cancel returns to text without starting a +session. Its dock provides collapse/expand and **End voice mode**; **Thinking** means +Brunch is working, while **Speaking** reflects local audio activity rather than a +completed turn. The microphone stays open during playback and the input ribbon follows +its level where the browser supports it. If the browser blocks remote playback, the +dock keeps the warning visible and offers **Play voice audio**; selecting it retries +playback from that user gesture. Ending Voice stops both sessions, microphone capture, +and playback; closing the panel ends the experiment rather than pausing it. A +connection error returns to setup without retrying. The detailed turn-taking and +playback controls below apply to Realtime-based Brunch Voice, not the Live experiment. While a session runs, the composer is replaced by a low-profile Voice dock at the foot of the panel: a ribbon that fades out at both ends and one short state -- **Connecting**, **Listening**, @@ -89,6 +92,11 @@ flicker above the line. While the assistant speaks the ribbon takes on a restrai motion instead, colour crossfading as the turn changes hands, so which side holds it is readable at a glance. It flattens to near a line whenever nobody holds the turn. +For Brunch-backed Live sessions, **Thinking** follows an active submitted or streaming +request, even without microphone activity. **Speaking** takes precedence during playback; +once work and playback are idle, the dock returns to **Listening**. This status does not +mute the microphone or announce progress aloud. + Voice failures and recovery warnings, including unconfirmed submissions and input that was not retained, join the Voice warning indicator rather than appearing as global notifications. Hover to preview or click to read the complete details, including while the panel is collapsed. Distinct From da58094e93ba3e0df1b531f5e4013e4821aca1b7 Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Tue, 15 Sep 2026 12:27:29 +0200 Subject: [PATCH 2/8] Keep Live sessions active after Brunch stops Co-authored-by: Cursor --- .../live-conversation-control.test.tsx | 22 +++++++++++++++---- .../live-conversation-control.tsx | 17 +++++++------- 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.test.tsx b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.test.tsx index 8c314811712..30329d1710a 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.test.tsx +++ b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.test.tsx @@ -22,11 +22,15 @@ import { import type { FlueClient, FlueConversationState } from "@flue/sdk"; import type { PetrinautAiVoiceModeContext } from "@hashintel/petrinaut/ui"; +const liveConversationMocks = vi.hoisted(() => ({ + stop: vi.fn(async () => {}), +})); + vi.mock("./live-conversation", () => ({ createLiveConversation: vi.fn(() => ({ retryPlayback: vi.fn(async () => {}), start: vi.fn(async () => {}), - stop: vi.fn(async () => {}), + stop: liveConversationMocks.stop, appendCommentary: vi.fn(() => true), appendInstructions: vi.fn(() => true), })), @@ -152,6 +156,7 @@ test.each(["submitted", "streaming"] as const)( expect(props.reportVoiceSessionState).toHaveBeenLastCalledWith( expect.objectContaining({ phase: "listening" }), ); + expect(liveConversationMocks.stop).not.toHaveBeenCalled(); rerender( , ); @@ -184,7 +189,11 @@ test("reuses setup and reports failure to the host dock and notification surface expect(createLiveConversation).not.toHaveBeenCalled(); await start(); expect(props.reportVoiceSessionState).toHaveBeenLastCalledWith( - expect.objectContaining({ phase: "connecting", notice: null }), + expect.objectContaining({ + phase: "connecting", + notice: null, + warningMessage: null, + }), ); const onState = vi.mocked(createLiveConversation).mock.calls[0]![0]; act(() => onState({ phase: "connected", message: null })); @@ -197,6 +206,7 @@ test("reuses setup and reports failure to the host dock and notification surface microphoneMuted: false, errorMessage: null, notice: null, + warningMessage: null, }); act(() => onState({ @@ -284,6 +294,7 @@ test("reuses setup and reports failure to the host dock and notification surface microphoneLevel: 0, microphoneMuted: true, notice: null, + warningMessage: null, }); expect(screen.getByText(connectionError)).toBeTruthy(); expect( @@ -660,6 +671,7 @@ test.each(["commentary", "instructions"] as const)( phase: "listening", errorMessage: null, notice: null, + warningMessage: null, }), ); } @@ -673,14 +685,16 @@ test.each(["commentary", "instructions"] as const)( expect.objectContaining({ phase: "listening", errorMessage: null, - notice: expect.stringContaining(text) as unknown, + notice: null, + warningMessage: expect.stringContaining(text) as unknown, }), ); for (const nextStatus of ["unknown", "accepted"] as const) { act(() => call[4]({ ...result, eventId: "later", status: nextStatus })); expect(props.reportVoiceSessionState).toHaveBeenLastCalledWith( expect.objectContaining({ - notice: expect.stringContaining(text) as unknown, + notice: null, + warningMessage: expect.stringContaining(text) as unknown, }), ); } diff --git a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.tsx b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.tsx index b244bb8e653..03466b5c96f 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.tsx +++ b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.tsx @@ -57,7 +57,7 @@ export const LiveConversationControl = ({ subscribeToStopRequested, }: LiveControlsContext) => { const [consented, setConsented] = useState(false); - const [notice, setNotice] = useState(null); + const [warningMessage, setWarningMessage] = useState(null); const [state, setState] = useState({ phase: "idle", message: null, @@ -139,11 +139,11 @@ export const LiveConversationControl = ({ ); useEffect(() => { - if (stopped || inputMode !== "voice" || !isAiAssistantOpen) { + if (inputMode !== "voice" || !isAiAssistantOpen) { bridge.current?.stop(); void session.current?.stop(); } - }, [stopped, inputMode, isAiAssistantOpen]); + }, [inputMode, isAiAssistantOpen]); useEffect( () => @@ -181,7 +181,8 @@ export const LiveConversationControl = ({ phase === "error" ? 0 : (activity?.microphoneLevel ?? 0), microphoneMuted: phase === "error", errorMessage: phase === "error" ? message : null, - notice: playbackBlocked ? message : notice, + notice: playbackBlocked ? message : null, + warningMessage, ...(playbackBlocked ? { canRetryPlayback: true } : {}), } : null, @@ -195,7 +196,7 @@ export const LiveConversationControl = ({ activity, status, stopped, - notice, + warningMessage, reportVoiceSessionState, ]); @@ -230,7 +231,7 @@ export const LiveConversationControl = ({ if (!consented || phase === "stopping" || sessionActive.current) return; sessionActive.current = true; setConsented(false); - setNotice(null); + setWarningMessage(null); setState({ phase: "connecting", message: null }); const next = createLiveConversation( (nextState) => { @@ -273,7 +274,7 @@ export const LiveConversationControl = ({ result.status === "local-failure" ? "could not be sent to Live locally" : "was rejected by Live"; - setNotice( + setWarningMessage( `The ${label} ${outcome}. Check the conversation; no automatic retry or replay was made. Acceptance does not confirm playback.`, ); }, @@ -282,7 +283,7 @@ export const LiveConversationControl = ({ submit: (input) => latest.current.submit(input), appendCommentary: next.appendCommentary, appendInstructions: next.appendInstructions, - notice: setNotice, + notice: setWarningMessage, }); bridge.current.update(latest.current.chat); session.current = next; From 4a237b40f9c9692373cc95258ddcda4aae740008 Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Tue, 15 Sep 2026 13:24:14 +0200 Subject: [PATCH 3/8] Separate Brunch response stops from Live sessions Co-authored-by: Cursor --- .../live-brunch-bridge.test.ts | 20 ++++++++++ .../app/voice-interview/live-brunch-bridge.ts | 38 ++++++++++++------- .../live-conversation-control.test.tsx | 4 +- .../live-conversation-control.tsx | 4 +- 4 files changed, 49 insertions(+), 17 deletions(-) diff --git a/apps/petrinaut-website/src/main/app/voice-interview/live-brunch-bridge.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/live-brunch-bridge.test.ts index 02ebb572981..bfc8eda9b1f 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/live-brunch-bridge.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/live-brunch-bridge.test.ts @@ -733,6 +733,26 @@ test("a locally stopped or failed turn cannot turn earlier successful prose into expect(fixture.appendCommentary).not.toHaveBeenCalled(); }); +test("repeated stopped snapshots retain a later turn", async () => { + const fixture = setup(); + await fixture.bridge.accept({ id: "one", text: "First" }); + fixture.update({ stopped: true }); + await fixture.bridge.accept({ id: "two", text: "Second" }); + fixture.update({ stopped: true }); + fixture.bridge.responseStarted(started); + fixture.update({ status: "streaming" }); + fixture.bridge.responseCompleted({ + ...started, + position: { batch: 2, index: 0 }, + }); + fixture.update({ segments: [segment()], settlements: completed }); + expect(fixture.submit).toHaveBeenCalledTimes(2); + expect(fixture.appendCommentary).toHaveBeenCalledExactlyOnceWith( + segment().text, + null, + ); +}); + test("a locally refused long commentary is offered intact once without truncation or replay", async () => { const fixture = setup(); fixture.appendCommentary.mockReturnValue(false); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/live-brunch-bridge.ts b/apps/petrinaut-website/src/main/app/voice-interview/live-brunch-bridge.ts index 19a5eb40362..9dfc1c5c7b4 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/live-brunch-bridge.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/live-brunch-bridge.ts @@ -80,6 +80,12 @@ export class LiveBrunchBridge { this.#unclaimedDelegations.clear(); } + public stopResponse(): void { + if (this.#abort.signal.aborted) return; + this.#chat = { ...this.#chat, stopped: true }; + this.#interruptTurns(); + } + public acceptDelegation(delegationId: string): void { if (this.#abort.signal.aborted) return; const turn = [...this.#turns].findLast( @@ -253,20 +259,10 @@ export class LiveBrunchBridge { public update(chat: Chat): void { if (this.#abort.signal.aborted) return; + const stopped = chat.stopped === true && this.#chat.stopped !== true; this.#chat = chat; - if (chat.stopped || chat.status === "error") { - for (const turn of this.#turns) { - logLiveDiagnostic("brunch.interrupted", { - inputId: turn.inputId, - submissionId: turn.submissionId, - stopped: chat.stopped === true, - status: chat.status, - }); - } - if (!chat.stopped) { - for (const turn of this.#turns) this.#unserved(turn.delegationId); - } - this.#turns.clear(); + if (stopped || chat.status === "error") { + this.#interruptTurns(); return; } if (chat.status === "submitted" || chat.status === "streaming") { @@ -275,6 +271,22 @@ export class LiveBrunchBridge { this.#settle(); } + #interruptTurns(): void { + for (const turn of this.#turns) { + logLiveDiagnostic("brunch.interrupted", { + inputId: turn.inputId, + submissionId: turn.submissionId, + stopped: this.#chat.stopped === true, + status: this.#chat.status, + }); + } + if (!this.#chat.stopped) { + for (const turn of this.#turns) this.#unserved(turn.delegationId); + } + this.#turns.clear(); + this.#waitingForComposer = undefined; + } + #settle(): void { if ( this.#abort.signal.aborted || diff --git a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.test.tsx b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.test.tsx index 30329d1710a..3b975b69888 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.test.tsx +++ b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.test.tsx @@ -495,9 +495,9 @@ test("final transcription enters the real admission helper and only its settled "delegation-1", ); act(() => tracker.recordStopRequested()); - expect(session.stop).toHaveBeenCalled(); + expect(session.stop).not.toHaveBeenCalled(); await act(async () => call[2]({ id: "late", text: "Late transcription" })); - expect(props.submitVoiceInput).toHaveBeenCalledOnce(); + expect(props.submitVoiceInput).toHaveBeenCalledTimes(2); }); test.each(["answer", "folded-answer"])( diff --git a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.tsx b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.tsx index 03466b5c96f..0bb33d632af 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.tsx +++ b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.tsx @@ -133,9 +133,9 @@ export const LiveConversationControl = ({ useEffect( () => subscribeToStopRequested?.(() => { - void end(); + bridge.current?.stopResponse(); }), - [end, subscribeToStopRequested], + [subscribeToStopRequested], ); useEffect(() => { From f6ab04f31bc02124e2eb795a4f30c7805bb73afc Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Tue, 15 Sep 2026 13:44:35 +0200 Subject: [PATCH 4/8] Resolve Live turn correlation edge cases Co-authored-by: Cursor --- .../live-brunch-bridge.test.ts | 41 +++++++++++++++++++ .../app/voice-interview/live-brunch-bridge.ts | 33 ++++++++++----- 2 files changed, 64 insertions(+), 10 deletions(-) diff --git a/apps/petrinaut-website/src/main/app/voice-interview/live-brunch-bridge.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/live-brunch-bridge.test.ts index bfc8eda9b1f..efd15152d90 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/live-brunch-bridge.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/live-brunch-bridge.test.ts @@ -165,6 +165,21 @@ test("offers one frozen correlated answer only after complete settlement, never ); }); +test("offers a completed correlated answer across a ready-to-ready transition", async () => { + const fixture = setup(); + await fixture.bridge.accept({ id: "one", text: "Seven, not four" }); + fixture.bridge.responseStarted(started); + fixture.bridge.responseCompleted({ + ...started, + position: { batch: 2, index: 0 }, + }); + fixture.update({ segments: [segment()], settlements: completed }); + expect(fixture.appendCommentary).toHaveBeenCalledExactlyOnceWith( + segment().text, + null, + ); +}); + test("waits for observed prose rendered after ready settlement", async () => { const fixture = setup(); await fixture.bridge.accept({ id: "one", text: "Seven, not four" }); @@ -372,6 +387,28 @@ test("duplicates, empty input and one waiting composer submission never create a expect(fixture.submit).toHaveBeenCalledTimes(2); }); +test("empty finalized input resolves its delegation before a later turn", async () => { + const fixture = setup(); + fixture.bridge.acceptDelegation("empty-delegation"); + await fixture.bridge.accept({ id: "empty", text: " " }); + await fixture.bridge.accept({ id: "next", text: "Continue" }); + fixture.update({ status: "streaming" }); + fixture.bridge.responseStarted(started); + fixture.bridge.responseCompleted({ + ...started, + position: { batch: 2, index: 0 }, + }); + fixture.update({ segments: [segment()], settlements: completed }); + expect(fixture.appendInstructions).toHaveBeenCalledExactlyOnceWith( + expect.stringContaining("No usable speech"), + "empty-delegation", + ); + expect(fixture.appendCommentary).toHaveBeenCalledExactlyOnceWith( + segment().text, + null, + ); +}); + test("uncertain admission is visible and never automatically replayed", async () => { const fixture = setup(); fixture.submit.mockRejectedValueOnce(new Error("Unknown admission")); @@ -533,6 +570,10 @@ test("two inputs answered by one submission offer its canonical prose only once" segment().text, "first-delegation", ); + expect(fixture.appendInstructions).toHaveBeenCalledExactlyOnceWith( + expect.stringContaining("already delivered"), + "second-delegation", + ); expect(fixture.notice).not.toHaveBeenCalledWith( expect.stringContaining("without a spoken answer"), ); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/live-brunch-bridge.ts b/apps/petrinaut-website/src/main/app/voice-interview/live-brunch-bridge.ts index 9dfc1c5c7b4..61e08d433e8 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/live-brunch-bridge.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/live-brunch-bridge.ts @@ -28,7 +28,6 @@ interface Turn { readonly baselineMessages: ReadonlySet; delegationId: string | null; submissionId?: string; - sawBusy: boolean; } type Submit = ConstructorParameters< @@ -121,15 +120,22 @@ export class LiveBrunchBridge { return; } this.#seenInputs.add(input.id); + const delegationId = [...this.#unclaimedDelegations].at(-1) ?? null; + if (delegationId !== null) this.#unclaimedDelegations.delete(delegationId); if (!input.text.trim()) { logLiveDiagnostic("input.ignored", { inputId: input.id, + delegationId, reason: "empty", }); + if (delegationId !== null) { + this.#dependencies.appendInstructions( + "No usable speech was captured for this turn. Ask the person to continue without assuming an answer.", + delegationId, + ); + } return; } - const delegationId = [...this.#unclaimedDelegations].at(-1) ?? null; - if (delegationId !== null) this.#unclaimedDelegations.delete(delegationId); if ( input.text.length > 32_000 || this.#waitingForComposer || @@ -157,8 +163,6 @@ export class LiveBrunchBridge { ...this.#chat.segments.map((segment) => segment.messageId), ...(this.#chat.snapshot?.messages.map((message) => message.id) ?? []), ]), - sawBusy: - this.#chat.status === "submitted" || this.#chat.status === "streaming", }; this.#waitingForComposer = turn; this.#turns.add(turn); @@ -265,9 +269,6 @@ export class LiveBrunchBridge { this.#interruptTurns(); return; } - if (chat.status === "submitted" || chat.status === "streaming") { - for (const turn of this.#turns) turn.sawBusy = true; - } this.#settle(); } @@ -346,7 +347,6 @@ export class LiveBrunchBridge { this.#unserved(turn.delegationId); continue; } - if (!turn.sawBusy) continue; if (settlements.some((settlement) => !settlement)) continue; if ( [...messages].some((id) => @@ -517,7 +517,20 @@ export class LiveBrunchBridge { const segments = sourceSegments.filter( (segment) => !this.#offeredSegments.has(segment.id), ); - if (!segments.length) continue; + if (!segments.length) { + logLiveDiagnostic("brunch.already-offered", { + inputId: turn.inputId, + submissionId: turn.submissionId, + delegationId: turn.delegationId, + }); + if (turn.delegationId !== null) { + this.#dependencies.appendInstructions( + "This answer was already delivered through another live turn. Continue the interview without repeating it.", + turn.delegationId, + ); + } + continue; + } for (const segment of segments) this.#offeredSegments.add(segment.id); // Freeze complete prose once. Sending is neither exact relay nor playback proof. const source = segments.map((segment) => segment.text).join("\n\n"); From 3657d8d69cbb20353bee3edaff34641d56ddcf85 Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Tue, 15 Sep 2026 14:03:17 +0200 Subject: [PATCH 5/8] Handle absent Brunch history in assistant tests Co-authored-by: Cursor --- .../main/app/local-storage-demo/local-storage-demo-app.test.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx index 07dbcbb31f0..e2ec3ee7939 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx @@ -2079,7 +2079,7 @@ describe("assistant selection", () => { const brunch = currentAssistant(); // Brunch reads Flue history, which holds none of the stock turn; and a // Brunch-side message write never reaches the local store. - expect(brunch.messages).not.toContainEqual(stockMessage); + expect(brunch.messages ?? []).not.toContainEqual(stockMessage); act(() => brunch.onMessages?.([ { id: "brunch-1", role: "user", parts: [] } as PetrinautAiMessage, From 25b4e96002a2f873eb3ee1b3298e54ae3958e6cd Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Tue, 15 Sep 2026 14:06:48 +0200 Subject: [PATCH 6/8] Preserve Live autoplay coverage after restacking Co-authored-by: Cursor --- .../src/main/app/voice-interview/live-conversation.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation.test.ts index 2475ef39793..c32048784ee 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation.test.ts @@ -40,6 +40,7 @@ const setup = () => { }); vi.stubGlobal("RTCPeerConnection", RTCPeerConnectionMock); const audio = { + autoplay: false, srcObject: null, muted: false, paused: false, From 45e4c95313e084853d8f69a012e5153ba4b75793 Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Tue, 15 Sep 2026 16:08:09 +0200 Subject: [PATCH 7/8] Resolve Live delegations when stopping Brunch Co-authored-by: Cursor --- .../live-brunch-bridge.test.ts | 64 +++++++++++++++++++ .../app/voice-interview/live-brunch-bridge.ts | 7 +- 2 files changed, 68 insertions(+), 3 deletions(-) diff --git a/apps/petrinaut-website/src/main/app/voice-interview/live-brunch-bridge.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/live-brunch-bridge.test.ts index efd15152d90..149fb6024cf 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/live-brunch-bridge.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/live-brunch-bridge.test.ts @@ -756,6 +756,70 @@ test("Stop withdraws only unsubmitted input and suppresses late settlements and expect(fixture.submit).toHaveBeenCalledOnce(); }); +test("response-only Stop resolves attached and unclaimed delegations before a later utterance", async () => { + const fixture = setup(); + fixture.bridge.acceptDelegation("attached"); + await fixture.bridge.accept({ id: "one", text: "First" }); + fixture.bridge.acceptDelegation("unclaimed"); + + fixture.bridge.stopResponse(); + + expect(fixture.appendInstructions.mock.calls).toEqual([ + [ + expect.stringContaining( + "Ask the person to continue. Do not claim the work completed", + ), + "attached", + ], + [ + expect.stringContaining( + "Ask the person to continue. Do not claim the work completed", + ), + "unclaimed", + ], + ]); + + fixture.submit.mockImplementationOnce(async (input) => { + input.onAdmission("second"); + return { + kind: "message", + messageId: "second-input", + submissionId: "second", + }; + }); + await fixture.bridge.accept({ id: "two", text: "Continue" }); + const secondResponse = { + ...started, + messageId: "second-answer", + submissionId: "second", + }; + fixture.bridge.responseStarted(secondResponse); + fixture.bridge.update({ + status: "streaming", + canAcceptVoiceInput: true, + segments: [], + settlements: [], + }); + fixture.bridge.responseCompleted({ + ...secondResponse, + position: { batch: 2, index: 0 }, + }); + fixture.update({ + segments: [ + { + ...segment("Continue with the next question."), + messageId: "second-answer", + submissionIds: ["second"], + }, + ], + settlements: [{ submissionId: "second", outcome: "completed" }], + }); + expect(fixture.appendCommentary).toHaveBeenCalledExactlyOnceWith( + "Continue with the next question.", + null, + ); +}); + test("a locally stopped or failed turn cannot turn earlier successful prose into success speech", async () => { const fixture = setup(); await fixture.bridge.accept({ id: "one", text: "First" }); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/live-brunch-bridge.ts b/apps/petrinaut-website/src/main/app/voice-interview/live-brunch-bridge.ts index 61e08d433e8..b2ca277f086 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/live-brunch-bridge.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/live-brunch-bridge.ts @@ -280,11 +280,12 @@ export class LiveBrunchBridge { stopped: this.#chat.stopped === true, status: this.#chat.status, }); + this.#unserved(turn.delegationId); } - if (!this.#chat.stopped) { - for (const turn of this.#turns) this.#unserved(turn.delegationId); - } + for (const delegationId of this.#unclaimedDelegations) + this.#unserved(delegationId); this.#turns.clear(); + this.#unclaimedDelegations.clear(); this.#waitingForComposer = undefined; } From d9b6d6acf87f4d41822a62f0501921a232200821 Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Tue, 15 Sep 2026 17:13:34 +0200 Subject: [PATCH 8/8] Align restacked Live playback coverage Co-authored-by: Cursor --- .../app/voice-interview/live-conversation-control.test.tsx | 1 + .../src/main/app/voice-interview/live-conversation.test.ts | 5 +---- .../src/main/app/voice-interview/live-conversation.ts | 6 ++---- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.test.tsx b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.test.tsx index 3b975b69888..d6b09aa88ac 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.test.tsx +++ b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation-control.test.tsx @@ -265,6 +265,7 @@ test("reuses setup and reports failure to the host dock and notification surface microphoneMuted: false, errorMessage: null, notice: playbackNotice, + warningMessage: null, }); if (!props.registerVoiceModeSessionControls) throw new Error("Session control registration was not provided"); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation.test.ts index c32048784ee..c3d0950775c 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation.test.ts @@ -787,10 +787,7 @@ test.each(["rejects", "throws"] as const)( const fixture = setup(); fixture.peers[0]!.getStats.mockResolvedValue( new Map([ - [ - "input", - { type: "media-source", kind: "audio", audioLevel: 0.42 }, - ], + ["input", { type: "media-source", kind: "audio", audioLevel: 0.42 }], ]), ); if (failureMode === "rejects") { diff --git a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation.ts b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation.ts index d821cd22f05..acc8c77f2a5 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/live-conversation.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/live-conversation.ts @@ -221,8 +221,7 @@ export const createLiveConversation = ( } else if (state === "connected" && recoveryTimers.has(kind)) { clearTimeout(recoveryTimers.get(kind)); recoveryTimers.delete(kind); - if (recoveryTimers.size === 0) - onState(activeState("connected")); + if (recoveryTimers.size === 0) onState(activeState("connected")); } }; @@ -311,8 +310,7 @@ export const createLiveConversation = ( // A peer may have disconnected before the last session-ready event. peers.forEach((_, connectionKind) => handleConnectionState(connectionKind)); if (stopping) return; - if (recoveryTimers.size === 0) - onState(activeState("connected")); + if (recoveryTimers.size === 0) onState(activeState("connected")); activityTimer = setTimeout(() => void sampleActivity(), 100); flushFinalizedInputs(); };