diff --git a/apps/petrinaut-website/README.md b/apps/petrinaut-website/README.md index b66405ed9cf..319df516d0a 100644 --- a/apps/petrinaut-website/README.md +++ b/apps/petrinaut-website/README.md @@ -103,6 +103,25 @@ The Brunch deployment must allow the website origin through its microphone permission. Denying permission leaves the existing text composer available and does not submit anything to Brunch. +When the preview cannot continue, the status panel distinguishes microphone +permission, microphone device, interrupted request, network, timeout, invalid +response, and unavailable/disabled failures. Permission and device failures +identify what to fix; network, timeout, and interrupted requests offer a +reconnect; invalid responses include a diagnostic reference for an operator; +and unavailable voice leaves the text composer as the fallback. Speech failures +always leave the canonical response visible to read. + +Realtime connection and Speech requests carry a random `x-request-id` through +the browser and server route, and the existing Brunch transport sends the same +header on each chat request so Brunch's privacy-safe request inspection can +correlate that boundary. Browser and server diagnostics report operation, +stage, outcome, duration, request ID, and—where applicable—status or sanitized +error code. Voice responses also expose privacy-safe `Server-Timing` metrics. +These diagnostics never record audio, SDP, transcript or prompt contents, +canonical speech text, credentials, or provider response bodies. This +controlled-preview evidence does not enable production: production remains +unconditionally disabled by the server policy. + ## Testing the API against the built output A plain `yarn build && yarn vite preview` only serves the static `dist/` assets - `/api/chat` will 404 because the dev plugin is not loaded by `vite preview`. Use one of the options below to exercise the production code path locally. diff --git a/apps/petrinaut-website/api/voice/realtime-call.ts b/apps/petrinaut-website/api/voice/realtime-call.ts index b9b1b1f78ee..dd6dee68187 100644 --- a/apps/petrinaut-website/api/voice/realtime-call.ts +++ b/apps/petrinaut-website/api/voice/realtime-call.ts @@ -1,4 +1,5 @@ import { createOpenAIRealtimeCallHandler } from "../../src/server/voice/openai-realtime-call"; +import { reportVoiceDiagnostic } from "../../src/voice-diagnostics"; declare const process: { env: Record; @@ -8,5 +9,6 @@ export default { fetch: createOpenAIRealtimeCallHandler({ environment: process.env, fetch: globalThis.fetch.bind(globalThis), + reportDiagnostic: reportVoiceDiagnostic, }), }; diff --git a/apps/petrinaut-website/api/voice/speech.ts b/apps/petrinaut-website/api/voice/speech.ts index a02b0f1b8b2..62e4a580e18 100644 --- a/apps/petrinaut-website/api/voice/speech.ts +++ b/apps/petrinaut-website/api/voice/speech.ts @@ -1,4 +1,5 @@ import { createOpenAISpeechHandler } from "../../src/server/voice/openai-speech"; +import { reportVoiceDiagnostic } from "../../src/voice-diagnostics"; declare const process: { env: Record; @@ -8,5 +9,6 @@ export default { fetch: createOpenAISpeechHandler({ environment: process.env, fetch: globalThis.fetch.bind(globalThis), + reportDiagnostic: reportVoiceDiagnostic, }), }; diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx index 619709b1d04..c14f42df3de 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx @@ -7,8 +7,20 @@ import { describe, expect, test, vi } from "vitest"; import { VoiceInterviewControl } from "../voice-interview/voice-interview-control"; import { getBrunchVoiceComposerControl } from "./local-storage-demo-app"; +const defaultTransportOptions = vi.hoisted(() => ({ + current: null as unknown, +})); + +vi.mock("./brunch-principal", () => ({ + getOrCreateBrunchPrincipal: () => "test-principal", +})); + vi.mock("@hashintel/petrinaut/ui", () => ({ - DefaultChatTransport: class {}, + DefaultChatTransport: class { + public constructor(options: unknown) { + defaultTransportOptions.current = options; + } + }, Petrinaut: () => null, WalkthroughProvider: ({ children }: { children: ReactNode }) => children, definePetrinautAiInteractiveTool: (definition: unknown) => definition, @@ -42,4 +54,14 @@ describe("local storage demo Brunch voice integration", () => { } expect(control.type).toBe(VoiceInterviewControl); }); + + test("correlates the existing Brunch transport request", () => { + const options = defaultTransportOptions.current as { + readonly headers: () => Record; + }; + + expect(options.headers()["x-request-id"]).toMatch( + /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/u, + ); + }); }); diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx index 47715043984..5dfd53abc82 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx @@ -23,6 +23,7 @@ import { WalkthroughProvider, } from "@hashintel/petrinaut/ui"; +import { VOICE_REQUEST_ID_HEADER } from "../../../voice-diagnostics"; import { useSentryFeedbackAction } from "../sentry-feedback-button"; import { VoiceInterviewControl } from "../voice-interview/voice-interview-control"; import { brunchAskInteractiveTool } from "./brunch-ask-interactive-tool"; @@ -116,6 +117,7 @@ const stockChatTransport = new DefaultChatTransport({ api: brunchPreviewConfig.chatEndpoint, headers: () => ({ [BRUNCH_PRINCIPAL_HEADER]: brunchPrincipal, + [VOICE_REQUEST_ID_HEADER]: crypto.randomUUID(), }), }); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts index 322d3beaccb..c1b249b1cd0 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts @@ -26,7 +26,8 @@ class FakeDataChannel extends EventTarget { } } -const createHarness = () => { +const createHarness = (connectionTimeoutMs = 15_000) => { + let requestNumber = 0; const channels: FakeDataChannel[] = []; const peers: Array<{ addTrack: ReturnType; @@ -54,6 +55,7 @@ const createHarness = () => { getTracks: () => [track], } as unknown as MediaStream; }); + const reportDiagnostic = vi.fn(); const createPeerConnection = () => { const channel = new FakeDataChannel(); channels.push(channel); @@ -76,15 +78,27 @@ const createHarness = () => { return peer as unknown as RTCPeerConnection; }; const session = new OpenAIRealtimeSession({ - connectionTimeoutMs: 15_000, + connectionTimeoutMs, + createRequestId: () => `voice-request-${++requestNumber}`, createPeerConnection, fetch, getUserMedia, + now: () => 100, + reportDiagnostic, }); const events: OpenAIRealtimeSessionEvent[] = []; session.subscribe((event) => events.push(event)); - return { channels, events, fetch, getUserMedia, peers, session, tracks }; + return { + channels, + events, + fetch, + getUserMedia, + peers, + reportDiagnostic, + session, + tracks, + }; }; describe("OpenAIRealtimeSession", () => { @@ -109,7 +123,10 @@ describe("OpenAIRealtimeSession", () => { "/api/voice/realtime-call", expect.objectContaining({ body: "v=0\r\no=browser offer", - headers: { "content-type": "application/sdp" }, + headers: { + "content-type": "application/sdp", + "x-request-id": "voice-request-1", + }, method: "POST", }), ); @@ -123,6 +140,13 @@ describe("OpenAIRealtimeSession", () => { harness.session.setMicrophoneEnabled(true); expect(harness.tracks[0]!.enabled).toBe(true); + expect(harness.reportDiagnostic).toHaveBeenCalledWith({ + durationMs: 0, + operation: "connection", + outcome: "success", + requestId: "voice-request-1", + stage: "browser", + }); }); test("emits only strict input transcription events with stable source identity", async () => { @@ -164,6 +188,13 @@ describe("OpenAIRealtimeSession", () => { }, ]); expect(harness.tracks[0]!.enabled).toBe(false); + expect(harness.reportDiagnostic).toHaveBeenLastCalledWith({ + durationMs: 0, + operation: "transcription", + outcome: "success", + requestId: "voice-request-2", + stage: "browser", + }); }); test("surfaces failed input transcription as a recoverable error", async () => { @@ -179,7 +210,10 @@ describe("OpenAIRealtimeSession", () => { expect(harness.events).toEqual([ { - message: "Voice transcription failed. Try reconnecting.", + code: "invalid-response", + message: + "The transcription service returned an invalid response. Try again; if it continues, give the diagnostic reference to an operator.", + requestId: "voice-request-1", type: "error", }, ]); @@ -189,6 +223,14 @@ describe("OpenAIRealtimeSession", () => { expect(harness.tracks[0]!.stop).toHaveBeenCalledOnce(); expect(harness.channels[0]!.close).toHaveBeenCalledOnce(); expect(harness.peers[0]!.close).toHaveBeenCalledOnce(); + expect(harness.reportDiagnostic).toHaveBeenLastCalledWith({ + durationMs: 0, + errorCode: "invalid-response", + operation: "transcription", + outcome: "failure", + requestId: "voice-request-1", + stage: "browser", + }); }); test("surfaces OpenAI data-channel errors without exposing diagnostics", async () => { @@ -202,7 +244,10 @@ describe("OpenAIRealtimeSession", () => { expect(harness.events).toEqual([ { - message: "The voice service reported an error. Try reconnecting.", + code: "invalid-response", + message: + "The transcription service returned an invalid response. Try again; if it continues, give the diagnostic reference to an operator.", + requestId: "voice-request-1", type: "error", }, ]); @@ -261,7 +306,364 @@ describe("OpenAIRealtimeSession", () => { ); await expect(harness.session.connect()).rejects.toThrow( - "Microphone access is required to start voice input.", + "Allow microphone access in your browser settings, then reconnect voice input.", + ); + expect(harness.reportDiagnostic).toHaveBeenCalledWith({ + durationMs: 0, + errorCode: "microphone-permission", + operation: "connection", + outcome: "failure", + requestId: "voice-request-1", + stage: "browser", + }); + }); + + test("distinguishes microphone device failures from permission denial", async () => { + const harness = createHarness(); + harness.getUserMedia.mockRejectedValueOnce( + new DOMException("private device detail", "NotFoundError"), + ); + + await expect(harness.session.connect()).rejects.toMatchObject({ + code: "microphone-device", + message: + "No usable microphone was found. Connect or select one, then reconnect voice input.", + requestId: "voice-request-1", + }); + expect(JSON.stringify(harness.reportDiagnostic.mock.calls)).not.toContain( + "private device detail", + ); + }); + + test("classifies browser network failures without exposing thrown details", async () => { + const harness = createHarness(); + harness.fetch.mockRejectedValueOnce( + new Error("private SDP and credential diagnostics"), + ); + + await expect(harness.session.connect()).rejects.toMatchObject({ + code: "network", + message: + "The voice connection could not be reached. Check your connection, then reconnect voice input.", + requestId: "voice-request-1", + }); + expect(harness.tracks[0]!.stop).toHaveBeenCalledOnce(); + expect(JSON.stringify(harness.reportDiagnostic.mock.calls)).not.toContain( + "private SDP and credential diagnostics", + ); + }); + + test("classifies a data channel that closes during startup as a network failure", async () => { + const harness = createHarness(); + let resolveFetch: ((response: Response) => void) | undefined; + harness.fetch.mockImplementationOnce( + () => + new Promise((resolve) => { + resolveFetch = resolve; + }), + ); + const connection = harness.session.connect(); + await vi.waitFor(() => expect(harness.fetch).toHaveBeenCalledOnce()); + harness.peers[0]!.setRemoteDescription.mockResolvedValueOnce(undefined); + resolveFetch?.( + new Response("v=0\r\no=OpenAI answer", { + headers: { "content-type": "application/sdp" }, + }), + ); + await vi.waitFor(() => + expect(harness.peers[0]?.setRemoteDescription).toHaveBeenCalledOnce(), + ); + await Promise.resolve(); + + harness.channels[0]!.dispatchEvent(new Event("close")); + + await expect(connection).rejects.toMatchObject({ + code: "network", + requestId: "voice-request-1", + }); + expect(harness.tracks[0]!.stop).toHaveBeenCalledOnce(); + expect(harness.channels[0]!.close).toHaveBeenCalledOnce(); + expect(harness.peers[0]!.close).toHaveBeenCalledOnce(); + }); + + test("rejects a provider error received before startup completes", async () => { + const harness = createHarness(); + let resolveFetch: ((response: Response) => void) | undefined; + harness.fetch.mockImplementationOnce( + () => + new Promise((resolve) => { + resolveFetch = resolve; + }), + ); + const connection = expect(harness.session.connect()).rejects.toMatchObject({ + code: "invalid-response", + requestId: "voice-request-1", + }); + await vi.waitFor(() => expect(harness.fetch).toHaveBeenCalledOnce()); + harness.peers[0]!.setRemoteDescription.mockImplementationOnce(async () => { + harness.channels[0]!.open(); + harness.channels[0]!.receive({ + type: "error", + error: { message: "private provider diagnostic" }, + }); + }); + resolveFetch?.( + new Response("v=0\r\no=OpenAI answer", { + headers: { "content-type": "application/sdp" }, + }), + ); + + await connection; + expect(harness.tracks[0]!.stop).toHaveBeenCalledOnce(); + expect(JSON.stringify(harness.reportDiagnostic.mock.calls)).not.toContain( + "private provider diagnostic", + ); + }); + + test("preserves a peer failure while the realtime call is pending", async () => { + const harness = createHarness(); + harness.fetch.mockImplementationOnce( + (_input, init) => + new Promise((_resolve, reject) => { + init?.signal?.addEventListener("abort", () => { + reject(new DOMException("aborted", "AbortError")); + }); + }), + ); + const connection = harness.session.connect(); + await vi.waitFor(() => expect(harness.fetch).toHaveBeenCalledOnce()); + + harness.peers[0]!.connectionState = "failed"; + harness.peers[0]!.onconnectionstatechange?.(); + + await expect(connection).rejects.toMatchObject({ + code: "network", + requestId: "voice-request-1", + }); + expect(harness.reportDiagnostic).toHaveBeenLastCalledWith( + expect.objectContaining({ + errorCode: "network", + outcome: "failure", + }), + ); + }); + + test("preserves a provider failure while waiting for the data channel", async () => { + const harness = createHarness(); + let resolveFetch: ((response: Response) => void) | undefined; + harness.fetch.mockImplementationOnce( + () => + new Promise((resolve) => { + resolveFetch = resolve; + }), + ); + const connection = harness.session.connect(); + await vi.waitFor(() => expect(harness.fetch).toHaveBeenCalledOnce()); + harness.peers[0]!.setRemoteDescription.mockResolvedValueOnce(undefined); + resolveFetch?.( + new Response("v=0\r\no=OpenAI answer", { + headers: { "content-type": "application/sdp" }, + }), + ); + await vi.waitFor(() => + expect(harness.peers[0]!.setRemoteDescription).toHaveBeenCalledOnce(), + ); + await Promise.resolve(); + + harness.channels[0]!.receive({ + error: { message: "private provider diagnostic" }, + type: "error", + }); + + await expect(connection).rejects.toMatchObject({ + code: "invalid-response", + requestId: "voice-request-1", + }); + expect(harness.reportDiagnostic).toHaveBeenLastCalledWith( + expect.objectContaining({ + errorCode: "invalid-response", + outcome: "failure", + }), + ); + }); + + test("rejects a data channel already closed after negotiation", async () => { + const harness = createHarness(1_000); + let resolveFetch: ((response: Response) => void) | undefined; + harness.fetch.mockImplementationOnce( + () => + new Promise((resolve) => { + resolveFetch = resolve; + }), + ); + const connection = expect(harness.session.connect()).rejects.toMatchObject({ + code: "network", + requestId: "voice-request-1", + }); + await vi.waitFor(() => expect(harness.fetch).toHaveBeenCalledOnce()); + harness.peers[0]!.setRemoteDescription.mockImplementationOnce(async () => { + harness.channels[0]!.close(); + }); + resolveFetch?.( + new Response("v=0\r\no=OpenAI answer", { + headers: { "content-type": "application/sdp" }, + }), + ); + + await connection; + }); + + test("sanitizes unexpected browser startup failures", async () => { + const harness = createHarness(); + let resolveFetch: ((response: Response) => void) | undefined; + harness.fetch.mockImplementationOnce( + () => + new Promise((resolve) => { + resolveFetch = resolve; + }), + ); + const connection = harness.session.connect(); + await vi.waitFor(() => expect(harness.fetch).toHaveBeenCalledOnce()); + harness.peers[0]!.setRemoteDescription.mockRejectedValueOnce( + new Error("private browser and SDP diagnostics"), + ); + resolveFetch?.( + new Response("v=0\r\no=private provider answer", { + headers: { "content-type": "application/sdp" }, + }), + ); + + await expect(connection).rejects.toMatchObject({ + code: "invalid-response", + message: + "The voice connection returned an invalid response. Try again; if it continues, give the diagnostic reference to an operator.", + requestId: "voice-request-1", + }); + expect(JSON.stringify(harness.reportDiagnostic.mock.calls)).not.toContain( + "private browser and SDP diagnostics", + ); + }); + + test("surfaces disabled voice as unavailable without reading the response body", async () => { + const harness = createHarness(); + const serverRequestId = "00000000-0000-4000-8000-000000000021"; + harness.fetch.mockResolvedValueOnce( + new Response("private provider response", { + headers: { + "x-petrinaut-voice-error": "unavailable", + "x-request-id": serverRequestId, + }, + status: 404, + }), + ); + + await expect(harness.session.connect()).rejects.toMatchObject({ + code: "unavailable", + requestId: serverRequestId, + }); + expect(JSON.stringify(harness.reportDiagnostic.mock.calls)).not.toContain( + "private provider response", + ); + }); + + test("classifies an explicit startup abort and cleans media resources", async () => { + const harness = createHarness(); + harness.fetch.mockImplementationOnce( + (_input, init) => + new Promise((_resolve, reject) => { + init?.signal?.addEventListener("abort", () => + reject(new DOMException("private abort detail", "AbortError")), + ); + }), + ); + + const connection = harness.session + .connect() + .catch((error: unknown) => error); + await vi.waitFor(() => expect(harness.fetch).toHaveBeenCalledOnce()); + await harness.session.disconnect(); + + await expect(connection).resolves.toMatchObject({ + code: "request-aborted", + requestId: "voice-request-1", + }); + expect(harness.tracks[0]!.stop).toHaveBeenCalledOnce(); + expect(harness.channels[0]!.close).toHaveBeenCalledOnce(); + expect(harness.peers[0]!.close).toHaveBeenCalledOnce(); + }); + + test("classifies disconnect while reading the SDP answer as aborted", async () => { + const harness = createHarness(); + let rejectAnswerRead: ((reason?: unknown) => void) | undefined; + harness.fetch.mockResolvedValueOnce({ + headers: new Headers({ "content-type": "application/sdp" }), + ok: true, + text: () => + new Promise((_resolve, reject) => { + rejectAnswerRead = reject; + }), + } as Response); + const connection = harness.session + .connect() + .catch((error: unknown) => error); + await vi.waitFor(() => expect(rejectAnswerRead).toBeTypeOf("function")); + + await harness.session.disconnect(); + rejectAnswerRead?.(new Error("private response read failure")); + + await expect(connection).resolves.toMatchObject({ + code: "request-aborted", + requestId: "voice-request-1", + }); + expect(harness.reportDiagnostic).toHaveBeenCalledWith( + expect.objectContaining({ + errorCode: "request-aborted", + outcome: "aborted", + }), + ); + }); + + test("classifies disconnect while applying the SDP answer as aborted", async () => { + const harness = createHarness(); + let resolveFetch: ((response: Response) => void) | undefined; + let rejectRemoteDescription: ((reason?: unknown) => void) | undefined; + harness.fetch.mockImplementationOnce( + () => + new Promise((resolve) => { + resolveFetch = resolve; + }), + ); + const connection = harness.session + .connect() + .catch((error: unknown) => error); + await vi.waitFor(() => expect(harness.fetch).toHaveBeenCalledOnce()); + const remoteDescription = new Promise((_resolve, reject) => { + rejectRemoteDescription = reject; + }); + harness.peers[0]!.setRemoteDescription.mockReturnValueOnce( + remoteDescription, + ); + resolveFetch?.( + new Response("v=0\r\no=OpenAI answer", { + headers: { "content-type": "application/sdp" }, + }), + ); + await vi.waitFor(() => + expect(harness.peers[0]!.setRemoteDescription).toHaveBeenCalledOnce(), + ); + + await harness.session.disconnect(); + rejectRemoteDescription?.(new Error("private SDP application failure")); + + await expect(connection).resolves.toMatchObject({ + code: "request-aborted", + requestId: "voice-request-1", + }); + expect(harness.reportDiagnostic).toHaveBeenCalledWith( + expect.objectContaining({ + errorCode: "request-aborted", + outcome: "aborted", + }), ); }); @@ -278,7 +680,7 @@ describe("OpenAIRealtimeSession", () => { ); const connection = expect(harness.session.connect()).rejects.toThrow( - "The voice connection timed out. Try reconnecting.", + "The voice connection timed out. Check your connection, then reconnect voice input.", ); await vi.advanceTimersByTimeAsync(15_000); @@ -286,6 +688,13 @@ describe("OpenAIRealtimeSession", () => { expect(harness.tracks[0]!.stop).toHaveBeenCalledOnce(); expect(harness.channels[0]!.close).toHaveBeenCalledOnce(); expect(harness.peers[0]!.close).toHaveBeenCalledOnce(); + expect(harness.reportDiagnostic).toHaveBeenCalledWith( + expect.objectContaining({ + errorCode: "timeout", + outcome: "failure", + requestId: "voice-request-1", + }), + ); }); test("times out when abort occurs before waiting for the data channel", async () => { @@ -293,6 +702,7 @@ describe("OpenAIRealtimeSession", () => { const harness = createHarness(); let resolveAnswer: ((answer: string) => void) | undefined; harness.fetch.mockResolvedValueOnce({ + headers: new Headers({ "content-type": "application/sdp" }), ok: true, text: () => new Promise((resolve) => { @@ -314,9 +724,12 @@ describe("OpenAIRealtimeSession", () => { resolveAnswer?.("v=0\r\no=late OpenAI answer"); await vi.waitFor(() => expect(settled).toBe(true)); - await expect(connection).resolves.toEqual( - new Error("The voice connection timed out. Try reconnecting."), - ); + await expect(connection).resolves.toMatchObject({ + code: "timeout", + message: + "The voice connection timed out. Check your connection, then reconnect voice input.", + requestId: "voice-request-1", + }); }); test("times out a stalled permission prompt and stops a late media stream", async () => { @@ -349,9 +762,12 @@ describe("OpenAIRealtimeSession", () => { const error = await connection; expect(settledAtTimeout).toBe(true); - expect(error).toEqual( - new Error("The voice connection timed out. Try reconnecting."), - ); + expect(error).toMatchObject({ + code: "timeout", + message: + "The voice connection timed out. Check your connection, then reconnect voice input.", + requestId: "voice-request-1", + }); expect(lateTrack.stop).toHaveBeenCalledOnce(); }); @@ -369,9 +785,55 @@ describe("OpenAIRealtimeSession", () => { expect(harness.peers[0]!.close).toHaveBeenCalledOnce(); expect(harness.events).toEqual([ { - message: "The voice connection failed. Try reconnecting.", + code: "network", + message: + "The voice connection could not be reached. Check your connection, then reconnect voice input.", + requestId: "voice-request-1", type: "error", }, ]); + expect(harness.reportDiagnostic).toHaveBeenLastCalledWith({ + durationMs: 0, + errorCode: "network", + operation: "connection", + outcome: "failure", + requestId: "voice-request-1", + stage: "browser", + }); + }); + + test("fails closed on a malformed completed provider transcript", async () => { + const harness = createHarness(); + await harness.session.connect(); + harness.session.setMicrophoneEnabled(true); + + harness.channels[0]!.receive({ + type: "conversation.item.input_audio_transcription.completed", + item_id: "item-a", + content_index: 0, + transcript: { private: "provider response body" }, + }); + + expect(harness.tracks[0]!.stop).toHaveBeenCalledOnce(); + expect(harness.events).toEqual([ + { + code: "invalid-response", + message: + "The transcription service returned an invalid response. Try again; if it continues, give the diagnostic reference to an operator.", + requestId: "voice-request-2", + type: "error", + }, + ]); + expect(harness.reportDiagnostic).toHaveBeenLastCalledWith({ + durationMs: 0, + errorCode: "invalid-response", + operation: "transcription", + outcome: "failure", + requestId: "voice-request-2", + stage: "browser", + }); + expect(JSON.stringify(harness.reportDiagnostic.mock.calls)).not.toContain( + "provider response body", + ); }); }); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts index 78dfd774469..b351e9a5785 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts @@ -1,3 +1,16 @@ +import { + createVoiceRequestId, + VoiceError, + VOICE_REQUEST_ID_HEADER, + voiceDiagnosticOutcome, + voiceDurationMs, + voiceErrorFromResponse, + voiceErrorMessage, + type VoiceDiagnosticReporter, + type VoiceErrorCode, + type VoiceOperation, +} from "../../../voice-diagnostics"; + export interface OpenAIRealtimeTranscriptKey { readonly connectionEpoch: number; readonly contentIndex: number; @@ -15,19 +28,32 @@ export type OpenAIRealtimeSessionEvent = readonly text: string; readonly type: "partial" | "completed"; } - | { readonly message: string; readonly type: "error" }; + | { + readonly code: VoiceErrorCode; + readonly message: string; + readonly requestId: string; + readonly type: "error"; + }; interface OpenAIRealtimeSessionDependencies { readonly connectionTimeoutMs: number; + readonly createRequestId?: () => string; readonly createPeerConnection: () => RTCPeerConnection; readonly fetch: typeof globalThis.fetch; readonly getUserMedia: ( constraints: MediaStreamConstraints, ) => Promise; + readonly now?: () => number; + readonly reportDiagnostic?: VoiceDiagnosticReporter; } type SessionListener = (event: OpenAIRealtimeSessionEvent) => void; +interface TranscriptionTiming { + readonly requestId: string; + readonly startedAt: number; +} + const asRecord = (value: unknown): Record | null => typeof value === "object" && value !== null ? (value as Record) @@ -81,6 +107,8 @@ export class OpenAIRealtimeSession { #abortController: AbortController | null = null; #activeEpoch: number | null = null; #connected = false; + #connectedAt: number | null = null; + #connectionRequestId: string | null = null; #dataChannel: RTCDataChannel | null = null; #epoch = 0; #mediaStream: MediaStream | null = null; @@ -88,6 +116,7 @@ export class OpenAIRealtimeSession { #microphoneTrack: MediaStreamTrack | null = null; #peerConnection: RTCPeerConnection | null = null; #unexpectedCloseListener: (() => void) | null = null; + readonly #transcriptionTimings = new Map(); public constructor(dependencies: OpenAIRealtimeSessionDependencies) { this.#dependencies = dependencies; @@ -101,8 +130,12 @@ export class OpenAIRealtimeSession { public async connect(): Promise { this.#releaseResources(); + const requestId = + this.#dependencies.createRequestId?.() ?? createVoiceRequestId(); + const startedAt = this.#now(); const connectionEpoch = ++this.#epoch; this.#activeEpoch = connectionEpoch; + this.#connectionRequestId = requestId; const abortController = new AbortController(); this.#abortController = abortController; const timeoutError = new DOMException( @@ -136,23 +169,43 @@ export class OpenAIRealtimeSession { abortController.signal, ); } catch (error) { - if (error instanceof DOMException && error.name === "NotAllowedError") { - throw new Error( - "Microphone access is required to start voice input.", + if ( + error instanceof DOMException && + (error.name === "NotAllowedError" || error.name === "SecurityError") + ) { + throw new VoiceError( + "connection", + "microphone-permission", + requestId, ); } - throw error; + if ( + error instanceof DOMException && + [ + "DevicesNotFoundError", + "NotFoundError", + "NotReadableError", + "OverconstrainedError", + "TrackStartError", + ].includes(error.name) + ) { + throw new VoiceError("connection", "microphone-device", requestId); + } + if (abortController.signal.aborted) { + throw error; + } + throw new VoiceError("connection", "microphone-device", requestId); } if (this.#activeEpoch !== connectionEpoch) { stopStream(mediaStream); - throw new DOMException("Connection replaced", "AbortError"); + throw new VoiceError("connection", "request-aborted", requestId); } this.#mediaStream = mediaStream; const [microphoneTrack] = mediaStream.getAudioTracks(); if (!microphoneTrack) { - throw new Error("No microphone is available for voice input."); + throw new VoiceError("connection", "microphone-device", requestId); } microphoneTrack.enabled = false; this.#microphoneTrack = microphoneTrack; @@ -165,7 +218,7 @@ export class OpenAIRealtimeSession { this.#activeEpoch === connectionEpoch && peerConnection.connectionState === "failed" ) { - this.#handleConnectionFailure(); + this.#handleConnectionFailure("network", "connection"); } }; @@ -178,60 +231,114 @@ export class OpenAIRealtimeSession { }; const unexpectedCloseListener = () => { if (this.#activeEpoch === connectionEpoch && this.#connected) { - this.#handleConnectionFailure(); + this.#handleConnectionFailure("network", "connection"); } }; this.#messageListener = messageListener; this.#unexpectedCloseListener = unexpectedCloseListener; dataChannel.addEventListener("message", messageListener); dataChannel.addEventListener("close", unexpectedCloseListener); + dataChannel.addEventListener("error", unexpectedCloseListener); const offer = await peerConnection.createOffer(); await peerConnection.setLocalDescription(offer); const offerSdp = peerConnection.localDescription?.sdp ?? offer.sdp; if (!offerSdp) { - throw new Error("The browser could not create a voice connection."); + throw new VoiceError("connection", "invalid-response", requestId); } - const response = await this.#dependencies.fetch( - "/api/voice/realtime-call", - { + let response: Response; + try { + response = await this.#dependencies.fetch("/api/voice/realtime-call", { body: offerSdp, - headers: { "content-type": "application/sdp" }, + headers: { + "content-type": "application/sdp", + [VOICE_REQUEST_ID_HEADER]: requestId, + }, method: "POST", signal: abortController.signal, - }, - ); + }); + } catch (error) { + if (abortController.signal.aborted) { + throw error; + } + throw new VoiceError("connection", "network", requestId); + } if (!response.ok) { - throw new Error("The voice connection could not be established."); + await response.body?.cancel(); + throw voiceErrorFromResponse(response, "connection", requestId); + } + const contentType = response.headers + .get("content-type") + ?.split(";", 1)[0] + ?.trim() + .toLowerCase(); + if (contentType !== "application/sdp") { + await response.body?.cancel(); + throw new VoiceError("connection", "invalid-response", requestId); + } + let answerSdp: string; + try { + answerSdp = await response.text(); + } catch (error) { + if (abortController.signal.aborted) { + throw error; + } + throw new VoiceError("connection", "network", requestId); } - const answerSdp = await response.text(); - if (!answerSdp.trim()) { - throw new Error("The voice connection returned an invalid answer."); + if (!answerSdp.trim() || !answerSdp.trimStart().startsWith("v=0")) { + throw new VoiceError("connection", "invalid-response", requestId); } - await peerConnection.setRemoteDescription({ - sdp: answerSdp, - type: "answer", - }); - await this.#waitForDataChannelOpen(dataChannel, abortController.signal); + try { + await peerConnection.setRemoteDescription({ + sdp: answerSdp, + type: "answer", + }); + } catch (error) { + if (abortController.signal.aborted) { + throw error; + } + throw new VoiceError("connection", "invalid-response", requestId); + } + await this.#waitForDataChannelOpen( + dataChannel, + abortController.signal, + requestId, + ); + if (abortController.signal.aborted) { + throw abortController.signal.reason; + } if (this.#activeEpoch !== connectionEpoch) { - throw new DOMException("Connection replaced", "AbortError"); + throw new VoiceError("connection", "request-aborted", requestId); } this.#connected = true; + this.#connectedAt = this.#now(); + this.#reportDiagnostic("connection", requestId, startedAt, undefined); return connectionEpoch; } catch (error) { + const voiceError = + abortController.signal.reason === timeoutError + ? new VoiceError("connection", "timeout", requestId) + : abortController.signal.reason instanceof VoiceError + ? abortController.signal.reason + : error instanceof VoiceError + ? error + : abortController.signal.aborted || + (error instanceof DOMException && error.name === "AbortError") + ? new VoiceError("connection", "request-aborted", requestId) + : new VoiceError("connection", "invalid-response", requestId); if (this.#activeEpoch === connectionEpoch) { this.#releaseResources(); } - if (abortController.signal.reason === timeoutError) { - throw new Error("The voice connection timed out. Try reconnecting."); - } - if (error instanceof Error && !(error instanceof DOMException)) { - throw error; - } - throw new Error("The voice connection could not be established."); + this.#reportDiagnostic( + "connection", + requestId, + startedAt, + voiceError.code, + ); + throw voiceError; } finally { globalThis.clearTimeout(timeout); if (this.#abortController === abortController) { @@ -256,13 +363,55 @@ export class OpenAIRealtimeSession { } } - #handleConnectionFailure(): void { + #handleConnectionFailure( + code: VoiceErrorCode, + operation: VoiceOperation, + ): void { + const transcriptionTiming = + operation === "transcription" + ? this.#transcriptionTimings.values().next().value + : undefined; + const requestId = + transcriptionTiming?.requestId ?? + this.#connectionRequestId ?? + this.#dependencies.createRequestId?.() ?? + createVoiceRequestId(); if (!this.#connected) { + this.#abortController?.abort(new VoiceError(operation, code, requestId)); return; } + if (operation === "transcription") { + if (this.#transcriptionTimings.size === 0) { + this.#reportDiagnostic( + operation, + requestId, + this.#connectedAt ?? this.#now(), + code, + ); + } else { + for (const timing of this.#transcriptionTimings.values()) { + this.#reportDiagnostic( + operation, + timing.requestId, + timing.startedAt, + code, + ); + } + this.#transcriptionTimings.clear(); + } + } else { + this.#reportDiagnostic( + operation, + requestId, + this.#connectedAt ?? this.#now(), + code, + ); + } this.#releaseResources(); this.#emit({ - message: "The voice connection failed. Try reconnecting.", + code, + message: voiceErrorMessage(operation, code), + requestId, type: "error", }); } @@ -273,10 +422,19 @@ export class OpenAIRealtimeSession { return; } + if ( + parsed.type === "error" || + parsed.type === "conversation.item.input_audio_transcription.failed" + ) { + this.#handleConnectionFailure("invalid-response", "transcription"); + return; + } + if (parsed.type === "input_audio_buffer.committed") { if (typeof parsed.item_id !== "string" || !parsed.item_id) { return; } + this.#startTranscription(parsed.item_id); this.setMicrophoneEnabled(false); this.#emit({ connectionEpoch, @@ -286,24 +444,6 @@ export class OpenAIRealtimeSession { return; } - if (parsed.type === "error") { - this.#releaseResources(); - this.#emit({ - message: "The voice service reported an error. Try reconnecting.", - type: "error", - }); - return; - } - - if (parsed.type === "conversation.item.input_audio_transcription.failed") { - this.#releaseResources(); - this.#emit({ - message: "Voice transcription failed. Try reconnecting.", - type: "error", - }); - return; - } - const transcriptEventType = parsed.type === "conversation.item.input_audio_transcription.delta" ? "partial" @@ -311,23 +451,33 @@ export class OpenAIRealtimeSession { "conversation.item.input_audio_transcription.completed" ? "completed" : null; + if (transcriptEventType === null) { + return; + } if ( - transcriptEventType === null || typeof parsed.item_id !== "string" || !parsed.item_id || !Number.isInteger(parsed.content_index) || (parsed.content_index as number) < 0 ) { + if (transcriptEventType === "completed") { + this.#handleConnectionFailure("invalid-response", "transcription"); + } return; } + this.#startTranscription(parsed.item_id); const text = transcriptEventType === "partial" ? parsed.delta : parsed.transcript; if (typeof text !== "string") { + if (transcriptEventType === "completed") { + this.#handleConnectionFailure("invalid-response", "transcription"); + } return; } if (transcriptEventType === "completed") { this.setMicrophoneEnabled(false); + this.#finishTranscription(parsed.item_id); } this.#emit({ key: { @@ -340,9 +490,59 @@ export class OpenAIRealtimeSession { }); } + #now(): number { + return this.#dependencies.now?.() ?? performance.now(); + } + + #reportDiagnostic( + operation: VoiceOperation, + requestId: string, + startedAt: number, + errorCode?: VoiceErrorCode, + ): void { + this.#dependencies.reportDiagnostic?.({ + durationMs: voiceDurationMs(startedAt, this.#now()), + ...(errorCode === undefined ? {} : { errorCode }), + operation, + outcome: voiceDiagnosticOutcome(errorCode), + requestId, + stage: "browser", + }); + } + + #startTranscription(itemId: string): void { + if (!this.#transcriptionTimings.has(itemId)) { + this.#transcriptionTimings.set(itemId, { + requestId: + this.#dependencies.createRequestId?.() ?? createVoiceRequestId(), + startedAt: this.#now(), + }); + } + } + + #finishTranscription(itemId: string): void { + const timing = this.#transcriptionTimings.get(itemId); + if (!timing) { + return; + } + this.#transcriptionTimings.delete(itemId); + this.#reportDiagnostic("transcription", timing.requestId, timing.startedAt); + } + #releaseResources(): void { + for (const timing of this.#transcriptionTimings.values()) { + this.#reportDiagnostic( + "transcription", + timing.requestId, + timing.startedAt, + "request-aborted", + ); + } + this.#transcriptionTimings.clear(); this.#activeEpoch = null; this.#connected = false; + this.#connectedAt = null; + this.#connectionRequestId = null; this.#abortController?.abort(); this.#abortController = null; @@ -354,6 +554,10 @@ export class OpenAIRealtimeSession { "close", this.#unexpectedCloseListener, ); + this.#dataChannel.removeEventListener( + "error", + this.#unexpectedCloseListener, + ); } this.#messageListener = null; this.#unexpectedCloseListener = null; @@ -378,6 +582,7 @@ export class OpenAIRealtimeSession { #waitForDataChannelOpen( dataChannel: RTCDataChannel, signal: AbortSignal, + requestId: string, ): Promise { if (signal.aborted) { return Promise.reject(signal.reason); @@ -385,24 +590,32 @@ export class OpenAIRealtimeSession { if (dataChannel.readyState === "open") { return Promise.resolve(); } + if ( + dataChannel.readyState === "closing" || + dataChannel.readyState === "closed" + ) { + return Promise.reject(new VoiceError("connection", "network", requestId)); + } return new Promise((resolve, reject) => { const handleResult = (event: Event) => { dataChannel.removeEventListener("open", handleResult); dataChannel.removeEventListener("error", handleResult); + dataChannel.removeEventListener("close", handleResult); signal.removeEventListener("abort", handleResult); if (event.type === "open") { resolve(); } else if (event.type === "abort") { - reject(new DOMException("Connection aborted", "AbortError")); + reject(signal.reason); } else { - reject(new Error("The voice event channel failed to open.")); + reject(new VoiceError("connection", "network", requestId)); } }; dataChannel.addEventListener("open", handleResult); dataChannel.addEventListener("error", handleResult); + dataChannel.addEventListener("close", handleResult); signal.addEventListener("abort", handleResult, { once: true }); }); } diff --git a/apps/petrinaut-website/src/main/app/voice-interview/speech-playback-controller.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/speech-playback-controller.test.ts index 8349c64f699..3e6158449a9 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/speech-playback-controller.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/speech-playback-controller.test.ts @@ -49,11 +49,15 @@ const createHarness = ( const audio = createAudioHarness(); const createAudio = vi.fn(() => audio.audio); const createObjectURL = vi.fn(() => "blob:canonical-speech"); + const reportDiagnostic = vi.fn(); const revokeObjectURL = vi.fn(); const controller = new SpeechPlaybackController({ createAudio, createObjectURL, + createRequestId: () => "voice-speech-request", fetch, + now: () => 100, + reportDiagnostic, revokeObjectURL, }); return { @@ -62,6 +66,7 @@ const createHarness = ( createAudio, createObjectURL, fetch, + reportDiagnostic, revokeObjectURL, }; }; @@ -81,7 +86,10 @@ describe("SpeechPlaybackController", () => { expect(request).toMatchObject({ body: JSON.stringify({ segmentId: segment.id, text: segment.text }), cache: "no-store", - headers: { "content-type": "application/json" }, + headers: { + "content-type": "application/json", + "x-request-id": "voice-speech-request", + }, method: "POST", }); expect(request?.signal).toBeInstanceOf(AbortSignal); @@ -94,6 +102,29 @@ describe("SpeechPlaybackController", () => { expect(harness.revokeObjectURL).toHaveBeenCalledWith( "blob:canonical-speech", ); + expect(harness.reportDiagnostic.mock.calls).toEqual([ + [ + { + durationMs: 0, + operation: "speech", + outcome: "success", + requestId: "voice-speech-request", + stage: "browser", + }, + ], + [ + { + durationMs: 0, + operation: "speech", + outcome: "success", + requestId: "voice-speech-request", + stage: "playback", + }, + ], + ]); + expect(JSON.stringify(harness.reportDiagnostic.mock.calls)).not.toContain( + segment.text, + ); }); test("rejects failed and non-audio speech responses without creating audio", async () => { @@ -109,24 +140,74 @@ describe("SpeechPlaybackController", () => { const harness = createHarness(fetch); await expect(harness.controller.play(segment)).rejects.toThrow( - "The response could not be spoken. Read the visible text instead.", + "The speech service returned an invalid response. Read the visible response instead.", ); await expect(harness.controller.play(segment)).rejects.toThrow( - "The response could not be spoken. Read the visible text instead.", + "The speech service returned an invalid response. Read the visible response instead.", ); await expect(harness.controller.play(segment)).rejects.toThrow( - "The response could not be spoken. Read the visible text instead.", + "The speech service returned an invalid response. Read the visible response instead.", ); expect(harness.createAudio).not.toHaveBeenCalled(); }); + test("preserves an upstream response-body abort as a request abort", async () => { + const response = new Response(new Uint8Array([1]), { + headers: { "content-type": "audio/mpeg" }, + }); + vi.spyOn(response, "blob").mockRejectedValue( + new DOMException("upstream aborted", "AbortError"), + ); + const fetch = vi.fn(async () => response); + const harness = createHarness(fetch); + + await expect(harness.controller.play(segment)).rejects.toMatchObject({ + code: "request-aborted", + requestId: "voice-speech-request", + }); + expect(harness.createAudio).not.toHaveBeenCalled(); + expect(harness.reportDiagnostic).toHaveBeenLastCalledWith( + expect.objectContaining({ + errorCode: "request-aborted", + outcome: "aborted", + requestId: "voice-speech-request", + stage: "browser", + }), + ); + }); + + test("rejects untrusted server-only diagnostics and request references", async () => { + const fetch = vi.fn( + async () => + new Response("private provider response", { + headers: { + "x-petrinaut-voice-error": "microphone-permission", + "x-request-id": "private transcript used as a request id", + }, + status: 502, + }), + ); + const harness = createHarness(fetch); + + await expect(harness.controller.play(segment)).rejects.toMatchObject({ + code: "invalid-response", + requestId: "voice-speech-request", + }); + expect(JSON.stringify(harness.reportDiagnostic.mock.calls)).not.toContain( + "private transcript used as a request id", + ); + expect(JSON.stringify(harness.reportDiagnostic.mock.calls)).not.toContain( + "private provider response", + ); + }); + test("rejects text that does not match its canonical fingerprint", async () => { const harness = createHarness(); await expect( harness.controller.play({ ...segment, text: "Tampered text" }), ).rejects.toThrow( - "The response could not be spoken. Read the visible text instead.", + "The speech service returned an invalid response. Read the visible response instead.", ); expect(harness.fetch).not.toHaveBeenCalled(); @@ -155,6 +236,14 @@ describe("SpeechPlaybackController", () => { ); await Promise.resolve(); expect(harness.createAudio).not.toHaveBeenCalled(); + expect(harness.reportDiagnostic).toHaveBeenCalledWith( + expect.objectContaining({ + errorCode: "request-aborted", + outcome: "aborted", + requestId: "voice-speech-request", + stage: "browser", + }), + ); }); test("pauses active audio, revokes its URL, and rejects stale completion on cancel", async () => { @@ -170,6 +259,13 @@ describe("SpeechPlaybackController", () => { expect(harness.revokeObjectURL).toHaveBeenCalledWith( "blob:canonical-speech", ); + expect(harness.reportDiagnostic).toHaveBeenLastCalledWith( + expect.objectContaining({ + errorCode: "request-aborted", + outcome: "aborted", + stage: "playback", + }), + ); }); test("turns audio startup and playback errors into the visible-text fallback", async () => { @@ -179,7 +275,7 @@ describe("SpeechPlaybackController", () => { ); await expect(harness.controller.play(segment)).rejects.toThrow( - "The response could not be spoken. Read the visible text instead.", + "The speech service returned an invalid response. Read the visible response instead.", ); expect(harness.revokeObjectURL).toHaveBeenCalledOnce(); @@ -190,8 +286,41 @@ describe("SpeechPlaybackController", () => { ); secondHarness.audio.emit("error"); await expect(playback).rejects.toThrow( - "The response could not be spoken. Read the visible text instead.", + "The speech service returned an invalid response. Read the visible response instead.", ); expect(secondHarness.revokeObjectURL).toHaveBeenCalledOnce(); }); + + test("revokes the object URL when the audio element cannot be created", async () => { + const harness = createHarness(); + harness.createAudio.mockImplementationOnce(() => { + throw new Error("audio construction failed"); + }); + + await expect(harness.controller.play(segment)).rejects.toMatchObject({ + code: "invalid-response", + requestId: "voice-speech-request", + }); + + expect(harness.revokeObjectURL).toHaveBeenCalledWith( + "blob:canonical-speech", + ); + }); + + test("classifies browser network failures without leaking diagnostics", async () => { + const fetch = vi.fn(async () => { + throw new Error("private browser network detail"); + }); + const harness = createHarness(fetch); + + await expect(harness.controller.play(segment)).rejects.toMatchObject({ + code: "network", + message: + "The speech service could not be reached. Read the visible response instead.", + requestId: "voice-speech-request", + }); + expect(JSON.stringify(harness.reportDiagnostic.mock.calls)).not.toContain( + "private browser network detail", + ); + }); }); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/speech-playback-controller.ts b/apps/petrinaut-website/src/main/app/voice-interview/speech-playback-controller.ts index d6cbc3936f4..5d29d0d6be9 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/speech-playback-controller.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/speech-playback-controller.ts @@ -1,11 +1,18 @@ +import { + createVoiceRequestId, + VoiceError, + VOICE_REQUEST_ID_HEADER, + voiceDiagnosticOutcome, + voiceDurationMs, + voiceErrorFromResponse, + type VoiceDiagnosticReporter, + type VoiceErrorCode, +} from "../../../voice-diagnostics"; import { hashCanonicalSpeechText, type CanonicalSpeechSegment, } from "./canonical-speech"; -const SPEECH_ERROR_MESSAGE = - "The response could not be spoken. Read the visible text instead."; - interface SpeechAudio { addEventListener(type: "ended" | "error", listener: () => void): void; pause(): void; @@ -16,7 +23,10 @@ interface SpeechAudio { interface SpeechPlaybackDependencies { readonly createAudio: (source: string) => SpeechAudio; readonly createObjectURL: (blob: Blob) => string; + readonly createRequestId?: () => string; readonly fetch: typeof globalThis.fetch; + readonly now?: () => number; + readonly reportDiagnostic?: VoiceDiagnosticReporter; readonly revokeObjectURL: (url: string) => void; } @@ -29,7 +39,8 @@ interface ActiveAudio { readonly generation: number; } -const fallbackError = (): Error => new Error(SPEECH_ERROR_MESSAGE); +const fallbackError = (requestId: string): VoiceError => + new VoiceError("speech", "invalid-response", requestId); const abortError = (): DOMException => new DOMException("Speech playback was canceled.", "AbortError"); @@ -75,44 +86,95 @@ export class SpeechPlaybackController { events: SpeechPlaybackEvents = {}, ): Promise { this.cancel(); + const requestId = + this.#dependencies.createRequestId?.() ?? createVoiceRequestId(); + const requestStartedAt = this.#now(); + let requestReported = false; + let playbackStartedAt: number | null = null; if ( segment.contentHash !== hashCanonicalSpeechText(segment.text) || !segment.id.endsWith(`:${segment.contentHash}`) ) { - throw fallbackError(); + this.#reportDiagnostic( + "browser", + requestId, + requestStartedAt, + "invalid-response", + ); + throw fallbackError(requestId); } const generation = this.#generation; const abortController = new AbortController(); this.#abortController = abortController; try { - const response = await waitForAbort( - this.#dependencies.fetch("/api/voice/speech", { - body: JSON.stringify({ segmentId: segment.id, text: segment.text }), - cache: "no-store", - headers: { "content-type": "application/json" }, - method: "POST", - signal: abortController.signal, - }), - abortController.signal, - ); + let response: Response; + try { + response = await waitForAbort( + this.#dependencies.fetch("/api/voice/speech", { + body: JSON.stringify({ + segmentId: segment.id, + text: segment.text, + }), + cache: "no-store", + headers: { + "content-type": "application/json", + [VOICE_REQUEST_ID_HEADER]: requestId, + }, + method: "POST", + signal: abortController.signal, + }), + abortController.signal, + ); + } catch (error) { + if (isAbortError(error)) { + throw error; + } + throw new VoiceError("speech", "network", requestId); + } + if (!response.ok) { + await response.body?.cancel(); + throw voiceErrorFromResponse(response, "speech", requestId); + } const contentType = response.headers .get("content-type") ?.split(";", 1)[0] ?.trim() .toLowerCase(); - if (!response.ok || contentType !== "audio/mpeg") { + if (contentType !== "audio/mpeg") { await response.body?.cancel(); - throw fallbackError(); + throw fallbackError(requestId); } - const blob = await waitForAbort(response.blob(), abortController.signal); + let blob: Blob; + try { + blob = await waitForAbort(response.blob(), abortController.signal); + } catch (error) { + if (isAbortError(error)) { + if (abortController.signal.aborted) { + throw error; + } + throw new VoiceError("speech", "request-aborted", requestId); + } + throw new VoiceError("speech", "network", requestId); + } if (generation !== this.#generation || blob.size === 0) { - throw generation === this.#generation ? fallbackError() : abortError(); + throw generation === this.#generation + ? fallbackError(requestId) + : abortError(); } + this.#reportDiagnostic("browser", requestId, requestStartedAt); + requestReported = true; + playbackStartedAt = this.#now(); const objectUrl = this.#dependencies.createObjectURL(blob); - const audio = this.#dependencies.createAudio(objectUrl); + let audio: SpeechAudio; + try { + audio = this.#dependencies.createAudio(objectUrl); + } catch { + this.#dependencies.revokeObjectURL(objectUrl); + throw fallbackError(requestId); + } await new Promise((resolve, reject) => { let settled = false; let cleanup = () => undefined; @@ -124,8 +186,13 @@ export class SpeechPlaybackController { cleanup(); finish(); }; - const handleEnded = () => settle(resolve); - const handleError = () => settle(() => reject(fallbackError())); + const handleEnded = () => + settle(() => { + this.#reportDiagnostic("playback", requestId, playbackStartedAt!); + resolve(); + }); + const handleError = () => + settle(() => reject(fallbackError(requestId))); cleanup = () => { audio.removeEventListener("ended", handleEnded); audio.removeEventListener("error", handleError); @@ -150,10 +217,30 @@ export class SpeechPlaybackController { }, handleError); }); } catch (error) { + const errorCode = isAbortError(error) + ? "request-aborted" + : error instanceof VoiceError + ? error.code + : "invalid-response"; + if (!requestReported) { + this.#reportDiagnostic( + "browser", + requestId, + requestStartedAt, + errorCode, + ); + } else if (playbackStartedAt !== null) { + this.#reportDiagnostic( + "playback", + requestId, + playbackStartedAt, + errorCode, + ); + } if (isAbortError(error)) { throw error; } - throw fallbackError(); + throw error instanceof VoiceError ? error : fallbackError(requestId); } finally { if (this.#abortController === abortController) { this.#abortController = null; @@ -161,6 +248,26 @@ export class SpeechPlaybackController { } } + #now(): number { + return this.#dependencies.now?.() ?? performance.now(); + } + + #reportDiagnostic( + stage: "browser" | "playback", + requestId: string, + startedAt: number, + errorCode?: VoiceErrorCode, + ): void { + this.#dependencies.reportDiagnostic?.({ + durationMs: voiceDurationMs(startedAt, this.#now()), + ...(errorCode === undefined ? {} : { errorCode }), + operation: "speech", + outcome: voiceDiagnosticOutcome(errorCode), + requestId, + stage, + }); + } + public cancel(): void { ++this.#generation; const reason = abortError(); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx index 21c9411095f..e3ae3e95e2c 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx @@ -54,7 +54,9 @@ describe("voice interview control", () => { onStart={vi.fn()} onSubmitCorrection={vi.fn()} snapshot={{ + errorCode: null, errorMessage: "", + errorRequestId: "", lastCommittedText: "", partialText: "", phase: "idle", @@ -77,7 +79,9 @@ describe("voice interview control", () => { onStart={vi.fn()} onSubmitCorrection={vi.fn()} snapshot={{ + errorCode: null, errorMessage: "", + errorRequestId: "", lastCommittedText: "The support lead closes it.", partialText: "The next activity", phase: "listening", @@ -106,7 +110,10 @@ describe("voice interview control", () => { onStart={vi.fn()} onSubmitCorrection={vi.fn()} snapshot={{ - errorMessage: "Microphone access is required.", + errorCode: "microphone-permission", + errorMessage: + "Allow microphone access in your browser settings, then reconnect voice input.", + errorRequestId: "voice-request-permission", lastCommittedText: "", partialText: "", phase: "recoverable-error", @@ -114,7 +121,11 @@ describe("voice interview control", () => { />, ); - expect(html).toContain("Microphone off. Microphone access is required."); + expect(html).toContain( + "Microphone off. Allow microphone access in your browser settings, then reconnect voice input.", + ); + expect(html).toContain("Error code: microphone-permission."); + expect(html).toContain("Diagnostic reference: voice-request-permission."); expect(html).toContain("Reconnect voice input"); }); @@ -166,7 +177,7 @@ describe("voice interview control", () => { expect(getUserMedia).toHaveBeenCalledOnce(); expect(container.textContent).toContain( - "Microphone access is required to start voice input.", + "Allow microphone access in your browser settings, then reconnect voice input.", ); expect( container.querySelector('button[aria-label="Reconnect voice input"]'), @@ -188,7 +199,9 @@ describe("voice interview control", () => { onStart={vi.fn()} onSubmitCorrection={vi.fn()} snapshot={{ + errorCode: null, errorMessage: "", + errorRequestId: "", lastCommittedText: "", partialText: "", phase, diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx index 7eec4491ae5..632be7dac53 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx @@ -10,6 +10,7 @@ import { FaMicrophone, FaMicrophoneSlash } from "react-icons/fa6"; import { Button } from "@hashintel/ds-components"; import { css } from "@hashintel/ds-helpers/css"; +import { reportVoiceDiagnostic } from "../../../voice-diagnostics"; import { selectCanonicalSpeechSegments } from "./canonical-speech"; import { OpenAIRealtimeSession } from "./openai-realtime-session"; import { SpeechPlaybackController } from "./speech-playback-controller"; @@ -174,8 +175,17 @@ const statusText = (snapshot: VoiceTurnSnapshot): string => { return "Microphone off. Creating AI-generated speech."; case "playing": return "Microphone off. Playing AI-generated speech."; - case "recoverable-error": - return `Microphone off. ${snapshot.errorMessage}`; + case "recoverable-error": { + const diagnostic = + snapshot.errorCode === null + ? "" + : ` Error code: ${snapshot.errorCode}.${ + snapshot.errorRequestId + ? ` Diagnostic reference: ${snapshot.errorRequestId}.` + : "" + }`; + return `Microphone off. ${snapshot.errorMessage}${diagnostic}`; + } } }; @@ -325,11 +335,13 @@ const AvailableVoiceInterviewControl = ({ fetch: globalThis.fetch.bind(globalThis), getUserMedia: (constraints) => navigator.mediaDevices.getUserMedia(constraints), + reportDiagnostic: reportVoiceDiagnostic, }); const playback = new SpeechPlaybackController({ createAudio: (source) => new Audio(source), createObjectURL: (blob) => URL.createObjectURL(blob), fetch: globalThis.fetch.bind(globalThis), + reportDiagnostic: reportVoiceDiagnostic, revokeObjectURL: (url) => URL.revokeObjectURL(url), }); const controller = new VoiceTurnController({ diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts new file mode 100644 index 00000000000..eddbfd80a52 --- /dev/null +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts @@ -0,0 +1,309 @@ +import { describe, expect, test, vi } from "vitest"; + +import { createOpenAIRealtimeCallHandler } from "../../../server/voice/openai-realtime-call"; +import { createOpenAISpeechHandler } from "../../../server/voice/openai-speech"; +import { + VOICE_REQUEST_ID_HEADER, + type VoiceDiagnosticEvent, +} from "../../../voice-diagnostics"; +import { selectCanonicalSpeechSegments } from "./canonical-speech"; +import { OpenAIRealtimeSession } from "./openai-realtime-session"; +import { SpeechPlaybackController } from "./speech-playback-controller"; +import { VoiceTurnController } from "./voice-turn-controller"; + +import type { PetrinautAiMessage } from "@hashintel/petrinaut/ui"; + +const origin = "https://petrinaut.test"; +const browserOffer = "v=0\r\na=private-browser-sdp\r\n"; +const providerAnswer = "v=0\r\na=private-provider-sdp\r\n"; +const finalizedTranscript = "Private finalized transcript."; +const canonicalSpeech = "Private canonical assistant response."; +const requestIds = [ + "00000000-0000-4000-8000-000000000011", + "00000000-0000-4000-8000-000000000012", + "00000000-0000-4000-8000-000000000013", +] as const; + +class FakeDataChannel extends EventTarget { + public readyState: RTCDataChannelState = "connecting"; + + public close(): void { + this.readyState = "closed"; + } + + public open(): void { + this.readyState = "open"; + this.dispatchEvent(new Event("open")); + } + + public receive(payload: unknown): void { + const event = new Event("message"); + Object.defineProperty(event, "data", { value: JSON.stringify(payload) }); + this.dispatchEvent(event); + } +} + +const createAudioHarness = () => { + const listeners = new Map void>>(); + const audio = { + addEventListener: vi.fn((type: string, listener: () => void) => { + const eventListeners = listeners.get(type) ?? new Set(); + eventListeners.add(listener); + listeners.set(type, eventListeners); + }), + pause: vi.fn(), + play: vi.fn(async () => undefined), + removeEventListener: vi.fn((type: string, listener: () => void) => { + listeners.get(type)?.delete(listener); + }), + }; + + return { + audio, + end: () => { + for (const listener of listeners.get("ended") ?? []) { + listener(); + } + }, + }; +}; + +describe("controlled voice preview", () => { + test("crosses the mocked browser, voice, Brunch, and canonical-speech boundaries", async () => { + const diagnostics: VoiceDiagnosticEvent[] = []; + const reportDiagnostic = (event: VoiceDiagnosticEvent) => + diagnostics.push(event); + const upstreamRealtimeFetch = vi.fn( + async () => + new Response(providerAnswer, { + headers: { "content-type": "text/plain" }, + }), + ); + const upstreamSpeechFetch = vi.fn( + async () => + new Response(new Uint8Array([1, 2, 3]), { + headers: { "content-type": "audio/mpeg" }, + }), + ); + const environment = { + OPENAI_VOICE_API_KEY: "private-provider-credential", + PETRINAUT_OPENAI_VOICE_ENABLED: "true", + VERCEL_ENV: "preview", + }; + let now = 0; + const clock = () => ++now; + const realtimeHandler = createOpenAIRealtimeCallHandler({ + environment, + fetch: upstreamRealtimeFetch, + now: clock, + reportDiagnostic, + }); + const speechHandler = createOpenAISpeechHandler({ + environment, + fetch: upstreamSpeechFetch, + now: clock, + reportDiagnostic, + }); + const browserRequests: Array<{ + readonly path: string; + readonly requestId: string | null; + readonly responseRequestId: string | null; + readonly serverTiming: string | null; + }> = []; + const browserFetch = vi.fn(async (input, init) => { + const url = new URL( + input instanceof Request + ? input.url + : input instanceof URL + ? input.href + : input, + origin, + ); + const headers = new Headers(init?.headers); + headers.set("origin", origin); + const request = new Request(url, { ...init, headers }); + const response = + url.pathname === "/api/voice/realtime-call" + ? await realtimeHandler(request) + : await speechHandler(request); + browserRequests.push({ + path: url.pathname, + requestId: request.headers.get(VOICE_REQUEST_ID_HEADER), + responseRequestId: response.headers.get(VOICE_REQUEST_ID_HEADER), + serverTiming: response.headers.get("server-timing"), + }); + return response; + }); + + const dataChannel = new FakeDataChannel(); + const track = { enabled: true, stop: vi.fn() }; + const mediaStream = { + getAudioTracks: () => [track], + getTracks: () => [track], + } as unknown as MediaStream; + const peer = { + addTrack: vi.fn(), + close: vi.fn(), + connectionState: "new" as RTCPeerConnectionState, + createDataChannel: vi.fn(() => dataChannel), + createOffer: vi.fn(async () => ({ + sdp: browserOffer, + type: "offer", + })), + localDescription: null as RTCSessionDescription | null, + onconnectionstatechange: null as (() => void) | null, + setLocalDescription: vi.fn( + async (description: RTCSessionDescriptionInit) => { + peer.localDescription = description as RTCSessionDescription; + }, + ), + setRemoteDescription: vi.fn(async () => dataChannel.open()), + }; + let requestNumber = 0; + const createRequestId = () => requestIds[requestNumber++]!; + const session = new OpenAIRealtimeSession({ + connectionTimeoutMs: 15_000, + createPeerConnection: () => peer as unknown as RTCPeerConnection, + createRequestId, + fetch: browserFetch, + getUserMedia: async () => mediaStream, + now: clock, + reportDiagnostic, + }); + const audio = createAudioHarness(); + const revokeObjectURL = vi.fn(); + const playback = new SpeechPlaybackController({ + createAudio: () => audio.audio, + createObjectURL: () => "blob:voice-integration", + createRequestId, + fetch: browserFetch, + now: clock, + reportDiagnostic, + revokeObjectURL, + }); + const submitText = vi.fn(async () => ({ + kind: "message" as const, + messageId: "voice-message", + })); + const controller = new VoiceTurnController({ + conversationId: "preview-conversation", + playback, + session, + submitText, + }); + + await controller.start(); + dataChannel.receive({ + item_id: "provider-item", + type: "input_audio_buffer.committed", + }); + dataChannel.receive({ + content_index: 0, + item_id: "provider-item", + transcript: finalizedTranscript, + type: "conversation.item.input_audio_transcription.completed", + }); + await vi.waitFor(() => expect(submitText).toHaveBeenCalledOnce()); + expect(submitText).toHaveBeenCalledWith({ + id: "voice:preview-conversation:1:provider-item:0", + text: finalizedTranscript, + }); + + controller.updateChat({ canonicalSegments: [], status: "submitted" }); + const messages = [ + { + id: "assistant-message", + parts: [{ state: "done", text: canonicalSpeech, type: "text" }], + role: "assistant", + }, + ] satisfies PetrinautAiMessage[]; + const canonicalSegments = selectCanonicalSpeechSegments(messages); + controller.updateChat({ canonicalSegments, status: "streaming" }); + controller.updateChat({ canonicalSegments, status: "ready" }); + + await vi.waitFor(() => expect(audio.audio.play).toHaveBeenCalledOnce()); + expect(controller.getSnapshot().phase).toBe("playing"); + audio.end(); + await vi.waitFor(() => + expect(controller.getSnapshot().phase).toBe("listening"), + ); + + expect( + browserRequests.map(({ serverTiming: _, ...request }) => request), + ).toEqual([ + { + path: "/api/voice/realtime-call", + requestId: requestIds[0], + responseRequestId: requestIds[0], + }, + { + path: "/api/voice/speech", + requestId: requestIds[2], + responseRequestId: requestIds[2], + }, + ]); + expect(browserRequests[0]?.serverTiming).toMatch( + /^petrinaut_voice_connection;dur=\d+(?:\.\d+)?$/u, + ); + expect(browserRequests[1]?.serverTiming).toMatch( + /^petrinaut_voice_speech;dur=\d+(?:\.\d+)?$/u, + ); + expect(upstreamRealtimeFetch).toHaveBeenCalledOnce(); + const realtimeForm = upstreamRealtimeFetch.mock.calls[0]?.[1] + ?.body as FormData; + expect(realtimeForm.get("sdp")).toBe(browserOffer); + expect(upstreamSpeechFetch).toHaveBeenCalledOnce(); + expect( + JSON.parse(upstreamSpeechFetch.mock.calls[0]?.[1]?.body as string), + ).toMatchObject({ input: canonicalSpeech }); + expect(diagnostics).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + operation: "connection", + requestId: requestIds[0], + stage: "browser", + }), + expect.objectContaining({ + operation: "connection", + requestId: requestIds[0], + stage: "server", + }), + expect.objectContaining({ + operation: "transcription", + requestId: requestIds[1], + stage: "browser", + }), + expect.objectContaining({ + operation: "speech", + requestId: requestIds[2], + stage: "browser", + }), + expect.objectContaining({ + operation: "speech", + requestId: requestIds[2], + stage: "server", + }), + expect.objectContaining({ + operation: "speech", + requestId: requestIds[2], + stage: "playback", + }), + ]), + ); + const serializedDiagnostics = JSON.stringify(diagnostics); + for (const privateValue of [ + browserOffer, + providerAnswer, + finalizedTranscript, + canonicalSpeech, + environment.OPENAI_VOICE_API_KEY, + ]) { + expect(serializedDiagnostics).not.toContain(privateValue); + } + + await controller.end(); + expect(track.stop).toHaveBeenCalledOnce(); + expect(peer.close).toHaveBeenCalledOnce(); + expect(revokeObjectURL).toHaveBeenCalledWith("blob:voice-integration"); + }); +}); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts index 49158855858..e2718623e39 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts @@ -1,5 +1,6 @@ import { describe, expect, test, vi } from "vitest"; +import { VoiceError } from "../../../voice-diagnostics"; import { createVoiceMessageId, VoiceTurnController, @@ -655,12 +656,18 @@ describe("VoiceTurnController", () => { await harness.controller.start(); harness.emit({ - message: "The voice connection failed. Try reconnecting.", + code: "network", + message: + "The voice connection could not be reached. Check your connection, then reconnect voice input.", + requestId: "voice-request-network", type: "error", }); expect(harness.controller.getSnapshot()).toMatchObject({ - errorMessage: "The voice connection failed. Try reconnecting.", + errorCode: "network", + errorMessage: + "The voice connection could not be reached. Check your connection, then reconnect voice input.", + errorRequestId: "voice-request-network", phase: "recoverable-error", }); expect(harness.session.setMicrophoneEnabled).toHaveBeenLastCalledWith( @@ -686,7 +693,9 @@ describe("VoiceTurnController", () => { await vi.waitFor(() => expect(harness.submitText).toHaveBeenCalledOnce()); harness.emit({ + code: "network", message: "The voice connection failed. Try reconnecting.", + requestId: "voice-request-late-delivery", type: "error", }); finishDelivery?.(); @@ -903,9 +912,7 @@ describe("VoiceTurnController", () => { test("keeps the microphone closed and visible text available when speech fails", async () => { const harness = createHarness(); harness.playback.play.mockRejectedValueOnce( - new Error( - "The response could not be spoken. Read the visible text instead.", - ), + new VoiceError("speech", "network", "voice-request-speech"), ); await harness.controller.start(); const response = canonicalSegment( @@ -920,8 +927,10 @@ describe("VoiceTurnController", () => { await vi.waitFor(() => expect(harness.controller.getSnapshot()).toMatchObject({ + errorCode: "network", errorMessage: - "The response could not be spoken. Read the visible text instead.", + "The speech service could not be reached. Read the visible response instead.", + errorRequestId: "voice-request-speech", phase: "recoverable-error", }), ); @@ -931,21 +940,72 @@ describe("VoiceTurnController", () => { ); }); - test("does not overwrite a speech failure when delivery resolves later", async () => { + test("does not let late delivery completion overwrite active speech", async () => { const harness = createHarness(); let finishDelivery: (() => void) | undefined; - const delivery = new Promise<{ kind: "message" }>((resolve) => { - finishDelivery = () => resolve({ kind: "message" }); + let finishPlayback: (() => void) | undefined; + harness.submitText.mockImplementationOnce( + () => + new Promise((resolve) => { + finishDelivery = () => resolve({ kind: "message" as const }); + }), + ); + harness.playback.play.mockImplementationOnce(async (_segment, events) => { + events?.onPlaying?.(); + await new Promise((resolve) => { + finishPlayback = resolve; + }); }); - harness.submitText.mockImplementationOnce(() => delivery); + await harness.controller.start(); + harness.emit({ + key: key(1, "deferred-delivery"), + text: "A finalized answer", + type: "completed", + }); + await vi.waitFor(() => expect(harness.submitText).toHaveBeenCalledOnce()); + harness.controller.updateChat({ + canonicalSegments: [], + status: "streaming", + }); + harness.controller.updateChat({ + canonicalSegments: [ + canonicalSegment("canonical-speech:deferred:text%3A0:fnv1a32:12345678"), + ], + status: "ready", + }); + await vi.waitFor(() => + expect(harness.controller.getSnapshot().phase).toBe("playing"), + ); + + finishDelivery?.(); + await Promise.resolve(); + + expect(harness.controller.getSnapshot().phase).toBe("playing"); + expect(harness.session.setMicrophoneEnabled).toHaveBeenLastCalledWith( + false, + ); + + finishPlayback?.(); + await vi.waitFor(() => + expect(harness.controller.getSnapshot().phase).toBe("listening"), + ); + }); + + test("does not let late delivery completion clear a speech failure", async () => { + const harness = createHarness(); + let finishDelivery: (() => void) | undefined; + harness.submitText.mockImplementationOnce( + () => + new Promise((resolve) => { + finishDelivery = () => resolve({ kind: "message" as const }); + }), + ); harness.playback.play.mockRejectedValueOnce( - new Error( - "The response could not be spoken. Read the visible text instead.", - ), + new VoiceError("speech", "network", "voice-request-speech-failure"), ); await harness.controller.start(); harness.emit({ - key: key(1, "answer"), + key: key(1, "deferred-delivery"), text: "A finalized answer", type: "completed", }); @@ -956,21 +1016,26 @@ describe("VoiceTurnController", () => { }); harness.controller.updateChat({ canonicalSegments: [ - canonicalSegment("canonical-speech:failed:text%3A0:fnv1a32:12345678"), + canonicalSegment( + "canonical-speech:failed-deferred:text%3A0:fnv1a32:12345678", + ), ], status: "ready", }); await vi.waitFor(() => - expect(harness.controller.getSnapshot().phase).toBe("recoverable-error"), + expect(harness.controller.getSnapshot()).toMatchObject({ + errorCode: "network", + errorRequestId: "voice-request-speech-failure", + phase: "recoverable-error", + }), ); finishDelivery?.(); - await delivery; await Promise.resolve(); expect(harness.controller.getSnapshot()).toMatchObject({ - errorMessage: - "The response could not be spoken. Read the visible text instead.", + errorCode: "network", + errorRequestId: "voice-request-speech-failure", phase: "recoverable-error", }); expect(harness.session.setMicrophoneEnabled).toHaveBeenLastCalledWith( @@ -981,9 +1046,7 @@ describe("VoiceTurnController", () => { test("does not flush a pending transcript after speech fails", async () => { const harness = createHarness(); harness.playback.play.mockRejectedValueOnce( - new Error( - "The response could not be spoken. Read the visible text instead.", - ), + new VoiceError("speech", "network", "voice-request-pending-speech"), ); await harness.controller.start(); updateChatStatus(harness.controller, "streaming"); @@ -1011,8 +1074,8 @@ describe("VoiceTurnController", () => { expect(harness.submitText).not.toHaveBeenCalled(); expect(harness.controller.getSnapshot()).toMatchObject({ - errorMessage: - "The response could not be spoken. Read the visible text instead.", + errorCode: "network", + errorRequestId: "voice-request-pending-speech", phase: "recoverable-error", }); }); @@ -1025,9 +1088,7 @@ describe("VoiceTurnController", () => { }); harness.submitText.mockImplementationOnce(() => delivery); harness.playback.play.mockRejectedValueOnce( - new Error( - "The response could not be spoken. Read the visible text instead.", - ), + new VoiceError("speech", "network", "voice-request-in-flight-speech"), ); await harness.controller.start(); harness.emit({ @@ -1054,8 +1115,8 @@ describe("VoiceTurnController", () => { await Promise.resolve(); expect(harness.controller.getSnapshot()).toMatchObject({ - errorMessage: - "The response could not be spoken. Read the visible text instead.", + errorCode: "network", + errorRequestId: "voice-request-in-flight-speech", phase: "recoverable-error", }); expect(harness.session.setMicrophoneEnabled).toHaveBeenLastCalledWith( diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts index b7ae5af42cf..c9312833616 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts @@ -1,3 +1,5 @@ +import { VoiceError, type VoiceErrorCode } from "../../../voice-diagnostics"; + import type { CanonicalSpeechSegment } from "./canonical-speech"; import type { OpenAIRealtimeSessionEvent, @@ -16,7 +18,9 @@ export type VoiceTurnPhase = | "recoverable-error"; export interface VoiceTurnSnapshot { + readonly errorCode: VoiceErrorCode | null; readonly errorMessage: string; + readonly errorRequestId: string; readonly lastCommittedText: string; readonly partialText: string; readonly phase: VoiceTurnPhase; @@ -75,7 +79,9 @@ export const createVoiceMessageId = ( ].join(":"); const initialSnapshot: VoiceTurnSnapshot = { + errorCode: null, errorMessage: "", + errorRequestId: "", lastCommittedText: "", partialText: "", phase: "idle", @@ -159,12 +165,15 @@ export class VoiceTurnController { if (generation !== this.#generation) { return; } + const voiceError = + error instanceof VoiceError + ? error + : new VoiceError("connection", "invalid-response", ""); this.#session.setMicrophoneEnabled(false); this.#update({ - errorMessage: - error instanceof Error - ? error.message - : "Voice input could not be started. Try reconnecting.", + errorCode: voiceError.code, + errorMessage: voiceError.message, + errorRequestId: voiceError.requestId, phase: "recoverable-error", }); } @@ -359,7 +368,12 @@ export class VoiceTurnController { this.#speechQueue.length = 0; this.#playback.cancel(); this.#session.setMicrophoneEnabled(false); - this.#update({ errorMessage: event.message, phase: "recoverable-error" }); + this.#update({ + errorCode: event.code, + errorMessage: event.message, + errorRequestId: event.requestId, + phase: "recoverable-error", + }); return; } if (this.#pendingDelivery !== null) { @@ -487,20 +501,25 @@ export class VoiceTurnController { } }, }); - } catch { + } catch (error) { if ( generation !== this.#generation || this.#speechLoopGeneration !== generation ) { return; } + const voiceError = + error instanceof VoiceError + ? error + : new VoiceError("speech", "invalid-response", ""); this.#activeSpeechSegmentId = null; this.#speechLoopGeneration = null; this.#speechQueue.length = 0; this.#session.setMicrophoneEnabled(false); this.#update({ - errorMessage: - "The response could not be spoken. Read the visible text instead.", + errorCode: voiceError.code, + errorMessage: voiceError.message, + errorRequestId: voiceError.requestId, phase: "recoverable-error", }); return; @@ -570,7 +589,11 @@ export class VoiceTurnController { } #update(update: Partial): void { - this.#snapshot = { ...this.#snapshot, ...update }; + const clearedError = + update.errorMessage !== undefined && !("errorCode" in update) + ? { errorCode: null, errorRequestId: "" } + : {}; + this.#snapshot = { ...this.#snapshot, ...clearedError, ...update }; for (const listener of this.#listeners) { listener(this.#snapshot); } diff --git a/apps/petrinaut-website/src/server/voice/openai-realtime-call.test.ts b/apps/petrinaut-website/src/server/voice/openai-realtime-call.test.ts index 7c2a182ec96..29cd5eb6b1b 100644 --- a/apps/petrinaut-website/src/server/voice/openai-realtime-call.test.ts +++ b/apps/petrinaut-website/src/server/voice/openai-realtime-call.test.ts @@ -1,5 +1,9 @@ import { afterEach, describe, expect, test, vi } from "vitest"; +import { + VOICE_ERROR_CODE_HEADER, + VOICE_REQUEST_ID_HEADER, +} from "../../voice-diagnostics"; import { createOpenAIRealtimeCallHandler } from "./openai-realtime-call"; const enabledEnvironment = { @@ -7,6 +11,8 @@ const enabledEnvironment = { PETRINAUT_OPENAI_VOICE_ENABLED: "true", VERCEL_ENV: "preview", }; +const requestId = "00000000-0000-4000-8000-000000000001"; +const generatedRequestId = "00000000-0000-4000-8000-000000000002"; const createRequest = ( body = "v=0\r\no=- 0 0 IN IP4 127.0.0.1\r\n", @@ -17,6 +23,7 @@ const createRequest = ( headers: { "content-type": "application/sdp", origin: "https://petrinaut.test", + [VOICE_REQUEST_ID_HEADER]: requestId, }, method: "POST", ...overrides, @@ -80,6 +87,7 @@ describe("OpenAI Realtime call handler", () => { }); test("forwards only the SDP and server-owned transcription policy", async () => { + const reportDiagnostic = vi.fn(); const fetch = vi.fn( async () => new Response("v=0\r\no=OpenAI answer", { @@ -89,6 +97,8 @@ describe("OpenAI Realtime call handler", () => { const handler = createOpenAIRealtimeCallHandler({ environment: enabledEnvironment, fetch, + now: () => 100, + reportDiagnostic, }); const response = await handler(createRequest()); @@ -96,6 +106,10 @@ describe("OpenAI Realtime call handler", () => { expect(response.status).toBe(200); expect(response.headers.get("cache-control")).toBe("no-store"); expect(response.headers.get("content-type")).toContain("application/sdp"); + expect(response.headers.get(VOICE_REQUEST_ID_HEADER)).toBe(requestId); + expect(response.headers.get("server-timing")).toBe( + "petrinaut_voice_connection;dur=0", + ); expect(await response.text()).toBe("v=0\r\no=OpenAI answer"); expect(fetch).toHaveBeenCalledOnce(); @@ -122,6 +136,53 @@ describe("OpenAI Realtime call handler", () => { }, }); expect(session as string).not.toContain("response.create"); + expect(reportDiagnostic).toHaveBeenCalledWith({ + durationMs: 0, + operation: "connection", + outcome: "success", + requestId, + stage: "server", + status: 200, + }); + expect(JSON.stringify(reportDiagnostic.mock.calls)).not.toContain( + "browser offer", + ); + }); + + test("replaces an untrusted request reference before diagnostics", async () => { + const fetch = vi.fn( + async () => + new Response("v=0\r\no=OpenAI answer", { + headers: { "content-type": "text/plain" }, + }), + ); + const reportDiagnostic = vi.fn(); + const handler = createOpenAIRealtimeCallHandler({ + createRequestId: () => generatedRequestId, + environment: enabledEnvironment, + fetch, + reportDiagnostic, + }); + + const response = await handler( + createRequest(undefined, { + headers: { + "content-type": "application/sdp", + origin: "https://petrinaut.test", + [VOICE_REQUEST_ID_HEADER]: "private transcript as correlation", + }, + }), + ); + + expect(response.headers.get(VOICE_REQUEST_ID_HEADER)).toBe( + generatedRequestId, + ); + expect(reportDiagnostic).toHaveBeenCalledWith( + expect.objectContaining({ requestId: generatedRequestId }), + ); + expect(JSON.stringify(reportDiagnostic.mock.calls)).not.toContain( + "private transcript as correlation", + ); }); test("sanitizes upstream failures", async () => { @@ -137,8 +198,13 @@ describe("OpenAI Realtime call handler", () => { expect(response.status).toBe(502); const responseBody = await response.text(); - expect(responseBody).toBe("The voice connection could not be established."); + expect(responseBody).toBe( + "The voice connection returned an invalid response. Try again; if it continues, give the diagnostic reference to an operator.", + ); expect(responseBody).not.toContain("secret"); + expect(response.headers.get(VOICE_ERROR_CODE_HEADER)).toBe( + "invalid-response", + ); expect(response.headers.get("cache-control")).toBe("no-store"); }); @@ -158,7 +224,7 @@ describe("OpenAI Realtime call handler", () => { expect(response.status).toBe(502); expect(await response.text()).toBe( - "The voice connection could not be established.", + "The voice connection returned an invalid response. Try again; if it continues, give the diagnostic reference to an operator.", ); }); @@ -175,9 +241,12 @@ describe("OpenAI Realtime call handler", () => { const response = await handler(request); - expect(response.status).toBe(502); + expect(response.status).toBe(400); expect(await response.text()).toBe( - "The voice connection could not be established.", + "The voice connection returned an invalid response. Try again; if it continues, give the diagnostic reference to an operator.", + ); + expect(response.headers.get(VOICE_ERROR_CODE_HEADER)).toBe( + "invalid-response", ); expect(fetch).not.toHaveBeenCalled(); }); @@ -227,8 +296,9 @@ describe("OpenAI Realtime call handler", () => { const response = await responsePromise; expect(response.status).toBe(504); expect(await response.text()).toBe( - "The voice connection could not be established.", + "The voice connection timed out. Check your connection, then reconnect voice input.", ); + expect(response.headers.get(VOICE_ERROR_CODE_HEADER)).toBe("timeout"); expect(response.headers.get("cache-control")).toBe("no-store"); }); @@ -258,6 +328,56 @@ describe("OpenAI Realtime call handler", () => { const response = await responsePromise; expect(fetch.mock.calls[0]?.[1]?.signal?.aborted).toBe(true); expect(response.status).toBe(502); + expect(response.headers.get(VOICE_ERROR_CODE_HEADER)).toBe( + "request-aborted", + ); expect(response.headers.get("cache-control")).toBe("no-store"); }); + + test("does not start an unabortable call for a pre-aborted request", async () => { + const requestAbortController = new AbortController(); + requestAbortController.abort(); + const fetch = vi.fn(async (_input, init) => { + expect(init?.signal?.aborted).toBe(true); + throw new DOMException("aborted", "AbortError"); + }); + const handler = createOpenAIRealtimeCallHandler({ + environment: enabledEnvironment, + fetch, + }); + + const response = await handler( + createRequest(undefined, { signal: requestAbortController.signal }), + ); + + expect(fetch).not.toHaveBeenCalled(); + expect(response.status).toBe(502); + expect(response.headers.get(VOICE_ERROR_CODE_HEADER)).toBe( + "request-aborted", + ); + }); + + test("classifies network failures without exposing upstream diagnostics", async () => { + const fetch = vi.fn(async () => { + throw new Error("private network diagnostics"); + }); + const reportDiagnostic = vi.fn(); + const handler = createOpenAIRealtimeCallHandler({ + environment: enabledEnvironment, + fetch, + now: () => 100, + reportDiagnostic, + }); + + const response = await handler(createRequest()); + + expect(response.status).toBe(502); + expect(response.headers.get(VOICE_ERROR_CODE_HEADER)).toBe("network"); + expect(await response.text()).toBe( + "The voice connection could not be reached. Check your connection, then reconnect voice input.", + ); + expect(JSON.stringify(reportDiagnostic.mock.calls)).not.toContain( + "private network diagnostics", + ); + }); }); diff --git a/apps/petrinaut-website/src/server/voice/openai-realtime-call.ts b/apps/petrinaut-website/src/server/voice/openai-realtime-call.ts index c2bb864e37b..7af43efe03b 100644 --- a/apps/petrinaut-website/src/server/voice/openai-realtime-call.ts +++ b/apps/petrinaut-website/src/server/voice/openai-realtime-call.ts @@ -1,14 +1,18 @@ +import { + voiceErrorMessage, + type VoiceDiagnosticReporter, + type VoiceErrorCode, +} from "../../voice-diagnostics"; import { createOpenAITranscriptionSession, getOpenAIVoiceAvailability, OPENAI_REALTIME_CONNECTION_TIMEOUT_MS, } from "./openai-voice-policy"; +import { createVoiceRequestDiagnostics } from "./voice-request-diagnostics"; const OPENAI_REALTIME_CALLS_ENDPOINT = "https://api.openai.com/v1/realtime/calls"; const MAX_SDP_BYTES = 65_536; -const CONNECTION_ERROR_MESSAGE = - "The voice connection could not be established."; const timeoutError = new DOMException("Upstream timed out", "TimeoutError"); interface VoiceEnvironment { @@ -18,8 +22,11 @@ interface VoiceEnvironment { } interface OpenAIRealtimeCallDependencies { + readonly createRequestId?: () => string; readonly environment: VoiceEnvironment; readonly fetch: typeof globalThis.fetch; + readonly now?: () => number; + readonly reportDiagnostic?: VoiceDiagnosticReporter; } const response = ( @@ -52,15 +59,37 @@ const readSdpOffer = async (request: Request): Promise => { }; export const createOpenAIRealtimeCallHandler = - ({ environment, fetch }: OpenAIRealtimeCallDependencies) => + ({ + createRequestId, + environment, + fetch, + now, + reportDiagnostic, + }: OpenAIRealtimeCallDependencies) => async (request: Request): Promise => { + const diagnostics = createVoiceRequestDiagnostics(request, "connection", { + createRequestId, + now, + reportDiagnostic, + }); + const voiceFailure = ( + errorCode: VoiceErrorCode, + status: number, + ): Response => + diagnostics.respond( + response(voiceErrorMessage("connection", errorCode), status), + errorCode, + ); + if (request.method !== "POST") { - return response("Method not allowed.", 405, { allow: "POST" }); + return diagnostics.respond( + response("Method not allowed.", 405, { allow: "POST" }), + ); } const requestOrigin = new URL(request.url).origin; if (request.headers.get("origin") !== requestOrigin) { - return response("Forbidden.", 403); + return diagnostics.respond(response("Forbidden.", 403)); } const contentType = request.headers @@ -69,11 +98,13 @@ export const createOpenAIRealtimeCallHandler = ?.trim() .toLowerCase(); if (contentType !== "application/sdp") { - return response("The request must contain an SDP offer.", 415); + return diagnostics.respond( + response("The request must contain an SDP offer.", 415), + ); } if (!getOpenAIVoiceAvailability(environment).available) { - return response("Not found.", 404); + return voiceFailure("unavailable", 404); } const abortController = new AbortController(); @@ -91,10 +122,28 @@ export const createOpenAIRealtimeCallHandler = try { abortController.signal.throwIfAborted(); - const sdp = await readSdpOffer(request); + let sdp: string | Response; + try { + sdp = await readSdpOffer(request); + } catch { + const errorCode = + abortController.signal.reason === timeoutError + ? "timeout" + : request.signal.aborted + ? "request-aborted" + : "invalid-response"; + return voiceFailure( + errorCode, + errorCode === "timeout" + ? 504 + : errorCode === "request-aborted" + ? 502 + : 400, + ); + } abortController.signal.throwIfAborted(); if (sdp instanceof Response) { - return sdp; + return diagnostics.respond(sdp); } const session = createOpenAITranscriptionSession(); @@ -111,7 +160,8 @@ export const createOpenAIRealtimeCallHandler = signal: abortController.signal, }); if (!upstreamResponse.ok) { - return response(CONNECTION_ERROR_MESSAGE, 502); + await upstreamResponse.body?.cancel(); + return voiceFailure("invalid-response", 502); } const upstreamContentType = upstreamResponse.headers .get("content-type") @@ -122,20 +172,26 @@ export const createOpenAIRealtimeCallHandler = upstreamContentType !== "application/sdp" && upstreamContentType !== "text/plain" ) { - return response(CONNECTION_ERROR_MESSAGE, 502); + await upstreamResponse.body?.cancel(); + return voiceFailure("invalid-response", 502); } const answer = await upstreamResponse.text(); if (!answer.trim() || !answer.trimStart().startsWith("v=0")) { - return response(CONNECTION_ERROR_MESSAGE, 502); + return voiceFailure("invalid-response", 502); } - return response(answer, 200, { "content-type": "application/sdp" }); - } catch { - return response( - CONNECTION_ERROR_MESSAGE, - abortController.signal.reason === timeoutError ? 504 : 502, + return diagnostics.respond( + response(answer, 200, { "content-type": "application/sdp" }), ); + } catch { + const errorCode = + abortController.signal.reason === timeoutError + ? "timeout" + : request.signal.aborted + ? "request-aborted" + : "network"; + return voiceFailure(errorCode, errorCode === "timeout" ? 504 : 502); } finally { globalThis.clearTimeout(timeout); request.signal.removeEventListener("abort", abortForRequest); diff --git a/apps/petrinaut-website/src/server/voice/openai-speech.test.ts b/apps/petrinaut-website/src/server/voice/openai-speech.test.ts index 6b8fab97139..ec52cc81947 100644 --- a/apps/petrinaut-website/src/server/voice/openai-speech.test.ts +++ b/apps/petrinaut-website/src/server/voice/openai-speech.test.ts @@ -1,5 +1,9 @@ import { afterEach, describe, expect, test, vi } from "vitest"; +import { + VOICE_ERROR_CODE_HEADER, + VOICE_REQUEST_ID_HEADER, +} from "../../voice-diagnostics"; import { createOpenAISpeechHandler, OPENAI_SPEECH_TIMEOUT_MS, @@ -10,6 +14,7 @@ const enabledEnvironment = { PETRINAUT_OPENAI_VOICE_ENABLED: "true", VERCEL_ENV: "preview", }; +const requestId = "00000000-0000-4000-8000-000000000003"; const validSpeechRequest = { segmentId: "canonical-speech:assistant-1:text%3A0:fnv1a32:69f1e741", @@ -25,6 +30,7 @@ const createRequest = ( headers: { "content-type": "application/json", origin: "https://petrinaut.test", + [VOICE_REQUEST_ID_HEADER]: requestId, }, method: "POST", ...overrides, @@ -152,6 +158,7 @@ describe("OpenAI Speech handler", () => { }); test("streams audio for the exact canonical text with fixed server policy", async () => { + const reportDiagnostic = vi.fn(); const firstChunk = new Uint8Array([1, 2, 3]); const secondChunk = new Uint8Array([4, 5]); const fetch = vi.fn( @@ -170,6 +177,8 @@ describe("OpenAI Speech handler", () => { const handler = createOpenAISpeechHandler({ environment: enabledEnvironment, fetch, + now: () => 100, + reportDiagnostic, }); const response = await handler(createRequest()); @@ -177,6 +186,10 @@ describe("OpenAI Speech handler", () => { expect(response.status).toBe(200); expect(response.headers.get("cache-control")).toBe("no-store"); expect(response.headers.get("content-type")).toBe("audio/mpeg"); + expect(response.headers.get(VOICE_REQUEST_ID_HEADER)).toBe(requestId); + expect(response.headers.get("server-timing")).toBe( + "petrinaut_voice_speech;dur=0", + ); expect(fetch).toHaveBeenCalledOnce(); const [url, request] = fetch.mock.calls[0]!; expect(url).toBe("https://api.openai.com/v1/audio/speech"); @@ -205,6 +218,17 @@ describe("OpenAI Speech handler", () => { expect(new Uint8Array(await response.arrayBuffer())).toEqual( new Uint8Array([1, 2, 3, 4, 5]), ); + expect(reportDiagnostic).toHaveBeenCalledWith({ + durationMs: 0, + operation: "speech", + outcome: "success", + requestId, + stage: "server", + status: 200, + }); + expect(JSON.stringify(reportDiagnostic.mock.calls)).not.toContain( + validSpeechRequest.text, + ); }); test("sanitizes upstream and non-audio failures", async () => { @@ -233,7 +257,10 @@ describe("OpenAI Speech handler", () => { for (const response of [upstreamFailure, nonAudio, wrongAudioFormat]) { expect(response.status).toBe(502); expect(await response.text()).toBe( - "The response could not be spoken. Read the visible text instead.", + "The speech service returned an invalid response. Read the visible response instead. Try again; if it continues, give the diagnostic reference to an operator.", + ); + expect(response.headers.get(VOICE_ERROR_CODE_HEADER)).toBe( + "invalid-response", ); expect(response.headers.get("cache-control")).toBe("no-store"); } @@ -260,8 +287,9 @@ describe("OpenAI Speech handler", () => { const response = await responsePromise; expect(response.status).toBe(504); expect(await response.text()).toBe( - "The response could not be spoken. Read the visible text instead.", + "The speech service timed out. Read the visible response instead.", ); + expect(response.headers.get(VOICE_ERROR_CODE_HEADER)).toBe("timeout"); }); test("does not apply the response timeout to an active audio stream", async () => { @@ -319,6 +347,78 @@ describe("OpenAI Speech handler", () => { const response = await responsePromise; expect(fetch.mock.calls[0]?.[1]?.signal?.aborted).toBe(true); expect(response.status).toBe(502); + expect(response.headers.get(VOICE_ERROR_CODE_HEADER)).toBe( + "request-aborted", + ); + }); + + test("does not start an unabortable synthesis for a pre-aborted request", async () => { + const requestAbortController = new AbortController(); + requestAbortController.abort(); + const fetch = vi.fn(async (_input, init) => { + expect(init?.signal?.aborted).toBe(true); + throw new DOMException("aborted", "AbortError"); + }); + const handler = createOpenAISpeechHandler({ + environment: enabledEnvironment, + fetch, + }); + + const response = await handler( + createRequest(undefined, { signal: requestAbortController.signal }), + ); + + expect(fetch).not.toHaveBeenCalled(); + expect(response.status).toBe(502); + expect(response.headers.get(VOICE_ERROR_CODE_HEADER)).toBe( + "request-aborted", + ); + }); + + test("classifies a browser abort while streaming as interrupted", async () => { + const requestAbortController = new AbortController(); + let upstreamController: + | ReadableStreamDefaultController + | undefined; + const reportDiagnostic = vi.fn(); + const fetch = vi.fn(async (_input, init) => { + const body = new ReadableStream({ + start(controller) { + upstreamController = controller; + init?.signal?.addEventListener("abort", () => { + controller.error(init.signal!.reason); + }); + }, + }); + return new Response(body, { + headers: { "content-type": "audio/mpeg" }, + }); + }); + const handler = createOpenAISpeechHandler({ + environment: enabledEnvironment, + fetch, + reportDiagnostic, + }); + const response = await handler( + createRequest(undefined, { signal: requestAbortController.signal }), + ); + const reader = response.body!.getReader(); + const read = reader.read(); + + requestAbortController.abort(); + + await expect(read).rejects.toMatchObject({ name: "AbortError" }); + expect(upstreamController).toBeDefined(); + expect(reportDiagnostic).toHaveBeenCalledWith( + expect.objectContaining({ + errorCode: "request-aborted", + operation: "speech", + outcome: "aborted", + requestId, + stage: "server", + status: 200, + }), + ); }); test("cancels the OpenAI stream when browser playback stops reading", async () => { @@ -348,4 +448,28 @@ describe("OpenAI Speech handler", () => { expect(fetch.mock.calls[0]?.[1]?.signal?.aborted).toBe(true); expect(upstreamCancel).toHaveBeenCalledWith("playback stopped"); }); + + test("classifies network failures without exposing upstream diagnostics", async () => { + const fetch = vi.fn(async () => { + throw new Error("private speech network diagnostics"); + }); + const reportDiagnostic = vi.fn(); + const handler = createOpenAISpeechHandler({ + environment: enabledEnvironment, + fetch, + now: () => 100, + reportDiagnostic, + }); + + const response = await handler(createRequest()); + + expect(response.status).toBe(502); + expect(response.headers.get(VOICE_ERROR_CODE_HEADER)).toBe("network"); + expect(await response.text()).toBe( + "The speech service could not be reached. Read the visible response instead.", + ); + expect(JSON.stringify(reportDiagnostic.mock.calls)).not.toContain( + "private speech network diagnostics", + ); + }); }); diff --git a/apps/petrinaut-website/src/server/voice/openai-speech.ts b/apps/petrinaut-website/src/server/voice/openai-speech.ts index 1f8639af452..5aadd168db3 100644 --- a/apps/petrinaut-website/src/server/voice/openai-speech.ts +++ b/apps/petrinaut-website/src/server/voice/openai-speech.ts @@ -1,12 +1,17 @@ import { hashCanonicalSpeechText } from "../../canonical-speech-fingerprint"; +import { + voiceErrorMessage, + type VoiceDiagnosticReporter, + type VoiceErrorCode, +} from "../../voice-diagnostics"; import { getOpenAIVoiceAvailability } from "./openai-voice-policy"; +import { createVoiceRequestDiagnostics } from "./voice-request-diagnostics"; const OPENAI_SPEECH_ENDPOINT = "https://api.openai.com/v1/audio/speech"; const MAX_REQUEST_BYTES = 32_768; const MAX_SPEECH_CHARACTERS = 4_096; -const SPEECH_ERROR_MESSAGE = - "The response could not be spoken. Read the visible text instead."; const timeoutError = new DOMException("Upstream timed out", "TimeoutError"); +const requestAbortError = new DOMException("Request aborted", "AbortError"); export const OPENAI_SPEECH_TIMEOUT_MS = 25_000; @@ -18,8 +23,11 @@ interface VoiceEnvironment { } interface OpenAISpeechDependencies { + readonly createRequestId?: () => string; readonly environment: VoiceEnvironment; readonly fetch: typeof globalThis.fetch; + readonly now?: () => number; + readonly reportDiagnostic?: VoiceDiagnosticReporter; } interface SpeechRequest { @@ -108,14 +116,14 @@ const isSpeechRequest = (value: unknown): value is SpeechRequest => { const proxyAudioStream = ( upstreamBody: ReadableStream, abortController: AbortController, - cleanup: () => void, + finishRequest: (errorCode?: VoiceErrorCode) => void, ): ReadableStream => { const reader = upstreamBody.getReader(); let finished = false; - const finish = () => { + const finish = (errorCode?: VoiceErrorCode) => { if (!finished) { finished = true; - cleanup(); + finishRequest(errorCode); } }; @@ -125,7 +133,7 @@ const proxyAudioStream = ( try { await reader.cancel(reason); } finally { - finish(); + finish("request-aborted"); } }, async pull(controller) { @@ -139,21 +147,49 @@ const proxyAudioStream = ( controller.enqueue(value); } catch (error) { controller.error(error); - finish(); + finish( + abortController.signal.reason === timeoutError + ? "timeout" + : abortController.signal.reason === requestAbortError + ? "request-aborted" + : "network", + ); } }, }); }; export const createOpenAISpeechHandler = - ({ environment, fetch }: OpenAISpeechDependencies) => + ({ + createRequestId, + environment, + fetch, + now, + reportDiagnostic, + }: OpenAISpeechDependencies) => async (request: Request): Promise => { + const diagnostics = createVoiceRequestDiagnostics(request, "speech", { + createRequestId, + now, + reportDiagnostic, + }); + const voiceFailure = ( + errorCode: VoiceErrorCode, + status: number, + ): Response => + diagnostics.respond( + response(voiceErrorMessage("speech", errorCode), status), + errorCode, + ); + if (request.method !== "POST") { - return response("Method not allowed.", 405, { allow: "POST" }); + return diagnostics.respond( + response("Method not allowed.", 405, { allow: "POST" }), + ); } if (request.headers.get("origin") !== new URL(request.url).origin) { - return response("Forbidden.", 403); + return diagnostics.respond(response("Forbidden.", 403)); } const contentType = request.headers @@ -162,15 +198,17 @@ export const createOpenAISpeechHandler = ?.trim() .toLowerCase(); if (contentType !== "application/json") { - return response("The request must contain JSON.", 415); + return diagnostics.respond( + response("The request must contain JSON.", 415), + ); } if (!getOpenAIVoiceAvailability(environment).available) { - return response("Not found.", 404); + return voiceFailure("unavailable", 404); } const abortController = new AbortController(); - const abortForRequest = () => abortController.abort(); + const abortForRequest = () => abortController.abort(requestAbortError); request.signal.addEventListener("abort", abortForRequest, { once: true }); if (request.signal.aborted) { abortForRequest(); @@ -186,20 +224,24 @@ export const createOpenAISpeechHandler = abortController.signal.throwIfAborted(); if (body instanceof Response) { removeRequestAbortListener(); - return body; + return diagnostics.respond(body); } parsedBody = JSON.parse(body); } catch { removeRequestAbortListener(); if (abortController.signal.aborted) { - return response(SPEECH_ERROR_MESSAGE, 502); + return voiceFailure("request-aborted", 502); } - return response("The speech request is invalid.", 400); + return diagnostics.respond( + response("The speech request is invalid.", 400), + ); } if (!isSpeechRequest(parsedBody)) { removeRequestAbortListener(); - return response("The speech request is invalid.", 400); + return diagnostics.respond( + response("The speech request is invalid.", 400), + ); } const abortForTimeout = () => abortController.abort(timeoutError); @@ -214,6 +256,10 @@ export const createOpenAISpeechHandler = clearSpeechTimeout(); removeRequestAbortListener(); }; + const finishStreamingRequest = (errorCode?: VoiceErrorCode) => { + cleanup(); + diagnostics.finish(200, errorCode); + }; try { abortController.signal.throwIfAborted(); @@ -244,20 +290,29 @@ export const createOpenAISpeechHandler = ) { await upstreamResponse.body?.cancel(); cleanup(); - return response(SPEECH_ERROR_MESSAGE, 502); + return voiceFailure("invalid-response", 502); } clearSpeechTimeout(); - return response( - proxyAudioStream(upstreamResponse.body, abortController, cleanup), - 200, - { "content-type": "audio/mpeg" }, + return diagnostics.decorate( + response( + proxyAudioStream( + upstreamResponse.body, + abortController, + finishStreamingRequest, + ), + 200, + { "content-type": "audio/mpeg" }, + ), ); } catch { cleanup(); - return response( - SPEECH_ERROR_MESSAGE, - abortController.signal.reason === timeoutError ? 504 : 502, - ); + const errorCode = + abortController.signal.reason === timeoutError + ? "timeout" + : request.signal.aborted + ? "request-aborted" + : "network"; + return voiceFailure(errorCode, errorCode === "timeout" ? 504 : 502); } }; diff --git a/apps/petrinaut-website/src/server/voice/voice-request-diagnostics.ts b/apps/petrinaut-website/src/server/voice/voice-request-diagnostics.ts new file mode 100644 index 00000000000..0673aa2e170 --- /dev/null +++ b/apps/petrinaut-website/src/server/voice/voice-request-diagnostics.ts @@ -0,0 +1,77 @@ +import { + createVoiceRequestId, + resolveVoiceRequestId, + VOICE_ERROR_CODE_HEADER, + VOICE_REQUEST_ID_HEADER, + voiceDiagnosticOutcome, + voiceDurationMs, + type VoiceDiagnosticReporter, + type VoiceErrorCode, + type VoiceOperation, +} from "../../voice-diagnostics"; + +interface VoiceRequestDiagnosticDependencies { + readonly createRequestId?: () => string; + readonly now?: () => number; + readonly reportDiagnostic?: VoiceDiagnosticReporter; +} + +export const createVoiceRequestDiagnostics = ( + request: Request, + operation: VoiceOperation, + { + createRequestId = createVoiceRequestId, + now = () => performance.now(), + reportDiagnostic, + }: VoiceRequestDiagnosticDependencies = {}, +) => { + const requestId = resolveVoiceRequestId( + request.headers.get(VOICE_REQUEST_ID_HEADER), + createRequestId, + ); + const startedAt = now(); + let finished = false; + + const elapsed = () => voiceDurationMs(startedAt, now()); + const decorate = ( + response: Response, + errorCode?: VoiceErrorCode, + ): Response => { + response.headers.set(VOICE_REQUEST_ID_HEADER, requestId); + if (errorCode !== undefined) { + response.headers.set(VOICE_ERROR_CODE_HEADER, errorCode); + } + response.headers.append( + "server-timing", + `petrinaut_voice_${operation};dur=${elapsed()}`, + ); + return response; + }; + const finish = (status: number, errorCode?: VoiceErrorCode): void => { + if (finished) { + return; + } + finished = true; + reportDiagnostic?.({ + durationMs: elapsed(), + ...(errorCode === undefined ? {} : { errorCode }), + operation, + outcome: + errorCode === undefined && status >= 400 + ? "failure" + : voiceDiagnosticOutcome(errorCode), + requestId, + stage: "server", + status, + }); + }; + const respond = ( + response: Response, + errorCode?: VoiceErrorCode, + ): Response => { + finish(response.status, errorCode); + return decorate(response, errorCode); + }; + + return { decorate, finish, requestId, respond }; +}; diff --git a/apps/petrinaut-website/src/voice-diagnostics.test.ts b/apps/petrinaut-website/src/voice-diagnostics.test.ts new file mode 100644 index 00000000000..4007ecde320 --- /dev/null +++ b/apps/petrinaut-website/src/voice-diagnostics.test.ts @@ -0,0 +1,28 @@ +import { describe, expect, test } from "vitest"; + +import { resolveVoiceRequestId } from "./voice-diagnostics"; + +const generatedRequestId = "00000000-0000-4000-8000-000000000099"; + +describe("resolveVoiceRequestId", () => { + test.each([ + "00000000-0000-4000-8000-000000000001", + "ABCDEF01-2345-4ABC-BDEF-0123456789AB", + ])("preserves a valid UUID-v4 request ID: %s", (requestId) => { + expect(resolveVoiceRequestId(requestId, () => generatedRequestId)).toBe( + requestId, + ); + }); + + test.each([ + "00000000-0000-5000-8000-000000000001", + "00000000-0000-4000-7000-000000000001", + "g0000000-0000-4000-8000-000000000001", + "000000000000-4000-8000-000000000001", + "00000000-0000-4000-8000-000000000001-extra", + ])("replaces an invalid request ID: %s", (requestId) => { + expect(resolveVoiceRequestId(requestId, () => generatedRequestId)).toBe( + generatedRequestId, + ); + }); +}); diff --git a/apps/petrinaut-website/src/voice-diagnostics.ts b/apps/petrinaut-website/src/voice-diagnostics.ts new file mode 100644 index 00000000000..be74ef0b911 --- /dev/null +++ b/apps/petrinaut-website/src/voice-diagnostics.ts @@ -0,0 +1,179 @@ +export const VOICE_ERROR_CODE_HEADER = "x-petrinaut-voice-error"; +export const VOICE_REQUEST_ID_HEADER = "x-request-id"; + +export const voiceErrorCodes = [ + "microphone-permission", + "microphone-device", + "request-aborted", + "network", + "timeout", + "invalid-response", + "unavailable", +] as const; + +export type VoiceErrorCode = (typeof voiceErrorCodes)[number]; +export type VoiceOperation = "connection" | "transcription" | "speech"; + +export interface VoiceDiagnosticEvent { + readonly durationMs: number; + readonly errorCode?: VoiceErrorCode; + readonly operation: VoiceOperation; + readonly outcome: "success" | "failure" | "aborted"; + readonly requestId: string; + readonly stage: "browser" | "playback" | "server"; + readonly status?: number; +} + +export type VoiceDiagnosticReporter = (event: VoiceDiagnosticEvent) => void; + +const serverVoiceErrorCodes = [ + "request-aborted", + "network", + "timeout", + "invalid-response", + "unavailable", +] as const satisfies readonly VoiceErrorCode[]; + +export const createVoiceRequestId = (): string => crypto.randomUUID(); + +const isAsciiHexadecimalDigit = (character: string): boolean => { + const codePoint = character.charCodeAt(0); + return ( + (codePoint >= 48 && codePoint <= 57) || + (codePoint >= 65 && codePoint <= 70) || + (codePoint >= 97 && codePoint <= 102) + ); +}; + +const isVoiceRequestId = (value: string): boolean => { + if ( + value.length !== 36 || + value[8] !== "-" || + value[13] !== "-" || + value[14] !== "4" || + value[18] !== "-" || + value[23] !== "-" || + !["8", "9", "a", "b"].includes(value[19]?.toLowerCase() ?? "") + ) { + return false; + } + + for (let index = 0; index < value.length; index++) { + if (index === 8 || index === 13 || index === 18 || index === 23) { + continue; + } + if (!isAsciiHexadecimalDigit(value[index]!)) { + return false; + } + } + + return true; +}; + +export const resolveVoiceRequestId = ( + value: string | null | undefined, + createRequestId: () => string = createVoiceRequestId, +): string => + value !== null && value !== undefined && isVoiceRequestId(value) + ? value + : createRequestId(); + +export const voiceDurationMs = ( + startedAt: number, + finishedAt: number, +): number => Math.max(0, Math.round((finishedAt - startedAt) * 10) / 10); + +export const voiceDiagnosticOutcome = ( + errorCode?: VoiceErrorCode, +): VoiceDiagnosticEvent["outcome"] => + errorCode === undefined + ? "success" + : errorCode === "request-aborted" + ? "aborted" + : "failure"; + +export const reportVoiceDiagnostic: VoiceDiagnosticReporter = (event) => { + // The event type permits scalar operational metadata only. Never add + // provider payloads, SDP, audio, prompts, or spoken/transcribed text here. + // oxlint-disable-next-line no-console -- preview diagnostics use the website's existing runtime logger. + console.info("[Petrinaut voice]", JSON.stringify(event)); +}; + +const isServerVoiceErrorCode = ( + value: unknown, +): value is (typeof serverVoiceErrorCodes)[number] => + typeof value === "string" && + (serverVoiceErrorCodes as readonly string[]).includes(value); + +export const voiceErrorMessage = ( + operation: VoiceOperation, + code: VoiceErrorCode, +): string => { + if (code === "microphone-permission") { + return "Allow microphone access in your browser settings, then reconnect voice input."; + } + if (code === "microphone-device") { + return "No usable microphone was found. Connect or select one, then reconnect voice input."; + } + + const visibleTextFallback = + operation === "speech" ? " Read the visible response instead." : ""; + const reconnect = + operation === "speech" + ? "" + : " Check your connection, then reconnect voice input."; + const subject = + operation === "connection" ? "voice connection" : `${operation} service`; + + switch (code) { + case "request-aborted": + return `The ${subject} request was interrupted.${visibleTextFallback}${reconnect}`; + case "network": + return `The ${subject} could not be reached.${visibleTextFallback}${reconnect}`; + case "timeout": + return `The ${subject} timed out.${visibleTextFallback}${reconnect}`; + case "invalid-response": + return `The ${subject} returned an invalid response.${visibleTextFallback} Try again; if it continues, give the diagnostic reference to an operator.`; + case "unavailable": + return `The ${subject} preview is unavailable or disabled.${visibleTextFallback} Continue with the text composer.`; + } +}; + +export class VoiceError extends Error { + public readonly code: VoiceErrorCode; + public readonly requestId: string; + + public constructor( + operation: VoiceOperation, + code: VoiceErrorCode, + requestId: string, + ) { + super(voiceErrorMessage(operation, code)); + this.name = "VoiceError"; + this.code = code; + this.requestId = requestId; + } +} + +export const voiceErrorFromResponse = ( + response: Response, + operation: VoiceOperation, + fallbackRequestId: string, +): VoiceError => { + const headerCode = response.headers.get(VOICE_ERROR_CODE_HEADER); + const code = isServerVoiceErrorCode(headerCode) + ? headerCode + : response.status === 404 + ? "unavailable" + : response.status === 504 + ? "timeout" + : "invalid-response"; + const responseRequestId = response.headers.get(VOICE_REQUEST_ID_HEADER); + return new VoiceError( + operation, + code, + responseRequestId !== null && isVoiceRequestId(responseRequestId) + ? responseRequestId + : fallbackRequestId, + ); +}; diff --git a/libs/@hashintel/brunch-agent/docs/adr/0009-openai-voice-ui-turn-shell.md b/libs/@hashintel/brunch-agent/docs/adr/0009-openai-voice-ui-turn-shell.md index ed8625bd55d..2d346d43a38 100644 --- a/libs/@hashintel/brunch-agent/docs/adr/0009-openai-voice-ui-turn-shell.md +++ b/libs/@hashintel/brunch-agent/docs/adr/0009-openai-voice-ui-turn-shell.md @@ -17,10 +17,11 @@ either preserve that boundary or create a second conversation authority in the a Only the first choice preserves Brunch's durable history, captures, pending asks, completion, and projection contracts. -The first rollout is a disabled preview. The production contracts for authenticated ownership, -distributed quotas, telemetry, replay, and final Petrinaut projection do not all exist yet. The -preview therefore needs a boundary that permits input and output experiments without claiming -production recovery or public availability. +The first rollout is a disabled preview. FE-1505's privacy-safe Brunch spans exist in this +ancestry, but the production contracts for authenticated ownership, distributed quotas, complete +voice telemetry, replay, and final Petrinaut projection do not all exist yet. The preview +therefore needs a boundary that permits input and output experiments without claiming production +recovery or public availability. ## Decision @@ -54,8 +55,8 @@ production recovery or public availability. 7. **The preview fails closed and is disabled by default.** Voice is unavailable when server policy, credentials, or the Brunch transport are unavailable. Text chat remains available. Public production remains disabled until FE-1439, FE-1420, platform authentication, - distributed quotas, FE-1505 telemetry, and FE-1438/FE-1440 completion and projection contracts - are available and consumed. + distributed quotas, production voice telemetry building on FE-1505, and FE-1438/FE-1440 + completion and projection contracts are available and consumed. ## Consequences @@ -93,6 +94,27 @@ production recovery or public availability. failure keeps the canonical response visible, closes the microphone, and requires an explicit recovery action. +## Controlled-preview reliability follow-up evidence + +- Browser and server failures use only actionable categories: microphone permission, microphone + device, interrupted request, network, timeout, invalid provider response, and unavailable or + disabled. Provider bodies and thrown details remain suppressed; the UI gives recovery guidance + plus a sanitized code and request reference where applicable. +- Realtime connection and Speech requests share a validated random `x-request-id` across browser + and website server diagnostics. Transcript completion has a content-free browser timing, voice + routes expose `Server-Timing`, and the existing Petrinaut-to-Brunch transport supplies + `x-request-id` to FE-1505's content-suppressed Brunch inspection path. Events contain only + operation, stage, outcome, duration, request ID, and optional status or error code—never audio, + SDP, transcripts, prompts, canonical speech text, credentials, or provider response bodies. +- Focused tests cover startup, permission/device and network failures, abort, timeout, malformed + provider responses, reconnect, and media/playback cleanup. One local integration test exercises + browser session setup through both app-owned voice handlers, a completed transcript at the + Brunch composer boundary, canonical response selection, Speech streaming, and playback using + only local fakes. +- This evidence is a reliability follow-up above Voice PR 3, not production PR 4. It does not test + real OpenAI media or write to a configured remote Brunch conversation, does not add recovery or + rollout infrastructure, and does not change the production-disabled policy. + ## Revisit condition Revisit if the unified OpenAI WebRTC initialization API cannot enforce server-owned transcription diff --git a/libs/@hashintel/petrinaut/docs/ai-assistant.md b/libs/@hashintel/petrinaut/docs/ai-assistant.md index 8b76dc57393..ae47c8ad1c0 100644 --- a/libs/@hashintel/petrinaut/docs/ai-assistant.md +++ b/libs/@hashintel/petrinaut/docs/ai-assistant.md @@ -35,6 +35,13 @@ working or a response is playing. Spoken responses use an AI-generated OpenAI vo in the voice status panel. If speech fails, the response remains visible to read and the voice control offers recovery instead of changing or regenerating the text. +If voice cannot continue, the status panel identifies the kind of problem. For microphone +permission or device errors, allow access or connect/select a microphone before reconnecting. For +an interrupted request, network error, or timeout, check the connection and choose **Reconnect +voice input**. If the preview is unavailable, continue with the text composer. An invalid service +response includes a diagnostic reference you can give to an operator; that reference and its +diagnostic record do not contain your transcript or the response being spoken. + **Clear AI chat** via the delete button in the top right of the panel: wipes the conversation, stops any in-flight stream, and tells the host app to forget the messages (if the host persists them). ## What the assistant can do