diff --git a/.changeset/stable-composer-controls.md b/.changeset/stable-composer-controls.md index 1bccd0dbc6c..36cc6644f20 100644 --- a/.changeset/stable-composer-controls.md +++ b/.changeset/stable-composer-controls.md @@ -2,7 +2,16 @@ "@hashintel/petrinaut": patch --- -Add a generic host-rendered AI composer control with stable finalized-text submission, -conversation identity, stop handling, schema-validated interactive-tool text mapping, and an -explicit separate-message target for corrections. Add a queue-aware voice submission path so a -finalized spoken turn is retained while another response settles. +Add generic host-rendered AI composer controls and a persistent interview stage with docked and +detached placements, protected active conversations, keyboard fallback, and one-answer buffering +while the normal chat stream settles. Include stable finalized-text submission, conversation +identity, stop handling, schema-validated interactive-tool text mapping, explicit separate-message +targeting for corrections, and a queue-aware voice submission path. Add the Chat / Interview mode +switch and export `PetrinautAiInteractionMode`, with the selected interaction mode and mode-change +callback available to host-rendered interview stages. `renderComposerControl` remains a supported +public seam for hosts that only need their own control beside the message box, independently of the +interview stage. + +Simplify Interview mode with a circular microphone waveform, compact transcript states that +distinguish recording, sending, sent, and undelivered answers, phase-specific icon controls, and +recovery that names the kind of failure before offering reconnect. diff --git a/apps/petrinaut-website/src/main/app/brunch-sweep-output.ts b/apps/petrinaut-website/src/main/app/brunch-sweep-output.ts new file mode 100644 index 00000000000..a200704591b --- /dev/null +++ b/apps/petrinaut-website/src/main/app/brunch-sweep-output.ts @@ -0,0 +1,69 @@ +import { z } from "zod"; + +const completionFailureSchema = z.object({ + diagnostic: z.string(), + nodeId: z.string().optional(), + kind: z.string().optional(), + slot: z.string().optional(), + requirement: z.string(), + actual: z.string(), + message: z.string(), + captureIds: z.array(z.string()), +}); + +const completionReportSchema = z.object({ + complete: z.boolean(), + pluginVersion: z.string(), + revision: z.string(), + failures: z.array(completionFailureSchema), + sliceNodeIds: z.array(z.string()), + outsideSlice: z.array( + z.object({ + nodeId: z.string(), + kind: z.string(), + open: z.array(completionFailureSchema), + }), + ), +}); + +const captureSchema = z.object({ + id: z.string(), + status: z.enum(["active", "superseded", "retracted"]), + epistemicStatus: z.string(), + confidence: z.string(), + content: z.union([ + z.object({ value: z.unknown() }), + z.object({ absence: z.string() }), + ]), + evidence: z.array(z.object({ excerpt: z.string() })).optional(), + basis: z + .object({ + type: z.string(), + description: z.string(), + }) + .optional(), + alternativeGroup: z.string().optional(), + supersedes: z.string().optional(), +}); + +/** Shape of the Brunch sweep client tool's output, as read by the app. */ +export const sweepOutputSchema = z.discriminatedUnion("status", [ + z.object({ status: z.literal("no-settled-range") }), + z.object({ + status: z.literal("refused"), + refusal: z.object({ + code: z.string(), + message: z.string(), + }), + }), + z.object({ + status: z.literal("applied"), + appliedCaptureIds: z.array(z.string()), + captures: z.array(captureSchema), + completion: completionReportSchema.optional(), + }), +]); + +export type SweepCompletionFailure = z.infer; +export type SweepCompletionReport = z.infer; +export type SweepCapture = z.infer; diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-panel-transport.ts b/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-panel-transport.ts index 5cc856f6507..6069ee67f2a 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-panel-transport.ts +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-panel-transport.ts @@ -1,4 +1,130 @@ +import { SWEEP_TOOL_NAME } from "@hashintel/brunch-agent-transport-aisdk/client-tools"; + +import { sweepOutputSchema } from "../brunch-sweep-output"; + +import type { + SweepCapture, + SweepCompletionFailure, + SweepCompletionReport, +} from "../brunch-sweep-output"; import type { PetrinautAiChatTransport } from "@hashintel/petrinaut/ui"; +import type { UIMessageChunk } from "ai"; + +const formatFailure = (failure: SweepCompletionFailure): string => { + const location = + failure.nodeId === undefined + ? "" + : ` at ${failure.nodeId}${failure.slot === undefined ? "" : `.${failure.slot}`}`; + const captures = + failure.captureIds.length === 0 + ? "" + : ` Captures: ${failure.captureIds.join(", ")}`; + return `Completion gap [${failure.diagnostic}]${location}: needs ${failure.requirement}; actual ${failure.actual}. ${failure.message}${captures}`; +}; + +const formatCapture = (capture: SweepCapture): string => { + const content = + "value" in capture.content + ? JSON.stringify(capture.content.value) + : `absence: ${capture.content.absence}`; + const provenance = + capture.evidence !== undefined + ? capture.evidence.map((evidence) => `“${evidence.excerpt}”`).join("; ") + : capture.basis === undefined + ? "no provenance" + : `${capture.basis.type}: ${capture.basis.description}`; + const history = [ + capture.alternativeGroup === undefined + ? undefined + : `alternative group ${capture.alternativeGroup}`, + capture.supersedes === undefined + ? undefined + : `supersedes ${capture.supersedes}`, + ].filter((fact) => fact !== undefined); + return `Capture ${capture.id} (${capture.status}; ${capture.epistemicStatus}; confidence ${capture.confidence}): ${content} — ${provenance}${history.length === 0 ? "" : `; ${history.join("; ")}`}`; +}; + +const formatCompletion = (report: SweepCompletionReport): string[] => [ + `Completion: ${report.complete ? "complete" : "incomplete"} · plugin ${report.pluginVersion} · revision ${report.revision}`, + `Completion slice: ${report.sliceNodeIds.join(", ") || "none"}`, + ...report.failures.map(formatFailure), + ...report.outsideSlice.flatMap((node) => [ + `Outside completion slice: ${node.nodeId} (${node.kind}); ${node.open.length} open requirement${node.open.length === 1 ? "" : "s"}`, + ...node.open.map((failure) => `Outside-slice ${formatFailure(failure)}`), + ]), +]; + +const summarizeSweepOutput = ( + output: unknown, +): + | { + readonly title: string; + readonly detail: string; + readonly items?: readonly string[]; + } + | undefined => { + const parsed = sweepOutputSchema.safeParse(output); + if (!parsed.success) return undefined; + + const sweep = parsed.data; + switch (sweep.status) { + case "no-settled-range": + return { + title: "No settled range to sweep", + detail: "The conversation has no settled user entries.", + }; + case "refused": + return { + title: "Sweep refused", + detail: sweep.refusal.message, + items: [`Refusal: ${sweep.refusal.code}`], + }; + case "applied": + return { + title: "Sweep applied", + detail: `${sweep.appliedCaptureIds.length} new capture${sweep.appliedCaptureIds.length === 1 ? "" : "s"} · ${sweep.captures.length} total · ${sweep.completion?.complete === true ? "complete" : "incomplete"}`, + items: [ + ...sweep.captures.map(formatCapture), + ...(sweep.completion === undefined + ? [] + : formatCompletion(sweep.completion)), + ], + }; + } +}; + +const decorateBrunchStream = ( + stream: ReadableStream, +): ReadableStream => { + const toolNamesByCallId = new Map(); + return stream.pipeThrough( + new TransformStream({ + transform(chunk, controller) { + if (chunk.type === "tool-input-available") { + toolNamesByCallId.set(chunk.toolCallId, chunk.toolName); + } + if ( + chunk.type === "tool-output-available" && + toolNamesByCallId.get(chunk.toolCallId) === SWEEP_TOOL_NAME + ) { + const summary = summarizeSweepOutput(chunk.output); + if ( + summary !== undefined && + typeof chunk.output === "object" && + chunk.output !== null + ) { + controller.enqueue({ + ...chunk, + output: { ...chunk.output, ...summary }, + }); + return; + } + } + controller.enqueue(chunk); + }, + }), + ); +}; /** * Pin Petrinaut's stock transport to one stable conversation id so reload, @@ -8,8 +134,18 @@ export const createBrunchPanelTransport = ( transport: PetrinautAiChatTransport, conversationId: string, ): PetrinautAiChatTransport => ({ - reconnectToStream: (options) => - transport.reconnectToStream({ ...options, chatId: conversationId }), - sendMessages: (options) => - transport.sendMessages({ ...options, chatId: conversationId }), + reconnectToStream: async (options) => { + const stream = await transport.reconnectToStream({ + ...options, + chatId: conversationId, + }); + return stream === null ? null : decorateBrunchStream(stream); + }, + sendMessages: async (options) => + decorateBrunchStream( + await transport.sendMessages({ + ...options, + chatId: conversationId, + }), + ), }); diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx index c14f42df3de..53948c0074a 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx @@ -5,7 +5,7 @@ import { isValidElement, type ReactNode } from "react"; import { describe, expect, test, vi } from "vitest"; import { VoiceInterviewControl } from "../voice-interview/voice-interview-control"; -import { getBrunchVoiceComposerControl } from "./local-storage-demo-app"; +import { getBrunchVoiceInterviewStage } from "./local-storage-demo-app"; const defaultTransportOptions = vi.hoisted(() => ({ current: null as unknown, @@ -28,16 +28,28 @@ vi.mock("@hashintel/petrinaut/ui", () => ({ describe("local storage demo Brunch voice integration", () => { test("does not install voice on the generic local chat fallback", () => { - expect(getBrunchVoiceComposerControl(false)).toBeUndefined(); + expect(getBrunchVoiceInterviewStage(null)).toBeUndefined(); }); test("installs the app-owned voice control for a configured Brunch transport", () => { - const renderControl = getBrunchVoiceComposerControl(true); - const control = renderControl?.({ + const config = { available: true as const, connectionTimeoutMs: 15_000 }; + const stage = getBrunchVoiceInterviewStage(config); + const control = stage?.({ + canAcceptInterviewAnswer: true, conversationId: "petrinaut-preview:net-1", + focusComposer: vi.fn(), + interactionMode: "chat", messages: [], + openSidebar: vi.fn(), + placement: "sidebar", + setActive: vi.fn(), + setInteractionMode: vi.fn(), status: "ready", stop: vi.fn(async () => undefined), + submitInterviewAnswer: vi.fn(async () => ({ + kind: "message" as const, + messageId: "message-1", + })), submitText: vi.fn(async () => ({ kind: "message" as const, messageId: "message-1", @@ -52,7 +64,10 @@ describe("local storage demo Brunch voice integration", () => { if (!isValidElement(control)) { throw new Error("Expected the configured composer control to render."); } - expect(control.type).toBe(VoiceInterviewControl); + expect(control).toMatchObject({ + props: { config }, + type: VoiceInterviewControl, + }); }); test("correlates the existing Brunch transport request", () => { diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx index 5dfd53abc82..adf36eb0860 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx @@ -17,15 +17,19 @@ import { import { DefaultChatTransport, Petrinaut, - type PetrinautAiComposerControl, - type PetrinautAiComposerControlContext, + type PetrinautAiInterviewStage, + type PetrinautAiInterviewStageContext, type PetrinautAiMessage, WalkthroughProvider, } from "@hashintel/petrinaut/ui"; import { VOICE_REQUEST_ID_HEADER } from "../../../voice-diagnostics"; import { useSentryFeedbackAction } from "../sentry-feedback-button"; -import { VoiceInterviewControl } from "../voice-interview/voice-interview-control"; +import { + loadOpenAIVoiceConfig, + type OpenAIVoiceConfig, + VoiceInterviewControl, +} from "../voice-interview/voice-interview-control"; import { brunchAskInteractiveTool } from "./brunch-ask-interactive-tool"; import { getOrCreateBrunchConversationId } from "./brunch-conversation-id"; import { createBrunchPanelTransport } from "./brunch-panel-transport"; @@ -91,18 +95,14 @@ const brunchPreviewConfig = resolveBrunchPreviewConfig( import.meta.env.VITE_BRUNCH_CHAT_ENDPOINT, ); -const renderBrunchVoiceComposerControl = ( - context: PetrinautAiComposerControlContext, -) => ; - -export const getBrunchVoiceComposerControl = ( - isBrunchConfigured: boolean, -): PetrinautAiComposerControl | undefined => - isBrunchConfigured ? renderBrunchVoiceComposerControl : undefined; - -const brunchVoiceComposerControl = getBrunchVoiceComposerControl( - brunchPreviewConfig.isBrunchConfigured, -); +export const getBrunchVoiceInterviewStage = ( + config: OpenAIVoiceConfig | null | undefined, +): PetrinautAiInterviewStage | undefined => + config + ? (context: PetrinautAiInterviewStageContext) => ( + + ) + : undefined; const createHandle = (net: SDCPNInLocalStorage): PetrinautDocHandle => createJsonDocHandle({ @@ -154,11 +154,38 @@ const createActiveHandle = (net: SDCPNInLocalStorage): ActiveHandle => ({ */ export const LocalStorageDemoApp = () => { const sentryFeedbackAction = useSentryFeedbackAction(); + const [openAIVoiceConfig, setOpenAIVoiceConfig] = + useState(); const { aiMessagesByNetId, setAiMessagesByNetId } = useLocalStorageAiMessages(); const { storedSDCPNs, setStoredSDCPNs } = useLocalStorageSDCPNs(); const storedSDCPNsForDisplay = getStoredSDCPNsForDisplay(storedSDCPNs); + useEffect(() => { + if (!brunchPreviewConfig.isBrunchConfigured) { + // eslint-disable-next-line react-hooks-js/set-state-in-effect -- Resolve the loading sentinel when voice is not configured. + setOpenAIVoiceConfig(null); + return; + } + + const abortController = new AbortController(); + void loadOpenAIVoiceConfig( + globalThis.fetch.bind(globalThis), + abortController.signal, + ).then((config) => { + if (!abortController.signal.aborted) { + setOpenAIVoiceConfig(config); + } + }); + + return () => abortController.abort(); + }, []); + + const brunchVoiceInterviewStage = useMemo( + () => getBrunchVoiceInterviewStage(openAIVoiceConfig), + [openAIVoiceConfig], + ); + // Pick the most recently modified net const mostRecentlyModifiedNet = Object.values(storedSDCPNsForDisplay).sort( @@ -334,14 +361,15 @@ export const LocalStorageDemoApp = () => { return next; }); }, - ...(brunchVoiceComposerControl + ...(brunchVoiceInterviewStage ? { - renderComposerControl: brunchVoiceComposerControl, + renderInterviewStage: brunchVoiceInterviewStage, } : {}), }), [ aiMessagesByNetId, + brunchVoiceInterviewStage, conversationId, currentNetId, flueHistory.messages, diff --git a/apps/petrinaut-website/src/main/app/voice-interview/interview-coverage.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/interview-coverage.test.ts new file mode 100644 index 00000000000..84d329b5062 --- /dev/null +++ b/apps/petrinaut-website/src/main/app/voice-interview/interview-coverage.test.ts @@ -0,0 +1,126 @@ +import { describe, expect, test } from "vitest"; + +import { SWEEP_TOOL_NAME } from "@hashintel/brunch-agent-transport-aisdk/client-tools"; + +import { selectInterviewCoverage } from "./interview-coverage"; + +import type { PetrinautAiMessage } from "@hashintel/petrinaut/ui"; + +describe("interview coverage", () => { + test("uses only authoritative completion results for covered and open topics", () => { + const messages = [ + { + id: "assistant-sweep", + role: "assistant", + parts: [ + { + type: "dynamic-tool", + toolCallId: "sweep-1", + toolName: SWEEP_TOOL_NAME, + state: "output-available", + input: {}, + output: { + status: "applied", + appliedCaptureIds: ["capture-owner"], + captures: [ + { + id: "capture-owner", + status: "active", + epistemicStatus: "explicit", + confidence: "high", + content: { + value: { + type: "slot-asserted", + kind: "activity", + node: "approval", + slot: "who performs it", + precision: "named", + assertion: { value: "shift lead" }, + }, + }, + }, + { + id: "capture-old", + status: "superseded", + epistemicStatus: "explicit", + confidence: "high", + content: { + value: { + type: "slot-asserted", + kind: "activity", + node: "approval", + slot: "how long it takes", + assertion: { value: "one hour" }, + }, + }, + }, + ], + completion: { + complete: false, + pluginVersion: "sdcpn/1", + revision: "revision-1", + failures: [ + { + diagnostic: "unaddressed", + nodeId: "activity:approval", + kind: "activity", + slot: "how long it takes", + requirement: "spread", + actual: "not mentioned", + message: "Duration is still unknown.", + captureIds: [], + }, + ], + sliceNodeIds: ["activity:approval", "activity:dispatch"], + outsideSlice: [], + }, + }, + }, + ], + }, + ] as unknown as PetrinautAiMessage[]; + + expect(selectInterviewCoverage(messages)).toEqual({ + complete: false, + covered: ["dispatch"], + stillExploring: ["approval — how long it takes"], + }); + }); + + test("names each covered topic once when node identifiers share a label", () => { + const messages = [ + { + id: "assistant-sweep", + role: "assistant", + parts: [ + { + type: "dynamic-tool", + toolCallId: "sweep-2", + toolName: SWEEP_TOOL_NAME, + state: "output-available", + input: {}, + output: { + status: "applied", + appliedCaptureIds: [], + captures: [], + completion: { + complete: true, + pluginVersion: "sdcpn/1", + revision: "revision-2", + failures: [], + sliceNodeIds: ["activity:approval", "object:approval"], + outsideSlice: [], + }, + }, + }, + ], + }, + ] as unknown as PetrinautAiMessage[]; + + expect(selectInterviewCoverage(messages)?.covered).toEqual(["approval"]); + }); + + test("omits coverage when no validated completion report exists", () => { + expect(selectInterviewCoverage([])).toBeNull(); + }); +}); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/interview-coverage.ts b/apps/petrinaut-website/src/main/app/voice-interview/interview-coverage.ts new file mode 100644 index 00000000000..5f1ef5b4164 --- /dev/null +++ b/apps/petrinaut-website/src/main/app/voice-interview/interview-coverage.ts @@ -0,0 +1,68 @@ +import { SWEEP_TOOL_NAME } from "@hashintel/brunch-agent-transport-aisdk/client-tools"; + +import { sweepOutputSchema } from "../brunch-sweep-output"; + +import type { PetrinautAiMessage } from "@hashintel/petrinaut/ui"; + +export interface InterviewCoverage { + readonly complete: boolean; + readonly covered: readonly string[]; + readonly stillExploring: readonly string[]; +} + +const unique = (items: string[]): string[] => [...new Set(items)]; + +const nodeLabel = (nodeId: string): string => { + const separator = nodeId.indexOf(":"); + return separator === -1 ? nodeId : nodeId.slice(separator + 1); +}; + +export const selectInterviewCoverage = ( + messages: PetrinautAiMessage[], +): InterviewCoverage | null => { + for (const message of messages.toReversed()) { + for (const part of message.parts.toReversed()) { + if ( + part.type !== "dynamic-tool" || + part.toolName !== SWEEP_TOOL_NAME || + part.state !== "output-available" + ) { + continue; + } + const parsed = sweepOutputSchema.safeParse(part.output); + if ( + !parsed.success || + parsed.data.status !== "applied" || + parsed.data.completion === undefined + ) { + continue; + } + + const { completion } = parsed.data; + const nodesWithFailures = new Set( + completion.failures.flatMap((failure) => + failure.nodeId === undefined ? [] : [failure.nodeId], + ), + ); + const covered = unique( + completion.sliceNodeIds + .filter((nodeId) => !nodesWithFailures.has(nodeId)) + .map(nodeLabel), + ); + const stillExploring = unique( + completion.failures.map((failure) => + failure.nodeId === undefined + ? failure.message + : `${nodeLabel(failure.nodeId)} — ${failure.slot ?? failure.message}`, + ), + ); + + return { + complete: completion.complete, + covered, + stillExploring, + }; + } + } + return null; +}; diff --git a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts index c1b249b1cd0..926d553cade 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts @@ -11,6 +11,7 @@ class FakeDataChannel extends EventTarget { public readonly close = vi.fn(() => { this.readyState = "closed"; }); + public readonly send = vi.fn(); public open() { this.readyState = "open"; @@ -26,8 +27,36 @@ class FakeDataChannel extends EventTarget { } } -const createHarness = (connectionTimeoutMs = 15_000) => { +const createHarness = ({ + connectionTimeoutMs = 15_000, + createAudioContext, +}: { + readonly connectionTimeoutMs?: number; + readonly createAudioContext?: () => AudioContext; +} = {}) => { let requestNumber = 0; + const animationFrames: FrameRequestCallback[] = []; + const analyser = { + fftSize: 0, + getByteTimeDomainData: vi.fn((data: Uint8Array) => { + data.fill(160); + }), + }; + const mediaSource = { connect: vi.fn() }; + const tracks: Array<{ enabled: boolean; stop: ReturnType }> = + []; + const trackEnabledWhenMeterConnected: boolean[] = []; + const audioContext = { + close: vi.fn(async () => undefined), + createAnalyser: vi.fn(() => analyser), + createMediaStreamSource: vi.fn(() => { + trackEnabledWhenMeterConnected.push(tracks.at(-1)?.enabled ?? true); + return mediaSource; + }), + resume: vi.fn(async () => undefined), + state: "suspended" as AudioContextState, + }; + const cancelAnimationFrame = vi.fn(); const channels: FakeDataChannel[] = []; const peers: Array<{ addTrack: ReturnType; @@ -39,8 +68,6 @@ const createHarness = (connectionTimeoutMs = 15_000) => { setLocalDescription: ReturnType; setRemoteDescription: ReturnType Promise>>; }> = []; - const tracks: Array<{ enabled: boolean; stop: ReturnType }> = - []; const fetch = vi.fn( async () => new Response("v=0\r\no=OpenAI answer", { @@ -78,18 +105,29 @@ const createHarness = (connectionTimeoutMs = 15_000) => { return peer as unknown as RTCPeerConnection; }; const session = new OpenAIRealtimeSession({ + cancelAnimationFrame, connectionTimeoutMs, + createAudioContext: + createAudioContext ?? (() => audioContext as unknown as AudioContext), createRequestId: () => `voice-request-${++requestNumber}`, createPeerConnection, fetch, getUserMedia, now: () => 100, reportDiagnostic, + requestAnimationFrame: (callback) => { + animationFrames.push(callback); + return animationFrames.length; + }, }); const events: OpenAIRealtimeSessionEvent[] = []; session.subscribe((event) => events.push(event)); return { + analyser, + animationFrames, + audioContext, + cancelAnimationFrame, channels, events, fetch, @@ -97,6 +135,7 @@ const createHarness = (connectionTimeoutMs = 15_000) => { peers, reportDiagnostic, session, + trackEnabledWhenMeterConnected, tracks, }; }; @@ -119,6 +158,7 @@ describe("OpenAIRealtimeSession", () => { }, }); expect(harness.tracks[0]!.enabled).toBe(false); + expect(harness.trackEnabledWhenMeterConnected).toEqual([false]); expect(harness.fetch).toHaveBeenCalledWith( "/api/voice/realtime-call", expect.objectContaining({ @@ -149,6 +189,126 @@ describe("OpenAIRealtimeSession", () => { }); }); + test("reports real input level only while the microphone track is enabled", async () => { + const harness = createHarness(); + await harness.session.connect(); + + expect(harness.animationFrames).toHaveLength(0); + harness.session.setMicrophoneEnabled(true); + expect(harness.animationFrames).toHaveLength(1); + harness.animationFrames.shift()?.(0); + + expect(harness.events.at(-1)).toMatchObject({ + type: "microphone-level", + }); + expect((harness.events.at(-1) as { level: number }).level).toBeGreaterThan( + 0, + ); + + harness.session.setMicrophoneEnabled(false); + expect(harness.tracks[0]!.enabled).toBe(false); + expect(harness.cancelAnimationFrame).toHaveBeenCalled(); + expect(harness.events.at(-1)).toEqual({ + level: 0, + type: "microphone-level", + }); + }); + + test("quantizes input levels and skips unchanged meter frames", async () => { + const harness = createHarness(); + harness.analyser.getByteTimeDomainData.mockImplementation( + (data: Uint8Array) => { + data.fill(200); + }, + ); + await harness.session.connect(); + harness.session.setMicrophoneEnabled(true); + + const levels = () => + harness.events + .filter((event) => event.type === "microphone-level") + .map((event) => (event as { level: number }).level); + + harness.animationFrames.shift()?.(0); + harness.animationFrames.shift()?.(0); + harness.animationFrames.shift()?.(0); + + expect(levels()).toEqual([0.56]); + + harness.analyser.getByteTimeDomainData.mockImplementation( + (data: Uint8Array) => { + data.fill(160); + }, + ); + harness.animationFrames.shift()?.(0); + + expect(levels()).toEqual([0.56, 0.25]); + }); + + test("resumes a suspended input meter before waiting for microphone access", async () => { + const harness = createHarness(); + const track = { enabled: true, stop: vi.fn() }; + const stream = { + getAudioTracks: () => [track], + getTracks: () => [track], + } as unknown as MediaStream; + let resolveMedia: ((mediaStream: MediaStream) => void) | undefined; + harness.getUserMedia.mockImplementationOnce( + () => + new Promise((resolve) => { + resolveMedia = resolve; + }), + ); + + const connection = harness.session.connect(); + const resumeCallsBeforeMedia = + harness.audioContext.resume.mock.calls.length; + const sourceCallsBeforeMedia = + harness.audioContext.createMediaStreamSource.mock.calls.length; + resolveMedia?.(stream); + await connection; + + expect(resumeCallsBeforeMedia).toBe(1); + expect(sourceCallsBeforeMedia).toBe(0); + expect(harness.audioContext.createMediaStreamSource).toHaveBeenCalledWith( + stream, + ); + }); + + test("connects without metering when audio context construction throws", async () => { + const harness = createHarness({ + createAudioContext: () => { + throw new Error("AudioContext unavailable"); + }, + }); + + await expect(harness.session.connect()).resolves.toBe(1); + harness.session.setMicrophoneEnabled(true); + + expect(harness.fetch).toHaveBeenCalledOnce(); + expect(harness.peers[0]!.addTrack).toHaveBeenCalledOnce(); + expect(harness.tracks[0]!.enabled).toBe(true); + expect(harness.animationFrames).toHaveLength(0); + expect(harness.events).toEqual([]); + }); + + test("connects without metering when meter initialization throws", async () => { + const harness = createHarness(); + harness.audioContext.createMediaStreamSource.mockImplementationOnce(() => { + throw new Error("Media stream source unavailable"); + }); + + await expect(harness.session.connect()).resolves.toBe(1); + harness.session.setMicrophoneEnabled(true); + + expect(harness.fetch).toHaveBeenCalledOnce(); + expect(harness.peers[0]!.addTrack).toHaveBeenCalledOnce(); + expect(harness.tracks[0]!.enabled).toBe(true); + expect(harness.audioContext.close).toHaveBeenCalledOnce(); + expect(harness.animationFrames).toHaveLength(0); + expect(harness.events).toEqual([]); + }); + test("emits only strict input transcription events with stable source identity", async () => { const harness = createHarness(); await harness.session.connect(); @@ -488,7 +648,7 @@ describe("OpenAIRealtimeSession", () => { }); test("rejects a data channel already closed after negotiation", async () => { - const harness = createHarness(1_000); + const harness = createHarness({ connectionTimeoutMs: 1_000 }); let resolveFetch: ((response: Response) => void) | undefined; harness.fetch.mockImplementationOnce( () => diff --git a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts index b351e9a5785..a27585f89e4 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts @@ -28,6 +28,7 @@ export type OpenAIRealtimeSessionEvent = readonly text: string; readonly type: "partial" | "completed"; } + | { readonly level: number; readonly type: "microphone-level" } | { readonly code: VoiceErrorCode; readonly message: string; @@ -36,7 +37,9 @@ export type OpenAIRealtimeSessionEvent = }; interface OpenAIRealtimeSessionDependencies { + readonly cancelAnimationFrame: (handle: number) => void; readonly connectionTimeoutMs: number; + readonly createAudioContext: () => AudioContext; readonly createRequestId?: () => string; readonly createPeerConnection: () => RTCPeerConnection; readonly fetch: typeof globalThis.fetch; @@ -45,6 +48,7 @@ interface OpenAIRealtimeSessionDependencies { ) => Promise; readonly now?: () => number; readonly reportDiagnostic?: VoiceDiagnosticReporter; + readonly requestAnimationFrame: (callback: FrameRequestCallback) => number; } type SessionListener = (event: OpenAIRealtimeSessionEvent) => void; @@ -104,6 +108,8 @@ const waitForAbort = ( export class OpenAIRealtimeSession { readonly #dependencies: OpenAIRealtimeSessionDependencies; readonly #listeners = new Set(); + #analyser: AnalyserNode | null = null; + #audioContext: AudioContext | null = null; #abortController: AbortController | null = null; #activeEpoch: number | null = null; #connected = false; @@ -112,6 +118,10 @@ export class OpenAIRealtimeSession { #dataChannel: RTCDataChannel | null = null; #epoch = 0; #mediaStream: MediaStream | null = null; + #meterFrame: number | null = null; + #meterHasSample = false; + #meterLevel = 0; + #meterSamples: Uint8Array | null = null; #messageListener: ((event: MessageEvent) => void) | null = null; #microphoneTrack: MediaStreamTrack | null = null; #peerConnection: RTCPeerConnection | null = null; @@ -147,6 +157,21 @@ export class OpenAIRealtimeSession { }, this.#dependencies.connectionTimeoutMs); try { + let audioContext: AudioContext | null = null; + try { + audioContext = this.#dependencies.createAudioContext(); + this.#audioContext = audioContext; + if (audioContext.state === "suspended") { + try { + void audioContext.resume().catch(() => undefined); + } catch { + // Input metering is optional and must not block voice connection. + } + } + } catch { + // Input metering is optional and must not block voice connection. + } + let mediaStream: MediaStream; try { const mediaStreamPromise = this.#dependencies.getUserMedia({ @@ -209,6 +234,13 @@ export class OpenAIRealtimeSession { } microphoneTrack.enabled = false; this.#microphoneTrack = microphoneTrack; + if (audioContext) { + try { + this.#initializeMeter(audioContext, mediaStream); + } catch { + this.#releaseMeterResources(); + } + } const peerConnection = this.#dependencies.createPeerConnection(); this.#peerConnection = peerConnection; @@ -349,7 +381,13 @@ export class OpenAIRealtimeSession { public setMicrophoneEnabled(enabled: boolean): void { if (this.#microphoneTrack) { - this.#microphoneTrack.enabled = enabled && this.#connected; + const isEnabled = enabled && this.#connected; + this.#microphoneTrack.enabled = isEnabled; + if (isEnabled) { + this.#startMeter(); + } else { + this.#stopMeter(); + } } } @@ -416,6 +454,81 @@ export class OpenAIRealtimeSession { }); } + #initializeMeter(audioContext: AudioContext, mediaStream: MediaStream): void { + const analyser = audioContext.createAnalyser(); + analyser.fftSize = 256; + audioContext.createMediaStreamSource(mediaStream).connect(analyser); + this.#analyser = analyser; + this.#meterSamples = new Uint8Array(analyser.fftSize); + } + + #startMeter(): void { + if (this.#meterFrame !== null || !this.#analyser || !this.#meterSamples) { + return; + } + + const sample = () => { + if ( + !this.#microphoneTrack?.enabled || + !this.#analyser || + !this.#meterSamples + ) { + this.#stopMeter(); + return; + } + this.#analyser.getByteTimeDomainData(this.#meterSamples); + let squaredTotal = 0; + for (const value of this.#meterSamples) { + const normalized = (value - 128) / 128; + squaredTotal += normalized * normalized; + } + // The meter only drives a five-bar waveform, so two decimals is all the + // resolution a listener can use. Quantizing and skipping repeats keeps + // an every-animation-frame sample from re-rendering the interview. + const level = + Math.round( + Math.min(1, Math.sqrt(squaredTotal / this.#meterSamples.length)) * + 100, + ) / 100; + if (level !== this.#meterLevel) { + this.#meterLevel = level; + this.#emit({ level, type: "microphone-level" }); + } + this.#meterHasSample = true; + this.#meterFrame = this.#dependencies.requestAnimationFrame(sample); + }; + + this.#meterFrame = this.#dependencies.requestAnimationFrame(sample); + } + + #stopMeter(): void { + if (this.#meterFrame === null) { + return; + } + this.#dependencies.cancelAnimationFrame(this.#meterFrame); + this.#meterFrame = null; + this.#meterLevel = 0; + if (this.#meterHasSample) { + this.#emit({ level: 0, type: "microphone-level" }); + this.#meterHasSample = false; + } + } + + #releaseMeterResources(): void { + this.#stopMeter(); + this.#analyser = null; + this.#meterSamples = null; + const audioContext = this.#audioContext; + this.#audioContext = null; + if (audioContext) { + try { + void audioContext.close().catch(() => undefined); + } catch { + // Input metering cleanup is best-effort. + } + } + } + #handleMessage(event: MessageEvent, connectionEpoch: number): void { const parsed = parseRealtimeEvent(event.data); if (!parsed || typeof parsed.type !== "string") { @@ -545,6 +658,7 @@ export class OpenAIRealtimeSession { this.#connectionRequestId = null; this.#abortController?.abort(); this.#abortController = null; + this.#releaseMeterResources(); if (this.#dataChannel && this.#messageListener) { this.#dataChannel.removeEventListener("message", this.#messageListener); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx index e3ae3e95e2c..8bc0dc29fb9 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx @@ -1,37 +1,212 @@ /** * @vitest-environment jsdom */ -import { act, StrictMode } from "react"; -import { createRoot } from "react-dom/client"; +import { + cleanup, + fireEvent, + render, + screen, + waitFor, +} from "@testing-library/react"; +import { StrictMode, useState } from "react"; import { renderToStaticMarkup } from "react-dom/server"; import { afterEach, describe, expect, test, vi } from "vitest"; +import { OpenAIRealtimeSession } from "./openai-realtime-session"; import { + acknowledgeVoiceInterviewDisclosure, + isVoiceInterviewDisclosureAcknowledged, loadOpenAIVoiceConfig, + VOICE_INTERVIEW_DISCLOSURE_STORAGE_KEY, VoiceInterviewControl, VoiceInterviewControlView, + type VoiceInterviewControlViewProps, } from "./voice-interview-control"; -describe("voice interview control", () => { - afterEach(() => { - vi.unstubAllGlobals(); +import type { VoiceTurnSnapshot } from "./voice-turn-controller"; +import type { PetrinautAiInterviewStageContext } from "@hashintel/petrinaut/ui"; + +const snapshot = { + canReviseLastAnswer: false, + currentQuestion: "What happens after approval?", + errorCode: null, + errorMessage: "", + errorRequestId: "", + lastAnswerDelivery: "none" as const, + lastCommittedText: "", + microphoneEnabled: true, + microphoneLevel: 0.24, + partialText: "The request goes to", + phase: "listening" as const, +}; + +const config = { available: true as const, connectionTimeoutMs: 15_000 }; + +const viewProps = ( + overrides: Partial = {}, +): VoiceInterviewControlViewProps => ({ + consented: true, + correction: "", + coverage: null, + editing: false, + microphoneCheck: "", + onCheckMicrophone: vi.fn(), + onConsentChange: vi.fn(), + onCorrectionChange: vi.fn(), + onDoneSpeaking: vi.fn(), + onEdit: vi.fn(), + onEnd: vi.fn(), + onExpand: vi.fn(), + onInterrupt: vi.fn(), + onMinimize: vi.fn(), + onPause: vi.fn(), + onReconnect: vi.fn(), + onRedo: vi.fn(), + onResume: vi.fn(), + onStart: vi.fn(), + onSubmitCorrection: vi.fn(), + onTypeInstead: vi.fn(), + placement: "sidebar", + presentation: "full", + snapshot, + ...overrides, +}); + +const StatefulVoiceInterviewHarness = ({ + onFocusComposer = vi.fn(), + onOpenSidebar, +}: { + onFocusComposer?: () => void; + onOpenSidebar: () => void; +}) => { + "use no memo"; + + const [active, setActive] = useState(false); + const [interactionMode, setInteractionMode] = + useState("chat"); + const [sidebarOpenRequests, setSidebarOpenRequests] = useState(0); + const context: PetrinautAiInterviewStageContext = { + canAcceptInterviewAnswer: true, + conversationId: "interview-test", + focusComposer: onFocusComposer, + interactionMode, + messages: [], + openSidebar: () => { + onOpenSidebar(); + setSidebarOpenRequests((requests) => requests + 1); + }, + placement: "sidebar", + setActive, + setInteractionMode, + status: "ready", + stop: vi.fn(async () => undefined), + submitInterviewAnswer: vi.fn(async () => ({ + kind: "message" as const, + messageId: "voice-answer", + })), + submitText: vi.fn(async () => ({ + kind: "message" as const, + messageId: "typed-answer", + })), + submitVoiceInput: vi.fn(async () => ({ + kind: "message" as const, + messageId: "voice-answer", + })), + }; + + return ( + <> + + + {active ? "Interview active" : "Interview inactive"} + + {interactionMode === "chat" ? "Chat mode" : "Interview mode"} + + {sidebarOpenRequests} sidebar open requests + + + ); +}; + +const stubUnavailableMicrophone = () => { + const getUserMedia = vi.fn(async () => { + throw new DOMException("Permission denied", "NotAllowedError"); + }); + vi.stubGlobal( + "fetch", + vi.fn(async () => + Response.json({ available: true, connectionTimeoutMs: 15_000 }), + ), + ); + vi.stubGlobal( + "AudioContext", + class { + public readonly state = "suspended"; + public readonly close = vi.fn(async () => undefined); + public readonly resume = vi.fn(async () => undefined); + }, + ); + vi.stubGlobal("navigator", { + mediaDevices: { getUserMedia }, }); + return getUserMedia; +}; + +afterEach(() => { + cleanup(); + vi.unstubAllGlobals(); + window.localStorage.clear(); +}); + +describe("voice interview stage", () => { + test("stores and reads the current disclosure acknowledgement", () => { + const values = new Map(); + const storage = { + getItem: (key: string) => values.get(key) ?? null, + setItem: (key: string, value: string) => values.set(key, value), + }; - test("loads only a schema-valid, available server configuration", async () => { + expect(isVoiceInterviewDisclosureAcknowledged(storage)).toBe(false); + acknowledgeVoiceInterviewDisclosure(storage); + expect(values.get(VOICE_INTERVIEW_DISCLOSURE_STORAGE_KEY)).toBe( + "acknowledged", + ); + expect(isVoiceInterviewDisclosureAcknowledged(storage)).toBe(true); + }); + + test("fails safe when disclosure storage is unavailable", () => { + const unavailableStorage = { + getItem: () => { + throw new DOMException("Blocked", "SecurityError"); + }, + setItem: () => { + throw new DOMException("Blocked", "SecurityError"); + }, + }; + + expect(isVoiceInterviewDisclosureAcknowledged(unavailableStorage)).toBe( + false, + ); + expect(() => + acknowledgeVoiceInterviewDisclosure(unavailableStorage), + ).not.toThrow(); + }); + + test("loads only a schema-valid available server configuration", async () => { const fetch = vi.fn(async () => Response.json({ available: true, connectionTimeoutMs: 15_000 }), ); - await expect(loadOpenAIVoiceConfig(fetch)).resolves.toEqual({ available: true, connectionTimeoutMs: 15_000, }); const [url, request] = fetch.mock.calls[0]!; expect(url).toBe("/api/voice/config"); - expect(request).toMatchObject({ - cache: "no-store", - method: "GET", - }); + expect(request).toMatchObject({ cache: "no-store", method: "GET" }); expect(request?.signal).toBeInstanceOf(AbortSignal); fetch.mockResolvedValueOnce( @@ -44,183 +219,873 @@ describe("voice interview control", () => { await expect(loadOpenAIVoiceConfig(fetch)).resolves.toBeNull(); }); - test("renders an accessible idle voice action and live status", () => { + test("shows disclosure and requires consent before starting", () => { const html = renderToStaticMarkup( , + ); + render( + , ); - expect(html).toContain("Start voice input"); - expect(html).toContain('aria-live="polite"'); - expect(html).toContain("Voice input is off."); + expect(html).toContain("Voice interview"); + expect(html).toContain("Talk through your process with AI"); + expect(html).toContain("transcribed by OpenAI"); + expect(html).toContain("keeps finalized answers"); + expect(html).toContain("not the audio"); + expect(html.indexOf("Start interview")).toBeLessThan( + html.indexOf("Check microphone"), + ); + expect(html).toMatch(/]*disabled[^>]*>Start interview/u); + expect(html).toContain('aria-label="Use text instead"'); + expect(html).toContain("Check microphone"); + expect(html).toContain("pos_absolute"); + expect(html).not.toContain("pos_fixed"); + + const textButton = screen.getByRole("button", { name: "Use text instead" }); + expect(textButton.querySelector("svg")).not.toBeNull(); + expect(textButton.parentElement?.getAttribute("data-scope")).toBe( + "tooltip", + ); + expect(textButton.textContent.replaceAll("\u200B", "").trim()).toBe(""); }); - test("labels the half-duplex listening state and keeps partial text visibly provisional", () => { + test("keeps diagnostic recovery details visible without reopening the microphone", () => { const html = renderToStaticMarkup( , - ); - - expect(html).toContain("Microphone on. Listening."); - expect(html).toContain("Live transcript (not sent)"); - expect(html).toContain("The next activity"); + {...viewProps({ + snapshot: { + ...snapshot, + errorCode: "microphone-permission", + errorMessage: + "Allow microphone access in your browser settings, then reconnect voice input.", + errorRequestId: "voice-request-permission", + microphoneEnabled: false, + microphoneLevel: 0, + partialText: "", + phase: "recoverable-error", + }, + })} + />, + ); + + expect(html).toContain("We couldn’t reconnect the microphone"); expect(html).toContain( - "Microphone on. Listening. Live transcript (not sent): The next activity", + "Allow microphone access in your browser settings, then reconnect voice input.", + ); + expect(html).toContain("Technical details"); + expect(html).toContain("microphone-permission"); + expect(html).toContain("voice-request-permission"); + expect(html).toContain(">Reconnect<"); + expect(html).toContain('aria-label="Use text instead"'); + expect(html).not.toContain(">Type instead<"); + }); + + test("starts in the full stage and keeps recovery visible under Strict Mode", async () => { + const getUserMedia = stubUnavailableMicrophone(); + const openSidebar = vi.fn(); + + render( + + + , + ); + + fireEvent.click(screen.getByRole("button", { name: "Select Interview" })); + fireEvent.click(screen.getByRole("checkbox")); + fireEvent.click(screen.getByRole("button", { name: "Start interview" })); + + expect( + await screen.findByRole("region", { name: "Voice interview stage" }), + ).not.toBeNull(); + expect( + await screen.findByText( + /Microphone off · Allow microphone access in your browser settings, then reconnect voice input\./u, + ), + ).not.toBeNull(); + expect(screen.getByRole("button", { name: "Reconnect" })).not.toBeNull(); + expect(screen.getByText("Interview active")).not.toBeNull(); + expect(screen.getByText("Interview mode")).not.toBeNull(); + expect(screen.getByText("1 sidebar open requests")).not.toBeNull(); + expect(openSidebar).toHaveBeenCalledOnce(); + expect(getUserMedia).toHaveBeenCalledOnce(); + + fireEvent.click(screen.getByRole("button", { name: "Select Chat" })); + expect( + screen.getByRole("region", { name: "Voice interview mini bar" }), + ).not.toBeNull(); + expect(screen.getByText("Chat mode")).not.toBeNull(); + expect(openSidebar).toHaveBeenCalledOnce(); + }); + + test("uses full Interview and compact Chat presentations without ending", async () => { + window.localStorage.setItem( + VOICE_INTERVIEW_DISCLOSURE_STORAGE_KEY, + "acknowledged", + ); + const getUserMedia = stubUnavailableMicrophone(); + render(); + + fireEvent.click(screen.getByRole("button", { name: "Select Interview" })); + expect( + await screen.findByRole("region", { name: "Voice interview stage" }), + ).not.toBeNull(); + + fireEvent.click( + screen.getByRole("button", { name: "Minimize voice interview" }), + ); + expect( + screen.getByRole("region", { name: "Voice interview mini bar" }), + ).not.toBeNull(); + expect(screen.getByText("Interview active")).not.toBeNull(); + + fireEvent.click( + screen.getByRole("button", { name: /Expand voice interview/u }), ); - expect(html).toContain("End voice input"); - expect(html).toContain("Correct last voice answer"); - expect(html).toContain("Send correction"); + expect( + screen.getByRole("region", { name: "Voice interview stage" }), + ).not.toBeNull(); + expect(screen.getByText("Interview mode")).not.toBeNull(); + expect(getUserMedia).toHaveBeenCalledOnce(); }); - test("offers reconnection without reopening the microphone after failure", () => { + test("ends the interview and returns to Chat", async () => { + window.localStorage.setItem( + VOICE_INTERVIEW_DISCLOSURE_STORAGE_KEY, + "acknowledged", + ); + stubUnavailableMicrophone(); + render(); + + fireEvent.click(screen.getByRole("button", { name: "Select Interview" })); + expect( + await screen.findByRole("region", { name: "Voice interview stage" }), + ).not.toBeNull(); + + fireEvent.click(screen.getByRole("button", { name: "End interview" })); + + expect(screen.getByText("Chat mode")).not.toBeNull(); + expect(screen.getByText("Interview inactive")).not.toBeNull(); + await waitFor(() => { + expect( + screen.queryByRole("region", { name: "Voice interview stage" }), + ).toBeNull(); + expect( + screen.queryByRole("region", { name: "Voice interview mini bar" }), + ).toBeNull(); + }); + }); + + test("restarts when Interview is reselected before teardown completes", async () => { + window.localStorage.setItem( + VOICE_INTERVIEW_DISCLOSURE_STORAGE_KEY, + "acknowledged", + ); + const connect = vi + .spyOn(OpenAIRealtimeSession.prototype, "connect") + .mockResolvedValue(1); + let finishDisconnect: (() => void) | undefined; + vi.spyOn(OpenAIRealtimeSession.prototype, "disconnect") + .mockImplementationOnce( + () => + new Promise((resolve) => { + finishDisconnect = resolve; + }), + ) + .mockResolvedValue(undefined); + vi.spyOn( + OpenAIRealtimeSession.prototype, + "setMicrophoneEnabled", + ).mockImplementation(() => {}); + render(); + + fireEvent.click(screen.getByRole("button", { name: "Select Interview" })); + await screen.findByText("Listening"); + fireEvent.click(screen.getByRole("button", { name: "End interview" })); + + expect(screen.getByText("Chat mode")).not.toBeNull(); + expect(screen.getByText("Interview inactive")).not.toBeNull(); + expect( + screen.queryByRole("region", { name: "Voice interview mini bar" }), + ).toBeNull(); + + fireEvent.click(screen.getByRole("button", { name: "Select Interview" })); + + expect(connect).toHaveBeenCalledOnce(); + expect(screen.getByText("Interview inactive")).not.toBeNull(); + finishDisconnect?.(); + + await waitFor(() => expect(connect).toHaveBeenCalledTimes(2)); + expect(screen.getByText("Interview active")).not.toBeNull(); + expect(screen.getByText("Listening")).not.toBeNull(); + }); + + test("records acknowledgement only when the interview starts", async () => { + window.localStorage.clear(); + stubUnavailableMicrophone(); + render(); + + fireEvent.click(screen.getByRole("button", { name: "Select Interview" })); + fireEvent.click(screen.getByRole("button", { name: "Check microphone" })); + expect( + window.localStorage.getItem(VOICE_INTERVIEW_DISCLOSURE_STORAGE_KEY), + ).toBeNull(); + + fireEvent.click(screen.getByRole("checkbox")); + fireEvent.click(screen.getByRole("button", { name: "Start interview" })); + expect( + window.localStorage.getItem(VOICE_INTERVIEW_DISCLOSURE_STORAGE_KEY), + ).toBe("acknowledged"); + }); + + test("does not record acknowledgement when choosing text instead", async () => { + window.localStorage.clear(); + const focusComposer = vi.fn(); + vi.stubGlobal( + "fetch", + vi.fn(async () => + Response.json({ available: true, connectionTimeoutMs: 15_000 }), + ), + ); + render( + , + ); + + fireEvent.click(screen.getByRole("button", { name: "Select Interview" })); + fireEvent.click(screen.getByRole("button", { name: "Use text instead" })); + expect( + window.localStorage.getItem(VOICE_INTERVIEW_DISCLOSURE_STORAGE_KEY), + ).toBeNull(); + expect(screen.getByText("Chat mode")).not.toBeNull(); + expect(focusComposer).toHaveBeenCalledOnce(); + }); + + test("uses text from an active interview without ending the session", async () => { + window.localStorage.setItem( + VOICE_INTERVIEW_DISCLOSURE_STORAGE_KEY, + "acknowledged", + ); + const focusComposer = vi.fn(); + stubUnavailableMicrophone(); + render( + , + ); + + fireEvent.click(screen.getByRole("button", { name: "Select Interview" })); + expect( + await screen.findByRole("region", { name: "Voice interview stage" }), + ).not.toBeNull(); + + fireEvent.click(screen.getByRole("button", { name: "Use text instead" })); + + expect(screen.getByText("Chat mode")).not.toBeNull(); + expect(screen.getByText("Interview active")).not.toBeNull(); + expect( + screen.getByRole("region", { name: "Voice interview mini bar" }), + ).not.toBeNull(); + expect(focusComposer).toHaveBeenCalledOnce(); + }); + + test("skips the disclosure after it has been acknowledged", async () => { + stubUnavailableMicrophone(); + window.localStorage.setItem( + VOICE_INTERVIEW_DISCLOSURE_STORAGE_KEY, + "acknowledged", + ); + render(); + + fireEvent.click(screen.getByRole("button", { name: "Select Interview" })); + + expect( + screen.queryByRole("region", { name: "Start voice interview" }), + ).toBeNull(); + expect( + await screen.findByRole("region", { name: "Voice interview stage" }), + ).not.toBeNull(); + }); + + test("keeps the question visible and names microphone level", () => { const html = renderToStaticMarkup( + , + ); + + expect(html).toContain("What happens after approval?"); + expect(html).toContain("Live transcript"); + expect(html).toContain("Listening"); + expect(html).toContain("Microphone input level: Medium"); + expect(html).toContain('aria-label="Done speaking"'); + expect(html).toContain("motionReduce:vis_hidden"); + expect(html).toContain("pos_relative"); + expect(html).not.toContain("pos_fixed"); + expect(html).toContain('aria-live="polite"'); + }); + + test("centers a circular microphone and waveform without visible level copy", () => { + render(); + + expect(screen.getByTestId("voice-microphone-focal")).not.toBeNull(); + expect(screen.getByTestId("voice-waveform")).not.toBeNull(); + expect(screen.getByText("Listening")).not.toBeNull(); + + const accessibleLevel = screen.getByText("Microphone input level: Medium"); + expect(accessibleLevel.className).toContain("pos_absolute"); + expect( + screen.queryByText("Microphone on · Listening", { + selector: ":not([role='status'])", + }), + ).toBeNull(); + }); + + test("shows compact recording and sent transcript statuses", () => { + const rendered = render(); + + expect(screen.getByText("Live transcript")).not.toBeNull(); + expect(screen.getByText("Recording")).not.toBeNull(); + expect(screen.queryByText("What we’re hearing · Not sent yet")).toBeNull(); + expect(screen.getByRole("status").textContent).toContain("Not sent yet"); + + rendered.rerender( , ); - expect(html).toContain( - "Microphone off. Allow microphone access in your browser settings, then reconnect voice input.", + expect(screen.getByText("Your answer")).not.toBeNull(); + expect(screen.getByText("Sent")).not.toBeNull(); + expect(screen.getByText("The shift lead assigns an owner.")).not.toBeNull(); + }); + + test("shows a sending status while the answer is still being delivered", () => { + render( + , ); - expect(html).toContain("Error code: microphone-permission."); - expect(html).toContain("Diagnostic reference: voice-request-permission."); - expect(html).toContain("Reconnect voice input"); + + expect(screen.getByText("Your answer")).not.toBeNull(); + expect(screen.getByText("Sending")).not.toBeNull(); + expect(screen.queryByText("Sent")).toBeNull(); }); - test("remains interactive after Strict Mode replays its effects", async () => { - const fetch = vi.fn(async () => - Response.json({ available: true, connectionTimeoutMs: 15_000 }), + test("shows a not-sent status when delivery failed", () => { + render( + , ); - const getUserMedia = vi.fn(async () => { - throw new DOMException("Permission denied", "NotAllowedError"); - }); - vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true); - vi.stubGlobal("fetch", fetch); - vi.stubGlobal("navigator", { mediaDevices: { getUserMedia } }); - - const container = document.createElement("div"); - document.body.append(container); - const root = createRoot(container); - - try { - await act(async () => { - root.render( - - undefined)} - submitText={vi.fn(async () => ({ - kind: "message" as const, - messageId: "message-1", - }))} - submitVoiceInput={vi.fn(async () => ({ - kind: "message" as const, - messageId: "voice-message-1", - }))} - /> - , - ); - }); - - const startButton = container.querySelector( - 'button[aria-label="Start voice input"]', - ); - expect(startButton).not.toBeNull(); - await act(async () => { - startButton!.click(); - }); + expect(screen.getByText("Not sent")).not.toBeNull(); + expect(screen.queryByText("Sent")).toBeNull(); + expect(screen.getByText("The shift lead assigns an owner.")).not.toBeNull(); + }); + + test("uses voice-app icon controls while listening", () => { + render(); + + for (const name of ["Use text instead", "Done speaking", "Pause"]) { + const button = screen.getByRole("button", { name }); + expect(button.querySelector("svg")).not.toBeNull(); + expect(button.parentElement?.getAttribute("data-scope")).toBe("tooltip"); + } + + expect( + screen + .getByRole("button", { name: "Done speaking" }) + .textContent.replaceAll("\u200B", "") + .trim(), + ).toBe(""); + }); + + test("orders the full listening actions as keyboard, done speaking, then pause", () => { + const html = renderToStaticMarkup( + , + ); + + expect(html.indexOf('aria-label="Use text instead"')).toBeLessThan( + html.indexOf('aria-label="Done speaking"'), + ); + expect(html.indexOf('aria-label="Done speaking"')).toBeLessThan( + html.indexOf('aria-label="Pause"'), + ); + }); + + test("offers only resume and keyboard actions while paused", () => { + render( + , + ); + + expect(screen.getByText("Paused")).not.toBeNull(); + expect( + screen.getByRole("button", { name: "Resume listening" }), + ).not.toBeNull(); + expect( + screen.getByRole("button", { name: "Use text instead" }), + ).not.toBeNull(); + expect(screen.queryByRole("button", { name: "Done speaking" })).toBeNull(); + expect(screen.queryByRole("button", { name: "Pause" })).toBeNull(); + expect( + screen.queryByRole("button", { name: "Interrupt and speak" }), + ).toBeNull(); + }); + + test("shows the waveform only while the microphone is listening", () => { + const rendered = render(); - expect(getUserMedia).toHaveBeenCalledOnce(); - expect(container.textContent).toContain( - "Allow microphone access in your browser settings, then reconnect voice input.", + expect(screen.getByTestId("voice-waveform")).not.toBeNull(); + + for (const phase of ["paused", "playing", "waiting"] as const) { + rendered.rerender( + , ); + expect(screen.queryByTestId("voice-waveform")).toBeNull(); + expect(screen.queryByText(/Microphone input level:/u)).toBeNull(); + } + }); + + test("keeps reconnect visible and makes secondary recovery icon-only", () => { + render( + , + ); + + expect( + screen.getByText("We couldn’t reconnect the microphone"), + ).not.toBeNull(); + expect( + screen.getByRole("button", { name: "Reconnect" }).textContent, + ).toContain("Reconnect"); + expect( + screen + .getByRole("button", { name: "Use text instead" }) + .querySelector("svg"), + ).not.toBeNull(); + expect(screen.getByText("Technical details")).not.toBeNull(); + }); + + test("names the recovery problem for each error family", () => { + const recovery = ( + errorCode: VoiceTurnSnapshot["errorCode"], + errorMessage: string, + ) => ( + + ); + + const rendered = render( + recovery( + "microphone-device", + "Connect or select a microphone, then reconnect voice input.", + ), + ); + expect( + screen.getByText("We couldn’t reconnect the microphone"), + ).not.toBeNull(); + expect(screen.getByText("Microphone unavailable")).not.toBeNull(); + expect( + screen.getByText( + "Connect or select a microphone, then reconnect voice input.", + ), + ).not.toBeNull(); + + rendered.rerender( + recovery( + "timeout", + "The voice connection timed out. Check your connection, then reconnect voice input.", + ), + ); + expect(screen.getByText("We lost the voice connection")).not.toBeNull(); + expect(screen.getByText("Connection paused")).not.toBeNull(); + + rendered.rerender( + recovery( + "invalid-response", + "The interview could not accept that answer. Use the composer to retry.", + ), + ); + expect(screen.getByText("The interview couldn’t continue")).not.toBeNull(); + expect(screen.getByText("Interview paused")).not.toBeNull(); + + rendered.rerender( + recovery( + null, + "The interview could not accept that answer. Use the composer to retry.", + ), + ); + expect(screen.getByText("The interview couldn’t continue")).not.toBeNull(); + expect(screen.getByText("Interview paused")).not.toBeNull(); + expect( + screen.queryByText("We couldn’t reconnect the microphone"), + ).toBeNull(); + }); + + test("renders icons for the listening controls", () => { + render(); + + for (const name of [ + "Minimize voice interview", + "End interview", + "Done speaking", + "Pause", + ]) { expect( - container.querySelector('button[aria-label="Reconnect voice input"]'), + screen.getByRole("button", { name }).querySelector("svg"), ).not.toBeNull(); - } finally { - await act(async () => root.unmount()); - container.remove(); } }); - test("announces synthesis, playback, and the AI-generated voice disclosure", () => { - const renderPhase = (phase: "synthesizing" | "playing") => - renderToStaticMarkup( - { + const waitingHtml = renderToStaticMarkup( + , - ); + phase: "waiting", + }, + })} + />, + ); - const synthesizing = renderPhase("synthesizing"); - expect(synthesizing).toContain( - "Microphone off. Creating AI-generated speech.", + expect(waitingHtml).not.toContain( + "Microphone input level unavailable while microphone is off", ); - expect(synthesizing).toContain( - "Spoken responses use an AI-generated OpenAI voice.", + }); + + test("renders committed repair actions separately from pause, minimize, and end", () => { + const html = renderToStaticMarkup( + , ); - const playing = renderPhase("playing"); - expect(playing).toContain("Microphone off. Playing AI-generated speech."); - expect(playing).toContain( - "Spoken responses use an AI-generated OpenAI voice.", + for (const name of [ + "Minimize voice interview", + "End interview", + "Redo answer", + "Edit text", + "Use text instead", + ]) { + expect(html).toContain(name); + } + }); + + test("enables repair actions only while the last answer can be revised", () => { + const rendered = render( + , + ); + + expect( + screen + .getByRole("button", { name: "Redo answer" }) + .hasAttribute("disabled"), + ).toBe(true); + expect( + screen + .getByRole("button", { name: "Edit text" }) + .hasAttribute("disabled"), + ).toBe(true); + + rendered.rerender( + , + ); + + expect( + screen + .getByRole("button", { name: "Redo answer" }) + .hasAttribute("disabled"), + ).toBe(false); + expect( + screen + .getByRole("button", { name: "Edit text" }) + .hasAttribute("disabled"), + ).toBe(false); + }); + + test("offers deterministic interrupt instead of listening during playback", () => { + const html = renderToStaticMarkup( + , + ); + + expect(html).toContain("Microphone off · Interviewer speaking"); + expect(html).toContain('aria-label="Interrupt and speak"'); + expect(html).not.toContain(">Pause<"); + }); + + test("uses a detached bottom mini bar with independent expand, type, pause, and end controls", () => { + render( + , + ); + + expect( + screen.getByRole("region", { name: "Voice interview mini bar" }), + ).not.toBeNull(); + expect( + screen.getByRole("button", { + name: "Expand voice interview. Microphone on · Listening. Question: What happens after approval?", + }), + ).not.toBeNull(); + expect(screen.getByText("Listening")).not.toBeNull(); + expect(screen.queryByText("Microphone on · Listening")).toBeNull(); + expect(screen.getByText("What happens after approval?")).not.toBeNull(); + expect( + screen.getByRole("button", { name: "Done speaking" }), + ).not.toBeNull(); + expect(screen.getByRole("button", { name: "Pause" })).not.toBeNull(); + expect( + screen.getByRole("button", { name: "Use text instead" }), + ).not.toBeNull(); + expect( + screen.getByRole("button", { name: "End interview" }), + ).not.toBeNull(); + + for (const name of [ + "Done speaking", + "Pause", + "Use text instead", + "End interview", + ]) { + const button = screen.getByRole("button", { name }); + expect(button.querySelector("svg")).not.toBeNull(); + expect(button.parentElement?.getAttribute("data-scope")).toBe("tooltip"); + } + + const html = renderToStaticMarkup( + , + ); + expect(html).toContain("--voice-interview-right"); + expect(html).toContain("[@media_(min-width:_768px)]"); + expect(html).not.toContain("md:right_4"); + }); + + test("shows only the valid compact phase action", () => { + const rendered = render( + , + ); + + expect( + screen.getByRole("button", { name: "Resume listening" }), + ).not.toBeNull(); + expect(screen.queryByRole("button", { name: "Done speaking" })).toBeNull(); + expect( + screen + .getByRole("button", { name: "Resume listening" }) + .querySelector("svg"), + ).not.toBeNull(); + expect( + screen + .getByRole("button", { name: "Resume listening" }) + .parentElement?.getAttribute("data-scope"), + ).toBe("tooltip"); + + rendered.rerender( + , + ); + expect( + screen.getByRole("button", { name: "Interrupt and speak" }), + ).not.toBeNull(); + expect(screen.queryByRole("button", { name: "Pause" })).toBeNull(); + expect( + screen + .getByRole("button", { name: "Interrupt and speak" }) + .querySelector("svg"), + ).not.toBeNull(); + expect( + screen + .getByRole("button", { name: "Interrupt and speak" }) + .parentElement?.getAttribute("data-scope"), + ).toBe("tooltip"); + }); + + test("announces compact question and provisional transcript context", () => { + render( + , + ); + + expect( + screen.getByRole("button", { + name: "Expand voice interview. Microphone on · Listening. Question: What happens after approval?", + }), + ).not.toBeNull(); + expect(screen.getByRole("status").textContent).toBe( + "Microphone on · Listening. Question: What happens after approval? Not sent yet: The request goes to", + ); + }); + + test("shows authoritative covered and still-exploring facts without a question count", () => { + const html = renderToStaticMarkup( + , ); + + expect(html).toContain("Covered"); + expect(html).toContain("Still exploring"); + expect(html).not.toMatch(/\d+ of \d+/u); + }); + + test("keeps interview coverage as a low-emphasis details row", () => { + const html = renderToStaticMarkup( + , + ); + + expect(html).toMatch(/
{ + if (typeof window === "undefined") { + return null; + } + try { + return window.localStorage; + } catch { + return null; + } +}; + +export const isVoiceInterviewDisclosureAcknowledged = ( + storage: Pick< + Storage, + "getItem" + > | null = getVoiceInterviewDisclosureStorage(), +): boolean => { + try { + return ( + storage?.getItem(VOICE_INTERVIEW_DISCLOSURE_STORAGE_KEY) === + VOICE_INTERVIEW_DISCLOSURE_ACKNOWLEDGED + ); + } catch { + return false; + } +}; + +export const acknowledgeVoiceInterviewDisclosure = ( + storage: Pick< + Storage, + "setItem" + > | null = getVoiceInterviewDisclosureStorage(), +): void => { + try { + storage?.setItem( + VOICE_INTERVIEW_DISCLOSURE_STORAGE_KEY, + VOICE_INTERVIEW_DISCLOSURE_ACKNOWLEDGED, + ); + } catch { + // Storage is optional; the disclosure will appear again next time. + } +}; + +type Presentation = "start" | "full" | "mini"; + const isRecord = (value: unknown): value is Record => typeof value === "object" && value !== null; @@ -39,10 +113,7 @@ export const loadOpenAIVoiceConfig = async ( method: "GET", signal, }); - if (!response.ok) { - return null; - } - + if (!response.ok) return null; const body: unknown = await response.json(); if ( !isRecord(body) || @@ -62,77 +133,307 @@ export const loadOpenAIVoiceConfig = async ( } }; -const controlStyle = css({ - position: "relative", - flexShrink: "0", +const rootStyle = cva({ + base: { + zIndex: "overlay", + pointerEvents: "auto", + }, + variants: { + presentation: { + start: { + position: "absolute", + right: "0", + bottom: "[-2px]", + width: "full", + }, + full: { + position: "relative", + width: "full", + }, + mini: { + position: "relative", + width: "full", + }, + detached: { + position: "fixed", + "--voice-interview-right": "0px", + "--voice-interview-bottom": "0px", + "--voice-interview-left": "0px", + "--voice-interview-width": "100%", + right: "[var(--voice-interview-right)]", + bottom: "[var(--voice-interview-bottom)]", + left: "[var(--voice-interview-left)]", + width: "[var(--voice-interview-width)]", + "@media (min-width: 768px)": { + "--voice-interview-right": "var(--spacing-4)", + "--voice-interview-bottom": "var(--spacing-4)", + "--voice-interview-left": "auto", + "--voice-interview-width": "440px", + }, + }, + }, + }, }); -const panelStyle = css({ - position: "absolute", - right: "0", - bottom: "[calc(100% + 8px)]", - zIndex: "overlay", +const cardStyle = css({ display: "flex", - width: "[280px]", flexDirection: "column", - gap: "2", + gap: "3", + padding: "4", + borderWidth: "thin", + borderStyle: "solid", + borderColor: "neutral.a20", + borderTopLeftRadius: "xl", + borderTopRightRadius: "xl", + borderBottomRightRadius: "xl", + borderBottomLeftRadius: "xl", + backgroundColor: "neutral.s00", + boxShadow: "xl", +}); + +const stageStyle = css({ + display: "flex", + maxHeight: "[72vh]", + flexDirection: "column", + gap: "3", padding: "3", + overflowY: "auto", borderWidth: "thin", borderStyle: "solid", borderColor: "neutral.a20", - borderRadius: "lg", backgroundColor: "neutral.s00", - boxShadow: "lg", + boxShadow: "[0 -8px 24px rgba(0,0,0,0.06)]", + borderRadius: "lg", }); -const statusStyle = css({ - color: "neutral.s90", - fontSize: "xs", - fontWeight: "medium", - lineHeight: "relaxed", +const headerStyle = css({ + display: "flex", + alignItems: "center", + justifyContent: "flex-end", + gap: "2", +}); + +const startHeaderStyle = css({ + display: "flex", + alignItems: "center", + gap: "2", +}); + +const titleStyle = css({ + flex: "1", + color: "neutral.s100", + fontSize: "sm", + fontWeight: "semibold", }); -const disclosureStyle = css({ - color: "neutral.s70", +const subtitleStyle = css({ + color: "neutral.s80", fontSize: "xs", - lineHeight: "relaxed", + lineHeight: "snug", }); -const liveRegionStyle = css({ - position: "absolute", - width: "[1px]", - height: "[1px]", - padding: "0", - margin: "[-1px]", +const questionStyle = css({ + color: "neutral.s110", + fontSize: "lg", + fontWeight: "semibold", + lineHeight: "snug", +}); + +const contextStyle = css({ + display: "block", overflow: "hidden", - clip: "[rect(0, 0, 0, 0)]", + color: "neutral.s80", + fontSize: "xs", + lineHeight: "snug", + textOverflow: "ellipsis", whiteSpace: "nowrap", - borderWidth: "0", }); -const partialStyle = css({ +const miniTextStyle = css({ + display: "flex", + minWidth: "0", + flexDirection: "column", +}); + +const focalAreaStyle = css({ display: "flex", + minHeight: "[150px]", flexDirection: "column", + alignItems: "center", + justifyContent: "center", + gap: "3", +}); + +const focalCircleStyle = cva({ + base: { + position: "relative", + display: "flex", + width: "[108px]", + height: "[108px]", + flexDirection: "column", + alignItems: "center", + justifyContent: "center", + gap: "2", + borderWidth: "thin", + borderStyle: "solid", + borderRadius: "full", + _before: { + content: '""', + position: "absolute", + inset: "[-10px]", + borderWidth: "thin", + borderStyle: "solid", + borderColor: "blue.a20", + borderRadius: "full", + }, + }, + variants: { + tone: { + active: { + borderColor: "blue.a30", + backgroundColor: "blue.a10", + color: "blue.s90", + boxShadow: "[0 14px 30px rgba(35,125,181,0.14)]", + }, + idle: { + borderColor: "neutral.a20", + backgroundColor: "neutral.s20", + color: "neutral.s80", + }, + success: { + borderColor: "green.a30", + backgroundColor: "green.a10", + color: "green.s90", + }, + error: { + borderColor: "red.a30", + backgroundColor: "red.a10", + color: "red.s90", + }, + }, + }, +}); + +const shortStateStyle = css({ + display: "flex", + alignItems: "center", + gap: "2", + color: "neutral.s90", + fontSize: "xs", + fontWeight: "semibold", +}); + +const recordingDotStyle = css({ + width: "[7px]", + height: "[7px]", + borderRadius: "full", + backgroundColor: "green.s70", + boxShadow: "[0 0 0 4px rgba(24,168,120,0.10)]", +}); + +const transcriptHeaderStyle = css({ + display: "flex", + alignItems: "center", + justifyContent: "space-between", + gap: "2", +}); + +const recordingTranscriptDotStyle = css({ + width: "[7px]", + height: "[7px]", + borderRadius: "full", + backgroundColor: "red.s70", +}); + +const transcriptActionsStyle = css({ + display: "flex", + justifyContent: "flex-end", gap: "1", - padding: "2", - borderRadius: "md", +}); + +const transcriptStateStyle = cva({ + base: { + display: "flex", + alignItems: "center", + gap: "1", + fontSize: "xs", + fontWeight: "semibold", + }, + variants: { + state: { + recording: { color: "red.s80" }, + sending: { color: "neutral.s80" }, + sent: { color: "green.s90" }, + unsent: { color: "red.s90" }, + }, + }, +}); + +const meterStyle = css({ + display: "flex", + height: "[34px]", + alignItems: "center", + gap: "1", + _motionReduce: { visibility: "hidden" }, +}); + +const meterBarStyle = css({ + width: "[5px]", + minHeight: "[4px]", + borderRadius: "full", + backgroundColor: "blue.s70", + transition: "[height 80ms linear]", + _motionReduce: { transition: "[none]" }, +}); + +const statusStyle = css({ + color: "neutral.s90", + fontSize: "sm", + fontWeight: "medium", +}); + +const transcriptStyle = css({ + display: "flex", + flexDirection: "column", + gap: "1", + padding: "2.5", + borderRadius: "lg", backgroundColor: "neutral.s10", color: "neutral.s100", fontSize: "sm", }); -const partialLabelStyle = css({ +const labelStyle = css({ color: "neutral.s80", fontSize: "xs", + fontWeight: "semibold", }); -const correctionFormStyle = css({ +const recoveryStyle = css({ display: "flex", flexDirection: "column", + gap: "1", + padding: "2.5", + borderRadius: "lg", + backgroundColor: "red.a10", + color: "neutral.s100", + fontSize: "sm", +}); + +const secondaryDetailsStyle = css({ + color: "neutral.s80", + fontSize: "xs", + _open: { color: "neutral.s90" }, +}); + +const actionsStyle = css({ + display: "flex", + flexWrap: "wrap", + alignItems: "center", gap: "2", }); -const correctionInputStyle = css({ +const inputStyle = css({ width: "full", paddingX: "2", paddingY: "1.5", @@ -144,75 +445,593 @@ const correctionInputStyle = css({ color: "neutral.s100", fontSize: "sm", _focusVisible: { - borderColor: "blue.a70", outline: "2px solid", - outlineColor: "blue.a30", + outlineColor: "blue.a40", outlineOffset: "[1px]", }, }); -const panelActionsStyle = css({ +const miniStyle = css({ display: "flex", - justifyContent: "flex-end", + minHeight: "[60px]", + alignItems: "center", + gap: "2", + padding: "2", + borderWidth: "thin", + borderStyle: "solid", + borderColor: "neutral.a20", + borderTopLeftRadius: "lg", + borderTopRightRadius: "lg", + borderBottomRightRadius: "lg", + borderBottomLeftRadius: "lg", + backgroundColor: "neutral.s00", + boxShadow: "lg", +}); + +const miniExpandStyle = css({ + display: "flex", + minWidth: "0", + flex: "1", + alignItems: "center", gap: "2", + padding: "2", + color: "neutral.s100", + textAlign: "left", + background: "[transparent]", + border: "none", + cursor: "pointer", + _focusVisible: { outline: "2px solid", outlineColor: "blue.a50" }, +}); + +const liveRegionStyle = css({ + position: "absolute", + width: "[1px]", + height: "[1px]", + padding: "0", + margin: "[-1px]", + overflow: "hidden", + clip: "[rect(0,0,0,0)]", + whiteSpace: "nowrap", + borderWidth: "0", }); const statusText = (snapshot: VoiceTurnSnapshot): string => { switch (snapshot.phase) { case "idle": - return "Voice input is off."; + return "Microphone off · Interview not started"; case "connecting": - return "Microphone off. Connecting voice input."; + return "Microphone off · Joining the interview"; case "listening": - return "Microphone on. Listening."; + return "Microphone on · Listening"; + case "paused": + return "Microphone off · Paused"; case "transcribing": - return "Microphone off. Finalizing the transcript."; + return "Microphone off · Finishing your answer"; case "delivering": - return "Microphone off. Sending the finalized transcript to Brunch."; + return "Microphone off · Answer recorded"; case "waiting": - return "Microphone off. Waiting for Brunch."; + return "Microphone off · Writing that down"; case "synthesizing": - return "Microphone off. Creating AI-generated speech."; + return "Microphone off · Preparing the next question"; case "playing": - return "Microphone off. Playing AI-generated speech."; + return "Microphone off · Interviewer speaking"; case "recoverable-error": { - const diagnostic = - snapshot.errorCode === null - ? "" - : ` Error code: ${snapshot.errorCode}.${ - snapshot.errorRequestId - ? ` Diagnostic reference: ${snapshot.errorRequestId}.` - : "" - }`; - return `Microphone off. ${snapshot.errorMessage}${diagnostic}`; + return `Microphone off · ${snapshot.errorMessage}`; + } + } +}; + +type RecoveryErrorFamily = "connection" | "interview" | "microphone"; + +const recoveryErrorFamily = ( + errorCode: VoiceTurnSnapshot["errorCode"], +): RecoveryErrorFamily => { + switch (errorCode) { + case "microphone-permission": + case "microphone-device": + return "microphone"; + case "network": + case "timeout": + case "request-aborted": + return "connection"; + default: + return "interview"; + } +}; + +const shortStatusText = (snapshot: VoiceTurnSnapshot): string => { + switch (snapshot.phase) { + case "idle": + return "Ready"; + case "connecting": + return "Connecting"; + case "listening": + return "Listening"; + case "paused": + return "Paused"; + case "transcribing": + return "Finishing answer"; + case "delivering": + return "Answer recorded"; + case "waiting": + return "Writing that down"; + case "synthesizing": + return "Preparing next question"; + case "playing": + return "Interviewer speaking"; + case "recoverable-error": { + switch (recoveryErrorFamily(snapshot.errorCode)) { + case "microphone": + return "Microphone unavailable"; + case "connection": + return "Connection paused"; + case "interview": + return "Interview paused"; + } } } }; -interface VoiceInterviewControlViewProps { +const inputLevelText = (level: number): string => + level >= 0.35 + ? "High" + : level >= 0.12 + ? "Medium" + : level > 0 + ? "Low" + : "Quiet"; + +const Meter = ({ snapshot }: { snapshot: VoiceTurnSnapshot }) => { + const level = snapshot.microphoneLevel; + return ( + <> + + + {`Microphone input level: ${inputLevelText(level)}`} + + + ); +}; + +const focalIcon = (phase: VoiceTurnPhase): ReactNode => { + switch (phase) { + case "listening": + return