From 9d0d35d45b3ac05c8df1459f2441c4c497e90218 Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Thu, 27 Aug 2026 00:07:35 +0200 Subject: [PATCH 1/8] Add OpenAI Realtime voice input Initialize transcription-only WebRTC sessions through an app-owned, fail-closed server boundary. Admit only stable finalized transcript items through the existing Petrinaut composer and Brunch transport, with half-duplex lifecycle, stale-event rejection, correction handling, preview gating, and accessible controls. Amp-Thread-ID: https://ampcode.com/threads/T-01a03fb3-fd3d-737f-b4c6-1fc9282950bf Co-authored-by: Amp --- apps/petrinaut-website/.env.example | 3 + apps/petrinaut-website/README.md | 45 +- apps/petrinaut-website/api/voice/config.ts | 9 + .../api/voice/realtime-call.ts | 12 + .../brunch-ask-interactive-tool.test.ts | 14 + .../brunch-ask-interactive-tool.tsx | 151 +++++++ .../local-storage-demo/brunch-ask-mapping.ts | 11 + .../brunch-preview-config.test.ts | 33 ++ .../brunch-preview-config.ts | 9 + .../local-storage-demo-app.test.tsx | 41 ++ .../local-storage-demo-app.tsx | 32 +- .../openai-realtime-session.test.ts | 298 +++++++++++++ .../openai-realtime-session.ts | 388 +++++++++++++++++ .../voice-interview-control.test.tsx | 110 +++++ .../voice-interview-control.tsx | 389 +++++++++++++++++ .../voice-turn-controller.test.ts | 401 ++++++++++++++++++ .../voice-interview/voice-turn-controller.ts | 374 ++++++++++++++++ .../server/voice/openai-realtime-call.test.ts | 223 ++++++++++ .../src/server/voice/openai-realtime-call.ts | 138 ++++++ .../server/voice/openai-voice-config.test.ts | 38 ++ .../src/server/voice/openai-voice-config.ts | 22 + .../server/voice/openai-voice-policy.test.ts | 96 +++++ .../src/server/voice/openai-voice-policy.ts | 58 +++ apps/petrinaut-website/src/vite-env.d.ts | 1 + apps/petrinaut-website/vercel.json | 3 + apps/petrinaut-website/vite.config.ts | 2 + 26 files changed, 2889 insertions(+), 12 deletions(-) create mode 100644 apps/petrinaut-website/api/voice/config.ts create mode 100644 apps/petrinaut-website/api/voice/realtime-call.ts create mode 100644 apps/petrinaut-website/src/main/app/local-storage-demo/brunch-ask-interactive-tool.test.ts create mode 100644 apps/petrinaut-website/src/main/app/local-storage-demo/brunch-ask-interactive-tool.tsx create mode 100644 apps/petrinaut-website/src/main/app/local-storage-demo/brunch-ask-mapping.ts create mode 100644 apps/petrinaut-website/src/main/app/local-storage-demo/brunch-preview-config.test.ts create mode 100644 apps/petrinaut-website/src/main/app/local-storage-demo/brunch-preview-config.ts create mode 100644 apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx create mode 100644 apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts create mode 100644 apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts create mode 100644 apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx create mode 100644 apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx create mode 100644 apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts create mode 100644 apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts create mode 100644 apps/petrinaut-website/src/server/voice/openai-realtime-call.test.ts create mode 100644 apps/petrinaut-website/src/server/voice/openai-realtime-call.ts create mode 100644 apps/petrinaut-website/src/server/voice/openai-voice-config.test.ts create mode 100644 apps/petrinaut-website/src/server/voice/openai-voice-config.ts create mode 100644 apps/petrinaut-website/src/server/voice/openai-voice-policy.test.ts create mode 100644 apps/petrinaut-website/src/server/voice/openai-voice-policy.ts diff --git a/apps/petrinaut-website/.env.example b/apps/petrinaut-website/.env.example index dfb9d72a1b6..2e3a4e81d8a 100644 --- a/apps/petrinaut-website/.env.example +++ b/apps/petrinaut-website/.env.example @@ -1 +1,4 @@ OPENAI_API_KEY=sk-xxxx +OPENAI_VOICE_API_KEY= +PETRINAUT_OPENAI_VOICE_ENABLED=false +VITE_BRUNCH_CHAT_ENDPOINT= diff --git a/apps/petrinaut-website/README.md b/apps/petrinaut-website/README.md index 83e8927096e..a9f53674254 100644 --- a/apps/petrinaut-website/README.md +++ b/apps/petrinaut-website/README.md @@ -7,7 +7,8 @@ role: Demo site and embed host for the Petrinaut editor A website for demoing Petrinaut (libs/@hashintel/petrinaut). -A SPA with API functions for AI assistance and JSON oEmbed discovery. +A SPA with API functions for AI assistance, voice initialization, and JSON +oEmbed discovery. ## Quickstart @@ -18,7 +19,7 @@ cp .env.example .env.local turbo run dev ``` -The dev server runs at [http://localhost:5173](http://localhost:5173). A plugin in `vite.config.ts` loads the API function. +The dev server runs at [http://localhost:5173](http://localhost:5173). A plugin in `vite.config.ts` loads the API functions. In production, the functions in the `api` folder are automatically deployed as Vercel Functions. @@ -60,15 +61,37 @@ provides a fake optimizer for isolated UI development. ## Environment variables -| Name | Required | Used by | Notes | -| ----------------------------- | ---------------- | ---------------- | --------------------------------------------------------- | -| `OPENAI_API_KEY` | for chat to work | `api/chat.ts` | OpenAI key the function uses to call `streamText`. | -| `PETRINAUT_AI_MODEL` | no | `api/chat.ts` | Overrides the default OpenAI model id. | -| `PETRINAUT_OPT_ORIGIN` | no | `vite.config.ts` | Overrides the local optimizer proxy target. | -| `VITE_PETRINAUT_OPT_PROVIDER` | no | website | Set to `service` to enable the optimization route. | -| `SENTRY_DSN` | no | `vite.config.ts` | Wired into the bundle via `__SENTRY_DSN__` at build time. | - -Local values live in `.env.local`; Vite's `loadEnv` (see [`vite.config.ts`](vite.config.ts)) copies them into `process.env` for both the dev server and the chat function. In production, set these in the Vercel project settings. +| Name | Required | Used by | Notes | +| -------------------------------- | ---------------- | ---------------- | ---------------------------------------------------------- | +| `OPENAI_API_KEY` | for chat to work | `api/chat.ts` | OpenAI key the function uses to call `streamText`. | +| `OPENAI_VOICE_API_KEY` | for voice input | voice API | Dedicated OpenAI key used only by the Realtime call proxy. | +| `PETRINAUT_OPENAI_VOICE_ENABLED` | no | voice API | Set to `true` to enable voice outside production. | +| `PETRINAUT_AI_MODEL` | no | `api/chat.ts` | Overrides the default OpenAI model id. | +| `PETRINAUT_OPT_ORIGIN` | no | `vite.config.ts` | Overrides the local optimizer proxy target. | +| `VITE_BRUNCH_CHAT_ENDPOINT` | for voice input | website | Full Brunch Petrinaut chat endpoint used by the panel. | +| `VITE_PETRINAUT_OPT_PROVIDER` | no | website | Set to `service` to enable the optimization route. | +| `SENTRY_DSN` | no | `vite.config.ts` | Wired into the bundle via `__SENTRY_DSN__` at build time. | + +Local values live in `.env.local`; Vite's `loadEnv` (see [`vite.config.ts`](vite.config.ts)) copies them into `process.env` for both the dev server and the API functions. In production, set these in the Vercel project settings. + +### Brunch voice-input preview + +Voice input is disabled by default and always unavailable when `VERCEL_ENV` is +`production`. To exercise the preview locally or in a Vercel preview, set a +real `VITE_BRUNCH_CHAT_ENDPOINT`, `PETRINAUT_OPENAI_VOICE_ENABLED=true`, and a +dedicated `OPENAI_VOICE_API_KEY`. The browser sends its SDP offer to this app; +the server initializes an OpenAI transcription-only Realtime session and keeps +the provider key, language, vocabulary, and VAD policy private. + +Only finalized transcripts enter the existing Petrinaut composer and Brunch AI +SDK transport. Partial transcripts remain display-only. The preview derives a +stable conversation id from the locally saved net; it is diagnostic identity, +not production authentication or conversation authority. + +The Brunch deployment must allow the website origin through its +`BRUNCH_PETRINAUT_ORIGINS` setting. Starting voice input requests browser +microphone permission. Denying permission leaves the existing text composer +available and does not submit anything to Brunch. ## Testing the API against the built output diff --git a/apps/petrinaut-website/api/voice/config.ts b/apps/petrinaut-website/api/voice/config.ts new file mode 100644 index 00000000000..c6239de88db --- /dev/null +++ b/apps/petrinaut-website/api/voice/config.ts @@ -0,0 +1,9 @@ +import { createOpenAIVoiceConfigHandler } from "../../src/server/voice/openai-voice-config"; + +declare const process: { + env: Record; +}; + +export default { + fetch: createOpenAIVoiceConfigHandler(process.env), +}; diff --git a/apps/petrinaut-website/api/voice/realtime-call.ts b/apps/petrinaut-website/api/voice/realtime-call.ts new file mode 100644 index 00000000000..b9b1b1f78ee --- /dev/null +++ b/apps/petrinaut-website/api/voice/realtime-call.ts @@ -0,0 +1,12 @@ +import { createOpenAIRealtimeCallHandler } from "../../src/server/voice/openai-realtime-call"; + +declare const process: { + env: Record; +}; + +export default { + fetch: createOpenAIRealtimeCallHandler({ + environment: process.env, + fetch: globalThis.fetch.bind(globalThis), + }), +}; diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-ask-interactive-tool.test.ts b/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-ask-interactive-tool.test.ts new file mode 100644 index 00000000000..c7db816a6b9 --- /dev/null +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-ask-interactive-tool.test.ts @@ -0,0 +1,14 @@ +import { describe, expect, test } from "vitest"; + +import { brunchAskFromComposerText } from "./brunch-ask-mapping"; + +describe("Brunch ask composer mapping", () => { + test("maps finalized composer text to the pending ask answer", () => { + expect( + brunchAskFromComposerText({ + input: { question: "Who triages the incident?" }, + text: "The support lead.", + }), + ).toEqual({ answer: "The support lead." }); + }); +}); diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-ask-interactive-tool.tsx b/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-ask-interactive-tool.tsx new file mode 100644 index 00000000000..1e8bed24fc6 --- /dev/null +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-ask-interactive-tool.tsx @@ -0,0 +1,151 @@ +import { type FormEvent, useId, useState } from "react"; + +import { + ASK_TOOL_NAME, + type BrunchAskInput, + type BrunchAskOutput, + parseBrunchAskInput, + parseBrunchAskOutput, +} from "@hashintel/brunch-agent-transport-aisdk/client-tools"; +import { css } from "@hashintel/ds-helpers/css"; +import { + definePetrinautAiInteractiveTool, + type PetrinautAiInteractiveToolWidgetProps, +} from "@hashintel/petrinaut/ui"; + +import { brunchAskFromComposerText } from "./brunch-ask-mapping"; + +const containerStyle = css({ + display: "flex", + flexDirection: "column", + gap: "2", + padding: "3", + borderWidth: "thin", + borderStyle: "solid", + borderColor: "blue.a30", + borderRadius: "lg", + backgroundColor: "blue.a10", +}); + +const questionStyle = css({ + color: "neutral.s100", + fontSize: "sm", + fontWeight: "medium", + lineHeight: "relaxed", +}); + +const formStyle = css({ + display: "flex", + flexDirection: "column", + gap: "2", +}); + +const labelStyle = css({ + color: "neutral.s90", + fontSize: "xs", + fontWeight: "medium", +}); + +const textareaStyle = css({ + width: "full", + minHeight: "20", + padding: "2", + borderWidth: "thin", + borderStyle: "solid", + borderColor: "neutral.a30", + borderRadius: "md", + backgroundColor: "neutral.s00", + color: "neutral.s100", + fontSize: "sm", + resize: "vertical", + _focusVisible: { + borderColor: "blue.a70", + outline: "2px solid", + outlineColor: "blue.a30", + outlineOffset: "[1px]", + }, +}); + +const submitButtonStyle = css({ + alignSelf: "flex-end", + paddingX: "3", + paddingY: "2", + borderRadius: "md", + backgroundColor: "blue.a85", + color: "white", + cursor: "pointer", + fontSize: "sm", + fontWeight: "medium", + _hover: { + backgroundColor: "blue.a100", + }, + _disabled: { + cursor: "not-allowed", + opacity: 0.45, + }, +}); + +const answerStyle = css({ + padding: "2", + borderRadius: "md", + backgroundColor: "neutral.s00", + color: "neutral.s90", + fontSize: "sm", +}); + +const BrunchAskWidget = ({ + input, + state, + submit, + submittedOutput, +}: PetrinautAiInteractiveToolWidgetProps) => { + const answerId = useId(); + const [answer, setAnswer] = useState(""); + + const onSubmit = (event: FormEvent) => { + event.preventDefault(); + const trimmedAnswer = answer.trim(); + if (!trimmedAnswer) { + return; + } + submit({ answer: trimmedAnswer }); + }; + + return ( +
+

{input.question}

+ {state === "submitted" ? ( +

{submittedOutput.answer}

+ ) : ( +
+ +