H-6763: Add OpenAI Realtime voice input to Brunch

.changeset/stable-composer-controls.md

@@ -4,4 +4,5 @@
 
 Add a generic host-rendered AI composer control with stable finalized-text submission,
 conversation identity, stop handling, schema-validated interactive-tool text mapping, and an
-explicit separate-message target for corrections.
+explicit separate-message target for corrections. Add a queue-aware voice submission path so a
+finalized spoken turn is retained while another response settles.

apps/petrinaut-website/.env.example

@@ -1 +1,4 @@
 OPENAI_API_KEY=sk-xxxx
+OPENAI_VOICE_API_KEY=
+PETRINAUT_OPENAI_VOICE_ENABLED=false
+VITE_BRUNCH_CHAT_ENDPOINT=

apps/petrinaut-website/README.md

@@ -7,7 +7,8 @@ role: Demo site and embed host for the Petrinaut editor
 
 A website for demoing Petrinaut (libs/@hashintel/petrinaut).
 
-A SPA with API functions for AI assistance and JSON oEmbed discovery.
+A SPA with API functions for AI assistance, voice initialization, and JSON
+oEmbed discovery.
 
 ## Quickstart
 
@@ -18,7 +19,7 @@ cp .env.example .env.local
 turbo run dev
 ```
 
-The dev server runs at [http://localhost:5173](http://localhost:5173). A plugin in `vite.config.ts` loads the API function.
+The dev server runs at [http://localhost:5173](http://localhost:5173). A plugin in `vite.config.ts` loads the API functions.
 
 In production, the functions in the `api` folder are automatically deployed as
 Vercel Functions.
@@ -60,15 +61,39 @@ provides a fake optimizer for isolated UI development.
 
 ## Environment variables
 
-| Name                          | Required         | Used by          | Notes                                                     |
-| ----------------------------- | ---------------- | ---------------- | --------------------------------------------------------- |
-| `OPENAI_API_KEY`              | for chat to work | `api/chat.ts`    | OpenAI key the function uses to call `streamText`.        |
-| `PETRINAUT_AI_MODEL`          | no               | `api/chat.ts`    | Overrides the default OpenAI model id.                    |
-| `PETRINAUT_OPT_ORIGIN`        | no               | `vite.config.ts` | Overrides the local optimizer proxy target.               |
-| `VITE_PETRINAUT_OPT_PROVIDER` | no               | website          | Set to `service` to enable the optimization route.        |
-| `SENTRY_DSN`                  | no               | `vite.config.ts` | Wired into the bundle via `__SENTRY_DSN__` at build time. |
-
-Local values live in `.env.local`; Vite's `loadEnv` (see [`vite.config.ts`](vite.config.ts)) copies them into `process.env` for both the dev server and the chat function. In production, set these in the Vercel project settings.
+| Name                             | Required         | Used by          | Notes                                                      |
+| -------------------------------- | ---------------- | ---------------- | ---------------------------------------------------------- |
+| `OPENAI_API_KEY`                 | for chat to work | `api/chat.ts`    | OpenAI key the function uses to call `streamText`.         |
+| `OPENAI_VOICE_API_KEY`           | for voice input  | voice API        | Dedicated OpenAI key used only by the Realtime call proxy. |
+| `PETRINAUT_OPENAI_VOICE_ENABLED` | no               | voice API        | Set to `true` to enable voice outside production.          |
+| `PETRINAUT_AI_MODEL`             | no               | `api/chat.ts`    | Overrides the default OpenAI model id.                     |
+| `PETRINAUT_OPT_ORIGIN`           | no               | `vite.config.ts` | Overrides the local optimizer proxy target.                |
+| `VITE_BRUNCH_CHAT_ENDPOINT`      | for voice input  | website          | Full Brunch Petrinaut chat endpoint used by the panel.     |
+| `VITE_PETRINAUT_OPT_PROVIDER`    | no               | website          | Set to `service` to enable the optimization route.         |
+| `SENTRY_DSN`                     | no               | `vite.config.ts` | Wired into the bundle via `__SENTRY_DSN__` at build time.  |
+
+Local values live in `.env.local`; Vite's `loadEnv` (see [`vite.config.ts`](vite.config.ts)) copies them into `process.env` for both the dev server and the API functions. In production, set these in the Vercel project settings.
+
+### Brunch voice-input preview
+
+Voice input is disabled by default and always unavailable when `VERCEL_ENV` is
+`production`. To exercise the preview locally or in a Vercel preview, set a
+real `VITE_BRUNCH_CHAT_ENDPOINT`, `PETRINAUT_OPENAI_VOICE_ENABLED=true`, and a
+dedicated `OPENAI_VOICE_API_KEY`. The browser sends its SDP offer to this app;
+the server initializes an OpenAI transcription-only Realtime session and keeps
+the provider key, model, language, and vocabulary policy private. The session
+uses `gpt-live-transcribe`'s default server VAD because OpenAI's unified call
+currently times out when explicit turn detection is included during setup.
+
+Only finalized transcripts enter the existing Petrinaut composer and Brunch AI
+SDK transport. Partial transcripts remain display-only. The preview derives a
+stable conversation id from the locally saved net; it is diagnostic identity,
+not production authentication or conversation authority.
+
+The Brunch deployment must allow the website origin through its
+`BRUNCH_PETRINAUT_ORIGINS` setting. Starting voice input requests browser
+microphone permission. Denying permission leaves the existing text composer
+available and does not submit anything to Brunch.
 
 ## Testing the API against the built output
 

apps/petrinaut-website/api/voice/config.ts

@@ -0,0 +1,9 @@
+import { createOpenAIVoiceConfigHandler } from "../../src/server/voice/openai-voice-config";
+
+declare const process: {
+  env: Record<string, string | undefined>;
+};
+
+export default {
+  fetch: createOpenAIVoiceConfigHandler(process.env),
+};

apps/petrinaut-website/api/voice/realtime-call.ts

@@ -0,0 +1,12 @@
+import { createOpenAIRealtimeCallHandler } from "../../src/server/voice/openai-realtime-call";
+
+declare const process: {
+  env: Record<string, string | undefined>;
+};
+
+export default {
+  fetch: createOpenAIRealtimeCallHandler({
+    environment: process.env,
+    fetch: globalThis.fetch.bind(globalThis),
+  }),
+};

apps/petrinaut-website/src/main/app/local-storage-demo/brunch-ask-interactive-tool.test.ts

@@ -0,0 +1,14 @@
+import { describe, expect, test } from "vitest";
+
+import { brunchAskFromComposerText } from "./brunch-ask-mapping";
+
+describe("Brunch ask composer mapping", () => {
+  test("maps finalized composer text to the pending ask answer", () => {
+    expect(
+      brunchAskFromComposerText({
+        input: { question: "Who triages the incident?" },
+        text: "The support lead.",
+      }),
+    ).toEqual({ answer: "The support lead." });
+  });
+});

apps/petrinaut-website/src/main/app/local-storage-demo/brunch-ask-interactive-tool.tsx

@@ -0,0 +1,151 @@
+import { type FormEvent, useId, useState } from "react";
+
+import {
+  ASK_TOOL_NAME,
+  type BrunchAskInput,
+  type BrunchAskOutput,
+  parseBrunchAskInput,
+  parseBrunchAskOutput,
+} from "@hashintel/brunch-agent-transport-aisdk/client-tools";
+import { css } from "@hashintel/ds-helpers/css";
+import {
+  definePetrinautAiInteractiveTool,
+  type PetrinautAiInteractiveToolWidgetProps,
+} from "@hashintel/petrinaut/ui";
+
+import { brunchAskFromComposerText } from "./brunch-ask-mapping";
+
+const containerStyle = css({
+  display: "flex",
+  flexDirection: "column",
+  gap: "2",
+  padding: "3",
+  borderWidth: "thin",
+  borderStyle: "solid",
+  borderColor: "blue.a30",
+  borderRadius: "lg",
+  backgroundColor: "blue.a10",
+});
+
+const questionStyle = css({
+  color: "neutral.s100",
+  fontSize: "sm",
+  fontWeight: "medium",
+  lineHeight: "relaxed",
+});
+
+const formStyle = css({
+  display: "flex",
+  flexDirection: "column",
+  gap: "2",
+});
+
+const labelStyle = css({
+  color: "neutral.s90",
+  fontSize: "xs",
+  fontWeight: "medium",
+});
+
+const textareaStyle = css({
+  width: "full",
+  minHeight: "20",
+  padding: "2",
+  borderWidth: "thin",
+  borderStyle: "solid",
+  borderColor: "neutral.a30",
+  borderRadius: "md",
+  backgroundColor: "neutral.s00",
+  color: "neutral.s100",
+  fontSize: "sm",
+  resize: "vertical",
+  _focusVisible: {
+    borderColor: "blue.a70",
+    outline: "2px solid",
+    outlineColor: "blue.a30",
+    outlineOffset: "[1px]",
+  },
+});
+
+const submitButtonStyle = css({
+  alignSelf: "flex-end",
+  paddingX: "3",
+  paddingY: "2",
+  borderRadius: "md",
+  backgroundColor: "blue.a85",
+  color: "white",
+  cursor: "pointer",
+  fontSize: "sm",
+  fontWeight: "medium",
+  _hover: {
+    backgroundColor: "blue.a100",
+  },
+  _disabled: {
+    cursor: "not-allowed",
+    opacity: 0.45,
+  },
+});
+
+const answerStyle = css({
+  padding: "2",
+  borderRadius: "md",
+  backgroundColor: "neutral.s00",
+  color: "neutral.s90",
+  fontSize: "sm",
+});
+
+const BrunchAskWidget = ({
+  input,
+  state,
+  submit,
+  submittedOutput,
+}: PetrinautAiInteractiveToolWidgetProps<BrunchAskInput, BrunchAskOutput>) => {
+  const answerId = useId();
+  const [answer, setAnswer] = useState("");
+
+  const onSubmit = (event: FormEvent<HTMLFormElement>) => {
+    event.preventDefault();
+    const trimmedAnswer = answer.trim();
+    if (!trimmedAnswer) {
+      return;
+    }
+    submit({ answer: trimmedAnswer });
+  };
+
+  return (
+    <section className={containerStyle}>
+      <p className={questionStyle}>{input.question}</p>
+      {state === "submitted" ? (
+        <p className={answerStyle}>{submittedOutput.answer}</p>
+      ) : (
+        <form className={formStyle} onSubmit={onSubmit}>
+          <label className={labelStyle} htmlFor={answerId}>
+            Your answer
+          </label>
+          <textarea
+            className={textareaStyle}
+            id={answerId}
+            onChange={(event) => setAnswer(event.target.value)}
+            placeholder="Write what you know; uncertainty is useful too."
+            rows={3}
+            value={answer}
+          />
+          <button
+            className={submitButtonStyle}
+            disabled={answer.trim().length === 0}
+            type="submit"
+          >
+            Send answer
+          </button>
+        </form>
+      )}
+    </section>
+  );
+};
+
+export const brunchAskInteractiveTool = definePetrinautAiInteractiveTool({
+  toolName: ASK_TOOL_NAME,
+  inputSchema: { parse: parseBrunchAskInput },
+  outputSchema: { parse: parseBrunchAskOutput },
+  fromComposerText: brunchAskFromComposerText,
+  component: BrunchAskWidget,
+});

apps/petrinaut-website/src/main/app/local-storage-demo/brunch-ask-mapping.ts

@@ -0,0 +1,11 @@
+import type {
+  BrunchAskInput,
+  BrunchAskOutput,
+} from "@hashintel/brunch-agent-transport-aisdk/client-tools";
+
+export const brunchAskFromComposerText = ({
+  text,
+}: {
+  input: BrunchAskInput;
+  text: string;
+}): BrunchAskOutput => ({ answer: text });

apps/petrinaut-website/src/main/app/local-storage-demo/brunch-preview-config.test.ts

@@ -0,0 +1,33 @@
+import { describe, expect, test } from "vitest";
+
+import {
+  createBrunchPreviewConversationId,
+  resolveBrunchPreviewConfig,
+} from "./brunch-preview-config";
+
+describe("Brunch preview host configuration", () => {
+  test("keeps the local generic chat fallback voice-free", () => {
+    expect(resolveBrunchPreviewConfig(undefined)).toEqual({
+      chatEndpoint: "/api/chat",
+      isBrunchConfigured: false,
+    });
+  });
+
+  test("uses an explicitly configured Brunch transport endpoint", () => {
+    expect(
+      resolveBrunchPreviewConfig("  https://brunch.test/api/petrinaut/chat  "),
+    ).toEqual({
+      chatEndpoint: "https://brunch.test/api/petrinaut/chat",
+      isBrunchConfigured: true,
+    });
+  });
+
+  test("derives a stable preview conversation identity from the saved net", () => {
+    expect(createBrunchPreviewConversationId("net-123")).toBe(
+      "petrinaut-preview:net-123",
+    );
+    expect(createBrunchPreviewConversationId("net-123")).toBe(
+      "petrinaut-preview:net-123",
+    );
+  });
+});

apps/petrinaut-website/src/main/app/local-storage-demo/brunch-preview-config.ts

@@ -0,0 +1,9 @@
+export const resolveBrunchPreviewConfig = (endpoint: string | undefined) => {
+  const configuredEndpoint = endpoint?.trim();
+  return configuredEndpoint
+    ? { chatEndpoint: configuredEndpoint, isBrunchConfigured: true }
+    : { chatEndpoint: "/api/chat", isBrunchConfigured: false };
+};
+
+export const createBrunchPreviewConversationId = (netId: string): string =>
+  `petrinaut-preview:${netId}`;

apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx

@@ -0,0 +1,45 @@
+/**
+ * @vitest-environment jsdom
+ */
+import { isValidElement, type ReactNode } from "react";
+import { describe, expect, test, vi } from "vitest";
+
+import { VoiceInterviewControl } from "../voice-interview/voice-interview-control";
+import { getBrunchVoiceComposerControl } from "./local-storage-demo-app";
+
+vi.mock("@hashintel/petrinaut/ui", () => ({
+  DefaultChatTransport: class {},
+  Petrinaut: () => null,
+  WalkthroughProvider: ({ children }: { children: ReactNode }) => children,
+  definePetrinautAiInteractiveTool: (definition: unknown) => definition,
+}));
+
+describe("local storage demo Brunch voice integration", () => {
+  test("does not install voice on the generic local chat fallback", () => {
+    expect(getBrunchVoiceComposerControl(false)).toBeUndefined();
+  });
+
+  test("installs the app-owned voice control for a configured Brunch transport", () => {
+    const renderControl = getBrunchVoiceComposerControl(true);
+    const control = renderControl?.({
+      conversationId: "petrinaut-preview:net-1",
+      messages: [],
+      status: "ready",
+      stop: vi.fn(async () => undefined),
+      submitText: vi.fn(async () => ({
+        kind: "message" as const,
+        messageId: "message-1",
+      })),
+      submitVoiceInput: vi.fn(async () => ({
+        kind: "message" as const,
+        messageId: "voice-message-1",
+      })),
+    });
+
+    expect(isValidElement(control)).toBe(true);
+    if (!isValidElement(control)) {
+      throw new Error("Expected the configured composer control to render.");
+    }
+    expect(control.type).toBe(VoiceInterviewControl);
+  });
+});