From 69fd1563fe703fc196fef3b29963fbd37cb29d9c Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Wed, 26 Aug 2026 23:11:21 +0200 Subject: [PATCH 1/6] Add generic Petrinaut composer submission Expose stable host-owned composer controls, conversation identity, and schema-validated text mapping through the existing AI SDK chat path. Record the app-owned OpenAI voice boundary and the production contracts that gate rollout. Amp-Thread-ID: https://ampcode.com/threads/T-01a03fb3-fd3d-737f-b4c6-1fc9282950bf Co-authored-by: Amp --- .changeset/stable-composer-controls.md | 6 + .../adr/0009-openai-voice-ui-turn-shell.md | 77 +++ .../docs/specs/petrinaut-integration.md | 36 +- libs/@hashintel/petrinaut/CHANGELOG.md | 7 + .../@hashintel/petrinaut/docs/ai-assistant.md | 11 +- libs/@hashintel/petrinaut/src/main.ts | 6 + libs/@hashintel/petrinaut/src/ui/index.ts | 6 + .../@hashintel/petrinaut/src/ui/petrinaut.tsx | 5 + .../ui/types/ai-assistant-composer-control.ts | 33 ++ .../src/ui/types/ai-interactive-tool.ts | 40 +- .../Editor/panels/ai-assistant-panel.test.tsx | 493 +++++++++++++++++- .../Editor/panels/ai-assistant-panel.tsx | 241 ++++++++- .../ai-assistant-contents.test.tsx | 28 + .../ai-assistant-contents.tsx | 12 +- .../interactive-tools/registry.test.tsx | 80 +++ .../interactive-tools/registry.ts | 1 + .../interactive-tools/types.ts | 2 + 17 files changed, 1043 insertions(+), 41 deletions(-) create mode 100644 .changeset/stable-composer-controls.md create mode 100644 libs/@hashintel/brunch-agent/docs/adr/0009-openai-voice-ui-turn-shell.md create mode 100644 libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts diff --git a/.changeset/stable-composer-controls.md b/.changeset/stable-composer-controls.md new file mode 100644 index 00000000000..fc2542aadd2 --- /dev/null +++ b/.changeset/stable-composer-controls.md @@ -0,0 +1,6 @@ +--- +"@hashintel/petrinaut": patch +--- + +Add a generic host-rendered AI composer control with stable finalized-text submission, +conversation identity, stop handling, and schema-validated interactive-tool text mapping. diff --git a/libs/@hashintel/brunch-agent/docs/adr/0009-openai-voice-ui-turn-shell.md b/libs/@hashintel/brunch-agent/docs/adr/0009-openai-voice-ui-turn-shell.md new file mode 100644 index 00000000000..4d7ea30471f --- /dev/null +++ b/libs/@hashintel/brunch-agent/docs/adr/0009-openai-voice-ui-turn-shell.md @@ -0,0 +1,77 @@ +# ADR-0009: OpenAI voice uses the app-owned UI turn shell + +Date: 2026-08-26 +Status: accepted for the bounded H-6763 preview stack +Extends: [ADR-0004](0004-in-petrinaut-staging-and-the-monorepo-import.md), which keeps Brunch and +Petrinaut composition in applications and reusable libraries mutually unaware +Preserves: [ADR-0003](0003-three-register-ir.md), which makes Brunch's capture fold authoritative, +and the [Petrinaut integration attach contract](../specs/petrinaut-integration.md#attach-contract) +Decided on: `kostandin/h-6763-petrinaut-composer-api`, from the approved H-6763 OpenAI voice plan + +## Context + +H-6763 adds spoken input and output to a Brunch elicitation shown in Petrinaut. The existing +production boundary already has the semantics the feature needs: one AI SDK UI-message stream, +the correlated `brunch_ask` tool, and a principal-owned conversation. A voice implementation can +either preserve that boundary or create a second conversation authority in the audio provider. +Only the first choice preserves Brunch's durable history, captures, pending asks, completion, and +projection contracts. + +The first rollout is a disabled preview. The production contracts for authenticated ownership, +distributed quotas, telemetry, replay, and final Petrinaut projection do not all exist yet. The +preview therefore needs a boundary that permits input and output experiments without claiming +production recovery or public availability. + +## Decision + +1. **The host application owns voice.** `apps/petrinaut-website` owns OpenAI policy, WebRTC, + transcript event parsing, half-duplex state, playback, feature UI, and server routes. + `@hashintel/petrinaut` exposes only a generic composer control and finalized-text submission + seam. Brunch packages contain no provider code. +2. **OpenAI is the only runtime voice provider.** Do not add a provider abstraction, selector, + compatibility layer, or ElevenLabs dependency, configuration, route, script, test, or + diagnostic. +3. **Realtime is transcription-only.** The server atomically combines browser SDP with trusted + transcription policy and calls OpenAI's unified WebRTC initialization endpoint. The fixed + model is `gpt-live-transcribe`. Semantic VAD begins at low eagerness as a tunable evaluation + setting. Provider keys, prompts, vocabulary, language policy, and model selection remain + server-side. Realtime never generates assistant responses. +4. **Completed provider items are the only admitted audio input.** Partials are display-only. + Completed items are keyed by connection epoch, provider item ID, and content index, then enter + the existing Petrinaut composer and AI SDK transport once. A pending `brunch_ask` uses its + existing correlated tool-output path; otherwise the final becomes a stable-ID user message. +5. **The interaction is half-duplex.** The microphone is closed while Brunch is handling a turn + and while speech is synthesized or playing. Barge-in and simultaneous listening and playback + are out of scope. +6. **Speech receives canonical Brunch text exactly.** A dedicated OpenAI Speech request receives + only finalized assistant text or the validated `brunch_ask.input.question` selected from the + AI SDK message structure. The application does not scrape rendered DOM, ask Realtime to "say + exactly," or ask a model to rewrite the text. Failure leaves the same text visible. +7. **The preview fails closed and is disabled by default.** Voice is unavailable when server + policy, credentials, or the Brunch transport are unavailable. Text chat remains available. + Public production remains disabled until FE-1439, FE-1420, platform authentication, + distributed quotas, FE-1505 telemetry, and FE-1438/FE-1440 completion and projection contracts + are available and consumed. + +## Consequences + +- Petrinaut's public API gains a generic host control with stable `submitText`, `stop`, messages, + status, and optional conversation identity. Interactive tools may opt into a schema-validated + text-to-output mapper. Keyboard and alternate finalized text therefore cannot bypass ask + correlation. +- OpenAI implementation names and policy stay in `apps/petrinaut-website`. The existing + `transport-aisdk` package remains the sole browser-to-Brunch conversation transport. +- Preview PRs may demonstrate transcription and exact canonical speech before production + recovery exists, but they may not claim durable redelivery, authenticated access, distributed + rate enforcement, production telemetry, or final projection. +- A later provider choice, full duplex, sentence-level streaming speech, mobile support, or + acoustic-pronunciation guarantee requires a new decision. Exact lexical input to Speech is the + enforceable fidelity contract in this record. + +## Revisit condition + +Revisit if the unified OpenAI WebRTC initialization API cannot enforce server-owned transcription +policy; if the generic composer seam cannot preserve the existing AI SDK and `brunch_ask` paths; +or if production authentication, replay, telemetry, or projection contracts require a boundary +change rather than an application adapter. Do not address any of these by making browser or +provider history authoritative. diff --git a/libs/@hashintel/brunch-agent/docs/specs/petrinaut-integration.md b/libs/@hashintel/brunch-agent/docs/specs/petrinaut-integration.md index 256b247ee2a..18aa6b6e281 100644 --- a/libs/@hashintel/brunch-agent/docs/specs/petrinaut-integration.md +++ b/libs/@hashintel/brunch-agent/docs/specs/petrinaut-integration.md @@ -5,7 +5,7 @@ `recommendation-demo-vehicle.md` as the September staging plan · **Evidence base**: the Petrinaut survey (FE-1358, `research/petrinaut-survey.md`), re-verified against `hashintel/hash` source on 2026-08-18 · **Amended**: FE-1506 (stable UI and voice attach -contract). +contract), H-6763 / ADR-0009 (generic composer submission and app-owned voice boundary). ## Problem Statement @@ -33,7 +33,7 @@ stream chunks and knows nothing about Flue. ## Seams -One primary seam, three supporting ones — all existing except the brunch server's front door, +One primary seam, four supporting ones — all existing except the brunch server's front door, which the design needs anyway: 1. **The ChatTransport wire seam** (primary; the contract-test surface): the AI SDK @@ -45,6 +45,11 @@ which the design needs anyway: 3. **The storage port seam** (ADR-0002 N5): the owner key is tested as store-level refusals. 4. **The artifact seam** (`parseSDCPNFile` / `sdcpnFileSchema`): unchanged; net validity checked in CI through the pure parser. +5. **The generic composer seam**: a host may render a control beside Petrinaut's text composer and + receive stable `submitText` and `stop` callbacks plus conversation state. Finalized alternate + text uses the same AI SDK `useChat` instance as keyboard input. When exactly one unresolved + interactive tool registers a schema-validated text mapper, `submitText` completes that tool; + otherwise it submits a stable-ID user message. Ambiguous mapped tools are refused. ## Attach Contract @@ -59,10 +64,14 @@ The panel and the voice edge attach to Brunch through one stable surface: `x-brunch-principal` header. The current UI shell keeps that value in localStorage so it is stable across reloads; replacing the local UID with authenticated identity must preserve the same request-level ownership semantics. +4. **Composer submission**: Petrinaut accepts an optional stable conversation ID and host composer + control. Keyboard and alternate finalized text both enter the same `submitText` function. A + pending `brunch_ask` is answered only through the existing correlated tool-output path; text is + not silently downgraded to an ordinary user message when more than one mapped ask is pending. -These three parts change only with notice to the panel and voice-edge owners. A provider-specific -voice requirement does not silently alter this surface; it arrives as a generic UI-shell extension -or triggers an explicit contract revision. +These four parts change only with notice to the panel and voice-edge owners. A provider-specific +voice requirement does not silently alter this surface; provider code and policy remain in the +host application under ADR-0009, while reusable Petrinaut and Brunch packages stay provider-free. ## User Stories @@ -112,6 +121,8 @@ or triggers an explicit contract revision. 21. As a future petrinaut-website maintainer, I want brunch-specific wiring contained at the app level (as the existing Actual-mode brunch-demo route already is), so that removing or evolving it never archaeology-digs through the library. +22. As a Petrinaut host, I want finalized alternate input to share keyboard submission and pending + interactive-tool correlation, so that a host control cannot create a second conversation path. ## Implementation Decisions @@ -160,6 +171,18 @@ or triggers an explicit contract revision. is a generic host-supplied-handlers extension to the `aiAssistant` prop (post-import, per ADR-0004's boundary discipline). +**Generic composer control** + +- `@hashintel/petrinaut` accepts an optional conversation ID and host render callback. The callback + receives current AI SDK messages and status plus stable `submitText` and `stop` functions. +- A host interactive tool may define `fromComposerText({ input, text })`. Petrinaut parses the + pending input, invokes the mapper, and parses its output before submitting the correlated tool + result. Unknown or unmapped tools preserve ordinary message submission; multiple eligible tools + fail visibly rather than guessing. +- The seam is provider- and elicitor-agnostic. OpenAI WebRTC, transcription policy, speech, and + half-duplex state belong to `apps/petrinaut-website`; Brunch remains behind the existing + transport. See [ADR-0009](../adr/0009-openai-voice-ui-turn-shell.md). + **Identity and storage** - The principal is ui-shell-owned: the demo site mints a random UID into localStorage and @@ -207,7 +230,8 @@ or triggers an explicit contract revision. ## Out of Scope -- Voice (conditional nice-to-have per FE-1359's tiers; unchanged by the pivot). +- Provider-specific voice behavior in Petrinaut or Brunch. The app-owned, disabled H-6763 preview + is governed by ADR-0009; production recovery and rollout wait for its named prerequisites. - HASH-app integration (design-for via the principal and adapter abstractions; no build). - The interpretation-render panel's visual design and placement (app-level UI vs. `PetrinautSlots` — decided when the demo-site wiring starts, after the spikes). diff --git a/libs/@hashintel/petrinaut/CHANGELOG.md b/libs/@hashintel/petrinaut/CHANGELOG.md index e1a311d94c1..af768cdfbfb 100644 --- a/libs/@hashintel/petrinaut/CHANGELOG.md +++ b/libs/@hashintel/petrinaut/CHANGELOG.md @@ -1,5 +1,12 @@ # @hashintel/petrinaut +## Unreleased + +### Patch Changes + +- Add a generic host-rendered AI composer control with stable finalized-text submission, + conversation identity, stop handling, and schema-validated interactive-tool text mapping. + ## 0.0.19 ### Patch Changes diff --git a/libs/@hashintel/petrinaut/docs/ai-assistant.md b/libs/@hashintel/petrinaut/docs/ai-assistant.md index 926e839142c..4ea9f3aabf4 100644 --- a/libs/@hashintel/petrinaut/docs/ai-assistant.md +++ b/libs/@hashintel/petrinaut/docs/ai-assistant.md @@ -19,6 +19,12 @@ While a response is streaming you can: - Press **Stop AI response** (the send button turns into a stop icon) to halt the current response. - Type your next message in the composer -- it is queued for after the current response ends. +The application embedding Petrinaut may place an additional control beside the message box. For +example, a host can offer another way to enter finalized text. Text submitted by that control +behaves like text sent with the keyboard: it joins the same conversation and, when an inline +question is waiting for an answer, completes that question rather than starting an unrelated +message. + **Clear AI chat** via the delete button in the top right of the panel: wipes the conversation, stops any in-flight stream, and tells the host app to forget the messages (if the host persists them). ## What the assistant can do @@ -56,4 +62,7 @@ When the assistant edits code surfaces (lambdas, kernels, dynamics, visualizers, ## Host configuration -Whether the assistant is available, where the conversation is stored (in-memory, in your host app's database, or anywhere else), and the model behind it are all controlled by the host application that embeds Petrinaut. Read-only documents and the simulate-mode restrictions described above always apply when applicable. +Whether the assistant is available, which additional composer controls appear, where the +conversation is stored (in-memory, in your host app's database, or anywhere else), and the model +behind it are all controlled by the host application that embeds Petrinaut. Read-only documents +and the simulate-mode restrictions described above always apply when applicable. diff --git a/libs/@hashintel/petrinaut/src/main.ts b/libs/@hashintel/petrinaut/src/main.ts index f55229d1f3c..cd47445a1ea 100644 --- a/libs/@hashintel/petrinaut/src/main.ts +++ b/libs/@hashintel/petrinaut/src/main.ts @@ -17,6 +17,12 @@ export { PetrinautOptimizationContext } from "./react/optimization-context"; export type { PetrinautSlots } from "./ui/types/petrinaut-slots"; export type { ViewportAction } from "./ui/types/viewport-action"; +export type { + PetrinautAiComposerControl, + PetrinautAiComposerControlContext, + PetrinautAiComposerStatus, + PetrinautAiComposerSubmitTextResult, +} from "./ui/types/ai-assistant-composer-control"; export type { PetrinautNavigationAction, PetrinautNavigationController, diff --git a/libs/@hashintel/petrinaut/src/ui/index.ts b/libs/@hashintel/petrinaut/src/ui/index.ts index 6dd28fd99e1..7526c884b7f 100644 --- a/libs/@hashintel/petrinaut/src/ui/index.ts +++ b/libs/@hashintel/petrinaut/src/ui/index.ts @@ -16,6 +16,12 @@ export type { PetrinautAiChatTransport, PetrinautProps, } from "./petrinaut"; +export type { + PetrinautAiComposerControl, + PetrinautAiComposerControlContext, + PetrinautAiComposerStatus, + PetrinautAiComposerSubmitTextResult, +} from "./types/ai-assistant-composer-control"; export type { PetrinautNavigationAction, PetrinautNavigationController, diff --git a/libs/@hashintel/petrinaut/src/ui/petrinaut.tsx b/libs/@hashintel/petrinaut/src/ui/petrinaut.tsx index 9822ec6ef0e..e639475599e 100644 --- a/libs/@hashintel/petrinaut/src/ui/petrinaut.tsx +++ b/libs/@hashintel/petrinaut/src/ui/petrinaut.tsx @@ -29,6 +29,7 @@ const editorRootStyle = css({ backgroundColor: "neutral.s25", }); +import type { PetrinautAiComposerControl } from "./types/ai-assistant-composer-control"; import type { PetrinautAiInteractiveTool } from "./types/ai-interactive-tool"; import type { PetrinautAiMessage, @@ -38,11 +39,15 @@ import type { export type PetrinautAiChatTransport = PetrinautAiTransport; export type PetrinautAiAssistant = { + /** Stable identity for this host-owned conversation. */ + conversationId?: string; /** Host-owned dynamic tools that render inline in the AI conversation. */ interactiveTools?: readonly PetrinautAiInteractiveTool[]; messages?: PetrinautAiMessage[]; onClearMessages?: () => void; onMessages?: (messages: PetrinautAiMessage[]) => void; + /** Render a host-owned control inside the assistant composer. */ + renderComposerControl?: PetrinautAiComposerControl; transport: PetrinautAiTransport; }; diff --git a/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts b/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts new file mode 100644 index 00000000000..cddc6229056 --- /dev/null +++ b/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts @@ -0,0 +1,33 @@ +import type { PetrinautAiMessage } from "../views/Editor/panels/ai-assistant-panel/types"; +import type { ReactNode } from "react"; + +/** Current lifecycle state of Petrinaut's AI SDK conversation. */ +export type PetrinautAiComposerStatus = + | "submitted" + | "streaming" + | "ready" + | "error"; + +/** Outcome from submitting finalized text through the assistant composer. */ +export type PetrinautAiComposerSubmitTextResult = + | { kind: "message"; messageId: string } + | { kind: "interactive-tool"; toolCallId: string }; + +/** Stable controls and current conversation state supplied to a host control. */ +export type PetrinautAiComposerControlContext = { + conversationId?: string; + messages: PetrinautAiMessage[]; + status: PetrinautAiComposerStatus; + /** Call from an event handler or effect, never while rendering. */ + stop: () => Promise; + /** Call from an event handler or effect, never while rendering. */ + submitText: (params: { + id?: string; + text: string; + }) => Promise; +}; + +/** Render callback for a host-owned control inside the assistant composer. */ +export type PetrinautAiComposerControl = ( + context: PetrinautAiComposerControlContext, +) => ReactNode; diff --git a/libs/@hashintel/petrinaut/src/ui/types/ai-interactive-tool.ts b/libs/@hashintel/petrinaut/src/ui/types/ai-interactive-tool.ts index 568ccdcc34a..ccd5aa95fff 100644 --- a/libs/@hashintel/petrinaut/src/ui/types/ai-interactive-tool.ts +++ b/libs/@hashintel/petrinaut/src/ui/types/ai-interactive-tool.ts @@ -39,6 +39,12 @@ export type PetrinautAiInteractiveToolDefinition = { inputSchema: PetrinautAiInteractiveToolSchema; /** Runtime contract for the widget's submitted output. */ outputSchema: PetrinautAiInteractiveToolSchema; + /** + * Optionally map text submitted through the assistant composer to this + * tool's output. Petrinaut validates both the pending input and mapped + * output before completing the tool call. + */ + fromComposerText?: (params: { input: Input; text: string }) => Output; /** Inline component shown while awaiting input and after submission. */ component: ComponentType< PetrinautAiInteractiveToolWidgetProps @@ -49,6 +55,7 @@ type ErasedInteractiveToolDefinition = { toolName: string; parseInput: (value: unknown) => unknown; parseOutput: (value: unknown) => unknown; + fromComposerText?: (params: { input: unknown; text: string }) => unknown; component: ComponentType< PetrinautAiInteractiveToolWidgetProps >; @@ -68,17 +75,30 @@ export type PetrinautAiInteractiveTool = { */ export const definePetrinautAiInteractiveTool = ( definition: PetrinautAiInteractiveToolDefinition, -): PetrinautAiInteractiveTool => ({ - toolName: definition.toolName, - [interactiveToolDefinition]: { +): PetrinautAiInteractiveTool => { + const fromComposerText = definition.fromComposerText; + + return { toolName: definition.toolName, - parseInput: (value) => definition.inputSchema.parse(value), - parseOutput: (value) => definition.outputSchema.parse(value), - component: definition.component as ComponentType< - PetrinautAiInteractiveToolWidgetProps - >, - }, -}); + [interactiveToolDefinition]: { + toolName: definition.toolName, + parseInput: (value) => definition.inputSchema.parse(value), + parseOutput: (value) => definition.outputSchema.parse(value), + fromComposerText: fromComposerText + ? ({ input, text }) => + definition.outputSchema.parse( + fromComposerText({ + input: definition.inputSchema.parse(input), + text, + }), + ) + : undefined, + component: definition.component as ComponentType< + PetrinautAiInteractiveToolWidgetProps + >, + }, + }; +}; /** @internal */ export const getPetrinautAiInteractiveToolDefinition = ( diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx index b9484cf7427..f2c219c6c0b 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx @@ -30,6 +30,7 @@ import { import { definePetrinautAiInteractiveTool } from "../../../types/ai-interactive-tool"; import { AiAssistantPanel } from "./ai-assistant-panel"; +import type { PetrinautAiAssistant } from "../../../petrinaut"; import type { PetrinautAiMessage, PetrinautAiTransport, @@ -104,7 +105,497 @@ const streamChunks = ( }, }); -afterEach(cleanup); +const textChunks = (id: string, text: string): UIMessageChunk[] => [ + { type: "start-step" }, + { type: "text-start", id }, + { type: "text-delta", id, delta: text }, + { type: "text-end", id }, +]; + +const testInstances: ReturnType[] = []; + +const renderTestPanel = ({ + aiAssistant, + initialMessage, +}: { + aiAssistant: PetrinautAiAssistant; + initialMessage?: string; +}) => { + const handle = createJsonDocHandle({ + id: "ai-assistant-panel-test", + initial: emptySDCPN, + }); + const instance = createPetrinaut({ document: handle }); + testInstances.push(instance); + const sdcpnContext: SDCPNContextValue = { + createNewNet: () => {}, + existingNets: [], + loadPetriNet: () => {}, + petriNetId: "ai-assistant-panel-test", + petriNetDefinition: emptySDCPN, + readonly: false, + extensions: DEFAULT_PETRINAUT_EXTENSIONS, + setTitle: () => {}, + title: "AI assistant panel test", + getItemType: () => null, + }; + + render( + + + + + + + , + ); +}; + +afterEach(() => { + cleanup(); + for (const instance of testInstances.splice(0)) { + instance.dispose(); + } +}); + +describe("AiAssistantPanel composer submissions", () => { + test("shares one stable submission path between alternate and keyboard text", async () => { + const requestMessages: PetrinautAiMessage[][] = []; + const chatIds: string[] = []; + const transport: PetrinautAiTransport = { + reconnectToStream: () => Promise.resolve(null), + sendMessages: vi.fn(({ chatId, messages }) => { + chatIds.push(chatId); + requestMessages.push(structuredClone(messages)); + const responseId = `response-${requestMessages.length}`; + return Promise.resolve( + streamChunks([ + { type: "start-step" }, + { type: "text-start", id: responseId }, + { + type: "text-delta", + id: responseId, + delta: `Response ${requestMessages.length}`, + }, + { type: "text-end", id: responseId }, + ]), + ); + }), + }; + const submitTextReferences = new Set(); + const stopReferences = new Set(); + const observedConversationIds = new Set(); + const aiAssistant: PetrinautAiAssistant = { + conversationId: "voice-conversation-1", + renderComposerControl: (context) => { + submitTextReferences.add(context.submitText); + stopReferences.add(context.stop); + observedConversationIds.add(context.conversationId); + + return ( + + ); + }, + transport, + }; + + renderTestPanel({ aiAssistant }); + + fireEvent.click( + screen.getByRole("button", { name: "Submit alternate text" }), + ); + await screen.findByText("Response 1"); + + const textarea = screen.getByRole("textbox", { + name: "Message AI assistant", + }); + fireEvent.change(textarea, { target: { value: "Keyboard turn" } }); + fireEvent.keyDown(textarea, { key: "Enter" }); + await screen.findByText("Response 2"); + + expect(chatIds).toEqual(["voice-conversation-1", "voice-conversation-1"]); + expect(requestMessages[0]?.[0]).toMatchObject({ + id: "voice-turn-1", + parts: [{ text: "Alternate turn", type: "text" }], + role: "user", + }); + expect(requestMessages[1]?.at(-1)).toMatchObject({ + parts: [{ text: "Keyboard turn", type: "text" }], + role: "user", + }); + expect(requestMessages[1]?.at(-1)?.id).not.toBe("voice-turn-1"); + expect(submitTextReferences.size).toBe(1); + expect(stopReferences.size).toBe(1); + expect(observedConversationIds).toEqual(new Set(["voice-conversation-1"])); + }); + + test("stops the current response through the host composer control", async () => { + const aborted = vi.fn(); + const transport: PetrinautAiTransport = { + reconnectToStream: () => Promise.resolve(null), + sendMessages: vi.fn( + ({ + abortSignal, + }: Parameters[0]) => + Promise.resolve( + new ReadableStream({ + start(controller) { + controller.enqueue({ type: "start-step" }); + controller.enqueue({ type: "text-start", id: "partial" }); + controller.enqueue({ + type: "text-delta", + id: "partial", + delta: "Partial response", + }); + abortSignal?.addEventListener("abort", () => { + aborted(); + controller.error(new DOMException("Aborted", "AbortError")); + }); + }, + }), + ), + ), + }; + + renderTestPanel({ + aiAssistant: { + renderComposerControl: ({ stop }) => ( + + ), + transport, + }, + initialMessage: "Start a long response", + }); + await screen.findByText("Partial response"); + + fireEvent.click( + screen.getByRole("button", { name: "Stop from host control" }), + ); + + await waitFor(() => expect(aborted).toHaveBeenCalledOnce()); + expect(await screen.findByText("Response stopped")).not.toBeNull(); + }); + + test("maps text to one unresolved host tool before sending another message", async () => { + const requestMessages: PetrinautAiMessage[][] = []; + const transport: PetrinautAiTransport = { + reconnectToStream: () => Promise.resolve(null), + sendMessages: vi.fn(({ messages }) => { + requestMessages.push(structuredClone(messages)); + + if (requestMessages.length === 1) { + return Promise.resolve( + streamChunks([ + { type: "start-step" }, + { + type: "tool-input-available", + dynamic: true, + toolCallId: "question-1", + toolName: "answerQuestion", + input: { question: "Which environment?" }, + }, + ]), + ); + } + + return Promise.resolve( + streamChunks( + textChunks( + `response-${requestMessages.length}`, + requestMessages.length === 2 + ? "Tool answer accepted" + : "Ordinary text accepted", + ), + ), + ); + }), + }; + const results: unknown[] = []; + const hostTool = definePetrinautAiInteractiveTool({ + toolName: "answerQuestion", + inputSchema: { + parse: (raw: unknown) => raw as { question: string }, + }, + outputSchema: { + parse: (raw: unknown) => raw as { answer: string }, + }, + fromComposerText: ({ text }) => ({ answer: text }), + component: ({ input }) => {input.question}, + }); + const aiAssistant: PetrinautAiAssistant = { + interactiveTools: [hostTool], + renderComposerControl: ({ submitText }) => ( + + ), + transport, + }; + + renderTestPanel({ aiAssistant, initialMessage: "Start questions" }); + await screen.findByText("Which environment?"); + + fireEvent.click(screen.getByRole("button", { name: "Submit answer text" })); + await screen.findByText("Tool answer accepted"); + + const followUpParts = requestMessages[1]?.at(-1)?.parts ?? []; + expect( + followUpParts.filter( + (part) => + part.type === "dynamic-tool" && + part.toolCallId === "question-1" && + part.state === "output-available", + ), + ).toEqual([ + expect.objectContaining({ + output: { answer: "Production" }, + toolCallId: "question-1", + }), + ]); + expect( + requestMessages[1]?.some( + (message) => + message.role === "user" && + message.parts.some( + (part) => part.type === "text" && part.text === "Production", + ), + ), + ).toBe(false); + expect(results).toEqual([ + { kind: "interactive-tool", toolCallId: "question-1" }, + ]); + + fireEvent.click(screen.getByRole("button", { name: "Submit answer text" })); + await screen.findByText("Ordinary text accepted"); + + expect(requestMessages[2]?.at(-1)).toMatchObject({ + id: "voice-answer-1", + parts: [{ text: "Production", type: "text" }], + role: "user", + }); + expect(results).toEqual([ + { kind: "interactive-tool", toolCallId: "question-1" }, + { kind: "message", messageId: "voice-answer-1" }, + ]); + }); + + test("falls back to a normal message when a pending tool has no text mapper", async () => { + const requestMessages: PetrinautAiMessage[][] = []; + const transport: PetrinautAiTransport = { + reconnectToStream: () => Promise.resolve(null), + sendMessages: vi.fn(({ messages }) => { + requestMessages.push(structuredClone(messages)); + return Promise.resolve( + streamChunks( + requestMessages.length === 1 + ? [ + { type: "start-step" }, + { + type: "tool-input-available", + dynamic: true, + toolCallId: "confirmation-1", + toolName: "confirmRelease", + input: { question: "Ship this change?" }, + }, + ] + : textChunks("ordinary-response", "Ordinary message received"), + ), + ); + }), + }; + const hostTool = definePetrinautAiInteractiveTool({ + toolName: "confirmRelease", + inputSchema: { + parse: (raw: unknown) => raw as { question: string }, + }, + outputSchema: { + parse: (raw: unknown) => raw as { approved: boolean }, + }, + component: ({ input }) => {input.question}, + }); + + renderTestPanel({ + aiAssistant: { + interactiveTools: [hostTool], + renderComposerControl: ({ submitText }) => ( + + ), + transport, + }, + initialMessage: "Start confirmation", + }); + await screen.findByText("Ship this change?"); + + fireEvent.click( + screen.getByRole("button", { name: "Submit unmapped text" }), + ); + await screen.findByText("Ordinary message received"); + + expect(requestMessages[1]?.at(-1)).toMatchObject({ + id: "ordinary-1", + parts: [{ text: "Not an answer", type: "text" }], + role: "user", + }); + }); + + test("falls back to a normal message for an unknown pending tool", async () => { + const requestMessages: PetrinautAiMessage[][] = []; + const transport: PetrinautAiTransport = { + reconnectToStream: () => Promise.resolve(null), + sendMessages: vi.fn(({ messages }) => { + requestMessages.push(structuredClone(messages)); + return Promise.resolve( + streamChunks(textChunks("ordinary-response", "Message received")), + ); + }), + }; + + renderTestPanel({ + aiAssistant: { + messages: [ + { + id: "assistant-with-unknown-tool", + role: "assistant", + parts: [ + { + type: "dynamic-tool", + state: "input-available", + toolCallId: "unknown-1", + toolName: "unknownTool", + input: {}, + }, + ], + }, + ], + renderComposerControl: ({ submitText }) => ( + + ), + transport, + }, + }); + + fireEvent.click( + screen.getByRole("button", { name: "Submit unknown-tool text" }), + ); + await screen.findByText("Message received"); + + expect(requestMessages[0]?.at(-1)).toMatchObject({ + id: "ordinary-2", + parts: [{ text: "Ordinary text", type: "text" }], + role: "user", + }); + }); + + test("rejects ambiguous pending text mappings without submitting", async () => { + const transport: PetrinautAiTransport = { + reconnectToStream: () => Promise.resolve(null), + sendMessages: vi.fn(() => + Promise.resolve( + streamChunks([ + { type: "start-step" }, + { + type: "tool-input-available", + dynamic: true, + toolCallId: "question-1", + toolName: "answerQuestion", + input: { question: "First question?" }, + }, + { + type: "tool-input-available", + dynamic: true, + toolCallId: "question-2", + toolName: "answerQuestion", + input: { question: "Second question?" }, + }, + ]), + ), + ), + }; + const hostTool = definePetrinautAiInteractiveTool({ + toolName: "answerQuestion", + inputSchema: { + parse: (raw: unknown) => raw as { question: string }, + }, + outputSchema: { + parse: (raw: unknown) => raw as { answer: string }, + }, + fromComposerText: ({ text }) => ({ answer: text }), + component: ({ input }) => {input.question}, + }); + + renderTestPanel({ + aiAssistant: { + interactiveTools: [hostTool], + renderComposerControl: ({ submitText }) => ( + + ), + transport, + }, + initialMessage: "Start ambiguous questions", + }); + await screen.findByText("First question?"); + await screen.findByText("Second question?"); + + fireEvent.click( + screen.getByRole("button", { name: "Submit ambiguous text" }), + ); + + expect( + await screen.findByText( + "Text matches more than one pending interactive AI tool.", + ), + ).not.toBeNull(); + expect(transport.sendMessages).toHaveBeenCalledOnce(); + }); +}); describe("AiAssistantPanel host interactive tools", () => { test("adds one dynamic output and sends one automatic follow-up", async () => { diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx index ee333801bff..c9c323c8533 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx @@ -1,6 +1,6 @@ import { useChat } from "@ai-sdk/react"; -import { lastAssistantMessageIsCompleteWithToolCalls } from "ai"; -import { use, useEffect, useRef, useState } from "react"; +import { generateId, lastAssistantMessageIsCompleteWithToolCalls } from "ai"; +import { use, useCallback, useEffect, useRef, useState } from "react"; import { aiCommandActionInputSchemas, @@ -54,6 +54,10 @@ import { } from "./ai-assistant-panel/tool-summaries"; import type { PetrinautAiAssistant } from "../../../petrinaut"; +import type { + PetrinautAiComposerControlContext, + PetrinautAiComposerSubmitTextResult, +} from "../../../types/ai-assistant-composer-control"; import type { PetrinautAiMessage } from "./ai-assistant-panel/types"; export type { @@ -322,6 +326,9 @@ export const AiAssistantPanel = ({ status, stop, } = useChat({ + ...(aiAssistant.conversationId === undefined + ? {} + : { id: aiAssistant.conversationId }), messages: aiAssistant.messages, transport: diagnosticsTransportState.transport, sendAutomaticallyWhen: lastAssistantMessageIsCompleteWithToolCalls, @@ -557,6 +564,197 @@ export const AiAssistantPanel = ({ }, }); + const composerSubmissionStateRef = useRef({ + addToolOutput, + interactiveTools: aiAssistant.interactiveTools, + messages, + sendMessage, + status, + }); + useEffect(() => { + composerSubmissionStateRef.current = { + addToolOutput, + interactiveTools: aiAssistant.interactiveTools, + messages, + sendMessage, + status, + }; + }, [ + addToolOutput, + aiAssistant.interactiveTools, + messages, + sendMessage, + status, + ]); + + const composerToolSubmissionsRef = useRef(new Set()); + useEffect(() => { + const pendingToolCallIds = new Set(); + for (const message of messages) { + for (const part of message.parts) { + if (part.type === "dynamic-tool" && part.state === "input-available") { + pendingToolCallIds.add(part.toolCallId); + } + } + } + + for (const toolCallId of composerToolSubmissionsRef.current) { + if (!pendingToolCallIds.has(toolCallId)) { + composerToolSubmissionsRef.current.delete(toolCallId); + } + } + }, [messages]); + + // Hosts retain this callback across transport-driven rerenders, so its + // identity is part of the public composer-control contract. The ref keeps + // its implementation current without forcing host controls to resubscribe. + const submitText = useCallback( + async ({ + id, + text, + }: { + id?: string; + text: string; + }): Promise => { + const trimmed = text.trim(); + if (!trimmed) { + const submissionError = new Error( + "AI assistant text must not be empty.", + ); + setStreamError(submissionError); + throw submissionError; + } + + const { + addToolOutput: submitToolOutput, + interactiveTools, + messages: currentMessages, + sendMessage: submitMessage, + status: currentStatus, + } = composerSubmissionStateRef.current; + if (currentStatus === "submitted" || currentStatus === "streaming") { + const submissionError = new Error( + "Wait for the current AI response before submitting more text.", + ); + setStreamError(submissionError); + throw submissionError; + } + + const mappedToolCalls: { + input: unknown; + mapText: (params: { input: unknown; text: string }) => unknown; + toolCallId: string; + toolName: string; + }[] = []; + for (const message of currentMessages) { + for (const part of message.parts) { + if ( + part.type !== "dynamic-tool" || + part.state !== "input-available" + ) { + continue; + } + + const definition = getInteractiveTool( + part.toolName, + part.input, + interactiveTools, + ); + if (!definition?.fromComposerText) { + continue; + } + + mappedToolCalls.push({ + input: part.input, + mapText: definition.fromComposerText, + toolCallId: part.toolCallId, + toolName: part.toolName, + }); + } + } + + if (mappedToolCalls.length > 1) { + const submissionError = new Error( + "Text matches more than one pending interactive AI tool.", + ); + setStreamError(submissionError); + throw submissionError; + } + + const mappedToolCall = mappedToolCalls[0]; + if (mappedToolCall) { + if (composerToolSubmissionsRef.current.has(mappedToolCall.toolCallId)) { + const submissionError = new Error( + "This interactive AI tool is already being submitted.", + ); + setStreamError(submissionError); + throw submissionError; + } + + let output: unknown; + try { + output = mappedToolCall.mapText({ + input: mappedToolCall.input, + text: trimmed, + }); + } catch (caught) { + const submissionError = + caught instanceof Error ? caught : new Error(String(caught)); + setStreamError(submissionError); + throw submissionError; + } + + setInput(""); + setStreamError(null); + setStopped(false); + stopRequestedRef.current = false; + composerToolSubmissionsRef.current.add(mappedToolCall.toolCallId); + try { + await addDynamicToolOutput(submitToolOutput, { + output, + tool: mappedToolCall.toolName, + toolCallId: mappedToolCall.toolCallId, + }); + } catch (caught) { + composerToolSubmissionsRef.current.delete(mappedToolCall.toolCallId); + const submissionError = + caught instanceof Error ? caught : new Error(String(caught)); + setStreamError(submissionError); + throw submissionError; + } + + return { + kind: "interactive-tool", + toolCallId: mappedToolCall.toolCallId, + }; + } + + const messageId = id ?? generateId(); + setInput(""); + setStreamError(null); + setStopped(false); + stopRequestedRef.current = false; + await submitMessage({ + id: messageId, + parts: [{ text: trimmed, type: "text" }], + role: "user", + }); + return { kind: "message", messageId }; + }, + [], + ); + + const stopStateRef = useRef(stop); + useEffect(() => { + stopStateRef.current = stop; + }, [stop]); + + // Like submitText, stop is exposed to host controls and must stay stable. + const stopComposer = useCallback(async () => { + stopRequestedRef.current = true; + await stopStateRef.current(); + }, []); + useEffect(() => { const trimmedInitialMessage = initialMessage?.trim(); if (!trimmedInitialMessage) { @@ -606,8 +804,24 @@ export const AiAssistantPanel = ({ : STARTER_CHIPS : REVIEW_CHIPS; + const composerControlContext: PetrinautAiComposerControlContext = { + conversationId: aiAssistant.conversationId, + messages, + status, + stop: stopComposer, + submitText, + }; + /* eslint-disable react-hooks-js/refs -- The public render prop receives + stable event callbacks that read their refs only when the host invokes + them from an event handler or effect. */ + const composerControl = aiAssistant.renderComposerControl?.( + composerControlContext, + ); + /* eslint-enable react-hooks-js/refs */ + return ( { - const trimmed = prompt.trim(); - if (!trimmed) { - return; - } - setInput(""); - setStreamError(null); - setStopped(false); - stopRequestedRef.current = false; - void sendMessage({ text: trimmed }); + void submitText({ text: prompt }).catch(() => {}); }} onStop={() => { // Flag the deliberate stop, then abort. The actual settling of the // partial transcript and the "Response stopped" note happen in // `onFinish`, once the SDK has fully unwound the stream — finalizing // here would race the chunks the SDK is still flushing. - stopRequestedRef.current = true; - void stop(); + void stopComposer(); }} onSubmit={() => { - const trimmed = input.trim(); - if (!trimmed) { - return; - } - setInput(""); - setStreamError(null); - setStopped(false); - stopRequestedRef.current = false; - void sendMessage({ text: trimmed }); + void submitText({ text: input }).catch(() => {}); }} promptChips={promptChips} rightOffset={hasSelection ? propertiesPanelWidth + PANEL_MARGIN : 0} diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx index 62b0f0a53e7..c2467a8c9ed 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx @@ -68,6 +68,34 @@ describe("AiAssistantContents", () => { expect(renderMarkdown).toHaveBeenCalledOnce(); }); + test("renders a host composer control between the textarea and send button", () => { + render( + + Alternate + + } + input="" + messages={[]} + onClose={noop} + onInputChange={noop} + onStop={noop} + onSubmit={noop} + status="ready" + />, + ); + + const textarea = screen.getByRole("textbox", { + name: "Message AI assistant", + }); + const control = screen.getByRole("button", { name: "Alternate input" }); + const sendButton = screen.getByRole("button", { name: "Send message" }); + + expect(textarea.nextElementSibling).toBe(control); + expect(control.nextElementSibling?.contains(sendButton)).toBe(true); + }); + test("renders a host interactive tool and submits its validated output once", () => { const parseOutput = vi.fn((raw: unknown) => { if ( diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.tsx index 4bc85d1ac7e..40dbb4c9890 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.tsx @@ -1,4 +1,11 @@ -import { memo, type RefObject, useEffect, useRef, useState } from "react"; +import { + memo, + type ReactNode, + type RefObject, + useEffect, + useRef, + useState, +} from "react"; import ReactMarkdown from "react-markdown"; import { Button } from "@hashintel/ds-components"; @@ -27,6 +34,7 @@ type AiAssistantStatus = "submitted" | "streaming" | "ready" | "error"; const EMPTY_INTERACTIVE_TOOLS: readonly PetrinautAiInteractiveTool[] = []; export type AiAssistantContentsProps = { + composerControl?: ReactNode; error?: Error; input: string; interactiveTools?: readonly PetrinautAiInteractiveTool[]; @@ -381,6 +389,7 @@ const AiAssistantMessage = memo( AiAssistantMessage.displayName = "AiAssistantMessage"; export const AiAssistantContents = ({ + composerControl, error, input, interactiveTools = EMPTY_INTERACTIVE_TOOLS, @@ -588,6 +597,7 @@ export const AiAssistantContents = ({ } aria-label="Message AI assistant" /> + {composerControl} + ), + transport, + }, + initialMessage: "Start questions", + }); + await screen.findByText("Which environment?"); + + fireEvent.click(screen.getByRole("button", { name: "Submit correction" })); + await screen.findByText("Correction accepted"); + + expect(requestMessages[1]?.at(-1)).toMatchObject({ + id: "correction-1", + parts: [{ text: "Correction: staging, not production.", type: "text" }], + role: "user", + }); + expect( + requestMessages[1]?.some((message) => + message.parts.some( + (part) => + part.type === "dynamic-tool" && + part.toolCallId === "question-1" && + part.state === "output-available", + ), + ), + ).toBe(false); + expect(results).toEqual([{ kind: "message", messageId: "correction-1" }]); + }); + test("falls back to a normal message when a pending tool has no text mapper", async () => { const requestMessages: PetrinautAiMessage[][] = []; const transport: PetrinautAiTransport = { diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx index c9c323c8533..cb64250318f 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx @@ -611,9 +611,11 @@ export const AiAssistantPanel = ({ const submitText = useCallback( async ({ id, + target = "auto", text, }: { id?: string; + target?: "auto" | "message"; text: string; }): Promise => { const trimmed = text.trim(); @@ -646,7 +648,7 @@ export const AiAssistantPanel = ({ toolCallId: string; toolName: string; }[] = []; - for (const message of currentMessages) { + for (const message of target === "auto" ? currentMessages : []) { for (const part of message.parts) { if ( part.type !== "dynamic-tool" || From 7273cef55b5b51a5168d2943e7881a62d7641b8e Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Thu, 27 Aug 2026 01:00:44 +0200 Subject: [PATCH 3/6] Keep composer controls on the current conversation --- .../adr/0009-openai-voice-ui-turn-shell.md | 7 +- .../docs/specs/petrinaut-integration.md | 24 ++-- .../@hashintel/petrinaut/src/ui/petrinaut.tsx | 2 +- .../ui/types/ai-assistant-composer-control.ts | 3 +- .../Editor/panels/ai-assistant-panel.test.tsx | 105 +++++++++++++++++- .../Editor/panels/ai-assistant-panel.tsx | 30 ++--- 6 files changed, 129 insertions(+), 42 deletions(-) diff --git a/libs/@hashintel/brunch-agent/docs/adr/0009-openai-voice-ui-turn-shell.md b/libs/@hashintel/brunch-agent/docs/adr/0009-openai-voice-ui-turn-shell.md index 4ab2bb4e6ba..2ffcd99e737 100644 --- a/libs/@hashintel/brunch-agent/docs/adr/0009-openai-voice-ui-turn-shell.md +++ b/libs/@hashintel/brunch-agent/docs/adr/0009-openai-voice-ui-turn-shell.md @@ -58,9 +58,10 @@ production recovery or public availability. ## Consequences - Petrinaut's public API gains a generic host control with stable `submitText`, `stop`, messages, - status, and optional conversation identity. Interactive tools may opt into a schema-validated - text-to-output mapper. Keyboard and alternate finalized text therefore cannot bypass ask - correlation by default; the host may explicitly target a separate message for a correction. + status, and the effective host-supplied or generated conversation identity. Interactive tools may + opt into a schema-validated text-to-output mapper. Keyboard and alternate finalized text therefore + cannot bypass ask correlation by default; the host may explicitly target a separate message for + a correction. - OpenAI implementation names and policy stay in `apps/petrinaut-website`. The existing `transport-aisdk` package remains the sole browser-to-Brunch conversation transport. - Preview PRs may demonstrate transcription and exact canonical speech before production diff --git a/libs/@hashintel/brunch-agent/docs/specs/petrinaut-integration.md b/libs/@hashintel/brunch-agent/docs/specs/petrinaut-integration.md index 4ff9a09be8c..bbf7c2cc20b 100644 --- a/libs/@hashintel/brunch-agent/docs/specs/petrinaut-integration.md +++ b/libs/@hashintel/brunch-agent/docs/specs/petrinaut-integration.md @@ -46,11 +46,12 @@ which the design needs anyway: 4. **The artifact seam** (`parseSDCPNFile` / `sdcpnFileSchema`): unchanged; net validity checked in CI through the pure parser. 5. **The generic composer seam**: a host may render a control beside Petrinaut's text composer and - receive stable `submitText` and `stop` callbacks plus conversation state. Finalized alternate - text uses the same AI SDK `useChat` instance as keyboard input. When exactly one unresolved - interactive tool registers a schema-validated text mapper, `submitText` completes that tool; - otherwise it submits a stable-ID user message. Ambiguous mapped tools are refused. A host may - explicitly target an ordinary message for a correction that must not answer the pending tool. + receive stable `submitText` and `stop` callbacks plus the effective AI SDK conversation identity + and state. Finalized alternate text uses the same AI SDK `useChat` instance as keyboard input. + When exactly one unresolved interactive tool registers a schema-validated text mapper, + `submitText` completes that tool; otherwise it submits a stable-ID user message. Ambiguous mapped + tools are refused. A host may explicitly target an ordinary message for a correction that must + not answer the pending tool. ## Attach Contract @@ -66,11 +67,11 @@ The panel and the voice edge attach to Brunch through one stable surface: stable across reloads; replacing the local UID with authenticated identity must preserve the same request-level ownership semantics. 4. **Composer submission**: Petrinaut accepts an optional stable conversation ID and host composer - control. Keyboard and alternate finalized text both enter the same `submitText` function. A - pending `brunch_ask` is answered only through the existing correlated tool-output path; text is - not silently downgraded to an ordinary user message when more than one mapped ask is pending. - Explicit corrections target new messages rather than silently mutating or answering another - pending ask. + control, then exposes the effective host-supplied or generated identity to that control. Keyboard + and alternate finalized text both enter the same `submitText` function. A pending `brunch_ask` + is answered only through the existing correlated tool-output path; text is not silently + downgraded to an ordinary user message when more than one mapped ask is pending. Explicit + corrections target new messages rather than silently mutating or answering another pending ask. These four parts change only with notice to the panel and voice-edge owners. A provider-specific voice requirement does not silently alter this surface; provider code and policy remain in the @@ -177,7 +178,8 @@ host application under ADR-0009, while reusable Petrinaut and Brunch packages st **Generic composer control** - `@hashintel/petrinaut` accepts an optional conversation ID and host render callback. The callback - receives current AI SDK messages and status plus stable `submitText` and `stop` functions. + receives the effective host-supplied or generated AI SDK conversation identity, current messages + and status, plus stable `submitText` and `stop` functions. - A host interactive tool may define `fromComposerText({ input, text })`. Petrinaut parses the pending input, invokes the mapper, and parses its output before submitting the correlated tool result. Unknown or unmapped tools preserve ordinary message submission; multiple eligible tools diff --git a/libs/@hashintel/petrinaut/src/ui/petrinaut.tsx b/libs/@hashintel/petrinaut/src/ui/petrinaut.tsx index e639475599e..f2de6e6d265 100644 --- a/libs/@hashintel/petrinaut/src/ui/petrinaut.tsx +++ b/libs/@hashintel/petrinaut/src/ui/petrinaut.tsx @@ -39,7 +39,7 @@ import type { export type PetrinautAiChatTransport = PetrinautAiTransport; export type PetrinautAiAssistant = { - /** Stable identity for this host-owned conversation. */ + /** Optional host-owned identity; `useChat` generates one when omitted. */ conversationId?: string; /** Host-owned dynamic tools that render inline in the AI conversation. */ interactiveTools?: readonly PetrinautAiInteractiveTool[]; diff --git a/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts b/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts index 8a3904a58a7..d1fb912ae5d 100644 --- a/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts +++ b/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts @@ -15,7 +15,8 @@ export type PetrinautAiComposerSubmitTextResult = /** Stable controls and current conversation state supplied to a host control. */ export type PetrinautAiComposerControlContext = { - conversationId?: string; + /** Effective AI SDK identity, whether host-supplied or generated by `useChat`. */ + conversationId: string; messages: PetrinautAiMessage[]; status: PetrinautAiComposerStatus; /** Call from an event handler or effect, never while rendering. */ diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx index b45a6024064..51c3db8bc4d 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx @@ -8,6 +8,7 @@ import { screen, waitFor, } from "@testing-library/react"; +import { useEffect } from "react"; import { afterEach, describe, expect, test, vi } from "vitest"; import { @@ -31,6 +32,7 @@ import { definePetrinautAiInteractiveTool } from "../../../types/ai-interactive- import { AiAssistantPanel } from "./ai-assistant-panel"; import type { PetrinautAiAssistant } from "../../../petrinaut"; +import type { PetrinautAiComposerControlContext } from "../../../types/ai-assistant-composer-control"; import type { PetrinautAiMessage, PetrinautAiTransport, @@ -112,6 +114,22 @@ const textChunks = (id: string, text: string): UIMessageChunk[] => [ { type: "text-end", id }, ]; +const SubmitForSecondConversation = ({ + conversationId, + submitText, +}: Pick< + PetrinautAiComposerControlContext, + "conversationId" | "submitText" +>) => { + useEffect(() => { + if (conversationId === "conversation-2") { + void submitText({ id: "second-turn", text: "Second conversation" }); + } + }, [conversationId, submitText]); + + return null; +}; + const testInstances: ReturnType[] = []; const renderTestPanel = ({ @@ -140,18 +158,25 @@ const renderTestPanel = ({ getItemType: () => null, }; - render( + const renderPanel = (nextAiAssistant: PetrinautAiAssistant) => ( - , + ); + const rendered = render(renderPanel(aiAssistant)); + + return { + ...rendered, + rerenderPanel: (nextAiAssistant: PetrinautAiAssistant) => + rendered.rerender(renderPanel(nextAiAssistant)), + }; }; afterEach(() => { @@ -162,6 +187,80 @@ afterEach(() => { }); describe("AiAssistantPanel composer submissions", () => { + test("exposes the generated useChat conversation identity to host controls", async () => { + const chatIds: string[] = []; + const observedConversationIds = new Set(); + const transport: PetrinautAiTransport = { + reconnectToStream: () => Promise.resolve(null), + sendMessages: vi.fn(({ chatId }) => { + chatIds.push(chatId); + return Promise.resolve( + streamChunks( + textChunks("generated-id-response", "Generated ID used"), + ), + ); + }), + }; + + renderTestPanel({ + aiAssistant: { + renderComposerControl: (context) => { + observedConversationIds.add(context.conversationId); + return ( + + ); + }, + transport, + }, + }); + + fireEvent.click( + screen.getByRole("button", { name: "Submit with generated identity" }), + ); + await screen.findByText("Generated ID used"); + + expect(chatIds).toHaveLength(1); + expect(chatIds[0]).toBeTruthy(); + expect(observedConversationIds).toEqual(new Set(chatIds)); + }); + + test("submits to the current chat when the conversation identity changes", async () => { + const chatIds: string[] = []; + const transport: PetrinautAiTransport = { + reconnectToStream: () => Promise.resolve(null), + sendMessages: vi.fn(({ chatId }) => { + chatIds.push(chatId); + return Promise.resolve( + streamChunks(textChunks("second-response", "Second chat used")), + ); + }), + }; + const createAiAssistant = ( + conversationId: string, + ): PetrinautAiAssistant => ({ + conversationId, + renderComposerControl: (context) => ( + + ), + transport, + }); + const rendered = renderTestPanel({ + aiAssistant: createAiAssistant("conversation-1"), + }); + + rendered.rerenderPanel(createAiAssistant("conversation-2")); + await waitFor(() => expect(chatIds).toHaveLength(1)); + + expect(chatIds).toEqual(["conversation-2"]); + }); + test("shares one stable submission path between alternate and keyboard text", async () => { const requestMessages: PetrinautAiMessage[][] = []; const chatIds: string[] = []; diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx index cb64250318f..6bd6394dbea 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx @@ -17,6 +17,7 @@ import { setNetTitleToolName, } from "@hashintel/petrinaut-core"; +import { useLatest } from "../../../../react/hooks/use-latest"; import { PetrinautInstanceContext } from "../../../../react/instance-context"; import { LanguageClientContext } from "../../../../react/lsp/context"; import { @@ -319,6 +320,7 @@ export const AiAssistantPanel = ({ const { error, + id: conversationId, messages, addToolOutput, sendMessage, @@ -564,28 +566,13 @@ export const AiAssistantPanel = ({ }, }); - const composerSubmissionStateRef = useRef({ + const composerSubmissionStateRef = useLatest({ addToolOutput, interactiveTools: aiAssistant.interactiveTools, messages, sendMessage, status, }); - useEffect(() => { - composerSubmissionStateRef.current = { - addToolOutput, - interactiveTools: aiAssistant.interactiveTools, - messages, - sendMessage, - status, - }; - }, [ - addToolOutput, - aiAssistant.interactiveTools, - messages, - sendMessage, - status, - ]); const composerToolSubmissionsRef = useRef(new Set()); useEffect(() => { @@ -743,19 +730,16 @@ export const AiAssistantPanel = ({ }); return { kind: "message", messageId }; }, - [], + [composerSubmissionStateRef], ); - const stopStateRef = useRef(stop); - useEffect(() => { - stopStateRef.current = stop; - }, [stop]); + const stopStateRef = useLatest(stop); // Like submitText, stop is exposed to host controls and must stay stable. const stopComposer = useCallback(async () => { stopRequestedRef.current = true; await stopStateRef.current(); - }, []); + }, [stopStateRef]); useEffect(() => { const trimmedInitialMessage = initialMessage?.trim(); @@ -807,7 +791,7 @@ export const AiAssistantPanel = ({ : REVIEW_CHIPS; const composerControlContext: PetrinautAiComposerControlContext = { - conversationId: aiAssistant.conversationId, + conversationId, messages, status, stop: stopComposer, From 71ad0cc9b0feb459bf0098ca53309da27a2d3e67 Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Thu, 27 Aug 2026 14:28:23 +0200 Subject: [PATCH 4/6] Prevent host controls from submitting composer drafts Co-authored-by: Cursor --- .../ai-assistant-contents.test.tsx | 32 +++++++++++++++++++ .../ai-assistant-contents.tsx | 7 +++- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx index c2467a8c9ed..80d7ea3a7f5 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx @@ -96,6 +96,38 @@ describe("AiAssistantContents", () => { expect(control.nextElementSibling?.contains(sendButton)).toBe(true); }); + test("does not submit the draft when a host composer button omits its type", () => { + const onSubmit = vi.fn(); + render( + , + ); + + fireEvent.click( + screen.getByRole("button", { + name: "Alternate input", + }), + ); + + expect(onSubmit).not.toHaveBeenCalled(); + + fireEvent.click(screen.getByRole("button", { name: "Send message" })); + expect(onSubmit).toHaveBeenCalledOnce(); + }); + test("renders a host interactive tool and submits its validated output once", () => { const parseOutput = vi.fn((raw: unknown) => { if ( diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.tsx index 40dbb4c9890..76b0d154f9a 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.tsx @@ -559,7 +559,11 @@ export const AiAssistantContents = ({
{ event.preventDefault(); - if (canSubmit) { + const submitter = (event.nativeEvent as SubmitEvent).submitter; + if ( + canSubmit && + submitter?.hasAttribute("data-ai-assistant-submit") + ) { onSubmit(); } }} @@ -599,6 +603,7 @@ export const AiAssistantContents = ({ /> {composerControl} + ), + }); + + renderTestPanel({ + aiAssistant: { + interactiveTools: [hostTool], + renderComposerControl: ({ status, stop }) => ( + <> + + {status} + + ), + transport, + }, + initialMessage: "Ask for confirmation", + }); + + await screen.findByRole("button", { name: "Continue?" }); + await waitFor(() => + expect(screen.getByTestId("host-status").textContent).toBe("ready"), + ); + + fireEvent.click(screen.getByRole("button", { name: "Stop while idle" })); + fireEvent.click(screen.getByRole("button", { name: "Continue?" })); + + await waitFor(() => + expect(transport.sendMessages).toHaveBeenCalledTimes(2), + ); + await waitFor(() => + expect(screen.getByTestId("host-status").textContent).toBe("ready"), + ); + expect(screen.queryByText("Response stopped")).toBeNull(); + }); + test("maps text to one unresolved host tool before sending another message", async () => { const requestMessages: PetrinautAiMessage[][] = []; const transport: PetrinautAiTransport = { diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx index 3c77cb0d13e..39af27effd6 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx @@ -733,12 +733,18 @@ export const AiAssistantPanel = ({ [composerSubmissionStateRef], ); - const stopStateRef = useLatest(stop); + const stopStateRef = useLatest({ status, stop }); // Like submitText, stop is exposed to host controls and must stay stable. const stopComposer = useCallback(async () => { + const { status: currentStatus, stop: stopCurrentResponse } = + stopStateRef.current; + if (currentStatus !== "submitted" && currentStatus !== "streaming") { + return; + } + stopRequestedRef.current = true; - await stopStateRef.current(); + await stopCurrentResponse(); }, [stopStateRef]); useEffect(() => {