From f4adfded2e87fc7d83dde3f1ab972b46c65a56a8 Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Thu, 3 Sep 2026 23:58:47 +0200 Subject: [PATCH 01/52] Authorize Voice safety on the unified Flue route Amp-Thread-ID: https://ampcode.com/threads/T-01a06941-3ee7-7482-a82e-8a16c7141785 Co-authored-by: Amp --- libs/@hashintel/brunch-agent/MISSION.md | 198 ++++++++---------- .../donor-behavior-matrix.md | 54 +++++ 2 files changed, 138 insertions(+), 114 deletions(-) create mode 100644 libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/donor-behavior-matrix.md diff --git a/libs/@hashintel/brunch-agent/MISSION.md b/libs/@hashintel/brunch-agent/MISSION.md index 2d09a6215ad..8c38f0298bf 100644 --- a/libs/@hashintel/brunch-agent/MISSION.md +++ b/libs/@hashintel/brunch-agent/MISSION.md @@ -1,163 +1,133 @@ -# Mission 5 — one Flue conversation route for Voice and the typed panel +# Mission 5 successor — Voice safety and UX parity on the unified Flue route ## Status -**Live as of 2026-09-03** for [FE-1574](https://linear.app/hash/issue/FE-1574/let-voice-speak-through-canonical-brunch-conversations) on `ln/fe-1574-direct-voice-flue`, stacked directly on the closed Mission 4 branch. This is the sole execution authority for the branch. The builder implementation now routes typed panel and finalized Voice turns through one browser `FlueClient` at `/agents/chat/:instanceId`, projects canonical replies into Petrinaut and TTS, uses durable Flue abort for explicit Stop, and rehydrates canonical conversation state through SDK observation. The former Brunch `/api/chat` handler and projector are deleted. Mission acceptance remains open until the required human demo and proof-leaf-8 bundle exist. +**Live as of 2026-09-03** for [FE-1580](https://linear.app/hash/issue/FE-1580/harden-voice-safety-and-ux-on-the-unified-flue-route) on `kostandin/fe-1580-harden-voice-safety-and-ux-on-the-unified-flue-route`, stacked directly on [PR #9528](https://github.com/hashintel/hash/pull/9528) at the GitHub-verified head `036fb06ee83e840fd7a87dd1bc758a2dd49ddd4c`. This file is the sole execution authority for the successor branch. The parent remains the authority for its own admission/Stop races, stream cancellation, hydration overwrite, client-tool classification, response/submission correlation, and PR/CI defects; this branch must restack onto parent fixes rather than repair them. -The accepted departure base remains Mission 4's package-composed `ChatAgent`: `useBrunchAgent()` mounts core's independent `elicitation` capability and `useSdcpnPlugin()` mounts the SDCPN job contribution. `@hashintel/brunch-agent-transport-aisdk` is now the browser adapter over public `@flue/sdk`, not a server handler. The current external Voice evidence remains PR [#9496](https://github.com/hashintel/hash/pull/9496) at `c7fe8a2e68e8fdc37018b21ec2e9daf4e9ef7c82`, PR [#9507](https://github.com/hashintel/hash/pull/9507) at `252b9dbb0c77fae8cee45a506f09cac3e20c381c`, and PR [#9512](https://github.com/hashintel/hash/pull/9512) at `d13535d1077b3a78d6a1411031b7d0a0a78e3144`. They are read-only source evidence, not branches to rewrite or wholesale architecture to restore. +The owner selected **half-duplex turn ownership** on 2026-09-03. While canonical assistant audio is pending or playing, the microphone is closed. The explicit **Your turn** action cancels output, waits for provider acknowledgement and response settlement, and only then opens a fresh input turn. Automatic duplex barge-in is rejected because assistant playback can become a false user turn. -Cold-start reads are [`docs/evidence/implementations/mission-4-voice-integration-handoff.md`](docs/evidence/implementations/mission-4-voice-integration-handoff.md), [`packages/transport-aisdk/src/index.ts`](packages/transport-aisdk/src/index.ts), [`packages/transport-aisdk/src/ui-stream.ts`](packages/transport-aisdk/src/ui-stream.ts), [`packages/transport-aisdk/src/transcript.ts`](packages/transport-aisdk/src/transcript.ts), [`apps/brunch-agent/src/app.ts`](../../../apps/brunch-agent/src/app.ts), [`apps/brunch-agent/src/http/ownership.ts`](../../../apps/brunch-agent/src/http/ownership.ts), [`apps/brunch-agent/test/petrinaut-chat.integration.ts`](../../../apps/brunch-agent/test/petrinaut-chat.integration.ts), [`apps/brunch-agent/test/architecture/boundaries.integration.ts`](../../../apps/brunch-agent/test/architecture/boundaries.integration.ts), [`apps/petrinaut-website/src/main/app/local-storage-demo/`](../../../apps/petrinaut-website/src/main/app/local-storage-demo/), [`apps/petrinaut-website/src/main/app/voice-interview/`](../../../apps/petrinaut-website/src/main/app/voice-interview/), and the installed Flue 2.0.3 documentation at `node_modules/@flue/sdk/docs/sdk/flue-client.md`, `node_modules/@flue/sdk/docs/reference/streaming-protocol.md`, and `node_modules/@flue/sdk/docs/guide/react.md`. +### Turn-ownership decision -## Imperative - -Make the mounted Flue conversation route the only product door into a Brunch conversation, and make Voice a faithful audio projection of that one canonical conversation. One finalized spoken answer and one typed panel message must both enter the owning Flue conversation through `@flue/sdk` against `/agents/chat/:instanceId`, and the corresponding canonical Brunch response must reach visible text and TTS without another model rewriting the text. Do this now because Mission 4 established the canonical agent composition while two transports still exist to the same conversation: the Voice preview's AI SDK composer path and the server-side `/api/chat` adapter, which admits through a different code path than the SDK does. Routing Voice onto Flue while keeping `/api/chat` for typed text would harden the split into two routes, two ownership rules, and two protocols; the least mechanism is one route, with the AI SDK reduced to the panel's rendering contract behind a host-supplied browser `ChatTransport`. - -### Product-manager litmus +1. **Adopted — half-duplex explicit handoff.** It gives assistant playback exclusive ownership, makes cancellation settlement a visible boundary, and guarantees fresh post-handoff capture. The product cost is one extra **Your turn** action and barrier latency when the user interrupts. +2. **Rejected — automatic duplex barge-in.** It offers the most conversational interruption and avoids an explicit control, but an open microphone can transcribe assistant playback as a user answer, and capture can race an unsettled cancellation. This risk is unacceptable for authoritative completed transcripts. -Adopted on restack onto the parent spine's 2026-09-03 litmus reframing. A product manager who did not watch the work must be able to notice the advance; the single-route consolidation, the browser `ChatTransport`, the deleted `/api/chat` door, and the repurposed transport package are internal sequencing and must not be presented as the advance. +Sections independent of canonical hydration may proceed. The hard-reload witness is blocked until the parent prevents its once-per-conversation hydration from overwriting a locally submitted turn. Prepare that witness but do not patch the parent defect here. -**Release note:** in the Petrinaut Brunch panel you can type or speak to Brunch in one conversation; what you hear is exactly what Brunch wrote; **Stop** really stops Brunch rather than just hiding its answer; and reopening the panel shows the same conversation you left, without re-sending or replaying anything. +The pinned donor-behavior decision record is the [FE-1580 donor matrix](docs/evidence/implementations/mission-5-voice-safety-parity/donor-behavior-matrix.md). Donor PRs are read-only evidence at their named heads; semantic reimplementation is required, never merge or cherry-pick. -**Demo script (no engineer present), on the deployment posture available at cut time — the local `yarn dev:brunch` pair with the Brunch preview selected:** open the panel and type one message; read the reply. Start Voice mode and speak one answer; see exactly one new user message appear, then see Brunch's reply appear as text and hear the same words read aloud. Speak over it once; playback stops and the text stays. Ask a second question and press **Stop** while Brunch is still working; the conversation shows that turn as stopped, not as an answer. Close the panel and reopen the same conversation: the typed turn, the spoken turn, and the stopped turn are all there exactly as you saw them, nothing replays, and nothing is sent again. +## Imperative -**Previously impossible:** Stop only cancelled the browser request while Brunch kept working, so reopening the panel showed a full answer you had stopped; typed and spoken turns entered Brunch through different doors, so a spoken turn could be held or ordered differently from a typed one. +Make Voice safe and product-complete on the one Flue conversation route established by the parent. Only a completed provider transcription may become a spoken answer; one logical typed or Voice delivery must admit at most one Flue turn; assistant output must yield the microphone through an acknowledged cancellation barrier; exact canonical responses must be replayable; and Voice attribution must survive canonical hydration and reopen. -**Completion:** the mission is complete at the contract stratum below — when a product manager can run this demo script end to end and proof leaf 8's witness bundle records it — not when the first typed or spoken turn crosses the route. The first green typed-panel tracer and the first green Voice tracer are internal milestones. +Voice path B is the **only admissible submission shape**: -### Recut rationale +```text +Voice completed transcript +→ Voice controller validates one keyed transcript identity +→ panel submitVoiceInputWithAdmission +→ panel submitVoiceInput +→ shared useChat submitText +→ host-supplied Flue ChatTransport +→ client.send({ message, idempotencyKey, signal }) +→ /agents/chat/:instanceId +``` -Inspected at the real boundary on 2026-09-03 (`node_modules/@flue/sdk/docs/reference/streaming-protocol.md`, `packages/transport-aisdk/src/index.ts`, `packages/transport-aisdk/src/ui-stream.ts`, `apps/petrinaut-website/src/main/app/local-storage-demo/brunch-panel-transport.ts`, `node_modules/ai/dist/index.d.ts` `ChatTransport`): +Voice may not call `FlueClient.send()` directly and may not own a second mutable transcript. A direct-send fallback would recreate the second admission path this stack exists to remove. -- Flue's SSE does not remove the need for translation while the panel speaks `useChat`: Flue admits one `DeliveredMessage` with a 202 and streams `ConversationStreamChunk` batches on a separate, never-ending offset-resumed read; the AI SDK posts the whole `messages` array and expects one finite `UIMessageChunk` stream per turn. Request shape, vocabulary, and lifecycle all differ, and `@flue/*` ships no AI SDK adapter. -- At the recut, the adapter was one translation cut across two homes by the transport topology gate: AI SDK request framing, CORS, and principal parsing in `transport-aisdk` (then Flue-free), and the load-bearing `ConversationStreamChunk → UIMessageChunk` projection plus snapshot → UI messages in `apps/brunch-agent/src/conversation/`. -- A Hono-level relocation would have re-expressed the same admit → follow → project → terminate join as a server handler; it would have removed nothing and kept two routes. It was rejected. -- AI SDK `ChatTransport` is a client-side interface, and Petrinaut already accepts and wraps host-supplied transports. The landed browser transport over the same `createFlueClient()` that Voice uses owns the projector, removes `/api/chat` and its handler, and makes typed and spoken turns share one admission path, one ownership guard, and one protocol. +The parent's claim that Flue 2.0.3 cannot accept caller idempotency is false. The installed `@flue/sdk` 2.0.3 typings expose `AgentPromptOptions.idempotencyKey?: string`, `AgentSendResult.deduplicated?: boolean`, and the 409 `submission_conflict` response with the existing `submissionId` in `FlueApiError.body.error.meta.submissionId`. The invariant is **at most one admitted turn**, not exactly one invocation of `send()`. ## Throughline -The real boundary is the local Petrinaut surface driven by `yarn dev:brunch` — both its typed panel and its Voice control — through one same-origin-proxied Flue route into the mounted production `ChatAgent`, back to visible Petrinaut text and, for Voice, audible playback: +The production throughline is the local Petrinaut Brunch surface driven by `yarn dev:brunch`: ```text - Petrinaut typed panel (`useChat`) Petrinaut microphone - → host-supplied browser `ChatTransport` → OpenAI Realtime provisional STT + turn detection - (`sendMessages` → one user text or → one validated finalized `continue_interview` answer - one client-tool-result signal) - └──────────────┬────────────────────────────┘ -→ one browser `createFlueClient()` per selected principal + logical conversation id -→ one supported `send()` admission at the same-origin proxied `/agents/chat/:instanceId` route - (Voice may enter via the panel's transport — preferred, one visible store — or call `send()` directly; see fog-line) -→ `agentOwnershipGuard` (the only ownership check) → `createAgentRouter(ChatAgent)` -→ current `ChatAgent` with `useBrunchAgent()` + `useSdcpnPlugin()` -→ SDK reads: `wait(admission, { onEvent })` for the panel's finite per-turn stream, - `observe({ live: "sse" })` for canonical state and reopen -→ response parts correlated by server-issued `submissionId` -→ panel: existing `ConversationStreamChunk → UIMessageChunk` projector, terminated on `submission-settled` -→ Voice: canonical completed Brunch text displayed and passed unchanged as TTS input -→ local playback/observation cancellation or explicit conversation-wide `abort()` -→ observation rehydration after reopening the same logical conversation +OpenAI Realtime microphone input +→ semantic VAD marks an input boundary but creates no model response +→ conversation.item.input_audio_transcription.completed +→ keyed transcript authority (connection epoch, item id, content index) +→ half-duplex Voice controller and shared panel submission path B +→ browser AI SDK ChatTransport over the memoized FlueClient +→ idempotent Flue admission on the same-origin /agents/chat/:instanceId proxy +→ agentOwnershipGuard → mounted Brunch ChatAgent +→ submission-correlated canonical response and settlement +→ exact canonical segment queue → visible panel text and TTS +→ replay controls and durable Voice provenance +→ observe({ live: "sse" }) hydration and reopen ``` -The `/api/chat` route, `createPetrinautChatHandler`, the in-process `init().dispatch()/read()` admission path, the `GET ?id=` history door, and the `/api/chat` Vite proxy are removed from the Brunch app; the local launcher proxies `/agents/chat/*` instead. `@hashintel/brunch-agent-transport-aisdk` is repurposed as the browser-side adapter: it exports the projector, the snapshot → UI-message projection, the header names, and a `ChatTransport` factory over a caller-supplied `FlueClient`, and depends on `ai` and the public `@flue/sdk` client only. The Petrinaut panel itself stays on `useChat`; it is not rewritten onto `@flue/react`. +Realtime exposes no tools, uses `tool_choice: "none"`, and configures semantic VAD with `create_response: false`. Model function-call arguments are ignored even if a provider violates the policy. Provisional transcription is display-only and disappears without submission. -`conversationId` is the stable logical reference selected by the Petrinaut host. The current principal plus that id mechanically derives the Flue instance path and ownership headers. `submissionId` correlates one admitted answer and its settlement. Flue offsets remain opaque. `uid` identifies one current incarnation and may guard a known continuation, but it must not replace the logical conversation id or be presented as durable user identity. +Local playback cancellation, local observation cancellation, the HTTP request `AbortSignal`, and durable conversation-wide `FlueClient.abort()` remain separate operations. The first three never masquerade as durable Stop; durable Stop never appears as a Voice transcription or playback failure. -For maintained Voice state beyond what the panel already holds, use the SDK's materialized `observe()` surface, or `useFlueAgent()` over the same memoized client if the React seam earns it. Use `readSubmissionReply()`, `read()`, or `wait({ onEvent })` only for submission-scoped extraction, not as a parallel transcript reducer; the panel transport's per-submission projection into `useChat`'s own store is that submission-scoped use, not a second transcript. Do not parse SSE, calculate offsets, retry stream chunks, or hand-pick the latest message. The tracer may stream canonical text visibly through the observation, but TTS begins only from completed speakable segments and may remain settlement-gated; token-by-token speech is not part of this claim. +## Proof -The first tracer is text-turn-only at the Brunch boundary. It does not require the temporary `brunch_ask` client-tool shim: a finalized spoken answer is a direct Flue user message, and canonical plain assistant text is sufficient to prove the transport. If the real tracer cannot preserve answer correlation without structured questions, stop and present that observed strain before mounting the suspended capability. +This mission closes the Voice safety and UX-parity stratum on the parent's route. It does not establish production identity, remote deployment, structured questions, a live `brunch_ask` capability, response simplification, workpiece mutation, or fixes for the parent's named defects. -### Contract stratum and readiness gate +### Product-manager litmus -Close the **one-route conversation transport stratum**: one typed turn and one finalized Voice turn each admitted once through `@flue/sdk` at the mounted route, canonical output, client-tool follow-up as a signal, local playback cancellation, durable abort, visible failure, and same-conversation reopen — with no second server-side door remaining. +**Release note:** Voice now submits only what the microphone actually transcribed, waits for a safe **Your turn** handoff before listening over Brunch, can replay the exact full response or question, and keeps its Voice attribution after reopening the conversation. -Order the tracers so the cheaper one proves the route first: the typed panel over the browser transport (it reuses the existing projector and has an existing integration scenario to re-express), then Voice. After each end-to-end turn works, enumerate the lateral obligations it exposes and close those required to make the visible claim true: duplicate finalization, ambiguous admission, submission/reply correlation, client-tool resume correlation, reconnect and replay, local cancellation versus durable abort races, fatal ownership errors, and canonical text/TTS-input correspondence. Carry broader speech ergonomics, multi-turn barge-in tuning, structured questions, and remote identity and exposure only to the named deferred owners below. +**Demo script:** run `yarn dev:brunch` and select the Brunch preview. Speak one answer and see exactly one matching user turn. While Brunch is speaking, confirm the microphone remains closed, choose **Your turn**, wait for the handoff, and speak again. Use the playback menu to read the full response and repeat the question verbatim. Start another turn, press durable **Stop** before settlement, and see a stopped turn rather than a Voice error. After the parent hydration blocker is fixed, hard-reload the settled conversation and confirm the same Voice attribution remains without resubmission or replay. -## Proof +**Previously impossible:** model-generated function arguments rather than completed audio transcription could become the answer; assistant playback could create a false user turn; cancellation could reopen capture before the provider settled; replay controls and durable Voice attribution were incomplete. -This proof establishes that one real local typed turn and one real local Voice turn each cross the supported Flue conversation protocol at the single mounted route into the current canonical Brunch agent, that the typed turn returns as one finite AI SDK stream and the Voice turn as one visible and spoken canonical response, both with bounded cancellation and recovery semantics, and that no server-side AI SDK door remains. It does **not** establish trusted production authentication, remote deployment, broad Voice UX, structured-question transport, Petrinaut client-tool mutation, workpiece viability, or that the `useChat` panel itself is removable. +**Completion:** sections 1–6 close when their tests and focused checks pass. Mission acceptance additionally requires the real microphone, handoff, Stop, hard-reload, and same-origin route witness after the parent hydration fix. Mocked or server-only proof cannot substitute for that witness. -1. **Typed panel over the browser Flue transport.** A typed panel submission calls `send()` exactly once with one `kind: "user"` message; a completed client-tool follow-up calls `send()` exactly once with one `kind: "signal"` `client-tool-result` message and resumes the same assistant message id; the returned `UIMessageChunk` stream carries the same start/step/part/finish sequence the former `/api/chat` integration asserted, and terminates on that submission's `submission-settled`. Reopen hydration comes from `observe({ live: "sse" })` through `snapshotToUiMessages`. Oracle: the current `apps/brunch-agent/test/petrinaut-chat.integration.ts` scenario re-expressed through the browser transport against the in-process `app.fetch` of the real `app.ts` (Flue route, ownership guard, faux provider), preserving its text, reasoning, server-tool, and client-tool-resume assertions; the relocated projector and transcript unit tests; and the outer witness typing one message in the real panel with the network ledger showing only `/agents/chat/:instanceId` traffic. -2. **Direct finalized admission.** A completed Realtime `continue_interview` call invokes Flue `send()` exactly once with one `kind: "user"` message; provisional transcript events, duplicated provider terminal events, stale epochs, and repeated tool-call delivery never enter history. The admitted server `submissionId` becomes the turn correlation key. A lost or ambiguous admission is surfaced and never blindly resent. Oracle: named cases in `apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.test.ts`, including `admits one finalized Realtime answer through Flue once`, plus canonical snapshot inspection showing exactly one matching visible user message. -3. **No server-side AI SDK door.** The Brunch app no longer mounts `/api/chat`; `createPetrinautChatHandler`, `PETRINAUT_CHAT_ROUTE`, the in-process `init()` admission path, and the `/api/chat` proxy are deleted; the repurposed transport package contains no `Request`/`Response` handler. The real Voice turn reaches the mounted route through `@flue/sdk` — either through the panel's Flue transport or by a direct `send()` — and makes no submission over any non-Flue protocol. Oracle: `apps/brunch-agent/test/build-artifact.test.ts` asserting the built server answers `/api/chat` with Hono's 404 and still serves the Flue route; the retained browser network ledger from the outer witness; and a focused integration case in `apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts` named `admits a Voice turn only through the Flue route`. -4. **Canonical visible and TTS output.** The materialized response selected for the admitted `submissionId` is the source for visible text and TTS input. Completed visible assistant text is preserved in part order; reasoning and non-speech parts are not promoted to spoken text. The exact string array sent for canonical speech equals the selected Brunch text, and no response-preparation or simplification generation call occurs. Oracle: `apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.test.ts`, `openai-realtime-session.test.ts`, and the outer artifact comparison of canonical snapshot text to the recorded TTS request payload; the human witness confirms that playback begins but does not claim synthesized audio is a verbatim recording. -5. **Cancellation and abort stay distinct.** Barge-in, pause, or panel closure stops local playback/observation according to the existing Voice contract without rewriting or deleting canonical history. An explicit durable Stop action uses Flue's conversation-wide `abort()` when work is unsettled, and history/observation exposes an `aborted` settlement; an abort that loses the race to completion remains completed. Oracle: named local-versus-durable cancellation cases in `voice-turn-controller.test.ts` and `voice-preview.integration.test.ts`, plus the retained Voice event ledger and Flue settlement. -6. **Reopen resumes without replay.** Reopening the same selected conversation rehydrates its canonical messages and settlements from Flue, does not submit another user message, and does not automatically replay settled audio. An absent conversation, fatal 401/403, reconnecting stream, and settled conversation are visibly distinguishable. Oracle: a named `rehydrates the settled Voice turn without resubmission or playback` case over the SDK observation in `realtime-brunch-bridge.test.ts`, followed by the outer witness reopening the panel and comparing the second view with the same snapshot. -7. **Architecture and boundary integrity.** The built app still mounts `useBrunchAgent()` plus `useSdcpnPlugin()`, excludes the obsolete app-local stub agent, derives ownership from principal plus logical conversation id in exactly one guard, and reaches Flue locally through a same-origin protocol-preserving proxy rather than a newly public route. The transport package's runtime dependencies are exactly `ai` and `@flue/sdk`; it imports no `@flue/runtime`, core, plugin, or binding module. Oracle: `apps/brunch-agent/test/build-artifact.test.ts`, `apps/brunch-agent/test/agent-ownership.test.ts`, the transport case in `apps/brunch-agent/test/architecture/boundaries.integration.ts` (`transports consume their wire encoder and the public Flue client only — never core, a binding, or the runtime`), SDCPN packaging tests, and browser inspection of the claimed local route and headers. -8. **Real Voice witness and retained proof bundle.** With `yarn dev:brunch`, a human speaks one answer, sees exactly one matching user message, sees and hears the canonical Brunch response begin, interrupts playback once, exercises Stop on one unsettled turn, and reopens the original settled turn. Retain under `docs/evidence/implementations/mission-5-direct-voice-flue/` the witness record, sanitized Voice event ledger, network route summary, canonical Flue snapshot, settlement outcomes, source/build commit, and hashes. Oracle: human adjudication against that bundle; mocked browser or server-only evidence cannot satisfy this leaf. -9. **Focused repository verification and truthful docs.** Brunch app, website, core/plugin, transport, and Petrinaut checks pass; end-user and operator prose describes the single route that actually shipped and preserves the distinction between canonical text and generated audio; no surviving prose or comment names `/api/chat` as a Brunch door. Oracle: `yarn exec turbo run lint:tsc lint:eslint test:unit build --filter @apps/brunch-agent --filter @apps/petrinaut-website --filter @hashintel/petrinaut --filter @hashintel/brunch-agent --filter @hashintel/brunch-agent-plugin-sdcpn --filter @hashintel/brunch-agent-transport-aisdk`, `yarn workspace @local/petrinaut-arch-docs lint:arch-docs` if a Petrinaut architectural boundary changes, `rg -n "api/chat" apps/brunch-agent libs/@hashintel/brunch-agent/packages apps/petrinaut-website/src/main/app/local-storage-demo` returning only the stock Petrinaut route and frozen Mission 4 evidence, inspection of `apps/petrinaut-website/README.md` and `libs/@hashintel/petrinaut/docs/ai-assistant.md`, and a patch changeset if the published Petrinaut package changes. +1. **Completed-transcript authority and half-duplex ownership.** Realtime session configuration has no tools, no model-created semantic-VAD response, and no automatic interruption policy. Only a unique completed transcript can reach the shared panel submission path. Duplicate, empty, failed, unavailable, stale, playback-overlapping/pre-handoff, and over-limit transcripts do not submit and produce the specified passive or recoverable notice. The microphone remains closed through assistant output and cancellation; **Your turn** opens only a post-barrier input turn. Oracle: transplanted-first cases in `openai-realtime-session.test.ts`, `realtime-brunch-bridge.test.ts`, `voice-turn-controller.test.ts`, `voice-interview-control.test.tsx`, and `voice-preview.integration.test.ts`. +2. **Idempotent admission.** Typed turns derive a stable key from the AI SDK message id; Voice turns derive it from connection epoch, item id, and content index. A repeated same-payload key converges on the original receipt, including `deduplicated: true`; a 409 `submission_conflict` surfaces the original `submissionId` without admitting another turn. An admission whose outcome cannot be established stays visibly ambiguous and is never automatically retried. Oracle: transport unit tests plus Voice integration tests asserting one admitted submission rather than one `send()` invocation. +3. **Acknowledged cancellation barrier.** `cancelOutput()` resolves only after input/output buffer clears, matching provider acknowledgements, and all targeted response terminal events. The latest mute preference wins while it settles. Audio captured before the handoff cannot submit afterward. Durable Stop remains a stopped Flue turn rather than a Voice failure. Oracle: donor-adapted session/controller race tests and the unsettled-Stop integration case. +4. **Canonical replay.** `canReadFullResponse`, `canRepeatQuestion`, `readFullResponse()`, `repeatQuestion()`, and the playback menu retain and enqueue exact canonical segments without a simplifier. Replay enables only after the matching response terminal and output completion and remains disabled during submission, capture, cancellation, pause, and errors. Oracle: canonical speech, controller, panel, and integration tests comparing segment identity and text. +5. **Durable Voice provenance.** An assistant message may retain multiple `voiceToolCallIds`; one failed sibling origin does not erase successful origins. Attribution survives canonical projection, hydration, and reopen through a supported Flue representation or deterministic reconstruction from durable correlation facts, never through user text. Oracle: snapshot projection, panel provenance, partial-failure, hydration, and reopen tests. +6. **Dormant ask removal.** If still present after restacking, the website does not register `brunchAskInteractiveTool` for Voice and canonical speech does not recognize `"brunch-ask"`. No spoken ask answer can enter a wait state the transport cannot resume. Oracle: registration/canonical-speech negative tests and a repository search showing no mounted Voice `brunch_ask` surface. +7. **Real witness and same-origin route.** After the parent hydration fix lands, a human performs one microphone turn, explicit interruption/handoff, durable Stop on an unsettled turn, and hard reload of a settled turn. The retained network route summary proves the absolute Flue `streamUrl` remains on the same-origin proxy. Oracle: `witness.md`, sanitized `voice-events.jsonl`, `network-routes.json`, canonical `flue-snapshot.json`, `settlements.json`, commit manifest, and hashes under `docs/evidence/implementations/mission-5-voice-safety-parity/`. +8. **Focused repository verification and truthful docs.** The requested four-workspace Turbo command passes. `apps/petrinaut-website/README.md` and `libs/@hashintel/petrinaut/docs/ai-assistant.md` describe half-duplex handoff, replay, Stop, transcript rejection, and durable attribution. If the published Petrinaut package changes, exactly one patch changeset covers it. Oracle: the command recorded in the PR and changeset inspection. ## Constraints -- Preserve Mission 4's current core/plugin/app composition and authored skill packaging. Voice reconciliation must never restore the deleted app-local `ChatAgent`, concise stub prompt, YAML plugin machinery, or a second model-facing agent. -- Flue history is the sole canonical conversation record. Voice owns media capture, provisional display, turn finalization, TTS, playback, and local interaction state; it owns no durable transcript and may not splice into stock-assistant history. -- Use `@flue/sdk`/`@flue/react` directly for shell-facing conversation transport. No adapter may re-specify Flue offsets, retries, materialization, settlement, or recovery. The browser `ChatTransport` is a projection over the public `FlueClient` (`send()`, `wait()`/`observe()`, `history()`), never a second HTTP client; it reads chunks only through the SDK's `onEvent`/observation surfaces. -- One product route. `/agents/chat/:instanceId` behind `agentOwnershipGuard` is the sole door for typed, Voice, diagnostic, and evaluation traffic; no route, handler, or package may accept a conversation turn over another protocol. The stock Petrinaut `/api/chat` (the website's own OpenAI function) is untouched and must not be borrowed. -- The Petrinaut panel remains on `useChat`; the AI SDK is its rendering contract, supplied a transport by the host. Do not rewrite the panel onto `@flue/react`, and do not add a second transcript store beside `useChat`'s messages for the typed panel. -- Submit only the validated finalized answer. Provisional transcription and audio remain ephemeral. One Voice finalization causes at most one `send()` call; because Flue 2.0.3 does not accept a caller idempotency key on `send()`, ambiguous admission must remain visible and must not trigger an automatic retry. -- Brunch owns canonical response content. TTS may synthesize audio from exact selected text, but no second model may summarize, shorten, paraphrase, or select replacement wording for the tracer. -- Local playback cancellation, local observation cancellation, HTTP request cancellation, and Flue's conversation-wide durable abort are distinct operations and must remain distinguishable in code, UI state, evidence, and tests. -- The current browser-minted local principal is an ownership discriminator, not trusted authentication. The outer proof is local and same-origin; it must not expose `/agents/chat/:id` publicly or claim production identity, authorization, CORS, deployment, or recovery. -- `@hashintel/brunch-agent-transport-aisdk` survives only as the browser-side adapter and the home of the projector, snapshot projection, and header names. Its runtime dependencies are `ai` and `@flue/sdk`; it never imports `@flue/runtime`, core, a plugin, or a binding, and client-tool names reach it as caller-supplied options. The boundary test's transport gate is amended to say exactly that — this is the one accepted topology-gate change of the recut. -- Do not mount the suspended `brunch_ask` capability merely to preserve the divergent preview stack. Re-entry requires observed plain-turn correlation strain and an owner decision consistent with the structured-question planning contract. -- External Voice branches and their issues/PRs remain read-only evidence. Port only behavior that serves this mission, preserve relevant provenance in commits, and do not rewrite, close, or represent those records as accepted wholesale. -- Record admission, first canonical text, first TTS request/audio, and settlement latency without transcript, prompt, tool, SDP, audio, credential, or response-body content in ordinary telemetry. -- No implementation begins until this authority cut is committed separately. Material changes to this contract require owner review and another focused authority commit before dependent implementation. +- Preserve the parent's one product route, memoized Flue client, browser `ChatTransport`, shared panel `useChat`, path-B Voice submission, canonical speech selection, durable Stop seam, and SDK observation hydration. Do not rebuild them. +- Transplant relevant regression tests before implementation. Reimplement donor behavior semantically against the current Flue path; donor branches and PRs are never merged, cherry-picked, rebased, rewritten, retargeted, or closed by this implementation. +- Derive one deterministic admission key per logical delivery. Treat `deduplicated` as successful convergence and `submission_conflict` as evidence of the already-admitted submission. Do not automatically retry an ambiguous admission. +- Normalize completed transcripts exactly once with trim plus Unicode whitespace collapse, then enforce the 32,000-code-point bound. Provisional text remains ephemeral and display-only. +- The half-duplex microphone is closed during output, cancellation, pause, error, and submission states. A cancellation promise is part of the turn boundary, not a cosmetic animation state. +- Brunch canonical text is never summarized, shortened, paraphrased, or regenerated for speech or replay. +- Preserve every surviving Voice origin independently. Provenance must use supported Flue data or deterministic durable correlation; never encode it in visible user text. +- Do not fix the parent's admission/Stop races, stream cancellation, hydration overwrite, client-tool classification, response/submission correlation, CI, title, or body. Restack onto Lu's fixes; report any blocker. +- Keep local playback cancellation, local observation cancellation, HTTP request cancellation, and durable `abort()` distinguishable in code, UI, tests, and evidence. +- No simplifier, structured questions, live `brunch_ask`, Petri-net generation/mutation, FE-1575 workpiece work, production identity, CORS/remote deployment, or panel `useChat` removal. ### Expected touched paths ```text -~ libs/@hashintel/brunch-agent/packages/transport-aisdk/src/ delete the HTTP handler; add ChatTransport factory over FlueClient; receive ui-stream + snapshotToUiMessages + headers -~ libs/@hashintel/brunch-agent/packages/transport-aisdk/package.json deps become ai + @flue/sdk; drop valibot if unused -~ libs/@hashintel/brunch-agent/packages/transport-aisdk/test/ replace chat-handler/golden with transport + projector + snapshot tests -~ apps/brunch-agent/test/architecture/boundaries.integration.ts amend the transport gate -- apps/brunch-agent/src/http/petrinaut-chat.ts server-side door removed -- apps/brunch-agent/src/conversation/ui-stream.ts moves into the transport package -~ apps/brunch-agent/src/conversation/transcript.ts snapshotToUiMessages moves out; formatFlueTranscript stays for the CLI -~ apps/brunch-agent/src/http/routes.ts, local-origins.ts drop PETRINAUT_CHAT_ROUTE and the /api/chat proxy; proxy /agents/chat/* -~ apps/brunch-agent/src/app.ts remove the /api/chat mount and app-transport closure -~ apps/brunch-agent/package.json drop the `ai` devDependency if nothing else uses it -- apps/brunch-agent/test/petrinaut-chat.test.ts, petrinaut-chat.integration.ts, petrinaut-chat-result.ts, flue-ui-stream.test.ts re-expressed against the browser transport / relocated -~ apps/brunch-agent/test/build-artifact.test.ts, local-dev-origins.test.ts single-route assertions -~ apps/brunch-agent/petrinaut-local.vite.config.ts same-origin Flue-route proxy for the local real surface -~ apps/petrinaut-website/src/main/app/local-storage-demo/ createFlueClient composition, browser transport, delete use-flue-chat-history -~ apps/petrinaut-website/src/main/app/voice-interview/ direct Flue admission, materialized response, cancellation, reopen -~ apps/petrinaut-website/package.json add @flue/sdk -~ yarn.lock workspace dependency update -? apps/brunch-agent/src/http/ownership.ts, src/conversation/identity*.ts only if the identity-contract home (fog-line) moves -? libs/@hashintel/petrinaut/src/ui/ smallest public panel seam only if host composition cannot remain local -~ apps/petrinaut-website/README.md operator-facing route and preview behavior -~ libs/@hashintel/petrinaut/docs/ai-assistant.md user-visible Voice behavior -~ libs/@hashintel/brunch-agent/MISSION.next.md reconcile the production-door, restricted-ingress, and adapter-removal statements -+ libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-direct-voice-flue/ retained proof bundle -? .changeset/ one patch changeset only if the published Petrinaut package changes +~ apps/petrinaut-website/src/main/app/voice-interview/ transcript authority, half-duplex state, cancellation, replay tests/code +~ apps/petrinaut-website/src/main/app/local-storage-demo/ path-B correlation and dormant ask removal if still present +~ apps/petrinaut-website/src/server/voice/ Realtime policy tests/code +~ libs/@hashintel/brunch-agent/packages/transport-aisdk/ stable idempotency and canonical projection/provenance tests/code +~ libs/@hashintel/petrinaut/src/react/voice-session/ public Voice state required by the panel +~ libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ Your turn, replay menu, durable provenance +~ apps/petrinaut-website/README.md operator behavior +~ libs/@hashintel/petrinaut/docs/ai-assistant.md end-user behavior +? .changeset/ one patch changeset if published Petrinaut changes ++ libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/ donor matrix and gated witness ``` ## Fog-line -- How a finalized Voice answer enters Flue. Two shapes are admissible: **(B, preferred)** Voice submits through the panel's own Flue `ChatTransport` (`useChat.sendMessage` → `send()`), so the panel's `useChat` messages remain the single visible store and Voice's `observe()` shrinks to selecting completed canonical text for TTS — or reads the panel's completed assistant message and drops `observe()` entirely; **(A)** Voice calls `send()` directly and keeps its own `observe()` state, with the panel rehydrating. Start with B; fall back to A only if the existing hold-while-streaming, epoch, or TTS-correlation semantics demonstrably strain under the panel's transport, and record the observed strain. Either way, one finalization is one `send()`, and if the chosen shape would create two mutable transcript stores or a second custom reducer, stop and reorient at the panel boundary. -- The smallest honest home for the browser-safe principal + logical-conversation-id → Flue-instance-id contract and the two ownership header names. `apps/brunch-agent/src/conversation/identity-web.ts` proves the algorithm; the website must not gain an app-to-app source import, and core must not own HTTP header names. The leading candidate is the repurposed transport package, which already exports the principal header; a website-local copy pinned by an equality test against the app is the fallback. Do not create a new package to hold two strings and a hash. -- Whether `wait(admission, { onEvent })` alone gives the browser transport a clean finite per-turn stream, or whether the panel needs `observe()` for reconnect during a turn. `wait()` rejects on failed/aborted settlement and on `terminal_event_missing`; the transport must map those to `error`/`abort` chunks rather than throwing past `useChat`. The first real disconnect mid-turn decides; do not pre-build reconnect machinery. -- Whether `reconnectToStream` should return `null` (observation-only rehydration, the current behavior) or resume an unsettled submission after reload. Start with `null` plus SDK observation; re-enter only if the witness observes a lost in-flight turn. -- The exact subset of PRs #9496, #9507, and #9512 to port after semantic comparison with the current branch. Their useful Voice state-machine behavior is evidence; their app-local agent topology, temporary ask shim, and generative preparation are not presumed requirements. -- The bounded speech-selection policy if the exercised Brunch response contains multiple completed text blocks or an interactive part. Begin with canonical completed visible text in order; if this produces duplicate, misleading, or unspeakable output, retain the mismatch and seek a Brunch-owned deterministic presentation rule rather than another generator. -- Whether the existing Stop affordance can express both local Voice interruption and explicit conversation-wide durable abort without misleading the user. The first real race decides the smallest UI distinction. -- Whether a same-origin local proxy can carry every SDK history/SSE/send/abort route unchanged. A crisp protocol or middleware blocker permits the thinnest route correction; it does not permit rebuilding the AI SDK adapter under another name. +- **Parent movement.** Lu owns #9528 and may push more commits. Before each implementation phase, compare the GitHub head and restack this branch; an observed parent change is adopted only through restack, never copied into this branch. +- **Durable provenance representation.** Inspect the current Flue event/snapshot vocabulary and existing correlation facts before choosing between a supported persisted metadata field and deterministic reconstruction. If neither can preserve multiple origins without a second authority or user-text encoding, stop with the exact missing platform seam. +- **Conflict normalization.** The installed SDK exposes the 409 contract through `FlueApiError.body: unknown`. Narrow only the documented envelope needed to recover `error.meta.submissionId`; do not create a general error protocol or infer success from prose. +- **Dormant `brunch_ask`.** Remove or gate only the parent surfaces that remain after the next restack. If Lu has already removed them, record the parent commit and make no duplicate change. +- **Hard-reload witness.** Hold proof leaf 7 until the parent's hydration overwrite is fixed. Unit/integration tests for this branch may cover deterministic projection, but they cannot substitute for the blocked real witness. ## Stop or reorient -Stop and surface the evidence if the implementation creates a second conversation authority, keeps or re-adds any server-side route that accepts a conversation turn over a non-Flue protocol, submits provisional STT, automatically retries an ambiguous admission, rewrites canonical text through another model, hand-rolls stream recovery, computes offsets, restores the old stub agent, or activates `brunch_ask` without observed need and owner approval. - -Stop if the browser transport cannot preserve the current client-tool resume semantics (completed client-tool parts on the referenced assistant message → one signal send → continuation of the same assistant message id) without a server-side helper; that is evidence the resume contract needs redesign, not permission to reintroduce `/api/chat`. +Stop and report if the work would require direct Voice `send()`, a second transcript or conversation authority, hand-rolled stream offsets/recovery, automatic retry after ambiguous admission, canonical text rewriting, a live structured-question path, or any excluded parent fix. -Stop at the boundary if direct Flue state cannot reach the existing visible panel without duplicated mutable history; decide the UI ownership seam before adding synchronization machinery. Stop if local cancellation accidentally aborts durable work, explicit Stop only cancels a browser request while the provider keeps spending, a stale response is spoken after conversation/epoch change, or reopen resubmits or replays a settled turn. +Stop if half-duplex handoff cannot guarantee that pre-handoff audio is rejected and post-barrier audio is fresh, or if provider acknowledgements cannot bound `cancelOutput()` without inventing events. Stop if provenance requires visible-text smuggling or a second durable store. Stop if replay can enable before both matching terminal conditions, or if local cancellation invokes durable abort. -Stop rather than widen if the real route requires public unauthenticated exposure, production identity work, remote deployment, Petrinaut mutation tools, workpiece/projection state, or a whole assistant rewrite. Those are not hidden prerequisites to this transport tracer. +Do not manufacture the hard-reload witness while the parent hydration defect remains. Retain the blocker and wait for a new parent head. ## Deferred -- **`useChat` panel removal:** with the server-side door gone, the AI SDK survives only as the Petrinaut panel's rendering contract behind a host-supplied transport. Whether Petrinaut ever drops `useChat` is a Petrinaut product decision, not a Brunch transport question; Brunch carries no further obligation here. -- **Restricted-ingress rule for the Flue route:** Mission 8's landed contract denied `/agents/chat/:id` publicly and routed restricted traffic through `/api/chat`. This recut makes the Flue route the only product route, so that rule must be re-expressed as the FE-1423 gates applying directly to `/agents/chat/:id`. Record the re-expression in `MISSION.next.md`; the release/deployment gate owns its enforcement. -- **Structured questions:** core-owned question semantics, binding, rendering, correlated reply, and resumed tool execution remain in the shared future-planning record. Re-enter when plain Voice turns demonstrably cannot preserve a required interaction. -- **Broader Voice quality:** multi-turn barge-in tuning, long-response ergonomics, optional deterministic spoken presentation, accessibility breadth, and response optimisation re-enter after measured strain on the direct canonical route. -- **Remote/public operation:** trusted identity and authorization, origin policy, hosted Flue reachability, rate/spend controls, replacement recovery, and remote observability remain with the Mission 8 release/deployment gate or a separately cut successor. -- **Product-data work:** prepared workpiece/Petrinaut viability remains Mission 6; capture-backed review remains Mission 7; automatic traceable projection remains Mission 9. This mission carries no document mutation or provenance claim beyond canonical conversation history. -- **Host breadth:** stock/Brunch picker behavior, session switching beyond the selected local Brunch conversation, and HASH embed parity wait for the first visible consumer that makes them load-bearing. +- The real witness and same-origin absolute-`streamUrl` observation wait on the parent hydration fix; once unblocked, they are part of this mission rather than a successor. +- Donor retirement waits until this replacement is accepted and each donor owner explicitly approves closure. Do not close #9496, #9500, #9507, or #9512 as an implementation side effect, and never close stakeholder-owned H-6763. +- Response preparation/simplification, structured questions, Petri-net work, FE-1575, production identity, CORS/remote deployment, and panel migration away from `useChat` remain in their existing owners or the future mission spine. diff --git a/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/donor-behavior-matrix.md b/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/donor-behavior-matrix.md new file mode 100644 index 00000000000..b651ceb3cb0 --- /dev/null +++ b/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/donor-behavior-matrix.md @@ -0,0 +1,54 @@ +# FE-1580 donor-behavior matrix + +## Decision frame + +This record pins the semantic disposition of the Voice donor branches for the live [FE-1580 mission](../../../../MISSION.md). The parent and donors are read-only source evidence at these exact heads: + +| Source | Pinned head | Role | +| --- | --- | --- | +| Parent PR [#9528](https://github.com/hashintel/hash/pull/9528) | `036fb06ee83e840fd7a87dd1bc758a2dd49ddd4c` | Unified Flue route and path-B departure base | +| Donor PR [#9496](https://github.com/hashintel/hash/pull/9496) | `c7fe8a2e68e8fdc37018b21ec2e9daf4e9ef7c82` | Canonical TTS queue and replay mechanics | +| Donor PR [#9500](https://github.com/hashintel/hash/pull/9500) | `935aa9f02a5ac635a50eb8bc130edb3e258af8e4` | Completed-transcript authority | +| Donor PR [#9507](https://github.com/hashintel/hash/pull/9507) | `252b9dbb0c77fae8cee45a506f09cac3e20c381c` | Temporary `brunch_ask` shim, excluded | +| Donor PR [#9512](https://github.com/hashintel/hash/pull/9512) | `d13535d1077b3a78d6a1411031b7d0a0a78e3144` | Half-duplex cancellation, replay UX, and provenance | + +No source is merged, cherry-picked, rebased, retargeted, rewritten, or closed by the implementation. Tests are transplanted first and adapted to the one Flue submission route; production behavior is reimplemented semantically. + +The owner selected half-duplex turn ownership on 2026-09-03: assistant output owns the audio turn until **Your turn** completes an acknowledged cancellation barrier. Automatic duplex is not an admissible fallback. + +## Behavior disposition + +| Source | Behavior | Disposition | Reason | Outstanding adaptation or proof | +| --- | --- | --- | --- | --- | +| #9528 | One `/agents/chat/:instanceId` product route, browser `ChatTransport`, one memoized client, path-B Voice submission through shared `useChat` | **Adopt** | This is the departure architecture and prevents a second admission authority. | Restack onto every new parent head; verify no successor code calls `send()` directly from Voice. | +| #9528 | Direct Voice `send()` as a fog-line fallback | **Reject** | It creates a second admission path and mutable coordination surface. The parent has already proved path B. | Mission authority now permits path B only. | +| #9528 | Claim that Flue 2.0.3 lacks caller idempotency | **Reject as factually false** | Installed typings expose `AgentPromptOptions.idempotencyKey`, `AgentSendResult.deduplicated`, and 409 `submission_conflict` with the existing `submissionId`. | Add transport tests and implementation for same-payload convergence, conflict recovery, and visible ambiguous admission. | +| #9528 | Parent-owned admission/Stop races, stream cancellation, hydration overwrite, client-tool classification, response/submission correlation, and PR/CI repairs | **Reject from successor scope** | Lu owns these defects; changing them here would make the stack compete with its parent. | Report blockers and restack onto fixes. The real reload witness remains blocked by hydration overwrite. | +| #9496 | Serialized canonical speech queue, retained exact source segments, response/output terminal gating | **Adopt mechanics** | Replay and ordinary TTS need one lifecycle-safe queue, and exact text preserves canonical authority. | Remove every preparation/simplifier dependency while adapting queue tests to the parent's canonical Flue segments. | +| #9496 | `canReadFullResponse`, `canRepeatQuestion`, `readFullResponse()`, `repeatQuestion()`, playback menu | **Adopt** | These are the missing Voice UX-parity controls. | Gate them during submission, capture, cancellation, pause, and errors; require matching response terminal plus output completion. | +| #9496 | Realtime-generated concise response preparation or any fallback that rewrites canonical text | **Reject** | Response simplification is a non-goal and violates exact canonical speech. | Tests compare retained segment ids and exact queued strings; no preparation API remains on this path. | +| #9500 | No Realtime tools, `tool_choice: "none"`, semantic VAD `create_response: false` | **Adopt** | Realtime detects/transcribes and renders supplied TTS only; it must not generate user meaning. | Adapt policy, session, and integration tests to the parent route. | +| #9500 | Only `conversation.item.input_audio_transcription.completed` can submit; model function arguments ignored | **Adopt** | Shape validation cannot prove model-generated arguments match the audio. | Transplant regression tests before replacing the parent bridge authority. | +| #9500 | Transcript identity `(connectionEpoch, itemId, contentIndex)`, stable submission id, trim plus Unicode whitespace collapse, 32,000-code-point limit | **Adopt** | This gives one deterministic logical Voice delivery and one normalization boundary. | Feed the derived identity through path B as the Flue idempotency key. | +| #9500 | Explicit duplicate, empty, failed, unavailable, and over-limit rejection; passive/recoverable not-heard UI; provisional display only | **Adopt** | Rejected audio must never become a turn, while ordinary silence/failure must not poison the session. | Preserve the reason-specific bridge events and controller recovery behavior. | +| #9500 | Silently settling ownership by discarding every playback-overlapping utterance without an explicit handoff | **Supersede** | It avoids echo but leaves users without a deliberate way to take the turn. | Use #9512 half-duplex `canTakeTurn`/`takeTurn()` and reject all speech captured before the completed handoff. | +| #9500 | `brunch_ask` answer/tool correlation and preparation code inherited from its base | **Reject** | Structured questions and response preparation are excluded. | Correlate the Voice delivery to its path-B submission and canonical response/question facts instead. | +| #9507 | Temporary `brunch_ask` registration, widget, correlated spoken ask answer, transcript formatting | **Reject entire shim** | The current transport only admits the supported follow-up set; a spoken ask can otherwise wait forever. Structured questions are a separate product decision. | Remove or gate dormant `brunchAskInteractiveTool` and `"brunch-ask"` canonical-speech recognition only if still present after restack. | +| #9512 | Half-duplex `canTakeTurn`, `takeTurn()`, `"cancelling"` output state, and **Your turn** control | **Adopt by owner decision** | It makes output/input ownership explicit and prevents assistant playback from becoming a false user turn. | Adapt public Voice state and panel controls without importing donor topology or ask widgets. | +| #9512 | Promise-returning `cancelOutput()` that waits for input/output clears, matching acknowledgements, and response terminal events | **Adopt** | The microphone cannot safely reopen on a fire-and-forget cancel. | Transplant acknowledgement/race tests first; preserve latest mute preference and fresh post-handoff capture. | +| #9512 | Replay availability tied to exact retained source, terminal response, and output completion | **Adopt with #9496 mechanics** | This closes replay races without changing canonical content. | Unify with the parent's segment/submission correlation rather than donor ask correlation. | +| #9512 | Voice answer icon/provenance before interactive answers | **Adopt semantically, broaden cardinality** | Live attribution is useful but one origin per assistant message is insufficient after coalesced or sibling Voice deliveries. | Use `voiceToolCallIds`; preserve successful siblings on partial failure and survive hydration/reopen. | +| #9512 | App-local agent topology, temporary ask UI, response preparation, or donor-specific host composition | **Reject** | The parent owns the one Flue route and current host composition; these mechanisms are obsolete or non-goals. | Reuse only state-machine, cancellation, replay, and attribution behavior. | + +## Outstanding acceptance ledger + +| Area | Required closing evidence | Current state | +| --- | --- | --- | +| Transcript authority | Transplanted-first session, bridge, controller, and integration regressions pass on path B. | Open for implementation. | +| Admission idempotency | Typed and Voice logical replays converge on one `submissionId`; conflict metadata is narrowed safely; ambiguous outcome does not retry. | Open for implementation. | +| Cancellation barrier | Buffer acknowledgements and targeted response terminals settle before capture; stale/pre-handoff audio cannot submit; latest mute choice wins. | Open for implementation. | +| Canonical replay | Exact segment queue and playback menu pass availability/race tests without a simplifier. | Open for implementation. | +| Durable provenance | Multiple origins and partial failure survive projection, hydration, and reopen without user-text encoding. | Representation remains at the mission fog-line pending Flue inspection. | +| Dormant ask | No mounted Voice ask capability remains, or the parent commit that removed it is recorded. | Recheck after the next parent restack. | +| Real witness | Microphone, handoff, unsettled Stop, reload, canonical snapshot, settlement, and same-origin absolute-`streamUrl` artifacts are retained with hashes. | Blocked by the parent's hydration overwrite. | +| Donor retirement | Replacement accepted and each donor owner explicitly approves closure. | Deferred; no donor or stakeholder issue may be closed now. | From 988482cfb4efdb86e84e660ff7518e1d561e91fa Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Fri, 4 Sep 2026 00:53:58 +0200 Subject: [PATCH 02/52] Pin unified Voice safety behavior Co-authored-by: Amp Amp-Thread-ID: https://ampcode.com/threads/T-01a06941-3ee7-7482-a82e-8a16c7141785 --- .../brunch-panel-transport.test.ts | 1 + .../voice-interview/canonical-speech.test.ts | 25 +- .../openai-realtime-session.test.ts | 244 +++-- .../realtime-brunch-bridge.test.ts | 877 ++++-------------- .../voice-preview.integration.test.ts | 182 ++-- .../voice-session-state.test.ts | 31 + .../voice-turn-controller.test.ts | 349 ++++++- .../server/voice/openai-realtime-call.test.ts | 10 +- .../server/voice/openai-voice-policy.test.ts | 44 +- .../test/chat-transport.test.ts | 86 ++ .../transport-aisdk/test/transcript.test.ts | 59 ++ .../Editor/panels/ai-assistant-panel.test.tsx | 139 ++- .../ai-assistant-contents.test.tsx | 104 ++- 13 files changed, 1222 insertions(+), 929 deletions(-) diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-panel-transport.test.ts b/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-panel-transport.test.ts index 6df757d8614..0b05c0d21da 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-panel-transport.test.ts +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-panel-transport.test.ts @@ -71,6 +71,7 @@ test("delegates one typed message to the supplied Flue conversation", async () = expect(send).toHaveBeenCalledOnce(); expect(send).toHaveBeenCalledWith({ + idempotencyKey: "ai-sdk:user-1", message: { kind: "user", body: "Typed tracer." }, signal: undefined, }); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.test.ts index dc12cda21c6..f688e22039e 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.test.ts @@ -1,7 +1,5 @@ import { describe, expect, test } from "vitest"; -import { ASK_TOOL_NAME } from "@hashintel/brunch-agent/client-tools"; - import { hashCanonicalSpeechText, selectCanonicalSpeechSegments, @@ -90,7 +88,7 @@ describe("canonical speech selection", () => { ]); }); - test("selects one exact validated brunch_ask question", () => { + test("does not treat structured tool input as canonical speech", () => { const messages = [ { id: "assistant-ask", @@ -99,21 +97,21 @@ describe("canonical speech selection", () => { { type: "dynamic-tool", toolCallId: "ask-1", - toolName: ASK_TOOL_NAME, + toolName: "brunch_ask", state: "input-available", input: { question: "Which operator confirms the batch?" }, }, { type: "dynamic-tool", toolCallId: "ask-malformed", - toolName: ASK_TOOL_NAME, + toolName: "brunch_ask", state: "input-available", input: { question: 42 }, }, { type: "dynamic-tool", toolCallId: "ask-submitted", - toolName: ASK_TOOL_NAME, + toolName: "brunch_ask", state: "output-available", input: { question: "Do not repeat an answered question." }, output: { answer: "Already answered." }, @@ -129,20 +127,7 @@ describe("canonical speech selection", () => { }, ] satisfies PetrinautAiMessage[]; - const selected = select(messages); - const contentHash = hashCanonicalSpeechText( - "Which operator confirms the batch?", - ); - expect(selected).toEqual([ - { - contentHash, - id: `canonical-speech:assistant-ask:ask-1:${contentHash}`, - messageId: "assistant-ask", - partId: "ask-1", - source: "brunch-ask", - text: "Which operator confirms the batch?", - }, - ]); + expect(select(messages)).toEqual([]); }); test("uses stable source identity plus an exact-text fingerprint", () => { diff --git a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts index 70f7bc4088f..5f78aabc74c 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts @@ -36,7 +36,7 @@ const canonicalSegment = ( id, messageId: `message-${id}`, partId: id, - source: "brunch-ask", + source: "assistant-text", text, }); @@ -221,7 +221,7 @@ describe("OpenAIRealtimeSession", () => { expect(harness.peers[0]!.close).toHaveBeenCalledOnce(); }); - test("keeps the microphone active through playback and reports automatic interruption", async () => { + test("keeps the microphone closed and rejects audio detected during playback", async () => { const harness = createHarness(); await harness.session.connect(); harness.session.setMicrophoneEnabled(true); @@ -242,31 +242,143 @@ describe("OpenAIRealtimeSession", () => { item_id: "item-user", type: "input_audio_buffer.speech_started", }); + channel.receive({ + response_id: "response-canonical", + type: "output_audio_buffer.stopped", + }); + channel.receive({ + content_index: 0, + item_id: "item-user", + transcript: "Assistant echo must not submit.", + type: "conversation.item.input_audio_transcription.completed", + }); - expect(harness.localTracks[0]!.enabled).toBe(true); - expect(harness.events).toEqual( - expect.arrayContaining([ - { - connectionEpoch: 1, - responseId: "response-canonical", - speechRequestId: "canonical-1-1", - type: "output-started", - }, - { - connectionEpoch: 1, - itemId: "item-user", - type: "input-speech-started", - }, - { - connectionEpoch: 1, - responseId: "response-canonical", - type: "output-interrupted", - }, - ]), + expect(harness.events).not.toContainEqual( + expect.objectContaining({ itemId: "item-user", type: "completed" }), + ); + expect(harness.events).not.toContainEqual( + expect.objectContaining({ + itemId: "item-user", + type: "input-speech-started", + }), ); }); - test("parses streamed tool arguments and the completed GA response output", async () => { + test("restores only the latest microphone preference after playback", async () => { + const harness = createHarness(); + await harness.session.connect(); + harness.session.setMicrophoneEnabled(true); + harness.session.speakCanonical([ + canonicalSegment("ask-1", "What happens next?"), + ]); + const channel = harness.channels[0]!; + authorizeLatestSpeechResponse(channel, "response-canonical"); + channel.receive({ + response_id: "response-canonical", + type: "output_audio_buffer.started", + }); + + expect(harness.localTracks[0]!.enabled).toBe(false); + harness.session.setMicrophoneEnabled(false); + channel.receive({ + response_id: "response-canonical", + type: "output_audio_buffer.stopped", + }); + + expect(harness.localTracks[0]!.enabled).toBe(false); + }); + + test("waits for input, output, and response settlement before completing handoff", async () => { + const harness = createHarness(); + await harness.session.connect(); + harness.session.setMicrophoneEnabled(true); + const channel = harness.channels[0]!; + channel.receive({ + audio_start_ms: 40, + item_id: "item-before-handoff", + type: "input_audio_buffer.speech_started", + }); + harness.session.speakCanonical([ + canonicalSegment("ask-handoff", "What happens next?"), + ]); + authorizeLatestSpeechResponse(channel, "response-handoff"); + channel.receive({ + response_id: "response-handoff", + type: "output_audio_buffer.started", + }); + + const cancellation = Promise.resolve(harness.session.cancelOutput()); + let settled = false; + void cancellation.then(() => { + settled = true; + }); + + expect(harness.localTracks[0]!.enabled).toBe(false); + expect(sentEvents(channel).slice(-3)).toEqual([ + { type: "input_audio_buffer.clear" }, + expect.objectContaining({ + response_id: "response-handoff", + type: "response.cancel", + }), + { type: "output_audio_buffer.clear" }, + ]); + channel.receive({ + content_index: 0, + item_id: "item-before-handoff", + transcript: "This began too early.", + type: "conversation.item.input_audio_transcription.completed", + }); + channel.receive({ type: "input_audio_buffer.cleared" }); + channel.receive({ + response_id: "response-handoff", + type: "output_audio_buffer.cleared", + }); + await Promise.resolve(); + expect(settled).toBe(false); + expect(harness.localTracks[0]!.enabled).toBe(false); + + channel.receive({ + response: { + id: "response-handoff", + output: [], + status: "cancelled", + }, + type: "response.done", + }); + await cancellation; + + expect(harness.localTracks[0]!.enabled).toBe(true); + expect( + harness.events.some( + (event) => + event.type === "completed" && + event.key.itemId === "item-before-handoff", + ), + ).toBe(false); + + channel.receive({ + audio_start_ms: 120, + item_id: "item-after-handoff", + type: "input_audio_buffer.speech_started", + }); + channel.receive({ + content_index: 0, + item_id: "item-after-handoff", + transcript: "This began after the handoff.", + type: "conversation.item.input_audio_transcription.completed", + }); + expect(harness.events).toContainEqual({ + key: { + connectionEpoch: 1, + contentIndex: 0, + itemId: "item-after-handoff", + }, + text: "This began after the handoff.", + type: "completed", + }); + }); + + test("never exposes model function arguments as user input", async () => { const harness = createHarness(); await harness.session.connect(); const channel = harness.channels[0]!; @@ -287,6 +399,8 @@ describe("OpenAIRealtimeSession", () => { response_id: "response-tool", type: "response.function_call_arguments.delta", }); + expect(harness.events).toEqual([]); + channel.receive({ response: { id: "response-tool", @@ -305,54 +419,8 @@ describe("OpenAIRealtimeSession", () => { }); expect(harness.events).toEqual([ - { - callId: "call-1", - connectionEpoch: 1, - delta: '{"answer":"Approved"}', - itemId: "item-function", - responseId: "response-tool", - type: "tool-arguments-delta", - }, - { - arguments: '{"answer":"Approved"}', - callId: "call-1", - connectionEpoch: 1, - itemId: "item-function", - name: "continue_interview", - responseId: "response-tool", - type: "tool-arguments-done", - }, - { - connectionEpoch: 1, - responseId: "response-tool", - status: "completed", - type: "response-terminal", - }, - ]); - - harness.session.completeFunctionCall("call-1", [ - canonicalSegment("ask-2", "Who acts next?"), + expect.objectContaining({ code: "invalid-response", type: "error" }), ]); - const [functionOutput, responseCreate] = sentEvents(channel).slice(-2); - expect(functionOutput).toEqual({ - type: "conversation.item.create", - item: { - type: "function_call_output", - call_id: "call-1", - output: JSON.stringify({ response_text: ["Who acts next?"] }), - }, - }); - expect(responseCreate).toMatchObject({ - type: "response.create", - response: { - instructions: - "Speak only the response_text strings supplied by Petrinaut, in array order and verbatim. Deliver them as a warm, calm, curious, confident, concise, and professionally neutral expert interviewer, at a measured conversational pace with natural emphasis. Never sound robotic, fawning, rushed, overenthusiastic, or patronizing. Do not add, remove, paraphrase, acknowledge, or explain anything.", - output_modalities: ["audio"], - parallel_tool_calls: false, - tool_choice: "none", - tools: [], - }, - }); }); test("closes a stopped function call without requesting speech", async () => { @@ -383,18 +451,30 @@ describe("OpenAIRealtimeSession", () => { await harness.session.connect(); const channel = harness.channels[0]!; - harness.session.completeFunctionCall("call-exact", [ + harness.session.speakCanonical([ canonicalSegment("ask-exact", " Exact Brunch text.\n"), ]); - expect(sentEvents(channel)[0]).toEqual({ - type: "conversation.item.create", - item: { - type: "function_call_output", - call_id: "call-exact", - output: JSON.stringify({ - response_text: [" Exact Brunch text.\n"], - }), + expect(sentEvents(channel)[0]).toMatchObject({ + type: "response.create", + response: { + conversation: "none", + input: [ + { + type: "message", + role: "system", + content: [ + { + type: "input_text", + text: JSON.stringify({ + response_text: [" Exact Brunch text.\n"], + }), + }, + ], + }, + ], + tool_choice: "none", + tools: [], }, }); const sentCount = sentEvents(channel).length; @@ -459,7 +539,7 @@ describe("OpenAIRealtimeSession", () => { ]); const responseCreate = sentEvents(channel)[0]!; - harness.session.cancelOutput(); + void harness.session.cancelOutput(); expect( sentEvents(channel).filter(({ type }) => type === "response.cancel"), @@ -606,7 +686,7 @@ describe("OpenAIRealtimeSession", () => { type: "output_audio_buffer.started", }); - harness.session.cancelOutput(); + void harness.session.cancelOutput(); const cancelEvent = sentEvents(channel).findLast( ({ type }) => type === "response.cancel", )!; @@ -650,7 +730,7 @@ describe("OpenAIRealtimeSession", () => { type: "output_audio_buffer.started", }); - harness.session.cancelOutput(); + void harness.session.cancelOutput(); const cancelEvent = sentEvents(channel).findLast( ({ type }) => type === "response.cancel", )!; diff --git a/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.test.ts index b65fc510f9c..4deef6ec07d 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.test.ts @@ -7,26 +7,54 @@ import { } from "./realtime-brunch-bridge"; import type { CanonicalSpeechSegment } from "./canonical-speech"; -import type { OpenAIRealtimeSessionEvent } from "./openai-realtime-session"; +import type { + OpenAIRealtimeSessionEvent, + OpenAIRealtimeTranscriptKey, +} from "./openai-realtime-session"; const segment = ( id: string, text: string, - source: CanonicalSpeechSegment["source"] = "brunch-ask", + submissionId?: string, ): CanonicalSpeechSegment => ({ contentHash: "fnv1a32:12345678", id, messageId: `message-${id}`, partId: id, - source, + source: "assistant-text", + ...(submissionId === undefined ? {} : { submissionId }), text, }); +const transcriptKey = ( + connectionEpoch: number, + itemId = "user-item-1", + contentIndex = 0, +): OpenAIRealtimeTranscriptKey => ({ connectionEpoch, contentIndex, itemId }); + +const completedTranscript = ( + connectionEpoch: number, + text = "The supervisor approves it.", + itemId = "user-item-1", + contentIndex = 0, +): Extract => ({ + key: transcriptKey(connectionEpoch, itemId, contentIndex), + text, + type: "completed", +}); + +const failedTranscript = ( + connectionEpoch: number, + itemId = "user-item-1", +): Extract => ({ + key: transcriptKey(connectionEpoch, itemId), + type: "transcription-failed", +}); + const createHarness = () => { let listener: ((event: OpenAIRealtimeSessionEvent) => void) | undefined; const session = { completeFunctionCall: vi.fn(), - completeFunctionCallWithoutResponse: vi.fn(), speakCanonical: vi.fn(), subscribe: vi.fn((next: (event: OpenAIRealtimeSessionEvent) => void) => { listener = next; @@ -39,10 +67,14 @@ const createHarness = () => { ConstructorParameters< typeof RealtimeBrunchBridge >[0]["submitInterviewAnswer"] - >(async () => ({ - kind: "interactive-tool", - toolCallId: "ask-current", - })); + >(async (input) => { + input.onAdmission("submission-voice-1"); + return { + kind: "message", + messageId: input.id, + submissionId: "submission-voice-1", + }; + }); const bridge = new RealtimeBrunchBridge({ session, submitInterviewAnswer, @@ -59,89 +91,29 @@ const createHarness = () => { }; }; -const toolDelta = ( - connectionEpoch: number, - delta: string, -): Extract => ({ - callId: "call-1", - connectionEpoch, - delta, - itemId: "function-item-1", - responseId: "response-1", - type: "tool-arguments-delta", -}); - -const toolDone = ( - connectionEpoch: number, - argumentsJson = '{"answer":"The supervisor approves it."}', -): Extract => ({ - arguments: argumentsJson, - callId: "call-1", - connectionEpoch, - itemId: "function-item-1", - name: "continue_interview", - responseId: "response-1", - type: "tool-arguments-done", -}); - -const responseTerminal = ( - connectionEpoch: number, - status: "cancelled" | "completed" | "failed" | "incomplete", - responseId = "response-1", -): Extract => ({ - connectionEpoch, - responseId, - status, - type: "response-terminal", -}); +const startReady = ( + harness: ReturnType, + connectionEpoch = 3, +): void => { + harness.bridge.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [], + status: "ready", + }); + harness.bridge.start(connectionEpoch); +}; describe("RealtimeBrunchBridge", () => { - test("speaks the current canonical turn without replaying history", () => { + test("rehydrates settled canonical speech without submission or playback", () => { const harness = createHarness(); - const historical = segment( - "history", - "Do not replay this.", - "assistant-text", - ); - const preamble = { - ...segment("preamble", "Thanks. One more question.", "assistant-text"), - messageId: "message-current-turn", - }; - const question = { - ...segment("ask-current", "What happens after approval?"), - messageId: "message-current-turn", - }; harness.bridge.updateChat({ canAcceptInterviewAnswer: true, - canonicalSegments: [historical, preamble, question], + canonicalSegments: [ + segment("settled", "Already delivered.", "submission-settled"), + ], status: "ready", }); - harness.bridge.start(4); - - expect(harness.session.speakCanonical).toHaveBeenCalledOnce(); - expect(harness.session.speakCanonical).toHaveBeenCalledWith([ - preamble, - question, - ]); - }); - - test("rehydrates the settled Voice turn without resubmission or playback", () => { - const harness = createHarness(); - const settledResponse = { - ...segment( - "settled-response", - "This canonical response was already delivered.", - "assistant-text", - ), - submissionIds: ["submission-settled"], - }; - - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [settledResponse], - status: "ready", - }); harness.bridge.start(9); expect(harness.submitInterviewAnswer).not.toHaveBeenCalled(); @@ -150,706 +122,265 @@ describe("RealtimeBrunchBridge", () => { expect(harness.events).toEqual([]); }); - test("streams and validates one tool call, preserves ask correlation, and waits for canonical Brunch output", async () => { + test("submits only a completed transcript through the user admission target", async () => { const harness = createHarness(); - const question = segment("ask-current", "What happens after approval?"); - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [question], - status: "ready", - }); - harness.bridge.start(7); - harness.session.speakCanonical.mockClear(); + startReady(harness, 7); + const key = transcriptKey(7); + + harness.emit({ key, text: "The supervisor", type: "partial" }); + harness.emit({ + arguments: '{"answer":"Fabricated answer"}', + callId: "legacy-call", + connectionEpoch: 7, + itemId: "legacy-item", + name: "continue_interview", + responseId: "legacy-response", + type: "tool-arguments-done", + } as unknown as OpenAIRealtimeSessionEvent); + expect(harness.submitInterviewAnswer).not.toHaveBeenCalled(); - harness.emit(toolDelta(7, '{"answer":"The supervisor')); - harness.emit(toolDelta(7, ' approves it."}')); - harness.emit(toolDone(7)); + harness.emit(completedTranscript(7, " The supervisor\napproves it. ")); await vi.waitFor(() => expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce(), ); + const deliveryId = createRealtimeSubmissionId(key); expect(harness.submitInterviewAnswer).toHaveBeenCalledWith( expect.objectContaining({ - admissionTarget: { - kind: "client-tool-result", - messageId: "message-ask-current", - }, - id: createRealtimeSubmissionId(7, "call-1"), + admissionTarget: { kind: "user", messageId: deliveryId }, + id: deliveryId, text: "The supervisor approves it.", }), ); - expect(harness.session.completeFunctionCall).not.toHaveBeenCalled(); - - harness.bridge.updateChat({ - canAcceptInterviewAnswer: false, - canonicalSegments: [question], - status: "streaming", - }); - const acknowledgement = segment( - "acknowledgement", - "Thanks. I have recorded that.", - "assistant-text", - ); - const nextQuestion = segment( - "ask-next", - "Who is informed next?", - "brunch-ask", - ); - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [question, acknowledgement, nextQuestion], - status: "ready", + expect(harness.events).toContainEqual({ + answer: "The supervisor approves it.", + deliveryId, + type: "submission-started", }); - - await vi.waitFor(() => - expect(harness.session.completeFunctionCall).toHaveBeenCalledWith( - "call-1", - [acknowledgement, nextQuestion], - ), - ); - expect(harness.events.map(({ type }) => type)).toEqual([ - "submission-started", - "submission-accepted", - "canonical-text-ready", - "submission-settled", - "canonical-response-ready", - ]); + expect(JSON.stringify(harness.events)).not.toContain("Fabricated answer"); }); - test("emits the real admission before composer submission completes", async () => { - const harness = createHarness(); - let finishSubmission: (() => void) | undefined; - harness.submitInterviewAnswer.mockImplementationOnce(async () => { - await new Promise((resolve) => { - finishSubmission = resolve; - }); - return { - kind: "interactive-tool", - toolCallId: "ask-current", - }; - }); - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [segment("ask-current", "Question")], - status: "ready", - }); - harness.bridge.start(7); - harness.emit(toolDone(7)); - - await vi.waitFor(() => - expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce(), - ); - const submission = harness.submitInterviewAnswer.mock.calls[0]?.[0]; - expect(submission).toBeDefined(); - submission?.onAdmission("submission-early"); - submission?.onAdmission("submission-early"); - - expect(harness.events).toEqual([ - expect.objectContaining({ type: "submission-started" }), - { - callId: "call-1", - submissionId: "submission-early", - type: "submission-admitted", - }, - ]); - - finishSubmission?.(); - await vi.waitFor(() => - expect(harness.events).toContainEqual( - expect.objectContaining({ type: "submission-accepted" }), - ), - ); - const question = segment("ask-current", "Question"); - harness.bridge.updateChat({ - canAcceptInterviewAnswer: false, - canonicalSegments: [question], - status: "streaming", - }); - const unrelated = segment("unrelated", "Do not select this."); - const correlated = { - ...segment("correlated", "Select this response."), - submissionIds: ["submission-early"], - }; - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [question, unrelated, correlated], - status: "ready", - }); - expect(harness.session.completeFunctionCall).toHaveBeenCalledWith( - "call-1", - [correlated], - ); - harness.bridge.stop(); - submission?.onAdmission("submission-stale"); + test("derives stable delivery identity from epoch, item, and content index", () => { expect( - harness.events.filter(({ type }) => type === "submission-admitted"), - ).toHaveLength(1); + createRealtimeSubmissionId(transcriptKey(12, "item/with spaces", 4)), + ).toBe("voice-realtime:12:item%2Fwith%20spaces:4"); }); - test("admits one finalized Realtime answer through Flue once", async () => { + test("submits duplicate completed transcript events exactly once", async () => { const harness = createHarness(); - harness.submitInterviewAnswer.mockResolvedValueOnce({ - kind: "message", - messageId: "message-kickoff", - submissionId: "submission-voice-1", - }); - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [], - status: "ready", - }); - harness.bridge.start(7); + startReady(harness); + const transcript = completedTranscript(3); - harness.emit(toolDone(7, '{"answer":"Battery charger workflow"}')); - harness.emit(toolDone(7, '{"answer":"Battery charger workflow"}')); + harness.emit(transcript); + harness.emit(transcript); await vi.waitFor(() => - expect(harness.submitInterviewAnswer).toHaveBeenCalledWith( - expect.objectContaining({ - admissionTarget: { - kind: "user", - messageId: createRealtimeSubmissionId(7, "call-1"), - }, - id: createRealtimeSubmissionId(7, "call-1"), - text: "Battery charger workflow", - }), - ), + expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce(), ); - harness.bridge.updateChat({ - canAcceptInterviewAnswer: false, - canonicalSegments: [], - status: "submitted", - }); - const unrelated = { - ...segment("unrelated", "Do not speak this response."), - submissionIds: ["submission-other"], - }; - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [unrelated], - status: "ready", + expect(harness.events).toContainEqual({ + reason: "duplicate", + type: "transcript-rejected", }); - expect(harness.session.completeFunctionCall).not.toHaveBeenCalled(); - const firstQuestion = { - ...segment("ask-first", "What starts the battery charger workflow?"), - submissionIds: ["submission-voice-1"], - }; - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [unrelated, firstQuestion], - status: "ready", - }); - - expect(harness.session.completeFunctionCall).toHaveBeenCalledWith( - "call-1", - [firstQuestion], - ); - expect(harness.events.map(({ type }) => type)).toEqual([ - "submission-started", - "submission-accepted", - "canonical-text-ready", - "submission-settled", - "canonical-response-ready", - ]); }); - test("rejects a submission result that disagrees with the transport admission", async () => { + test.each([ + ["", "empty"], + [" \n\t ", "empty"], + ["a".repeat(32_001), "over-limit"], + ] as const)( + "rejects an invalid completed transcript as %s", + (text, reason) => { + const harness = createHarness(); + startReady(harness); + + harness.emit(completedTranscript(3, text)); + + expect(harness.submitInterviewAnswer).not.toHaveBeenCalled(); + expect(harness.events).toEqual([{ reason, type: "transcript-rejected" }]); + }, + ); + + test("rejects a failed transcript and accepts the next keyed turn", async () => { const harness = createHarness(); - harness.submitInterviewAnswer.mockImplementationOnce(async (input) => { - input.onAdmission("submission-early"); - return { - kind: "message", - messageId: input.id, - submissionId: "submission-other", - }; - }); - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [], - status: "ready", - }); - harness.bridge.start(7); - harness.emit(toolDone(7)); + startReady(harness); + + harness.emit(failedTranscript(3, "failed-item")); + expect(harness.events).toEqual([ + { reason: "failed", type: "transcript-rejected" }, + ]); + harness.emit(completedTranscript(3, "Retried answer.", "retry-item")); await vi.waitFor(() => - expect(harness.events).toContainEqual( - expect.objectContaining({ - code: "interview-correlation", - type: "error", - }), + expect(harness.submitInterviewAnswer).toHaveBeenCalledWith( + expect.objectContaining({ text: "Retried answer." }), ), ); }); - test("requires a correlated Brunch busy cycle before accepting new canonical segments", async () => { + test("rejects completed transcripts while the shared submission path is unavailable", () => { const harness = createHarness(); - const question = segment("ask-current", "What happens after approval?"); - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [question], - status: "ready", - }); - harness.bridge.start(7); - harness.emit(toolDone(7)); - await vi.waitFor(() => - expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce(), - ); - const unrelated = segment( - "unrelated", - "An unrelated canonical update.", - "assistant-text", - ); - - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [question, unrelated], - status: "ready", - }); - - expect(harness.session.completeFunctionCall).not.toHaveBeenCalled(); - harness.bridge.updateChat({ canAcceptInterviewAnswer: false, - canonicalSegments: [question, unrelated], - status: "submitted", - }); - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [question, unrelated], - status: "ready", - }); - - expect(harness.session.completeFunctionCall).toHaveBeenCalledWith( - "call-1", - [unrelated], - ); - }); - - test("rejects streamed arguments whose response or item identity changes", async () => { - const harness = createHarness(); - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [segment("ask-current", "Question")], - status: "ready", + canonicalSegments: [], + status: "streaming", }); harness.bridge.start(3); - harness.emit(toolDelta(3, '{"answer":"Answer"}')); - harness.emit({ - ...toolDone(3, '{"answer":"Answer"}'), - responseId: "response-2", - }); - await Promise.resolve(); + harness.emit(completedTranscript(3)); expect(harness.submitInterviewAnswer).not.toHaveBeenCalled(); expect(harness.events).toEqual([ - expect.objectContaining({ - code: "interview-correlation", - type: "error", - }), + { reason: "unavailable", type: "transcript-rejected" }, ]); }); - test("rejects concurrent argument streams before either can submit", async () => { + test("ignores transcripts from an inactive connection epoch", () => { const harness = createHarness(); - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [segment("ask-current", "Question")], - status: "ready", - }); - harness.bridge.start(3); - harness.emit(toolDelta(3, '{"answer":"First"}')); + startReady(harness, 2); - harness.emit({ - ...toolDelta(3, '{"answer":"Second"}'), - callId: "call-2", - itemId: "function-item-2", - }); + harness.emit(completedTranscript(1, "Stale answer")); + harness.emit(failedTranscript(1, "stale-failed")); expect(harness.submitInterviewAnswer).not.toHaveBeenCalled(); - expect(harness.events).toEqual([ - expect.objectContaining({ type: "error" }), - ]); + expect(harness.events).toEqual([]); }); - test("discards a cancelled argument stream without poisoning the next answer", async () => { + test("correlates the admitted submission with exact canonical response segments", async () => { const harness = createHarness(); - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [segment("ask-current", "Question")], - status: "ready", - }); - harness.bridge.start(3); - harness.emit(toolDelta(3, '{"answer":"Cancelled"}')); - harness.emit(responseTerminal(3, "cancelled")); - harness.emit(toolDone(3, '{"answer":"Cancelled"}')); - - harness.emit({ - ...toolDelta(3, '{"answer":"Accepted"}'), - callId: "call-2", - itemId: "function-item-2", - responseId: "response-2", - }); - harness.emit({ - ...toolDone(3, '{"answer":"Accepted"}'), - callId: "call-2", - itemId: "function-item-2", - responseId: "response-2", - }); - + startReady(harness, 7); + harness.emit(completedTranscript(7)); await vi.waitFor(() => expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce(), ); - expect(harness.submitInterviewAnswer).toHaveBeenCalledWith( - expect.objectContaining({ - admissionTarget: { - kind: "client-tool-result", - messageId: "message-ask-current", - }, - id: createRealtimeSubmissionId(3, "call-2"), - text: "Accepted", - }), - ); - expect(harness.events).not.toContainEqual( - expect.objectContaining({ type: "error" }), - ); - }); - - test("rejects an unfinished argument stream from a completed response", () => { - const harness = createHarness(); - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [segment("ask-current", "Question")], - status: "ready", - }); - harness.bridge.start(3); - harness.emit(toolDelta(3, '{"answer":"Incomplete')); + const input = harness.submitInterviewAnswer.mock.calls[0]?.[0]; + expect(input).toBeDefined(); - harness.emit(responseTerminal(3, "completed")); - - expect(harness.submitInterviewAnswer).not.toHaveBeenCalled(); - expect(harness.events).toEqual([ - expect.objectContaining({ - code: "interview-correlation", - type: "error", - }), - ]); - }); - - test("rejects duplicate, stale, overlapping, and malformed calls without another Brunch submission", async () => { - const harness = createHarness(); - const question = segment("ask-current", "What happens after approval?"); + input?.onAdmission("submission-voice-1"); harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [question], - status: "ready", + canAcceptInterviewAnswer: false, + canonicalSegments: [], + status: "submitted", }); - harness.bridge.start(2); - - harness.emit(toolDone(1)); - harness.emit(toolDelta(2, '{"answer":"The supervisor approves it."}')); - harness.emit(toolDone(2)); - harness.emit(toolDone(2)); - await vi.waitFor(() => - expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce(), + const unrelated = segment( + "unrelated", + "Do not speak this.", + "submission-other", + ); + const correlated = segment( + "correlated", + "Speak this canonical response.", + "submission-voice-1", ); - - harness.emit({ - ...toolDone(2, '{"answer":"Overlapping"}'), - callId: "call-2", - itemId: "function-item-2", - }); - - expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce(); - expect(harness.events.at(-1)).toMatchObject({ type: "error" }); - }); - - test.each([ - ["wrong tool", { ...toolDone(3), name: "invent_question" }], - ["invalid JSON", toolDone(3, "not-json")], - ["extra property", toolDone(3, '{"answer":"Valid","extra":true}')], - ["empty answer", toolDone(3, '{"answer":" "}')], - ])("rejects %s arguments", async (_label, event) => { - const harness = createHarness(); harness.bridge.updateChat({ canAcceptInterviewAnswer: true, - canonicalSegments: [segment("ask-current", "Question")], + canonicalSegments: [unrelated, correlated], status: "ready", }); - harness.bridge.start(3); - - harness.emit(event); - await Promise.resolve(); - expect(harness.submitInterviewAnswer).not.toHaveBeenCalled(); - expect(harness.events).toEqual([ - expect.objectContaining({ type: "error" }), + const deliveryId = createRealtimeSubmissionId(transcriptKey(7)); + expect(harness.session.speakCanonical).toHaveBeenCalledWith([correlated]); + expect(harness.session.completeFunctionCall).not.toHaveBeenCalled(); + expect(harness.events.map(({ type }) => type)).toEqual([ + "submission-started", + "submission-admitted", + "submission-accepted", + "canonical-text-ready", + "submission-settled", + "canonical-response-ready", ]); - }); - - test("rejects a composer result that does not match the pending brunch_ask", async () => { - const harness = createHarness(); - harness.submitInterviewAnswer.mockResolvedValueOnce({ - kind: "interactive-tool", - toolCallId: "another-ask", - }); - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [segment("ask-current", "Question")], - status: "ready", + expect(harness.events.at(-1)).toEqual({ + deliveryId, + segments: [correlated], + type: "canonical-response-ready", }); - harness.bridge.start(5); - - harness.emit(toolDone(5)); - - await vi.waitFor(() => - expect(harness.events.at(-1)).toMatchObject({ type: "error" }), - ); - expect(harness.session.completeFunctionCall).not.toHaveBeenCalled(); }); - test("records first canonical text before the turn settles", async () => { + test("does not start speech cancelled while its correlated response is pending", async () => { const harness = createHarness(); - const question = segment("ask-current", "Question"); - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [question], - status: "ready", - }); - harness.bridge.start(7); - harness.emit(toolDone(7)); + startReady(harness, 7); + harness.emit(completedTranscript(7)); await vi.waitFor(() => expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce(), ); - harness.submitInterviewAnswer.mock.calls[0]?.[0].onAdmission( - "submission-text", - ); - await vi.waitFor(() => - expect(harness.events).toContainEqual( - expect.objectContaining({ type: "submission-accepted" }), - ), - ); - harness.bridge.updateChat({ - canAcceptInterviewAnswer: false, - canonicalSegments: [question], - status: "streaming", - }); - const firstText = { - ...segment("first", "First completed block.", "assistant-text"), - submissionIds: ["submission-text"], - }; harness.bridge.updateChat({ canAcceptInterviewAnswer: false, - canonicalSegments: [question, firstText], + canonicalSegments: [], status: "streaming", }); - const typesWhileStreaming = harness.events.map(({ type }) => type); - expect(typesWhileStreaming).toContain("canonical-text-ready"); - expect(typesWhileStreaming).not.toContain("submission-settled"); - expect(harness.session.completeFunctionCall).not.toHaveBeenCalled(); + harness.bridge.cancelPendingSpeech(); - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [question, firstText], - status: "ready", - }); - expect( - harness.events.filter(({ type }) => type === "canonical-text-ready"), - ).toHaveLength(1); - expect(harness.events.map(({ type }) => type)).toContain( - "submission-settled", + const correlated = segment( + "correlated", + "Retain this without speaking it.", + "submission-voice-1", ); - expect(harness.session.completeFunctionCall).toHaveBeenCalledWith( - "call-1", - [firstText], - ); - }); - - test("closes a durably stopped Voice turn without speaking", async () => { - const harness = createHarness(); - const question = segment("ask-current", "Question"); harness.bridge.updateChat({ canAcceptInterviewAnswer: true, - canonicalSegments: [question], + canonicalSegments: [correlated], status: "ready", }); - harness.bridge.start(7); - harness.emit(toolDone(7)); - await vi.waitFor(() => - expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce(), - ); - harness.submitInterviewAnswer.mock.calls[0]?.[0].onAdmission( - "submission-stopped", - ); - await vi.waitFor(() => - expect(harness.events).toContainEqual( - expect.objectContaining({ type: "submission-accepted" }), - ), - ); - harness.bridge.updateChat({ - canAcceptInterviewAnswer: false, - canonicalSegments: [question], - status: "streaming", - }); - // A completed step with no text is not a stop: the panel may still be - // sending the client-tool follow-up that carries the reply. - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [question], - settlements: [ - { outcome: "completed", submissionId: "submission-stopped" }, - ], - status: "ready", - }); - expect( - harness.session.completeFunctionCallWithoutResponse, - ).not.toHaveBeenCalled(); - expect(harness.events.map(({ type }) => type)).not.toContain( - "submission-settled", - ); - - const speechRequestsBeforeStop = - harness.session.speakCanonical.mock.calls.length; - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [question], - settlements: [{ outcome: "aborted", submissionId: "submission-stopped" }], - status: "ready", + expect(harness.session.speakCanonical).not.toHaveBeenCalled(); + expect(harness.events.at(-1)).toMatchObject({ + segments: [correlated], + speechCancelled: true, + type: "canonical-response-ready", }); - expect( - harness.session.completeFunctionCallWithoutResponse, - ).toHaveBeenCalledWith("call-1", "aborted"); - expect(harness.session.completeFunctionCall).not.toHaveBeenCalled(); - expect(harness.session.speakCanonical).toHaveBeenCalledTimes( - speechRequestsBeforeStop, - ); - expect(harness.events.map(({ type }) => type)).toEqual( - expect.arrayContaining(["submission-settled", "submission-stopped"]), - ); - expect(harness.events.some(({ type }) => type === "error")).toBe(false); }); - test("speaks the folded continuation that answers a Voice brunch_ask follow-up", async () => { + test("rejects a path-B result that does not preserve the delivery identity", async () => { const harness = createHarness(); - const question = { - ...segment("ask-current", "Question"), - submissionIds: ["submission-question"], - }; - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [question], - status: "ready", + harness.submitInterviewAnswer.mockResolvedValueOnce({ + kind: "message", + messageId: "different-message", + submissionId: "submission-voice-1", }); - harness.bridge.start(7); - harness.emit(toolDone(7)); - await vi.waitFor(() => - expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce(), - ); - harness.submitInterviewAnswer.mock.calls[0]?.[0].onAdmission( - "submission-answer", - ); + startReady(harness); + + harness.emit(completedTranscript(3)); + await vi.waitFor(() => expect(harness.events).toContainEqual( - expect.objectContaining({ type: "submission-accepted" }), + expect.objectContaining({ + code: "interview-correlation", + type: "error", + }), ), ); - harness.bridge.updateChat({ - canAcceptInterviewAnswer: false, - canonicalSegments: [question], - status: "streaming", - }); - - // The continuation is projected onto the message that asked, so that - // message is now written by both submissions. - const nextQuestion = { - ...segment("ask-next", "Next question"), - messageId: question.messageId, - submissionIds: ["submission-question", "submission-answer"], - }; - const askedAgain = { - ...question, - submissionIds: nextQuestion.submissionIds, - }; - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [askedAgain, nextQuestion], - status: "ready", - }); - - expect(harness.session.completeFunctionCall).toHaveBeenCalledWith( - "call-1", - [nextQuestion], - ); }); - test("speaks a reply that arrives through a client-tool follow-up", async () => { + test("requires a shared chat busy cycle before accepting new canonical text", async () => { const harness = createHarness(); - harness.submitInterviewAnswer.mockResolvedValueOnce({ - kind: "message", - messageId: "message-kickoff", - submissionId: "submission-voice-1", - }); - harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [], - status: "ready", - }); - harness.bridge.start(7); - harness.emit(toolDone(7, '{"answer":"Read the guide first."}')); + startReady(harness); + harness.emit(completedTranscript(3)); await vi.waitFor(() => - expect(harness.events).toContainEqual( - expect.objectContaining({ type: "submission-accepted" }), - ), + expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce(), + ); + const response = segment( + "response", + "Canonical response.", + "submission-voice-1", ); - harness.bridge.updateChat({ - canAcceptInterviewAnswer: false, - canonicalSegments: [], - status: "submitted", - }); - // Brunch read a doc mid-turn; the panel's follow-up submission finished - // the same assistant message. - const reply = { - ...segment("reply", "The guide says hello.", "assistant-text"), - submissionIds: ["submission-voice-1", "submission-doc-follow-up"], - }; harness.bridge.updateChat({ canAcceptInterviewAnswer: true, - canonicalSegments: [reply], + canonicalSegments: [response], status: "ready", }); + expect(harness.session.speakCanonical).not.toHaveBeenCalled(); - expect(harness.session.completeFunctionCall).toHaveBeenCalledWith( - "call-1", - [reply], - ); - }); - - test("speaks new canonical text turns without creating a Realtime tool result", () => { - const harness = createHarness(); - const question = segment("ask-current", "Question"); harness.bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [question], - status: "ready", + canAcceptInterviewAnswer: false, + canonicalSegments: [response], + status: "streaming", }); - harness.bridge.start(8); - harness.session.speakCanonical.mockClear(); - const response = segment( - "typed-response", - "Canonical response", - "assistant-text", - ); - harness.bridge.updateChat({ canAcceptInterviewAnswer: true, - canonicalSegments: [question, response], + canonicalSegments: [response], status: "ready", }); expect(harness.session.speakCanonical).toHaveBeenCalledWith([response]); - expect(harness.session.completeFunctionCall).not.toHaveBeenCalled(); }); }); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts index 6e4a0a63095..2bf0d41fd00 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts @@ -73,11 +73,9 @@ const initialMessages = [ id: "initial-question-message", parts: [ { - input: { question: "What happens after approval?" }, - state: "input-available", - toolCallId: "ask-current", - toolName: "brunch_ask", - type: "dynamic-tool", + state: "done", + text: "What happens after approval?", + type: "text", }, ], role: "assistant", @@ -95,11 +93,9 @@ const responseMessages = [ id: "next-question-message", parts: [ { - input: { question: canonicalQuestion }, - state: "input-available", - toolCallId: "ask-next", - toolName: "brunch_ask", - type: "dynamic-tool", + state: "done", + text: canonicalQuestion, + type: "text", }, ], role: "assistant", @@ -107,7 +103,7 @@ const responseMessages = [ ] satisfies PetrinautAiMessage[]; describe("controlled voice preview", () => { - test("bridges one Realtime tool call through Brunch and back to canonical duplex audio", async () => { + test("bridges one completed transcript through Brunch and back to canonical half-duplex audio", async () => { const diagnostics: VoiceDiagnosticEvent[] = []; const reportDiagnostic = (event: VoiceDiagnosticEvent) => diagnostics.push(event); @@ -211,10 +207,16 @@ describe("controlled voice preview", () => { reportDiagnostic, requestAnimationFrame: vi.fn(() => 1), }); - const submitInterviewAnswer = vi.fn(async () => ({ - kind: "interactive-tool" as const, - toolCallId: "ask-current", - })); + const submissionId = "submission-voice-1"; + type SubmitInterviewAnswer = ConstructorParameters< + typeof RealtimeBrunchBridge + >[0]["submitInterviewAnswer"]; + const submitInterviewAnswer = vi.fn( + async ({ id, onAdmission }) => { + onAdmission(submissionId); + return { kind: "message", messageId: id, submissionId }; + }, + ); const bridge = new RealtimeBrunchBridge({ session, submitInterviewAnswer, @@ -222,32 +224,53 @@ describe("controlled voice preview", () => { const controller = new VoiceTurnController({ bridge, session, - submitText: submitInterviewAnswer, + submitText: vi.fn(async () => ({ kind: "message" as const })), }); + await controller.start(); + const initialSegments = selectCanonicalSpeechSegments(initialMessages); controller.updateChat({ canAcceptInterviewAnswer: true, - canonicalSegments: selectCanonicalSpeechSegments(initialMessages), + canonicalSegments: initialSegments, status: "ready", }); - - await controller.start(); authorizeLatestSpeechResponse(dataChannel, "response-initial-question"); dataChannel.receive({ response_id: "response-initial-question", type: "output_audio_buffer.started", }); + expect(controller.getSnapshot()).toMatchObject({ + canTakeTurn: true, + microphoneEnabled: true, + output: "speaking", + }); + expect(track.enabled).toBe(false); + + const handoff = controller.takeTurn(); dataChannel.receive({ audio_start_ms: 300, - item_id: "user-item", + item_id: "playback-overlap", type: "input_audio_buffer.speech_started", }); + dataChannel.receive({ + content_index: 0, + item_id: "playback-overlap", + transcript: "Playback must not become input.", + type: "conversation.item.input_audio_transcription.completed", + }); + dataChannel.receive({ type: "input_audio_buffer.cleared" }); dataChannel.receive({ response: { id: "response-initial-question", + output: [], status: "cancelled", }, type: "response.done", }); + dataChannel.receive({ + response_id: "response-initial-question", + type: "output_audio_buffer.cleared", + }); + await handoff; expect(controller.getSnapshot()).toMatchObject({ input: "listening", microphoneEnabled: true, @@ -255,39 +278,31 @@ describe("controlled voice preview", () => { }); dataChannel.receive({ - call_id: "call-1", - delta: `{"answer":"${spokenAnswer}"}`, - item_id: "function-item-1", - output_index: 0, - response_id: "response-tool-1", - type: "response.function_call_arguments.delta", + audio_start_ms: 500, + item_id: "user-item", + type: "input_audio_buffer.speech_started", }); dataChannel.receive({ - response: { - id: "response-tool-1", - output: [ - { - arguments: `{"answer":"${spokenAnswer}"}`, - call_id: "call-1", - id: "function-item-1", - name: "continue_interview", - status: "completed", - type: "function_call", - }, - ], - status: "completed", - }, - type: "response.done", + content_index: 0, + delta: "The supervisor", + item_id: "user-item", + type: "conversation.item.input_audio_transcription.delta", + }); + dataChannel.receive({ + content_index: 0, + item_id: "user-item", + transcript: spokenAnswer, + type: "conversation.item.input_audio_transcription.completed", }); await vi.waitFor(() => expect(submitInterviewAnswer).toHaveBeenCalledWith( expect.objectContaining({ admissionTarget: { - kind: "client-tool-result", - messageId: "initial-question-message", + kind: "user", + messageId: "voice-realtime:1:user-item:0", }, - id: "voice-realtime:1:call-1", + id: "voice-realtime:1:user-item:0", text: spokenAnswer, }), ), @@ -301,29 +316,43 @@ describe("controlled voice preview", () => { controller.updateChat({ canAcceptInterviewAnswer: false, - canonicalSegments: selectCanonicalSpeechSegments(initialMessages), + canonicalSegments: initialSegments, status: "streaming", }); + const initialSegmentIds = new Set(initialSegments.map(({ id }) => id)); + const correlatedSegments = selectCanonicalSpeechSegments( + responseMessages, + ).map((segment) => + initialSegmentIds.has(segment.id) + ? segment + : { ...segment, submissionId }, + ); controller.updateChat({ canAcceptInterviewAnswer: true, - canonicalSegments: selectCanonicalSpeechSegments(responseMessages), + canonicalSegments: correlatedSegments, status: "ready", }); - const [functionOutput, responseCreate] = sentEvents(dataChannel).slice(-2); - expect(functionOutput).toEqual({ - type: "conversation.item.create", - item: { - type: "function_call_output", - call_id: "call-1", - output: JSON.stringify({ - response_text: [canonicalReply, canonicalQuestion], - }), - }, - }); + const responseCreate = sentEvents(dataChannel).findLast( + ({ type }) => type === "response.create", + ); expect(responseCreate).toMatchObject({ type: "response.create", response: { + input: [ + { + content: [ + { + text: JSON.stringify({ + response_text: [canonicalReply, canonicalQuestion], + }), + type: "input_text", + }, + ], + role: "system", + type: "message", + }, + ], output_modalities: ["audio"], tool_choice: "none", tools: [], @@ -341,6 +370,7 @@ describe("controlled voice preview", () => { microphoneEnabled: true, output: "speaking", }); + expect(track.enabled).toBe(false); const remoteTrack = { kind: "audio", stop: vi.fn() }; const remoteStream = { @@ -373,11 +403,13 @@ describe("controlled voice preview", () => { turn_detection: { type: "semantic_vad", eagerness: "low", - create_response: true, - interrupt_response: true, + create_response: false, + interrupt_response: false, }, }, }, + tool_choice: "none", + tools: [], }); expect(diagnostics).toEqual( expect.arrayContaining([ @@ -443,8 +475,6 @@ describe("controlled voice preview", () => { | undefined; const bridge = new RealtimeBrunchBridge({ session: { - completeFunctionCall: vi.fn(), - completeFunctionCallWithoutResponse: vi.fn(), speakCanonical: vi.fn(), subscribe: (listener) => { realtimeListener = listener; @@ -457,6 +487,7 @@ describe("controlled voice preview", () => { admissionTarget, id, onAdmission, + signal, text, }) => { const unsubscribe = tracker.subscribeToAdmission( @@ -475,7 +506,7 @@ describe("controlled voice preview", () => { parts: [{ type: "text", text }], }, ], - abortSignal: undefined, + abortSignal: signal, }); try { await stream.pipeTo(new WritableStream()); @@ -498,14 +529,14 @@ describe("controlled voice preview", () => { }); bridge.start(1); - const finalized = { - arguments: '{"answer":"The supervisor approves it."}', - callId: "call-1", - connectionEpoch: 1, - itemId: "function-item-1", - name: "continue_interview", - responseId: "response-1", - type: "tool-arguments-done" as const, + const finalized: OpenAIRealtimeSessionEvent = { + key: { + connectionEpoch: 1, + contentIndex: 0, + itemId: "input-item-1", + }, + text: spokenAnswer, + type: "completed", }; realtimeListener?.(finalized); realtimeListener?.(finalized); @@ -513,7 +544,7 @@ describe("controlled voice preview", () => { await vi.waitFor(() => expect(send).toHaveBeenCalledOnce()); await vi.waitFor(() => expect(bridgeEvents).toContainEqual({ - callId: "call-1", + deliveryId: "voice-realtime:1:input-item-1:0", submissionId: admission.submissionId, type: "submission-admitted", }), @@ -521,10 +552,13 @@ describe("controlled voice preview", () => { expect(bridgeEvents).not.toContainEqual( expect.objectContaining({ type: "submission-accepted" }), ); - expect(send).toHaveBeenCalledWith({ - message: { kind: "user", body: "The supervisor approves it." }, - signal: undefined, + expect(send).toHaveBeenCalledOnce(); + const sendInput = send.mock.calls[0]?.[0]; + expect(sendInput).toMatchObject({ + idempotencyKey: "ai-sdk:voice-realtime:1:input-item-1:0", + message: { kind: "user", body: spokenAnswer }, }); + expect(sendInput?.signal).toBeInstanceOf(AbortSignal); expect(admission.streamUrl).toContain("/agents/chat/"); settleSubmission?.(); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.test.ts index d637a762607..912282facab 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.test.ts @@ -5,6 +5,9 @@ import { toVoiceSessionState } from "./voice-session-state"; import type { VoiceTurnSnapshot } from "./voice-turn-controller"; const listeningSnapshot = { + canReadFullResponse: false, + canRepeatQuestion: false, + canTakeTurn: false, canReviseLastAnswer: false, connection: "connected", currentQuestion: "What happens after approval?", @@ -12,6 +15,7 @@ const listeningSnapshot = { errorMessage: "", errorRequestId: "", input: "listening", + inputNotice: "none", lastAnswerDelivery: "none", lastCommittedText: "", microphoneEnabled: true, @@ -30,13 +34,40 @@ describe("toVoiceSessionState", () => { test("reports a listening turn with its microphone level", () => { expect(mapSnapshot()).toEqual({ + canReadFullResponse: false, + canRepeatQuestion: false, + canTakeTurn: false, errorMessage: null, microphoneLevel: 0.24, microphoneMuted: false, + notice: null, phase: "listening", }); }); + test("publishes safe handoff and canonical playback availability", () => { + expect( + mapSnapshot({ + canReadFullResponse: true, + canRepeatQuestion: true, + canTakeTurn: true, + }), + ).toMatchObject({ + canReadFullResponse: true, + canRepeatQuestion: true, + canTakeTurn: true, + }); + }); + + test("describes recoverable transcript rejections", () => { + expect(mapSnapshot({ inputNotice: "not-heard" })?.notice).toBe( + "We didn't catch that. Please try again.", + ); + expect(mapSnapshot({ inputNotice: "too-long" })?.notice).toBe( + "That answer is too long. Please try a shorter response.", + ); + }); + test("hands the turn to the assistant while it speaks", () => { expect(mapSnapshot({ output: "speaking", partialText: "" })).toMatchObject({ phase: "speaking", diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts index 08dbffb2e7a..d73538c3a2a 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts @@ -16,10 +16,11 @@ const createHarness = () => { | undefined; let bridgeListener: ((event: RealtimeBrunchBridgeEvent) => void) | undefined; const session = { - cancelOutput: vi.fn(), + cancelOutput: vi.fn<() => Promise>(async () => undefined), connect: vi.fn(async () => ++epoch), disconnect: vi.fn(async () => undefined), setMicrophoneEnabled: vi.fn(), + speakCanonical: vi.fn(), subscribe: vi.fn( (listener: (event: OpenAIRealtimeSessionEvent) => void) => { sessionListener = listener; @@ -30,6 +31,7 @@ const createHarness = () => { ), }; const bridge = { + cancelPendingSpeech: vi.fn(), start: vi.fn(), stop: vi.fn(), subscribe: vi.fn((listener: (event: RealtimeBrunchBridgeEvent) => void) => { @@ -73,7 +75,7 @@ const question = ( id, messageId: `message-${id}`, partId: id, - source: "brunch-ask", + source: "assistant-text", text, }); @@ -84,33 +86,33 @@ describe("VoiceTurnController", () => { harness.emitBridge({ answer: "Private finalized answer", - callId: "call-opaque", + deliveryId: "call-opaque", type: "submission-started", }); harness.advanceTime(10); harness.emitBridge({ - callId: "call-opaque", + deliveryId: "call-opaque", submissionId: "submission-opaque", type: "submission-admitted", }); harness.emitBridge({ - callId: "call-opaque", + deliveryId: "call-opaque", submissionId: "submission-opaque", type: "submission-admitted", }); harness.advanceTime(10); harness.emitBridge({ answer: "Private finalized answer", - callId: "call-opaque", + deliveryId: "call-opaque", type: "submission-accepted", }); harness.emitBridge({ - callId: "call-opaque", + deliveryId: "call-opaque", type: "canonical-text-ready", }); harness.advanceTime(10); harness.emitBridge({ - callId: "call-opaque", + deliveryId: "call-opaque", type: "submission-settled", }); harness.advanceTime(10); @@ -167,7 +169,7 @@ describe("VoiceTurnController", () => { await harness.controller.end(); harness.emitBridge({ - callId: "call-opaque", + deliveryId: "call-opaque", type: "submission-settled", }); harness.emitSession(outputStarted); @@ -199,7 +201,7 @@ describe("VoiceTurnController", () => { }); }); - test("keeps capture active while the interviewer speaks and interrupts automatically", async () => { + test("tracks assistant playback without admitting automatic barge-in", async () => { const harness = createHarness(); await harness.controller.start(); @@ -222,18 +224,243 @@ describe("VoiceTurnController", () => { }); expect(harness.controller.getSnapshot()).toMatchObject({ microphoneEnabled: true, - output: "interrupted", + output: "speaking", }); expect(harness.session.cancelOutput).not.toHaveBeenCalled(); }); - test("represents submitting and output independently without closing capture", async () => { + test("hands off an active response once and applies the latest mute preference after cancellation", async () => { const harness = createHarness(); + let finishCancellation: (() => void) | undefined; + harness.controller.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [question("ask-handoff")], + status: "ready", + }); await harness.controller.start(); + harness.session.cancelOutput.mockImplementationOnce( + () => + new Promise((resolve) => { + finishCancellation = resolve; + }), + ); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-handoff", + speechRequestId: "speech-handoff", + type: "output-started", + }); + harness.session.cancelOutput.mockClear(); + + expect(harness.controller.getSnapshot().canTakeTurn).toBe(true); + const handoff = harness.controller.takeTurn(); + const repeatedHandoff = harness.controller.takeTurn(); + + expect(repeatedHandoff).toBe(handoff); + expect(harness.bridge.cancelPendingSpeech).toHaveBeenCalledOnce(); + expect(harness.session.cancelOutput).toHaveBeenCalledOnce(); + expect(harness.session.setMicrophoneEnabled).toHaveBeenLastCalledWith( + false, + ); + expect(harness.controller.getSnapshot()).toMatchObject({ + canTakeTurn: false, + output: "cancelling", + }); + + harness.session.setMicrophoneEnabled.mockClear(); + harness.controller.setMicrophoneMuted(true); + harness.controller.setMicrophoneMuted(false); + expect(harness.session.setMicrophoneEnabled).not.toHaveBeenCalled(); + expect(harness.controller.getSnapshot().microphoneEnabled).toBe(true); + + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-handoff", + type: "output-interrupted", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-handoff", + status: "cancelled", + type: "response-terminal", + }); + expect(harness.controller.getSnapshot().output).toBe("cancelling"); + + finishCancellation?.(); + await handoff; + + expect(harness.session.setMicrophoneEnabled).toHaveBeenCalledOnce(); + expect(harness.session.setMicrophoneEnabled).toHaveBeenCalledWith(true); + expect(harness.controller.getSnapshot()).toMatchObject({ + canTakeTurn: false, + microphoneEnabled: true, + output: "interrupted", + }); + }); + + test("keeps the user turn when cancelled pending speech settles later", async () => { + const harness = createHarness(); + harness.controller.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [question("ask-handoff")], + status: "ready", + }); + await harness.controller.start(); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-handoff", + speechRequestId: "speech-handoff", + type: "output-started", + }); + + await harness.controller.takeTurn(); + harness.session.setMicrophoneEnabled.mockClear(); + harness.emitBridge({ + deliveryId: "voice-1", + segments: [question("ask-late", "Retained late response")], + speechCancelled: true, + type: "canonical-response-ready", + }); + + expect(harness.session.setMicrophoneEnabled).not.toHaveBeenCalled(); + expect(harness.controller.getSnapshot()).toMatchObject({ + input: "listening", + microphoneEnabled: true, + output: "interrupted", + }); + }); + + test("replays exact canonical response segments only after matching audio and response settlement", async () => { + const harness = createHarness(); + const context = question("context", "Approval is required before release."); + const nextQuestion = question("ask-replay", "Who approves release?"); + await harness.controller.start(); + + harness.emitBridge({ + deliveryId: "voice-1", + segments: [context, nextQuestion], + type: "canonical-response-ready", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-source", + speechRequestId: "speech-source", + type: "output-started", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-source", + type: "output-stopped", + }); + + expect(harness.controller.getSnapshot()).toMatchObject({ + canReadFullResponse: false, + canRepeatQuestion: false, + }); + harness.controller.readFullResponse(); + harness.controller.repeatQuestion(); + expect(harness.session.speakCanonical).not.toHaveBeenCalled(); + + harness.emitSession({ + connectionEpoch: 1, + responseId: "unrelated-response", + status: "completed", + type: "response-terminal", + }); + expect(harness.controller.getSnapshot().canRepeatQuestion).toBe(false); + + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-source", + status: "completed", + type: "response-terminal", + }); + expect(harness.controller.getSnapshot()).toMatchObject({ + canReadFullResponse: true, + canRepeatQuestion: true, + }); + + harness.controller.repeatQuestion(); + expect(harness.session.speakCanonical).toHaveBeenCalledWith([nextQuestion]); + expect(harness.controller.getSnapshot()).toMatchObject({ + canReadFullResponse: false, + canRepeatQuestion: false, + output: "waiting-for-tool", + }); + + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-replay", + speechRequestId: "speech-replay", + type: "output-started", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-replay", + status: "completed", + type: "response-terminal", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-replay", + type: "output-stopped", + }); + harness.controller.readFullResponse(); + + expect(harness.session.speakCanonical).toHaveBeenNthCalledWith(2, [ + context, + nextQuestion, + ]); + }); + + test("disables replay while the user is capturing input", async () => { + const harness = createHarness(); + const segment = question("ask-capture"); + await harness.controller.start(); + harness.emitBridge({ + deliveryId: "voice-1", + segments: [segment], + type: "canonical-response-ready", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-source", + speechRequestId: "speech-source", + type: "output-started", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-source", + type: "output-stopped", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-source", + status: "completed", + type: "response-terminal", + }); + expect(harness.controller.getSnapshot().canRepeatQuestion).toBe(true); + + harness.emitSession({ + connectionEpoch: 1, + itemId: "item-user", + type: "input-speech-started", + }); + + expect(harness.controller.getSnapshot()).toMatchObject({ + canReadFullResponse: false, + canRepeatQuestion: false, + }); + }); + + test("keeps capture closed from submission until canonical output settles", async () => { + const harness = createHarness(); + await harness.controller.start(); + harness.session.setMicrophoneEnabled.mockClear(); harness.emitBridge({ answer: "The supervisor approves it.", - callId: "call-1", + deliveryId: "call-1", type: "submission-started", }); expect(harness.controller.getSnapshot()).toMatchObject({ @@ -243,13 +470,16 @@ describe("VoiceTurnController", () => { microphoneEnabled: true, output: "waiting-for-tool", }); + expect(harness.session.setMicrophoneEnabled).toHaveBeenLastCalledWith( + false, + ); harness.emitBridge({ answer: "The supervisor approves it.", - callId: "call-1", + deliveryId: "call-1", type: "submission-accepted", }); harness.emitBridge({ - callId: "call-1", + deliveryId: "call-1", segments: [question("ask-2", "Who acts next?")], type: "canonical-response-ready", }); @@ -260,9 +490,33 @@ describe("VoiceTurnController", () => { microphoneEnabled: true, output: "waiting-for-tool", }); - expect(harness.session.setMicrophoneEnabled).not.toHaveBeenCalledWith( - false, - ); + expect(harness.session.setMicrophoneEnabled).not.toHaveBeenCalledWith(true); + + harness.emitSession({ + connectionEpoch: 1, + speechRequestId: "speech-next", + type: "canonical-speech-requested", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-next", + speechRequestId: "speech-next", + type: "output-started", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-next", + status: "completed", + type: "response-terminal", + }); + expect(harness.session.setMicrophoneEnabled).not.toHaveBeenCalledWith(true); + + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-next", + type: "output-stopped", + }); + expect(harness.session.setMicrophoneEnabled).toHaveBeenLastCalledWith(true); }); test("returns to listening after a durably stopped turn without speaking", async () => { @@ -304,12 +558,12 @@ describe("VoiceTurnController", () => { await harness.controller.start(); harness.emitBridge({ answer: "The supervisor approves it.", - callId: "call-1", + deliveryId: "call-1", type: "submission-started", }); harness.controller.pause(); - harness.controller.resume(); + await harness.controller.resume(); expect(harness.controller.getSnapshot()).toMatchObject({ input: "submitting", @@ -319,11 +573,11 @@ describe("VoiceTurnController", () => { harness.emitBridge({ answer: "The supervisor approves it.", - callId: "call-1", + deliveryId: "call-1", type: "submission-accepted", }); harness.emitBridge({ - callId: "call-1", + deliveryId: "call-1", segments: [question("ask-2", "Who acts next?")], type: "canonical-response-ready", }); @@ -344,7 +598,7 @@ describe("VoiceTurnController", () => { await harness.controller.start(); harness.emitBridge({ answer: "The supervisor approves it.", - callId: "call-1", + deliveryId: "call-1", type: "submission-started", }); @@ -355,11 +609,11 @@ describe("VoiceTurnController", () => { }); harness.emitBridge({ answer: "The supervisor approves it.", - callId: "call-1", + deliveryId: "call-1", type: "submission-accepted", }); harness.emitBridge({ - callId: "call-1", + deliveryId: "call-1", segments: [question("ask-2", "Who acts next?")], type: "canonical-response-ready", }); @@ -381,13 +635,13 @@ describe("VoiceTurnController", () => { await harness.controller.start(); harness.emitBridge({ answer: "The supervisor approves it.", - callId: "call-1", + deliveryId: "call-1", type: "submission-started", }); harness.controller.pause(); harness.emitBridge({ - callId: "call-1", + deliveryId: "call-1", segments: [question("ask-2", "Who acts next?")], type: "canonical-response-ready", }); @@ -399,7 +653,7 @@ describe("VoiceTurnController", () => { }); expect(harness.session.cancelOutput).toHaveBeenCalledTimes(2); - harness.controller.resume(); + await harness.controller.resume(); expect(harness.controller.getSnapshot()).toMatchObject({ input: "listening", microphoneEnabled: true, @@ -434,6 +688,33 @@ describe("VoiceTurnController", () => { expect(harness.submitText).not.toHaveBeenCalled(); }); + test.each(["empty", "failed"] as const)( + "reports a recoverable not-heard notice for a %s transcript", + async (reason) => { + const harness = createHarness(); + await harness.controller.start(); + harness.emitSession({ + connectionEpoch: 1, + itemId: "item-1", + type: "input-speech-started", + }); + harness.emitSession({ + key: { connectionEpoch: 1, contentIndex: 0, itemId: "item-1" }, + text: "Provisional words", + type: "partial", + }); + + harness.emitBridge({ reason, type: "transcript-rejected" }); + + expect(harness.controller.getSnapshot()).toMatchObject({ + input: "listening", + inputNotice: "not-heard", + partialText: "", + }); + expect(harness.submitText).not.toHaveBeenCalled(); + }, + ); + test("keeps completed display transcripts until submission and rejects late events", async () => { const harness = createHarness(); await harness.controller.start(); @@ -449,11 +730,11 @@ describe("VoiceTurnController", () => { }); harness.emitBridge({ answer: "First answer", - callId: "call-1", + deliveryId: "call-1", type: "submission-started", }); harness.emitBridge({ - callId: "call-1", + deliveryId: "call-1", segments: [question("ask-2", "Who acts next?")], type: "canonical-response-ready", }); @@ -539,7 +820,7 @@ describe("VoiceTurnController", () => { output: "interrupted", }); - harness.controller.resume(); + await harness.controller.resume(); expect(harness.controller.getSnapshot()).toMatchObject({ input: "listening", microphoneEnabled: true, @@ -617,7 +898,7 @@ describe("VoiceTurnController", () => { output: "idle", }); - harness.controller.resume(); + await harness.controller.resume(); expect(harness.bridge.start).toHaveBeenCalledWith(1); expect(harness.controller.getSnapshot()).toMatchObject({ input: "listening", @@ -766,7 +1047,7 @@ describe("VoiceTurnController", () => { await harness.controller.start(); harness.emitBridge({ answer: "The supervisor approves it.", - callId: "call-1", + deliveryId: "call-1", type: "submission-started", }); harness.emitBridge({ @@ -835,7 +1116,7 @@ describe("VoiceTurnController", () => { await bridgeFailure.controller.start(); bridgeFailure.emitBridge({ answer: "Pending answer", - callId: "call-1", + deliveryId: "call-1", type: "submission-started", }); bridgeFailure.emitBridge({ diff --git a/apps/petrinaut-website/src/server/voice/openai-realtime-call.test.ts b/apps/petrinaut-website/src/server/voice/openai-realtime-call.test.ts index 38188a178c3..c7f4f94c284 100644 --- a/apps/petrinaut-website/src/server/voice/openai-realtime-call.test.ts +++ b/apps/petrinaut-website/src/server/voice/openai-realtime-call.test.ts @@ -86,7 +86,7 @@ describe("OpenAI Realtime call handler", () => { expect(fetch).not.toHaveBeenCalled(); }); - test("forwards only the SDP and server-owned duplex Realtime policy", async () => { + test("forwards only the SDP and server-owned half-duplex Realtime policy", async () => { const reportDiagnostic = vi.fn(); const fetch = vi.fn( async () => @@ -131,16 +131,16 @@ describe("OpenAI Realtime call handler", () => { type: "realtime", model: "gpt-realtime-2", output_modalities: ["audio"], - tool_choice: "required", - tools: [{ name: "continue_interview", type: "function" }], + tool_choice: "none", + tools: [], audio: { input: { transcription: { model: "gpt-4o-transcribe", language: "en" }, turn_detection: { type: "semantic_vad", eagerness: "low", - create_response: true, - interrupt_response: true, + create_response: false, + interrupt_response: false, }, }, }, diff --git a/apps/petrinaut-website/src/server/voice/openai-voice-policy.test.ts b/apps/petrinaut-website/src/server/voice/openai-voice-policy.test.ts index 1cdd3f0fead..523b8c85f17 100644 --- a/apps/petrinaut-website/src/server/voice/openai-voice-policy.test.ts +++ b/apps/petrinaut-website/src/server/voice/openai-voice-policy.test.ts @@ -46,18 +46,18 @@ describe("OpenAI voice policy", () => { ).toEqual({ available: true, connectionTimeoutMs: 15_000 }); }); - test("owns the trusted GPT-Realtime-2 duplex session policy", () => { - expect(OPENAI_REALTIME_POLICY_VERSION).toBe("brunch-control-plane-v1"); + test("owns the trusted GPT-Realtime-2 half-duplex session policy", () => { + expect(OPENAI_REALTIME_POLICY_VERSION).toBe("brunch-control-plane-v3"); expect(createOpenAIRealtimeSession()).toEqual({ type: "realtime", model: "gpt-realtime-2", output_modalities: ["audio"], reasoning: { effort: "low" }, parallel_tool_calls: false, - tool_choice: "required", + tool_choice: "none", instructions: `# Role and objective -You are the realtime voice of an expert interviewer for process-model elicitation. The person speaking is the domain expert. Listen attentively, submit each complete spoken answer to Brunch, and deliver Brunch's next interview turn. +You are the realtime voice of an expert interviewer for process-model elicitation. The person speaking is the domain expert. Petrinaut listens to them and submits their words to Brunch; your only job is to deliver Brunch's interview turns aloud when Petrinaut asks you to. # Personality and delivery @@ -65,29 +65,16 @@ Sound warm, calm, curious, confident, concise, and professionally neutral. Speak # Authority -Brunch is the sole authority for interview state, questions, captures, completion, and business decisions. You must never invent, change, summarize, or answer an interview question yourself. +Brunch is the sole authority for interview state, questions, captures, completion, and business decisions. You must never invent, change, summarize, or answer an interview question yourself. You must never restate, guess, or fill in what the speaker said. # Turn handling -After semantic turn detection finds that the user has finished a complete spoken answer, call continue_interview exactly once with that answer. Do not speak, emit a preamble, or emit conversational text before calling the tool. +Never respond on your own after the speaker stops talking. Petrinaut transcribes their words and decides what happens next. Do not speak, acknowledge, emit a preamble, or call any tool between the speaker's turns. # Canonical output -After the tool result arrives, speak only its response_text strings, in array order and verbatim. Do not add, remove, paraphrase, acknowledge, or explain anything. Never call another tool while speaking a tool result.`, - tools: [ - { - type: "function", - name: "continue_interview", - description: - "Submit the user's complete spoken answer to the authoritative Brunch interview.", - parameters: { - type: "object", - additionalProperties: false, - properties: { answer: { type: "string" } }, - required: ["answer"], - }, - }, - ], +When Petrinaut supplies response_text, speak only those strings, in array order and verbatim. Do not add, remove, paraphrase, acknowledge, or explain anything.`, + tools: [], audio: { input: { noise_reduction: { type: "far_field" }, @@ -100,8 +87,8 @@ After the tool result arrives, speak only its response_text strings, in array or turn_detection: { type: "semantic_vad", eagerness: "low", - create_response: true, - interrupt_response: true, + create_response: false, + interrupt_response: false, }, }, output: { voice: "marin" }, @@ -109,12 +96,17 @@ After the tool result arrives, speak only its response_text strings, in array or }); }); - test("allows no provider-owned interview decisions or unrestricted tools", () => { - const serializedPolicy = JSON.stringify(createOpenAIRealtimeSession()); + test("lets Realtime neither answer for the user nor call tools between turns", () => { + const policy = createOpenAIRealtimeSession(); + const serializedPolicy = JSON.stringify(policy); expect(serializedPolicy).not.toContain("response.create"); expect(serializedPolicy).not.toContain("gpt-realtime-1.5"); + expect(serializedPolicy).not.toContain("continue_interview"); expect(serializedPolicy).not.toContain('"tool_choice":"auto"'); - expect(createOpenAIRealtimeSession().tools).toHaveLength(1); + expect(serializedPolicy).not.toContain('"tool_choice":"required"'); + expect(policy.tools).toHaveLength(0); + expect(policy.audio.input.turn_detection.create_response).toBe(false); + expect(policy.audio.input.transcription.model).toBe("gpt-4o-transcribe"); }); }); diff --git a/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/chat-transport.test.ts b/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/chat-transport.test.ts index 3b982b101be..bf882baf95e 100644 --- a/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/chat-transport.test.ts +++ b/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/chat-transport.test.ts @@ -427,3 +427,89 @@ test("reports a client-tool continuation against the resumed assistant id", asyn submissionId: admission.submissionId, }); }); + +test("replays a stable typed or Voice message with the same idempotency key", async () => { + const seenKeys = new Set(); + let admittedTurns = 0; + const send = vi.fn(async (options) => { + const key = options.idempotencyKey; + if (key === undefined || !seenKeys.has(key)) { + admittedTurns += 1; + if (key !== undefined) seenKeys.add(key); + return admission; + } + return { ...admission, deduplicated: true }; + }); + const wait = vi.fn(async () => undefined); + const transport = createFlueChatTransport({ + client: { send, wait } as Pick as FlueClient, + clientToolNames: new Set(), + }); + const typedTurn = sendOptions([ + { + id: "typed-message-1", + role: "user", + parts: [{ type: "text", text: "Admit this once." }], + }, + ]); + + const firstStream = await transport.sendMessages(typedTurn); + const replayedStream = await transport.sendMessages(typedTurn); + await Promise.all([readChunks(firstStream), readChunks(replayedStream)]); + + expect(send).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ idempotencyKey: "ai-sdk:typed-message-1" }), + ); + expect(send).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ idempotencyKey: "ai-sdk:typed-message-1" }), + ); + expect(admittedTurns).toBe(1); + + const voiceTurn = sendOptions([ + { + id: "voice-realtime:7:item%2F1:0", + role: "user", + parts: [{ type: "text", text: "Voice transcript." }], + }, + ]); + await readChunks(await transport.sendMessages(voiceTurn)); + expect(send).toHaveBeenLastCalledWith( + expect.objectContaining({ + idempotencyKey: "ai-sdk:voice-realtime:7:item%2F1:0", + }), + ); +}); + +test("reports an idempotency conflict as a definite existing admission", async () => { + const send = vi.fn(async () => { + throw new FlueApiError(409, { + error: { + details: "", + message: "The delivery key already names another payload.", + meta: { submissionId: "submission-existing" }, + type: "submission_conflict", + }, + }); + }); + const transport = createFlueChatTransport({ + client: { send } as Pick as FlueClient, + clientToolNames: new Set(), + }); + + await expect( + transport.sendMessages( + sendOptions([ + { + id: "user-conflict", + role: "user", + parts: [{ type: "text", text: "Changed payload." }], + }, + ]), + ), + ).rejects.toThrow( + "delivery key already belongs to admitted submission submission-existing", + ); + expect(send).toHaveBeenCalledOnce(); +}); diff --git a/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/transcript.test.ts b/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/transcript.test.ts index fe15a52470a..eafaf90e3b5 100644 --- a/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/transcript.test.ts +++ b/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/transcript.test.ts @@ -85,6 +85,65 @@ test("uses a recorded browser result even when it is null", () => { ]); }); +test("reconstructs durable voice provenance for each browser result", () => { + const snapshot: FlueConversationSnapshot = { + ...snapshotWithPendingClientTool, + messages: [ + { + ...snapshotWithPendingClientTool.messages[0]!, + parts: [ + ...snapshotWithPendingClientTool.messages[0]!.parts, + { + type: "dynamic-tool", + toolCallId: "tool-doc-2", + toolName: "readPetrinautDoc", + state: "output-available", + input: { doc: "ai-assistant" }, + output: { awaiting: "client" }, + }, + ], + }, + { + id: "signal-voice-results", + role: "system", + purpose: "dispatch", + display: "hidden", + signal: { tagName: CLIENT_TOOL_RESULT_SIGNAL }, + parts: [ + { + type: "text", + text: JSON.stringify([ + { + toolCallId: "tool-doc-1", + toolName: "readPetrinautDoc", + output: "First guide", + source: "voice", + }, + { + toolCallId: "tool-doc-2", + toolName: "readPetrinautDoc", + output: "Second guide", + source: "voice", + }, + ]), + state: "done", + }, + ], + }, + ], + }; + + expect(snapshotToUiMessages(snapshot, projectionOptions)).toEqual([ + expect.objectContaining({ + id: "assistant-1", + metadata: { + source: "voice", + voiceToolCallIds: ["tool-doc-1", "tool-doc-2"], + }, + }), + ]); +}); + test("keeps Flue data parts on the AI SDK message", () => { const snapshot: FlueConversationSnapshot = { v: 1, diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx index b4659e40a8c..d5fc9dd6d9c 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx @@ -1352,7 +1352,10 @@ describe("AiAssistantPanel composer submissions", () => { expect(latestVoiceContext?.status).toBe("streaming"); expect(latestVoiceContext?.canAcceptVoiceInput).toBe(true); expect(requests[1]?.at(-1)).toMatchObject({ - metadata: { source: "voice", toolCallId: "queued-question" }, + metadata: { + source: "voice", + voiceToolCallIds: ["queued-question"], + }, role: "assistant", }); expect( @@ -2425,7 +2428,10 @@ describe("AiAssistantPanel composer submissions", () => { ), ); expect(containingMessage).toMatchObject({ - metadata: { source: "voice", toolCallId: "question-voice" }, + metadata: { + source: "voice", + voiceToolCallIds: ["question-voice"], + }, }); expect( containingMessage?.parts.find( @@ -2449,12 +2455,139 @@ describe("AiAssistantPanel composer submissions", () => { expect(onMessages.mock.lastCall?.[0]).toEqual( expect.arrayContaining([ expect.objectContaining({ - metadata: { source: "voice", toolCallId: "question-voice" }, + metadata: { + source: "voice", + voiceToolCallIds: ["question-voice"], + }, }), ]), ); }); + test("retains every voice tool origin on one assistant message", async () => { + let latestMessages = [ + { + id: "assistant-voice-questions", + parts: [ + { + input: { question: "Who approves it?" }, + state: "input-available", + toolCallId: "voice-question-1", + toolName: "answerQuestion", + type: "dynamic-tool", + }, + { + input: { question: "Who acts next?" }, + state: "input-available", + toolCallId: "voice-question-2", + toolName: "answerQuestion", + type: "dynamic-tool", + }, + ], + role: "assistant", + }, + ] as unknown as PetrinautAiMessage[]; + const updateMessages = ( + updater: (messages: PetrinautAiMessage[]) => PetrinautAiMessage[], + ) => { + latestMessages = updater(latestMessages); + }; + const addToolOutput = vi.fn().mockResolvedValue(undefined); + + for (const toolCallId of ["voice-question-1", "voice-question-2"]) { + await addMappedToolOutput({ + addToolOutput, + currentMessages: latestMessages, + params: { + output: { answer: toolCallId }, + tool: "answerQuestion", + toolCallId, + }, + source: "voice", + updateMessages, + }); + } + + expect(latestMessages[0]?.metadata).toEqual({ + source: "voice", + voiceToolCallIds: ["voice-question-1", "voice-question-2"], + }); + }); + + test("preserves sibling voice provenance when another tool output rejects", async () => { + let latestMessages = [ + { + id: "assistant-voice-questions", + parts: [ + { + input: { question: "Who approves it?" }, + state: "input-available", + toolCallId: "voice-question-1", + toolName: "answerQuestion", + type: "dynamic-tool", + }, + { + input: { question: "Who acts next?" }, + state: "input-available", + toolCallId: "voice-question-2", + toolName: "answerQuestion", + type: "dynamic-tool", + }, + ], + role: "assistant", + }, + ] as unknown as PetrinautAiMessage[]; + const updateMessages = ( + updater: (messages: PetrinautAiMessage[]) => PetrinautAiMessage[], + ) => { + latestMessages = updater(latestMessages); + }; + let rejectFirstSubmission: ((reason?: unknown) => void) | undefined; + const addToolOutput = vi + .fn() + .mockImplementationOnce( + () => + new Promise((_resolve, reject) => { + rejectFirstSubmission = reject; + }), + ) + .mockResolvedValueOnce(undefined); + + const firstSubmission = addMappedToolOutput({ + addToolOutput, + currentMessages: latestMessages, + params: { + output: { answer: "The shift lead" }, + tool: "answerQuestion", + toolCallId: "voice-question-1", + }, + source: "voice", + updateMessages, + }); + const firstSubmissionRejection = expect(firstSubmission).rejects.toThrow( + "First voice tool output rejected.", + ); + + await addMappedToolOutput({ + addToolOutput, + currentMessages: latestMessages, + params: { + output: { answer: "The release manager" }, + tool: "answerQuestion", + toolCallId: "voice-question-2", + }, + source: "voice", + updateMessages, + }); + rejectFirstSubmission?.(new Error("First voice tool output rejected.")); + await firstSubmissionRejection; + + expect(latestMessages[0]?.metadata).toEqual({ + source: "voice", + voiceToolCallIds: ["voice-question-2"], + }); + }); + test("rolls back failed tool provenance before a typed retry", async () => { let latestMessages = [ { diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx index ee2d2d8c6f7..caf6dfff543 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx @@ -50,6 +50,14 @@ const noop = () => {}; // instead of letting jsdom log a not-implemented error per render. beforeAll(() => { vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue(null); + vi.stubGlobal( + "ResizeObserver", + class { + public disconnect() {} + public observe() {} + public unobserve() {} + }, + ); }); afterEach(() => { @@ -234,17 +242,23 @@ describe("AiAssistantContents", () => { ).not.toBeNull(); }); - test("keeps the session's controls in the dock", () => { + test("keeps handoff and canonical playback controls in the Voice dock", async () => { const store = createVoiceSessionStore(); const actions = { end: vi.fn(), pause: vi.fn(), + readFullResponse: vi.fn(), reconnect: vi.fn(), + repeatQuestion: vi.fn(), resume: vi.fn(), setMicrophoneMuted: vi.fn(), + takeTurn: vi.fn(), }; store.setActions(actions); store.setState({ + canReadFullResponse: true, + canRepeatQuestion: true, + canTakeTurn: true, errorMessage: null, microphoneLevel: 0.4, microphoneMuted: false, @@ -273,9 +287,45 @@ describe("AiAssistantContents", () => { fireEvent.click( within(dock).getByRole("button", { name: "End voice mode" }), ); + fireEvent.click(within(dock).getByRole("button", { name: "Your turn" })); expect(actions.setMicrophoneMuted).toHaveBeenCalledWith(true); expect(actions.end).toHaveBeenCalledOnce(); + expect(actions.takeTurn).toHaveBeenCalledOnce(); + + fireEvent.click( + within(dock).getByRole("button", { name: "Voice playback options" }), + ); + const repeatQuestion = await screen.findByRole("menuitem", { + name: "Repeat question", + }); + const repeatMenu = screen.getByRole("menu"); + fireEvent.keyDown(repeatMenu, { key: "ArrowDown" }); + await waitFor(() => + expect(repeatMenu.getAttribute("aria-activedescendant")).toBe( + repeatQuestion.id, + ), + ); + fireEvent.keyDown(repeatMenu, { key: "Enter" }); + await waitFor(() => expect(actions.repeatQuestion).toHaveBeenCalledOnce()); + + fireEvent.click( + within(dock).getByRole("button", { name: "Voice playback options" }), + ); + const readFullResponse = await screen.findByRole("menuitem", { + name: "Read full response", + }); + const fullResponseMenu = screen.getByRole("menu"); + fireEvent.keyDown(fullResponseMenu, { key: "End" }); + await waitFor(() => + expect(fullResponseMenu.getAttribute("aria-activedescendant")).toBe( + readFullResponse.id, + ), + ); + fireEvent.keyDown(fullResponseMenu, { key: "Enter" }); + await waitFor(() => + expect(actions.readFullResponse).toHaveBeenCalledOnce(), + ); act(() => { store.setState({ @@ -292,6 +342,20 @@ describe("AiAssistantContents", () => { ); expect(actions.setMicrophoneMuted).toHaveBeenLastCalledWith(false); + + act(() => { + store.setState({ + errorMessage: null, + microphoneLevel: 0, + microphoneMuted: false, + notice: "We didn't catch that. Please try again.", + phase: "listening", + }); + }); + expect( + within(dock).getAllByText("We didn't catch that. Please try again."), + ).not.toHaveLength(0); + expect(dock.getAttribute("data-voice-notice")).toBe("visible"); }); test("shows a voice recovery failure as a toast", async () => { @@ -554,7 +618,7 @@ describe("AiAssistantContents", () => { ).toBeNull(); }); - test("marks only the exact submitted interactive-tool answer named by voice metadata", () => { + test("marks every submitted interactive-tool answer named by voice metadata", () => { const hostTool = definePetrinautAiInteractiveTool({ toolName: "answerQuestion", inputSchema: { @@ -570,7 +634,10 @@ describe("AiAssistantContents", () => { const messages = [ { id: "assistant-questions", - metadata: { source: "voice", toolCallId: "question-voice" }, + metadata: { + source: "voice", + voiceToolCallIds: ["question-voice-1", "question-voice-2"], + }, role: "assistant", parts: [ { @@ -585,10 +652,18 @@ describe("AiAssistantContents", () => { type: "dynamic-tool", toolName: "answerQuestion", state: "output-available", - toolCallId: "question-voice", + toolCallId: "question-voice-1", input: { question: "Who approves it?" }, output: { answer: "The shift lead" }, }, + { + type: "dynamic-tool", + toolName: "answerQuestion", + state: "output-available", + toolCallId: "question-voice-2", + input: { question: "Who acts next?" }, + output: { answer: "The dispatcher" }, + }, ], }, ] as unknown as PetrinautAiMessage[]; @@ -606,13 +681,18 @@ describe("AiAssistantContents", () => { />, ); - expect( - within( - screen - .getByText("question-voice: The shift lead") - .closest("[data-tool-call-id]")!, - ).getByTestId("voice-input-provenance"), - ).not.toBeNull(); + for (const [toolCallId, answer] of [ + ["question-voice-1", "The shift lead"], + ["question-voice-2", "The dispatcher"], + ]) { + expect( + within( + screen + .getByText(`${toolCallId}: ${answer}`) + .closest("[data-tool-call-id]")!, + ).getByTestId("voice-input-provenance"), + ).not.toBeNull(); + } expect( within( screen @@ -620,7 +700,7 @@ describe("AiAssistantContents", () => { .closest("[data-tool-call-id]")!, ).queryByTestId("voice-input-provenance"), ).toBeNull(); - expect(screen.getAllByTestId("voice-input-provenance")).toHaveLength(1); + expect(screen.getAllByTestId("voice-input-provenance")).toHaveLength(2); expect(screen.queryByText("The shift lead", { exact: true })).toBeNull(); expect(container.querySelectorAll('[data-role="user"]')).toHaveLength(0); }); From 4883b21f01d9bd9b26f0242adb87b3d1ef6b0d93 Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Fri, 4 Sep 2026 00:56:43 +0200 Subject: [PATCH 03/52] Harden Voice turns on the unified Flue route Co-authored-by: Amp Amp-Thread-ID: https://ampcode.com/threads/T-01a06941-3ee7-7482-a82e-8a16c7141785 --- .changeset/flue-voice-safety.md | 5 + apps/petrinaut-website/README.md | 54 ++- .../local-storage-demo-app.tsx | 3 +- .../app/voice-interview/canonical-speech.ts | 30 +- .../openai-realtime-session.ts | 329 ++++++++++-------- .../voice-interview/realtime-brunch-bridge.ts | 315 ++++++----------- .../voice-interview-control.tsx | 7 +- .../voice-interview/voice-session-state.ts | 10 + .../voice-interview/voice-turn-controller.ts | 308 ++++++++++++++-- .../src/server/voice/openai-voice-policy.ts | 37 +- libs/@hashintel/brunch-agent/MISSION.md | 29 +- .../donor-behavior-matrix.md | 14 +- .../provenance-blocker.md | 39 +++ .../witness-blocker.md | 32 ++ .../packages/transport-aisdk/src/index.ts | 57 +++ .../transport-aisdk/src/transcript.ts | 55 ++- .../@hashintel/petrinaut/docs/ai-assistant.md | 38 +- .../src/react/voice-session/store.ts | 3 + .../src/react/voice-session/types.ts | 8 + .../react/voice-session/use-voice-session.ts | 40 +++ .../ui/types/ai-assistant-composer-control.ts | 6 + .../Editor/components/voice-session-labels.ts | 4 + .../Editor/panels/ai-assistant-panel.tsx | 66 +++- .../ai-assistant-contents/tool-list.tsx | 3 +- .../ai-assistant-contents/voice-dock.tsx | 64 +++- .../voice-dock/playback-menu.tsx | 47 +++ .../Editor/panels/ai-assistant-panel/types.ts | 4 +- 27 files changed, 1082 insertions(+), 525 deletions(-) create mode 100644 .changeset/flue-voice-safety.md create mode 100644 libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/provenance-blocker.md create mode 100644 libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/witness-blocker.md create mode 100644 libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/voice-dock/playback-menu.tsx diff --git a/.changeset/flue-voice-safety.md b/.changeset/flue-voice-safety.md new file mode 100644 index 00000000000..4aeeb7920a4 --- /dev/null +++ b/.changeset/flue-voice-safety.md @@ -0,0 +1,5 @@ +--- +"@hashintel/petrinaut": patch +--- + +Add half-duplex Voice handoff, canonical replay controls, recoverable transcript notices, and multi-origin Voice attribution for client-tool results. diff --git a/apps/petrinaut-website/README.md b/apps/petrinaut-website/README.md index 8e31217aa9e..47ed6dc659a 100644 --- a/apps/petrinaut-website/README.md +++ b/apps/petrinaut-website/README.md @@ -89,7 +89,18 @@ disclosure before requesting microphone access. The disclosure also provides a microphone check and is remembered in browser storage only after Voice mode starts. -When Brunch is selected, typed and finalized spoken turns both enter the same mounted Flue conversation route. **Stop** requests a durable Brunch abort before the panel cancels its local response stream. Closing or speaking over Voice playback only stops local media; it does not alter canonical conversation history. Reopening the same net restores its observed Flue conversation without resubmitting a turn or replaying settled audio. +When Brunch is selected, typed turns and completed Voice transcripts both enter +the same mounted Flue conversation route. Each logical turn carries a stable +delivery key so a replayed request converges on the existing admission instead +of creating another turn. If admission cannot be confirmed, the UI reports the +ambiguity and does not retry automatically. **Stop** requests a durable Brunch +abort before the panel cancels its local response stream. Local playback +cancellation remains separate and does not alter canonical history. Reopening +the same net restores its observed Flue conversation without resubmitting a +turn or replaying settled audio. Voice-origin client-tool results retain their +markers through Flue history. Direct spoken user turns remain canonical text +after reopening, but Flue 2.0.3 does not yet expose the caller delivery metadata +needed to restore their Voice chip. An active session stays at the end of the transcript. Its compact divider shows a waveform and **Connecting**, **Listening**, **Speaking**, **Paused**, or a @@ -103,24 +114,39 @@ The text composer remains available. Sending typed text ends Voice mode first, then submits the draft exactly once through the same conversation; a failed handoff restores the draft. Closing the assistant pauses capture and speech before hiding it. Reopening preserves the mounted session in **Paused** state. -**Pause** and **End voice mode** live under **Voice mode actions**, while -**Resume** or **Reconnect** appears as the primary action when applicable. +The dock exposes **Your turn** while canonical audio owns the turn. That action +clears pending input and output, waits for the provider's matching +acknowledgements and response terminal event, and only then opens the +microphone for fresh capture. Its playback menu offers **Repeat question** and +**Read full response** once the matching response and audio output have both +finished. Replay enqueues the exact retained canonical segments and is disabled +during capture, submission, cancellation, pause, and errors. The browser sends its SDP offer to this app; the server initializes a trusted `gpt-realtime-2` audio-input/audio-output session through OpenAI's unified -Realtime call endpoint. The provider key, model, instructions, tools, language, -and vocabulary policy stay server-side. The session uses semantic VAD with low -eagerness so natural thinking pauses are less likely to end an answer early. - -Realtime is the disposable media plane: it carries continuous microphone and remote audio, detects complete turns, and handles barge-in. Brunch remains the control plane and sole authority for questions, captures, state, completion, and durable history. The browser bridge accepts only the configured `continue_interview` function, validates and serializes its arguments, rejects duplicate or stale calls, and submits the answer through Petrinaut's shared composer path with pending-question correlation. +Realtime call endpoint. The provider key, model, instructions, language, and +vocabulary policy stay server-side. Realtime exposes no tools, uses +`tool_choice: "none"`, and configures semantic VAD to detect an input boundary +without creating a model response. + +Realtime is the disposable media plane: it carries microphone and remote audio, +detects complete turns, and transcribes input. Brunch remains the control plane +and sole authority for questions, captures, state, completion, and durable +history. The bridge accepts only +`conversation.item.input_audio_transcription.completed` as an answer, ignores +model function arguments, and submits the normalized transcript through +Petrinaut's shared composer path. Connection epoch, item id, and content index +form its stable identity. Duplicate, empty, failed, unavailable, and over-limit +transcripts never submit; recoverable failures leave a not-heard or too-long +notice in the dock. Provisional transcription remains display-only. The bridge waits for the correlated Brunch turn before returning canonical -speech segments to Realtime. It then requests audio with tools disabled and -instructs Realtime to speak only those segments. Generated audio is not a -verbatim record: canonical Brunch text remains visible and authoritative. The -microphone stays active while the interviewer speaks and while Brunch is -working. Speaking over assistant audio interrupts playback automatically; -WebRTC truncates provider-side unheard audio without changing Brunch history. +speech segments to Realtime. It instructs Realtime to speak only those +segments. Generated audio is not a verbatim recording: canonical Brunch text +remains visible and authoritative. Voice is half-duplex: the physical +microphone is closed while the interviewer speaks, while Brunch is working, and +through cancellation. Audio captured before a **Your turn** handoff is +discarded and cannot become a later answer. The local Brunch preview reaches the mounted route through its same-origin, protocol-preserving proxy; this does not establish remote authentication or public ingress. Denying microphone permission leaves the text composer available and submits nothing to Brunch. When Voice mode cannot continue, the inline recovery state distinguishes microphone, connection, and other Voice failures, explains the next action, and offers **Reconnect** where appropriate. Sanitized error codes and diagnostic references remain collapsed under **Technical details**. diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx index 39e4b466fa3..3072fd0a5fc 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx @@ -45,7 +45,6 @@ import { type OpenAIVoiceConfig, VoiceInterviewControl, } from "../voice-interview/voice-interview-control"; -import { brunchAskInteractiveTool } from "./brunch-ask-interactive-tool"; import { getOrCreateBrunchConversationId } from "./brunch-conversation-id"; import { BrunchPanelConversationTracker, @@ -549,7 +548,7 @@ export const LocalStorageDemoApp = ({ () => ({ ...(conversationId === null ? {} : { conversationId }), canClearMessages: flueClientPromise === null, - interactiveTools: brunchInteractiveTools, + interactiveTools: [], transport: petrinautAiChatTransport, ...(flueClientPromise === null ? {} diff --git a/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.ts b/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.ts index b99a0afdf30..d50c298a7fb 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.ts @@ -1,8 +1,3 @@ -import { - ASK_TOOL_NAME, - parseBrunchAskInput, -} from "@hashintel/brunch-agent/client-tools"; - import { hashCanonicalSpeechText } from "../../../canonical-speech-fingerprint"; import type { AgentSendResult } from "@flue/sdk"; @@ -15,7 +10,7 @@ export interface CanonicalSpeechSegment { readonly id: string; readonly messageId: string; readonly partId: string; - readonly source: "assistant-text" | "brunch-ask"; + readonly source: "assistant-text"; /** * Every Flue submission that wrote to this segment's message: the one that * started it plus any client-tool continuation projected back onto it. @@ -70,29 +65,6 @@ export const selectCanonicalSpeechSegments = ( part.text, ), ); - continue; - } - - if ( - part.type !== "dynamic-tool" || - part.toolName !== ASK_TOOL_NAME || - part.state !== "input-available" - ) { - continue; - } - - try { - const input = parseBrunchAskInput(part.input); - segments.push( - createSegment( - message.id, - part.toolCallId, - "brunch-ask", - input.question, - ), - ); - } catch { - // Malformed tool inputs remain visible as tool errors; they are not spoken. } } } diff --git a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts index 99b66aac151..6d86a66e793 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts @@ -19,13 +19,6 @@ export interface OpenAIRealtimeTranscriptKey { readonly itemId: string; } -interface RealtimeToolEventIdentity { - readonly callId: string; - readonly connectionEpoch: number; - readonly itemId: string; - readonly responseId: string; -} - export type OpenAIRealtimeSessionEvent = | { readonly key: OpenAIRealtimeTranscriptKey; @@ -74,15 +67,6 @@ export type OpenAIRealtimeSessionEvent = readonly status: "cancelled" | "completed" | "failed" | "incomplete"; readonly type: "response-terminal"; } - | (RealtimeToolEventIdentity & { - readonly delta: string; - readonly type: "tool-arguments-delta"; - }) - | (RealtimeToolEventIdentity & { - readonly arguments: string; - readonly name: string; - readonly type: "tool-arguments-done"; - }) | { readonly code: VoiceErrorCode; readonly message: string; @@ -192,17 +176,21 @@ const waitForAbort = ( }; export class OpenAIRealtimeSession { + readonly #acceptedInputItemIds = new Set(); readonly #dependencies: OpenAIRealtimeSessionDependencies; readonly #activeResponseIds = new Set(); readonly #listeners = new Set(); readonly #authorizedResponseIds = new Set(); readonly #cancelledCanonicalResponseIds = new Set(); readonly #cancelledSpeechRequestIds = new Set(); + readonly #cancelOutputAwaitingRequestIds = new Set(); + readonly #cancelOutputAwaitingResponseIds = new Set(); readonly #canonicalResponseIds = new Set(); readonly #canonicalSpeechQueue: CanonicalSpeechRequest[] = []; readonly #completedResponseCancelEventIds = new Set(); readonly #pendingClientEvents = new Map(); readonly #pendingSpeechRequests = new Map(); + readonly #playbackOverlappingInputItemIds = new Set(); readonly #remoteStreams = new Set(); readonly #speechRequestIds = new Map(); readonly #speechTimings = new Map(); @@ -214,6 +202,10 @@ export class OpenAIRealtimeSession { #connected = false; #connectedAt: number | null = null; #clientEventSequence = 0; + #cancelOutputAwaitingInputBufferClear = false; + #cancelOutputAwaitingOutputBufferResponseId: string | null = null; + #cancelOutputPromise: Promise | null = null; + #cancelOutputResolve: (() => void) | null = null; #connectionRequestId: string | null = null; #dataChannel: RTCDataChannel | null = null; #epoch = 0; @@ -223,9 +215,11 @@ export class OpenAIRealtimeSession { #meterHasSample = false; #meterLevel = 0; #meterSamples: Uint8Array | null = null; + #microphoneRequested = false; #microphoneTrack: MediaStreamTrack | null = null; #peerConnection: RTCPeerConnection | null = null; #remoteAudio: RemoteAudio | null = null; + #requireInputSpeechStart = false; #responseCreateEventId: string | null = null; #responseTerminalSequence = 0; #speakingResponseId: string | null = null; @@ -393,92 +387,75 @@ export class OpenAIRealtimeSession { } public setMicrophoneEnabled(enabled: boolean): void { - if (!this.#microphoneTrack) { - return; - } - const isEnabled = enabled && this.#connected; - this.#microphoneTrack.enabled = isEnabled; - if (isEnabled) { - this.#startMeter(); - } else { - this.#stopMeter(); - } + this.#microphoneRequested = enabled && this.#connected; + this.#syncMicrophoneTrack(); } public speakCanonical(segments: CanonicalSpeechSegment[]): void { this.#requestCanonicalSpeech(segments, true); } - public completeFunctionCall( - callId: string, - segments: CanonicalSpeechSegment[], - ): void { - if (!callId) { - throw new VoiceError("speech", "invalid-response", ""); + public cancelOutput(): Promise { + if (!this.#connected || this.#dataChannel?.readyState !== "open") { + return Promise.resolve(); } - const responseText = this.#canonicalResponseText(segments); - this.#send({ - type: "conversation.item.create", - item: { - type: "function_call_output", - call_id: callId, - output: JSON.stringify({ response_text: responseText }), - }, - }); - this.#requestCanonicalSpeech(segments, false); - } - - /** - * Close a Realtime function call whose Brunch turn settled without a reply. - * The call output records the settlement so the model does not wait on it, - * and no speech is requested: a stopped turn has no canonical text to speak. - */ - public completeFunctionCallWithoutResponse( - callId: string, - outcome: "aborted" | "failed", - ): void { - if (!callId) { - throw new VoiceError("speech", "invalid-response", ""); + if (this.#cancelOutputPromise) { + return this.#cancelOutputPromise; } - this.#send({ - type: "conversation.item.create", - item: { - type: "function_call_output", - call_id: callId, - output: JSON.stringify({ response_text: [], outcome }), - }, - }); - } - public cancelOutput(): void { - if (!this.#connected || this.#dataChannel?.readyState !== "open") { - return; + const cancelOutputPromise = new Promise((resolve) => { + this.#cancelOutputResolve = resolve; + }); + this.#cancelOutputPromise = cancelOutputPromise; + this.#cancelOutputAwaitingInputBufferClear = true; + this.#cancelOutputAwaitingOutputBufferResponseId = this.#speakingResponseId; + this.#requireInputSpeechStart = true; + for (const itemId of this.#acceptedInputItemIds) { + this.#playbackOverlappingInputItemIds.add(itemId); } + this.#acceptedInputItemIds.clear(); + this.#syncMicrophoneTrack(); - for (const request of this.#canonicalSpeechQueue.splice(0)) { - this.#cancelPendingSpeechRequest(request.speechRequestId); - } + try { + this.#send({ type: "input_audio_buffer.clear" }); - if (this.#responseCreateEventId !== null) { - const pendingEvent = this.#pendingClientEvents.get( - this.#responseCreateEventId, - ); - if (pendingEvent?.kind === "response-create") { - this.#cancelledSpeechRequestIds.add( - pendingEvent.request.speechRequestId, + for (const request of this.#canonicalSpeechQueue.splice(0)) { + this.#cancelPendingSpeechRequest(request.speechRequestId); + } + + if (this.#responseCreateEventId !== null) { + const pendingEvent = this.#pendingClientEvents.get( + this.#responseCreateEventId, ); + if (pendingEvent?.kind === "response-create") { + this.#cancelledSpeechRequestIds.add( + pendingEvent.request.speechRequestId, + ); + this.#cancelOutputAwaitingRequestIds.add( + pendingEvent.request.speechRequestId, + ); + } } - } - for (const responseId of this.#canonicalResponseIds) { - if ( - this.#activeResponseIds.has(responseId) && - !this.#cancelledCanonicalResponseIds.has(responseId) - ) { - this.#cancelledCanonicalResponseIds.add(responseId); - this.#cancelOutputResponse(responseId); + for (const responseId of this.#canonicalResponseIds) { + if ( + this.#activeResponseIds.has(responseId) && + !this.#cancelledCanonicalResponseIds.has(responseId) + ) { + this.#cancelOutputAwaitingResponseIds.add(responseId); + this.#cancelledCanonicalResponseIds.add(responseId); + this.#cancelResponse(responseId); + } } + + this.#send({ type: "output_audio_buffer.clear" }); + } catch { + this.#finishOutputCancellation(true); + this.#handleConnectionFailure("network", "speech"); } + + this.#finishOutputCancellation(); + return cancelOutputPromise; } #cancelOutputResponse(responseId: string): void { @@ -643,6 +620,12 @@ export class OpenAIRealtimeSession { this.#handleResponseDone(parsed, connectionEpoch); return; } + if (parsed.type === "input_audio_buffer.cleared") { + this.#acceptedInputItemIds.clear(); + this.#cancelOutputAwaitingInputBufferClear = false; + this.#finishOutputCancellation(); + return; + } if (parsed.type === "input_audio_buffer.committed") { const itemId = nonEmptyString(parsed.item_id); if (itemId) this.#startTranscription(itemId); @@ -651,23 +634,22 @@ export class OpenAIRealtimeSession { if (parsed.type === "input_audio_buffer.speech_started") { const itemId = nonEmptyString(parsed.item_id); if (!itemId || nonNegativeInteger(parsed.audio_start_ms) === null) return; + if (this.#speakingResponseId || !this.#microphoneTrack?.enabled) { + this.#playbackOverlappingInputItemIds.add(itemId); + return; + } + this.#acceptedInputItemIds.add(itemId); this.#emit({ connectionEpoch, itemId, type: "input-speech-started", }); - if (this.#speakingResponseId) { - this.#emit({ - connectionEpoch, - responseId: this.#speakingResponseId, - type: "output-interrupted", - }); - } return; } if (parsed.type === "input_audio_buffer.speech_stopped") { const itemId = nonEmptyString(parsed.item_id); if (!itemId || nonNegativeInteger(parsed.audio_end_ms) === null) return; + if (this.#playbackOverlappingInputItemIds.has(itemId)) return; this.#emit({ connectionEpoch, itemId, @@ -677,15 +659,12 @@ export class OpenAIRealtimeSession { } if ( parsed.type === "output_audio_buffer.started" || - parsed.type === "output_audio_buffer.stopped" + parsed.type === "output_audio_buffer.stopped" || + parsed.type === "output_audio_buffer.cleared" ) { this.#handleOutputBufferEvent(parsed, connectionEpoch); return; } - if (parsed.type === "response.function_call_arguments.delta") { - this.#handleToolEvent(parsed, connectionEpoch); - return; - } if ( parsed.type === "conversation.item.input_audio_transcription.delta" || parsed.type === "conversation.item.input_audio_transcription.completed" || @@ -707,6 +686,9 @@ export class OpenAIRealtimeSession { if (metadata?.petrinaut_kind !== "canonical-speech" || !speechRequestId) { return; } + if (this.#cancelOutputAwaitingRequestIds.delete(speechRequestId)) { + this.#cancelOutputAwaitingResponseIds.add(responseId); + } this.#completeResponseCreateEvent(speechRequestId); this.#canonicalResponseIds.add(responseId); if (this.#cancelledSpeechRequestIds.delete(speechRequestId)) { @@ -762,6 +744,10 @@ export class OpenAIRealtimeSession { ) { this.#pendingClientEvents.delete(sourceEventId); this.#completedResponseCancelEventIds.delete(sourceEventId); + if (pendingEvent?.kind === "response-cancel") { + this.#cancelOutputAwaitingResponseIds.delete(pendingEvent.responseId); + this.#finishOutputCancellation(); + } return; } @@ -780,7 +766,11 @@ export class OpenAIRealtimeSession { pendingEvent.request.speechRequestId, ) ) { + this.#cancelOutputAwaitingRequestIds.delete( + pendingEvent.request.speechRequestId, + ); this.#cancelPendingSpeechRequest(pendingEvent.request.speechRequestId); + this.#finishOutputCancellation(); return; } this.#canonicalSpeechQueue.unshift(pendingEvent.request); @@ -815,6 +805,8 @@ export class OpenAIRealtimeSession { } this.#responseTerminalSequence += 1; this.#activeResponseIds.delete(responseId); + this.#cancelOutputAwaitingResponseIds.delete(responseId); + this.#finishOutputCancellation(); this.#clearResponseCancelEvents(responseId); this.#waitingForResponseTerminal = false; @@ -836,44 +828,10 @@ export class OpenAIRealtimeSession { this.#handleConnectionFailure("invalid-response", "connection"); return; } - const functionCalls = output - .map(asRecord) - .filter( - (item): item is Record => - item?.type === "function_call", - ); - if ( - functionCalls.length > 1 || - (functionCalls.length > 0 && this.#canonicalResponseIds.has(responseId)) - ) { + if (output.some((item) => asRecord(item)?.type === "function_call")) { this.#handleConnectionFailure("invalid-response", "connection"); return; } - for (const item of functionCalls) { - const argumentsJson = nonEmptyString(item.arguments); - const callId = nonEmptyString(item.call_id); - const itemId = nonEmptyString(item.id); - const name = nonEmptyString(item.name); - if ( - !argumentsJson || - !callId || - !itemId || - !name || - (item.status !== undefined && item.status !== "completed") - ) { - this.#handleConnectionFailure("invalid-response", "connection"); - return; - } - this.#emit({ - arguments: argumentsJson, - callId, - connectionEpoch, - itemId, - name, - responseId, - type: "tool-arguments-done", - }); - } this.#emit({ connectionEpoch, responseId, @@ -941,6 +899,9 @@ export class OpenAIRealtimeSession { if (!responseId) return; if (event.type === "output_audio_buffer.started") { if (this.#cancelledCanonicalResponseIds.has(responseId)) { + if (this.#cancelOutputPromise) { + this.#cancelOutputAwaitingOutputBufferResponseId = responseId; + } this.#send({ type: "output_audio_buffer.clear" }); return; } @@ -950,6 +911,7 @@ export class OpenAIRealtimeSession { return; } this.#speakingResponseId = responseId; + this.#syncMicrophoneTrack(); const speechRequestId = this.#speechRequestIds.get(responseId); if (!speechRequestId) { this.#handleConnectionFailure("invalid-response", "connection"); @@ -964,35 +926,24 @@ export class OpenAIRealtimeSession { return; } const wasSpeaking = this.#speakingResponseId === responseId; + const wasCleared = event.type === "output_audio_buffer.cleared"; this.#finishSpeech( responseId, - this.#cancelledCanonicalResponseIds.has(responseId) + wasCleared || this.#cancelledCanonicalResponseIds.has(responseId) ? "request-aborted" : undefined, ); if (wasSpeaking) { - this.#emit({ connectionEpoch, responseId, type: "output-stopped" }); + this.#emit({ + connectionEpoch, + responseId, + type: wasCleared ? "output-interrupted" : "output-stopped", + }); + } + if (this.#cancelOutputAwaitingOutputBufferResponseId === responseId) { + this.#cancelOutputAwaitingOutputBufferResponseId = null; + this.#finishOutputCancellation(); } - } - - #handleToolEvent( - event: Record, - connectionEpoch: number, - ): void { - const callId = nonEmptyString(event.call_id); - const itemId = nonEmptyString(event.item_id); - const responseId = nonEmptyString(event.response_id); - const outputIndex = nonNegativeInteger(event.output_index); - if (!callId || !itemId || !responseId || outputIndex === null) return; - if (typeof event.delta !== "string") return; - this.#emit({ - callId, - connectionEpoch, - delta: event.delta, - itemId, - responseId, - type: "tool-arguments-delta", - }); } #handleTranscriptEvent( @@ -1003,9 +954,30 @@ export class OpenAIRealtimeSession { const contentIndex = nonNegativeInteger(event.content_index); if (!itemId || contentIndex === null) return; const key = { connectionEpoch, contentIndex, itemId }; + const overlapsPlayback = + this.#playbackOverlappingInputItemIds.has(itemId) || + (this.#requireInputSpeechStart && + !this.#acceptedInputItemIds.has(itemId)); + if (overlapsPlayback) { + if ( + event.type === + "conversation.item.input_audio_transcription.completed" || + event.type === "conversation.item.input_audio_transcription.failed" + ) { + this.#finishTranscription( + itemId, + event.type === "conversation.item.input_audio_transcription.failed" + ? "invalid-response" + : undefined, + ); + this.#acceptedInputItemIds.delete(itemId); + } + return; + } this.#startTranscription(itemId); if (event.type === "conversation.item.input_audio_transcription.failed") { this.#finishTranscription(itemId, "invalid-response"); + this.#acceptedInputItemIds.delete(itemId); this.#emit({ key, type: "transcription-failed" }); return; } @@ -1018,6 +990,7 @@ export class OpenAIRealtimeSession { event.type === "conversation.item.input_audio_transcription.completed" ) { this.#finishTranscription(itemId); + this.#acceptedInputItemIds.delete(itemId); } this.#emit({ key, @@ -1058,7 +1031,31 @@ export class OpenAIRealtimeSession { this.#authorizedResponseIds.delete(responseId); if (this.#speakingResponseId === responseId) { this.#speakingResponseId = null; + this.#syncMicrophoneTrack(); + } + } + + #finishOutputCancellation(force = false): void { + if ( + !this.#cancelOutputPromise || + (!force && + (this.#cancelOutputAwaitingInputBufferClear || + this.#cancelOutputAwaitingOutputBufferResponseId !== null || + this.#cancelOutputAwaitingRequestIds.size > 0 || + this.#cancelOutputAwaitingResponseIds.size > 0)) + ) { + return; } + + const resolve = this.#cancelOutputResolve; + this.#cancelOutputPromise = null; + this.#cancelOutputResolve = null; + this.#cancelOutputAwaitingInputBufferClear = false; + this.#cancelOutputAwaitingOutputBufferResponseId = null; + this.#cancelOutputAwaitingRequestIds.clear(); + this.#cancelOutputAwaitingResponseIds.clear(); + this.#syncMicrophoneTrack(); + resolve?.(); } #emit(event: OpenAIRealtimeSessionEvent): void { @@ -1239,6 +1236,23 @@ export class OpenAIRealtimeSession { this.#meterFrame = this.#dependencies.requestAnimationFrame(sample); } + #syncMicrophoneTrack(): void { + if (!this.#microphoneTrack) { + return; + } + const enabled = + this.#microphoneRequested && + this.#connected && + this.#cancelOutputPromise === null && + this.#speakingResponseId === null; + this.#microphoneTrack.enabled = enabled; + if (enabled) { + this.#startMeter(); + } else { + this.#stopMeter(); + } + } + #stopMeter(): void { if (this.#meterFrame === null) return; this.#dependencies.cancelAnimationFrame(this.#meterFrame); @@ -1328,13 +1342,17 @@ export class OpenAIRealtimeSession { ); } this.#transcriptionTimings.clear(); + this.#acceptedInputItemIds.clear(); this.#activeResponseIds.clear(); this.#cancelledCanonicalResponseIds.clear(); this.#cancelledSpeechRequestIds.clear(); + this.#cancelOutputAwaitingRequestIds.clear(); + this.#cancelOutputAwaitingResponseIds.clear(); this.#canonicalSpeechQueue.length = 0; this.#completedResponseCancelEventIds.clear(); this.#pendingClientEvents.clear(); this.#pendingSpeechRequests.clear(); + this.#playbackOverlappingInputItemIds.clear(); this.#speechTimings.clear(); this.#speechRequestIds.clear(); this.#authorizedResponseIds.clear(); @@ -1342,6 +1360,8 @@ export class OpenAIRealtimeSession { this.#responseCreateEventId = null; this.#responseTerminalSequence = 0; this.#speakingResponseId = null; + this.#microphoneRequested = false; + this.#requireInputSpeechStart = false; this.#waitingForResponseTerminal = false; this.#activeEpoch = null; this.#connected = false; @@ -1388,6 +1408,7 @@ export class OpenAIRealtimeSession { this.#mediaStream = null; } this.#microphoneTrack = null; + this.#finishOutputCancellation(true); } #waitForDataChannelOpen( diff --git a/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.ts b/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.ts index a7b0014cc59..3eded3bd273 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.ts @@ -1,5 +1,8 @@ import type { CanonicalSpeechSegment } from "./canonical-speech"; -import type { OpenAIRealtimeSessionEvent } from "./openai-realtime-session"; +import type { + OpenAIRealtimeSessionEvent, + OpenAIRealtimeTranscriptKey, +} from "./openai-realtime-session"; import type { AgentSendResult, FlueConversationSettlement } from "@flue/sdk"; import type { FlueChatTransportOptions } from "@hashintel/brunch-agent-transport-aisdk"; import type { @@ -21,14 +24,6 @@ interface ChatUpdate { } interface RealtimeBridgeSession { - completeFunctionCall( - callId: string, - segments: CanonicalSpeechSegment[], - ): void; - completeFunctionCallWithoutResponse( - callId: string, - outcome: Exclude, - ): void; speakCanonical(segments: CanonicalSpeechSegment[]): void; subscribe(listener: (event: OpenAIRealtimeSessionEvent) => void): () => void; } @@ -67,64 +62,68 @@ interface RealtimeBrunchBridgeDependencies { interface ActiveSubmission { readonly abortController: AbortController; readonly baselineSegmentIds: ReadonlySet; - readonly callId: string; - readonly epoch: number; - readonly pendingQuestionId: string | null; - readonly pendingQuestionMessageId: string | null; + readonly deliveryId: string; correlated: boolean; firstTextEmitted: boolean; sawBusyChatStatus: boolean; + speechCancelled: boolean; submissionId: AgentSendResult["submissionId"] | null; } -interface ArgumentStream { - readonly chunks: string[]; - readonly itemId: string; - readonly responseId: string; -} - export type RealtimeBridgeErrorCode = | "interview-correlation" | "interview-response" | "interview-submission"; +export type RealtimeTranscriptRejectionReason = + | "duplicate" + | "empty" + | "failed" + | "over-limit" + | "unavailable"; + export type RealtimeBrunchBridgeEvent = | { readonly answer: string; - readonly callId: string; + readonly deliveryId: string; readonly type: "submission-started"; } | { readonly answer: string; - readonly callId: string; + readonly deliveryId: string; readonly type: "submission-accepted"; } | { - readonly callId: string; + readonly deliveryId: string; readonly submissionId: AgentSendResult["submissionId"]; readonly type: "submission-admitted"; } | { - readonly callId: string; + readonly deliveryId: string; readonly segments: CanonicalSpeechSegment[]; + readonly speechCancelled?: true; readonly type: "canonical-response-ready"; } | { - readonly callId: string; + readonly deliveryId: string; readonly type: "canonical-text-ready"; } | { - readonly callId: string; + readonly deliveryId: string; readonly type: "submission-settled"; } | { - readonly callId: string; + readonly deliveryId: string; readonly outcome: Exclude< VoiceSubmissionSettlement["outcome"], "completed" >; readonly type: "submission-stopped"; } + | { + readonly reason: RealtimeTranscriptRejectionReason; + readonly type: "transcript-rejected"; + } | { readonly code: RealtimeBridgeErrorCode; readonly message: string; @@ -137,45 +136,27 @@ const INVALID_BRIDGE_EVENT = "The voice response could not be matched to the interview. Reconnect voice or use text instead."; const ANSWER_LIMIT = 32_000; -export const createRealtimeSubmissionId = ( - connectionEpoch: number, - callId: string, -): string => `voice-realtime:${connectionEpoch}:${encodeURIComponent(callId)}`; - -const latestPendingQuestion = ( - segments: CanonicalSpeechSegment[], -): CanonicalSpeechSegment | undefined => - segments.findLast(({ source }) => source === "brunch-ask"); - -const parseContinueInterviewArguments = ( - argumentsJson: string, -): string | null => { - try { - const value: unknown = JSON.parse(argumentsJson); - if (typeof value !== "object" || value === null || Array.isArray(value)) { - return null; - } - const record = value as Record; - if (Object.keys(record).length !== 1 || typeof record.answer !== "string") { - return null; - } - const answer = record.answer.trim(); - return answer && Array.from(answer).length <= ANSWER_LIMIT ? answer : null; - } catch { - return null; - } -}; +export const createRealtimeSubmissionId = ({ + connectionEpoch, + contentIndex, + itemId, +}: OpenAIRealtimeTranscriptKey): string => + `voice-realtime:${connectionEpoch}:${encodeURIComponent(itemId)}:${contentIndex}`; + +const transcriptKeyId = (key: OpenAIRealtimeTranscriptKey): string => + createRealtimeSubmissionId(key); + +const normalizeTranscript = (transcript: string): string => + transcript.trim().replace(/\s+/gu, " "); export class RealtimeBrunchBridge { - readonly #argumentDeltas = new Map(); readonly #listeners = new Set(); - readonly #processedCalls = new Set(); + readonly #processedTranscripts = new Set(); readonly #session: RealtimeBridgeSession; readonly #submitInterviewAnswer: ( input: SubmitInterviewAnswerInput, ) => Promise; readonly #seenSegmentIds = new Set(); - readonly #terminalResponseIds = new Set(); #activeEpoch: number | null = null; #activeSubmission: ActiveSubmission | null = null; #chat: ChatUpdate = { @@ -199,27 +180,22 @@ export class RealtimeBrunchBridge { return () => this.#listeners.delete(listener); } + public cancelPendingSpeech(): void { + if (this.#activeSubmission) { + this.#activeSubmission.speechCancelled = true; + } + } + public start(connectionEpoch: number): void { ++this.#generation; this.#activeSubmission?.abortController.abort(); this.#activeEpoch = connectionEpoch; this.#activeSubmission = null; - this.#argumentDeltas.clear(); - this.#processedCalls.clear(); + this.#processedTranscripts.clear(); this.#seenSegmentIds.clear(); - this.#terminalResponseIds.clear(); for (const segment of this.#chat.canonicalSegments) { this.#seenSegmentIds.add(segment.id); } - - const question = latestPendingQuestion(this.#chat.canonicalSegments); - if (question) { - this.#session.speakCanonical( - this.#chat.canonicalSegments.filter( - ({ messageId }) => messageId === question.messageId, - ), - ); - } } public stop(): void { @@ -227,8 +203,7 @@ export class RealtimeBrunchBridge { this.#activeSubmission?.abortController.abort(); this.#activeEpoch = null; this.#activeSubmission = null; - this.#argumentDeltas.clear(); - this.#terminalResponseIds.clear(); + this.#processedTranscripts.clear(); } public updateChat(update: ChatUpdate): void { @@ -276,6 +251,10 @@ export class RealtimeBrunchBridge { } } + #rejectTranscript(reason: RealtimeTranscriptRejectionReason): void { + this.#emit({ reason, type: "transcript-rejected" }); + } + #fail( message: string, code: RealtimeBridgeErrorCode = "interview-correlation", @@ -283,160 +262,82 @@ export class RealtimeBrunchBridge { ++this.#generation; this.#activeSubmission?.abortController.abort(); this.#activeSubmission = null; - this.#argumentDeltas.clear(); this.#emit({ code, message, type: "error" }); } #handleSessionEvent(event: OpenAIRealtimeSessionEvent): void { - if ( - !("connectionEpoch" in event) || - event.connectionEpoch !== this.#activeEpoch - ) { - return; - } - if (event.type === "response-terminal") { - this.#handleResponseTerminal(event); + if (event.type !== "completed" && event.type !== "transcription-failed") { return; } - if ( - event.type !== "tool-arguments-delta" && - event.type !== "tool-arguments-done" - ) { + if (event.key.connectionEpoch !== this.#activeEpoch) { return; } - const responseKey = `${event.connectionEpoch}:${event.responseId}`; - if (this.#terminalResponseIds.has(responseKey)) { - return; - } - const callKey = `${event.connectionEpoch}:${event.callId}`; - if (this.#processedCalls.has(callKey)) { - return; - } - if (event.type === "tool-arguments-delta") { - const stream = this.#argumentDeltas.get(callKey); - if (!stream && this.#argumentDeltas.size > 0) { - this.#processedCalls.add(callKey); - this.#fail(INVALID_BRIDGE_EVENT); - return; - } - if ( - stream && - (stream.itemId !== event.itemId || - stream.responseId !== event.responseId) - ) { - this.#processedCalls.add(callKey); - this.#fail(INVALID_BRIDGE_EVENT); - return; - } - if (stream) { - stream.chunks.push(event.delta); - } else { - this.#argumentDeltas.set(callKey, { - chunks: [event.delta], - itemId: event.itemId, - responseId: event.responseId, - }); - } + const keyId = transcriptKeyId(event.key); + if (this.#processedTranscripts.has(keyId)) { + this.#rejectTranscript("duplicate"); return; } + this.#processedTranscripts.add(keyId); - this.#processedCalls.add(callKey); - const stream = this.#argumentDeltas.get(callKey); - if (!stream && this.#argumentDeltas.size > 0) { - this.#fail(INVALID_BRIDGE_EVENT); + if (event.type === "transcription-failed") { + this.#rejectTranscript("failed"); return; } - this.#argumentDeltas.delete(callKey); if ( this.#activeSubmission || - event.name !== "continue_interview" || - (stream !== undefined && - (stream.itemId !== event.itemId || - stream.responseId !== event.responseId || - stream.chunks.join("") !== event.arguments)) + !this.#chat.canAcceptInterviewAnswer || + this.#chat.status !== "ready" ) { - this.#fail(INVALID_BRIDGE_EVENT); + this.#rejectTranscript("unavailable"); return; } - const answer = parseContinueInterviewArguments(event.arguments); - const question = latestPendingQuestion(this.#chat.canonicalSegments); - if ( - !answer || - !this.#chat.canAcceptInterviewAnswer || - (!question && this.#chat.status !== "ready") - ) { - this.#fail(INVALID_BRIDGE_EVENT); + const answer = normalizeTranscript(event.text); + if (answer.length === 0) { + this.#rejectTranscript("empty"); + return; + } + if (Array.from(answer).length > ANSWER_LIMIT) { + this.#rejectTranscript("over-limit"); return; } + const deliveryId = createRealtimeSubmissionId(event.key); const generation = this.#generation; this.#activeSubmission = { abortController: new AbortController(), baselineSegmentIds: new Set( this.#chat.canonicalSegments.map(({ id }) => id), ), - callId: event.callId, correlated: false, - epoch: event.connectionEpoch, + deliveryId, firstTextEmitted: false, - pendingQuestionId: question?.partId ?? null, - pendingQuestionMessageId: question?.messageId ?? null, sawBusyChatStatus: false, + speechCancelled: false, submissionId: null, }; - this.#emit({ answer, callId: event.callId, type: "submission-started" }); - void this.#submit(event, answer, generation); - } - - #handleResponseTerminal( - event: Extract, - ): void { - const responseKey = `${event.connectionEpoch}:${event.responseId}`; - const matchingStreams = [...this.#argumentDeltas].filter( - ([, stream]) => stream.responseId === event.responseId, - ); - if (event.status === "completed" && matchingStreams.length > 0) { - this.#fail(INVALID_BRIDGE_EVENT); - return; - } - - for (const [callKey] of matchingStreams) { - this.#argumentDeltas.delete(callKey); - this.#processedCalls.add(callKey); - } - this.#terminalResponseIds.add(responseKey); + this.#emit({ answer, deliveryId, type: "submission-started" }); + void this.#submit(answer, deliveryId, generation); } async #submit( - event: Extract, answer: string, + deliveryId: string, generation: number, ): Promise { try { const activeAtSubmission = this.#activeSubmission; if (!activeAtSubmission) return; - const voiceMessageId = createRealtimeSubmissionId( - event.connectionEpoch, - event.callId, - ); const result = await this.#submitInterviewAnswer({ - admissionTarget: - activeAtSubmission.pendingQuestionMessageId === null - ? { kind: "user", messageId: voiceMessageId } - : { - kind: "client-tool-result", - messageId: activeAtSubmission.pendingQuestionMessageId, - }, - id: voiceMessageId, + admissionTarget: { kind: "user", messageId: deliveryId }, + id: deliveryId, onAdmission: (submissionId) => { const active = this.#activeSubmission; if ( generation !== this.#generation || !active || - active.callId !== event.callId || - active.epoch !== event.connectionEpoch + active.deliveryId !== deliveryId ) { return; } @@ -448,7 +349,7 @@ export class RealtimeBrunchBridge { } active.submissionId = submissionId; this.#emit({ - callId: event.callId, + deliveryId, submissionId, type: "submission-admitted", }); @@ -460,22 +361,15 @@ export class RealtimeBrunchBridge { if ( generation !== this.#generation || !active || - active.callId !== event.callId || - active.epoch !== event.connectionEpoch + active.deliveryId !== deliveryId ) { return; } - const resultMatchesSubmission = - active.pendingQuestionId === null - ? result.kind === "message" - : result.kind === "interactive-tool" && - result.toolCallId === active.pendingQuestionId; - if (!resultMatchesSubmission) { + if (result.kind !== "message" || result.messageId !== deliveryId) { this.#fail(INVALID_BRIDGE_EVENT); return; } - const resultSubmissionId = - result.kind === "message" ? (result.submissionId ?? null) : null; + const resultSubmissionId = result.submissionId ?? null; if ( active.submissionId !== null && resultSubmissionId !== null && @@ -486,11 +380,7 @@ export class RealtimeBrunchBridge { } active.submissionId ??= resultSubmissionId; active.correlated = true; - this.#emit({ - answer, - callId: event.callId, - type: "submission-accepted", - }); + this.#emit({ answer, deliveryId, type: "submission-accepted" }); this.#completeCorrelatedSubmission(); } catch { if (generation === this.#generation) { @@ -521,7 +411,10 @@ export class RealtimeBrunchBridge { // Completed canonical text can land while the turn is still streaming; // record that instant separately from settlement. active.firstTextEmitted = true; - this.#emit({ callId: active.callId, type: "canonical-text-ready" }); + this.#emit({ + deliveryId: active.deliveryId, + type: "canonical-text-ready", + }); } if (this.#chat.status !== "ready") { return; @@ -531,20 +424,26 @@ export class RealtimeBrunchBridge { return; } - this.#emit({ callId: active.callId, type: "submission-settled" }); - try { - this.#session.completeFunctionCall(active.callId, responseSegments); - } catch { - this.#fail(INVALID_BRIDGE_EVENT); - return; + this.#emit({ + deliveryId: active.deliveryId, + type: "submission-settled", + }); + if (!active.speechCancelled) { + try { + this.#session.speakCanonical(responseSegments); + } catch { + this.#fail(INVALID_BRIDGE_EVENT); + return; + } } for (const segment of responseSegments) { this.#seenSegmentIds.add(segment.id); } this.#activeSubmission = null; this.#emit({ - callId: active.callId, + deliveryId: active.deliveryId, segments: responseSegments, + ...(active.speechCancelled ? { speechCancelled: true as const } : {}), type: "canonical-response-ready", }); } @@ -563,19 +462,13 @@ export class RealtimeBrunchBridge { if (settlement === undefined || settlement.outcome === "completed") { return; } - this.#emit({ callId: active.callId, type: "submission-settled" }); - try { - this.#session.completeFunctionCallWithoutResponse( - active.callId, - settlement.outcome, - ); - } catch { - this.#fail(INVALID_BRIDGE_EVENT); - return; - } + this.#emit({ + deliveryId: active.deliveryId, + type: "submission-settled", + }); this.#activeSubmission = null; this.#emit({ - callId: active.callId, + deliveryId: active.deliveryId, outcome: settlement.outcome, type: "submission-stopped", }); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx index 5206ac7d8c1..2576a789974 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx @@ -425,12 +425,17 @@ const AvailableVoiceInterviewControl = ({ registerVoiceModeControls({ end: () => store.controller.end(), pause: () => store.controller.pause(), + readFullResponse: () => store.controller.readFullResponse(), reconnect: () => { void store.controller.reconnect(); }, - resume: () => store.controller.resume(), + repeatQuestion: () => store.controller.repeatQuestion(), + resume: () => { + void store.controller.resume(); + }, setMicrophoneMuted: (muted) => store.controller.setMicrophoneMuted(muted), + takeTurn: () => store.controller.takeTurn(), }), [registerVoiceModeControls, store], ); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.ts index e1a06464eb0..5d8bfce2219 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-session-state.ts @@ -65,6 +65,7 @@ const phaseOf = ( return "speaking"; } if ( + snapshot.output === "cancelling" || snapshot.output === "waiting-for-tool" || snapshot.input === "submitting" ) { @@ -93,6 +94,9 @@ export const toVoiceSessionState = ({ } return { + canReadFullResponse: snapshot.canReadFullResponse, + canRepeatQuestion: snapshot.canRepeatQuestion, + canTakeTurn: snapshot.canTakeTurn, errorMessage: snapshot.connection === "error" ? errorMessageOf(snapshot) : null, microphoneMuted: @@ -100,6 +104,12 @@ export const toVoiceSessionState = ({ snapshot.input !== "paused" && !snapshot.microphoneEnabled, microphoneLevel: snapshot.microphoneLevel, + notice: + snapshot.inputNotice === "not-heard" + ? "We didn't catch that. Please try again." + : snapshot.inputNotice === "too-long" + ? "That answer is too long. Please try a shorter response." + : null, phase: phaseOf(snapshot), }; }; diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts index 0f4e42cbac6..94210a7c3d7 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts @@ -19,13 +19,18 @@ export type VoiceConnectionState = | "error"; export type VoiceInputState = "listening" | "paused" | "submitting"; export type VoiceOutputState = + | "cancelling" | "idle" | "waiting-for-tool" | "speaking" | "interrupted"; export type VoiceAnswerDelivery = "none" | "pending" | "delivered" | "failed"; +export type VoiceInputNotice = "none" | "not-heard" | "too-long"; export interface VoiceTurnSnapshot { + readonly canReadFullResponse: boolean; + readonly canRepeatQuestion: boolean; + readonly canTakeTurn: boolean; readonly canReviseLastAnswer: boolean; readonly connection: VoiceConnectionState; readonly currentQuestion: string; @@ -33,6 +38,7 @@ export interface VoiceTurnSnapshot { readonly errorMessage: string; readonly errorRequestId: string; readonly input: VoiceInputState; + readonly inputNotice: VoiceInputNotice; readonly lastAnswerDelivery: VoiceAnswerDelivery; readonly lastCommittedText: string; readonly microphoneEnabled: boolean; @@ -57,14 +63,16 @@ export interface VoiceLatencyEvent { } interface RealtimeSession { - cancelOutput(): void; + cancelOutput(): Promise; connect(): Promise; disconnect(): Promise; setMicrophoneEnabled(enabled: boolean): void; + speakCanonical(segments: CanonicalSpeechSegment[]): void; subscribe(listener: (event: OpenAIRealtimeSessionEvent) => void): () => void; } interface RealtimeBridge { + cancelPendingSpeech(): void; start(connectionEpoch: number): void; stop(): void; subscribe(listener: (event: RealtimeBrunchBridgeEvent) => void): () => void; @@ -97,6 +105,9 @@ interface ChatUpdate { type SnapshotListener = (snapshot: VoiceTurnSnapshot) => void; const initialSnapshot: VoiceTurnSnapshot = { + canReadFullResponse: false, + canRepeatQuestion: false, + canTakeTurn: false, canReviseLastAnswer: false, connection: "idle", currentQuestion: "", @@ -104,6 +115,7 @@ const initialSnapshot: VoiceTurnSnapshot = { errorMessage: "", errorRequestId: "", input: "paused", + inputNotice: "none", lastAnswerDelivery: "none", lastCommittedText: "", microphoneEnabled: false, @@ -114,8 +126,7 @@ const initialSnapshot: VoiceTurnSnapshot = { const latestQuestion = ( segments: CanonicalSpeechSegment[], -): CanonicalSpeechSegment | undefined => - segments.findLast(({ source }) => source === "brunch-ask"); +): CanonicalSpeechSegment | undefined => segments.at(-1); export class VoiceTurnController { readonly #bridge: RealtimeBridge; @@ -125,17 +136,24 @@ export class VoiceTurnController { readonly #session: RealtimeSession; readonly #submitText: (input: SubmitTextInput) => Promise; #activeEpoch: number | null = null; + #activeSpeechOutputEnded = false; + #activeSpeechResponseId: string | null = null; + #activeSpeechResponseTerminal = false; #answerFinalizedAt: number | null = null; #answeredQuestionId: string | null = null; #bridgeStarted = false; #currentQuestionId: string | null = null; #generation = 0; #inputStateOnResume: Exclude | null = null; + #inputTurnPending = false; #latencyCorrelationId: string | null = null; + #lastResponseSegments: CanonicalSpeechSegment[] = []; + #outputCancellationPromise: Promise | null = null; #pauseRequested = false; readonly #recordedLatencyEvents = new Set(); #snapshot = initialSnapshot; #submittingQuestionId: string | null = null; + #takingTurnPromise: Promise | null = null; #teardownPromise: Promise | null = null; #transcriptItemId: string | null = null; #transcriptKey: string | null = null; @@ -195,8 +213,13 @@ export class VoiceTurnController { } this.#inputStateOnResume = null; + this.#inputTurnPending = false; + this.#outputCancellationPromise = null; this.#pauseRequested = false; this.#bridgeStarted = false; + this.#activeSpeechOutputEnded = false; + this.#activeSpeechResponseId = null; + this.#activeSpeechResponseTerminal = false; this.#update({ connection: "connecting", errorCode: null, @@ -234,14 +257,21 @@ export class VoiceTurnController { public async end(): Promise { ++this.#generation; this.#activeEpoch = null; + this.#activeSpeechOutputEnded = false; + this.#activeSpeechResponseId = null; + this.#activeSpeechResponseTerminal = false; this.#answerFinalizedAt = null; this.#answeredQuestionId = null; this.#bridgeStarted = false; this.#currentQuestionId = null; this.#inputStateOnResume = null; + this.#inputTurnPending = false; this.#latencyCorrelationId = null; + this.#lastResponseSegments = []; + this.#outputCancellationPromise = null; this.#recordedLatencyEvents.clear(); this.#submittingQuestionId = null; + this.#takingTurnPromise = null; this.#pauseRequested = false; this.#transcriptItemId = null; this.#transcriptKey = null; @@ -292,7 +322,7 @@ export class VoiceTurnController { this.#inputStateOnResume = this.#snapshot.input; this.#pauseRequested = true; const output = this.#snapshot.output === "idle" ? "idle" : "interrupted"; - this.#session.cancelOutput(); + void this.#cancelOutput(); this.#session.setMicrophoneEnabled(false); this.#update({ input: "paused", @@ -315,17 +345,51 @@ export class VoiceTurnController { ) { return; } - this.#session.setMicrophoneEnabled(!muted); + if ( + this.#takingTurnPromise === null && + this.#outputCancellationPromise === null && + this.#activeSpeechResponseId === null && + (this.#snapshot.output === "idle" || + this.#snapshot.output === "interrupted") + ) { + this.#session.setMicrophoneEnabled(!muted); + } this.#update({ microphoneEnabled: !muted, microphoneLevel: 0 }); } - public resume(): void { + public async resume(): Promise { if ( this.#snapshot.connection !== "connected" || this.#snapshot.input !== "paused" ) { return; } + const generation = this.#generation; + while (this.#outputCancellationPromise || this.#takingTurnPromise) { + try { + await (this.#outputCancellationPromise ?? this.#takingTurnPromise); + } catch (error) { + if (generation !== this.#generation) return; + const voiceError = + error instanceof VoiceError + ? error + : new VoiceError("speech", "network", ""); + this.#setError( + voiceError.message, + voiceError.code, + voiceError.requestId, + ); + return; + } + const snapshotAfterCancellation = this.getSnapshot(); + if ( + generation !== this.#generation || + snapshotAfterCancellation.connection !== "connected" || + snapshotAfterCancellation.input !== "paused" + ) { + return; + } + } const input = this.#inputStateOnResume ?? "listening"; this.#inputStateOnResume = null; this.#pauseRequested = false; @@ -382,6 +446,73 @@ export class VoiceTurnController { } } + public readFullResponse(): void { + if (!this.#snapshot.canReadFullResponse) return; + this.#update({ output: "waiting-for-tool" }); + this.#session.speakCanonical([...this.#lastResponseSegments]); + } + + public repeatQuestion(): void { + const question = this.#lastResponseSegments.at(-1); + if (!question || !this.#snapshot.canRepeatQuestion) return; + this.#update({ output: "waiting-for-tool" }); + this.#session.speakCanonical([question]); + } + + /** + * Hands the turn to the user only after provider cancellation has cleared + * input and output and the active response has reached a terminal state. + */ + public takeTurn(): Promise { + if (this.#takingTurnPromise) return this.#takingTurnPromise; + if (!this.#snapshot.canTakeTurn) return Promise.resolve(); + + const generation = this.#generation; + this.#bridge.cancelPendingSpeech(); + this.#session.setMicrophoneEnabled(false); + this.#inputTurnPending = false; + this.#transcriptItemId = null; + this.#transcriptKey = null; + this.#update({ output: "cancelling", partialText: "" }); + + const takingTurnPromise = this.#session + .cancelOutput() + .then(() => { + if ( + generation !== this.#generation || + this.#snapshot.connection !== "connected" || + this.#snapshot.input === "paused" + ) { + return; + } + this.#activeSpeechOutputEnded = false; + this.#activeSpeechResponseId = null; + this.#activeSpeechResponseTerminal = false; + this.#session.setMicrophoneEnabled(this.#snapshot.microphoneEnabled); + this.#update({ output: "interrupted" }); + }) + .catch((error: unknown) => { + if (generation !== this.#generation) return; + const voiceError = + error instanceof VoiceError + ? error + : new VoiceError("speech", "network", ""); + this.#setError( + voiceError.message, + voiceError.code, + voiceError.requestId, + ); + }) + .finally(() => { + if (this.#takingTurnPromise === takingTurnPromise) { + this.#takingTurnPromise = null; + } + }); + this.#takingTurnPromise = takingTurnPromise; + this.#update({}); + return takingTurnPromise; + } + public updateChat(update: ChatUpdate): void { const question = latestQuestion(update.canonicalSegments); if (question && question.id !== this.#currentQuestionId) { @@ -391,7 +522,7 @@ export class VoiceTurnController { } this.#bridge.updateChat(update); if (this.#snapshot.input === "paused") { - this.#session.cancelOutput(); + void this.#cancelOutput(); } } @@ -406,15 +537,18 @@ export class VoiceTurnController { if (paused) { this.#inputStateOnResume = "submitting"; } + this.#inputTurnPending = false; this.#answerFinalizedAt = this.#now(); - this.#latencyCorrelationId = event.callId; + this.#latencyCorrelationId = event.deliveryId; this.#recordedLatencyEvents.clear(); this.#submittingQuestionId = this.#currentQuestionId; this.#transcriptItemId = null; this.#transcriptKey = null; this.#ttsSpeechRequestId = null; + this.#session.setMicrophoneEnabled(false); this.#update({ input: paused ? "paused" : "submitting", + inputNotice: "none", lastAnswerDelivery: "pending", lastCommittedText: event.answer, output: "waiting-for-tool", @@ -422,6 +556,18 @@ export class VoiceTurnController { }); return; } + if (event.type === "transcript-rejected") { + if (event.reason === "duplicate" || event.reason === "unavailable") { + return; + } + this.#transcriptItemId = null; + this.#transcriptKey = null; + this.#update({ + inputNotice: event.reason === "over-limit" ? "too-long" : "not-heard", + partialText: "", + }); + return; + } if (event.type === "submission-accepted") { this.#answeredQuestionId = this.#submittingQuestionId; this.#submittingQuestionId = null; @@ -429,15 +575,29 @@ export class VoiceTurnController { return; } if (event.type === "submission-admitted") { - this.#recordLatency("submission-admitted", event.callId); + this.#recordLatency("submission-admitted", event.deliveryId); return; } if (event.type === "canonical-text-ready") { - this.#recordLatency("first-canonical-text", event.callId); + this.#recordLatency("first-canonical-text", event.deliveryId); return; } if (event.type === "submission-settled") { - this.#recordLatency("submission-settled", event.callId); + this.#recordLatency("submission-settled", event.deliveryId); + return; + } + this.#lastResponseSegments = [...event.segments]; + const question = event.segments.at(-1); + if (event.speechCancelled) { + const paused = this.#snapshot.input === "paused"; + if (paused) { + this.#inputStateOnResume = "listening"; + } + this.#update({ + input: paused ? "paused" : "listening", + output: "interrupted", + }); + if (question) this.#recordLatency("answer-ready", question.id); return; } if (event.type === "submission-stopped") { @@ -456,15 +616,12 @@ export class VoiceTurnController { const paused = this.#snapshot.input === "paused"; if (paused) { this.#inputStateOnResume = "listening"; - this.#session.cancelOutput(); + void this.#cancelOutput(); } this.#update({ input: paused ? "paused" : "listening", output: paused ? "interrupted" : "waiting-for-tool", }); - const question = event.segments.findLast( - ({ source }) => source === "brunch-ask", - ); if (question) this.#recordLatency("answer-ready", question.id); } @@ -486,6 +643,7 @@ export class VoiceTurnController { return; } if (event.type === "canonical-speech-requested") { + this.#session.setMicrophoneEnabled(false); if ( this.#latencyCorrelationId !== null && this.#ttsSpeechRequestId === null @@ -496,8 +654,11 @@ export class VoiceTurnController { return; } if (event.type === "output-started") { + this.#activeSpeechOutputEnded = false; + this.#activeSpeechResponseId = event.responseId; + this.#activeSpeechResponseTerminal = false; if (this.#snapshot.input === "paused") { - this.#session.cancelOutput(); + void this.#cancelOutput(); this.#update({ output: "interrupted" }); return; } @@ -514,32 +675,56 @@ export class VoiceTurnController { return; } if (event.type === "output-stopped") { - this.#update({ output: "idle" }); + if (event.responseId !== this.#activeSpeechResponseId) return; + this.#activeSpeechOutputEnded = true; + if (this.#activeSpeechResponseTerminal) { + this.#clearSettledSpeech(); + } + this.#update({ + output: this.#takingTurnPromise ? "cancelling" : "idle", + }); if (this.#currentQuestionId) { this.#recordLatency("question-spoken", this.#currentQuestionId); } return; } if (event.type === "output-interrupted") { - this.#update({ output: "interrupted" }); + if (event.responseId !== this.#activeSpeechResponseId) return; + this.#activeSpeechOutputEnded = true; + if (this.#activeSpeechResponseTerminal) { + this.#clearSettledSpeech(); + } + this.#update({ + output: this.#takingTurnPromise ? "cancelling" : "interrupted", + }); return; } if (event.type === "input-speech-started") { + if ( + this.#takingTurnPromise || + this.#snapshot.output === "speaking" || + this.#snapshot.output === "cancelling" + ) { + return; + } + this.#inputTurnPending = true; this.#transcriptItemId = event.itemId; this.#transcriptKey = null; - if (this.#snapshot.output === "speaking") { - this.#update({ output: "interrupted", partialText: "" }); - } else { - this.#update({ partialText: "" }); + this.#update({ inputNotice: "none", partialText: "" }); + return; + } + if (event.type === "response-terminal") { + if (event.responseId === this.#activeSpeechResponseId) { + if (this.#activeSpeechOutputEnded) { + this.#clearSettledSpeech(); + } else { + this.#activeSpeechResponseTerminal = true; + } + this.#update({}); } return; } - if ( - event.type === "input-speech-stopped" || - event.type === "response-terminal" || - event.type === "tool-arguments-delta" || - event.type === "tool-arguments-done" - ) { + if (event.type === "input-speech-stopped") { return; } @@ -547,6 +732,7 @@ export class VoiceTurnController { if (event.key.connectionEpoch !== this.#activeEpoch) return; if (event.key.itemId !== this.#transcriptItemId) return; if (event.type === "transcription-failed") { + this.#inputTurnPending = false; this.#transcriptItemId = null; this.#transcriptKey = null; this.#update({ partialText: "" }); @@ -560,6 +746,7 @@ export class VoiceTurnController { }); return; } + this.#inputTurnPending = false; this.#transcriptItemId = null; this.#transcriptKey = null; this.#update({ @@ -574,9 +761,15 @@ export class VoiceTurnController { ): void { ++this.#generation; this.#activeEpoch = null; + this.#activeSpeechOutputEnded = false; + this.#activeSpeechResponseId = null; + this.#activeSpeechResponseTerminal = false; this.#inputStateOnResume = null; + this.#inputTurnPending = false; this.#latencyCorrelationId = null; + this.#outputCancellationPromise = null; this.#recordedLatencyEvents.clear(); + this.#takingTurnPromise = null; this.#bridgeStarted = false; this.#transcriptItemId = null; this.#transcriptKey = null; @@ -601,6 +794,37 @@ export class VoiceTurnController { }); } + #cancelOutput(): Promise { + const cancellationPromise = this.#session.cancelOutput(); + this.#outputCancellationPromise = cancellationPromise; + void cancellationPromise.then( + () => { + if (this.#outputCancellationPromise === cancellationPromise) { + this.#outputCancellationPromise = null; + } + }, + () => { + if (this.#outputCancellationPromise === cancellationPromise) { + this.#outputCancellationPromise = null; + } + }, + ); + return cancellationPromise; + } + + #clearSettledSpeech(): void { + this.#activeSpeechOutputEnded = false; + this.#activeSpeechResponseId = null; + this.#activeSpeechResponseTerminal = false; + if ( + this.#snapshot.connection === "connected" && + this.#snapshot.input === "listening" && + this.#takingTurnPromise === null + ) { + this.#session.setMicrophoneEnabled(this.#snapshot.microphoneEnabled); + } + } + #recordLatency(name: VoiceLatencyEvent["name"], correlationId: string): void { if (this.#answerFinalizedAt === null) return; const eventKey = `${correlationId}:${name}`; @@ -623,14 +847,42 @@ export class VoiceTurnController { ); } + #canReplay(snapshot: VoiceTurnSnapshot): boolean { + return ( + snapshot.connection === "connected" && + snapshot.input === "listening" && + !this.#inputTurnPending && + this.#activeSpeechResponseId === null && + (snapshot.output === "idle" || snapshot.output === "interrupted") + ); + } + + #canTakeTurn(snapshot: VoiceTurnSnapshot): boolean { + return ( + snapshot.connection === "connected" && + snapshot.input !== "paused" && + (snapshot.output === "waiting-for-tool" || + snapshot.output === "speaking") && + this.#currentQuestionId !== null && + this.#currentQuestionId !== this.#answeredQuestionId && + this.#currentQuestionId !== this.#submittingQuestionId && + Boolean(snapshot.currentQuestion) && + this.#takingTurnPromise === null + ); + } + #isPauseRequested(): boolean { return this.#pauseRequested; } #update(update: Partial): void { const snapshot = { ...this.#snapshot, ...update }; + const canReplay = this.#canReplay(snapshot); this.#snapshot = { ...snapshot, + canReadFullResponse: canReplay && this.#lastResponseSegments.length > 0, + canRepeatQuestion: canReplay && this.#lastResponseSegments.length > 0, + canTakeTurn: this.#canTakeTurn(snapshot), canReviseLastAnswer: this.#canReviseLastAnswer(snapshot), }; for (const listener of this.#listeners) listener(this.#snapshot); diff --git a/apps/petrinaut-website/src/server/voice/openai-voice-policy.ts b/apps/petrinaut-website/src/server/voice/openai-voice-policy.ts index fc401fc0c7b..653b622dc3a 100644 --- a/apps/petrinaut-website/src/server/voice/openai-voice-policy.ts +++ b/apps/petrinaut-website/src/server/voice/openai-voice-policy.ts @@ -1,5 +1,5 @@ export const OPENAI_REALTIME_CONNECTION_TIMEOUT_MS = 15_000; -export const OPENAI_REALTIME_POLICY_VERSION = "brunch-control-plane-v1"; +export const OPENAI_REALTIME_POLICY_VERSION = "brunch-control-plane-v3"; interface VoiceEnvironment { readonly NODE_ENV?: string; @@ -24,7 +24,7 @@ export const getOpenAIVoiceAvailability = (environment: VoiceEnvironment) => ({ const REALTIME_INSTRUCTIONS = `# Role and objective -You are the realtime voice of an expert interviewer for process-model elicitation. The person speaking is the domain expert. Listen attentively, submit each complete spoken answer to Brunch, and deliver Brunch's next interview turn. +You are the realtime voice of an expert interviewer for process-model elicitation. The person speaking is the domain expert. Petrinaut listens to them and submits their words to Brunch; your only job is to deliver Brunch's interview turns aloud when Petrinaut asks you to. # Personality and delivery @@ -32,38 +32,31 @@ Sound warm, calm, curious, confident, concise, and professionally neutral. Speak # Authority -Brunch is the sole authority for interview state, questions, captures, completion, and business decisions. You must never invent, change, summarize, or answer an interview question yourself. +Brunch is the sole authority for interview state, questions, captures, completion, and business decisions. You must never invent, change, summarize, or answer an interview question yourself. You must never restate, guess, or fill in what the speaker said. # Turn handling -After semantic turn detection finds that the user has finished a complete spoken answer, call continue_interview exactly once with that answer. Do not speak, emit a preamble, or emit conversational text before calling the tool. +Never respond on your own after the speaker stops talking. Petrinaut transcribes their words and decides what happens next. Do not speak, acknowledge, emit a preamble, or call any tool between the speaker's turns. # Canonical output -After the tool result arrives, speak only its response_text strings, in array order and verbatim. Do not add, remove, paraphrase, acknowledge, or explain anything. Never call another tool while speaking a tool result.`; +When Petrinaut supplies response_text, speak only those strings, in array order and verbatim. Do not add, remove, paraphrase, acknowledge, or explain anything.`; +/** + * The completed input transcription is the only source of the user's answer. + * Semantic VAD therefore commits audio without creating a response or + * interrupting playback, and the Realtime model has no tools with which to + * manufacture an answer. + */ export const createOpenAIRealtimeSession = () => ({ type: "realtime" as const, model: "gpt-realtime-2", output_modalities: ["audio"] as const, reasoning: { effort: "low" as const }, parallel_tool_calls: false, - tool_choice: "required" as const, + tool_choice: "none" as const, instructions: REALTIME_INSTRUCTIONS, - tools: [ - { - type: "function" as const, - name: "continue_interview", - description: - "Submit the user's complete spoken answer to the authoritative Brunch interview.", - parameters: { - type: "object" as const, - additionalProperties: false, - properties: { answer: { type: "string" as const } }, - required: ["answer"] as const, - }, - }, - ], + tools: [] as const, audio: { input: { noise_reduction: { type: "far_field" as const }, @@ -76,8 +69,8 @@ export const createOpenAIRealtimeSession = () => ({ turn_detection: { type: "semantic_vad" as const, eagerness: "low" as const, - create_response: true, - interrupt_response: true, + create_response: false, + interrupt_response: false, }, }, output: { voice: "marin" as const }, diff --git a/libs/@hashintel/brunch-agent/MISSION.md b/libs/@hashintel/brunch-agent/MISSION.md index 8c38f0298bf..0daf2b514e9 100644 --- a/libs/@hashintel/brunch-agent/MISSION.md +++ b/libs/@hashintel/brunch-agent/MISSION.md @@ -13,6 +13,16 @@ The owner selected **half-duplex turn ownership** on 2026-09-03. While canonical Sections independent of canonical hydration may proceed. The hard-reload witness is blocked until the parent prevents its once-per-conversation hydration from overwriting a locally submitted turn. Prepare that witness but do not patch the parent defect here. +Sections 1–5 and dormant-ask cleanup are implemented with focused regression +coverage. Section 6 is complete only for the supported client-tool-result path: +Flue signals persist each Voice-origin tool-call id beside its output, and +canonical projection reconstructs multiple surviving origins. Direct spoken +user attribution is blocked because Flue 2.0.3 projects the generated +`submissionId` but neither caller metadata nor the caller idempotency key. The +discarded browser-correlation implementation would have violated the explicit +second-durable-store stop condition. The focused repository check passes; the +real witness remains blocked by the parent hydration defect. + The pinned donor-behavior decision record is the [FE-1580 donor matrix](docs/evidence/implementations/mission-5-voice-safety-parity/donor-behavior-matrix.md). Donor PRs are read-only evidence at their named heads; semantic reimplementation is required, never merge or cherry-pick. ## Imperative @@ -51,7 +61,7 @@ OpenAI Realtime microphone input → agentOwnershipGuard → mounted Brunch ChatAgent → submission-correlated canonical response and settlement → exact canonical segment queue → visible panel text and TTS -→ replay controls and durable Voice provenance +→ replay controls and supported client-tool Voice provenance → observe({ live: "sse" }) hydration and reopen ``` @@ -65,22 +75,22 @@ This mission closes the Voice safety and UX-parity stratum on the parent's route ### Product-manager litmus -**Release note:** Voice now submits only what the microphone actually transcribed, waits for a safe **Your turn** handoff before listening over Brunch, can replay the exact full response or question, and keeps its Voice attribution after reopening the conversation. +**Release note:** Voice now submits only what the microphone actually transcribed, waits for a safe **Your turn** handoff before listening over Brunch, and can replay the exact full response or question. Client-tool Voice origins survive canonical reopen; restoring the Voice chip on direct spoken user messages remains blocked on Flue projection support. -**Demo script:** run `yarn dev:brunch` and select the Brunch preview. Speak one answer and see exactly one matching user turn. While Brunch is speaking, confirm the microphone remains closed, choose **Your turn**, wait for the handoff, and speak again. Use the playback menu to read the full response and repeat the question verbatim. Start another turn, press durable **Stop** before settlement, and see a stopped turn rather than a Voice error. After the parent hydration blocker is fixed, hard-reload the settled conversation and confirm the same Voice attribution remains without resubmission or replay. +**Demo script:** run `yarn dev:brunch` and select the Brunch preview. Speak one answer and see exactly one matching user turn. While Brunch is speaking, confirm the microphone remains closed, choose **Your turn**, wait for the handoff, and speak again. Use the playback menu to read the full response and repeat the question verbatim. Start another turn, press durable **Stop** before settlement, and see a stopped turn rather than a Voice error. After the parent hydration blocker is fixed, hard-reload the settled conversation and confirm the canonical turn remains without resubmission or replay; direct-user Voice-chip restoration additionally waits on the Flue projection seam. -**Previously impossible:** model-generated function arguments rather than completed audio transcription could become the answer; assistant playback could create a false user turn; cancellation could reopen capture before the provider settled; replay controls and durable Voice attribution were incomplete. +**Previously impossible:** model-generated function arguments rather than completed audio transcription could become the answer; assistant playback could create a false user turn; cancellation could reopen capture before the provider settled; replay controls and multi-origin client-tool Voice attribution were incomplete. -**Completion:** sections 1–6 close when their tests and focused checks pass. Mission acceptance additionally requires the real microphone, handoff, Stop, hard-reload, and same-origin route witness after the parent hydration fix. Mocked or server-only proof cannot substitute for that witness. +**Completion:** sections 1–5 and the supported portion of section 6 close when their tests and focused checks pass. Direct-user provenance needs the Flue re-entry seam recorded below. Mission acceptance additionally requires the real microphone, handoff, Stop, hard-reload, and same-origin route witness after the parent hydration fix. Mocked or server-only proof cannot substitute for that witness. 1. **Completed-transcript authority and half-duplex ownership.** Realtime session configuration has no tools, no model-created semantic-VAD response, and no automatic interruption policy. Only a unique completed transcript can reach the shared panel submission path. Duplicate, empty, failed, unavailable, stale, playback-overlapping/pre-handoff, and over-limit transcripts do not submit and produce the specified passive or recoverable notice. The microphone remains closed through assistant output and cancellation; **Your turn** opens only a post-barrier input turn. Oracle: transplanted-first cases in `openai-realtime-session.test.ts`, `realtime-brunch-bridge.test.ts`, `voice-turn-controller.test.ts`, `voice-interview-control.test.tsx`, and `voice-preview.integration.test.ts`. 2. **Idempotent admission.** Typed turns derive a stable key from the AI SDK message id; Voice turns derive it from connection epoch, item id, and content index. A repeated same-payload key converges on the original receipt, including `deduplicated: true`; a 409 `submission_conflict` surfaces the original `submissionId` without admitting another turn. An admission whose outcome cannot be established stays visibly ambiguous and is never automatically retried. Oracle: transport unit tests plus Voice integration tests asserting one admitted submission rather than one `send()` invocation. 3. **Acknowledged cancellation barrier.** `cancelOutput()` resolves only after input/output buffer clears, matching provider acknowledgements, and all targeted response terminal events. The latest mute preference wins while it settles. Audio captured before the handoff cannot submit afterward. Durable Stop remains a stopped Flue turn rather than a Voice failure. Oracle: donor-adapted session/controller race tests and the unsettled-Stop integration case. 4. **Canonical replay.** `canReadFullResponse`, `canRepeatQuestion`, `readFullResponse()`, `repeatQuestion()`, and the playback menu retain and enqueue exact canonical segments without a simplifier. Replay enables only after the matching response terminal and output completion and remains disabled during submission, capture, cancellation, pause, and errors. Oracle: canonical speech, controller, panel, and integration tests comparing segment identity and text. -5. **Durable Voice provenance.** An assistant message may retain multiple `voiceToolCallIds`; one failed sibling origin does not erase successful origins. Attribution survives canonical projection, hydration, and reopen through a supported Flue representation or deterministic reconstruction from durable correlation facts, never through user text. Oracle: snapshot projection, panel provenance, partial-failure, hydration, and reopen tests. +5. **Durable Voice provenance.** An assistant message may retain multiple `voiceToolCallIds`; one failed sibling origin does not erase successful origins. Persisted Flue client-tool-result signals support deterministic reconstruction after hydration and reopen. Direct spoken user messages remain Voice-attributed only while live because the canonical snapshot omits their caller origin. Re-entry requires a supported Flue user-message metadata/idempotency projection; browser storage and user-text encoding are rejected. Oracle: snapshot projection and panel partial-failure tests for supported origins, plus the [blocker record](docs/evidence/implementations/mission-5-voice-safety-parity/provenance-blocker.md). 6. **Dormant ask removal.** If still present after restacking, the website does not register `brunchAskInteractiveTool` for Voice and canonical speech does not recognize `"brunch-ask"`. No spoken ask answer can enter a wait state the transport cannot resume. Oracle: registration/canonical-speech negative tests and a repository search showing no mounted Voice `brunch_ask` surface. 7. **Real witness and same-origin route.** After the parent hydration fix lands, a human performs one microphone turn, explicit interruption/handoff, durable Stop on an unsettled turn, and hard reload of a settled turn. The retained network route summary proves the absolute Flue `streamUrl` remains on the same-origin proxy. Oracle: `witness.md`, sanitized `voice-events.jsonl`, `network-routes.json`, canonical `flue-snapshot.json`, `settlements.json`, commit manifest, and hashes under `docs/evidence/implementations/mission-5-voice-safety-parity/`. -8. **Focused repository verification and truthful docs.** The requested four-workspace Turbo command passes. `apps/petrinaut-website/README.md` and `libs/@hashintel/petrinaut/docs/ai-assistant.md` describe half-duplex handoff, replay, Stop, transcript rejection, and durable attribution. If the published Petrinaut package changes, exactly one patch changeset covers it. Oracle: the command recorded in the PR and changeset inspection. +8. **Focused repository verification and truthful docs.** The requested four-workspace Turbo command passes. `apps/petrinaut-website/README.md` and `libs/@hashintel/petrinaut/docs/ai-assistant.md` describe half-duplex handoff, replay, Stop, transcript rejection, and the direct-user attribution limitation. If the published Petrinaut package changes, exactly one patch changeset covers it. Oracle: the command recorded in the PR and changeset inspection. ## Constraints @@ -113,21 +123,22 @@ This mission closes the Voice safety and UX-parity stratum on the parent's route ## Fog-line - **Parent movement.** Lu owns #9528 and may push more commits. Before each implementation phase, compare the GitHub head and restack this branch; an observed parent change is adopted only through restack, never copied into this branch. -- **Durable provenance representation.** Inspect the current Flue event/snapshot vocabulary and existing correlation facts before choosing between a supported persisted metadata field and deterministic reconstruction. If neither can preserve multiple origins without a second authority or user-text encoding, stop with the exact missing platform seam. - **Conflict normalization.** The installed SDK exposes the 409 contract through `FlueApiError.body: unknown`. Narrow only the documented envelope needed to recover `error.meta.submissionId`; do not create a general error protocol or infer success from prose. - **Dormant `brunch_ask`.** Remove or gate only the parent surfaces that remain after the next restack. If Lu has already removed them, record the parent commit and make no duplicate change. +- **Direct-user Voice provenance.** Flue 2.0.3's canonical user message exposes a generated `submissionId` but not caller metadata or `idempotencyKey`. Re-enter only when a supported durable correlation seam exists or the owner explicitly changes the representation; do not add browser persistence or encode origin in user text. - **Hard-reload witness.** Hold proof leaf 7 until the parent's hydration overwrite is fixed. Unit/integration tests for this branch may cover deterministic projection, but they cannot substitute for the blocked real witness. ## Stop or reorient Stop and report if the work would require direct Voice `send()`, a second transcript or conversation authority, hand-rolled stream offsets/recovery, automatic retry after ambiguous admission, canonical text rewriting, a live structured-question path, or any excluded parent fix. -Stop if half-duplex handoff cannot guarantee that pre-handoff audio is rejected and post-barrier audio is fresh, or if provider acknowledgements cannot bound `cancelOutput()` without inventing events. Stop if provenance requires visible-text smuggling or a second durable store. Stop if replay can enable before both matching terminal conditions, or if local cancellation invokes durable abort. +Stop if half-duplex handoff cannot guarantee that pre-handoff audio is rejected and post-barrier audio is fresh, or if provider acknowledgements cannot bound `cancelOutput()` without inventing events. The provenance stop condition has fired for direct spoken user turns: the browser-store implementation was removed and the unsupported leaf is recorded as blocked. Stop if replay can enable before both matching terminal conditions, or if local cancellation invokes durable abort. Do not manufacture the hard-reload witness while the parent hydration defect remains. Retain the blocker and wait for a new parent head. ## Deferred - The real witness and same-origin absolute-`streamUrl` observation wait on the parent hydration fix; once unblocked, they are part of this mission rather than a successor. +- Direct-user Voice attribution after canonical hydration waits on a supported Flue caller-metadata or idempotency projection seam, or an explicit owner decision to change the durable input representation. - Donor retirement waits until this replacement is accepted and each donor owner explicitly approves closure. Do not close #9496, #9500, #9507, or #9512 as an implementation side effect, and never close stakeholder-owned H-6763. - Response preparation/simplification, structured questions, Petri-net work, FE-1575, production identity, CORS/remote deployment, and panel migration away from `useChat` remain in their existing owners or the future mission spine. diff --git a/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/donor-behavior-matrix.md b/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/donor-behavior-matrix.md index b651ceb3cb0..7143461cb51 100644 --- a/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/donor-behavior-matrix.md +++ b/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/donor-behavior-matrix.md @@ -37,18 +37,18 @@ The owner selected half-duplex turn ownership on 2026-09-03: assistant output ow | #9512 | Half-duplex `canTakeTurn`, `takeTurn()`, `"cancelling"` output state, and **Your turn** control | **Adopt by owner decision** | It makes output/input ownership explicit and prevents assistant playback from becoming a false user turn. | Adapt public Voice state and panel controls without importing donor topology or ask widgets. | | #9512 | Promise-returning `cancelOutput()` that waits for input/output clears, matching acknowledgements, and response terminal events | **Adopt** | The microphone cannot safely reopen on a fire-and-forget cancel. | Transplant acknowledgement/race tests first; preserve latest mute preference and fresh post-handoff capture. | | #9512 | Replay availability tied to exact retained source, terminal response, and output completion | **Adopt with #9496 mechanics** | This closes replay races without changing canonical content. | Unify with the parent's segment/submission correlation rather than donor ask correlation. | -| #9512 | Voice answer icon/provenance before interactive answers | **Adopt semantically, broaden cardinality** | Live attribution is useful but one origin per assistant message is insufficient after coalesced or sibling Voice deliveries. | Use `voiceToolCallIds`; preserve successful siblings on partial failure and survive hydration/reopen. | +| #9512 | Voice answer icon/provenance before interactive answers | **Partially adopt; blocked for direct user turns** | Live attribution is useful but one origin per assistant message is insufficient after coalesced or sibling Voice deliveries. Flue's client-tool result signal can durably carry those origins. Its direct-user delivery and snapshot types expose no caller metadata or idempotency key, so a direct spoken user message cannot be identified after reopen without a forbidden second store or text encoding. | Keep `voiceToolCallIds`, preserve successful siblings on partial failure, and reconstruct supported tool-result origins from Flue signals. Re-enter direct-user attribution only when Flue provides a supported durable correlation seam. | | #9512 | App-local agent topology, temporary ask UI, response preparation, or donor-specific host composition | **Reject** | The parent owns the one Flue route and current host composition; these mechanisms are obsolete or non-goals. | Reuse only state-machine, cancellation, replay, and attribution behavior. | ## Outstanding acceptance ledger | Area | Required closing evidence | Current state | | --- | --- | --- | -| Transcript authority | Transplanted-first session, bridge, controller, and integration regressions pass on path B. | Open for implementation. | -| Admission idempotency | Typed and Voice logical replays converge on one `submissionId`; conflict metadata is narrowed safely; ambiguous outcome does not retry. | Open for implementation. | -| Cancellation barrier | Buffer acknowledgements and targeted response terminals settle before capture; stale/pre-handoff audio cannot submit; latest mute choice wins. | Open for implementation. | -| Canonical replay | Exact segment queue and playback menu pass availability/race tests without a simplifier. | Open for implementation. | -| Durable provenance | Multiple origins and partial failure survive projection, hydration, and reopen without user-text encoding. | Representation remains at the mission fog-line pending Flue inspection. | -| Dormant ask | No mounted Voice ask capability remains, or the parent commit that removed it is recorded. | Recheck after the next parent restack. | +| Transcript authority | Transplanted-first session, bridge, controller, and integration regressions pass on path B. | Implemented. Focused policy, session, bridge, controller, control, and preview integration tests pass using completed transcripts only. | +| Admission idempotency | Typed and Voice logical replays converge on one `submissionId`; conflict metadata is narrowed safely; ambiguous outcome does not retry. | Implemented. Transport tests cover stable typed/Voice keys, deduplicated receipts, 409 conflicts, and non-retried ambiguity. | +| Cancellation barrier | Buffer acknowledgements and targeted response terminals settle before capture; stale/pre-handoff audio cannot submit; latest mute choice wins. | Implemented. Session/controller races and the preview integration cover acknowledged handoff and discarded overlap. | +| Canonical replay | Exact segment queue and playback menu pass availability/race tests without a simplifier. | Implemented. Controller and panel tests cover both replay actions, exact segments, terminal gating, and disabled states. | +| Durable provenance | Multiple origins and partial failure survive projection, hydration, and reopen without user-text encoding. | Partially implemented for assistant client-tool results through persisted Flue signals; multiple sibling origins survive projection and partial failure. Direct spoken user attribution is blocked because Flue 2.0.3 snapshots do not expose caller idempotency or user-message metadata. The rejected browser store would violate mission authority. | +| Dormant ask | No mounted Voice ask capability remains, or the parent commit that removed it is recorded. | Implemented. The website no longer registers the ask widget and canonical speech ignores `brunch_ask`; dormant source files remain unmounted. | | Real witness | Microphone, handoff, unsettled Stop, reload, canonical snapshot, settlement, and same-origin absolute-`streamUrl` artifacts are retained with hashes. | Blocked by the parent's hydration overwrite. | | Donor retirement | Replacement accepted and each donor owner explicitly approves closure. | Deferred; no donor or stakeholder issue may be closed now. | diff --git a/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/provenance-blocker.md b/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/provenance-blocker.md new file mode 100644 index 00000000000..93bd15c5994 --- /dev/null +++ b/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/provenance-blocker.md @@ -0,0 +1,39 @@ +# FE-1580 direct-user Voice provenance blocker + +## Observed boundary + +Flue 2.0.3 can durably preserve Voice provenance for client-tool results: the +existing client-tool result signal carries each Voice-origin tool-call id, and +canonical snapshot projection can reconstruct every surviving sibling origin. + +The corresponding direct-user seam does not exist in the installed public +contract: + +- `DeliveredMessage` user input accepts only `body` and image `attachments`; +- the caller's `idempotencyKey` is accepted for admission but is not projected + into `FlueConversationMessage` or `FlueConversationSettlement`; +- materialized user messages expose the generated `submissionId`, but no Voice + source metadata; and +- snapshot `metadata` is agent-authored response metadata, not caller-authored + user-message metadata. + +The discarded implementation persisted Voice `submissionId` values in browser +storage and correlated them after hydration. That would create a second durable +store, which the mission explicitly names as a stop condition. Encoding the +origin in visible user text is also prohibited. Replacing the canonical direct +user message with a hidden Flue signal would change the delivery semantics and +require a synthetic second transcript projection, so it is not a transparent +representation of the existing path-B turn. + +## Current disposition + +Direct spoken user turns still render with a Voice chip while their AI SDK +message metadata is live. Their canonical text and submission survive Flue +hydration, but the Voice chip cannot be reconstructed after reopen. This portion +of section 6 is blocked rather than reported as complete. + +Re-enter only when Flue projects caller metadata or the caller idempotency key +onto the canonical direct-user message, or when the product owner explicitly +authorizes a different durable representation. The oracle is a snapshot-only +test that reconstructs the Voice marker after a fresh process with no browser +correlation state. diff --git a/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/witness-blocker.md b/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/witness-blocker.md new file mode 100644 index 00000000000..2a84dae70c4 --- /dev/null +++ b/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/witness-blocker.md @@ -0,0 +1,32 @@ +# FE-1580 real-witness blocker + +## Current disposition + +The real Voice witness has **not** been run and no witness bundle is claimed. +Sections 1–5 and the supported client-tool portion of section 6 have focused +automated coverage, but that coverage cannot replace the microphone, handoff, +unsettled Stop, hard-reload, and network-route witness required for mission +acceptance. Direct-user Voice attribution has a separate +[Flue projection blocker](provenance-blocker.md). + +The parent branch still runs canonical hydration once per conversation in a way +that can overwrite a locally submitted turn. The successor may not fix that +parent-owned defect. A hard-reload recording made before the parent fix would +therefore be unable to establish the required durability claim. + +## Re-entry gate + +After [PR #9528](https://github.com/hashintel/hash/pull/9528) lands a fix and this +branch is restacked onto its new verified head: + +1. run one real microphone turn and retain the sanitized Voice event ledger; +2. use **Your turn** during output and retain cancellation acknowledgements; +3. durably stop an unsettled turn and retain its stopped settlement; +4. hard-reload a settled Voice turn and retain the canonical Flue snapshot; +5. retain a network route summary proving the absolute Flue `streamUrl` remains + on the same-origin proxy; and +6. record the exact commits and hashes for every retained artifact. + +Until then, `witness.md`, `voice-events.jsonl`, `network-routes.json`, +`flue-snapshot.json`, and `settlements.json` are intentionally absent rather +than populated with simulated evidence. diff --git a/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/index.ts b/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/index.ts index 93b19f1e1be..c29f85526dd 100644 --- a/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/index.ts +++ b/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/index.ts @@ -26,6 +26,7 @@ export interface ClientToolResult { readonly toolCallId: string; readonly toolName: string; readonly output: unknown; + readonly source?: "voice"; } export interface FlueChatTransportOptions { @@ -42,6 +43,11 @@ export interface FlueChatTransportOptions { }) => void; } +const asRecord = (value: unknown): Record | null => + typeof value === "object" && value !== null && !Array.isArray(value) + ? (value as Record) + : null; + const completedClientToolResults = ( messages: readonly UIMessage[], assistantMessageId: string, @@ -54,6 +60,17 @@ const completedClientToolResults = ( if (assistantMessage === undefined) { return []; } + const metadata = asRecord(assistantMessage.metadata); + const voiceToolCallIds = new Set( + Array.isArray(metadata?.voiceToolCallIds) + ? metadata.voiceToolCallIds.filter( + (toolCallId): toolCallId is string => typeof toolCallId === "string", + ) + : [], + ); + if (typeof metadata?.toolCallId === "string") { + voiceToolCallIds.add(metadata.toolCallId); + } return assistantMessage.parts.flatMap((part): ClientToolResult[] => { if (!isToolUIPart(part)) return []; const toolName = getToolName(part); @@ -70,6 +87,9 @@ const completedClientToolResults = ( toolCallId: part.toolCallId, toolName, output: part.output, + ...(voiceToolCallIds.has(part.toolCallId) + ? { source: "voice" as const } + : {}), }, ]; }); @@ -97,11 +117,30 @@ const finalUserMessage = ( const isAbortError = (error: unknown): boolean => error instanceof Error && error.name === "AbortError"; +const conflictingSubmissionId = (error: FlueApiError): string | null => { + if (error.status !== 409) return null; + const body = asRecord(error.body); + const errorBody = asRecord(body?.error); + const metadata = asRecord(errorBody?.meta); + return errorBody?.type === "submission_conflict" && + typeof metadata?.submissionId === "string" && + metadata.submissionId.length > 0 + ? metadata.submissionId + : null; +}; + const admissionError = (error: unknown): Error => { if (isAbortError(error)) { return error as Error; } if (error instanceof FlueApiError) { + const existingSubmissionId = conflictingSubmissionId(error); + if (existingSubmissionId !== null) { + return new Error( + `The delivery key already belongs to admitted submission ${existingSubmissionId}; the changed payload was not admitted.`, + { cause: error }, + ); + } return new Error( `Brunch rejected the message before admission (HTTP ${error.status}).`, { cause: error }, @@ -262,13 +301,31 @@ export const createFlueChatTransport = < toolCallIds: toolResults .map((result) => result.toolCallId) .join(","), + ...(toolResults.some(({ source }) => source === "voice") + ? { + voiceToolCallIds: toolResults + .filter(({ source }) => source === "voice") + .map(({ toolCallId }) => toolCallId) + .join(","), + } + : {}), }, }; })(); + const idempotencyKey = + messageId === undefined + ? `ai-sdk:${userMessage!.id}` + : `ai-sdk-tool:${messageId}:${toolResults + .map(({ toolCallId }) => toolCallId) + .join(",")}`; + if (Array.from(idempotencyKey).length > 256) { + throw new Error("The submitted message identity is too long."); + } let admission: AgentSendResult; try { admission = await options.client.send({ + idempotencyKey, message, signal: abortSignal, }); diff --git a/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/transcript.ts b/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/transcript.ts index 089fc3dd4bc..64ba4243aba 100644 --- a/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/transcript.ts +++ b/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/transcript.ts @@ -9,10 +9,16 @@ import type { UIMessage } from "ai"; type UiMessagePart = UIMessage["parts"][number]; +export interface UiHistoryMessageMetadata { + readonly source: "voice"; + readonly voiceToolCallIds?: readonly string[]; +} + export type UiHistoryMessage = Omit< - UIMessage, + UIMessage, "metadata" | "parts" | "role" > & { + metadata?: UiHistoryMessageMetadata; role: Extract; parts: UiMessagePart[]; }; @@ -33,11 +39,16 @@ const isFlueDataPart = ( const isRecord = (value: unknown): value is Record => typeof value === "object" && value !== null; +interface ClientToolResult { + readonly output: unknown; + readonly source?: "voice"; +} + const clientToolResultsFrom = ( snapshot: Pick, signalName: string, -): ReadonlyMap => { - const outputsByCallId = new Map(); +): ReadonlyMap => { + const resultsByCallId = new Map(); for (const message of snapshot.messages) { if (message.purpose !== "dispatch") continue; if (message.signal?.tagName !== signalName) continue; @@ -63,19 +74,22 @@ const clientToolResultsFrom = ( ) { continue; } - outputsByCallId.set(result.toolCallId, result.output); + resultsByCallId.set(result.toolCallId, { + output: result.output, + ...(result.source === "voice" ? { source: "voice" } : {}), + }); } } - return outputsByCallId; + return resultsByCallId; }; const toolPartFrom = ( part: Extract, clientToolNames: ReadonlySet, - clientOutputs: ReadonlyMap, + clientResults: ReadonlyMap, ): UiMessagePart => { const isClientTool = clientToolNames.has(part.toolName); - const hasClientOutput = clientOutputs.has(part.toolCallId); + const hasClientOutput = clientResults.has(part.toolCallId); if (part.state === "output-error") { return { type: `tool-${part.toolName}`, @@ -95,7 +109,7 @@ const toolPartFrom = ( }; } const output = isClientTool - ? clientOutputs.get(part.toolCallId) + ? clientResults.get(part.toolCallId)?.output : part.state === "output-available" ? part.output : undefined; @@ -121,7 +135,7 @@ const toolPartFrom = ( const partsFrom = ( message: FlueConversationMessage, options: SnapshotToUiMessagesOptions, - clientOutputs: ReadonlyMap, + clientResults: ReadonlyMap, ): UiMessagePart[] => { const parts: UiMessagePart[] = []; for (const part of message.parts) { @@ -134,7 +148,7 @@ const partsFrom = ( continue; } if (part.type === "dynamic-tool") { - parts.push(toolPartFrom(part, options.clientToolNames, clientOutputs)); + parts.push(toolPartFrom(part, options.clientToolNames, clientResults)); continue; } if (part.type === "file") { @@ -159,7 +173,7 @@ export const snapshotToUiMessages = ( snapshot: Pick, options: SnapshotToUiMessagesOptions, ): UiHistoryMessage[] => { - const clientOutputs = clientToolResultsFrom( + const clientResults = clientToolResultsFrom( snapshot, CLIENT_TOOL_RESULT_SIGNAL, ); @@ -180,7 +194,7 @@ export const snapshotToUiMessages = ( if (message.display !== "visible") continue; if (message.purpose !== "user" && message.purpose !== "assistant") continue; if (message.role !== "user" && message.role !== "assistant") continue; - const parts = partsFrom(message, options, clientOutputs); + const parts = partsFrom(message, options, clientResults); if (message.role === "user") { resumableAssistant = undefined; continuationPending = false; @@ -195,10 +209,27 @@ export const snapshotToUiMessages = ( continuationPending = false; continue; } + const voiceToolCallIds = + message.role === "assistant" + ? message.parts.flatMap((part) => + part.type === "dynamic-tool" && + clientResults.get(part.toolCallId)?.source === "voice" + ? [part.toolCallId] + : [], + ) + : []; const projected: UiHistoryMessage = { id: message.id, role: message.role, parts, + ...(voiceToolCallIds.length > 0 + ? { + metadata: { + source: "voice", + voiceToolCallIds, + }, + } + : {}), }; messages.push(projected); if (message.role === "assistant") { diff --git a/libs/@hashintel/petrinaut/docs/ai-assistant.md b/libs/@hashintel/petrinaut/docs/ai-assistant.md index 6d56acb96e8..e4d1ebf9fa8 100644 --- a/libs/@hashintel/petrinaut/docs/ai-assistant.md +++ b/libs/@hashintel/petrinaut/docs/ai-assistant.md @@ -20,11 +20,11 @@ While a response is streaming you can: - Type your next message in the composer -- it is queued for after the current response ends. The application embedding Petrinaut may place an additional control beside the message box. For example, a host can offer another way to enter finalized text. Text submitted by that control behaves like text sent with the keyboard: it joins the same conversation and, when an inline question is waiting for an answer, completes that question rather than starting an unrelated message. A host can explicitly submit a separate message instead when the text is a correction or other follow-up that must not answer the pending question. -If the host offers voice input, a finalized spoken turn is held while an existing response finishes and is submitted when the conversation is ready. +If the host offers voice input, only a finalized transcript captured while Voice owns the input turn can be submitted. Voice waits while an existing response finishes or yields through the host's handoff control. If an assistant request fails, Petrinaut shows the error in a brief toast rather than adding it to the conversation. Retry from the composer when the assistant is ready. -Hosts may provide canonical conversation rehydration. In that case, reopening the same assistant shows its settled and stopped turns without resubmitting a message or replaying Voice audio. +Hosts may provide canonical conversation rehydration. In that case, reopening the same assistant shows its settled and stopped turns without resubmitting a message or replaying Voice audio. Voice markers attached to client-tool results survive that history. A direct spoken user message remains in the transcript after reopening, but its **Voice** chip may not be restored by the current Brunch host. When the Brunch voice preview is enabled and available, an empty composer shows a waveform action titled **Start voice mode**. Typing non-whitespace text replaces it with **Send**. The same dynamic @@ -55,23 +55,29 @@ the session ends, the held turns appear together under a **Voice session · N tu finalized answers and canonical Brunch text become chat history; provisional transcription and Realtime audio are ephemeral. Finalized spoken user messages carry a small **Voice** chip in front of the words themselves, and the exact inline answer completed by speech carries the same chip, so Voice -provenance remains visible without duplicating an answer. - -The microphone stays on while the interviewer speaks, so speaking naturally interrupts the audio -and starts listening to you; you do not need to select an interrupt action. Semantic voice detection -finishes each answer automatically after a natural pause and is tuned to allow longer thinking -pauses. There is no required done-speaking action. - -Every session control lives in the dock: **Show transcription in chat** on the left, and on the right -**Mute microphone** (**Unmute microphone** once muted) beside **End voice mode**. Muting stops -sending audio without ending the turn, so the assistant plays out whatever it is saying and unmuting -drops you straight back into the conversation. **Resume voice mode** replaces the microphone action -while a session is paused, and **Reconnect voice mode** replaces it after a failure. Nothing is added -to the canvas toolbar. Sending non-empty typed text from the +provenance remains visible without duplicating an answer while the session is mounted. + +Voice is half-duplex. The microphone is closed while the interviewer speaks or the assistant is +working, which prevents playback from becoming a false answer. Select **Your turn** to interrupt: +the dock shows the handoff as thinking while it clears pending audio and waits for the provider to +finish cancellation, then opens a fresh input turn. Audio captured before that completed handoff is +discarded. Semantic voice detection finishes your answer automatically after a natural pause, so +there is no required done-speaking action. Duplicate, empty, failed, or unavailable transcripts are +not submitted; the dock asks you to try again. An overlong answer instead asks for a shorter response. +Provisional words remain display-only until the provider completes their transcript. + +Every session control lives in the dock: **Show transcription in chat** and **Voice playback +options** on the left, and the available handoff, microphone, recovery, and end actions on the right. +The playback menu offers **Repeat question** and **Read full response** after the matching response +and speech have both finished. Both replay the exact retained canonical words; they stay unavailable +during active capture, submission, cancellation, pause, and errors. **Mute microphone** becomes +**Unmute microphone** once muted, and your latest choice applies when a handoff settles. **Resume voice mode** +replaces the microphone action while a session is paused, and **Reconnect voice mode** replaces it +after a failure. Nothing is added to the canvas toolbar. Sending non-empty typed text from the composer or first-run prompt ends Voice mode before it sends the message once through the same conversation; repeated send actions are ignored while that short handoff completes. -The interviewer uses a warm, calm, curious, and professionally neutral voice and treats you as the authority on your system. Brunch still chooses every question and interview decision; OpenAI only delivers its words. The question and finalized response shown in the Petrinaut conversation are authoritative. The speech request receives that exact Brunch text in part order; synthesized audio is generated from it but is not a verbatim recording. Interrupting audio does not undo the visible response or change the interview's saved history. +The interviewer uses a warm, calm, curious, and professionally neutral voice and treats you as the authority on your system. Brunch still chooses every question and interview decision; OpenAI only transcribes your completed input and delivers Brunch's words. The question and finalized response shown in the Petrinaut conversation are authoritative. The speech request receives that exact Brunch text in part order; synthesized audio is generated from it but is not a verbatim recording. Interrupting audio does not undo the visible response or change the interview's saved history. Closing the AI panel pauses microphone capture and active speech, then hides the dock until you reopen the panel. The same mounted session stays paused; choose **Resume voice mode** when you are diff --git a/libs/@hashintel/petrinaut/src/react/voice-session/store.ts b/libs/@hashintel/petrinaut/src/react/voice-session/store.ts index bcdc4e0ee48..73e502c20bb 100644 --- a/libs/@hashintel/petrinaut/src/react/voice-session/store.ts +++ b/libs/@hashintel/petrinaut/src/react/voice-session/store.ts @@ -7,9 +7,12 @@ import type { export type VoiceSessionActions = { end: () => void; pause: () => void; + readFullResponse?: () => void; reconnect: () => void; + repeatQuestion?: () => void; resume: () => void; setMicrophoneMuted: (muted: boolean) => void; + takeTurn?: () => Promise | void; }; export type VoiceSessionSnapshot = { diff --git a/libs/@hashintel/petrinaut/src/react/voice-session/types.ts b/libs/@hashintel/petrinaut/src/react/voice-session/types.ts index bd1a26f425e..b72bee75f8c 100644 --- a/libs/@hashintel/petrinaut/src/react/voice-session/types.ts +++ b/libs/@hashintel/petrinaut/src/react/voice-session/types.ts @@ -16,10 +16,18 @@ export type PetrinautAiVoiceSessionPhase = * effect: it changes at microphone-sampling rate. */ export type PetrinautAiVoiceSessionState = { + /** Whether the current canonical assistant response is safe to replay. */ + canReadFullResponse?: boolean; + /** Whether the final segment of the canonical response is safe to repeat. */ + canRepeatQuestion?: boolean; + /** Whether the user can cancel Voice output and start their turn. */ + canTakeTurn?: boolean; errorMessage: string | null; /** Whether microphone capture is muted independently of whose turn it is. */ microphoneMuted: boolean; /** Normalized 0–1 input level driving the listening indicator. */ microphoneLevel: number; + /** Recoverable feedback about an utterance which was not submitted. */ + notice?: string | null; phase: PetrinautAiVoiceSessionPhase; }; diff --git a/libs/@hashintel/petrinaut/src/react/voice-session/use-voice-session.ts b/libs/@hashintel/petrinaut/src/react/voice-session/use-voice-session.ts index a95671ce61d..3f88e964621 100644 --- a/libs/@hashintel/petrinaut/src/react/voice-session/use-voice-session.ts +++ b/libs/@hashintel/petrinaut/src/react/voice-session/use-voice-session.ts @@ -62,3 +62,43 @@ export const useVoiceSessionActions = (): VoiceSessionActions | null => { () => null, ); }; + +export const useVoiceSessionCanReadFullResponse = (): boolean => { + const store = use(VoiceSessionContext); + + return useSyncExternalStore( + store.subscribe, + () => store.getSnapshot().state?.canReadFullResponse ?? false, + () => false, + ); +}; + +export const useVoiceSessionCanRepeatQuestion = (): boolean => { + const store = use(VoiceSessionContext); + + return useSyncExternalStore( + store.subscribe, + () => store.getSnapshot().state?.canRepeatQuestion ?? false, + () => false, + ); +}; + +export const useVoiceSessionCanTakeTurn = (): boolean => { + const store = use(VoiceSessionContext); + + return useSyncExternalStore( + store.subscribe, + () => store.getSnapshot().state?.canTakeTurn ?? false, + () => false, + ); +}; + +export const useVoiceSessionNotice = (): string | null => { + const store = use(VoiceSessionContext); + + return useSyncExternalStore( + store.subscribe, + () => store.getSnapshot().state?.notice ?? null, + () => null, + ); +}; diff --git a/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts b/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts index 7ef27d515f5..de3acc3a4ae 100644 --- a/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts +++ b/libs/@hashintel/petrinaut/src/ui/types/ai-assistant-composer-control.ts @@ -61,12 +61,18 @@ export type PetrinautAiVoiceModeControls = { reconnect: () => void; /** Resumes microphone capture after `pause`. */ resume: () => void; + /** Replays the exact retained canonical assistant response when available. */ + readFullResponse?: () => void; + /** Replays the final segment of the retained canonical response. */ + repeatQuestion?: () => void; /** * Stops or restarts microphone capture while the session keeps running, so * the assistant carries on speaking. Unlike `pause`, which suspends the * whole session when Petrinaut closes the panel. */ setMicrophoneMuted: (muted: boolean) => void; + /** Cancels Voice output and hands the live microphone turn to the user. */ + takeTurn?: () => Promise | void; }; /** Stable controls and conversation state supplied to a host-owned Voice mode. */ diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/components/voice-session-labels.ts b/libs/@hashintel/petrinaut/src/ui/views/Editor/components/voice-session-labels.ts index eaf5dabb078..a6b6fb3f0c2 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/components/voice-session-labels.ts +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/components/voice-session-labels.ts @@ -29,7 +29,11 @@ export const voiceSessionActionLabels = { end: "End voice mode", mute: "Mute microphone", pause: "Pause voice mode", + playbackOptions: "Voice playback options", + readFullResponse: "Read full response", reconnect: "Reconnect voice mode", + repeatQuestion: "Repeat question", resume: "Resume voice mode", + takeTurn: "Your turn", unmute: "Unmute microphone", } as const; diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx index 6c515cc2adf..c97800049d8 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx @@ -119,10 +119,30 @@ const markVoiceToolOrigin = ( ): PetrinautAiMessage[] => messages.map((message) => message.id === messageId - ? { - ...message, - metadata: { ...message.metadata, source: "voice", toolCallId }, - } + ? (() => { + const previousToolCallIds = + message.metadata?.source === "voice" + ? [ + ...(message.metadata.voiceToolCallIds ?? []), + ...(message.metadata.toolCallId + ? [message.metadata.toolCallId] + : []), + ] + : []; + const { toolCallId: _legacyToolCallId, ...previousMetadata } = + message.metadata ?? {}; + + return { + ...message, + metadata: { + ...previousMetadata, + source: "voice", + voiceToolCallIds: [ + ...new Set([...previousToolCallIds, toolCallId]), + ], + }, + }; + })() : message, ); @@ -211,8 +231,42 @@ export const addMappedToolOutput = async ({ latestMessages.map((message) => message.id === containingMessage.id && message.metadata?.source === "voice" && - message.metadata.toolCallId === params.toolCallId - ? { ...message, metadata: previousMetadata } + (message.metadata.voiceToolCallIds?.includes(params.toolCallId) === + true || + message.metadata.toolCallId === params.toolCallId) + ? (() => { + const attributionAlreadyPresent = + previousMetadata?.source === "voice" && + (previousMetadata.voiceToolCallIds?.includes( + params.toolCallId, + ) === true || + previousMetadata.toolCallId === params.toolCallId); + const voiceToolCallIds = [ + ...(message.metadata.voiceToolCallIds ?? []), + ...(message.metadata.toolCallId + ? [message.metadata.toolCallId] + : []), + ]; + const remainingVoiceToolCallIds = attributionAlreadyPresent + ? voiceToolCallIds + : voiceToolCallIds.filter( + (candidateToolCallId) => + candidateToolCallId !== params.toolCallId, + ); + if (remainingVoiceToolCallIds.length === 0) { + return { ...message, metadata: previousMetadata }; + } + const { toolCallId: _legacyToolCallId, ...metadata } = + message.metadata; + + return { + ...message, + metadata: { + ...metadata, + voiceToolCallIds: [...new Set(remainingVoiceToolCallIds)], + }, + }; + })() : message, ), ); diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/tool-list.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/tool-list.tsx index f566adab421..39920911d39 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/tool-list.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/tool-list.tsx @@ -497,7 +497,8 @@ export const toToolRenderItem = ( state === "output-available" && typeof part.toolCallId === "string" && message.metadata?.source === "voice" && - message.metadata.toolCallId === part.toolCallId, + (message.metadata.voiceToolCallIds?.includes(part.toolCallId) === true || + message.metadata.toolCallId === part.toolCallId), errorText: state === "output-error" && typeof part.errorText === "string" ? part.errorText diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/voice-dock.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/voice-dock.tsx index db2177be73f..50679be8230 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/voice-dock.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/voice-dock.tsx @@ -3,7 +3,11 @@ import { css, cva } from "@hashintel/ds-helpers/css"; import { useVoiceSessionActions, + useVoiceSessionCanReadFullResponse, + useVoiceSessionCanRepeatQuestion, + useVoiceSessionCanTakeTurn, useVoiceSessionMicrophoneMuted, + useVoiceSessionNotice, useVoiceSessionPhase, } from "../../../../../../react/voice-session/use-voice-session"; import { LiveVoiceSessionIndicator } from "../../../components/voice-session-indicator"; @@ -13,6 +17,7 @@ import { } from "../../../components/voice-session-labels"; import { aiFooterMinHeight } from "./footer-height"; import { MicrophoneIcon } from "./voice-dock/microphone-icon"; +import { VoicePlaybackMenu } from "./voice-dock/playback-menu"; import { TranscriptionIcon } from "./voice-dock/transcription-icon"; import type { VoiceSessionActions } from "../../../../../../react/voice-session/store"; @@ -100,9 +105,13 @@ const visuallyHiddenStyle = css({ export type VoiceDockProps = { actions: VoiceSessionActions | null; + canReadFullResponse: boolean; + canRepeatQuestion: boolean; + canTakeTurn: boolean; /** Rendered instead of the live indicator when the caller supplies one. */ indicator?: ReactNode; microphoneMuted: boolean; + notice: string | null; onTranscriptionToggle: () => void; phase: PetrinautAiVoiceSessionPhase; /** Whether spoken turns are currently written into the conversation live. */ @@ -117,8 +126,12 @@ export type VoiceDockProps = { */ export const VoiceDock = ({ actions, + canReadFullResponse, + canRepeatQuestion, + canTakeTurn, indicator, microphoneMuted, + notice, onTranscriptionToggle, phase, transcriptionShown, @@ -135,33 +148,52 @@ export const VoiceDock = ({ aria-label="Voice session" className={dockStyle} data-phase={phase} + data-voice-notice={notice ? "visible" : undefined} data-testid="ai-voice-dock" > {actions !== null && ( - + ), + transport, + }, + }); + + fireEvent.click( + screen.getByRole("button", { name: "Submit normalized Voice payload" }), + ); + await screen.findByText("Voice message accepted"); + + expect(requestMessages[0]?.at(-1)).toMatchObject({ + id: "voice-realtime:3:item-1:0", + metadata: { source: "voice" }, + parts: [{ text: " Already normalized upstream ", type: "text" }], + role: "user", + }); + }); + test("marks the exact pending tool as voice-origin without a user message", async () => { const requestMessages: PetrinautAiMessage[][] = []; const onMessages = vi.fn(); diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx index dc015dcfe0b..c138cf7738c 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx @@ -899,8 +899,8 @@ export const AiAssistantPanel = ({ target?: "auto" | "message"; text: string; }): Promise => { - const trimmed = text.trim(); - if (!trimmed) { + const submissionText = source === "voice" ? text : text.trim(); + if (!submissionText.trim()) { const submissionError = new Error( "AI assistant text must not be empty.", ); @@ -979,7 +979,7 @@ export const AiAssistantPanel = ({ try { output = mappedToolCall.mapText({ input: mappedToolCall.input, - text: trimmed, + text: submissionText, }); } catch (caught) { const submissionError = @@ -1033,7 +1033,7 @@ export const AiAssistantPanel = ({ await submitMessage({ id: messageId, ...(source === "voice" ? { metadata: { source } } : {}), - parts: [{ text: trimmed, type: "text" }], + parts: [{ text: submissionText, type: "text" }], role: "user", }); return { kind: "message", messageId }; From dc3f6715da832a5bb80f7d8174c59ab9b884679a Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Fri, 4 Sep 2026 01:47:05 +0200 Subject: [PATCH 10/52] Prove dormant ask and Voice provenance on reopen Amp-Thread-ID: https://ampcode.com/threads/T-01a06941-3ee7-7482-a82e-8a16c7141785 Co-authored-by: Amp --- .../local-storage-demo-app.test.tsx | 49 ++++++++++- .../use-flue-chat-history.test.ts | 84 +++++++++++++++++++ 2 files changed, 132 insertions(+), 1 deletion(-) diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx index 18ad15887f5..00348665e7f 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx @@ -1,7 +1,7 @@ /** * @vitest-environment jsdom */ -import { act, cleanup, render } from "@testing-library/react"; +import { act, cleanup, render, waitFor } from "@testing-library/react"; import { isValidElement, type ReactNode } from "react"; import { afterEach, describe, expect, test, vi } from "vitest"; @@ -19,10 +19,24 @@ import { import type { FlueClient } from "@flue/sdk"; import type { PetrinautNavigationController } from "@hashintel/petrinaut/react"; +import type { PetrinautAiAssistant } from "@hashintel/petrinaut/ui"; const defaultTransportOptions = vi.hoisted(() => ({ current: null as unknown, })); +const flueClientMock = vi.hoisted(() => ({ current: null as unknown })); +const renderedPetrinaut = vi.hoisted(() => ({ aiAssistant: null as unknown })); + +vi.mock("@flue/sdk", () => ({ + createFlueClient: () => flueClientMock.current, +})); + +vi.mock("./brunch-preview-config", () => ({ + resolveBrunchPreviewConfig: () => ({ + chatEndpoint: "/agents/chat", + isBrunchConfigured: true, + }), +})); const editorProps = vi.hoisted(() => ({ current: null as { @@ -46,6 +60,7 @@ vi.mock("@hashintel/petrinaut/ui", () => ({ }, Petrinaut: (props: Record) => { editorProps.current = props; + renderedPetrinaut.aiAssistant = props.aiAssistant; return null; }, WalkthroughProvider: ({ children }: { children: ReactNode }) => children, @@ -110,6 +125,38 @@ describe("local storage demo Brunch voice integration", () => { unsubscribe(); }); + test("registers no brunch_ask tool in the production Brunch preview", async () => { + flueClientMock.current = { + observe: () => ({ + close: vi.fn(), + getSnapshot: () => ({ phase: "absent" }), + refresh: vi.fn(), + subscribe: () => () => undefined, + }), + }; + vi.stubGlobal( + "fetch", + vi.fn(async () => + Response.json({ available: false }), + ), + ); + + const rendered = render(); + await waitFor(() => expect(renderedPetrinaut.aiAssistant).not.toBeNull()); + const aiAssistant = renderedPetrinaut.aiAssistant as PetrinautAiAssistant; + + expect(aiAssistant.requestStop).toBeTypeOf("function"); + expect(aiAssistant.interactiveTools).toEqual([]); + expect( + aiAssistant.interactiveTools?.some( + ({ toolName }) => toolName === "brunch_ask", + ), + ).toBe(false); + + rendered.unmount(); + vi.unstubAllGlobals(); + }); + test("correlates the existing Brunch transport request", () => { const options = defaultTransportOptions.current as { readonly headers: () => Record; diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/use-flue-chat-history.test.ts b/apps/petrinaut-website/src/main/app/local-storage-demo/use-flue-chat-history.test.ts index f21c7a30033..c29ad8a16d9 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/use-flue-chat-history.test.ts +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/use-flue-chat-history.test.ts @@ -104,6 +104,90 @@ test("exposes the canonical settlement index for Voice correlation", async () => ]); }); +test("preserves every persisted Voice tool origin across hydration and reopen", async () => { + const harness = createObservationHarness({ + conversation: { + conversationId: "conversation-1", + settlements: [], + messages: [ + { + id: "assistant-voice-tools", + role: "assistant", + purpose: "assistant", + display: "visible", + parts: [ + { + type: "dynamic-tool", + toolCallId: "tool-doc-1", + toolName: "readPetrinautDoc", + state: "output-available", + input: { doc: "ai-assistant" }, + output: { awaiting: "client" }, + }, + { + type: "dynamic-tool", + toolCallId: "tool-doc-2", + toolName: "readPetrinautDoc", + state: "output-available", + input: { doc: "ai-assistant" }, + output: { awaiting: "client" }, + }, + ], + }, + { + id: "signal-voice-results", + role: "system", + purpose: "dispatch", + display: "hidden", + signal: { tagName: "client-tool-result" }, + parts: [ + { + type: "text", + text: JSON.stringify([ + { + toolCallId: "tool-doc-1", + toolName: "readPetrinautDoc", + output: "First guide", + source: "voice", + }, + { + toolCallId: "tool-doc-2", + toolName: "readPetrinautDoc", + output: "Second guide", + source: "voice", + }, + ]), + state: "done", + }, + ], + }, + ], + }, + offset: "offset-voice", + phase: "live", + error: undefined, + }); + const firstOpen = renderHook(() => + useFlueChatHistory(harness.clientPromise, "conversation-1"), + ); + + await waitFor(() => expect(firstOpen.result.current.ready).toBe(true)); + expect(firstOpen.result.current.messages?.[0]?.metadata).toEqual({ + source: "voice", + voiceToolCallIds: ["tool-doc-1", "tool-doc-2"], + }); + firstOpen.unmount(); + + const reopened = renderHook(() => + useFlueChatHistory(harness.clientPromise, "conversation-1"), + ); + await waitFor(() => expect(reopened.result.current.ready).toBe(true)); + expect(reopened.result.current.messages?.[0]?.metadata).toEqual({ + source: "voice", + voiceToolCallIds: ["tool-doc-1", "tool-doc-2"], + }); +}); + test("asks nothing of the generic chat route, which keeps no history", () => { const { result } = renderHook(() => useFlueChatHistory(null, "conversation-1"), From 4342d623f4f4f0cd764d62d91cf16f86a6fc7e0c Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Fri, 4 Sep 2026 01:48:48 +0200 Subject: [PATCH 11/52] Prove durable Stop through the Flue host seam Amp-Thread-ID: https://ampcode.com/threads/T-01a06941-3ee7-7482-a82e-8a16c7141785 Co-authored-by: Amp --- .../local-storage-demo-app.test.tsx | 86 ++++++++++++++++++- 1 file changed, 84 insertions(+), 2 deletions(-) diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx index 00348665e7f..f2dc2eff39f 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx @@ -1,7 +1,7 @@ /** * @vitest-environment jsdom */ -import { act, cleanup, render, waitFor } from "@testing-library/react"; +import { act, cleanup, render, screen, waitFor } from "@testing-library/react"; import { isValidElement, type ReactNode } from "react"; import { afterEach, describe, expect, test, vi } from "vitest"; @@ -9,6 +9,7 @@ import { defaultPetrinautNavigationHistoryPolicy } from "@hashintel/petrinaut/re import { FlueChatAdmissionError } from "@hashintel/brunch-agent-transport-aisdk"; +import { OpenAIRealtimeSession } from "../voice-interview/openai-realtime-session"; import { VoiceInterviewControl } from "../voice-interview/voice-interview-control"; import { BrunchPanelConversationTracker } from "./brunch-panel-transport"; import { @@ -17,7 +18,10 @@ import { requestFlueStop, } from "./local-storage-demo-app"; -import type { FlueClient } from "@flue/sdk"; +import type { + AgentConversationObservationSnapshot, + FlueClient, +} from "@flue/sdk"; import type { PetrinautNavigationController } from "@hashintel/petrinaut/react"; import type { PetrinautAiAssistant } from "@hashintel/petrinaut/ui"; @@ -126,6 +130,7 @@ describe("local storage demo Brunch voice integration", () => { }); test("registers no brunch_ask tool in the production Brunch preview", async () => { + renderedPetrinaut.aiAssistant = null; flueClientMock.current = { observe: () => ({ close: vi.fn(), @@ -157,6 +162,83 @@ describe("local storage demo Brunch voice integration", () => { vi.unstubAllGlobals(); }); + test("keeps durable Flue Stop distinct from local playback cancellation", async () => { + renderedPetrinaut.aiAssistant = null; + let snapshot: AgentConversationObservationSnapshot = { + conversation: { + conversationId: "conversation-stop", + settlements: [], + messages: [], + }, + offset: "offset-before-stop", + phase: "live" as const, + error: undefined, + }; + const listeners = new Set<() => void>(); + const localPlaybackCancellation = vi.spyOn( + OpenAIRealtimeSession.prototype, + "cancelOutput", + ); + const abort = vi.fn(async () => { + snapshot = { + conversation: { + conversationId: "conversation-stop", + settlements: [ + { submissionId: "submission-stop", outcome: "aborted" as const }, + ], + messages: [], + }, + offset: "offset-after-stop", + phase: "live" as const, + error: undefined, + }; + for (const listener of listeners) listener(); + return { aborted: true }; + }); + flueClientMock.current = { + abort, + observe: () => ({ + close: vi.fn(), + getSnapshot: () => snapshot, + refresh: vi.fn(), + subscribe: (listener: () => void) => { + listeners.add(listener); + return () => listeners.delete(listener); + }, + }), + }; + vi.stubGlobal( + "fetch", + vi.fn(async () => + Response.json({ available: false }), + ), + ); + + const rendered = render(); + await waitFor(() => + expect( + (renderedPetrinaut.aiAssistant as PetrinautAiAssistant).requestStop, + ).toBeTypeOf("function"), + ); + const aiAssistant = renderedPetrinaut.aiAssistant as PetrinautAiAssistant; + + await expect(aiAssistant.requestStop?.()).resolves.toBe("stop-requested"); + expect(abort).toHaveBeenCalledOnce(); + expect(localPlaybackCancellation).not.toHaveBeenCalled(); + await waitFor(() => { + const currentAssistant = + renderedPetrinaut.aiAssistant as PetrinautAiAssistant; + const status = currentAssistant.renderComposerControl?.({} as never); + if (!status) throw new Error("Expected the Brunch status control."); + render(status); + expect(screen.getByText("Last Brunch response stopped.")).not.toBeNull(); + }); + + rendered.unmount(); + localPlaybackCancellation.mockRestore(); + vi.unstubAllGlobals(); + }); + test("correlates the existing Brunch transport request", () => { const options = defaultTransportOptions.current as { readonly headers: () => Record; From f3c2852a8da08651a2e20866b40aaf3b9725461a Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Fri, 4 Sep 2026 01:50:28 +0200 Subject: [PATCH 12/52] Reconcile the Voice safety acceptance record Amp-Thread-ID: https://ampcode.com/threads/T-01a06941-3ee7-7482-a82e-8a16c7141785 Co-authored-by: Amp --- libs/@hashintel/brunch-agent/MISSION.md | 4 +- .../donor-behavior-matrix.md | 41 +++++++++++++------ .../provenance-blocker.md | 3 ++ .../witness-blocker.md | 13 +++--- 4 files changed, 41 insertions(+), 20 deletions(-) diff --git a/libs/@hashintel/brunch-agent/MISSION.md b/libs/@hashintel/brunch-agent/MISSION.md index 87b8f4ef9b9..0329d697b0a 100644 --- a/libs/@hashintel/brunch-agent/MISSION.md +++ b/libs/@hashintel/brunch-agent/MISSION.md @@ -69,7 +69,7 @@ OpenAI Realtime microphone input → observe({ live: "sse" }) hydration and reopen ``` -Realtime exposes no tools, uses `tool_choice: "none"`, and configures semantic VAD with `create_response: false`. Model function-call arguments are ignored even if a provider violates the policy. Provisional transcription is display-only and disappears without submission. +Realtime exposes no tools, uses `tool_choice: "none"`, and configures semantic VAD with `create_response: false`. Model function-call arguments are ignored even if a provider violates the policy. Provisional transcription is display-only and disappears without submission. OpenAI permits transcription completion for any committed audio item and does not guarantee completion order across turns; this mission deliberately accepts only an item whose matching `speech_started` boundary occurred during the current input turn. A boundaryless or completion-before-boundary item remains rejected rather than gaining authority retroactively. Local playback cancellation, local observation cancellation, the HTTP request `AbortSignal`, and durable conversation-wide `FlueClient.abort()` remain separate operations. The first three never masquerade as durable Stop; durable Stop never appears as a Voice transcription or playback failure. @@ -101,7 +101,7 @@ This mission closes the Voice safety and UX-parity stratum on the parent's route - Preserve the parent's one product route, memoized Flue client, browser `ChatTransport`, shared panel `useChat`, path-B Voice submission, canonical speech selection, durable Stop seam, and SDK observation hydration. Do not rebuild them. - Transplant relevant regression tests before implementation. Reimplement donor behavior semantically against the current Flue path; donor branches and PRs are never merged, cherry-picked, rebased, rewritten, retargeted, or closed by this implementation. - Derive one deterministic admission key per logical delivery. Treat `deduplicated` as successful convergence and `submission_conflict` as evidence of the already-admitted submission. Do not automatically retry an ambiguous admission. -- Normalize completed transcripts exactly once with trim plus Unicode whitespace collapse, then enforce the 32,000-code-point bound. Provisional text remains ephemeral and display-only. +- Normalize completed transcripts exactly once in the Realtime bridge with trim plus Unicode whitespace collapse, then enforce the 32,000-code-point bound. The generic panel validates but does not mutate that already-normalized Voice payload. Provisional text remains ephemeral and display-only. - The half-duplex microphone is closed during output, cancellation, pause, error, and submission states. A cancellation promise is part of the turn boundary, not a cosmetic animation state. - Brunch canonical text is never summarized, shortened, paraphrased, or regenerated for speech or replay. - Preserve every surviving Voice origin independently. Provenance must use supported Flue data or deterministic durable correlation; never encode it in visible user text. diff --git a/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/donor-behavior-matrix.md b/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/donor-behavior-matrix.md index 7d9891aa4d9..6b179e3f849 100644 --- a/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/donor-behavior-matrix.md +++ b/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/donor-behavior-matrix.md @@ -22,34 +22,49 @@ The owner selected half-duplex turn ownership on 2026-09-03: assistant output ow | --- | --- | --- | --- | --- | | #9528 | One `/agents/chat/:instanceId` product route, browser `ChatTransport`, one memoized client, path-B Voice submission through shared `useChat` | **Adopt** | This is the departure architecture and prevents a second admission authority. | Restack onto every new parent head; verify no successor code calls `send()` directly from Voice. | | #9528 | Direct Voice `send()` as a fog-line fallback | **Reject** | It creates a second admission path and mutable coordination surface. The parent has already proved path B. | Mission authority now permits path B only. | -| #9528 | Claim that Flue 2.0.3 lacks caller idempotency | **Reject as factually false** | Installed typings expose `AgentPromptOptions.idempotencyKey`, `AgentSendResult.deduplicated`, and 409 `submission_conflict` with the existing `submissionId`. | Add transport tests and implementation for same-payload convergence, conflict recovery, and visible ambiguous admission. | +| #9528 | Claim that Flue 2.0.3 lacks caller idempotency | **Reject as factually false** | Installed typings expose `AgentPromptOptions.idempotencyKey`, `AgentSendResult.deduplicated`, and 409 `submission_conflict` with the existing `submissionId`. | Implemented with transport convergence/conflict tests and typed Voice admission outcomes. | | #9528 | Parent-owned admission/Stop races, stream cancellation, hydration overwrite, client-tool classification, response/submission correlation, and PR/CI repairs | **Reject from successor scope** | Lu owns these defects; changing them here would make the stack compete with its parent. | Report blockers and restack onto fixes. The real reload witness remains blocked by hydration overwrite. | -| #9496 | Serialized canonical speech queue, retained exact source segments, response/output terminal gating | **Adopt mechanics** | Replay and ordinary TTS need one lifecycle-safe queue, and exact text preserves canonical authority. | Remove every preparation/simplifier dependency while adapting queue tests to the parent's canonical Flue segments. | -| #9496 | `canReadFullResponse`, `readFullResponse()`, exact full-response playback menu | **Adopt** | Exact full-response replay is supported by retained canonical segment identity. | Gate it during submission, capture, cancellation, pause, and errors; require matching response terminal plus output completion. | +| #9496 | Serialized canonical speech queue, retained exact source segments, response/output terminal gating | **Adopt mechanics** | Replay and ordinary TTS need one lifecycle-safe queue, and exact text preserves canonical authority. | Implemented without a preparation/simplifier dependency; exact-segment and queue tests pass. | +| #9496 | `canReadFullResponse`, `readFullResponse()`, exact full-response playback menu | **Adopt** | Exact full-response replay is supported by retained canonical segment identity. | Implemented with idle-state and matching response/output terminal gates. | | #9496 | `canRepeatQuestion`, `repeatQuestion()` implemented as replaying the final canonical segment | **Block** | Canonical Brunch segments provide no deterministic question marker. The final segment may be ordinary prose and is not authority for question identity. | Keep the production action unavailable. Re-enter only through a supported Brunch-owned marker or correlation rule without restoring `brunch_ask` or structured questions. | | #9496 | Realtime-generated concise response preparation or any fallback that rewrites canonical text | **Reject** | Response simplification is a non-goal and violates exact canonical speech. | Tests compare retained segment ids and exact queued strings; no preparation API remains on this path. | -| #9500 | No Realtime tools, `tool_choice: "none"`, semantic VAD `create_response: false` | **Adopt** | Realtime detects/transcribes and renders supplied TTS only; it must not generate user meaning. | Adapt policy, session, and integration tests to the parent route. | -| #9500 | Only `conversation.item.input_audio_transcription.completed` can submit; model function arguments ignored | **Adopt** | Shape validation cannot prove model-generated arguments match the audio. | Transplant regression tests before replacing the parent bridge authority. | -| #9500 | Transcript identity `(connectionEpoch, itemId, contentIndex)`, stable submission id, trim plus Unicode whitespace collapse, 32,000-code-point limit | **Adopt** | This gives one deterministic logical Voice delivery and one normalization boundary. | Feed the derived identity through path B as the Flue idempotency key. | -| #9500 | Explicit duplicate, empty, failed, unavailable, and over-limit rejection; passive/recoverable not-heard UI; provisional display only | **Adopt** | Rejected audio must never become a turn, while ordinary silence/failure must not poison the session. | Preserve the reason-specific bridge events and controller recovery behavior. | +| #9500 | No Realtime tools, `tool_choice: "none"`, semantic VAD `create_response: false` | **Adopt** | Realtime detects/transcribes and renders supplied TTS only; it must not generate user meaning. | Implemented in policy, session, and production-preview integration tests. | +| #9500 | Only `conversation.item.input_audio_transcription.completed` can submit; model function arguments ignored | **Adopt** | Shape validation cannot prove model-generated arguments match the audio. | Implemented with current-turn speech-boundary, stale, reordered, and late-output rejection tests. | +| #9500 | Transcript identity `(connectionEpoch, itemId, contentIndex)`, stable submission id, trim plus Unicode whitespace collapse, 32,000-code-point limit | **Adopt** | This gives one deterministic logical Voice delivery and one normalization boundary. | Implemented through path B; the panel preserves the bridge-normalized payload unchanged. | +| #9500 | Explicit duplicate, empty, failed, unavailable, and over-limit rejection; passive/recoverable not-heard UI; provisional display only | **Adopt** | Rejected audio must never become a turn, while ordinary silence/failure must not poison the session. | Implemented with reason-specific bridge/controller UI coverage. | | #9500 | Silently settling ownership by discarding every playback-overlapping utterance without an explicit handoff | **Supersede** | It avoids echo but leaves users without a deliberate way to take the turn. | Use #9512 half-duplex `canTakeTurn`/`takeTurn()` and reject all speech captured before the completed handoff. | | #9500 | `brunch_ask` answer/tool correlation and preparation code inherited from its base | **Reject** | Structured questions and response preparation are excluded. | Correlate the Voice delivery to its path-B submission and canonical response facts; question replay remains blocked without a supported marker. | | #9507 | Temporary `brunch_ask` registration, widget, correlated spoken ask answer, transcript formatting | **Reject entire shim** | The current transport only admits the supported follow-up set; a spoken ask can otherwise wait forever. Structured questions are a separate product decision. | Remove or gate dormant `brunchAskInteractiveTool` and `"brunch-ask"` canonical-speech recognition only if still present after restack. | -| #9512 | Half-duplex `canTakeTurn`, `takeTurn()`, `"cancelling"` output state, and **Your turn** control | **Adopt by owner decision** | It makes output/input ownership explicit and prevents assistant playback from becoming a false user turn. | Adapt public Voice state and panel controls without importing donor topology or ask widgets. | -| #9512 | Promise-returning `cancelOutput()` that waits for input/output clears, matching acknowledgements, and response terminal events | **Adopt** | The microphone cannot safely reopen on a fire-and-forget cancel. | Transplant acknowledgement/race tests first; preserve latest mute preference and fresh post-handoff capture. | -| #9512 | Full-response replay availability tied to exact retained source, terminal response, and output completion | **Adopt with #9496 mechanics** | This closes replay races without changing canonical content. | Unify with the parent's segment/submission correlation rather than donor ask correlation. Question replay is excluded until a supported marker exists. | +| #9512 | Half-duplex `canTakeTurn`, `takeTurn()`, `"cancelling"` output state, and **Your turn** control | **Adopt by owner decision** | It makes output/input ownership explicit and prevents assistant playback from becoming a false user turn. | Implemented through the public Voice store and production panel registration path. | +| #9512 | Promise-returning `cancelOutput()` that waits for input/output clears, matching acknowledgements, and response terminal events | **Adopt** | The microphone cannot safely reopen on a fire-and-forget cancel. | Implemented with acknowledgement/race tests, latest-mute behavior, and fresh post-handoff capture. | +| #9512 | Full-response replay availability tied to exact retained source, terminal response, and output completion | **Adopt with #9496 mechanics** | This closes replay races without changing canonical content. | Implemented against parent segment/submission correlation. Question replay is excluded until a supported marker exists. | | #9512 | Voice answer icon/provenance before interactive answers | **Partially adopt; blocked for direct user turns** | Live attribution is useful but one origin per assistant message is insufficient after coalesced or sibling Voice deliveries. Flue's client-tool result signal can durably carry those origins. Its direct-user delivery and snapshot types expose no caller metadata or idempotency key, so a direct spoken user message cannot be identified after reopen without a forbidden second store or text encoding. | Keep `voiceToolCallIds`, preserve successful siblings on partial failure, and reconstruct supported tool-result origins from Flue signals. Re-enter direct-user attribution only when Flue provides a supported durable correlation seam. | | #9512 | App-local agent topology, temporary ask UI, response preparation, or donor-specific host composition | **Reject** | The parent owns the one Flue route and current host composition; these mechanisms are obsolete or non-goals. | Reuse only state-machine, cancellation, replay, and attribution behavior. | +## Adopted-behavior replacement coverage + +| Adopted behavior | Replacement implementation | Regression test | Production integration proof | Status | +| --- | --- | --- | --- | --- | +| One path-B Flue admission route | `local-storage-demo-app.tsx`, `brunch-panel-transport.ts`, transport `src/index.ts` | `brunch-panel-transport.test.ts`, `chat-transport.test.ts` | `voice-preview.integration.test.ts` crosses completed transcript → panel submission → Flue transport → canonical speech | **Implemented**; parent defects remain downstack | +| Stable admission identity and typed outcomes | transport `src/index.ts`, `brunch-panel-transport.ts`, `submitVoiceInputWithAdmission`, `realtime-brunch-bridge.ts` | transport admission cases; bridge/controller cases for rejected, conflict, ambiguous, and local abort | production preview 500 case observes one `send()` and visible ambiguous Voice state | **Implemented** | +| Exact canonical TTS queue and full-response replay | `openai-realtime-session.ts`, `voice-turn-controller.ts`, Petrinaut playback menu | session queue/cancellation cases; controller exact-segment and terminal-gating cases; panel action tests | real host registration exposes `readFullResponse`; panel forwards it through `voiceSessionStore` | **Implemented** | +| Repeat-question replay | none: no supported question identity exists | controller and host-registration negative cases | Brunch Voice host omits `repeatQuestion`; panel leaves the generic item disabled | **Blocked**, not mislabeled as final-segment replay | +| Disabled Realtime generation/tools | `openai-voice-policy.ts`, `openai-realtime-session.ts` | policy/session tests reject tools and function arguments | controlled production preview negotiates the server policy and emits only canonical speech | **Implemented** | +| Completed-transcript authority | `openai-realtime-session.ts`, `realtime-brunch-bridge.ts` | missing/stale/reordered boundary, keyed identity, normalization, duplicate/failure/limit, and late-output cases | controlled production preview submits one completed transcript through path B | **Implemented**; provider-valid boundaryless commits are intentionally rejected by mission policy | +| Half-duplex acknowledged handoff | `openai-realtime-session.ts`, `voice-turn-controller.ts`, Voice public store/dock | input/output clear acknowledgement, targeted response terminal, latest mute, stale/pre-handoff rejection | panel registration tests exercise **Your turn**; preview integration proves fresh post-handoff capture | **Implemented** | +| Durable Stop distinct from local cancellation | app `requestFlueStop`, panel `stopComposer`, session `cancelOutput` | panel durable-before-local Stop, controller/session local-cancel cases, app host Stop case | configured Brunch app invokes `FlueClient.abort()`, observes an aborted settlement, and does not invoke local playback cancellation | **Implemented**; parent-owned Stop races excluded | +| Multi-origin Voice client-tool provenance | panel `addMappedToolOutput`, transport client-tool-result signal/projection, `useFlueChatHistory` | sibling partial-failure, persisted-signal projection, hydration/reopen cases | configured app consumes the public Flue observation and restores every `voiceToolCallId` | **Implemented for client-tool results**; direct-user marker **blocked** | +| No live `brunch_ask` | Brunch app registers `interactiveTools: []`; canonical speech selector ignores the ask name | canonical-speech negative case and configured-app registration negative case | captured production Brunch `PetrinautAiAssistant` has no ask tool while retaining Flue Stop wiring | **Implemented exclusion** | + ## Outstanding acceptance ledger | Area | Required closing evidence | Current state | | --- | --- | --- | -| Transcript authority | Transplanted-first session, bridge, controller, and integration regressions pass on path B. | Implemented for completed-transcript authority and late-output invalidation. Matching `speech_started` boundary authority and reordered-event coverage remain under audit. | +| Transcript authority | Transplanted-first session, bridge, controller, and integration regressions pass on path B. | Implemented. Matching current-turn `speech_started`, stale/reordered boundaries, late-output invalidation, exact bridge normalization, and unchanged panel payload are covered. | | Admission idempotency | Typed and Voice logical replays converge on one `submissionId`; conflict metadata is narrowed safely; ambiguous outcome does not retry. | Implemented. Transport tests cover stable typed/Voice keys, deduplicated receipts, 409 conflicts, and non-retried ambiguity. | -| Cancellation barrier | Buffer acknowledgements and targeted response terminals settle before capture; stale/pre-handoff audio cannot submit; latest mute choice wins. | Implemented at session/controller level. Production panel-seam gating and unsettled durable-Stop integration proof remain outstanding. | +| Cancellation barrier | Buffer acknowledgements and targeted response terminals settle before capture; stale/pre-handoff audio cannot submit; latest mute choice wins. | Implemented. Session/controller races cover the barrier and mute preference; panel registration and configured-app Stop cases cover the production host seams. | | Canonical replay | Exact segment queue and playback menu pass availability/race tests without a simplifier. | Full-response replay is implemented with exact segments and terminal gating. **Repeat question** is blocked because no supported deterministic question marker exists; the production action must remain unavailable rather than replaying the final prose segment. | | Durable provenance | Multiple origins and partial failure survive projection, hydration, and reopen without user-text encoding. | Partially implemented for assistant client-tool results through persisted Flue signals; multiple sibling origins survive projection and partial failure. Direct spoken user attribution is blocked because Flue 2.0.3 snapshots do not expose caller idempotency or user-message metadata. The rejected browser store would violate mission authority. | -| Dormant ask | No mounted Voice ask capability remains, or the parent commit that removed it is recorded. | Canonical speech ignores `brunch_ask` and dormant source remains unmounted. A registration-level negative production-preview test remains outstanding. | +| Dormant ask | No mounted Voice ask capability remains, or the parent commit that removed it is recorded. | Implemented exclusion. Canonical speech ignores `brunch_ask`, and a configured-app registration test proves the production Brunch assistant supplies no ask tool. Dormant source remains unmounted. | | Real witness | Microphone, handoff, unsettled Stop, reload, canonical snapshot, settlement, and same-origin absolute-`streamUrl` artifacts are retained with hashes. | Blocked by the parent's hydration overwrite. | | Donor retirement | Replacement accepted and each donor owner explicitly approves closure. | Deferred; no donor or stakeholder issue may be closed now. | diff --git a/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/provenance-blocker.md b/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/provenance-blocker.md index 93bd15c5994..27668dd37fb 100644 --- a/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/provenance-blocker.md +++ b/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/provenance-blocker.md @@ -5,6 +5,9 @@ Flue 2.0.3 can durably preserve Voice provenance for client-tool results: the existing client-tool result signal carries each Voice-origin tool-call id, and canonical snapshot projection can reconstruct every surviving sibling origin. +Regression coverage preserves successful siblings after a partial failure, +projects both origins from the persisted signal, and restores them through the +production observation hook after unmount and reopen. The corresponding direct-user seam does not exist in the installed public contract: diff --git a/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/witness-blocker.md b/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/witness-blocker.md index 2a84dae70c4..d13f8632262 100644 --- a/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/witness-blocker.md +++ b/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/witness-blocker.md @@ -3,11 +3,14 @@ ## Current disposition The real Voice witness has **not** been run and no witness bundle is claimed. -Sections 1–5 and the supported client-tool portion of section 6 have focused -automated coverage, but that coverage cannot replace the microphone, handoff, -unsettled Stop, hard-reload, and network-route witness required for mission -acceptance. Direct-user Voice attribution has a separate -[Flue projection blocker](provenance-blocker.md). +Completed-transcript authority, admission idempotency, half-duplex handoff, +acknowledged cancellation, exact full-response replay, durable Stop, dormant-ask +exclusion, and the supported client-tool portion of Voice provenance have +focused automated coverage. **Repeat question** is separately blocked by the +lack of a deterministic canonical question marker. Automated coverage cannot +replace the microphone, handoff, unsettled Stop, hard-reload, and network-route +witness required for mission acceptance. Direct-user Voice attribution has a +separate [Flue projection blocker](provenance-blocker.md). The parent branch still runs canonical hydration once per conversation in a way that can overwrite a locally submitted turn. The successor may not fix that From 58053cad176e74defcbabd9ed0be7df44456e172 Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Fri, 4 Sep 2026 02:16:04 +0200 Subject: [PATCH 13/52] Clarify canonical speech turn ownership Amp-Thread-ID: https://ampcode.com/threads/T-01a06941-3ee7-7482-a82e-8a16c7141785 Co-authored-by: Amp --- libs/@hashintel/brunch-agent/MISSION.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/libs/@hashintel/brunch-agent/MISSION.md b/libs/@hashintel/brunch-agent/MISSION.md index 0329d697b0a..bab04fded50 100644 --- a/libs/@hashintel/brunch-agent/MISSION.md +++ b/libs/@hashintel/brunch-agent/MISSION.md @@ -4,7 +4,7 @@ **Live as of 2026-09-03** for [FE-1580](https://linear.app/hash/issue/FE-1580/harden-voice-safety-and-ux-on-the-unified-flue-route) on `kostandin/fe-1580-harden-voice-safety-and-ux-on-the-unified-flue-route`, stacked directly on [PR #9528](https://github.com/hashintel/hash/pull/9528) at the GitHub-verified head `036fb06ee83e840fd7a87dd1bc758a2dd49ddd4c`. This file is the sole execution authority for the successor branch. The parent remains the authority for its own admission/Stop races, stream cancellation, hydration overwrite, client-tool classification, response/submission correlation, and PR/CI defects; this branch must restack onto parent fixes rather than repair them. -The owner selected **half-duplex turn ownership** on 2026-09-03. While canonical assistant audio is pending or playing, the microphone is closed. The explicit **Your turn** action cancels output, waits for provider acknowledgement and response settlement, and only then opens a fresh input turn. Automatic duplex barge-in is rejected because assistant playback can become a false user turn. +The owner selected **half-duplex turn ownership** on 2026-09-03. While canonical assistant audio is pending or playing, the microphone is closed. Ownership transfers away from input as soon as canonical speech is requested, before `response.create` is sent: every accepted unfinished input item becomes stale and provisional transcript state is cleared. The explicit **Your turn** action cancels output, waits for provider acknowledgement and response settlement, and only then opens a fresh input turn. Automatic duplex barge-in is rejected because assistant playback can become a false user turn. ### Turn-ownership decision @@ -17,7 +17,7 @@ Completed-transcript authority, half-duplex ownership, admission idempotency, the cancellation barrier, and exact full-response replay have focused regression coverage. **Repeat question** remains blocked: canonical Brunch speech does not carry a deterministic question marker, so treating the final text segment as a -question would replay ordinary prose under a false label. Section 6 is complete +question would replay ordinary prose under a false label. Proof item 5 is complete only for the supported client-tool-result path: Flue signals persist each Voice-origin tool-call id beside its output, and canonical projection reconstructs multiple surviving origins. Direct spoken user attribution is @@ -69,7 +69,7 @@ OpenAI Realtime microphone input → observe({ live: "sse" }) hydration and reopen ``` -Realtime exposes no tools, uses `tool_choice: "none"`, and configures semantic VAD with `create_response: false`. Model function-call arguments are ignored even if a provider violates the policy. Provisional transcription is display-only and disappears without submission. OpenAI permits transcription completion for any committed audio item and does not guarantee completion order across turns; this mission deliberately accepts only an item whose matching `speech_started` boundary occurred during the current input turn. A boundaryless or completion-before-boundary item remains rejected rather than gaining authority retroactively. +Realtime exposes no tools, uses `tool_choice: "none"`, and configures semantic VAD with `create_response: false`. Model function-call arguments are ignored even if a provider violates the policy. Provisional transcription is display-only and disappears without submission. OpenAI permits transcription completion for any committed audio item and does not guarantee completion order across turns; this mission deliberately accepts only an item whose matching `speech_started` boundary occurred during the current input turn. A boundaryless or completion-before-boundary item remains rejected rather than gaining authority retroactively. Requesting canonical speech ends that input turn before `response.create`: unfinished accepted items and their provisional display state are invalidated even if their transcription completes before output audio starts. Local playback cancellation, local observation cancellation, the HTTP request `AbortSignal`, and durable conversation-wide `FlueClient.abort()` remain separate operations. The first three never masquerade as durable Stop; durable Stop never appears as a Voice transcription or playback failure. @@ -87,7 +87,7 @@ This mission closes the Voice safety and UX-parity stratum on the parent's route **Completion:** the implemented portions close when their tests and focused checks pass. **Repeat question** needs the Brunch-owned marker recorded below, and direct-user provenance needs the Flue re-entry seam recorded below. Mission acceptance additionally requires the real microphone, handoff, Stop, hard-reload, and same-origin route witness after the parent hydration fix. Mocked or server-only proof cannot substitute for that witness. -1. **Completed-transcript authority and half-duplex ownership.** Realtime session configuration has no tools, no model-created semantic-VAD response, and no automatic interruption policy. Only a unique completed transcript can reach the shared panel submission path. Duplicate, empty, failed, unavailable, stale, playback-overlapping/pre-handoff, and over-limit transcripts do not submit and produce the specified passive or recoverable notice. The microphone remains closed through assistant output and cancellation; **Your turn** opens only a post-barrier input turn. Oracle: transplanted-first cases in `openai-realtime-session.test.ts`, `realtime-brunch-bridge.test.ts`, `voice-turn-controller.test.ts`, `voice-interview-control.test.tsx`, and `voice-preview.integration.test.ts`. +1. **Completed-transcript authority and half-duplex ownership.** Realtime session configuration has no tools, no model-created semantic-VAD response, and no automatic interruption policy. Only a unique completed transcript can reach the shared panel submission path. Duplicate, empty, failed, unavailable, stale, canonical-speech-overlapping/pre-handoff, playback-overlapping, and over-limit transcripts do not submit and produce the specified passive or recoverable notice. Before sending `response.create`, a canonical speech request invalidates every unfinished accepted item, clears bridge/controller transcript state and provisional UI, and closes the microphone; a completion in the interval before output starts cannot submit or regain authority. **Your turn** opens only a post-barrier input turn. Oracle: transplanted-first cases in `openai-realtime-session.test.ts`, `realtime-brunch-bridge.test.ts`, `voice-turn-controller.test.ts`, `voice-interview-control.test.tsx`, and `voice-preview.integration.test.ts`. 2. **Idempotent admission.** Typed turns derive a stable key from the AI SDK message id; Voice turns derive it from connection epoch, item id, and content index. A repeated same-payload key converges on the original receipt, including `deduplicated: true`; a 409 `submission_conflict` surfaces the original `submissionId` without admitting another turn. An admission whose outcome cannot be established stays visibly ambiguous and is never automatically retried. Oracle: transport unit tests plus Voice integration tests asserting one admitted submission rather than one `send()` invocation. 3. **Acknowledged cancellation barrier.** `cancelOutput()` resolves only after input/output buffer clears, matching provider acknowledgements, and all targeted response terminal events. The latest mute preference wins while it settles. Audio captured before the handoff cannot submit afterward. Durable Stop remains a stopped Flue turn rather than a Voice failure. Oracle: donor-adapted session/controller race tests and the unsettled-Stop integration case. 4. **Canonical full-response replay; question replay blocked.** `canReadFullResponse`, `readFullResponse()`, and the playback menu retain and enqueue all exact canonical segments in order without a simplifier. Full-response replay enables only after the matching response terminal and output completion and remains disabled during submission, capture, cancellation, pause, and errors. `canRepeatQuestion` remains false and the production Voice host does not offer `repeatQuestion()` because neither canonical segments nor Flue correlation expose a deterministic Brunch-owned question marker. The final text segment is not authority for question identity. Re-entry requires such a marker without restoring `brunch_ask` or structured questions. Oracle: canonical speech, controller, panel, and integration tests comparing segment identity and text, plus negative production-host coverage for the blocked question action. @@ -102,7 +102,7 @@ This mission closes the Voice safety and UX-parity stratum on the parent's route - Transplant relevant regression tests before implementation. Reimplement donor behavior semantically against the current Flue path; donor branches and PRs are never merged, cherry-picked, rebased, rewritten, retargeted, or closed by this implementation. - Derive one deterministic admission key per logical delivery. Treat `deduplicated` as successful convergence and `submission_conflict` as evidence of the already-admitted submission. Do not automatically retry an ambiguous admission. - Normalize completed transcripts exactly once in the Realtime bridge with trim plus Unicode whitespace collapse, then enforce the 32,000-code-point bound. The generic panel validates but does not mutate that already-normalized Voice payload. Provisional text remains ephemeral and display-only. -- The half-duplex microphone is closed during output, cancellation, pause, error, and submission states. A cancellation promise is part of the turn boundary, not a cosmetic animation state. +- The half-duplex microphone is closed from the canonical speech request through output, cancellation, pause, error, and submission states. The request invalidates accepted unfinished input before `response.create`; only an acknowledged **Your turn** handoff can establish fresh post-request capture. A cancellation promise is part of the turn boundary, not a cosmetic animation state. - Brunch canonical text is never summarized, shortened, paraphrased, or regenerated for speech or replay. - Preserve every surviving Voice origin independently. Provenance must use supported Flue data or deterministic durable correlation; never encode it in visible user text. - Do not fix the parent's admission/Stop races, stream cancellation, hydration overwrite, client-tool classification, response/submission correlation, CI, title, or body. Restack onto Lu's fixes; report any blocker. From 81da04684e69eeb1da8648a18a5c60508a5588ef Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Fri, 4 Sep 2026 02:19:32 +0200 Subject: [PATCH 14/52] Invalidate input before canonical speech Amp-Thread-ID: https://ampcode.com/threads/T-01a06941-3ee7-7482-a82e-8a16c7141785 Co-authored-by: Amp --- .../openai-realtime-session.test.ts | 95 ++++++++++++++++ .../openai-realtime-session.ts | 7 ++ .../realtime-brunch-bridge.test.ts | 59 ++++++++++ .../voice-interview/realtime-brunch-bridge.ts | 5 +- .../voice-preview.integration.test.ts | 106 ++++++++++++++---- .../voice-turn-controller.test.ts | 85 ++++++++++++++ .../voice-interview/voice-turn-controller.ts | 4 + 7 files changed, 339 insertions(+), 22 deletions(-) diff --git a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts index de913472f75..d9a00d030d7 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts @@ -316,6 +316,101 @@ describe("OpenAIRealtimeSession", () => { expect(harness.localTracks[0]!.enabled).toBe(false); }); + test("invalidates accepted input before requesting canonical speech output", async () => { + const harness = createHarness(); + await harness.session.connect(); + harness.session.setMicrophoneEnabled(true); + const channel = harness.channels[0]!; + let microphoneEnabledWhenResponseRequested: boolean | undefined; + channel.send.mockImplementation((payload: string) => { + if (JSON.parse(payload).type === "response.create") { + microphoneEnabledWhenResponseRequested = + harness.localTracks[0]!.enabled; + } + }); + + channel.receive({ + audio_start_ms: 80, + item_id: "item-before-request", + type: "input_audio_buffer.speech_started", + }); + channel.receive({ + content_index: 0, + delta: "This started before canonical speech", + item_id: "item-before-request", + type: "conversation.item.input_audio_transcription.delta", + }); + + harness.session.speakCanonical([ + canonicalSegment("ask-request", "What happens next?"), + ]); + expect(harness.events).toContainEqual( + expect.objectContaining({ type: "canonical-speech-requested" }), + ); + expect(microphoneEnabledWhenResponseRequested).toBe(false); + expect(harness.localTracks[0]!.enabled).toBe(false); + + channel.receive({ + content_index: 0, + item_id: "item-before-request", + transcript: "This completed before output started.", + type: "conversation.item.input_audio_transcription.completed", + }); + expect( + harness.events.some( + (event) => + event.type === "completed" && + event.key.itemId === "item-before-request", + ), + ).toBe(false); + + const handoff = harness.session.cancelOutput(); + authorizeLatestSpeechResponse(channel, "response-before-output"); + channel.receive({ type: "input_audio_buffer.cleared" }); + channel.receive({ + response: { + id: "response-before-output", + output: [], + status: "cancelled", + }, + type: "response.done", + }); + await handoff; + expect(harness.localTracks[0]!.enabled).toBe(true); + + channel.receive({ + content_index: 0, + item_id: "item-before-request", + transcript: "The stale item cannot recover authority.", + type: "conversation.item.input_audio_transcription.completed", + }); + channel.receive({ + audio_start_ms: 160, + item_id: "item-after-handoff", + type: "input_audio_buffer.speech_started", + }); + channel.receive({ + content_index: 0, + item_id: "item-after-handoff", + transcript: "This is fresh after the handoff.", + type: "conversation.item.input_audio_transcription.completed", + }); + + expect( + harness.events.filter((event) => event.type === "completed"), + ).toEqual([ + { + key: { + connectionEpoch: 1, + contentIndex: 0, + itemId: "item-after-handoff", + }, + text: "This is fresh after the handoff.", + type: "completed", + }, + ]); + }); + test("restores only the latest microphone preference after playback", async () => { const harness = createHarness(); await harness.session.connect(); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts index 87c9d8cf0d3..6490da54901 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.ts @@ -574,6 +574,11 @@ export class OpenAIRealtimeSession { request, responseTerminalSequence: this.#responseTerminalSequence, }); + for (const itemId of this.#acceptedInputItemIds) { + this.#playbackOverlappingInputItemIds.add(itemId); + } + this.#acceptedInputItemIds.clear(); + this.#syncMicrophoneTrack(); try { this.#send({ event_id: eventId, @@ -590,6 +595,7 @@ export class OpenAIRealtimeSession { } catch (error) { this.#responseCreateEventId = null; this.#pendingClientEvents.delete(eventId); + this.#syncMicrophoneTrack(); throw error; } } @@ -1252,6 +1258,7 @@ export class OpenAIRealtimeSession { this.#microphoneRequested && this.#connected && this.#cancelOutputPromise === null && + this.#responseCreateEventId === null && this.#speakingResponseId === null; this.#microphoneTrack.enabled = enabled; if (enabled) { diff --git a/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.test.ts index 2e1966dcddf..e19273418af 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.test.ts @@ -209,6 +209,65 @@ describe("RealtimeBrunchBridge", () => { expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce(); }); + test("rejects unfinished input as soon as canonical speech is requested", async () => { + const harness = createHarness(); + startReady(harness); + + harness.emit({ + connectionEpoch: 3, + itemId: "item-before-request", + type: "input-speech-started", + }); + harness.emit({ + connectionEpoch: 3, + speechRequestId: "speech-request", + type: "canonical-speech-requested", + }); + harness.emit( + completedTranscript( + 3, + "This completed before output started.", + "item-before-request", + ), + ); + + expect(harness.submitInterviewAnswer).not.toHaveBeenCalled(); + expect(harness.events).toContainEqual({ + reason: "unavailable", + type: "transcript-rejected", + }); + + harness.emit( + completedTranscript( + 3, + "The stale item cannot recover authority.", + "item-before-request", + ), + ); + expect(harness.submitInterviewAnswer).not.toHaveBeenCalled(); + + harness.emit({ + connectionEpoch: 3, + responseId: "response-request", + type: "output-interrupted", + }); + harness.emit({ + connectionEpoch: 3, + itemId: "item-after-handoff", + type: "input-speech-started", + }); + harness.emit( + completedTranscript(3, "This is fresh.", "item-after-handoff"), + ); + + await vi.waitFor(() => + expect(harness.submitInterviewAnswer).toHaveBeenCalledOnce(), + ); + expect(harness.submitInterviewAnswer).toHaveBeenCalledWith( + expect.objectContaining({ text: "This is fresh." }), + ); + }); + test("derives stable delivery identity from epoch, item, and content index", () => { expect( createRealtimeSubmissionId(transcriptKey(12, "item/with spaces", 4)), diff --git a/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.ts b/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.ts index dd76621df32..6e9f1787aca 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.ts @@ -337,7 +337,10 @@ export class RealtimeBrunchBridge { } return; } - if (event.type === "output-started") { + if ( + event.type === "canonical-speech-requested" || + event.type === "output-started" + ) { this.#outputActive = true; for (const itemId of this.#acceptedInputItemIds) { this.#playbackOverlappingInputItemIds.add(itemId); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts index 6afd6ad7e73..8fee24b6766 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts @@ -209,15 +209,58 @@ describe("controlled voice preview", () => { reportDiagnostic, requestAnimationFrame: vi.fn(() => 1), }); - const submissionId = "submission-voice-1"; + const admission: AgentSendResult = { + offset: "offset-voice-1", + streamUrl: "https://petrinaut.test/agents/chat/instance-1", + submissionId: "submission-voice-1", + uid: "uid-voice-1", + }; + const send = vi.fn(async () => admission); + const wait = vi.fn(async () => undefined); + const tracker = new BrunchPanelConversationTracker(); + const transport = createBrunchPanelTransport( + Promise.resolve({ send, wait } as Pick< + FlueClient, + "send" | "wait" + > as FlueClient), + tracker, + ); type SubmitInterviewAnswer = ConstructorParameters< typeof RealtimeBrunchBridge >[0]["submitInterviewAnswer"]; - const submitInterviewAnswer = vi.fn( - async ({ id, onAdmission }) => { - onAdmission(submissionId); - return { kind: "message", messageId: id, submissionId }; - }, + const submitInterviewAnswer = vi.fn((input) => + submitVoiceInputWithAdmission({ + input, + resolveInputSubmission: (messageId) => + tracker.submissionForInput(messageId), + submitVoiceInput: async ({ id, text }) => { + if (id === undefined) { + throw new Error("Voice message identity is required."); + } + const stream = await transport.sendMessages({ + abortSignal: input.signal, + chatId: "conversation-1", + messageId: undefined, + messages: [ + { + id, + metadata: { source: "voice" }, + parts: [{ text, type: "text" }], + role: "user", + }, + ], + trigger: "submit-message", + }); + void stream.pipeTo(new WritableStream()); + return { kind: "message", messageId: id }; + }, + subscribeToAdmission: (target, listener) => + tracker.subscribeToAdmission(target, ({ admission: admitted }) => + listener(admitted.submissionId), + ), + subscribeToAdmissionFailure: (target, listener) => + tracker.subscribeToAdmissionFailure(target, listener), + }), ); const bridge = new RealtimeBrunchBridge({ session, @@ -249,26 +292,38 @@ describe("controlled voice preview", () => { canonicalSegments: initialSegments, status: "ready", }); - authorizeLatestSpeechResponse(dataChannel, "response-initial-question"); - dataChannel.receive({ - response_id: "response-initial-question", - type: "output_audio_buffer.started", - }); dataChannel.receive({ content_index: 0, item_id: "pre-output-item", - transcript: "This completed during assistant playback.", + transcript: "This completed before output started.", type: "conversation.item.input_audio_transcription.completed", }); expect(controller.getSnapshot()).toMatchObject({ - canTakeTurn: true, lastCommittedText: "", microphoneEnabled: true, - output: "speaking", partialText: "", }); expect(track.enabled).toBe(false); expect(submitInterviewAnswer).not.toHaveBeenCalled(); + expect(send).not.toHaveBeenCalled(); + + dataChannel.receive({ + content_index: 0, + item_id: "pre-output-item", + transcript: "The stale item cannot recover authority.", + type: "conversation.item.input_audio_transcription.completed", + }); + expect(send).not.toHaveBeenCalled(); + + authorizeLatestSpeechResponse(dataChannel, "response-initial-question"); + dataChannel.receive({ + response_id: "response-initial-question", + type: "output_audio_buffer.started", + }); + expect(controller.getSnapshot()).toMatchObject({ + canTakeTurn: true, + output: "speaking", + }); const handoff = controller.takeTurn(); dataChannel.receive({ @@ -332,12 +387,21 @@ describe("controlled voice preview", () => { }), ), ); - expect(controller.getSnapshot()).toMatchObject({ - input: "submitting", - lastAnswerDelivery: "delivered", - microphoneEnabled: true, - output: "waiting-for-tool", - }); + await vi.waitFor(() => expect(send).toHaveBeenCalledOnce()); + expect(send).toHaveBeenCalledWith( + expect.objectContaining({ + idempotencyKey: "ai-sdk:voice-realtime:1:user-item:0", + message: { body: spokenAnswer, kind: "user" }, + }), + ); + await vi.waitFor(() => + expect(controller.getSnapshot()).toMatchObject({ + input: "submitting", + lastAnswerDelivery: "delivered", + microphoneEnabled: true, + output: "waiting-for-tool", + }), + ); controller.updateChat({ canAcceptInterviewAnswer: false, @@ -350,7 +414,7 @@ describe("controlled voice preview", () => { ).map((segment) => initialSegmentIds.has(segment.id) ? segment - : { ...segment, submissionId }, + : { ...segment, submissionId: admission.submissionId }, ); controller.updateChat({ canAcceptInterviewAnswer: true, diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts index bc623736104..3e663848da5 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts @@ -304,6 +304,91 @@ describe("VoiceTurnController", () => { }); }); + test("clears capture when canonical speech is requested before output starts", async () => { + const harness = createHarness(); + harness.controller.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [question("ask-request")], + status: "ready", + }); + await harness.controller.start(); + harness.emitBridge({ + deliveryId: "voice-request", + segments: [question("ask-request")], + type: "canonical-response-ready", + }); + harness.emitSession({ + connectionEpoch: 1, + itemId: "item-before-request", + type: "input-speech-started", + }); + harness.emitSession({ + key: { + connectionEpoch: 1, + contentIndex: 0, + itemId: "item-before-request", + }, + text: "Provisional pre-request words", + type: "partial", + }); + + harness.emitSession({ + connectionEpoch: 1, + speechRequestId: "speech-request", + type: "canonical-speech-requested", + }); + + expect(harness.session.setMicrophoneEnabled).toHaveBeenLastCalledWith( + false, + ); + expect(harness.controller.getSnapshot()).toMatchObject({ + canTakeTurn: true, + lastCommittedText: "", + partialText: "", + }); + harness.emitSession({ + key: { + connectionEpoch: 1, + contentIndex: 0, + itemId: "item-before-request", + }, + text: "This completed before output started.", + type: "completed", + }); + expect(harness.controller.getSnapshot()).toMatchObject({ + lastCommittedText: "", + partialText: "", + }); + expect(harness.submitText).not.toHaveBeenCalled(); + + await harness.controller.takeTurn(); + harness.emitSession({ + connectionEpoch: 1, + itemId: "item-after-handoff", + type: "input-speech-started", + }); + harness.emitSession({ + key: { + connectionEpoch: 1, + contentIndex: 0, + itemId: "item-after-handoff", + }, + text: "Fresh post-handoff answer.", + type: "completed", + }); + harness.emitBridge({ + answer: "Fresh post-handoff answer.", + deliveryId: "fresh-delivery", + type: "submission-started", + }); + + expect(harness.controller.getSnapshot()).toMatchObject({ + input: "submitting", + lastCommittedText: "Fresh post-handoff answer.", + partialText: "", + }); + }); + test("hands off an active response once and applies the latest mute preference after cancellation", async () => { const harness = createHarness(); let finishCancellation: (() => void) | undefined; diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts index adc152d280c..6040032869e 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts @@ -637,6 +637,10 @@ export class VoiceTurnController { } if (event.type === "canonical-speech-requested") { this.#session.setMicrophoneEnabled(false); + this.#inputTurnPending = false; + this.#transcriptItemId = null; + this.#transcriptKey = null; + this.#update({ partialText: "" }); if ( this.#latencyCorrelationId !== null && this.#ttsSpeechRequestId === null From 128d14404ec7be72a5dd4634945fff848562963f Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Fri, 4 Sep 2026 02:20:36 +0200 Subject: [PATCH 15/52] Prove Voice admission outcomes through the panel route Amp-Thread-ID: https://ampcode.com/threads/T-01a06941-3ee7-7482-a82e-8a16c7141785 Co-authored-by: Amp --- .../voice-preview.integration.test.ts | 206 ++++++++++++------ 1 file changed, 135 insertions(+), 71 deletions(-) diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts index 8fee24b6766..c000aeae4e2 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts @@ -104,6 +104,82 @@ const responseMessages = [ }, ] satisfies PetrinautAiMessage[]; +const createAdmissionOutcomeHarness = ( + client: Pick, +) => { + const tracker = new BrunchPanelConversationTracker(); + const transport = createBrunchPanelTransport( + Promise.resolve(client as FlueClient), + tracker, + ); + let realtimeListener: + | ((event: OpenAIRealtimeSessionEvent) => void) + | undefined; + const bridge = new RealtimeBrunchBridge({ + session: { + speakCanonical: vi.fn(), + subscribe: (listener) => { + realtimeListener = listener; + return () => { + realtimeListener = undefined; + }; + }, + }, + submitInterviewAnswer: (input) => + submitVoiceInputWithAdmission({ + input, + resolveInputSubmission: (messageId) => + tracker.submissionForInput(messageId), + submitVoiceInput: async ({ id, text }) => { + if (id === undefined) { + throw new Error("Voice message identity is required."); + } + void transport + .sendMessages({ + abortSignal: input.signal, + chatId: "conversation-1", + messageId: undefined, + messages: [ + { + id, + metadata: { source: "voice" }, + parts: [{ text, type: "text" }], + role: "user", + }, + ], + trigger: "submit-message", + }) + .catch(() => undefined); + return { kind: "message", messageId: id }; + }, + subscribeToAdmission: (target, listener) => + tracker.subscribeToAdmission(target, ({ admission }) => + listener(admission.submissionId), + ), + subscribeToAdmissionFailure: (target, listener) => + tracker.subscribeToAdmissionFailure(target, listener), + }), + }); + const events: RealtimeBrunchBridgeEvent[] = []; + bridge.subscribe((event) => events.push(event)); + bridge.updateChat({ + canAcceptInterviewAnswer: true, + canonicalSegments: [], + status: "ready", + }); + bridge.start(1); + + return { + emitCompletedTranscript: (itemId: string) => + realtimeListener?.({ + key: { connectionEpoch: 1, contentIndex: 0, itemId }, + text: spokenAnswer, + type: "completed", + }), + events, + }; +}; + describe("controlled voice preview", () => { test("bridges one completed transcript through Brunch and back to canonical half-duplex audio", async () => { const diagnostics: VoiceDiagnosticEvent[] = []; @@ -662,80 +738,13 @@ describe("controlled voice preview", () => { const send = vi.fn(async () => { throw new FlueApiError(500, ""); }); - const tracker = new BrunchPanelConversationTracker(); - const transport = createBrunchPanelTransport( - Promise.resolve({ send } as Pick as FlueClient), - tracker, - ); - let realtimeListener: - | ((event: OpenAIRealtimeSessionEvent) => void) - | undefined; - const bridge = new RealtimeBrunchBridge({ - session: { - speakCanonical: vi.fn(), - subscribe: (listener) => { - realtimeListener = listener; - return () => { - realtimeListener = undefined; - }; - }, - }, - submitInterviewAnswer: (input) => - submitVoiceInputWithAdmission({ - input, - resolveInputSubmission: (messageId) => - tracker.submissionForInput(messageId), - submitVoiceInput: async ({ id, text }) => { - if (id === undefined) { - throw new Error("Voice message identity is required."); - } - void transport - .sendMessages({ - trigger: "submit-message", - chatId: "conversation-1", - messageId: undefined, - messages: [ - { - id, - role: "user", - metadata: { source: "voice" }, - parts: [{ type: "text", text }], - }, - ], - abortSignal: input.signal, - }) - .catch(() => undefined); - return { kind: "message", messageId: id }; - }, - subscribeToAdmission: (target, listener) => - tracker.subscribeToAdmission(target, ({ admission }) => - listener(admission.submissionId), - ), - subscribeToAdmissionFailure: (target, listener) => - tracker.subscribeToAdmissionFailure(target, listener), - }), - }); - const events: RealtimeBrunchBridgeEvent[] = []; - bridge.subscribe((event) => events.push(event)); - bridge.updateChat({ - canAcceptInterviewAnswer: true, - canonicalSegments: [], - status: "ready", - }); - bridge.start(1); + const abort = vi.fn(async () => ({ aborted: true })); + const harness = createAdmissionOutcomeHarness({ abort, send }); - realtimeListener?.({ - key: { - connectionEpoch: 1, - contentIndex: 0, - itemId: "input-item-ambiguous", - }, - text: spokenAnswer, - type: "completed", - }); + harness.emitCompletedTranscript("input-item-ambiguous"); await vi.waitFor(() => - expect(events).toContainEqual({ + expect(harness.events).toContainEqual({ code: "admission-ambiguous", failure: { kind: "ambiguous" }, message: @@ -744,5 +753,60 @@ describe("controlled voice preview", () => { }), ); expect(send).toHaveBeenCalledOnce(); + expect(abort).not.toHaveBeenCalled(); + }); + + test("preserves a conflicting submission through the production admission path", async () => { + const send = vi.fn(async () => { + throw new FlueApiError(409, { + error: { + details: "", + message: "The delivery key already names another payload.", + meta: { submissionId: "submission-existing" }, + type: "submission_conflict", + }, + }); + }); + const abort = vi.fn(async () => ({ aborted: true })); + const harness = createAdmissionOutcomeHarness({ abort, send }); + + harness.emitCompletedTranscript("input-item-conflict"); + + await vi.waitFor(() => + expect(harness.events).toContainEqual({ + code: "admission-conflict", + failure: { + kind: "submission-conflict", + status: 409, + submissionId: "submission-existing", + }, + message: + "The delivery key already belongs to admitted submission submission-existing; the changed payload was not admitted.", + type: "error", + }), + ); + expect(send).toHaveBeenCalledOnce(); + expect(abort).not.toHaveBeenCalled(); + }); + + test("keeps local admission abort distinct from durable Flue abort", async () => { + const send = vi.fn(async () => { + throw new DOMException("Local admission cancelled", "AbortError"); + }); + const abort = vi.fn(async () => ({ aborted: true })); + const harness = createAdmissionOutcomeHarness({ abort, send }); + + harness.emitCompletedTranscript("input-item-aborted"); + + await vi.waitFor(() => + expect(harness.events).toContainEqual({ + code: "admission-aborted", + failure: { kind: "aborted" }, + message: "The local chat submission was cancelled.", + type: "error", + }), + ); + expect(send).toHaveBeenCalledOnce(); + expect(abort).not.toHaveBeenCalled(); }); }); From 161b78968520cb6bb217c2946fb72dc4bd89713c Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Fri, 4 Sep 2026 02:23:21 +0200 Subject: [PATCH 16/52] Complete acknowledged Voice turn handoff Amp-Thread-ID: https://ampcode.com/threads/T-01a06941-3ee7-7482-a82e-8a16c7141785 Co-authored-by: Amp --- .../main/app/voice-interview/realtime-brunch-bridge.test.ts | 6 +----- .../src/main/app/voice-interview/realtime-brunch-bridge.ts | 4 ++++ .../main/app/voice-interview/voice-turn-controller.test.ts | 2 ++ .../src/main/app/voice-interview/voice-turn-controller.ts | 2 ++ 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.test.ts index e19273418af..2f7ef6a851d 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.test.ts @@ -246,11 +246,7 @@ describe("RealtimeBrunchBridge", () => { ); expect(harness.submitInterviewAnswer).not.toHaveBeenCalled(); - harness.emit({ - connectionEpoch: 3, - responseId: "response-request", - type: "output-interrupted", - }); + harness.bridge.completeTurnHandoff(); harness.emit({ connectionEpoch: 3, itemId: "item-after-handoff", diff --git a/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.ts b/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.ts index 6e9f1787aca..ab9fc825272 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.ts @@ -225,6 +225,10 @@ export class RealtimeBrunchBridge { } } + public completeTurnHandoff(): void { + this.#outputActive = false; + } + public start(connectionEpoch: number): void { ++this.#generation; this.#activeSubmission?.abortController.abort(); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts index 3e663848da5..f30eb2e327e 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts @@ -32,6 +32,7 @@ const createHarness = () => { }; const bridge = { cancelPendingSpeech: vi.fn(), + completeTurnHandoff: vi.fn(), start: vi.fn(), stop: vi.fn(), subscribe: vi.fn((listener: (event: RealtimeBrunchBridgeEvent) => void) => { @@ -362,6 +363,7 @@ describe("VoiceTurnController", () => { expect(harness.submitText).not.toHaveBeenCalled(); await harness.controller.takeTurn(); + expect(harness.bridge.completeTurnHandoff).toHaveBeenCalledOnce(); harness.emitSession({ connectionEpoch: 1, itemId: "item-after-handoff", diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts index 6040032869e..7cd3513df58 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts @@ -73,6 +73,7 @@ interface RealtimeSession { interface RealtimeBridge { cancelPendingSpeech(): void; + completeTurnHandoff(): void; start(connectionEpoch: number): void; stop(): void; subscribe(listener: (event: RealtimeBrunchBridgeEvent) => void): () => void; @@ -481,6 +482,7 @@ export class VoiceTurnController { this.#activeSpeechOutputEnded = false; this.#activeSpeechResponseId = null; this.#activeSpeechResponseTerminal = false; + this.#bridge.completeTurnHandoff(); this.#session.setMicrophoneEnabled(this.#snapshot.microphoneEnabled); this.#update({ output: "interrupted" }); }) From 8114136ee4d5de85100512e91bbe14fbab68b0a1 Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Fri, 4 Sep 2026 02:26:04 +0200 Subject: [PATCH 17/52] Record Voice race verification --- apps/petrinaut-website/README.md | 14 +++++---- libs/@hashintel/brunch-agent/MISSION.md | 15 +++++++-- .../donor-behavior-matrix.md | 31 ++++++++++++++++--- .../provenance-blocker.md | 2 +- 4 files changed, 48 insertions(+), 14 deletions(-) diff --git a/apps/petrinaut-website/README.md b/apps/petrinaut-website/README.md index aad11268d79..874201a3622 100644 --- a/apps/petrinaut-website/README.md +++ b/apps/petrinaut-website/README.md @@ -95,12 +95,14 @@ delivery key so a replayed request converges on the existing admission instead of creating another turn. If admission cannot be confirmed, the UI reports the ambiguity and does not retry automatically. **Stop** requests a durable Brunch abort before the panel cancels its local response stream. Local playback -cancellation remains separate and does not alter canonical history. Reopening -the same net restores its observed Flue conversation without resubmitting a -turn or replaying settled audio. Voice-origin client-tool results retain their -markers through Flue history. Direct spoken user turns remain canonical text -after reopening, but Flue 2.0.3 does not yet expose the caller delivery metadata -needed to restore their Voice chip. +cancellation remains separate and does not alter canonical history. Canonical +Flue history is the source used when the same net is reopened. Automated +projection coverage does not resubmit turns or replay settled audio, but the +required real hard-reload witness remains blocked by the parent hydration fix, +so reload parity is not yet claimed for this preview. Voice-origin client-tool +results retain their markers in Flue history. Direct spoken user turns remain +canonical text, but Flue 2.0.3 does not yet expose the caller delivery metadata +needed to restore their Voice chip after reopening. An active session stays at the end of the transcript. Its compact divider shows a waveform and **Connecting**, **Listening**, **Speaking**, **Paused**, or a diff --git a/libs/@hashintel/brunch-agent/MISSION.md b/libs/@hashintel/brunch-agent/MISSION.md index bab04fded50..35f29721717 100644 --- a/libs/@hashintel/brunch-agent/MISSION.md +++ b/libs/@hashintel/brunch-agent/MISSION.md @@ -15,7 +15,8 @@ Sections independent of canonical hydration may proceed. The hard-reload witness Completed-transcript authority, half-duplex ownership, admission idempotency, the cancellation barrier, and exact full-response replay have focused regression -coverage. **Repeat question** remains blocked: canonical Brunch speech does not +coverage, including the interval between a canonical speech request and output +start. **Repeat question** remains blocked: canonical Brunch speech does not carry a deterministic question marker, so treating the final text segment as a question would replay ordinary prose under a false label. Proof item 5 is complete only for the supported client-tool-result path: Flue signals persist each @@ -27,6 +28,16 @@ browser-correlation implementation would have violated the explicit second-durable-store stop condition. The real witness remains blocked by the parent hydration defect. +The 2026-09-04 corrective verification covers the current 44-file successor +diff against #9528: the four race suites pass 94/94 tests, the filtered +production admission-outcome cases pass 3/3 tests, and the requested Turbo run +passes 30/30 tasks, including 243/243 website tests. Architecture validation +passes with 62 layers, 297 edges, 614 files, 63 generated pages, and 31 authored +pages. `git diff --check` and formatting of all formatter-owned successor files +pass. The root formatter remains red only outside this branch's diff: two +parent-owned #9528 files and an unrelated untracked `.cursor` plan. The exact +commands and dispositions are retained in the [donor matrix](docs/evidence/implementations/mission-5-voice-safety-parity/donor-behavior-matrix.md#corrective-verification). + The pinned donor-behavior decision record is the [FE-1580 donor matrix](docs/evidence/implementations/mission-5-voice-safety-parity/donor-behavior-matrix.md). Donor PRs are read-only evidence at their named heads; semantic reimplementation is required, never merge or cherry-pick. ## Imperative @@ -83,7 +94,7 @@ This mission closes the Voice safety and UX-parity stratum on the parent's route **Demo script:** run `yarn dev:brunch` and select the Brunch preview. Speak one answer and see exactly one matching user turn. While Brunch is speaking, confirm the microphone remains closed, choose **Your turn**, wait for the handoff, and speak again. Use the playback menu to read the full response exactly; **Repeat question** remains unavailable until Brunch provides a deterministic canonical question marker. Start another turn, press durable **Stop** before settlement, and see a stopped turn rather than a Voice error. After the parent hydration blocker is fixed, hard-reload the settled conversation and confirm the canonical turn remains without resubmission or replay; direct-user Voice-chip restoration additionally waits on the Flue projection seam. -**Previously impossible:** model-generated function arguments rather than completed audio transcription could become the answer; assistant playback could create a false user turn; cancellation could reopen capture before the provider settled; replay controls and multi-origin client-tool Voice attribution were incomplete. +**Previously impossible:** model-generated function arguments rather than completed audio transcription could become the answer; an accepted transcript could complete after canonical speech was requested but before output started; assistant playback could create a false user turn; cancellation could reopen capture before the provider settled; replay controls and multi-origin client-tool Voice attribution were incomplete. **Completion:** the implemented portions close when their tests and focused checks pass. **Repeat question** needs the Brunch-owned marker recorded below, and direct-user provenance needs the Flue re-entry seam recorded below. Mission acceptance additionally requires the real microphone, handoff, Stop, hard-reload, and same-origin route witness after the parent hydration fix. Mocked or server-only proof cannot substitute for that witness. diff --git a/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/donor-behavior-matrix.md b/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/donor-behavior-matrix.md index 6b179e3f849..5d4d4499d09 100644 --- a/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/donor-behavior-matrix.md +++ b/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/donor-behavior-matrix.md @@ -46,12 +46,12 @@ The owner selected half-duplex turn ownership on 2026-09-03: assistant output ow | Adopted behavior | Replacement implementation | Regression test | Production integration proof | Status | | --- | --- | --- | --- | --- | | One path-B Flue admission route | `local-storage-demo-app.tsx`, `brunch-panel-transport.ts`, transport `src/index.ts` | `brunch-panel-transport.test.ts`, `chat-transport.test.ts` | `voice-preview.integration.test.ts` crosses completed transcript → panel submission → Flue transport → canonical speech | **Implemented**; parent defects remain downstack | -| Stable admission identity and typed outcomes | transport `src/index.ts`, `brunch-panel-transport.ts`, `submitVoiceInputWithAdmission`, `realtime-brunch-bridge.ts` | transport admission cases; bridge/controller cases for rejected, conflict, ambiguous, and local abort | production preview 500 case observes one `send()` and visible ambiguous Voice state | **Implemented** | +| Stable admission identity and typed outcomes | transport `src/index.ts`, `brunch-panel-transport.ts`, `submitVoiceInputWithAdmission`, `realtime-brunch-bridge.ts` | transport admission cases; bridge/controller cases for rejected, conflict, ambiguous, and local abort | production preview carries 409 conflict, 500 ambiguity, and local abort through transport → tracker → `submitVoiceInputWithAdmission` → bridge; each observes one `send()`, and local abort never invokes durable `FlueClient.abort()` | **Implemented** | | Exact canonical TTS queue and full-response replay | `openai-realtime-session.ts`, `voice-turn-controller.ts`, Petrinaut playback menu | session queue/cancellation cases; controller exact-segment and terminal-gating cases; panel action tests | real host registration exposes `readFullResponse`; panel forwards it through `voiceSessionStore` | **Implemented** | | Repeat-question replay | none: no supported question identity exists | controller and host-registration negative cases | Brunch Voice host omits `repeatQuestion`; panel leaves the generic item disabled | **Blocked**, not mislabeled as final-segment replay | | Disabled Realtime generation/tools | `openai-voice-policy.ts`, `openai-realtime-session.ts` | policy/session tests reject tools and function arguments | controlled production preview negotiates the server policy and emits only canonical speech | **Implemented** | -| Completed-transcript authority | `openai-realtime-session.ts`, `realtime-brunch-bridge.ts` | missing/stale/reordered boundary, keyed identity, normalization, duplicate/failure/limit, and late-output cases | controlled production preview submits one completed transcript through path B | **Implemented**; provider-valid boundaryless commits are intentionally rejected by mission policy | -| Half-duplex acknowledged handoff | `openai-realtime-session.ts`, `voice-turn-controller.ts`, Voice public store/dock | input/output clear acknowledgement, targeted response terminal, latest mute, stale/pre-handoff rejection | panel registration tests exercise **Your turn**; preview integration proves fresh post-handoff capture | **Implemented** | +| Completed-transcript authority | `openai-realtime-session.ts`, `realtime-brunch-bridge.ts` | missing/stale/reordered boundary, keyed identity, normalization, duplicate/failure/limit, canonical-request-before-output, and late-output cases | controlled production preview proves a pre-request item cannot call Flue `send()` before output starts and only fresh post-handoff input submits through path B | **Implemented**; provider-valid boundaryless commits are intentionally rejected by mission policy | +| Half-duplex acknowledged handoff | `openai-realtime-session.ts`, `voice-turn-controller.ts`, Voice public store/dock | canonical-request invalidation, input/output clear acknowledgement, targeted response terminal, latest mute, stale/pre-handoff rejection | panel registration tests exercise **Your turn**; preview integration proves the microphone closes before `response.create` and fresh post-handoff capture submits once | **Implemented** | | Durable Stop distinct from local cancellation | app `requestFlueStop`, panel `stopComposer`, session `cancelOutput` | panel durable-before-local Stop, controller/session local-cancel cases, app host Stop case | configured Brunch app invokes `FlueClient.abort()`, observes an aborted settlement, and does not invoke local playback cancellation | **Implemented**; parent-owned Stop races excluded | | Multi-origin Voice client-tool provenance | panel `addMappedToolOutput`, transport client-tool-result signal/projection, `useFlueChatHistory` | sibling partial-failure, persisted-signal projection, hydration/reopen cases | configured app consumes the public Flue observation and restores every `voiceToolCallId` | **Implemented for client-tool results**; direct-user marker **blocked** | | No live `brunch_ask` | Brunch app registers `interactiveTools: []`; canonical speech selector ignores the ask name | canonical-speech negative case and configured-app registration negative case | captured production Brunch `PetrinautAiAssistant` has no ask tool while retaining Flue Stop wiring | **Implemented exclusion** | @@ -60,11 +60,32 @@ The owner selected half-duplex turn ownership on 2026-09-03: assistant output ow | Area | Required closing evidence | Current state | | --- | --- | --- | -| Transcript authority | Transplanted-first session, bridge, controller, and integration regressions pass on path B. | Implemented. Matching current-turn `speech_started`, stale/reordered boundaries, late-output invalidation, exact bridge normalization, and unchanged panel payload are covered. | -| Admission idempotency | Typed and Voice logical replays converge on one `submissionId`; conflict metadata is narrowed safely; ambiguous outcome does not retry. | Implemented. Transport tests cover stable typed/Voice keys, deduplicated receipts, 409 conflicts, and non-retried ambiguity. | +| Transcript authority | Transplanted-first session, bridge, controller, and integration regressions pass on path B. | Implemented. Matching current-turn `speech_started`, stale/reordered boundaries, canonical-speech-request and late-output invalidation, provisional UI clearing, exact bridge normalization, and unchanged panel payload are covered. | +| Admission idempotency | Typed and Voice logical replays converge on one `submissionId`; conflict metadata is narrowed safely; ambiguous outcome does not retry. | Implemented. Transport tests cover stable typed/Voice keys, deduplicated receipts, 409 conflicts, and non-retried ambiguity; production-path integration preserves the original conflict `submissionId` and keeps local admission abort distinct from durable abort. | | Cancellation barrier | Buffer acknowledgements and targeted response terminals settle before capture; stale/pre-handoff audio cannot submit; latest mute choice wins. | Implemented. Session/controller races cover the barrier and mute preference; panel registration and configured-app Stop cases cover the production host seams. | | Canonical replay | Exact segment queue and playback menu pass availability/race tests without a simplifier. | Full-response replay is implemented with exact segments and terminal gating. **Repeat question** is blocked because no supported deterministic question marker exists; the production action must remain unavailable rather than replaying the final prose segment. | | Durable provenance | Multiple origins and partial failure survive projection, hydration, and reopen without user-text encoding. | Partially implemented for assistant client-tool results through persisted Flue signals; multiple sibling origins survive projection and partial failure. Direct spoken user attribution is blocked because Flue 2.0.3 snapshots do not expose caller idempotency or user-message metadata. The rejected browser store would violate mission authority. | | Dormant ask | No mounted Voice ask capability remains, or the parent commit that removed it is recorded. | Implemented exclusion. Canonical speech ignores `brunch_ask`, and a configured-app registration test proves the production Brunch assistant supplies no ask tool. Dormant source remains unmounted. | | Real witness | Microphone, handoff, unsettled Stop, reload, canonical snapshot, settlement, and same-origin absolute-`streamUrl` artifacts are retained with hashes. | Blocked by the parent's hydration overwrite. | | Donor retirement | Replacement accepted and each donor owner explicitly approves closure. | Deferred; no donor or stakeholder issue may be closed now. | + +## Corrective verification + +Fresh local checks on 2026-09-04 cover the 44-file successor diff against the +verified #9528 head `036fb06ee83e840fd7a87dd1bc758a2dd49ddd4c`: + +| Command | Result | +| --- | --- | +| `yarn workspace @apps/petrinaut-website test:unit src/main/app/voice-interview/openai-realtime-session.test.ts src/main/app/voice-interview/realtime-brunch-bridge.test.ts src/main/app/voice-interview/voice-turn-controller.test.ts src/main/app/voice-interview/voice-preview.integration.test.ts` | Exit 0; 4 files and 94/94 tests passed. This includes the request-before-output race at session, bridge, controller, and production integration layers. | +| `yarn workspace @apps/petrinaut-website test:unit src/main/app/voice-interview/voice-preview.integration.test.ts -t 'ambiguous Flue admission\|conflicting submission\|local admission abort'` | Exit 0; 3/3 selected tests passed and 2 unrelated tests were filtered. Conflict retains the original `submissionId`; local abort remains distinct from durable abort; every path calls `send()` once. | +| `mise exec -- yarn exec turbo run lint:tsc lint:eslint test:unit build --filter @apps/brunch-agent --filter @apps/petrinaut-website --filter @hashintel/petrinaut --filter @hashintel/brunch-agent-transport-aisdk` | Exit 0; 30/30 tasks passed, including 31 website test files and 243/243 website tests. Website ESLint retains one non-failing warning at `voice-interview-control.tsx:480`. | +| `yarn workspace @local/petrinaut-arch-docs lint:arch-docs` | Exit 0; 62 layers, 297 edges, 614 files, 63 generated pages, and 31 authored pages. | +| `yarn exec oxfmt --check $(cat /tmp/fe1580-files)` over the 44 child-owned paths | Exit 0; all 40 formatter-owned files passed. The remaining four Brunch Markdown records are intentionally excluded by `oxfmt.config.ts` and pass repository whitespace validation. | +| `git diff --check` | Exit 0. | +| `yarn lint:format` | Exit 1 outside the successor diff only: parent-owned `apps/brunch-agent/src/conversation/identity.ts` and `apps/brunch-agent/test/conversation-identity.test.ts`, plus unrelated untracked `.cursor/plans/fe-1574_mission_recut_6f23f7cd.plan.md`. They were not modified. | + +No production Voice source under `apps/petrinaut-website/src/main/app/voice-interview` +calls `FlueClient.send()`; its only `.send()` is the OpenAI Realtime data +channel. Production Brunch registration supplies `interactiveTools: []`, and +canonical speech has no `brunch_ask` recognition. The dormant ask source remains +unmounted. diff --git a/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/provenance-blocker.md b/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/provenance-blocker.md index 27668dd37fb..7e9c3e6ea30 100644 --- a/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/provenance-blocker.md +++ b/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/provenance-blocker.md @@ -33,7 +33,7 @@ representation of the existing path-B turn. Direct spoken user turns still render with a Voice chip while their AI SDK message metadata is live. Their canonical text and submission survive Flue hydration, but the Voice chip cannot be reconstructed after reopen. This portion -of section 6 is blocked rather than reported as complete. +of proof item 5 is blocked rather than reported as complete. Re-enter only when Flue projects caller metadata or the caller idempotency key onto the canonical direct-user message, or when the product owner explicitly From b176777e8fb0c4fae1cc535bf7d6fae773d6640b Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Fri, 4 Sep 2026 09:34:47 +0200 Subject: [PATCH 18/52] Authorize exact Voice question markers Amp-Thread-ID: https://ampcode.com/threads/T-01a06941-3ee7-7482-a82e-8a16c7141785 Co-authored-by: Amp --- libs/@hashintel/brunch-agent/MISSION.md | 32 +++--- ...rker-and-provenance-decision-2026-09-04.md | 97 +++++++++++++++++++ 2 files changed, 117 insertions(+), 12 deletions(-) create mode 100644 libs/@hashintel/brunch-agent/docs/evidence/design/mission-5-question-marker-and-provenance-decision-2026-09-04.md diff --git a/libs/@hashintel/brunch-agent/MISSION.md b/libs/@hashintel/brunch-agent/MISSION.md index 35f29721717..3993f775191 100644 --- a/libs/@hashintel/brunch-agent/MISSION.md +++ b/libs/@hashintel/brunch-agent/MISSION.md @@ -2,10 +2,19 @@ ## Status -**Live as of 2026-09-03** for [FE-1580](https://linear.app/hash/issue/FE-1580/harden-voice-safety-and-ux-on-the-unified-flue-route) on `kostandin/fe-1580-harden-voice-safety-and-ux-on-the-unified-flue-route`, stacked directly on [PR #9528](https://github.com/hashintel/hash/pull/9528) at the GitHub-verified head `036fb06ee83e840fd7a87dd1bc758a2dd49ddd4c`. This file is the sole execution authority for the successor branch. The parent remains the authority for its own admission/Stop races, stream cancellation, hydration overwrite, client-tool classification, response/submission correlation, and PR/CI defects; this branch must restack onto parent fixes rather than repair them. +**Live as of 2026-09-04** for [FE-1580](https://linear.app/hash/issue/FE-1580/harden-voice-safety-and-ux-on-the-unified-flue-route) on `kostandin/fe-1580-harden-voice-safety-and-ux-on-the-unified-flue-route`, stacked directly on [PR #9528](https://github.com/hashintel/hash/pull/9528) at the GitHub-verified head `036fb06ee83e840fd7a87dd1bc758a2dd49ddd4c`. This file is the sole execution authority for the successor branch. The parent remains the authority for its own admission/Stop races, stream cancellation, hydration overwrite, client-tool classification, response/submission correlation, and PR/CI defects; this branch must restack onto parent fixes rather than repair them. The owner selected **half-duplex turn ownership** on 2026-09-03. While canonical assistant audio is pending or playing, the microphone is closed. Ownership transfers away from input as soon as canonical speech is requested, before `response.create` is sent: every accepted unfinished input item becomes stale and provisional transcript state is cleared. The explicit **Your turn** action cancels output, waits for provider acknowledgement and response settlement, and only then opens a fresh input turn. Automatic duplex barge-in is rejected because assistant playback can become a false user turn. +On 2026-09-04, the owner approved a non-interactive Brunch-owned question +marker for exact **Repeat question** replay. The marker is a server tool plus a +durable client data part: it identifies exact assistant-authored text but never +suspends for an answer, mounts `brunch_ask`, or creates a second Voice submission +path. For direct-user Voice provenance, the owner selected an upstream Flue +user-metadata contract rather than a local runtime patch or correlated sidecar +signal. The [decision record](docs/evidence/design/mission-5-question-marker-and-provenance-decision-2026-09-04.md) +defines the accepted and rejected mechanics. + ### Turn-ownership decision 1. **Adopted — half-duplex explicit handoff.** It gives assistant playback exclusive ownership, makes cancellation settlement a visible boundary, and guarantees fresh post-handoff capture. The product cost is one extra **Your turn** action and barrier latency when the user interrupts. @@ -16,9 +25,9 @@ Sections independent of canonical hydration may proceed. The hard-reload witness Completed-transcript authority, half-duplex ownership, admission idempotency, the cancellation barrier, and exact full-response replay have focused regression coverage, including the interval between a canonical speech request and output -start. **Repeat question** remains blocked: canonical Brunch speech does not -carry a deterministic question marker, so treating the final text segment as a -question would replay ordinary prose under a false label. Proof item 5 is complete +start. **Repeat question** is authorized but not yet implemented: only the +approved durable Brunch marker may enable it, and the final text segment remains +invalid question authority. Proof item 5 is complete only for the supported client-tool-result path: Flue signals persist each Voice-origin tool-call id beside its output, and canonical projection reconstructs multiple surviving origins. Direct spoken user attribution is @@ -90,7 +99,7 @@ This mission closes the Voice safety and UX-parity stratum on the parent's route ### Product-manager litmus -**Release note:** Voice now submits only what the microphone actually transcribed, waits for a safe **Your turn** handoff before listening over Brunch, and can replay the exact full response. Client-tool Voice origins survive canonical reopen; **Repeat question** waits on a supported canonical question marker, and restoring the Voice chip on direct spoken user messages remains blocked on Flue projection support. +**Release note:** Voice now submits only what the microphone actually transcribed, waits for a safe **Your turn** handoff before listening over Brunch, and can replay the exact full response. Client-tool Voice origins survive canonical reopen. **Repeat question** will replay only exact text carrying the approved Brunch-owned marker; restoring the Voice chip on direct spoken user messages remains blocked on an upstream Flue user-metadata contract. **Demo script:** run `yarn dev:brunch` and select the Brunch preview. Speak one answer and see exactly one matching user turn. While Brunch is speaking, confirm the microphone remains closed, choose **Your turn**, wait for the handoff, and speak again. Use the playback menu to read the full response exactly; **Repeat question** remains unavailable until Brunch provides a deterministic canonical question marker. Start another turn, press durable **Stop** before settlement, and see a stopped turn rather than a Voice error. After the parent hydration blocker is fixed, hard-reload the settled conversation and confirm the canonical turn remains without resubmission or replay; direct-user Voice-chip restoration additionally waits on the Flue projection seam. @@ -101,7 +110,7 @@ This mission closes the Voice safety and UX-parity stratum on the parent's route 1. **Completed-transcript authority and half-duplex ownership.** Realtime session configuration has no tools, no model-created semantic-VAD response, and no automatic interruption policy. Only a unique completed transcript can reach the shared panel submission path. Duplicate, empty, failed, unavailable, stale, canonical-speech-overlapping/pre-handoff, playback-overlapping, and over-limit transcripts do not submit and produce the specified passive or recoverable notice. Before sending `response.create`, a canonical speech request invalidates every unfinished accepted item, clears bridge/controller transcript state and provisional UI, and closes the microphone; a completion in the interval before output starts cannot submit or regain authority. **Your turn** opens only a post-barrier input turn. Oracle: transplanted-first cases in `openai-realtime-session.test.ts`, `realtime-brunch-bridge.test.ts`, `voice-turn-controller.test.ts`, `voice-interview-control.test.tsx`, and `voice-preview.integration.test.ts`. 2. **Idempotent admission.** Typed turns derive a stable key from the AI SDK message id; Voice turns derive it from connection epoch, item id, and content index. A repeated same-payload key converges on the original receipt, including `deduplicated: true`; a 409 `submission_conflict` surfaces the original `submissionId` without admitting another turn. An admission whose outcome cannot be established stays visibly ambiguous and is never automatically retried. Oracle: transport unit tests plus Voice integration tests asserting one admitted submission rather than one `send()` invocation. 3. **Acknowledged cancellation barrier.** `cancelOutput()` resolves only after input/output buffer clears, matching provider acknowledgements, and all targeted response terminal events. The latest mute preference wins while it settles. Audio captured before the handoff cannot submit afterward. Durable Stop remains a stopped Flue turn rather than a Voice failure. Oracle: donor-adapted session/controller race tests and the unsettled-Stop integration case. -4. **Canonical full-response replay; question replay blocked.** `canReadFullResponse`, `readFullResponse()`, and the playback menu retain and enqueue all exact canonical segments in order without a simplifier. Full-response replay enables only after the matching response terminal and output completion and remains disabled during submission, capture, cancellation, pause, and errors. `canRepeatQuestion` remains false and the production Voice host does not offer `repeatQuestion()` because neither canonical segments nor Flue correlation expose a deterministic Brunch-owned question marker. The final text segment is not authority for question identity. Re-entry requires such a marker without restoring `brunch_ask` or structured questions. Oracle: canonical speech, controller, panel, and integration tests comparing segment identity and text, plus negative production-host coverage for the blocked question action. +4. **Canonical full-response and exact question replay.** `canReadFullResponse`, `readFullResponse()`, and the playback menu retain and enqueue all exact canonical text segments in order without a simplifier. A non-interactive `brunch_mark_question` server tool writes a durable `data-brunch-question` marker containing exact question text and tool-call identity. The selector accepts it only when the same finalized assistant message contains that exact text; there is no final-segment or punctuation fallback. `repeatQuestion()` queues only the accepted marked segment. Both replay actions enable only after the correlated Brunch response settles, matching Realtime output completes, and input is idle, and remain disabled during submission, capture, cancellation, pause, and errors. The marker never accepts an answer or changes Voice path B. Oracle: core tool, live transport, snapshot projection, canonical speech, controller, panel-host, and production-preview tests comparing marker identity and exact queued text while proving `brunch_ask` remains unmounted. 5. **Durable Voice provenance.** An assistant message may retain multiple `voiceToolCallIds`; one failed sibling origin does not erase successful origins. Persisted Flue client-tool-result signals support deterministic reconstruction after hydration and reopen. Direct spoken user messages remain Voice-attributed only while live because the canonical snapshot omits their caller origin. Re-entry requires a supported Flue user-message metadata/idempotency projection; browser storage and user-text encoding are rejected. Oracle: snapshot projection and panel partial-failure tests for supported origins, plus the [blocker record](docs/evidence/implementations/mission-5-voice-safety-parity/provenance-blocker.md). 6. **Dormant ask removal.** If still present after restacking, the website does not register `brunchAskInteractiveTool` for Voice and canonical speech does not recognize `"brunch-ask"`. No spoken ask answer can enter a wait state the transport cannot resume. Oracle: registration/canonical-speech negative tests and a repository search showing no mounted Voice `brunch_ask` surface. 7. **Real witness and same-origin route.** After the parent hydration fix lands, a human performs one microphone turn, explicit interruption/handoff, durable Stop on an unsettled turn, and hard reload of a settled turn. The retained network route summary proves the absolute Flue `streamUrl` remains on the same-origin proxy. Oracle: `witness.md`, sanitized `voice-events.jsonl`, `network-routes.json`, canonical `flue-snapshot.json`, `settlements.json`, commit manifest, and hashes under `docs/evidence/implementations/mission-5-voice-safety-parity/`. @@ -118,7 +127,7 @@ This mission closes the Voice safety and UX-parity stratum on the parent's route - Preserve every surviving Voice origin independently. Provenance must use supported Flue data or deterministic durable correlation; never encode it in visible user text. - Do not fix the parent's admission/Stop races, stream cancellation, hydration overwrite, client-tool classification, response/submission correlation, CI, title, or body. Restack onto Lu's fixes; report any blocker. - Keep local playback cancellation, local observation cancellation, HTTP request cancellation, and durable `abort()` distinguishable in code, UI, tests, and evidence. -- No simplifier, structured questions, live `brunch_ask`, Petri-net generation/mutation, FE-1575 workpiece work, production identity, CORS/remote deployment, or panel `useChat` removal. +- No simplifier, interactive or suspending structured questions, live `brunch_ask`, Petri-net generation/mutation, FE-1575 workpiece work, production identity, CORS/remote deployment, or panel `useChat` removal. The approved non-interactive question marker annotates existing assistant prose only; it is not an answer path or affordance. ### Expected touched paths @@ -140,22 +149,21 @@ This mission closes the Voice safety and UX-parity stratum on the parent's route - **Parent movement.** Lu owns #9528 and may push more commits. Before each implementation phase, compare the GitHub head and restack this branch; an observed parent change is adopted only through restack, never copied into this branch. - **Conflict normalization.** The installed SDK exposes the 409 contract through `FlueApiError.body: unknown`. Narrow only the documented envelope needed to recover `error.meta.submissionId`; do not create a general error protocol or infer success from prose. - **Dormant `brunch_ask`.** Remove or gate only the parent surfaces that remain after the next restack. If Lu has already removed them, record the parent commit and make no duplicate change. -- **Repeat-question identity.** Canonical assistant segments expose exact text and segment identity but no Brunch-owned question marker. Keep the action unavailable rather than treating the final segment as a question. Re-enter only through a deterministic supported marker or correlation rule that does not restore `brunch_ask` or structured questions. -- **Direct-user Voice provenance.** Flue 2.0.3's canonical user message exposes a generated `submissionId` but not caller metadata or `idempotencyKey`. Re-enter only when a supported durable correlation seam exists or the owner explicitly changes the representation; do not add browser persistence or encode origin in user text. +- **Question-marker compliance.** The owner selected `brunch_mark_question` plus a durable client data part. The remaining implementation uncertainty is whether the model follows the instruction on every eligible question. Missing or unmatched markers must degrade by leaving **Repeat question** disabled; they never justify inference from final prose. Product proof covers structural correctness, not a universal model-compliance rate. +- **Direct-user Voice provenance.** Flue 2.0.3 and current upstream `main` expose a generated `submissionId` but not caller metadata or `idempotencyKey` on canonical user messages. The owner selected an upstream Flue user-metadata contract. Keep this leaf blocked until a released seam can be adopted; do not patch Flue locally, add a provenance signal admission, add sidecar persistence, or encode origin in user content. - **Hard-reload witness.** Hold proof leaf 7 until the parent's hydration overwrite is fixed. Unit/integration tests for this branch may cover deterministic projection, but they cannot substitute for the blocked real witness. ## Stop or reorient Stop and report if the work would require direct Voice `send()`, a second transcript or conversation authority, hand-rolled stream offsets/recovery, automatic retry after ambiguous admission, canonical text rewriting, a live structured-question path, or any excluded parent fix. -Stop if half-duplex handoff cannot guarantee that pre-handoff audio is rejected and post-barrier audio is fresh, or if provider acknowledgements cannot bound `cancelOutput()` without inventing events. The provenance stop condition has fired for direct spoken user turns: the browser-store implementation was removed and the unsupported leaf is recorded as blocked. The Repeat-question stop condition has also fired: do not relabel the final canonical segment as a question. Stop if full-response replay can enable before both matching terminal conditions, or if local cancellation invokes durable abort. +Stop if half-duplex handoff cannot guarantee that pre-handoff audio is rejected and post-barrier audio is fresh, or if provider acknowledgements cannot bound `cancelOutput()` without inventing events. The provenance stop condition has fired for direct spoken user turns: the browser-store implementation was removed and the unsupported leaf is recorded as blocked pending upstream Flue support. For **Repeat question**, stop rather than infer question identity when the approved marker is absent or does not exactly match finalized assistant text. Stop if either replay action can enable before both matching terminal conditions, or if local cancellation invokes durable abort. Do not manufacture the hard-reload witness while the parent hydration defect remains. Retain the blocker and wait for a new parent head. ## Deferred - The real witness and same-origin absolute-`streamUrl` observation wait on the parent hydration fix; once unblocked, they are part of this mission rather than a successor. -- **Repeat question** waits on a deterministic, supported Brunch-owned marker or correlation rule that distinguishes a question from ordinary canonical prose without restoring `brunch_ask` or structured questions. -- Direct-user Voice attribution after canonical hydration waits on a supported Flue caller-metadata or idempotency projection seam, or an explicit owner decision to change the durable input representation. +- Direct-user Voice attribution after canonical hydration waits on a released upstream Flue caller-metadata projection seam. The owner rejected a local Flue patch and correlated signal sidecar for this mission. - Donor retirement waits until this replacement is accepted and each donor owner explicitly approves closure. Do not close #9496, #9500, #9507, or #9512 as an implementation side effect, and never close stakeholder-owned H-6763. - Response preparation/simplification, structured questions, Petri-net work, FE-1575, production identity, CORS/remote deployment, and panel migration away from `useChat` remain in their existing owners or the future mission spine. diff --git a/libs/@hashintel/brunch-agent/docs/evidence/design/mission-5-question-marker-and-provenance-decision-2026-09-04.md b/libs/@hashintel/brunch-agent/docs/evidence/design/mission-5-question-marker-and-provenance-decision-2026-09-04.md new file mode 100644 index 00000000000..f664d2175f4 --- /dev/null +++ b/libs/@hashintel/brunch-agent/docs/evidence/design/mission-5-question-marker-and-provenance-decision-2026-09-04.md @@ -0,0 +1,97 @@ +# Mission 5 question replay and direct-user provenance decision + +## Decision + +The owner approved two changes to the live FE-1580 authority on 2026-09-04: + +1. Brunch may expose a non-interactive, model-facing question-marker tool. The + marker identifies exact assistant-authored question text for Voice replay, + but it never suspends a response, renders an answer affordance, accepts an + answer, or changes Voice path B. +2. Direct-user Voice provenance must wait for an upstream Flue contract that + durably projects caller metadata on a canonical `kind: "user"` message. This + branch must not patch Flue locally or approximate provenance with a second + signal admission, browser storage, or encoded user text. + +## Exact question marker + +Brunch core owns a `brunch_mark_question` server tool and a +`data-brunch-question` client marker. Before asking the user a direct question, +the model calls the tool with the exact question text. The tool writes a durable +data part containing that text and its Flue `toolCallId`, then returns a small +acknowledgement. It does not terminate the response. Brunch instructions require +the same exact text to appear in ordinary assistant prose after the tool call. + +The browser transport hides the marker tool's implementation call while +retaining the data part. This keeps an internal annotation out of Petrinaut's +tool-activity UI without creating another conversation representation. Both the +live stream and canonical snapshot projection apply the same hidden-tool rule. + +Canonical speech accepts a question marker only when all of these facts hold: + +- the marker has a non-empty string question and non-empty `toolCallId`; +- it belongs to an assistant message; +- the same assistant message contains the exact marked string in finalized + ordinary text; and +- the marker data part is complete and canonical, not provisional Voice state. + +Malformed, unmatched, stale, or absent markers do not enable **Repeat +question**. The final text segment and punctuation are never used as fallback +question authority. The selected question segment derives stable identity from +the assistant message id, marker tool-call id, and exact-text hash. Full-response +speech remains the ordered ordinary text segments and is not rewritten or +duplicated by the marker. + +The Voice controller carries the selected question separately from the full +response. **Repeat question** reuses the existing exact canonical queue and the +same settlement, output-completion, idle-input, submission, cancellation, +capture, pause, and error gates as **Read full response**. The control remains +disabled when the settled response has no matching marker. + +## Production proof + +Tests are written and observed failing before implementation. Closing evidence +must cover: + +- Brunch's built Flue agent mounting `brunch_mark_question` while continuing to + omit `brunch_ask`; +- a real server-tool call writing a durable `data-brunch-question` part; +- live transport and snapshot projection hiding the implementation tool while + retaining the marker; +- canonical selection rejecting malformed and unmatched markers and preserving + exact text and stable identity for a valid marker; +- the production Voice host registering `repeatQuestion` and the panel invoking + it only when `canRepeatQuestion` is true; and +- controller and preview integration proving exact question-only replay after + correlated Brunch settlement and matching Realtime output completion, with + every existing replay exclusion still enforced. + +## Direct-user Voice provenance + +Flue 2.0.3 and current upstream `main` accept only `body` and image +`attachments` on `kind: "user"`. The caller's idempotency key is irreversibly +hashed into `submissionId`; canonical snapshots do not expose that key or +caller-authored user metadata. Agent-authored response metadata cannot annotate +the canonical user message. + +The accepted route is an upstream Flue extension that admits caller metadata on +the user delivery, persists it atomically with the canonical user record, and +projects it on live and historical user messages. FE-1580 can adopt that seam +only after a released dependency is available and the branch is explicitly +authorized to upgrade. The closing oracle is a snapshot-only fresh-process test +that restores the Voice marker without browser correlation state. + +Rejected alternatives: + +- a local Yarn patch to Flue, because it forks substrate persistence and wire + projection inside this product PR; +- a correlated provenance signal, because it is a second, non-atomic admission + that can independently fail or wake the agent; +- browser or application sidecar storage, because it becomes a second durable + authority; and +- hidden transcript, attachment, or visible-text encoding, because it changes + the canonical user representation or smuggles metadata through content. + +Until the upstream contract is released and adopted, direct spoken user text +remains canonically durable but its Voice chip after reopen remains blocked and +must not be reported as complete. From 996049453802cea3ba9eafd502d29b3ee673031b Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Fri, 4 Sep 2026 09:36:43 +0200 Subject: [PATCH 19/52] Add exact Brunch question markers Amp-Thread-ID: https://ampcode.com/threads/T-01a06941-3ee7-7482-a82e-8a16c7141785 Co-authored-by: Amp --- .../brunch-agent/packages/core/src/flue.ts | 40 +++++++++- .../brunch-agent/packages/core/src/index.ts | 7 ++ .../packages/core/src/prompts/SYSTEM.md | 2 + .../packages/core/src/question-marker.ts | 20 +++++ .../core/test/question-marker.test.ts | 78 +++++++++++++++++++ 5 files changed, 143 insertions(+), 4 deletions(-) create mode 100644 libs/@hashintel/brunch-agent/packages/core/src/question-marker.ts create mode 100644 libs/@hashintel/brunch-agent/packages/core/test/question-marker.test.ts diff --git a/libs/@hashintel/brunch-agent/packages/core/src/flue.ts b/libs/@hashintel/brunch-agent/packages/core/src/flue.ts index cb020aaf9e0..9d57e9ea9e1 100644 --- a/libs/@hashintel/brunch-agent/packages/core/src/flue.ts +++ b/libs/@hashintel/brunch-agent/packages/core/src/flue.ts @@ -1,6 +1,20 @@ -import { useModel, useSkill } from "@flue/runtime"; +import { + defineTool, + useDataWriter, + useModel, + useSkill, + useTool, +} from "@flue/runtime"; +import * as v from "valibot"; import systemPrompt from "./prompts/SYSTEM.md?raw"; +import { + BRUNCH_QUESTION_DATA_NAME, + BRUNCH_QUESTION_TOOL_NAME, + BrunchQuestionDataSchema, + BrunchQuestionInputSchema, + type BrunchQuestionData, +} from "./question-marker"; import { ELICITATION_SKILL_NAME, elicitationSkill, @@ -10,14 +24,32 @@ import { skillFromMarkdown } from "./skills/skill-markdown"; /** * Mount the contributions owned by Brunch core and return its system prompt. * - * Core contributes the always-on universal prompt and one `elicitation` - * capability skill. It owns no model-facing tool; add one here only when it - * applies independently of the selected modelling formalism and host. + * Core contributes the always-on universal prompt, one `elicitation` + * capability skill, and the formalism-independent question marker. */ export function useBrunchAgent(model: string): string { useModel(model); useSkill(elicitationSkill); + const writeQuestion = useDataWriter(BRUNCH_QUESTION_DATA_NAME, { + schema: BrunchQuestionDataSchema, + }); + useTool(createBrunchQuestionMarkerTool(writeQuestion)); return systemPrompt.replace(/^\s+|\s+$/gu, ""); } +export const createBrunchQuestionMarkerTool = ( + writeQuestion: (question: BrunchQuestionData) => void, +) => + defineTool({ + name: BRUNCH_QUESTION_TOOL_NAME, + description: + "Mark the exact text of a direct question for accessible replay. Call this immediately before including that exact question in ordinary assistant prose. This marker does not ask or answer the question itself.", + input: BrunchQuestionInputSchema, + output: v.object({ marked: v.literal(true) }), + run({ data, toolCallId }) { + writeQuestion({ question: data.question, toolCallId }); + return { output: { marked: true as const } }; + }, + }); + export { ELICITATION_SKILL_NAME, elicitationSkill, skillFromMarkdown }; diff --git a/libs/@hashintel/brunch-agent/packages/core/src/index.ts b/libs/@hashintel/brunch-agent/packages/core/src/index.ts index f4021f55ef0..d272333fe81 100644 --- a/libs/@hashintel/brunch-agent/packages/core/src/index.ts +++ b/libs/@hashintel/brunch-agent/packages/core/src/index.ts @@ -41,6 +41,13 @@ export { toolPrefix, type Operation, } from "./conversation/naming"; +export { + BRUNCH_QUESTION_DATA_NAME, + BRUNCH_QUESTION_TOOL_NAME, + BrunchQuestionDataSchema, + BrunchQuestionInputSchema, + type BrunchQuestionData, +} from "./question-marker"; export { type HarnessReplyEvent, type ReplyPartKind, diff --git a/libs/@hashintel/brunch-agent/packages/core/src/prompts/SYSTEM.md b/libs/@hashintel/brunch-agent/packages/core/src/prompts/SYSTEM.md index 6c8049566df..99a6c65a011 100644 --- a/libs/@hashintel/brunch-agent/packages/core/src/prompts/SYSTEM.md +++ b/libs/@hashintel/brunch-agent/packages/core/src/prompts/SYSTEM.md @@ -10,6 +10,8 @@ Establish what the result must help the person decide, answer, compare, explain, Use the person's vocabulary and follow concrete cases rather than traversing a schema, template, or target representation. Do not open with a battery of independent questions; deepen one answerable thread at a time and group questions only when they share one frame. +Before asking the person a direct question, call `brunch_mark_question` with the exact question text. Then include the exact same question text in ordinary assistant prose. The marker only makes that text available for accessible replay; it does not wait for or accept the answer, so continue the same response normally after calling it. Do not mark headings, rhetorical questions, or prose that you will not present verbatim. + ## Authorship and uncertainty Keep what the person said distinct from your normalization, inference, assumption, proposal, transformation, or default. Do not invent content, silently increase precision, or treat assent to wording you supplied as independent evidence. When accounts differ, establish whether the relationship is correction, conflict, or contextual coexistence before reconciling them. diff --git a/libs/@hashintel/brunch-agent/packages/core/src/question-marker.ts b/libs/@hashintel/brunch-agent/packages/core/src/question-marker.ts new file mode 100644 index 00000000000..97914132d02 --- /dev/null +++ b/libs/@hashintel/brunch-agent/packages/core/src/question-marker.ts @@ -0,0 +1,20 @@ +import * as v from "valibot"; + +export const BRUNCH_QUESTION_TOOL_NAME = "brunch_mark_question"; +export const BRUNCH_QUESTION_DATA_NAME = "brunch-question"; + +const NonBlankStringSchema = v.pipe( + v.string(), + v.check((value) => /\S/u.test(value), "Expected a non-blank string."), +); + +export const BrunchQuestionInputSchema = v.object({ + question: NonBlankStringSchema, +}); + +export const BrunchQuestionDataSchema = v.object({ + question: NonBlankStringSchema, + toolCallId: NonBlankStringSchema, +}); + +export type BrunchQuestionData = v.InferOutput; diff --git a/libs/@hashintel/brunch-agent/packages/core/test/question-marker.test.ts b/libs/@hashintel/brunch-agent/packages/core/test/question-marker.test.ts new file mode 100644 index 00000000000..f040d8c6757 --- /dev/null +++ b/libs/@hashintel/brunch-agent/packages/core/test/question-marker.test.ts @@ -0,0 +1,78 @@ +import { readFile } from "node:fs/promises"; + +import * as v from "valibot"; +import { describe, expect, test, vi } from "vitest"; + +import { createBrunchQuestionMarkerTool } from "../src/flue"; +import { + BRUNCH_QUESTION_DATA_NAME, + BRUNCH_QUESTION_TOOL_NAME, + BrunchQuestionDataSchema, + BrunchQuestionInputSchema, +} from "../src/question-marker"; + +describe("the Brunch question marker", () => { + test("defines one non-interactive tool and data-part identity", () => { + expect(BRUNCH_QUESTION_TOOL_NAME).toBe("brunch_mark_question"); + expect(BRUNCH_QUESTION_DATA_NAME).toBe("brunch-question"); + }); + + test("preserves exact non-blank question text and tool-call identity", () => { + const question = " Which line should run this order? "; + + expect( + v.parse(BrunchQuestionInputSchema, { + question, + }), + ).toEqual({ question }); + expect( + v.parse(BrunchQuestionDataSchema, { + question, + toolCallId: "tool-question-1", + }), + ).toEqual({ question, toolCallId: "tool-question-1" }); + }); + + test("writes the exact marker without terminating or waiting for an answer", async () => { + const writeQuestion = vi.fn(); + const tool = createBrunchQuestionMarkerTool(writeQuestion); + + const result = await tool.run({ + data: { question: "Which line should run this order?" }, + log: { + error: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + }, + toolCallId: "tool-question-1", + }); + + expect(writeQuestion).toHaveBeenCalledOnce(); + expect(writeQuestion).toHaveBeenCalledWith({ + question: "Which line should run this order?", + toolCallId: "tool-question-1", + }); + expect(result).toEqual({ output: { marked: true } }); + }); + + test.each([ + { question: "" }, + { question: " " }, + { question: "What matters?", toolCallId: "" }, + { question: "What matters?", toolCallId: " " }, + ])("rejects an incomplete marker: %j", (marker) => { + expect(v.safeParse(BrunchQuestionDataSchema, marker).success).toBe(false); + }); + + test("instructs the model to mark and then reproduce the exact question in ordinary prose", async () => { + const systemPrompt = await readFile( + new URL("../src/prompts/SYSTEM.md", import.meta.url), + "utf8", + ); + + expect(systemPrompt).toContain("brunch_mark_question"); + expect(systemPrompt).toContain("exact same question text"); + expect(systemPrompt).toContain("ordinary assistant prose"); + expect(systemPrompt).toContain("does not wait for or accept the answer"); + }); +}); From da30f9243f4ead435c03f4fc7a6344afb33022eb Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Fri, 4 Sep 2026 09:40:37 +0200 Subject: [PATCH 20/52] Preserve hidden question markers across Flue Amp-Thread-ID: https://ampcode.com/threads/T-01a06941-3ee7-7482-a82e-8a16c7141785 Co-authored-by: Amp --- .../test/petrinaut-chat-result.ts | 6 ++ .../test/petrinaut-chat.integration.ts | 61 ++++++++++++++++- apps/brunch-agent/test/petrinaut-chat.test.ts | 17 +++++ .../brunch-panel-transport.ts | 2 + .../use-flue-chat-history.ts | 7 +- .../brunch-agent/packages/core/package.json | 4 ++ .../brunch-agent/packages/core/vite.config.ts | 3 + .../packages/transport-aisdk/src/index.ts | 2 + .../transport-aisdk/src/transcript.ts | 2 + .../packages/transport-aisdk/src/ui-stream.ts | 8 +++ .../transport-aisdk/test/transcript.test.ts | 47 +++++++++++++ .../transport-aisdk/test/ui-stream.test.ts | 68 +++++++++++++++++++ 12 files changed, 222 insertions(+), 5 deletions(-) diff --git a/apps/brunch-agent/test/petrinaut-chat-result.ts b/apps/brunch-agent/test/petrinaut-chat-result.ts index c7cc142689c..b1a70cb413c 100644 --- a/apps/brunch-agent/test/petrinaut-chat-result.ts +++ b/apps/brunch-agent/test/petrinaut-chat-result.ts @@ -21,6 +21,10 @@ export interface PetrinautChatResult { readonly resumedStatus: number; readonly resumedText: string; readonly resumedFinish: UIMessageChunk | undefined; + readonly questionMarkerLive: unknown; + readonly questionMarkerHistory: unknown; + readonly questionToolVisibleLive: boolean; + readonly questionToolVisibleHistory: boolean; readonly historyUserEntryCount: number; readonly historyClientToolResultCount: number; readonly historyGetStatus: number; @@ -51,5 +55,7 @@ export interface PetrinautChatResult { export interface PetrinautResumeResult { readonly historyGetStatus: number; readonly historyUserText: string; + readonly questionMarkerHistory: unknown; + readonly questionToolVisibleHistory: boolean; readonly transcript: string; } diff --git a/apps/brunch-agent/test/petrinaut-chat.integration.ts b/apps/brunch-agent/test/petrinaut-chat.integration.ts index 20750ca68a4..253fc8ff4d3 100644 --- a/apps/brunch-agent/test/petrinaut-chat.integration.ts +++ b/apps/brunch-agent/test/petrinaut-chat.integration.ts @@ -18,6 +18,10 @@ import { snapshotToUiMessages, } from "@hashintel/brunch-agent-transport-aisdk"; import { ELICITATION_SKILL_NAME } from "@hashintel/brunch-agent/flue"; +import { + BRUNCH_QUESTION_DATA_NAME, + BRUNCH_QUESTION_TOOL_NAME, +} from "@hashintel/brunch-agent/question-marker"; import { PING_TOOL_NAME } from "../src/agents/chat-agent/tools/ping.ts"; import { applyCaptureSweep } from "../src/capture/apply-sweep.ts"; @@ -43,6 +47,7 @@ const ACTIVATE_SKILL_TOOL_NAME = "activate_skill"; const CHAT_MODEL_ID = "claude-haiku-4-5"; const RUNBOOK_SKILL_NAME = "sdcpn-modelling"; const READ_SKILL_RESOURCE_TOOL_NAME = "read_skill_resource"; +const question = "Which documentation page should we inspect next?"; const principalKey = "principal-mission-1"; const conversationId = "conversation-mission-1"; @@ -79,6 +84,35 @@ const userTextFromHistory = ( .map((part) => part.text) .join(""); +const questionMarkerFromHistory = ( + messages: ReturnType, +): unknown => { + const marker = messages + .flatMap((message) => message.parts) + .find( + (part) => + part.type === `data-${BRUNCH_QUESTION_DATA_NAME}` && "data" in part, + ); + return marker !== undefined && "data" in marker ? marker.data : undefined; +}; + +const questionMarkerFromChunks = ( + chunks: readonly UIMessageChunk[], +): unknown => { + const marker = chunks.find( + (chunk) => + chunk.type === `data-${BRUNCH_QUESTION_DATA_NAME}` && "data" in chunk, + ); + return marker !== undefined && "data" in marker ? marker.data : undefined; +}; + +const questionToolVisibleInHistory = ( + messages: ReturnType, +): boolean => + messages + .flatMap((message) => message.parts) + .some((part) => part.type === `tool-${BRUNCH_QUESTION_TOOL_NAME}`); + const faux = fauxProvider({ provider: "anthropic", models: [{ id: CHAT_MODEL_ID, reasoning: true }], @@ -98,19 +132,24 @@ try { const panelTransport = createFlueChatTransport({ client: historyClient, clientToolNames, + hiddenToolNames: new Set([BRUNCH_QUESTION_TOOL_NAME]), }); const projectHistory = ( snapshot: Awaited>, ) => snapshotToUiMessages(snapshot, { clientToolNames, + hiddenToolNames: new Set([BRUNCH_QUESTION_TOOL_NAME]), }); if (process.env.BRUNCH_RESUME_PHASE === "1") { const snapshot = await historyClient.history(); + const historyMessages = projectHistory(snapshot); const result: PetrinautResumeResult = { historyGetStatus: 200, - historyUserText: userTextFromHistory(projectHistory(snapshot)), + historyUserText: userTextFromHistory(historyMessages), + questionMarkerHistory: questionMarkerFromHistory(historyMessages), + questionToolVisibleHistory: questionToolVisibleInHistory(historyMessages), transcript: formatFlueTranscript(snapshot), }; process.stdout.write(`PETRINAUT_RESUME_RESULT ${JSON.stringify(result)}\n`); @@ -196,9 +235,19 @@ try { ], { stopReason: "toolUse" }, ), + fauxAssistantMessage( + [ + fauxToolCall( + BRUNCH_QUESTION_TOOL_NAME, + { question }, + { id: "tool-question-1" }, + ), + ], + { stopReason: "toolUse" }, + ), fauxAssistantMessage([ fauxText( - "The guide says the assistant can read its own documentation pages.", + `The guide says the assistant can read its own documentation pages. ${question}`, ), ]), fauxAssistantMessage([ @@ -400,6 +449,14 @@ try { .map((chunk) => chunk.delta) .join(""), resumedFinish: resumedChunks.at(-1), + questionMarkerLive: questionMarkerFromChunks(resumedChunks), + questionMarkerHistory: questionMarkerFromHistory(historyMessages), + questionToolVisibleLive: resumedChunks.some( + (chunk) => + chunk.type === "tool-input-available" && + chunk.toolName === BRUNCH_QUESTION_TOOL_NAME, + ), + questionToolVisibleHistory: questionToolVisibleInHistory(historyMessages), historyUserEntryCount: userEntryIds.length, historyClientToolResultCount: clientToolResultCount, historyGetStatus: 200, diff --git a/apps/brunch-agent/test/petrinaut-chat.test.ts b/apps/brunch-agent/test/petrinaut-chat.test.ts index daf97db27ea..2ee108a9ac0 100644 --- a/apps/brunch-agent/test/petrinaut-chat.test.ts +++ b/apps/brunch-agent/test/petrinaut-chat.test.ts @@ -72,6 +72,16 @@ test("the browser transport streams the mounted Flue agent through server and cl type: "finish", finishReason: "stop", }); + expect(result.questionMarkerLive).toEqual({ + question: "Which documentation page should we inspect next?", + toolCallId: "tool-question-1", + }); + expect(result.questionToolVisibleLive).toBe(false); + expect(result.questionMarkerHistory).toEqual({ + question: "Which documentation page should we inspect next?", + toolCallId: "tool-question-1", + }); + expect(result.questionToolVisibleHistory).toBe(false); expect(result.historyUserEntryCount).toBe(1); expect(result.historyClientToolResultCount).toBe(1); @@ -107,6 +117,7 @@ test("the browser transport streams the mounted Flue agent through server and cl expect(result.interviewerToolNames).toContain("read_skill_resource"); expect(result.interviewerToolNames).toContain("ping"); expect(result.interviewerToolNames).toContain("readPetrinautDoc"); + expect(result.interviewerToolNames).toContain("brunch_mark_question"); expect(result.interviewerToolNames).not.toContain("brunch_ask"); expect(result.interviewerToolNames).not.toContain("sweep"); expect(result.interviewerToolNames).not.toContain("brunch_sweep"); @@ -151,10 +162,16 @@ test("the browser transport streams the mounted Flue agent through server and cl expect(resumeResult.historyUserText).toContain( "Run the FE-1435 transport probe.", ); + expect(resumeResult.questionMarkerHistory).toEqual({ + question: "Which documentation page should we inspect next?", + toolCallId: "tool-question-1", + }); + expect(resumeResult.questionToolVisibleHistory).toBe(false); expect(resumeResult.transcript).toContain("tool ping"); expect(resumeResult.transcript).toContain("tool readPetrinautDoc"); expect(resumeResult.transcript).toContain("tool activate_skill"); expect(resumeResult.transcript).toContain("tool read_skill_resource"); + expect(resumeResult.transcript).toContain("tool brunch_mark_question"); } finally { await rm(dbDirectory, { recursive: true, force: true }); } diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-panel-transport.ts b/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-panel-transport.ts index c6e0f24a308..076b16cbcb6 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-panel-transport.ts +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/brunch-panel-transport.ts @@ -3,6 +3,7 @@ import { FlueChatAdmissionError, } from "@hashintel/brunch-agent-transport-aisdk"; import { SWEEP_TOOL_NAME } from "@hashintel/brunch-agent/client-tools"; +import { BRUNCH_QUESTION_TOOL_NAME } from "@hashintel/brunch-agent/question-marker"; import { readPetrinautDocToolName } from "@hashintel/petrinaut-core"; import { sweepOutputSchema } from "../brunch-sweep-output"; @@ -276,6 +277,7 @@ export const createBrunchPanelTransport = ( const transport = createFlueChatTransport({ client, clientToolNames: new Set([readPetrinautDocToolName]), + hiddenToolNames: new Set([BRUNCH_QUESTION_TOOL_NAME]), onAdmission: (event) => { tracker.recordAdmission(event); hooks?.onAdmission?.(event.admission); diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/use-flue-chat-history.ts b/apps/petrinaut-website/src/main/app/local-storage-demo/use-flue-chat-history.ts index 7e3bf752055..65b914a5e3d 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/use-flue-chat-history.ts +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/use-flue-chat-history.ts @@ -1,8 +1,8 @@ import { useCallback, useEffect, useRef, useState } from "react"; import { snapshotToUiMessages } from "@hashintel/brunch-agent-transport-aisdk"; - -import { brunchClientToolNames } from "./brunch-client-tools"; +import { BRUNCH_QUESTION_TOOL_NAME } from "@hashintel/brunch-agent/question-marker"; +import { readPetrinautDocToolName } from "@hashintel/petrinaut-core"; import type { AgentConversationObservation, @@ -22,7 +22,8 @@ const projectPetrinautMessages = ( // The host owns this narrowing: its configured client-tool catalog is the // same catalog Petrinaut's message type exposes. snapshotToUiMessages(conversation, { - clientToolNames: brunchClientToolNames, + clientToolNames: new Set([readPetrinautDocToolName]), + hiddenToolNames: new Set([BRUNCH_QUESTION_TOOL_NAME]), }) as PetrinautAiMessage[]; export const useFlueChatHistory = ( diff --git a/libs/@hashintel/brunch-agent/packages/core/package.json b/libs/@hashintel/brunch-agent/packages/core/package.json index ca72a1d986e..f8c56ce978e 100644 --- a/libs/@hashintel/brunch-agent/packages/core/package.json +++ b/libs/@hashintel/brunch-agent/packages/core/package.json @@ -18,6 +18,10 @@ "types": "./src/flue.ts", "import": "./dist/flue.js" }, + "./question-marker": { + "types": "./src/question-marker.ts", + "import": "./dist/question-marker.js" + }, "./storage": { "types": "./src/storage.ts", "import": "./dist/storage.js" diff --git a/libs/@hashintel/brunch-agent/packages/core/vite.config.ts b/libs/@hashintel/brunch-agent/packages/core/vite.config.ts index 291e5b9c4bf..cd4e7eedf23 100644 --- a/libs/@hashintel/brunch-agent/packages/core/vite.config.ts +++ b/libs/@hashintel/brunch-agent/packages/core/vite.config.ts @@ -13,6 +13,9 @@ export default defineConfig({ ), flue: fileURLToPath(new URL("src/flue.ts", import.meta.url)), index: fileURLToPath(new URL("src/index.ts", import.meta.url)), + "question-marker": fileURLToPath( + new URL("src/question-marker.ts", import.meta.url), + ), storage: fileURLToPath(new URL("src/storage.ts", import.meta.url)), }, fileName: (_format, entryName) => `${entryName}.js`, diff --git a/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/index.ts b/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/index.ts index c185a1919b6..64d370955f6 100644 --- a/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/index.ts +++ b/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/index.ts @@ -32,6 +32,7 @@ export interface ClientToolResult { export interface FlueChatTransportOptions { readonly client: FlueClient; readonly clientToolNames: ReadonlySet; + readonly hiddenToolNames?: ReadonlySet; readonly onAdmission?: (event: { readonly admission: AgentSendResult; readonly kind: "client-tool-result" | "user"; @@ -267,6 +268,7 @@ const streamSubmission = ( const projector = createFlueUiStream({ submissionId: admission.submissionId, clientToolNames: options.clientToolNames, + hiddenToolNames: options.hiddenToolNames, write, }); diff --git a/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/transcript.ts b/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/transcript.ts index 64ba4243aba..8d27f09af1c 100644 --- a/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/transcript.ts +++ b/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/transcript.ts @@ -25,6 +25,7 @@ export type UiHistoryMessage = Omit< export interface SnapshotToUiMessagesOptions { readonly clientToolNames: ReadonlySet; + readonly hiddenToolNames?: ReadonlySet; } const unhandledConversationPart = (part: never): never => { @@ -148,6 +149,7 @@ const partsFrom = ( continue; } if (part.type === "dynamic-tool") { + if (options.hiddenToolNames?.has(part.toolName) === true) continue; parts.push(toolPartFrom(part, options.clientToolNames, clientResults)); continue; } diff --git a/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/ui-stream.ts b/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/ui-stream.ts index 7270be320b2..22f2864530c 100644 --- a/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/ui-stream.ts +++ b/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/ui-stream.ts @@ -4,6 +4,7 @@ import type { UIMessageChunk } from "ai"; export interface FlueUiStreamOptions { readonly submissionId: AgentSendResult["submissionId"]; readonly clientToolNames: ReadonlySet; + readonly hiddenToolNames?: ReadonlySet; readonly write: (chunk: UIMessageChunk) => void; } @@ -26,6 +27,7 @@ export const createFlueUiStream = ( let turnId: string | undefined; let partOrdinal = 0; let streamingPart: StreamingPart | undefined; + const hiddenToolCallIds = new Set(); const pendingClientToolCallIds = new Set(); const finishPart = (): void => { @@ -123,6 +125,10 @@ export const createFlueUiStream = ( if (!accepting || messageId === undefined) return; if (chunk.messageId !== messageId) return; finishPart(); + if (options.hiddenToolNames?.has(chunk.toolName) === true) { + hiddenToolCallIds.add(chunk.toolCallId); + return; + } const isClientTool = options.clientToolNames.has(chunk.toolName); if (isClientTool) pendingClientToolCallIds.add(chunk.toolCallId); options.write({ @@ -136,6 +142,7 @@ export const createFlueUiStream = ( } case "tool-output": { if (!accepting || messageId === undefined) return; + if (hiddenToolCallIds.has(chunk.toolCallId)) return; if (pendingClientToolCallIds.has(chunk.toolCallId)) return; options.write({ type: "tool-output-available", @@ -147,6 +154,7 @@ export const createFlueUiStream = ( } case "tool-output-error": { if (!accepting || messageId === undefined) return; + if (hiddenToolCallIds.has(chunk.toolCallId)) return; if (pendingClientToolCallIds.has(chunk.toolCallId)) return; options.write({ type: "tool-output-error", diff --git a/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/transcript.test.ts b/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/transcript.test.ts index eafaf90e3b5..7ab457e4e62 100644 --- a/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/transcript.test.ts +++ b/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/transcript.test.ts @@ -294,3 +294,50 @@ test("folds a client-tool continuation into the assistant message it resumed", ( }, ]); }); + +test("hides a question-marker tool while retaining its durable data", () => { + const question = "Which line should run this order?"; + const snapshot: FlueConversationSnapshot = { + v: 1, + conversationId: "conversation-1", + offset: "0", + messages: [ + { + id: "assistant-question", + role: "assistant", + purpose: "assistant", + display: "visible", + parts: [ + { + type: "dynamic-tool", + toolCallId: "tool-question-1", + toolName: "brunch_mark_question", + state: "output-available", + input: { question }, + output: { marked: true }, + }, + { + type: "data-brunch-question", + data: { question, toolCallId: "tool-question-1" }, + }, + { type: "text", text: question, state: "done" }, + ], + }, + ], + settlements: [], + }; + + expect(snapshotToUiMessages(snapshot, projectionOptions)).toEqual([ + { + id: "assistant-question", + role: "assistant", + parts: [ + { + type: "data-brunch-question", + data: { question, toolCallId: "tool-question-1" }, + }, + { type: "text", text: question, state: "done" }, + ], + }, + ]); +}); diff --git a/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/ui-stream.test.ts b/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/ui-stream.test.ts index c567fcd9b38..5f6c26ecf20 100644 --- a/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/ui-stream.test.ts +++ b/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/ui-stream.test.ts @@ -9,11 +9,13 @@ const position = (index: number) => ({ batch: 1, index }); const project = ( chunks: readonly ConversationStreamChunk[], + hiddenToolNames: ReadonlySet = new Set(), ): UIMessageChunk[] => { const written: UIMessageChunk[] = []; const projector = createFlueUiStream({ submissionId: "submission-1", clientToolNames: new Set(["readPetrinautDoc"]), + hiddenToolNames, write: (chunk) => written.push(chunk), }); for (const chunk of chunks) projector.accept(chunk); @@ -64,6 +66,72 @@ test("projects data and metadata onto the AI SDK stream", () => { }); }); +test("hides an implementation tool while preserving its data marker", () => { + const written = project( + [ + { + type: "message-started", + conversationId: "conversation-1", + messageId: "message-1", + submissionId: "submission-1", + turnId: "turn-1", + position: position(0), + }, + { + type: "tool-input", + conversationId: "conversation-1", + messageId: "message-1", + toolCallId: "tool-question-1", + toolName: "brunch_mark_question", + input: { question: "Which line should run this order?" }, + position: position(1), + }, + { + type: "data-part", + conversationId: "conversation-1", + messageId: "message-1", + name: "brunch-question", + data: { + question: "Which line should run this order?", + toolCallId: "tool-question-1", + }, + position: position(2), + }, + { + type: "tool-output", + conversationId: "conversation-1", + toolCallId: "tool-question-1", + output: { marked: true }, + position: position(3), + }, + { + type: "submission-settled", + conversationId: "conversation-1", + submissionId: "submission-1", + outcome: "completed", + position: position(4), + }, + ], + new Set(["brunch_mark_question"]), + ); + + expect(written).toContainEqual({ + type: "data-brunch-question", + data: { + question: "Which line should run this order?", + toolCallId: "tool-question-1", + }, + }); + expect( + written.some( + (chunk) => + chunk.type === "tool-input-available" || + chunk.type === "tool-output-available" || + chunk.type === "tool-output-error", + ), + ).toBe(false); +}); + test("ignores observation catch-up chunks in a submission stream", () => { const written = project([ { From 8f76002f90460f7875be16108dc8a39297b55e53 Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Fri, 4 Sep 2026 09:46:53 +0200 Subject: [PATCH 21/52] Select exact Brunch questions for Voice replay Amp-Thread-ID: https://ampcode.com/threads/T-01a06941-3ee7-7482-a82e-8a16c7141785 Co-authored-by: Amp --- .../voice-interview/canonical-speech.test.ts | 122 ++++++++++++++++++ .../app/voice-interview/canonical-speech.ts | 50 ++++++- .../brunch-agent/packages/core/src/index.ts | 1 + .../packages/core/src/question-marker.ts | 8 ++ .../core/test/question-marker.test.ts | 12 ++ 5 files changed, 190 insertions(+), 3 deletions(-) diff --git a/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.test.ts index f688e22039e..50807c4ba97 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.test.ts @@ -2,6 +2,7 @@ import { describe, expect, test } from "vitest"; import { hashCanonicalSpeechText, + selectCanonicalSpeech, selectCanonicalSpeechSegments, } from "./canonical-speech"; @@ -130,6 +131,127 @@ describe("canonical speech selection", () => { expect(select(messages)).toEqual([]); }); + test("selects an exact marked question separately from full-response text", () => { + const question = "Which operator confirms the batch?"; + const selection = selectCanonicalSpeech([ + { + id: "assistant-question", + role: "assistant", + parts: [ + { + type: "data-brunch-question", + data: { question, toolCallId: "tool-question-1" }, + }, + { + type: "text", + text: `The batch is ready. ${question} I can explain the choices.`, + state: "done", + }, + ], + }, + ]); + + expect(selection.segments.map(({ text }) => text)).toEqual([ + `The batch is ready. ${question} I can explain the choices.`, + ]); + expect(selection.questionSegment).toEqual({ + contentHash: hashCanonicalSpeechText(question), + id: `canonical-speech:assistant-question:question%3Atool-question-1:${hashCanonicalSpeechText(question)}`, + messageId: "assistant-question", + partId: "question:tool-question-1", + source: "assistant-question", + text: question, + }); + }); + + test.each([ + { + name: "missing exact finalized prose", + parts: [ + { + type: "data-brunch-question" as const, + data: { + question: "Which operator confirms the batch?", + toolCallId: "tool-question-1", + }, + }, + { + type: "text" as const, + text: "A different question appears in the response.", + state: "done" as const, + }, + ], + }, + { + name: "only provisional prose", + parts: [ + { + type: "data-brunch-question" as const, + data: { + question: "Which operator confirms the batch?", + toolCallId: "tool-question-1", + }, + }, + { + type: "text" as const, + text: "Which operator confirms the batch?", + state: "streaming" as const, + }, + ], + }, + { + name: "blank marker identity", + parts: [ + { + type: "data-brunch-question" as const, + data: { + question: "Which operator confirms the batch?", + toolCallId: " ", + }, + }, + { + type: "text" as const, + text: "Which operator confirms the batch?", + state: "done" as const, + }, + ], + }, + ])("rejects a question marker with $name", ({ parts }) => { + expect( + selectCanonicalSpeech([ + { + id: "assistant-invalid-question", + role: "assistant", + parts, + }, + ]).questionSegment, + ).toBeUndefined(); + }); + + test("does not correlate a marker to text from another assistant message", () => { + const question = "Which operator confirms the batch?"; + + expect( + selectCanonicalSpeech([ + { + id: "assistant-marker", + role: "assistant", + parts: [ + { + type: "data-brunch-question", + data: { question, toolCallId: "tool-question-1" }, + }, + ], + }, + { + id: "assistant-text", + role: "assistant", + parts: [{ type: "text", text: question, state: "done" }], + }, + ]).questionSegment, + ).toBeUndefined(); + }); + test("uses stable source identity plus an exact-text fingerprint", () => { expect(hashCanonicalSpeechText("hello")).toBe("fnv1a32:4f9f2cab"); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.ts b/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.ts index d50c298a7fb..53e9fc63c49 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.ts @@ -1,3 +1,8 @@ +import { + BRUNCH_QUESTION_DATA_NAME, + parseBrunchQuestionData, +} from "@hashintel/brunch-agent/question-marker"; + import { hashCanonicalSpeechText } from "../../../canonical-speech-fingerprint"; import type { AgentSendResult } from "@flue/sdk"; @@ -10,7 +15,7 @@ export interface CanonicalSpeechSegment { readonly id: string; readonly messageId: string; readonly partId: string; - readonly source: "assistant-text"; + readonly source: "assistant-question" | "assistant-text"; /** * Every Flue submission that wrote to this segment's message: the one that * started it plus any client-tool continuation projected back onto it. @@ -43,14 +48,30 @@ const createSegment = ( export const selectCanonicalSpeechSegments = ( messages: PetrinautAiMessage[], -): CanonicalSpeechSegment[] => { +): CanonicalSpeechSegment[] => selectCanonicalSpeech(messages).segments; + +export interface CanonicalSpeechSelection { + readonly questionSegment?: CanonicalSpeechSegment; + readonly segments: CanonicalSpeechSegment[]; +} + +export const selectCanonicalSpeech = ( + messages: PetrinautAiMessage[], +): CanonicalSpeechSelection => { const segments: CanonicalSpeechSegment[] = []; + let questionSegment: CanonicalSpeechSegment | undefined; for (const message of messages) { if (message.role !== "assistant") { continue; } + const finalizedTexts = message.parts.flatMap((part) => + part.type === "text" && part.state !== "streaming" && part.text.trim() + ? [part.text] + : [], + ); + for (const [partIndex, part] of message.parts.entries()) { if ( part.type === "text" && @@ -67,7 +88,30 @@ export const selectCanonicalSpeechSegments = ( ); } } + + const questionMarkers = message.parts.flatMap((part) => { + if (part.type !== `data-${BRUNCH_QUESTION_DATA_NAME}`) { + return []; + } + + const marker = parseBrunchQuestionData(part.data); + + return marker && + finalizedTexts.some((text) => text.includes(marker.question)) + ? [marker] + : []; + }); + const latestQuestionMarker = questionMarkers.at(-1); + + if (latestQuestionMarker) { + questionSegment = createSegment( + message.id, + `question:${latestQuestionMarker.toolCallId}`, + "assistant-question", + latestQuestionMarker.question, + ); + } } - return segments; + return { questionSegment, segments }; }; diff --git a/libs/@hashintel/brunch-agent/packages/core/src/index.ts b/libs/@hashintel/brunch-agent/packages/core/src/index.ts index d272333fe81..25fd3fdc75c 100644 --- a/libs/@hashintel/brunch-agent/packages/core/src/index.ts +++ b/libs/@hashintel/brunch-agent/packages/core/src/index.ts @@ -46,6 +46,7 @@ export { BRUNCH_QUESTION_TOOL_NAME, BrunchQuestionDataSchema, BrunchQuestionInputSchema, + parseBrunchQuestionData, type BrunchQuestionData, } from "./question-marker"; export { diff --git a/libs/@hashintel/brunch-agent/packages/core/src/question-marker.ts b/libs/@hashintel/brunch-agent/packages/core/src/question-marker.ts index 97914132d02..ba6194c63f5 100644 --- a/libs/@hashintel/brunch-agent/packages/core/src/question-marker.ts +++ b/libs/@hashintel/brunch-agent/packages/core/src/question-marker.ts @@ -18,3 +18,11 @@ export const BrunchQuestionDataSchema = v.object({ }); export type BrunchQuestionData = v.InferOutput; + +export const parseBrunchQuestionData = ( + value: unknown, +): BrunchQuestionData | undefined => { + const result = v.safeParse(BrunchQuestionDataSchema, value); + + return result.success ? result.output : undefined; +}; diff --git a/libs/@hashintel/brunch-agent/packages/core/test/question-marker.test.ts b/libs/@hashintel/brunch-agent/packages/core/test/question-marker.test.ts index f040d8c6757..581ebf7239d 100644 --- a/libs/@hashintel/brunch-agent/packages/core/test/question-marker.test.ts +++ b/libs/@hashintel/brunch-agent/packages/core/test/question-marker.test.ts @@ -9,6 +9,7 @@ import { BRUNCH_QUESTION_TOOL_NAME, BrunchQuestionDataSchema, BrunchQuestionInputSchema, + parseBrunchQuestionData, } from "../src/question-marker"; describe("the Brunch question marker", () => { @@ -62,6 +63,17 @@ describe("the Brunch question marker", () => { { question: "What matters?", toolCallId: " " }, ])("rejects an incomplete marker: %j", (marker) => { expect(v.safeParse(BrunchQuestionDataSchema, marker).success).toBe(false); + expect(parseBrunchQuestionData(marker)).toBeUndefined(); + }); + + test("parses exact question data at the client projection boundary", () => { + const marker = { + question: " Which line should run this order? ", + toolCallId: "tool-question-1", + }; + + expect(parseBrunchQuestionData(marker)).toEqual(marker); + expect(parseBrunchQuestionData(null)).toBeUndefined(); }); test("instructs the model to mark and then reproduce the exact question in ordinary prose", async () => { From 2d71d3c4d7056cf2cd9810593af3625e86a634cc Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Fri, 4 Sep 2026 09:51:36 +0200 Subject: [PATCH 22/52] Replay exact Brunch questions in Voice Amp-Thread-ID: https://ampcode.com/threads/T-01a06941-3ee7-7482-a82e-8a16c7141785 Co-authored-by: Amp --- .../realtime-brunch-bridge.test.ts | 11 +++ .../voice-interview/realtime-brunch-bridge.ts | 13 ++++ .../voice-interview-control.test.tsx | 4 +- .../voice-interview-control.tsx | 23 +++--- .../voice-preview.integration.test.ts | 75 +++++++++++++++++-- .../voice-turn-controller.test.ts | 74 ++++++++++++++++++ .../voice-interview/voice-turn-controller.ts | 25 ++++--- 7 files changed, 198 insertions(+), 27 deletions(-) diff --git a/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.test.ts index 2f7ef6a851d..5e852d2aeeb 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.test.ts @@ -375,9 +375,19 @@ describe("RealtimeBrunchBridge", () => { "Speak this canonical response.", "submission-voice-1", ); + const correlatedQuestion: CanonicalSpeechSegment = { + ...segment( + "correlated-question", + "Which operator confirms the batch?", + "submission-voice-1", + ), + messageId: correlated.messageId, + source: "assistant-question", + }; harness.bridge.updateChat({ canAcceptInterviewAnswer: true, canonicalSegments: [unrelated, correlated], + questionSegment: correlatedQuestion, status: "ready", }); @@ -394,6 +404,7 @@ describe("RealtimeBrunchBridge", () => { ]); expect(harness.events.at(-1)).toEqual({ deliveryId, + questionSegment: correlatedQuestion, segments: [correlated], type: "canonical-response-ready", }); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.ts b/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.ts index ab9fc825272..c825414e676 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.ts @@ -23,6 +23,7 @@ export type VoiceSubmissionSettlement = Pick< interface ChatUpdate { readonly canAcceptInterviewAnswer: boolean; readonly canonicalSegments: CanonicalSpeechSegment[]; + readonly questionSegment?: CanonicalSpeechSegment; /** Flue's settlement index: the only witness that a turn ended short of a reply. */ readonly settlements?: readonly VoiceSubmissionSettlement[]; readonly status: PetrinautAiVoiceModeContext["status"]; @@ -115,6 +116,7 @@ export type RealtimeBrunchBridgeEvent = } | { readonly deliveryId: string; + readonly questionSegment?: CanonicalSpeechSegment; readonly segments: CanonicalSpeechSegment[]; readonly speechCancelled?: true; readonly type: "canonical-response-ready"; @@ -542,9 +544,20 @@ export class RealtimeBrunchBridge { for (const segment of responseSegments) { this.#seenSegmentIds.add(segment.id); } + const questionSegment = this.#chat.questionSegment; + const correlatedQuestion = + questionSegment && + responseSegments.some( + ({ messageId }) => messageId === questionSegment.messageId, + ) && + (active.submissionId === null || + (questionSegment.submissionIds?.includes(active.submissionId) ?? false)) + ? questionSegment + : undefined; this.#activeSubmission = null; this.#emit({ deliveryId: active.deliveryId, + ...(correlatedQuestion ? { questionSegment: correlatedQuestion } : {}), segments: responseSegments, ...(active.speechCancelled ? { speechCancelled: true as const } : {}), type: "canonical-response-ready", diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx index f679efa74b5..869628df9f6 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx @@ -448,7 +448,7 @@ describe("voice interview control", () => { expect(screen.getByText("Voice inactive")).not.toBeNull(); }); - test("does not register Repeat question without a canonical question marker", async () => { + test("registers replay controls that remain snapshot-gated", async () => { render(); await waitFor(() => expect(registeredVoiceModeControls).toBeDefined()); @@ -457,7 +457,7 @@ describe("voice interview control", () => { "function", ); expect(registeredVoiceModeControls?.takeTurn).toBeTypeOf("function"); - expect(registeredVoiceModeControls?.repeatQuestion).toBeUndefined(); + expect(registeredVoiceModeControls?.repeatQuestion).toBeTypeOf("function"); }); test("restarts when Voice is reselected before teardown completes", async () => { diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx index 5625fe15cdf..0943cbeb679 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx @@ -11,7 +11,7 @@ import { Button } from "@hashintel/ds-components"; import { css } from "@hashintel/ds-helpers/css"; import { reportVoiceDiagnostic } from "../../../voice-diagnostics"; -import { selectCanonicalSpeechSegments } from "./canonical-speech"; +import { selectCanonicalSpeech } from "./canonical-speech"; import { OpenAIRealtimeSession } from "./openai-realtime-session"; import { RealtimeBrunchBridge, @@ -25,6 +25,7 @@ import { type VoiceTurnSnapshot, } from "./voice-turn-controller"; +import type { CanonicalSpeechSegment } from "./canonical-speech"; import type { AgentSendResult } from "@flue/sdk"; import type { PetrinautAiVoiceModeContext } from "@hashintel/petrinaut/ui"; @@ -420,16 +421,19 @@ const AvailableVoiceInterviewControl = ({ subscribeToAdmission, subscribeToAdmissionFailure, ); + const canonicalSpeech = selectCanonicalSpeech(context.messages); + const correlateSegment = (segment: CanonicalSpeechSegment) => { + const submissionIds = resolveResponseSubmission?.(segment.messageId); + return submissionIds === undefined || submissionIds.length === 0 + ? segment + : { ...segment, submissionIds }; + }; store.controller.updateChat({ canAcceptInterviewAnswer: context.canAcceptVoiceInput, - canonicalSegments: selectCanonicalSpeechSegments(context.messages).map( - (segment) => { - const submissionIds = resolveResponseSubmission?.(segment.messageId); - return submissionIds === undefined || submissionIds.length === 0 - ? segment - : { ...segment, submissionIds }; - }, - ), + canonicalSegments: canonicalSpeech.segments.map(correlateSegment), + ...(canonicalSpeech.questionSegment + ? { questionSegment: correlateSegment(canonicalSpeech.questionSegment) } + : {}), settlements, status: context.status, }); @@ -457,6 +461,7 @@ const AvailableVoiceInterviewControl = ({ reconnect: () => { void store.controller.reconnect(); }, + repeatQuestion: () => store.controller.repeatQuestion(), resume: () => { void store.controller.resume(); }, diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts index c000aeae4e2..3210ddf5f66 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts @@ -10,12 +10,13 @@ import { BrunchPanelConversationTracker, createBrunchPanelTransport, } from "../local-storage-demo/brunch-panel-transport"; -import { selectCanonicalSpeechSegments } from "./canonical-speech"; +import { selectCanonicalSpeech } from "./canonical-speech"; import { OpenAIRealtimeSession } from "./openai-realtime-session"; import { RealtimeBrunchBridge } from "./realtime-brunch-bridge"; import { submitVoiceInputWithAdmission } from "./voice-interview-control"; import { VoiceTurnController } from "./voice-turn-controller"; +import type { CanonicalSpeechSegment } from "./canonical-speech"; import type { OpenAIRealtimeSessionEvent } from "./openai-realtime-session"; import type { RealtimeBrunchBridgeEvent } from "./realtime-brunch-bridge"; import type { AgentSendResult, FlueClient } from "@flue/sdk"; @@ -74,6 +75,13 @@ const initialMessages = [ { id: "initial-question-message", parts: [ + { + data: { + question: "What happens after approval?", + toolCallId: "tool-initial-question", + }, + type: "data-brunch-question", + }, { state: "done", text: "What happens after approval?", @@ -94,6 +102,13 @@ const responseMessages = [ { id: "next-question-message", parts: [ + { + data: { + question: canonicalQuestion, + toolCallId: "tool-next-question", + }, + type: "data-brunch-question", + }, { state: "done", text: canonicalQuestion, @@ -362,10 +377,12 @@ describe("controlled voice preview", () => { expect(controller.getSnapshot().partialText).toBe( "Speech started before output", ); - const initialSegments = selectCanonicalSpeechSegments(initialMessages); + const initialSelection = selectCanonicalSpeech(initialMessages); + const initialSegments = initialSelection.segments; controller.updateChat({ canAcceptInterviewAnswer: true, canonicalSegments: initialSegments, + questionSegment: initialSelection.questionSegment, status: "ready", }); dataChannel.receive({ @@ -485,16 +502,19 @@ describe("controlled voice preview", () => { status: "streaming", }); const initialSegmentIds = new Set(initialSegments.map(({ id }) => id)); - const correlatedSegments = selectCanonicalSpeechSegments( - responseMessages, - ).map((segment) => + const responseSelection = selectCanonicalSpeech(responseMessages); + const correlateResponse = (segment: CanonicalSpeechSegment) => initialSegmentIds.has(segment.id) ? segment - : { ...segment, submissionId: admission.submissionId }, - ); + : { ...segment, submissionId: admission.submissionId }; + const correlatedSegments = + responseSelection.segments.map(correlateResponse); controller.updateChat({ canAcceptInterviewAnswer: true, canonicalSegments: correlatedSegments, + questionSegment: responseSelection.questionSegment + ? correlateResponse(responseSelection.questionSegment) + : undefined, status: "ready", }); @@ -537,6 +557,47 @@ describe("controlled voice preview", () => { }); expect(track.enabled).toBe(false); + dataChannel.receive({ + response_id: "response-canonical-reply", + type: "output_audio_buffer.stopped", + }); + dataChannel.receive({ + response: { + id: "response-canonical-reply", + output: [], + status: "completed", + }, + type: "response.done", + }); + expect(controller.getSnapshot()).toMatchObject({ + canReadFullResponse: true, + canRepeatQuestion: true, + output: "idle", + }); + + controller.repeatQuestion(); + + const replayCreate = sentEvents(dataChannel).findLast( + ({ type }) => type === "response.create", + ); + expect(replayCreate).toMatchObject({ + response: { + input: [ + { + content: [ + { + text: JSON.stringify({ response_text: [canonicalQuestion] }), + type: "input_text", + }, + ], + role: "system", + type: "message", + }, + ], + }, + type: "response.create", + }); + const remoteTrack = { kind: "audio", stop: vi.fn() }; const remoteStream = { getTracks: () => [remoteTrack], diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts index f30eb2e327e..c8f404a7e23 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts @@ -80,6 +80,14 @@ const question = ( text, }); +const markedQuestion = ( + id: string, + text = "What happens after approval?", +): CanonicalSpeechSegment => ({ + ...question(id, text), + source: "assistant-question", +}); + describe("VoiceTurnController", () => { test("records the content-free Voice lifecycle once in causal order", async () => { const harness = createHarness(); @@ -187,6 +195,7 @@ describe("VoiceTurnController", () => { harness.controller.updateChat({ canAcceptInterviewAnswer: true, canonicalSegments: [question("ask-1")], + questionSegment: markedQuestion("ask-1"), status: "ready", }); @@ -235,6 +244,7 @@ describe("VoiceTurnController", () => { harness.controller.updateChat({ canAcceptInterviewAnswer: true, canonicalSegments: [question("ask-late-transcript")], + questionSegment: markedQuestion("ask-late-transcript"), status: "ready", }); await harness.controller.start(); @@ -310,6 +320,7 @@ describe("VoiceTurnController", () => { harness.controller.updateChat({ canAcceptInterviewAnswer: true, canonicalSegments: [question("ask-request")], + questionSegment: markedQuestion("ask-request"), status: "ready", }); await harness.controller.start(); @@ -397,6 +408,7 @@ describe("VoiceTurnController", () => { harness.controller.updateChat({ canAcceptInterviewAnswer: true, canonicalSegments: [question("ask-handoff")], + questionSegment: markedQuestion("ask-handoff"), status: "ready", }); await harness.controller.start(); @@ -465,6 +477,7 @@ describe("VoiceTurnController", () => { harness.controller.updateChat({ canAcceptInterviewAnswer: true, canonicalSegments: [question("ask-handoff")], + questionSegment: markedQuestion("ask-handoff"), status: "ready", }); await harness.controller.start(); @@ -551,6 +564,57 @@ describe("VoiceTurnController", () => { ]); }); + test("repeats only the exact Brunch-marked question after replay settles", async () => { + const harness = createHarness(); + const context = question("context", "Approval is required before release."); + const finalProse = question( + "response-prose", + "The approver is recorded. I can explain the escalation path.", + ); + const markedQuestion: CanonicalSpeechSegment = { + ...question("marked-question", "Who approves release?"), + messageId: finalProse.messageId, + source: "assistant-question", + }; + await harness.controller.start(); + + harness.emitBridge({ + deliveryId: "voice-1", + questionSegment: markedQuestion, + segments: [context, finalProse], + type: "canonical-response-ready", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-source", + speechRequestId: "speech-source", + type: "output-started", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-source", + type: "output-stopped", + }); + harness.emitSession({ + connectionEpoch: 1, + responseId: "response-source", + status: "completed", + type: "response-terminal", + }); + + expect(harness.controller.getSnapshot()).toMatchObject({ + canReadFullResponse: true, + canRepeatQuestion: true, + }); + + harness.controller.repeatQuestion(); + + expect(harness.session.speakCanonical).toHaveBeenCalledOnce(); + expect(harness.session.speakCanonical).toHaveBeenCalledWith([ + markedQuestion, + ]); + }); + test("disables replay while the user is capturing input", async () => { const harness = createHarness(); const segment = question("ask-capture"); @@ -734,6 +798,7 @@ describe("VoiceTurnController", () => { harness.controller.updateChat({ canAcceptInterviewAnswer: true, canonicalSegments: [question("ask-1")], + questionSegment: markedQuestion("ask-1"), status: "ready", }); await harness.controller.start(); @@ -746,6 +811,7 @@ describe("VoiceTurnController", () => { harness.controller.updateChat({ canAcceptInterviewAnswer: true, canonicalSegments: [question("ask-2", "Who acts next?")], + questionSegment: markedQuestion("ask-2", "Who acts next?"), status: "ready", }); harness.emitBridge({ @@ -1157,6 +1223,10 @@ describe("VoiceTurnController", () => { canonicalSegments: [ question("ask-reconnect", "What happens after approval?"), ], + questionSegment: markedQuestion( + "ask-reconnect", + "What happens after approval?", + ), status: "ready", }); await harness.controller.start(); @@ -1183,6 +1253,10 @@ describe("VoiceTurnController", () => { canonicalSegments: [ question("ask-failed-delivery", "What happens after approval?"), ], + questionSegment: markedQuestion( + "ask-failed-delivery", + "What happens after approval?", + ), status: "ready", }); await harness.controller.start(); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts index 7cd3513df58..84b24448e59 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts @@ -99,6 +99,7 @@ interface VoiceTurnControllerDependencies { interface ChatUpdate { readonly canAcceptInterviewAnswer: boolean; readonly canonicalSegments: CanonicalSpeechSegment[]; + readonly questionSegment?: CanonicalSpeechSegment; readonly settlements?: readonly VoiceSubmissionSettlement[]; readonly status: PetrinautAiVoiceModeContext["status"]; } @@ -125,10 +126,6 @@ const initialSnapshot: VoiceTurnSnapshot = { partialText: "", }; -const latestQuestion = ( - segments: CanonicalSpeechSegment[], -): CanonicalSpeechSegment | undefined => segments.at(-1); - export class VoiceTurnController { readonly #bridge: RealtimeBridge; readonly #listeners = new Set(); @@ -148,6 +145,7 @@ export class VoiceTurnController { #inputStateOnResume: Exclude | null = null; #inputTurnPending = false; #latencyCorrelationId: string | null = null; + #lastResponseQuestion: CanonicalSpeechSegment | null = null; #lastResponseSegments: CanonicalSpeechSegment[] = []; #outputCancellationPromise: Promise | null = null; #pauseRequested = false; @@ -268,6 +266,7 @@ export class VoiceTurnController { this.#inputStateOnResume = null; this.#inputTurnPending = false; this.#latencyCorrelationId = null; + this.#lastResponseQuestion = null; this.#lastResponseSegments = []; this.#outputCancellationPromise = null; this.#recordedLatencyEvents.clear(); @@ -453,6 +452,13 @@ export class VoiceTurnController { this.#session.speakCanonical([...this.#lastResponseSegments]); } + public repeatQuestion(): void { + if (!this.#snapshot.canRepeatQuestion || !this.#lastResponseQuestion) + return; + this.#update({ output: "waiting-for-tool" }); + this.#session.speakCanonical([this.#lastResponseQuestion]); + } + /** * Hands the turn to the user only after provider cancellation has cleared * input and output and the active response has reached a terminal state. @@ -509,7 +515,7 @@ export class VoiceTurnController { } public updateChat(update: ChatUpdate): void { - const question = latestQuestion(update.canonicalSegments); + const question = update.questionSegment; if (question && question.id !== this.#currentQuestionId) { this.#currentQuestionId = question.id; this.#update({ currentQuestion: question.text }); @@ -581,8 +587,9 @@ export class VoiceTurnController { this.#recordLatency("submission-settled", event.deliveryId); return; } + this.#lastResponseQuestion = event.questionSegment ?? null; this.#lastResponseSegments = [...event.segments]; - const question = event.segments.at(-1); + const responseEnd = event.segments.at(-1); if (event.speechCancelled) { const paused = this.#snapshot.input === "paused"; if (paused) { @@ -592,7 +599,7 @@ export class VoiceTurnController { input: paused ? "paused" : "listening", output: "interrupted", }); - if (question) this.#recordLatency("answer-ready", question.id); + if (responseEnd) this.#recordLatency("answer-ready", responseEnd.id); return; } if (event.type === "submission-stopped") { @@ -617,7 +624,7 @@ export class VoiceTurnController { input: paused ? "paused" : "listening", output: paused ? "interrupted" : "waiting-for-tool", }); - if (question) this.#recordLatency("answer-ready", question.id); + if (responseEnd) this.#recordLatency("answer-ready", responseEnd.id); } #handleSessionEvent(event: OpenAIRealtimeSessionEvent): void { @@ -883,7 +890,7 @@ export class VoiceTurnController { this.#snapshot = { ...snapshot, canReadFullResponse: canReplay && this.#lastResponseSegments.length > 0, - canRepeatQuestion: false, + canRepeatQuestion: canReplay && this.#lastResponseQuestion !== null, canTakeTurn: this.#canTakeTurn(snapshot), canReviseLastAnswer: this.#canReviseLastAnswer(snapshot), }; From 8f0d629d25e0bf8e36ea818d48fe8df819635e52 Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Fri, 4 Sep 2026 09:52:07 +0200 Subject: [PATCH 23/52] Record exact Voice question replay Amp-Thread-ID: https://ampcode.com/threads/T-01a06941-3ee7-7482-a82e-8a16c7141785 Co-authored-by: Amp --- apps/petrinaut-website/README.md | 6 ++++-- libs/@hashintel/brunch-agent/MISSION.md | 19 ++++++++++--------- .../donor-behavior-matrix.md | 10 +++++----- .../witness-blocker.md | 11 +++++------ .../@hashintel/petrinaut/docs/ai-assistant.md | 6 ++++-- 5 files changed, 28 insertions(+), 24 deletions(-) diff --git a/apps/petrinaut-website/README.md b/apps/petrinaut-website/README.md index 874201a3622..307d8bfa0fb 100644 --- a/apps/petrinaut-website/README.md +++ b/apps/petrinaut-website/README.md @@ -123,8 +123,10 @@ microphone for fresh capture. Its playback menu offers **Repeat question** and **Read full response**. Full-response replay becomes available once the matching response and audio output have both finished, enqueues all exact retained canonical segments in order, and is disabled during capture, submission, -cancellation, pause, and errors. **Repeat question** remains disabled because -canonical Brunch speech does not yet identify which segment is a question. +cancellation, pause, and errors. **Repeat question** has the same safety gates +and replays only exact question text carrying Brunch's non-interactive marker; +if the marker is missing, malformed, or does not match finalized prose, the +action stays disabled rather than guessing from the final segment. The browser sends its SDP offer to this app; the server initializes a trusted `gpt-realtime-2` audio-input/audio-output session through OpenAI's unified diff --git a/libs/@hashintel/brunch-agent/MISSION.md b/libs/@hashintel/brunch-agent/MISSION.md index 3993f775191..74a158b94c7 100644 --- a/libs/@hashintel/brunch-agent/MISSION.md +++ b/libs/@hashintel/brunch-agent/MISSION.md @@ -23,11 +23,12 @@ defines the accepted and rejected mechanics. Sections independent of canonical hydration may proceed. The hard-reload witness is blocked until the parent prevents its once-per-conversation hydration from overwriting a locally submitted turn. Prepare that witness but do not patch the parent defect here. Completed-transcript authority, half-duplex ownership, admission idempotency, -the cancellation barrier, and exact full-response replay have focused regression -coverage, including the interval between a canonical speech request and output -start. **Repeat question** is authorized but not yet implemented: only the -approved durable Brunch marker may enable it, and the final text segment remains -invalid question authority. Proof item 5 is complete +the cancellation barrier, exact full-response replay, and exact marked-question +replay have focused regression coverage, including the interval between a +canonical speech request and output start. **Repeat question** is enabled only +for an approved durable Brunch marker whose exact text appears in finalized +assistant prose from the same message; the final text segment remains invalid +question authority. Proof item 5 is complete only for the supported client-tool-result path: Flue signals persist each Voice-origin tool-call id beside its output, and canonical projection reconstructs multiple surviving origins. Direct spoken user attribution is @@ -99,13 +100,13 @@ This mission closes the Voice safety and UX-parity stratum on the parent's route ### Product-manager litmus -**Release note:** Voice now submits only what the microphone actually transcribed, waits for a safe **Your turn** handoff before listening over Brunch, and can replay the exact full response. Client-tool Voice origins survive canonical reopen. **Repeat question** will replay only exact text carrying the approved Brunch-owned marker; restoring the Voice chip on direct spoken user messages remains blocked on an upstream Flue user-metadata contract. +**Release note:** Voice now submits only what the microphone actually transcribed, waits for a safe **Your turn** handoff before listening over Brunch, and can replay the exact full response or exact Brunch-marked question. Client-tool Voice origins survive canonical reopen. Restoring the Voice chip on direct spoken user messages remains blocked on an upstream Flue user-metadata contract. -**Demo script:** run `yarn dev:brunch` and select the Brunch preview. Speak one answer and see exactly one matching user turn. While Brunch is speaking, confirm the microphone remains closed, choose **Your turn**, wait for the handoff, and speak again. Use the playback menu to read the full response exactly; **Repeat question** remains unavailable until Brunch provides a deterministic canonical question marker. Start another turn, press durable **Stop** before settlement, and see a stopped turn rather than a Voice error. After the parent hydration blocker is fixed, hard-reload the settled conversation and confirm the canonical turn remains without resubmission or replay; direct-user Voice-chip restoration additionally waits on the Flue projection seam. +**Demo script:** run `yarn dev:brunch` and select the Brunch preview. Speak one answer and see exactly one matching user turn. While Brunch is speaking, confirm the microphone remains closed, choose **Your turn**, wait for the handoff, and speak again. After the response and audio settle, use the playback menu to read the full response exactly and repeat only the exact Brunch-marked question; a missing or unmatched marker keeps that action disabled. Start another turn, press durable **Stop** before settlement, and see a stopped turn rather than a Voice error. After the parent hydration blocker is fixed, hard-reload the settled conversation and confirm the canonical turn remains without resubmission or replay; direct-user Voice-chip restoration additionally waits on the Flue projection seam. **Previously impossible:** model-generated function arguments rather than completed audio transcription could become the answer; an accepted transcript could complete after canonical speech was requested but before output started; assistant playback could create a false user turn; cancellation could reopen capture before the provider settled; replay controls and multi-origin client-tool Voice attribution were incomplete. -**Completion:** the implemented portions close when their tests and focused checks pass. **Repeat question** needs the Brunch-owned marker recorded below, and direct-user provenance needs the Flue re-entry seam recorded below. Mission acceptance additionally requires the real microphone, handoff, Stop, hard-reload, and same-origin route witness after the parent hydration fix. Mocked or server-only proof cannot substitute for that witness. +**Completion:** the implemented portions close when their tests and focused checks pass. Exact question replay uses the Brunch-owned marker recorded below; direct-user provenance still needs the Flue re-entry seam recorded below. Mission acceptance additionally requires the real microphone, handoff, Stop, hard-reload, and same-origin route witness after the parent hydration fix. Mocked or server-only proof cannot substitute for that witness. 1. **Completed-transcript authority and half-duplex ownership.** Realtime session configuration has no tools, no model-created semantic-VAD response, and no automatic interruption policy. Only a unique completed transcript can reach the shared panel submission path. Duplicate, empty, failed, unavailable, stale, canonical-speech-overlapping/pre-handoff, playback-overlapping, and over-limit transcripts do not submit and produce the specified passive or recoverable notice. Before sending `response.create`, a canonical speech request invalidates every unfinished accepted item, clears bridge/controller transcript state and provisional UI, and closes the microphone; a completion in the interval before output starts cannot submit or regain authority. **Your turn** opens only a post-barrier input turn. Oracle: transplanted-first cases in `openai-realtime-session.test.ts`, `realtime-brunch-bridge.test.ts`, `voice-turn-controller.test.ts`, `voice-interview-control.test.tsx`, and `voice-preview.integration.test.ts`. 2. **Idempotent admission.** Typed turns derive a stable key from the AI SDK message id; Voice turns derive it from connection epoch, item id, and content index. A repeated same-payload key converges on the original receipt, including `deduplicated: true`; a 409 `submission_conflict` surfaces the original `submissionId` without admitting another turn. An admission whose outcome cannot be established stays visibly ambiguous and is never automatically retried. Oracle: transport unit tests plus Voice integration tests asserting one admitted submission rather than one `send()` invocation. @@ -114,7 +115,7 @@ This mission closes the Voice safety and UX-parity stratum on the parent's route 5. **Durable Voice provenance.** An assistant message may retain multiple `voiceToolCallIds`; one failed sibling origin does not erase successful origins. Persisted Flue client-tool-result signals support deterministic reconstruction after hydration and reopen. Direct spoken user messages remain Voice-attributed only while live because the canonical snapshot omits their caller origin. Re-entry requires a supported Flue user-message metadata/idempotency projection; browser storage and user-text encoding are rejected. Oracle: snapshot projection and panel partial-failure tests for supported origins, plus the [blocker record](docs/evidence/implementations/mission-5-voice-safety-parity/provenance-blocker.md). 6. **Dormant ask removal.** If still present after restacking, the website does not register `brunchAskInteractiveTool` for Voice and canonical speech does not recognize `"brunch-ask"`. No spoken ask answer can enter a wait state the transport cannot resume. Oracle: registration/canonical-speech negative tests and a repository search showing no mounted Voice `brunch_ask` surface. 7. **Real witness and same-origin route.** After the parent hydration fix lands, a human performs one microphone turn, explicit interruption/handoff, durable Stop on an unsettled turn, and hard reload of a settled turn. The retained network route summary proves the absolute Flue `streamUrl` remains on the same-origin proxy. Oracle: `witness.md`, sanitized `voice-events.jsonl`, `network-routes.json`, canonical `flue-snapshot.json`, `settlements.json`, commit manifest, and hashes under `docs/evidence/implementations/mission-5-voice-safety-parity/`. -8. **Focused repository verification and truthful docs.** The requested four-workspace Turbo command passes. `apps/petrinaut-website/README.md` and `libs/@hashintel/petrinaut/docs/ai-assistant.md` describe half-duplex handoff, exact full-response replay, the blocked question action, Stop, transcript rejection, and the direct-user attribution limitation. If the published Petrinaut package changes, exactly one patch changeset covers it. Oracle: the command recorded in the PR and changeset inspection. +8. **Focused repository verification and truthful docs.** The requested four-workspace Turbo command passes. `apps/petrinaut-website/README.md` and `libs/@hashintel/petrinaut/docs/ai-assistant.md` describe half-duplex handoff, exact full-response and marked-question replay, Stop, transcript rejection, and the direct-user attribution limitation. If the published Petrinaut package changes, exactly one patch changeset covers it. Oracle: the command recorded in the PR and changeset inspection. ## Constraints diff --git a/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/donor-behavior-matrix.md b/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/donor-behavior-matrix.md index 5d4d4499d09..ecb8bb91c23 100644 --- a/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/donor-behavior-matrix.md +++ b/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/donor-behavior-matrix.md @@ -26,18 +26,18 @@ The owner selected half-duplex turn ownership on 2026-09-03: assistant output ow | #9528 | Parent-owned admission/Stop races, stream cancellation, hydration overwrite, client-tool classification, response/submission correlation, and PR/CI repairs | **Reject from successor scope** | Lu owns these defects; changing them here would make the stack compete with its parent. | Report blockers and restack onto fixes. The real reload witness remains blocked by hydration overwrite. | | #9496 | Serialized canonical speech queue, retained exact source segments, response/output terminal gating | **Adopt mechanics** | Replay and ordinary TTS need one lifecycle-safe queue, and exact text preserves canonical authority. | Implemented without a preparation/simplifier dependency; exact-segment and queue tests pass. | | #9496 | `canReadFullResponse`, `readFullResponse()`, exact full-response playback menu | **Adopt** | Exact full-response replay is supported by retained canonical segment identity. | Implemented with idle-state and matching response/output terminal gates. | -| #9496 | `canRepeatQuestion`, `repeatQuestion()` implemented as replaying the final canonical segment | **Block** | Canonical Brunch segments provide no deterministic question marker. The final segment may be ordinary prose and is not authority for question identity. | Keep the production action unavailable. Re-enter only through a supported Brunch-owned marker or correlation rule without restoring `brunch_ask` or structured questions. | +| #9496 | `canRepeatQuestion`, `repeatQuestion()`, and playback-menu action | **Adopt UX; reject final-segment inference** | The final segment may be ordinary prose and is not authority for question identity. The approved `brunch_mark_question` data marker now supplies deterministic identity without accepting an answer. | Implemented by replaying only exact marked text found in finalized prose from the same assistant message; a missing or unmatched marker leaves the action disabled. | | #9496 | Realtime-generated concise response preparation or any fallback that rewrites canonical text | **Reject** | Response simplification is a non-goal and violates exact canonical speech. | Tests compare retained segment ids and exact queued strings; no preparation API remains on this path. | | #9500 | No Realtime tools, `tool_choice: "none"`, semantic VAD `create_response: false` | **Adopt** | Realtime detects/transcribes and renders supplied TTS only; it must not generate user meaning. | Implemented in policy, session, and production-preview integration tests. | | #9500 | Only `conversation.item.input_audio_transcription.completed` can submit; model function arguments ignored | **Adopt** | Shape validation cannot prove model-generated arguments match the audio. | Implemented with current-turn speech-boundary, stale, reordered, and late-output rejection tests. | | #9500 | Transcript identity `(connectionEpoch, itemId, contentIndex)`, stable submission id, trim plus Unicode whitespace collapse, 32,000-code-point limit | **Adopt** | This gives one deterministic logical Voice delivery and one normalization boundary. | Implemented through path B; the panel preserves the bridge-normalized payload unchanged. | | #9500 | Explicit duplicate, empty, failed, unavailable, and over-limit rejection; passive/recoverable not-heard UI; provisional display only | **Adopt** | Rejected audio must never become a turn, while ordinary silence/failure must not poison the session. | Implemented with reason-specific bridge/controller UI coverage. | | #9500 | Silently settling ownership by discarding every playback-overlapping utterance without an explicit handoff | **Supersede** | It avoids echo but leaves users without a deliberate way to take the turn. | Use #9512 half-duplex `canTakeTurn`/`takeTurn()` and reject all speech captured before the completed handoff. | -| #9500 | `brunch_ask` answer/tool correlation and preparation code inherited from its base | **Reject** | Structured questions and response preparation are excluded. | Correlate the Voice delivery to its path-B submission and canonical response facts; question replay remains blocked without a supported marker. | +| #9500 | `brunch_ask` answer/tool correlation and preparation code inherited from its base | **Reject** | Structured questions and response preparation are excluded. | Correlate the Voice delivery to its path-B submission and canonical response facts; exact question replay uses the non-interactive marker instead. | | #9507 | Temporary `brunch_ask` registration, widget, correlated spoken ask answer, transcript formatting | **Reject entire shim** | The current transport only admits the supported follow-up set; a spoken ask can otherwise wait forever. Structured questions are a separate product decision. | Remove or gate dormant `brunchAskInteractiveTool` and `"brunch-ask"` canonical-speech recognition only if still present after restack. | | #9512 | Half-duplex `canTakeTurn`, `takeTurn()`, `"cancelling"` output state, and **Your turn** control | **Adopt by owner decision** | It makes output/input ownership explicit and prevents assistant playback from becoming a false user turn. | Implemented through the public Voice store and production panel registration path. | | #9512 | Promise-returning `cancelOutput()` that waits for input/output clears, matching acknowledgements, and response terminal events | **Adopt** | The microphone cannot safely reopen on a fire-and-forget cancel. | Implemented with acknowledgement/race tests, latest-mute behavior, and fresh post-handoff capture. | -| #9512 | Full-response replay availability tied to exact retained source, terminal response, and output completion | **Adopt with #9496 mechanics** | This closes replay races without changing canonical content. | Implemented against parent segment/submission correlation. Question replay is excluded until a supported marker exists. | +| #9512 | Replay availability tied to exact retained source, terminal response, and output completion | **Adopt with #9496 mechanics** | This closes replay races without changing canonical content. | Implemented against parent segment/submission correlation for exact full-response and marked-question replay. | | #9512 | Voice answer icon/provenance before interactive answers | **Partially adopt; blocked for direct user turns** | Live attribution is useful but one origin per assistant message is insufficient after coalesced or sibling Voice deliveries. Flue's client-tool result signal can durably carry those origins. Its direct-user delivery and snapshot types expose no caller metadata or idempotency key, so a direct spoken user message cannot be identified after reopen without a forbidden second store or text encoding. | Keep `voiceToolCallIds`, preserve successful siblings on partial failure, and reconstruct supported tool-result origins from Flue signals. Re-enter direct-user attribution only when Flue provides a supported durable correlation seam. | | #9512 | App-local agent topology, temporary ask UI, response preparation, or donor-specific host composition | **Reject** | The parent owns the one Flue route and current host composition; these mechanisms are obsolete or non-goals. | Reuse only state-machine, cancellation, replay, and attribution behavior. | @@ -48,7 +48,7 @@ The owner selected half-duplex turn ownership on 2026-09-03: assistant output ow | One path-B Flue admission route | `local-storage-demo-app.tsx`, `brunch-panel-transport.ts`, transport `src/index.ts` | `brunch-panel-transport.test.ts`, `chat-transport.test.ts` | `voice-preview.integration.test.ts` crosses completed transcript → panel submission → Flue transport → canonical speech | **Implemented**; parent defects remain downstack | | Stable admission identity and typed outcomes | transport `src/index.ts`, `brunch-panel-transport.ts`, `submitVoiceInputWithAdmission`, `realtime-brunch-bridge.ts` | transport admission cases; bridge/controller cases for rejected, conflict, ambiguous, and local abort | production preview carries 409 conflict, 500 ambiguity, and local abort through transport → tracker → `submitVoiceInputWithAdmission` → bridge; each observes one `send()`, and local abort never invokes durable `FlueClient.abort()` | **Implemented** | | Exact canonical TTS queue and full-response replay | `openai-realtime-session.ts`, `voice-turn-controller.ts`, Petrinaut playback menu | session queue/cancellation cases; controller exact-segment and terminal-gating cases; panel action tests | real host registration exposes `readFullResponse`; panel forwards it through `voiceSessionStore` | **Implemented** | -| Repeat-question replay | none: no supported question identity exists | controller and host-registration negative cases | Brunch Voice host omits `repeatQuestion`; panel leaves the generic item disabled | **Blocked**, not mislabeled as final-segment replay | +| Exact question replay | core `brunch_mark_question` tool/data contract; transport hidden-tool projection; `canonical-speech.ts`; bridge/controller; Voice host callback | core marker tests; live/snapshot transport projection tests; canonical selector malformed/unmatched/cross-message cases; controller final-segment negative and exact-marker replay cases | real Flue integration persists and reopens `data-brunch-question` while hiding the marker tool; controlled Voice preview carries the marker through response correlation and queues only the exact question; panel host forwards the action | **Implemented**; missing or unmatched markers fail closed | | Disabled Realtime generation/tools | `openai-voice-policy.ts`, `openai-realtime-session.ts` | policy/session tests reject tools and function arguments | controlled production preview negotiates the server policy and emits only canonical speech | **Implemented** | | Completed-transcript authority | `openai-realtime-session.ts`, `realtime-brunch-bridge.ts` | missing/stale/reordered boundary, keyed identity, normalization, duplicate/failure/limit, canonical-request-before-output, and late-output cases | controlled production preview proves a pre-request item cannot call Flue `send()` before output starts and only fresh post-handoff input submits through path B | **Implemented**; provider-valid boundaryless commits are intentionally rejected by mission policy | | Half-duplex acknowledged handoff | `openai-realtime-session.ts`, `voice-turn-controller.ts`, Voice public store/dock | canonical-request invalidation, input/output clear acknowledgement, targeted response terminal, latest mute, stale/pre-handoff rejection | panel registration tests exercise **Your turn**; preview integration proves the microphone closes before `response.create` and fresh post-handoff capture submits once | **Implemented** | @@ -63,7 +63,7 @@ The owner selected half-duplex turn ownership on 2026-09-03: assistant output ow | Transcript authority | Transplanted-first session, bridge, controller, and integration regressions pass on path B. | Implemented. Matching current-turn `speech_started`, stale/reordered boundaries, canonical-speech-request and late-output invalidation, provisional UI clearing, exact bridge normalization, and unchanged panel payload are covered. | | Admission idempotency | Typed and Voice logical replays converge on one `submissionId`; conflict metadata is narrowed safely; ambiguous outcome does not retry. | Implemented. Transport tests cover stable typed/Voice keys, deduplicated receipts, 409 conflicts, and non-retried ambiguity; production-path integration preserves the original conflict `submissionId` and keeps local admission abort distinct from durable abort. | | Cancellation barrier | Buffer acknowledgements and targeted response terminals settle before capture; stale/pre-handoff audio cannot submit; latest mute choice wins. | Implemented. Session/controller races cover the barrier and mute preference; panel registration and configured-app Stop cases cover the production host seams. | -| Canonical replay | Exact segment queue and playback menu pass availability/race tests without a simplifier. | Full-response replay is implemented with exact segments and terminal gating. **Repeat question** is blocked because no supported deterministic question marker exists; the production action must remain unavailable rather than replaying the final prose segment. | +| Canonical replay | Exact segment queue and playback menu pass availability/race tests without a simplifier. | Full-response replay preserves every exact segment. **Repeat question** uses only a durable non-interactive Brunch marker that exactly matches finalized prose in the same assistant message; final-segment inference remains rejected. Both actions share terminal/output/input gating. | | Durable provenance | Multiple origins and partial failure survive projection, hydration, and reopen without user-text encoding. | Partially implemented for assistant client-tool results through persisted Flue signals; multiple sibling origins survive projection and partial failure. Direct spoken user attribution is blocked because Flue 2.0.3 snapshots do not expose caller idempotency or user-message metadata. The rejected browser store would violate mission authority. | | Dormant ask | No mounted Voice ask capability remains, or the parent commit that removed it is recorded. | Implemented exclusion. Canonical speech ignores `brunch_ask`, and a configured-app registration test proves the production Brunch assistant supplies no ask tool. Dormant source remains unmounted. | | Real witness | Microphone, handoff, unsettled Stop, reload, canonical snapshot, settlement, and same-origin absolute-`streamUrl` artifacts are retained with hashes. | Blocked by the parent's hydration overwrite. | diff --git a/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/witness-blocker.md b/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/witness-blocker.md index d13f8632262..561a23ec530 100644 --- a/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/witness-blocker.md +++ b/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/witness-blocker.md @@ -5,12 +5,11 @@ The real Voice witness has **not** been run and no witness bundle is claimed. Completed-transcript authority, admission idempotency, half-duplex handoff, acknowledged cancellation, exact full-response replay, durable Stop, dormant-ask -exclusion, and the supported client-tool portion of Voice provenance have -focused automated coverage. **Repeat question** is separately blocked by the -lack of a deterministic canonical question marker. Automated coverage cannot -replace the microphone, handoff, unsettled Stop, hard-reload, and network-route -witness required for mission acceptance. Direct-user Voice attribution has a -separate [Flue projection blocker](provenance-blocker.md). +exclusion, exact Brunch-marked question replay, and the supported client-tool +portion of Voice provenance have focused automated coverage. Automated coverage +cannot replace the microphone, handoff, unsettled Stop, hard-reload, and +network-route witness required for mission acceptance. Direct-user Voice +attribution has a separate [Flue projection blocker](provenance-blocker.md). The parent branch still runs canonical hydration once per conversation in a way that can overwrite a locally submitted turn. The successor may not fix that diff --git a/libs/@hashintel/petrinaut/docs/ai-assistant.md b/libs/@hashintel/petrinaut/docs/ai-assistant.md index 5083a52e8a7..5d12436c413 100644 --- a/libs/@hashintel/petrinaut/docs/ai-assistant.md +++ b/libs/@hashintel/petrinaut/docs/ai-assistant.md @@ -69,8 +69,10 @@ Provisional words remain display-only until the provider completes their transcr Every session control lives in the dock: **Show transcription in chat** and **Voice playback options** on the left, and the available handoff, microphone, recovery, and end actions on the right. **Read full response** becomes available after the matching response and speech have both finished -and replays every exact retained canonical segment in order. **Repeat question** stays disabled in -the Brunch preview because its canonical speech does not yet identify which segment is a question. +and replays every exact retained canonical segment in order. **Repeat question** uses the same +availability gates and replays only exact question text explicitly marked by Brunch. It stays +disabled when that marker is missing or does not match finalized assistant text rather than +guessing that the final segment is a question. Playback stays unavailable during active capture, submission, cancellation, pause, and errors. **Mute microphone** becomes **Unmute microphone** once muted, and your latest choice applies when a handoff settles. **Resume voice mode** replaces the microphone action while a session is paused, and **Reconnect voice mode** replaces it From 54ab6aba1e3ff18b22916fff34f6d07fa6816b38 Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Fri, 4 Sep 2026 09:54:54 +0200 Subject: [PATCH 24/52] Satisfy Voice replay lint checks Amp-Thread-ID: https://ampcode.com/threads/T-01a06941-3ee7-7482-a82e-8a16c7141785 Co-authored-by: Amp --- .../src/main/app/voice-interview/canonical-speech.ts | 8 ++++---- .../app/voice-interview/voice-turn-controller.test.ts | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.ts b/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.ts index 53e9fc63c49..fd466e1448d 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/canonical-speech.ts @@ -46,10 +46,6 @@ const createSegment = ( }; }; -export const selectCanonicalSpeechSegments = ( - messages: PetrinautAiMessage[], -): CanonicalSpeechSegment[] => selectCanonicalSpeech(messages).segments; - export interface CanonicalSpeechSelection { readonly questionSegment?: CanonicalSpeechSegment; readonly segments: CanonicalSpeechSegment[]; @@ -115,3 +111,7 @@ export const selectCanonicalSpeech = ( return { questionSegment, segments }; }; + +export const selectCanonicalSpeechSegments = ( + messages: PetrinautAiMessage[], +): CanonicalSpeechSegment[] => selectCanonicalSpeech(messages).segments; diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts index c8f404a7e23..170602de8b3 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts @@ -571,7 +571,7 @@ describe("VoiceTurnController", () => { "response-prose", "The approver is recorded. I can explain the escalation path.", ); - const markedQuestion: CanonicalSpeechSegment = { + const exactQuestion: CanonicalSpeechSegment = { ...question("marked-question", "Who approves release?"), messageId: finalProse.messageId, source: "assistant-question", @@ -580,7 +580,7 @@ describe("VoiceTurnController", () => { harness.emitBridge({ deliveryId: "voice-1", - questionSegment: markedQuestion, + questionSegment: exactQuestion, segments: [context, finalProse], type: "canonical-response-ready", }); @@ -611,7 +611,7 @@ describe("VoiceTurnController", () => { expect(harness.session.speakCanonical).toHaveBeenCalledOnce(); expect(harness.session.speakCanonical).toHaveBeenCalledWith([ - markedQuestion, + exactQuestion, ]); }); From b0845464bf7c10c4ba00c9f680ca2a3f5c485e99 Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Fri, 4 Sep 2026 09:59:49 +0200 Subject: [PATCH 25/52] Refresh Mission 5 acceptance records Amp-Thread-ID: https://ampcode.com/threads/T-01a06941-3ee7-7482-a82e-8a16c7141785 Co-authored-by: Amp --- libs/@hashintel/brunch-agent/MISSION.md | 6 +++--- .../donor-behavior-matrix.md | 12 ++++++++---- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/libs/@hashintel/brunch-agent/MISSION.md b/libs/@hashintel/brunch-agent/MISSION.md index 74a158b94c7..a6a310bce77 100644 --- a/libs/@hashintel/brunch-agent/MISSION.md +++ b/libs/@hashintel/brunch-agent/MISSION.md @@ -38,10 +38,10 @@ browser-correlation implementation would have violated the explicit second-durable-store stop condition. The real witness remains blocked by the parent hydration defect. -The 2026-09-04 corrective verification covers the current 44-file successor -diff against #9528: the four race suites pass 94/94 tests, the filtered +The 2026-09-04 corrective verification covers the current 58-file successor +diff against #9528: the four focused race cases pass 4/4 tests, the filtered production admission-outcome cases pass 3/3 tests, and the requested Turbo run -passes 30/30 tasks, including 243/243 website tests. Architecture validation +passes 30/30 tasks, including 249/249 website tests. Architecture validation passes with 62 layers, 297 edges, 614 files, 63 generated pages, and 31 authored pages. `git diff --check` and formatting of all formatter-owned successor files pass. The root formatter remains red only outside this branch's diff: two diff --git a/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/donor-behavior-matrix.md b/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/donor-behavior-matrix.md index ecb8bb91c23..d4e35f922e4 100644 --- a/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/donor-behavior-matrix.md +++ b/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/donor-behavior-matrix.md @@ -71,16 +71,20 @@ The owner selected half-duplex turn ownership on 2026-09-03: assistant output ow ## Corrective verification -Fresh local checks on 2026-09-04 cover the 44-file successor diff against the +Fresh local checks on 2026-09-04 cover the 58-file successor diff against the verified #9528 head `036fb06ee83e840fd7a87dd1bc758a2dd49ddd4c`: | Command | Result | | --- | --- | -| `yarn workspace @apps/petrinaut-website test:unit src/main/app/voice-interview/openai-realtime-session.test.ts src/main/app/voice-interview/realtime-brunch-bridge.test.ts src/main/app/voice-interview/voice-turn-controller.test.ts src/main/app/voice-interview/voice-preview.integration.test.ts` | Exit 0; 4 files and 94/94 tests passed. This includes the request-before-output race at session, bridge, controller, and production integration layers. | +| `yarn workspace @apps/petrinaut-website test:unit src/main/app/voice-interview/openai-realtime-session.test.ts src/main/app/voice-interview/realtime-brunch-bridge.test.ts src/main/app/voice-interview/voice-turn-controller.test.ts src/main/app/voice-interview/voice-preview.integration.test.ts -t 'invalidates accepted input before requesting canonical speech output\|rejects unfinished input as soon as canonical speech is requested\|clears capture when canonical speech is requested before output starts\|bridges one completed transcript through Brunch and back to canonical half-duplex audio'` | Exit 0; 4/4 selected tests passed and 91 unrelated tests were filtered across four files. This covers the request-before-output race at session, bridge, controller, and production integration layers. | | `yarn workspace @apps/petrinaut-website test:unit src/main/app/voice-interview/voice-preview.integration.test.ts -t 'ambiguous Flue admission\|conflicting submission\|local admission abort'` | Exit 0; 3/3 selected tests passed and 2 unrelated tests were filtered. Conflict retains the original `submissionId`; local abort remains distinct from durable abort; every path calls `send()` once. | -| `mise exec -- yarn exec turbo run lint:tsc lint:eslint test:unit build --filter @apps/brunch-agent --filter @apps/petrinaut-website --filter @hashintel/petrinaut --filter @hashintel/brunch-agent-transport-aisdk` | Exit 0; 30/30 tasks passed, including 31 website test files and 243/243 website tests. Website ESLint retains one non-failing warning at `voice-interview-control.tsx:480`. | +| `yarn workspace @hashintel/brunch-agent test:unit test/question-marker.test.ts` | Exit 0; 9/9 exact question-marker tests passed. | +| `yarn workspace @hashintel/brunch-agent-transport-aisdk test:unit` | Exit 0; 22/22 transport tests passed, including live and snapshot marker projection. | +| `yarn workspace @apps/brunch-agent test:unit test/petrinaut-chat.test.ts` | Exit 0; 1/1 real-Flue integration test passed, including exact marker persistence through fresh-process reopen while marker tools remain hidden. | +| `yarn workspace @hashintel/petrinaut test:unit --run src/ui/views/Editor/panels/ai-assistant-panel.test.tsx` | Exit 0; 40/40 production host-registration and panel tests passed. | +| `mise exec -- yarn exec turbo run lint:tsc lint:eslint test:unit build --filter @apps/brunch-agent --filter @apps/petrinaut-website --filter @hashintel/petrinaut --filter @hashintel/brunch-agent-transport-aisdk` | Exit 0; 30/30 tasks passed, including 31 website test files and 249/249 website tests. Website ESLint retains one non-failing warning at `voice-interview-control.tsx:485`. | | `yarn workspace @local/petrinaut-arch-docs lint:arch-docs` | Exit 0; 62 layers, 297 edges, 614 files, 63 generated pages, and 31 authored pages. | -| `yarn exec oxfmt --check $(cat /tmp/fe1580-files)` over the 44 child-owned paths | Exit 0; all 40 formatter-owned files passed. The remaining four Brunch Markdown records are intentionally excluded by `oxfmt.config.ts` and pass repository whitespace validation. | +| `git diff --name-only -z ln/fe-1574-direct-voice-flue...HEAD \| xargs -0 yarn exec oxfmt --check` | Exit 0; all 52 formatter-owned files in the 58-file successor diff passed. The six unmatched documentation files are excluded by `oxfmt.config.ts` and pass repository whitespace validation. | | `git diff --check` | Exit 0. | | `yarn lint:format` | Exit 1 outside the successor diff only: parent-owned `apps/brunch-agent/src/conversation/identity.ts` and `apps/brunch-agent/test/conversation-identity.test.ts`, plus unrelated untracked `.cursor/plans/fe-1574_mission_recut_6f23f7cd.plan.md`. They were not modified. | From f7e0be9a1c881b837bae3c136a2c7b5cd97f5af6 Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Fri, 4 Sep 2026 11:17:30 +0200 Subject: [PATCH 26/52] Remove Brunch status from the composer Keep Voice controls and durable Stop wiring while removing the conversation status mount. Preserve the dormant brunch_ask exclusion after adopting the latest parent head. Amp-Thread-ID: https://ampcode.com/threads/T-01a06941-3ee7-7482-a82e-8a16c7141785 Co-authored-by: Amp --- .../local-storage-demo-app.test.tsx | 14 ++-- .../local-storage-demo-app.tsx | 76 ------------------- 2 files changed, 5 insertions(+), 85 deletions(-) diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx index f2dc2eff39f..21eb06313c8 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx @@ -1,7 +1,7 @@ /** * @vitest-environment jsdom */ -import { act, cleanup, render, screen, waitFor } from "@testing-library/react"; +import { act, cleanup, render, waitFor } from "@testing-library/react"; import { isValidElement, type ReactNode } from "react"; import { afterEach, describe, expect, test, vi } from "vitest"; @@ -225,14 +225,10 @@ describe("local storage demo Brunch voice integration", () => { await expect(aiAssistant.requestStop?.()).resolves.toBe("stop-requested"); expect(abort).toHaveBeenCalledOnce(); expect(localPlaybackCancellation).not.toHaveBeenCalled(); - await waitFor(() => { - const currentAssistant = - renderedPetrinaut.aiAssistant as PetrinautAiAssistant; - const status = currentAssistant.renderComposerControl?.({} as never); - if (!status) throw new Error("Expected the Brunch status control."); - render(status); - expect(screen.getByText("Last Brunch response stopped.")).not.toBeNull(); - }); + expect( + (renderedPetrinaut.aiAssistant as PetrinautAiAssistant) + .renderComposerControl, + ).toBeUndefined(); rendered.unmount(); localPlaybackCancellation.mockRestore(); diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx index 92ddf4ec804..42d20127b44 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx @@ -25,7 +25,6 @@ import { import { DefaultChatTransport, Petrinaut, - type PetrinautAiInteractiveTool, type PetrinautAiMessage, type PetrinautAiStopResult, type PetrinautAiVoiceMode, @@ -158,11 +157,6 @@ const createHandle = (net: SDCPNInLocalStorage): PetrinautDocHandle => const brunchPrincipal = getOrCreateBrunchPrincipal(); -/** Every widget here must answer a tool named in `brunchClientToolNames`. */ -export const brunchInteractiveTools: readonly PetrinautAiInteractiveTool[] = [ - brunchAskInteractiveTool, -]; - const stockChatTransport = new DefaultChatTransport({ api: brunchPreviewConfig.chatEndpoint, headers: () => ({ @@ -227,64 +221,6 @@ const createActiveHandle = (net: SDCPNInLocalStorage): ActiveHandle => ({ fallbackNet: net, }); -type FlueChatHistory = ReturnType; - -const errorStatus = (error: Error | undefined): number | undefined => { - if ( - error !== undefined && - "status" in error && - typeof error.status === "number" - ) { - return error.status; - } - return undefined; -}; - -const BrunchConversationStatus = ({ - error, - latestSettlement, - phase, - refresh, -}: Pick< - FlueChatHistory, - "error" | "latestSettlement" | "phase" | "refresh" ->) => { - if (phase === undefined) return null; - - const label = - phase === "loading" - ? "Loading Brunch conversation…" - : phase === "connecting" - ? "Reconnecting to Brunch…" - : phase === "absent" - ? "New Brunch conversation" - : phase === "error" - ? errorStatus(error) === 401 || errorStatus(error) === 403 - ? "Brunch access was denied." - : "Brunch conversation unavailable." - : phase === "closed" - ? "Brunch conversation closed." - : latestSettlement?.outcome === "aborted" - ? "Last Brunch response stopped." - : latestSettlement?.outcome === "failed" - ? "Last Brunch response failed." - : "Brunch conversation ready."; - - return ( - - {label} - {phase === "error" && ( - <> - {" "} - - - )} - - ); -}; - /** * The demo's own palette command, registered beside Petrinaut's: picking it * in the palette starts a fresh net. @@ -562,18 +498,6 @@ export const LocalStorageDemoApp = ({ requestStop: () => requestFlueStop(flueClientPromise, conversationTracker), }), - ...(flueClientPromise === null - ? {} - : { - renderComposerControl: () => ( - - ), - }), messages: flueClientPromise === null ? currentNetId From 820cce06f9ccc90590f90471fcc03aff309d985c Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Fri, 4 Sep 2026 11:19:18 +0200 Subject: [PATCH 27/52] Reconcile Voice safety with the latest Flue parent Retain multi-submission response correlation, direct completed-transcript authority, durable stopped-turn handling, idempotent sends, and hidden question markers after restacking onto the latest parent head. Amp-Thread-ID: https://ampcode.com/threads/T-01a06941-3ee7-7482-a82e-8a16c7141785 Co-authored-by: Amp --- .../openai-realtime-session.test.ts | 23 ---------------- .../realtime-brunch-bridge.test.ts | 5 +--- .../voice-preview.integration.test.ts | 2 +- .../voice-turn-controller.test.ts | 10 +++---- .../voice-interview/voice-turn-controller.ts | 26 +++++++++---------- .../test/chat-transport.test.ts | 2 ++ .../transport-aisdk/test/transcript.test.ts | 1 + 7 files changed, 23 insertions(+), 46 deletions(-) diff --git a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts index d9a00d030d7..332bf6a3f73 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/openai-realtime-session.test.ts @@ -570,29 +570,6 @@ describe("OpenAIRealtimeSession", () => { ]); }); - test("closes a stopped function call without requesting speech", async () => { - const harness = createHarness(); - await harness.session.connect(); - const channel = harness.channels[0]!; - const sentBefore = sentEvents(channel).length; - - harness.session.completeFunctionCallWithoutResponse( - "call-stopped", - "aborted", - ); - - expect(sentEvents(channel).slice(sentBefore)).toEqual([ - { - type: "conversation.item.create", - item: { - type: "function_call_output", - call_id: "call-stopped", - output: JSON.stringify({ response_text: [], outcome: "aborted" }), - }, - }, - ]); - }); - test("preserves exact canonical whitespace while rejecting blank speech", async () => { const harness = createHarness(); await harness.session.connect(); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.test.ts index 5e852d2aeeb..03b8d570b83 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/realtime-brunch-bridge.test.ts @@ -24,7 +24,7 @@ const segment = ( messageId: `message-${id}`, partId: id, source: "assistant-text", - ...(submissionId === undefined ? {} : { submissionId }), + ...(submissionId === undefined ? {} : { submissionIds: [submissionId] }), text, }); @@ -56,7 +56,6 @@ const failedTranscript = ( const createHarness = () => { let listener: ((event: OpenAIRealtimeSessionEvent) => void) | undefined; const session = { - completeFunctionCall: vi.fn(), speakCanonical: vi.fn(), subscribe: vi.fn((next: (event: OpenAIRealtimeSessionEvent) => void) => { listener = next; @@ -120,7 +119,6 @@ describe("RealtimeBrunchBridge", () => { expect(harness.submitInterviewAnswer).not.toHaveBeenCalled(); expect(harness.session.speakCanonical).not.toHaveBeenCalled(); - expect(harness.session.completeFunctionCall).not.toHaveBeenCalled(); expect(harness.events).toEqual([]); }); @@ -393,7 +391,6 @@ describe("RealtimeBrunchBridge", () => { const deliveryId = createRealtimeSubmissionId(transcriptKey(7)); expect(harness.session.speakCanonical).toHaveBeenCalledWith([correlated]); - expect(harness.session.completeFunctionCall).not.toHaveBeenCalled(); expect(harness.events.map(({ type }) => type)).toEqual([ "submission-started", "submission-admitted", diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts index 3210ddf5f66..b067de20f55 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-preview.integration.test.ts @@ -506,7 +506,7 @@ describe("controlled voice preview", () => { const correlateResponse = (segment: CanonicalSpeechSegment) => initialSegmentIds.has(segment.id) ? segment - : { ...segment, submissionId: admission.submissionId }; + : { ...segment, submissionIds: [admission.submissionId] }; const correlatedSegments = responseSelection.segments.map(correlateResponse); controller.updateChat({ diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts index 170602de8b3..1d59f5ce45e 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.test.ts @@ -730,18 +730,18 @@ describe("VoiceTurnController", () => { harness.emitBridge({ answer: "Stop this one.", - callId: "call-1", + deliveryId: "voice-1", type: "submission-started", }); harness.emitBridge({ answer: "Stop this one.", - callId: "call-1", + deliveryId: "voice-1", type: "submission-accepted", }); harness.advanceTime(40); - harness.emitBridge({ callId: "call-1", type: "submission-settled" }); + harness.emitBridge({ deliveryId: "voice-1", type: "submission-settled" }); harness.emitBridge({ - callId: "call-1", + deliveryId: "voice-1", outcome: "aborted", type: "submission-stopped", }); @@ -752,7 +752,7 @@ describe("VoiceTurnController", () => { output: "idle", }); expect(harness.latencyEvents).toContainEqual({ - correlationId: "call-1", + correlationId: "voice-1", elapsedMs: 40, name: "submission-settled", }); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts index 84b24448e59..1be24210183 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-turn-controller.ts @@ -587,6 +587,19 @@ export class VoiceTurnController { this.#recordLatency("submission-settled", event.deliveryId); return; } + if (event.type === "submission-stopped") { + // Brunch was stopped before it replied: nothing to speak, and the + // interviewer is free to listen again. + const pausedWhileStopped = this.#snapshot.input === "paused"; + if (pausedWhileStopped) { + this.#inputStateOnResume = "listening"; + } + this.#update({ + input: pausedWhileStopped ? "paused" : "listening", + output: "idle", + }); + return; + } this.#lastResponseQuestion = event.questionSegment ?? null; this.#lastResponseSegments = [...event.segments]; const responseEnd = event.segments.at(-1); @@ -602,19 +615,6 @@ export class VoiceTurnController { if (responseEnd) this.#recordLatency("answer-ready", responseEnd.id); return; } - if (event.type === "submission-stopped") { - // Brunch was stopped before it replied: nothing to speak, and the - // interviewer is free to listen again. - const pausedWhileStopped = this.#snapshot.input === "paused"; - if (pausedWhileStopped) { - this.#inputStateOnResume = "listening"; - } - this.#update({ - input: pausedWhileStopped ? "paused" : "listening", - output: "idle", - }); - return; - } const paused = this.#snapshot.input === "paused"; if (paused) { this.#inputStateOnResume = "listening"; diff --git a/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/chat-transport.test.ts b/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/chat-transport.test.ts index 21aa1292efa..dc8edeb898b 100644 --- a/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/chat-transport.test.ts +++ b/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/chat-transport.test.ts @@ -114,6 +114,7 @@ test("admits one user message and projects a finite per-turn stream", async () = expect(send).toHaveBeenCalledOnce(); expect(send).toHaveBeenCalledWith({ + idempotencyKey: "ai-sdk:user-1", message: { kind: "user", body: "Run the transport tracer." }, signal: undefined, }); @@ -158,6 +159,7 @@ test("admits one client-tool result signal and resumes its assistant id", async ); expect(send).toHaveBeenCalledWith({ + idempotencyKey: "ai-sdk-tool:assistant-original:tool-1", message: { kind: "signal", type: "client-tool-result", diff --git a/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/transcript.test.ts b/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/transcript.test.ts index 7ab457e4e62..c41a8de7f0f 100644 --- a/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/transcript.test.ts +++ b/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/transcript.test.ts @@ -31,6 +31,7 @@ const snapshotWithPendingClientTool: FlueConversationSnapshot = { const projectionOptions = { clientToolNames: new Set(["readPetrinautDoc"]), + hiddenToolNames: new Set(["brunch_mark_question"]), }; test("leaves an unfinished client tool available to run", () => { From 5ef5619c49eb46f0429c8b871c68f4ec356c7f53 Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Fri, 4 Sep 2026 11:20:59 +0200 Subject: [PATCH 28/52] Remove obsolete status dependencies Amp-Thread-ID: https://ampcode.com/threads/T-01a06941-3ee7-7482-a82e-8a16c7141785 Co-authored-by: Amp --- .../main/app/local-storage-demo/local-storage-demo-app.tsx | 4 ---- 1 file changed, 4 deletions(-) diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx index 42d20127b44..e9a648a1e5b 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.tsx @@ -538,11 +538,7 @@ export const LocalStorageDemoApp = ({ conversationId, currentNetId, flueClientPromise, - flueHistory.error, - flueHistory.latestSettlement, flueHistory.messages, - flueHistory.phase, - flueHistory.refresh, petrinautAiChatTransport, setAiMessagesByNetId, ], From 96a4b870a1070d3f2ae2ef164fc373b50c4a7b68 Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Fri, 4 Sep 2026 11:26:55 +0200 Subject: [PATCH 29/52] Type Brunch question marker mocks Amp-Thread-ID: https://ampcode.com/threads/T-01a06941-3ee7-7482-a82e-8a16c7141785 Co-authored-by: Amp --- .../packages/core/test/question-marker.test.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/libs/@hashintel/brunch-agent/packages/core/test/question-marker.test.ts b/libs/@hashintel/brunch-agent/packages/core/test/question-marker.test.ts index 581ebf7239d..b1da49f3049 100644 --- a/libs/@hashintel/brunch-agent/packages/core/test/question-marker.test.ts +++ b/libs/@hashintel/brunch-agent/packages/core/test/question-marker.test.ts @@ -10,8 +10,11 @@ import { BrunchQuestionDataSchema, BrunchQuestionInputSchema, parseBrunchQuestionData, + type BrunchQuestionData, } from "../src/question-marker"; +import type { FlueLogger } from "@flue/runtime"; + describe("the Brunch question marker", () => { test("defines one non-interactive tool and data-part identity", () => { expect(BRUNCH_QUESTION_TOOL_NAME).toBe("brunch_mark_question"); @@ -35,15 +38,15 @@ describe("the Brunch question marker", () => { }); test("writes the exact marker without terminating or waiting for an answer", async () => { - const writeQuestion = vi.fn(); + const writeQuestion = vi.fn<(question: BrunchQuestionData) => void>(); const tool = createBrunchQuestionMarkerTool(writeQuestion); const result = await tool.run({ data: { question: "Which line should run this order?" }, log: { - error: vi.fn(), - info: vi.fn(), - warn: vi.fn(), + error: vi.fn(), + info: vi.fn(), + warn: vi.fn(), }, toolCallId: "tool-question-1", }); From 75fa6957a9cc08db9c3d5be1daf75ce6cd6f9ab3 Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Fri, 4 Sep 2026 11:29:18 +0200 Subject: [PATCH 30/52] Restore the Voice witness and latency authority Amp-Thread-ID: https://ampcode.com/threads/T-01a06941-3ee7-7482-a82e-8a16c7141785 Co-authored-by: Amp --- libs/@hashintel/brunch-agent/MISSION.md | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/libs/@hashintel/brunch-agent/MISSION.md b/libs/@hashintel/brunch-agent/MISSION.md index a6a310bce77..d032b3a50e6 100644 --- a/libs/@hashintel/brunch-agent/MISSION.md +++ b/libs/@hashintel/brunch-agent/MISSION.md @@ -2,7 +2,7 @@ ## Status -**Live as of 2026-09-04** for [FE-1580](https://linear.app/hash/issue/FE-1580/harden-voice-safety-and-ux-on-the-unified-flue-route) on `kostandin/fe-1580-harden-voice-safety-and-ux-on-the-unified-flue-route`, stacked directly on [PR #9528](https://github.com/hashintel/hash/pull/9528) at the GitHub-verified head `036fb06ee83e840fd7a87dd1bc758a2dd49ddd4c`. This file is the sole execution authority for the successor branch. The parent remains the authority for its own admission/Stop races, stream cancellation, hydration overwrite, client-tool classification, response/submission correlation, and PR/CI defects; this branch must restack onto parent fixes rather than repair them. +**Live as of 2026-09-04** for [FE-1580](https://linear.app/hash/issue/FE-1580/harden-voice-safety-and-ux-on-the-unified-flue-route) on `kostandin/fe-1580-harden-voice-safety-and-ux-on-the-unified-flue-route`, stacked directly on [PR #9528](https://github.com/hashintel/hash/pull/9528) at the GitHub-verified head `6ca81b7bc4d6c112ff3936c38a7209d07f773b8a`. This file is the sole execution authority for the successor branch. The restack adopts the parent's canonical hydration overwrite guard, multi-submission response correlation, settlement-driven durable Stop, aligned live/snapshot projection, queued Voice-input cancellation, and client-tool continuation behavior. The parent remains the authority for defects in those mechanisms; this branch must restack onto further parent fixes rather than repair them. The owner selected **half-duplex turn ownership** on 2026-09-03. While canonical assistant audio is pending or playing, the microphone is closed. Ownership transfers away from input as soon as canonical speech is requested, before `response.create` is sent: every accepted unfinished input item becomes stale and provisional transcript state is cleared. The explicit **Your turn** action cancels output, waits for provider acknowledgement and response settlement, and only then opens a fresh input turn. Automatic duplex barge-in is rejected because assistant playback can become a false user turn. @@ -20,7 +20,7 @@ defines the accepted and rejected mechanics. 1. **Adopted — half-duplex explicit handoff.** It gives assistant playback exclusive ownership, makes cancellation settlement a visible boundary, and guarantees fresh post-handoff capture. The product cost is one extra **Your turn** action and barrier latency when the user interrupts. 2. **Rejected — automatic duplex barge-in.** It offers the most conversational interruption and avoids an explicit control, but an open microphone can transcribe assistant playback as a user answer, and capture can race an unsettled cancellation. This risk is unacceptable for authoritative completed transcripts. -Sections independent of canonical hydration may proceed. The hard-reload witness is blocked until the parent prevents its once-per-conversation hydration from overwriting a locally submitted turn. Prepare that witness but do not patch the parent defect here. +The parent now prevents its once-per-conversation canonical hydration from overwriting a locally submitted turn. The real microphone, handoff, Stop, hard-reload, and same-origin witness is therefore executable but remains unproved until the human witness and retained artifact bundle are complete. Completed-transcript authority, half-duplex ownership, admission idempotency, the cancellation barrier, exact full-response replay, and exact marked-question @@ -35,8 +35,9 @@ reconstructs multiple surviving origins. Direct spoken user attribution is blocked because Flue 2.0.3 projects the generated `submissionId` but neither caller metadata nor the caller idempotency key. The discarded browser-correlation implementation would have violated the explicit -second-durable-store stop condition. The real witness remains blocked by the -parent hydration defect. +second-durable-store stop condition. The restacked hydration guard removes the +old parent blocker, but no real witness claim is valid until the retained human +evidence exists. The 2026-09-04 corrective verification covers the current 58-file successor diff against #9528: the four focused race cases pass 4/4 tests, the filtered @@ -102,11 +103,11 @@ This mission closes the Voice safety and UX-parity stratum on the parent's route **Release note:** Voice now submits only what the microphone actually transcribed, waits for a safe **Your turn** handoff before listening over Brunch, and can replay the exact full response or exact Brunch-marked question. Client-tool Voice origins survive canonical reopen. Restoring the Voice chip on direct spoken user messages remains blocked on an upstream Flue user-metadata contract. -**Demo script:** run `yarn dev:brunch` and select the Brunch preview. Speak one answer and see exactly one matching user turn. While Brunch is speaking, confirm the microphone remains closed, choose **Your turn**, wait for the handoff, and speak again. After the response and audio settle, use the playback menu to read the full response exactly and repeat only the exact Brunch-marked question; a missing or unmatched marker keeps that action disabled. Start another turn, press durable **Stop** before settlement, and see a stopped turn rather than a Voice error. After the parent hydration blocker is fixed, hard-reload the settled conversation and confirm the canonical turn remains without resubmission or replay; direct-user Voice-chip restoration additionally waits on the Flue projection seam. +**Demo script:** run `yarn dev:brunch` and select the Brunch preview. Speak one answer and see exactly one matching user turn. While Brunch is speaking, confirm the microphone remains closed, choose **Your turn**, wait for the handoff, and speak again. After the response and audio settle, use the playback menu to read the full response exactly and repeat only the exact Brunch-marked question; a missing or unmatched marker keeps that action disabled. Start another turn, press durable **Stop** before settlement, and see a stopped turn rather than a Voice error. Hard-reload the settled conversation and confirm the canonical turn remains without resubmission or replay; direct-user Voice-chip restoration additionally waits on the Flue projection seam. **Previously impossible:** model-generated function arguments rather than completed audio transcription could become the answer; an accepted transcript could complete after canonical speech was requested but before output started; assistant playback could create a false user turn; cancellation could reopen capture before the provider settled; replay controls and multi-origin client-tool Voice attribution were incomplete. -**Completion:** the implemented portions close when their tests and focused checks pass. Exact question replay uses the Brunch-owned marker recorded below; direct-user provenance still needs the Flue re-entry seam recorded below. Mission acceptance additionally requires the real microphone, handoff, Stop, hard-reload, and same-origin route witness after the parent hydration fix. Mocked or server-only proof cannot substitute for that witness. +**Completion:** the implemented portions close when their tests and focused checks pass. Exact question replay uses the Brunch-owned marker recorded below; direct-user provenance still needs the Flue re-entry seam recorded below. Mission acceptance additionally requires the real microphone, handoff, Stop, hard-reload, and same-origin route witness, plus the comparative Voice latency gate. Mocked or server-only proof cannot substitute for that witness or for real audible-latency samples. 1. **Completed-transcript authority and half-duplex ownership.** Realtime session configuration has no tools, no model-created semantic-VAD response, and no automatic interruption policy. Only a unique completed transcript can reach the shared panel submission path. Duplicate, empty, failed, unavailable, stale, canonical-speech-overlapping/pre-handoff, playback-overlapping, and over-limit transcripts do not submit and produce the specified passive or recoverable notice. Before sending `response.create`, a canonical speech request invalidates every unfinished accepted item, clears bridge/controller transcript state and provisional UI, and closes the microphone; a completion in the interval before output starts cannot submit or regain authority. **Your turn** opens only a post-barrier input turn. Oracle: transplanted-first cases in `openai-realtime-session.test.ts`, `realtime-brunch-bridge.test.ts`, `voice-turn-controller.test.ts`, `voice-interview-control.test.tsx`, and `voice-preview.integration.test.ts`. 2. **Idempotent admission.** Typed turns derive a stable key from the AI SDK message id; Voice turns derive it from connection epoch, item id, and content index. A repeated same-payload key converges on the original receipt, including `deduplicated: true`; a 409 `submission_conflict` surfaces the original `submissionId` without admitting another turn. An admission whose outcome cannot be established stays visibly ambiguous and is never automatically retried. Oracle: transport unit tests plus Voice integration tests asserting one admitted submission rather than one `send()` invocation. @@ -114,8 +115,9 @@ This mission closes the Voice safety and UX-parity stratum on the parent's route 4. **Canonical full-response and exact question replay.** `canReadFullResponse`, `readFullResponse()`, and the playback menu retain and enqueue all exact canonical text segments in order without a simplifier. A non-interactive `brunch_mark_question` server tool writes a durable `data-brunch-question` marker containing exact question text and tool-call identity. The selector accepts it only when the same finalized assistant message contains that exact text; there is no final-segment or punctuation fallback. `repeatQuestion()` queues only the accepted marked segment. Both replay actions enable only after the correlated Brunch response settles, matching Realtime output completes, and input is idle, and remain disabled during submission, capture, cancellation, pause, and errors. The marker never accepts an answer or changes Voice path B. Oracle: core tool, live transport, snapshot projection, canonical speech, controller, panel-host, and production-preview tests comparing marker identity and exact queued text while proving `brunch_ask` remains unmounted. 5. **Durable Voice provenance.** An assistant message may retain multiple `voiceToolCallIds`; one failed sibling origin does not erase successful origins. Persisted Flue client-tool-result signals support deterministic reconstruction after hydration and reopen. Direct spoken user messages remain Voice-attributed only while live because the canonical snapshot omits their caller origin. Re-entry requires a supported Flue user-message metadata/idempotency projection; browser storage and user-text encoding are rejected. Oracle: snapshot projection and panel partial-failure tests for supported origins, plus the [blocker record](docs/evidence/implementations/mission-5-voice-safety-parity/provenance-blocker.md). 6. **Dormant ask removal.** If still present after restacking, the website does not register `brunchAskInteractiveTool` for Voice and canonical speech does not recognize `"brunch-ask"`. No spoken ask answer can enter a wait state the transport cannot resume. Oracle: registration/canonical-speech negative tests and a repository search showing no mounted Voice `brunch_ask` surface. -7. **Real witness and same-origin route.** After the parent hydration fix lands, a human performs one microphone turn, explicit interruption/handoff, durable Stop on an unsettled turn, and hard reload of a settled turn. The retained network route summary proves the absolute Flue `streamUrl` remains on the same-origin proxy. Oracle: `witness.md`, sanitized `voice-events.jsonl`, `network-routes.json`, canonical `flue-snapshot.json`, `settlements.json`, commit manifest, and hashes under `docs/evidence/implementations/mission-5-voice-safety-parity/`. -8. **Focused repository verification and truthful docs.** The requested four-workspace Turbo command passes. `apps/petrinaut-website/README.md` and `libs/@hashintel/petrinaut/docs/ai-assistant.md` describe half-duplex handoff, exact full-response and marked-question replay, Stop, transcript rejection, and the direct-user attribution limitation. If the published Petrinaut package changes, exactly one patch changeset covers it. Oracle: the command recorded in the PR and changeset inspection. +7. **Real witness and same-origin route.** A human performs one microphone turn, explicit interruption/handoff, durable Stop on an unsettled turn, and hard reload of a settled turn. The retained network route summary proves the absolute Flue `streamUrl` remains on the same-origin proxy. Oracle: `witness.md`, sanitized `voice-events.jsonl`, `network-routes.json`, canonical `flue-snapshot.json`, `settlements.json`, commit manifest, and hashes under `docs/evidence/implementations/mission-5-voice-safety-parity/`. +8. **Comparative Voice latency.** Ten comparable real-audio trials at pinned donor #9496 head `c7fe8a2e68e8fdc37018b21ec2e9daf4e9ef7c82` and ten at the final restacked candidate use the same machine, browser, microphone/input phrase, model configuration, warm/cold-start policy, and finalized-speech-to-first-audible-canonical-TTS boundary. The candidate median must not regress, and its p95 regression must remain below 20%. The donor runs from an isolated worktree without changing its branch. Oracle: retained raw sanitized samples, calculation method, environment, both commit SHAs, median, and p95; a comparison that cannot be run reliably leaves this proof incomplete. +9. **Focused repository verification and truthful docs.** The requested seven-workspace Turbo command passes, including `@hashintel/brunch-agent`, `@hashintel/brunch-agent-binding-flue`, and `@hashintel/brunch-agent-plugin-sdcpn`. `apps/petrinaut-website/README.md` and `libs/@hashintel/petrinaut/docs/ai-assistant.md` describe half-duplex handoff, exact full-response and marked-question replay, Stop, transcript rejection, and the direct-user attribution limitation. If the published Petrinaut package changes, exactly one patch changeset covers it. Oracle: the command recorded in the PR and changeset inspection. ## Constraints @@ -152,7 +154,7 @@ This mission closes the Voice safety and UX-parity stratum on the parent's route - **Dormant `brunch_ask`.** Remove or gate only the parent surfaces that remain after the next restack. If Lu has already removed them, record the parent commit and make no duplicate change. - **Question-marker compliance.** The owner selected `brunch_mark_question` plus a durable client data part. The remaining implementation uncertainty is whether the model follows the instruction on every eligible question. Missing or unmatched markers must degrade by leaving **Repeat question** disabled; they never justify inference from final prose. Product proof covers structural correctness, not a universal model-compliance rate. - **Direct-user Voice provenance.** Flue 2.0.3 and current upstream `main` expose a generated `submissionId` but not caller metadata or `idempotencyKey` on canonical user messages. The owner selected an upstream Flue user-metadata contract. Keep this leaf blocked until a released seam can be adopted; do not patch Flue locally, add a provenance signal admission, add sidecar persistence, or encode origin in user content. -- **Hard-reload witness.** Hold proof leaf 7 until the parent's hydration overwrite is fixed. Unit/integration tests for this branch may cover deterministic projection, but they cannot substitute for the blocked real witness. +- **Human product evidence.** The hydration overwrite guard is present after the restack, so proof leaf 7 may run. Unit/integration tests still cannot substitute for the real microphone/hard-reload witness, and server or synthetic timing cannot substitute for proof leaf 8's first-audible-audio measurements. ## Stop or reorient @@ -160,11 +162,11 @@ Stop and report if the work would require direct Voice `send()`, a second transc Stop if half-duplex handoff cannot guarantee that pre-handoff audio is rejected and post-barrier audio is fresh, or if provider acknowledgements cannot bound `cancelOutput()` without inventing events. The provenance stop condition has fired for direct spoken user turns: the browser-store implementation was removed and the unsupported leaf is recorded as blocked pending upstream Flue support. For **Repeat question**, stop rather than infer question identity when the approved marker is absent or does not exactly match finalized assistant text. Stop if either replay action can enable before both matching terminal conditions, or if local cancellation invokes durable abort. -Do not manufacture the hard-reload witness while the parent hydration defect remains. Retain the blocker and wait for a new parent head. +Do not manufacture the hard-reload witness or latency samples. If the human/browser environment cannot produce reliable observations, retain an incomplete evidence record and request the missing action explicitly. ## Deferred -- The real witness and same-origin absolute-`streamUrl` observation wait on the parent hydration fix; once unblocked, they are part of this mission rather than a successor. +- The real witness, same-origin absolute-`streamUrl` observation, and comparative latency gate require human browser and microphone evidence; they are part of this mission rather than a successor. - Direct-user Voice attribution after canonical hydration waits on a released upstream Flue caller-metadata projection seam. The owner rejected a local Flue patch and correlated signal sidecar for this mission. - Donor retirement waits until this replacement is accepted and each donor owner explicitly approves closure. Do not close #9496, #9500, #9507, or #9512 as an implementation side effect, and never close stakeholder-owned H-6763. - Response preparation/simplification, structured questions, Petri-net work, FE-1575, production identity, CORS/remote deployment, and panel migration away from `useChat` remain in their existing owners or the future mission spine. From 63acc341bf045460ad66d5273c106a4742607dde Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Fri, 4 Sep 2026 11:38:57 +0200 Subject: [PATCH 31/52] Prepare the final Voice acceptance evidence Amp-Thread-ID: https://ampcode.com/threads/T-01a06941-3ee7-7482-a82e-8a16c7141785 Co-authored-by: Amp --- apps/petrinaut-website/README.md | 12 ++--- libs/@hashintel/brunch-agent/MISSION.md | 17 +++---- .../donor-behavior-matrix.md | 32 +++++++------ .../provenance-blocker.md | 4 ++ .../witness-blocker.md | 46 +++++++++++++------ 5 files changed, 68 insertions(+), 43 deletions(-) diff --git a/apps/petrinaut-website/README.md b/apps/petrinaut-website/README.md index 307d8bfa0fb..1f6919f3a32 100644 --- a/apps/petrinaut-website/README.md +++ b/apps/petrinaut-website/README.md @@ -97,12 +97,12 @@ ambiguity and does not retry automatically. **Stop** requests a durable Brunch abort before the panel cancels its local response stream. Local playback cancellation remains separate and does not alter canonical history. Canonical Flue history is the source used when the same net is reopened. Automated -projection coverage does not resubmit turns or replay settled audio, but the -required real hard-reload witness remains blocked by the parent hydration fix, -so reload parity is not yet claimed for this preview. Voice-origin client-tool -results retain their markers in Flue history. Direct spoken user turns remain -canonical text, but Flue 2.0.3 does not yet expose the caller delivery metadata -needed to restore their Voice chip after reopening. +coverage guards a locally submitted turn from an older hydration snapshot and +does not resubmit turns or replay settled audio. The real hard-reload witness is +still pending, so reload parity is not yet claimed for this preview. +Voice-origin client-tool results retain their markers in Flue history. Direct +spoken user turns remain canonical text, but Flue 2.0.3 does not yet expose the +caller delivery metadata needed to restore their Voice chip after reopening. An active session stays at the end of the transcript. Its compact divider shows a waveform and **Connecting**, **Listening**, **Speaking**, **Paused**, or a diff --git a/libs/@hashintel/brunch-agent/MISSION.md b/libs/@hashintel/brunch-agent/MISSION.md index d032b3a50e6..31bee7ce8a4 100644 --- a/libs/@hashintel/brunch-agent/MISSION.md +++ b/libs/@hashintel/brunch-agent/MISSION.md @@ -40,14 +40,15 @@ old parent blocker, but no real witness claim is valid until the retained human evidence exists. The 2026-09-04 corrective verification covers the current 58-file successor -diff against #9528: the four focused race cases pass 4/4 tests, the filtered -production admission-outcome cases pass 3/3 tests, and the requested Turbo run -passes 30/30 tasks, including 249/249 website tests. Architecture validation -passes with 62 layers, 297 edges, 614 files, 63 generated pages, and 31 authored -pages. `git diff --check` and formatting of all formatter-owned successor files -pass. The root formatter remains red only outside this branch's diff: two -parent-owned #9528 files and an unrelated untracked `.cursor` plan. The exact -commands and dispositions are retained in the [donor matrix](docs/evidence/implementations/mission-5-voice-safety-parity/donor-behavior-matrix.md#corrective-verification). +diff against #9528 head `6ca81b7bc4`: the four focused race cases pass 4/4 +tests, the filtered production admission-outcome cases pass 3/3 tests, and the +complete seven-workspace Turbo run passes 39/39 tasks, including 254/254 website +tests and the Brunch core package. Architecture validation passes with 62 +layers, 297 edges, 615 files, 63 generated pages, and 31 authored pages. `git +diff --check` and formatting of all 52 formatter-owned successor files pass. The +root formatter remains red only outside this branch's diff on an unrelated +untracked `.cursor` plan. The exact commands and dispositions are retained in +the [donor matrix](docs/evidence/implementations/mission-5-voice-safety-parity/donor-behavior-matrix.md#corrective-verification). The pinned donor-behavior decision record is the [FE-1580 donor matrix](docs/evidence/implementations/mission-5-voice-safety-parity/donor-behavior-matrix.md). Donor PRs are read-only evidence at their named heads; semantic reimplementation is required, never merge or cherry-pick. diff --git a/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/donor-behavior-matrix.md b/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/donor-behavior-matrix.md index d4e35f922e4..d8401984e08 100644 --- a/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/donor-behavior-matrix.md +++ b/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/donor-behavior-matrix.md @@ -6,7 +6,7 @@ This record pins the semantic disposition of the Voice donor branches for the li | Source | Pinned head | Role | | --- | --- | --- | -| Parent PR [#9528](https://github.com/hashintel/hash/pull/9528) | `036fb06ee83e840fd7a87dd1bc758a2dd49ddd4c` | Unified Flue route and path-B departure base | +| Parent PR [#9528](https://github.com/hashintel/hash/pull/9528) | `6ca81b7bc4d6c112ff3936c38a7209d07f773b8a` | Unified Flue route and path-B departure base | | Donor PR [#9496](https://github.com/hashintel/hash/pull/9496) | `c7fe8a2e68e8fdc37018b21ec2e9daf4e9ef7c82` | Canonical TTS queue and replay mechanics | | Donor PR [#9500](https://github.com/hashintel/hash/pull/9500) | `935aa9f02a5ac635a50eb8bc130edb3e258af8e4` | Completed-transcript authority | | Donor PR [#9507](https://github.com/hashintel/hash/pull/9507) | `252b9dbb0c77fae8cee45a506f09cac3e20c381c` | Temporary `brunch_ask` shim, excluded | @@ -23,7 +23,7 @@ The owner selected half-duplex turn ownership on 2026-09-03: assistant output ow | #9528 | One `/agents/chat/:instanceId` product route, browser `ChatTransport`, one memoized client, path-B Voice submission through shared `useChat` | **Adopt** | This is the departure architecture and prevents a second admission authority. | Restack onto every new parent head; verify no successor code calls `send()` directly from Voice. | | #9528 | Direct Voice `send()` as a fog-line fallback | **Reject** | It creates a second admission path and mutable coordination surface. The parent has already proved path B. | Mission authority now permits path B only. | | #9528 | Claim that Flue 2.0.3 lacks caller idempotency | **Reject as factually false** | Installed typings expose `AgentPromptOptions.idempotencyKey`, `AgentSendResult.deduplicated`, and 409 `submission_conflict` with the existing `submissionId`. | Implemented with transport convergence/conflict tests and typed Voice admission outcomes. | -| #9528 | Parent-owned admission/Stop races, stream cancellation, hydration overwrite, client-tool classification, response/submission correlation, and PR/CI repairs | **Reject from successor scope** | Lu owns these defects; changing them here would make the stack compete with its parent. | Report blockers and restack onto fixes. The real reload witness remains blocked by hydration overwrite. | +| #9528 | Canonical hydration guard, multi-submission response correlation, settlement-driven durable Stop, aligned live/snapshot projection, queued Voice-input cancellation, and client-tool continuation | **Adopt through restack** | These mechanisms remain parent-owned and must enter the successor through the stack rather than copied fixes. | Restacked onto `6ca81b7bc4`; hydration no longer blocks the real witness. Further defects in these mechanisms remain parent scope. | | #9496 | Serialized canonical speech queue, retained exact source segments, response/output terminal gating | **Adopt mechanics** | Replay and ordinary TTS need one lifecycle-safe queue, and exact text preserves canonical authority. | Implemented without a preparation/simplifier dependency; exact-segment and queue tests pass. | | #9496 | `canReadFullResponse`, `readFullResponse()`, exact full-response playback menu | **Adopt** | Exact full-response replay is supported by retained canonical segment identity. | Implemented with idle-state and matching response/output terminal gates. | | #9496 | `canRepeatQuestion`, `repeatQuestion()`, and playback-menu action | **Adopt UX; reject final-segment inference** | The final segment may be ordinary prose and is not authority for question identity. The approved `brunch_mark_question` data marker now supplies deterministic identity without accepting an answer. | Implemented by replaying only exact marked text found in finalized prose from the same assistant message; a missing or unmatched marker leaves the action disabled. | @@ -66,27 +66,31 @@ The owner selected half-duplex turn ownership on 2026-09-03: assistant output ow | Canonical replay | Exact segment queue and playback menu pass availability/race tests without a simplifier. | Full-response replay preserves every exact segment. **Repeat question** uses only a durable non-interactive Brunch marker that exactly matches finalized prose in the same assistant message; final-segment inference remains rejected. Both actions share terminal/output/input gating. | | Durable provenance | Multiple origins and partial failure survive projection, hydration, and reopen without user-text encoding. | Partially implemented for assistant client-tool results through persisted Flue signals; multiple sibling origins survive projection and partial failure. Direct spoken user attribution is blocked because Flue 2.0.3 snapshots do not expose caller idempotency or user-message metadata. The rejected browser store would violate mission authority. | | Dormant ask | No mounted Voice ask capability remains, or the parent commit that removed it is recorded. | Implemented exclusion. Canonical speech ignores `brunch_ask`, and a configured-app registration test proves the production Brunch assistant supplies no ask tool. Dormant source remains unmounted. | -| Real witness | Microphone, handoff, unsettled Stop, reload, canonical snapshot, settlement, and same-origin absolute-`streamUrl` artifacts are retained with hashes. | Blocked by the parent's hydration overwrite. | +| Real witness | Microphone, handoff, unsettled Stop, reload, canonical snapshot, settlement, and same-origin absolute-`streamUrl` artifacts are retained with hashes. | Parent hydration blocker resolved by restack; human browser/microphone run and retained artifacts remain outstanding. | +| Comparative latency | Ten pinned #9496 trials and ten final-candidate trials retain raw finalized-speech-to-first-audible-canonical-TTS samples and show no median regression with p95 regression below 20%. | Donor isolated worktree is prepared and its five focused Voice suites pass 108/108 after dependency build. Twenty comparable human audible trials and statistics remain outstanding. | | Donor retirement | Replacement accepted and each donor owner explicitly approves closure. | Deferred; no donor or stakeholder issue may be closed now. | ## Corrective verification Fresh local checks on 2026-09-04 cover the 58-file successor diff against the -verified #9528 head `036fb06ee83e840fd7a87dd1bc758a2dd49ddd4c`: +verified #9528 head `6ca81b7bc4d6c112ff3936c38a7209d07f773b8a`: | Command | Result | | --- | --- | -| `yarn workspace @apps/petrinaut-website test:unit src/main/app/voice-interview/openai-realtime-session.test.ts src/main/app/voice-interview/realtime-brunch-bridge.test.ts src/main/app/voice-interview/voice-turn-controller.test.ts src/main/app/voice-interview/voice-preview.integration.test.ts -t 'invalidates accepted input before requesting canonical speech output\|rejects unfinished input as soon as canonical speech is requested\|clears capture when canonical speech is requested before output starts\|bridges one completed transcript through Brunch and back to canonical half-duplex audio'` | Exit 0; 4/4 selected tests passed and 91 unrelated tests were filtered across four files. This covers the request-before-output race at session, bridge, controller, and production integration layers. | -| `yarn workspace @apps/petrinaut-website test:unit src/main/app/voice-interview/voice-preview.integration.test.ts -t 'ambiguous Flue admission\|conflicting submission\|local admission abort'` | Exit 0; 3/3 selected tests passed and 2 unrelated tests were filtered. Conflict retains the original `submissionId`; local abort remains distinct from durable abort; every path calls `send()` once. | -| `yarn workspace @hashintel/brunch-agent test:unit test/question-marker.test.ts` | Exit 0; 9/9 exact question-marker tests passed. | -| `yarn workspace @hashintel/brunch-agent-transport-aisdk test:unit` | Exit 0; 22/22 transport tests passed, including live and snapshot marker projection. | -| `yarn workspace @apps/brunch-agent test:unit test/petrinaut-chat.test.ts` | Exit 0; 1/1 real-Flue integration test passed, including exact marker persistence through fresh-process reopen while marker tools remain hidden. | -| `yarn workspace @hashintel/petrinaut test:unit --run src/ui/views/Editor/panels/ai-assistant-panel.test.tsx` | Exit 0; 40/40 production host-registration and panel tests passed. | -| `mise exec -- yarn exec turbo run lint:tsc lint:eslint test:unit build --filter @apps/brunch-agent --filter @apps/petrinaut-website --filter @hashintel/petrinaut --filter @hashintel/brunch-agent-transport-aisdk` | Exit 0; 30/30 tasks passed, including 31 website test files and 249/249 website tests. Website ESLint retains one non-failing warning at `voice-interview-control.tsx:485`. | -| `yarn workspace @local/petrinaut-arch-docs lint:arch-docs` | Exit 0; 62 layers, 297 edges, 614 files, 63 generated pages, and 31 authored pages. | -| `git diff --name-only -z ln/fe-1574-direct-voice-flue...HEAD \| xargs -0 yarn exec oxfmt --check` | Exit 0; all 52 formatter-owned files in the 58-file successor diff passed. The six unmatched documentation files are excluded by `oxfmt.config.ts` and pass repository whitespace validation. | +| `mise exec -- yarn workspace @apps/petrinaut-website test:unit src/main/app/voice-interview/openai-realtime-session.test.ts src/main/app/voice-interview/realtime-brunch-bridge.test.ts src/main/app/voice-interview/voice-turn-controller.test.ts src/main/app/voice-interview/voice-preview.integration.test.ts -t 'invalidates accepted input before requesting canonical speech output\|rejects unfinished input as soon as canonical speech is requested\|clears capture when canonical speech is requested before output starts\|bridges one completed transcript through Brunch and back to canonical half-duplex audio'` | Exit 0; 4/4 selected tests passed and 92 unrelated tests were filtered across four files. This covers the request-before-output race at session, bridge, controller, and production integration layers. | +| `mise exec -- yarn workspace @apps/petrinaut-website test:unit src/main/app/voice-interview/voice-preview.integration.test.ts -t 'ambiguous Flue admission\|conflicting submission\|local admission abort'` | Exit 0; 3/3 selected tests passed and 2 unrelated tests were filtered. Conflict retains the original `submissionId`; local abort remains distinct from durable abort; every path calls `send()` once. | +| `mise exec -- yarn workspace @hashintel/brunch-agent test:unit test/question-marker.test.ts` | Exit 0; 9/9 exact question-marker tests passed. | +| `mise exec -- yarn workspace @hashintel/brunch-agent-transport-aisdk test:unit` | Exit 0; 26/26 transport tests passed, including live and snapshot marker projection. | +| `mise exec -- yarn workspace @apps/brunch-agent test:unit test/petrinaut-chat.test.ts` | Exit 0; 1/1 real-Flue integration test passed, including exact marker persistence through fresh-process reopen while marker tools remain hidden. | +| `mise exec -- yarn workspace @hashintel/petrinaut test:unit --run src/ui/views/Editor/panels/ai-assistant-panel.test.tsx` | Exit 0; 44/44 production host-registration and panel tests passed. | +| `mise exec -- yarn workspace @apps/petrinaut-website test:unit src/main/app/voice-interview/canonical-speech.test.ts src/main/app/voice-interview/openai-realtime-session.test.ts src/main/app/voice-interview/voice-turn-controller.test.ts` | Exit 0; 79/79 exact replay, queue, terminal-gating, and turn-controller tests passed. | +| `mise exec -- yarn exec turbo run lint:tsc lint:eslint test:unit build --filter @hashintel/brunch-agent` | Exit 0; 5/5 tasks passed, including 10/10 test files and 86/86 tests; the four question-marker mock lint failures are resolved with production-interface signatures. | +| `mise exec -- yarn exec turbo run lint:tsc lint:eslint test:unit build --filter @apps/brunch-agent --filter @apps/petrinaut-website --filter @hashintel/petrinaut --filter @hashintel/brunch-agent --filter @hashintel/brunch-agent-binding-flue --filter @hashintel/brunch-agent-plugin-sdcpn --filter @hashintel/brunch-agent-transport-aisdk` | Exit 0; 39/39 tasks passed, including 16/16 Brunch app files with 79/79 tests and 31/31 website files with 254/254 tests. Website ESLint retains one non-failing warning at `voice-interview-control.tsx:493`. | +| `yarn workspace @local/petrinaut-arch-docs lint:arch-docs` | Exit 0; 62 layers, 297 edges, 615 files, 63 generated pages, and 31 authored pages. | +| `git diff --name-only -z 6ca81b7bc4d6c112ff3936c38a7209d07f773b8a...HEAD \| xargs -0 yarn exec oxfmt --check` | Exit 0; all 52 formatter-owned files in the 58-file successor diff passed. The six unmatched documentation files are excluded by `oxfmt.config.ts` and pass repository whitespace validation. | | `git diff --check` | Exit 0. | -| `yarn lint:format` | Exit 1 outside the successor diff only: parent-owned `apps/brunch-agent/src/conversation/identity.ts` and `apps/brunch-agent/test/conversation-identity.test.ts`, plus unrelated untracked `.cursor/plans/fe-1574_mission_recut_6f23f7cd.plan.md`. They were not modified. | +| `yarn lint:format` | Exit 1 outside the successor diff only: unrelated untracked `.cursor/plans/fe-1574_mission_recut_6f23f7cd.plan.md`. It was not modified. | +| In isolated detached worktree `/Users/kostandin/Projects/hashdev/worktrees/fe-1580-latency-baseline-9496`: `mise exec -- yarn exec turbo run build --filter '@apps/petrinaut-website^...'`, then `mise exec -- yarn workspace @apps/petrinaut-website test:unit src/main/app/voice-interview/canonical-speech.test.ts src/main/app/voice-interview/openai-realtime-session.test.ts src/main/app/voice-interview/realtime-brunch-bridge.test.ts src/main/app/voice-interview/voice-turn-controller.test.ts src/main/app/voice-interview/voice-preview.integration.test.ts` | Exit 0; dependency build passed 14/14 tasks, then all 5/5 donor Voice files and 108/108 tests passed at pinned #9496 head. The isolated donor and candidate panels return HTTP 200 on ports 4916 and 4915 respectively; real audible samples remain uncollected. | No production Voice source under `apps/petrinaut-website/src/main/app/voice-interview` calls `FlueClient.send()`; its only `.send()` is the OpenAI Realtime data diff --git a/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/provenance-blocker.md b/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/provenance-blocker.md index 7e9c3e6ea30..b3e9e897b07 100644 --- a/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/provenance-blocker.md +++ b/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/provenance-blocker.md @@ -40,3 +40,7 @@ onto the canonical direct-user message, or when the product owner explicitly authorizes a different durable representation. The oracle is a snapshot-only test that reconstructs the Voice marker after a fresh process with no browser correlation state. + +The restacked branch still installs `@flue/sdk` 2.0.3 with this same public +shape. No supported projection seam or owner-approved deferral has been +recorded, so direct-user reopen attribution remains blocked. diff --git a/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/witness-blocker.md b/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/witness-blocker.md index 561a23ec530..92e59acf1d8 100644 --- a/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/witness-blocker.md +++ b/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/witness-blocker.md @@ -1,4 +1,4 @@ -# FE-1580 real-witness blocker +# FE-1580 human-evidence gate ## Current disposition @@ -11,24 +11,40 @@ cannot replace the microphone, handoff, unsettled Stop, hard-reload, and network-route witness required for mission acceptance. Direct-user Voice attribution has a separate [Flue projection blocker](provenance-blocker.md). -The parent branch still runs canonical hydration once per conversation in a way -that can overwrite a locally submitted turn. The successor may not fix that -parent-owned defect. A hard-reload recording made before the parent fix would -therefore be unable to establish the required durability claim. +The successor is restacked onto [PR #9528](https://github.com/hashintel/hash/pull/9528) +head `6ca81b7bc4d6c112ff3936c38a7209d07f773b8a`. That parent now guards its +once-per-conversation hydration from replacing a locally visible assistant +response with an older canonical snapshot, so hydration no longer blocks this +witness. The remaining gate is the required human browser and microphone run. ## Re-entry gate -After [PR #9528](https://github.com/hashintel/hash/pull/9528) lands a fix and this -branch is restacked onto its new verified head: +Using the final source/build commit: -1. run one real microphone turn and retain the sanitized Voice event ledger; -2. use **Your turn** during output and retain cancellation acknowledgements; -3. durably stop an unsettled turn and retain its stopped settlement; -4. hard-reload a settled Voice turn and retain the canonical Flue snapshot; -5. retain a network route summary proving the absolute Flue `streamUrl` remains - on the same-origin proxy; and -6. record the exact commits and hashes for every retained artifact. +1. submit one typed turn; +2. run one real microphone turn and confirm exactly one matching user message; +3. confirm visible text and synthesized speech use the same canonical response; +4. use **Your turn** during output and retain cancellation acknowledgements; +5. confirm pre-handoff audio cannot submit and fresh post-handoff speech can; +6. durably stop an unsettled turn and retain its stopped settlement; +7. replay the exact full response and exact marked question; +8. hard-reload the settled conversation and confirm no resubmission or + automatic replay; +9. retain the canonical Flue snapshot and settlement index; +10. retain a network route summary proving the absolute Flue `streamUrl` + remains on the same-origin proxy; and +11. record the exact source/build and evidence commits plus hashes for every + retained artifact. + +The comparative latency proof also requires ten audible trials at pinned donor +#9496 head `c7fe8a2e68e8fdc37018b21ec2e9daf4e9ef7c82` and ten at the final +candidate. Both sets use the same machine, browser, microphone/input phrase, +model configuration, warm/cold-start policy, and finalized-speech-to-first- +audible-canonical-TTS boundary. Raw sanitized samples, the calculation method, +environment, commit identities, median, and p95 must be retained; the candidate +median may not regress and p95 regression must remain below 20%. Until then, `witness.md`, `voice-events.jsonl`, `network-routes.json`, `flue-snapshot.json`, and `settlements.json` are intentionally absent rather -than populated with simulated evidence. +than populated with simulated evidence. Latency samples and statistics are also +intentionally absent until the comparable human trials run. From 9938283a19ab20567ad6b4c96330ea392243c16f Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Fri, 4 Sep 2026 11:46:16 +0200 Subject: [PATCH 32/52] Adapt Voice host tests to parent navigation Amp-Thread-ID: https://ampcode.com/threads/T-01a06941-3ee7-7482-a82e-8a16c7141785 Co-authored-by: Amp --- .../local-storage-demo-app.test.tsx | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx index 21eb06313c8..84162a0ae3b 100644 --- a/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx +++ b/apps/petrinaut-website/src/main/app/local-storage-demo/local-storage-demo-app.test.tsx @@ -5,9 +5,8 @@ import { act, cleanup, render, waitFor } from "@testing-library/react"; import { isValidElement, type ReactNode } from "react"; import { afterEach, describe, expect, test, vi } from "vitest"; -import { defaultPetrinautNavigationHistoryPolicy } from "@hashintel/petrinaut/react"; - import { FlueChatAdmissionError } from "@hashintel/brunch-agent-transport-aisdk"; +import { defaultPetrinautNavigationHistoryPolicy } from "@hashintel/petrinaut/react"; import { OpenAIRealtimeSession } from "../voice-interview/openai-realtime-session"; import { VoiceInterviewControl } from "../voice-interview/voice-interview-control"; @@ -146,7 +145,9 @@ describe("local storage demo Brunch voice integration", () => { ), ); - const rendered = render(); + const rendered = render( + {}} search={{}} />, + ); await waitFor(() => expect(renderedPetrinaut.aiAssistant).not.toBeNull()); const aiAssistant = renderedPetrinaut.aiAssistant as PetrinautAiAssistant; @@ -214,7 +215,9 @@ describe("local storage demo Brunch voice integration", () => { ), ); - const rendered = render(); + const rendered = render( + {}} search={{}} />, + ); await waitFor(() => expect( (renderedPetrinaut.aiAssistant as PetrinautAiAssistant).requestStop, From 60d41c5923bb7a978aa19511dc076fe5407ed646 Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Fri, 4 Sep 2026 11:50:01 +0200 Subject: [PATCH 33/52] Record the final Voice restack verification Amp-Thread-ID: https://ampcode.com/threads/T-01a06941-3ee7-7482-a82e-8a16c7141785 Co-authored-by: Amp --- libs/@hashintel/brunch-agent/MISSION.md | 10 +++++----- .../donor-behavior-matrix.md | 18 ++++++++++++------ .../witness-blocker.md | 2 +- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/libs/@hashintel/brunch-agent/MISSION.md b/libs/@hashintel/brunch-agent/MISSION.md index 31bee7ce8a4..824449b867c 100644 --- a/libs/@hashintel/brunch-agent/MISSION.md +++ b/libs/@hashintel/brunch-agent/MISSION.md @@ -2,7 +2,7 @@ ## Status -**Live as of 2026-09-04** for [FE-1580](https://linear.app/hash/issue/FE-1580/harden-voice-safety-and-ux-on-the-unified-flue-route) on `kostandin/fe-1580-harden-voice-safety-and-ux-on-the-unified-flue-route`, stacked directly on [PR #9528](https://github.com/hashintel/hash/pull/9528) at the GitHub-verified head `6ca81b7bc4d6c112ff3936c38a7209d07f773b8a`. This file is the sole execution authority for the successor branch. The restack adopts the parent's canonical hydration overwrite guard, multi-submission response correlation, settlement-driven durable Stop, aligned live/snapshot projection, queued Voice-input cancellation, and client-tool continuation behavior. The parent remains the authority for defects in those mechanisms; this branch must restack onto further parent fixes rather than repair them. +**Live as of 2026-09-04** for [FE-1580](https://linear.app/hash/issue/FE-1580/harden-voice-safety-and-ux-on-the-unified-flue-route) on `kostandin/fe-1580-harden-voice-safety-and-ux-on-the-unified-flue-route`, stacked directly on [PR #9528](https://github.com/hashintel/hash/pull/9528) at the GitHub-verified head `eecbe99e201fd8cb78d9b719e789b6abd373ed1b`. This file is the sole execution authority for the successor branch. The restack adopts the parent's canonical hydration overwrite guard, multi-submission response correlation, settlement-driven durable Stop, aligned live/snapshot projection, queued Voice-input cancellation, and client-tool continuation behavior. The parent remains the authority for defects in those mechanisms; this branch must restack onto further parent fixes rather than repair them. The owner selected **half-duplex turn ownership** on 2026-09-03. While canonical assistant audio is pending or playing, the microphone is closed. Ownership transfers away from input as soon as canonical speech is requested, before `response.create` is sent: every accepted unfinished input item becomes stale and provisional transcript state is cleared. The explicit **Your turn** action cancels output, waits for provider acknowledgement and response settlement, and only then opens a fresh input turn. Automatic duplex barge-in is rejected because assistant playback can become a false user turn. @@ -40,11 +40,11 @@ old parent blocker, but no real witness claim is valid until the retained human evidence exists. The 2026-09-04 corrective verification covers the current 58-file successor -diff against #9528 head `6ca81b7bc4`: the four focused race cases pass 4/4 +diff against #9528 head `eecbe99e201f`: the four focused race cases pass 4/4 tests, the filtered production admission-outcome cases pass 3/3 tests, and the -complete seven-workspace Turbo run passes 39/39 tasks, including 254/254 website -tests and the Brunch core package. Architecture validation passes with 62 -layers, 297 edges, 615 files, 63 generated pages, and 31 authored pages. `git +complete seven-workspace Turbo run passes 39/39 tasks, including 279/279 website +tests and the Brunch core package. Architecture validation passes with 68 +layers, 337 edges, 692 files, 69 generated pages, and 38 authored pages. `git diff --check` and formatting of all 52 formatter-owned successor files pass. The root formatter remains red only outside this branch's diff on an unrelated untracked `.cursor` plan. The exact commands and dispositions are retained in diff --git a/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/donor-behavior-matrix.md b/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/donor-behavior-matrix.md index d8401984e08..3321a1c538c 100644 --- a/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/donor-behavior-matrix.md +++ b/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/donor-behavior-matrix.md @@ -6,7 +6,7 @@ This record pins the semantic disposition of the Voice donor branches for the li | Source | Pinned head | Role | | --- | --- | --- | -| Parent PR [#9528](https://github.com/hashintel/hash/pull/9528) | `6ca81b7bc4d6c112ff3936c38a7209d07f773b8a` | Unified Flue route and path-B departure base | +| Parent PR [#9528](https://github.com/hashintel/hash/pull/9528) | `eecbe99e201fd8cb78d9b719e789b6abd373ed1b` | Unified Flue route and path-B departure base | | Donor PR [#9496](https://github.com/hashintel/hash/pull/9496) | `c7fe8a2e68e8fdc37018b21ec2e9daf4e9ef7c82` | Canonical TTS queue and replay mechanics | | Donor PR [#9500](https://github.com/hashintel/hash/pull/9500) | `935aa9f02a5ac635a50eb8bc130edb3e258af8e4` | Completed-transcript authority | | Donor PR [#9507](https://github.com/hashintel/hash/pull/9507) | `252b9dbb0c77fae8cee45a506f09cac3e20c381c` | Temporary `brunch_ask` shim, excluded | @@ -23,7 +23,7 @@ The owner selected half-duplex turn ownership on 2026-09-03: assistant output ow | #9528 | One `/agents/chat/:instanceId` product route, browser `ChatTransport`, one memoized client, path-B Voice submission through shared `useChat` | **Adopt** | This is the departure architecture and prevents a second admission authority. | Restack onto every new parent head; verify no successor code calls `send()` directly from Voice. | | #9528 | Direct Voice `send()` as a fog-line fallback | **Reject** | It creates a second admission path and mutable coordination surface. The parent has already proved path B. | Mission authority now permits path B only. | | #9528 | Claim that Flue 2.0.3 lacks caller idempotency | **Reject as factually false** | Installed typings expose `AgentPromptOptions.idempotencyKey`, `AgentSendResult.deduplicated`, and 409 `submission_conflict` with the existing `submissionId`. | Implemented with transport convergence/conflict tests and typed Voice admission outcomes. | -| #9528 | Canonical hydration guard, multi-submission response correlation, settlement-driven durable Stop, aligned live/snapshot projection, queued Voice-input cancellation, and client-tool continuation | **Adopt through restack** | These mechanisms remain parent-owned and must enter the successor through the stack rather than copied fixes. | Restacked onto `6ca81b7bc4`; hydration no longer blocks the real witness. Further defects in these mechanisms remain parent scope. | +| #9528 | Canonical hydration guard, multi-submission response correlation, settlement-driven durable Stop, aligned live/snapshot projection, queued Voice-input cancellation, and client-tool continuation | **Adopt through restack** | These mechanisms remain parent-owned and must enter the successor through the stack rather than copied fixes. | Restacked onto `eecbe99e201f`; hydration no longer blocks the real witness. Further defects in these mechanisms remain parent scope. | | #9496 | Serialized canonical speech queue, retained exact source segments, response/output terminal gating | **Adopt mechanics** | Replay and ordinary TTS need one lifecycle-safe queue, and exact text preserves canonical authority. | Implemented without a preparation/simplifier dependency; exact-segment and queue tests pass. | | #9496 | `canReadFullResponse`, `readFullResponse()`, exact full-response playback menu | **Adopt** | Exact full-response replay is supported by retained canonical segment identity. | Implemented with idle-state and matching response/output terminal gates. | | #9496 | `canRepeatQuestion`, `repeatQuestion()`, and playback-menu action | **Adopt UX; reject final-segment inference** | The final segment may be ordinary prose and is not authority for question identity. The approved `brunch_mark_question` data marker now supplies deterministic identity without accepting an answer. | Implemented by replaying only exact marked text found in finalized prose from the same assistant message; a missing or unmatched marker leaves the action disabled. | @@ -73,7 +73,13 @@ The owner selected half-duplex turn ownership on 2026-09-03: assistant output ow ## Corrective verification Fresh local checks on 2026-09-04 cover the 58-file successor diff against the -verified #9528 head `6ca81b7bc4d6c112ff3936c38a7209d07f773b8a`: +verified #9528 head `eecbe99e201fd8cb78d9b719e789b6abd373ed1b`. Graphite replayed one +repeatedly touched integration-test conflict while restacking: the semantic +resolution keeps the parent's required URL-navigation props and tests together +with the successor's admission, dormant-ask, durable-Stop, and status-removal +proof. No production-source conflict was resolved by choosing either side +wholesale. The verified code head before this evidence-only update is +`9938283a19ab20567ad6b4c96330ea392243c16f`: | Command | Result | | --- | --- | @@ -85,9 +91,9 @@ verified #9528 head `6ca81b7bc4d6c112ff3936c38a7209d07f773b8a`: | `mise exec -- yarn workspace @hashintel/petrinaut test:unit --run src/ui/views/Editor/panels/ai-assistant-panel.test.tsx` | Exit 0; 44/44 production host-registration and panel tests passed. | | `mise exec -- yarn workspace @apps/petrinaut-website test:unit src/main/app/voice-interview/canonical-speech.test.ts src/main/app/voice-interview/openai-realtime-session.test.ts src/main/app/voice-interview/voice-turn-controller.test.ts` | Exit 0; 79/79 exact replay, queue, terminal-gating, and turn-controller tests passed. | | `mise exec -- yarn exec turbo run lint:tsc lint:eslint test:unit build --filter @hashintel/brunch-agent` | Exit 0; 5/5 tasks passed, including 10/10 test files and 86/86 tests; the four question-marker mock lint failures are resolved with production-interface signatures. | -| `mise exec -- yarn exec turbo run lint:tsc lint:eslint test:unit build --filter @apps/brunch-agent --filter @apps/petrinaut-website --filter @hashintel/petrinaut --filter @hashintel/brunch-agent --filter @hashintel/brunch-agent-binding-flue --filter @hashintel/brunch-agent-plugin-sdcpn --filter @hashintel/brunch-agent-transport-aisdk` | Exit 0; 39/39 tasks passed, including 16/16 Brunch app files with 79/79 tests and 31/31 website files with 254/254 tests. Website ESLint retains one non-failing warning at `voice-interview-control.tsx:493`. | -| `yarn workspace @local/petrinaut-arch-docs lint:arch-docs` | Exit 0; 62 layers, 297 edges, 615 files, 63 generated pages, and 31 authored pages. | -| `git diff --name-only -z 6ca81b7bc4d6c112ff3936c38a7209d07f773b8a...HEAD \| xargs -0 yarn exec oxfmt --check` | Exit 0; all 52 formatter-owned files in the 58-file successor diff passed. The six unmatched documentation files are excluded by `oxfmt.config.ts` and pass repository whitespace validation. | +| `mise exec -- yarn exec turbo run lint:tsc lint:eslint test:unit build --filter @apps/brunch-agent --filter @apps/petrinaut-website --filter @hashintel/petrinaut --filter @hashintel/brunch-agent --filter @hashintel/brunch-agent-binding-flue --filter @hashintel/brunch-agent-plugin-sdcpn --filter @hashintel/brunch-agent-transport-aisdk` | Exit 0; 39/39 tasks passed, including 16/16 Brunch app files with 79/79 tests, 10/10 Brunch core files with 86/86 tests, 3/3 transport files with 26/26 tests, 5/5 binding files with 18/18 tests, 2/2 plugin files with 8/8 tests, 72/72 Petrinaut files with 616/616 tests, and 31/31 website files with 279/279 tests. Website ESLint retains one non-failing warning at `voice-interview-control.tsx:493`; other inherited warnings remain outside this successor's corrective scope. | +| `yarn workspace @local/petrinaut-arch-docs lint:arch-docs` | Exit 0; 68 layers, 337 edges, 692 files, 69 generated pages, and 38 authored pages. | +| `git diff --name-only -z eecbe99e201fd8cb78d9b719e789b6abd373ed1b...HEAD \| xargs -0 yarn exec oxfmt --check` | Exit 0; all 52 formatter-owned files in the 58-file successor diff passed. The six unmatched documentation files are excluded by `oxfmt.config.ts` and pass repository whitespace validation. | | `git diff --check` | Exit 0. | | `yarn lint:format` | Exit 1 outside the successor diff only: unrelated untracked `.cursor/plans/fe-1574_mission_recut_6f23f7cd.plan.md`. It was not modified. | | In isolated detached worktree `/Users/kostandin/Projects/hashdev/worktrees/fe-1580-latency-baseline-9496`: `mise exec -- yarn exec turbo run build --filter '@apps/petrinaut-website^...'`, then `mise exec -- yarn workspace @apps/petrinaut-website test:unit src/main/app/voice-interview/canonical-speech.test.ts src/main/app/voice-interview/openai-realtime-session.test.ts src/main/app/voice-interview/realtime-brunch-bridge.test.ts src/main/app/voice-interview/voice-turn-controller.test.ts src/main/app/voice-interview/voice-preview.integration.test.ts` | Exit 0; dependency build passed 14/14 tasks, then all 5/5 donor Voice files and 108/108 tests passed at pinned #9496 head. The isolated donor and candidate panels return HTTP 200 on ports 4916 and 4915 respectively; real audible samples remain uncollected. | diff --git a/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/witness-blocker.md b/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/witness-blocker.md index 92e59acf1d8..1ea7de3378d 100644 --- a/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/witness-blocker.md +++ b/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/witness-blocker.md @@ -12,7 +12,7 @@ network-route witness required for mission acceptance. Direct-user Voice attribution has a separate [Flue projection blocker](provenance-blocker.md). The successor is restacked onto [PR #9528](https://github.com/hashintel/hash/pull/9528) -head `6ca81b7bc4d6c112ff3936c38a7209d07f773b8a`. That parent now guards its +head `eecbe99e201fd8cb78d9b719e789b6abd373ed1b`. That parent now guards its once-per-conversation hydration from replacing a locally visible assistant response with an older canonical snapshot, so hydration no longer blocks this witness. The remaining gate is the required human browser and microphone run. From a04e142361354f90998e586843baf492919d681d Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Fri, 4 Sep 2026 11:54:54 +0200 Subject: [PATCH 34/52] Record Voice launcher remediation side quest Co-authored-by: Cursor --- libs/@hashintel/brunch-agent/SIDE_QUEST.md | 53 ++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 libs/@hashintel/brunch-agent/SIDE_QUEST.md diff --git a/libs/@hashintel/brunch-agent/SIDE_QUEST.md b/libs/@hashintel/brunch-agent/SIDE_QUEST.md new file mode 100644 index 00000000000..84524c226d3 --- /dev/null +++ b/libs/@hashintel/brunch-agent/SIDE_QUEST.md @@ -0,0 +1,53 @@ +# Side quest — Preserve Voice APIs in the Brunch launcher + +## Relationship to the live mission + +The real `yarn dev:brunch` witness required by Mission 5 exposed a bounded launcher +regression: the Brunch-specific Vite config removes Petrinaut's entire +`petrinaut-api-dev` plugin, so `/api/voice/config` returns transformed source +instead of the handler's JSON response. The owner directed this remediation to +PR #9531 rather than its parent. This side quest changes only local launcher +wiring and does not change the unified Flue conversation route or Voice turn +semantics. + +## Imperative + +Make the Brunch-configured Petrinaut panel retain the website's API plugin while +continuing to proxy `/agents/chat/*` to the Brunch server, so the existing Voice +availability and Realtime-call handlers can run under `yarn dev:brunch`. + +## Throughline + +```text +yarn dev:brunch +→ apps/brunch-agent/petrinaut-local.vite.config.ts +→ real apps/petrinaut-website Vite config and petrinaut-api-dev plugin +→ /api/voice/config and /api/voice/realtime-call + +panel /agents/chat/* +→ unchanged same-origin proxy +→ Brunch Flue server +``` + +## Proof + +- A config-level regression test loads the real merged Brunch panel config and + finds the `petrinaut-api-dev` plugin alongside the `/agents/chat` proxy. +- The focused Brunch test suite passes. +- With the required Voice environment enabled, a real local + `/api/voice/config` request returns the handler's JSON availability envelope, + not source text. + +## Constraints + +- Do not alter the stock Petrinaut `/api/chat` handler or route. +- Do not proxy `/api/voice/*` to the Brunch server. +- Preserve `/agents/chat` as the only Brunch conversation route. +- Do not change Voice policy, conversation state, or production deployment. + +## Stop conditions and budget + +Stop if retaining the website plugin causes `/api/chat` to intercept +`/agents/chat`, changes the Flue proxy, or requires production routing work. +Budget: one launcher config change, one focused regression test, and the minimum +documentation/evidence update required to close this side quest. From 011b8bc59fa742a69997f591abcb36fd6935c5e4 Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Fri, 4 Sep 2026 11:57:08 +0200 Subject: [PATCH 35/52] Preserve Voice APIs in Brunch launcher Co-authored-by: Cursor --- .../petrinaut-local.vite.config.ts | 46 ++++------------ .../test/local-dev-origins.test.ts | 38 +++++++++++++ libs/@hashintel/brunch-agent/MISSION.md | 9 ++++ libs/@hashintel/brunch-agent/SIDE_QUEST.md | 53 ------------------- .../witness-blocker.md | 11 ++++ 5 files changed, 67 insertions(+), 90 deletions(-) delete mode 100644 libs/@hashintel/brunch-agent/SIDE_QUEST.md diff --git a/apps/brunch-agent/petrinaut-local.vite.config.ts b/apps/brunch-agent/petrinaut-local.vite.config.ts index 78036a815ae..281f11b4565 100644 --- a/apps/brunch-agent/petrinaut-local.vite.config.ts +++ b/apps/brunch-agent/petrinaut-local.vite.config.ts @@ -1,43 +1,21 @@ /** * Local FE-1436 panel launcher. * - * Loads the real hash Petrinaut website config, removes only the website's - * stock `/api/chat` dev handler, and proxies Brunch's mounted Flue route to - * the committed application server. The real panel, wrappers, and editor - * stay untouched; hash's tracked checkout stays clean. + * Loads the real hash Petrinaut website config, including its stock API + * handlers, and proxies Brunch's mounted Flue route to the committed + * application server. The real panel, wrappers, and editor stay untouched; + * hash's tracked checkout stays clean. */ import { join, resolve } from "node:path"; -import { - defineConfig, - loadConfigFromFile, - mergeConfig, - type PluginOption, -} from "vite"; +import { defineConfig, loadConfigFromFile, mergeConfig } from "vite"; import { defaultChatOrigin, petrinautLocalServer, } from "./src/http/local-origins.ts"; -const withoutIncumbentChatHandler = ( - plugins: readonly PluginOption[], -): PluginOption[] => - plugins.filter((plugin) => { - if ( - plugin === false || - plugin === null || - plugin === undefined || - Array.isArray(plugin) || - typeof plugin !== "object" || - !("name" in plugin) - ) { - return true; - } - return plugin.name !== "petrinaut-api-dev"; - }); - export default defineConfig(async (environment) => { const websiteRoot = process.env.PETRINAUT_WEBSITE_ROOT; if (!websiteRoot) { @@ -58,14 +36,8 @@ export default defineConfig(async (environment) => { throw new Error(`Could not load Petrinaut's Vite config from ${root}.`); const chatOrigin = process.env.BRUNCH_CHAT_ORIGIN ?? defaultChatOrigin; - return mergeConfig( - { - ...loaded.config, - plugins: withoutIncumbentChatHandler(loaded.config.plugins ?? []), - }, - { - root, - server: petrinautLocalServer(chatOrigin), - }, - ); + return mergeConfig(loaded.config, { + root, + server: petrinautLocalServer(chatOrigin), + }); }); diff --git a/apps/brunch-agent/test/local-dev-origins.test.ts b/apps/brunch-agent/test/local-dev-origins.test.ts index ddf0d734ec3..38f957f1174 100644 --- a/apps/brunch-agent/test/local-dev-origins.test.ts +++ b/apps/brunch-agent/test/local-dev-origins.test.ts @@ -1,7 +1,9 @@ import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; import { expect, test } from "vitest"; +import petrinautLocalConfig from "../petrinaut-local.vite.config.ts"; import { defaultChatOrigin, localChatListen, @@ -64,3 +66,39 @@ test("petrinaut:dev proxies the mounted Flue conversation route", () => { 'VITE_BRUNCH_CHAT_ENDPOINT ??= "/agents/chat"', ); }); + +test("petrinaut:dev retains the website API handlers needed by Voice", async () => { + const previousCwd = process.cwd(); + const previousWebsiteRoot = process.env.PETRINAUT_WEBSITE_ROOT; + const previousChatEndpoint = process.env.VITE_BRUNCH_CHAT_ENDPOINT; + process.env.PETRINAUT_WEBSITE_ROOT = fileURLToPath( + new URL("../../petrinaut-website", import.meta.url), + ); + + try { + const config = await petrinautLocalConfig({ + command: "serve", + isPreview: false, + isSsrBuild: false, + mode: "test", + }); + + expect(config.plugins).toEqual( + expect.arrayContaining([ + expect.objectContaining({ name: "petrinaut-api-dev" }), + ]), + ); + } finally { + process.chdir(previousCwd); + if (previousWebsiteRoot === undefined) { + delete process.env.PETRINAUT_WEBSITE_ROOT; + } else { + process.env.PETRINAUT_WEBSITE_ROOT = previousWebsiteRoot; + } + if (previousChatEndpoint === undefined) { + delete process.env.VITE_BRUNCH_CHAT_ENDPOINT; + } else { + process.env.VITE_BRUNCH_CHAT_ENDPOINT = previousChatEndpoint; + } + } +}); diff --git a/libs/@hashintel/brunch-agent/MISSION.md b/libs/@hashintel/brunch-agent/MISSION.md index 824449b867c..67a01e47704 100644 --- a/libs/@hashintel/brunch-agent/MISSION.md +++ b/libs/@hashintel/brunch-agent/MISSION.md @@ -22,6 +22,13 @@ defines the accepted and rejected mechanics. The parent now prevents its once-per-conversation canonical hydration from overwriting a locally submitted turn. The real microphone, handoff, Stop, hard-reload, and same-origin witness is therefore executable but remains unproved until the human witness and retained artifact bundle are complete. +On 2026-09-04, the owner directed PR #9531 to close one bounded launcher +defect exposed while starting that witness. The Brunch panel config now retains +Petrinaut's stock API plugin so `/api/voice/config` and +`/api/voice/realtime-call` remain available beside the unchanged +`/agents/chat/*` Flue proxy. This is an explicit exception to the parent-defect +constraint below; it changes no conversation or Voice-turn semantics. + Completed-transcript authority, half-duplex ownership, admission idempotency, the cancellation barrier, exact full-response replay, and exact marked-question replay have focused regression coverage, including the interval between a @@ -142,6 +149,8 @@ This mission closes the Voice safety and UX-parity stratum on the parent's route ~ libs/@hashintel/brunch-agent/packages/transport-aisdk/ stable idempotency and canonical projection/provenance tests/code ~ libs/@hashintel/petrinaut/src/react/voice-session/ public Voice state required by the panel ~ libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ Your turn, replay menu, durable provenance +~ apps/brunch-agent/petrinaut-local.vite.config.ts retain Petrinaut Voice API handlers beside the Flue proxy +~ apps/brunch-agent/test/local-dev-origins.test.ts real merged-config launcher regression ~ apps/petrinaut-website/README.md operator behavior ~ libs/@hashintel/petrinaut/docs/ai-assistant.md end-user behavior ? .changeset/ one patch changeset if published Petrinaut changes diff --git a/libs/@hashintel/brunch-agent/SIDE_QUEST.md b/libs/@hashintel/brunch-agent/SIDE_QUEST.md deleted file mode 100644 index 84524c226d3..00000000000 --- a/libs/@hashintel/brunch-agent/SIDE_QUEST.md +++ /dev/null @@ -1,53 +0,0 @@ -# Side quest — Preserve Voice APIs in the Brunch launcher - -## Relationship to the live mission - -The real `yarn dev:brunch` witness required by Mission 5 exposed a bounded launcher -regression: the Brunch-specific Vite config removes Petrinaut's entire -`petrinaut-api-dev` plugin, so `/api/voice/config` returns transformed source -instead of the handler's JSON response. The owner directed this remediation to -PR #9531 rather than its parent. This side quest changes only local launcher -wiring and does not change the unified Flue conversation route or Voice turn -semantics. - -## Imperative - -Make the Brunch-configured Petrinaut panel retain the website's API plugin while -continuing to proxy `/agents/chat/*` to the Brunch server, so the existing Voice -availability and Realtime-call handlers can run under `yarn dev:brunch`. - -## Throughline - -```text -yarn dev:brunch -→ apps/brunch-agent/petrinaut-local.vite.config.ts -→ real apps/petrinaut-website Vite config and petrinaut-api-dev plugin -→ /api/voice/config and /api/voice/realtime-call - -panel /agents/chat/* -→ unchanged same-origin proxy -→ Brunch Flue server -``` - -## Proof - -- A config-level regression test loads the real merged Brunch panel config and - finds the `petrinaut-api-dev` plugin alongside the `/agents/chat` proxy. -- The focused Brunch test suite passes. -- With the required Voice environment enabled, a real local - `/api/voice/config` request returns the handler's JSON availability envelope, - not source text. - -## Constraints - -- Do not alter the stock Petrinaut `/api/chat` handler or route. -- Do not proxy `/api/voice/*` to the Brunch server. -- Preserve `/agents/chat` as the only Brunch conversation route. -- Do not change Voice policy, conversation state, or production deployment. - -## Stop conditions and budget - -Stop if retaining the website plugin causes `/api/chat` to intercept -`/agents/chat`, changes the Flue proxy, or requires production routing work. -Budget: one launcher config change, one focused regression test, and the minimum -documentation/evidence update required to close this side quest. diff --git a/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/witness-blocker.md b/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/witness-blocker.md index 1ea7de3378d..e15c60a36d5 100644 --- a/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/witness-blocker.md +++ b/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/witness-blocker.md @@ -17,6 +17,17 @@ once-per-conversation hydration from replacing a locally visible assistant response with an older canonical snapshot, so hydration no longer blocks this witness. The remaining gate is the required human browser and microphone run. +An owner-directed PR #9531 side quest also removed a local launcher blocker +found at the real boundary on 2026-09-04. The Brunch-specific Vite config had +removed Petrinaut's entire `petrinaut-api-dev` plugin, so +`/api/voice/config` returned transformed module source instead of the handler's +JSON. The launcher now retains the website API plugin while continuing to +proxy only `/agents/chat/*` to Brunch. A config-level regression test loads the +real merged config, and an isolated `yarn dev:brunch` panel process with an +enabled non-secret test environment returned +`{"available":true,"connectionTimeoutMs":15000}`. This proves local Voice API +wiring only; it does not satisfy the human witness below. + ## Re-entry gate Using the final source/build commit: From 3f3b106ee82cdff319e04a152526eb73b640a027 Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Fri, 4 Sep 2026 12:25:15 +0200 Subject: [PATCH 36/52] Document compact Voice consent card Co-authored-by: Cursor --- ...09-04-voice-consent-compact-card-design.md | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 docs/superpowers/specs/2026-09-04-voice-consent-compact-card-design.md diff --git a/docs/superpowers/specs/2026-09-04-voice-consent-compact-card-design.md b/docs/superpowers/specs/2026-09-04-voice-consent-compact-card-design.md new file mode 100644 index 00000000000..5f604266e28 --- /dev/null +++ b/docs/superpowers/specs/2026-09-04-voice-consent-compact-card-design.md @@ -0,0 +1,68 @@ +# Voice consent compact card + +## Goal + +Replace PR #9531's visually loose Voice consent block with a compact setup card +that matches Petrinaut's AI assistant panel. Preserve the existing disclosure, +explicit acknowledgement, microphone check, focus behavior, and start flow. + +## Design + +The pre-session Voice surface remains pinned between the transcript and composer. +It becomes a quiet white card inside the panel's subtle footer surface, using the +same border, radius, shadow, spacing, and typography language as the composer and +message cards. + +The card contains: + +1. A small brand-tinted Voice icon beside the title. +2. The title **Start a voice conversation** and subtitle **Talk through your + process with AI**. +3. Concise disclosure copy: **OpenAI processes live audio and speaks the + interviewer’s words. Petrinaut saves finalized answers—not audio.** +4. A design-system checkbox in a muted inset row with the label **I understand + how voice data is handled.** +5. A compact brand **Start voice** button and subtle **Test microphone** button. + +The card must remain legible at the assistant's 320 px minimum width. Actions may +wrap without changing their order. + +## Interaction and states + +- **Start voice** is disabled until the checkbox is selected. +- **Test microphone** does not store consent or start Voice. +- While testing, the secondary button shows a loading state and duplicate checks + are prevented. +- Success and failure replace a reserved status line below the actions, avoiding + layout jump. The status uses polite live-region semantics. +- Starting Voice stores the existing versioned acknowledgement, hides the card, + and starts the existing controller exactly once. +- Keyboard focus continues to move to the consent section when it appears. + +## Accessibility + +- Use `@hashintel/ds-components` `Checkbox` and `Button` rather than native + unstyled controls. +- Keep the section's existing accessible name. +- Associate status text with the microphone-check action through the live region. +- Preserve visible focus states supplied by the design system. +- Do not rely on color alone for status. + +## Scope + +The implementation is limited to +`apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx` +and its focused tests. No Voice transport, consent persistence, session, panel +layout, or live Voice dock behavior changes. + +The existing Petrinaut AI-assistant documentation remains behaviorally accurate; +no user-guide copy change is required. + +## Verification + +- Extend the focused test to cover the design-system checkbox, compact labels, + disabled start state, microphone-check loading guard, success/failure status, + and unchanged acknowledgement behavior. +- Run the focused Voice control test. +- Run Petrinaut website type checking and linting. +- Inspect the card in the real 500 px panel and at the 320 px minimum width. From cfa0441bd375906c2c63c00c0556b740ead3d08f Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Fri, 4 Sep 2026 12:28:13 +0200 Subject: [PATCH 37/52] Make Voice launcher test CI-hermetic Amp-Thread-ID: https://ampcode.com/threads/T-01a06941-3ee7-7482-a82e-8a16c7141785 Co-authored-by: Amp --- .../petrinaut-local.vite.config.ts | 28 +++++++++-- .../test/local-dev-origins.test.ts | 49 ++++++------------- 2 files changed, 39 insertions(+), 38 deletions(-) diff --git a/apps/brunch-agent/petrinaut-local.vite.config.ts b/apps/brunch-agent/petrinaut-local.vite.config.ts index 281f11b4565..ea37867c9c5 100644 --- a/apps/brunch-agent/petrinaut-local.vite.config.ts +++ b/apps/brunch-agent/petrinaut-local.vite.config.ts @@ -9,13 +9,34 @@ import { join, resolve } from "node:path"; -import { defineConfig, loadConfigFromFile, mergeConfig } from "vite"; +import { + defineConfig, + loadConfigFromFile, + mergeConfig, + type UserConfig, +} from "vite"; import { defaultChatOrigin, petrinautLocalServer, } from "./src/http/local-origins.ts"; +interface PetrinautPanelConfigOptions { + readonly chatOrigin: string; + readonly loadedConfig: UserConfig; + readonly root: string; +} + +export const mergePetrinautPanelConfig = ({ + chatOrigin, + loadedConfig, + root, +}: PetrinautPanelConfigOptions): UserConfig => + mergeConfig(loadedConfig, { + root, + server: petrinautLocalServer(chatOrigin), + }); + export default defineConfig(async (environment) => { const websiteRoot = process.env.PETRINAUT_WEBSITE_ROOT; if (!websiteRoot) { @@ -36,8 +57,9 @@ export default defineConfig(async (environment) => { throw new Error(`Could not load Petrinaut's Vite config from ${root}.`); const chatOrigin = process.env.BRUNCH_CHAT_ORIGIN ?? defaultChatOrigin; - return mergeConfig(loaded.config, { + return mergePetrinautPanelConfig({ + chatOrigin, + loadedConfig: loaded.config, root, - server: petrinautLocalServer(chatOrigin), }); }); diff --git a/apps/brunch-agent/test/local-dev-origins.test.ts b/apps/brunch-agent/test/local-dev-origins.test.ts index 38f957f1174..56da1c3f41b 100644 --- a/apps/brunch-agent/test/local-dev-origins.test.ts +++ b/apps/brunch-agent/test/local-dev-origins.test.ts @@ -1,9 +1,8 @@ import { readFileSync } from "node:fs"; -import { fileURLToPath } from "node:url"; import { expect, test } from "vitest"; -import petrinautLocalConfig from "../petrinaut-local.vite.config.ts"; +import { mergePetrinautPanelConfig } from "../petrinaut-local.vite.config.ts"; import { defaultChatOrigin, localChatListen, @@ -67,38 +66,18 @@ test("petrinaut:dev proxies the mounted Flue conversation route", () => { ); }); -test("petrinaut:dev retains the website API handlers needed by Voice", async () => { - const previousCwd = process.cwd(); - const previousWebsiteRoot = process.env.PETRINAUT_WEBSITE_ROOT; - const previousChatEndpoint = process.env.VITE_BRUNCH_CHAT_ENDPOINT; - process.env.PETRINAUT_WEBSITE_ROOT = fileURLToPath( - new URL("../../petrinaut-website", import.meta.url), - ); - - try { - const config = await petrinautLocalConfig({ - command: "serve", - isPreview: false, - isSsrBuild: false, - mode: "test", - }); +test("petrinaut:dev retains the website API handlers needed by Voice", () => { + const config = mergePetrinautPanelConfig({ + chatOrigin: defaultChatOrigin, + loadedConfig: { + plugins: [{ name: "petrinaut-api-dev" }], + }, + root: "/test/petrinaut-website", + }); - expect(config.plugins).toEqual( - expect.arrayContaining([ - expect.objectContaining({ name: "petrinaut-api-dev" }), - ]), - ); - } finally { - process.chdir(previousCwd); - if (previousWebsiteRoot === undefined) { - delete process.env.PETRINAUT_WEBSITE_ROOT; - } else { - process.env.PETRINAUT_WEBSITE_ROOT = previousWebsiteRoot; - } - if (previousChatEndpoint === undefined) { - delete process.env.VITE_BRUNCH_CHAT_ENDPOINT; - } else { - process.env.VITE_BRUNCH_CHAT_ENDPOINT = previousChatEndpoint; - } - } + expect(config.plugins).toEqual( + expect.arrayContaining([ + expect.objectContaining({ name: "petrinaut-api-dev" }), + ]), + ); }); From 7a1f74642f62f803e98f793ab1b250fa61dbc3a6 Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Fri, 4 Sep 2026 12:35:41 +0200 Subject: [PATCH 38/52] Plan compact Voice consent card Co-authored-by: Cursor --- .../2026-09-04-voice-consent-compact-card.md | 377 ++++++++++++++++++ 1 file changed, 377 insertions(+) create mode 100644 docs/superpowers/plans/2026-09-04-voice-consent-compact-card.md diff --git a/docs/superpowers/plans/2026-09-04-voice-consent-compact-card.md b/docs/superpowers/plans/2026-09-04-voice-consent-compact-card.md new file mode 100644 index 00000000000..2c6783a1507 --- /dev/null +++ b/docs/superpowers/plans/2026-09-04-voice-consent-compact-card.md @@ -0,0 +1,377 @@ +# Voice Consent Compact Card Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Replace PR #9531's loose first-use Voice disclosure with the approved compact Petrinaut setup card without changing consent or session semantics. + +**Architecture:** Keep the pre-session surface inside `VoiceInterviewControl`, where its behavior already lives. Restyle that surface with Petrinaut design-system primitives and add only local microphone-check pending state; the panel slot, persisted acknowledgement, and Voice controller remain unchanged. + +**Tech Stack:** React 19, TypeScript, Panda CSS, `@hashintel/ds-components`, Vitest, Testing Library. + +## Global Constraints + +- Keep the surface pinned between the transcript and composer. +- Use the approved title, disclosure, checkbox, and button copy exactly. +- Use `Checkbox` and `Button` from `@hashintel/ds-components`. +- Preserve the versioned acknowledgement, focus behavior, and one-start flow. +- Keep the layout legible at the assistant's 320 px minimum width. +- Do not modify Voice transport, session, panel layout, or live Voice dock behavior. + +--- + +## File structure + +- Modify `apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx`: render and style the compact disclosure card and guard the local microphone check while pending. +- Modify `apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx`: pin the approved copy, accessible controls, pending microphone behavior, and unchanged acknowledgement semantics. + +### Task 1: Build the compact Voice setup card + +**Files:** + +- Modify: `apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx:10,207-306,405-407,535-558` +- Test: `apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx:373-422,556-571` + +**Interfaces:** + +- Consumes: `Button` and `Checkbox` from `@hashintel/ds-components`; existing `consented`, `microphoneCheck`, `onCheckMicrophone`, `onConsentChange`, and `onStart` values. +- Produces: the existing `VoiceInterviewDisclosure` surface with one additional `checkingMicrophone: boolean` prop. + +- [ ] **Step 1: Update the first-use disclosure test to pin the approved accessible UI** + +Replace the old title and button assertions with: + +```tsx +const disclosure = screen.getByRole("region", { + name: "Voice mode consent", +}); +expect( + within(disclosure).getByText("Start a voice conversation"), +).not.toBeNull(); +expect( + within(disclosure).getByText( + "OpenAI processes live audio and speaks the interviewer’s words. Petrinaut saves finalized answers—not audio.", + ), +).not.toBeNull(); + +const consent = within(disclosure).getByRole("checkbox", { + name: "I understand how voice data is handled.", +}); +const start = within(disclosure).getByRole("button", { + name: "Start voice", +}); +expect(start.hasAttribute("disabled")).toBe(true); +fireEvent.click(consent); +expect(start.hasAttribute("disabled")).toBe(false); +expect( + within(disclosure).getByRole("button", { name: "Test microphone" }), +).not.toBeNull(); +expect(document.activeElement).toBe(disclosure); +``` + +Update the later acknowledgement test to query **Test microphone** and **Start voice**. + +- [ ] **Step 2: Add a failing test for one pending microphone check** + +Add a test with a controllable `getUserMedia` promise: + +```tsx +test("keeps one microphone check pending and reports its result", async () => { + let resolveCheck: ((stream: MediaStream) => void) | undefined; + const getUserMedia = vi.fn( + () => + new Promise((resolve) => { + resolveCheck = resolve; + }), + ); + vi.stubGlobal("navigator", { mediaDevices: { getUserMedia } }); + render(); + + fireEvent.click(screen.getByRole("button", { name: "Select Voice" })); + const check = screen.getByRole("button", { name: "Test microphone" }); + fireEvent.click(check); + fireEvent.click(check); + + expect(getUserMedia).toHaveBeenCalledOnce(); + expect(check.getAttribute("aria-busy")).toBe("true"); + + resolveCheck?.({ getTracks: () => [] } as unknown as MediaStream); + + expect(await screen.findByText("Microphone ready.")).not.toBeNull(); + await waitFor(() => expect(check.getAttribute("aria-busy")).toBe("false")); +}); +``` + +- [ ] **Step 3: Run the focused test and verify the new assertions fail** + +Run: + +```bash +yarn workspace @apps/petrinaut-website test:unit src/main/app/voice-interview/voice-interview-control.test.tsx +``` + +Expected: the suite fails because the approved copy, button names, design-system checkbox, and pending-check guard are not implemented. + +- [ ] **Step 4: Replace the native checkbox and loose disclosure markup** + +Import the design-system checkbox: + +```tsx +import { Button, Checkbox } from "@hashintel/ds-components"; +``` + +Add a local decorative waveform matching Petrinaut's composer action: + +```tsx +const VoiceModeIcon = () => ( + +); +``` + +Replace the current disclosure styles with: + +```tsx +const disclosureFrameStyle = css({ + width: "full", + padding: "2", + borderTopWidth: "thin", + borderTopStyle: "solid", + borderTopColor: "neutral.a20", + backgroundColor: "neutral.bg.subtle", + color: "neutral.s100", + _focus: { outline: "none" }, +}); + +const disclosureCardStyle = css({ + display: "flex", + flexDirection: "column", + gap: "2", + padding: "3", + borderWidth: "thin", + borderStyle: "solid", + borderColor: "neutral.a20", + borderRadius: "xl", + backgroundColor: "neutral.s00", + boxShadow: + "[0px 0px 0px 1px rgba(0,0,0,0.03), 0px 8px 16px -12px rgba(0,0,0,0.18)]", +}); + +const disclosureHeaderStyle = css({ + display: "flex", + alignItems: "center", + gap: "2", +}); + +const disclosureIconStyle = css({ + display: "inline-flex", + width: "7", + height: "7", + flexShrink: "0", + alignItems: "center", + justifyContent: "center", + borderRadius: "lg", + backgroundColor: "blue.a20", + color: "blue.s90", +}); + +const disclosureTitleStyle = css({ + display: "flex", + minWidth: "[0]", + flexDirection: "column", + gap: "0.5", +}); + +const disclosureHeadingStyle = css({ + fontSize: "sm", + fontWeight: "semibold", + lineHeight: "tight", +}); + +const disclosureSubtitleStyle = css({ + color: "neutral.s80", + fontSize: "xs", +}); + +const disclosureCopyStyle = css({ + color: "neutral.s90", + fontSize: "xs", + lineHeight: "relaxed", +}); + +const disclosureConsentStyle = css({ + width: "full", + padding: "2", + borderRadius: "lg", + backgroundColor: "neutral.a10", + color: "neutral.s100", +}); + +const disclosureActionsStyle = css({ + display: "flex", + flexWrap: "wrap", + alignItems: "center", + gap: "2", +}); + +const disclosureStatusStyle = css({ + minHeight: "[18px]", + color: "neutral.s80", + fontSize: "xs", + lineHeight: "relaxed", +}); +``` + +Render the approved card: + +```tsx +
+
+
+ + + +
+ + Start a voice conversation + + + Talk through your process with AI + +
+
+

+ OpenAI processes live audio and speaks the interviewer’s words. Petrinaut + saves finalized answers—not audio. +

+ +
+ + +
+
+ {microphoneCheck} +
+
+
+``` + +- [ ] **Step 5: Guard the local microphone check while pending** + +Add and pass the pending state: + +```tsx +const [checkingMicrophone, setCheckingMicrophone] = useState(false); +``` + +```tsx + +``` + +Implement the guarded callback: + +```tsx +onCheckMicrophone={() => { + if (checkingMicrophone) { + return; + } + setCheckingMicrophone(true); + setMicrophoneCheck(""); + void navigator.mediaDevices + .getUserMedia({ audio: true }) + .then( + (stream) => { + for (const track of stream.getTracks()) { + track.stop(); + } + setMicrophoneCheck("Microphone ready."); + }, + () => setMicrophoneCheck("Microphone access was not available."), + ) + .finally(() => setCheckingMicrophone(false)); +}} +``` + +- [ ] **Step 6: Run focused tests** + +Run: + +```bash +yarn workspace @apps/petrinaut-website test:unit src/main/app/voice-interview/voice-interview-control.test.tsx +``` + +Expected: 15 tests pass with no failures. + +- [ ] **Step 7: Run type checking and linting** + +Run: + +```bash +yarn workspace @apps/petrinaut-website lint:tsc +yarn workspace @apps/petrinaut-website lint:eslint +``` + +Expected: both commands exit 0. Existing non-failing warnings may remain unchanged. + +- [ ] **Step 8: Inspect responsive layout** + +Open the local Petrinaut preview and verify at 500 px and 320 px assistant widths: + +- the card remains between transcript and composer; +- the title and copy wrap without horizontal overflow; +- actions remain ordered **Start voice**, then **Test microphone**; +- focus, disabled, loading, success, and failure states are legible; +- selecting **Start voice** transitions to the existing live Voice dock. + +- [ ] **Step 9: Commit the implementation** + +```bash +git add \ + apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx \ + apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx +git commit -m "Polish Voice consent setup" +``` From c7efd7da31f8f3801914d8ed67d4b2279e98cc98 Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Fri, 4 Sep 2026 12:45:51 +0200 Subject: [PATCH 39/52] Polish Voice consent setup Co-authored-by: Cursor --- .../voice-interview-control.test.tsx | 80 +++++-- .../voice-interview-control.tsx | 201 +++++++++++++----- 2 files changed, 220 insertions(+), 61 deletions(-) diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx index 869628df9f6..111b6471de0 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx @@ -371,7 +371,7 @@ describe("voice interview control", () => { await expect(loadOpenAIVoiceConfig(fetch)).resolves.toBeNull(); }); - test("keeps the first-use disclosure inline without a text-handoff action", () => { + test("keeps the first-use disclosure inline without a text-handoff action", async () => { render(); fireEvent.click(screen.getByRole("button", { name: "Select Voice" })); @@ -379,16 +379,33 @@ describe("voice interview control", () => { const disclosure = screen.getByRole("region", { name: "Voice mode consent", }); - expect(disclosure).not.toBeNull(); - expect(within(disclosure).getByText("Voice mode")).not.toBeNull(); expect( - screen.getByText("OpenAI processes live audio", { exact: false }), + within(disclosure).getByText("Start a voice conversation"), ).not.toBeNull(); expect( - screen - .getByRole("button", { name: "Start voice mode" }) - .hasAttribute("disabled"), - ).toBe(true); + within(disclosure).getByText( + "OpenAI processes live audio and speaks the interviewer’s words. Petrinaut saves finalized answers—not audio.", + ), + ).not.toBeNull(); + + const consent = within(disclosure).getByRole("checkbox", { + name: "I understand how voice data is handled.", + }); + const start = within(disclosure).getByRole("button", { + name: "Start voice", + }); + expect(start.hasAttribute("disabled")).toBe(true); + fireEvent.click(consent); + await waitFor(() => + expect( + within(disclosure) + .getByRole("button", { name: "Start voice" }) + .hasAttribute("disabled"), + ).toBe(false), + ); + expect( + within(disclosure).getByRole("button", { name: "Test microphone" }), + ).not.toBeNull(); expect( screen.queryByRole("button", { name: "Use text instead" }), ).toBeNull(); @@ -408,7 +425,14 @@ describe("voice interview control", () => { fireEvent.click(screen.getByRole("button", { name: "Select Voice" })); fireEvent.click(screen.getByRole("checkbox")); - fireEvent.click(screen.getByRole("button", { name: "Start voice mode" })); + await waitFor(() => + expect( + screen + .getByRole("button", { name: "Start voice" }) + .hasAttribute("disabled"), + ).toBe(false), + ); + fireEvent.click(screen.getByRole("button", { name: "Start voice" })); expect(await screen.findByText("Session: error")).not.toBeNull(); expect(screen.getByText("Voice active")).not.toBeNull(); @@ -424,6 +448,31 @@ describe("voice interview control", () => { expect(screen.getByText("Session: error")).not.toBeNull(); }); + test("keeps one microphone check pending and reports its result", async () => { + let resolveCheck: ((stream: MediaStream) => void) | undefined; + const getUserMedia = vi.fn( + () => + new Promise((resolve) => { + resolveCheck = resolve; + }), + ); + vi.stubGlobal("navigator", { mediaDevices: { getUserMedia } }); + render(); + + fireEvent.click(screen.getByRole("button", { name: "Select Voice" })); + const check = screen.getByRole("button", { name: "Test microphone" }); + fireEvent.click(check); + fireEvent.click(check); + + expect(getUserMedia).toHaveBeenCalledOnce(); + expect(check.getAttribute("aria-busy")).toBe("true"); + + resolveCheck?.({ getTracks: () => [] } as unknown as MediaStream); + + expect(await screen.findByText("Microphone ready.")).not.toBeNull(); + await waitFor(() => expect(check.getAttribute("aria-busy")).toBe("false")); + }); + test("starts directly after acknowledgement and ends through the registered control", async () => { window.localStorage.setItem( VOICE_INTERVIEW_DISCLOSURE_STORAGE_KEY, @@ -553,18 +602,25 @@ describe("voice interview control", () => { expect(screen.getByText("Panel closed")).not.toBeNull(); }); - test("records acknowledgement only when the interview starts", () => { + test("records acknowledgement only when the interview starts", async () => { stubUnavailableMicrophone(); render(); fireEvent.click(screen.getByRole("button", { name: "Select Voice" })); - fireEvent.click(screen.getByRole("button", { name: "Check microphone" })); + fireEvent.click(screen.getByRole("button", { name: "Test microphone" })); expect( window.localStorage.getItem(VOICE_INTERVIEW_DISCLOSURE_STORAGE_KEY), ).toBeNull(); fireEvent.click(screen.getByRole("checkbox")); - fireEvent.click(screen.getByRole("button", { name: "Start voice mode" })); + await waitFor(() => + expect( + screen + .getByRole("button", { name: "Start voice" }) + .hasAttribute("disabled"), + ).toBe(false), + ); + fireEvent.click(screen.getByRole("button", { name: "Start voice" })); expect( window.localStorage.getItem(VOICE_INTERVIEW_DISCLOSURE_STORAGE_KEY), ).toBe("acknowledged"); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx index 0943cbeb679..b87a8296fe3 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx @@ -7,7 +7,7 @@ import { } from "react"; import { FlueChatAdmissionError } from "@hashintel/brunch-agent-transport-aisdk"; -import { Button } from "@hashintel/ds-components"; +import { Button, Checkbox } from "@hashintel/ds-components"; import { css } from "@hashintel/ds-helpers/css"; import { reportVoiceDiagnostic } from "../../../voice-diagnostics"; @@ -205,31 +205,82 @@ export const loadOpenAIVoiceConfig = async ( } }; -const disclosureStyle = css({ - display: "flex", +const VoiceModeIcon = () => ( + +); + +const disclosureFrameStyle = css({ width: "full", - flexDirection: "column", - gap: "2", - paddingX: "2", - paddingY: "2", + padding: "2", borderTopWidth: "thin", borderTopStyle: "solid", borderTopColor: "neutral.a20", + backgroundColor: "neutral.bg.subtle", color: "neutral.s100", + _focus: { outline: "none" }, +}); + +const disclosureCardStyle = css({ + display: "flex", + flexDirection: "column", + gap: "2", + padding: "3", + borderWidth: "thin", + borderStyle: "solid", + borderColor: "neutral.a20", + borderRadius: "xl", + backgroundColor: "neutral.s00", + boxShadow: + "[0px 0px 0px 1px rgba(0,0,0,0.03), 0px 8px 16px -12px rgba(0,0,0,0.18)]", +}); + +const disclosureHeaderStyle = css({ + display: "flex", + alignItems: "center", + gap: "2", +}); + +const disclosureIconStyle = css({ + display: "inline-flex", + width: "7", + height: "7", + flexShrink: "0", + alignItems: "center", + justifyContent: "center", + borderRadius: "lg", + backgroundColor: "blue.a20", + color: "blue.s90", }); const disclosureTitleStyle = css({ display: "flex", + minWidth: "[0]", flexDirection: "column", - gap: "1", + gap: "0.5", +}); + +const disclosureHeadingStyle = css({ fontSize: "sm", fontWeight: "semibold", + lineHeight: "tight", }); const disclosureSubtitleStyle = css({ color: "neutral.s80", fontSize: "xs", - fontWeight: "normal", }); const disclosureCopyStyle = css({ @@ -238,6 +289,14 @@ const disclosureCopyStyle = css({ lineHeight: "relaxed", }); +const disclosureConsentStyle = css({ + width: "full", + padding: "2", + borderRadius: "lg", + backgroundColor: "neutral.a10", + color: "neutral.s100", +}); + const disclosureActionsStyle = css({ display: "flex", flexWrap: "wrap", @@ -245,13 +304,22 @@ const disclosureActionsStyle = css({ gap: "2", }); +const disclosureStatusStyle = css({ + minHeight: "[18px]", + color: "neutral.s80", + fontSize: "xs", + lineHeight: "relaxed", +}); + const VoiceInterviewDisclosure = ({ + checkingMicrophone, consented, microphoneCheck, onCheckMicrophone, onConsentChange, onStart, }: { + readonly checkingMicrophone: boolean; readonly consented: boolean; readonly microphoneCheck: string; readonly onCheckMicrophone: () => void; @@ -267,40 +335,65 @@ const VoiceInterviewDisclosure = ({ return (
-
- Voice mode - - Talk through your process with AI - -
-

- OpenAI processes live audio and speaks the interviewer’s words. - Petrinaut keeps finalized answers in this conversation, not the audio. -

- - {microphoneCheck && ( -

- {microphoneCheck} +

+
+ + + +
+ + Start a voice conversation + + + Talk through your process with AI + +
+
+

+ OpenAI processes live audio and speaks the interviewer’s words. + Petrinaut saves finalized answers—not audio.

- )} -
- - + +
+ + +
+
+ {microphoneCheck} +
); @@ -405,6 +498,7 @@ const AvailableVoiceInterviewControl = ({ const [showDisclosure, setShowDisclosure] = useState(false); const [consented, setConsented] = useState(false); const [microphoneCheck, setMicrophoneCheck] = useState(""); + const [checkingMicrophone, setCheckingMicrophone] = useState(false); const handledVoiceSelectionRef = useRef(false); const { inputMode, @@ -534,19 +628,28 @@ const AvailableVoiceInterviewControl = ({ return ( { - setMicrophoneCheck("Checking microphone…"); - void navigator.mediaDevices.getUserMedia({ audio: true }).then( - (stream) => { - for (const track of stream.getTracks()) { - track.stop(); - } - setMicrophoneCheck("Microphone ready."); - }, - () => setMicrophoneCheck("Microphone access was not available."), - ); + if (checkingMicrophone) { + return; + } + setCheckingMicrophone(true); + setMicrophoneCheck(""); + void navigator.mediaDevices + .getUserMedia({ audio: true }) + .then( + (stream) => { + for (const track of stream.getTracks()) { + track.stop(); + } + setMicrophoneCheck("Microphone ready."); + }, + () => + setMicrophoneCheck("Microphone access was not available."), + ) + .finally(() => setCheckingMicrophone(false)); }} onConsentChange={setConsented} onStart={() => { From fa27f5860cd8c0561c0b2ebb1661f211505f66af Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Fri, 4 Sep 2026 12:50:56 +0200 Subject: [PATCH 40/52] Handle unavailable microphone checks Co-authored-by: Cursor --- .../voice-interview-control.test.tsx | 42 +++++++++++++++++++ .../voice-interview-control.tsx | 30 ++++++++----- 2 files changed, 61 insertions(+), 11 deletions(-) diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx index 111b6471de0..6ce89128ad9 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx @@ -473,6 +473,48 @@ describe("voice interview control", () => { await waitFor(() => expect(check.getAttribute("aria-busy")).toBe("false")); }); + test.each([ + { + failure: "media devices are missing", + stubMedia: () => vi.stubGlobal("navigator", {}), + }, + { + failure: "getUserMedia throws synchronously", + stubMedia: () => + vi.stubGlobal("navigator", { + mediaDevices: { + getUserMedia: () => { + throw new DOMException("Unavailable", "NotSupportedError"); + }, + }, + }), + }, + ])( + "reports an accessible microphone failure when $failure", + async ({ stubMedia }) => { + stubMedia(); + render(); + + fireEvent.click(screen.getByRole("button", { name: "Select Voice" })); + const disclosure = screen.getByRole("region", { + name: "Voice mode consent", + }); + const check = within(disclosure).getByRole("button", { + name: "Test microphone", + }); + + fireEvent.click(check); + + const status = await within(disclosure).findByText( + "Microphone access was not available.", + ); + expect(status.getAttribute("aria-live")).toBe("polite"); + expect(status.getAttribute("aria-atomic")).toBe("true"); + expect(check.getAttribute("aria-describedby")).toBe(status.id); + await waitFor(() => expect(check.getAttribute("aria-busy")).toBe("false")); + }, + ); + test("starts directly after acknowledgement and ends through the registered control", async () => { window.localStorage.setItem( VOICE_INTERVIEW_DISCLOSURE_STORAGE_KEY, diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx index b87a8296fe3..5d32e6f4e97 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx @@ -637,17 +637,25 @@ const AvailableVoiceInterviewControl = ({ } setCheckingMicrophone(true); setMicrophoneCheck(""); - void navigator.mediaDevices - .getUserMedia({ audio: true }) - .then( - (stream) => { - for (const track of stream.getTracks()) { - track.stop(); - } - setMicrophoneCheck("Microphone ready."); - }, - () => - setMicrophoneCheck("Microphone access was not available."), + let microphoneCheckPromise: Promise; + try { + const getUserMedia = navigator.mediaDevices?.getUserMedia; + microphoneCheckPromise = + getUserMedia === undefined + ? Promise.reject(new Error("Microphone access is unavailable.")) + : getUserMedia.call(navigator.mediaDevices, { audio: true }); + } catch (error) { + microphoneCheckPromise = Promise.reject(error); + } + void microphoneCheckPromise + .then((stream) => { + for (const track of stream.getTracks()) { + track.stop(); + } + setMicrophoneCheck("Microphone ready."); + }) + .catch(() => + setMicrophoneCheck("Microphone access was not available."), ) .finally(() => setCheckingMicrophone(false)); }} From 8a904c40201a2dbb87bb73bf8790141c0d51cce0 Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Fri, 4 Sep 2026 12:54:00 +0200 Subject: [PATCH 41/52] Fix Voice verification errors Co-authored-by: Cursor --- .../app/voice-interview/voice-interview-control.test.tsx | 4 +++- .../main/app/voice-interview/voice-interview-control.tsx | 8 +++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx index 6ce89128ad9..632a4332269 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx @@ -511,7 +511,9 @@ describe("voice interview control", () => { expect(status.getAttribute("aria-live")).toBe("polite"); expect(status.getAttribute("aria-atomic")).toBe("true"); expect(check.getAttribute("aria-describedby")).toBe(status.id); - await waitFor(() => expect(check.getAttribute("aria-busy")).toBe("false")); + await waitFor(() => + expect(check.getAttribute("aria-busy")).toBe("false"), + ); }, ); diff --git a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx index 5d32e6f4e97..a78cbab7496 100644 --- a/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx +++ b/apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx @@ -639,11 +639,13 @@ const AvailableVoiceInterviewControl = ({ setMicrophoneCheck(""); let microphoneCheckPromise: Promise; try { - const getUserMedia = navigator.mediaDevices?.getUserMedia; + const { mediaDevices } = navigator as { + readonly mediaDevices?: MediaDevices; + }; microphoneCheckPromise = - getUserMedia === undefined + mediaDevices === undefined ? Promise.reject(new Error("Microphone access is unavailable.")) - : getUserMedia.call(navigator.mediaDevices, { audio: true }); + : mediaDevices.getUserMedia({ audio: true }); } catch (error) { microphoneCheckPromise = Promise.reject(error); } From 46b66bc58d08b094dc0ea11a7cd38cf3218e35c4 Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Fri, 4 Sep 2026 13:06:27 +0200 Subject: [PATCH 42/52] Format Voice consent implementation plan Amp-Thread-ID: https://ampcode.com/threads/T-01a06941-3ee7-7482-a82e-8a16c7141785 Co-authored-by: Amp --- .../plans/2026-09-04-voice-consent-compact-card.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/superpowers/plans/2026-09-04-voice-consent-compact-card.md b/docs/superpowers/plans/2026-09-04-voice-consent-compact-card.md index 2c6783a1507..8b5b39ee709 100644 --- a/docs/superpowers/plans/2026-09-04-voice-consent-compact-card.md +++ b/docs/superpowers/plans/2026-09-04-voice-consent-compact-card.md @@ -123,7 +123,13 @@ Add a local decorative waveform matching Petrinaut's composer action: ```tsx const VoiceModeIcon = () => ( -
-
- - - -
- - Start a voice conversation - - - Talk through your process with AI - -
-
-

- OpenAI processes live audio and speaks the interviewer’s words. Petrinaut - saves finalized answers—not audio. -

- -
- - -
-
- {microphoneCheck} -
-
- -``` - -- [ ] **Step 5: Guard the local microphone check while pending** - -Add and pass the pending state: - -```tsx -const [checkingMicrophone, setCheckingMicrophone] = useState(false); -``` - -```tsx - -``` - -Implement the guarded callback: - -```tsx -onCheckMicrophone={() => { - if (checkingMicrophone) { - return; - } - setCheckingMicrophone(true); - setMicrophoneCheck(""); - void navigator.mediaDevices - .getUserMedia({ audio: true }) - .then( - (stream) => { - for (const track of stream.getTracks()) { - track.stop(); - } - setMicrophoneCheck("Microphone ready."); - }, - () => setMicrophoneCheck("Microphone access was not available."), - ) - .finally(() => setCheckingMicrophone(false)); -}} -``` - -- [ ] **Step 6: Run focused tests** - -Run: - -```bash -yarn workspace @apps/petrinaut-website test:unit src/main/app/voice-interview/voice-interview-control.test.tsx -``` - -Expected: 15 tests pass with no failures. - -- [ ] **Step 7: Run type checking and linting** - -Run: - -```bash -yarn workspace @apps/petrinaut-website lint:tsc -yarn workspace @apps/petrinaut-website lint:eslint -``` - -Expected: both commands exit 0. Existing non-failing warnings may remain unchanged. - -- [ ] **Step 8: Inspect responsive layout** - -Open the local Petrinaut preview and verify at 500 px and 320 px assistant widths: - -- the card remains between transcript and composer; -- the title and copy wrap without horizontal overflow; -- actions remain ordered **Start voice**, then **Test microphone**; -- focus, disabled, loading, success, and failure states are legible; -- selecting **Start voice** transitions to the existing live Voice dock. - -- [ ] **Step 9: Commit the implementation** - -```bash -git add \ - apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx \ - apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.test.tsx -git commit -m "Polish Voice consent setup" -``` diff --git a/docs/superpowers/specs/2026-09-04-voice-consent-compact-card-design.md b/docs/superpowers/specs/2026-09-04-voice-consent-compact-card-design.md deleted file mode 100644 index 5f604266e28..00000000000 --- a/docs/superpowers/specs/2026-09-04-voice-consent-compact-card-design.md +++ /dev/null @@ -1,68 +0,0 @@ -# Voice consent compact card - -## Goal - -Replace PR #9531's visually loose Voice consent block with a compact setup card -that matches Petrinaut's AI assistant panel. Preserve the existing disclosure, -explicit acknowledgement, microphone check, focus behavior, and start flow. - -## Design - -The pre-session Voice surface remains pinned between the transcript and composer. -It becomes a quiet white card inside the panel's subtle footer surface, using the -same border, radius, shadow, spacing, and typography language as the composer and -message cards. - -The card contains: - -1. A small brand-tinted Voice icon beside the title. -2. The title **Start a voice conversation** and subtitle **Talk through your - process with AI**. -3. Concise disclosure copy: **OpenAI processes live audio and speaks the - interviewer’s words. Petrinaut saves finalized answers—not audio.** -4. A design-system checkbox in a muted inset row with the label **I understand - how voice data is handled.** -5. A compact brand **Start voice** button and subtle **Test microphone** button. - -The card must remain legible at the assistant's 320 px minimum width. Actions may -wrap without changing their order. - -## Interaction and states - -- **Start voice** is disabled until the checkbox is selected. -- **Test microphone** does not store consent or start Voice. -- While testing, the secondary button shows a loading state and duplicate checks - are prevented. -- Success and failure replace a reserved status line below the actions, avoiding - layout jump. The status uses polite live-region semantics. -- Starting Voice stores the existing versioned acknowledgement, hides the card, - and starts the existing controller exactly once. -- Keyboard focus continues to move to the consent section when it appears. - -## Accessibility - -- Use `@hashintel/ds-components` `Checkbox` and `Button` rather than native - unstyled controls. -- Keep the section's existing accessible name. -- Associate status text with the microphone-check action through the live region. -- Preserve visible focus states supplied by the design system. -- Do not rely on color alone for status. - -## Scope - -The implementation is limited to -`apps/petrinaut-website/src/main/app/voice-interview/voice-interview-control.tsx` -and its focused tests. No Voice transport, consent persistence, session, panel -layout, or live Voice dock behavior changes. - -The existing Petrinaut AI-assistant documentation remains behaviorally accurate; -no user-guide copy change is required. - -## Verification - -- Extend the focused test to cover the design-system checkbox, compact labels, - disabled start state, microphone-check loading guard, success/failure status, - and unchanged acknowledgement behavior. -- Run the focused Voice control test. -- Run Petrinaut website type checking and linting. -- Inspect the card in the real 500 px panel and at the 320 px minimum width. From 9d753b2068d35e6f093e350ca55ec15f99ebaf2a Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Fri, 4 Sep 2026 17:12:51 +0200 Subject: [PATCH 44/52] Prebuild Brunch dependencies before local development Amp-Thread-ID: https://ampcode.com/threads/T-01a06cf5-75d4-7763-addd-4a881479b20f Co-authored-by: Amp --- apps/brunch-agent/test/local-dev-origins.test.ts | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/brunch-agent/test/local-dev-origins.test.ts b/apps/brunch-agent/test/local-dev-origins.test.ts index 56da1c3f41b..8a20c937cae 100644 --- a/apps/brunch-agent/test/local-dev-origins.test.ts +++ b/apps/brunch-agent/test/local-dev-origins.test.ts @@ -22,7 +22,7 @@ test("one documented root command starts the Brunch server and Petrinaut panel", }; expect(rootPackage.scripts["dev:brunch"]).toBe( - "CARGO_TERM_PROGRESS_WHEN=never turbo run build --filter '@apps/petrinaut-website^...' && npm-run-all --parallel dev:brunch:server dev:brunch:panel", + "CARGO_TERM_PROGRESS_WHEN=never turbo run build --filter '@apps/brunch-agent^...' --filter '@apps/petrinaut-website^...' && npm-run-all --parallel dev:brunch:server dev:brunch:panel", ); expect(rootPackage.scripts["dev:brunch:server"]).toBe( "yarn workspace @apps/brunch-agent dev", diff --git a/package.json b/package.json index 2888e18031b..c0423bf6437 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,7 @@ "dev": "CARGO_TERM_PROGRESS_WHEN=never turbo dev --log-order stream --filter '@apps/hash-api' --filter '@apps/hash-frontend' --", "dev:backend": "CARGO_TERM_PROGRESS_WHEN=never turbo dev --log-order stream --filter '@apps/hash-api' --", "dev:backend:api": "CARGO_TERM_PROGRESS_WHEN=never turbo dev --log-order stream --filter '@apps/hash-api' --", - "dev:brunch": "CARGO_TERM_PROGRESS_WHEN=never turbo run build --filter '@apps/petrinaut-website^...' && npm-run-all --parallel dev:brunch:server dev:brunch:panel", + "dev:brunch": "CARGO_TERM_PROGRESS_WHEN=never turbo run build --filter '@apps/brunch-agent^...' --filter '@apps/petrinaut-website^...' && npm-run-all --parallel dev:brunch:server dev:brunch:panel", "dev:brunch:panel": "PETRINAUT_WEBSITE_ROOT=\"$PWD/apps/petrinaut-website\" yarn workspace @apps/brunch-agent petrinaut:dev", "dev:brunch:server": "yarn workspace @apps/brunch-agent dev", "dev:frontend": "CARGO_TERM_PROGRESS_WHEN=never turbo dev --log-order stream --filter '@apps/hash-frontend' --", From 9a1afcfaed699c82a655ef1046942d951403213c Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Fri, 4 Sep 2026 17:49:37 +0200 Subject: [PATCH 45/52] Authorize bounded Petrinaut error and transcript UX work Amp-Thread-ID: https://ampcode.com/threads/T-01a06d1b-91ad-741d-ae26-ca27b1ba20c5 Co-authored-by: Amp --- libs/@hashintel/brunch-agent/MISSION.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/libs/@hashintel/brunch-agent/MISSION.md b/libs/@hashintel/brunch-agent/MISSION.md index 67a01e47704..4a29dd28fde 100644 --- a/libs/@hashintel/brunch-agent/MISSION.md +++ b/libs/@hashintel/brunch-agent/MISSION.md @@ -29,6 +29,14 @@ Petrinaut's stock API plugin so `/api/voice/config` and `/agents/chat/*` Flue proxy. This is an explicit exception to the parent-defect constraint below; it changes no conversation or Voice-turn semantics. +On 2026-09-04, the owner directed PR #9531 to close two more bounded Petrinaut +defects: preserve real Flue and browser-tool errors through the transport and +display them in full, and replace deferred Voice-transcript disclosure with +immediate transcript display plus a dock-only collapsed live-session view. +These are explicit exceptions to the parent-defect constraint below; they +change no conversation authority, submission path, or Voice-turn semantics, +and local playback cancellation remains separate from durable abort. + Completed-transcript authority, half-duplex ownership, admission idempotency, the cancellation barrier, exact full-response replay, and exact marked-question replay have focused regression coverage, including the interval between a From 60d49135e4b657f6404a74f239d64d83d358796d Mon Sep 17 00:00:00 2001 From: Kostandin Angjellari Date: Fri, 4 Sep 2026 18:22:26 +0200 Subject: [PATCH 46/52] Show live Voice transcripts and complete errors Amp-Thread-ID: https://ampcode.com/threads/T-01a06d1b-91ad-741d-ae26-ca27b1ba20c5 Co-authored-by: Amp --- .changeset/flue-voice-safety.md | 1 + libs/@hashintel/brunch-agent/MISSION.md | 15 +- .../donor-behavior-matrix.md | 11 +- .../transport-aisdk/src/error-text.ts | 81 ++++++++ .../packages/transport-aisdk/src/index.ts | 3 +- .../packages/transport-aisdk/src/ui-stream.ts | 4 +- .../test/chat-transport.test.ts | 12 +- .../transport-aisdk/test/ui-stream.test.ts | 68 +++++++ .../@hashintel/petrinaut/docs/ai-assistant.md | 23 +-- libs/@hashintel/petrinaut/src/panda-preset.ts | 4 - .../src/react/notifications/context.ts | 1 + .../src/react/notifications/provider.test.tsx | 76 +++++++ .../src/react/notifications/provider.tsx | 20 +- .../src/react/notifications/toaster.tsx | 91 ++++++++- .../Editor/components/voice-session-labels.ts | 2 + .../Editor/panels/ai-assistant-panel.test.tsx | 35 +++- .../Editor/panels/ai-assistant-panel.tsx | 34 +++- .../ai-assistant-contents.stories.tsx | 24 ++- .../ai-assistant-contents.test.tsx | 106 +++++++--- .../ai-assistant-contents.tsx | 192 +++++------------- .../defer-voice-messages.test.ts | 109 ---------- .../defer-voice-messages.ts | 75 ------- .../ai-assistant-contents/tool-list.tsx | 13 +- .../ai-assistant-contents/voice-dock.tsx | 42 ++-- .../voice-dock/transcription-icon.tsx | 17 -- 25 files changed, 600 insertions(+), 459 deletions(-) create mode 100644 libs/@hashintel/brunch-agent/packages/transport-aisdk/src/error-text.ts create mode 100644 libs/@hashintel/petrinaut/src/react/notifications/provider.test.tsx delete mode 100644 libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/defer-voice-messages.test.ts delete mode 100644 libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/defer-voice-messages.ts delete mode 100644 libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents/voice-dock/transcription-icon.tsx diff --git a/.changeset/flue-voice-safety.md b/.changeset/flue-voice-safety.md index f1ab1400607..bf7108e0ff2 100644 --- a/.changeset/flue-voice-safety.md +++ b/.changeset/flue-voice-safety.md @@ -3,3 +3,4 @@ --- Add half-duplex Voice handoff, exact full-response replay, recoverable transcript notices, and multi-origin Voice attribution for client-tool results. +Voice transcripts now appear live with a dock-only collapse mode, while assistant and tool failures retain and display their complete details in persistent, copyable notifications and inline tool cards. diff --git a/libs/@hashintel/brunch-agent/MISSION.md b/libs/@hashintel/brunch-agent/MISSION.md index 4a29dd28fde..fe0acaa3aeb 100644 --- a/libs/@hashintel/brunch-agent/MISSION.md +++ b/libs/@hashintel/brunch-agent/MISSION.md @@ -54,16 +54,15 @@ second-durable-store stop condition. The restacked hydration guard removes the old parent blocker, but no real witness claim is valid until the retained human evidence exists. -The 2026-09-04 corrective verification covers the current 58-file successor +The 2026-09-04 corrective verification covers the current 72-file successor diff against #9528 head `eecbe99e201f`: the four focused race cases pass 4/4 tests, the filtered production admission-outcome cases pass 3/3 tests, and the -complete seven-workspace Turbo run passes 39/39 tasks, including 279/279 website -tests and the Brunch core package. Architecture validation passes with 68 -layers, 337 edges, 692 files, 69 generated pages, and 38 authored pages. `git -diff --check` and formatting of all 52 formatter-owned successor files pass. The -root formatter remains red only outside this branch's diff on an unrelated -untracked `.cursor` plan. The exact commands and dispositions are retained in -the [donor matrix](docs/evidence/implementations/mission-5-voice-safety-parity/donor-behavior-matrix.md#corrective-verification). +complete seven-workspace Turbo run passes 39/39 tasks and 1,121/1,121 tests, +including 282/282 website tests and the Brunch core package. Architecture +validation passes with 68 layers, 337 edges, 690 files, 69 generated pages, and +38 authored pages. `git diff --check` passes, and the root formatter accepts all +5,527 matched files. The exact commands and dispositions are retained in the +[donor matrix](docs/evidence/implementations/mission-5-voice-safety-parity/donor-behavior-matrix.md#corrective-verification). The pinned donor-behavior decision record is the [FE-1580 donor matrix](docs/evidence/implementations/mission-5-voice-safety-parity/donor-behavior-matrix.md). Donor PRs are read-only evidence at their named heads; semantic reimplementation is required, never merge or cherry-pick. diff --git a/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/donor-behavior-matrix.md b/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/donor-behavior-matrix.md index 3321a1c538c..ce35160f062 100644 --- a/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/donor-behavior-matrix.md +++ b/libs/@hashintel/brunch-agent/docs/evidence/implementations/mission-5-voice-safety-parity/donor-behavior-matrix.md @@ -72,13 +72,13 @@ The owner selected half-duplex turn ownership on 2026-09-03: assistant output ow ## Corrective verification -Fresh local checks on 2026-09-04 cover the 58-file successor diff against the +Fresh local checks on 2026-09-04 cover the 72-file successor diff against the verified #9528 head `eecbe99e201fd8cb78d9b719e789b6abd373ed1b`. Graphite replayed one repeatedly touched integration-test conflict while restacking: the semantic resolution keeps the parent's required URL-navigation props and tests together with the successor's admission, dormant-ask, durable-Stop, and status-removal proof. No production-source conflict was resolved by choosing either side -wholesale. The verified code head before this evidence-only update is +wholesale. The earlier verified code head before this evidence-only update is `9938283a19ab20567ad6b4c96330ea392243c16f`: | Command | Result | @@ -91,11 +91,10 @@ wholesale. The verified code head before this evidence-only update is | `mise exec -- yarn workspace @hashintel/petrinaut test:unit --run src/ui/views/Editor/panels/ai-assistant-panel.test.tsx` | Exit 0; 44/44 production host-registration and panel tests passed. | | `mise exec -- yarn workspace @apps/petrinaut-website test:unit src/main/app/voice-interview/canonical-speech.test.ts src/main/app/voice-interview/openai-realtime-session.test.ts src/main/app/voice-interview/voice-turn-controller.test.ts` | Exit 0; 79/79 exact replay, queue, terminal-gating, and turn-controller tests passed. | | `mise exec -- yarn exec turbo run lint:tsc lint:eslint test:unit build --filter @hashintel/brunch-agent` | Exit 0; 5/5 tasks passed, including 10/10 test files and 86/86 tests; the four question-marker mock lint failures are resolved with production-interface signatures. | -| `mise exec -- yarn exec turbo run lint:tsc lint:eslint test:unit build --filter @apps/brunch-agent --filter @apps/petrinaut-website --filter @hashintel/petrinaut --filter @hashintel/brunch-agent --filter @hashintel/brunch-agent-binding-flue --filter @hashintel/brunch-agent-plugin-sdcpn --filter @hashintel/brunch-agent-transport-aisdk` | Exit 0; 39/39 tasks passed, including 16/16 Brunch app files with 79/79 tests, 10/10 Brunch core files with 86/86 tests, 3/3 transport files with 26/26 tests, 5/5 binding files with 18/18 tests, 2/2 plugin files with 8/8 tests, 72/72 Petrinaut files with 616/616 tests, and 31/31 website files with 279/279 tests. Website ESLint retains one non-failing warning at `voice-interview-control.tsx:493`; other inherited warnings remain outside this successor's corrective scope. | -| `yarn workspace @local/petrinaut-arch-docs lint:arch-docs` | Exit 0; 68 layers, 337 edges, 692 files, 69 generated pages, and 38 authored pages. | -| `git diff --name-only -z eecbe99e201fd8cb78d9b719e789b6abd373ed1b...HEAD \| xargs -0 yarn exec oxfmt --check` | Exit 0; all 52 formatter-owned files in the 58-file successor diff passed. The six unmatched documentation files are excluded by `oxfmt.config.ts` and pass repository whitespace validation. | +| `mise exec -- yarn exec turbo run lint:tsc lint:eslint test:unit build --filter @apps/brunch-agent --filter @apps/petrinaut-website --filter @hashintel/petrinaut --filter @hashintel/brunch-agent --filter @hashintel/brunch-agent-binding-flue --filter @hashintel/brunch-agent-plugin-sdcpn --filter @hashintel/brunch-agent-transport-aisdk` | Exit 0; 39/39 tasks and 1,121/1,121 tests passed: 16/16 Brunch app files with 80/80 tests, 10/10 Brunch core files with 86/86 tests, 3/3 transport files with 32/32 tests, 5/5 binding files with 18/18 tests, 2/2 plugin files with 8/8 tests, 72/72 Petrinaut files with 615/615 tests, and 31/31 website files with 282/282 tests. Website ESLint retains one non-failing warning at `voice-interview-control.tsx:587`; other inherited warnings remain outside this successor's corrective scope. | +| `yarn workspace @local/petrinaut-arch-docs lint:arch-docs` | Exit 0; 68 layers, 337 edges, 690 files, 69 generated pages, and 38 authored pages. | +| `yarn lint:format` | Exit 0; all 5,527 matched repository files use the correct format. | | `git diff --check` | Exit 0. | -| `yarn lint:format` | Exit 1 outside the successor diff only: unrelated untracked `.cursor/plans/fe-1574_mission_recut_6f23f7cd.plan.md`. It was not modified. | | In isolated detached worktree `/Users/kostandin/Projects/hashdev/worktrees/fe-1580-latency-baseline-9496`: `mise exec -- yarn exec turbo run build --filter '@apps/petrinaut-website^...'`, then `mise exec -- yarn workspace @apps/petrinaut-website test:unit src/main/app/voice-interview/canonical-speech.test.ts src/main/app/voice-interview/openai-realtime-session.test.ts src/main/app/voice-interview/realtime-brunch-bridge.test.ts src/main/app/voice-interview/voice-turn-controller.test.ts src/main/app/voice-interview/voice-preview.integration.test.ts` | Exit 0; dependency build passed 14/14 tasks, then all 5/5 donor Voice files and 108/108 tests passed at pinned #9496 head. The isolated donor and candidate panels return HTTP 200 on ports 4916 and 4915 respectively; real audible samples remain uncollected. | No production Voice source under `apps/petrinaut-website/src/main/app/voice-interview` diff --git a/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/error-text.ts b/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/error-text.ts new file mode 100644 index 00000000000..16b440c79cc --- /dev/null +++ b/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/error-text.ts @@ -0,0 +1,81 @@ +const maxErrorTextLength = 10_000; + +const nonEmptyText = (value: string): string | null => + value.trim().length > 0 ? value : null; + +const isPlainObject = (value: unknown): value is Record => { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + return false; + } + + const prototype = Object.getPrototypeOf(value) as unknown; + return prototype === Object.prototype || prototype === null; +}; + +const serializePlainObject = ( + value: Record, + seen: WeakSet, +): string | null => { + try { + const serialized: unknown = JSON.stringify( + value, + (_key, nestedValue: unknown) => { + if (typeof nestedValue === "bigint") { + return nestedValue.toString(); + } + if (typeof nestedValue !== "object" || nestedValue === null) { + return nestedValue; + } + if (seen.has(nestedValue)) { + return "[Circular]"; + } + seen.add(nestedValue); + return nestedValue; + }, + ); + return typeof serialized === "string" ? serialized : null; + } catch { + return null; + } +}; + +const serializeErrorValue = ( + value: unknown, + seen: WeakSet, +): string | null => { + if (typeof value === "string") { + return nonEmptyText(value); + } + if (value instanceof Error) { + if (seen.has(value)) { + return "[Circular]"; + } + seen.add(value); + + const message = nonEmptyText(value.message); + const cause = + value.cause === undefined ? null : serializeErrorValue(value.cause, seen); + if (message !== null && cause !== null) { + return `${message}\nCaused by: ${cause}`; + } + return message ?? cause; + } + if (isPlainObject(value)) { + return serializePlainObject(value, seen); + } + return null; +}; + +export const serializeErrorText = ( + error: unknown, + fallback = "The chat turn failed.", +): string => { + const serialized = serializeErrorValue(error, new WeakSet()); + if (serialized === null) { + return fallback; + } + if (serialized.length <= maxErrorTextLength) { + return serialized; + } + return `${serialized.slice(0, maxErrorTextLength - 1)}…`; +}; diff --git a/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/index.ts b/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/index.ts index 64d370955f6..e27aa988424 100644 --- a/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/index.ts +++ b/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/index.ts @@ -2,6 +2,7 @@ import { FlueApiError, FlueExecutionError } from "@flue/sdk"; import { getToolName, isToolUIPart } from "ai"; import { CLIENT_TOOL_RESULT_SIGNAL } from "./client-tool-result"; +import { serializeErrorText } from "./error-text"; import { createFlueUiStream } from "./ui-stream"; import type { AgentSendResult, DeliveredMessage, FlueClient } from "@flue/sdk"; @@ -222,7 +223,7 @@ const streamFailureChunk = ( error instanceof FlueExecutionError && error.failure === "terminal_event_missing" ? "The chat stream ended before the turn settled." - : "The chat turn failed.", + : serializeErrorText(error), }; }; diff --git a/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/ui-stream.ts b/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/ui-stream.ts index 22f2864530c..860fb2d4c88 100644 --- a/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/ui-stream.ts +++ b/libs/@hashintel/brunch-agent/packages/transport-aisdk/src/ui-stream.ts @@ -1,3 +1,5 @@ +import { serializeErrorText } from "./error-text"; + import type { AgentSendResult, ConversationStreamChunk } from "@flue/sdk"; import type { UIMessageChunk } from "ai"; @@ -88,7 +90,7 @@ export const createFlueUiStream = ( case "failed": options.write({ type: "error", - errorText: "The chat turn failed.", + errorText: serializeErrorText(chunk.error), }); break; case "aborted": diff --git a/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/chat-transport.test.ts b/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/chat-transport.test.ts index dc8edeb898b..91fd5b18f69 100644 --- a/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/chat-transport.test.ts +++ b/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/chat-transport.test.ts @@ -196,12 +196,14 @@ test("starts with history-only reconnection", async () => { test.each([ [ "failed", - new FlueExecutionError({ - target: "agent_submission", - targetId: admission.submissionId, - failure: "failed", + new Error("Elicitor tool failed.", { + cause: { field: "answer", reason: "Required" }, }), - { type: "error", errorText: "The chat turn failed." }, + { + type: "error", + errorText: + 'Elicitor tool failed.\nCaused by: {"field":"answer","reason":"Required"}', + }, ], [ "aborted", diff --git a/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/ui-stream.test.ts b/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/ui-stream.test.ts index 5f6c26ecf20..999a8ef498a 100644 --- a/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/ui-stream.test.ts +++ b/libs/@hashintel/brunch-agent/packages/transport-aisdk/test/ui-stream.test.ts @@ -186,3 +186,71 @@ test("ignores observation catch-up chunks in a submission stream", () => { "finish", ]); }); + +test.each([ + { + error: new Error("Elicitor failed.", { + cause: "The requested field is required.", + }), + expected: "Elicitor failed.\nCaused by: The requested field is required.", + shape: "Error with cause", + }, + { + error: "The elicitor rejected the answer.", + expected: "The elicitor rejected the answer.", + shape: "string", + }, + { + error: { field: "answer", reason: "Required" }, + expected: '{"field":"answer","reason":"Required"}', + shape: "plain object", + }, + { + error: 503, + expected: "The chat turn failed.", + shape: "unsupported value", + }, + { + error: "", + expected: "The chat turn failed.", + shape: "empty string", + }, +])("preserves a failed submission's $shape error", ({ error, expected }) => { + const written = project([ + { + type: "submission-settled", + conversationId: "conversation-1", + submissionId: "submission-1", + outcome: "failed", + error, + position: position(0), + }, + ]); + + expect(written).toEqual([{ type: "error", errorText: expected }]); + expect(written).not.toContainEqual({ + type: "error", + errorText: "[object Object]", + }); +}); + +test("bounds cyclic failed-submission objects", () => { + const cyclicError: Record = { reason: "Recursive failure" }; + cyclicError.self = cyclicError; + cyclicError.payload = "x".repeat(20_000); + + const written = project([ + { + type: "submission-settled", + conversationId: "conversation-1", + submissionId: "submission-1", + outcome: "failed", + error: cyclicError, + position: position(0), + }, + ]); + + const failure = written.find((chunk) => chunk.type === "error"); + expect(failure?.errorText).toContain('"self":"[Circular]"'); + expect(failure?.errorText.length).toBeLessThanOrEqual(10_000); +}); diff --git a/libs/@hashintel/petrinaut/docs/ai-assistant.md b/libs/@hashintel/petrinaut/docs/ai-assistant.md index 5d12436c413..2f04b775a45 100644 --- a/libs/@hashintel/petrinaut/docs/ai-assistant.md +++ b/libs/@hashintel/petrinaut/docs/ai-assistant.md @@ -22,7 +22,7 @@ While a response is streaming you can: The application embedding Petrinaut may place an additional control beside the message box. For example, a host can offer another way to enter finalized text. Text submitted by that control behaves like text sent with the keyboard: it joins the same conversation and, when an inline question is waiting for an answer, completes that question rather than starting an unrelated message. A host can explicitly submit a separate message instead when the text is a correction or other follow-up that must not answer the pending question. If the host offers voice input, only a finalized transcript captured while Voice owns the input turn can be submitted. Voice waits while an existing response finishes or yields through the host's handoff control. -If an assistant request fails, Petrinaut shows the error in a brief toast rather than adding it to the conversation. Retry from the composer when the assistant is ready. +If an assistant request fails, Petrinaut shows the complete error in a persistent toast rather than adding it to the conversation. Long errors wrap, diagnostic details can be copied, and the toast stays open until you close it. Retry from the composer when the assistant is ready. Hosts may provide canonical conversation rehydration. In that case, reopening the same assistant shows its settled and stopped turns without resubmitting a message or replaying Voice audio. Voice markers attached to client-tool results survive that history. A direct spoken user message remains in the transcript after reopening, but its **Voice** chip may not be restored by the current Brunch host. @@ -46,12 +46,10 @@ flicker above the line. While the assistant speaks the ribbon takes on a restrai motion instead, colour crossfading as the turn changes hands, so which side holds it is readable at a glance. It flattens to near a line whenever nobody holds the turn. -The conversation itself stays still. Spoken turns are written to it as they happen, because that is -what runs the tools that edit the net, but they stay hidden until the session ends rather than -scrolling the transcript mid-sentence. **Show transcription in chat** lets them through as they land -instead; turning it off holds them back again, and it starts off with each session. Two things are -never held back either way: anything you typed, and any inline question waiting for your answer. When -the session ends, the held turns appear together under a **Voice session · N turns** divider. Only +Spoken turns appear in the conversation as soon as their finalized text arrives, so the transcript +stays current while the session runs and tools that edit the net remain visible. Select **Collapse +voice session** to reduce the panel to the Voice dock alone; this hides the AI header, transcript, and +host Voice region without ending the session. Select **Expand voice session** to restore them. Only finalized answers and canonical Brunch text become chat history; provisional transcription and Realtime audio are ephemeral. Finalized spoken user messages carry a small **Voice** chip in front of the words themselves, and the exact inline answer completed by speech carries the same chip, so Voice @@ -66,8 +64,9 @@ there is no required done-speaking action. Duplicate, empty, failed, or unavaila not submitted; the dock asks you to try again. An overlong answer instead asks for a shorter response. Provisional words remain display-only until the provider completes their transcript. -Every session control lives in the dock: **Show transcription in chat** and **Voice playback -options** on the left, and the available handoff, microphone, recovery, and end actions on the right. +Every session control lives in the dock: **Collapse voice session** / **Expand voice session** and +**Voice playback options** on the left, and the available handoff, microphone, recovery, and end +actions on the right. **Read full response** becomes available after the matching response and speech have both finished and replays every exact retained canonical segment in order. **Repeat question** uses the same availability gates and replays only exact question text explicitly marked by Brunch. It stays @@ -88,8 +87,8 @@ ready. **Clear AI chat** is unavailable while a Voice session is active. If voice cannot continue, the status reads **Voice interrupted** and the actionable error arrives as -a toast that names the microphone, connection, or Voice failure in one sentence, followed by any -diagnostic reference in parentheses. **Reconnect voice mode** replaces the microphone action until +a persistent toast that names the microphone, connection, or Voice failure in one sentence, followed +by any diagnostic reference in parentheses. **Reconnect voice mode** replaces the microphone action until the session recovers. For microphone permission or device errors, allow access or connect/select a microphone before reconnecting. For an interrupted request, network error, or timeout, check the connection and reconnect. If the preview is unavailable, continue with the text composer. An invalid @@ -102,7 +101,7 @@ When no interview is active and the host permits clearing, **Clear AI chat** via ## What the assistant can do -The assistant has tools for inspecting and modifying the current net. You'll see one card per tool call inline in the conversation: +The assistant has tools for inspecting and modifying the current net. You'll see one card per tool call inline in the conversation. A failed tool card leads with its complete error instead of hiding it behind a hover tooltip: - **Read tools** (neutral, expandable) –– for checking the current net state and active Petrinaut extensions at any point, for compilation errors, and for reading the user guide. - **Mutation tools** (green for additions/updates, red for deletions) -- "Added place X", "Updated transition Y", "Removed metric Z", and so on. Multiple successive mutations group under a collapsible "N changes" header. diff --git a/libs/@hashintel/petrinaut/src/panda-preset.ts b/libs/@hashintel/petrinaut/src/panda-preset.ts index 2cdc120abf8..7d4ab7bd23c 100644 --- a/libs/@hashintel/petrinaut/src/panda-preset.ts +++ b/libs/@hashintel/petrinaut/src/panda-preset.ts @@ -122,10 +122,6 @@ export const petrinautPandaPreset = { from: { opacity: "1", transform: "translateX(0)" }, to: { opacity: "0", transform: "translateX(100px)" }, }, - petrinautVoiceReveal: { - from: { opacity: "0", transform: "translateY(10px)" }, - to: { opacity: "1", transform: "translateY(0)" }, - }, petrinautVoiceSwap: { from: { opacity: "0" }, to: { opacity: "1" }, diff --git a/libs/@hashintel/petrinaut/src/react/notifications/context.ts b/libs/@hashintel/petrinaut/src/react/notifications/context.ts index 9911f709022..5f508982ae4 100644 --- a/libs/@hashintel/petrinaut/src/react/notifications/context.ts +++ b/libs/@hashintel/petrinaut/src/react/notifications/context.ts @@ -3,6 +3,7 @@ import { createContext } from "react"; export type NotificationTone = "error" | "neutral" | "success"; export type AddNotificationInput = { + detail?: string; message: string; tone?: NotificationTone; durationMs?: number; diff --git a/libs/@hashintel/petrinaut/src/react/notifications/provider.test.tsx b/libs/@hashintel/petrinaut/src/react/notifications/provider.test.tsx new file mode 100644 index 00000000000..1dda6521daa --- /dev/null +++ b/libs/@hashintel/petrinaut/src/react/notifications/provider.test.tsx @@ -0,0 +1,76 @@ +/** + * @vitest-environment jsdom + */ +import { + cleanup, + fireEvent, + render, + screen, + waitFor, +} from "@testing-library/react"; +import { use } from "react"; +import { afterEach, expect, test, vi } from "vitest"; + +import { NotificationsContext } from "./context"; +import { NotificationsProvider } from "./provider"; +import { notificationsToaster } from "./toaster"; + +afterEach(() => { + cleanup(); + vi.restoreAllMocks(); +}); + +test("keeps error notifications open while preserving the default for other tones", async () => { + const createToast = vi.spyOn(notificationsToaster, "create"); + const Trigger = () => { + const { addNotification } = use(NotificationsContext); + + return ( + <> + + + + ); + }; + + render( + + + , + ); + + fireEvent.click(screen.getByRole("button", { name: "Error" })); + fireEvent.click(screen.getByRole("button", { name: "Success" })); + + await waitFor(() => expect(createToast).toHaveBeenCalledTimes(2)); + expect(createToast).toHaveBeenNthCalledWith(1, { + description: "The complete elicitor failure.", + duration: Infinity, + id: "notification-0", + title: "AI assistant error", + type: "error", + }); + expect(createToast).toHaveBeenNthCalledWith(2, { + description: undefined, + duration: 3000, + id: "notification-1", + title: "Saved", + type: "success", + }); +}); diff --git a/libs/@hashintel/petrinaut/src/react/notifications/provider.tsx b/libs/@hashintel/petrinaut/src/react/notifications/provider.tsx index f6d39467b8b..945584fe104 100644 --- a/libs/@hashintel/petrinaut/src/react/notifications/provider.tsx +++ b/libs/@hashintel/petrinaut/src/react/notifications/provider.tsx @@ -16,22 +16,28 @@ export const NotificationsProvider = ({ }: { children: ReactNode; }) => { - function dismissNotification(id: string) { + const dismissNotification = (id: string) => { queueMicrotask(() => { notificationsToaster.dismiss(id); }); - } + }; - function addNotification({ + const addNotification = ({ + detail, durationMs, message, tone = "success", - }: AddNotificationInput) { - const id = `notification-${nextNotificationId++}`; - const effectiveDurationMs = durationMs ?? DEFAULT_NOTIFICATION_DURATION_MS; + }: AddNotificationInput) => { + const id = `notification-${nextNotificationId}`; + nextNotificationId += 1; + const effectiveDurationMs = + tone === "error" + ? Infinity + : (durationMs ?? DEFAULT_NOTIFICATION_DURATION_MS); queueMicrotask(() => { notificationsToaster.create({ + description: detail, duration: effectiveDurationMs, id, title: message, @@ -40,7 +46,7 @@ export const NotificationsProvider = ({ }); return id; - } + }; useEffect(() => { return () => { diff --git a/libs/@hashintel/petrinaut/src/react/notifications/toaster.tsx b/libs/@hashintel/petrinaut/src/react/notifications/toaster.tsx index bc26dc8403b..fd2261b2033 100644 --- a/libs/@hashintel/petrinaut/src/react/notifications/toaster.tsx +++ b/libs/@hashintel/petrinaut/src/react/notifications/toaster.tsx @@ -5,7 +5,7 @@ import { createToaster, } from "@ark-ui/react/toast"; -import { usePortalContainerRef } from "@hashintel/ds-components"; +import { Button, usePortalContainerRef } from "@hashintel/ds-components"; import { css } from "@hashintel/ds-helpers/css"; export const notificationsToaster = createToaster({ @@ -24,10 +24,11 @@ const toastRootStyle = css({ transition: "[translate 300ms, scale 300ms, opacity 300ms, box-shadow 300ms]", transitionTimingFunction: "[cubic-bezier(0.21, 1.02, 0.73, 1)]", display: "flex", - alignItems: "center", + alignItems: "flex-start", + gap: "2", minHeight: "[26px]", width: "[max-content]", - maxWidth: "[320px]", + maxWidth: "[min(480px, calc(100vw - 32px))]", borderRadius: "lg", boxShadow: "[0 8px 24px rgba(0, 0, 0, 0.24)]", paddingX: "4", @@ -44,23 +45,93 @@ const toastRootStyle = css({ }, }); +const toastContentStyle = css({ + display: "flex", + flex: "[1]", + minWidth: "[0]", + flexDirection: "column", + gap: "1", +}); + const toastTitleStyle = css({ overflow: "hidden", - textOverflow: "ellipsis", - whiteSpace: "nowrap", + overflowWrap: "anywhere", + lineClamp: "4", fontSize: "xs", fontWeight: "medium", lineHeight: "[14px]", }); +const toastDescriptionStyle = css({ + maxHeight: "[240px]", + overflow: "auto", + overflowWrap: "anywhere", + whiteSpace: "pre-wrap", + fontSize: "xs", + lineHeight: "[18px]", + userSelect: "text", +}); + +const toastActionsStyle = css({ + display: "flex", + flexShrink: "[0]", + gap: "1", +}); + +const toastActionStyle = css({ + color: "neutral.s00", + _hover: { + color: "neutral.s00", + }, +}); + export const NotificationsToaster = () => ( - {(toast) => ( - - {toast.title} - - )} + {(toast) => { + const detail = + typeof toast.description === "string" ? toast.description : undefined; + + return ( + +
+ + {toast.title} + + {detail && ( + + {detail} + + )} +
+
+ {detail && ( +
+ + ); + }} ); diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/components/voice-session-labels.ts b/libs/@hashintel/petrinaut/src/ui/views/Editor/components/voice-session-labels.ts index a6b6fb3f0c2..6b3fae02943 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/components/voice-session-labels.ts +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/components/voice-session-labels.ts @@ -26,7 +26,9 @@ export const voiceSessionStatusLabel = ( }; export const voiceSessionActionLabels = { + collapse: "Collapse voice session", end: "End voice mode", + expand: "Expand voice session", mute: "Mute microphone", pause: "Pause voice mode", playbackOptions: "Voice playback options", diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx index 25cda413ab6..f264a529b64 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.test.tsx @@ -16,6 +16,7 @@ import { DEFAULT_PETRINAUT_EXTENSIONS, createJsonDocHandle, createPetrinaut, + getLatestNetDefinitionToolName, type SDCPN, } from "@hashintel/petrinaut-core"; @@ -31,7 +32,11 @@ import { type SDCPNContextValue, } from "../../../../react/state/sdcpn-context"; import { definePetrinautAiInteractiveTool } from "../../../types/ai-interactive-tool"; -import { addMappedToolOutput, AiAssistantPanel } from "./ai-assistant-panel"; +import { + addMappedToolOutput, + AiAssistantPanel, + safelyAddToolOutput, +} from "./ai-assistant-panel"; import type { PetrinautAiAssistant } from "../../../petrinaut"; import type { @@ -2894,6 +2899,34 @@ describe("AiAssistantPanel composer submissions", () => { expect(latestMessages[0]?.metadata).toBeUndefined(); }); + test("reports browser tool-output rejections through the AI SDK error state", async () => { + const addToolOutput = vi + .fn() + .mockRejectedValueOnce(new Error("The browser tool rejected its output.")) + .mockResolvedValueOnce(undefined); + + safelyAddToolOutput( + addToolOutput as Parameters[0], + { + tool: getLatestNetDefinitionToolName, + toolCallId: "tool-browser-failure", + output: { + definition: emptySDCPN, + extensions: DEFAULT_PETRINAUT_EXTENSIONS, + title: "Failure fixture", + }, + }, + ); + + await waitFor(() => expect(addToolOutput).toHaveBeenCalledTimes(2)); + expect(addToolOutput).toHaveBeenLastCalledWith({ + errorText: "The browser tool rejected its output.", + state: "output-error", + tool: getLatestNetDefinitionToolName, + toolCallId: "tool-browser-failure", + }); + }); + test("sends review chips as messages while an interactive tool is pending", async () => { const requestMessages: PetrinautAiMessage[][] = []; const transport: PetrinautAiTransport = { diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx index c138cf7738c..d44ac65b280 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel.tsx @@ -155,7 +155,25 @@ const isPetrinautAiCommandToolName = ( toolName: string, ): toolName is AiCommandActionName => toolName in aiCommandActionInputSchemas; -const safelyAddToolOutput = ( +const browserToolErrorText = (error: unknown): string => { + if (error instanceof Error && error.message.trim().length > 0) { + return error.message; + } + if (typeof error === "string" && error.trim().length > 0) { + return error; + } + try { + const serialized: unknown = JSON.stringify(error); + if (typeof serialized === "string" && serialized.length > 0) { + return serialized; + } + } catch { + // Fall through to the stable fallback for cyclic values. + } + return "The browser tool failed."; +}; + +export const safelyAddToolOutput = ( addToolOutput: ReturnType< typeof useChat >["addToolOutput"], @@ -163,10 +181,16 @@ const safelyAddToolOutput = ( ReturnType>["addToolOutput"] >[0], ) => { - // Failures here surface in the UI as an errored tool call (with the - // error message on hover), so we just swallow the rejection to avoid an - // unhandled-promise warning. - void Promise.resolve(addToolOutput(params)).catch(() => {}); + void Promise.resolve(addToolOutput(params)).catch((error: unknown) => { + void Promise.resolve( + addToolOutput({ + errorText: browserToolErrorText(error), + state: "output-error", + tool: params.tool, + toolCallId: params.toolCallId, + }), + ).catch(() => {}); + }); }; const addDynamicToolOutput = ( diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.stories.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.stories.tsx index c21fe5af927..c69b35aafc3 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.stories.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.stories.tsx @@ -1,4 +1,5 @@ import { type ReactNode, useState } from "react"; +import { userEvent, within } from "storybook/test"; import { Button } from "@hashintel/ds-components"; import { css } from "@hashintel/ds-helpers/css"; @@ -343,6 +344,25 @@ export const VoiceSessionListening: Story = { ), }; +export const VoiceSessionCollapsed: Story = { + render: () => ( + } + voiceModeAvailable + voiceSession={liveSession({ microphoneLevel: 0.6 })} + /> + ), + play: async ({ canvasElement }) => { + await userEvent.click( + within(canvasElement).getByRole("button", { + name: "Collapse voice session", + }), + ); + }, +}; + export const VoiceSessionSpeaking: Story = { render: () => ( + ...singleToolCallMessage, + parts: singleToolCallMessage.parts.map((part) => part.type.startsWith("tool-") ? { ...part, diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx index caf6dfff543..3afdda4a8cc 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.test.tsx @@ -44,6 +44,10 @@ vi.mock("react-markdown", async (importOriginal) => { }); const noop = () => {}; +const initialClipboardDescriptor = Object.getOwnPropertyDescriptor( + navigator, + "clipboard", +); // The voice ribbon asks for a 2D context on mount. jsdom has no canvas, and // answering with `null` takes the same branch a browser without one would, @@ -64,14 +68,28 @@ afterEach(() => { cleanup(); vi.clearAllMocks(); vi.useRealTimers(); + if (initialClipboardDescriptor === undefined) { + Reflect.deleteProperty(navigator, "clipboard"); + } else { + Object.defineProperty(navigator, "clipboard", initialClipboardDescriptor); + } }); describe("AiAssistantContents", () => { test("shows assistant errors as toasts instead of transcript messages", async () => { + const writeText = vi.fn().mockResolvedValue(undefined); + Object.defineProperty(navigator, "clipboard", { + configurable: true, + value: { writeText }, + }); render( { ); const toast = await waitFor(() => { - const element = document.querySelector( + const element = document.querySelector( '[data-scope="toast"][data-part="root"]', ); expect(element).not.toBeNull(); return element!; }); - expect(toast.textContent).toBe("Failed to fetch"); + expect( + toast.querySelector('[data-scope="toast"][data-part="title"]') + ?.textContent, + ).toBe("AI assistant error"); + expect( + toast.querySelector('[data-scope="toast"][data-part="description"]') + ?.textContent, + ).toBe( + 'Elicitor failed.\nCaused by: {"field":"answer","reason":"Required"}', + ); + fireEvent.click( + within(toast).getByRole("button", { name: "Copy details" }), + ); + expect(writeText).toHaveBeenCalledWith( + 'Elicitor failed.\nCaused by: {"field":"answer","reason":"Required"}', + ); expect( within(screen.getByTestId("ai-transcript")).queryByText( - "Failed to fetch", + "AI assistant error", ), ).toBeNull(); + fireEvent.click( + within(toast).getByRole("button", { name: "Close notification" }), + ); + await waitFor(() => + expect(toast.getAttribute("data-state")).toBe("closed"), + ); }); test("keeps one Voice mode slot mounted above the composer when the panel closes", () => { @@ -145,7 +184,7 @@ describe("AiAssistantContents", () => { expect(voiceModeUnmounts).toBe(0); }); - test("swaps the composer for the dock while a session runs, and defers its spoken turns", () => { + test("shows spoken turns live and collapses an active session without unmounting the panel", () => { const store = createVoiceSessionStore(); const actions = { end: vi.fn(), @@ -178,6 +217,7 @@ describe("AiAssistantContents", () => { onStop={noop} onSubmit={noop} status="ready" + voiceMode={
Host Voice controls
} /> ); @@ -213,29 +253,44 @@ describe("AiAssistantContents", () => { ] as PetrinautAiMessage[]), ); - expect(screen.queryByText("Spoken request")).toBeNull(); - expect(screen.queryByText("Spoken reply")).toBeNull(); + expect(screen.getByText("Spoken request")).not.toBeNull(); + expect(screen.getByText("Spoken reply")).not.toBeNull(); expect(screen.getByText("Typed aside")).not.toBeNull(); - // The dock's transcription action writes the held turns into the chat - // mid-session, and holds them back again when it is turned off. + const transcript = screen.getByTestId("ai-transcript"); + const voiceMode = screen.getByTestId("ai-voice-mode"); + const header = screen + .getByRole("button", { name: "Close AI assistant" }) + .closest("div")!; + fireEvent.click( - within(dock).getByRole("button", { name: "Show transcription in chat" }), + within(dock).getByRole("button", { name: "Collapse voice session" }), ); - expect(screen.getByText("Spoken request")).not.toBeNull(); - expect(screen.getByText("Spoken reply")).not.toBeNull(); - expect(screen.queryByText("Voice session · 1 turn")).toBeNull(); + + expect(screen.getByTestId("ai-transcript")).toBe(transcript); + expect(screen.getByTestId("ai-voice-mode")).toBe(voiceMode); + expect( + screen + .getByRole("button", { name: "Close AI assistant", hidden: true }) + .closest("div"), + ).toBe(header); + expect(transcript.className).toContain("d_none"); + expect(voiceMode.className).toContain("d_none"); + expect(header.className).toContain("d_none"); fireEvent.click( - within(dock).getByRole("button", { name: "Hide transcription in chat" }), + within(dock).getByRole("button", { name: "Expand voice session" }), ); - expect(screen.queryByText("Spoken request")).toBeNull(); + + expect(transcript.className).not.toContain("d_none"); + expect(voiceMode.className).not.toContain("d_none"); + expect(header.className).not.toContain("d_none"); + expect(screen.getByText("Spoken request")).not.toBeNull(); act(() => store.setState(null)); expect(screen.getByText("Spoken request")).not.toBeNull(); expect(screen.getByText("Spoken reply")).not.toBeNull(); - expect(screen.getByText("Voice session · 1 turn")).not.toBeNull(); expect(screen.queryByRole("region", { name: "Voice session" })).toBeNull(); expect( screen.getByRole("textbox", { name: "Message AI assistant" }), @@ -383,15 +438,16 @@ describe("AiAssistantContents", () => { ); const toast = await waitFor(() => { - const element = document.querySelector( + const element = document.querySelector( '[data-scope="toast"][data-part="root"]', ); expect(element).not.toBeNull(); return element!; }); - expect(toast.textContent).toBe( - "Microphone unavailable. Check your browser permissions.", - ); + expect( + toast.querySelector('[data-scope="toast"][data-part="title"]') + ?.textContent, + ).toBe("Microphone unavailable. Check your browser permissions."); }); test("does not repeat a voice error toast until the session recovers", () => { @@ -1674,7 +1730,7 @@ describe("AiAssistantContents", () => { expect(screen.getByRole("button", { name: /2 changes/u })).not.toBeNull(); }); - test("labels failed tool calls as errored", () => { + test("shows failed tool-call errors inline", () => { const messages: PetrinautAiMessage[] = [ { id: "assistant-1", @@ -1705,9 +1761,11 @@ describe("AiAssistantContents", () => { />, ); - expect( - screen.getByRole("button", { name: /deleteItemsByIds errored/u }), - ).not.toBeNull(); + const tool = screen.getByRole("button", { + name: /Validation failed.*deleteItemsByIds/u, + }); + expect(tool).not.toBeNull(); + expect(tool.getAttribute("title")).toBeNull(); }); test("expands deleted item summaries", () => { diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.tsx index 42cd71bc79e..f705ef7b05a 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/ai-assistant-panel/ai-assistant-contents.tsx @@ -1,5 +1,4 @@ import { - Fragment, memo, type ReactNode, type RefObject, @@ -14,8 +13,10 @@ import ReactMarkdown from "react-markdown"; import { Button, Icon } from "@hashintel/ds-components"; import { css, cva } from "@hashintel/ds-helpers/css"; -import { NotificationsContext } from "../../../../../react/notifications/context"; -import { VoiceSessionContext } from "../../../../../react/voice-session/context"; +import { + NotificationsContext, + type AddNotificationInput, +} from "../../../../../react/notifications/context"; import { useVoiceSessionErrorMessage, useVoiceSessionPhase, @@ -23,7 +24,6 @@ import { import { AiAssistantIcon } from "../../../../components/ai-assistant-icon"; import { ResizeHandle } from "../../../../resize/resize-handle"; import { AiVoiceModeIcon } from "../../components/ai-voice-mode-button"; -import { partitionVoiceSessionMessages } from "./ai-assistant-contents/defer-voice-messages"; import { aiFooterMinHeight } from "./ai-assistant-contents/footer-height"; import { getMessageRenderItems } from "./ai-assistant-contents/get-message-render-items"; import { @@ -48,6 +48,11 @@ type AiAssistantStatus = "submitted" | "streaming" | "ready" | "error"; const EMPTY_INTERACTIVE_TOOLS: readonly PetrinautAiInteractiveTool[] = []; +const errorNotification = ( + message: string, + detail?: string, +): AddNotificationInput => ({ detail, message, tone: "error" }); + export type AiAssistantContentsProps = { clearMessagesDisabled?: boolean; composerControl?: ReactNode; @@ -90,6 +95,9 @@ const shellStyle = cva({ }, }, variants: { + collapsed: { + true: {}, + }, open: { true: { top: "0", @@ -117,6 +125,16 @@ const shellStyle = cva({ }, }, }, + compoundVariants: [ + { + collapsed: true, + open: true, + css: { + top: "[auto]", + height: `[${aiFooterMinHeight + 16}px]`, + }, + }, + ], }); // Tracks the card's inset within the padded shell, so the resize handle @@ -261,40 +279,6 @@ const messageStyle = cva({ textAlign: "right", }, }, - // Spoken turns land in the transcript together once the session ends, so - // they arrive with a single entrance rather than appearing out of nowhere. - revealed: { - true: { - animationName: "[petrinautVoiceReveal]", - animationDuration: "[420ms]", - animationTimingFunction: "[cubic-bezier(0.22, 0.9, 0.3, 1)]", - "@media (prefers-reduced-motion: reduce)": { - animationName: "[none]", - }, - }, - }, - }, -}); - -const voiceSessionMetaStyle = css({ - display: "flex", - alignItems: "center", - gap: "2", - paddingX: "1", - color: "neutral.s90", - fontSize: "xs", - fontWeight: "medium", - _before: { - flex: "[1]", - height: "[1px]", - backgroundColor: "neutral.a30", - content: '""', - }, - _after: { - flex: "[1]", - height: "[1px]", - backgroundColor: "neutral.a30", - content: '""', }, }); @@ -441,12 +425,10 @@ const AiAssistantMessage = memo( handlersRef, interactiveTools, message, - revealed = false, }: { handlersRef: MessageHandlersRef; interactiveTools: readonly PetrinautAiInteractiveTool[]; message: PetrinautAiMessage; - revealed?: boolean; }) => { const role = message.role === "user" ? "user" : "assistant"; const renderItems = getMessageRenderItems(message, interactiveTools); @@ -459,7 +441,7 @@ const AiAssistantMessage = memo( return (
@@ -542,7 +524,6 @@ export const AiAssistantContents = ({ voiceModeAvailable = false, }: AiAssistantContentsProps) => { const { addNotification } = use(NotificationsContext); - const voiceSessionStore = use(VoiceSessionContext); const voiceSessionPhase = useVoiceSessionPhase(); const voiceSessionErrorMessage = useVoiceSessionErrorMessage(); const isVoiceSessionLive = voiceSessionPhase !== null; @@ -601,76 +582,8 @@ export const AiAssistantContents = ({ variant: "solid", }; - // Index of the first message belonging to the current or most recent voice - // session. Everything from here on is held back while that session runs, and - // revealed together once it ends. - const [sessionBaselineIndex, setSessionBaselineIndex] = useState< - number | null - >(() => - voiceSessionStore.getSnapshot().state === null ? null : messages.length, - ); - - // Off by default: the dock's transcription action writes spoken turns into - // the conversation as they land instead of holding them to the end. - const [transcriptionShown, setTranscriptionShown] = useState(false); - - const messageCountRef = useRef(messages.length); - useEffect(() => { - messageCountRef.current = messages.length; - }, [messages]); - - // Read from the store rather than from a render effect, so the baseline is - // captured on the event that starts the session instead of a render that - // happens to observe it. - useEffect(() => { - let wasLive = voiceSessionStore.getSnapshot().state !== null; - - return voiceSessionStore.subscribe(() => { - const isLive = voiceSessionStore.getSnapshot().state !== null; - if (isLive === wasLive) { - return; - } - wasLive = isLive; - - if (isLive) { - setSessionBaselineIndex(messageCountRef.current); - setTranscriptionShown(false); - } - }); - }, [voiceSessionStore]); - - const isHoldingVoiceTurns = isVoiceSessionLive && !transcriptionShown; - - const sessionPartition = - sessionBaselineIndex === null - ? null - : partitionVoiceSessionMessages({ - deferredFromIndex: sessionBaselineIndex, - interactiveTools, - messages, - }); - - const visibleMessages = - isHoldingVoiceTurns && sessionPartition !== null - ? sessionPartition.visible - : messages; - - // Held turns become "revealed" once they are let through — by the - // transcription action mid-session, or by the session ending — so they carry - // the entrance animation either way. - const revealedIds = new Set( - isHoldingVoiceTurns || sessionPartition === null - ? [] - : sessionPartition.deferred.map((message) => message.id), - ); - // The divider counts a finished session, so it waits for the session to end - // rather than growing a turn at a time under a live transcription. - const firstRevealedMessageId = isVoiceSessionLive - ? undefined - : visibleMessages.find((message) => revealedIds.has(message.id))?.id; - const revealedVoiceTurnCount = visibleMessages.filter( - (message) => revealedIds.has(message.id) && message.role === "user", - ).length; + const [voiceDockCollapsed, setVoiceDockCollapsed] = useState(false); + const isVoiceDockCollapsed = isVoiceSessionLive && voiceDockCollapsed; const [assistantWidth, setAssistantWidth] = useState(defaultAssistantWidth); @@ -686,10 +599,7 @@ export const AiAssistantContents = ({ return; } notifiedErrorRef.current = error; - addNotification({ - message: error.message, - tone: "error", - }); + addNotification(errorNotification("AI assistant error", error.message)); }, [addNotification, error]); // Voice failures (microphone denied, connection dropped) are reported by the @@ -709,10 +619,7 @@ export const AiAssistantContents = ({ } notifiedVoiceErrorRef.current = voiceSessionErrorMessage; - addNotification({ - message: voiceSessionErrorMessage, - tone: "error", - }); + addNotification(errorNotification(voiceSessionErrorMessage)); }, [addNotification, voiceSessionErrorMessage, voiceSessionPhase]); const inputRef = useRef(null); @@ -810,7 +717,10 @@ export const AiAssistantContents = ({