From 4de1dec1bab92a72af7cc24e2f5c0208b68154f0 Mon Sep 17 00:00:00 2001 From: wipke Date: Wed, 23 Sep 2026 16:17:01 -0400 Subject: [PATCH] Show the spreadsheet preview in a realtime voice conversation A voice reply's text is a transcript of what the model said aloud, so it never contains the [fig:N] marker the client swaps for the preview picture. The preview was built and sent with the reply, but never drawn. The preview tool now asks the host to show each picture it registers, and the realtime runner appends the marker to the next spoken turn. The document prompt also sends preview requests to the tabular agent and tells a voice model not to read markers aloud. --- .../Orchestration/AIInvocationContext.cs | 35 +++ .../Realtime/RealtimeChatSessionRunner.cs | 43 +++- .../Handlers/DocumentOrchestrationHandler.cs | 1 + .../Tools/PreviewTabularDataTool.cs | 12 + .../Prompts/document-availability.md | 7 + .../RealtimeChatSessionRunnerTests.cs | 209 ++++++++++++++++++ 6 files changed, 304 insertions(+), 3 deletions(-) diff --git a/src/Abstractions/CrestApps.Core.AI.Abstractions/Orchestration/AIInvocationContext.cs b/src/Abstractions/CrestApps.Core.AI.Abstractions/Orchestration/AIInvocationContext.cs index 88267ee5..e362da44 100644 --- a/src/Abstractions/CrestApps.Core.AI.Abstractions/Orchestration/AIInvocationContext.cs +++ b/src/Abstractions/CrestApps.Core.AI.Abstractions/Orchestration/AIInvocationContext.cs @@ -1,3 +1,4 @@ +using System.Collections.Concurrent; using CrestApps.Core.AI.Models; using CrestApps.Core.AI.Tooling; @@ -29,6 +30,7 @@ namespace CrestApps.Core.AI.Orchestration; /// public sealed class AIInvocationContext { + private readonly ConcurrentQueue _pendingFigureMarkers = new(); private int _referenceIndex; private List _disposeCallbacks; @@ -95,6 +97,39 @@ public int NextReferenceIndex() return Interlocked.Increment(ref _referenceIndex); } + /// + /// Asks the host to show the picture registered under in . + /// Used where the model cannot write the marker itself, such as a spoken realtime reply. Thread-safe. + /// + /// The picture marker, for example [fig:1]. + public void RequestFigureDisplay(string marker) + { + ArgumentException.ThrowIfNullOrWhiteSpace(marker); + + _pendingFigureMarkers.Enqueue(marker); + } + + /// + /// Removes and returns the markers passed to since the last call. + /// + /// The pending markers in request order. + public IReadOnlyList TakeFigureDisplayRequests() + { + if (_pendingFigureMarkers.IsEmpty) + { + return []; + } + + var markers = new List(); + + while (_pendingFigureMarkers.TryDequeue(out var marker)) + { + markers.Add(marker); + } + + return markers; + } + /// /// Registers a callback to run when the invocation scope is disposed (i.e., when the /// request/prompt completes). Used to release request-scoped resources such as in-memory diff --git a/src/Primitives/CrestApps.Core.AI.Chat/Realtime/RealtimeChatSessionRunner.cs b/src/Primitives/CrestApps.Core.AI.Chat/Realtime/RealtimeChatSessionRunner.cs index b6d12a86..8575f740 100644 --- a/src/Primitives/CrestApps.Core.AI.Chat/Realtime/RealtimeChatSessionRunner.cs +++ b/src/Primitives/CrestApps.Core.AI.Chat/Realtime/RealtimeChatSessionRunner.cs @@ -699,7 +699,7 @@ await conversation.UpdateTurnDetectionAsync( } // The stream ended (session closed). Persist any assistant turn that never received a done event. - await FlushAssistantTurnAsync(context, turnStore, sink, sessionId, turn, finalText: null, cancellationToken); + await FlushAssistantTurnAsync(context, turnStore, sink, sessionId, turn, finalText: null, cancellationToken, endOfSession: true); } // Ends the session when neither side has spoken for the configured idle window, nobody is mid-utterance, and @@ -943,21 +943,58 @@ private async Task FlushAssistantTurnAsync( string sessionId, AssistantTurn turn, string? finalText, - CancellationToken cancellationToken) + CancellationToken cancellationToken, + bool endOfSession = false) { // Persist when deltas were accumulated, or when a final transcript arrived even without deltas // (some providers emit only the completed transcript). - if (!turn.HasContent && string.IsNullOrWhiteSpace(finalText)) + if (!turn.HasContent && string.IsNullOrWhiteSpace(finalText) && !endOfSession) { return; } var content = !string.IsNullOrWhiteSpace(finalText) ? finalText! : turn.Builder.ToString(); var messageId = turn.MessageId ?? UniqueId.GenerateId(); + + // Requested pictures go on the next spoken turn (never a tool-only response), or on a picture-only turn at + // session end. Taken before the snapshot so every requested reference is in it. + IReadOnlyList requestedFigures = !string.IsNullOrWhiteSpace(content) || endOfSession + ? AIInvocationScope.Current?.TakeFigureDisplayRequests() ?? [] + : []; var references = SnapshotReferences(); turn.Reset(); + var figureMarkers = new List(requestedFigures.Count); + + foreach (var marker in requestedFigures) + { + // Skip markers already in the text, repeated, or without a picture the client can draw. + if (content.Contains(marker, StringComparison.Ordinal) || + figureMarkers.Contains(marker, StringComparer.OrdinalIgnoreCase) || + references is null || + !references.TryGetValue(marker, out var reference) || + !reference.IsImage || + string.IsNullOrWhiteSpace(reference.Link)) + { + continue; + } + + figureMarkers.Add(marker); + } + + if (figureMarkers.Count > 0) + { + var figureLine = string.Join(' ', figureMarkers); + + // Sent as a transcript delta for the live client and saved with the turn for history. + var delta = string.IsNullOrWhiteSpace(content) ? figureLine : "\n\n" + figureLine; + + await sink.AssistantTranscriptDeltaAsync(sessionId, messageId, delta, messageId, references, cancellationToken); + + content = string.IsNullOrWhiteSpace(content) ? figureLine : content + delta; + } + if (string.IsNullOrWhiteSpace(content)) { return; diff --git a/src/Primitives/CrestApps.Core.AI.Documents/Handlers/DocumentOrchestrationHandler.cs b/src/Primitives/CrestApps.Core.AI.Documents/Handlers/DocumentOrchestrationHandler.cs index 64ab2f0d..a368b12f 100644 --- a/src/Primitives/CrestApps.Core.AI.Documents/Handlers/DocumentOrchestrationHandler.cs +++ b/src/Primitives/CrestApps.Core.AI.Documents/Handlers/DocumentOrchestrationHandler.cs @@ -208,6 +208,7 @@ sessionObj is AIChatSession session && ["visionUserSuppliedDocuments"] = visionUserSuppliedDocuments ?? [], ["tabularAgentName"] = TabularDataAgentProvider.AgentName, ["isInScope"] = ragMetadata?.IsInScope == true, + ["isRealtime"] = context.OrchestrationContext.ExecutionMode == OrchestrationExecutionMode.Realtime, }; var header = await _templateService.RenderAsync(AITemplateIds.DocumentAvailability, arguments, cancellationToken); diff --git a/src/Primitives/CrestApps.Core.AI.Documents/Tools/PreviewTabularDataTool.cs b/src/Primitives/CrestApps.Core.AI.Documents/Tools/PreviewTabularDataTool.cs index ce0116fc..7c4f8adf 100644 --- a/src/Primitives/CrestApps.Core.AI.Documents/Tools/PreviewTabularDataTool.cs +++ b/src/Primitives/CrestApps.Core.AI.Documents/Tools/PreviewTabularDataTool.cs @@ -128,6 +128,15 @@ protected override async ValueTask InvokeCoreAsync( if (!string.IsNullOrEmpty(cachedResponse)) { + // Asked again, so show the cached pictures again. + foreach (var marker in AIInvocationScope.Current.ToolReferences.Keys) + { + if (cachedResponse.Contains(marker, StringComparison.Ordinal)) + { + AIInvocationScope.Current.RequestFigureDisplay(marker); + } + } + if (logger.IsEnabled(LogLevel.Debug)) { logger.LogDebug("AI tool '{ToolName}' returned a cached preview for this turn.", Name); @@ -504,6 +513,9 @@ private static async Task BuildImageResponseAsync( foreach (var (marker, reference) in pending) { invocationContext.ToolReferences[marker] = reference; + + // A spoken reply never contains the marker, so ask the host to show the picture. + invocationContext.RequestFigureDisplay(marker); } var response = new StringBuilder(); diff --git a/src/Primitives/CrestApps.Core.AI/Templates/Prompts/document-availability.md b/src/Primitives/CrestApps.Core.AI/Templates/Prompts/document-availability.md index 391dfb71..f9d16160 100644 --- a/src/Primitives/CrestApps.Core.AI/Templates/Prompts/document-availability.md +++ b/src/Primitives/CrestApps.Core.AI/Templates/Prompts/document-availability.md @@ -7,6 +7,7 @@ Parameters: - userSuppliedDocuments: array of non-image session/user-level ChatDocumentInfo objects that are user-visible uploads/attachments. - tabularUserSuppliedDocuments: array of tabular session/user-level ChatDocumentInfo objects handled by the Tabular Data Agent. - visionUserSuppliedDocuments: array of image session/user-level ChatDocumentInfo objects with text analysis available via document tools. + - isRealtime: boolean indicating a realtime voice session. IsListable: false Category: Documents --- @@ -36,11 +37,17 @@ Use `inspect_image` only when you need pixel-level detail that the text analysis The user has uploaded the following tabular data files. Use the `{{ tabularAgentName | default: "tabular-data-agent" }}` agent for spreadsheet/table tasks, including summaries, row counts, column descriptions, filtering, calculations, percentages, aggregates, transformations, and any question that references a column name or code. Do not answer tabular-data questions from document text alone; delegate to the tabular agent so it can inspect the SQL workspace and run queries. +This includes requests to see the data ("show me the file", "preview it", "what does it look like"): delegate and ask the agent for a preview. Describing the sheets from document metadata is not a preview. + Delegate EVERY follow-up about this data as well, not only the first request. A message such as "add a column", "also sort it", "now format that", or "make it a chart" refers to the live table and requires the agent again; the earlier answer in this conversation is not a substitute for re-running the work. A file you described in an earlier turn does not still exist to be amended — each delivered file is produced by one tool call, and changing it means producing a new one. This includes a request that only changes how the file LOOKS — "freeze the header", "shade alternating rows", "make that column currency", "widen the columns", "highlight the negatives", "add a total row". These read like small cosmetic touches, but you cannot apply one: the styling lives in a file that only the agent can rebuild, so answering such a message yourself leaves the user with a described change and nothing to download. A follow-up that begins with "also", "now", or "and" is still its own request and needs the agent just as much as the first one did. Never state that a file has been created, updated, or is ready for download unless a tool returned a download marker in THIS turn, and always return that marker exactly as given. Naming a file you did not just produce leaves the user with a link that does not work, or no link at all. +{% if isRealtime %} + +This is a spoken conversation. When the agent's reply contains a picture marker such as `[fig:1]`, the picture is shown on the user's screen automatically. This overrides any instruction to copy picture markers: never read a marker aloud. Tell the user the preview is on their screen and briefly describe it. Do not call `view_document_figure` for a `[fig:N]` marker. +{% endif %} ### Available tabular files: {% for doc in tabularUserSuppliedDocuments %} diff --git a/tests/CrestApps.Core.Tests/Core/Realtime/RealtimeChatSessionRunnerTests.cs b/tests/CrestApps.Core.Tests/Core/Realtime/RealtimeChatSessionRunnerTests.cs index 54020efa..5ffc9afb 100644 --- a/tests/CrestApps.Core.Tests/Core/Realtime/RealtimeChatSessionRunnerTests.cs +++ b/tests/CrestApps.Core.Tests/Core/Realtime/RealtimeChatSessionRunnerTests.cs @@ -1321,6 +1321,208 @@ await runner.RunAsync( Assert.Null(reference.Link); } + [Fact] + public async Task RunAsync_PlacesAPictureAToolAskedToShowOnTheNextSpokenReply() + { + // A spoken reply never contains the marker, so the runner appends it. + var profile = new AIProfile { Type = AIProfileType.Chat }; + var session = new AIChatSession { SessionId = "session-1" }; + var (store, persisted) = CreateStore(); + var sink = new RecordingSink(); + + using var scope = AIInvocationScope.Begin(); + + // The tool-only response must not take the picture into a bubble of its own. + var toolResponseCompleted = Evt(RealtimeConversationEventType.ResponseCompleted); + var conversation = new FakeConversation( + [ + Evt(RealtimeConversationEventType.ResponseStarted), + toolResponseCompleted, + Evt(RealtimeConversationEventType.ResponseStarted), + Evt(RealtimeConversationEventType.AssistantTranscriptDelta, text: "Here is the preview."), + Evt(RealtimeConversationEventType.AssistantTranscriptDone, text: "Here is the preview."), + Evt(RealtimeConversationEventType.ResponseCompleted), + ]) + { + BeforeEvent = evt => + { + if (ReferenceEquals(evt, toolResponseCompleted)) + { + scope.Context.ToolReferences["[fig:1]"] = new AICompletionReference { Index = 1, Title = "Sheet1", Link = "/ai/documents/preview-1/download", IsImage = true }; + scope.Context.RequestFigureDisplay("[fig:1]"); + } + }, + }; + + var runner = new RealtimeChatSessionRunner(new FakeOrchestrator(conversation), TimeProvider.System, NullLogger.Instance); + + await runner.RunAsync( + new RealtimeChatRunContext { Resource = profile, SessionId = session.SessionId, ChatSession = session }, + new ChatSessionRealtimeTurnStore(store.Object), + PendingAudio(TestContext.Current.CancellationToken), + sink, + TestContext.Current.CancellationToken); + + // One bubble, with the marker both streamed and saved. + var turn = Assert.Single(persisted); + Assert.Equal("Here is the preview.\n\n[fig:1]", turn.Content); + Assert.True(turn.References!["[fig:1]"].IsImage); + Assert.Equal(["Here is the preview.", "\n\n[fig:1]"], sink.AssistantDeltas); + Assert.Single(sink.AssistantCompleted); + Assert.Empty(scope.Context.TakeFigureDisplayRequests()); + } + + [Fact] + public async Task RunAsync_DoesNotRepeatAPictureTheReplyAlreadyNames() + { + // A cascaded chat leg can write the marker itself; it must not be added twice. + var profile = new AIProfile { Type = AIProfileType.Chat }; + var session = new AIChatSession { SessionId = "session-1" }; + var (store, persisted) = CreateStore(); + var sink = new RecordingSink(); + + using var scope = AIInvocationScope.Begin(); + scope.Context.ToolReferences["[fig:1]"] = new AICompletionReference { Index = 1, Title = "Sheet1", Link = "/ai/documents/preview-1/download", IsImage = true }; + scope.Context.RequestFigureDisplay("[fig:1]"); + + var conversation = new FakeConversation( + [ + Evt(RealtimeConversationEventType.AssistantTranscriptDelta, text: "Here it is: [fig:1]"), + Evt(RealtimeConversationEventType.AssistantTranscriptDone, text: "Here it is: [fig:1]"), + ]); + + var runner = new RealtimeChatSessionRunner(new FakeOrchestrator(conversation), TimeProvider.System, NullLogger.Instance); + + await runner.RunAsync( + new RealtimeChatRunContext { Resource = profile, SessionId = session.SessionId, ChatSession = session }, + new ChatSessionRealtimeTurnStore(store.Object), + PendingAudio(TestContext.Current.CancellationToken), + sink, + TestContext.Current.CancellationToken); + + Assert.Equal("Here it is: [fig:1]", Assert.Single(persisted).Content); + Assert.Equal(["Here it is: [fig:1]"], sink.AssistantDeltas); + } + + [Fact] + public async Task RunAsync_ShowsAPictureAgainWhenItIsAskedForAgain() + { + // A repeat request is answered from the preview cache with the same marker; it is shown again. + var profile = new AIProfile { Type = AIProfileType.Chat }; + var session = new AIChatSession { SessionId = "session-1" }; + var (store, persisted) = CreateStore(); + + using var scope = AIInvocationScope.Begin(); + scope.Context.ToolReferences["[fig:1]"] = new AICompletionReference { Index = 1, Title = "Sheet1", Link = "/ai/documents/preview-1/download", IsImage = true }; + scope.Context.RequestFigureDisplay("[fig:1]"); + + var secondToolResponseCompleted = Evt(RealtimeConversationEventType.ResponseCompleted); + var conversation = new FakeConversation( + [ + Evt(RealtimeConversationEventType.AssistantTranscriptDone, text: "Here is the preview."), + Evt(RealtimeConversationEventType.AssistantTranscriptDone, text: "Glad it helped."), + Evt(RealtimeConversationEventType.ResponseStarted), + secondToolResponseCompleted, + Evt(RealtimeConversationEventType.AssistantTranscriptDone, text: "Here it is again."), + ]) + { + BeforeEvent = evt => + { + if (ReferenceEquals(evt, secondToolResponseCompleted)) + { + scope.Context.RequestFigureDisplay("[fig:1]"); + } + }, + }; + + var runner = new RealtimeChatSessionRunner(new FakeOrchestrator(conversation), TimeProvider.System, NullLogger.Instance); + + await runner.RunAsync( + new RealtimeChatRunContext { Resource = profile, SessionId = session.SessionId, ChatSession = session }, + new ChatSessionRealtimeTurnStore(store.Object), + PendingAudio(TestContext.Current.CancellationToken), + new RecordingSink(), + TestContext.Current.CancellationToken); + + // The reply in between does not get the picture. + Assert.Equal( + ["Here is the preview.\n\n[fig:1]", "Glad it helped.", "Here it is again.\n\n[fig:1]"], + persisted.Select(prompt => prompt.Content)); + } + + [Fact] + public async Task RunAsync_WhenTheSessionEndsBeforeTheReply_StillShowsThePicture() + { + // The session ended before the reply, so the picture is saved as its own turn. + var profile = new AIProfile { Type = AIProfileType.Chat }; + var session = new AIChatSession { SessionId = "session-1" }; + var (store, persisted) = CreateStore(); + var sink = new RecordingSink(); + + using var scope = AIInvocationScope.Begin(); + + var toolResponseCompleted = Evt(RealtimeConversationEventType.ResponseCompleted); + var conversation = new FakeConversation( + [ + Evt(RealtimeConversationEventType.ResponseStarted), + toolResponseCompleted, + ]) + { + BeforeEvent = evt => + { + if (ReferenceEquals(evt, toolResponseCompleted)) + { + scope.Context.ToolReferences["[fig:1]"] = new AICompletionReference { Index = 1, Title = "Sheet1", Link = "/ai/documents/preview-1/download", IsImage = true }; + scope.Context.RequestFigureDisplay("[fig:1]"); + } + }, + }; + + var runner = new RealtimeChatSessionRunner(new FakeOrchestrator(conversation), TimeProvider.System, NullLogger.Instance); + + await runner.RunAsync( + new RealtimeChatRunContext { Resource = profile, SessionId = session.SessionId, ChatSession = session }, + new ChatSessionRealtimeTurnStore(store.Object), + PendingAudio(TestContext.Current.CancellationToken), + sink, + TestContext.Current.CancellationToken); + + Assert.Equal("[fig:1]", Assert.Single(persisted).Content); + Assert.Equal(["[fig:1]"], sink.AssistantDeltas); + Assert.Single(sink.AssistantCompleted); + } + + [Fact] + public async Task RunAsync_PlacesOnlyPicturesAToolAskedToShowAndTheHostCanServe() + { + // Unrequested, unservable, and unregistered pictures are not added. + var profile = new AIProfile { Type = AIProfileType.Chat }; + var session = new AIChatSession { SessionId = "session-1" }; + var (store, persisted) = CreateStore(); + + using var scope = AIInvocationScope.Begin(); + scope.Context.ToolReferences["[fig:1]"] = new AICompletionReference { Index = 1, Title = "Retrieved figure", Link = "/figures/1", IsImage = true }; + scope.Context.ToolReferences["[fig:2]"] = new AICompletionReference { Index = 2, Title = "Unservable preview", IsImage = true }; + scope.Context.RequestFigureDisplay("[fig:2]"); + scope.Context.RequestFigureDisplay("[fig:3]"); + + var conversation = new FakeConversation( + [ + Evt(RealtimeConversationEventType.AssistantTranscriptDone, text: "Here is the answer."), + ]); + + var runner = new RealtimeChatSessionRunner(new FakeOrchestrator(conversation), TimeProvider.System, NullLogger.Instance); + + await runner.RunAsync( + new RealtimeChatRunContext { Resource = profile, SessionId = session.SessionId, ChatSession = session }, + new ChatSessionRealtimeTurnStore(store.Object), + PendingAudio(TestContext.Current.CancellationToken), + new RecordingSink(), + TestContext.Current.CancellationToken); + + Assert.Equal("Here is the answer.", Assert.Single(persisted).Content); + } + private sealed class FixedLinkResolver : IAIReferenceLinkResolver { private readonly string _prefix; @@ -1414,6 +1616,11 @@ public FakeConversation(IReadOnlyList events) /// public bool HoldOpen { get; init; } + /// + /// Runs before each scripted event is raised, for example to simulate a tool call. + /// + public Action? BeforeEvent { get; init; } + /// /// A second batch of events a test can release once the first has been consumed, so it can put virtual /// time between them — the provider raising nothing at all while the user keeps talking, for instance. @@ -1514,6 +1721,8 @@ public async IAsyncEnumerable GetEventsAsync([Enumera { cancellationToken.ThrowIfCancellationRequested(); + BeforeEvent?.Invoke(evt); + yield return evt; await Task.Yield();