diff --git a/src/Abstractions/CrestApps.Core.AI.Abstractions/Orchestration/AIInvocationContext.cs b/src/Abstractions/CrestApps.Core.AI.Abstractions/Orchestration/AIInvocationContext.cs index 88267ee5..e362da44 100644 --- a/src/Abstractions/CrestApps.Core.AI.Abstractions/Orchestration/AIInvocationContext.cs +++ b/src/Abstractions/CrestApps.Core.AI.Abstractions/Orchestration/AIInvocationContext.cs @@ -1,3 +1,4 @@ +using System.Collections.Concurrent; using CrestApps.Core.AI.Models; using CrestApps.Core.AI.Tooling; @@ -29,6 +30,7 @@ namespace CrestApps.Core.AI.Orchestration; /// public sealed class AIInvocationContext { + private readonly ConcurrentQueue _pendingFigureMarkers = new(); private int _referenceIndex; private List _disposeCallbacks; @@ -95,6 +97,39 @@ public int NextReferenceIndex() return Interlocked.Increment(ref _referenceIndex); } + /// + /// Asks the host to show the picture registered under in . + /// Used where the model cannot write the marker itself, such as a spoken realtime reply. Thread-safe. + /// + /// The picture marker, for example [fig:1]. + public void RequestFigureDisplay(string marker) + { + ArgumentException.ThrowIfNullOrWhiteSpace(marker); + + _pendingFigureMarkers.Enqueue(marker); + } + + /// + /// Removes and returns the markers passed to since the last call. + /// + /// The pending markers in request order. + public IReadOnlyList TakeFigureDisplayRequests() + { + if (_pendingFigureMarkers.IsEmpty) + { + return []; + } + + var markers = new List(); + + while (_pendingFigureMarkers.TryDequeue(out var marker)) + { + markers.Add(marker); + } + + return markers; + } + /// /// Registers a callback to run when the invocation scope is disposed (i.e., when the /// request/prompt completes). Used to release request-scoped resources such as in-memory diff --git a/src/Primitives/CrestApps.Core.AI.Chat/Realtime/RealtimeChatSessionRunner.cs b/src/Primitives/CrestApps.Core.AI.Chat/Realtime/RealtimeChatSessionRunner.cs index b6d12a86..8575f740 100644 --- a/src/Primitives/CrestApps.Core.AI.Chat/Realtime/RealtimeChatSessionRunner.cs +++ b/src/Primitives/CrestApps.Core.AI.Chat/Realtime/RealtimeChatSessionRunner.cs @@ -699,7 +699,7 @@ await conversation.UpdateTurnDetectionAsync( } // The stream ended (session closed). Persist any assistant turn that never received a done event. - await FlushAssistantTurnAsync(context, turnStore, sink, sessionId, turn, finalText: null, cancellationToken); + await FlushAssistantTurnAsync(context, turnStore, sink, sessionId, turn, finalText: null, cancellationToken, endOfSession: true); } // Ends the session when neither side has spoken for the configured idle window, nobody is mid-utterance, and @@ -943,21 +943,58 @@ private async Task FlushAssistantTurnAsync( string sessionId, AssistantTurn turn, string? finalText, - CancellationToken cancellationToken) + CancellationToken cancellationToken, + bool endOfSession = false) { // Persist when deltas were accumulated, or when a final transcript arrived even without deltas // (some providers emit only the completed transcript). - if (!turn.HasContent && string.IsNullOrWhiteSpace(finalText)) + if (!turn.HasContent && string.IsNullOrWhiteSpace(finalText) && !endOfSession) { return; } var content = !string.IsNullOrWhiteSpace(finalText) ? finalText! : turn.Builder.ToString(); var messageId = turn.MessageId ?? UniqueId.GenerateId(); + + // Requested pictures go on the next spoken turn (never a tool-only response), or on a picture-only turn at + // session end. Taken before the snapshot so every requested reference is in it. + IReadOnlyList requestedFigures = !string.IsNullOrWhiteSpace(content) || endOfSession + ? AIInvocationScope.Current?.TakeFigureDisplayRequests() ?? [] + : []; var references = SnapshotReferences(); turn.Reset(); + var figureMarkers = new List(requestedFigures.Count); + + foreach (var marker in requestedFigures) + { + // Skip markers already in the text, repeated, or without a picture the client can draw. + if (content.Contains(marker, StringComparison.Ordinal) || + figureMarkers.Contains(marker, StringComparer.OrdinalIgnoreCase) || + references is null || + !references.TryGetValue(marker, out var reference) || + !reference.IsImage || + string.IsNullOrWhiteSpace(reference.Link)) + { + continue; + } + + figureMarkers.Add(marker); + } + + if (figureMarkers.Count > 0) + { + var figureLine = string.Join(' ', figureMarkers); + + // Sent as a transcript delta for the live client and saved with the turn for history. + var delta = string.IsNullOrWhiteSpace(content) ? figureLine : "\n\n" + figureLine; + + await sink.AssistantTranscriptDeltaAsync(sessionId, messageId, delta, messageId, references, cancellationToken); + + content = string.IsNullOrWhiteSpace(content) ? figureLine : content + delta; + } + if (string.IsNullOrWhiteSpace(content)) { return; diff --git a/src/Primitives/CrestApps.Core.AI.Documents/Handlers/DocumentOrchestrationHandler.cs b/src/Primitives/CrestApps.Core.AI.Documents/Handlers/DocumentOrchestrationHandler.cs index 64ab2f0d..a368b12f 100644 --- a/src/Primitives/CrestApps.Core.AI.Documents/Handlers/DocumentOrchestrationHandler.cs +++ b/src/Primitives/CrestApps.Core.AI.Documents/Handlers/DocumentOrchestrationHandler.cs @@ -208,6 +208,7 @@ sessionObj is AIChatSession session && ["visionUserSuppliedDocuments"] = visionUserSuppliedDocuments ?? [], ["tabularAgentName"] = TabularDataAgentProvider.AgentName, ["isInScope"] = ragMetadata?.IsInScope == true, + ["isRealtime"] = context.OrchestrationContext.ExecutionMode == OrchestrationExecutionMode.Realtime, }; var header = await _templateService.RenderAsync(AITemplateIds.DocumentAvailability, arguments, cancellationToken); diff --git a/src/Primitives/CrestApps.Core.AI.Documents/Tools/PreviewTabularDataTool.cs b/src/Primitives/CrestApps.Core.AI.Documents/Tools/PreviewTabularDataTool.cs index ce0116fc..7c4f8adf 100644 --- a/src/Primitives/CrestApps.Core.AI.Documents/Tools/PreviewTabularDataTool.cs +++ b/src/Primitives/CrestApps.Core.AI.Documents/Tools/PreviewTabularDataTool.cs @@ -128,6 +128,15 @@ protected override async ValueTask InvokeCoreAsync( if (!string.IsNullOrEmpty(cachedResponse)) { + // Asked again, so show the cached pictures again. + foreach (var marker in AIInvocationScope.Current.ToolReferences.Keys) + { + if (cachedResponse.Contains(marker, StringComparison.Ordinal)) + { + AIInvocationScope.Current.RequestFigureDisplay(marker); + } + } + if (logger.IsEnabled(LogLevel.Debug)) { logger.LogDebug("AI tool '{ToolName}' returned a cached preview for this turn.", Name); @@ -504,6 +513,9 @@ private static async Task BuildImageResponseAsync( foreach (var (marker, reference) in pending) { invocationContext.ToolReferences[marker] = reference; + + // A spoken reply never contains the marker, so ask the host to show the picture. + invocationContext.RequestFigureDisplay(marker); } var response = new StringBuilder(); diff --git a/src/Primitives/CrestApps.Core.AI/Templates/Prompts/document-availability.md b/src/Primitives/CrestApps.Core.AI/Templates/Prompts/document-availability.md index 391dfb71..f9d16160 100644 --- a/src/Primitives/CrestApps.Core.AI/Templates/Prompts/document-availability.md +++ b/src/Primitives/CrestApps.Core.AI/Templates/Prompts/document-availability.md @@ -7,6 +7,7 @@ Parameters: - userSuppliedDocuments: array of non-image session/user-level ChatDocumentInfo objects that are user-visible uploads/attachments. - tabularUserSuppliedDocuments: array of tabular session/user-level ChatDocumentInfo objects handled by the Tabular Data Agent. - visionUserSuppliedDocuments: array of image session/user-level ChatDocumentInfo objects with text analysis available via document tools. + - isRealtime: boolean indicating a realtime voice session. IsListable: false Category: Documents --- @@ -36,11 +37,17 @@ Use `inspect_image` only when you need pixel-level detail that the text analysis The user has uploaded the following tabular data files. Use the `{{ tabularAgentName | default: "tabular-data-agent" }}` agent for spreadsheet/table tasks, including summaries, row counts, column descriptions, filtering, calculations, percentages, aggregates, transformations, and any question that references a column name or code. Do not answer tabular-data questions from document text alone; delegate to the tabular agent so it can inspect the SQL workspace and run queries. +This includes requests to see the data ("show me the file", "preview it", "what does it look like"): delegate and ask the agent for a preview. Describing the sheets from document metadata is not a preview. + Delegate EVERY follow-up about this data as well, not only the first request. A message such as "add a column", "also sort it", "now format that", or "make it a chart" refers to the live table and requires the agent again; the earlier answer in this conversation is not a substitute for re-running the work. A file you described in an earlier turn does not still exist to be amended — each delivered file is produced by one tool call, and changing it means producing a new one. This includes a request that only changes how the file LOOKS — "freeze the header", "shade alternating rows", "make that column currency", "widen the columns", "highlight the negatives", "add a total row". These read like small cosmetic touches, but you cannot apply one: the styling lives in a file that only the agent can rebuild, so answering such a message yourself leaves the user with a described change and nothing to download. A follow-up that begins with "also", "now", or "and" is still its own request and needs the agent just as much as the first one did. Never state that a file has been created, updated, or is ready for download unless a tool returned a download marker in THIS turn, and always return that marker exactly as given. Naming a file you did not just produce leaves the user with a link that does not work, or no link at all. +{% if isRealtime %} + +This is a spoken conversation. When the agent's reply contains a picture marker such as `[fig:1]`, the picture is shown on the user's screen automatically. This overrides any instruction to copy picture markers: never read a marker aloud. Tell the user the preview is on their screen and briefly describe it. Do not call `view_document_figure` for a `[fig:N]` marker. +{% endif %} ### Available tabular files: {% for doc in tabularUserSuppliedDocuments %} diff --git a/tests/CrestApps.Core.Tests/Core/Realtime/RealtimeChatSessionRunnerTests.cs b/tests/CrestApps.Core.Tests/Core/Realtime/RealtimeChatSessionRunnerTests.cs index 54020efa..5ffc9afb 100644 --- a/tests/CrestApps.Core.Tests/Core/Realtime/RealtimeChatSessionRunnerTests.cs +++ b/tests/CrestApps.Core.Tests/Core/Realtime/RealtimeChatSessionRunnerTests.cs @@ -1321,6 +1321,208 @@ await runner.RunAsync( Assert.Null(reference.Link); } + [Fact] + public async Task RunAsync_PlacesAPictureAToolAskedToShowOnTheNextSpokenReply() + { + // A spoken reply never contains the marker, so the runner appends it. + var profile = new AIProfile { Type = AIProfileType.Chat }; + var session = new AIChatSession { SessionId = "session-1" }; + var (store, persisted) = CreateStore(); + var sink = new RecordingSink(); + + using var scope = AIInvocationScope.Begin(); + + // The tool-only response must not take the picture into a bubble of its own. + var toolResponseCompleted = Evt(RealtimeConversationEventType.ResponseCompleted); + var conversation = new FakeConversation( + [ + Evt(RealtimeConversationEventType.ResponseStarted), + toolResponseCompleted, + Evt(RealtimeConversationEventType.ResponseStarted), + Evt(RealtimeConversationEventType.AssistantTranscriptDelta, text: "Here is the preview."), + Evt(RealtimeConversationEventType.AssistantTranscriptDone, text: "Here is the preview."), + Evt(RealtimeConversationEventType.ResponseCompleted), + ]) + { + BeforeEvent = evt => + { + if (ReferenceEquals(evt, toolResponseCompleted)) + { + scope.Context.ToolReferences["[fig:1]"] = new AICompletionReference { Index = 1, Title = "Sheet1", Link = "/ai/documents/preview-1/download", IsImage = true }; + scope.Context.RequestFigureDisplay("[fig:1]"); + } + }, + }; + + var runner = new RealtimeChatSessionRunner(new FakeOrchestrator(conversation), TimeProvider.System, NullLogger.Instance); + + await runner.RunAsync( + new RealtimeChatRunContext { Resource = profile, SessionId = session.SessionId, ChatSession = session }, + new ChatSessionRealtimeTurnStore(store.Object), + PendingAudio(TestContext.Current.CancellationToken), + sink, + TestContext.Current.CancellationToken); + + // One bubble, with the marker both streamed and saved. + var turn = Assert.Single(persisted); + Assert.Equal("Here is the preview.\n\n[fig:1]", turn.Content); + Assert.True(turn.References!["[fig:1]"].IsImage); + Assert.Equal(["Here is the preview.", "\n\n[fig:1]"], sink.AssistantDeltas); + Assert.Single(sink.AssistantCompleted); + Assert.Empty(scope.Context.TakeFigureDisplayRequests()); + } + + [Fact] + public async Task RunAsync_DoesNotRepeatAPictureTheReplyAlreadyNames() + { + // A cascaded chat leg can write the marker itself; it must not be added twice. + var profile = new AIProfile { Type = AIProfileType.Chat }; + var session = new AIChatSession { SessionId = "session-1" }; + var (store, persisted) = CreateStore(); + var sink = new RecordingSink(); + + using var scope = AIInvocationScope.Begin(); + scope.Context.ToolReferences["[fig:1]"] = new AICompletionReference { Index = 1, Title = "Sheet1", Link = "/ai/documents/preview-1/download", IsImage = true }; + scope.Context.RequestFigureDisplay("[fig:1]"); + + var conversation = new FakeConversation( + [ + Evt(RealtimeConversationEventType.AssistantTranscriptDelta, text: "Here it is: [fig:1]"), + Evt(RealtimeConversationEventType.AssistantTranscriptDone, text: "Here it is: [fig:1]"), + ]); + + var runner = new RealtimeChatSessionRunner(new FakeOrchestrator(conversation), TimeProvider.System, NullLogger.Instance); + + await runner.RunAsync( + new RealtimeChatRunContext { Resource = profile, SessionId = session.SessionId, ChatSession = session }, + new ChatSessionRealtimeTurnStore(store.Object), + PendingAudio(TestContext.Current.CancellationToken), + sink, + TestContext.Current.CancellationToken); + + Assert.Equal("Here it is: [fig:1]", Assert.Single(persisted).Content); + Assert.Equal(["Here it is: [fig:1]"], sink.AssistantDeltas); + } + + [Fact] + public async Task RunAsync_ShowsAPictureAgainWhenItIsAskedForAgain() + { + // A repeat request is answered from the preview cache with the same marker; it is shown again. + var profile = new AIProfile { Type = AIProfileType.Chat }; + var session = new AIChatSession { SessionId = "session-1" }; + var (store, persisted) = CreateStore(); + + using var scope = AIInvocationScope.Begin(); + scope.Context.ToolReferences["[fig:1]"] = new AICompletionReference { Index = 1, Title = "Sheet1", Link = "/ai/documents/preview-1/download", IsImage = true }; + scope.Context.RequestFigureDisplay("[fig:1]"); + + var secondToolResponseCompleted = Evt(RealtimeConversationEventType.ResponseCompleted); + var conversation = new FakeConversation( + [ + Evt(RealtimeConversationEventType.AssistantTranscriptDone, text: "Here is the preview."), + Evt(RealtimeConversationEventType.AssistantTranscriptDone, text: "Glad it helped."), + Evt(RealtimeConversationEventType.ResponseStarted), + secondToolResponseCompleted, + Evt(RealtimeConversationEventType.AssistantTranscriptDone, text: "Here it is again."), + ]) + { + BeforeEvent = evt => + { + if (ReferenceEquals(evt, secondToolResponseCompleted)) + { + scope.Context.RequestFigureDisplay("[fig:1]"); + } + }, + }; + + var runner = new RealtimeChatSessionRunner(new FakeOrchestrator(conversation), TimeProvider.System, NullLogger.Instance); + + await runner.RunAsync( + new RealtimeChatRunContext { Resource = profile, SessionId = session.SessionId, ChatSession = session }, + new ChatSessionRealtimeTurnStore(store.Object), + PendingAudio(TestContext.Current.CancellationToken), + new RecordingSink(), + TestContext.Current.CancellationToken); + + // The reply in between does not get the picture. + Assert.Equal( + ["Here is the preview.\n\n[fig:1]", "Glad it helped.", "Here it is again.\n\n[fig:1]"], + persisted.Select(prompt => prompt.Content)); + } + + [Fact] + public async Task RunAsync_WhenTheSessionEndsBeforeTheReply_StillShowsThePicture() + { + // The session ended before the reply, so the picture is saved as its own turn. + var profile = new AIProfile { Type = AIProfileType.Chat }; + var session = new AIChatSession { SessionId = "session-1" }; + var (store, persisted) = CreateStore(); + var sink = new RecordingSink(); + + using var scope = AIInvocationScope.Begin(); + + var toolResponseCompleted = Evt(RealtimeConversationEventType.ResponseCompleted); + var conversation = new FakeConversation( + [ + Evt(RealtimeConversationEventType.ResponseStarted), + toolResponseCompleted, + ]) + { + BeforeEvent = evt => + { + if (ReferenceEquals(evt, toolResponseCompleted)) + { + scope.Context.ToolReferences["[fig:1]"] = new AICompletionReference { Index = 1, Title = "Sheet1", Link = "/ai/documents/preview-1/download", IsImage = true }; + scope.Context.RequestFigureDisplay("[fig:1]"); + } + }, + }; + + var runner = new RealtimeChatSessionRunner(new FakeOrchestrator(conversation), TimeProvider.System, NullLogger.Instance); + + await runner.RunAsync( + new RealtimeChatRunContext { Resource = profile, SessionId = session.SessionId, ChatSession = session }, + new ChatSessionRealtimeTurnStore(store.Object), + PendingAudio(TestContext.Current.CancellationToken), + sink, + TestContext.Current.CancellationToken); + + Assert.Equal("[fig:1]", Assert.Single(persisted).Content); + Assert.Equal(["[fig:1]"], sink.AssistantDeltas); + Assert.Single(sink.AssistantCompleted); + } + + [Fact] + public async Task RunAsync_PlacesOnlyPicturesAToolAskedToShowAndTheHostCanServe() + { + // Unrequested, unservable, and unregistered pictures are not added. + var profile = new AIProfile { Type = AIProfileType.Chat }; + var session = new AIChatSession { SessionId = "session-1" }; + var (store, persisted) = CreateStore(); + + using var scope = AIInvocationScope.Begin(); + scope.Context.ToolReferences["[fig:1]"] = new AICompletionReference { Index = 1, Title = "Retrieved figure", Link = "/figures/1", IsImage = true }; + scope.Context.ToolReferences["[fig:2]"] = new AICompletionReference { Index = 2, Title = "Unservable preview", IsImage = true }; + scope.Context.RequestFigureDisplay("[fig:2]"); + scope.Context.RequestFigureDisplay("[fig:3]"); + + var conversation = new FakeConversation( + [ + Evt(RealtimeConversationEventType.AssistantTranscriptDone, text: "Here is the answer."), + ]); + + var runner = new RealtimeChatSessionRunner(new FakeOrchestrator(conversation), TimeProvider.System, NullLogger.Instance); + + await runner.RunAsync( + new RealtimeChatRunContext { Resource = profile, SessionId = session.SessionId, ChatSession = session }, + new ChatSessionRealtimeTurnStore(store.Object), + PendingAudio(TestContext.Current.CancellationToken), + new RecordingSink(), + TestContext.Current.CancellationToken); + + Assert.Equal("Here is the answer.", Assert.Single(persisted).Content); + } + private sealed class FixedLinkResolver : IAIReferenceLinkResolver { private readonly string _prefix; @@ -1414,6 +1616,11 @@ public FakeConversation(IReadOnlyList events) /// public bool HoldOpen { get; init; } + /// + /// Runs before each scripted event is raised, for example to simulate a tool call. + /// + public Action? BeforeEvent { get; init; } + /// /// A second batch of events a test can release once the first has been consumed, so it can put virtual /// time between them — the provider raising nothing at all while the user keeps talking, for instance. @@ -1514,6 +1721,8 @@ public async IAsyncEnumerable GetEventsAsync([Enumera { cancellationToken.ThrowIfCancellationRequested(); + BeforeEvent?.Invoke(evt); + yield return evt; await Task.Yield();