Allow Realtime voice conversation to call preview tool

src/Abstractions/CrestApps.Core.AI.Abstractions/Orchestration/AIInvocationContext.cs

@@ -1,3 +1,4 @@
+using System.Collections.Concurrent;
 using CrestApps.Core.AI.Models;
 using CrestApps.Core.AI.Tooling;
 
@@ -29,6 +30,7 @@ namespace CrestApps.Core.AI.Orchestration;
 /// </summary>
 public sealed class AIInvocationContext
 {
+    private readonly ConcurrentQueue<string> _pendingFigureMarkers = new();
     private int _referenceIndex;
     private List<Action> _disposeCallbacks;
 
@@ -95,6 +97,39 @@ public int NextReferenceIndex()
         return Interlocked.Increment(ref _referenceIndex);
     }
 
+    /// <summary>
+    /// Asks the host to show the picture registered under <paramref name="marker"/> in <see cref="ToolReferences"/>.
+    /// Used where the model cannot write the marker itself, such as a spoken realtime reply. Thread-safe.
+    /// </summary>
+    /// <param name="marker">The picture marker, for example <c>[fig:1]</c>.</param>
+    public void RequestFigureDisplay(string marker)
+    {
+        ArgumentException.ThrowIfNullOrWhiteSpace(marker);
+
+        _pendingFigureMarkers.Enqueue(marker);
+    }
+
+    /// <summary>
+    /// Removes and returns the markers passed to <see cref="RequestFigureDisplay"/> since the last call.
+    /// </summary>
+    /// <returns>The pending markers in request order.</returns>
+    public IReadOnlyList<string> TakeFigureDisplayRequests()
+    {
+        if (_pendingFigureMarkers.IsEmpty)
+        {
+            return [];
+        }
+
+        var markers = new List<string>();
+
+        while (_pendingFigureMarkers.TryDequeue(out var marker))
+        {
+            markers.Add(marker);
+        }
+
+        return markers;
+    }
+
     /// <summary>
     /// Registers a callback to run when the invocation scope is disposed (i.e., when the
     /// request/prompt completes). Used to release request-scoped resources such as in-memory

src/Primitives/CrestApps.Core.AI.Chat/Realtime/RealtimeChatSessionRunner.cs

@@ -699,7 +699,7 @@ await conversation.UpdateTurnDetectionAsync(
         }
 
         // The stream ended (session closed). Persist any assistant turn that never received a done event.
-        await FlushAssistantTurnAsync(context, turnStore, sink, sessionId, turn, finalText: null, cancellationToken);
+        await FlushAssistantTurnAsync(context, turnStore, sink, sessionId, turn, finalText: null, cancellationToken, endOfSession: true);
     }
 
     // Ends the session when neither side has spoken for the configured idle window, nobody is mid-utterance, and
@@ -943,21 +943,58 @@ private async Task FlushAssistantTurnAsync(
         string sessionId,
         AssistantTurn turn,
         string? finalText,
-        CancellationToken cancellationToken)
+        CancellationToken cancellationToken,
+        bool endOfSession = false)
     {
         // Persist when deltas were accumulated, or when a final transcript arrived even without deltas
         // (some providers emit only the completed transcript).
-        if (!turn.HasContent && string.IsNullOrWhiteSpace(finalText))
+        if (!turn.HasContent && string.IsNullOrWhiteSpace(finalText) && !endOfSession)
         {
             return;
         }
 
         var content = !string.IsNullOrWhiteSpace(finalText) ? finalText! : turn.Builder.ToString();
         var messageId = turn.MessageId ?? UniqueId.GenerateId();
+
+        // Requested pictures go on the next spoken turn (never a tool-only response), or on a picture-only turn at
+        // session end. Taken before the snapshot so every requested reference is in it.
+        IReadOnlyList<string> requestedFigures = !string.IsNullOrWhiteSpace(content) || endOfSession
+            ? AIInvocationScope.Current?.TakeFigureDisplayRequests() ?? []
+            : [];
         var references = SnapshotReferences();
 
         turn.Reset();
 
+        var figureMarkers = new List<string>(requestedFigures.Count);
+
+        foreach (var marker in requestedFigures)
+        {
+            // Skip markers already in the text, repeated, or without a picture the client can draw.
+            if (content.Contains(marker, StringComparison.Ordinal) ||
+                figureMarkers.Contains(marker, StringComparer.OrdinalIgnoreCase) ||
+                references is null ||
+                !references.TryGetValue(marker, out var reference) ||
+                !reference.IsImage ||
+                string.IsNullOrWhiteSpace(reference.Link))
+            {
+                continue;
+            }
+
+            figureMarkers.Add(marker);
+        }
+
+        if (figureMarkers.Count > 0)
+        {
+            var figureLine = string.Join(' ', figureMarkers);
+
+            // Sent as a transcript delta for the live client and saved with the turn for history.
+            var delta = string.IsNullOrWhiteSpace(content) ? figureLine : "\n\n" + figureLine;
+
+            await sink.AssistantTranscriptDeltaAsync(sessionId, messageId, delta, messageId, references, cancellationToken);
+
+            content = string.IsNullOrWhiteSpace(content) ? figureLine : content + delta;
+        }
+
         if (string.IsNullOrWhiteSpace(content))
         {
             return;

src/Primitives/CrestApps.Core.AI.Documents/Handlers/DocumentOrchestrationHandler.cs

@@ -208,6 +208,7 @@ sessionObj is AIChatSession session &&
             ["visionUserSuppliedDocuments"] = visionUserSuppliedDocuments ?? [],
             ["tabularAgentName"] = TabularDataAgentProvider.AgentName,
             ["isInScope"] = ragMetadata?.IsInScope == true,
+            ["isRealtime"] = context.OrchestrationContext.ExecutionMode == OrchestrationExecutionMode.Realtime,
         };
 
         var header = await _templateService.RenderAsync(AITemplateIds.DocumentAvailability, arguments, cancellationToken);

src/Primitives/CrestApps.Core.AI.Documents/Tools/PreviewTabularDataTool.cs

@@ -128,6 +128,15 @@ protected override async ValueTask<object> InvokeCoreAsync(
 
         if (!string.IsNullOrEmpty(cachedResponse))
         {
+            // Asked again, so show the cached pictures again.
+            foreach (var marker in AIInvocationScope.Current.ToolReferences.Keys)
+            {
+                if (cachedResponse.Contains(marker, StringComparison.Ordinal))
+                {
+                    AIInvocationScope.Current.RequestFigureDisplay(marker);
+                }
+            }
+
             if (logger.IsEnabled(LogLevel.Debug))
             {
                 logger.LogDebug("AI tool '{ToolName}' returned a cached preview for this turn.", Name);
@@ -504,6 +513,9 @@ private static async Task<string> BuildImageResponseAsync(
         foreach (var (marker, reference) in pending)
         {
             invocationContext.ToolReferences[marker] = reference;
+
+            // A spoken reply never contains the marker, so ask the host to show the picture.
+            invocationContext.RequestFigureDisplay(marker);
         }
 
         var response = new StringBuilder();

src/Primitives/CrestApps.Core.AI/Templates/Prompts/document-availability.md

@@ -7,6 +7,7 @@ Parameters:
   - userSuppliedDocuments: array of non-image session/user-level ChatDocumentInfo objects that are user-visible uploads/attachments.
   - tabularUserSuppliedDocuments: array of tabular session/user-level ChatDocumentInfo objects handled by the Tabular Data Agent.
   - visionUserSuppliedDocuments: array of image session/user-level ChatDocumentInfo objects with text analysis available via document tools.
+  - isRealtime: boolean indicating a realtime voice session.
 IsListable: false
 Category: Documents
 ---
@@ -36,11 +37,17 @@ Use `inspect_image` only when you need pixel-level detail that the text analysis
 The user has uploaded the following tabular data files.
 Use the `{{ tabularAgentName | default: "tabular-data-agent" }}` agent for spreadsheet/table tasks, including summaries, row counts, column descriptions, filtering, calculations, percentages, aggregates, transformations, and any question that references a column name or code. Do not answer tabular-data questions from document text alone; delegate to the tabular agent so it can inspect the SQL workspace and run queries.
 
+This includes requests to see the data ("show me the file", "preview it", "what does it look like"): delegate and ask the agent for a preview. Describing the sheets from document metadata is not a preview.
+
 Delegate EVERY follow-up about this data as well, not only the first request. A message such as "add a column", "also sort it", "now format that", or "make it a chart" refers to the live table and requires the agent again; the earlier answer in this conversation is not a substitute for re-running the work. A file you described in an earlier turn does not still exist to be amended — each delivered file is produced by one tool call, and changing it means producing a new one.
 
 This includes a request that only changes how the file LOOKS — "freeze the header", "shade alternating rows", "make that column currency", "widen the columns", "highlight the negatives", "add a total row". These read like small cosmetic touches, but you cannot apply one: the styling lives in a file that only the agent can rebuild, so answering such a message yourself leaves the user with a described change and nothing to download. A follow-up that begins with "also", "now", or "and" is still its own request and needs the agent just as much as the first one did.
 
 Never state that a file has been created, updated, or is ready for download unless a tool returned a download marker in THIS turn, and always return that marker exactly as given. Naming a file you did not just produce leaves the user with a link that does not work, or no link at all.
+{% if isRealtime %}
+
+This is a spoken conversation. When the agent's reply contains a picture marker such as `[fig:1]`, the picture is shown on the user's screen automatically. This overrides any instruction to copy picture markers: never read a marker aloud. Tell the user the preview is on their screen and briefly describe it. Do not call `view_document_figure` for a `[fig:N]` marker.
+{% endif %}
 
 ### Available tabular files:
 {% for doc in tabularUserSuppliedDocuments %}