[PoC] feat(anthropic): images in tool results

Sources/LLMProviderKit/Models/LLMModels.swift

@@ -57,6 +57,13 @@ public struct LLMMessage: Sendable, Equatable {
     public static func tool(_ content: String, toolCallId: String) -> Self {
         LLMMessage(role: .tool, content: content, toolCallId: toolCallId)
     }
+
+    /// Create a tool-result message carrying images (e.g. a screenshot a tool
+    /// returned) so a vision-capable model can see them. Providers place the
+    /// images per their API's rules for tool results.
+    public static func tool(_ content: String, images: [LLMImage], toolCallId: String) -> Self {
+        LLMMessage(role: .tool, content: content, images: images, toolCallId: toolCallId)
+    }
 }
 
 /// An image payload attached to a message for vision-capable models.

Sources/LLMProviderKitAnthropic/AnthropicProvider.swift

@@ -59,11 +59,31 @@ public struct AnthropicProvider: LLMProvider {
             if msg.role == .tool {
                 var blocks: [[String: Any]] = []
                 if let toolCallId = msg.toolCallId {
-                    blocks.append([
+                    var toolResult: [String: Any] = [
                         "type": "tool_result",
-                        "tool_use_id": toolCallId,
-                        "content": msg.content
-                    ])
+                        "tool_use_id": toolCallId
+                    ]
+                    if msg.images.isEmpty {
+                        toolResult["content"] = msg.content
+                    } else {
+                        // Anthropic allows tool_result.content to be an array of
+                        // text + image blocks — keeps the image in the same user
+                        // turn as the result (no alternation break) and correlated
+                        // with the tool call.
+                        var inner: [[String: Any]] = [["type": "text", "text": msg.content]]
+                        for img in msg.images {
+                            inner.append([
+                                "type": "image",
+                                "source": [
+                                    "type": "base64",
+                                    "media_type": img.mimeType,
+                                    "data": img.base64
+                                ]
+                            ])
+                        }
+                        toolResult["content"] = inner
+                    }
+                    blocks.append(toolResult)
                 } else {
                     blocks.append(["type": "text", "text": msg.content])
                 }

Tests/LLMProviderKitTests/LiveAnthropicToolImageTests.swift

@@ -0,0 +1,61 @@
+import Foundation
+import CoreGraphics
+import ImageIO
+import UniformTypeIdentifiers
+@testable import LLMProviderKit
+@testable import LLMProviderKitAnthropic
+import Testing
+
+/// LIVE test (gated on ANTHROPIC_API_KEY): proves the real Anthropic API accepts
+/// our tool-result-with-image structure AND that the model actually sees the
+/// image. Sends a solid-red screenshot as a tool result and expects the model to
+/// name the color "red".
+///
+/// Run with:  ANTHROPIC_API_KEY=sk-ant-… swift test --filter LiveAnthropicToolImageTests
+struct LiveAnthropicToolImageTests {
+
+    /// A solid-color PNG rendered without UIKit/AppKit (CoreGraphics + ImageIO).
+    private func solidPNG(red: CGFloat, green: CGFloat, blue: CGFloat, size: Int = 64) -> Data {
+        let cs = CGColorSpaceCreateDeviceRGB()
+        let ctx = CGContext(data: nil, width: size, height: size, bitsPerComponent: 8,
+                            bytesPerRow: 0, space: cs,
+                            bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue)!
+        ctx.setFillColor(CGColor(red: red, green: green, blue: blue, alpha: 1))
+        ctx.fill(CGRect(x: 0, y: 0, width: size, height: size))
+        let image = ctx.makeImage()!
+        let out = NSMutableData()
+        let dest = CGImageDestinationCreateWithData(out, UTType.png.identifier as CFString, 1, nil)!
+        CGImageDestinationAddImage(dest, image, nil)
+        CGImageDestinationFinalize(dest)
+        return out as Data
+    }
+
+    @Test(.enabled(if: ProcessInfo.processInfo.environment["ANTHROPIC_API_KEY"] != nil))
+    func modelSeesToolResultImage() async throws {
+        let key = ProcessInfo.processInfo.environment["ANTHROPIC_API_KEY"]!
+        let model = "claude-3-5-sonnet-20241022"
+        let provider = AnthropicProvider(configuration:
+            AnthropicProvider.anthropic(apiKey: key, model: model))
+
+        let screenshot = LLMImage(data: solidPNG(red: 1, green: 0, blue: 0), mimeType: "image/png")
+        let tool = LLMToolDefinition(
+            name: "take_snapshot",
+            description: "Capture a screenshot of the current page.",
+            parameters: ["type": "object", "properties": [:]])
+
+        let messages: [LLMMessage] = [
+            .user("Call take_snapshot, then answer with ONLY the single dominant color you see in the screenshot."),
+            .assistant(content: "", toolCalls: [
+                LLMToolCall(id: "tu_1", name: "take_snapshot", arguments: "{}")
+            ]),
+            .tool("Screenshot captured.", images: [screenshot], toolCallId: "tu_1"),
+        ]
+
+        let request = LLMRequest(model: model, messages: messages, maxTokens: 100, tools: [tool])
+        let response = try await provider.complete(request)
+
+        let text = response.text.lowercased()
+        // The API accepted the tool_result-with-image AND the model saw a red image.
+        #expect(text.contains("red"), "Expected the model to see red; got: \(response.text)")
+    }
+}

Tests/LLMProviderKitTests/ToolResultImageTests.swift

@@ -0,0 +1,69 @@
+import Foundation
+@testable import LLMProviderKit
+@testable import LLMProviderKitAnthropic
+import Testing
+
+/// Proves the tricky Anthropic path for multimodal tool results: an image a tool
+/// returned rides inside the `tool_result.content` array in the SAME user turn,
+/// so it reaches a vision model without breaking Anthropic's user/assistant
+/// alternation. This is a deterministic check on the exact request we'd send.
+struct ToolResultImageTests {
+
+    private func decodedBody(_ messages: [LLMMessage]) throws -> [String: Any] {
+        let provider = AnthropicProvider(configuration:
+            AnthropicProvider.anthropic(apiKey: "test", model: "claude-3-5-sonnet-20241022"))
+        let request = LLMRequest(model: "claude-3-5-sonnet-20241022", messages: messages)
+        let body = try #require(try provider.prepareRequest(request, stream: false).httpBody)
+        return try #require(try JSONSerialization.jsonObject(with: body) as? [String: Any])
+    }
+
+    @Test func toolResultImageRidesInsideToolResultContent() throws {
+        // A minimal 1x1 PNG stands in for a browser screenshot.
+        let png = Data(base64Encoded:
+            "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==")!
+        let image = LLMImage(data: png, mimeType: "image/png")
+
+        let messages: [LLMMessage] = [
+            .user("Take a screenshot and describe it"),
+            .assistant(content: "", toolCalls: [
+                LLMToolCall(id: "tu_1", name: "take_snapshot", arguments: "{}")
+            ]),
+            .tool("Screenshot captured.", images: [image], toolCallId: "tu_1"),
+        ]
+
+        let body = try decodedBody(messages)
+        let msgs = try #require(body["messages"] as? [[String: Any]])
+
+        // Alternation: user → assistant → user (tool results map to a user turn).
+        #expect(msgs.map { $0["role"] as? String } == ["user", "assistant", "user"])
+
+        // The last user turn's tool_result block carries the image inline.
+        let toolTurn = msgs[2]
+        let blocks = try #require(toolTurn["content"] as? [[String: Any]])
+        let toolResult = try #require(blocks.first { $0["type"] as? String == "tool_result" })
+        #expect(toolResult["tool_use_id"] as? String == "tu_1")
+
+        let inner = try #require(toolResult["content"] as? [[String: Any]])
+        let textBlock = try #require(inner.first { $0["type"] as? String == "text" })
+        #expect(textBlock["text"] as? String == "Screenshot captured.")
+
+        let imageBlock = try #require(inner.first { $0["type"] as? String == "image" })
+        let source = try #require(imageBlock["source"] as? [String: Any])
+        #expect(source["type"] as? String == "base64")
+        #expect(source["media_type"] as? String == "image/png")
+        #expect(source["data"] as? String == image.base64)
+    }
+
+    @Test func textOnlyToolResultStaysAString() throws {
+        // Without images, tool_result.content remains a plain string (unchanged).
+        let messages: [LLMMessage] = [
+            .assistant(content: "", toolCalls: [LLMToolCall(id: "tu_9", name: "noop", arguments: "{}")]),
+            .tool("done", toolCallId: "tu_9"),
+        ]
+        let body = try decodedBody(messages)
+        let msgs = try #require(body["messages"] as? [[String: Any]])
+        let blocks = try #require(msgs[1]["content"] as? [[String: Any]])
+        let toolResult = try #require(blocks.first { $0["type"] as? String == "tool_result" })
+        #expect(toolResult["content"] as? String == "done")
+    }
+}