diff --git a/Sources/LLMProviderKit/Models/LLMModels.swift b/Sources/LLMProviderKit/Models/LLMModels.swift index d55fd1d..99e626f 100644 --- a/Sources/LLMProviderKit/Models/LLMModels.swift +++ b/Sources/LLMProviderKit/Models/LLMModels.swift @@ -57,6 +57,13 @@ public struct LLMMessage: Sendable, Equatable { public static func tool(_ content: String, toolCallId: String) -> Self { LLMMessage(role: .tool, content: content, toolCallId: toolCallId) } + + /// Create a tool-result message carrying images (e.g. a screenshot a tool + /// returned) so a vision-capable model can see them. Providers place the + /// images per their API's rules for tool results. + public static func tool(_ content: String, images: [LLMImage], toolCallId: String) -> Self { + LLMMessage(role: .tool, content: content, images: images, toolCallId: toolCallId) + } } /// An image payload attached to a message for vision-capable models. diff --git a/Sources/LLMProviderKitAnthropic/AnthropicProvider.swift b/Sources/LLMProviderKitAnthropic/AnthropicProvider.swift index dd5dfe6..ba74b8a 100644 --- a/Sources/LLMProviderKitAnthropic/AnthropicProvider.swift +++ b/Sources/LLMProviderKitAnthropic/AnthropicProvider.swift @@ -59,11 +59,31 @@ public struct AnthropicProvider: LLMProvider { if msg.role == .tool { var blocks: [[String: Any]] = [] if let toolCallId = msg.toolCallId { - blocks.append([ + var toolResult: [String: Any] = [ "type": "tool_result", - "tool_use_id": toolCallId, - "content": msg.content - ]) + "tool_use_id": toolCallId + ] + if msg.images.isEmpty { + toolResult["content"] = msg.content + } else { + // Anthropic allows tool_result.content to be an array of + // text + image blocks — keeps the image in the same user + // turn as the result (no alternation break) and correlated + // with the tool call. + var inner: [[String: Any]] = [["type": "text", "text": msg.content]] + for img in msg.images { + inner.append([ + "type": "image", + "source": [ + "type": "base64", + "media_type": img.mimeType, + "data": img.base64 + ] + ]) + } + toolResult["content"] = inner + } + blocks.append(toolResult) } else { blocks.append(["type": "text", "text": msg.content]) } diff --git a/Tests/LLMProviderKitTests/LiveAnthropicToolImageTests.swift b/Tests/LLMProviderKitTests/LiveAnthropicToolImageTests.swift new file mode 100644 index 0000000..8105fe9 --- /dev/null +++ b/Tests/LLMProviderKitTests/LiveAnthropicToolImageTests.swift @@ -0,0 +1,61 @@ +import Foundation +import CoreGraphics +import ImageIO +import UniformTypeIdentifiers +@testable import LLMProviderKit +@testable import LLMProviderKitAnthropic +import Testing + +/// LIVE test (gated on ANTHROPIC_API_KEY): proves the real Anthropic API accepts +/// our tool-result-with-image structure AND that the model actually sees the +/// image. Sends a solid-red screenshot as a tool result and expects the model to +/// name the color "red". +/// +/// Run with: ANTHROPIC_API_KEY=sk-ant-… swift test --filter LiveAnthropicToolImageTests +struct LiveAnthropicToolImageTests { + + /// A solid-color PNG rendered without UIKit/AppKit (CoreGraphics + ImageIO). + private func solidPNG(red: CGFloat, green: CGFloat, blue: CGFloat, size: Int = 64) -> Data { + let cs = CGColorSpaceCreateDeviceRGB() + let ctx = CGContext(data: nil, width: size, height: size, bitsPerComponent: 8, + bytesPerRow: 0, space: cs, + bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue)! + ctx.setFillColor(CGColor(red: red, green: green, blue: blue, alpha: 1)) + ctx.fill(CGRect(x: 0, y: 0, width: size, height: size)) + let image = ctx.makeImage()! + let out = NSMutableData() + let dest = CGImageDestinationCreateWithData(out, UTType.png.identifier as CFString, 1, nil)! + CGImageDestinationAddImage(dest, image, nil) + CGImageDestinationFinalize(dest) + return out as Data + } + + @Test(.enabled(if: ProcessInfo.processInfo.environment["ANTHROPIC_API_KEY"] != nil)) + func modelSeesToolResultImage() async throws { + let key = ProcessInfo.processInfo.environment["ANTHROPIC_API_KEY"]! + let model = "claude-3-5-sonnet-20241022" + let provider = AnthropicProvider(configuration: + AnthropicProvider.anthropic(apiKey: key, model: model)) + + let screenshot = LLMImage(data: solidPNG(red: 1, green: 0, blue: 0), mimeType: "image/png") + let tool = LLMToolDefinition( + name: "take_snapshot", + description: "Capture a screenshot of the current page.", + parameters: ["type": "object", "properties": [:]]) + + let messages: [LLMMessage] = [ + .user("Call take_snapshot, then answer with ONLY the single dominant color you see in the screenshot."), + .assistant(content: "", toolCalls: [ + LLMToolCall(id: "tu_1", name: "take_snapshot", arguments: "{}") + ]), + .tool("Screenshot captured.", images: [screenshot], toolCallId: "tu_1"), + ] + + let request = LLMRequest(model: model, messages: messages, maxTokens: 100, tools: [tool]) + let response = try await provider.complete(request) + + let text = response.text.lowercased() + // The API accepted the tool_result-with-image AND the model saw a red image. + #expect(text.contains("red"), "Expected the model to see red; got: \(response.text)") + } +} diff --git a/Tests/LLMProviderKitTests/ToolResultImageTests.swift b/Tests/LLMProviderKitTests/ToolResultImageTests.swift new file mode 100644 index 0000000..428e372 --- /dev/null +++ b/Tests/LLMProviderKitTests/ToolResultImageTests.swift @@ -0,0 +1,69 @@ +import Foundation +@testable import LLMProviderKit +@testable import LLMProviderKitAnthropic +import Testing + +/// Proves the tricky Anthropic path for multimodal tool results: an image a tool +/// returned rides inside the `tool_result.content` array in the SAME user turn, +/// so it reaches a vision model without breaking Anthropic's user/assistant +/// alternation. This is a deterministic check on the exact request we'd send. +struct ToolResultImageTests { + + private func decodedBody(_ messages: [LLMMessage]) throws -> [String: Any] { + let provider = AnthropicProvider(configuration: + AnthropicProvider.anthropic(apiKey: "test", model: "claude-3-5-sonnet-20241022")) + let request = LLMRequest(model: "claude-3-5-sonnet-20241022", messages: messages) + let body = try #require(try provider.prepareRequest(request, stream: false).httpBody) + return try #require(try JSONSerialization.jsonObject(with: body) as? [String: Any]) + } + + @Test func toolResultImageRidesInsideToolResultContent() throws { + // A minimal 1x1 PNG stands in for a browser screenshot. + let png = Data(base64Encoded: + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==")! + let image = LLMImage(data: png, mimeType: "image/png") + + let messages: [LLMMessage] = [ + .user("Take a screenshot and describe it"), + .assistant(content: "", toolCalls: [ + LLMToolCall(id: "tu_1", name: "take_snapshot", arguments: "{}") + ]), + .tool("Screenshot captured.", images: [image], toolCallId: "tu_1"), + ] + + let body = try decodedBody(messages) + let msgs = try #require(body["messages"] as? [[String: Any]]) + + // Alternation: user → assistant → user (tool results map to a user turn). + #expect(msgs.map { $0["role"] as? String } == ["user", "assistant", "user"]) + + // The last user turn's tool_result block carries the image inline. + let toolTurn = msgs[2] + let blocks = try #require(toolTurn["content"] as? [[String: Any]]) + let toolResult = try #require(blocks.first { $0["type"] as? String == "tool_result" }) + #expect(toolResult["tool_use_id"] as? String == "tu_1") + + let inner = try #require(toolResult["content"] as? [[String: Any]]) + let textBlock = try #require(inner.first { $0["type"] as? String == "text" }) + #expect(textBlock["text"] as? String == "Screenshot captured.") + + let imageBlock = try #require(inner.first { $0["type"] as? String == "image" }) + let source = try #require(imageBlock["source"] as? [String: Any]) + #expect(source["type"] as? String == "base64") + #expect(source["media_type"] as? String == "image/png") + #expect(source["data"] as? String == image.base64) + } + + @Test func textOnlyToolResultStaysAString() throws { + // Without images, tool_result.content remains a plain string (unchanged). + let messages: [LLMMessage] = [ + .assistant(content: "", toolCalls: [LLMToolCall(id: "tu_9", name: "noop", arguments: "{}")]), + .tool("done", toolCallId: "tu_9"), + ] + let body = try decodedBody(messages) + let msgs = try #require(body["messages"] as? [[String: Any]]) + let blocks = try #require(msgs[1]["content"] as? [[String: Any]]) + let toolResult = try #require(blocks.first { $0["type"] as? String == "tool_result" }) + #expect(toolResult["content"] as? String == "done") + } +}