diff --git a/CHANGELOG.md b/CHANGELOG.md index e383a9f..793ce94 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ All notable changes to LLMProviderKit will be documented in this file. - Send Gemini tool results (`functionResponse` parts) in a `user` turn instead of `model`; Gemini only accepts `user`/`model` roles and mishandled tool results sent as `model`. ### Added +- **In-process provider support.** `complete(_:)` and `stream(_:)` are now `LLMProvider` protocol requirements (with the existing HTTP implementations as defaults), so an on-device provider — e.g. an MLX or llama.cpp backend — can override them directly and be dispatched correctly through `any LLMProvider`, instead of the extension methods being statically bound to the HTTP path. The HTTP-shaped hooks (`prepareRequest`, `parseStreamLine`, `parseResponse`) gain default (throwing/empty) implementations so an in-process provider only implements `complete`/`stream` + `name`/`configuration`. Existing HTTP providers are unchanged. Regression coverage: an in-process provider overrides `complete`/`stream` and wins dispatch through the existential. - Latest curated models: OpenAI GPT-5.6 (`gpt-5.6`), Gemini 3.6 Flash (`gemini-3.6-flash`) and Gemini 3.5 Flash-Lite (`gemini-3.5-flash-lite`), and Anthropic Opus 4.7 / Opus 4.6. Gemini's default preset model is now Gemini 3.6 Flash. - Regression coverage for non-ASCII streaming text. - Regression coverage for the Anthropic version header and Gemini tool-result role. diff --git a/Sources/LLMProviderKit/Protocols/LLMProvider.swift b/Sources/LLMProviderKit/Protocols/LLMProvider.swift index 2fceeba..efd7ac0 100644 --- a/Sources/LLMProviderKit/Protocols/LLMProvider.swift +++ b/Sources/LLMProviderKit/Protocols/LLMProvider.swift @@ -43,11 +43,35 @@ public protocol LLMProvider: Sendable { /// Not every provider exposes a model list endpoint. The default /// implementation throws `LLMError.unsupportedOperation`. func availableModels() async throws -> [LLMModelInfo] + + /// Non-streaming completion. Declared as a requirement (with a default HTTP + /// implementation below) so an **in-process** provider — e.g. an on-device + /// MLX/llama.cpp backend — can override it directly instead of going through + /// `prepareRequest`/`parseResponse`. HTTP providers rely on the default. + func complete(_ request: LLMRequest) async throws -> LLMResponse + + /// Streaming completion. A requirement for the same reason as `complete`. + func stream(_ request: LLMRequest) -> AsyncThrowingStream } extension LLMProvider { public var urlSession: URLSession { .shared } + // Default (throwing) implementations of the HTTP-shaped hooks, so a provider + // that overrides `complete`/`stream` (in-process, no URLRequest) doesn't have + // to implement them. HTTP providers implement all three as before. + public func prepareRequest(_ request: LLMRequest, stream: Bool) throws -> URLRequest { + throw LLMError.unsupportedOperation("\(Self.name) does not build URL requests (override complete/stream instead).") + } + + public func parseStreamLine(_ line: String, request: LLMRequest) throws -> [LLMStreamChunk] { + [] + } + + public func parseResponse(_ data: Data, request: LLMRequest) throws -> LLMResponse { + throw LLMError.unsupportedOperation("\(Self.name) does not parse HTTP responses (override complete instead).") + } + public func resolvedModel(for request: LLMRequest) async throws -> String { if !request.model.isEmpty { return request.model } if let defaultModel = configuration.defaultModel, !defaultModel.isEmpty { return defaultModel } diff --git a/Tests/LLMProviderKitTests/LLMKitTests.swift b/Tests/LLMProviderKitTests/LLMKitTests.swift index 02edb58..3c7c694 100644 --- a/Tests/LLMProviderKitTests/LLMKitTests.swift +++ b/Tests/LLMProviderKitTests/LLMKitTests.swift @@ -764,3 +764,64 @@ extension ProviderTests { #expect(secondArgs["message"] as? String == "SwiftAgentKit") } } + +// MARK: - In-process provider (the pattern an on-device MLX backend uses) + +/// A provider that generates in-process — no HTTP. It overrides `complete`/ +/// `stream` and never touches `prepareRequest`/`parseResponse`. This is exactly +/// how an `MLXProvider` will plug in. +struct EchoLocalProvider: LLMProvider { + static let name = "echo-local" + let configuration: LLMProviderConfiguration + + private func reply(_ request: LLMRequest) -> String { + "echo: " + (request.messages.last(where: { $0.role == .user })?.content ?? "") + } + + func complete(_ request: LLMRequest) async throws -> LLMResponse { + LLMResponse(text: reply(request), finishReason: .stop, request: request, providerName: Self.name) + } + + func stream(_ request: LLMRequest) -> AsyncThrowingStream { + let text = reply(request) + return AsyncThrowingStream { continuation in + continuation.yield(.text(text)) + continuation.yield(.finish(reason: .stop, usage: nil)) + continuation.finish() + } + } +} + +struct InProcessProviderTests { + private func request() -> LLMRequest { + LLMRequest(model: "local", messages: [LLMMessage(role: .user, content: "hi there")]) + } + + /// Called through the existential `any LLMProvider`, the override must win — + /// proving `complete` is a dynamically-dispatched requirement, not a static + /// extension method (which would call the HTTP default and throw). + @Test func inProcessCompleteIsDynamicallyDispatched() async throws { + let provider: any LLMProvider = EchoLocalProvider(configuration: LLMProviderConfiguration(name: "echo-local", baseURL: URL(string: "inprocess://local")!)) + let response = try await provider.complete(request()) + #expect(response.text == "echo: hi there") + #expect(response.providerName == "echo-local") + } + + @Test func inProcessStreamIsDynamicallyDispatched() async throws { + let provider: any LLMProvider = EchoLocalProvider(configuration: LLMProviderConfiguration(name: "echo-local", baseURL: URL(string: "inprocess://local")!)) + var text = "" + for try await chunk in provider.stream(request()) { + if case .text(let t) = chunk { text += t } + } + #expect(text == "echo: hi there") + } + + /// The default HTTP hooks now exist, so an in-process provider that doesn't + /// implement them fails loudly (unsupported) rather than failing to compile. + @Test func httpHooksHaveThrowingDefaults() throws { + let provider = EchoLocalProvider(configuration: LLMProviderConfiguration(name: "echo-local", baseURL: URL(string: "inprocess://local")!)) + #expect(throws: (any Error).self) { + _ = try provider.prepareRequest(request(), stream: false) + } + } +}