From 102e0c987a7155d6704b7a9fdc52d006cd798b3a Mon Sep 17 00:00:00 2001 From: Ayman Hamed Date: Tue, 28 Jul 2026 14:33:26 +0300 Subject: [PATCH] fix(context): ledger receipts preserve the tool's conclusion (head+tail) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A ContextSift receipt summarized a tool result with its first N chars, so a failure captured the START of a traceback and dropped the ACTUAL error at the end. Once the exchange was compacted into the ledger, the agent could no longer tell whether the tool ran or why it failed — it would forget the outcome of its own tool call and re-run from scratch. The summary now keeps a head AND a tail (default length 200 -> 320), so the conclusion — a shell exit line, a written path, or the exception at the bottom of a traceback — survives compaction. Regression: a failure's exception text (at the tail) appears in the ledger receipt; existing externalization/recency tests tightened to check the raw output isn't kept INLINE (a conclusion snippet in the ledger is expected). Co-Authored-By: Claude Opus 4.8 --- CHANGELOG.md | 1 + .../Context/ContextManager.swift | 20 +++++++++-- .../SwiftAgentKitTests.swift | 35 ++++++++++++++++--- 3 files changed, 50 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a490db1..87723d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ All notable changes to SwiftAgentKit will be documented in this file. ## Unreleased ### Fixed +- **ContextSift ledger receipts now preserve the tool's CONCLUSION.** A receipt's one-line summary was the first N characters of the output — so for a failure it captured `"Traceback (most recent call last): File …"` and **threw away the actual error at the end** (`UnicodeEncodeError: …`). Once a tool exchange was compacted into the ledger, the agent could no longer tell whether the tool succeeded or why it failed (observed: an agent that had run a script and gotten a traceback later saying "I don't see evidence it was executed" and re-running from scratch). The summary now keeps a **head AND a tail** (default length raised 200 → 320), so the conclusion — a shell exit line, a written path, or the exception at the bottom of a traceback — survives compaction. Regression coverage added (a failure's exception text appears in the ledger receipt). - **ContextSift is now budget-aware and recency-preserving.** `ContextManager` keeps the whole conversation inline while under `inlineBudgetChars` (default 16000); above it, it externalizes whole tool exchanges **oldest-first, keeping the most recent tool results inline**. Previously it externalized *all* completed tool exchanges unconditionally — so on a multi-step task the model lost the results it had just produced (e.g. a traceback it needed to fix), breaking the run → read-error → fix → rerun loop (observed: an agent diagnosing a script error, then greeting instead of fixing it). Whole exchanges are evicted together so tool_call/result pairing stays valid; a single huge result still trips the budget and offloads. Regression coverage added (small stays inline; oldest evicted while recent kept; large externalizes). - **Streaming tool calls no longer double-generate.** When a streaming turn already delivers complete tool calls (name present), `executeTurn` now uses them directly instead of re-issuing a non-streaming `complete()` for the arguments. The re-issue still happens for providers that only *signal* tool use without streaming usable args (e.g. some HTTP providers). This fixes in-process providers such as **MLX**, where the re-issue ran a *second* full on-device generation that — at a non-zero temperature — could diverge and return empty/different tool calls, leaving the turn stuck with no output. Also halves the compute per tool turn for local models. The stream's `.toolCall` payloads (previously discarded) are now collected and used. Regression coverage added (an in-process provider that streams a complete tool call drives the loop with zero `complete()` calls). - Bumped the `LLMProviderKit` floor to `0.1.0-alpha.7` (in-process-provider support: overridable `complete`/`stream` + default HTTP hooks). diff --git a/Sources/SwiftAgentKit/Context/ContextManager.swift b/Sources/SwiftAgentKit/Context/ContextManager.swift index e7de399..20c8044 100644 --- a/Sources/SwiftAgentKit/Context/ContextManager.swift +++ b/Sources/SwiftAgentKit/Context/ContextManager.swift @@ -53,7 +53,7 @@ public final class ContextManager: @unchecked Sendable { store: any ArtifactStore = InMemoryArtifactStore(), maxActiveResultChars: Int = 8_000, ledgerEntries: Int = 20, - summaryLength: Int = 200, + summaryLength: Int = 320, inlineBudgetChars: Int = 16_000 ) { self.store = store @@ -205,7 +205,7 @@ public final class ContextManager: @unchecked Sendable { if let cached = cachedReceipt(result.toolCallId) { return cached } let name = result.toolName ?? "tool" - let summary = singleLine(String(result.result.prefix(summaryLength))) + let summary = conclusionSummary(result.result) var artifactIDs: [String] = [] // Don't spill retrieval-tool output to a new artifact — that would nest // artifacts of artifacts and never surface the real content. @@ -275,4 +275,20 @@ public final class ContextManager: @unchecked Sendable { private func singleLine(_ text: String) -> String { text.replacingOccurrences(of: "\n", with: " ").trimmingCharacters(in: .whitespaces) } + + /// A one-line receipt summary that preserves the tool's CONCLUSION. For long + /// output the meaningful result is often at the END (a shell exit line, a + /// written path, or — for a failure — the exception at the bottom of a + /// traceback), so we keep a head AND a tail rather than only the first N + /// characters. This is what lets the agent still know the outcome of a tool + /// call after its raw output has been compacted out of context. + private func conclusionSummary(_ text: String) -> String { + let flat = singleLine(text) + guard flat.count > summaryLength else { return flat } + let headLen = summaryLength / 2 + let tailLen = summaryLength - headLen + let head = String(flat.prefix(headLen)).trimmingCharacters(in: .whitespaces) + let tail = String(flat.suffix(tailLen)).trimmingCharacters(in: .whitespaces) + return "\(head) … \(tail)" + } } diff --git a/Tests/SwiftAgentKitTests/SwiftAgentKitTests.swift b/Tests/SwiftAgentKitTests/SwiftAgentKitTests.swift index d32798d..23fd635 100644 --- a/Tests/SwiftAgentKitTests/SwiftAgentKitTests.swift +++ b/Tests/SwiftAgentKitTests/SwiftAgentKitTests.swift @@ -1725,9 +1725,10 @@ private func firstArtifactID(in text: String) -> String? { let out = await manager.modelMessages(messages) { $0 } - // The full tool output must NOT be resent in any model message… - #expect(out.allSatisfy { !$0.content.contains("DEEP_NEEDLE_END") }) - // …but a ledger + artifact reference must appear in the (single) system block. + // The raw tool output must NOT be resent as an inline tool message (a short + // conclusion snippet in the ledger is fine — and desired). + #expect(!out.contains { $0.role == .tool && $0.content.contains("DEEP_NEEDLE_END") }) + // …and a ledger + artifact reference must appear in the (single) system block. let system = out.first { $0.role == .system } #expect(system != nil) #expect(system?.content.contains("tool ledger") == true) @@ -1793,6 +1794,30 @@ private func firstArtifactID(in text: String) -> String? { #expect(out.allSatisfy { !$0.content.contains("tool ledger") }) } +@Test func testReceiptSummaryCapturesConclusionAtTail() async { + // A failed tool exchange, externalized to the ledger. The ledger receipt must + // preserve the CONCLUSION (the exception at the END of a traceback), not just + // the first N chars — otherwise the agent can't tell what happened. + let manager = ContextManager(inlineBudgetChars: 0) // force externalization + let traceback = "Traceback (most recent call last):\n" + + String(repeating: " File \"build.py\", line 12, in \n", count: 30) + + "UnicodeEncodeError: 'latin-1' codec can't encode character BULLET_CONCLUSION" + let messages: [AgentMessage] = [ + .user("run the build"), + .assistant(content: "", toolCalls: [AgentToolCall(id: "c1", name: "run_shell")]), + .tool(results: [.error(toolCallId: "c1", toolName: "run_shell", message: traceback)]), + .assistant("that failed"), // exchange is completed (not active) + .user("why did it fail?"), + ] + + let out = await manager.modelMessages(messages) { $0 } + let system = out.first { $0.role == .system }?.content ?? "" + + #expect(system.contains("BULLET_CONCLUSION")) // the tail (real error) survived + #expect(system.contains("Traceback")) // the head too + #expect(system.contains("ERROR")) // marked as a failure +} + @Test func testContextManagerEvictsOldestKeepsRecent() async { // Over budget with two completed tool exchanges: the OLD one is externalized // to the ledger, the RECENT tool result stays inline (so an iterative @@ -1812,7 +1837,9 @@ private func firstArtifactID(in text: String) -> String? { let out = await manager.modelMessages(messages) { $0 } #expect(out.contains { $0.role == .tool && $0.content.contains("RECENT_MARKER_kept") }) - #expect(out.allSatisfy { !$0.content.contains("OLDTAIL") }) + // The old exchange is externalized — not kept as an inline tool message + // (a conclusion snippet in the ledger is fine). + #expect(!out.contains { $0.role == .tool && $0.content.contains("OLDTAIL") }) #expect(out.contains { $0.role == .system && $0.content.contains("ledger") }) }