diff --git a/packages/core/src/common/openai-message-converter.ts b/packages/core/src/common/openai-message-converter.ts index 97999045..24ea5abf 100644 --- a/packages/core/src/common/openai-message-converter.ts +++ b/packages/core/src/common/openai-message-converter.ts @@ -276,3 +276,47 @@ export class OpenAIMessageConverter { ); } } + +/** + * Estimate the token count of an OpenAI messages array (chars / 4 heuristic). + * Used as a pre-send budget guard so requests never exceed the model context + * window even when the reported usage (activeTokens) undercounts. (#269) + */ +export function estimateOpenAIMessagesTokens(messages: ChatCompletionMessageParam[]): number { + let chars = 0; + for (const message of messages) { + chars += estimateMessageChars(message); + } + return Math.max(1, Math.ceil(chars / 4)); +} + +function estimateMessageChars(message: ChatCompletionMessageParam): number { + let chars = 0; + const content = message.content; + if (typeof content === "string") { + chars += content.length; + } else if (Array.isArray(content)) { + for (const part of content) { + if (part.type === "text") { + chars += part.text.length; + } else if (part.type === "image_url") { + const url = (part as { image_url?: { url?: unknown } }).image_url?.url; + if (typeof url === "string") { + chars += url.length; + } + } + } + } + + const params = message as unknown as { tool_calls?: unknown[]; tool_call_id?: string; name?: string }; + if (params.name) { + chars += params.name.length; + } + if (params.tool_call_id) { + chars += params.tool_call_id.length; + } + if (Array.isArray(params.tool_calls)) { + chars += JSON.stringify(params.tool_calls).length; + } + return chars; +} diff --git a/packages/core/src/session.ts b/packages/core/src/session.ts index b9252eaa..4f9b6f1f 100644 --- a/packages/core/src/session.ts +++ b/packages/core/src/session.ts @@ -30,6 +30,7 @@ import { import { McpManager } from "./mcp/mcp-manager"; import { getDefaultAutoCompactWindow, + getDefaultContextWindow, type McpServerConfig, type PermissionScope, type PermissionSettings, @@ -54,7 +55,7 @@ import { } from "./common/permissions"; import { clearSessionWorkingDir } from "./tools/bash-handler"; import { reportNewPrompt } from "./common/telemetry"; -import { OpenAIMessageConverter } from "./common/openai-message-converter"; +import { estimateOpenAIMessagesTokens, OpenAIMessageConverter } from "./common/openai-message-converter"; import { supportsMultimodal } from "./common/model-capabilities"; export type { PermissionScope } from "./settings"; @@ -1395,9 +1396,18 @@ ${agentInstructions} } } + let requestMessages = this.prepareSessionMessagesForRequest(this.listSessionMessages(sessionId)); + let messages = this.messageConverter.buildMessages(requestMessages, thinkingEnabled, model); const compactPromptTokenThreshold = this.getResolvedSettings().autoCompactWindow ?? getCompactPromptTokenThreshold(model); - if (session.activeTokens > compactPromptTokenThreshold) { + const estimatedTokens = estimateOpenAIMessagesTokens(messages); + const contextWindow = this.getResolvedSettings().contextWindow ?? getDefaultContextWindow(model); + // Pre-send token budget guard (#269): compact when the reported + // activeTokens exceed the auto-compact window OR when the estimated + // request size would exceed the model context window (the estimate + // catches cases where activeTokens undercounts, e.g. large tool + // outputs stored in history). + if (session.activeTokens > compactPromptTokenThreshold || estimatedTokens > contextWindow) { const message = this.buildAssistantMessage( sessionId, "The conversation is getting long, compacting...", @@ -1406,13 +1416,11 @@ ${agentInstructions} message.meta = { asThinking: true }; this.onAssistantMessage(message, false); await this.compactSession(sessionId, sessionController.signal); + // Rebuild messages after compaction so the request stays within budget. + requestMessages = this.prepareSessionMessagesForRequest(this.listSessionMessages(sessionId)); + messages = this.messageConverter.buildMessages(requestMessages, thinkingEnabled, model); } - const messages = this.messageConverter.buildMessages( - this.prepareSessionMessagesForRequest(this.listSessionMessages(sessionId)), - thinkingEnabled, - model - ); const thinkingOptions = buildThinkingRequestOptions(thinkingEnabled, baseURL, reasoningEffort); const response = await this.createChatCompletionStream( client, @@ -1590,30 +1598,45 @@ ${agentInstructions} const compactPrompt = getCompactPrompt(sessionMessages.slice(startIndex, endIndex)); const thinkingOptions = buildThinkingRequestOptions(thinkingEnabled, baseURL, reasoningEffort); - const response = await this.createChatCompletionStream( - client, - { - model, - ...(temperature !== undefined ? { temperature } : {}), - messages: [{ role: "user", content: compactPrompt }], - ...thinkingOptions, - }, - signal ? { signal } : undefined, - sessionId, - { - enabled: debugLogEnabled, - location: "SessionManager.compactSession", - baseURL, - params: { temperature, thinkingEnabled, reasoningEffort }, + + let compactedSummary = ""; + let responseUsage: ModelUsage | null = null; + try { + const response = await this.createChatCompletionStream( + client, + { + model, + ...(temperature !== undefined ? { temperature } : {}), + messages: [{ role: "user", content: compactPrompt }], + ...thinkingOptions, + }, + signal ? { signal } : undefined, + sessionId, + { + enabled: debugLogEnabled, + location: "SessionManager.compactSession", + baseURL, + params: { temperature, thinkingEnabled, reasoningEffort }, + } + ); + this.throwIfAborted(signal); + const rawLlmResponse = response.choices?.[0]?.message?.content; + const llmResponse = typeof rawLlmResponse === "string" ? rawLlmResponse : ""; + compactedSummary = llmResponse.replace(/[\s\S]*?<\/analysis>/gi, "").trim(); + responseUsage = response.usage ?? null; + } catch (error) { + // Degraded compaction (#269): if the summarization call itself fails + // (e.g. the request already exceeds the model context limit), still + // drop the oldest messages so the session can continue instead of + // deadlocking on repeated 400 errors. + if (signal?.aborted || this.isInterrupted(sessionId)) { + throw error; } - ); - this.throwIfAborted(signal); - const rawLlmResponse = response.choices?.[0]?.message?.content; - const llmResponse = typeof rawLlmResponse === "string" ? rawLlmResponse : ""; - const compactedSummary = llmResponse.replace(/[\s\S]*?<\/analysis>/gi, "").trim(); + compactedSummary = + "(conversation summary unavailable — earlier messages were truncated to stay within the context window.)"; + } const now = new Date().toISOString(); - const responseUsage = response.usage ?? null; this.updateSessionEntry(sessionId, (entry) => ({ ...entry, usage: accumulateUsage(entry.usage, responseUsage), diff --git a/packages/core/src/tests/openai-message-converter.test.ts b/packages/core/src/tests/openai-message-converter.test.ts index a54c213d..f7e900f4 100644 --- a/packages/core/src/tests/openai-message-converter.test.ts +++ b/packages/core/src/tests/openai-message-converter.test.ts @@ -1,6 +1,7 @@ import { test } from "node:test"; import assert from "node:assert/strict"; -import { OpenAIMessageConverter } from "../common/openai-message-converter"; +import type { ChatCompletionMessageParam } from "openai/resources/chat/completions"; +import { estimateOpenAIMessagesTokens, OpenAIMessageConverter } from "../common/openai-message-converter"; import type { SessionMessage } from "../session"; // --------------------------------------------------------------------------- @@ -506,3 +507,41 @@ test("OpenAIMessageConverter.findToolFunction handles null/empty toolCalls", () const toolCalls = [null, undefined, { noId: true }]; assert.equal(c.findToolFunction(toolCalls as unknown[], "call-1"), null); }); + +// --------------------------------------------------------------------------- +// estimateOpenAIMessagesTokens (#269) +// --------------------------------------------------------------------------- + +test("estimateOpenAIMessagesTokens estimates tokens from text content", () => { + const messages = [ + { role: "user", content: "x".repeat(400) }, + { role: "assistant", content: "y".repeat(200) }, + ] as unknown as ChatCompletionMessageParam[]; + assert.equal(estimateOpenAIMessagesTokens(messages), Math.ceil(600 / 4)); +}); + +test("estimateOpenAIMessagesTokens counts image_url data URIs", () => { + const messages = [ + { + role: "user", + content: [ + { type: "text", text: "hi" }, + { type: "image_url", image_url: { url: "data:image/png;base64," + "A".repeat(100) } }, + ], + }, + ] as unknown as ChatCompletionMessageParam[]; + // 2 chars text + 122 chars data-URI ("data:image/png;base64," is 22 chars) => 124 chars + assert.equal(estimateOpenAIMessagesTokens(messages), Math.ceil(124 / 4)); +}); + +test("estimateOpenAIMessagesTokens includes tool call payloads", () => { + const messages = [ + { + role: "assistant", + content: "", + tool_calls: [{ id: "call-1", type: "function", function: { name: "read", arguments: '{"file_path":"/tmp/a"}' } }], + }, + ] as unknown as ChatCompletionMessageParam[]; + const tokens = estimateOpenAIMessagesTokens(messages); + assert.ok(tokens >= 1); +});