diff --git a/src/adapters/openai-chat.ts b/src/adapters/openai-chat.ts index 74c9042219..8275a5f3de 100644 --- a/src/adapters/openai-chat.ts +++ b/src/adapters/openai-chat.ts @@ -289,10 +289,20 @@ function invalidChoicesEvent(usage?: OcxUsage): Extract { +function invalidToolCallsEvent( + rawToolCalls: unknown, + mode: "stream" | "response", + usage?: OcxUsage, +): Extract { + const diagnostic = diagnoseInvalidToolCalls(rawToolCalls, mode); + const detail = diagnostic + ? ` (${diagnostic.reason}${diagnostic.callIndex !== undefined ? `; callIndex=${diagnostic.callIndex}` : ""}; valueType=${diagnostic.valueType})` + : ""; return { type: "error", - message: "upstream response contained invalid tool calls", + status: 502, + errorType: "upstream_error", + message: `upstream response contained invalid tool calls${detail}`, ...(usage !== undefined ? { usage } : {}), }; } @@ -333,6 +343,83 @@ type InvalidToolCallReason = | "tool_call_function_name_blank" | "tool_call_function_arguments_invalid"; +type InvalidToolCallDiagnostic = { + reason: InvalidToolCallReason; + callIndex?: number; + valueType: string; +}; + +type InvalidFieldShape = + | { + kind: "object"; + knownKeys: string[]; + knownFieldTypes: Record; + hasUnknownKeys: boolean; + } + | { + kind: "array"; + length: number; + }; + +const SAFE_TOOL_CALL_SHAPE_KEYS = [ + "name", + "type", + "value", + "function", + "arguments", + "id", + "index", +] as const; +const SAFE_TOOL_CALL_SHAPE_KEY_SET = new Set(SAFE_TOOL_CALL_SHAPE_KEYS); + +function structuralValueType(value: unknown): string { + return value === null ? "null" : Array.isArray(value) ? "array" : typeof value; +} + +function invalidToolCallField(rawToolCalls: unknown, diagnostic: InvalidToolCallDiagnostic): unknown { + if (diagnostic.reason === "tool_calls_not_array") return rawToolCalls; + if (!Array.isArray(rawToolCalls) || diagnostic.callIndex === undefined) return undefined; + + const rawToolCall = rawToolCalls[diagnostic.callIndex]; + if (diagnostic.reason === "tool_call_not_object") return rawToolCall; + if (!isRecord(rawToolCall)) return undefined; + if (diagnostic.reason === "tool_call_function_not_object") return rawToolCall.function; + + const rawFunction = rawToolCall.function; + switch (diagnostic.reason) { + case "tool_call_id_invalid": + return rawToolCall.id; + case "tool_call_function_name_invalid": + return isRecord(rawFunction) ? rawFunction.name : undefined; + case "tool_call_function_arguments_invalid": + return isRecord(rawFunction) ? rawFunction.arguments : undefined; + default: + return undefined; + } +} + +function fingerprintInvalidField(value: unknown): InvalidFieldShape | undefined { + if (Array.isArray(value)) return { kind: "array", length: value.length }; + if (!isRecord(value)) return undefined; + + const knownKeys: string[] = []; + const knownFieldTypes: Record = {}; + for (const key of SAFE_TOOL_CALL_SHAPE_KEYS) { + if (!Object.hasOwn(value, key)) continue; + knownKeys.push(key); + knownFieldTypes[key] = structuralValueType(value[key]); + } + + let hasUnknownKeys = false; + for (const key of Object.keys(value)) { + if (!SAFE_TOOL_CALL_SHAPE_KEY_SET.has(key)) { + hasUnknownKeys = true; + break; + } + } + return { kind: "object", knownKeys, knownFieldTypes, hasUnknownKeys }; +} + /** * Streamed string fields are absent when null or undefined (#1731): OpenAI-compatible * streamers repeat already-sent `id`/`name`/`arguments` as null on continuation deltas. @@ -350,7 +437,7 @@ function isInvalidStreamStringField(value: unknown): boolean { function diagnoseInvalidToolCalls( rawToolCalls: unknown, mode: "stream" | "response", -): { reason: InvalidToolCallReason; callIndex?: number; valueType: string } | undefined { +): InvalidToolCallDiagnostic | undefined { if (!Array.isArray(rawToolCalls)) { return { reason: "tool_calls_not_array", valueType: rawToolCalls === null ? "null" : typeof rawToolCalls }; } @@ -426,8 +513,15 @@ function diagnoseInvalidToolCalls( } function logInvalidToolCalls(mode: "stream" | "response", rawToolCalls: unknown): void { + if (!isDebugEnabled()) return; const diagnostic = diagnoseInvalidToolCalls(rawToolCalls, mode); - if (diagnostic) debugProviderDiagnostic("openai-chat", "invalid-tool-calls", { mode, ...diagnostic }); + if (!diagnostic) return; + const fieldShape = fingerprintInvalidField(invalidToolCallField(rawToolCalls, diagnostic)); + debugProviderDiagnostic("openai-chat", "invalid-tool-calls", { + mode, + ...diagnostic, + ...(fieldShape ? { fieldShape } : {}), + }); } function developerSystemText(message: OcxMessage): string | undefined { @@ -1184,7 +1278,6 @@ export function createOpenAIChatAdapter(provider: OcxProviderConfig): ProviderAd buildRequest(parsed: OcxParsedRequest) { const { url, headers, hasCredential } = openAIChatTransport(provider); - const messages = messagesToChatFormat(parsed, provider); const tools = toolsToChatFormatForProvider(parsed, provider); const toolChoice = toolChoiceToChatFormat(parsed.options.toolChoice, parsed.context.tools, provider); @@ -1479,12 +1572,12 @@ export function createOpenAIChatAdapter(provider: OcxProviderConfig): ProviderAd // tolerated as absent because OpenAI-compatible providers may emit it as stream padding. if (!Array.isArray(rawToolCalls)) { logInvalidToolCalls("stream", rawToolCalls); - return yield* terminateWithError(invalidToolCallsEvent(pendingUsage)); + return yield* terminateWithError(invalidToolCallsEvent(rawToolCalls, "stream", pendingUsage)); } for (const rawToolCall of rawToolCalls) { if (!isRecord(rawToolCall)) { logInvalidToolCalls("stream", rawToolCalls); - return yield* terminateWithError(invalidToolCallsEvent(pendingUsage)); + return yield* terminateWithError(invalidToolCallsEvent(rawToolCalls, "stream", pendingUsage)); } const tc = rawToolCall as { index?: number; @@ -1499,7 +1592,7 @@ export function createOpenAIChatAdapter(provider: OcxProviderConfig): ProviderAd if (rawFunction !== undefined && rawFunction !== null) { if (!isRecord(rawFunction)) { logInvalidToolCalls("stream", rawToolCalls); - return yield* terminateWithError(invalidToolCallsEvent(pendingUsage)); + return yield* terminateWithError(invalidToolCallsEvent(rawToolCalls, "stream", pendingUsage)); } const rawName = rawFunction.name; const rawArguments = rawFunction.arguments; @@ -1508,12 +1601,12 @@ export function createOpenAIChatAdapter(provider: OcxProviderConfig): ProviderAd // non-string value still fails closed before entering the accumulator. if (isInvalidStreamStringField(rawName) || isInvalidStreamStringField(rawArguments)) { logInvalidToolCalls("stream", rawToolCalls); - return yield* terminateWithError(invalidToolCallsEvent(pendingUsage)); + return yield* terminateWithError(invalidToolCallsEvent(rawToolCalls, "stream", pendingUsage)); } } if (isInvalidStreamStringField(tc.id)) { logInvalidToolCalls("stream", rawToolCalls); - return yield* terminateWithError(invalidToolCallsEvent(pendingUsage)); + return yield* terminateWithError(invalidToolCallsEvent(rawToolCalls, "stream", pendingUsage)); } const key = typeof tc.index === "number" ? `i:${tc.index}` @@ -1675,12 +1768,12 @@ export function createOpenAIChatAdapter(provider: OcxProviderConfig): ProviderAd if (rawToolCalls !== undefined && rawToolCalls !== null) { if (!Array.isArray(rawToolCalls)) { logInvalidToolCalls("response", rawToolCalls); - return [invalidToolCallsEvent(usage)]; + return [invalidToolCallsEvent(rawToolCalls, "response", usage)]; } for (const rawToolCall of rawToolCalls) { if (!isRecord(rawToolCall) || !isRecord(rawToolCall.function)) { logInvalidToolCalls("response", rawToolCalls); - return [invalidToolCallsEvent(usage)]; + return [invalidToolCallsEvent(rawToolCalls, "response", usage)]; } const id = rawToolCall.id; const name = rawToolCall.function.name; @@ -1691,7 +1784,7 @@ export function createOpenAIChatAdapter(provider: OcxProviderConfig): ProviderAd if (typeof id !== "string" || typeof name !== "string" || typeof args !== "string" || name.trim().length === 0) { logInvalidToolCalls("response", rawToolCalls); - return [invalidToolCallsEvent(usage)]; + return [invalidToolCallsEvent(rawToolCalls, "response", usage)]; } events.push({ type: "tool_call_start", id, name }); events.push({ type: "tool_call_delta", arguments: args }); diff --git a/tests/openai-chat-hardening.test.ts b/tests/openai-chat-hardening.test.ts index e8675c7983..ab731ab35b 100644 --- a/tests/openai-chat-hardening.test.ts +++ b/tests/openai-chat-hardening.test.ts @@ -244,18 +244,20 @@ describe("openai-chat non-stream response hardening", () => { test("rejects malformed nested tool calls without throwing", async () => { const adapter = createOpenAIChatAdapter(provider()); - for (const toolCalls of [ - { unexpected: true }, - [null], - [{ id: "call_missing_function" }], - ]) { + for (const [toolCalls, message] of [ + [{ unexpected: true }, "upstream response contained invalid tool calls (tool_calls_not_array; valueType=object)"], + [[null], "upstream response contained invalid tool calls (tool_call_not_object; callIndex=0; valueType=null)"], + [[{ id: "call_missing_function" }], "upstream response contained invalid tool calls (tool_call_function_not_object; callIndex=0; valueType=undefined)"], + ] as const) { const events = await adapter.parseResponse!(new Response(JSON.stringify({ choices: [{ message: { role: "assistant", tool_calls: toolCalls } }], usage: { prompt_tokens: 7, completion_tokens: 2 }, }))); expect(events).toEqual([{ type: "error", - message: "upstream response contained invalid tool calls", + status: 502, + errorType: "upstream_error", + message, usage: { inputTokens: 7, outputTokens: 2 }, }]); } @@ -272,7 +274,12 @@ describe("openai-chat non-stream response hardening", () => { }] } }], }))); - expect(events).toEqual([{ type: "error", message: "upstream response contained invalid tool calls" }]); + expect(events).toEqual([{ + type: "error", + status: 502, + errorType: "upstream_error", + message: "upstream response contained invalid tool calls (tool_call_function_arguments_invalid; callIndex=0; valueType=object)", + }]); const lines = getDebugLogEntries().map(entry => entry.line).join("\n"); expect(lines).toContain("[ocx:openai-chat:invalid-tool-calls]"); expect(lines).toContain('"mode":"response"'); @@ -393,7 +400,10 @@ describe("openai-chat stream response hardening", () => { test("malformed nested streaming tool calls are terminal errors", async () => { const adapter = createOpenAIChatAdapter(provider()); - for (const toolCalls of [{ unexpected: true }, [null]]) { + for (const [toolCalls, message] of [ + [{ unexpected: true }, "upstream response contained invalid tool calls (tool_calls_not_array; valueType=object)"], + [[null], "upstream response contained invalid tool calls (tool_call_not_object; callIndex=0; valueType=null)"], + ] as const) { const response = new Response([ `data: ${JSON.stringify({ choices: [{ delta: { tool_calls: toolCalls } }], @@ -405,7 +415,9 @@ describe("openai-chat stream response hardening", () => { const events = await collect(adapter.parseStream(response)); expect(events).toEqual([{ type: "error", - message: "upstream response contained invalid tool calls", + status: 502, + errorType: "upstream_error", + message, usage: { inputTokens: 7, outputTokens: 2 }, }]); } @@ -424,7 +436,12 @@ describe("openai-chat stream response hardening", () => { ].join("")); const events = await collect(adapter.parseStream(response)); - expect(events).toEqual([{ type: "error", message: "upstream response contained invalid tool calls" }]); + expect(events).toEqual([{ + type: "error", + status: 502, + errorType: "upstream_error", + message: "upstream response contained invalid tool calls (tool_call_not_object; callIndex=1; valueType=null)", + }]); const lines = getDebugLogEntries().map(entry => entry.line).join("\n"); expect(lines).toContain('"mode":"stream"'); expect(lines).toContain('"reason":"tool_call_not_object"'); @@ -446,7 +463,12 @@ describe("openai-chat stream response hardening", () => { ].join("")); const events = await collect(adapter.parseStream(response)); - expect(events).toEqual([{ type: "error", message: "upstream response contained invalid tool calls" }]); + expect(events).toEqual([{ + type: "error", + status: 502, + errorType: "upstream_error", + message: "upstream response contained invalid tool calls (tool_call_not_object; callIndex=1; valueType=null)", + }]); const lines = getDebugLogEntries().map(entry => entry.line).join("\n"); // The null-padded continuation delta at index 0 is accepted by the accumulator, so the // diagnostic must point at index 1 rather than claiming the padding was the defect. diff --git a/tests/openai-chat-invalid-tool-call-diagnostics.test.ts b/tests/openai-chat-invalid-tool-call-diagnostics.test.ts new file mode 100644 index 0000000000..4a8704ab48 --- /dev/null +++ b/tests/openai-chat-invalid-tool-call-diagnostics.test.ts @@ -0,0 +1,166 @@ +import { afterEach, beforeEach, expect, test } from "bun:test"; +import { createOpenAIChatAdapter as createOpenAIChatAdapterProduction } from "../src/adapters/openai-chat"; +import { getDebugLogEntries, resetDebugLogBufferForTests } from "../src/lib/debug-log-buffer"; +import { resetDebugSettingsForTests } from "../src/lib/debug-settings"; +import type { AdapterEvent, OcxProviderConfig } from "../src/types"; +import { withTestTranslatorBudget } from "./helpers/translator-budget"; + +const createOpenAIChatAdapter = (...args: Parameters) => + withTestTranslatorBudget(createOpenAIChatAdapterProduction(...args)); + +const provider: OcxProviderConfig = { + adapter: "openai-chat", + baseUrl: "https://example.test/v1", + apiKey: "sk-test", + authMode: "key", +}; + +const previousDebug = process.env.OCX_DEBUG; + +beforeEach(() => { + resetDebugSettingsForTests(); + resetDebugLogBufferForTests(); + delete process.env.OCX_DEBUG; +}); + +afterEach(() => { + resetDebugSettingsForTests(); + resetDebugLogBufferForTests(); + if (previousDebug === undefined) delete process.env.OCX_DEBUG; + else process.env.OCX_DEBUG = previousDebug; +}); + +async function collect(stream: AsyncGenerator): Promise { + const events: AdapterEvent[] = []; + for await (const event of stream) events.push(event); + return events; +} + +function requireErrorEvent(events: AdapterEvent[]): Extract { + const event = events.find((candidate): candidate is Extract => + candidate.type === "error"); + if (!event) throw new Error("expected an error event"); + return event; +} + +function malformedNameResponse(name: unknown, args = "{}"): Response { + return new Response( + `data: ${JSON.stringify({ + choices: [{ + index: 0, + delta: { + tool_calls: [{ + index: 0, + id: "call_1", + function: { name, arguments: args }, + }], + }, + }], + })}\n\n`, + ); +} + +test("object-valued streamed function.name is a 502 with value-free compatibility detail", async () => { + const privateNameValue = "must-not-reach-diagnostics"; + const privateArguments = "must-not-reach-diagnostics-either"; + const adapter = createOpenAIChatAdapter(provider); + const events = await collect(adapter.parseStream(malformedNameResponse( + { privateNameValue }, + JSON.stringify({ value: privateArguments }), + ))); + + expect(events).toEqual([{ + type: "error", + status: 502, + errorType: "upstream_error", + message: "upstream response contained invalid tool calls (tool_call_function_name_invalid; callIndex=0; valueType=object)", + }]); + const serialized = JSON.stringify(events); + expect(serialized).not.toContain(privateNameValue); + expect(serialized).not.toContain(privateArguments); + expect(serialized).not.toContain("call_1"); + expect(getDebugLogEntries()).toEqual([]); +}); + +test("provider debug fingerprints only allowlisted object structure for an invalid field", async () => { + process.env.OCX_DEBUG = "1"; + const privateNestedValue = "must-not-reach-provider-debug"; + const privateUnknownKey = "must-not-reach-provider-debug-as-a-key"; + const privateUnknownValue = "must-not-reach-provider-debug-as-a-value"; + const privateArguments = "must-not-reach-provider-debug-arguments"; + const adapter = createOpenAIChatAdapter(provider); + + const events = await collect(adapter.parseStream(malformedNameResponse({ + name: privateNestedValue, + [privateUnknownKey]: privateUnknownValue, + }, JSON.stringify({ privateArguments })))); + + expect(requireErrorEvent(events).message).not.toContain("fieldShape"); + const lines = getDebugLogEntries().map(entry => entry.line).join("\n"); + expect(lines).toContain('[ocx:openai-chat:invalid-tool-calls]'); + expect(lines).toContain('"reason":"tool_call_function_name_invalid"'); + expect(lines).toContain('"fieldShape":{"kind":"object"'); + expect(lines).toContain('"knownKeys":["name"]'); + expect(lines).toContain('"knownFieldTypes":{"name":"string"}'); + expect(lines).toContain('"hasUnknownKeys":true'); + expect(lines).not.toContain(privateNestedValue); + expect(lines).not.toContain(privateUnknownKey); + expect(lines).not.toContain(privateUnknownValue); + expect(lines).not.toContain(privateArguments); + expect(lines).not.toContain("call_1"); +}); + +test("provider debug fingerprints invalid arrays by length without retaining elements", async () => { + process.env.OCX_DEBUG = "1"; + const privateElement = "must-not-reach-array-fingerprint"; + const adapter = createOpenAIChatAdapter(provider); + + const events = await collect(adapter.parseStream(malformedNameResponse([privateElement, { privateElement }]))); + + expect(requireErrorEvent(events).message).not.toContain("fieldShape"); + const lines = getDebugLogEntries().map(entry => entry.line).join("\n"); + expect(lines).toContain('"fieldShape":{"kind":"array","length":2}'); + expect(lines).not.toContain(privateElement); +}); + +test("provider debug fingerprints buffered invalid fields with the same privacy boundary", async () => { + process.env.OCX_DEBUG = "1"; + const privateNestedValue = "must-not-reach-buffered-provider-debug"; + const privateUnknownKey = "must-not-reach-buffered-provider-debug-as-key"; + const privateUnknownValue = "must-not-reach-buffered-provider-debug-as-value"; + const privateArguments = "must-not-reach-buffered-provider-debug-arguments"; + const adapter = createOpenAIChatAdapter(provider); + + const events = await adapter.parseResponse!(new Response(JSON.stringify({ + choices: [{ + message: { + role: "assistant", + tool_calls: [{ + id: "call_buffered_private", + type: "function", + function: { + name: { + name: privateNestedValue, + [privateUnknownKey]: privateUnknownValue, + }, + arguments: JSON.stringify({ privateArguments }), + }, + }], + }, + }], + }))); + + expect(requireErrorEvent(events).message).not.toContain("fieldShape"); + const lines = getDebugLogEntries().map(entry => entry.line).join("\n"); + expect(lines).toContain('"mode":"response"'); + expect(lines).toContain('"reason":"tool_call_function_name_invalid"'); + expect(lines).toContain('"fieldShape":{"kind":"object"'); + expect(lines).toContain('"knownKeys":["name"]'); + expect(lines).toContain('"knownFieldTypes":{"name":"string"}'); + expect(lines).toContain('"hasUnknownKeys":true'); + expect(lines).not.toContain(privateNestedValue); + expect(lines).not.toContain(privateUnknownKey); + expect(lines).not.toContain(privateUnknownValue); + expect(lines).not.toContain(privateArguments); + expect(lines).not.toContain("call_buffered_private"); +});