diff --git a/docs-site/src/content/docs/guides/providers.md b/docs-site/src/content/docs/guides/providers.md index e1451dcfbd..d1eec12e1a 100644 --- a/docs-site/src/content/docs/guides/providers.md +++ b/docs-site/src/content/docs/guides/providers.md @@ -189,6 +189,15 @@ headers — see [Adapters](/reference/adapters/)). Pool mode overwrites only aut client identity (for example `originator`, session, or thread headers) when the caller did not send them. +For account-switch compatibility diagnosis, enabling provider debug (`ocx debug provider on`) adds +one `[ocx:codex:affinity]` line per canonical ChatGPT forward response. The line contains header +presence, coarse size buckets, process-local HMAC equality tags, safe summaries of known top-level +turn fields, and a count of unknown turn fields. It never includes raw credentials, account ids, +attestation values, thread/session ids, turn metadata, or request bodies; the tags intentionally +change after every proxy restart. Use `ocx debug provider logs -f` while +reproducing the two requests, then run `ocx debug provider off`. This capture is observation-only and +does not strip metadata, retry a request, switch accounts, reset a thread, or otherwise affect routing. + **Diagnostics and reauth.** Human `ocx status` prints an OAuth health block (redacted account ids, no tokens). `ocx doctor` adds an OAuth reliability section with writable-store / single-flight checks and WARN rows that include a recovery Action. When an OAuth provider account needs reauthentication, run diff --git a/src/codex/affinity-debug.ts b/src/codex/affinity-debug.ts new file mode 100644 index 0000000000..f91f6e3369 --- /dev/null +++ b/src/codex/affinity-debug.ts @@ -0,0 +1,162 @@ +/** + * Opt-in Codex affinity diagnostics for account-switch compatibility failures. + * + * The provider debug stream may compare values only within the current process. It never emits + * raw header values, credentials, account identifiers, or durable unsalted hashes. + */ +import { createHmac, randomBytes } from "node:crypto"; +import { isDebugEnabled } from "../lib/debug-settings"; +import { debugProviderDiagnostic } from "../lib/debug"; + +const MAX_TAGGED_VALUE_BYTES = 16 * 1024; +const RUN_KEY = randomBytes(32); +let nextSequence = 0; + +const SAFE_AFFINITY_HEADERS = [ + "openai-beta", + "originator", + "session_id", + "session-id", + "thread-id", + "x-client-request-id", + "x-codex-beta-features", + "x-codex-installation-id", + "x-codex-parent-thread-id", + "x-codex-turn-metadata", + "x-codex-turn-state", + "x-codex-window-id", + "x-openai-subagent", + "x-responsesapi-include-timing-metrics", +] as const; + +const KNOWN_TURN_FIELDS = [ + "forked_from_thread_id", + "parent_thread_id", + "request_kind", + "session_id", + "subagent_kind", + "thread_id", +] as const; + +type SizeBucket = "0" | "1-31" | "32-127" | "128-511" | "512-2047" | "2048-16384" | "oversized"; + +export interface CodexAffinityValueTag { + name: string; + size: SizeBucket; + tag?: string; +} + +export interface CodexAffinityJsonSummary { + shape: "absent" | "malformed" | "oversized" | "array" | "scalar" | "object"; + knownFields?: Array; + unknownFieldCount?: number; +} + +export interface CodexAffinityDiagnosticInput { + inboundHeaders: Headers; + outboundHeaders: HeadersInit; + authKind: "main" | "pool" | "main-pool"; + accountMode: "direct" | "pool" | undefined; + fixedAccount: boolean; + credentialSubstituted: boolean; + accountGatedModel: boolean; + wireModelNormalized: boolean; + status: number; +} + +function sizeBucket(bytes: number): SizeBucket { + if (bytes === 0) return "0"; + if (bytes <= 31) return "1-31"; + if (bytes <= 127) return "32-127"; + if (bytes <= 511) return "128-511"; + if (bytes <= 2047) return "512-2047"; + if (bytes <= MAX_TAGGED_VALUE_BYTES) return "2048-16384"; + return "oversized"; +} + +function valueTag(name: string, value: string): CodexAffinityValueTag { + const bytes = Buffer.byteLength(value, "utf8"); + const size = sizeBucket(bytes); + if (size === "oversized") return { name, size }; + const tag = createHmac("sha256", RUN_KEY) + .update(name) + .update("\0") + .update(value) + .digest("hex") + .slice(0, 12); + return { name, size, tag }; +} + +function headerTags(headers: Headers): CodexAffinityValueTag[] { + const rows: CodexAffinityValueTag[] = []; + for (const name of SAFE_AFFINITY_HEADERS) { + const value = headers.get(name); + if (value !== null) rows.push(valueTag(name, value)); + } + return rows; +} + +function jsonSummary(headers: Headers, name: "x-codex-turn-metadata" | "x-codex-turn-state"): CodexAffinityJsonSummary { + const raw = headers.get(name); + if (raw === null) return { shape: "absent" }; + if (Buffer.byteLength(raw, "utf8") > MAX_TAGGED_VALUE_BYTES) return { shape: "oversized" }; + let parsed: unknown; + try { + parsed = JSON.parse(raw) as unknown; + } catch { + return { shape: "malformed" }; + } + if (Array.isArray(parsed)) return { shape: "array" }; + if (!parsed || typeof parsed !== "object") return { shape: "scalar" }; + const record = parsed as Record; + const knownFields: NonNullable = []; + for (const field of KNOWN_TURN_FIELDS) { + if (!Object.hasOwn(record, field)) continue; + const value = record[field]; + if (value === null) { + knownFields.push({ name: field, kind: "null", size: "0" }); + } else { + const scalarKind = typeof value; + if (scalarKind === "string" || scalarKind === "number" || scalarKind === "boolean") { + knownFields.push({ ...valueTag(`${name}.${field}`, String(value)), name: field, kind: scalarKind }); + continue; + } + knownFields.push({ name: field, kind: Array.isArray(value) ? "array" : "object", size: "0" }); + } + } + const known = new Set(KNOWN_TURN_FIELDS); + const unknownFieldCount = Object.keys(record).filter(key => !known.has(key)).length; + return { + shape: "object", + ...(knownFields.length > 0 ? { knownFields } : {}), + ...(unknownFieldCount > 0 ? { unknownFieldCount } : {}), + }; +} + +/** Emit one observation-only, privacy-bounded provider-debug record. */ +export function captureCodexAffinityDiagnostic(input: CodexAffinityDiagnosticInput): void { + if (!isDebugEnabled()) return; + try { + const outbound = new Headers(input.outboundHeaders); + debugProviderDiagnostic("codex", "affinity", { + sequence: ++nextSequence, + authKind: input.authKind, + accountMode: input.accountMode ?? "unset", + fixedAccount: input.fixedAccount, + credentialSubstituted: input.credentialSubstituted, + accountGatedModel: input.accountGatedModel, + wireModelNormalized: input.wireModelNormalized, + status: input.status, + inbound: headerTags(input.inboundHeaders), + outbound: headerTags(outbound), + inboundTurnMetadata: jsonSummary(input.inboundHeaders, "x-codex-turn-metadata"), + outboundTurnMetadata: jsonSummary(outbound, "x-codex-turn-metadata"), + inboundTurnState: jsonSummary(input.inboundHeaders, "x-codex-turn-state"), + outboundTurnState: jsonSummary(outbound, "x-codex-turn-state"), + }); + } catch { + // Diagnostics must never affect request handling. + } +} + +export const CODEX_AFFINITY_DEBUG_SAFE_HEADERS = SAFE_AFFINITY_HEADERS; diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index 3c7773e4e4..87c0722b11 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -115,6 +115,7 @@ import { resolveCodexModelEntitlements, } from "../../codex/model-entitlements"; import { ACCOUNT_GATED_NATIVE_OPENAI_MODELS } from "../../codex/catalog/native-models"; +import { captureCodexAffinityDiagnostic } from "../../codex/affinity-debug"; import { computeQuotaCooldown, formatCodexProviderForLog, @@ -521,6 +522,11 @@ interface CodexPoolAccountRetryArgs { connectMs: number; passthroughEstimate?: number; stream: boolean; + onResponse?: ( + response: Response, + authCtx: Extract, + request: Awaited["buildRequest"]>>, + ) => void; } type CodexPoolAccountRetryResult = @@ -761,6 +767,7 @@ async function retryCodexPoolOnAlternateAccount( route.provider.authMode === "forward", ); retrySendCount += 1; + args.onResponse?.(upstreamResponse, retryAuthCtx, request); if (!retrySameConfirmedAccount || retrySendCount >= maxRetrySends) break; if (!await shouldRetryCodexPoolAccountModel400( upstreamResponse, @@ -1141,7 +1148,7 @@ function unreadableEncryptedAgentTaskResponse(): Response { } type ResponsesAuthResolution = - | { ok: true; authCtx: CodexAuthContext; headers: Headers } + | { ok: true; authCtx: CodexAuthContext; headers: Headers; substituteMainCredential: boolean } | { ok: false; response: Response }; /** @@ -1205,6 +1212,7 @@ async function resolveResponsesCodexAuth( ok: true, authCtx, headers: materializeCodexUpstreamAuth(req.headers, authCtx, { substituteMainCredential }), + substituteMainCredential, }; } catch (err) { if (err instanceof CodexAccountCooldownError) { @@ -2217,11 +2225,13 @@ async function handleResponsesInner( pendingHostAdmissionLease = admission.lease; } + let substituteMainCredential = false; { const finalAuth = await resolveResponsesCodexAuth(req, config, route, options); if (!finalAuth.ok) return finalAuth.response; authCtx = finalAuth.authCtx; selectedForwardHeaders = finalAuth.headers; + substituteMainCredential = finalAuth.substituteMainCredential; } route.provider = applyCodexAuthContextToProvider(route.provider, authCtx, route.codexAccountMode); @@ -2884,6 +2894,29 @@ async function handleResponsesInner( } } + const captureAffinityResponse = ( + response: Response, + captureAuthCtx: CodexAuthContext = authCtx, + captureRequest: Awaited> = request, + credentialSubstituted = substituteMainCredential + || captureAuthCtx.kind === "pool" + || captureAuthCtx.kind === "main-pool", + ): void => { + if (!isCanonicalOpenAiForwardProvider(route.provider)) return; + captureCodexAffinityDiagnostic({ + inboundHeaders: req.headers, + outboundHeaders: captureRequest.headers, + authKind: captureAuthCtx.kind, + accountMode: route.codexAccountMode, + fixedAccount: isFixedCodexAccount(captureAuthCtx), + credentialSubstituted, + accountGatedModel: ACCOUNT_GATED_NATIVE_OPENAI_MODELS.has(route.modelId), + wireModelNormalized: parsed.modelId !== route.modelId, + status: response.status, + }); + }; + captureAffinityResponse(upstreamResponse); + if (usesCodexForwardPoolAuth(authCtx, route.provider)) { let poolRetryOutcome: number | undefined; if (await shouldRetryCodexPoolAccountModel400( @@ -2912,6 +2945,9 @@ async function handleResponsesInner( connectMs, passthroughEstimate, stream: parsed.stream, + onResponse: (response, retryAuthCtx, retryRequest) => { + captureAffinityResponse(response, retryAuthCtx, retryRequest, true); + }, }); if (retry.kind === "transport") { authCtx = retry.authCtx; diff --git a/structure/08_openai-provider-tiers.md b/structure/08_openai-provider-tiers.md index a43325674b..21cef4e954 100644 --- a/structure/08_openai-provider-tiers.md +++ b/structure/08_openai-provider-tiers.md @@ -183,6 +183,27 @@ preserving a stale one would block every later migration. usage, model visibility, subagent state, and injection state retain the selected virtual id. - Compact preserves provider/selected identity but sends the base model without a reasoning object. +## Process-local affinity diagnostics + +Provider debug capture includes one `[ocx:codex:affinity]` record for each canonical ChatGPT +forward response before account-model retry selection. The record compares only an explicit safe +header-name allowlist. Values are represented by size buckets and 12-character HMAC equality tags +under a random process-local key; raw credentials, account ids, attestation values, thread/session +ids, turn metadata, and request bodies never enter the record. Known top-level turn-metadata fields +use the same process-local tags, while unknown fields contribute only a count. Oversized values are +classified without hashing. The diagnostic is observational: it cannot strip headers, retry, +switch accounts, reset threads, or mutate affinity. + +```text +[Decision Log] +- 목적과 의도: Identify which combined Codex affinity values survive a Plus-to-K12 credential substitution without collecting private thread or account data. +- 기존 구현 및 제약 조건: Pool auth intentionally copies the curated caller metadata and replaces only authorization plus chatgpt-account-id. Individual header probes did not reproduce the workspace denial, while raw captures would expose account-bound identifiers. +- 검토한 주요 대안: Delete all affinity metadata; log raw values; persist ordinary hashes; perform automatic header-ablation retries; or emit process-local keyed equality evidence only when provider debug is enabled. +- 선택한 방식: Emit bounded pre-stream diagnostics with a random per-process HMAC key, a fixed non-credential header allowlist, known turn-field summaries, and no request mutation. +- 다른 대안 대신 이 방식을 선택한 이유: Equality across two requests in one run is enough to narrow the incompatible combination; process-local HMACs prevent durable correlation and make offline guessing useless, while observation-only capture cannot change production semantics. +- 장점, 단점 및 영향: Maintainers can compare a Plus success and exact-K12 denial safely. Tags cannot be compared across restarts, and the diagnostic does not itself identify an upstream policy rule or fix the rejection. +``` + ## Account identity and store concurrency Pool mode needs stable public names and a store that survives concurrent refresh: diff --git a/tests/codex-affinity-debug.test.ts b/tests/codex-affinity-debug.test.ts new file mode 100644 index 0000000000..42221f95b2 --- /dev/null +++ b/tests/codex-affinity-debug.test.ts @@ -0,0 +1,138 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { + captureCodexAffinityDiagnostic, + CODEX_AFFINITY_DEBUG_SAFE_HEADERS, +} from "../src/codex/affinity-debug"; +import { getDebugLogEntries, resetDebugLogBufferForTests } from "../src/lib/debug-log-buffer"; +import { resetDebugSettingsForTests, setDebugSettings } from "../src/lib/debug-settings"; + +function diagnosticPayload(): Record { + const line = getDebugLogEntries().at(-1)?.line ?? ""; + const prefix = "[ocx:codex:affinity] "; + expect(line.startsWith(prefix)).toBe(true); + return JSON.parse(line.slice(prefix.length)) as Record; +} + +beforeEach(() => { + resetDebugSettingsForTests(); + resetDebugLogBufferForTests(); +}); + +afterEach(() => { + resetDebugSettingsForTests(); + resetDebugLogBufferForTests(); +}); + +describe("Codex affinity diagnostics", () => { + test("stays silent unless provider debug is explicitly enabled", () => { + captureCodexAffinityDiagnostic({ + inboundHeaders: new Headers({ session_id: "private-session" }), + outboundHeaders: new Headers({ session_id: "private-session" }), + authKind: "pool", + accountMode: "pool", + fixedAccount: false, + credentialSubstituted: true, + accountGatedModel: false, + wireModelNormalized: false, + status: 200, + }); + expect(getDebugLogEntries()).toEqual([]); + }); + + test("emits only process-local equality tags and bounded known turn-field summaries", () => { + setDebugSettings({ debug: true }); + const turnMetadata = JSON.stringify({ + session_id: "session-private", + thread_id: "thread-private", + parent_thread_id: "parent-private", + forked_from_thread_id: "fork-private", + request_kind: "turn", + private_unknown_key: "must-not-appear", + }); + captureCodexAffinityDiagnostic({ + inboundHeaders: new Headers({ + authorization: "Bearer inbound-secret", + "chatgpt-account-id": "account-inbound-secret", + "x-oai-attestation": "attestation-secret", + session_id: "session-private", + "x-codex-turn-metadata": turnMetadata, + }), + outboundHeaders: new Headers({ + authorization: "Bearer outbound-secret", + "chatgpt-account-id": "account-outbound-secret", + "x-oai-attestation": "attestation-secret", + session_id: "session-private", + "x-codex-turn-metadata": turnMetadata, + }), + authKind: "pool", + accountMode: "pool", + fixedAccount: true, + credentialSubstituted: true, + accountGatedModel: true, + wireModelNormalized: false, + status: 403, + }); + + const line = getDebugLogEntries().at(-1)?.line ?? ""; + for (const secret of [ + "inbound-secret", + "outbound-secret", + "account-inbound-secret", + "account-outbound-secret", + "attestation-secret", + "session-private", + "thread-private", + "parent-private", + "fork-private", + "must-not-appear", + ]) expect(line).not.toContain(secret); + + const payload = diagnosticPayload(); + expect(payload).toMatchObject({ + authKind: "pool", + accountMode: "pool", + fixedAccount: true, + credentialSubstituted: true, + accountGatedModel: true, + wireModelNormalized: false, + status: 403, + }); + const inbound = payload.inbound as Array<{ name: string; tag?: string }>; + const outbound = payload.outbound as Array<{ name: string; tag?: string }>; + expect(inbound.map(row => row.name)).toEqual(["session_id", "x-codex-turn-metadata"]); + expect(outbound.map(row => row.name)).toEqual(["session_id", "x-codex-turn-metadata"]); + expect(inbound[0]?.tag).toBe(outbound[0]?.tag); + expect(payload.inboundTurnMetadata).toMatchObject({ shape: "object", unknownFieldCount: 1 }); + }); + + test("different values receive different tags and oversized values receive no digest", () => { + setDebugSettings({ debug: true }); + captureCodexAffinityDiagnostic({ + inboundHeaders: new Headers({ session_id: "first" }), + outboundHeaders: new Headers({ session_id: "second", "x-codex-turn-state": "x".repeat(16 * 1024 + 1) }), + authKind: "main", + accountMode: "direct", + fixedAccount: false, + credentialSubstituted: false, + accountGatedModel: false, + wireModelNormalized: false, + status: 200, + }); + const payload = diagnosticPayload(); + const inbound = payload.inbound as Array<{ name: string; tag?: string }>; + const outbound = payload.outbound as Array<{ name: string; size: string; tag?: string }>; + expect(inbound.find(row => row.name === "session_id")?.tag) + .not.toBe(outbound.find(row => row.name === "session_id")?.tag); + expect(outbound.find(row => row.name === "x-codex-turn-state")).toEqual({ + name: "x-codex-turn-state", + size: "oversized", + }); + expect(payload.outboundTurnState).toEqual({ shape: "oversized" }); + }); + + test("the diagnostic allowlist never includes credential or attestation headers", () => { + expect(CODEX_AFFINITY_DEBUG_SAFE_HEADERS).not.toContain("authorization"); + expect(CODEX_AFFINITY_DEBUG_SAFE_HEADERS).not.toContain("chatgpt-account-id"); + expect(CODEX_AFFINITY_DEBUG_SAFE_HEADERS).not.toContain("x-oai-attestation"); + }); +}); diff --git a/tests/codex-envkey-admission-substitution.test.ts b/tests/codex-envkey-admission-substitution.test.ts index 08ab22245c..0ee133c5ff 100644 --- a/tests/codex-envkey-admission-substitution.test.ts +++ b/tests/codex-envkey-admission-substitution.test.ts @@ -5,6 +5,8 @@ import { join } from "node:path"; import { saveConfig } from "../src/config"; import { startServer } from "../src/server"; import type { OcxConfig } from "../src/types"; +import { getDebugLogEntries, resetDebugLogBufferForTests } from "../src/lib/debug-log-buffer"; +import { resetDebugSettingsForTests, setDebugSettings } from "../src/lib/debug-settings"; /** * #1686 end to end: a Codex client injected with `env_key` presents the proxy admission @@ -68,6 +70,8 @@ beforeEach(() => { process.env.OPENCODEX_HOME = ocxHome; process.env.CODEX_HOME = codexHome; delete process.env.OPENCODEX_API_AUTH_TOKEN; + resetDebugSettingsForTests(); + resetDebugLogBufferForTests(); upstreamAuth = []; globalThis.fetch = (async (input, init) => { const raw = input instanceof Request ? input.url : String(input); @@ -83,6 +87,8 @@ beforeEach(() => { afterEach(() => { globalThis.fetch = originalFetch; + resetDebugSettingsForTests(); + resetDebugLogBufferForTests(); if (previousOcxHome === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = previousOcxHome; if (previousCodexHome === undefined) delete process.env.CODEX_HOME; @@ -105,6 +111,7 @@ async function postResponses(url: string | URL, authorization: string): Promise< describe("#1686 env_key bearer admission reaches Direct with substitution", () => { test("an admission bearer is served and the stored main credential goes upstream", async () => { + setDebugSettings({ debug: true }); saveConfig(directConfig()); const stored = liveJwt(); writeStoredMain(stored); @@ -119,6 +126,16 @@ describe("#1686 env_key bearer admission reaches Direct with substitution", () = expect(upstreamAuth).toEqual([`Bearer ${stored}`]); // The proof that matters: our own secret never reached the wire. expect(upstreamAuth.join("|")).not.toContain(ADMISSION_SECRET); + const affinityLine = getDebugLogEntries() + .map(entry => entry.line) + .find(line => line.startsWith("[ocx:codex:affinity] ")); + expect(affinityLine).toBeDefined(); + expect(JSON.parse(affinityLine!.slice("[ocx:codex:affinity] ".length))).toMatchObject({ + authKind: "main", + accountMode: "direct", + credentialSubstituted: true, + status: 200, + }); } finally { await server.stop(true); } @@ -156,4 +173,3 @@ describe("#1686 env_key bearer admission reaches Direct with substitution", () = } }); }); - diff --git a/tests/server-auth.test.ts b/tests/server-auth.test.ts index cf50d5250d..c3577540e8 100644 --- a/tests/server-auth.test.ts +++ b/tests/server-auth.test.ts @@ -45,6 +45,8 @@ import { configuredAdminToken } from "../src/lib/admin-secrets"; import { SYSTEM_RESTART_CAPABILITY_VERSION } from "../src/lib/system-restart-contract"; import { LOCAL_PROVIDER_RELOAD_CAPABILITY_VERSION } from "../src/lib/local-provider-reload-contract"; import { resetCodexModelEntitlementCacheForTests } from "../src/codex/model-entitlements"; +import { getDebugLogEntries, resetDebugLogBufferForTests } from "../src/lib/debug-log-buffer"; +import { resetDebugSettingsForTests, setDebugSettings } from "../src/lib/debug-settings"; import { watchdogMs } from "./helpers/ci-watchdog"; const previousApiToken = process.env.OPENCODEX_API_AUTH_TOKEN; @@ -143,6 +145,8 @@ afterEach(() => { clearAccountNeedsReauth("pool-b"); clearAccountQuota(); resetCodexModelEntitlementCacheForTests(); + resetDebugSettingsForTests(); + resetDebugLogBufferForTests(); if (existsSync(TEST_DIR)) rmSync(TEST_DIR, { recursive: true }); }); @@ -2107,6 +2111,7 @@ describe("server local API auth", () => { }); test("Activation A: allow-listed 400 retries once on another eligible pool account", async () => { + setDebugSettings({ debug: true }); const harness = await startPoolRetryHarness(accountId => accountId === "acct-pool-a" ? rejectionResponse(unsupportedModelBody()) : Response.json({ id: "retry-success", status: "completed", output: [] })); @@ -2118,6 +2123,18 @@ describe("server local API auth", () => { expect(getCodexUpstreamHealth("pool-a")).toBeNull(); expect(getCodexUpstreamHealth("pool-b")).toBeNull(); expect(harness.config.activeCodexAccountId).toBe("pool-a"); + const affinity = getDebugLogEntries() + .map(entry => entry.line) + .filter(line => line.startsWith("[ocx:codex:affinity] ")) + .map(line => JSON.parse(line.slice("[ocx:codex:affinity] ".length)) as { + status: number; + authKind: string; + credentialSubstituted: boolean; + }); + expect(affinity).toEqual([ + expect.objectContaining({ status: 400, authKind: "pool", credentialSubstituted: true }), + expect.objectContaining({ status: 200, authKind: "pool", credentialSubstituted: true }), + ]); } finally { await stopPoolRetryHarness(harness); }