From b5d8537c21bde80c63183927b5c4bac496d78925 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Wed, 12 Aug 2026 21:37:22 +0200 Subject: [PATCH 1/5] test(agents): cover routed fallback after recovery --- tests/agent-task-recovery-fallback.test.ts | 65 ++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 tests/agent-task-recovery-fallback.test.ts diff --git a/tests/agent-task-recovery-fallback.test.ts b/tests/agent-task-recovery-fallback.test.ts new file mode 100644 index 0000000000..7d7043de13 --- /dev/null +++ b/tests/agent-task-recovery-fallback.test.ts @@ -0,0 +1,65 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { + noteSubagentModelFailure, + resetSubagentModelFallbackStateForTests, +} from "../src/codex/subagent-model-fallback"; +import { resetAgentTaskRecoveryState } from "../src/server/responses/agent-task-recovery"; +import { + codexHeaders, + encryptedInput, + originalFetch, + post, + providerResponse, + recoverySse, + routedConfig, +} from "./helpers/agent-task-recovery"; + +describe("agent task recovery fallback routing", () => { + beforeEach(() => { + resetAgentTaskRecoveryState(); + resetSubagentModelFallbackStateForTests(); + }); + + afterEach(() => { + globalThis.fetch = originalFetch; + resetAgentTaskRecoveryState(); + resetSubagentModelFallbackStateForTests(); + }); + + test("reapplies routed fallback after encrypted task recovery", async () => { + const config = routedConfig(); + config.subagentModelFallback = ["xai/grok-4.6"]; + noteSubagentModelFailure("xai/grok-4.5", "429", config); + + const fetchedUrls: string[] = []; + const providerModels: string[] = []; + globalThis.fetch = (async (input, init) => { + const url = String(input); + fetchedUrls.push(url); + if (url.includes("chatgpt.com")) { + return new Response(recoverySse("Dispatch this recovered task through the healthy fallback."), { + status: 200, + headers: { "content-type": "text/event-stream" }, + }); + } + + const raw = typeof init?.body === "string" ? init.body : "{}"; + const body = JSON.parse(raw) as { model?: string }; + providerModels.push(body.model ?? ""); + return providerResponse(); + }) as typeof fetch; + + const response = await post( + config, + "xai/grok-4.5", + encryptedInput(), + codexHeaders(), + ); + + expect(response.status).toBe(200); + expect(fetchedUrls).toHaveLength(2); + expect(fetchedUrls[0]).toContain("chatgpt.com/backend-api/codex"); + expect(fetchedUrls[1]).toContain("api.x.ai"); + expect(providerModels).toEqual(["grok-4.6"]); + }); +}); From 0aead199e91a79732bd027c0f317507563c0fecb Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Wed, 12 Aug 2026 21:45:24 +0200 Subject: [PATCH 2/5] chore: apply recovery fallback fix --- .../_maintainer-patch-recovery-fallback.yml | 112 ++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 .github/workflows/_maintainer-patch-recovery-fallback.yml diff --git a/.github/workflows/_maintainer-patch-recovery-fallback.yml b/.github/workflows/_maintainer-patch-recovery-fallback.yml new file mode 100644 index 0000000000..c43efc52fe --- /dev/null +++ b/.github/workflows/_maintainer-patch-recovery-fallback.yml @@ -0,0 +1,112 @@ +name: Maintainer recovery fallback patch + +on: + push: + branches: + - fix/recovery-routed-fallback + +permissions: + contents: write + +jobs: + patch: + if: ${{ github.actor == 'Wibias' }} + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + with: + ref: fix/recovery-routed-fallback + fetch-depth: 0 + - name: Apply focused fix + shell: bash + run: | + python - <<'PY' + from pathlib import Path + + path = Path('src/server/responses/core.ts') + text = path.read_text() + + old = ''' let selectedForwardHeaders = req.headers; + let subagentFallbackAccountId = config.activeCodexAccountId ?? null; + let subagentQuotaFailureModel = parsed.modelId; + ''' + new = ''' let selectedForwardHeaders = req.headers; + let subagentFallbackAccountId = config.activeCodexAccountId ?? null; + let subagentFallbackPreviewAccountId: string | null | undefined; + let subagentQuotaFailureModel = parsed.modelId; + ''' + if old not in text: + raise SystemExit('fallback state anchor not found') + text = text.replace(old, new, 1) + + old = ''' subagentFallbackAccountId = previewAccountId ?? config.activeCodexAccountId ?? null; + const fallback = applySubagentModelFallback( + ''' + new = ''' subagentFallbackPreviewAccountId = previewAccountId; + subagentFallbackAccountId = previewAccountId ?? config.activeCodexAccountId ?? null; + const fallback = applySubagentModelFallback( + ''' + if old not in text: + raise SystemExit('preview account anchor not found') + text = text.replace(old, new, 1) + + old = ''' markBodyNonPersistable(parsed._rawBody); + toolBridgeMaps = buildToolBridgeMaps(parsed, translatorBudget); + ''' + new = ''' markBodyNonPersistable(parsed._rawBody); + + // The ciphertext-only pass intentionally excludes routed candidates. Once recovery + // makes the assignment readable, run selection again with the full configured chain + // and keep the route in sync with any newly selected fallback. + const fallback = applySubagentModelFallback( + parsed, + req.headers, + config, + subagentFallbackPreviewAccountId, + Date.now(), + false, + previewSelectionOptions, + ); + if (fallback) { + (logCtx as unknown as Record).subagentModelFallbackFrom = fallback.from; + (logCtx as unknown as Record).subagentModelFallbackTo = fallback.to; + if (isInjectionDebugEnabled()) { + injectionDebugLog(`[opencodex] subagent model fallback ${fallback.from} -> ${fallback.to}`); + } + } + subagentQuotaFailureModel = fallback?.to ?? parsed.modelId; + + if (fallback?.to && !slugsEquivalent(fallback.to, route.modelId)) { + try { + route = routeModel(config, fallback.to, evidenceFromBody(parsed._rawBody)); + logCtx.routeDecision = route.routeDecision; + } catch (err) { + if (err instanceof NoAvailableComboTargetsError) { + return comboUnavailableResponse(err.message); + } + if (err instanceof NoEligiblePolicyCandidateError) { + logCtx.routeDecision = err.trace; + } + return formatErrorResponse( + 404, + "invalid_request_error", + err instanceof Error ? err.message : String(err), + ); + } + } + toolBridgeMaps = buildToolBridgeMaps(parsed, translatorBudget); + ''' + if old not in text: + raise SystemExit('recovery reparse anchor not found') + text = text.replace(old, new, 1) + + path.write_text(text) + PY + + rm .github/workflows/_maintainer-patch-recovery-fallback.yml + git config user.name "Wibias" + git config user.email "37517432+Wibias@users.noreply.github.com" + git add src/server/responses/core.ts .github/workflows/_maintainer-patch-recovery-fallback.yml + git commit -m "fix(agents): reroute recovered tasks through fallback" + git push origin HEAD:fix/recovery-routed-fallback From 11c4c554961ac122383cc146126d36ad30a8fdfe Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Wed, 12 Aug 2026 21:46:32 +0200 Subject: [PATCH 3/5] chore: retry recovery fallback patch --- .../_maintainer-patch-recovery-fallback.yml | 136 +++++++++--------- 1 file changed, 64 insertions(+), 72 deletions(-) diff --git a/.github/workflows/_maintainer-patch-recovery-fallback.yml b/.github/workflows/_maintainer-patch-recovery-fallback.yml index c43efc52fe..6ff84abdc9 100644 --- a/.github/workflows/_maintainer-patch-recovery-fallback.yml +++ b/.github/workflows/_maintainer-patch-recovery-fallback.yml @@ -24,82 +24,74 @@ jobs: python - <<'PY' from pathlib import Path - path = Path('src/server/responses/core.ts') + path = Path("src/server/responses/core.ts") text = path.read_text() - old = ''' let selectedForwardHeaders = req.headers; - let subagentFallbackAccountId = config.activeCodexAccountId ?? null; - let subagentQuotaFailureModel = parsed.modelId; - ''' - new = ''' let selectedForwardHeaders = req.headers; - let subagentFallbackAccountId = config.activeCodexAccountId ?? null; - let subagentFallbackPreviewAccountId: string | null | undefined; - let subagentQuotaFailureModel = parsed.modelId; - ''' - if old not in text: - raise SystemExit('fallback state anchor not found') - text = text.replace(old, new, 1) + state_anchor = " let subagentFallbackAccountId = config.activeCodexAccountId ?? null;" + if text.count(state_anchor) != 1: + raise SystemExit("fallback state anchor not unique") + text = text.replace( + state_anchor, + state_anchor + "\n let subagentFallbackPreviewAccountId: string | null | undefined;", + 1, + ) - old = ''' subagentFallbackAccountId = previewAccountId ?? config.activeCodexAccountId ?? null; - const fallback = applySubagentModelFallback( - ''' - new = ''' subagentFallbackPreviewAccountId = previewAccountId; - subagentFallbackAccountId = previewAccountId ?? config.activeCodexAccountId ?? null; - const fallback = applySubagentModelFallback( - ''' - if old not in text: - raise SystemExit('preview account anchor not found') - text = text.replace(old, new, 1) + preview_anchor = " subagentFallbackAccountId = previewAccountId ?? config.activeCodexAccountId ?? null;" + if text.count(preview_anchor) != 1: + raise SystemExit("preview account anchor not unique") + text = text.replace( + preview_anchor, + " subagentFallbackPreviewAccountId = previewAccountId;\n" + preview_anchor, + 1, + ) - old = ''' markBodyNonPersistable(parsed._rawBody); - toolBridgeMaps = buildToolBridgeMaps(parsed, translatorBudget); - ''' - new = ''' markBodyNonPersistable(parsed._rawBody); - - // The ciphertext-only pass intentionally excludes routed candidates. Once recovery - // makes the assignment readable, run selection again with the full configured chain - // and keep the route in sync with any newly selected fallback. - const fallback = applySubagentModelFallback( - parsed, - req.headers, - config, - subagentFallbackPreviewAccountId, - Date.now(), - false, - previewSelectionOptions, - ); - if (fallback) { - (logCtx as unknown as Record).subagentModelFallbackFrom = fallback.from; - (logCtx as unknown as Record).subagentModelFallbackTo = fallback.to; - if (isInjectionDebugEnabled()) { - injectionDebugLog(`[opencodex] subagent model fallback ${fallback.from} -> ${fallback.to}`); - } - } - subagentQuotaFailureModel = fallback?.to ?? parsed.modelId; - - if (fallback?.to && !slugsEquivalent(fallback.to, route.modelId)) { - try { - route = routeModel(config, fallback.to, evidenceFromBody(parsed._rawBody)); - logCtx.routeDecision = route.routeDecision; - } catch (err) { - if (err instanceof NoAvailableComboTargetsError) { - return comboUnavailableResponse(err.message); - } - if (err instanceof NoEligiblePolicyCandidateError) { - logCtx.routeDecision = err.trace; - } - return formatErrorResponse( - 404, - "invalid_request_error", - err instanceof Error ? err.message : String(err), - ); - } - } - toolBridgeMaps = buildToolBridgeMaps(parsed, translatorBudget); - ''' - if old not in text: - raise SystemExit('recovery reparse anchor not found') - text = text.replace(old, new, 1) + recovery_anchor = " markBodyNonPersistable(parsed._rawBody);" + if text.count(recovery_anchor) != 1: + raise SystemExit("recovery reparse anchor not unique") + recovery_lines = [ + recovery_anchor, + "", + " // The ciphertext-only pass intentionally excludes routed candidates. Once recovery", + " // makes the assignment readable, run selection again with the full configured chain", + " // and keep the route in sync with any newly selected fallback.", + " const fallback = applySubagentModelFallback(", + " parsed,", + " req.headers,", + " config,", + " subagentFallbackPreviewAccountId,", + " Date.now(),", + " false,", + " previewSelectionOptions,", + " );", + " if (fallback) {", + " (logCtx as unknown as Record).subagentModelFallbackFrom = fallback.from;", + " (logCtx as unknown as Record).subagentModelFallbackTo = fallback.to;", + " if (isInjectionDebugEnabled()) {", + " injectionDebugLog(`[opencodex] subagent model fallback ${fallback.from} -> ${fallback.to}`);", + " }", + " }", + " subagentQuotaFailureModel = fallback?.to ?? parsed.modelId;", + "", + " if (fallback?.to && !slugsEquivalent(fallback.to, route.modelId)) {", + " try {", + " route = routeModel(config, fallback.to, evidenceFromBody(parsed._rawBody));", + " logCtx.routeDecision = route.routeDecision;", + " } catch (err) {", + " if (err instanceof NoAvailableComboTargetsError) {", + " return comboUnavailableResponse(err.message);", + " }", + " if (err instanceof NoEligiblePolicyCandidateError) {", + " logCtx.routeDecision = err.trace;", + " }", + " return formatErrorResponse(", + " 404,", + " \"invalid_request_error\",", + " err instanceof Error ? err.message : String(err),", + " );", + " }", + " }", + ] + text = text.replace(recovery_anchor, "\n".join(recovery_lines), 1) path.write_text(text) PY From 2b773b7497211589f976d1dbb56d0dddd522d754 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Wed, 12 Aug 2026 19:47:04 +0000 Subject: [PATCH 4/5] fix(agents): reroute recovered tasks through fallback --- .../_maintainer-patch-recovery-fallback.yml | 104 ------------------ src/server/responses/core.ts | 42 +++++++ 2 files changed, 42 insertions(+), 104 deletions(-) delete mode 100644 .github/workflows/_maintainer-patch-recovery-fallback.yml diff --git a/.github/workflows/_maintainer-patch-recovery-fallback.yml b/.github/workflows/_maintainer-patch-recovery-fallback.yml deleted file mode 100644 index 6ff84abdc9..0000000000 --- a/.github/workflows/_maintainer-patch-recovery-fallback.yml +++ /dev/null @@ -1,104 +0,0 @@ -name: Maintainer recovery fallback patch - -on: - push: - branches: - - fix/recovery-routed-fallback - -permissions: - contents: write - -jobs: - patch: - if: ${{ github.actor == 'Wibias' }} - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 - with: - ref: fix/recovery-routed-fallback - fetch-depth: 0 - - name: Apply focused fix - shell: bash - run: | - python - <<'PY' - from pathlib import Path - - path = Path("src/server/responses/core.ts") - text = path.read_text() - - state_anchor = " let subagentFallbackAccountId = config.activeCodexAccountId ?? null;" - if text.count(state_anchor) != 1: - raise SystemExit("fallback state anchor not unique") - text = text.replace( - state_anchor, - state_anchor + "\n let subagentFallbackPreviewAccountId: string | null | undefined;", - 1, - ) - - preview_anchor = " subagentFallbackAccountId = previewAccountId ?? config.activeCodexAccountId ?? null;" - if text.count(preview_anchor) != 1: - raise SystemExit("preview account anchor not unique") - text = text.replace( - preview_anchor, - " subagentFallbackPreviewAccountId = previewAccountId;\n" + preview_anchor, - 1, - ) - - recovery_anchor = " markBodyNonPersistable(parsed._rawBody);" - if text.count(recovery_anchor) != 1: - raise SystemExit("recovery reparse anchor not unique") - recovery_lines = [ - recovery_anchor, - "", - " // The ciphertext-only pass intentionally excludes routed candidates. Once recovery", - " // makes the assignment readable, run selection again with the full configured chain", - " // and keep the route in sync with any newly selected fallback.", - " const fallback = applySubagentModelFallback(", - " parsed,", - " req.headers,", - " config,", - " subagentFallbackPreviewAccountId,", - " Date.now(),", - " false,", - " previewSelectionOptions,", - " );", - " if (fallback) {", - " (logCtx as unknown as Record).subagentModelFallbackFrom = fallback.from;", - " (logCtx as unknown as Record).subagentModelFallbackTo = fallback.to;", - " if (isInjectionDebugEnabled()) {", - " injectionDebugLog(`[opencodex] subagent model fallback ${fallback.from} -> ${fallback.to}`);", - " }", - " }", - " subagentQuotaFailureModel = fallback?.to ?? parsed.modelId;", - "", - " if (fallback?.to && !slugsEquivalent(fallback.to, route.modelId)) {", - " try {", - " route = routeModel(config, fallback.to, evidenceFromBody(parsed._rawBody));", - " logCtx.routeDecision = route.routeDecision;", - " } catch (err) {", - " if (err instanceof NoAvailableComboTargetsError) {", - " return comboUnavailableResponse(err.message);", - " }", - " if (err instanceof NoEligiblePolicyCandidateError) {", - " logCtx.routeDecision = err.trace;", - " }", - " return formatErrorResponse(", - " 404,", - " \"invalid_request_error\",", - " err instanceof Error ? err.message : String(err),", - " );", - " }", - " }", - ] - text = text.replace(recovery_anchor, "\n".join(recovery_lines), 1) - - path.write_text(text) - PY - - rm .github/workflows/_maintainer-patch-recovery-fallback.yml - git config user.name "Wibias" - git config user.email "37517432+Wibias@users.noreply.github.com" - git add src/server/responses/core.ts .github/workflows/_maintainer-patch-recovery-fallback.yml - git commit -m "fix(agents): reroute recovered tasks through fallback" - git push origin HEAD:fix/recovery-routed-fallback diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index 7bd5cb1074..af4caad913 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -1627,6 +1627,7 @@ async function handleResponsesInner( }; let selectedForwardHeaders = req.headers; let subagentFallbackAccountId = config.activeCodexAccountId ?? null; + let subagentFallbackPreviewAccountId: string | null | undefined; let subagentQuotaFailureModel = parsed.modelId; const parentThreadId = req.headers.get("x-codex-parent-thread-id")?.trim() ?? null; @@ -1652,6 +1653,7 @@ async function handleResponsesInner( undefined, previewSelectionOptions, ); + subagentFallbackPreviewAccountId = previewAccountId; subagentFallbackAccountId = previewAccountId ?? config.activeCodexAccountId ?? null; const fallback = applySubagentModelFallback( parsed, @@ -1737,6 +1739,46 @@ async function handleResponsesInner( // text. Bar it from the continuation cache before any recording path can reach it — // that cache is persisted to disk, which would defeat the recovery cache's TTL. markBodyNonPersistable(parsed._rawBody); + + // The ciphertext-only pass intentionally excludes routed candidates. Once recovery + // makes the assignment readable, run selection again with the full configured chain + // and keep the route in sync with any newly selected fallback. + const fallback = applySubagentModelFallback( + parsed, + req.headers, + config, + subagentFallbackPreviewAccountId, + Date.now(), + false, + previewSelectionOptions, + ); + if (fallback) { + (logCtx as unknown as Record).subagentModelFallbackFrom = fallback.from; + (logCtx as unknown as Record).subagentModelFallbackTo = fallback.to; + if (isInjectionDebugEnabled()) { + injectionDebugLog(`[opencodex] subagent model fallback ${fallback.from} -> ${fallback.to}`); + } + } + subagentQuotaFailureModel = fallback?.to ?? parsed.modelId; + + if (fallback?.to && !slugsEquivalent(fallback.to, route.modelId)) { + try { + route = routeModel(config, fallback.to, evidenceFromBody(parsed._rawBody)); + logCtx.routeDecision = route.routeDecision; + } catch (err) { + if (err instanceof NoAvailableComboTargetsError) { + return comboUnavailableResponse(err.message); + } + if (err instanceof NoEligiblePolicyCandidateError) { + logCtx.routeDecision = err.trace; + } + return formatErrorResponse( + 404, + "invalid_request_error", + err instanceof Error ? err.message : String(err), + ); + } + } toolBridgeMaps = buildToolBridgeMaps(parsed, translatorBudget); } catch { unreadableEncryptedAgentTask = true; From 1f0d57690b769015e3665a71df623d5a2fd0f8eb Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Wed, 12 Aug 2026 21:48:17 +0200 Subject: [PATCH 5/5] test(agents): clarify recovered fallback regression --- tests/agent-task-recovery-fallback.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/agent-task-recovery-fallback.test.ts b/tests/agent-task-recovery-fallback.test.ts index 7d7043de13..62b7053c2a 100644 --- a/tests/agent-task-recovery-fallback.test.ts +++ b/tests/agent-task-recovery-fallback.test.ts @@ -26,7 +26,7 @@ describe("agent task recovery fallback routing", () => { resetSubagentModelFallbackStateForTests(); }); - test("reapplies routed fallback after encrypted task recovery", async () => { + test("routes a recovered task through the healthy routed fallback", async () => { const config = routedConfig(); config.subagentModelFallback = ["xai/grok-4.6"]; noteSubagentModelFailure("xai/grok-4.5", "429", config);