fix(responses): routed-destination sanitize + opaque-state recovery series (#2254 rebased)

src/adapters/base.ts

@@ -68,10 +68,12 @@ export interface AdapterRequest {
     method: string;
     headers: Record<string, string>;
     body: string;
-    /** Custom-tool names actually lowered to upstream function calls while building this request. */
+    /** Final upstream wire names of custom tools lowered to functions while building this request. */
     convertedRoutedCustomToolNames?: ReadonlySet<string>;
     /** Client tool-search names actually lowered to upstream function calls for this request. */
     convertedRoutedToolSearchNames?: ReadonlySet<string>;
+    /** Upstream-only aliases for namespace tools flattened in this request. */
+    convertedRoutedNamespaceToolAliases?: ReadonlyMap<string, { namespace: string; name: string }>;
     /** Releases observation of a serialized request body after its final fetch attempt settles. */
     releaseBodyObservation?: () => void;
     /** Exact reasoning parameter emitted by the adapter, for request-log diagnostics only. */

src/adapters/openai-responses.ts

@@ -2,16 +2,22 @@ import { createHash } from "node:crypto";
 import type { IncomingMeta, ProviderAdapter } from "./base";
 import { namespacedToolName, type AdapterEvent, type OcxParsedRequest, type OcxProviderConfig, type OcxUsage, type TierDecision } from "../types";
 import { catalogModelSupportsReasoningSummaries } from "../codex/catalog";
-import { COMPACT_PROMPT, decodeCompactionSummary, SUMMARY_PREFIX } from "../responses/compaction";
+import { COMPACT_PROMPT, compactionItemToText, decodeCompactionSummary, isCompactionItemType } from "../responses/compaction";
 import { collectResponsesToolGroups } from "../responses/tool-groups";
 import { isHostedToolUnsupportedForModel } from "../responses/hosted-tool-policy";
 import { decodeServerSentEvents } from "../lib/sse-decoder";
-import { CODEX_FORWARD_BASE_URL, isCanonicalOpenAiForwardProvider } from "../providers/openai-tiers";
+import {
+  CODEX_FORWARD_BASE_URL,
+  destinationDecodesNativeCompactionBlob,
+  isCanonicalOpenAiForwardProvider,
+  isOpenAiOperatedResponsesDestination,
+} from "../providers/openai-tiers";
 import { OCX_REASONING_PREFIX } from "../responses/reasoning-envelope";
 import { modelRecordValue } from "../reasoning-effort";
 import type { TranslatorBudget } from "../lib/translator-budget";
 import { rewriteRoutedCustomToolsForUpstream } from "../responses/custom-tool-compat";
 import { rewriteRoutedToolSearchForUpstream } from "../responses/tool-search-compat";
+import { rewriteRoutedNamespaceToolsForUpstream } from "../responses/namespace-tool-compat";
 import { openaiResponsesUrl } from "./openai-responses-url";
 import {
   createAdapterTierMetadata,
@@ -41,7 +47,11 @@ export const FORWARD_HEADERS = [
 
 export function sanitizeReasoningInputContent(
   body: unknown,
-  opts?: { preserveRawReasoningContent?: boolean },
+  opts?: {
+    preserveRawReasoningContent?: boolean;
+    dropNullContentChannel?: boolean;
+    stripEncryptedContent?: boolean;
+  },
 ): unknown {
   if (!body || typeof body !== "object" || Array.isArray(body)) return body;
   const raw = body as Record<string, unknown>;
@@ -56,24 +66,49 @@ export function sanitizeReasoningInputContent(
     // ocxr1 envelopes are proxy-minted (Anthropic signatures), not OpenAI encryption — the native
     // backend cannot decrypt them and would reject the request. Strip regardless of content shape.
     const hasOcxEnvelope = typeof rec.encrypted_content === "string" && rec.encrypted_content.startsWith(OCX_REASONING_PREFIX);
-    if (!hasRawContent && !hasOcxEnvelope) return item;
-    if (hasOcxEnvelope) {
-      changed = true;
-      const next: Record<string, unknown> = { ...rec };
-      delete next.encrypted_content;
-      if (!opts?.preserveRawReasoningContent) next.content = [];
-      return next;
+    const hasOutputStatus = Object.prototype.hasOwnProperty.call(rec, "status");
+    const hasEncryptedContent = Object.prototype.hasOwnProperty.call(rec, "encrypted_content");
+    const stripEncryptedContent = hasOcxEnvelope
+      || (opts?.stripEncryptedContent === true && hasEncryptedContent);
+    // Codex serializes an absent reasoning content channel as `"content": null`. The field is
+    // optional and null carries nothing, but a strict gateway rejects the item on its declared type
+    // — xAI answers `Could not decode the compaction blob`, naming the sibling `encrypted_content`
+    // rather than the field it actually refused, which is why this reads as a blob failure. Drop the
+    // key so the item matches the shape the upstream issued.
+    //
+    // Gated to routed destinations. An OpenAI-operated backend binds the blob to the item's exact
+    // shape, so deleting a field there invalidates it (`The encrypted content ... could not be
+    // verified`); the two requirements are exactly opposed, and a live regression proved it. That
+    // gate is also why this drop may touch an item that keeps its blob: xAI demonstrably accepts its
+    // own blob without the null channel, and the destinations that bind blobs to item shape never
+    // reach this branch. This is independent of the output-only status removal below.
+    const dropNullContentChannel = opts?.dropNullContentChannel === true
+      && "content" in rec && !Array.isArray(rec.content);
+    // `status` is output-only. Measured OpenAI reasoning items never contain it, and Grok accepts
+    // its own encrypted_content with status removed. Keeping a foreign status beside a retained
+    // blob makes OpenAI reject the field before blob validation, starving the provenance recovery
+    // of the opaque-blob error it needs. Content blanking remains the separate pre-existing rule.
+    const stripOutputStatus = hasOutputStatus;
+    const blankContent = !dropNullContentChannel
+      && !opts?.preserveRawReasoningContent
+      && (hasRawContent || hasOcxEnvelope);
+    if (!blankContent && !stripOutputStatus && !stripEncryptedContent && !dropNullContentChannel) {
+      return item;
     }
+    changed = true;
+    const next: Record<string, unknown> = { ...rec };
+    if (dropNullContentChannel) delete next.content;
+    if (stripOutputStatus) delete next.status;
+    if (stripEncryptedContent) delete next.encrypted_content;
     // Routed models can produce raw `reasoning_text` output items. Codex echoes those in later
     // native GPT requests, but ChatGPT's Responses backend accepts reasoning input only with empty
     // `content`; keep summaries/ids and drop the raw content so native passthrough does not 400.
     // DeepSeek's Responses API instead ACCEPTS plaintext reasoning replay (its compatibility
     // guide merges reasoning items into the adjacent assistant message), so providers flagged
     // `preserveResponsesReasoningContent` keep it — deleting valid replay content there breaks
     // continuations after tool calls (issue #875 family).
-    if (opts?.preserveRawReasoningContent) return item;
-    changed = true;
-    return { ...rec, content: [] };
+    if (blankContent) next.content = [];
+    return next;
   });
 
   return changed ? { ...raw, input } : body;
@@ -120,6 +155,66 @@ function stripInvalidItemIds(body: unknown): unknown {
   return changed ? { ...body, input } : body;
 }
 
+/**
+ * Codex-private tool fields that only the ChatGPT backend understands.
+ *
+ * A third-party Responses gateway validates its schema and rejects the whole request before
+ * inference — xAI answers `Argument not supported: external_web_access` — so these are removed at
+ * the noncanonical boundary while the tool and every public option stay.
+ *
+ * Keep this a table. Each private bit Codex attaches has so far arrived as its own bespoke strip
+ * with its own traversal, and the traversals disagreed about which containers they covered; a new
+ * one should be a row here instead. `toolTypes` omitted means the field is private on any tool.
+ */
+const CANONICAL_ONLY_TOOL_FIELDS: readonly { field: string; toolTypes?: ReadonlySet<string> }[] = [
+  // ChatGPT's browsing policy bit. The public hosted tool is enabled by its presence alone.
+  { field: "external_web_access", toolTypes: new Set(["web_search", "web_search_preview"]) },
+  // Deferred-discovery marker. `activateDeferredTool` clears it only for tools a `tool_search_output`
+  // already loaded, so a still-deferred declaration — including one promoted out of a namespace
+  // group — otherwise reaches the wire carrying it.
+  { field: "defer_loading" },
+];
+
+function stripCanonicalOnlyToolFields(body: unknown): unknown {
+  if (!isPlainObject(body)) return body;
+
+  const rewriteTools = (tools: unknown[]): unknown[] => {
+    let changed = false;
+    const rewritten = tools.map(tool => {
+      if (!isPlainObject(tool)) return tool;
+      let next = tool;
+      for (const { field, toolTypes } of CANONICAL_ONLY_TOOL_FIELDS) {
+        if (!Object.hasOwn(next, field)) continue;
+        if (toolTypes && (typeof next.type !== "string" || !toolTypes.has(next.type))) continue;
+        const { [field]: _private, ...rest } = next;
+        next = rest;
+      }
+      if (next === tool) return tool;
+      changed = true;
+      return next;
+    });
+    return changed ? rewritten : tools;
+  };
+
+  let rewrittenBody = body;
+  if (Array.isArray(body.tools)) {
+    const tools = rewriteTools(body.tools);
+    if (tools !== body.tools) rewrittenBody = { ...rewrittenBody, tools };
+  }
+  if (!Array.isArray(body.input)) return rewrittenBody;
+
+  let input: unknown[] | undefined;
+  for (let index = 0; index < body.input.length; index += 1) {
+    const item = body.input[index];
+    if (!isPlainObject(item) || item.type !== "additional_tools" || !Array.isArray(item.tools)) continue;
+    const tools = rewriteTools(item.tools);
+    if (tools === item.tools) continue;
+    input ??= [...body.input];
+    input[index] = { ...item, tools };
+  }
+  return input ? { ...rewrittenBody, input } : rewrittenBody;
+}
+
 /**
  * When `store` is false, the upstream API does not persist response items. Any item ID
  * forwarded in `input` is then interpreted as a reference to a stored item that does not
@@ -143,25 +238,41 @@ function stripItemIdsWhenUnstored(body: unknown): unknown {
 }
 
 /**
- * Replace proxy-minted compaction items (`encrypted_content` starting with `ocx1:`) with plain
- * user messages before forwarding to the ChatGPT backend. Our envelope is transparent base64, not
- * OpenAI encryption — the native backend cannot decrypt it and would reject the request. Real
- * OpenAI-encrypted compaction items are forwarded untouched.
+ * Normalize replayed compaction items for the destination backend.
+ *
+ * A compaction item carries an `encrypted_content` blob the client replays verbatim on every later
+ * turn, and only the backend that minted it can decode it. Proxy-minted `ocx1:` envelopes are
+ * transparent base64 rather than encryption, so no upstream can read them and they always become
+ * plain user messages. Native blobs have multiple possible minters, so a destination's ability to
+ * decode its own blobs does not make a blob from a previous serving identity portable. On a known
+ * identity mismatch the blob degrades to the same note the bridged parser uses, even when the
+ * destination normally accepts native blobs. Without a known mismatch, the destination capability
+ * keeps the existing behavior.
+ *
+ * A bare `context_compaction` marker carries no blob and is forwarded untouched.
  */
-function scrubOcxCompactionItems(body: unknown): unknown {
+function scrubOcxCompactionItems(
+  body: unknown,
+  destinationDecodesNativeBlob: boolean,
+  threadServingIdentityChanged: boolean,
+): unknown {
   if (!isPlainObject(body) || !Array.isArray(body.input)) return body;
 
   let changed = false;
   const input = body.input.map(item => {
-    if (!isPlainObject(item)) return item;
-    if (item.type !== "compaction" && item.type !== "compaction_summary" && item.type !== "context_compaction") return item;
-    const decoded = typeof item.encrypted_content === "string" ? decodeCompactionSummary(item.encrypted_content) : null;
-    if (decoded === null) return item;
+    if (!isPlainObject(item) || !isCompactionItemType(item.type)) return item;
+    const encrypted = typeof item.encrypted_content === "string" ? item.encrypted_content : undefined;
+    if (encrypted === undefined) return item;
+    if (
+      decodeCompactionSummary(encrypted) === null
+      && destinationDecodesNativeBlob
+      && !threadServingIdentityChanged
+    ) return item;
     changed = true;
     return {
       type: "message",
       role: "user",
-      content: [{ type: "input_text", text: `${SUMMARY_PREFIX}\n\n${decoded}` }],
+      content: [{ type: "input_text", text: compactionItemToText(encrypted) }],
     };
   });
 
@@ -1530,6 +1641,7 @@ export function createResponsesPassthroughAdapter(provider: OcxProviderConfig):
       const forward = provider.authMode === "forward";
       let convertedRoutedCustomToolNames: Set<string> | undefined;
       let convertedRoutedToolSearchNames: Set<string> | undefined;
+      let convertedRoutedNamespaceToolAliases: Map<string, { namespace: string; name: string }> | undefined;
       const unexpandedMiss = !!parsed.previousResponseId && parsed._previousResponseInputExpanded !== true;
       let outBody = stripPreviousResponseId(
         parsed._rawBody,
@@ -1602,7 +1714,26 @@ export function createResponsesPassthroughAdapter(provider: OcxProviderConfig):
           outBody = stripOpenAiOnlyWebSearchFields(outBody);
         }
       }
-      const sanitizedBody = normalizeToolSchemas(stripSparkCompatibility(stripUnsupportedReasoningParams(stripItemIdsWhenUnstored(stripInvalidItemIds(stripUnsupportedHostedTools(sanitizeReasoningInputContent(scrubOcxCompactionItems(outBody), { preserveRawReasoningContent: provider.preserveResponsesReasoningContent === true })))))));
+      if (!isCanonicalOpenAiForwardProvider(provider)) {
+        // Codex 0.147 emits private namespace tool groups, while public/third-party Responses
+        // gateways accept only flat tool variants. Run after custom/tool-search lowering so
+        // namespace children already carry their final public kind before they are promoted.
+        const rewritten = rewriteRoutedNamespaceToolsForUpstream(outBody);
+        outBody = rewritten.body;
+        convertedRoutedNamespaceToolAliases = rewritten.aliases;
+        // Last, so promoted namespace children are also cleared of Codex-private fields.
+        outBody = stripCanonicalOnlyToolFields(outBody);
+      }
+      const threadServingIdentityChanged = parsed._stripReasoningEncryptedContent === true;
+      const sanitizedBody = normalizeToolSchemas(stripSparkCompatibility(stripUnsupportedReasoningParams(stripItemIdsWhenUnstored(stripInvalidItemIds(stripUnsupportedHostedTools(sanitizeReasoningInputContent(scrubOcxCompactionItems(
+        outBody,
+        destinationDecodesNativeCompactionBlob(provider),
+        threadServingIdentityChanged,
+      ), {
+        preserveRawReasoningContent: provider.preserveResponsesReasoningContent === true,
+        dropNullContentChannel: !isOpenAiOperatedResponsesDestination(provider),
+        stripEncryptedContent: threadServingIdentityChanged,
+      })))))));
       const finalBody = stripDisabledReasoningSummaries(
         normalizeConfiguredReasoningSummaryDelivery(sanitizedBody, provider, parsed.modelId),
         provider,
@@ -1630,6 +1761,7 @@ export function createResponsesPassthroughAdapter(provider: OcxProviderConfig):
         releaseBodyObservation,
         ...(convertedRoutedCustomToolNames ? { convertedRoutedCustomToolNames } : {}),
         ...(convertedRoutedToolSearchNames ? { convertedRoutedToolSearchNames } : {}),
+        ...(convertedRoutedNamespaceToolAliases ? { convertedRoutedNamespaceToolAliases } : {}),
         ...(tierLog ? { tierLog } : {}),
       };
     },

src/config.ts

@@ -715,6 +715,7 @@ const providerConfigSchema = z.object({
   supportsServiceTier: z.boolean().optional(),
   modelSupportsServiceTier: z.record(z.string().min(1), z.boolean()).optional(),
   preserveResponsesReasoningContent: z.boolean().optional(),
+  decodesNativeCompactionBlobs: z.boolean().optional(),
   allowPrivateNetwork: z.boolean().optional(),
   // The management API accepts `null` as "clear this", so a config written before the POST
   // canonicalization below can hold one on disk. Rejecting it here would send the operator

src/providers/openai-tiers.ts

@@ -54,6 +54,36 @@ export function supportsNativeResponsesCompactEndpoint(
     && normalizedBaseUrl(provider.baseUrl) === OPENAI_API_BASE_URL;
 }
 
+/**
+ * Whether this destination is an OpenAI-operated Responses backend — the canonical ChatGPT Codex
+ * surface or the official OpenAI API.
+ *
+ * Deliberately not keyed on `authMode === "forward"`: a noncanonical forward provider does not
+ * receive the caller's credentials (see the forward-header gate in the Responses adapter), so
+ * forward auth says nothing about which backend is on the other end.
+ */
+export function isOpenAiOperatedResponsesDestination(provider: OcxProviderConfig): boolean {
+  if (isCanonicalOpenAiForwardProvider(provider)) return true;
+  return provider.adapter === "openai-responses"
+    && normalizedBaseUrl(provider.baseUrl) === OPENAI_API_BASE_URL;
+}
+
+/**
+ * Whether this destination can decode a native (non-`ocx1:`) compaction blob.
+ *
+ * Only the backend that minted a blob can decode it. `authMode: "forward"` alone is not a signal:
+ * the adapter forwards caller credentials only to the canonical ChatGPT Codex surface, while a
+ * noncanonical forward provider receives no caller credentials and may point at any backend.
+ *
+ * Relay only to an OpenAI-operated destination or a destination whose operator explicitly opts in.
+ * Keyed by destination rather than provider id: a blob's issuer is the URL that produced it, not the
+ * local config key a replay travels under.
+ */
+export function destinationDecodesNativeCompactionBlob(provider: OcxProviderConfig): boolean {
+  return isOpenAiOperatedResponsesDestination(provider)
+    || provider.decodesNativeCompactionBlobs === true;
+}
+
 export interface OpenAiTierMigrationProjection {
   config: OcxConfig;
   changed: boolean;

src/responses/compaction.ts

@@ -33,6 +33,24 @@ export const SUMMARY_PREFIX = "Another language model started to solve this prob
 
 export const OPAQUE_COMPACTION_NOTE = "[earlier conversation was compacted; the summary is stored in a format this model cannot read]";
 
+/**
+ * Item types in the compact wire family. Each carries an `encrypted_content` blob the client
+ * replays verbatim on every later turn, and the minting backend verifies it is unmodified.
+ *
+ * Keep this the only enumeration: a copy that listed just `compaction` let the response-side
+ * field backfill synthesize ids into the other two, which the client then replayed as "modified
+ * from the compact response".
+ */
+const COMPACTION_ITEM_TYPES: ReadonlySet<string> = new Set([
+  "compaction",
+  "compaction_summary",
+  "context_compaction",
+]);
+
+export function isCompactionItemType(type: unknown): boolean {
+  return typeof type === "string" && COMPACTION_ITEM_TYPES.has(type);
+}
+
 export function encodeCompactionSummary(summary: string): string {
   return OCX_COMPACTION_PREFIX + Buffer.from(summary, "utf-8").toString("base64");
 }