Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 41 additions & 8 deletions src/adapters/openai-chat.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,15 @@ import { buildNonOpenAIToolCatalogNudgeForTools, shouldInjectNonOpenAIToolCatalo
import { openRouterProviderPayload, resolveOpenRouterRouting } from "../providers/openrouter-routing";
import {
canForwardForeignServiceTierForChatModel,
fastPolicyForModel,
supportsServiceTierForModel,
} from "../providers/service-tier";
import {
canonicalFastTierMarker,
createAdapterTierMetadata,
decideTier,
type AdapterTierMetadata,
type ResolvedFastPolicy,
} from "../providers/fastwire";
import { openaiChatCompletionsUrl } from "./openai-chat-url";
import { stripResponsesOnlyEncryptedMarker } from "./responses-tool-schema";
Expand Down Expand Up @@ -96,6 +99,8 @@ export function buildOpenAIChatPassthroughRequest(
rawBody: Record<string, unknown>,
modelId: string,
stream: boolean,
fastPolicy: ResolvedFastPolicy = fastPolicyForModel(provider, modelId, undefined, "chat"),
fastMode?: boolean,
): AdapterRequest {
const { url, headers, hasCredential } = openAIChatTransport(provider);

Expand Down Expand Up @@ -123,7 +128,16 @@ export function buildOpenAIChatPassthroughRequest(
// `<listed>:<tag>` siblings the operator never opted out, silently returning prose.
if (provider.noStructuredOutputModels?.includes(modelId)) delete body.response_format;

if (provider.chatServiceTier && rawBody.service_tier !== undefined) {
// Run the same complete Fast policy as the translated Chat path, including explicit
// fastMode and foreign-tier handling. On inherited canonical Fast, the passthrough still
// retains the caller's exact spelling; forced Fast uses the policy-owned wire value.
const callerTier = typeof rawBody.service_tier === "string" ? rawBody.service_tier : undefined;
const tierDecision = decideTier(fastPolicy, fastMode, callerTier);
if (tierDecision.kind === "set") {
body.service_tier = fastMode === undefined && canonicalFastTierMarker(callerTier) !== undefined
? callerTier
: tierDecision.value;
} else if (tierDecision.kind === "forward-caller" && rawBody.service_tier !== undefined) {
body.service_tier = rawBody.service_tier;
}
if (provider.promptCacheKey && rawBody.prompt_cache_key !== undefined) {
Expand Down Expand Up @@ -1290,6 +1304,26 @@ function thinkingBudgetForEffort(parsed: OcxParsedRequest, reasoningEffort: stri
return fraction === undefined ? undefined : Math.max(1, Math.floor(maxBudget * fraction));
}

function canSerializeOpenAIChatServiceTier(
provider: OcxProviderConfig,
modelId: string,
serviceTier: unknown,
tierDecision?: OcxParsedRequest["options"]["tierDecision"],
): boolean {
if (serviceTier === undefined) return false;
if (tierDecision !== undefined) {
return tierDecision.kind === "set" || tierDecision.kind === "forward-caller";
}
// No decision from the router means this call did not go through the tier state machine, so
// ask that machine rather than re-deriving a looser answer beside it. The previous fallback
// returned true whenever foreign forwarding was allowed at all, which let a caller tier
// reach the wire in cases `decideTier` would have dropped — the two paths disagreeing is
// precisely the bug, so there is now only one authority.
const callerTier = typeof serviceTier === "string" ? serviceTier : undefined;
const decision = decideTier(fastPolicyForModel(provider, modelId, undefined, "chat"), undefined, callerTier);
return decision.kind === "set" || decision.kind === "forward-caller";
}

export function createOpenAIChatAdapter(provider: OcxProviderConfig): ProviderAdapter {
return {
name: "openai-chat",
Expand All @@ -1312,13 +1346,12 @@ export function createOpenAIChatAdapter(provider: OcxProviderConfig): ProviderAd
// unclassified Chat routes remain behind the caller-forwarding opt-in.
const serviceTier = parsed.options.serviceTier;
const tierDecision = parsed.options.tierDecision;
const callerCanonicalFast = canonicalFastTierMarker(serviceTier) !== undefined;
const callerTierForwardAllowed = canForwardForeignServiceTierForChatModel(provider, parsed.modelId);
const canonicalFastCapability = callerCanonicalFast
&& supportsServiceTierForModel(provider, parsed.modelId) === true;
const canSerializeServiceTier = tierDecision?.kind === "set"
|| tierDecision?.kind === "forward-caller"
|| (tierDecision === undefined && (callerTierForwardAllowed || canonicalFastCapability));
const canSerializeServiceTier = canSerializeOpenAIChatServiceTier(
provider,
parsed.modelId,
serviceTier,
tierDecision,
);
if (canSerializeServiceTier && serviceTier !== undefined) {
body.service_tier = serviceTier;
}
Expand Down
13 changes: 11 additions & 2 deletions src/server/chat-native.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import {
rateLimitRetryPolicyFor,
rotateProviderTransportOn429,
} from "../providers/key-failover";
import { fastPolicyForModel } from "../providers/service-tier";
import type { RouteResult } from "../router";
import type { OcxConfig, OcxProviderConfig } from "../types";
import { fetchWithHeaderTimeout, providerFetch, safeHostLabel } from "./responses/fetch-helpers";
Expand Down Expand Up @@ -154,8 +155,16 @@ export async function handleNativeChatCompletions(options: HandleNativeChatOptio
translatorBudget.chargeRetained(bytes, { kind: "request_copies" });
retainedRequestBytes = bytes;
};
const buildActiveRequest = () => buildOpenAIChatPassthroughRequest(
activeProvider,
options.chatBody,
route.modelId,
requestedStream,
fastPolicyForModel(activeProvider, route.modelId, route.providerName, "chat"),
config.fastMode,
);
try {
activeRequest = buildOpenAIChatPassthroughRequest(activeProvider, options.chatBody, route.modelId, requestedStream);
activeRequest = buildActiveRequest();
retainRequest(activeRequest);
} catch (error) {
releaseRetainedRequest();
Expand Down Expand Up @@ -222,7 +231,7 @@ export async function handleNativeChatCompletions(options: HandleNativeChatOptio
activeProvider = rotated;
activeAdapter = createOpenAIChatAdapter(activeProvider);
releaseRetainedRequest();
activeRequest = buildOpenAIChatPassthroughRequest(activeProvider, options.chatBody, route.modelId, requestedStream);
activeRequest = buildActiveRequest();
retainRequest(activeRequest);
response = await send(activeRequest, "key-429");
}
Expand Down
26 changes: 15 additions & 11 deletions tests/fastwire-characterization-wire.test.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { afterEach, describe, expect, spyOn, test } from "bun:test";
import { buildOpenAIChatPassthroughRequest } from "../src/adapters/openai-chat";
import { chatCompletionsToResponsesBody } from "../src/chat/inbound";
import { fastPolicyForModel } from "../src/providers/service-tier";
import * as adapterResolveModule from "../src/server/adapter-resolve";
import type { RequestLogContext } from "../src/server/request-log";
import { handleResponses } from "../src/server/responses/core";
Expand Down Expand Up @@ -364,27 +365,30 @@ describe("FastWire characterization: rawBody observation point", () => {
});

describe("FastWire characterization: known bugs", () => {
test("characterization (known bug): native chat passthrough ignores exact-model false", () => {
test("characterization: native chat passthrough honors exact-model false", () => {
// FastWire #1886 native-chat policy fix: exact-model false now strips the caller tier.
const provider: OcxProviderConfig = {
adapter: "openai-chat",
baseUrl: "https://native-chat.example.test/v1",
authMode: "key",
apiKey: "sk-test",
supportsServiceTier: true,
chatServiceTier: true,
modelSupportsServiceTier: { model: false },
};
const request = buildOpenAIChatPassthroughRequest(
{
adapter: "openai-chat",
baseUrl: "https://native-chat.example.test/v1",
authMode: "key",
apiKey: "sk-test",
supportsServiceTier: true,
chatServiceTier: true,
modelSupportsServiceTier: { model: false },
},
provider,
{
model: "model",
messages: [{ role: "user", content: "ping" }],
service_tier: "flex",
},
"model",
false,
fastPolicyForModel(provider, "model", "native-chat", "chat"),
);
const body = JSON.parse(request.body) as Record<string, unknown>;
expect(body.service_tier).toBe("flex");
expect(body).not.toHaveProperty("service_tier");
});

test("characterization: chat-to-responses conversion preserves service_tier", () => {
Expand Down
Loading
Loading