From c49cb11aed900e255a2771e77de768423c78a11b Mon Sep 17 00:00:00 2001 From: Nguyen Thanh Dat Date: Tue, 18 Aug 2026 23:15:56 +0700 Subject: [PATCH] fix(lab): report the model gates the way the adapters actually match them `resolveProductionBehaviorValues` is documented as "authoritative effective values emitted by the production route/model/adapter resolver", and its hash is the behavior fingerprint that keys Lab evidence. Ten of its rows are membership tests over the provider's `no*Models`-style lists, and they went through a local function includesModel(list, modelId) { return Array.isArray(list) && list.includes(modelId); } while every runtime gate those rows describe matches through `modelInList`, which also accepts a bare entry for a tagged id. ollama-cloud serves `gpt-oss:120b`, `qwen3-coder:480b`, `qwen3.5:397b` and `gemma4:31b`, and the same registry row writes the bare `gpt-oss` into noVisionModels -- the bare-prefix form is how these lists are meant to be written. So with `noTemperatureModels: ["gpt-oss"]` the adapter omitted temperature from the request for `gpt-oss:120b` while the report said `sampling.omitTemperature: false`. Same for omitTopP, omitPenalties, reasoning.budgetMode, reasoning.splitMode, reasoning.toggleMode, reasoning.supported, reasoning.replayMode's two flags, and tools.choiceRestrictions. The blast radius is confined to subjects that were being described wrongly. Fingerprints measured on the same config, before and after: gpt-oss:120b 54154e19bd2c8ee4 -> 45a73577c5c257f9 (was wrong) gpt-oss 45a73577c5c257f9 -> 45a73577c5c257f9 (unchanged) glm-5.3 54154e19bd2c8ee4 -> 54154e19bd2c8ee4 (unchanged) Note the first line against the third: a model whose sampling gates were applied hashed identically to one where they were not. It now hashes with `gpt-oss`, the id it actually behaves like. Recorded evidence for unaffected subjects keeps its fingerprint, so `resolverVersion` is left at 2 rather than invalidating every recorded subject globally -- say the word if you would rather draw a clean generation boundary and I will bump it. Tests assert the wire the adapter really builds first, then hold the report to that same wire, so the pair cannot drift apart silently. The five report cases fail on current dev. Co-Authored-By: Claude Opus 5 --- src/routing/compatibility/behavior.ts | 12 ++- ...uting-compatibility-model-matching.test.ts | 74 +++++++++++++++++++ 2 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 tests/routing-compatibility-model-matching.test.ts diff --git a/src/routing/compatibility/behavior.ts b/src/routing/compatibility/behavior.ts index 553e531bb1..fce2ff004d 100644 --- a/src/routing/compatibility/behavior.ts +++ b/src/routing/compatibility/behavior.ts @@ -1,3 +1,4 @@ +import { modelInList } from "../../types"; import type { OcxConfig, OcxProviderConfig } from "../../types"; import { PROVIDER_REGISTRY } from "../../providers/registry"; import { fastPolicyForModel, serviceTierSupportForModel } from "../../providers/service-tier"; @@ -37,8 +38,17 @@ function behaviorRow(source: LabBehaviorSource, value: unknown) { return { source, value }; } +/** + * Membership for the provider's `no*Models`-style lists. + * + * Delegates to modelInList so the report matches the wire: every runtime gate these + * rows describe (openai-chat's sampling/reasoning/tool-choice gates, reasoning-effort's + * noReasoningModels) matches through modelInList, which also accepts a bare entry for a + * tagged id. ollama-cloud serves `gpt-oss:120b` and lists the bare `gpt-oss`, so an + * exact-only check here reported "temperature is sent" on a request that omits it. + */ function includesModel(list: string[] | undefined, modelId: string): boolean { - return Array.isArray(list) && list.includes(modelId); + return modelInList(list, modelId); } function modelValue(map: Record | undefined, modelId: string): T | undefined { diff --git a/tests/routing-compatibility-model-matching.test.ts b/tests/routing-compatibility-model-matching.test.ts new file mode 100644 index 0000000000..1dee27be25 --- /dev/null +++ b/tests/routing-compatibility-model-matching.test.ts @@ -0,0 +1,74 @@ +/** + * The compatibility behavior report is documented as "authoritative effective values + * emitted by the production route/model/adapter resolver", and its hash keys Lab + * evidence. These cases hold it to that: for one routed model they assert the wire the + * adapter actually builds, then assert the report describes that same wire. + */ +import { describe, expect, test } from "bun:test"; +import { resolveProductionBehaviorValues } from "../src/routing/compatibility/behavior"; +import { createOpenAIChatAdapter } from "../src/adapters/openai-chat"; +import type { OcxConfig, OcxParsedRequest, OcxProviderConfig } from "../src/types"; + +// ollama-cloud ships `gpt-oss:120b` verbatim (src/providers/registry.ts) and the same +// registry row lists the bare `gpt-oss` in noVisionModels, i.e. the bare-prefix form is +// the documented, intended way to write these lists. +const MODEL = "gpt-oss:120b"; + +const effective: OcxProviderConfig = { + adapter: "openai-chat", + baseUrl: "https://ollama.com/v1", + apiKey: "sk-test", + authMode: "key", + noTemperatureModels: ["gpt-oss"], + noTopPModels: ["gpt-oss"], + noPenaltyModels: ["gpt-oss"], + thinkingBudgetModels: ["gpt-oss"], + autoToolChoiceOnlyModels: ["gpt-oss"], +}; + +const config = { providers: { "ollama-cloud": effective } } as unknown as OcxConfig; + +const values = () => + resolveProductionBehaviorValues(config, "ollama-cloud", MODEL, effective, "salt")!; + +function wire(): Record { + const parsed: OcxParsedRequest = { + modelId: MODEL, + context: { messages: [{ role: "user", content: "hi", timestamp: 0 }] }, + stream: false, + options: { temperature: 0.5, topP: 0.9, presencePenalty: 0.2, frequencyPenalty: 0.2 }, + }; + return JSON.parse(createOpenAIChatAdapter(effective).buildRequest(parsed).body as string); +} + +describe("behavior report must agree with the wire the adapter actually builds", () => { + test("adapter really does omit these for the :tag model (ground truth)", () => { + const body = wire(); + expect(body.temperature).toBeUndefined(); + expect(body.top_p).toBeUndefined(); + expect(body.presence_penalty).toBeUndefined(); + expect(body.frequency_penalty).toBeUndefined(); + }); + + test("report agrees: sampling.omitTemperature", () => { + expect(values()["sampling.omitTemperature"]!.value).toBe(true); + }); + test("report agrees: sampling.omitTopP", () => { + expect(values()["sampling.omitTopP"]!.value).toBe(true); + }); + test("report agrees: sampling.omitPenalties", () => { + expect(values()["sampling.omitPenalties"]!.value).toBe(true); + }); + test("report agrees: reasoning.budgetMode", () => { + expect(values()["reasoning.budgetMode"]!.value).toBe(true); + }); + test("report agrees: tools.choiceRestrictions", () => { + expect(values()["tools.choiceRestrictions"]!.value).toEqual(["auto"]); + }); + + test("an unlisted model reports false (control)", () => { + const v = resolveProductionBehaviorValues(config, "ollama-cloud", "glm-5.3", effective, "salt")!; + expect(v["sampling.omitTemperature"]!.value).toBe(false); + expect(v["reasoning.budgetMode"]!.value).toBe(false); + }); +});