From fea77c162ef917950e2466364022c99f5076dfa5 Mon Sep 17 00:00:00 2001 From: NotXf1le <89696340+NotXf1le@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:42:48 +0200 Subject: [PATCH] Expose missing-logprob probes in MCP --- packages/choosekit-mcp/README.md | 3 ++- packages/choosekit-mcp/package-lock.json | 12 ++++++------ packages/choosekit-mcp/package.json | 4 ++-- packages/choosekit-mcp/src/cli.ts | 1 + packages/choosekit-mcp/src/config.ts | 6 ++++++ packages/choosekit-mcp/tests/cli.test.mjs | 4 ++++ packages/choosekit-mcp/tests/package.test.mjs | 1 + 7 files changed, 22 insertions(+), 9 deletions(-) diff --git a/packages/choosekit-mcp/README.md b/packages/choosekit-mcp/README.md index 34cda70..50b2ca6 100644 --- a/packages/choosekit-mcp/README.md +++ b/packages/choosekit-mcp/README.md @@ -15,6 +15,7 @@ Select and configure the backend when the MCP process starts. `CHOOSEKIT_BACKEND | All | `CHOOSEKIT_MODE` | `labels` | `labels` for every backend; `minimal-prefix` is llama.cpp only | | All | `CHOOSEKIT_IMAGE_ROOT` | Current working directory | Root directory for files supplied through `imagePaths` | | llama.cpp | `CHOOSEKIT_BASE_URL` | Required | Base URL of the llama.cpp server, for example `http://127.0.0.1:8080` | +| llama.cpp | `CHOOSEKIT_PROBE_MISSING_LOGPROBS` | `false` | Set to `true` to score choices outside the returned top logprobs with additional requests | | Ollama | `CHOOSEKIT_BASE_URL` | `http://127.0.0.1:11434` | Base URL of the Ollama server | | OpenRouter | `OPENROUTER_API_KEY` | Required | OpenRouter API key | | OpenRouter | `OPENROUTER_PROVIDER` | Not set | Pins one provider and disables fallback | @@ -93,7 +94,7 @@ Relative paths are resolved from `CHOOSEKIT_IMAGE_ROOT`, which defaults to the M With `labels`, choosekit maps the supplied choice keys to A/B/C labels for scoring, so each description must contain the option's full meaning. With `minimal-prefix`, it scores the shortest token prefixes that distinguish the original keys. When no supplied option may apply, add an explicit choice such as `insufficient_information`. -The result contains the selected key, the complete normalized distribution, raw scores, margin, entropy, token-boundary rollback, and backend usage when available. A score is `null` when the upstream backend did not return a log probability for that choice; its probability in the distribution is `0`. Probabilities represent relative preference among the supplied choices. Estimating correctness requires separate calibration. +The result contains the selected key, the normalized distribution, raw scores, margin, entropy, token-boundary rollback, and backend usage when available. A score is `null` when the upstream backend did not return a log probability for that choice; its probability in the distribution is `0`. Probabilities represent relative preference among the supplied choices. Estimating correctness requires separate calibration. ## Requirements diff --git a/packages/choosekit-mcp/package-lock.json b/packages/choosekit-mcp/package-lock.json index 832a955..fbb4abe 100644 --- a/packages/choosekit-mcp/package-lock.json +++ b/packages/choosekit-mcp/package-lock.json @@ -1,16 +1,16 @@ { "name": "choosekit-mcp", - "version": "0.3.0", + "version": "0.4.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "choosekit-mcp", - "version": "0.3.0", + "version": "0.4.0", "license": "Apache-2.0", "dependencies": { "@modelcontextprotocol/server": "2.0.0", - "choosekit": "^0.6.0", + "choosekit": "^0.7.0", "zod": "^4.6.4" }, "bin": { @@ -60,9 +60,9 @@ } }, "node_modules/choosekit": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/choosekit/-/choosekit-0.6.0.tgz", - "integrity": "sha512-dhTRxTbS5uCQapuxNLL1csglaKzsc87u7KaKqyIEuE5vfunOH13+k7k6JoJ2CRzBiwaH2IC/rHItc3Jfgr9txA==", + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/choosekit/-/choosekit-0.7.0.tgz", + "integrity": "sha512-BbJt3psfsi9JRUvdfB2qUT6huECmvjTgQ+A+b17DnOmoJg4hdbr4eYJOYS9HobugRrRXXrAkC9ij6Z3vvWBbpA==", "license": "Apache-2.0", "engines": { "node": ">=20" diff --git a/packages/choosekit-mcp/package.json b/packages/choosekit-mcp/package.json index ab1f955..700ce76 100644 --- a/packages/choosekit-mcp/package.json +++ b/packages/choosekit-mcp/package.json @@ -1,6 +1,6 @@ { "name": "choosekit-mcp", - "version": "0.3.0", + "version": "0.4.0", "description": "Use choosekit as an MCP server.", "license": "Apache-2.0", "author": { @@ -37,7 +37,7 @@ }, "dependencies": { "@modelcontextprotocol/server": "2.0.0", - "choosekit": "^0.6.0", + "choosekit": "^0.7.0", "zod": "^4.6.4" }, "devDependencies": { diff --git a/packages/choosekit-mcp/src/cli.ts b/packages/choosekit-mcp/src/cli.ts index 38566fe..03188e8 100644 --- a/packages/choosekit-mcp/src/cli.ts +++ b/packages/choosekit-mcp/src/cli.ts @@ -18,6 +18,7 @@ try { baseURL: config.baseURL, ...(config.model === undefined ? {} : { model: config.model }), mode: config.mode, + probeMissingLogprobs: config.probeMissingLogprobs, }); break; case "ollama": diff --git a/packages/choosekit-mcp/src/config.ts b/packages/choosekit-mcp/src/config.ts index c25be6d..16104a4 100644 --- a/packages/choosekit-mcp/src/config.ts +++ b/packages/choosekit-mcp/src/config.ts @@ -9,6 +9,7 @@ export type Config = readonly imageRoot?: string; readonly model?: string; readonly mode: "labels" | "minimal-prefix"; + readonly probeMissingLogprobs: boolean; } | { readonly backend: "openrouter"; @@ -89,11 +90,16 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): Config { if (mode !== "labels" && mode !== "minimal-prefix") { throw new TypeError("CHOOSEKIT_MODE must be labels or minimal-prefix."); } + const probe = env.CHOOSEKIT_PROBE_MISSING_LOGPROBS?.trim() ?? "false"; + if (probe !== "true" && probe !== "false") { + throw new TypeError("CHOOSEKIT_PROBE_MISSING_LOGPROBS must be true or false."); + } return { backend, baseURL, ...(imageRoot === undefined ? {} : { imageRoot }), ...(model === undefined ? {} : { model }), mode, + probeMissingLogprobs: probe === "true", }; } diff --git a/packages/choosekit-mcp/tests/cli.test.mjs b/packages/choosekit-mcp/tests/cli.test.mjs index 69b6a9e..89ffa27 100644 --- a/packages/choosekit-mcp/tests/cli.test.mjs +++ b/packages/choosekit-mcp/tests/cli.test.mjs @@ -25,6 +25,7 @@ const configurationKeys = [ "CHOOSEKIT_IMAGE_ROOT", "CHOOSEKIT_MODEL", "CHOOSEKIT_MODE", + "CHOOSEKIT_PROBE_MISSING_LOGPROBS", "OPENROUTER_API_KEY", "OPENROUTER_PROVIDER", ]; @@ -134,6 +135,8 @@ test("reports invalid environment configuration without writing to stdout", () = "CHOOSEKIT_BASE_URL must be a valid URL."], ["invalid mode", { CHOOSEKIT_MODE: "keys" }, "CHOOSEKIT_MODE must be labels or minimal-prefix."], + ["invalid missing-logprob setting", { CHOOSEKIT_PROBE_MISSING_LOGPROBS: "1" }, + "CHOOSEKIT_PROBE_MISSING_LOGPROBS must be true or false."], ["unknown backend", { CHOOSEKIT_BACKEND: "other" }, "CHOOSEKIT_BACKEND must be llama-cpp, ollama, or openrouter."], ["Ollama without model", { @@ -296,6 +299,7 @@ test("serves a minimal-prefix choice using the configured llama.cpp endpoint", a CHOOSEKIT_BASE_URL: `http://127.0.0.1:${port}`, CHOOSEKIT_MODEL: "fixture-model", CHOOSEKIT_MODE: "minimal-prefix", + CHOOSEKIT_PROBE_MISSING_LOGPROBS: "true", })); t.after(() => process_.close()); await initialize(process_); diff --git a/packages/choosekit-mcp/tests/package.test.mjs b/packages/choosekit-mcp/tests/package.test.mjs index fc67c5a..0fb4618 100644 --- a/packages/choosekit-mcp/tests/package.test.mjs +++ b/packages/choosekit-mcp/tests/package.test.mjs @@ -18,6 +18,7 @@ function llamaEnvironment() { "CHOOSEKIT_BASE_URL", "CHOOSEKIT_MODEL", "CHOOSEKIT_MODE", + "CHOOSEKIT_PROBE_MISSING_LOGPROBS", "OPENROUTER_API_KEY", "OPENROUTER_PROVIDER", ]) delete env[key];